// What the harness sends the server and learns from it, for the operations // checks (spec/OPERATIONS.md): each request, with the trace context it was // sent in, its answer and how long it took; and every secret the run // handled, which the server's log must never hold. // // Every fetch to the server goes through `watch`'s wrapper; a request made // with node:http calls `begin` and `end` itself. import { createHash } from 'node:crypto' import { readdirSync, readFileSync } from 'node:fs' import { join } from 'node:path' import { SEED } from './args.mjs' /// Wall-clock milliseconds, with a fraction: of the clock as it is when this /// module loads, which lib/clients.mjs's aging of `performance.now` (for the /// reference's caches) does not move. const realNow = performance.now.bind(performance) export const now = () => performance.timeOrigin + realNow() /// Each request the harness sent the server, in order: `{n, life, traceId, /// spanId, sampled, method, path, route, params, status, started, ended, /// token, commit, did}`, and of a fetch, once its body is read, `bytes`, /// how many it read, `length`, its `Content-Length` if any, and `short`, /// true if the body failed before it ended. `life` is the server process it /// went to (`server.lives`), `params` its query, `status` 0 until it is /// answered, or for good if it never was; `token` the bearer token it /// carried; `commit` whether its JSON answer says a commit was made, and /// `did` the DID it names. export const sent = [] /// Every secret the run handled: passwords, tokens, invite codes, keys, /// emails, and the canary record values the server must not log. export const secrets = new Set() /// The DID each bearer token the server gave out is for. export const tokenDids = new Map() let serial = 0 let origin = null let lifeOf = () => 0 /// The trace context of the harness's `n`th request: ids made from the seed /// and `n`, not drawn, so that the requests of a seed stay the same; every /// tenth with its sampled flag clear. export function traceFor(n) { const h = createHash('sha256').update(`trace ${SEED} ${n}`).digest('hex') return { traceId: h.slice(0, 32), spanId: h.slice(32, 48), sampled: n % 10 !== 0 } } /// A request about to go to the server: noted, with a `traceparent` added /// to `headers` (a plain object, lowercase names, or a Headers) unless it /// has one already. Answers the note, which `end` completes. export function begin(method, url, headers, body) { const u = new URL(url) const get = (k) => (headers instanceof Headers ? headers.get(k) : headers[k]) const set = (k, v) => (headers instanceof Headers ? headers.set(k, v) : (headers[k] = v)) const n = ++serial let ctx = traceFor(n) const given = get('traceparent') if (given) { const m = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/.exec(given) ctx = m ? { traceId: m[1], spanId: m[2], sampled: (parseInt(m[3], 16) & 1) === 1 } : { traceId: null, spanId: null, sampled: false } } else set('traceparent', `00-${ctx.traceId}-${ctx.spanId}-${ctx.sampled ? '01' : '00'}`) const auth = get('authorization') noteAuth(auth) // Each proof of a request sent two, which a Headers joins with commas. const dpop = get('dpop') noteAuth(dpop, true) if (typeof dpop === 'string') for (const p of dpop.split(/,\s*/)) noteSecret(p) noteCookies(get('cookie')) const type = get('content-type') ?? '' const text = typeof body === 'string' ? body : body instanceof Uint8Array && body.length < 1 << 20 ? Buffer.from(body).toString('utf8') : body instanceof URLSearchParams ? body.toString() : null if (text !== null && /x-www-form-urlencoded/i.test(type)) noteFields(new URLSearchParams(text)) else if (typeof body === 'string' || (text !== null && /json/.test(type))) noteText(text) const rec = { n, life: lifeOf(), ...ctx, method: (method ?? 'GET').toUpperCase(), path: u.pathname, route: routeOf(u.pathname), params: Object.fromEntries(u.searchParams), status: 0, started: now(), ended: null, token: /^Bearer (\S+)$/.exec(auth ?? '')?.[1] ?? null, commit: false, did: null, } sent.push(rec) return rec } /// The answer to `rec`: its status, and its body's text if it may hold /// JSON or be a page, once it has been read whole. export function end(rec, status, text) { rec.status = status rec.ended = now() if (typeof text !== 'string' || !text) return const values = jsonIn(text) if (!values.length) notePage(text) for (const v of values) { learn(v) if (v && typeof v === 'object' && !Array.isArray(v)) { if (v.commit && typeof v.commit === 'object') rec.commit = true if (typeof v.did === 'string') rec.did = v.did } } } /// Note the secrets in an HTML page: the fields of its forms (an OAuth /// `form_post` page carries its code so), and the JSON its script declares. function notePage(html) { const unescape = (s) => s.replace(/"/g, '"').replace(/'/g, "'").replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&') for (const m of html.matchAll(/]*\bname="([^"]*)"[^>]*\bvalue="([^"]*)"/gi)) noteFields({ [unescape(m[1])]: unescape(m[2]) }) for (const m of html.matchAll(/=\s*JSON\.parse\(("(?:[^"\\]|\\.)*")\)/g)) { try { walk(JSON.parse(JSON.parse(m[1]))) } catch {} } } /// Note the secrets in an answer of the server's, however it came (over /// XRPC, or from an MCP tool or CLI command), and the account each token /// it gives out is for. export function learn(v) { walk(v) if (v && typeof v === 'object' && typeof v.did === 'string') for (const k of ['accessJwt', 'refreshJwt']) if (typeof v[k] === 'string') tokenDids.set(v[k], v.did) } /// Send every fetch to `serverUrl` through `begin` and `end`. `life` says /// which of the server's processes is running. export function watch(serverUrl, life) { origin = new URL(serverUrl).origin lifeOf = life const plain = globalThis.fetch globalThis.fetch = async (input, init = {}) => { const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) if (url.origin !== origin) return plain(input, init) const headers = new Headers(init.headers ?? (input instanceof Request ? input.headers : undefined)) const rec = begin(init.method ?? (input instanceof Request ? input.method : 'GET'), url, headers, init.body) let res try { res = await plain(input, { ...init, headers }) } catch (err) { rec.ended = now() throw err } rec.status = res.status noteUrl(res.headers.get('location')) noteSetCookies(res.headers.getSetCookie()) if (res.redirected) noteUrl(res.url) const type = res.headers.get('content-type') ?? '' const length = res.headers.get('content-length') if (length !== null) rec.length = Number(length) rec.read = readCounted(res.clone(), rec).then((buf) => end(rec, res.status, buf && /json|event-stream|text/.test(type) && buf.byteLength < 1 << 22 ? buf.toString('utf8') : '')) return res } } /// The body of `res`, read whole, its bytes counted on `rec` as they come; /// null, with `rec.short`, if it failed before it ended. async function readCounted(res, rec) { rec.bytes = 0 if (!res.body) return Buffer.alloc(0) const chunks = [] try { for await (const chunk of res.body) { chunks.push(chunk) rec.bytes += chunk.byteLength } } catch { rec.short = true rec.ended = now() return null } return Buffer.concat(chunks) } /// Every answer read whole. export const settled = () => Promise.all(sent.map((r) => r.read).filter(Boolean)) // ------------------------------------------------------------ secrets /// A secret, and for a JWT its signature too, which a log of part of it /// would hold. export function noteSecret(s) { if (typeof s !== 'string' || s.length < 8) return secrets.add(s) const parts = s.split('.') if (parts.length === 3 && parts[2].length >= 16) secrets.add(parts[2]) } /// Note the secrets in an `Authorization` header (or, `whole`, a header /// that is one, as `DPoP`). export function noteAuth(h, whole = false) { if (typeof h !== 'string') return if (whole) return noteSecret(h) const [scheme, value] = h.split(' ') if (/^bearer$/i.test(scheme)) noteSecret(value) else if (/^basic$/i.test(scheme) && value) { noteSecret(value) const decoded = Buffer.from(value, 'base64').toString('utf8') noteSecret(decoded.slice(decoded.indexOf(':') + 1)) } else noteSecret(value) } /// Note the values of a `Cookie` header: a device's and a session's ids /// (spec/OAUTH.md, "Devices") are secrets. export function noteCookies(h) { if (typeof h !== 'string') return for (const pair of h.split(/;\s*/)) noteSecret(pair.slice(pair.indexOf('=') + 1)) } /// Note the values of an answer's `Set-Cookie` headers. export function noteSetCookies(all) { for (const c of all ?? []) noteCookies(c.split(';')[0]) } /// The keys whose string values are secrets: passwords, tokens, codes /// (an email's one-time code at OAuth's sign-in, `emailOtp`, too), /// credentials and keys, assertions and attestations, and emails. const SECRET_KEY = /(password|jwt|token|code|otp|credential|secret|email|privatekey|assertion|attestation)s?$/i /// Note the secrets in a form's or a URL's parameters (a URLSearchParams, /// or a plain object): those under a secret's key. export function noteFields(params) { for (const [k, v] of params instanceof URLSearchParams ? params : Object.entries(params ?? {})) if (SECRET_KEY.test(k)) noteSecret(v) } /// Note the secrets in a URL's query and fragment, as an OAuth redirect /// carries its code or token. export function noteUrl(url) { if (typeof url !== 'string' || !/[?#]/.test(url)) return let u try { u = new URL(url) } catch { return } noteFields(new URLSearchParams(u.search)) noteFields(new URLSearchParams(u.hash.slice(1))) } /// Whether `url` goes to the server, not the reference or anything else. export const toServer = (url) => origin !== null && new URL(url).origin === origin /// Note the secrets in a JSON value: the strings under a secret's key, in /// arrays too, and in strings that are JSON themselves, as an MCP tool's /// answer holds its text. export function walk(v, secret = false) { if (typeof v === 'string') { if (secret) noteSecret(v) else if (/^[[{]/.test(v)) for (const x of jsonIn(v)) walk(x) else if (/^[a-z][a-z0-9+.-]*:/i.test(v)) noteUrl(v) } else if (Array.isArray(v)) for (const x of v) walk(x, secret) else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) walk(x, SECRET_KEY.test(k)) } /// Note the secrets in a request's or an answer's text. export function noteText(text) { for (const v of jsonIn(text)) walk(v) } /// The JSON values in `text`: the whole of it, or each `data:` of an event /// stream. function jsonIn(text) { const out = [] try { out.push(JSON.parse(text)) } catch { for (const line of text.split(/\r?\n/)) { if (!line.startsWith('data:')) continue try { out.push(JSON.parse(line.slice(5))) } catch {} } } return out } // ------------------------------------------------------------ routes /// The methods the server serves: those whose lexicons are in spec/lexicons /// (spec/PDS.md, "What it serves"). export const SERVED = new Set() for (const file of readdirSync(new URL('../../spec/lexicons', import.meta.url), { recursive: true })) { if (!String(file).endsWith('.json')) continue const lex = JSON.parse(readFileSync(join(new URL('../../spec/lexicons', import.meta.url).pathname, String(file)), 'utf8')) if (['query', 'procedure', 'subscription'].includes(lex.defs?.main?.type)) SERVED.add(lex.id) } /// The routes outside /xrpc/ the server serves, as spec/OPERATIONS.md lists them. export const ROUTES = [ '/', '/mcp', '/.well-known/atproto-did', '/.well-known/oauth-protected-resource', '/.well-known/oauth-authorization-server', '/oauth/jwks', '/oauth/par', '/oauth/token', '/oauth/revoke', '/oauth/authorize', '/oauth/authorize/redirect', '/@atproto/oauth-provider/~api/sign-in', '/@atproto/oauth-provider/~api/consent', '/@atproto/oauth-provider/~api/reject', '/@atproto/oauth-provider/~api/sign-out', '/@atproto/oauth-provider/~api/passkey-options', '/@atproto/oauth-provider/~api/passkey-sign-in', '/oauth/passkey', '/@atproto/oauth-provider/~api/passkey-register', '/.well-known/oauth-protected-resource/mcp', '/.well-known/oauth-authorization-server/oauth/mcp', '/oauth/mcp/authorize', '/oauth/mcp/token', '/oauth/mcp/revoke', ] /// The server's own routes under /xrpc/, of no lexicon: its health, the /// admin's reads, and the account's OAuth and passkey routes /// (spec/OPERATIONS.md, "Health", "The admin's reads"; spec/OAUTH-ACCOUNT.md; /// spec/PASSKEYS.md). export const OWN_XRPC = ['_health', '_logs', '_stats', '_oauthSessions', '_oauthRevokeSession', '_oauthDevices', '_oauthRevokeDevice', '_passkeyList', '_passkeyAdd', '_passkeyRemove'] /// A path's route, as the server's metrics and log name it /// (spec/OPERATIONS.md): every path under `/oauth/assets/` is one route, /// whatever its name. export function routeOf(path) { if (path.startsWith('/xrpc/') && OWN_XRPC.includes(path.slice(6))) return path.slice(6) if (path.startsWith('/xrpc/')) return SERVED.has(path.slice(6)) ? path.slice(6) : '_unknown' if (path.startsWith('/oauth/assets/')) return ASSETS return ROUTES.includes(path) ? path : '_unknown' } /// The route of the authorization pages' assets (spec/OAUTH-PAGES.md). export const ASSETS = '/oauth/assets/{name}' /// A route's `http.route`, or null for `_unknown`. export const httpRoute = (route) => (route === '_unknown' ? null : route.startsWith('/') ? route : `/xrpc/${route}`)