~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
330 lines · 13.5 KB Raw History
// What the harness sends the server and learns from it, for the operations
// checks (spec/OPERATIONS.md): each request, with the trace context it was
// sent in, its answer and how long it took; and every secret the run
// handled, which the server's log must never hold.
//
// Every fetch to the server goes through `watch`'s wrapper; a request made
// with node:http calls `begin` and `end` itself.

import { createHash } from 'node:crypto'
import { readdirSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import { SEED } from './args.mjs'

/// Wall-clock milliseconds, with a fraction: of the clock as it is when this
/// module loads, which lib/clients.mjs's aging of `performance.now` (for the
/// reference's caches) does not move.
const realNow = performance.now.bind(performance)
export const now = () => performance.timeOrigin + realNow()

/// Each request the harness sent the server, in order: `{n, life, traceId,
/// spanId, sampled, method, path, route, params, status, started, ended,
/// token, commit, did}`, and of a fetch, once its body is read, `bytes`,
/// how many it read, `length`, its `Content-Length` if any, and `short`,
/// true if the body failed before it ended. `life` is the server process it
/// went to (`server.lives`), `params` its query, `status` 0 until it is
/// answered, or for good if it never was; `token` the bearer token it
/// carried; `commit` whether its JSON answer says a commit was made, and
/// `did` the DID it names.
export const sent = []

/// Every secret the run handled: passwords, tokens, invite codes, keys,
/// emails, and the canary record values the server must not log.
export const secrets = new Set()

/// The DID each bearer token the server gave out is for.
export const tokenDids = new Map()

let serial = 0
let origin = null
let lifeOf = () => 0

/// The trace context of the harness's `n`th request: ids made from the seed
/// and `n`, not drawn, so that the requests of a seed stay the same; every
/// tenth with its sampled flag clear.
export function traceFor(n) {
  const h = createHash('sha256').update(`trace ${SEED} ${n}`).digest('hex')
  return { traceId: h.slice(0, 32), spanId: h.slice(32, 48), sampled: n % 10 !== 0 }
}

/// A request about to go to the server: noted, with a `traceparent` added
/// to `headers` (a plain object, lowercase names, or a Headers) unless it
/// has one already. Answers the note, which `end` completes.
export function begin(method, url, headers, body) {
  const u = new URL(url)
  const get = (k) => (headers instanceof Headers ? headers.get(k) : headers[k])
  const set = (k, v) => (headers instanceof Headers ? headers.set(k, v) : (headers[k] = v))
  const n = ++serial
  let ctx = traceFor(n)
  const given = get('traceparent')
  if (given) {
    const m = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/.exec(given)
    ctx = m ? { traceId: m[1], spanId: m[2], sampled: (parseInt(m[3], 16) & 1) === 1 } : { traceId: null, spanId: null, sampled: false }
  } else set('traceparent', `00-${ctx.traceId}-${ctx.spanId}-${ctx.sampled ? '01' : '00'}`)
  const auth = get('authorization')
  noteAuth(auth)
  // Each proof of a request sent two, which a Headers joins with commas.
  const dpop = get('dpop')
  noteAuth(dpop, true)
  if (typeof dpop === 'string') for (const p of dpop.split(/,\s*/)) noteSecret(p)
  noteCookies(get('cookie'))
  const type = get('content-type') ?? ''
  const text = typeof body === 'string' ? body : body instanceof Uint8Array && body.length < 1 << 20 ? Buffer.from(body).toString('utf8') : body instanceof URLSearchParams ? body.toString() : null
  if (text !== null && /x-www-form-urlencoded/i.test(type)) noteFields(new URLSearchParams(text))
  else if (typeof body === 'string' || (text !== null && /json/.test(type))) noteText(text)
  const rec = {
    n, life: lifeOf(), ...ctx, method: (method ?? 'GET').toUpperCase(), path: u.pathname, route: routeOf(u.pathname), params: Object.fromEntries(u.searchParams),
    status: 0, started: now(), ended: null, token: /^Bearer (\S+)$/.exec(auth ?? '')?.[1] ?? null, commit: false, did: null,
  }
  sent.push(rec)
  return rec
}

/// The answer to `rec`: its status, and its body's text if it may hold
/// JSON or be a page, once it has been read whole.
export function end(rec, status, text) {
  rec.status = status
  rec.ended = now()
  if (typeof text !== 'string' || !text) return
  const values = jsonIn(text)
  if (!values.length) notePage(text)
  for (const v of values) {
    learn(v)
    if (v && typeof v === 'object' && !Array.isArray(v)) {
      if (v.commit && typeof v.commit === 'object') rec.commit = true
      if (typeof v.did === 'string') rec.did = v.did
    }
  }
}

/// Note the secrets in an HTML page: the fields of its forms (an OAuth
/// `form_post` page carries its code so), and the JSON its script declares.
function notePage(html) {
  const unescape = (s) => s.replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&amp;/g, '&')
  for (const m of html.matchAll(/<input\b[^>]*\bname="([^"]*)"[^>]*\bvalue="([^"]*)"/gi)) noteFields({ [unescape(m[1])]: unescape(m[2]) })
  for (const m of html.matchAll(/=\s*JSON\.parse\(("(?:[^"\\]|\\.)*")\)/g)) {
    try {
      walk(JSON.parse(JSON.parse(m[1])))
    } catch {}
  }
}

/// Note the secrets in an answer of the server's, however it came (over
/// XRPC, or from an MCP tool or CLI command), and the account each token
/// it gives out is for.
export function learn(v) {
  walk(v)
  if (v && typeof v === 'object' && typeof v.did === 'string') for (const k of ['accessJwt', 'refreshJwt']) if (typeof v[k] === 'string') tokenDids.set(v[k], v.did)
}

/// Send every fetch to `serverUrl` through `begin` and `end`. `life` says
/// which of the server's processes is running.
export function watch(serverUrl, life) {
  origin = new URL(serverUrl).origin
  lifeOf = life
  const plain = globalThis.fetch
  globalThis.fetch = async (input, init = {}) => {
    const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url)
    if (url.origin !== origin) return plain(input, init)
    const headers = new Headers(init.headers ?? (input instanceof Request ? input.headers : undefined))
    const rec = begin(init.method ?? (input instanceof Request ? input.method : 'GET'), url, headers, init.body)
    let res
    try {
      res = await plain(input, { ...init, headers })
    } catch (err) {
      rec.ended = now()
      throw err
    }
    rec.status = res.status
    noteUrl(res.headers.get('location'))
    noteSetCookies(res.headers.getSetCookie())
    if (res.redirected) noteUrl(res.url)
    const type = res.headers.get('content-type') ?? ''
    const length = res.headers.get('content-length')
    if (length !== null) rec.length = Number(length)
    rec.read = readCounted(res.clone(), rec).then((buf) => end(rec, res.status, buf && /json|event-stream|text/.test(type) && buf.byteLength < 1 << 22 ? buf.toString('utf8') : ''))
    return res
  }
}

/// The body of `res`, read whole, its bytes counted on `rec` as they come;
/// null, with `rec.short`, if it failed before it ended.
async function readCounted(res, rec) {
  rec.bytes = 0
  if (!res.body) return Buffer.alloc(0)
  const chunks = []
  try {
    for await (const chunk of res.body) {
      chunks.push(chunk)
      rec.bytes += chunk.byteLength
    }
  } catch {
    rec.short = true
    rec.ended = now()
    return null
  }
  return Buffer.concat(chunks)
}

/// Every answer read whole.
export const settled = () => Promise.all(sent.map((r) => r.read).filter(Boolean))

// ------------------------------------------------------------ secrets

/// A secret, and for a JWT its signature too, which a log of part of it
/// would hold.
export function noteSecret(s) {
  if (typeof s !== 'string' || s.length < 8) return
  secrets.add(s)
  const parts = s.split('.')
  if (parts.length === 3 && parts[2].length >= 16) secrets.add(parts[2])
}

/// Note the secrets in an `Authorization` header (or, `whole`, a header
/// that is one, as `DPoP`).
export function noteAuth(h, whole = false) {
  if (typeof h !== 'string') return
  if (whole) return noteSecret(h)
  const [scheme, value] = h.split(' ')
  if (/^bearer$/i.test(scheme)) noteSecret(value)
  else if (/^basic$/i.test(scheme) && value) {
    noteSecret(value)
    const decoded = Buffer.from(value, 'base64').toString('utf8')
    noteSecret(decoded.slice(decoded.indexOf(':') + 1))
  } else noteSecret(value)
}

/// Note the values of a `Cookie` header: a device's and a session's ids
/// (spec/OAUTH.md, "Devices") are secrets.
export function noteCookies(h) {
  if (typeof h !== 'string') return
  for (const pair of h.split(/;\s*/)) noteSecret(pair.slice(pair.indexOf('=') + 1))
}

/// Note the values of an answer's `Set-Cookie` headers.
export function noteSetCookies(all) {
  for (const c of all ?? []) noteCookies(c.split(';')[0])
}

/// The keys whose string values are secrets: passwords, tokens, codes
/// (an email's one-time code at OAuth's sign-in, `emailOtp`, too),
/// credentials and keys, assertions and attestations, and emails.
const SECRET_KEY = /(password|jwt|token|code|otp|credential|secret|email|privatekey|assertion|attestation)s?$/i

/// Note the secrets in a form's or a URL's parameters (a URLSearchParams,
/// or a plain object): those under a secret's key.
export function noteFields(params) {
  for (const [k, v] of params instanceof URLSearchParams ? params : Object.entries(params ?? {})) if (SECRET_KEY.test(k)) noteSecret(v)
}

/// Note the secrets in a URL's query and fragment, as an OAuth redirect
/// carries its code or token.
export function noteUrl(url) {
  if (typeof url !== 'string' || !/[?#]/.test(url)) return
  let u
  try {
    u = new URL(url)
  } catch {
    return
  }
  noteFields(new URLSearchParams(u.search))
  noteFields(new URLSearchParams(u.hash.slice(1)))
}

/// Whether `url` goes to the server, not the reference or anything else.
export const toServer = (url) => origin !== null && new URL(url).origin === origin

/// Note the secrets in a JSON value: the strings under a secret's key, in
/// arrays too, and in strings that are JSON themselves, as an MCP tool's
/// answer holds its text.
export function walk(v, secret = false) {
  if (typeof v === 'string') {
    if (secret) noteSecret(v)
    else if (/^[[{]/.test(v)) for (const x of jsonIn(v)) walk(x)
    else if (/^[a-z][a-z0-9+.-]*:/i.test(v)) noteUrl(v)
  } else if (Array.isArray(v)) for (const x of v) walk(x, secret)
  else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) walk(x, SECRET_KEY.test(k))
}

/// Note the secrets in a request's or an answer's text.
export function noteText(text) {
  for (const v of jsonIn(text)) walk(v)
}

/// The JSON values in `text`: the whole of it, or each `data:` of an event
/// stream.
function jsonIn(text) {
  const out = []
  try {
    out.push(JSON.parse(text))
  } catch {
    for (const line of text.split(/\r?\n/)) {
      if (!line.startsWith('data:')) continue
      try {
        out.push(JSON.parse(line.slice(5)))
      } catch {}
    }
  }
  return out
}

// ------------------------------------------------------------ routes

/// The methods the server serves: those whose lexicons are in spec/lexicons
/// (spec/PDS.md, "What it serves").
export const SERVED = new Set()
for (const file of readdirSync(new URL('../../spec/lexicons', import.meta.url), { recursive: true })) {
  if (!String(file).endsWith('.json')) continue
  const lex = JSON.parse(readFileSync(join(new URL('../../spec/lexicons', import.meta.url).pathname, String(file)), 'utf8'))
  if (['query', 'procedure', 'subscription'].includes(lex.defs?.main?.type)) SERVED.add(lex.id)
}

/// The routes outside /xrpc/ the server serves, as spec/OPERATIONS.md lists them.
export const ROUTES = [
  '/',
  '/mcp',
  '/.well-known/atproto-did',
  '/.well-known/oauth-protected-resource',
  '/.well-known/oauth-authorization-server',
  '/oauth/jwks',
  '/oauth/par',
  '/oauth/token',
  '/oauth/revoke',
  '/oauth/authorize',
  '/oauth/authorize/redirect',
  '/@atproto/oauth-provider/~api/sign-in',
  '/@atproto/oauth-provider/~api/consent',
  '/@atproto/oauth-provider/~api/reject',
  '/@atproto/oauth-provider/~api/sign-out',
  '/@atproto/oauth-provider/~api/passkey-options',
  '/@atproto/oauth-provider/~api/passkey-sign-in',
  '/oauth/passkey',
  '/@atproto/oauth-provider/~api/passkey-register',
  '/.well-known/oauth-protected-resource/mcp',
  '/.well-known/oauth-authorization-server/oauth/mcp',
  '/oauth/mcp/authorize',
  '/oauth/mcp/token',
  '/oauth/mcp/revoke',
]

/// The server's own routes under /xrpc/, of no lexicon: its health, the
/// admin's reads, and the account's OAuth and passkey routes
/// (spec/OPERATIONS.md, "Health", "The admin's reads"; spec/OAUTH-ACCOUNT.md;
/// spec/PASSKEYS.md).
export const OWN_XRPC = ['_health', '_logs', '_stats', '_oauthSessions', '_oauthRevokeSession', '_oauthDevices', '_oauthRevokeDevice', '_passkeyList', '_passkeyAdd', '_passkeyRemove']

/// A path's route, as the server's metrics and log name it
/// (spec/OPERATIONS.md): every path under `/oauth/assets/` is one route,
/// whatever its name.
export function routeOf(path) {
  if (path.startsWith('/xrpc/') && OWN_XRPC.includes(path.slice(6))) return path.slice(6)
  if (path.startsWith('/xrpc/')) return SERVED.has(path.slice(6)) ? path.slice(6) : '_unknown'
  if (path.startsWith('/oauth/assets/')) return ASSETS
  return ROUTES.includes(path) ? path : '_unknown'
}

/// The route of the authorization pages' assets (spec/OAUTH-PAGES.md).
export const ASSETS = '/oauth/assets/{name}'

/// A route's `http.route`, or null for `_unknown`.
export const httpRoute = (route) => (route === '_unknown' ? null : route.startsWith('/') ? route : `/xrpc/${route}`)