~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
260 lines · 15.5 KB Raw History
// The server's log (spec/OPERATIONS.md), checked at the end of every run:
// each line one JSON object with the fields the spec gives; each start's
// lines as the spec orders them, with `recovered` after a start the last
// server did not shut down; a line for every request the harness sent, with
// that request's trace, route, status and a time no longer than the harness
// waited; nothing on standard error but why a start exited 1; and none of
// the run's secrets anywhere the server wrote: standard output, standard
// error, and the files it kept its log in (`PDS_LOG_DIR`).

import { readFileSync } from 'node:fs'
import { server } from '../lib/processes.mjs'
import { OTLP_KEY } from '../lib/adapters/monolith.mjs'
import { LEVELS, keptFiles } from '../lib/admin.mjs'
import { now, secrets, sent, settled, tokenDids } from '../lib/observe.mjs'
import { count, differ } from '../lib/report.mjs'
import { accounts } from '../lib/state.mjs'

const TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
const TRACE_ID = /^(?!0{32})[0-9a-f]{32}$/
/// The methods that take a token and act as its account, whose lines must
/// name it.
const AUTHED = new Set([
  'com.atproto.repo.createRecord', 'com.atproto.repo.putRecord', 'com.atproto.repo.deleteRecord', 'com.atproto.repo.applyWrites',
  'com.atproto.repo.uploadBlob', 'com.atproto.server.getSession', 'com.atproto.server.refreshSession', 'com.atproto.server.deleteSession',
  'com.atproto.server.createAppPassword', 'com.atproto.server.listAppPasswords', 'com.atproto.server.revokeAppPassword',
  'com.atproto.server.getAccountInviteCodes', 'com.atproto.identity.updateHandle', 'com.atproto.identity.getRecommendedDidCredentials',
])
const GET_BLOB = 'com.atproto.sync.getBlob'

/// The server failing spec/OPERATIONS.md.
export const broke = (what, detail, want, got) => differ(`the server's ${what}`, detail, want, got, 'spec/OPERATIONS.md')

/// The least level a start of the server writes.
export const levelOf = (life) => (life.env.PDS_LOG_LEVEL || 'info')
const writes = (life, level) => levelOf(life) !== 'off' && LEVELS.indexOf(level) >= LEVELS.indexOf(levelOf(life))

const slice = (buf, from, to) => buf.subarray(from, to).toString('utf8')

/// The server's log as it is now, by start: each of `server.lives` with
/// `lines`, `{text, obj}` (`obj` null if the line is no JSON object), and
/// `stderr`, what it wrote to standard error.
export function readLog() {
  const out = readFileSync(server.logPath)
  const err = readFileSync(server.errPath)
  return server.lives.map((life, i) => {
    const next = server.lives[i + 1]
    const text = slice(out, life.outFrom, next ? next.outFrom : out.length)
    const pieces = text.split('\n')
    const tail = pieces.pop()
    const lines = pieces.map((t) => {
      let obj = null
      try {
        const v = JSON.parse(t)
        if (v && typeof v === 'object' && !Array.isArray(v)) obj = v
      } catch {}
      return { text: t, obj }
    })
    if (tail) lines.push({ text: tail, obj: null, unended: true })
    return { ...life, n: i, lines, stderr: slice(err, life.errFrom, next ? next.errFrom : err.length) }
  })
}

/// Hold each line to the spec's fields, as the start that wrote it was
/// configured (spec/OPERATIONS.md, "The log", "Every line", "Events").
async function checkLines(lives) {
  for (const life of lives) {
    for (const [i, { text, obj, unended }] of life.lines.entries()) {
      const at = { start: life.n + 1, line: i + 1, text: text.slice(0, 300) }
      if (unended) await broke('log', at, 'every line ended by \\n', 'a line cut short')
      if (!obj) await broke('log', at, 'every line one JSON object', 'not JSON')
      const bad = (want) => broke('log', at, want, obj)
      if (typeof obj.time !== 'string' || !TIME.test(obj.time)) await bad('`time`, RFC 3339 in UTC with milliseconds')
      const t = Date.parse(obj.time)
      if (t < life.started - 1000 || t > (life.ended ?? now()) + 1000) await bad(`a \`time\` while the server ran, ${new Date(life.started).toISOString()} to ${new Date(life.ended ?? now()).toISOString()}`)
      if (!LEVELS.includes(obj.level)) await bad(`\`level\` one of ${LEVELS.join(', ')}`)
      if (!writes(life, obj.level)) await bad(`no line below PDS_LOG_LEVEL ${levelOf(life)}`)
      if (typeof obj.msg !== 'string' || !obj.msg) await bad('a `msg`')
      if (Object.values(obj).some((v) => v === null)) await bad('no field null')
      const int = (v) => Number.isInteger(v)
      if (obj.msg === 'request') {
        if (typeof obj.method !== 'string' || !int(obj.status) || obj.status < 100 || obj.status > 599) await bad('`method`, a string, and `status`, an HTTP status')
        if (typeof obj.ms !== 'number' || !(obj.ms >= 0) || !Number.isFinite(obj.ms)) await bad('`ms`, a number of 0 or more')
        if (typeof obj.trace_id !== 'string' || !TRACE_ID.test(obj.trace_id)) await bad('`trace_id`, 32 lowercase hex digits, not all 0')
        if ('did' in obj && (typeof obj.did !== 'string' || !obj.did.startsWith('did:'))) await bad('`did` a DID, when there is one')
        if (obj.method === GET_BLOB) {
          if ('repo' in obj && (typeof obj.repo !== 'string' || !obj.repo.startsWith('did:') || typeof obj.cid !== 'string' || !obj.cid)) await bad('`repo` a DID and `cid` a CID, together')
          if ('cid' in obj && !('repo' in obj)) await bad('`cid` only with `repo`')
          if ('bytes' in obj && (!int(obj.bytes) || obj.bytes < 0 || !('repo' in obj) || obj.status !== 200)) await bad('`bytes`, a number of 0 or more, with `repo` and `cid`, of a 200')
        } else if (['repo', 'cid', 'bytes'].some((k) => k in obj)) await bad('no `repo`, `cid` or `bytes` but of getBlob')
        if (obj.level !== (obj.status >= 500 ? 'error' : 'info')) await bad('`level` info, or error for a status of 500 or more')
      } else if (obj.msg === 'start') {
        if (obj.level !== 'info' || typeof obj.version !== 'string' || !obj.version || obj.port !== Number(life.env.PDS_PORT)) await bad(`level info, a \`version\`, and \`port\` ${life.env.PDS_PORT}`)
        const mp = life.env.PDS_METRICS_PORT ? Number(life.env.PDS_METRICS_PORT) : undefined
        if (obj.metrics_port !== mp) await bad(mp ? `\`metrics_port\` ${mp}` : 'no `metrics_port`, as PDS_METRICS_PORT is unset')
        if (!['together', 'each'].includes(obj.sync) || (life.env.PDS_SYNC && obj.sync !== life.env.PDS_SYNC)) await bad(`\`sync\` ${life.env.PDS_SYNC ?? 'together or each'}`)
      } else if (obj.msg === 'recovered') {
        if (obj.level !== 'warn' || !int(obj.dropped_bytes) || obj.dropped_bytes < 0) await bad('level warn and `dropped_bytes`, a number of 0 or more')
      } else if (obj.msg === 'shutdown') {
        if (obj.level !== 'info' || !['SIGTERM', 'SIGINT'].includes(obj.signal)) await bad('level info and `signal` SIGTERM or SIGINT')
      } else if (obj.msg === 'plc failure') {
        if (obj.level !== 'warn' || typeof obj.error !== 'string' || ('status' in obj && !int(obj.status))) await bad('level warn, an `error`, and `status` a number if any')
      } else if (obj.msg === 'trace export failed') {
        if (obj.level !== 'warn' || typeof obj.error !== 'string') await bad('level warn and an `error`')
      }
    }
  }
}

/// Each start's lines in the spec's order: none from a start that exited 1
/// on its environment; `start` first; `recovered` next if, and only if, the
/// last start that ran did not shut down; `shutdown` last after a SIGTERM
/// it exited 0 on (spec/OPERATIONS.md, "Events").
async function checkStarts(lives) {
  let previous = null
  for (const life of lives) {
    const msgs = life.lines.map((l) => l.obj?.msg)
    const at = { start: life.n + 1, env: { PDS_LOG_LEVEL: life.env.PDS_LOG_LEVEL } }
    if (life.exited?.code === 1 && life.how === 'exit' && !msgs.includes('start')) {
      if (msgs.length) await broke('log', at, 'nothing on standard output from a start that exits 1 on its environment', life.lines.slice(0, 3).map((l) => l.text))
      continue
    }
    if (writes(life, 'info') && msgs[0] !== 'start') await broke('log', at, 'a `start` line first', life.lines.slice(0, 2).map((l) => l.text))
    if (msgs.filter((m) => m === 'start').length > 1) await broke('log', at, 'one `start` line', msgs.filter((m) => m === 'start').length)
    const unclean = previous && !(previous.how === 'SIGTERM' && previous.exited?.code === 0)
    const recovered = msgs.indexOf('recovered')
    if (writes(life, 'warn')) {
      if (unclean && recovered === -1) await broke('log', { ...at, last: previous.how }, 'a `recovered` line: the last server did not shut down', msgs.slice(0, 3))
      if (!unclean && recovered !== -1) await broke('log', { ...at, last: previous ? `exited ${JSON.stringify(previous.exited)} on ${previous.how}` : 'none, the directory was empty' }, 'no `recovered` line', life.lines[recovered].text)
      if (recovered !== -1 && msgs.slice(0, recovered).some((m) => m === 'request')) await broke('log', at, '`recovered` before any request', msgs.slice(0, recovered + 1))
    }
    if (life.how === 'SIGTERM' && life.exited?.code === 0 && writes(life, 'info') && msgs.at(-1) !== 'shutdown') await broke('log', at, 'a `shutdown` line last', life.lines.slice(-2).map((l) => l.text))
    previous = life
  }
}

/// A line for every request the harness sent and had answered, found by
/// its trace id (spec/OPERATIONS.md, "Events").
async function checkRequests(lives) {
  await settled()
  const byTrace = new Map()
  for (const life of lives) for (const { obj } of life.lines) if (obj?.msg === 'request') byTrace.set(obj.trace_id, [...(byTrace.get(obj.trace_id) ?? []), obj])
  let checked = 0
  for (const rec of sent) {
    if (!rec.status || !rec.traceId || rec.ended === null) continue
    const life = lives[rec.life]
    const level = rec.status >= 500 ? 'error' : 'info'
    // A body cut short once the server it came from was signalled to stop,
    // as when it was killed, may have no line: the server writes one as the
    // body ends, and a server killed writes nothing more.
    const stopped = life.signalled ?? life.ended
    if (rec.short && stopped != null && rec.ended >= stopped) continue
    // The request may have reached the next start, if the one it was sent
    // to was killed first: then its answer came after that one exited.
    const next = life.ended !== null && rec.ended > life.ended ? lives[rec.life + 1] : undefined
    if (!writes(life, level) || (next && !writes(next, level))) continue
    const request = { method: rec.method, path: rec.path, traceparent: `00-${rec.traceId}-${rec.spanId}-${rec.sampled ? '01' : '00'}`, status: rec.status }
    const lines = byTrace.get(rec.traceId) ?? []
    if (lines.length !== 1) await broke('log', request, 'one `request` line with the request\'s trace id', lines.length ? lines : 'none')
    const [line] = lines
    const took = rec.ended - rec.started
    if (line.method !== rec.route || line.status !== rec.status) await broke('log', request, { method: rec.route, status: rec.status }, line)
    if (line.ms > took + 20) await broke('log', request, `\`ms\` no more than the ${took.toFixed(1)} ms the harness waited for the whole answer`, line)
    const did = rec.status < 300 && AUTHED.has(rec.route) ? tokenDids.get(rec.token) : undefined
    if (did && line.did !== did) await broke('log', request, { did }, line)
    if (rec.route === GET_BLOB) await blobFields(rec, request, line)
    checked += 1
  }
  count('log lines of the harness\'s requests', checked > 0)
}

/// A getBlob's `repo`, `cid` and `bytes`: of the blob it asked for, when
/// answered 200 or 500, and of a 200 the bytes the harness read, or, of an
/// answer that ended short, at least those and fewer than it said it had
/// (spec/OPERATIONS.md, "Events").
async function blobFields(rec, request, line) {
  const got = { repo: line.repo, cid: line.cid, bytes: line.bytes }
  const want = {}
  if (rec.status === 200 || rec.status === 500) Object.assign(want, { repo: rec.params.did, cid: rec.params.cid })
  if (rec.status === 200) {
    if (rec.bytes === undefined) want.bytes = Number.isInteger(line.bytes) ? line.bytes : 'a number'
    else if (rec.short) {
      if (!(line.bytes >= rec.bytes && line.bytes < rec.length)) await broke('log', request, `\`bytes\` no fewer than the ${rec.bytes} the harness read of an answer that ended short, and fewer than its Content-Length ${rec.length}`, got)
      want.bytes = line.bytes
    } else want.bytes = rec.bytes
  }
  if (got.repo !== want.repo || got.cid !== want.cid || got.bytes !== want.bytes) await broke('log', request, want, got)
}

/// Nothing on standard error, but from a start that exited 1
/// (spec/OPERATIONS.md, "The log").
async function checkStderr(lives) {
  for (const life of lives) {
    if (life.exited?.code === 1 && life.how === 'exit') continue
    if (life.stderr) await broke('standard error', { start: life.n + 1, ended: life.how ?? 'running' }, 'nothing: standard error is only for why `pds serve` exits 1', life.stderr.slice(0, 500))
  }
}

/// The length every secret has at least (`noteSecret`): one in a text
/// starts with as many characters that some secret starts with.
const SECRET_START = 8

/// What finds a secret of `all` in a text, or null: in one pass over the
/// text, looking up the characters at each place among the secrets' starts,
/// rather than a search of the text for each secret.
function secretFinder(all) {
  const byStart = new Map()
  const short = []
  for (const s of all) {
    if (s.length < SECRET_START) short.push(s)
    else if (byStart.has(s.slice(0, SECRET_START))) byStart.get(s.slice(0, SECRET_START)).push(s)
    else byStart.set(s.slice(0, SECRET_START), [s])
  }
  return (text) => {
    for (let i = 0; i + SECRET_START <= text.length; i++) {
      const found = byStart.get(text.slice(i, i + SECRET_START))?.find((s) => text.startsWith(s, i))
      if (found) return found
    }
    return short.find((s) => text.includes(s)) ?? null
  }
}

/// None of the run's secrets, nor any account's password or email, nor a
/// key the server was given, nor the text of a record the harness wrote,
/// anywhere the server wrote: its lines, its standard error, and the files
/// in each `PDS_LOG_DIR` it was given (spec/OPERATIONS.md, "What is never
/// logged", "Kept").
async function checkSecrets(lives) {
  const all = new Set(secrets)
  for (const a of accounts) for (const s of [a.password, a.email]) if (typeof s === 'string' && s.length >= 8) all.add(s)
  for (const life of lives) for (const k of ['PDS_PLC_ROTATION_KEY_HEX', 'PDS_ADMIN_PASSWORD']) if (life.env[k]?.length >= 8) all.add(life.env[k])
  all.add(OTLP_KEY)
  const find = secretFinder(all)
  const named = (s) => `${s.slice(0, 4)}… (${s.length} characters)`
  for (const life of lives) {
    for (const { text } of life.lines) {
      const s = find(text)
      if (s !== null) await broke('log', { start: life.n + 1, secret: named(s) }, 'no secret in any line', text.slice(0, 300))
    }
    const s = find(life.stderr)
    if (s !== null) await broke('standard error', { start: life.n + 1, secret: named(s) }, 'no secret', life.stderr.slice(0, 300))
  }
  for (const dir of new Set(lives.map((l) => l.env.PDS_LOG_DIR).filter(Boolean))) {
    for (const { path, text } of keptFiles(dir)) {
      const s = find(text)
      if (s !== null) await broke('kept log', { PDS_LOG_DIR: dir, file: path, secret: named(s) }, 'no secret in any file', text.split('\n').find((l) => l.includes(s))?.slice(0, 300) ?? '')
    }
  }
  count(`secrets not logged (${all.size})`, true)
}

/// The whole log, as it is now.
export async function checkLog() {
  const lives = readLog()
  await checkLines(lives)
  await checkStarts(lives)
  await checkRequests(lives)
  await checkStderr(lives)
  await checkSecrets(lives)
  count('log lines', lives.reduce((n, l) => n + l.lines.length, 0) > 0)
}