chadtmiller.com/monolith
An atproto PDS
0
0
// The server's log (spec/OPERATIONS.md), checked at the end of every run:
// each line one JSON object with the fields the spec gives; each start's
// lines as the spec orders them, with `recovered` after a start the last
// server did not shut down; a line for every request the harness sent, with
// that request's trace, route, status and a time no longer than the harness
// waited; nothing on standard error but why a start exited 1; and none of
// the run's secrets anywhere the server wrote: standard output, standard
// error, and the files it kept its log in (`PDS_LOG_DIR`).
import { readFileSync } from 'node:fs'
import { server } from '../lib/processes.mjs'
import { OTLP_KEY } from '../lib/adapters/monolith.mjs'
import { LEVELS, keptFiles } from '../lib/admin.mjs'
import { now, secrets, sent, settled, tokenDids } from '../lib/observe.mjs'
import { count, differ } from '../lib/report.mjs'
import { accounts } from '../lib/state.mjs'
const TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
const TRACE_ID = /^(?!0{32})[0-9a-f]{32}$/
/// The methods that take a token and act as its account, whose lines must
/// name it.
const AUTHED = new Set([
'com.atproto.repo.createRecord', 'com.atproto.repo.putRecord', 'com.atproto.repo.deleteRecord', 'com.atproto.repo.applyWrites',
'com.atproto.repo.uploadBlob', 'com.atproto.server.getSession', 'com.atproto.server.refreshSession', 'com.atproto.server.deleteSession',
'com.atproto.server.createAppPassword', 'com.atproto.server.listAppPasswords', 'com.atproto.server.revokeAppPassword',
'com.atproto.server.getAccountInviteCodes', 'com.atproto.identity.updateHandle', 'com.atproto.identity.getRecommendedDidCredentials',
])
const GET_BLOB = 'com.atproto.sync.getBlob'
/// The server failing spec/OPERATIONS.md.
export const broke = (what, detail, want, got) => differ(`the server's ${what}`, detail, want, got, 'spec/OPERATIONS.md')
/// The least level a start of the server writes.
export const levelOf = (life) => (life.env.PDS_LOG_LEVEL || 'info')
const writes = (life, level) => levelOf(life) !== 'off' && LEVELS.indexOf(level) >= LEVELS.indexOf(levelOf(life))
const slice = (buf, from, to) => buf.subarray(from, to).toString('utf8')
/// The server's log as it is now, by start: each of `server.lives` with
/// `lines`, `{text, obj}` (`obj` null if the line is no JSON object), and
/// `stderr`, what it wrote to standard error.
export function readLog() {
const out = readFileSync(server.logPath)
const err = readFileSync(server.errPath)
return server.lives.map((life, i) => {
const next = server.lives[i + 1]
const text = slice(out, life.outFrom, next ? next.outFrom : out.length)
const pieces = text.split('\n')
const tail = pieces.pop()
const lines = pieces.map((t) => {
let obj = null
try {
const v = JSON.parse(t)
if (v && typeof v === 'object' && !Array.isArray(v)) obj = v
} catch {}
return { text: t, obj }
})
if (tail) lines.push({ text: tail, obj: null, unended: true })
return { ...life, n: i, lines, stderr: slice(err, life.errFrom, next ? next.errFrom : err.length) }
})
}
/// Hold each line to the spec's fields, as the start that wrote it was
/// configured (spec/OPERATIONS.md, "The log", "Every line", "Events").
async function checkLines(lives) {
for (const life of lives) {
for (const [i, { text, obj, unended }] of life.lines.entries()) {
const at = { start: life.n + 1, line: i + 1, text: text.slice(0, 300) }
if (unended) await broke('log', at, 'every line ended by \\n', 'a line cut short')
if (!obj) await broke('log', at, 'every line one JSON object', 'not JSON')
const bad = (want) => broke('log', at, want, obj)
if (typeof obj.time !== 'string' || !TIME.test(obj.time)) await bad('`time`, RFC 3339 in UTC with milliseconds')
const t = Date.parse(obj.time)
if (t < life.started - 1000 || t > (life.ended ?? now()) + 1000) await bad(`a \`time\` while the server ran, ${new Date(life.started).toISOString()} to ${new Date(life.ended ?? now()).toISOString()}`)
if (!LEVELS.includes(obj.level)) await bad(`\`level\` one of ${LEVELS.join(', ')}`)
if (!writes(life, obj.level)) await bad(`no line below PDS_LOG_LEVEL ${levelOf(life)}`)
if (typeof obj.msg !== 'string' || !obj.msg) await bad('a `msg`')
if (Object.values(obj).some((v) => v === null)) await bad('no field null')
const int = (v) => Number.isInteger(v)
if (obj.msg === 'request') {
if (typeof obj.method !== 'string' || !int(obj.status) || obj.status < 100 || obj.status > 599) await bad('`method`, a string, and `status`, an HTTP status')
if (typeof obj.ms !== 'number' || !(obj.ms >= 0) || !Number.isFinite(obj.ms)) await bad('`ms`, a number of 0 or more')
if (typeof obj.trace_id !== 'string' || !TRACE_ID.test(obj.trace_id)) await bad('`trace_id`, 32 lowercase hex digits, not all 0')
if ('did' in obj && (typeof obj.did !== 'string' || !obj.did.startsWith('did:'))) await bad('`did` a DID, when there is one')
if (obj.method === GET_BLOB) {
if ('repo' in obj && (typeof obj.repo !== 'string' || !obj.repo.startsWith('did:') || typeof obj.cid !== 'string' || !obj.cid)) await bad('`repo` a DID and `cid` a CID, together')
if ('cid' in obj && !('repo' in obj)) await bad('`cid` only with `repo`')
if ('bytes' in obj && (!int(obj.bytes) || obj.bytes < 0 || !('repo' in obj) || obj.status !== 200)) await bad('`bytes`, a number of 0 or more, with `repo` and `cid`, of a 200')
} else if (['repo', 'cid', 'bytes'].some((k) => k in obj)) await bad('no `repo`, `cid` or `bytes` but of getBlob')
if (obj.level !== (obj.status >= 500 ? 'error' : 'info')) await bad('`level` info, or error for a status of 500 or more')
} else if (obj.msg === 'start') {
if (obj.level !== 'info' || typeof obj.version !== 'string' || !obj.version || obj.port !== Number(life.env.PDS_PORT)) await bad(`level info, a \`version\`, and \`port\` ${life.env.PDS_PORT}`)
const mp = life.env.PDS_METRICS_PORT ? Number(life.env.PDS_METRICS_PORT) : undefined
if (obj.metrics_port !== mp) await bad(mp ? `\`metrics_port\` ${mp}` : 'no `metrics_port`, as PDS_METRICS_PORT is unset')
if (!['together', 'each'].includes(obj.sync) || (life.env.PDS_SYNC && obj.sync !== life.env.PDS_SYNC)) await bad(`\`sync\` ${life.env.PDS_SYNC ?? 'together or each'}`)
} else if (obj.msg === 'recovered') {
if (obj.level !== 'warn' || !int(obj.dropped_bytes) || obj.dropped_bytes < 0) await bad('level warn and `dropped_bytes`, a number of 0 or more')
} else if (obj.msg === 'shutdown') {
if (obj.level !== 'info' || !['SIGTERM', 'SIGINT'].includes(obj.signal)) await bad('level info and `signal` SIGTERM or SIGINT')
} else if (obj.msg === 'plc failure') {
if (obj.level !== 'warn' || typeof obj.error !== 'string' || ('status' in obj && !int(obj.status))) await bad('level warn, an `error`, and `status` a number if any')
} else if (obj.msg === 'trace export failed') {
if (obj.level !== 'warn' || typeof obj.error !== 'string') await bad('level warn and an `error`')
}
}
}
}
/// Each start's lines in the spec's order: none from a start that exited 1
/// on its environment; `start` first; `recovered` next if, and only if, the
/// last start that ran did not shut down; `shutdown` last after a SIGTERM
/// it exited 0 on (spec/OPERATIONS.md, "Events").
async function checkStarts(lives) {
let previous = null
for (const life of lives) {
const msgs = life.lines.map((l) => l.obj?.msg)
const at = { start: life.n + 1, env: { PDS_LOG_LEVEL: life.env.PDS_LOG_LEVEL } }
if (life.exited?.code === 1 && life.how === 'exit' && !msgs.includes('start')) {
if (msgs.length) await broke('log', at, 'nothing on standard output from a start that exits 1 on its environment', life.lines.slice(0, 3).map((l) => l.text))
continue
}
if (writes(life, 'info') && msgs[0] !== 'start') await broke('log', at, 'a `start` line first', life.lines.slice(0, 2).map((l) => l.text))
if (msgs.filter((m) => m === 'start').length > 1) await broke('log', at, 'one `start` line', msgs.filter((m) => m === 'start').length)
const unclean = previous && !(previous.how === 'SIGTERM' && previous.exited?.code === 0)
const recovered = msgs.indexOf('recovered')
if (writes(life, 'warn')) {
if (unclean && recovered === -1) await broke('log', { ...at, last: previous.how }, 'a `recovered` line: the last server did not shut down', msgs.slice(0, 3))
if (!unclean && recovered !== -1) await broke('log', { ...at, last: previous ? `exited ${JSON.stringify(previous.exited)} on ${previous.how}` : 'none, the directory was empty' }, 'no `recovered` line', life.lines[recovered].text)
if (recovered !== -1 && msgs.slice(0, recovered).some((m) => m === 'request')) await broke('log', at, '`recovered` before any request', msgs.slice(0, recovered + 1))
}
if (life.how === 'SIGTERM' && life.exited?.code === 0 && writes(life, 'info') && msgs.at(-1) !== 'shutdown') await broke('log', at, 'a `shutdown` line last', life.lines.slice(-2).map((l) => l.text))
previous = life
}
}
/// A line for every request the harness sent and had answered, found by
/// its trace id (spec/OPERATIONS.md, "Events").
async function checkRequests(lives) {
await settled()
const byTrace = new Map()
for (const life of lives) for (const { obj } of life.lines) if (obj?.msg === 'request') byTrace.set(obj.trace_id, [...(byTrace.get(obj.trace_id) ?? []), obj])
let checked = 0
for (const rec of sent) {
if (!rec.status || !rec.traceId || rec.ended === null) continue
const life = lives[rec.life]
const level = rec.status >= 500 ? 'error' : 'info'
// A body cut short once the server it came from was signalled to stop,
// as when it was killed, may have no line: the server writes one as the
// body ends, and a server killed writes nothing more.
const stopped = life.signalled ?? life.ended
if (rec.short && stopped != null && rec.ended >= stopped) continue
// The request may have reached the next start, if the one it was sent
// to was killed first: then its answer came after that one exited.
const next = life.ended !== null && rec.ended > life.ended ? lives[rec.life + 1] : undefined
if (!writes(life, level) || (next && !writes(next, level))) continue
const request = { method: rec.method, path: rec.path, traceparent: `00-${rec.traceId}-${rec.spanId}-${rec.sampled ? '01' : '00'}`, status: rec.status }
const lines = byTrace.get(rec.traceId) ?? []
if (lines.length !== 1) await broke('log', request, 'one `request` line with the request\'s trace id', lines.length ? lines : 'none')
const [line] = lines
const took = rec.ended - rec.started
if (line.method !== rec.route || line.status !== rec.status) await broke('log', request, { method: rec.route, status: rec.status }, line)
if (line.ms > took + 20) await broke('log', request, `\`ms\` no more than the ${took.toFixed(1)} ms the harness waited for the whole answer`, line)
const did = rec.status < 300 && AUTHED.has(rec.route) ? tokenDids.get(rec.token) : undefined
if (did && line.did !== did) await broke('log', request, { did }, line)
if (rec.route === GET_BLOB) await blobFields(rec, request, line)
checked += 1
}
count('log lines of the harness\'s requests', checked > 0)
}
/// A getBlob's `repo`, `cid` and `bytes`: of the blob it asked for, when
/// answered 200 or 500, and of a 200 the bytes the harness read, or, of an
/// answer that ended short, at least those and fewer than it said it had
/// (spec/OPERATIONS.md, "Events").
async function blobFields(rec, request, line) {
const got = { repo: line.repo, cid: line.cid, bytes: line.bytes }
const want = {}
if (rec.status === 200 || rec.status === 500) Object.assign(want, { repo: rec.params.did, cid: rec.params.cid })
if (rec.status === 200) {
if (rec.bytes === undefined) want.bytes = Number.isInteger(line.bytes) ? line.bytes : 'a number'
else if (rec.short) {
if (!(line.bytes >= rec.bytes && line.bytes < rec.length)) await broke('log', request, `\`bytes\` no fewer than the ${rec.bytes} the harness read of an answer that ended short, and fewer than its Content-Length ${rec.length}`, got)
want.bytes = line.bytes
} else want.bytes = rec.bytes
}
if (got.repo !== want.repo || got.cid !== want.cid || got.bytes !== want.bytes) await broke('log', request, want, got)
}
/// Nothing on standard error, but from a start that exited 1
/// (spec/OPERATIONS.md, "The log").
async function checkStderr(lives) {
for (const life of lives) {
if (life.exited?.code === 1 && life.how === 'exit') continue
if (life.stderr) await broke('standard error', { start: life.n + 1, ended: life.how ?? 'running' }, 'nothing: standard error is only for why `pds serve` exits 1', life.stderr.slice(0, 500))
}
}
/// The length every secret has at least (`noteSecret`): one in a text
/// starts with as many characters that some secret starts with.
const SECRET_START = 8
/// What finds a secret of `all` in a text, or null: in one pass over the
/// text, looking up the characters at each place among the secrets' starts,
/// rather than a search of the text for each secret.
function secretFinder(all) {
const byStart = new Map()
const short = []
for (const s of all) {
if (s.length < SECRET_START) short.push(s)
else if (byStart.has(s.slice(0, SECRET_START))) byStart.get(s.slice(0, SECRET_START)).push(s)
else byStart.set(s.slice(0, SECRET_START), [s])
}
return (text) => {
for (let i = 0; i + SECRET_START <= text.length; i++) {
const found = byStart.get(text.slice(i, i + SECRET_START))?.find((s) => text.startsWith(s, i))
if (found) return found
}
return short.find((s) => text.includes(s)) ?? null
}
}
/// None of the run's secrets, nor any account's password or email, nor a
/// key the server was given, nor the text of a record the harness wrote,
/// anywhere the server wrote: its lines, its standard error, and the files
/// in each `PDS_LOG_DIR` it was given (spec/OPERATIONS.md, "What is never
/// logged", "Kept").
async function checkSecrets(lives) {
const all = new Set(secrets)
for (const a of accounts) for (const s of [a.password, a.email]) if (typeof s === 'string' && s.length >= 8) all.add(s)
for (const life of lives) for (const k of ['PDS_PLC_ROTATION_KEY_HEX', 'PDS_ADMIN_PASSWORD']) if (life.env[k]?.length >= 8) all.add(life.env[k])
all.add(OTLP_KEY)
const find = secretFinder(all)
const named = (s) => `${s.slice(0, 4)}… (${s.length} characters)`
for (const life of lives) {
for (const { text } of life.lines) {
const s = find(text)
if (s !== null) await broke('log', { start: life.n + 1, secret: named(s) }, 'no secret in any line', text.slice(0, 300))
}
const s = find(life.stderr)
if (s !== null) await broke('standard error', { start: life.n + 1, secret: named(s) }, 'no secret', life.stderr.slice(0, 300))
}
for (const dir of new Set(lives.map((l) => l.env.PDS_LOG_DIR).filter(Boolean))) {
for (const { path, text } of keptFiles(dir)) {
const s = find(text)
if (s !== null) await broke('kept log', { PDS_LOG_DIR: dir, file: path, secret: named(s) }, 'no secret in any file', text.split('\n').find((l) => l.includes(s))?.slice(0, 300) ?? '')
}
}
count(`secrets not logged (${all.size})`, true)
}
/// The whole log, as it is now.
export async function checkLog() {
const lives = readLog()
await checkLines(lives)
await checkStarts(lives)
await checkRequests(lives)
await checkStderr(lives)
await checkSecrets(lives)
count('log lines', lives.reduce((n, l) => n + l.lines.length, 0) > 0)
}