chadtmiller.com/monolith
An atproto PDS
0
0
// A compatibility run (compare.mjs --area, which compat.mjs runs): the core
// checks of one area, compared with the reference as compare.mjs compares
// them, but going on past a difference. Each area is a run of its own, on a
// fresh pair of servers, so that what differs in one does not spill into
// another: its own steps, drawn by weight as compare.mjs draws its steps,
// and the steps it needs others for (writes, for the repository, sync and
// the firehose), whose differences are kept apart. A check is one step;
// it goes on past what differs (lib/report.mjs keeps it), is counted
// failed if anything did, and the run goes on. What it found is written to
// `--result` as JSON.
//
// Only the core is here (compat/README.md, the map): the email flows,
// invites, rate limits, the admin lifecycle and proxying are not, nor
// Monolith's own rules (its commands, logs, traces and metrics, store and
// scale, the exact numbers of its limits, its timing floors, and the
// reference's page API).
import { readFileSync, writeFileSync } from 'node:fs'
import { AREA, CHECK_EVERY, FIREHOSE, IMPL, RESULT, SEED, STEPS } from './args.mjs'
import { below, chance, pick } from './random.mjs'
import { fail, plcLog } from './plc.mjs'
import { adapter, cand, ref, shutdown } from './processes.mjs'
import { allowed, count, counts, differ, found, run } from './report.mjs'
import { can, passwordFor } from './server.mjs'
import { accounts, didOf } from './state.mjs'
import { both, shown, xrpc } from './xrpc.mjs'
import { accountRace, createAccount, createAccountRefused, createSession, crossWrites, made, newDetails } from '../checks/accounts.mjs'
import { blobRecord, getBlob, listBlobs, uploadBlob } from '../checks/blobs.mjs'
import { checkCrawls, getBlocks, getGoneBlocks, getRepoStatus, listRepos, noCrawlsYet, syncGetRecord } from '../checks/federation.mjs'
import { checkFirehose, checkReplay, startFirehose } from '../checks/firehose.mjs'
import { handleRace, recommendedCredentials, resolveHandle, updateHandle, updateHandleNoToken, updateHandleRefused, wellKnownHandle } from '../checks/identity.mjs'
import { oddRequests } from '../checks/odd.mjs'
import { checkOAuthMetadata, oauthClientLibrary, oauthDpopRefused, oauthGetSession, oauthLogin, oauthParRefused, oauthRefresh, oauthRevoke, oauthRevokeRefused, oauthTokenRefused } from '../checks/oauth.mjs'
import { oauthScopedCall, oauthScopedEdges, oauthScopedRefused } from '../checks/oauth-scopes.mjs'
import { oauthSetCall, oauthSetLogin, oauthSetPar } from '../checks/permission-sets.mjs'
import { oauthAssertionRefused, oauthClientCache, oauthClientDocument, oauthClientLogin, oauthConfidentialSession } from '../checks/oauth-clients.mjs'
import { applyWrites, concurrent, createRecord, createWithTid, deleteRecord, describeRepo, getRecord, listRecords, putRecord, syncHeads } from '../checks/records.mjs'
import { checkRepo, checkRepos } from '../checks/repository.mjs'
import { describeServer, getSession } from '../checks/session.mjs'
import { appPasswords, deleteSession, refreshSession, sessionRace, wrongTokens } from '../checks/sessions.mjs'
import { duplicates, validatedBatch, validatedElsewhere, validatedWrite } from '../checks/validation.mjs'
/// The record writes the repository, sync and firehose areas need.
const WRITES = [[createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8]]
const newAccount = Object.defineProperty(() => createAccount(), 'name', { value: 'createAccount' })
const firehoseChecks = Object.defineProperty(async () => {
await checkFirehose()
if (chance(0.3)) await checkReplay()
}, 'name', { value: 'checkFirehose' })
const crawlsSoFar = Object.defineProperty(() => checkCrawls(0), 'name', { value: 'checkCrawls' })
/// The one account of a server that hosts one (spec/COMPAT.md, "One
/// account"): the reference makes it with createAccount, the server through
/// its adapter's `claim`, with the same handle and email and the password
/// the server was started with. Their answers are not compared, nor is the
/// server's genesis operation; the account's signing keys, and the server's
/// rotation key, are read from the PLC directory. Then it is checked as a
/// new account is, from getSession on. Answers the account, or null.
async function claimAccount() {
const CREATE = 'com.atproto.server.createAccount'
const { handle, email } = newDetails(true)
const body = { handle, email, password: cand.password }
const [a, b] = await Promise.all([xrpc(ref, CREATE, { body }), cand.claim({ handle, email })])
count('createAccount (the server\'s one account)', a.status === 200 && b.status === 200)
if (a.status !== 200) await differ(CREATE, body, 'an account', a.body, 'the reference refuses an account the harness means to make')
if (b.status !== 200) await differ('the server\'s one account, made by its adapter', { handle, email }, 'an account', b.body, 'the adapter fails its own step')
if (a.status !== 200 || b.status !== 200) return null
const acct = made(body, a, b)
for (const side of [ref, cand]) {
const log = await plcLog(didOf(side, acct))
const op = Array.isArray(log.body) ? log.body[0] : null
if (!op) {
await fail(side, 'PLC log', shown({ body }), 'a genesis operation', log)
continue
}
acct.sides[side.name].key = op.verificationMethods?.atproto
if (side === cand) {
acct.sides[side.name].kem = op.verificationMethods?.atproto_kem
cand.rotationKey ??= op.rotationKeys?.[0]
}
}
await both('getSession of a new account', 'com.atproto.server.getSession', { token: acct })
await syncHeads(acct)
await checkRepo(acct)
return acct
}
/// The areas, in the order the report lists them. Each has `about`, what it
/// checks; `steps`, `[check, weight, needs]`, drawn as compare.mjs draws
/// its steps, `needs` the settings (lib/adapters/, `unsupported`) a
/// check depends on; `support`, `[step, weight, needs]` drawn among them,
/// whose differences are kept apart; `pre`, checks of the area before the
/// steps (and after the first accounts unless `beforeAccounts`); and
/// `every`, checks of the area every `--check-every` steps and at the end.
/// `accounts` is the need of a check that makes an account or needs a
/// second one (spec/COMPAT.md, "One account").
export const AREAS = {
oauth: {
about: 'OAuth, the protocol: the server\'s metadata, PAR, the code exchange with PKCE, DPoP and its nonces, refresh and its rotation, revocation, DPoP-bound calls to XRPC, and a client app\'s own library',
pre: [[checkOAuthMetadata]],
steps: [[oauthLogin, 4], [oauthParRefused, 3], [oauthDpopRefused, 3], [oauthTokenRefused, 2], [oauthRevokeRefused, 1], [oauthRefresh, 3], [oauthRevoke, 2], [oauthGetSession, 2], [oauthClientLibrary, 1, ['address']]],
},
'oauth scopes': {
about: 'OAuth scopes, enforced on every method: calls with an OAuth session\'s token, and with one thing wrong in its proof, token or scheme',
steps: [[oauthScopedCall, 8], [oauthScopedRefused, 4], [oauthScopedEdges, 1]],
},
'oauth permission sets': {
about: 'OAuth permission sets (`include:` scopes), resolved from the harness\'s lexicon authority: PAR with sets that resolve and do not, the page\'s data, consent keeping or dropping each, the token\'s expanded scope, and calls the sets\' grants allow or refuse',
steps: [[oauthSetPar, 2, ['lexiconAuthority']], [oauthSetLogin, 2, ['lexiconAuthority']], [oauthSetCall, 3, ['lexiconAuthority']]],
},
'oauth clients': {
about: 'OAuth clients\' metadata documents: fetched from the harness\'s web, kept and refused as the reference does; confidential clients and their assertions',
steps: [[oauthClientDocument, 4, ['clientFetch']], [oauthClientCache, 1, ['clientFetch']], [oauthClientLogin, 3, ['clientFetch']], [oauthAssertionRefused, 3, ['clientFetch']], [oauthConfidentialSession, 2, ['clientFetch']]],
},
sessions: {
about: 'Sessions: createSession, getSession, refreshSession, app passwords and their scopes, deleteSession, tokens of the wrong kind, and describeServer',
steps: [[createSession, 3], [getSession, 1], [describeServer, 1, ['handleDomains']], [refreshSession, 3], [deleteSession, 2], [wrongTokens, 2], [appPasswords, 4], [sessionRace, 1]],
},
accounts: {
about: 'Accounts: createAccount, the DID it registers with the PLC directory and its genesis operation, and what it refuses',
steps: [[newAccount, 4, ['accounts']], [createAccountRefused, 4, ['accounts']], [accountRace, 1, ['accounts']]],
},
records: {
about: 'Records: createRecord, putRecord, deleteRecord, applyWrites, swapCommit and swapRecord, getRecord, listRecords, describeRepo, writes to another account\'s repository, and requests the lexicons refuse',
steps: [[createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8], [getRecord, 10], [listRecords, 6], [describeRepo, 2], [createWithTid, 2], [concurrent, 6], [oddRequests, 3], [crossWrites, 3, ['accounts']]],
},
validation: {
about: 'Records checked against their lexicons: known types valid and not, `validate`, duplicates replaced, in applyWrites too',
steps: [[validatedWrite, 10], [duplicates, 5], [validatedBatch, 4], [validatedElsewhere, 1]],
},
repository: {
about: 'The repository: each account\'s whole repository from getRepo, verified (signature, MST, every record) by the reference\'s library against the DID\'s key, its MST root the reference\'s, getRepo since, getLatestCommit',
support: WRITES,
pre: [[checkRepos]],
every: [[checkRepos]],
},
sync: {
about: 'Sync: listRepos, getRepoStatus, getBlocks, sync.getRecord with its proof, and requestCrawl',
pre: [[noCrawlsYet, ['hostname'], true], [crawlsSoFar, ['hostname']]],
steps: [[listRepos, 2], [getRepoStatus, 2], [getBlocks, 3], [getGoneBlocks, 2], [syncGetRecord, 3]],
support: WRITES,
every: [[crawlsSoFar, ['hostname']]],
},
firehose: {
about: 'The firehose: each side\'s subscribeRepos from the start, every event checked as a consumer would, in order, with its blocks, and replays from a cursor',
support: [...WRITES, [newAccount, 1, ['accounts']], [updateHandle, 2]],
every: [[firehoseChecks]],
},
identity: {
about: 'Identity: handles, their changes and the PLC operations they make, resolveHandle, /.well-known/atproto-did, getRecommendedDidCredentials, and DID documents',
steps: [[updateHandle, 3], [updateHandleRefused, 3], [updateHandleNoToken, 1], [resolveHandle, 3], [wellKnownHandle, 2], [recommendedCredentials, 1, ['recoveryKey']], [describeRepo, 2], [handleRace, 1, ['accounts']]],
},
blobs: {
about: 'Blobs: uploadBlob, records that reference them, getBlob, listBlobs',
steps: [[uploadBlob, 4], [blobRecord, 5], [getBlob, 3], [listBlobs, 2]],
},
}
/// Why `needs` cannot be met by the server, or null.
const missing = (needs = []) => needs.map((n) => adapter.unsupported[n] && `${n}: ${adapter.unsupported[n]}`).filter(Boolean)
/// What the run found, of its own checks and of its supporting steps: how
/// many ran, differed, and could not go on; the first difference; and by
/// check, the same, with each one's first difference.
const own = { checks: 0, failed: 0, errors: 0, first: null, byCheck: {} }
const support = { checks: 0, failed: 0, errors: 0, first: null, byCheck: {} }
const skipped = {}
/// An exception a check did not expect, as an answer of a shape it cannot
/// read, kept as a difference is.
const exception = (e, name) => ({ name, step: run.step, what: `the check could not go on: ${String(e?.message ?? e).split('\n')[0]}`, note: 'an exception, not a comparison: what the server answered was of a shape the check cannot read, or the harness is wrong', request: null, reference: null, server: String(e?.stack ?? e).split('\n').slice(0, 6).join('\n') })
/// Run `fn` as one check of the area (or a supporting step, `mine` false).
/// It goes on past what differs (lib/report.mjs keeps it): one with a
/// difference is counted failed, and its first difference kept; one that
/// throws, or does not end within 3 minutes, with none, is counted as not
/// going on. Then the run goes on.
async function check(fn, args, mine) {
const name = fn.name || 'anonymous'
const tally = mine ? own : support
const by = (tally.byCheck[name] ??= { checks: 0, failed: 0, errors: 0, first: null, whats: {} })
tally.checks++
by.checks++
found.length = 0
let timer, error = null
try {
await Promise.race([fn(...args), new Promise((_, no) => (timer = setTimeout(() => no(new Error('no answer within 3 minutes')), 180_000)))])
} catch (e) {
error = e
} finally {
clearTimeout(timer)
}
const diffs = found.splice(0)
if (!diffs.length && !error) return
// Each thing that differed, how often, the exception it ended in after
// them too.
const whats = [...diffs.map((d) => d.what), ...(error && diffs.length ? [`then could not go on: ${String(error?.message ?? error).split('\n')[0]}`] : [])]
for (const w of whats) by.whats[w] = (by.whats[w] ?? 0) + 1
const f = diffs.length ? { name, ...diffs[0], ...(diffs.length > 1 ? { more: diffs.length - 1 } : {}) } : exception(error, name)
const kind = diffs.length ? 'failed' : 'errors'
tally[kind]++
by[kind]++
tally.first ??= f
by.first ??= f
console.error(`step ${run.step}: ${name}: ${f.what}${f.note ? ` (${f.note})` : ''}`)
}
/// Each of `list`, `[check, needs, beforeAccounts]`, run unless the server
/// cannot meet its needs, which is noted.
async function runAll(list, mine, when) {
for (const [fn, needs, before] of list ?? []) {
if (Boolean(before) !== when) continue
const why = missing(needs)
if (why.length) skipped[fn.name] = why
else await check(fn, [], mine)
}
}
/// The area's run, and its result written; then the run ends.
export async function areaRun() {
const area = AREAS[AREA]
if (!area) {
console.error(`no area ${AREA}; there are ${Object.keys(AREAS).join(', ')}`)
await shutdown(2)
}
const drawn = []
const drawable = (list, mine) => {
for (const [fn, w, needs] of list ?? []) {
const why = missing(needs)
if (why.length) skipped[fn.name] = why
else drawn.push([fn, w, mine])
}
}
drawable(area.steps, true)
drawable(area.support, false)
if (FIREHOSE) await check(startFirehose, [], true)
await runAll(area.pre, true, true)
// The first accounts, as compare.mjs makes them, each a supporting step;
// as many more as it takes (up to ten tries), when the server refuses
// one the reference makes, which is then not kept. A server that hosts
// one account has that one, made once, through its adapter.
if (!can('accounts')) {
await check(Object.defineProperty(() => claimAccount(), 'name', { value: 'the server\'s one account' }), [], false)
} else {
const first = Object.defineProperty(() => createAccount({ withPassword: true, password: passwordFor }), 'name', { value: 'createAccount (a first account)' })
const want = 2 + below(2)
for (let tries = 0; accounts.length < want && tries < 10; tries++) await check(first, [], false)
}
for (const acct of accounts) await check(describeRepo, [acct], false)
await runAll(area.pre, true, false)
const total = drawn.reduce((n, [, w]) => n + w, 0)
for (run.step = 1; run.step <= STEPS; run.step++) {
if (drawn.length && accounts.length) {
let r = below(total)
for (const [fn, w, mine] of drawn) {
if ((r -= w) < 0) {
await check(fn, [pick(accounts)], mine)
break
}
}
}
if (run.step % CHECK_EVERY === 0) await runAll(area.every, true, false)
if (run.step % 100 === 0) console.error(`${run.step} steps`)
}
run.step = STEPS
await runAll(area.every, true, false)
const pkg = (name) => JSON.parse(readFileSync(new URL(`../node_modules/${name}/package.json`, import.meta.url), 'utf8')).version
const result = {
impl: IMPL,
area: AREA,
about: area.about,
seed: SEED,
steps: STEPS,
reference: `@atproto/pds ${pkg('@atproto/pds')} (through @atproto/dev-env ${pkg('@atproto/dev-env')})`,
server: await adapter.info(),
own,
support,
skipped,
unsupported: adapter.unsupported,
allowed,
accounts: accounts.length,
requests: Object.values(counts).reduce((n, { ok, error }) => n + ok + error, 0),
}
// A difference may hold what a firehose frame decodes to: BigInts and bytes.
const plain = (k, v) => (typeof v === 'bigint' ? `${v}` : v instanceof Uint8Array ? `<${v.length} bytes>` : v)
writeFileSync(RESULT, `${JSON.stringify(result, plain, 2)}\n`)
console.error(`\n${AREA}: ${own.checks - own.failed - own.errors} of ${own.checks} checks agree, ${own.failed} differ, ${own.errors} could not go on; supporting steps: ${support.failed + support.errors} of ${support.checks} failed`)
await shutdown(0)
}