// A compatibility run (compare.mjs --area, which compat.mjs runs): the core // checks of one area, compared with the reference as compare.mjs compares // them, but going on past a difference. Each area is a run of its own, on a // fresh pair of servers, so that what differs in one does not spill into // another: its own steps, drawn by weight as compare.mjs draws its steps, // and the steps it needs others for (writes, for the repository, sync and // the firehose), whose differences are kept apart. A check is one step; // it goes on past what differs (lib/report.mjs keeps it), is counted // failed if anything did, and the run goes on. What it found is written to // `--result` as JSON. // // Only the core is here (compat/README.md, the map): the email flows, // invites, rate limits, the admin lifecycle and proxying are not, nor // Monolith's own rules (its commands, logs, traces and metrics, store and // scale, the exact numbers of its limits, its timing floors, and the // reference's page API). import { readFileSync, writeFileSync } from 'node:fs' import { AREA, CHECK_EVERY, FIREHOSE, IMPL, RESULT, SEED, STEPS } from './args.mjs' import { below, chance, pick } from './random.mjs' import { fail, plcLog } from './plc.mjs' import { adapter, cand, ref, shutdown } from './processes.mjs' import { allowed, count, counts, differ, found, run } from './report.mjs' import { can, passwordFor } from './server.mjs' import { accounts, didOf } from './state.mjs' import { both, shown, xrpc } from './xrpc.mjs' import { accountRace, createAccount, createAccountRefused, createSession, crossWrites, made, newDetails } from '../checks/accounts.mjs' import { blobRecord, getBlob, listBlobs, uploadBlob } from '../checks/blobs.mjs' import { checkCrawls, getBlocks, getGoneBlocks, getRepoStatus, listRepos, noCrawlsYet, syncGetRecord } from '../checks/federation.mjs' import { checkFirehose, checkReplay, startFirehose } from '../checks/firehose.mjs' import { handleRace, recommendedCredentials, resolveHandle, updateHandle, updateHandleNoToken, updateHandleRefused, wellKnownHandle } from '../checks/identity.mjs' import { oddRequests } from '../checks/odd.mjs' import { checkOAuthMetadata, oauthClientLibrary, oauthDpopRefused, oauthGetSession, oauthLogin, oauthParRefused, oauthRefresh, oauthRevoke, oauthRevokeRefused, oauthTokenRefused } from '../checks/oauth.mjs' import { oauthScopedCall, oauthScopedEdges, oauthScopedRefused } from '../checks/oauth-scopes.mjs' import { oauthSetCall, oauthSetLogin, oauthSetPar } from '../checks/permission-sets.mjs' import { oauthAssertionRefused, oauthClientCache, oauthClientDocument, oauthClientLogin, oauthConfidentialSession } from '../checks/oauth-clients.mjs' import { applyWrites, concurrent, createRecord, createWithTid, deleteRecord, describeRepo, getRecord, listRecords, putRecord, syncHeads } from '../checks/records.mjs' import { checkRepo, checkRepos } from '../checks/repository.mjs' import { describeServer, getSession } from '../checks/session.mjs' import { appPasswords, deleteSession, refreshSession, sessionRace, wrongTokens } from '../checks/sessions.mjs' import { duplicates, validatedBatch, validatedElsewhere, validatedWrite } from '../checks/validation.mjs' /// The record writes the repository, sync and firehose areas need. const WRITES = [[createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8]] const newAccount = Object.defineProperty(() => createAccount(), 'name', { value: 'createAccount' }) const firehoseChecks = Object.defineProperty(async () => { await checkFirehose() if (chance(0.3)) await checkReplay() }, 'name', { value: 'checkFirehose' }) const crawlsSoFar = Object.defineProperty(() => checkCrawls(0), 'name', { value: 'checkCrawls' }) /// The one account of a server that hosts one (spec/COMPAT.md, "One /// account"): the reference makes it with createAccount, the server through /// its adapter's `claim`, with the same handle and email and the password /// the server was started with. Their answers are not compared, nor is the /// server's genesis operation; the account's signing keys, and the server's /// rotation key, are read from the PLC directory. Then it is checked as a /// new account is, from getSession on. Answers the account, or null. async function claimAccount() { const CREATE = 'com.atproto.server.createAccount' const { handle, email } = newDetails(true) const body = { handle, email, password: cand.password } const [a, b] = await Promise.all([xrpc(ref, CREATE, { body }), cand.claim({ handle, email })]) count('createAccount (the server\'s one account)', a.status === 200 && b.status === 200) if (a.status !== 200) await differ(CREATE, body, 'an account', a.body, 'the reference refuses an account the harness means to make') if (b.status !== 200) await differ('the server\'s one account, made by its adapter', { handle, email }, 'an account', b.body, 'the adapter fails its own step') if (a.status !== 200 || b.status !== 200) return null const acct = made(body, a, b) for (const side of [ref, cand]) { const log = await plcLog(didOf(side, acct)) const op = Array.isArray(log.body) ? log.body[0] : null if (!op) { await fail(side, 'PLC log', shown({ body }), 'a genesis operation', log) continue } acct.sides[side.name].key = op.verificationMethods?.atproto if (side === cand) { acct.sides[side.name].kem = op.verificationMethods?.atproto_kem cand.rotationKey ??= op.rotationKeys?.[0] } } await both('getSession of a new account', 'com.atproto.server.getSession', { token: acct }) await syncHeads(acct) await checkRepo(acct) return acct } /// The areas, in the order the report lists them. Each has `about`, what it /// checks; `steps`, `[check, weight, needs]`, drawn as compare.mjs draws /// its steps, `needs` the settings (lib/adapters/, `unsupported`) a /// check depends on; `support`, `[step, weight, needs]` drawn among them, /// whose differences are kept apart; `pre`, checks of the area before the /// steps (and after the first accounts unless `beforeAccounts`); and /// `every`, checks of the area every `--check-every` steps and at the end. /// `accounts` is the need of a check that makes an account or needs a /// second one (spec/COMPAT.md, "One account"). export const AREAS = { oauth: { about: 'OAuth, the protocol: the server\'s metadata, PAR, the code exchange with PKCE, DPoP and its nonces, refresh and its rotation, revocation, DPoP-bound calls to XRPC, and a client app\'s own library', pre: [[checkOAuthMetadata]], steps: [[oauthLogin, 4], [oauthParRefused, 3], [oauthDpopRefused, 3], [oauthTokenRefused, 2], [oauthRevokeRefused, 1], [oauthRefresh, 3], [oauthRevoke, 2], [oauthGetSession, 2], [oauthClientLibrary, 1, ['address']]], }, 'oauth scopes': { about: 'OAuth scopes, enforced on every method: calls with an OAuth session\'s token, and with one thing wrong in its proof, token or scheme', steps: [[oauthScopedCall, 8], [oauthScopedRefused, 4], [oauthScopedEdges, 1]], }, 'oauth permission sets': { about: 'OAuth permission sets (`include:` scopes), resolved from the harness\'s lexicon authority: PAR with sets that resolve and do not, the page\'s data, consent keeping or dropping each, the token\'s expanded scope, and calls the sets\' grants allow or refuse', steps: [[oauthSetPar, 2, ['lexiconAuthority']], [oauthSetLogin, 2, ['lexiconAuthority']], [oauthSetCall, 3, ['lexiconAuthority']]], }, 'oauth clients': { about: 'OAuth clients\' metadata documents: fetched from the harness\'s web, kept and refused as the reference does; confidential clients and their assertions', steps: [[oauthClientDocument, 4, ['clientFetch']], [oauthClientCache, 1, ['clientFetch']], [oauthClientLogin, 3, ['clientFetch']], [oauthAssertionRefused, 3, ['clientFetch']], [oauthConfidentialSession, 2, ['clientFetch']]], }, sessions: { about: 'Sessions: createSession, getSession, refreshSession, app passwords and their scopes, deleteSession, tokens of the wrong kind, and describeServer', steps: [[createSession, 3], [getSession, 1], [describeServer, 1, ['handleDomains']], [refreshSession, 3], [deleteSession, 2], [wrongTokens, 2], [appPasswords, 4], [sessionRace, 1]], }, accounts: { about: 'Accounts: createAccount, the DID it registers with the PLC directory and its genesis operation, and what it refuses', steps: [[newAccount, 4, ['accounts']], [createAccountRefused, 4, ['accounts']], [accountRace, 1, ['accounts']]], }, records: { about: 'Records: createRecord, putRecord, deleteRecord, applyWrites, swapCommit and swapRecord, getRecord, listRecords, describeRepo, writes to another account\'s repository, and requests the lexicons refuse', steps: [[createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8], [getRecord, 10], [listRecords, 6], [describeRepo, 2], [createWithTid, 2], [concurrent, 6], [oddRequests, 3], [crossWrites, 3, ['accounts']]], }, validation: { about: 'Records checked against their lexicons: known types valid and not, `validate`, duplicates replaced, in applyWrites too', steps: [[validatedWrite, 10], [duplicates, 5], [validatedBatch, 4], [validatedElsewhere, 1]], }, repository: { about: 'The repository: each account\'s whole repository from getRepo, verified (signature, MST, every record) by the reference\'s library against the DID\'s key, its MST root the reference\'s, getRepo since, getLatestCommit', support: WRITES, pre: [[checkRepos]], every: [[checkRepos]], }, sync: { about: 'Sync: listRepos, getRepoStatus, getBlocks, sync.getRecord with its proof, and requestCrawl', pre: [[noCrawlsYet, ['hostname'], true], [crawlsSoFar, ['hostname']]], steps: [[listRepos, 2], [getRepoStatus, 2], [getBlocks, 3], [getGoneBlocks, 2], [syncGetRecord, 3]], support: WRITES, every: [[crawlsSoFar, ['hostname']]], }, firehose: { about: 'The firehose: each side\'s subscribeRepos from the start, every event checked as a consumer would, in order, with its blocks, and replays from a cursor', support: [...WRITES, [newAccount, 1, ['accounts']], [updateHandle, 2]], every: [[firehoseChecks]], }, identity: { about: 'Identity: handles, their changes and the PLC operations they make, resolveHandle, /.well-known/atproto-did, getRecommendedDidCredentials, and DID documents', steps: [[updateHandle, 3], [updateHandleRefused, 3], [updateHandleNoToken, 1], [resolveHandle, 3], [wellKnownHandle, 2], [recommendedCredentials, 1, ['recoveryKey']], [describeRepo, 2], [handleRace, 1, ['accounts']]], }, blobs: { about: 'Blobs: uploadBlob, records that reference them, getBlob, listBlobs', steps: [[uploadBlob, 4], [blobRecord, 5], [getBlob, 3], [listBlobs, 2]], }, } /// Why `needs` cannot be met by the server, or null. const missing = (needs = []) => needs.map((n) => adapter.unsupported[n] && `${n}: ${adapter.unsupported[n]}`).filter(Boolean) /// What the run found, of its own checks and of its supporting steps: how /// many ran, differed, and could not go on; the first difference; and by /// check, the same, with each one's first difference. const own = { checks: 0, failed: 0, errors: 0, first: null, byCheck: {} } const support = { checks: 0, failed: 0, errors: 0, first: null, byCheck: {} } const skipped = {} /// An exception a check did not expect, as an answer of a shape it cannot /// read, kept as a difference is. const exception = (e, name) => ({ name, step: run.step, what: `the check could not go on: ${String(e?.message ?? e).split('\n')[0]}`, note: 'an exception, not a comparison: what the server answered was of a shape the check cannot read, or the harness is wrong', request: null, reference: null, server: String(e?.stack ?? e).split('\n').slice(0, 6).join('\n') }) /// Run `fn` as one check of the area (or a supporting step, `mine` false). /// It goes on past what differs (lib/report.mjs keeps it): one with a /// difference is counted failed, and its first difference kept; one that /// throws, or does not end within 3 minutes, with none, is counted as not /// going on. Then the run goes on. async function check(fn, args, mine) { const name = fn.name || 'anonymous' const tally = mine ? own : support const by = (tally.byCheck[name] ??= { checks: 0, failed: 0, errors: 0, first: null, whats: {} }) tally.checks++ by.checks++ found.length = 0 let timer, error = null try { await Promise.race([fn(...args), new Promise((_, no) => (timer = setTimeout(() => no(new Error('no answer within 3 minutes')), 180_000)))]) } catch (e) { error = e } finally { clearTimeout(timer) } const diffs = found.splice(0) if (!diffs.length && !error) return // Each thing that differed, how often, the exception it ended in after // them too. const whats = [...diffs.map((d) => d.what), ...(error && diffs.length ? [`then could not go on: ${String(error?.message ?? error).split('\n')[0]}`] : [])] for (const w of whats) by.whats[w] = (by.whats[w] ?? 0) + 1 const f = diffs.length ? { name, ...diffs[0], ...(diffs.length > 1 ? { more: diffs.length - 1 } : {}) } : exception(error, name) const kind = diffs.length ? 'failed' : 'errors' tally[kind]++ by[kind]++ tally.first ??= f by.first ??= f console.error(`step ${run.step}: ${name}: ${f.what}${f.note ? ` (${f.note})` : ''}`) } /// Each of `list`, `[check, needs, beforeAccounts]`, run unless the server /// cannot meet its needs, which is noted. async function runAll(list, mine, when) { for (const [fn, needs, before] of list ?? []) { if (Boolean(before) !== when) continue const why = missing(needs) if (why.length) skipped[fn.name] = why else await check(fn, [], mine) } } /// The area's run, and its result written; then the run ends. export async function areaRun() { const area = AREAS[AREA] if (!area) { console.error(`no area ${AREA}; there are ${Object.keys(AREAS).join(', ')}`) await shutdown(2) } const drawn = [] const drawable = (list, mine) => { for (const [fn, w, needs] of list ?? []) { const why = missing(needs) if (why.length) skipped[fn.name] = why else drawn.push([fn, w, mine]) } } drawable(area.steps, true) drawable(area.support, false) if (FIREHOSE) await check(startFirehose, [], true) await runAll(area.pre, true, true) // The first accounts, as compare.mjs makes them, each a supporting step; // as many more as it takes (up to ten tries), when the server refuses // one the reference makes, which is then not kept. A server that hosts // one account has that one, made once, through its adapter. if (!can('accounts')) { await check(Object.defineProperty(() => claimAccount(), 'name', { value: 'the server\'s one account' }), [], false) } else { const first = Object.defineProperty(() => createAccount({ withPassword: true, password: passwordFor }), 'name', { value: 'createAccount (a first account)' }) const want = 2 + below(2) for (let tries = 0; accounts.length < want && tries < 10; tries++) await check(first, [], false) } for (const acct of accounts) await check(describeRepo, [acct], false) await runAll(area.pre, true, false) const total = drawn.reduce((n, [, w]) => n + w, 0) for (run.step = 1; run.step <= STEPS; run.step++) { if (drawn.length && accounts.length) { let r = below(total) for (const [fn, w, mine] of drawn) { if ((r -= w) < 0) { await check(fn, [pick(accounts)], mine) break } } } if (run.step % CHECK_EVERY === 0) await runAll(area.every, true, false) if (run.step % 100 === 0) console.error(`${run.step} steps`) } run.step = STEPS await runAll(area.every, true, false) const pkg = (name) => JSON.parse(readFileSync(new URL(`../node_modules/${name}/package.json`, import.meta.url), 'utf8')).version const result = { impl: IMPL, area: AREA, about: area.about, seed: SEED, steps: STEPS, reference: `@atproto/pds ${pkg('@atproto/pds')} (through @atproto/dev-env ${pkg('@atproto/dev-env')})`, server: await adapter.info(), own, support, skipped, unsupported: adapter.unsupported, allowed, accounts: accounts.length, requests: Object.values(counts).reduce((n, { ok, error }) => n + ok + error, 0), } // A difference may hold what a firehose frame decodes to: BigInts and bytes. const plain = (k, v) => (typeof v === 'bigint' ? `${v}` : v instanceof Uint8Array ? `<${v.length} bytes>` : v) writeFileSync(RESULT, `${JSON.stringify(result, plain, 2)}\n`) console.error(`\n${AREA}: ${own.checks - own.failed - own.errors} of ${own.checks} checks agree, ${own.failed} differ, ${own.errors} could not go on; supporting steps: ${support.failed + support.errors} of ${support.checks} failed`) await shutdown(0) }