// Runs a server and the reference PDS side by side, sends both the same // random sequence of requests, and stops at the first answer that differs. // // node harness/compare.mjs [--steps 400] [--seed 1] [--check-every 10] [--crash-every N] [--firehose] [--commands] [--s3] [--limits | --invites | --lifecycle | --moves | --store | --scale] [--only name,...] [--impl name] [--area name --result file.json] -- // // The server is started with these in its environment (spec/PDS.md, "What // it serves", spec/ACCOUNTS.md, "Environment"), and with no accounts: // // PDS_PORT the port to listen on, on 127.0.0.1 (PDS_BIND_ADDRESS unset) // PDS_DATA_DIR an empty directory to keep its data in // PDS_HOSTNAME localhost // PDS_PUBLIC_URL http://127.0.0.1: // PDS_HANDLE_DOMAINS .test,.example, as the reference's // PDS_PLC_URL the reference's PLC directory // PDS_PLC_ROTATION_KEY_HEX a new secp256k1 rotation key, 32 bytes as hex // PDS_RECOVERY_DID_KEY the reference's recovery key, as a did:key // PDS_CRAWLERS crawlers the harness runs, which record each requestCrawl // PDS_OAUTH_CLIENT_FETCH the harness's web of OAuth client documents (spec/OAUTH-CLIENTS.md) // PDS_ADMIN_PASSWORD the reference's admin password // PDS_INVITE_REQUIRED false, or true with --invites // PDS_RATE_LIMITS_ENABLED false, or true with --limits // PDS_TRUSTED_PROXIES the proxies the reference trusts in X-Forwarded-For (spec/LIMITS.md) // PDS_DID_CACHE_SECS 0, but with --limits (spec/LIMITS.md) // PDS_BLOB_UPLOAD_LIMIT 6291459 with --limits, as the reference's then (spec/LIMITS.md) // PDS_DEV_MAIL_DIR an empty directory it writes each email to (spec/EMAIL.md) // PDS_EMAIL_SMTP_URL the harness's SMTP server (lib/smtp.mjs), with a user and password to sign in with, percent-encoded // PDS_EMAIL_FROM mailer@pds-harness.test, the address each email is from (spec/EMAIL.md, "SMTP") // PDS_BSKY_APP_VIEW_URL the AppView the harness runs (lib/services.mjs), as the reference's // PDS_BSKY_APP_VIEW_DID its did:web // PDS_BSKY_APP_VIEW_CDN_URL_PATTERN http://cdn.pds-harness.test/%s/%s/%s, as the reference's // PDS_DISABLE_SSRF_PROTECTION true, as the reference runs // PDS_REPO_BACKFILL_LIMIT_MS 5000 with --store, as the reference's (spec/STORE.md, "Environment") // PDS_CACHE_BYTES 16 MiB with --scale (spec/STORE.md, "Environment") // PDS_LEXICON_AUTHORITY_DID the harness's lexicon authority, a third PDS publishing its permission sets (lib/sets.mjs) // PDS_HANDLE_DNS_SERVER the harness's DNS server of handles (lib/handles.mjs, spec/IDENTITY.md) // PDS_HANDLE_FETCH the harness's web of handles' /.well-known/atproto-did // PDS_BLOBSTORE_S3_* with --s3, its bucket in the harness's S3 store, path-style, its keys, and timeouts of 2 s (spec/BLOBS.md) // // The harness makes the first accounts on both sides with createAccount, // and more as it goes; each step picks an account and uses its token. // What is compared, and what is let differ, is in spec/PDS.md and the pages it links. // --crash-every kills the server every N steps, sometimes with a write on its // way, and starts it again on the same data; --firehose checks both // firehoses too. With --commands, some requests go to the server through the // method's MCP tool or CLI command instead of XRPC (spec/COMMANDS.md), and // both are checked against the lexicons first. The server command's first // word is then the `pds` binary, whose other commands are the CLI // (spec/COMMANDS.md, "One binary"). // --limits and --invites are runs of their own, of checks rather than // random steps: with the rate limits on, on both sides, and then what the // server holds, held to spec/LIMITS.md; or with invite codes required. // --store is a run of its own too: a few accounts churned, the server killed // while it reorganizes its data, and its data directory's size and starts // held to spec/STORE.md. --scale is one too: the server alone, its memory // and starts measured with many accounts and a large repository. --pages is // one too: the authorization pages in Chromium on both sides, the same // flows on each, held to the reference and to spec/OAUTH-PAGES.md. // --only runs only the steps, or those checks, of the names given. // --lifecycle is a run of its own: accounts deactivated, taken down and // deleted, and what each state does to every method (spec/LIFECYCLE.md); // --steps counts its changes, 80 by default, and --crash-every kills the // server every N of them. // --moves is a run of its own: accounts moved to the server and away from // it, through a third PDS, and the methods of a move by account state // (spec/MIGRATION.md); --steps counts its moves, 6 by default. Not to be // mistaken for --migrate, below, which moves blobs, not accounts. // --s3 keeps both sides' blobs in buckets of an S3 store the harness runs // (lib/s3.mjs), with the random steps or --lifecycle: the store fails as // steps ask, for both alike, and the server's bucket and data directory // are held to spec/BLOBS.md, "Where blobs are kept". // --migrate is a run of its own: the server's blobs moved from its data // directory into a bucket and back by `pds migrate-blobs`, killed as it // goes, and every blob read as the reference's after each move. // --impl names another implementation's adapter (lib/adapters/), which // starts that server itself, given as much of this setup as it can take; // Monolith's is the default, the command after `--` with this environment. // --area runs only the core checks of that area (lib/areas.mjs), going on // past a difference, and writes what it found to --result: compat.mjs runs // it for each area and implementation (compat/README.md). // --tally writes, as the run ends, how many requests of each kind it made: // the same for two runs of one seed, to show a change left the run as it // was. // // This file is the run: the steps, and how often each is drawn. What they // share is in lib/: the arguments, the one random source every draw comes // from, the processes, the PLC directory, requests (lib/xrpc.mjs, and // lib/via.mjs for MCP tools and CLI commands), what the harness knows of the // accounts and their repositories, and the report. Each area's checks are in // checks/. import { AREA, CHECK_EVERY, COMMANDS, CRASH_EVERY, FIREHOSE, INVITES, LIFECYCLE, LIMITS, MIGRATE, MOVES, ONLY, PAGES, S3, SCALE, SEED, STEPS, STORE, UPGRADE, UPGRADE_WRITE } from './lib/args.mjs' import { below, pick } from './lib/random.mjs' import { shutdown, start } from './lib/processes.mjs' import { counts, run } from './lib/report.mjs' import { accounts, recordCount } from './lib/state.mjs' import { accountRace, createAccount, createAccountRefused, createFirstAccounts, createSession, crossWrites, withoutAccount } from './checks/accounts.mjs' import { checkCommands } from './checks/commands.mjs' import { checkMcpOAuth } from './checks/oauth-mcp.mjs' import { oauthAccountAuth, oauthAccountCommands, oauthAccountDevices, oauthAccountSessions } from './checks/oauth-account.mjs' import { passkeyAdding, passkeyAuth, passkeyCommands, passkeyRemoving, passkeySignInRefused, passkeySigningIn, passkeyStates } from './checks/passkeys.mjs' import { blobRecord, getBlob, listBlobs, uploadBlob } from './checks/blobs.mjs' import { appPasswordEmail, confirmEmail, emailFactor, passwordReset, requestEmailConfirmation, requestEmailUpdate, smtpRefused, updateEmail } from './checks/email.mjs' import { smtp } from './lib/server.mjs' import { crash, crashes } from './checks/crashes.mjs' import { snapshot } from './checks/backup.mjs' import { checkCrawls, getBlocks, getGoneBlocks, getRepoStatus, listRepos, noCrawlsYet, syncGetRecord } from './checks/federation.mjs' import { startFirehose } from './checks/firehose.mjs' import { handleRace, recommendedCredentials, resolveHandle, updateHandle, updateHandleNoToken, updateHandleRefused, wellKnownHandle } from './checks/identity.mjs' import { oddRequests } from './checks/odd.mjs' import { cors } from './checks/cors.mjs' import { checkOAuthMetadata, oauthAuthorizeRefused, oauthClientLibrary, oauthDepartures, oauthDeviceCookies, oauthDpopRefused, oauthGetSession, oauthLogin, oauthParRefused, oauthRefresh, oauthRevoke, oauthRevokeRefused, oauthTokenRefused } from './checks/oauth.mjs' import { oauthScopedCall, oauthScopedEdges, oauthScopedRefused } from './checks/oauth-scopes.mjs' import { oauthSetCall, oauthSetLogin, oauthSetPar } from './checks/permission-sets.mjs' import { oauthAssertionRefused, oauthClientCache, oauthClientDepartures, oauthClientDocument, oauthClientLogin, oauthConfidentialSession, oauthSilentSignIn } from './checks/oauth-clients.mjs' import { applyWrites, concurrent, createRecord, createWithTid, deleteRecord, describeRepo, getRecord, listRecords, putRecord } from './checks/records.mjs' import { checkRepos } from './checks/repository.mjs' import { describeServer, getSession } from './checks/session.mjs' import { appPasswordScopes, bigBatch, bodyCaps, inviteCodes, invitesRequired, limitsRun, loginTiming, oauthSignInTiming, refusedBodies } from './checks/limits.mjs' import { appPasswords, deleteSession, refreshSession, sessionRace, wrongTokens } from './checks/sessions.mjs' import { checkOperations, outage } from './checks/operations.mjs' import { movesRun } from './checks/moves.mjs' import { lifecycleRun } from './checks/lifecycle.mjs' import { chatCase, feedAndPush, foreignRecord, forwarded, preferences, readAfterWrite, refused, serviceAuth } from './checks/proxy.mjs' import { duplicates, validatedBatch, validatedElsewhere, validatedWrite } from './checks/validation.mjs' import { storeRun } from './checks/store.mjs' import { scaleRun } from './checks/scale.mjs' import { pagesRun } from './checks/pages.mjs' import { upgradeRun, upgradeWrite } from './checks/upgrade.mjs' import { S3_STEPS, blobstoreCounted, bucketHolds, dataDirHoldsNoBlobs } from './checks/blobstore.mjs' import { migrateRun } from './checks/migrate.mjs' import { areaRun } from './lib/areas.mjs' await start() if (AREA) await areaRun() if (LIMITS || INVITES) { await (LIMITS ? limitsRun() : invitesRequired()) await checkOperations() console.error(`\nall ${LIMITS ? 'limits' : 'invite'} checks agree, ${accounts.length} accounts`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(56)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (LIFECYCLE) { await lifecycleRun() if (S3) await bucketHolds('after the lifecycle run') await checkOperations() console.error(`\nall lifecycle checks agree (seed ${SEED}), ${accounts.length} accounts`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(72)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (MOVES) { await movesRun() await checkOperations() for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(80)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (STORE) { const { report, starts, cold } = await storeRun() await checkOperations() console.error(`\nall ${STEPS} store steps agree (seed ${SEED}), ${accounts.length} accounts, ${recordCount()} records${cold ? `, a cold first use read ${cold.read} bytes of at most ${cold.most}` : ', a cold first use\'s reads not counted here'}`) for (const line of report) console.error(` ${line}`) const p50 = (xs) => [...xs].sort((x, y) => x - y)[xs.length >> 1] const [ms, empty] = [starts.map((s) => s.ms), starts.map((s) => s.empty)] console.error(` ${starts.length} starts, p50 ${p50(ms)} ms, max ${Math.max(...ms)} ms, the last ${ms.at(-1)} ms; the empty starts around each, their median p50 ${p50(empty)} ms, max ${Math.max(...empty)} ms`) console.error(` ${crashes.count} crashes, ${crashes.reorganizing} while reorganizing, ${crashes.inFlight} with a write on its way (${crashes.landed} landed)`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(56)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (SCALE) { const { r0, t0, accounts: n, repo, events } = await scaleRun() console.error(`\nall scale checks hold: ${n} accounts, a repository of ${(repo / 1048576).toFixed(1)} MiB, a replay of ${(events / 1048576).toFixed(1)} MiB; empty start ${t0} ms, ${(r0 / 1048576).toFixed(1)} MiB`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(56)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (PAGES) { await createFirstAccounts() const flows = await pagesRun() console.error(`\nall ${flows} page flows agree (seed ${SEED}), ${accounts.length} accounts`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(56)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (UPGRADE_WRITE) { const { version, accounts: n, reads } = await upgradeWrite() console.error(`\nharness/upgrade/ written by ${version}: ${n} accounts, ${reads} reads`) await shutdown(0) } if (UPGRADE) { const { version, reads } = await upgradeRun() console.error(`\nall ${reads} reads answer as ${version} answered them, on its data directories shut down and killed`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(56)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (MIGRATE) { await migrateRun() await checkOperations() console.error(`\nall migration checks agree (seed ${SEED}), ${accounts.length} accounts`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(72)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) await shutdown(0) } if (FIREHOSE) await startFirehose() await noCrawlsYet() await createFirstAccounts() await checkCrawls(0) if (COMMANDS) { await checkCommands() await checkMcpOAuth() } for (const acct of accounts) await describeRepo(acct) await checkRepos() await checkOAuthMetadata() const STEPS_BY_WEIGHT = [ [createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8], [getRecord, 10], [listRecords, 6], [describeRepo, 2], [getSession, 1], [createWithTid, 2], [concurrent, 6], [oddRequests, 3], [cors, 3], [describeServer, 1], [() => createAccount(), 1], [createAccountRefused, 4], [accountRace, 1], [createSession, 3], [crossWrites, 3], [listRepos, 2], [getRepoStatus, 2], [getBlocks, 3], [getGoneBlocks, 2], [syncGetRecord, 3], [refreshSession, 3], [deleteSession, 2], [wrongTokens, 2], [appPasswords, 4], [sessionRace, 1], [updateHandle, 3], [updateHandleRefused, 3], [updateHandleNoToken, 1], [resolveHandle, 3], [wellKnownHandle, 2], [recommendedCredentials, 1], [uploadBlob, 4], [blobRecord, 5], [getBlob, 3], [listBlobs, 2], [oauthLogin, 4], [oauthParRefused, 3], [oauthDpopRefused, 3], [oauthAuthorizeRefused, 3], [oauthTokenRefused, 2], [oauthRevokeRefused, 1], [oauthRefresh, 3], [oauthRevoke, 2], [oauthGetSession, 2], [oauthClientLibrary, 1], [oauthDepartures, 1], [oauthDeviceCookies, 1], [oauthScopedCall, 8], [oauthScopedRefused, 4], [oauthScopedEdges, 1], [oauthSetPar, 2], [oauthSetLogin, 2], [oauthSetCall, 3], [oauthClientDocument, 4], [oauthClientCache, 1], [oauthClientLogin, 3], [oauthAssertionRefused, 3], [oauthConfidentialSession, 2], [oauthSilentSignIn, 2], [oauthClientDepartures, 1], [oauthAccountAuth, 1], [oauthAccountSessions, 1], [oauthAccountDevices, 1], [passkeyAuth, 1], [passkeyAdding, 1], [passkeySigningIn, 1], [passkeySignInRefused, 1], [passkeyRemoving, 1], [passkeyStates, 1], [bigBatch, 1], [bodyCaps, 1], [refusedBodies, 2], [inviteCodes, 4], [appPasswordScopes, 1], [loginTiming, 1], [oauthSignInTiming, 1], [requestEmailConfirmation, 2], [confirmEmail, 3], [requestEmailUpdate, 2], [updateEmail, 3], [emailFactor, 1], [passwordReset, 3], [appPasswordEmail, 1], [forwarded, 4], [refused, 3], [serviceAuth, 3], [chatCase, 1], [preferences, 4], [readAfterWrite, 3], [foreignRecord, 1], [feedAndPush, 2], [validatedWrite, 10], [duplicates, 5], [validatedBatch, 4], [validatedElsewhere, 1], ...(COMMANDS ? [[withoutAccount, 2], [oauthAccountCommands, 1], [passkeyCommands, 1]] : []), // An email its SMTP server refuses, for a server sending through one (spec/EMAIL.md, "SMTP"). ...(smtp() ? [[smtpRefused, 2]] : []), ...(S3 ? S3_STEPS : []), // Which account wins may differ between the sides (spec/IDENTITY.md, "At // once"). ...(FIREHOSE ? [] : [[handleRace, 1]]), ].filter(([fn]) => !ONLY || ONLY.has(fn.name)) const total = STEPS_BY_WEIGHT.reduce((n, [, w]) => n + w, 0) for (run.step = 1; run.step <= STEPS; run.step++) { let r = below(total) for (const [fn, w] of STEPS_BY_WEIGHT) { if ((r -= w) < 0) { await fn(pick(accounts)) break } } if (CRASH_EVERY && run.step % CRASH_EVERY === 0) { // Every second crash, a snapshot of the data directory taken first, // while it writes (spec/DURABILITY.md, "Backups"). if (run.step % (2 * CRASH_EVERY) === 0) await snapshot() await crash() } else if (run.step % CHECK_EVERY === 0) await checkRepos() await outage(run.step) if (run.step % 100 === 0) console.error(`${run.step} steps agree`) } await checkRepos() await checkCrawls(crashes.count) if (S3) { await bucketHolds('at the end') await dataDirHoldsNoBlobs('at the end') await blobstoreCounted() } await checkOperations() console.error(`\nall ${STEPS} steps agree (seed ${SEED}), ${accounts.length} accounts, ${recordCount()} records`) for (const [kind, { ok, error }] of Object.entries(counts).sort()) console.error(` ${kind.padEnd(56)} ${String(ok).padStart(5)} ok ${String(error).padStart(5)} refused`) if (crashes.count) { const ms = crashes.restartMs.sort((x, y) => x - y) console.error(` ${crashes.count} crashes, ${crashes.inFlight} with a write on its way (${crashes.landed} landed), restart p50 ${ms[ms.length >> 1]} ms, max ${ms[ms.length - 1]} ms`) } await shutdown(0)