chadtmiller.com/monolith
An atproto PDS
0
0
// Runs the gates, several at once, and says which failed, with each
// failure's whole output.
//
// node gates.mjs [--jobs N] <suite>... suites: quick, code, compare, commands, crash, perf
//
// The spaces run needs its own packages: `cd spaces && npm ci` once.
//
// The harness's runs are independent: each starts its own reference PDS and
// server on ports and directories of its own. --jobs (or GATES_JOBS) sets
// how many go at once; it defaults to one for each of the machine's cores,
// and four at most. A run takes about half a core: it waits most of its
// time, a third of a crash run's on the 350 and 400 ms both sides take
// over every login. `quick` is a short run of each kind, for while working.
// `code`, `compare` and `commands` are the gates every pull and push runs;
// `crash` and `perf`, the long crash runs and the runs that time the
// server, are the nightly gates, run on main's tip every night. `code` is
// cargo's checks of the source, with the cargo CARGO names (`cargo` by
// default), and the pages' (web/AGENTS.md). `perf` runs alone, after
// everything else has finished, as it
// times both servers.
import { spawn } from 'node:child_process'
import { availableParallelism } from 'node:os'
const server = ['../target/release/pds', 'serve']
/// A run, and about how many seconds it takes in the gates workflow: for
/// the order the runs start in alone, the longest first.
const run = (secs, ...argv) => ({ secs, argv: [...argv.map(String), '--', ...server] })
const cargo = (process.env.CARGO ?? 'cargo').split(' ')
/// A check that is no run of the server: a command as it is, or a harness
/// script with no server.
const check = (secs, ...argv) => ({ secs, argv: argv.map(String) })
const SUITES = {
quick: [
run(80, 'compare.mjs', '--steps', 400, '--seed', 1),
run(150, 'compare.mjs', '--steps', 400, '--seed', 21, '--crash-every', 15, '--firehose'),
run(55, 'compare.mjs', '--steps', 400, '--seed', 31, '--commands'),
run(10, 'compare.mjs', '--invites'),
run(60, 'compare.mjs', '--pages', '--seed', 1),
run(15, 'compare.mjs', '--upgrade'),
run(90, 'compare.mjs', '--steps', 200, '--seed', 61, '--crash-every', 25, '--s3'),
],
// The source (docs/guide/contributing.md, "The code gates"): formatted,
// clean of clippy's warnings, its dependencies' licences and advisories,
// and no unsafe code; the harness's citations of the spec (spec/PDS.md,
// "How it is checked"); the pages' components in their layers, their
// colours' contrast, and svelte-check.
code: [
check(5, ...cargo, 'fmt', '--all', '--check'),
check(120, ...cargo, 'clippy', '--workspace', '--all-targets', '--', '-D', 'warnings'),
check(5, ...cargo, 'deny', 'check'),
check(1, 'code.mjs'),
check(1, 'spec-coverage.mjs'),
check(1, 'layers.mjs'),
check(1, 'contrast.mjs'),
check(20, 'npm', '--prefix', 'web', 'run', 'check'),
],
// The server against the reference: plain; with invite codes required;
// with the rate limits on, then what the server holds (spec/LIMITS.md);
// the lifecycle run, of 30 changes, about 5 minutes beside the other runs
// (40 took 6); the moves run, of 6 moves of accounts (spec/MIGRATION.md); the
// store, churned and killed while it compacts
// (spec/STORE.md); the spaces run, against the pinned spaces alpha; and a
// short crashing run with the firehose, so that every pull has the server
// killed while it writes; the pages in a browser (spec/OAUTH-PAGES.md);
// the server on the previous version's data (spec/STORE.md,
// "Upgrades"); and blobs in a bucket, crashing and with accounts deleted
// (spec/BLOBS.md, "Where blobs are kept").
compare: [
run(155, 'spaces/compare.mjs', '--seed', 1),
...[[1, 80], [2, 55], [3, 85], [4, 80], [5, 65]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 400, '--seed', n)),
run(10, 'compare.mjs', '--invites'),
run(140, 'compare.mjs', '--limits'),
run(95, 'compare.mjs', '--steps', 600, '--seed', 51, '--crash-every', 40, '--store'),
run(185, 'compare.mjs', '--lifecycle', '--steps', 30),
run(90, 'compare.mjs', '--moves', '--steps', 6),
run(150, 'compare.mjs', '--steps', 400, '--seed', 24, '--crash-every', 15, '--firehose'),
...[1, 2].map((n) => run(60, 'compare.mjs', '--pages', '--seed', n)),
run(15, 'compare.mjs', '--upgrade'),
run(150, 'compare.mjs', '--steps', 400, '--seed', 61, '--crash-every', 25, '--s3'),
run(120, 'compare.mjs', '--lifecycle', '--steps', 12, '--s3'),
run(120, 'compare.mjs', '--migrate', '--seed', 71),
],
// Nightly: the server killed while it writes, every 15 steps, over long
// runs: plain, with the firehose, and with the firehose through the MCP
// tools and CLI commands.
crash: [
...[[21, 445], [22, 330]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 1000, '--seed', n, '--crash-every', 15)),
...[[21, 380], [22, 345], [23, 395]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 1000, '--seed', n, '--crash-every', 15, '--firehose')),
run(600, 'compare.mjs', '--steps', 1000, '--seed', 41, '--crash-every', 15, '--firehose', '--commands'),
],
// The methods through their MCP tools and CLI commands, against the
// reference and the lexicons; and MCP's conformance suite.
commands: [
...[[31, 55], [32, 55], [33, 55]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 400, '--seed', n, '--commands')),
run(7, 'conformance.mjs'),
],
// Nightly: the runs that measure time: the server's speed against the
// reference's and its own baseline, and what it holds once subscribers
// have come and gone (spec/OPERATIONS.md), its whole output printed even
// when it passes, so the baseline can be raised from a nightly log; and
// the scale run, which holds each start to twice the start on an empty
// data directory and 250 ms (spec/STORE.md): a start with 2000 accounts,
// 160 ms by itself, took 270 and 436 ms beside one crashing run, whose own
// server starts and syncs its disk again and again. Making its 100000
// accounts is most of the scale run's time.
perf: [{ ...run(120, 'perf.mjs'), whole: true }, run(900, 'compare.mjs', '--scale')],
}
const args = process.argv.slice(2)
const j = args.indexOf('--jobs')
const jobs = Number(j === -1 ? process.env.GATES_JOBS ?? Math.min(4, availableParallelism()) : args[j + 1])
const names = j === -1 ? args : args.filter((_, i) => i !== j && i !== j + 1)
if (!names.length || names.some((n) => !SUITES[n]) || !(jobs >= 1)) {
console.error(`usage: node gates.mjs [--jobs N] <${Object.keys(SUITES).join('|')}>...`)
process.exit(2)
}
// The longest runs first, so the short ones fill in around them; the perf
// suite's alone, one at a time, last.
const runs = names.flatMap((n) => SUITES[n])
const alone = (run) => SUITES.perf.includes(run)
const together = runs.filter((run) => !alone(run)).sort((x, y) => y.secs - x.secs)
const last = runs.filter(alone)
const harness = new URL('.', import.meta.url).pathname
const root = new URL('..', import.meta.url).pathname
/// A run: a harness script with node, in the harness; anything else as it
/// is, in the repository's root.
const argvOf = (run) => (run.argv[0].endsWith('.mjs') ? { argv: ['node', ...run.argv], cwd: harness } : { argv: run.argv, cwd: root })
const started = Date.now()
const failed = []
async function go(run) {
const t = Date.now()
const { argv, cwd } = argvOf(run)
const p = spawn(argv[0], argv.slice(1), { cwd, stdio: ['ignore', 'pipe', 'pipe'] })
let out = ''
p.stdout.on('data', (d) => (out += d))
p.stderr.on('data', (d) => (out += d))
const code = await new Promise((ok) => {
p.on('exit', ok)
p.on('error', (err) => ((out += String(err)), ok(-1)))
})
const secs = ((Date.now() - t) / 1000).toFixed(0)
const summary = out.trim().split('\n').find((l) => /^all |scenarios pass/.test(l)) ?? ''
if (code === 0) console.error(`ok ${secs.padStart(4)}s ${argv.join(' ')}\n ${run.whole ? out.trim().split('\n').join('\n ') : summary}`)
else {
failed.push(run)
console.error(`FAIL ${secs.padStart(4)}s ${argv.join(' ')}\n${out}`)
}
}
let next = 0
async function worker() {
while (next < together.length) await go(together[next++])
}
await Promise.all(Array.from({ length: Math.min(jobs, together.length) }, worker))
for (const run of last) await go(run)
const total = ((Date.now() - started) / 1000).toFixed(0)
console.error(failed.length ? `\n${failed.length} of ${runs.length} runs failed (${total}s)` : `\nall ${runs.length} runs pass (${total}s, ${jobs} at once)`)
process.exit(failed.length ? 1 : 0)