// Runs the gates, several at once, and says which failed, with each // failure's whole output. // // node gates.mjs [--jobs N] ... suites: quick, code, compare, commands, crash, perf // // The spaces run needs its own packages: `cd spaces && npm ci` once. // // The harness's runs are independent: each starts its own reference PDS and // server on ports and directories of its own. --jobs (or GATES_JOBS) sets // how many go at once; it defaults to one for each of the machine's cores, // and four at most. A run takes about half a core: it waits most of its // time, a third of a crash run's on the 350 and 400 ms both sides take // over every login. `quick` is a short run of each kind, for while working. // `code`, `compare` and `commands` are the gates every pull and push runs; // `crash` and `perf`, the long crash runs and the runs that time the // server, are the nightly gates, run on main's tip every night. `code` is // cargo's checks of the source, with the cargo CARGO names (`cargo` by // default), and the pages' (web/AGENTS.md). `perf` runs alone, after // everything else has finished, as it // times both servers. import { spawn } from 'node:child_process' import { availableParallelism } from 'node:os' const server = ['../target/release/pds', 'serve'] /// A run, and about how many seconds it takes in the gates workflow: for /// the order the runs start in alone, the longest first. const run = (secs, ...argv) => ({ secs, argv: [...argv.map(String), '--', ...server] }) const cargo = (process.env.CARGO ?? 'cargo').split(' ') /// A check that is no run of the server: a command as it is, or a harness /// script with no server. const check = (secs, ...argv) => ({ secs, argv: argv.map(String) }) const SUITES = { quick: [ run(80, 'compare.mjs', '--steps', 400, '--seed', 1), run(150, 'compare.mjs', '--steps', 400, '--seed', 21, '--crash-every', 15, '--firehose'), run(55, 'compare.mjs', '--steps', 400, '--seed', 31, '--commands'), run(10, 'compare.mjs', '--invites'), run(60, 'compare.mjs', '--pages', '--seed', 1), run(15, 'compare.mjs', '--upgrade'), run(90, 'compare.mjs', '--steps', 200, '--seed', 61, '--crash-every', 25, '--s3'), ], // The source (docs/guide/contributing.md, "The code gates"): formatted, // clean of clippy's warnings, its dependencies' licences and advisories, // and no unsafe code; the harness's citations of the spec (spec/PDS.md, // "How it is checked"); the pages' components in their layers, their // colours' contrast, and svelte-check. code: [ check(5, ...cargo, 'fmt', '--all', '--check'), check(120, ...cargo, 'clippy', '--workspace', '--all-targets', '--', '-D', 'warnings'), check(5, ...cargo, 'deny', 'check'), check(1, 'code.mjs'), check(1, 'spec-coverage.mjs'), check(1, 'layers.mjs'), check(1, 'contrast.mjs'), check(20, 'npm', '--prefix', 'web', 'run', 'check'), ], // The server against the reference: plain; with invite codes required; // with the rate limits on, then what the server holds (spec/LIMITS.md); // the lifecycle run, of 30 changes, about 5 minutes beside the other runs // (40 took 6); the moves run, of 6 moves of accounts (spec/MIGRATION.md); the // store, churned and killed while it compacts // (spec/STORE.md); the spaces run, against the pinned spaces alpha; and a // short crashing run with the firehose, so that every pull has the server // killed while it writes; the pages in a browser (spec/OAUTH-PAGES.md); // the server on the previous version's data (spec/STORE.md, // "Upgrades"); and blobs in a bucket, crashing and with accounts deleted // (spec/BLOBS.md, "Where blobs are kept"). compare: [ run(155, 'spaces/compare.mjs', '--seed', 1), ...[[1, 80], [2, 55], [3, 85], [4, 80], [5, 65]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 400, '--seed', n)), run(10, 'compare.mjs', '--invites'), run(140, 'compare.mjs', '--limits'), run(95, 'compare.mjs', '--steps', 600, '--seed', 51, '--crash-every', 40, '--store'), run(185, 'compare.mjs', '--lifecycle', '--steps', 30), run(90, 'compare.mjs', '--moves', '--steps', 6), run(150, 'compare.mjs', '--steps', 400, '--seed', 24, '--crash-every', 15, '--firehose'), ...[1, 2].map((n) => run(60, 'compare.mjs', '--pages', '--seed', n)), run(15, 'compare.mjs', '--upgrade'), run(150, 'compare.mjs', '--steps', 400, '--seed', 61, '--crash-every', 25, '--s3'), run(120, 'compare.mjs', '--lifecycle', '--steps', 12, '--s3'), run(120, 'compare.mjs', '--migrate', '--seed', 71), ], // Nightly: the server killed while it writes, every 15 steps, over long // runs: plain, with the firehose, and with the firehose through the MCP // tools and CLI commands. crash: [ ...[[21, 445], [22, 330]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 1000, '--seed', n, '--crash-every', 15)), ...[[21, 380], [22, 345], [23, 395]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 1000, '--seed', n, '--crash-every', 15, '--firehose')), run(600, 'compare.mjs', '--steps', 1000, '--seed', 41, '--crash-every', 15, '--firehose', '--commands'), ], // The methods through their MCP tools and CLI commands, against the // reference and the lexicons; and MCP's conformance suite. commands: [ ...[[31, 55], [32, 55], [33, 55]].map(([n, secs]) => run(secs, 'compare.mjs', '--steps', 400, '--seed', n, '--commands')), run(7, 'conformance.mjs'), ], // Nightly: the runs that measure time: the server's speed against the // reference's and its own baseline, and what it holds once subscribers // have come and gone (spec/OPERATIONS.md), its whole output printed even // when it passes, so the baseline can be raised from a nightly log; and // the scale run, which holds each start to twice the start on an empty // data directory and 250 ms (spec/STORE.md): a start with 2000 accounts, // 160 ms by itself, took 270 and 436 ms beside one crashing run, whose own // server starts and syncs its disk again and again. Making its 100000 // accounts is most of the scale run's time. perf: [{ ...run(120, 'perf.mjs'), whole: true }, run(900, 'compare.mjs', '--scale')], } const args = process.argv.slice(2) const j = args.indexOf('--jobs') const jobs = Number(j === -1 ? process.env.GATES_JOBS ?? Math.min(4, availableParallelism()) : args[j + 1]) const names = j === -1 ? args : args.filter((_, i) => i !== j && i !== j + 1) if (!names.length || names.some((n) => !SUITES[n]) || !(jobs >= 1)) { console.error(`usage: node gates.mjs [--jobs N] <${Object.keys(SUITES).join('|')}>...`) process.exit(2) } // The longest runs first, so the short ones fill in around them; the perf // suite's alone, one at a time, last. const runs = names.flatMap((n) => SUITES[n]) const alone = (run) => SUITES.perf.includes(run) const together = runs.filter((run) => !alone(run)).sort((x, y) => y.secs - x.secs) const last = runs.filter(alone) const harness = new URL('.', import.meta.url).pathname const root = new URL('..', import.meta.url).pathname /// A run: a harness script with node, in the harness; anything else as it /// is, in the repository's root. const argvOf = (run) => (run.argv[0].endsWith('.mjs') ? { argv: ['node', ...run.argv], cwd: harness } : { argv: run.argv, cwd: root }) const started = Date.now() const failed = [] async function go(run) { const t = Date.now() const { argv, cwd } = argvOf(run) const p = spawn(argv[0], argv.slice(1), { cwd, stdio: ['ignore', 'pipe', 'pipe'] }) let out = '' p.stdout.on('data', (d) => (out += d)) p.stderr.on('data', (d) => (out += d)) const code = await new Promise((ok) => { p.on('exit', ok) p.on('error', (err) => ((out += String(err)), ok(-1))) }) const secs = ((Date.now() - t) / 1000).toFixed(0) const summary = out.trim().split('\n').find((l) => /^all |scenarios pass/.test(l)) ?? '' if (code === 0) console.error(`ok ${secs.padStart(4)}s ${argv.join(' ')}\n ${run.whole ? out.trim().split('\n').join('\n ') : summary}`) else { failed.push(run) console.error(`FAIL ${secs.padStart(4)}s ${argv.join(' ')}\n${out}`) } } let next = 0 async function worker() { while (next < together.length) await go(together[next++]) } await Promise.all(Array.from({ length: Math.min(jobs, together.length) }, worker)) for (const run of last) await go(run) const total = ((Date.now() - started) / 1000).toFixed(0) console.error(failed.length ? `\n${failed.length} of ${runs.length} runs failed (${total}s)` : `\nall ${runs.length} runs pass (${total}s, ${jobs} at once)`) process.exit(failed.length ? 1 : 0)