chadtmiller.com/monolith
An atproto PDS
0
0
// How compatible a PDS is with the reference, area by area: compare.mjs's
// core checks, run against each implementation an adapter in
// lib/adapters/ names, going on past a difference (lib/areas.mjs). Each
// area of each implementation is a run of its own, beside a fresh
// reference, from the same seed; the tables go to standard output as
// Markdown, and the raw results to `--json`.
//
// node compat.mjs [--impls monolith,zds,vlpds,pds.js,tranquil] [--areas oauth,records,...] [--steps 200] [--seed 1] [--jobs 3] [--json out.json]
// node compat.mjs --report out.json (the tables again from a run's --json)
//
// Monolith runs from ../target/release/pds (build it first); zds, vlpds,
// pds.js and tranquil from the images `node bench.mjs --setup` builds and
// pulls (bench/impls.json). compat/README.md says what is checked, and what
// is not; compat/RESULTS.md is a run's tables.
import { spawn } from 'node:child_process'
import { existsSync, mkdtempSync, openSync, readFileSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { pins, root, sh } from './lib/impls.mjs'
const here = new URL('.', import.meta.url).pathname
/// A flag's value; given twice, the last wins, so `just compat --steps 50`
/// overrides the recipe's own.
const arg = (name, fallback) => {
const i = process.argv.lastIndexOf(name)
return i === -1 ? fallback : process.argv[i + 1]
}
/// The areas, in the report's order (lib/areas.mjs's AREAS; listed here
/// too, as importing that module reads compare.mjs's arguments).
const AREAS = ['oauth', 'oauth scopes', 'oauth permission sets', 'oauth clients', 'sessions', 'accounts', 'records', 'validation', 'repository', 'sync', 'firehose', 'identity', 'blobs']
const IMPLS = ['monolith', 'zds', 'vlpds', 'pds.js', 'tranquil']
/// Differences an implementation's own documentation says it makes on
/// purpose: by implementation, `[check, what, source]`, matched against a
/// difference's check and its `what` (substrings; null matches any). A
/// first difference one of these matches is marked so in the report.
const DELIBERATE = {
zds: [],
vlpds: [
['listRepos', 'listRepos', 'vlpds DESIGN.md:5905, listRepos in (slot, DID) order, not the order accounts were made'],
['getBlob', 'getBlob', 'vlpds DESIGN.md:5906-5907 and docs/blobs.md:65-66, an upload served before any record references it'],
],
'pds.js': [],
tranquil: [
[null, 'PLC genesis operation', 'tranquil example.toml:228-230, its configuration reference (docs/1_WELCOME_TO_TRANQUIL_PDS.md:16), PLC operations signed with each account\'s own signing key, which always stays in its rotation keys'],
[null, 'getRecommendedDidCredentials', 'tranquil example.toml:228-230, its configuration reference (docs/1_WELCOME_TO_TRANQUIL_PDS.md:16), the account\'s own signing key always one of its rotation keys'],
],
}
/// What spec/COMPAT.md lets an implementation other than Monolith have,
/// section by section: all the harness lets it, and nothing more.
const ALLOWED = [
['Taken out of every answer (spec/COMPAT.md, "Taken out of every answer")', 'as for Monolith: each account\'s DID, its signing key, the server\'s URL, each commit\'s `cid` and `rev`, tokens, cursors and error messages; the `error` name and the HTTP status must match, and a `didDoc` must be the PLC directory\'s.'],
['The KEM key (spec/COMPAT.md, "The KEM key")', 'an `atproto_kem` key may be published or not; one published must be a P-256 did:key other than the signing key, the same in getRecommendedDidCredentials. `spacesAlpha` is not checked.'],
['OAuth access tokens\' life (spec/COMPAT.md, "OAuth access tokens\' life")', '`expires_in` of 15 minutes or of 60.'],
['`describeServer` (spec/COMPAT.md, "describeServer")', 'only `did`, `availableUserDomains` and `inviteCodeRequired` are compared, as for Monolith.'],
['Exact numbers of limits (spec/COMPAT.md, "Exact numbers of limits")', 'no blob at the upload limit or one byte past it, and no PAR body of 100 KiB or one byte past it, is sent.'],
['Signing in (spec/COMPAT.md, "Signing in")', 'on the server\'s own page, through its adapter\'s hook; compared from the redirect to the client on. Where its page cannot narrow the scope, the scope asked is approved. The reference\'s page API checks are not run.'],
['Monolith\'s own departures (spec/COMPAT.md, "Monolith\'s own departures")', 'not held: OAuth\'s and OAuth clients\' departures, and `request_parameter_supported: false`.'],
['One account (spec/COMPAT.md, "One account")', 'a server that hosts one account makes it through its adapter, not createAccount, and each area has that one; the checks that make an account or need a second are not run, and its rotation key is read from the account\'s genesis operation.'],
['Password rules (spec/COMPAT.md, "Password rules")', 'where a server refuses passwords by a rule of its own, each area\'s first accounts are made, on both sides, with a password its adapter makes from the one drawn; every check sending a password of its own sends it as before.'],
['Its own address (spec/COMPAT.md, "Its own address")', 'a server that names itself by an https origin it is not reached at in the run has every request for that origin sent to the URL it is reached at, otherwise as before; its proofs and tokens still name the origin.'],
['Settings an implementation cannot be given (spec/COMPAT.md, the section of that name)', 'the checks needing one are not run, and each is named below with why.'],
]
if (process.argv[2] === '--report') process.stdout.write(markdown(JSON.parse(readFileSync(process.argv[3], 'utf8'))))
else await main()
async function main() {
const impls = arg('--impls', IMPLS.join(',')).split(',')
const areas = arg('--areas', AREAS.join(',')).split(',')
const steps = Number(arg('--steps', 200))
const seed = Number(arg('--seed', 1))
const jobs = Number(arg('--jobs', 3))
const out = arg('--json', null)
for (const a of areas) if (!AREAS.includes(a)) throw new Error(`no area ${a}; there are ${AREAS.join(', ')}`)
if (impls.includes('monolith') && !existsSync(`${root}target/release/pds`)) throw new Error('Monolith: no target/release/pds; run `just build` first (the pages in web/, then pds)')
const dir = mkdtempSync(join(tmpdir(), 'compat-'))
console.error(`runs' logs and results in ${dir}`)
const queue = impls.flatMap((impl) => areas.map((area) => ({ impl, area })))
const results = []
const runOne = async ({ impl, area }) => {
const tag = `${impl}-${area.replace(/ /g, '-')}`
const file = join(dir, `${tag}.json`)
const log = join(dir, `${tag}.log`)
const args = [join(here, 'compare.mjs'), '--impl', impl, '--area', area, '--steps', String(steps), '--seed', String(seed), '--result', file, ...(impl === 'monolith' ? ['--', `${root}target/release/pds`, 'serve'] : [])]
const started = Date.now()
const code = await new Promise((ok) => {
const fd = openSync(log, 'w')
spawn(process.execPath, args, { cwd: here, stdio: ['ignore', fd, fd] }).on('exit', ok)
})
const ms = Date.now() - started
if (existsSync(file)) {
results.push({ ...JSON.parse(readFileSync(file, 'utf8')), ms, log })
console.error(`${impl} ${area}: done in ${(ms / 1000).toFixed(0)} s`)
} else {
const tail = readFileSync(log, 'utf8').trim().split('\n').slice(-12).join('\n')
results.push({ impl, area, ended: `the run ended (exit ${code}) before it was done:\n${tail}`, ms, log })
console.error(`${impl} ${area}: ended (exit ${code}); its log is ${log}`)
}
}
await Promise.all(Array.from({ length: Math.max(1, jobs) }, async () => {
while (queue.length) await runOne(queue.shift())
}))
const pkg = (name) => JSON.parse(readFileSync(join(here, 'node_modules', name, 'package.json'), 'utf8')).version
const run = {
date: new Date().toISOString().slice(0, 10),
seed,
steps,
reference: `@atproto/pds ${pkg('@atproto/pds')} (through @atproto/dev-env ${pkg('@atproto/dev-env')}), Node ${process.version}`,
monolithCommit: await sh('git', ['rev-parse', '--short', 'HEAD'], root),
pins: pins.implementations,
impls,
areas,
results,
}
if (out) writeFileSync(out, `${JSON.stringify(run, null, 2)}\n`)
process.stdout.write(markdown(run))
}
/// A run's results as Markdown: the pinned versions; a table, an area per
/// row and an implementation per column; what each could not be given; the
/// differences let every implementation; and each area's first difference
/// on each implementation, in full.
function markdown(run) {
const by = (impl, area) => run.results.find((r) => r.impl === impl && r.area === area)
const lines = []
const p = (s = '') => lines.push(s)
p('## What ran')
p()
p(`Compared with the reference, **${run.reference}**; seed ${run.seed}, ${run.steps} steps per area, each area a run of its own on fresh servers (Monolith at commit \`${run.monolithCommit}\`, ${run.date}).`)
p()
p('| implementation | version | from |')
p('|---|---|---|')
for (const impl of run.impls) {
const info = run.results.find((r) => r.impl === impl && r.server)?.server
const pin = run.pins[impl]
p(`| ${impl} | ${info?.version ?? pin?.release ?? '?'} | ${(info?.source ?? pin?.source ?? '').replace(/\|/g, '\\|')} |`)
}
p()
p('## Summary')
p()
p('Each cell is the area\'s checks that agree with the reference, of those run. A check is one step of the area, drawn at random as compare.mjs draws them; it goes on past what differs, and counts as failed if anything did. "skipped" is an area whose every check depends on a setting the server cannot be given (below).')
p()
p(`| area | ${run.impls.join(' | ')} |`)
p(`|---|${run.impls.map(() => '---:').join('|')}|`)
for (const area of run.areas) {
const cells = run.impls.map((impl) => {
const r = by(impl, area)
if (!r) return '—'
if (r.ended) return 'did not finish'
if (!r.own.checks) return 'skipped'
const ok = r.own.checks - r.own.failed - r.own.errors
return ok === r.own.checks ? `${ok}/${r.own.checks}` : `**${ok}/${r.own.checks}**`
})
p(`| ${area} | ${cells.join(' | ')} |`)
}
p()
p('Supporting steps (the first accounts, and the writes the repository, sync and firehose areas need) are not counted in their area; where they differ, the area\'s section says so.')
p()
p('## What each implementation could not be given')
p()
for (const impl of run.impls) {
const u = run.results.find((r) => r.impl === impl && r.unsupported)?.unsupported ?? {}
if (!Object.keys(u).length) continue
p(`**${impl}**`)
p()
for (const [k, why] of Object.entries(u)) p(`- \`${k}\`: ${why}`)
p()
}
p('## What spec/COMPAT.md lets an implementation other than Monolith have')
p()
p('Monolith is held to all of spec/PDS.md and the pages it links; these are let every other implementation alike, and nothing else is.')
p()
for (const [what, how] of ALLOWED) p(`- **${what}**: ${how}`)
p()
const used = run.impls.map((impl) => {
const all = {}
for (const r of run.results.filter((x) => x.impl === impl)) for (const [k, n] of Object.entries(r.allowed ?? {})) all[k] = (all[k] ?? 0) + n
return [impl, all]
}).filter(([, all]) => Object.keys(all).length)
for (const [impl, all] of used) p(`${impl} made use of: ${Object.entries(all).map(([k, n]) => `${k} (${n} times)`).join('; ')}.`)
if (used.length) p()
p('## By area')
for (const area of run.areas) {
p()
p(`### ${area}`)
p()
const about = run.results.find((r) => r.area === area && r.about)?.about
if (about) p(`${about}.`)
for (const impl of run.impls) {
const r = by(impl, area)
if (!r) continue
p()
if (r.ended) {
p(`**${impl}**: the run did not finish.`)
p()
p('```')
p(r.ended)
p('```')
continue
}
const ok = r.own.checks - r.own.failed - r.own.errors
const skipped = Object.entries(r.skipped ?? {})
p(`**${impl}**: ${r.own.checks ? `${ok} of ${r.own.checks} checks agree` : 'no check ran'}${r.own.failed ? `, ${r.own.failed} differ` : ''}${r.own.errors ? `, ${r.own.errors} could not go on` : ''}.${skipped.length ? ` Skipped: ${skipped.map(([n, why]) => `\`${n}\` (${why.join('; ')})`).join(', ')}.` : ''}`)
const failing = Object.entries(r.own.byCheck).filter(([, c]) => c.first)
if (failing.length) {
p()
p('| check | failed | of | what differed (how often) |')
p('|---|--:|--:|---|')
for (const [name, c] of failing) {
const whats = Object.entries(Object.keys(c.whats ?? {}).length ? c.whats : { [c.first.what]: c.failed + c.errors }).sort((x, y) => y[1] - x[1])
const shown = whats.slice(0, 5).map(([w, n]) => `${w}${n > 1 ? ` (${n})` : ''}`).join('; ') + (whats.length > 5 ? `; and ${whats.length - 5} more` : '')
const mark = deliberate(impl, c.first)
p(`| ${name} | ${c.failed + c.errors} | ${c.checks} | ${shown.replace(/\|/g, '\\|')}${mark ? ` (documented as deliberate: ${mark})` : ''} |`)
}
}
if (r.own.first) {
p()
const mark = deliberate(impl, r.own.first)
p(`First difference, at step ${r.own.first.step} (${r.own.first.name}): ${r.own.first.what}${r.own.first.note ? `: ${r.own.first.note}` : ''}.${mark ? ` **Documented as deliberate**: ${mark}.` : ''}`)
p()
p('```')
p(`request: ${JSON.stringify(r.own.first.request, null, 2)}`)
p(`reference: ${JSON.stringify(r.own.first.reference, null, 2)}`)
p(`${impl}: ${typeof r.own.first.server === 'string' ? r.own.first.server : JSON.stringify(r.own.first.server, null, 2)}`)
p('```')
}
if (r.support.failed + r.support.errors) {
const s = r.support.first
p()
p(`Supporting steps: ${r.support.failed + r.support.errors} of ${r.support.checks} differed (not counted above); the first, ${s.name}: ${s.what}.`)
}
}
}
return `${lines.join('\n')}\n`
}
/// Whether `impl`'s documentation says the difference `f` is on purpose:
/// where, or null.
function deliberate(impl, f) {
const hit = (DELIBERATE[impl] ?? []).find(([check, what]) => (!check || f.name?.includes(check)) && (!what || String(f.what).includes(what)))
return hit ? hit[2] : null
}