// The performance step (spec/OPERATIONS.md, "Performance"): one fixed // workload against the reference PDS and against the server, each in a // process of its own, part by part, each part in three rounds with the sides // taking turns; it reports each side's best round of each part, its p50, p99 // and throughput, and the server's ratios to the reference's beside the // baseline's (perf-baseline.json). It fails a part where the server's p99 // ratio is more than 1.5 times the baseline's (or 1, if lower), or its // throughput ratio below 0.8 times the baseline's (or 1, if higher). Then, // against the server alone, subscribers come and go, cohort after cohort, // and it fails if the server then holds more descriptors, threads or memory // than after the first cohort. // // node perf.mjs -- // // The server is started as compare.mjs starts it (lib/env.mjs), with no // rate limits, registering its DIDs with the reference's PLC directory. import { execFileSync, spawn } from 'node:child_process' import { createHash } from 'node:crypto' import { mkdtempSync, openSync, readdirSync, readFileSync } from 'node:fs' import { createServer } from 'node:net' import { tmpdir } from 'node:os' import { join } from 'node:path' import { createInterface } from 'node:readline' /// The factor of the baseline's p99 ratio (or 1, if lower) the server's may /// be. const P99_FACTOR = 1.5 /// The factor of the baseline's throughput ratio (or 1, if higher) the /// server's must reach: run-to-run noise of a few percent, on parts bound by /// the same work on both sides, would turn a floor of 1 into flakes. const RATE_FACTOR = 0.8 /// Each part's ratios, the server's best over the reference's, as nightly /// runs on the runner measured them, the weakest of several. const BASELINE = JSON.parse(readFileSync(new URL('./perf-baseline.json', import.meta.url), 'utf8')).parts /// Subscribers come and go: cohorts of this many, after one to warm. const COHORT = 20 const COHORTS = 20 /// How much more memory the server may hold after the last cohort than /// after the first: what the allocator keeps of what it freed. const MEMORY_SLACK = 8 * 1048576 /// How long the server is left alone after the first cohort before what it /// holds is read: idle threads the parts before used end after 10 s, and /// what they held goes back with them. const IDLE_MS = 15000 /// How long the server has, after the last cohort, to give back what it held. const SETTLE_MS = 20000 // The reference, in a process of its own: started by the run, it says where // it listens on its first line, and stops when its input closes. if (process.argv[2] === '--reference') { const { TestNetworkNoAppView } = await import('@atproto/dev-env') const net = await TestNetworkNoAppView.create({ pds: { hostname: 'localhost', rateLimitsEnabled: false, inviteRequired: false } }) process.stdout.write(`${JSON.stringify({ url: net.pds.url, plcUrl: net.plc.url, recoveryKey: net.pds.ctx.cfg.identity.recoveryDidKey })}\n`) process.stdin.resume() process.stdin.on('end', async () => { await net.close() process.exit(0) }) } else await run() async function run() { const { command } = await import('./lib/args.mjs') const { below } = await import('./lib/random.mjs') const { serverEnv } = await import('./lib/env.mjs') const { cborDecodeMulti } = await import('@atproto/common') const freePort = () => new Promise((ok) => { const s = createServer() s.listen(0, '127.0.0.1', () => { const { port } = s.address() s.close(() => ok(port)) }) }) const refProc = spawn(process.execPath, [new URL(import.meta.url).pathname, '--reference'], { stdio: ['pipe', 'pipe', 'ignore'] }) const first = await new Promise((ok) => createInterface({ input: refProc.stdout }).once('line', ok)) const { url: refUrl, plcUrl, recoveryKey } = JSON.parse(first) const port = await freePort() const dataDir = mkdtempSync(join(tmpdir(), 'pds-perf-')) const logPath = join(mkdtempSync(join(tmpdir(), 'pds-perf-log-')), 'server.log') const { env } = await serverEnv({ port, dataDir, plc: plcUrl, recoveryKey }) const log = openSync(logPath, 'a') const serverProc = spawn(command[0], command.slice(1), { env: { ...process.env, ...env }, stdio: ['ignore', log, log] }) const serverUrl = `http://127.0.0.1:${port}` const stop = (code) => { serverProc.kill('SIGKILL') refProc.stdin.end() process.exit(code) } for (const event of ['uncaughtException', 'unhandledRejection']) process.on(event, (err) => { console.error(err) stop(1) }) for (const started = Date.now(); ; ) { try { if ((await fetch(`${serverUrl}/xrpc/com.atproto.server.describeServer`)).ok) break } catch {} if (serverProc.exitCode !== null || Date.now() - started > 30000) { console.error(`the server did not start; its log is ${logPath}`) stop(1) } await new Promise((ok) => setTimeout(ok, 20)) } const sides = [ { name: 'reference', url: refUrl, accounts: [], records: [] }, { name: 'server', url: serverUrl, accounts: [], records: [] }, ] /// One request, timed from sending it to having its answer whole; /// anything but a 2xx stops the run. async function call(side, nsid, { params, body, bytes, token, type } = {}) { const url = new URL(`${side.url}/xrpc/${nsid}`) for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v) const headers = {} if (token) headers.authorization = `Bearer ${token}` if (body !== undefined) headers['content-type'] = 'application/json' if (bytes) headers['content-type'] = type const t = performance.now() const r = await fetch(url, { method: body !== undefined || bytes ? 'POST' : 'GET', headers, body: bytes ?? (body === undefined ? undefined : JSON.stringify(body)) }) const buf = Buffer.from(await r.arrayBuffer()) const ms = performance.now() - t if (!r.ok) { console.error(`${side.name}: ${nsid} answered ${r.status}: ${buf.toString('utf8').slice(0, 300)}${side.name === 'server' ? `\nserver log: ${logPath}` : ''}`) stop(1) } return { ms, json: () => JSON.parse(buf.toString('utf8')) } } /// Requests `lo` to `hi`, `at` at once, each made by `make(i)`: their /// times, and how long they took together. async function load(lo, hi, at, make) { const times = [] let next = lo const t = performance.now() await Promise.all(Array.from({ length: at }, async () => { while (next < hi) times.push(await make(next++)) })) return { times, total: performance.now() - t } } const ACCOUNTS = 40 /// Each part: how many requests, and a round of them, `lo` to `hi`, on a /// side. const PARTS = { createAccount: [ACCOUNTS, (side, lo, hi) => load(lo, hi, 4, async (i) => { const r = await call(side, 'com.atproto.server.createAccount', { body: { handle: `perf${i}.test`, email: `perf${i}@example.com`, password: `perf-password-${i}` } }) const { did, accessJwt } = r.json() side.accounts[i] = { did, token: accessJwt } return r.ms })], createRecord: [2000, (side, lo, hi) => load(lo, hi, 16, async (i) => { const a = side.accounts[i % ACCOUNTS] const r = await call(side, 'com.atproto.repo.createRecord', { body: { repo: a.did, collection: 'com.example.note', record: { $type: 'com.example.note', text: `a note of about a hundred bytes, number ${i}, to read back later`, n: i } }, token: a.token }) side.records[i] = { repo: a.did, rkey: r.json().uri.split('/').pop() } return r.ms })], getRecord: [4000, (side, lo, hi, picks) => load(lo, hi, 16, async (i) => (await call(side, 'com.atproto.repo.getRecord', { params: { repo: side.records[picks[i]].repo, collection: 'com.example.note', rkey: side.records[picks[i]].rkey } })).ms)], getRepo: [200, (side, lo, hi) => load(lo, hi, 8, async (i) => (await call(side, 'com.atproto.sync.getRepo', { params: { did: side.accounts[i % ACCOUNTS].did } })).ms)], firehose: [200, async (side, lo, hi) => { // Each from an address of its own, as 20 clients would be: the server // holds 16 at most from one (spec/LIMITS.md). const subs = await Promise.all(Array.from({ length: 20 }, (_, n) => new Promise((ok, fail) => { const ws = new WebSocket(`${side.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`, { headers: { 'x-forwarded-for': `198.18.0.${n + 1}` } }) ws.binaryType = 'arraybuffer' const sub = { ws, got: new Map() } ws.onmessage = (m) => { const [header, body] = cborDecodeMulti(new Uint8Array(m.data)) if (header?.t !== '#commit') return for (const op of body.ops ?? []) { const k = /\/fh(\d+)x$/.exec(op.path) if (k && !sub.got.has(Number(k[1]))) sub.got.set(Number(k[1]), performance.now()) } } ws.onopen = () => ok(sub) ws.onerror = () => fail(new Error(`${side.name}: subscribeRepos`)) }))) const sentAt = [] const a = side.accounts[0] const t = performance.now() for (let i = lo; i < hi; i++) { sentAt[i] = performance.now() await call(side, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `fh${i}x`, record: { $type: 'com.example.note', text: 'fan-out', n: i } }, token: a.token }) } for (let w = 0; w < 100 && subs.some((s) => s.got.size < hi - lo); w++) await new Promise((ok) => setTimeout(ok, 100)) for (const s of subs) s.ws.close() const times = [] let last = t for (const s of subs) for (let i = lo; i < hi; i++) { const at = s.got.get(i) if (at === undefined) { console.error(`${side.name}: a subscriber did not get the commit of write ${i} within 10 s`) stop(1) } times.push(at - sentAt[i]) last = Math.max(last, at) } return { times, total: last - t } }], uploadBlob: [200, (side, lo, hi) => { const blob = (i) => { const out = Buffer.alloc(65536) for (let o = 0; o < out.length; o += 32) createHash('sha256').update(`blob ${i} ${o}`).digest().copy(out, o) return out } return load(lo, hi, 8, async (i) => (await call(side, 'com.atproto.repo.uploadBlob', { bytes: blob(i), type: 'application/octet-stream', token: side.accounts[i % ACCOUNTS].token })).ms) }], } const pct = (xs, p) => { const s = [...xs].sort((x, y) => x - y) return s[Math.min(s.length - 1, Math.ceil((p / 100) * s.length) - 1)] } // Each part in three rounds, the sides taking turns; each side judged by // its best round, as other work on the machine only ever slows a round. const ROUNDS = 3 const rows = [] const failed = [] for (const [part, [n, fn]] of Object.entries(PARTS)) { const picks = part === 'getRecord' ? Array.from({ length: n }, () => below(2000)) : null const rounds = { reference: [], server: [] } for (let k = 0; k < ROUNDS; k++) { const [lo, hi] = [Math.floor((n * k) / ROUNDS), Math.floor((n * (k + 1)) / ROUNDS)] for (const side of sides) { const { times, total } = await fn(side, lo, hi, picks) rounds[side.name].push({ p50: pct(times, 50), p99: pct(times, 99), rate: (times.length / total) * 1000 }) } } const best = (rs) => ({ p50: Math.min(...rs.map((x) => x.p50)), p99: Math.min(...rs.map((x) => x.p99)), rate: Math.max(...rs.map((x) => x.rate)) }) const [r, s] = [best(rounds.reference), best(rounds.server)] // The ratios, held to the baseline's, and never to less than the // reference's own. const got = { p99: s.p99 / r.p99, rate: s.rate / r.rate } const base = BASELINE[part] ?? { p99: 1, rate: 1 } const [p99Of, rateOf] = [Math.min(base.p99, 1), Math.max(base.rate, 1)] const why = [] if (got.p99 > P99_FACTOR * p99Of) why.push(`p99 ratio ${sig(got.p99)} (${s.p99.toFixed(2)} ms to the reference's ${r.p99.toFixed(2)} ms), over ${P99_FACTOR} × ${sig(p99Of)}`) if (got.rate < RATE_FACTOR * rateOf) why.push(`throughput ratio ${sig(got.rate)} (${s.rate.toFixed(0)}/s to the reference's ${r.rate.toFixed(0)}/s), below ${RATE_FACTOR} × ${sig(rateOf)}`) if (why.length) failed.push(`${part}: ${why.join('; ')}`) rows.push({ part, r, s, verdict: why.length ? 'FAIL' : 'ok', rounds, got, base }) } const f = (x) => x.toFixed(1).padStart(8) console.error(`each side's best of ${ROUNDS} rounds, the sides taking turns, and the server's ratios to the reference's (the baseline's):\n`) console.error(`${'part'.padEnd(14)}${'reference p50'.padStart(14)}${'p99'.padStart(9)}${'per s'.padStart(9)} ${'server p50'.padStart(11)}${'p99'.padStart(9)}${'per s'.padStart(9)} ${'p99 ratio'.padStart(18)}${'per s ratio'.padStart(18)}`) for (const { part, r, s, verdict, got, base } of rows) console.error(`${part.padEnd(14)}${f(r.p50).padStart(14)}${f(r.p99).padStart(9)}${r.rate.toFixed(0).padStart(9)} ${f(s.p50).padStart(11)}${f(s.p99).padStart(9)}${s.rate.toFixed(0).padStart(9)} ${`${sig(got.p99)} (${sig(base.p99)})`.padStart(18)}${`${sig(got.rate)} (${sig(base.rate)})`.padStart(18)} ${verdict}`) console.error('\neach round\'s p99, in ms:') for (const { part, rounds } of rows) console.error(`${part.padEnd(14)}reference ${rounds.reference.map((x) => x.p99.toFixed(1)).join(', ')}; server ${rounds.server.map((x) => x.p99.toFixed(1)).join(', ')}`) console.error(`\nthe ratios, as perf-baseline.json holds them: ${JSON.stringify(Object.fromEntries(rows.map(({ part, got }) => [part, { p99: sig(got.p99), rate: sig(got.rate) }])))}`) // Subscribers come and go, against the server alone: what it holds for // a cohort of subscribers it gives back once they have left. const server = sides[1] const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms)) let cohorts = 0 async function cohort() { const k = cohorts++ // Each from an address of its own, the same 20 each time: the server // holds 16 at most from one (spec/LIMITS.md). const subs = await Promise.all(Array.from({ length: COHORT }, (_, n) => new Promise((ok, fail) => { const ws = new WebSocket(`${server.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`, { headers: { 'x-forwarded-for': `198.18.1.${n + 1}` } }) ws.binaryType = 'arraybuffer' const sub = { ws, got: false, closed: new Promise((c) => ws.addEventListener('close', c)) } ws.onmessage = (m) => { const [header, body] = cborDecodeMulti(new Uint8Array(m.data)) if (header?.t === '#commit' && (body.ops ?? []).some((op) => op.path === `com.example.note/churn${k}`)) sub.got = true } ws.onopen = () => ok(sub) ws.onerror = () => fail(new Error('server: subscribeRepos')) }))) const a = server.accounts[0] await call(server, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `churn${k}`, record: { $type: 'com.example.note', text: 'come and go', n: k } }, token: a.token }) for (const t = Date.now(); subs.some((s) => !s.got); await sleep(5)) if (Date.now() - t > 10000) { console.error(`server: a subscriber of cohort ${k} did not get the commit of its write within 10 s`) stop(1) } for (const s of subs) s.ws.close() if (!(await Promise.race([Promise.all(subs.map((s) => s.closed)).then(() => true), sleep(10000).then(() => false)]))) { console.error(`server: a subscriber of cohort ${k} did not see its connection closed within 10 s of closing it`) stop(1) } } /// What the server process holds: its open file descriptors, threads and /// memory, resident or swapped out (Linux), or its footprint, resident or /// compressed (macOS), in bytes. const pid = String(serverProc.pid) const footprintPath = join(mkdtempSync(join(tmpdir(), 'pds-perf-footprint-')), 'footprint.json') function holds() { if (process.platform === 'linux') { const status = readFileSync(`/proc/${pid}/status`, 'utf8') const field = (name) => Number(new RegExp(`^${name}:\\s+(\\d+)`, 'm').exec(status)?.[1] ?? 0) return { fds: readdirSync(`/proc/${pid}/fd`).length, threads: field('Threads'), memory: (field('VmRSS') + field('VmSwap')) * 1024 } } const out = (cmd, ...args) => execFileSync(cmd, args, { encoding: 'utf8' }) out('footprint', '-p', pid, '-j', footprintPath) const fp = JSON.parse(readFileSync(footprintPath, 'utf8')) return { fds: out('lsof', '-n', '-P', '-a', '-p', pid, '-d', '0-1000000', '-F', 'f').split('\n').filter((l) => l.startsWith('f')).length, threads: out('ps', '-M', '-p', pid).trim().split('\n').length - 1, memory: fp.processes[0].footprint * fp['bytes per unit'], } } await cohort() await sleep(IDLE_MS) const warmed = holds() for (let c = 0; c < COHORTS; c++) await cohort() const within = (h) => h.fds <= warmed.fds && h.threads <= warmed.threads && h.memory <= warmed.memory + MEMORY_SLACK // The least of each reading: what is held for good is in every one, and // what the server starts and ends on its own in only some. const least = (a, b) => ({ fds: Math.min(a.fds, b.fds), threads: Math.min(a.threads, b.threads), memory: Math.min(a.memory, b.memory) }) const settling = Date.now() let left = holds() while (!within(left) && Date.now() - settling < SETTLE_MS) { await sleep(250) left = least(left, holds()) } const settled = ((Date.now() - settling) / 1000).toFixed(1) const mib = (x) => `${(x / 1048576).toFixed(1)} MiB` const held = [ ['descriptors', warmed.fds, left.fds, left.fds <= warmed.fds], ['threads', warmed.threads, left.threads, left.threads <= warmed.threads], ['memory', mib(warmed.memory), `${mib(left.memory)} (at most ${mib(MEMORY_SLACK)} more)`, left.memory <= warmed.memory + MEMORY_SLACK], ] console.error(`\nsubscribers come and go, ${COHORTS} cohorts of ${COHORT} after one to warm; what the server holds ${IDLE_MS / 1000} s after the first, and ${settled} s after the last:\n`) for (const [what, a, b, ok] of held) console.error(`${what.padEnd(14)}${String(a).padStart(12)}${String(b).padStart(36)} ${ok ? 'ok' : 'FAIL'}`) for (const [what, a, b, ok] of held) if (!ok) failed.push(`${what}: ${b} after the last cohort, ${a} after the first`) if (failed.length) console.error(`\nthe server is slower, or holds more, than spec/OPERATIONS.md, "Performance", allows:\n ${failed.join('\n ')}`) else console.error(`\nall ${rows.length} parts within the baseline's and the reference's (p99 ratio at most ${P99_FACTOR} ×, throughput ratio at least ${RATE_FACTOR} ×), and what ${COHORTS * COHORT} subscribers held given back`) stop(failed.length ? 1 : 0) } /// Three significant figures. function sig(x) { return Number(x.toPrecision(3)) }