chadtmiller.com/monolith
An atproto PDS
0
0
// The performance step (spec/OPERATIONS.md, "Performance"): one fixed
// workload against the reference PDS and against the server, each in a
// process of its own, part by part, each part in three rounds with the sides
// taking turns; it reports each side's best round of each part, its p50, p99
// and throughput, and the server's ratios to the reference's beside the
// baseline's (perf-baseline.json). It fails a part where the server's p99
// ratio is more than 1.5 times the baseline's (or 1, if lower), or its
// throughput ratio below 0.8 times the baseline's (or 1, if higher). Then,
// against the server alone, subscribers come and go, cohort after cohort,
// and it fails if the server then holds more descriptors, threads or memory
// than after the first cohort.
//
// node perf.mjs -- <server command...>
//
// The server is started as compare.mjs starts it (lib/env.mjs), with no
// rate limits, registering its DIDs with the reference's PLC directory.
import { execFileSync, spawn } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtempSync, openSync, readdirSync, readFileSync } from 'node:fs'
import { createServer } from 'node:net'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { createInterface } from 'node:readline'
/// The factor of the baseline's p99 ratio (or 1, if lower) the server's may
/// be.
const P99_FACTOR = 1.5
/// The factor of the baseline's throughput ratio (or 1, if higher) the
/// server's must reach: run-to-run noise of a few percent, on parts bound by
/// the same work on both sides, would turn a floor of 1 into flakes.
const RATE_FACTOR = 0.8
/// Each part's ratios, the server's best over the reference's, as nightly
/// runs on the runner measured them, the weakest of several.
const BASELINE = JSON.parse(readFileSync(new URL('./perf-baseline.json', import.meta.url), 'utf8')).parts
/// Subscribers come and go: cohorts of this many, after one to warm.
const COHORT = 20
const COHORTS = 20
/// How much more memory the server may hold after the last cohort than
/// after the first: what the allocator keeps of what it freed.
const MEMORY_SLACK = 8 * 1048576
/// How long the server is left alone after the first cohort before what it
/// holds is read: idle threads the parts before used end after 10 s, and
/// what they held goes back with them.
const IDLE_MS = 15000
/// How long the server has, after the last cohort, to give back what it held.
const SETTLE_MS = 20000
// The reference, in a process of its own: started by the run, it says where
// it listens on its first line, and stops when its input closes.
if (process.argv[2] === '--reference') {
const { TestNetworkNoAppView } = await import('@atproto/dev-env')
const net = await TestNetworkNoAppView.create({ pds: { hostname: 'localhost', rateLimitsEnabled: false, inviteRequired: false } })
process.stdout.write(`${JSON.stringify({ url: net.pds.url, plcUrl: net.plc.url, recoveryKey: net.pds.ctx.cfg.identity.recoveryDidKey })}\n`)
process.stdin.resume()
process.stdin.on('end', async () => {
await net.close()
process.exit(0)
})
} else await run()
async function run() {
const { command } = await import('./lib/args.mjs')
const { below } = await import('./lib/random.mjs')
const { serverEnv } = await import('./lib/env.mjs')
const { cborDecodeMulti } = await import('@atproto/common')
const freePort = () => new Promise((ok) => {
const s = createServer()
s.listen(0, '127.0.0.1', () => {
const { port } = s.address()
s.close(() => ok(port))
})
})
const refProc = spawn(process.execPath, [new URL(import.meta.url).pathname, '--reference'], { stdio: ['pipe', 'pipe', 'ignore'] })
const first = await new Promise((ok) => createInterface({ input: refProc.stdout }).once('line', ok))
const { url: refUrl, plcUrl, recoveryKey } = JSON.parse(first)
const port = await freePort()
const dataDir = mkdtempSync(join(tmpdir(), 'pds-perf-'))
const logPath = join(mkdtempSync(join(tmpdir(), 'pds-perf-log-')), 'server.log')
const { env } = await serverEnv({ port, dataDir, plc: plcUrl, recoveryKey })
const log = openSync(logPath, 'a')
const serverProc = spawn(command[0], command.slice(1), { env: { ...process.env, ...env }, stdio: ['ignore', log, log] })
const serverUrl = `http://127.0.0.1:${port}`
const stop = (code) => {
serverProc.kill('SIGKILL')
refProc.stdin.end()
process.exit(code)
}
for (const event of ['uncaughtException', 'unhandledRejection'])
process.on(event, (err) => {
console.error(err)
stop(1)
})
for (const started = Date.now(); ; ) {
try {
if ((await fetch(`${serverUrl}/xrpc/com.atproto.server.describeServer`)).ok) break
} catch {}
if (serverProc.exitCode !== null || Date.now() - started > 30000) {
console.error(`the server did not start; its log is ${logPath}`)
stop(1)
}
await new Promise((ok) => setTimeout(ok, 20))
}
const sides = [
{ name: 'reference', url: refUrl, accounts: [], records: [] },
{ name: 'server', url: serverUrl, accounts: [], records: [] },
]
/// One request, timed from sending it to having its answer whole;
/// anything but a 2xx stops the run.
async function call(side, nsid, { params, body, bytes, token, type } = {}) {
const url = new URL(`${side.url}/xrpc/${nsid}`)
for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v)
const headers = {}
if (token) headers.authorization = `Bearer ${token}`
if (body !== undefined) headers['content-type'] = 'application/json'
if (bytes) headers['content-type'] = type
const t = performance.now()
const r = await fetch(url, { method: body !== undefined || bytes ? 'POST' : 'GET', headers, body: bytes ?? (body === undefined ? undefined : JSON.stringify(body)) })
const buf = Buffer.from(await r.arrayBuffer())
const ms = performance.now() - t
if (!r.ok) {
console.error(`${side.name}: ${nsid} answered ${r.status}: ${buf.toString('utf8').slice(0, 300)}${side.name === 'server' ? `\nserver log: ${logPath}` : ''}`)
stop(1)
}
return { ms, json: () => JSON.parse(buf.toString('utf8')) }
}
/// Requests `lo` to `hi`, `at` at once, each made by `make(i)`: their
/// times, and how long they took together.
async function load(lo, hi, at, make) {
const times = []
let next = lo
const t = performance.now()
await Promise.all(Array.from({ length: at }, async () => {
while (next < hi) times.push(await make(next++))
}))
return { times, total: performance.now() - t }
}
const ACCOUNTS = 40
/// Each part: how many requests, and a round of them, `lo` to `hi`, on a
/// side.
const PARTS = {
createAccount: [ACCOUNTS, (side, lo, hi) => load(lo, hi, 4, async (i) => {
const r = await call(side, 'com.atproto.server.createAccount', { body: { handle: `perf${i}.test`, email: `perf${i}@example.com`, password: `perf-password-${i}` } })
const { did, accessJwt } = r.json()
side.accounts[i] = { did, token: accessJwt }
return r.ms
})],
createRecord: [2000, (side, lo, hi) => load(lo, hi, 16, async (i) => {
const a = side.accounts[i % ACCOUNTS]
const r = await call(side, 'com.atproto.repo.createRecord', { body: { repo: a.did, collection: 'com.example.note', record: { $type: 'com.example.note', text: `a note of about a hundred bytes, number ${i}, to read back later`, n: i } }, token: a.token })
side.records[i] = { repo: a.did, rkey: r.json().uri.split('/').pop() }
return r.ms
})],
getRecord: [4000, (side, lo, hi, picks) => load(lo, hi, 16, async (i) => (await call(side, 'com.atproto.repo.getRecord', { params: { repo: side.records[picks[i]].repo, collection: 'com.example.note', rkey: side.records[picks[i]].rkey } })).ms)],
getRepo: [200, (side, lo, hi) => load(lo, hi, 8, async (i) => (await call(side, 'com.atproto.sync.getRepo', { params: { did: side.accounts[i % ACCOUNTS].did } })).ms)],
firehose: [200, async (side, lo, hi) => {
// Each from an address of its own, as 20 clients would be: the server
// holds 16 at most from one (spec/LIMITS.md).
const subs = await Promise.all(Array.from({ length: 20 }, (_, n) => new Promise((ok, fail) => {
const ws = new WebSocket(`${side.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`, { headers: { 'x-forwarded-for': `198.18.0.${n + 1}` } })
ws.binaryType = 'arraybuffer'
const sub = { ws, got: new Map() }
ws.onmessage = (m) => {
const [header, body] = cborDecodeMulti(new Uint8Array(m.data))
if (header?.t !== '#commit') return
for (const op of body.ops ?? []) {
const k = /\/fh(\d+)x$/.exec(op.path)
if (k && !sub.got.has(Number(k[1]))) sub.got.set(Number(k[1]), performance.now())
}
}
ws.onopen = () => ok(sub)
ws.onerror = () => fail(new Error(`${side.name}: subscribeRepos`))
})))
const sentAt = []
const a = side.accounts[0]
const t = performance.now()
for (let i = lo; i < hi; i++) {
sentAt[i] = performance.now()
await call(side, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `fh${i}x`, record: { $type: 'com.example.note', text: 'fan-out', n: i } }, token: a.token })
}
for (let w = 0; w < 100 && subs.some((s) => s.got.size < hi - lo); w++) await new Promise((ok) => setTimeout(ok, 100))
for (const s of subs) s.ws.close()
const times = []
let last = t
for (const s of subs) for (let i = lo; i < hi; i++) {
const at = s.got.get(i)
if (at === undefined) {
console.error(`${side.name}: a subscriber did not get the commit of write ${i} within 10 s`)
stop(1)
}
times.push(at - sentAt[i])
last = Math.max(last, at)
}
return { times, total: last - t }
}],
uploadBlob: [200, (side, lo, hi) => {
const blob = (i) => {
const out = Buffer.alloc(65536)
for (let o = 0; o < out.length; o += 32) createHash('sha256').update(`blob ${i} ${o}`).digest().copy(out, o)
return out
}
return load(lo, hi, 8, async (i) => (await call(side, 'com.atproto.repo.uploadBlob', { bytes: blob(i), type: 'application/octet-stream', token: side.accounts[i % ACCOUNTS].token })).ms)
}],
}
const pct = (xs, p) => {
const s = [...xs].sort((x, y) => x - y)
return s[Math.min(s.length - 1, Math.ceil((p / 100) * s.length) - 1)]
}
// Each part in three rounds, the sides taking turns; each side judged by
// its best round, as other work on the machine only ever slows a round.
const ROUNDS = 3
const rows = []
const failed = []
for (const [part, [n, fn]] of Object.entries(PARTS)) {
const picks = part === 'getRecord' ? Array.from({ length: n }, () => below(2000)) : null
const rounds = { reference: [], server: [] }
for (let k = 0; k < ROUNDS; k++) {
const [lo, hi] = [Math.floor((n * k) / ROUNDS), Math.floor((n * (k + 1)) / ROUNDS)]
for (const side of sides) {
const { times, total } = await fn(side, lo, hi, picks)
rounds[side.name].push({ p50: pct(times, 50), p99: pct(times, 99), rate: (times.length / total) * 1000 })
}
}
const best = (rs) => ({ p50: Math.min(...rs.map((x) => x.p50)), p99: Math.min(...rs.map((x) => x.p99)), rate: Math.max(...rs.map((x) => x.rate)) })
const [r, s] = [best(rounds.reference), best(rounds.server)]
// The ratios, held to the baseline's, and never to less than the
// reference's own.
const got = { p99: s.p99 / r.p99, rate: s.rate / r.rate }
const base = BASELINE[part] ?? { p99: 1, rate: 1 }
const [p99Of, rateOf] = [Math.min(base.p99, 1), Math.max(base.rate, 1)]
const why = []
if (got.p99 > P99_FACTOR * p99Of) why.push(`p99 ratio ${sig(got.p99)} (${s.p99.toFixed(2)} ms to the reference's ${r.p99.toFixed(2)} ms), over ${P99_FACTOR} × ${sig(p99Of)}`)
if (got.rate < RATE_FACTOR * rateOf) why.push(`throughput ratio ${sig(got.rate)} (${s.rate.toFixed(0)}/s to the reference's ${r.rate.toFixed(0)}/s), below ${RATE_FACTOR} × ${sig(rateOf)}`)
if (why.length) failed.push(`${part}: ${why.join('; ')}`)
rows.push({ part, r, s, verdict: why.length ? 'FAIL' : 'ok', rounds, got, base })
}
const f = (x) => x.toFixed(1).padStart(8)
console.error(`each side's best of ${ROUNDS} rounds, the sides taking turns, and the server's ratios to the reference's (the baseline's):\n`)
console.error(`${'part'.padEnd(14)}${'reference p50'.padStart(14)}${'p99'.padStart(9)}${'per s'.padStart(9)} ${'server p50'.padStart(11)}${'p99'.padStart(9)}${'per s'.padStart(9)} ${'p99 ratio'.padStart(18)}${'per s ratio'.padStart(18)}`)
for (const { part, r, s, verdict, got, base } of rows) console.error(`${part.padEnd(14)}${f(r.p50).padStart(14)}${f(r.p99).padStart(9)}${r.rate.toFixed(0).padStart(9)} ${f(s.p50).padStart(11)}${f(s.p99).padStart(9)}${s.rate.toFixed(0).padStart(9)} ${`${sig(got.p99)} (${sig(base.p99)})`.padStart(18)}${`${sig(got.rate)} (${sig(base.rate)})`.padStart(18)} ${verdict}`)
console.error('\neach round\'s p99, in ms:')
for (const { part, rounds } of rows) console.error(`${part.padEnd(14)}reference ${rounds.reference.map((x) => x.p99.toFixed(1)).join(', ')}; server ${rounds.server.map((x) => x.p99.toFixed(1)).join(', ')}`)
console.error(`\nthe ratios, as perf-baseline.json holds them: ${JSON.stringify(Object.fromEntries(rows.map(({ part, got }) => [part, { p99: sig(got.p99), rate: sig(got.rate) }])))}`)
// Subscribers come and go, against the server alone: what it holds for
// a cohort of subscribers it gives back once they have left.
const server = sides[1]
const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
let cohorts = 0
async function cohort() {
const k = cohorts++
// Each from an address of its own, the same 20 each time: the server
// holds 16 at most from one (spec/LIMITS.md).
const subs = await Promise.all(Array.from({ length: COHORT }, (_, n) => new Promise((ok, fail) => {
const ws = new WebSocket(`${server.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`, { headers: { 'x-forwarded-for': `198.18.1.${n + 1}` } })
ws.binaryType = 'arraybuffer'
const sub = { ws, got: false, closed: new Promise((c) => ws.addEventListener('close', c)) }
ws.onmessage = (m) => {
const [header, body] = cborDecodeMulti(new Uint8Array(m.data))
if (header?.t === '#commit' && (body.ops ?? []).some((op) => op.path === `com.example.note/churn${k}`)) sub.got = true
}
ws.onopen = () => ok(sub)
ws.onerror = () => fail(new Error('server: subscribeRepos'))
})))
const a = server.accounts[0]
await call(server, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `churn${k}`, record: { $type: 'com.example.note', text: 'come and go', n: k } }, token: a.token })
for (const t = Date.now(); subs.some((s) => !s.got); await sleep(5))
if (Date.now() - t > 10000) {
console.error(`server: a subscriber of cohort ${k} did not get the commit of its write within 10 s`)
stop(1)
}
for (const s of subs) s.ws.close()
if (!(await Promise.race([Promise.all(subs.map((s) => s.closed)).then(() => true), sleep(10000).then(() => false)]))) {
console.error(`server: a subscriber of cohort ${k} did not see its connection closed within 10 s of closing it`)
stop(1)
}
}
/// What the server process holds: its open file descriptors, threads and
/// memory, resident or swapped out (Linux), or its footprint, resident or
/// compressed (macOS), in bytes.
const pid = String(serverProc.pid)
const footprintPath = join(mkdtempSync(join(tmpdir(), 'pds-perf-footprint-')), 'footprint.json')
function holds() {
if (process.platform === 'linux') {
const status = readFileSync(`/proc/${pid}/status`, 'utf8')
const field = (name) => Number(new RegExp(`^${name}:\\s+(\\d+)`, 'm').exec(status)?.[1] ?? 0)
return { fds: readdirSync(`/proc/${pid}/fd`).length, threads: field('Threads'), memory: (field('VmRSS') + field('VmSwap')) * 1024 }
}
const out = (cmd, ...args) => execFileSync(cmd, args, { encoding: 'utf8' })
out('footprint', '-p', pid, '-j', footprintPath)
const fp = JSON.parse(readFileSync(footprintPath, 'utf8'))
return {
fds: out('lsof', '-n', '-P', '-a', '-p', pid, '-d', '0-1000000', '-F', 'f').split('\n').filter((l) => l.startsWith('f')).length,
threads: out('ps', '-M', '-p', pid).trim().split('\n').length - 1,
memory: fp.processes[0].footprint * fp['bytes per unit'],
}
}
await cohort()
await sleep(IDLE_MS)
const warmed = holds()
for (let c = 0; c < COHORTS; c++) await cohort()
const within = (h) => h.fds <= warmed.fds && h.threads <= warmed.threads && h.memory <= warmed.memory + MEMORY_SLACK
// The least of each reading: what is held for good is in every one, and
// what the server starts and ends on its own in only some.
const least = (a, b) => ({ fds: Math.min(a.fds, b.fds), threads: Math.min(a.threads, b.threads), memory: Math.min(a.memory, b.memory) })
const settling = Date.now()
let left = holds()
while (!within(left) && Date.now() - settling < SETTLE_MS) {
await sleep(250)
left = least(left, holds())
}
const settled = ((Date.now() - settling) / 1000).toFixed(1)
const mib = (x) => `${(x / 1048576).toFixed(1)} MiB`
const held = [
['descriptors', warmed.fds, left.fds, left.fds <= warmed.fds],
['threads', warmed.threads, left.threads, left.threads <= warmed.threads],
['memory', mib(warmed.memory), `${mib(left.memory)} (at most ${mib(MEMORY_SLACK)} more)`, left.memory <= warmed.memory + MEMORY_SLACK],
]
console.error(`\nsubscribers come and go, ${COHORTS} cohorts of ${COHORT} after one to warm; what the server holds ${IDLE_MS / 1000} s after the first, and ${settled} s after the last:\n`)
for (const [what, a, b, ok] of held) console.error(`${what.padEnd(14)}${String(a).padStart(12)}${String(b).padStart(36)} ${ok ? 'ok' : 'FAIL'}`)
for (const [what, a, b, ok] of held) if (!ok) failed.push(`${what}: ${b} after the last cohort, ${a} after the first`)
if (failed.length) console.error(`\nthe server is slower, or holds more, than spec/OPERATIONS.md, "Performance", allows:\n ${failed.join('\n ')}`)
else console.error(`\nall ${rows.length} parts within the baseline's and the reference's (p99 ratio at most ${P99_FACTOR} ×, throughput ratio at least ${RATE_FACTOR} ×), and what ${COHORTS * COHORT} subscribers held given back`)
stop(failed.length ? 1 : 0)
}
/// Three significant figures.
function sig(x) {
return Number(x.toPrecision(3))
}