chadtmiller.com/monolith
An atproto PDS
0
0
// The store benchmark's third part (README.md): how the server's start, its
// memory, and the compaction of its service log follow the number of
// accounts. Prints a Markdown table.
//
// node service.mjs [accounts...] by default 10000 30000 100000
//
// Each count runs twice, its accounts without and with an email and a
// password. The accounts are not made through the server, which would take
// a PLC operation and a password hash each: their entries are written into
// the service log, `accounts.0.log`, as the server writes them (frames of
// `account` entries, `pds-server`'s `log`), none with a repository yet, so
// this knows the log's format and breaks when it changes. The server is
// `../../target/release/pds` (`just build`), or `PDS_BIN`; its data
// directories are under `BENCH_DIR`, by default `target/data` here, and are
// removed after.
import { execFileSync, spawn } from 'node:child_process'
import { createHash } from 'node:crypto'
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, statSync, writeSync } from 'node:fs'
import { createServer } from 'node:net'
import { join, resolve } from 'node:path'
const here = new URL('.', import.meta.url).pathname
const BIN = resolve(process.env.PDS_BIN ?? join(here, '../../target/release/pds'))
const ROOT = resolve(process.env.BENCH_DIR ?? join(here, 'target/data'))
const COUNTS = process.argv.slice(2).map(Number)
if (COUNTS.some((n) => !(n >= 1))) {
console.error('node service.mjs [accounts...]')
process.exit(2)
}
if (!COUNTS.length) COUNTS.push(10000, 30000, 100000)
if (!existsSync(BIN)) {
console.error(`no server at ${BIN}: \`just build\` first, or set PDS_BIN`)
process.exit(1)
}
/// The cache the scale run gives the server (spec/STORE.md, "How it is
/// checked"), and the memory rule's other parts.
const CACHE_BYTES = 16 * 1048576
const REQUEST_SLACK = 32 * 1048576
const PER_ACCOUNT = 4096
const START_SLACK = 250
const ADMIN = 'admin-pass'
const sha256 = (...parts) => parts.reduce((h, p) => h.update(p), createHash('sha256')).digest()
const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
const MiB = (bytes) => (bytes / 1048576).toFixed(1)
/// A frame of a log (`pds-store`): its payload's length, the first 8 bytes
/// of SHA-256 over the length and the payload, and the payload, a JSON
/// array of entries.
function frame(entries) {
const payload = Buffer.from(JSON.stringify(entries))
const head = Buffer.alloc(4)
head.writeUInt32LE(payload.length)
return Buffer.concat([head, sha256(head, payload).subarray(0, 8), payload])
}
/// The kinds of the entries in a log's frames.
function kinds(data) {
const out = new Set()
for (let pos = 0; data.length - pos >= 12; ) {
const n = data.readUInt32LE(pos)
for (const e of JSON.parse(data.subarray(pos + 12, pos + 12 + n))) out.add(e.t)
pos += 12 + n
}
return out
}
const B32 = 'abcdefghijklmnopqrstuvwxyz234567'
function base32(bytes, chars) {
let out = ''
for (let i = 0, bits = 0, acc = 0; out.length < chars; ) {
if (bits < 5) {
acc = (acc << 8) | bytes[i++]
bits += 8
}
out += B32[(acc >> (bits - 5)) & 31]
bits -= 5
}
return out
}
/// Account `i`'s entry, as `createAccount` writes it.
function account(i, emails, url) {
const h = sha256(String(i))
const did = `did:plc:${base32(h, 24)}`
const handle = `user${i}.test`
return {
t: 'account', did, handle,
email: emails ? `user${i}@example.com` : null,
password: emails ? `scrypt$${sha256('salt', h).toString('hex')}$${sha256('hash', h).toString('hex')}` : null,
key: sha256('key', h).toString('hex'),
kemKey: null,
didDoc: {
'@context': ['https://www.w3.org/ns/did/v1', 'https://w3id.org/security/multikey/v1', 'https://w3id.org/security/suites/secp256k1-2019/v1'],
id: did,
alsoKnownAs: [`at://${handle}`],
verificationMethod: [{ id: `${did}#atproto`, type: 'Multikey', controller: did, publicKeyMultibase: `zQ3sh${base32(sha256('pub', h), 43)}` }],
service: [{ id: '#atproto_pds', type: 'AtprotoPersonalDataServer', serviceEndpoint: url }],
},
}
}
const freePort = () => new Promise((ok) => {
const s = createServer().listen(0, '127.0.0.1', () => {
const { port } = s.address()
s.close(() => ok(port))
})
})
/// The server's resident memory, in bytes, as `ps` reports it.
const rss = (pid) => Number(execFileSync('ps', ['-o', 'rss=', '-p', String(pid)]).toString().trim()) * 1024
/// One count of accounts, with emails and passwords or without: a row.
async function measure(n, emails) {
const dir = join(ROOT, `service-${n}-${emails ? 'emails' : 'plain'}`)
rmSync(dir, { recursive: true, force: true })
mkdirSync(dir, { recursive: true })
const port = await freePort()
const url = `http://127.0.0.1:${port}`
const env = {
PATH: process.env.PATH, PDS_PORT: String(port), PDS_DATA_DIR: dir, PDS_HOSTNAME: 'localhost', PDS_PUBLIC_URL: url,
PDS_HANDLE_DOMAINS: '.test', PDS_PLC_URL: 'http://127.0.0.1:9', PDS_PLC_ROTATION_KEY_HEX: sha256('rotation').toString('hex'),
PDS_ADMIN_PASSWORD: ADMIN, PDS_INVITE_REQUIRED: 'false', PDS_RATE_LIMITS_ENABLED: 'false', PDS_CACHE_BYTES: String(CACHE_BYTES),
}
const out = openSync(`${dir}.out`, 'w')
/// Start the server: answers it, and how long until it answered describeServer.
const start = async () => {
const at = performance.now()
const proc = spawn(BIN, ['serve'], { env, stdio: ['ignore', out, out] })
let exited = null
proc.on('exit', (code) => (exited = code ?? 'killed'))
for (;;) {
if (exited !== null) throw new Error(`the server exited ${exited}: ${readFileSync(`${dir}.out`, 'utf8').slice(-2000)}`)
try {
const res = await fetch(`${url}/xrpc/com.atproto.server.describeServer`)
await res.arrayBuffer()
if (res.ok) return { proc, ms: performance.now() - at }
} catch {}
await sleep(2)
}
}
const stop = (proc) => new Promise((ok) => {
proc.on('exit', ok)
proc.kill('SIGTERM')
})
// An empty data directory: once unmeasured, as a binary's first run may
// be slow, then three times, the slowest start and its memory what the
// others are held against, as the scale run holds them.
await stop((await start()).proc)
let t0 = 0
let r0 = 0
for (let i = 0; i < 3; i++) {
const { proc, ms } = await start()
t0 = Math.max(t0, ms)
await sleep(300)
r0 = rss(proc.pid)
await stop(proc)
}
// The accounts, into the service log; and seqs reserved, as a data
// directory that ever numbered an event has them, so that no start reads
// every account's own log for the last.
const log = join(dir, 'accounts.0.log')
const fd = openSync(log, 'a')
if (!kinds(readFileSync(log)).has('seqs')) writeSync(fd, frame([{ t: 'seqs', upto: 1024 }]))
for (let i = 0; i < n; i += 32) {
writeSync(fd, frame(Array.from({ length: Math.min(32, n - i) }, (_, j) => account(i + j, emails, url))))
}
closeSync(fd)
const logBytes = statSync(log).size
// Started on that log, all of it replayed: a log no server is left with
// once it is past what it compacts at, so this is what replaying one
// costs, not a start it is held to.
const first = await start()
await sleep(300)
// One write to the service log makes its compaction due: its memory is
// sampled until the checkpoint is in place and the log it covers gone.
const checkpoint = join(dir, 'accounts.checkpoint')
const res = await fetch(`${url}/xrpc/com.atproto.server.createInviteCode`, {
method: 'POST', body: '{"useCount":1}',
headers: { 'content-type': 'application/json', authorization: `Basic ${Buffer.from(`admin:${ADMIN}`).toString('base64')}` },
})
if (!res.ok) throw new Error(`createInviteCode: ${res.status} ${await res.text()}`)
const at = performance.now()
let peak = rss(first.proc.pid)
let folded = null
while (performance.now() - at < 300_000) {
peak = Math.max(peak, rss(first.proc.pid))
if (existsSync(checkpoint) && !existsSync(log) && !existsSync(`${checkpoint}.tmp`)) {
folded = performance.now() - at
break
}
await sleep(5)
}
await stop(first.proc)
// Started on the checkpoint, as a server that holds these accounts starts.
const second = await start()
await sleep(300)
const idle = rss(second.proc.pid)
await stop(second.proc)
closeSync(out)
rmSync(dir, { recursive: true, force: true })
rmSync(`${dir}.out`, { force: true })
const ms = (x) => (x === null ? 'never' : x.toFixed(0))
const startBound = 2 * t0 + START_SLACK
const rssBound = r0 + REQUEST_SLACK + CACHE_BYTES + PER_ACCOUNT * n
const over = (x, bound, text) => (x > bound ? `**${text}**` : text)
console.log(`| ${n} | ${emails ? 'yes' : 'no'} | ${MiB(logBytes)} | ${ms(first.ms)} | ${over(second.ms, startBound, ms(second.ms))} | ${ms(startBound)} | ${MiB(idle)} | ${ms(folded)} | ${over(peak, rssBound, MiB(peak))} | ${MiB(rssBound)} |`)
}
console.log('| accounts | email and password | service log MiB | that log replayed at a start ms | start on the checkpoint ms | start bound ms | idle RSS MiB | compaction ms | peak RSS compacting MiB | RSS bound MiB |')
console.log('|--:|---|--:|--:|--:|--:|--:|--:|--:|--:|')
for (const n of COUNTS) {
for (const emails of [false, true]) await measure(n, emails)
}