chadtmiller.com/monolith
An atproto PDS
0
0
// A benchmark of PDS implementations, one at a time, on one machine, every
// one in Docker the same way: perf.mjs's workload with applyWrites,
// listRecords and more besides; the routes a client calls most; reads and
// writes by the number of callers; a mix of them at once; and subscribers
// coming and going. Each round is a fresh server on fresh data with a fresh
// PLC directory in memory, warmed the same way first. It reports each part's
// throughput, p50 and tail (best and median of the rounds), the server's
// start time (container start to the first answered describeServer), its RSS
// idle and at peak, and what the client itself can send (its ceiling). Then,
// apart, a storage round of each: a fresh server with a short backfill window
// W, the same warm-up and parts, and what it keeps on disk under load (once
// the parts end) and at rest (after a quiet of 2 W), against the accounts'
// live state (spec/STORE.md, "Bounds").
//
// node bench.mjs --setup (build and pull everything bench/impls.json pins)
// node bench.mjs [--rounds 3] [--storage-rounds 1] [--backfill-ms 10000] [--impls monolith,reference,vlpds] [--clients 4] [--busy] [--json out.json]
// node bench.mjs --report out.json (the tables again from a run's --json)
//
// Each implementation is an adapter in ADAPTERS: how to make its container
// against a PLC directory, on a port, and start and stop it; what to send
// createAccount; and what it is (version, commit, storage). To add one, add
// an adapter and its pin in bench/impls.json (bench/README.md); a part it
// does not serve goes in its `unsupported`, with why. A route a server does
// not serve is found in the warm-up, and its answer is the table's why.
import { spawnSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync, writeSync } from 'node:fs'
import { cpus, loadavg, release, totalmem, tmpdir, type } from 'node:os'
import { join } from 'node:path'
import { openPool, timedFetch } from './lib/loadpool.mjs'
import { VLPDS_MEM_MIB, containerLogs, containerPssKiB, containerRssKiB, dockerPort, fromDocker, imageId, pins, root, sh, shOk, startPostgres, startRustfs, tranquilCreateArgs, tranquilEnv, tranquilOwnEnv, vlpdsArgs, vlpdsCreateArgs, vlpdsEnv, vlpdsOwnEnv, zdsCreateArgs, zdsEnv, zdsOwnEnv } from './lib/impls.mjs'
const here = new URL('.', import.meta.url).pathname
// What each implementation runs, pinned: bench/impls.json (lib/impls.mjs).
const vendor = join(root, 'bench', 'vendor')
const pinnedImages = () => [...Object.values(pins.implementations).map((x) => x.image), ...Object.values(pins.images).map((x) => x.image)].filter(Boolean)
// The label an image built from this repository carries: the last commit
// that changed what it is built from (its pin's `inputs`), `-dirty` if any of
// those has changes not committed. A change elsewhere, as to the bench
// itself, leaves an image as good as it was.
const REVISION = 'org.opencontainers.image.revision'
const revision = (inputs) => {
const git = (...args) => spawnSync('git', [...args, '--', ...inputs], { cwd: root, encoding: 'utf8' }).stdout.trim()
return `${git('log', '-1', '--format=%H')}${git('status', '--porcelain') ? '-dirty' : ''}`
}
// The firehose under load: its writes, this many at once.
const LOADED_WRITERS = 16
// The numbers of callers each of the curves is measured at.
const CALLERS = [1, 4, 16, 64, 256]
// The concurrencies the client's ceiling is measured at: every one a part
// uses.
const CEILING_AT = [1, 4, 8, 16, 32, 64, 256]
// A part whose best rate is at least this share of the client's ceiling at
// the same concurrency may be held back by the client.
const NEAR_CEILING = 0.5
/// --setup: everything the bench runs, from the pins alone. Run again, it
/// pulls no image it has and builds only what changed.
function setup() {
const fail = (why) => {
console.error(`setup: ${why}`)
process.exit(1)
}
const run = (cmd, args, cwd = root) => {
console.error(`setup: (${cwd}) ${cmd} ${args.join(' ')}`)
const r = spawnSync(cmd, args, { cwd, stdio: 'inherit' })
if (r.error?.code === 'ENOENT') fail(`${cmd} is not installed (see bench/README.md, Prerequisites)`)
if (r.status !== 0) fail(`${cmd} ${args.join(' ')} failed (exit ${r.status})`)
}
const quiet = (cmd, args, cwd = root) => spawnSync(cmd, args, { cwd, encoding: 'utf8' })
const major = Number(process.versions.node.split('.')[0])
if (major < 24) fail(`Node ${process.versions.node}: the bench needs Node 24 or later`)
const needGit = Object.values(pins.implementations).some((x) => x.repo)
for (const [cmd, args, what] of [['rustup', ['--version'], 'rustup (Rust toolchains)'], ['npm', ['--version'], 'npm'], ['docker', ['--version'], 'Docker'], ...(needGit ? [['git', ['--version'], 'git']] : [])]) {
const r = quiet(cmd, args)
if (r.error || r.status !== 0) fail(`${what} is missing: \`${cmd} ${args.join(' ')}\` failed (see bench/README.md, Prerequisites)`)
}
if (quiet('docker', ['info']).status !== 0) fail('Docker is installed but its daemon does not answer (`docker info` failed): start Docker')
// Built here, outside Docker: what the compatibility check runs
// (compat.mjs).
for (const impl of Object.values(pins.implementations)) {
if (!impl.build) continue
const cwd = join(root, impl.cwd ?? '')
// Rust builds go through rustup at the toolchain pinned, whatever
// `cargo` is first on the PATH.
if (impl.toolchain) run('rustup', ['toolchain', 'install', impl.toolchain, '--profile', 'minimal', '--no-self-update'], cwd)
run(impl.build[0], impl.build.slice(1), cwd)
}
// Built from this repository into an image, every time: Docker's own
// cache leaves nothing to do when nothing changed. Labelled with the
// commit of what it is built from, so a run can tell an image older than it.
for (const impl of Object.values(pins.implementations)) {
if (!impl.context) continue
run('docker', ['build', '-t', impl.tag, '-f', join(root, impl.dockerfile), '--label', `${REVISION}=${revision(impl.inputs)}`, join(root, impl.context)])
}
// Built from another repository at its pinned commit, with its own
// Dockerfile, unless an image of that tag is there already.
for (const [name, impl] of Object.entries(pins.implementations)) {
if (!impl.repo || quiet('docker', ['image', 'inspect', impl.tag]).status === 0) continue
const dir = join(vendor, name)
if (!existsSync(join(dir, '.git'))) {
mkdirSync(dir, { recursive: true })
run('git', ['init', '-q'], dir)
}
if (quiet('git', ['rev-parse', 'HEAD'], dir).stdout?.trim() !== impl.commit) {
run('git', ['fetch', '-q', '--depth', '1', impl.repo, impl.commit], dir)
run('git', ['checkout', '-q', '--detach', impl.commit], dir)
}
run('docker', ['build', '-t', impl.tag, '-f', join(dir, impl.dockerfile), dir])
}
for (const image of pinnedImages()) {
if (quiet('docker', ['image', 'inspect', image]).status !== 0) run('docker', ['pull', image])
}
console.error('setup: done')
}
if (process.argv[2] === '--setup') setup()
// The tables again from a run's --json, without running anything.
else if (process.argv[2] === '--report') process.stdout.write(markdown(JSON.parse(readFileSync(process.argv[3], 'utf8'))))
else await main()
/// The tail a part's times are reported at: a percentile at least ten of
/// them lie above, or the slowest.
function tailOf(n) {
if (n >= 1000) return ['p99', 99]
if (n >= 200) return ['p95', 95]
if (n >= 100) return ['p90', 90]
return ['max', 100]
}
function pct(xs, p) {
const s = Array.from(xs).sort((x, y) => x - y)
return s[Math.min(s.length - 1, Math.max(0, Math.ceil((p / 100) * s.length) - 1))]
}
async function main() {
const { cborDecodeMulti } = await import('@atproto/common')
const { Database, PlcServer } = await import('@did-plc/server')
const { Secp256k1Keypair } = await import('@atproto/crypto')
const arg = (name, fallback) => {
const i = process.argv.indexOf(name)
return i === -1 ? fallback : process.argv[i + 1]
}
const ROUNDS = Number(arg('--rounds', 3))
const CLIENTS = Number(arg('--clients', 4))
// A free port below the ephemeral range, for a server in Docker
// (lib/impls.mjs).
const freePort = dockerPort
const vlpdsMemMiB = VLPDS_MEM_MIB
const rotationHex = async () => Buffer.from(await (await Secp256k1Keypair.create({ exportable: true })).export()).toString('hex')
const envArgs = (env) => Object.entries(env).flatMap(([k, v]) => ['-e', `${k}=${v}`])
const dockerAbout = async () => `Docker ${await sh('docker', ['version', '--format', '{{.Server.Version}}'])}, its Linux VM ${await sh('docker', ['info', '--format', '{{.NCPU}} CPUs, {{.MemTotal}} bytes'])}`
const imageAbout = async (tag) => `${tag} (${await sh('docker', ['image', 'inspect', '--format', '{{.Id}} {{.Os}}/{{.Architecture}}', tag])})`
const imageRevision = (tag) => sh('docker', ['image', 'inspect', '--format', `{{index .Config.Labels "${REVISION}"}}`, tag])
const running = (name) => async () => (await sh('docker', ['inspect', '--format', '{{.State.Running}}', name])) === 'false'
// The storage rounds, after an implementation's speed rounds: each a fresh
// server started with a short backfill window W, where it has a setting
// for one (`window` in its adapter), for its size at rest: after the
// parts, no request for 2 W + 1 s, then one write to each account, then
// the size sampled every 500 ms for 10 s, the least kept (spec/STORE.md,
// "Bounds"). Apart from the speed rounds, which run on each server's
// default window, as a short one changes what replay sends.
const STORAGE_ROUNDS = Number(arg('--storage-rounds', 1))
const BACKFILL_MS = Number(arg('--backfill-ms', 10000))
const QUIET_MS = 2 * BACKFILL_MS + 1000
const REST_SAMPLE_MS = 500
const REST_SAMPLES_MS = 10000
const sleep = (ms) => new Promise((ok) => setTimeout(ok, Math.max(0, ms)))
// vlpds's backfill window is its log segment retention, in whole seconds
// or milliseconds, with a pass every W / 2 (within its 1 s to 10 min).
const vlpdsRetention = (ms) => ({ retention: ms % 1000 ? `${ms}ms` : `${ms / 1000}s`, interval: `${Math.min(600, Math.max(1, Math.round(ms / 2000)))}s` })
/// The sum of the sizes of the files under `dir` in the running container
/// `name` (its volume): find and stat in the container, as root (stat, as
/// BusyBox's find, in RustFS's image, has no -printf); or, where its image
/// has no shell, in a container of the image `reader` given its volumes.
async function containerBytes(name, dir, reader) {
const where = reader ? ['run', '--rm', '-u', '0', '--volumes-from', name, '--entrypoint', 'sh', reader] : ['exec', '-u', '0', name, 'sh']
const out = await shOk('docker', [...where, '-c', 'find "$1" -type f -exec stat -c %s {} + 2>/dev/null; true', 'sh', dir])
return out.split('\n').reduce((sum, l) => sum + (Number(l) || 0), 0)
}
// ---- Adapters ----------------------------------------------------------
// Every server runs in a container of its own, made in `setup` before the
// start clock with a fresh volume; `start` only starts it. Each reaches
// the bench's PLC directory through host.docker.internal, and the bench
// reaches it on a loopback port; each is told its URL is that port.
// `comm` is the server's process in its container, whose RSS and threads
// are the server's; `volume` the directory in it that holds all it keeps
// (its log goes to `docker logs`, not there), and `reader`, where its
// image has no shell to read that with, an image that has one. `setup` is
// given `backfillMs` in a storage round only; `window` says how it sets
// the server's backfill window, or `noWindow` why nothing does.
const ADAPTERS = {
// Monolith as its deployment builds and runs it: the repository's
// Dockerfile (docs/guide/deploying.md), its data on the volume /data,
// given only what a local benchmark needs (as perf.mjs's, lib/env.mjs,
// which this cannot import, as its imports read compare.mjs's
// arguments): no rate limits, no invite codes, every proxy the reference
// trusts trusted; its DID cache as it comes.
'monolith': {
comm: 'pds',
volume: '/data',
window: 'PDS_REPO_BACKFILL_LIMIT_MS',
async info() {
const pin = pins.implementations['monolith']
return {
version: (await sh('docker', ['run', '--rm', pin.tag, '--version'])) || `workspace ${readFileSync(join(root, 'Cargo.toml'), 'utf8').match(/^version = "([^"]+)"/m)?.[1]}`,
commit: await imageRevision(pin.tag),
source: `this repository's Dockerfile, image ${await imageAbout(pin.tag)}; ${await dockerAbout()}`,
storage: 'its own append-only logs and files on the volume /data (spec/STORE.md); blobs as files there',
durability: 'every answered write synced before the answer, group-committed (spec/DURABILITY.md), with Rust\'s `sync_all`/`sync_data`, which on Linux are fsync(2)/fdatasync(2); on the Docker VM\'s virtual disk',
config: `docker run -v /data; the image's PDS_PORT=2583, PDS_BIND_ADDRESS=::, PDS_DATA_DIR=/data; ${Object.entries(monolithEnv({ port: '<port>', plcUrl: 'http://127.0.0.1:<bench PLC port>', rotation: '<fresh>' })).map(([k, v]) => `${k}=${v}`).join(' ')}`,
}
},
async setup({ port, plcUrl, log, backfillMs }) {
const name = `bench-monolith-${process.pid}-${Date.now()}`
const teardown = async () => {
await containerLogs(name, log)
await sh('docker', ['rm', '-f', '-v', name])
}
try {
const env = monolithEnv({ port, plcUrl, rotation: await rotationHex() })
if (backfillMs !== undefined) env.PDS_REPO_BACKFILL_LIMIT_MS = String(backfillMs)
await shOk('docker', ['create', '--name', name, '-p', `127.0.0.1:${port}:2583`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/data', ...envArgs(env), pins.implementations['monolith'].tag])
} catch (e) {
await teardown()
throw e
}
return { name, teardown }
},
async start({ port, aux }) {
await shOk('docker', ['start', aux.name])
return {
url: `http://127.0.0.1:${port}`,
exited: running(aux.name),
async stop() {
// SIGTERM: it shuts down within Docker's 10 s (docs/guide/deploying.md).
await sh('docker', ['stop', aux.name])
},
}
},
unsupported: {},
},
// The reference, as perf.mjs runs it (dev-env's TestPds), in the image
// bench/docker/reference.Dockerfile builds from the harness's packages:
// its data and blobs on the volume /data. It listens inside on the port
// it is reached at, as it did outside Docker.
reference: {
// Node names its main thread's process so.
comm: 'MainThread',
volume: '/data',
window: 'repoBackfillLimitMs (PDS_REPO_BACKFILL_LIMIT_MS)',
async info() {
const pin = pins.implementations.reference
const pkg = (name) => JSON.parse(readFileSync(`${here}node_modules/${name}/package.json`, 'utf8')).version
return {
version: `@atproto/pds ${pkg('@atproto/pds')} (through @atproto/dev-env ${pkg('@atproto/dev-env')}), Node ${await sh('docker', ['run', '--rm', '--entrypoint', 'node', pin.tag, '--version'])}`,
commit: `npm release (no git checkout), packages of harness/package-lock.json as at ${await imageRevision(pin.tag)}`,
source: `npm, pinned in harness/package-lock.json; image ${await imageAbout(pin.tag)}; ${await dockerAbout()}`,
storage: 'SQLite (better-sqlite3): one service DB, a sequencer DB, a DID cache DB, and one DB per account; blobs as files; all on the volume /data',
durability: 'SQLite WAL, synchronous=FULL (better-sqlite3 default): fsync(2) on each commit, on the Docker VM\'s virtual disk; blobs written with fs.writeFile, never fsynced',
config: 'docker run -v /data; PORT=<port>, PLC_URL=http://host.docker.internal:<bench PLC port>, and in the storage rounds only BACKFILL_MS=<W>; dev-env TestPds with rateLimitsEnabled false, inviteRequired false, hostname localhost, and in the storage rounds repoBackfillLimitMs W (harness/bench-reference.mjs)',
}
},
async setup({ port, plcUrl, log, backfillMs }) {
const name = `bench-reference-${process.pid}-${Date.now()}`
const teardown = async () => {
await containerLogs(name, log)
await sh('docker', ['rm', '-f', '-v', name])
}
try {
await shOk('docker', ['create', '--name', name, '-p', `127.0.0.1:${port}:${port}`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/data', ...envArgs({ PORT: port, PLC_URL: fromDocker(plcUrl), ...(backfillMs !== undefined && { BACKFILL_MS: backfillMs }) }), pins.implementations.reference.tag])
} catch (e) {
await teardown()
throw e
}
return { name, teardown }
},
async start({ port, aux }) {
await shOk('docker', ['start', aux.name])
return {
url: `http://127.0.0.1:${port}`,
exited: running(aux.name),
async stop() {
await sh('docker', ['stop', aux.name])
},
}
},
unsupported: {},
},
// Bluesky's Very Large PDS, its release image (bench/impls.json). Its
// only durable state is an S3 bucket: each round gets a fresh RustFS
// container (the store that passed vlpds-bucket-probe, bench/README.md)
// on a Docker network of its own, and a fresh vlpds container on it,
// both made in `setup`. Real secrets (not --dev-mode), fresh each round;
// PLC registration on.
vlpds: {
comm: 'vlpds',
// Its SST cache alone: everything else is in the bucket, beside it.
volume: '/var/lib/vlpds',
window: 'VLPDS_LOG_RETENTION (log segment retention: its firehose backfill window), with VLPDS_LOG_RETENTION_INTERVAL at W / 2 so that a retention pass falls within the quiet (its default 60 s would not)',
async info() {
const pin = pins.implementations.vlpds
const store = pins.images[pin.needs[0]]
return {
version: `${pin.release} (${await sh('docker', ['image', 'inspect', '--format', '{{index .Config.Labels "org.opencontainers.image.version"}}', pin.image])})`,
commit: pin.commit,
source: `release image ${pin.image}; S3 store ${store.image} (${store.release}); ${await dockerAbout()}`,
storage: 'everything in the S3 bucket (RustFS, one drive on a Docker volume): commit log segments, SlateDB state (one shard), blobs under blob/{did}/{cid}; local disk (a Docker volume) only an SST cache',
durability: 'a write is answered once the log segment holding it is PUT to the bucket; uploadBlob answers once the blob is PUT to the bucket. RustFS in its default `strict` durability mode fsyncs object data, xl.meta and the commit renames\' directories before answering a PUT, on the Docker VM\'s virtual disk',
config: `docker run --memory ${vlpdsMemMiB}m --memory-swap ${vlpdsMemMiB}m --stop-timeout 90 -v /var/lib/vlpds; ${Object.entries(vlpdsEnv).map(([k, v]) => `${k}=${v}`).join(' ')}; args ${vlpdsArgs.join(' ')}; VLPDS_PUBLIC_URL=http://127.0.0.1:<port>, VLPDS_PLC_URL=http://host.docker.internal:<bench PLC port>, VLPDS_S3_* the round's RustFS; in the storage rounds only, VLPDS_LOG_RETENTION=${vlpdsRetention(BACKFILL_MS).retention} and VLPDS_LOG_RETENTION_INTERVAL=${vlpdsRetention(BACKFILL_MS).interval} (the bench's backfill window W, and W / 2 between retention passes); fresh random JWT secret, admin and internal tokens, KEK and secp256k1 PLC rotation key each round; handle domain vlpds.test, invites not required, threads (I/O = cores, workers = cores / 2) all its defaults`,
}
},
async setup({ port, plcUrl, log, backfillMs }) {
const id = `bench-vlpds-${process.pid}-${Date.now()}`
const store = `${id}-s3`
const pds = `${id}-pds`
const teardown = async () => {
await containerLogs(pds, log)
await sh('docker', ['rm', '-f', '-v', pds])
await sh('docker', ['rm', '-f', '-v', store])
await sh('docker', ['network', 'rm', id])
}
try {
await shOk('docker', ['network', 'create', id])
const s3 = await startRustfs(id, store)
const env = { ...vlpdsEnv, ...vlpdsOwnEnv({ port, plcUrl, rotationHex: await rotationHex(), store: s3 }) }
if (backfillMs !== undefined) {
const { retention, interval } = vlpdsRetention(backfillMs)
Object.assign(env, { VLPDS_LOG_RETENTION: retention, VLPDS_LOG_RETENTION_INTERVAL: interval })
}
await shOk('docker', vlpdsCreateArgs({ name: pds, network: id, port, env }))
} catch (e) {
await teardown()
throw e
}
return {
name: pds,
// RustFS's processes' RSS, and its bucket (the files on its
// volume), apart from vlpds's.
beside: { name: 'RustFS', rssKiB: () => containerRssKiB(store), diskBytes: () => containerBytes(store, '/data') },
teardown,
}
},
async start({ port, aux }) {
await shOk('docker', ['start', aux.name])
return {
url: `http://127.0.0.1:${port}`,
exited: running(aux.name),
async stop() {
// Graceful (SIGTERM): it hands back its shard and fences its log.
await sh('docker', ['stop', '-t', '90', aux.name])
},
}
},
// Handles under its one handle domain, vlpds.test (its default).
accountBody: (i, tag) => ({ handle: `${tag}${i}.vlpds.test`, email: `${tag}${i}@example.com`, password: `bench-password-${i}` }),
unsupported: {},
},
// zds, a PDS in Zig on SQLite, its image built by --setup from its own
// Dockerfile at the pinned commit (bench/impls.json). Run as its docs
// run it (docs/deployment.md, fly.toml: the database and blobs under
// /data, on a volume), with a fresh rotation key, JWT secret and admin
// token each round; zds has no dev mode.
zds: {
comm: 'zds',
volume: '/data',
noWindow: 'zds has no setting for it: none of the ZDS_* variables its binary reads is a backfill window or an event retention, so at rest is measured with what it keeps by default',
async info() {
const pin = pins.implementations.zds
return {
version: `${pin.release} (build.zig.zon 0.7.8; /xrpc/_health reports it)`,
commit: pin.commit,
source: `${pin.repo} at ${pin.commit}, image ${await imageAbout(pin.tag)} built with its own ${pin.dockerfile}; ${await dockerAbout()}`,
storage: 'SQLite (bundled 3.53.0, through zqlite): one database, /data/zds.sqlite3, on one connection; record bodies as DAG-CBOR blocks; blobs as files under /data/blobs; both on a Docker volume',
durability: 'SQLite journal_mode=WAL with synchronous left at the compiled-in default FULL (zds sets no synchronous pragma; zqlite compiles SQLite with no SQLITE_DEFAULT_* flags), so each commit fsyncs the WAL; fullfsync does not apply on Linux. Blobs written with fopen/fwrite/fclose to a temp file and renamed into place, never fsynced. On the Docker VM\'s virtual disk',
config: `docker run -v /data; ${Object.entries(zdsEnv).map(([k, v]) => `${k}=${v}`).join(' ')}; ZDS_PUBLIC_URL=http://127.0.0.1:<port> (server DID derived: did:web:127.0.0.1%3A<port>), ZDS_PLC_DIRECTORY=http://host.docker.internal:<bench PLC port>; fresh random ZDS_JWT_SECRET, ZDS_ADMIN_TOKEN and secp256k1 ZDS_PLC_ROTATION_KEY each round; handle domains .test, invites not required, permissioned data off (its defaults); the Dockerfile's ZDS_HOST, ZDS_PORT=2583, ZDS_DB=/data/zds.sqlite3`,
}
},
async setup({ port, plcUrl, log }) {
const name = `bench-zds-${process.pid}-${Date.now()}`
const env = { ...zdsEnv, ...zdsOwnEnv({ port, plcUrl, rotationHex: await rotationHex() }) }
const teardown = async () => {
await containerLogs(name, log)
await sh('docker', ['rm', '-f', '-v', name])
}
try {
await shOk('docker', zdsCreateArgs({ name, port, env }))
} catch (e) {
await teardown()
throw e
}
return { name, teardown }
},
async start({ port, aux }) {
await shOk('docker', ['start', aux.name])
return {
url: `http://127.0.0.1:${port}`,
exited: running(aux.name),
async stop() {
// zds runs as the container's PID 1 with no SIGTERM handler, so
// `docker stop` would only SIGKILL it after its timeout: kill it.
await sh('docker', ['kill', aux.name])
},
}
},
unsupported: {
firehoseLoaded: 'its live firehose skips commits when writes come at once: of a warm-up of 40 putRecords, 4 at once to 4 accounts, each of 20 subscribers was sent 34 to 37 (its log: `subscribeRepos end start_cursor=466 end_cursor=506 frames=35`), so no lag can be timed for the rest',
},
},
// Tranquil, a PDS in Rust, its image built by --setup from its own
// Dockerfile at the pinned commit (bench/impls.json), on PostgreSQL, its
// default repository backend: each round a fresh Docker network, a fresh
// PostgreSQL container on it with its data on a fresh volume, and a
// fresh tranquil container, configured as lib/impls.mjs configures it,
// with fresh secrets; all made in `setup`.
tranquil: {
comm: 'tranquil-pds',
// Its blobs alone: everything else is in PostgreSQL, beside it.
volume: '/var/lib/tranquil-pds/blobs',
// Its image is distroless, with no shell: PostgreSQL's has one.
reader: pins.images.postgres.image,
noWindow: 'its backfill window is FIREHOSE_BACKFILL_HOURS, in whole hours (72 by default), and it prunes the events past EVENT_RETENTION_MAX_AGE_SECS (7 days by default, and never less than that window) once a minute at most: nothing sets a window of W with a pass in the quiet, so at rest is measured with what it keeps by default',
async info() {
const pin = pins.implementations.tranquil
const db = pins.images[pin.needs[0]]
return {
version: `${pin.release} (its Cargo.toml's workspace version 0.8.0)`,
commit: pin.commit,
source: `${pin.repo} at ${pin.commit}, image ${await imageAbout(pin.tag)} built with its own ${pin.dockerfile}; database ${db.image} (${db.release}); ${await dockerAbout()}`,
storage: 'PostgreSQL (its default repository backend), one database on a Docker volume, through a pool of 10 to 100 connections: accounts, records, repository blocks and the event sequence in its tables; blobs as files under /var/lib/tranquil-pds/blobs, on a Docker volume of its own; its cache in its process (ripple, one node)',
durability: 'PostgreSQL at its defaults, which tranquil leaves as they are: fsync on, synchronous_commit on, so a write is answered once its transaction\'s WAL is flushed (fdatasync(2), wal_sync_method\'s Linux default). Blobs written to a temp file, fsynced (sync_all) and renamed into place, the directory not fsynced. On the Docker VM\'s virtual disk',
config: `docker run -v /var/lib/tranquil-pds/blobs on a Docker network of its own; ${Object.entries(tranquilEnv).map(([k, v]) => `${k}=${v}`).join(' ')}; DATABASE_URL the round's PostgreSQL, PLC_DIRECTORY_URL=http://host.docker.internal:<bench PLC port>; fresh random JWT_SECRET, DPOP_SECRET and MASTER_KEY each round; handles under its hostname (user_handle_domains unset); the image's SERVER_HOST=[::], SERVER_PORT=3000; its binary UPX-compressed, as its Dockerfile builds it by default. PostgreSQL: ${db.release} with POSTGRES_USER=tranquil_pds, POSTGRES_DB=pds and a fresh password, its defaults otherwise (max_connections 100, shared_buffers 128 MB), its data on a fresh volume`,
}
},
async setup({ port, plcUrl, log }) {
const id = `bench-tranquil-${process.pid}-${Date.now()}`
const store = `${id}-db`
const pds = `${id}-pds`
const teardown = async () => {
await containerLogs(pds, log)
await sh('docker', ['rm', '-f', '-v', pds])
await sh('docker', ['rm', '-f', '-v', store])
await sh('docker', ['network', 'rm', id])
}
try {
await shOk('docker', ['network', 'create', id])
const db = await startPostgres(id, store)
await shOk('docker', tranquilCreateArgs({ name: pds, network: id, port, env: { ...tranquilEnv, ...tranquilOwnEnv({ plcUrl, db }) } }))
} catch (e) {
await teardown()
throw e
}
return {
name: pds,
// PostgreSQL's memory, and its data directory (the files on its
// volume), apart from tranquil's. Its memory is its processes'
// PSS, not their RSS: the sum of their RSS counts the shared
// buffers once in every backend that touched them.
beside: { name: 'PostgreSQL', rssKiB: () => containerPssKiB(store, 'postgres', 'postgres'), diskBytes: () => containerBytes(store, '/var/lib/postgresql') },
teardown,
}
},
async start({ port, aux }) {
await shOk('docker', ['start', aux.name])
return {
url: `http://127.0.0.1:${port}`,
exited: running(aux.name),
async stop() {
// SIGTERM: it shuts down gracefully on it.
await sh('docker', ['stop', aux.name])
},
}
},
// Handles under its one handle domain, its hostname; a password with
// an upper-case letter, as it requires one.
accountBody: (i, tag) => ({ handle: `${tag}${i}.tranquil.test`, email: `${tag}${i}@example.com`, password: `Bench-password-${i}` }),
unsupported: {},
},
}
const accountBody = (i, tag) => ({ handle: `${tag}${i}.test`, email: `${tag}${i}@example.com`, password: `bench-password-${i}` })
/// Monolith's environment besides its image's.
function monolithEnv({ port, plcUrl, rotation }) {
return {
PDS_HOSTNAME: 'localhost',
PDS_PUBLIC_URL: `http://127.0.0.1:${port}`,
PDS_HANDLE_DOMAINS: '.test,.example',
PDS_PLC_URL: fromDocker(plcUrl),
PDS_PLC_ROTATION_KEY_HEX: rotation,
PDS_ADMIN_PASSWORD: 'admin-pass',
PDS_INVITE_REQUIRED: 'false',
PDS_RATE_LIMITS_ENABLED: 'false',
PDS_TRUSTED_PROXIES: '127.0.0.0/8,::1/128,169.254.0.0/16,fe80::/10,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fc00::/7',
// An AppView, as every server on the network has (docs/guide/
// deploying.md): without one, getPreferences is not served
// (spec/PROXY.md). Nothing in the workload is forwarded to it, and
// nothing listens there.
PDS_BSKY_APP_VIEW_URL: 'http://127.0.0.1:9',
PDS_BSKY_APP_VIEW_DID: 'did:web:api.bsky.app',
}
}
/// A container's process `comm`: its RSS in KiB and its threads, as
/// `docker top` (ps in Docker's VM) has them.
async function processOf(name, comm) {
const rows = (await sh('docker', ['top', name, '-o', 'pid,nlwp,rss,comm'])).split('\n').slice(1).map((l) => l.trim().split(/\s+/))
const mine = rows.filter(([, , , c]) => c === comm)
return { threads: mine.reduce((s, [, t]) => s + (Number(t) || 0), 0), rssKiB: mine.reduce((s, [, , r]) => s + (Number(r) || 0), 0) }
}
// ---- The client --------------------------------------------------------
const pool = await openPool(CLIENTS)
/// A request to `nsid` (or, starting with `/`, that path) of the server.
function req(srv, nsid, { params, body, bytes, type, token, method } = {}) {
const url = new URL(nsid.startsWith('/') ? `${srv.url}${nsid}` : `${srv.url}/xrpc/${nsid}`)
for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v)
const headers = {}
if (token) headers.authorization = `Bearer ${token}`
if (body !== undefined) headers['content-type'] = 'application/json'
if (bytes) headers['content-type'] = type
return { method: method ?? (body !== undefined || bytes ? 'POST' : 'GET'), url: url.href, headers, body: bytes ?? (body === undefined ? undefined : JSON.stringify(body)) }
}
/// How many times the client waited for a local port (lib/loadpool.mjs),
/// all told: each part's is what this grew by over it.
let portWaits = 0
/// `requests`, `at` at once, from the client's threads: their times, how
/// long they took together, and with `keep` their answers. Anything but
/// a 2xx stops the run.
async function send(srv, requests, at, keep = false) {
const r = await pool.run(requests, at, { keep })
portWaits += r.portWaits
if (r.firstError) {
const { i, status, body } = r.firstError
throw new Error(`${srv.name}: ${new URL(requests[i].url).pathname} answered ${status}: ${body}\nserver log: ${srv.logPath}`)
}
return { times: r.times, total: r.total, bodies: r.bodies }
}
/// One request from this thread, timed from sending it to having its
/// answer whole; anything but a 2xx stops the run. For the firehose's
/// writes, whose answers are timed against what subscribers get, and for
/// what must wait on the answer before it.
async function call(srv, nsid, o = {}) {
const r = await timedFetch(req(srv, nsid, o))
portWaits += r.waits
if (r.status < 200 || r.status >= 300) throw new Error(`${srv.name}: ${nsid} answered ${r.status}: ${r.buf.toString('utf8').slice(0, 300)}\nserver log: ${srv.logPath}`)
return { ms: r.ms, size: r.buf.length, json: () => JSON.parse(r.buf.toString('utf8')) }
}
/// `n` calls from this thread, `at` at once, each made by `make(i)`:
/// their times, and how long they took together.
async function load(n, at, make) {
const times = []
let next = 0
const t = performance.now()
await Promise.all(Array.from({ length: at }, async () => {
while (next < n) times.push(await make(next++))
}))
return { times, total: performance.now() - t }
}
const blob = (i) => {
const out = Buffer.alloc(65536)
for (let o = 0; o < out.length; o += 32) createHash('sha256').update(`blob ${i} ${o}`).digest().copy(out, o)
return out
}
const note = (i) => ({ $type: 'com.example.note', text: `a note of about a hundred bytes, number ${i}, to read back later`, n: i })
const wsOf = (srv) => `${srv.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`
const pick = (xs, i) => xs[(i * 7919) % xs.length]
const own = (st, i) => st.accounts[i % st.accounts.length]
/// A subscriber to the firehose from the address 198.18.<k>, from
/// `cursor` if given, calling `onCommit(body)` for each #commit; answers
/// once it is connected. One that cannot connect tries again for 10 s,
/// as the client's ports may have run out for a moment (lib/loadpool.mjs);
/// then it fails.
async function subscribe(srv, k, onCommit, cursor) {
for (let tries = 1; ; tries++) {
try {
return await new Promise((ok, fail) => {
const ws = new WebSocket(`${wsOf(srv)}${cursor === undefined ? '' : `?cursor=${cursor}`}`, { headers: { 'x-forwarded-for': `198.18.${k >> 8}.${(k & 255) + 1}` } })
ws.binaryType = 'arraybuffer'
const sub = { ws, events: 0 }
ws.onmessage = (m) => {
sub.events++
const [header, body] = cborDecodeMulti(new Uint8Array(m.data))
if (header?.t === '#commit') onCommit(body, sub)
}
ws.onopen = () => ok(sub)
ws.onerror = () => fail(new Error(`${srv.name}: subscribeRepos${cursor === undefined ? '' : ` from cursor ${cursor}`}`))
})
} catch (e) {
if (tries === 40) throw e
await new Promise((ok) => setTimeout(ok, 250))
}
}
}
const until = async (done, ms) => {
for (let w = 0; w < ms / 50 && !done(); w++) await new Promise((ok) => setTimeout(ok, 50))
return done()
}
/// `subsN` subscribers, each from an address of its own (Monolith holds 16
/// at most from one, spec/LIMITS.md), and `n` putRecords, `writers` at
/// once (no more than there are accounts, so that they go to different
/// ones), over the accounts: each time is the firehose's lag, from a
/// write's answer to a subscriber having its commit, 0 when the commit
/// came first. No rate: the writes' would be the write path's, which the
/// write parts measure.
async function fanOut(srv, st, n, subsN, writers, key) {
const written = new RegExp(`/${st.tag}${key}(\\d+)x$`)
const subs = await Promise.all(Array.from({ length: subsN }, (_, k) => {
const got = new Map()
return subscribe(srv, k, (body) => {
for (const op of body.ops ?? []) {
const m = written.exec(op.path)
if (m && !got.has(Number(m[1]))) got.set(Number(m[1]), performance.now())
}
}).then((sub) => Object.assign(sub, { got }))
}))
const answeredAt = []
await load(n, Math.min(writers, st.accounts.length), async (i) => {
const a = own(st, i)
await call(srv, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `${st.tag}${key}${i}x`, record: { $type: 'com.example.note', text: 'fan-out', n: i } }, token: a.token })
answeredAt[i] = performance.now()
})
await until(() => subs.every((s) => s.got.size >= n), 10000)
for (const s of subs) s.ws.close()
const times = []
for (const s of subs) for (let i = 0; i < n; i++) {
const at = s.got.get(i)
if (at === undefined) throw new Error(`${srv.name}: a subscriber did not get the commit of write ${i} within 10 s`)
times.push(Math.max(0, at - answeredAt[i]))
}
return { times }
}
// ---- The workload ------------------------------------------------------
/// The parts, in order, each with its size in a measured round and how
/// many requests at once; a warm-up runs each at a tenth of its size.
/// `st` is the state of a run: its accounts, records and blobs.
const PARTS = {
createAccount: [40, 4, async (srv, st, n, at) => {
const body = srv.accountBody
const r = await send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.server.createAccount', { body: body(i, st.tag) })), at, true)
r.bodies.forEach((b, i) => {
const { did, accessJwt, refreshJwt } = JSON.parse(b)
const { handle, password } = body(i, st.tag)
st.accounts[i] = { did, handle, password, token: accessJwt, refresh: refreshJwt }
})
return r
}],
createRecord: [1000, 16, async (srv, st, n, at) => {
const r = await send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: own(st, i).did, collection: 'com.example.note', record: note(i) }, token: own(st, i).token })), at, true)
r.bodies.forEach((b, i) => {
st.records[i] = { repo: own(st, i).did, rkey: JSON.parse(b).uri.split('/').pop() }
})
return r
}],
applyWrites: [200, 8, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.applyWrites', { body: { repo: own(st, i).did, writes: Array.from({ length: 10 }, (_, k) => ({ $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.note', value: note(100000 + i * 10 + k) })) }, token: own(st, i).token })), at)],
getRecord: [2000, 16, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => getRecord(srv, st, i)), at)],
listRecords: [1000, 16, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => listRecords(srv, st, i)), at)],
getRepo: [1000, 8, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.sync.getRepo', { params: { did: own(st, i).did } })), at)],
// Then, not timed, a record of each account that refers to the first
// blob it uploaded, so that sync.getBlob serves it. The rest no record
// references: each account holds them for the blob TTL (an hour, past
// the round), so they count in its live state.
uploadBlob: [1000, 8, async (srv, st, n, at) => {
const r = await send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.uploadBlob', { bytes: blob(i), type: 'application/octet-stream', token: own(st, i).token })), at, true)
const firsts = st.accounts.map((_, k) => k).filter((k) => k < n)
await send(srv, firsts.map((k) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: st.accounts[k].did, collection: 'com.example.note', record: { ...note(300000 + k), blob: JSON.parse(r.bodies[k]).blob } }, token: st.accounts[k].token })), 8)
st.blobs = firsts.map((k) => ({ did: st.accounts[k].did, cid: JSON.parse(r.bodies[k]).blob.ref.$link }))
for (let i = 0; i < n; i++) if (!firsts.includes(i)) own(st, i).uploaded = (own(st, i).uploaded ?? 0) + 65536
return r
}],
// One write at a time, as one person posting: no other write to share
// its sync with. After the reads, so the repos they read are as before.
createRecordSerial: [1000, 1, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: own(st, i).did, collection: 'com.example.note', record: note(200000 + i) }, token: own(st, i).token })), at)],
// 20 subscribers; the writes one after another, then LOADED_WRITERS at once.
firehose: [100, 20, (srv, st, n, subsN) => fanOut(srv, st, n, subsN, 1, 'fh')],
firehoseLoaded: [400, 20, (srv, st, n, subsN) => fanOut(srv, st, n, subsN, LOADED_WRITERS, 'fl')],
// `subsN` subscribers from cursor 0, as relays catching up: each is sent
// every event the server has made, then a write made once all are
// connected. Each time is a subscriber's, from asking to that write's
// commit; the rate is the events sent to all of them before it, per
// second. 4 at once: zds and vlpds each serve 4 backfills at a time.
replay: [4, 4, async (srv, st, n, subsN) => {
const marker = new RegExp(`/${st.tag}replayx$`)
const t = performance.now()
const subs = await Promise.all(Array.from({ length: subsN }, (_, k) => subscribe(srv, k, (body, sub) => {
if (sub.done === undefined && (body.ops ?? []).some((op) => marker.test(op.path))) {
sub.done = performance.now()
sub.counted = sub.events
}
}, 0)))
const a = st.accounts[0]
await call(srv, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `${st.tag}replayx`, record: { $type: 'com.example.note', text: 'caught up' } }, token: a.token })
const caught = await until(() => subs.every((s) => s.done !== undefined), 60000)
for (const s of subs) s.ws.close()
if (!caught) throw new Error(`${srv.name}: a subscriber from cursor 0 did not catch up within 60 s`)
return { times: subs.map((s) => s.done - t), total: Math.max(...subs.map((s) => s.done)) - t, count: subs.reduce((sum, s) => sum + s.counted, 0) }
}],
}
const getRecord = (srv, st, i) => {
const rec = pick(st.records, i)
return req(srv, 'com.atproto.repo.getRecord', { params: { repo: rec.repo, collection: 'com.example.note', rkey: rec.rkey } })
}
const listRecords = (srv, st, i) => req(srv, 'com.atproto.repo.listRecords', { params: { repo: own(st, i).did, collection: 'com.example.note', limit: '50' } })
const createRecord = (srv, st, i, base) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: own(st, i).did, collection: 'com.example.note', record: note(base + i) }, token: own(st, i).token })
/// The routes a client calls most, besides the parts' (bench/README.md):
/// each a request made from the run's state, `n` of them, `at` at once.
const ROUTES = {
describeServer: [1000, 16, (srv) => req(srv, 'com.atproto.server.describeServer')],
oauthMetadata: [1000, 16, (srv) => req(srv, '/.well-known/oauth-authorization-server')],
createSession: [200, 8, (srv, st, i) => req(srv, 'com.atproto.server.createSession', { body: { identifier: own(st, i).handle, password: own(st, i).password } })],
getSession: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.server.getSession', { token: own(st, i).token })],
getPreferences: [1000, 16, (srv, st, i) => req(srv, 'app.bsky.actor.getPreferences', { token: own(st, i).token })],
getServiceAuth: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.server.getServiceAuth', { params: { aud: 'did:web:api.bsky.app', lxm: 'app.bsky.feed.getTimeline' }, token: own(st, i).token })],
resolveHandle: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.identity.resolveHandle', { params: { handle: own(st, i).handle } })],
describeRepo: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.repo.describeRepo', { params: { repo: own(st, i).did } })],
getLatestCommit: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.sync.getLatestCommit', { params: { did: own(st, i).did } })],
syncGetRecord: [1000, 16, (srv, st, i) => {
const rec = pick(st.records, i)
return req(srv, 'com.atproto.sync.getRecord', { params: { did: rec.repo, collection: 'com.example.note', rkey: rec.rkey } })
}],
listRepos: [1000, 16, (srv) => req(srv, 'com.atproto.sync.listRepos', { params: { limit: '100' } })],
getBlob: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.sync.getBlob', { params: pick(st.blobs, i) })],
// Each answer's tokens are the account's next ones: from this thread,
// never two of one account's at once. Last, as the old tokens may no
// longer be good.
refreshSession: [1000, 16, 'chain'],
}
async function route(srv, st, name, n, at) {
const [, , make] = ROUTES[name]
if (make !== 'chain') return send(srv, Array.from({ length: n }, (_, i) => make(srv, st, i)), at)
// Each of `at` callers has accounts of its own, taken in turn, so that
// no account's tokens are in two requests at once.
const callers = Math.min(at, st.accounts.length)
const times = []
let sent = 0
const t = performance.now()
await Promise.all(Array.from({ length: callers }, async (_, k) => {
const mine = st.accounts.filter((_, i) => i % callers === k)
for (let j = 0; sent < n; j++) {
sent++
const a = mine[j % mine.length]
const r = await call(srv, 'com.atproto.server.refreshSession', { method: 'POST', token: a.refresh })
const { accessJwt, refreshJwt } = r.json()
Object.assign(a, { token: accessJwt, refresh: refreshJwt })
times.push(r.ms)
}
}))
return { times, total: performance.now() - t }
}
/// A day's traffic in small: of each 100 requests, this many of each
/// route, from 32 callers at once, with 2 subscribers to the firehose
/// as relays would be.
const MIX = [['getSession', 15], ['getPreferences', 10], ['getRecord', 25], ['listRecords', 15], ['describeRepo', 5], ['getBlob', 10], ['getLatestCommit', 5], ['createRecord', 10], ['applyWrites', 3], ['uploadBlob', 2]]
// So that the least share has 200 requests: its p95 has ten above it.
const MIX_N = 10000
const MIX_AT = 32
// The routes in the order of a smooth weighted round robin: each share
// spread through the 100 as evenly as it goes.
const mixOrder = (() => {
const credit = MIX.map(() => 0)
return Array.from({ length: 100 }, () => {
MIX.forEach(([, w], k) => (credit[k] += w))
const k = credit.indexOf(Math.max(...credit))
credit[k] -= 100
return MIX[k][0]
})
})()
const mixRequest = (srv, st, name, i) => {
if (name === 'getRecord') return getRecord(srv, st, i)
if (name === 'listRecords') return listRecords(srv, st, i)
if (name === 'createRecord') return createRecord(srv, st, i, 400000)
if (name === 'applyWrites') return req(srv, 'com.atproto.repo.applyWrites', { body: { repo: own(st, i).did, writes: Array.from({ length: 10 }, (_, k) => ({ $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.note', value: note(500000 + i * 10 + k) })) }, token: own(st, i).token })
if (name === 'uploadBlob') return req(srv, 'com.atproto.repo.uploadBlob', { bytes: blob(600000 + i), type: 'application/octet-stream', token: own(st, i).token })
return ROUTES[name][2](srv, st, i)
}
async function mixed(srv, st) {
let frames = 0
const subs = await Promise.all([0, 1].map((k) => subscribe(srv, 200 + k, () => frames++)))
const names = Array.from({ length: MIX_N }, (_, i) => mixOrder[i % 100])
const r = await send(srv, names.map((name, i) => mixRequest(srv, st, name, i)), MIX_AT)
for (const s of subs) s.ws.close()
const routes = {}
for (const [name] of MIX) {
const times = names.flatMap((x, i) => (x === name ? [r.times[i]] : []))
routes[name] = { p50: pct(times, 50), p95: pct(times, 95), n: times.length }
}
return { rate: (MIX_N / r.total) * 1000, routes, frames }
}
/// getRecord and createRecord by the number of callers at once: at each,
/// what is answered per second and how long it takes, and how many were
/// answered with no 2xx (counted, not stopped at: a server that pushes
/// back under load says so here).
const CURVES = {
getRecord: [(c) => Math.max(2000, 40 * c), (srv, st, i) => getRecord(srv, st, i)],
createRecord: [(c) => Math.max(500, 4 * c), (srv, st, i) => createRecord(srv, st, i, 700000 + st.curveBase)],
}
async function curve(srv, st, name, callers) {
const [count, make] = CURVES[name]
const n = count(callers)
const r = await pool.run(Array.from({ length: n }, (_, i) => make(srv, st, i)), callers)
portWaits += r.portWaits
st.curveBase += n
const good = Array.from(r.times).filter((_, i) => r.statuses[i] >= 200 && r.statuses[i] < 300)
const [tailLabel, p] = tailOf(good.length)
return { n, rate: (good.length / r.total) * 1000, p50: pct(good, 50), tail: pct(good, p), tailLabel, errors: n - good.length, portWaits: r.portWaits, ...(r.firstError && { firstError: `${r.firstError.status}: ${r.firstError.body.slice(0, 160)}` }) }
}
/// Subscribers coming and going: a cohort of 20, each from an address of
/// its own, connects, has one write's commit, and goes; one cohort to
/// warm, then COHORTS more. The server's threads and RSS after the first
/// and after the last, each once it has had 2 s to let them go; and how
/// many subscribers it refused (a server that holds a subscriber's place
/// after it went refuses the next cohort's).
const COHORTS = 10
async function churn(srv, st, sample) {
let refused = 0
const cohort = async (c) => {
const marker = `${st.tag}churn${c}x`
const subs = (await Promise.allSettled(Array.from({ length: 20 }, (_, k) => subscribe(srv, 100 + k, (body, sub) => {
if ((body.ops ?? []).some((op) => op.path.endsWith(`/${marker}`))) sub.got = true
})))).flatMap((x) => (x.status === 'fulfilled' ? [x.value] : []))
refused += 20 - subs.length
const a = st.accounts[c % st.accounts.length]
await call(srv, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: marker, record: { $type: 'com.example.note', text: 'churn', n: c } }, token: a.token })
const all = await until(() => subs.every((s) => s.got), 10000)
for (const s of subs) s.ws.close()
if (!all) throw new Error(`${srv.name}: a subscriber of cohort ${c} did not get its commit within 10 s`)
}
const settle = async () => {
await new Promise((ok) => setTimeout(ok, 2000))
return sample()
}
await cohort(0)
const first = await settle()
for (let c = 1; c <= COHORTS; c++) await cohort(c)
const last = await settle()
return { first, last, subscribers: 20 * (COHORTS + 1), refused }
}
// ---- A round ------------------------------------------------------------
/// The accounts' live state (spec/STORE.md, "Bounds"), in bytes: of each,
/// its getRepo without `since`, the blobs listBlobs lists (getBlob's
/// bytes), and the uploads no record references, which it holds for the
/// blob TTL (an hour, past the round) and listBlobs does not list.
async function liveState(srv, accounts) {
const live = { repo: 0, blobs: 0, uploads: 0 }
for (const a of accounts) {
live.repo += (await call(srv, 'com.atproto.sync.getRepo', { params: { did: a.did } })).size
for (let cursor; ; ) {
const page = (await call(srv, 'com.atproto.sync.listBlobs', { params: { did: a.did, limit: '1000', ...(cursor && { cursor }) } })).json()
for (const cid of page.cids) live.blobs += (await call(srv, 'com.atproto.sync.getBlob', { params: { did: a.did, cid } })).size
cursor = page.cids.length ? page.cursor : undefined
if (!cursor) break
}
live.uploads += a.uploaded ?? 0
}
return { ...live, total: live.repo + live.blobs + live.uploads }
}
/// One round: a fresh PLC directory and server, warmed, then each part,
/// each route, the curves, the mix and the churn. A storage round
/// (`storage`) starts the server with the short backfill window, is
/// warmed with the parts alone, and after the parts measures what it keeps
/// on disk, answering only that: its speed is not kept, as the short
/// window changes replay.
async function round(name, adapter, storage = false) {
const backfillMs = storage ? BACKFILL_MS : undefined
const plcPort = await freePort()
const plc = PlcServer.create({ db: Database.mock(), port: plcPort })
await plc.start()
const plcUrl = `http://127.0.0.1:${plcPort}`
const dataDir = mkdtempSync(join(tmpdir(), `bench-${name}-`))
const logPath = join(dataDir, 'server.log')
const log = openSync(logPath, 'a')
const loadStart = loadavg()[0]
// The container is made before the start clock. A port picked free can
// be taken (an outgoing connection's ephemeral port) before Docker binds
// it: then all again on another.
let port
let aux
let s
let t0
for (let attempt = 1; ; attempt++) {
port = await freePort()
aux = null
try {
aux = await adapter.setup({ port, plcUrl, dataDir, log, backfillMs })
t0 = performance.now()
s = await adapter.start({ port, plcUrl, dataDir, log, aux })
break
} catch (e) {
await aux?.teardown()
if (!/address already in use/.test(e.message) || attempt === 3) {
await plc.destroy()
throw e
}
writeSync(log, `port ${port} was taken before Docker bound it; again on another\n`)
}
}
let ok = false
try {
const srv = { name, url: s.url, logPath, accountBody: adapter.accountBody ?? accountBody }
const sample = () => processOf(aux.name, adapter.comm)
let lastExitCheck = t0
for (;;) {
try {
if ((await fetch(`${srv.url}/xrpc/com.atproto.server.describeServer`)).ok) break
} catch {}
if (performance.now() - t0 > 60000) throw new Error(`${name} did not start; its log is ${logPath}`)
// A container's state is a `docker inspect` away: once a second.
if (performance.now() - lastExitCheck > 1000) {
lastExitCheck = performance.now()
if (await s.exited()) throw new Error(`${name} exited before it started; its log is ${logPath}`)
}
await new Promise((ok) => setTimeout(ok, 10))
}
const startMs = performance.now() - t0
await new Promise((ok) => setTimeout(ok, 2000))
const idleKiB = (await sample()).rssKiB
// Something beside the server with memory of its own (vlpds's store,
// tranquil's database).
const beside = aux.beside
const besideIdleKiB = beside ? await beside.rssKiB() : undefined
let peakKiB = idleKiB
let besidePeakKiB = besideIdleKiB
let sampling = true
const sampler = (async () => {
while (sampling) {
peakKiB = Math.max(peakKiB, (await sample()).rssKiB)
await new Promise((ok) => setTimeout(ok, 100))
}
})()
// Its own loop, as `docker top` is slow.
const besideSampler = (async () => {
while (beside && sampling) besidePeakKiB = Math.max(besidePeakKiB, await beside.rssKiB())
})()
const parts = {}
const routes = {}
const notServed = {}
const warm = { tag: 'warm', accounts: [], records: [], curveBase: 0 }
const st = { tag: 'bench', accounts: [], records: [], curveBase: 0 }
const summary = (times, total, count) => {
const [tailLabel, p] = tailOf(times.length)
return { p50: pct(times, 50), p95: pct(times, 95), tail: pct(times, p), tailLabel, ...(total !== undefined && { rate: ((count ?? times.length) / total) * 1000 }), n: times.length }
}
// The warm-up: each part and route at a tenth of its size, on
// accounts of its own. A route answered with no 2xx here is one the
// server does not serve.
const step = (what) => console.error(`${name}: ${what} (${((performance.now() - t0) / 1000).toFixed(0)} s)`)
step('warm-up')
for (const [part, [n, at, fn]] of Object.entries(PARTS)) {
if (adapter.unsupported[part]) continue
await fn(srv, warm, Math.max(1, Math.ceil(n / 10)), part === 'createAccount' ? at : Math.min(at, n))
}
for (const [r, [n, at]] of storage ? [] : Object.entries(ROUTES)) {
try {
await route(srv, warm, r, Math.ceil(n / 10), at)
} catch (e) {
notServed[r] = e.message.split('\nserver log')[0].replace(`${name}: `, '')
}
}
step('parts')
for (const [part, [n, at, fn]] of Object.entries(PARTS)) {
if (adapter.unsupported[part]) continue
const waited = portWaits
const { times, total, count } = await fn(srv, st, n, at)
parts[part] = { ...summary(times, total, count), portWaits: portWaits - waited }
}
if (storage) {
sampling = false
await sampler
await besideSampler
// What it keeps on disk: the files on its container's volume; and
// apart, what it needs beside it (vlpds's bucket, tranquil's
// database).
const besideDisk = beside?.diskBytes ? beside : null
const sizes = async () => ({ server: await containerBytes(aux.name, adapter.volume, adapter.reader), ...(besideDisk && { beside: await besideDisk.diskBytes() }) })
step('storage under load')
const underLoad = await sizes()
// At rest: no request for 2 W + 1 s, then a write to each account the
// round made, then the least of the sizes sampled for 10 s.
step('storage at rest')
const accounts = [...warm.accounts, ...st.accounts]
await sleep(QUIET_MS)
await send(srv, accounts.map((a, i) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: a.did, collection: 'com.example.note', record: note(800000 + i) }, token: a.token })), 8)
const atRest = await sizes()
for (let k = 1, t = performance.now(); k * REST_SAMPLE_MS <= REST_SAMPLES_MS; k++) {
await sleep(t + k * REST_SAMPLE_MS - performance.now())
const x = await sizes()
atRest.server = Math.min(atRest.server, x.server)
if (besideDisk) atRest.beside = Math.min(atRest.beside, x.beside)
}
const live = await liveState(srv, accounts)
ok = true
return {
accounts: accounts.length,
underLoadBytes: underLoad.server,
atRestBytes: atRest.server,
liveBytes: live.total,
live,
...(besideDisk && { beside: { name: besideDisk.name, underLoadBytes: underLoad.beside, atRestBytes: atRest.beside } }),
}
}
step('routes')
for (const [r, [n, at]] of Object.entries(ROUTES)) {
if (notServed[r]) continue
const waited = portWaits
const { times, total } = await route(srv, st, r, n, at)
routes[r] = { ...summary(times, total), portWaits: portWaits - waited }
}
step('by callers')
const curves = {}
for (const c of Object.keys(CURVES)) {
curves[c] = {}
for (const callers of CALLERS) curves[c][callers] = await curve(srv, st, c, callers)
}
const missing = MIX.map(([r]) => r).filter((r) => notServed[r] || (!ROUTES[r] && adapter.unsupported[r]))
if (!missing.length) step('the mix')
const waited = portWaits
const mix = missing.length ? { notServed: `it does not serve ${missing.join(', ')}` } : { ...(await mixed(srv, st)), portWaits: portWaits - waited }
step('subscribers coming and going')
const subscribers = await churn(srv, st, sample)
sampling = false
await sampler
await besideSampler
ok = true
return { startMs, idleKiB, peakKiB, ...(beside && { beside: { name: beside.name, idleKiB: besideIdleKiB, peakKiB: besidePeakKiB } }), load: [loadStart, loadavg()[0]], parts, routes, notServed, curves, mix, churn: subscribers }
} finally {
await s?.stop()
await aux?.teardown()
await plc.destroy()
// A failed round keeps its directory, for its log.
if (ok) rmSync(dataDir, { recursive: true, force: true })
}
}
/// What the client can send: a server in Docker, reached as the servers
/// are, that answers every request with the same 2 bytes and does nothing
/// else (Node's `net`, in the reference's image). At each concurrency the
/// parts use, the best of three.
async function ceiling() {
const port = await freePort()
const name = `bench-ceiling-${process.pid}-${Date.now()}`
const code = "const res=Buffer.from('HTTP/1.1 200 OK\\r\\ncontent-type: application/json\\r\\ncontent-length: 2\\r\\n\\r\\n{}');require('net').createServer((s)=>{let b='';s.on('data',(d)=>{b+=d.toString('latin1');let i;while((i=b.indexOf('\\r\\n\\r\\n'))!==-1){b=b.slice(i+4);s.write(res)}});s.on('error',()=>{})}).listen(Number(process.env.PORT),'0.0.0.0')"
await shOk('docker', ['run', '-d', '--name', name, '-p', `127.0.0.1:${port}:${port}`, '-e', `PORT=${port}`, '--entrypoint', 'node', pins.implementations.reference.tag, '-e', code])
try {
const url = `http://127.0.0.1:${port}/xrpc/_ceiling`
for (let t = performance.now(); ; ) {
try {
if ((await fetch(url)).ok) break
} catch {}
if (performance.now() - t > 30000) throw new Error('the ceiling\'s responder did not answer within 30 s')
await new Promise((ok) => setTimeout(ok, 50))
}
const out = {}
for (const at of CEILING_AT) {
const n = Math.max(5000, 100 * at)
let best = 0
for (let k = 0; k < 3; k++) {
const r = await pool.run(Array.from({ length: n }, () => ({ method: 'GET', url, headers: {} })), at)
if (r.firstError) throw new Error(`the ceiling's responder answered ${r.firstError.status}: ${r.firstError.body}`)
best = Math.max(best, (n / r.total) * 1000)
}
out[at] = best
}
return out
} finally {
await sh('docker', ['rm', '-f', name])
}
}
// ---- The run ------------------------------------------------------------
const impls = arg('--impls', Object.keys(ADAPTERS).join(',')).split(',')
// Run only what --setup built from the pins, and an image built from this
// repository only as what it is built from is now.
const out = arg('--json')
for (const name of impls) {
if (!ADAPTERS[name]) throw new Error(`no adapter ${name}; there are ${Object.keys(ADAPTERS).join(', ')}`)
const pin = pins.implementations[name]
if (!pin) throw new Error(`${name} has no entry in bench/impls.json`)
const missing = (what) => new Error(`${name}: ${what}; run \`node bench.mjs --setup\` first`)
for (const image of [pin.image, (pin.repo || pin.context) && pin.tag, ...(pin.needs ?? []).map((n) => pins.images[n].image)].filter(Boolean)) {
if (!(await imageId(image))) throw missing(`no Docker image ${image}`)
}
if (pin.context && (await imageRevision(pin.tag)) !== revision(pin.inputs)) throw missing(`its image ${pin.tag} was built from ${(await imageRevision(pin.tag)) || 'another tree'}, not what it is built from as it is now (${revision(pin.inputs)})`)
}
if (!(await imageId(pins.implementations.reference.tag))) throw new Error(`the client's ceiling is measured with the reference's image ${pins.implementations.reference.tag}; run \`node bench.mjs --setup\` first`)
// Other work on the machine makes every number its own: refuse to start
// on a busy one unless told to.
const cores = cpus().length
const [load1] = loadavg()
if (load1 > cores / 2 && !process.argv.includes('--busy')) throw new Error(`the machine is busy (load average ${load1.toFixed(1)} on ${cores} cores, over ${cores / 2}): its numbers would be the other work's. Wait until it is idle, or give --busy to run anyway and have the load recorded`)
const os = type() === 'Darwin' ? `${await sh('sw_vers', ['-productName'])} ${await sh('sw_vers', ['-productVersion'])}` : `${type()} ${release()}`
const results = {
machine: {
cpu: cpus()[0].model,
cores,
memGiB: Math.round(totalmem() / 2 ** 30),
os: `${os} (${process.arch})`,
node: process.version,
docker: await dockerAbout(),
loadAtStart: load1,
},
pins: { ...pins, 'Monolith commit': spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(), 'Monolith image built from': revision(pins.implementations['monolith'].inputs) },
rounds: ROUNDS,
clients: CLIENTS,
workload: {
parts: Object.fromEntries(Object.entries(PARTS).map(([p, [n, at]]) => [p, { n, at }])),
routes: Object.fromEntries(Object.entries(ROUTES).map(([p, [n, at]]) => [p, { n, at }])),
callers: CALLERS,
mix: { shares: Object.fromEntries(MIX), n: MIX_N, at: MIX_AT },
cohorts: COHORTS,
},
storage: { rounds: STORAGE_ROUNDS, backfillMs: BACKFILL_MS, quietMs: QUIET_MS, sampleMs: REST_SAMPLE_MS, samplesMs: REST_SAMPLES_MS },
impls: {},
}
try {
console.error(`the client's ceiling, from ${CLIENTS} threads`)
results.ceiling = await ceiling()
for (const name of impls) {
const adapter = ADAPTERS[name]
const r = { info: await adapter.info(), unsupported: adapter.unsupported, rounds: [] }
for (let k = 0; k < ROUNDS; k++) {
console.error(`${name}: round ${k + 1} of ${ROUNDS}`)
r.rounds.push(await round(name, adapter))
}
r.storage = { ...(adapter.window ? { window: adapter.window } : { noWindow: adapter.noWindow }), rounds: [] }
for (let k = 0; k < STORAGE_ROUNDS; k++) {
console.error(`${name}: storage round ${k + 1} of ${STORAGE_ROUNDS}`)
r.storage.rounds.push(await round(name, adapter, true))
}
results.impls[name] = r
// What is done so far, kept should a later implementation fail.
if (out) writeFileSync(out, `${JSON.stringify(results, null, 2)}\n`)
}
} finally {
await pool.close()
}
results.machine.loadAtEnd = loadavg()[0]
if (out) writeFileSync(out, `${JSON.stringify(results, null, 2)}\n`)
process.stdout.write(markdown(results))
}
/// The results as Markdown tables: the machine and the client's ceiling;
/// a summary, a column per implementation, the reference leftmost, as the
/// baseline; then per part, each implementation's best and median round;
/// the routes; the curves; the mix; the churn; start time and memory. The
/// reference comes first in every table.
function markdown(results) {
const med = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)]
const mib = (k) => (k / 1024).toFixed(1)
const ms = (x) => (x === undefined ? '' : x.toFixed(1))
const names = Object.keys(results.impls).sort((a, b) => (b === 'reference') - (a === 'reference'))
const lines = []
const { parts: workload, routes: routeLoad } = results.workload
const partsOf = Object.keys(workload)
const ceilingAt = (at) => results.ceiling?.[at]
// The best rate of a part's rounds `xs`. One near what the client can send
// at all is marked ‡: the client may be part of what holds it. One where
// the client waited for local ports is marked ◊.
const rateCell = (xs, at) => {
const rate = Math.max(...xs.map((x) => x.rate))
return `${rate.toFixed(0)}${ceilingAt(at) && rate >= NEAR_CEILING * ceilingAt(at) ? ' ‡' : ''}${xs.some((x) => x.portWaits) ? ' ◊' : ''}`
}
const nearNote = `‡: at least ${NEAR_CEILING * 100}% of the client's ceiling at the same concurrency (above): the client may be part of what limits it. ◊: the server closed each connection after its answer, the client opened one a request, and it ran out of local ports and waited for them (each closed connection holds its port for a while): the rate is held to how fast they came back, and the times are from when each request was sent.`
const loads = names.flatMap((n) => results.impls[n].rounds.flatMap((r) => r.load ?? []))
lines.push('### Machine and client', '')
lines.push(`- ${results.machine.cpu}, ${results.machine.cores} cores, ${results.machine.memGiB} GiB, ${results.machine.os}; Node ${results.machine.node}; ${results.machine.docker}.`)
lines.push(`- Load average (1 min): ${results.machine.loadAtStart.toFixed(1)} at the start, ${(results.machine.loadAtEnd ?? 0).toFixed(1)} at the end; ${Math.min(...loads).toFixed(1)} to ${Math.max(...loads).toFixed(1)} at the rounds' starts and ends.`)
lines.push(`- The client: ${results.clients} worker threads, each with its own connections, kept alive. Its ceiling, a responder in Docker answering every request with 2 bytes and doing nothing else, requests per second (the best of three):`, '')
lines.push(`| at once | ${Object.keys(results.ceiling).join(' | ')} |`, `|---|${Object.keys(results.ceiling).map(() => '--:').join('|')}|`)
lines.push(`| per s | ${Object.values(results.ceiling).map((x) => x.toFixed(0)).join(' | ')} |`, '')
const lagged = new Set(['firehose', 'firehoseLoaded'])
const lagOf = "each time is the firehose's lag, from a write's answer to a subscriber having its commit, 0 when the commit came first. No rate: the writes' would be the write path's, which the write parts measure."
const about = {
createRecordSerial: (n) => `${n} requests, one after another, as one person posting: no other write to share a sync with.`,
uploadBlob: (n, at) => `${n} requests, ${at} at once, of 64 KiB each; then, not timed, a record of each account referring to its first blob, for getBlob.`,
firehose: (n, at) => `${n} putRecords one after another, ${at} subscribers; ${lagOf}`,
firehoseLoaded: (n, at) => `${n} putRecords, ${LOADED_WRITERS} at once, each to a different account, ${at} subscribers; ${lagOf}`,
replay: (n, at) => `${at} subscribers from cursor 0, as relays catching up: each is sent every event the server made in the round, warm-up and all, then a write made once all are connected. Throughput is the events sent to them all up to that write's commit, per second; each time is a subscriber's, from asking to having that commit.`,
}
const best = (xs, k, lo) => (lo ? Math.min(...xs.map((x) => x[k])) : Math.max(...xs.map((x) => x[k])))
lines.push('### Summary', '', "Each part's requests per second (replay: events sent per second), the mix's, start time and memory, each the best round; for the firehose's live parts, the p50 of their lag, the best round. Detail follows.", '')
lines.push(`| | ${names.join(' | ')} |`, `|---|${names.map(() => '--:').join('|')}|`)
for (const part of partsOf) {
const lag = lagged.has(part)
const cell = (name) => {
const r = results.impls[name]
if (r.unsupported[part]) return 'not served'
const xs = r.rounds.map((x) => x.parts[part])
return lag ? best(xs, 'p50', true).toFixed(1) : part === 'replay' ? best(xs, 'rate').toFixed(0) : rateCell(xs, workload[part].at)
}
lines.push(`| ${lag ? `${part} lag p50 ms` : `${part} per s`} | ${names.map(cell).join(' | ')} |`)
}
lines.push(`| mix per s | ${names.map((name) => {
const xs = results.impls[name].rounds.map((x) => x.mix)
return xs[0].notServed ? 'not served' : rateCell(xs, results.workload.mix.at)
}).join(' | ')} |`)
const each = (f) => names.map((name) => f(results.impls[name].rounds)).join(' | ')
lines.push(`| start ms | ${each((rs) => best(rs, 'startMs', true).toFixed(0))} |`)
lines.push(`| idle RSS MiB (median) | ${each((rs) => mib(med(rs.map((x) => x.idleKiB))))} |`)
lines.push(`| peak RSS MiB | ${each((rs) => mib(best(rs, 'peakKiB', true)))} |`, '', nearNote, '')
for (const part of partsOf) {
const { n, at } = workload[part]
const lag = lagged.has(part)
const someRound = names.map((name) => results.impls[name].rounds[0].parts[part]).find(Boolean)
const tail = someRound?.tailLabel ?? 'tail'
lines.push(`#### ${part}`, '', about[part]?.(n, at) ?? `${n} requests, ${at} at once.`, '')
lines.push(
lag ? `| impl | p50 ms (best) | p50 ms (median) | ${tail} ms (best) | ${tail} ms (median) |` : `| impl | per s (best) | per s (median) | p50 ms (best) | p50 ms (median) | ${tail} ms (best) | ${tail} ms (median) |`,
lag ? '|---|--:|--:|--:|--:|' : '|---|--:|--:|--:|--:|--:|--:|',
)
for (const name of names) {
const r = results.impls[name]
if (r.unsupported[part]) {
lines.push(`| ${name} | not served: ${r.unsupported[part]} |${lag ? '|||' : '|||||'}`)
continue
}
const xs = r.rounds.map((x) => x.parts[part])
const g = (k) => xs.map((x) => x[k])
const rate = lag ? '' : `${part === 'replay' ? best(xs, 'rate').toFixed(0) : rateCell(xs, at)} | ${med(g('rate')).toFixed(0)} | `
lines.push(`| ${name} | ${rate}${ms(best(xs, 'p50', true))} | ${ms(med(g('p50')))} | ${ms(best(xs, 'tail', true))} | ${ms(med(g('tail')))} |`)
}
lines.push('')
}
lines.push('### Routes', '', 'The routes a client calls most, besides the parts above, each measured alone: requests per second, then the p50 and the tail of their times, each the best round. A route a server answered with no 2xx in the warm-up is not served, with its answer.', '')
const routeNames = Object.keys(routeLoad)
lines.push(`| route | n, at once | ${names.map((n) => `${n} per s | p50 ms | tail ms`).join(' | ')} |`, `|---|---|${names.map(() => '--:|--:|--:').join('|')}|`)
const why = []
for (const r of routeNames) {
const { n, at } = routeLoad[r]
const cells = names.map((name) => {
const rounds = results.impls[name].rounds
const notServed = rounds[0].notServed[r]
if (notServed) {
why.push(`${name}, ${r}: ${notServed}`)
return 'not served | | '
}
const xs = rounds.map((x) => x.routes[r])
return `${rateCell(xs, at)} | ${ms(best(xs, 'p50', true))} | ${ms(best(xs, 'tail', true))} (${xs[0].tailLabel})`
})
lines.push(`| ${r} | ${n}, ${at} | ${cells.join(' | ')} |`)
}
lines.push('', nearNote, '')
if (why.length) lines.push('Not served:', '', ...why.map((w) => `- ${w}`), '')
lines.push('### By the number of callers', '', `Each at ${results.workload.callers.join(', ')} callers at once: requests answered with a 2xx per second, and the p50 and tail of their times, the best round; and how many were answered otherwise, in the median round, counted rather than stopped at.`, '')
for (const c of ['getRecord', 'createRecord']) {
lines.push(`#### ${c}`, '', `| callers | ${names.join(' | ')} |`, `|--:|${names.map(() => '--:').join('|')}|`)
for (const callers of results.workload.callers) {
const cells = names.map((name) => {
const xs = results.impls[name].rounds.map((x) => x.curves[c][callers])
const errors = med(xs.map((x) => x.errors))
return `${rateCell(xs, callers)}/s, p50 ${ms(best(xs, 'p50', true))}, ${xs[0].tailLabel} ${ms(best(xs, 'tail', true))} ms${errors ? `, **${errors} not 2xx** (${xs.find((x) => x.firstError)?.firstError ?? ''})` : ''}`
})
lines.push(`| ${callers} | ${cells.join(' | ')} |`)
}
lines.push('')
}
const { shares, n: mixN, at: mixAt } = results.workload.mix
lines.push('### The mix', '', `${mixN} requests, ${mixAt} at once, of each 100 so many of each route, with 2 subscribers to the firehose: requests per second, then each route's p95 in the mix beside its p95 measured alone (the parts' and the routes' tables), the best round, in ms.`, '')
lines.push(`| route | of 100 | ${names.map((n) => `${n} p95 in mix | alone`).join(' | ')} |`, `|---|--:|${names.map(() => '--:|--:').join('|')}|`)
lines.push(`| all, per s | | ${names.map((name) => {
const xs = results.impls[name].rounds.map((x) => x.mix)
return xs[0].notServed ? `not served: ${xs[0].notServed} |` : `${rateCell(xs, mixAt)} |`
}).join(' | ')} |`)
for (const [r, share] of Object.entries(shares)) {
const cells = names.map((name) => {
const rounds = results.impls[name].rounds
if (rounds[0].mix.notServed) return ' | '
const inMix = rounds.map((x) => x.mix.routes[r])
const alone = rounds.map((x) => x.parts[r] ?? x.routes[r]).filter(Boolean)
return `${ms(best(inMix, 'p95', true))} | ${alone.length ? ms(best(alone, 'p95', true)) : ''}`
})
lines.push(`| ${r} | ${share} | ${cells.join(' | ')} |`)
}
lines.push('')
lines.push('### Subscribers coming and going', '', `${results.workload.cohorts + 1} cohorts of 20 subscribers, each from an address of its own: a cohort connects, has one write's commit and goes. The server's threads and RSS 2 s after the first cohort went and 2 s after the last, and how many of the subscribers it refused, the median round.`, '')
lines.push('| impl | threads after the first | after the last | RSS MiB after the first | after the last | refused, of all |', '|---|--:|--:|--:|--:|--:|')
for (const name of names) {
const xs = results.impls[name].rounds.map((x) => x.churn)
lines.push(`| ${name} | ${med(xs.map((x) => x.first.threads))} | ${med(xs.map((x) => x.last.threads))} | ${mib(med(xs.map((x) => x.first.rssKiB)))} | ${mib(med(xs.map((x) => x.last.rssKiB)))} | ${med(xs.map((x) => x.refused))} of ${xs[0].subscribers} |`)
}
lines.push('')
lines.push('### Start time and memory', '', "Start: from `docker start` of the server's container, made already, to its first answered `describeServer`, on an empty volume. Idle RSS: 2 s after start, before any request. Peak RSS: the most sampled (every ~100 ms) through the warm-up and the whole round. Each the server's own process in its container, from `docker top`.", '')
lines.push('| impl | start ms (best) | start ms (median) | idle RSS MiB (median) | peak RSS MiB (best) | peak RSS MiB (median) |', '|---|--:|--:|--:|--:|--:|')
for (const name of names) {
const rs = results.impls[name].rounds
lines.push(`| ${name} | ${best(rs, 'startMs', true).toFixed(0)} | ${med(rs.map((x) => x.startMs)).toFixed(0)} | ${mib(med(rs.map((x) => x.idleKiB)))} | ${mib(best(rs, 'peakKiB', true))} | ${mib(med(rs.map((x) => x.peakKiB)))} |`)
}
lines.push('')
const withBeside = names.filter((name) => results.impls[name].rounds.some((x) => x.beside))
if (withBeside.length) {
lines.push('Not in the table above: what a server needs running beside it, its RSS sampled apart (the sum over its container\'s processes, from `docker top`, as fast as that answers; for PostgreSQL, whose processes share its buffers, the sum of their PSS, which counts each shared page once).', '')
lines.push('| impl | beside it | idle MiB (median) | peak MiB (best) | peak MiB (median) |', '|---|---|--:|--:|--:|')
for (const name of withBeside) {
const xs = results.impls[name].rounds.map((x) => x.beside)
lines.push(`| ${name} | ${xs[0].name} | ${mib(med(xs.map((x) => x.idleKiB)))} | ${mib(best(xs, 'peakKiB', true))} | ${mib(med(xs.map((x) => x.peakKiB)))} |`)
}
lines.push('')
}
const stored = names.filter((name) => results.impls[name].storage?.rounds.length)
if (stored.length) {
const { backfillMs, quietMs, sampleMs, samplesMs } = results.storage
const mb = (b) => (b / 2 ** 20).toFixed(1)
lines.push('### Storage', '', `From each implementation's storage round${results.storage.rounds > 1 ? `s (${results.storage.rounds}, the median)` : ''}, apart from the rounds above: a fresh server started with a backfill window W of ${backfillMs / 1000} s where it has a setting for one, the same warm-up of the parts and the parts, whose speed is not kept (a short window changes what replay sends). What it keeps on disk: the sum of the sizes of the files on its container's volume (its log goes to \`docker logs\`, not there). Under load: once the parts end, after replay. At rest: after the parts, no request for 2 W + 1 s (${quietMs / 1000} s), then one createRecord to each account the round made, the warm-up's too, then the size sampled every ${sampleMs} ms for ${samplesMs / 1000} s, the least kept (spec/STORE.md, "Bounds"). Live state: of each of those accounts, its getRepo, the blobs listBlobs lists and the uploads no record references.`, '')
lines.push('| impl | backfill window | data under load MiB (median) | data at rest MiB (median) | live state MiB (median) | at rest ÷ live (median) |', '|---|---|--:|--:|--:|--:|')
const notes = []
for (const name of stored) {
const r = results.impls[name]
const ds = r.storage.rounds
if (r.storage.noWindow) notes.push(`${name}: ${r.storage.noWindow}.`)
lines.push(`| ${name} | ${r.storage.window ? `W = ${backfillMs / 1000} s` : 'its default: no setting'} | ${mb(med(ds.map((d) => d.underLoadBytes)))} | ${mb(med(ds.map((d) => d.atRestBytes)))} | ${mb(med(ds.map((d) => d.liveBytes)))} | ${med(ds.map((d) => d.atRestBytes / d.liveBytes)).toFixed(2)} |`)
}
lines.push('', ...notes.flatMap((n) => [n, '']))
const withBesideDisk = stored.filter((name) => results.impls[name].storage.rounds.some((d) => d.beside))
if (withBesideDisk.length) {
lines.push('Not in the table above: what a server keeps in what runs beside it (vlpds: its bucket, the files on RustFS\'s volume; tranquil: its database, the files on PostgreSQL\'s volume), measured the same way at the same moments; the last column counts the server\'s own and this together.', '')
lines.push('| impl | beside it | under load MiB (median) | at rest MiB (median) | both at rest ÷ live (median) |', '|---|---|--:|--:|--:|')
for (const name of withBesideDisk) {
const ds = results.impls[name].storage.rounds
lines.push(`| ${name} | ${ds[0].beside.name} | ${mb(med(ds.map((d) => d.beside.underLoadBytes)))} | ${mb(med(ds.map((d) => d.beside.atRestBytes)))} | ${med(ds.map((d) => (d.atRestBytes + d.beside.atRestBytes) / d.liveBytes)).toFixed(2)} |`)
}
lines.push('')
}
}
return `${lines.join('\n')}\n`
}