// A benchmark of PDS implementations, one at a time, on one machine, every // one in Docker the same way: perf.mjs's workload with applyWrites, // listRecords and more besides; the routes a client calls most; reads and // writes by the number of callers; a mix of them at once; and subscribers // coming and going. Each round is a fresh server on fresh data with a fresh // PLC directory in memory, warmed the same way first. It reports each part's // throughput, p50 and tail (best and median of the rounds), the server's // start time (container start to the first answered describeServer), its RSS // idle and at peak, and what the client itself can send (its ceiling). Then, // apart, a storage round of each: a fresh server with a short backfill window // W, the same warm-up and parts, and what it keeps on disk under load (once // the parts end) and at rest (after a quiet of 2 W), against the accounts' // live state (spec/STORE.md, "Bounds"). // // node bench.mjs --setup (build and pull everything bench/impls.json pins) // node bench.mjs [--rounds 3] [--storage-rounds 1] [--backfill-ms 10000] [--impls monolith,reference,vlpds] [--clients 4] [--busy] [--json out.json] // node bench.mjs --report out.json (the tables again from a run's --json) // // Each implementation is an adapter in ADAPTERS: how to make its container // against a PLC directory, on a port, and start and stop it; what to send // createAccount; and what it is (version, commit, storage). To add one, add // an adapter and its pin in bench/impls.json (bench/README.md); a part it // does not serve goes in its `unsupported`, with why. A route a server does // not serve is found in the warm-up, and its answer is the table's why. import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync, writeSync } from 'node:fs' import { cpus, loadavg, release, totalmem, tmpdir, type } from 'node:os' import { join } from 'node:path' import { openPool, timedFetch } from './lib/loadpool.mjs' import { VLPDS_MEM_MIB, containerLogs, containerPssKiB, containerRssKiB, dockerPort, fromDocker, imageId, pins, root, sh, shOk, startPostgres, startRustfs, tranquilCreateArgs, tranquilEnv, tranquilOwnEnv, vlpdsArgs, vlpdsCreateArgs, vlpdsEnv, vlpdsOwnEnv, zdsCreateArgs, zdsEnv, zdsOwnEnv } from './lib/impls.mjs' const here = new URL('.', import.meta.url).pathname // What each implementation runs, pinned: bench/impls.json (lib/impls.mjs). const vendor = join(root, 'bench', 'vendor') const pinnedImages = () => [...Object.values(pins.implementations).map((x) => x.image), ...Object.values(pins.images).map((x) => x.image)].filter(Boolean) // The label an image built from this repository carries: the last commit // that changed what it is built from (its pin's `inputs`), `-dirty` if any of // those has changes not committed. A change elsewhere, as to the bench // itself, leaves an image as good as it was. const REVISION = 'org.opencontainers.image.revision' const revision = (inputs) => { const git = (...args) => spawnSync('git', [...args, '--', ...inputs], { cwd: root, encoding: 'utf8' }).stdout.trim() return `${git('log', '-1', '--format=%H')}${git('status', '--porcelain') ? '-dirty' : ''}` } // The firehose under load: its writes, this many at once. const LOADED_WRITERS = 16 // The numbers of callers each of the curves is measured at. const CALLERS = [1, 4, 16, 64, 256] // The concurrencies the client's ceiling is measured at: every one a part // uses. const CEILING_AT = [1, 4, 8, 16, 32, 64, 256] // A part whose best rate is at least this share of the client's ceiling at // the same concurrency may be held back by the client. const NEAR_CEILING = 0.5 /// --setup: everything the bench runs, from the pins alone. Run again, it /// pulls no image it has and builds only what changed. function setup() { const fail = (why) => { console.error(`setup: ${why}`) process.exit(1) } const run = (cmd, args, cwd = root) => { console.error(`setup: (${cwd}) ${cmd} ${args.join(' ')}`) const r = spawnSync(cmd, args, { cwd, stdio: 'inherit' }) if (r.error?.code === 'ENOENT') fail(`${cmd} is not installed (see bench/README.md, Prerequisites)`) if (r.status !== 0) fail(`${cmd} ${args.join(' ')} failed (exit ${r.status})`) } const quiet = (cmd, args, cwd = root) => spawnSync(cmd, args, { cwd, encoding: 'utf8' }) const major = Number(process.versions.node.split('.')[0]) if (major < 24) fail(`Node ${process.versions.node}: the bench needs Node 24 or later`) const needGit = Object.values(pins.implementations).some((x) => x.repo) for (const [cmd, args, what] of [['rustup', ['--version'], 'rustup (Rust toolchains)'], ['npm', ['--version'], 'npm'], ['docker', ['--version'], 'Docker'], ...(needGit ? [['git', ['--version'], 'git']] : [])]) { const r = quiet(cmd, args) if (r.error || r.status !== 0) fail(`${what} is missing: \`${cmd} ${args.join(' ')}\` failed (see bench/README.md, Prerequisites)`) } if (quiet('docker', ['info']).status !== 0) fail('Docker is installed but its daemon does not answer (`docker info` failed): start Docker') // Built here, outside Docker: what the compatibility check runs // (compat.mjs). for (const impl of Object.values(pins.implementations)) { if (!impl.build) continue const cwd = join(root, impl.cwd ?? '') // Rust builds go through rustup at the toolchain pinned, whatever // `cargo` is first on the PATH. if (impl.toolchain) run('rustup', ['toolchain', 'install', impl.toolchain, '--profile', 'minimal', '--no-self-update'], cwd) run(impl.build[0], impl.build.slice(1), cwd) } // Built from this repository into an image, every time: Docker's own // cache leaves nothing to do when nothing changed. Labelled with the // commit of what it is built from, so a run can tell an image older than it. for (const impl of Object.values(pins.implementations)) { if (!impl.context) continue run('docker', ['build', '-t', impl.tag, '-f', join(root, impl.dockerfile), '--label', `${REVISION}=${revision(impl.inputs)}`, join(root, impl.context)]) } // Built from another repository at its pinned commit, with its own // Dockerfile, unless an image of that tag is there already. for (const [name, impl] of Object.entries(pins.implementations)) { if (!impl.repo || quiet('docker', ['image', 'inspect', impl.tag]).status === 0) continue const dir = join(vendor, name) if (!existsSync(join(dir, '.git'))) { mkdirSync(dir, { recursive: true }) run('git', ['init', '-q'], dir) } if (quiet('git', ['rev-parse', 'HEAD'], dir).stdout?.trim() !== impl.commit) { run('git', ['fetch', '-q', '--depth', '1', impl.repo, impl.commit], dir) run('git', ['checkout', '-q', '--detach', impl.commit], dir) } run('docker', ['build', '-t', impl.tag, '-f', join(dir, impl.dockerfile), dir]) } for (const image of pinnedImages()) { if (quiet('docker', ['image', 'inspect', image]).status !== 0) run('docker', ['pull', image]) } console.error('setup: done') } if (process.argv[2] === '--setup') setup() // The tables again from a run's --json, without running anything. else if (process.argv[2] === '--report') process.stdout.write(markdown(JSON.parse(readFileSync(process.argv[3], 'utf8')))) else await main() /// The tail a part's times are reported at: a percentile at least ten of /// them lie above, or the slowest. function tailOf(n) { if (n >= 1000) return ['p99', 99] if (n >= 200) return ['p95', 95] if (n >= 100) return ['p90', 90] return ['max', 100] } function pct(xs, p) { const s = Array.from(xs).sort((x, y) => x - y) return s[Math.min(s.length - 1, Math.max(0, Math.ceil((p / 100) * s.length) - 1))] } async function main() { const { cborDecodeMulti } = await import('@atproto/common') const { Database, PlcServer } = await import('@did-plc/server') const { Secp256k1Keypair } = await import('@atproto/crypto') const arg = (name, fallback) => { const i = process.argv.indexOf(name) return i === -1 ? fallback : process.argv[i + 1] } const ROUNDS = Number(arg('--rounds', 3)) const CLIENTS = Number(arg('--clients', 4)) // A free port below the ephemeral range, for a server in Docker // (lib/impls.mjs). const freePort = dockerPort const vlpdsMemMiB = VLPDS_MEM_MIB const rotationHex = async () => Buffer.from(await (await Secp256k1Keypair.create({ exportable: true })).export()).toString('hex') const envArgs = (env) => Object.entries(env).flatMap(([k, v]) => ['-e', `${k}=${v}`]) const dockerAbout = async () => `Docker ${await sh('docker', ['version', '--format', '{{.Server.Version}}'])}, its Linux VM ${await sh('docker', ['info', '--format', '{{.NCPU}} CPUs, {{.MemTotal}} bytes'])}` const imageAbout = async (tag) => `${tag} (${await sh('docker', ['image', 'inspect', '--format', '{{.Id}} {{.Os}}/{{.Architecture}}', tag])})` const imageRevision = (tag) => sh('docker', ['image', 'inspect', '--format', `{{index .Config.Labels "${REVISION}"}}`, tag]) const running = (name) => async () => (await sh('docker', ['inspect', '--format', '{{.State.Running}}', name])) === 'false' // The storage rounds, after an implementation's speed rounds: each a fresh // server started with a short backfill window W, where it has a setting // for one (`window` in its adapter), for its size at rest: after the // parts, no request for 2 W + 1 s, then one write to each account, then // the size sampled every 500 ms for 10 s, the least kept (spec/STORE.md, // "Bounds"). Apart from the speed rounds, which run on each server's // default window, as a short one changes what replay sends. const STORAGE_ROUNDS = Number(arg('--storage-rounds', 1)) const BACKFILL_MS = Number(arg('--backfill-ms', 10000)) const QUIET_MS = 2 * BACKFILL_MS + 1000 const REST_SAMPLE_MS = 500 const REST_SAMPLES_MS = 10000 const sleep = (ms) => new Promise((ok) => setTimeout(ok, Math.max(0, ms))) // vlpds's backfill window is its log segment retention, in whole seconds // or milliseconds, with a pass every W / 2 (within its 1 s to 10 min). const vlpdsRetention = (ms) => ({ retention: ms % 1000 ? `${ms}ms` : `${ms / 1000}s`, interval: `${Math.min(600, Math.max(1, Math.round(ms / 2000)))}s` }) /// The sum of the sizes of the files under `dir` in the running container /// `name` (its volume): find and stat in the container, as root (stat, as /// BusyBox's find, in RustFS's image, has no -printf); or, where its image /// has no shell, in a container of the image `reader` given its volumes. async function containerBytes(name, dir, reader) { const where = reader ? ['run', '--rm', '-u', '0', '--volumes-from', name, '--entrypoint', 'sh', reader] : ['exec', '-u', '0', name, 'sh'] const out = await shOk('docker', [...where, '-c', 'find "$1" -type f -exec stat -c %s {} + 2>/dev/null; true', 'sh', dir]) return out.split('\n').reduce((sum, l) => sum + (Number(l) || 0), 0) } // ---- Adapters ---------------------------------------------------------- // Every server runs in a container of its own, made in `setup` before the // start clock with a fresh volume; `start` only starts it. Each reaches // the bench's PLC directory through host.docker.internal, and the bench // reaches it on a loopback port; each is told its URL is that port. // `comm` is the server's process in its container, whose RSS and threads // are the server's; `volume` the directory in it that holds all it keeps // (its log goes to `docker logs`, not there), and `reader`, where its // image has no shell to read that with, an image that has one. `setup` is // given `backfillMs` in a storage round only; `window` says how it sets // the server's backfill window, or `noWindow` why nothing does. const ADAPTERS = { // Monolith as its deployment builds and runs it: the repository's // Dockerfile (docs/guide/deploying.md), its data on the volume /data, // given only what a local benchmark needs (as perf.mjs's, lib/env.mjs, // which this cannot import, as its imports read compare.mjs's // arguments): no rate limits, no invite codes, every proxy the reference // trusts trusted; its DID cache as it comes. 'monolith': { comm: 'pds', volume: '/data', window: 'PDS_REPO_BACKFILL_LIMIT_MS', async info() { const pin = pins.implementations['monolith'] return { version: (await sh('docker', ['run', '--rm', pin.tag, '--version'])) || `workspace ${readFileSync(join(root, 'Cargo.toml'), 'utf8').match(/^version = "([^"]+)"/m)?.[1]}`, commit: await imageRevision(pin.tag), source: `this repository's Dockerfile, image ${await imageAbout(pin.tag)}; ${await dockerAbout()}`, storage: 'its own append-only logs and files on the volume /data (spec/STORE.md); blobs as files there', durability: 'every answered write synced before the answer, group-committed (spec/DURABILITY.md), with Rust\'s `sync_all`/`sync_data`, which on Linux are fsync(2)/fdatasync(2); on the Docker VM\'s virtual disk', config: `docker run -v /data; the image's PDS_PORT=2583, PDS_BIND_ADDRESS=::, PDS_DATA_DIR=/data; ${Object.entries(monolithEnv({ port: '', plcUrl: 'http://127.0.0.1:', rotation: '' })).map(([k, v]) => `${k}=${v}`).join(' ')}`, } }, async setup({ port, plcUrl, log, backfillMs }) { const name = `bench-monolith-${process.pid}-${Date.now()}` const teardown = async () => { await containerLogs(name, log) await sh('docker', ['rm', '-f', '-v', name]) } try { const env = monolithEnv({ port, plcUrl, rotation: await rotationHex() }) if (backfillMs !== undefined) env.PDS_REPO_BACKFILL_LIMIT_MS = String(backfillMs) await shOk('docker', ['create', '--name', name, '-p', `127.0.0.1:${port}:2583`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/data', ...envArgs(env), pins.implementations['monolith'].tag]) } catch (e) { await teardown() throw e } return { name, teardown } }, async start({ port, aux }) { await shOk('docker', ['start', aux.name]) return { url: `http://127.0.0.1:${port}`, exited: running(aux.name), async stop() { // SIGTERM: it shuts down within Docker's 10 s (docs/guide/deploying.md). await sh('docker', ['stop', aux.name]) }, } }, unsupported: {}, }, // The reference, as perf.mjs runs it (dev-env's TestPds), in the image // bench/docker/reference.Dockerfile builds from the harness's packages: // its data and blobs on the volume /data. It listens inside on the port // it is reached at, as it did outside Docker. reference: { // Node names its main thread's process so. comm: 'MainThread', volume: '/data', window: 'repoBackfillLimitMs (PDS_REPO_BACKFILL_LIMIT_MS)', async info() { const pin = pins.implementations.reference const pkg = (name) => JSON.parse(readFileSync(`${here}node_modules/${name}/package.json`, 'utf8')).version return { version: `@atproto/pds ${pkg('@atproto/pds')} (through @atproto/dev-env ${pkg('@atproto/dev-env')}), Node ${await sh('docker', ['run', '--rm', '--entrypoint', 'node', pin.tag, '--version'])}`, commit: `npm release (no git checkout), packages of harness/package-lock.json as at ${await imageRevision(pin.tag)}`, source: `npm, pinned in harness/package-lock.json; image ${await imageAbout(pin.tag)}; ${await dockerAbout()}`, storage: 'SQLite (better-sqlite3): one service DB, a sequencer DB, a DID cache DB, and one DB per account; blobs as files; all on the volume /data', durability: 'SQLite WAL, synchronous=FULL (better-sqlite3 default): fsync(2) on each commit, on the Docker VM\'s virtual disk; blobs written with fs.writeFile, never fsynced', config: 'docker run -v /data; PORT=, PLC_URL=http://host.docker.internal:, and in the storage rounds only BACKFILL_MS=; dev-env TestPds with rateLimitsEnabled false, inviteRequired false, hostname localhost, and in the storage rounds repoBackfillLimitMs W (harness/bench-reference.mjs)', } }, async setup({ port, plcUrl, log, backfillMs }) { const name = `bench-reference-${process.pid}-${Date.now()}` const teardown = async () => { await containerLogs(name, log) await sh('docker', ['rm', '-f', '-v', name]) } try { await shOk('docker', ['create', '--name', name, '-p', `127.0.0.1:${port}:${port}`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/data', ...envArgs({ PORT: port, PLC_URL: fromDocker(plcUrl), ...(backfillMs !== undefined && { BACKFILL_MS: backfillMs }) }), pins.implementations.reference.tag]) } catch (e) { await teardown() throw e } return { name, teardown } }, async start({ port, aux }) { await shOk('docker', ['start', aux.name]) return { url: `http://127.0.0.1:${port}`, exited: running(aux.name), async stop() { await sh('docker', ['stop', aux.name]) }, } }, unsupported: {}, }, // Bluesky's Very Large PDS, its release image (bench/impls.json). Its // only durable state is an S3 bucket: each round gets a fresh RustFS // container (the store that passed vlpds-bucket-probe, bench/README.md) // on a Docker network of its own, and a fresh vlpds container on it, // both made in `setup`. Real secrets (not --dev-mode), fresh each round; // PLC registration on. vlpds: { comm: 'vlpds', // Its SST cache alone: everything else is in the bucket, beside it. volume: '/var/lib/vlpds', window: 'VLPDS_LOG_RETENTION (log segment retention: its firehose backfill window), with VLPDS_LOG_RETENTION_INTERVAL at W / 2 so that a retention pass falls within the quiet (its default 60 s would not)', async info() { const pin = pins.implementations.vlpds const store = pins.images[pin.needs[0]] return { version: `${pin.release} (${await sh('docker', ['image', 'inspect', '--format', '{{index .Config.Labels "org.opencontainers.image.version"}}', pin.image])})`, commit: pin.commit, source: `release image ${pin.image}; S3 store ${store.image} (${store.release}); ${await dockerAbout()}`, storage: 'everything in the S3 bucket (RustFS, one drive on a Docker volume): commit log segments, SlateDB state (one shard), blobs under blob/{did}/{cid}; local disk (a Docker volume) only an SST cache', durability: 'a write is answered once the log segment holding it is PUT to the bucket; uploadBlob answers once the blob is PUT to the bucket. RustFS in its default `strict` durability mode fsyncs object data, xl.meta and the commit renames\' directories before answering a PUT, on the Docker VM\'s virtual disk', config: `docker run --memory ${vlpdsMemMiB}m --memory-swap ${vlpdsMemMiB}m --stop-timeout 90 -v /var/lib/vlpds; ${Object.entries(vlpdsEnv).map(([k, v]) => `${k}=${v}`).join(' ')}; args ${vlpdsArgs.join(' ')}; VLPDS_PUBLIC_URL=http://127.0.0.1:, VLPDS_PLC_URL=http://host.docker.internal:, VLPDS_S3_* the round's RustFS; in the storage rounds only, VLPDS_LOG_RETENTION=${vlpdsRetention(BACKFILL_MS).retention} and VLPDS_LOG_RETENTION_INTERVAL=${vlpdsRetention(BACKFILL_MS).interval} (the bench's backfill window W, and W / 2 between retention passes); fresh random JWT secret, admin and internal tokens, KEK and secp256k1 PLC rotation key each round; handle domain vlpds.test, invites not required, threads (I/O = cores, workers = cores / 2) all its defaults`, } }, async setup({ port, plcUrl, log, backfillMs }) { const id = `bench-vlpds-${process.pid}-${Date.now()}` const store = `${id}-s3` const pds = `${id}-pds` const teardown = async () => { await containerLogs(pds, log) await sh('docker', ['rm', '-f', '-v', pds]) await sh('docker', ['rm', '-f', '-v', store]) await sh('docker', ['network', 'rm', id]) } try { await shOk('docker', ['network', 'create', id]) const s3 = await startRustfs(id, store) const env = { ...vlpdsEnv, ...vlpdsOwnEnv({ port, plcUrl, rotationHex: await rotationHex(), store: s3 }) } if (backfillMs !== undefined) { const { retention, interval } = vlpdsRetention(backfillMs) Object.assign(env, { VLPDS_LOG_RETENTION: retention, VLPDS_LOG_RETENTION_INTERVAL: interval }) } await shOk('docker', vlpdsCreateArgs({ name: pds, network: id, port, env })) } catch (e) { await teardown() throw e } return { name: pds, // RustFS's processes' RSS, and its bucket (the files on its // volume), apart from vlpds's. beside: { name: 'RustFS', rssKiB: () => containerRssKiB(store), diskBytes: () => containerBytes(store, '/data') }, teardown, } }, async start({ port, aux }) { await shOk('docker', ['start', aux.name]) return { url: `http://127.0.0.1:${port}`, exited: running(aux.name), async stop() { // Graceful (SIGTERM): it hands back its shard and fences its log. await sh('docker', ['stop', '-t', '90', aux.name]) }, } }, // Handles under its one handle domain, vlpds.test (its default). accountBody: (i, tag) => ({ handle: `${tag}${i}.vlpds.test`, email: `${tag}${i}@example.com`, password: `bench-password-${i}` }), unsupported: {}, }, // zds, a PDS in Zig on SQLite, its image built by --setup from its own // Dockerfile at the pinned commit (bench/impls.json). Run as its docs // run it (docs/deployment.md, fly.toml: the database and blobs under // /data, on a volume), with a fresh rotation key, JWT secret and admin // token each round; zds has no dev mode. zds: { comm: 'zds', volume: '/data', noWindow: 'zds has no setting for it: none of the ZDS_* variables its binary reads is a backfill window or an event retention, so at rest is measured with what it keeps by default', async info() { const pin = pins.implementations.zds return { version: `${pin.release} (build.zig.zon 0.7.8; /xrpc/_health reports it)`, commit: pin.commit, source: `${pin.repo} at ${pin.commit}, image ${await imageAbout(pin.tag)} built with its own ${pin.dockerfile}; ${await dockerAbout()}`, storage: 'SQLite (bundled 3.53.0, through zqlite): one database, /data/zds.sqlite3, on one connection; record bodies as DAG-CBOR blocks; blobs as files under /data/blobs; both on a Docker volume', durability: 'SQLite journal_mode=WAL with synchronous left at the compiled-in default FULL (zds sets no synchronous pragma; zqlite compiles SQLite with no SQLITE_DEFAULT_* flags), so each commit fsyncs the WAL; fullfsync does not apply on Linux. Blobs written with fopen/fwrite/fclose to a temp file and renamed into place, never fsynced. On the Docker VM\'s virtual disk', config: `docker run -v /data; ${Object.entries(zdsEnv).map(([k, v]) => `${k}=${v}`).join(' ')}; ZDS_PUBLIC_URL=http://127.0.0.1: (server DID derived: did:web:127.0.0.1%3A), ZDS_PLC_DIRECTORY=http://host.docker.internal:; fresh random ZDS_JWT_SECRET, ZDS_ADMIN_TOKEN and secp256k1 ZDS_PLC_ROTATION_KEY each round; handle domains .test, invites not required, permissioned data off (its defaults); the Dockerfile's ZDS_HOST, ZDS_PORT=2583, ZDS_DB=/data/zds.sqlite3`, } }, async setup({ port, plcUrl, log }) { const name = `bench-zds-${process.pid}-${Date.now()}` const env = { ...zdsEnv, ...zdsOwnEnv({ port, plcUrl, rotationHex: await rotationHex() }) } const teardown = async () => { await containerLogs(name, log) await sh('docker', ['rm', '-f', '-v', name]) } try { await shOk('docker', zdsCreateArgs({ name, port, env })) } catch (e) { await teardown() throw e } return { name, teardown } }, async start({ port, aux }) { await shOk('docker', ['start', aux.name]) return { url: `http://127.0.0.1:${port}`, exited: running(aux.name), async stop() { // zds runs as the container's PID 1 with no SIGTERM handler, so // `docker stop` would only SIGKILL it after its timeout: kill it. await sh('docker', ['kill', aux.name]) }, } }, unsupported: { firehoseLoaded: 'its live firehose skips commits when writes come at once: of a warm-up of 40 putRecords, 4 at once to 4 accounts, each of 20 subscribers was sent 34 to 37 (its log: `subscribeRepos end start_cursor=466 end_cursor=506 frames=35`), so no lag can be timed for the rest', }, }, // Tranquil, a PDS in Rust, its image built by --setup from its own // Dockerfile at the pinned commit (bench/impls.json), on PostgreSQL, its // default repository backend: each round a fresh Docker network, a fresh // PostgreSQL container on it with its data on a fresh volume, and a // fresh tranquil container, configured as lib/impls.mjs configures it, // with fresh secrets; all made in `setup`. tranquil: { comm: 'tranquil-pds', // Its blobs alone: everything else is in PostgreSQL, beside it. volume: '/var/lib/tranquil-pds/blobs', // Its image is distroless, with no shell: PostgreSQL's has one. reader: pins.images.postgres.image, noWindow: 'its backfill window is FIREHOSE_BACKFILL_HOURS, in whole hours (72 by default), and it prunes the events past EVENT_RETENTION_MAX_AGE_SECS (7 days by default, and never less than that window) once a minute at most: nothing sets a window of W with a pass in the quiet, so at rest is measured with what it keeps by default', async info() { const pin = pins.implementations.tranquil const db = pins.images[pin.needs[0]] return { version: `${pin.release} (its Cargo.toml's workspace version 0.8.0)`, commit: pin.commit, source: `${pin.repo} at ${pin.commit}, image ${await imageAbout(pin.tag)} built with its own ${pin.dockerfile}; database ${db.image} (${db.release}); ${await dockerAbout()}`, storage: 'PostgreSQL (its default repository backend), one database on a Docker volume, through a pool of 10 to 100 connections: accounts, records, repository blocks and the event sequence in its tables; blobs as files under /var/lib/tranquil-pds/blobs, on a Docker volume of its own; its cache in its process (ripple, one node)', durability: 'PostgreSQL at its defaults, which tranquil leaves as they are: fsync on, synchronous_commit on, so a write is answered once its transaction\'s WAL is flushed (fdatasync(2), wal_sync_method\'s Linux default). Blobs written to a temp file, fsynced (sync_all) and renamed into place, the directory not fsynced. On the Docker VM\'s virtual disk', config: `docker run -v /var/lib/tranquil-pds/blobs on a Docker network of its own; ${Object.entries(tranquilEnv).map(([k, v]) => `${k}=${v}`).join(' ')}; DATABASE_URL the round's PostgreSQL, PLC_DIRECTORY_URL=http://host.docker.internal:; fresh random JWT_SECRET, DPOP_SECRET and MASTER_KEY each round; handles under its hostname (user_handle_domains unset); the image's SERVER_HOST=[::], SERVER_PORT=3000; its binary UPX-compressed, as its Dockerfile builds it by default. PostgreSQL: ${db.release} with POSTGRES_USER=tranquil_pds, POSTGRES_DB=pds and a fresh password, its defaults otherwise (max_connections 100, shared_buffers 128 MB), its data on a fresh volume`, } }, async setup({ port, plcUrl, log }) { const id = `bench-tranquil-${process.pid}-${Date.now()}` const store = `${id}-db` const pds = `${id}-pds` const teardown = async () => { await containerLogs(pds, log) await sh('docker', ['rm', '-f', '-v', pds]) await sh('docker', ['rm', '-f', '-v', store]) await sh('docker', ['network', 'rm', id]) } try { await shOk('docker', ['network', 'create', id]) const db = await startPostgres(id, store) await shOk('docker', tranquilCreateArgs({ name: pds, network: id, port, env: { ...tranquilEnv, ...tranquilOwnEnv({ plcUrl, db }) } })) } catch (e) { await teardown() throw e } return { name: pds, // PostgreSQL's memory, and its data directory (the files on its // volume), apart from tranquil's. Its memory is its processes' // PSS, not their RSS: the sum of their RSS counts the shared // buffers once in every backend that touched them. beside: { name: 'PostgreSQL', rssKiB: () => containerPssKiB(store, 'postgres', 'postgres'), diskBytes: () => containerBytes(store, '/var/lib/postgresql') }, teardown, } }, async start({ port, aux }) { await shOk('docker', ['start', aux.name]) return { url: `http://127.0.0.1:${port}`, exited: running(aux.name), async stop() { // SIGTERM: it shuts down gracefully on it. await sh('docker', ['stop', aux.name]) }, } }, // Handles under its one handle domain, its hostname; a password with // an upper-case letter, as it requires one. accountBody: (i, tag) => ({ handle: `${tag}${i}.tranquil.test`, email: `${tag}${i}@example.com`, password: `Bench-password-${i}` }), unsupported: {}, }, } const accountBody = (i, tag) => ({ handle: `${tag}${i}.test`, email: `${tag}${i}@example.com`, password: `bench-password-${i}` }) /// Monolith's environment besides its image's. function monolithEnv({ port, plcUrl, rotation }) { return { PDS_HOSTNAME: 'localhost', PDS_PUBLIC_URL: `http://127.0.0.1:${port}`, PDS_HANDLE_DOMAINS: '.test,.example', PDS_PLC_URL: fromDocker(plcUrl), PDS_PLC_ROTATION_KEY_HEX: rotation, PDS_ADMIN_PASSWORD: 'admin-pass', PDS_INVITE_REQUIRED: 'false', PDS_RATE_LIMITS_ENABLED: 'false', PDS_TRUSTED_PROXIES: '127.0.0.0/8,::1/128,169.254.0.0/16,fe80::/10,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fc00::/7', // An AppView, as every server on the network has (docs/guide/ // deploying.md): without one, getPreferences is not served // (spec/PROXY.md). Nothing in the workload is forwarded to it, and // nothing listens there. PDS_BSKY_APP_VIEW_URL: 'http://127.0.0.1:9', PDS_BSKY_APP_VIEW_DID: 'did:web:api.bsky.app', } } /// A container's process `comm`: its RSS in KiB and its threads, as /// `docker top` (ps in Docker's VM) has them. async function processOf(name, comm) { const rows = (await sh('docker', ['top', name, '-o', 'pid,nlwp,rss,comm'])).split('\n').slice(1).map((l) => l.trim().split(/\s+/)) const mine = rows.filter(([, , , c]) => c === comm) return { threads: mine.reduce((s, [, t]) => s + (Number(t) || 0), 0), rssKiB: mine.reduce((s, [, , r]) => s + (Number(r) || 0), 0) } } // ---- The client -------------------------------------------------------- const pool = await openPool(CLIENTS) /// A request to `nsid` (or, starting with `/`, that path) of the server. function req(srv, nsid, { params, body, bytes, type, token, method } = {}) { const url = new URL(nsid.startsWith('/') ? `${srv.url}${nsid}` : `${srv.url}/xrpc/${nsid}`) for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v) const headers = {} if (token) headers.authorization = `Bearer ${token}` if (body !== undefined) headers['content-type'] = 'application/json' if (bytes) headers['content-type'] = type return { method: method ?? (body !== undefined || bytes ? 'POST' : 'GET'), url: url.href, headers, body: bytes ?? (body === undefined ? undefined : JSON.stringify(body)) } } /// How many times the client waited for a local port (lib/loadpool.mjs), /// all told: each part's is what this grew by over it. let portWaits = 0 /// `requests`, `at` at once, from the client's threads: their times, how /// long they took together, and with `keep` their answers. Anything but /// a 2xx stops the run. async function send(srv, requests, at, keep = false) { const r = await pool.run(requests, at, { keep }) portWaits += r.portWaits if (r.firstError) { const { i, status, body } = r.firstError throw new Error(`${srv.name}: ${new URL(requests[i].url).pathname} answered ${status}: ${body}\nserver log: ${srv.logPath}`) } return { times: r.times, total: r.total, bodies: r.bodies } } /// One request from this thread, timed from sending it to having its /// answer whole; anything but a 2xx stops the run. For the firehose's /// writes, whose answers are timed against what subscribers get, and for /// what must wait on the answer before it. async function call(srv, nsid, o = {}) { const r = await timedFetch(req(srv, nsid, o)) portWaits += r.waits if (r.status < 200 || r.status >= 300) throw new Error(`${srv.name}: ${nsid} answered ${r.status}: ${r.buf.toString('utf8').slice(0, 300)}\nserver log: ${srv.logPath}`) return { ms: r.ms, size: r.buf.length, json: () => JSON.parse(r.buf.toString('utf8')) } } /// `n` calls from this thread, `at` at once, each made by `make(i)`: /// their times, and how long they took together. async function load(n, at, make) { const times = [] let next = 0 const t = performance.now() await Promise.all(Array.from({ length: at }, async () => { while (next < n) times.push(await make(next++)) })) return { times, total: performance.now() - t } } const blob = (i) => { const out = Buffer.alloc(65536) for (let o = 0; o < out.length; o += 32) createHash('sha256').update(`blob ${i} ${o}`).digest().copy(out, o) return out } const note = (i) => ({ $type: 'com.example.note', text: `a note of about a hundred bytes, number ${i}, to read back later`, n: i }) const wsOf = (srv) => `${srv.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos` const pick = (xs, i) => xs[(i * 7919) % xs.length] const own = (st, i) => st.accounts[i % st.accounts.length] /// A subscriber to the firehose from the address 198.18., from /// `cursor` if given, calling `onCommit(body)` for each #commit; answers /// once it is connected. One that cannot connect tries again for 10 s, /// as the client's ports may have run out for a moment (lib/loadpool.mjs); /// then it fails. async function subscribe(srv, k, onCommit, cursor) { for (let tries = 1; ; tries++) { try { return await new Promise((ok, fail) => { const ws = new WebSocket(`${wsOf(srv)}${cursor === undefined ? '' : `?cursor=${cursor}`}`, { headers: { 'x-forwarded-for': `198.18.${k >> 8}.${(k & 255) + 1}` } }) ws.binaryType = 'arraybuffer' const sub = { ws, events: 0 } ws.onmessage = (m) => { sub.events++ const [header, body] = cborDecodeMulti(new Uint8Array(m.data)) if (header?.t === '#commit') onCommit(body, sub) } ws.onopen = () => ok(sub) ws.onerror = () => fail(new Error(`${srv.name}: subscribeRepos${cursor === undefined ? '' : ` from cursor ${cursor}`}`)) }) } catch (e) { if (tries === 40) throw e await new Promise((ok) => setTimeout(ok, 250)) } } } const until = async (done, ms) => { for (let w = 0; w < ms / 50 && !done(); w++) await new Promise((ok) => setTimeout(ok, 50)) return done() } /// `subsN` subscribers, each from an address of its own (Monolith holds 16 /// at most from one, spec/LIMITS.md), and `n` putRecords, `writers` at /// once (no more than there are accounts, so that they go to different /// ones), over the accounts: each time is the firehose's lag, from a /// write's answer to a subscriber having its commit, 0 when the commit /// came first. No rate: the writes' would be the write path's, which the /// write parts measure. async function fanOut(srv, st, n, subsN, writers, key) { const written = new RegExp(`/${st.tag}${key}(\\d+)x$`) const subs = await Promise.all(Array.from({ length: subsN }, (_, k) => { const got = new Map() return subscribe(srv, k, (body) => { for (const op of body.ops ?? []) { const m = written.exec(op.path) if (m && !got.has(Number(m[1]))) got.set(Number(m[1]), performance.now()) } }).then((sub) => Object.assign(sub, { got })) })) const answeredAt = [] await load(n, Math.min(writers, st.accounts.length), async (i) => { const a = own(st, i) await call(srv, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `${st.tag}${key}${i}x`, record: { $type: 'com.example.note', text: 'fan-out', n: i } }, token: a.token }) answeredAt[i] = performance.now() }) await until(() => subs.every((s) => s.got.size >= n), 10000) for (const s of subs) s.ws.close() const times = [] for (const s of subs) for (let i = 0; i < n; i++) { const at = s.got.get(i) if (at === undefined) throw new Error(`${srv.name}: a subscriber did not get the commit of write ${i} within 10 s`) times.push(Math.max(0, at - answeredAt[i])) } return { times } } // ---- The workload ------------------------------------------------------ /// The parts, in order, each with its size in a measured round and how /// many requests at once; a warm-up runs each at a tenth of its size. /// `st` is the state of a run: its accounts, records and blobs. const PARTS = { createAccount: [40, 4, async (srv, st, n, at) => { const body = srv.accountBody const r = await send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.server.createAccount', { body: body(i, st.tag) })), at, true) r.bodies.forEach((b, i) => { const { did, accessJwt, refreshJwt } = JSON.parse(b) const { handle, password } = body(i, st.tag) st.accounts[i] = { did, handle, password, token: accessJwt, refresh: refreshJwt } }) return r }], createRecord: [1000, 16, async (srv, st, n, at) => { const r = await send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: own(st, i).did, collection: 'com.example.note', record: note(i) }, token: own(st, i).token })), at, true) r.bodies.forEach((b, i) => { st.records[i] = { repo: own(st, i).did, rkey: JSON.parse(b).uri.split('/').pop() } }) return r }], applyWrites: [200, 8, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.applyWrites', { body: { repo: own(st, i).did, writes: Array.from({ length: 10 }, (_, k) => ({ $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.note', value: note(100000 + i * 10 + k) })) }, token: own(st, i).token })), at)], getRecord: [2000, 16, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => getRecord(srv, st, i)), at)], listRecords: [1000, 16, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => listRecords(srv, st, i)), at)], getRepo: [1000, 8, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.sync.getRepo', { params: { did: own(st, i).did } })), at)], // Then, not timed, a record of each account that refers to the first // blob it uploaded, so that sync.getBlob serves it. The rest no record // references: each account holds them for the blob TTL (an hour, past // the round), so they count in its live state. uploadBlob: [1000, 8, async (srv, st, n, at) => { const r = await send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.uploadBlob', { bytes: blob(i), type: 'application/octet-stream', token: own(st, i).token })), at, true) const firsts = st.accounts.map((_, k) => k).filter((k) => k < n) await send(srv, firsts.map((k) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: st.accounts[k].did, collection: 'com.example.note', record: { ...note(300000 + k), blob: JSON.parse(r.bodies[k]).blob } }, token: st.accounts[k].token })), 8) st.blobs = firsts.map((k) => ({ did: st.accounts[k].did, cid: JSON.parse(r.bodies[k]).blob.ref.$link })) for (let i = 0; i < n; i++) if (!firsts.includes(i)) own(st, i).uploaded = (own(st, i).uploaded ?? 0) + 65536 return r }], // One write at a time, as one person posting: no other write to share // its sync with. After the reads, so the repos they read are as before. createRecordSerial: [1000, 1, (srv, st, n, at) => send(srv, Array.from({ length: n }, (_, i) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: own(st, i).did, collection: 'com.example.note', record: note(200000 + i) }, token: own(st, i).token })), at)], // 20 subscribers; the writes one after another, then LOADED_WRITERS at once. firehose: [100, 20, (srv, st, n, subsN) => fanOut(srv, st, n, subsN, 1, 'fh')], firehoseLoaded: [400, 20, (srv, st, n, subsN) => fanOut(srv, st, n, subsN, LOADED_WRITERS, 'fl')], // `subsN` subscribers from cursor 0, as relays catching up: each is sent // every event the server has made, then a write made once all are // connected. Each time is a subscriber's, from asking to that write's // commit; the rate is the events sent to all of them before it, per // second. 4 at once: zds and vlpds each serve 4 backfills at a time. replay: [4, 4, async (srv, st, n, subsN) => { const marker = new RegExp(`/${st.tag}replayx$`) const t = performance.now() const subs = await Promise.all(Array.from({ length: subsN }, (_, k) => subscribe(srv, k, (body, sub) => { if (sub.done === undefined && (body.ops ?? []).some((op) => marker.test(op.path))) { sub.done = performance.now() sub.counted = sub.events } }, 0))) const a = st.accounts[0] await call(srv, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `${st.tag}replayx`, record: { $type: 'com.example.note', text: 'caught up' } }, token: a.token }) const caught = await until(() => subs.every((s) => s.done !== undefined), 60000) for (const s of subs) s.ws.close() if (!caught) throw new Error(`${srv.name}: a subscriber from cursor 0 did not catch up within 60 s`) return { times: subs.map((s) => s.done - t), total: Math.max(...subs.map((s) => s.done)) - t, count: subs.reduce((sum, s) => sum + s.counted, 0) } }], } const getRecord = (srv, st, i) => { const rec = pick(st.records, i) return req(srv, 'com.atproto.repo.getRecord', { params: { repo: rec.repo, collection: 'com.example.note', rkey: rec.rkey } }) } const listRecords = (srv, st, i) => req(srv, 'com.atproto.repo.listRecords', { params: { repo: own(st, i).did, collection: 'com.example.note', limit: '50' } }) const createRecord = (srv, st, i, base) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: own(st, i).did, collection: 'com.example.note', record: note(base + i) }, token: own(st, i).token }) /// The routes a client calls most, besides the parts' (bench/README.md): /// each a request made from the run's state, `n` of them, `at` at once. const ROUTES = { describeServer: [1000, 16, (srv) => req(srv, 'com.atproto.server.describeServer')], oauthMetadata: [1000, 16, (srv) => req(srv, '/.well-known/oauth-authorization-server')], createSession: [200, 8, (srv, st, i) => req(srv, 'com.atproto.server.createSession', { body: { identifier: own(st, i).handle, password: own(st, i).password } })], getSession: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.server.getSession', { token: own(st, i).token })], getPreferences: [1000, 16, (srv, st, i) => req(srv, 'app.bsky.actor.getPreferences', { token: own(st, i).token })], getServiceAuth: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.server.getServiceAuth', { params: { aud: 'did:web:api.bsky.app', lxm: 'app.bsky.feed.getTimeline' }, token: own(st, i).token })], resolveHandle: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.identity.resolveHandle', { params: { handle: own(st, i).handle } })], describeRepo: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.repo.describeRepo', { params: { repo: own(st, i).did } })], getLatestCommit: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.sync.getLatestCommit', { params: { did: own(st, i).did } })], syncGetRecord: [1000, 16, (srv, st, i) => { const rec = pick(st.records, i) return req(srv, 'com.atproto.sync.getRecord', { params: { did: rec.repo, collection: 'com.example.note', rkey: rec.rkey } }) }], listRepos: [1000, 16, (srv) => req(srv, 'com.atproto.sync.listRepos', { params: { limit: '100' } })], getBlob: [1000, 16, (srv, st, i) => req(srv, 'com.atproto.sync.getBlob', { params: pick(st.blobs, i) })], // Each answer's tokens are the account's next ones: from this thread, // never two of one account's at once. Last, as the old tokens may no // longer be good. refreshSession: [1000, 16, 'chain'], } async function route(srv, st, name, n, at) { const [, , make] = ROUTES[name] if (make !== 'chain') return send(srv, Array.from({ length: n }, (_, i) => make(srv, st, i)), at) // Each of `at` callers has accounts of its own, taken in turn, so that // no account's tokens are in two requests at once. const callers = Math.min(at, st.accounts.length) const times = [] let sent = 0 const t = performance.now() await Promise.all(Array.from({ length: callers }, async (_, k) => { const mine = st.accounts.filter((_, i) => i % callers === k) for (let j = 0; sent < n; j++) { sent++ const a = mine[j % mine.length] const r = await call(srv, 'com.atproto.server.refreshSession', { method: 'POST', token: a.refresh }) const { accessJwt, refreshJwt } = r.json() Object.assign(a, { token: accessJwt, refresh: refreshJwt }) times.push(r.ms) } })) return { times, total: performance.now() - t } } /// A day's traffic in small: of each 100 requests, this many of each /// route, from 32 callers at once, with 2 subscribers to the firehose /// as relays would be. const MIX = [['getSession', 15], ['getPreferences', 10], ['getRecord', 25], ['listRecords', 15], ['describeRepo', 5], ['getBlob', 10], ['getLatestCommit', 5], ['createRecord', 10], ['applyWrites', 3], ['uploadBlob', 2]] // So that the least share has 200 requests: its p95 has ten above it. const MIX_N = 10000 const MIX_AT = 32 // The routes in the order of a smooth weighted round robin: each share // spread through the 100 as evenly as it goes. const mixOrder = (() => { const credit = MIX.map(() => 0) return Array.from({ length: 100 }, () => { MIX.forEach(([, w], k) => (credit[k] += w)) const k = credit.indexOf(Math.max(...credit)) credit[k] -= 100 return MIX[k][0] }) })() const mixRequest = (srv, st, name, i) => { if (name === 'getRecord') return getRecord(srv, st, i) if (name === 'listRecords') return listRecords(srv, st, i) if (name === 'createRecord') return createRecord(srv, st, i, 400000) if (name === 'applyWrites') return req(srv, 'com.atproto.repo.applyWrites', { body: { repo: own(st, i).did, writes: Array.from({ length: 10 }, (_, k) => ({ $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.note', value: note(500000 + i * 10 + k) })) }, token: own(st, i).token }) if (name === 'uploadBlob') return req(srv, 'com.atproto.repo.uploadBlob', { bytes: blob(600000 + i), type: 'application/octet-stream', token: own(st, i).token }) return ROUTES[name][2](srv, st, i) } async function mixed(srv, st) { let frames = 0 const subs = await Promise.all([0, 1].map((k) => subscribe(srv, 200 + k, () => frames++))) const names = Array.from({ length: MIX_N }, (_, i) => mixOrder[i % 100]) const r = await send(srv, names.map((name, i) => mixRequest(srv, st, name, i)), MIX_AT) for (const s of subs) s.ws.close() const routes = {} for (const [name] of MIX) { const times = names.flatMap((x, i) => (x === name ? [r.times[i]] : [])) routes[name] = { p50: pct(times, 50), p95: pct(times, 95), n: times.length } } return { rate: (MIX_N / r.total) * 1000, routes, frames } } /// getRecord and createRecord by the number of callers at once: at each, /// what is answered per second and how long it takes, and how many were /// answered with no 2xx (counted, not stopped at: a server that pushes /// back under load says so here). const CURVES = { getRecord: [(c) => Math.max(2000, 40 * c), (srv, st, i) => getRecord(srv, st, i)], createRecord: [(c) => Math.max(500, 4 * c), (srv, st, i) => createRecord(srv, st, i, 700000 + st.curveBase)], } async function curve(srv, st, name, callers) { const [count, make] = CURVES[name] const n = count(callers) const r = await pool.run(Array.from({ length: n }, (_, i) => make(srv, st, i)), callers) portWaits += r.portWaits st.curveBase += n const good = Array.from(r.times).filter((_, i) => r.statuses[i] >= 200 && r.statuses[i] < 300) const [tailLabel, p] = tailOf(good.length) return { n, rate: (good.length / r.total) * 1000, p50: pct(good, 50), tail: pct(good, p), tailLabel, errors: n - good.length, portWaits: r.portWaits, ...(r.firstError && { firstError: `${r.firstError.status}: ${r.firstError.body.slice(0, 160)}` }) } } /// Subscribers coming and going: a cohort of 20, each from an address of /// its own, connects, has one write's commit, and goes; one cohort to /// warm, then COHORTS more. The server's threads and RSS after the first /// and after the last, each once it has had 2 s to let them go; and how /// many subscribers it refused (a server that holds a subscriber's place /// after it went refuses the next cohort's). const COHORTS = 10 async function churn(srv, st, sample) { let refused = 0 const cohort = async (c) => { const marker = `${st.tag}churn${c}x` const subs = (await Promise.allSettled(Array.from({ length: 20 }, (_, k) => subscribe(srv, 100 + k, (body, sub) => { if ((body.ops ?? []).some((op) => op.path.endsWith(`/${marker}`))) sub.got = true })))).flatMap((x) => (x.status === 'fulfilled' ? [x.value] : [])) refused += 20 - subs.length const a = st.accounts[c % st.accounts.length] await call(srv, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: marker, record: { $type: 'com.example.note', text: 'churn', n: c } }, token: a.token }) const all = await until(() => subs.every((s) => s.got), 10000) for (const s of subs) s.ws.close() if (!all) throw new Error(`${srv.name}: a subscriber of cohort ${c} did not get its commit within 10 s`) } const settle = async () => { await new Promise((ok) => setTimeout(ok, 2000)) return sample() } await cohort(0) const first = await settle() for (let c = 1; c <= COHORTS; c++) await cohort(c) const last = await settle() return { first, last, subscribers: 20 * (COHORTS + 1), refused } } // ---- A round ------------------------------------------------------------ /// The accounts' live state (spec/STORE.md, "Bounds"), in bytes: of each, /// its getRepo without `since`, the blobs listBlobs lists (getBlob's /// bytes), and the uploads no record references, which it holds for the /// blob TTL (an hour, past the round) and listBlobs does not list. async function liveState(srv, accounts) { const live = { repo: 0, blobs: 0, uploads: 0 } for (const a of accounts) { live.repo += (await call(srv, 'com.atproto.sync.getRepo', { params: { did: a.did } })).size for (let cursor; ; ) { const page = (await call(srv, 'com.atproto.sync.listBlobs', { params: { did: a.did, limit: '1000', ...(cursor && { cursor }) } })).json() for (const cid of page.cids) live.blobs += (await call(srv, 'com.atproto.sync.getBlob', { params: { did: a.did, cid } })).size cursor = page.cids.length ? page.cursor : undefined if (!cursor) break } live.uploads += a.uploaded ?? 0 } return { ...live, total: live.repo + live.blobs + live.uploads } } /// One round: a fresh PLC directory and server, warmed, then each part, /// each route, the curves, the mix and the churn. A storage round /// (`storage`) starts the server with the short backfill window, is /// warmed with the parts alone, and after the parts measures what it keeps /// on disk, answering only that: its speed is not kept, as the short /// window changes replay. async function round(name, adapter, storage = false) { const backfillMs = storage ? BACKFILL_MS : undefined const plcPort = await freePort() const plc = PlcServer.create({ db: Database.mock(), port: plcPort }) await plc.start() const plcUrl = `http://127.0.0.1:${plcPort}` const dataDir = mkdtempSync(join(tmpdir(), `bench-${name}-`)) const logPath = join(dataDir, 'server.log') const log = openSync(logPath, 'a') const loadStart = loadavg()[0] // The container is made before the start clock. A port picked free can // be taken (an outgoing connection's ephemeral port) before Docker binds // it: then all again on another. let port let aux let s let t0 for (let attempt = 1; ; attempt++) { port = await freePort() aux = null try { aux = await adapter.setup({ port, plcUrl, dataDir, log, backfillMs }) t0 = performance.now() s = await adapter.start({ port, plcUrl, dataDir, log, aux }) break } catch (e) { await aux?.teardown() if (!/address already in use/.test(e.message) || attempt === 3) { await plc.destroy() throw e } writeSync(log, `port ${port} was taken before Docker bound it; again on another\n`) } } let ok = false try { const srv = { name, url: s.url, logPath, accountBody: adapter.accountBody ?? accountBody } const sample = () => processOf(aux.name, adapter.comm) let lastExitCheck = t0 for (;;) { try { if ((await fetch(`${srv.url}/xrpc/com.atproto.server.describeServer`)).ok) break } catch {} if (performance.now() - t0 > 60000) throw new Error(`${name} did not start; its log is ${logPath}`) // A container's state is a `docker inspect` away: once a second. if (performance.now() - lastExitCheck > 1000) { lastExitCheck = performance.now() if (await s.exited()) throw new Error(`${name} exited before it started; its log is ${logPath}`) } await new Promise((ok) => setTimeout(ok, 10)) } const startMs = performance.now() - t0 await new Promise((ok) => setTimeout(ok, 2000)) const idleKiB = (await sample()).rssKiB // Something beside the server with memory of its own (vlpds's store, // tranquil's database). const beside = aux.beside const besideIdleKiB = beside ? await beside.rssKiB() : undefined let peakKiB = idleKiB let besidePeakKiB = besideIdleKiB let sampling = true const sampler = (async () => { while (sampling) { peakKiB = Math.max(peakKiB, (await sample()).rssKiB) await new Promise((ok) => setTimeout(ok, 100)) } })() // Its own loop, as `docker top` is slow. const besideSampler = (async () => { while (beside && sampling) besidePeakKiB = Math.max(besidePeakKiB, await beside.rssKiB()) })() const parts = {} const routes = {} const notServed = {} const warm = { tag: 'warm', accounts: [], records: [], curveBase: 0 } const st = { tag: 'bench', accounts: [], records: [], curveBase: 0 } const summary = (times, total, count) => { const [tailLabel, p] = tailOf(times.length) return { p50: pct(times, 50), p95: pct(times, 95), tail: pct(times, p), tailLabel, ...(total !== undefined && { rate: ((count ?? times.length) / total) * 1000 }), n: times.length } } // The warm-up: each part and route at a tenth of its size, on // accounts of its own. A route answered with no 2xx here is one the // server does not serve. const step = (what) => console.error(`${name}: ${what} (${((performance.now() - t0) / 1000).toFixed(0)} s)`) step('warm-up') for (const [part, [n, at, fn]] of Object.entries(PARTS)) { if (adapter.unsupported[part]) continue await fn(srv, warm, Math.max(1, Math.ceil(n / 10)), part === 'createAccount' ? at : Math.min(at, n)) } for (const [r, [n, at]] of storage ? [] : Object.entries(ROUTES)) { try { await route(srv, warm, r, Math.ceil(n / 10), at) } catch (e) { notServed[r] = e.message.split('\nserver log')[0].replace(`${name}: `, '') } } step('parts') for (const [part, [n, at, fn]] of Object.entries(PARTS)) { if (adapter.unsupported[part]) continue const waited = portWaits const { times, total, count } = await fn(srv, st, n, at) parts[part] = { ...summary(times, total, count), portWaits: portWaits - waited } } if (storage) { sampling = false await sampler await besideSampler // What it keeps on disk: the files on its container's volume; and // apart, what it needs beside it (vlpds's bucket, tranquil's // database). const besideDisk = beside?.diskBytes ? beside : null const sizes = async () => ({ server: await containerBytes(aux.name, adapter.volume, adapter.reader), ...(besideDisk && { beside: await besideDisk.diskBytes() }) }) step('storage under load') const underLoad = await sizes() // At rest: no request for 2 W + 1 s, then a write to each account the // round made, then the least of the sizes sampled for 10 s. step('storage at rest') const accounts = [...warm.accounts, ...st.accounts] await sleep(QUIET_MS) await send(srv, accounts.map((a, i) => req(srv, 'com.atproto.repo.createRecord', { body: { repo: a.did, collection: 'com.example.note', record: note(800000 + i) }, token: a.token })), 8) const atRest = await sizes() for (let k = 1, t = performance.now(); k * REST_SAMPLE_MS <= REST_SAMPLES_MS; k++) { await sleep(t + k * REST_SAMPLE_MS - performance.now()) const x = await sizes() atRest.server = Math.min(atRest.server, x.server) if (besideDisk) atRest.beside = Math.min(atRest.beside, x.beside) } const live = await liveState(srv, accounts) ok = true return { accounts: accounts.length, underLoadBytes: underLoad.server, atRestBytes: atRest.server, liveBytes: live.total, live, ...(besideDisk && { beside: { name: besideDisk.name, underLoadBytes: underLoad.beside, atRestBytes: atRest.beside } }), } } step('routes') for (const [r, [n, at]] of Object.entries(ROUTES)) { if (notServed[r]) continue const waited = portWaits const { times, total } = await route(srv, st, r, n, at) routes[r] = { ...summary(times, total), portWaits: portWaits - waited } } step('by callers') const curves = {} for (const c of Object.keys(CURVES)) { curves[c] = {} for (const callers of CALLERS) curves[c][callers] = await curve(srv, st, c, callers) } const missing = MIX.map(([r]) => r).filter((r) => notServed[r] || (!ROUTES[r] && adapter.unsupported[r])) if (!missing.length) step('the mix') const waited = portWaits const mix = missing.length ? { notServed: `it does not serve ${missing.join(', ')}` } : { ...(await mixed(srv, st)), portWaits: portWaits - waited } step('subscribers coming and going') const subscribers = await churn(srv, st, sample) sampling = false await sampler await besideSampler ok = true return { startMs, idleKiB, peakKiB, ...(beside && { beside: { name: beside.name, idleKiB: besideIdleKiB, peakKiB: besidePeakKiB } }), load: [loadStart, loadavg()[0]], parts, routes, notServed, curves, mix, churn: subscribers } } finally { await s?.stop() await aux?.teardown() await plc.destroy() // A failed round keeps its directory, for its log. if (ok) rmSync(dataDir, { recursive: true, force: true }) } } /// What the client can send: a server in Docker, reached as the servers /// are, that answers every request with the same 2 bytes and does nothing /// else (Node's `net`, in the reference's image). At each concurrency the /// parts use, the best of three. async function ceiling() { const port = await freePort() const name = `bench-ceiling-${process.pid}-${Date.now()}` const code = "const res=Buffer.from('HTTP/1.1 200 OK\\r\\ncontent-type: application/json\\r\\ncontent-length: 2\\r\\n\\r\\n{}');require('net').createServer((s)=>{let b='';s.on('data',(d)=>{b+=d.toString('latin1');let i;while((i=b.indexOf('\\r\\n\\r\\n'))!==-1){b=b.slice(i+4);s.write(res)}});s.on('error',()=>{})}).listen(Number(process.env.PORT),'0.0.0.0')" await shOk('docker', ['run', '-d', '--name', name, '-p', `127.0.0.1:${port}:${port}`, '-e', `PORT=${port}`, '--entrypoint', 'node', pins.implementations.reference.tag, '-e', code]) try { const url = `http://127.0.0.1:${port}/xrpc/_ceiling` for (let t = performance.now(); ; ) { try { if ((await fetch(url)).ok) break } catch {} if (performance.now() - t > 30000) throw new Error('the ceiling\'s responder did not answer within 30 s') await new Promise((ok) => setTimeout(ok, 50)) } const out = {} for (const at of CEILING_AT) { const n = Math.max(5000, 100 * at) let best = 0 for (let k = 0; k < 3; k++) { const r = await pool.run(Array.from({ length: n }, () => ({ method: 'GET', url, headers: {} })), at) if (r.firstError) throw new Error(`the ceiling's responder answered ${r.firstError.status}: ${r.firstError.body}`) best = Math.max(best, (n / r.total) * 1000) } out[at] = best } return out } finally { await sh('docker', ['rm', '-f', name]) } } // ---- The run ------------------------------------------------------------ const impls = arg('--impls', Object.keys(ADAPTERS).join(',')).split(',') // Run only what --setup built from the pins, and an image built from this // repository only as what it is built from is now. const out = arg('--json') for (const name of impls) { if (!ADAPTERS[name]) throw new Error(`no adapter ${name}; there are ${Object.keys(ADAPTERS).join(', ')}`) const pin = pins.implementations[name] if (!pin) throw new Error(`${name} has no entry in bench/impls.json`) const missing = (what) => new Error(`${name}: ${what}; run \`node bench.mjs --setup\` first`) for (const image of [pin.image, (pin.repo || pin.context) && pin.tag, ...(pin.needs ?? []).map((n) => pins.images[n].image)].filter(Boolean)) { if (!(await imageId(image))) throw missing(`no Docker image ${image}`) } if (pin.context && (await imageRevision(pin.tag)) !== revision(pin.inputs)) throw missing(`its image ${pin.tag} was built from ${(await imageRevision(pin.tag)) || 'another tree'}, not what it is built from as it is now (${revision(pin.inputs)})`) } if (!(await imageId(pins.implementations.reference.tag))) throw new Error(`the client's ceiling is measured with the reference's image ${pins.implementations.reference.tag}; run \`node bench.mjs --setup\` first`) // Other work on the machine makes every number its own: refuse to start // on a busy one unless told to. const cores = cpus().length const [load1] = loadavg() if (load1 > cores / 2 && !process.argv.includes('--busy')) throw new Error(`the machine is busy (load average ${load1.toFixed(1)} on ${cores} cores, over ${cores / 2}): its numbers would be the other work's. Wait until it is idle, or give --busy to run anyway and have the load recorded`) const os = type() === 'Darwin' ? `${await sh('sw_vers', ['-productName'])} ${await sh('sw_vers', ['-productVersion'])}` : `${type()} ${release()}` const results = { machine: { cpu: cpus()[0].model, cores, memGiB: Math.round(totalmem() / 2 ** 30), os: `${os} (${process.arch})`, node: process.version, docker: await dockerAbout(), loadAtStart: load1, }, pins: { ...pins, 'Monolith commit': spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(), 'Monolith image built from': revision(pins.implementations['monolith'].inputs) }, rounds: ROUNDS, clients: CLIENTS, workload: { parts: Object.fromEntries(Object.entries(PARTS).map(([p, [n, at]]) => [p, { n, at }])), routes: Object.fromEntries(Object.entries(ROUTES).map(([p, [n, at]]) => [p, { n, at }])), callers: CALLERS, mix: { shares: Object.fromEntries(MIX), n: MIX_N, at: MIX_AT }, cohorts: COHORTS, }, storage: { rounds: STORAGE_ROUNDS, backfillMs: BACKFILL_MS, quietMs: QUIET_MS, sampleMs: REST_SAMPLE_MS, samplesMs: REST_SAMPLES_MS }, impls: {}, } try { console.error(`the client's ceiling, from ${CLIENTS} threads`) results.ceiling = await ceiling() for (const name of impls) { const adapter = ADAPTERS[name] const r = { info: await adapter.info(), unsupported: adapter.unsupported, rounds: [] } for (let k = 0; k < ROUNDS; k++) { console.error(`${name}: round ${k + 1} of ${ROUNDS}`) r.rounds.push(await round(name, adapter)) } r.storage = { ...(adapter.window ? { window: adapter.window } : { noWindow: adapter.noWindow }), rounds: [] } for (let k = 0; k < STORAGE_ROUNDS; k++) { console.error(`${name}: storage round ${k + 1} of ${STORAGE_ROUNDS}`) r.storage.rounds.push(await round(name, adapter, true)) } results.impls[name] = r // What is done so far, kept should a later implementation fail. if (out) writeFileSync(out, `${JSON.stringify(results, null, 2)}\n`) } } finally { await pool.close() } results.machine.loadAtEnd = loadavg()[0] if (out) writeFileSync(out, `${JSON.stringify(results, null, 2)}\n`) process.stdout.write(markdown(results)) } /// The results as Markdown tables: the machine and the client's ceiling; /// a summary, a column per implementation, the reference leftmost, as the /// baseline; then per part, each implementation's best and median round; /// the routes; the curves; the mix; the churn; start time and memory. The /// reference comes first in every table. function markdown(results) { const med = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] const mib = (k) => (k / 1024).toFixed(1) const ms = (x) => (x === undefined ? '' : x.toFixed(1)) const names = Object.keys(results.impls).sort((a, b) => (b === 'reference') - (a === 'reference')) const lines = [] const { parts: workload, routes: routeLoad } = results.workload const partsOf = Object.keys(workload) const ceilingAt = (at) => results.ceiling?.[at] // The best rate of a part's rounds `xs`. One near what the client can send // at all is marked ‡: the client may be part of what holds it. One where // the client waited for local ports is marked ◊. const rateCell = (xs, at) => { const rate = Math.max(...xs.map((x) => x.rate)) return `${rate.toFixed(0)}${ceilingAt(at) && rate >= NEAR_CEILING * ceilingAt(at) ? ' ‡' : ''}${xs.some((x) => x.portWaits) ? ' ◊' : ''}` } const nearNote = `‡: at least ${NEAR_CEILING * 100}% of the client's ceiling at the same concurrency (above): the client may be part of what limits it. ◊: the server closed each connection after its answer, the client opened one a request, and it ran out of local ports and waited for them (each closed connection holds its port for a while): the rate is held to how fast they came back, and the times are from when each request was sent.` const loads = names.flatMap((n) => results.impls[n].rounds.flatMap((r) => r.load ?? [])) lines.push('### Machine and client', '') lines.push(`- ${results.machine.cpu}, ${results.machine.cores} cores, ${results.machine.memGiB} GiB, ${results.machine.os}; Node ${results.machine.node}; ${results.machine.docker}.`) lines.push(`- Load average (1 min): ${results.machine.loadAtStart.toFixed(1)} at the start, ${(results.machine.loadAtEnd ?? 0).toFixed(1)} at the end; ${Math.min(...loads).toFixed(1)} to ${Math.max(...loads).toFixed(1)} at the rounds' starts and ends.`) lines.push(`- The client: ${results.clients} worker threads, each with its own connections, kept alive. Its ceiling, a responder in Docker answering every request with 2 bytes and doing nothing else, requests per second (the best of three):`, '') lines.push(`| at once | ${Object.keys(results.ceiling).join(' | ')} |`, `|---|${Object.keys(results.ceiling).map(() => '--:').join('|')}|`) lines.push(`| per s | ${Object.values(results.ceiling).map((x) => x.toFixed(0)).join(' | ')} |`, '') const lagged = new Set(['firehose', 'firehoseLoaded']) const lagOf = "each time is the firehose's lag, from a write's answer to a subscriber having its commit, 0 when the commit came first. No rate: the writes' would be the write path's, which the write parts measure." const about = { createRecordSerial: (n) => `${n} requests, one after another, as one person posting: no other write to share a sync with.`, uploadBlob: (n, at) => `${n} requests, ${at} at once, of 64 KiB each; then, not timed, a record of each account referring to its first blob, for getBlob.`, firehose: (n, at) => `${n} putRecords one after another, ${at} subscribers; ${lagOf}`, firehoseLoaded: (n, at) => `${n} putRecords, ${LOADED_WRITERS} at once, each to a different account, ${at} subscribers; ${lagOf}`, replay: (n, at) => `${at} subscribers from cursor 0, as relays catching up: each is sent every event the server made in the round, warm-up and all, then a write made once all are connected. Throughput is the events sent to them all up to that write's commit, per second; each time is a subscriber's, from asking to having that commit.`, } const best = (xs, k, lo) => (lo ? Math.min(...xs.map((x) => x[k])) : Math.max(...xs.map((x) => x[k]))) lines.push('### Summary', '', "Each part's requests per second (replay: events sent per second), the mix's, start time and memory, each the best round; for the firehose's live parts, the p50 of their lag, the best round. Detail follows.", '') lines.push(`| | ${names.join(' | ')} |`, `|---|${names.map(() => '--:').join('|')}|`) for (const part of partsOf) { const lag = lagged.has(part) const cell = (name) => { const r = results.impls[name] if (r.unsupported[part]) return 'not served' const xs = r.rounds.map((x) => x.parts[part]) return lag ? best(xs, 'p50', true).toFixed(1) : part === 'replay' ? best(xs, 'rate').toFixed(0) : rateCell(xs, workload[part].at) } lines.push(`| ${lag ? `${part} lag p50 ms` : `${part} per s`} | ${names.map(cell).join(' | ')} |`) } lines.push(`| mix per s | ${names.map((name) => { const xs = results.impls[name].rounds.map((x) => x.mix) return xs[0].notServed ? 'not served' : rateCell(xs, results.workload.mix.at) }).join(' | ')} |`) const each = (f) => names.map((name) => f(results.impls[name].rounds)).join(' | ') lines.push(`| start ms | ${each((rs) => best(rs, 'startMs', true).toFixed(0))} |`) lines.push(`| idle RSS MiB (median) | ${each((rs) => mib(med(rs.map((x) => x.idleKiB))))} |`) lines.push(`| peak RSS MiB | ${each((rs) => mib(best(rs, 'peakKiB', true)))} |`, '', nearNote, '') for (const part of partsOf) { const { n, at } = workload[part] const lag = lagged.has(part) const someRound = names.map((name) => results.impls[name].rounds[0].parts[part]).find(Boolean) const tail = someRound?.tailLabel ?? 'tail' lines.push(`#### ${part}`, '', about[part]?.(n, at) ?? `${n} requests, ${at} at once.`, '') lines.push( lag ? `| impl | p50 ms (best) | p50 ms (median) | ${tail} ms (best) | ${tail} ms (median) |` : `| impl | per s (best) | per s (median) | p50 ms (best) | p50 ms (median) | ${tail} ms (best) | ${tail} ms (median) |`, lag ? '|---|--:|--:|--:|--:|' : '|---|--:|--:|--:|--:|--:|--:|', ) for (const name of names) { const r = results.impls[name] if (r.unsupported[part]) { lines.push(`| ${name} | not served: ${r.unsupported[part]} |${lag ? '|||' : '|||||'}`) continue } const xs = r.rounds.map((x) => x.parts[part]) const g = (k) => xs.map((x) => x[k]) const rate = lag ? '' : `${part === 'replay' ? best(xs, 'rate').toFixed(0) : rateCell(xs, at)} | ${med(g('rate')).toFixed(0)} | ` lines.push(`| ${name} | ${rate}${ms(best(xs, 'p50', true))} | ${ms(med(g('p50')))} | ${ms(best(xs, 'tail', true))} | ${ms(med(g('tail')))} |`) } lines.push('') } lines.push('### Routes', '', 'The routes a client calls most, besides the parts above, each measured alone: requests per second, then the p50 and the tail of their times, each the best round. A route a server answered with no 2xx in the warm-up is not served, with its answer.', '') const routeNames = Object.keys(routeLoad) lines.push(`| route | n, at once | ${names.map((n) => `${n} per s | p50 ms | tail ms`).join(' | ')} |`, `|---|---|${names.map(() => '--:|--:|--:').join('|')}|`) const why = [] for (const r of routeNames) { const { n, at } = routeLoad[r] const cells = names.map((name) => { const rounds = results.impls[name].rounds const notServed = rounds[0].notServed[r] if (notServed) { why.push(`${name}, ${r}: ${notServed}`) return 'not served | | ' } const xs = rounds.map((x) => x.routes[r]) return `${rateCell(xs, at)} | ${ms(best(xs, 'p50', true))} | ${ms(best(xs, 'tail', true))} (${xs[0].tailLabel})` }) lines.push(`| ${r} | ${n}, ${at} | ${cells.join(' | ')} |`) } lines.push('', nearNote, '') if (why.length) lines.push('Not served:', '', ...why.map((w) => `- ${w}`), '') lines.push('### By the number of callers', '', `Each at ${results.workload.callers.join(', ')} callers at once: requests answered with a 2xx per second, and the p50 and tail of their times, the best round; and how many were answered otherwise, in the median round, counted rather than stopped at.`, '') for (const c of ['getRecord', 'createRecord']) { lines.push(`#### ${c}`, '', `| callers | ${names.join(' | ')} |`, `|--:|${names.map(() => '--:').join('|')}|`) for (const callers of results.workload.callers) { const cells = names.map((name) => { const xs = results.impls[name].rounds.map((x) => x.curves[c][callers]) const errors = med(xs.map((x) => x.errors)) return `${rateCell(xs, callers)}/s, p50 ${ms(best(xs, 'p50', true))}, ${xs[0].tailLabel} ${ms(best(xs, 'tail', true))} ms${errors ? `, **${errors} not 2xx** (${xs.find((x) => x.firstError)?.firstError ?? ''})` : ''}` }) lines.push(`| ${callers} | ${cells.join(' | ')} |`) } lines.push('') } const { shares, n: mixN, at: mixAt } = results.workload.mix lines.push('### The mix', '', `${mixN} requests, ${mixAt} at once, of each 100 so many of each route, with 2 subscribers to the firehose: requests per second, then each route's p95 in the mix beside its p95 measured alone (the parts' and the routes' tables), the best round, in ms.`, '') lines.push(`| route | of 100 | ${names.map((n) => `${n} p95 in mix | alone`).join(' | ')} |`, `|---|--:|${names.map(() => '--:|--:').join('|')}|`) lines.push(`| all, per s | | ${names.map((name) => { const xs = results.impls[name].rounds.map((x) => x.mix) return xs[0].notServed ? `not served: ${xs[0].notServed} |` : `${rateCell(xs, mixAt)} |` }).join(' | ')} |`) for (const [r, share] of Object.entries(shares)) { const cells = names.map((name) => { const rounds = results.impls[name].rounds if (rounds[0].mix.notServed) return ' | ' const inMix = rounds.map((x) => x.mix.routes[r]) const alone = rounds.map((x) => x.parts[r] ?? x.routes[r]).filter(Boolean) return `${ms(best(inMix, 'p95', true))} | ${alone.length ? ms(best(alone, 'p95', true)) : ''}` }) lines.push(`| ${r} | ${share} | ${cells.join(' | ')} |`) } lines.push('') lines.push('### Subscribers coming and going', '', `${results.workload.cohorts + 1} cohorts of 20 subscribers, each from an address of its own: a cohort connects, has one write's commit and goes. The server's threads and RSS 2 s after the first cohort went and 2 s after the last, and how many of the subscribers it refused, the median round.`, '') lines.push('| impl | threads after the first | after the last | RSS MiB after the first | after the last | refused, of all |', '|---|--:|--:|--:|--:|--:|') for (const name of names) { const xs = results.impls[name].rounds.map((x) => x.churn) lines.push(`| ${name} | ${med(xs.map((x) => x.first.threads))} | ${med(xs.map((x) => x.last.threads))} | ${mib(med(xs.map((x) => x.first.rssKiB)))} | ${mib(med(xs.map((x) => x.last.rssKiB)))} | ${med(xs.map((x) => x.refused))} of ${xs[0].subscribers} |`) } lines.push('') lines.push('### Start time and memory', '', "Start: from `docker start` of the server's container, made already, to its first answered `describeServer`, on an empty volume. Idle RSS: 2 s after start, before any request. Peak RSS: the most sampled (every ~100 ms) through the warm-up and the whole round. Each the server's own process in its container, from `docker top`.", '') lines.push('| impl | start ms (best) | start ms (median) | idle RSS MiB (median) | peak RSS MiB (best) | peak RSS MiB (median) |', '|---|--:|--:|--:|--:|--:|') for (const name of names) { const rs = results.impls[name].rounds lines.push(`| ${name} | ${best(rs, 'startMs', true).toFixed(0)} | ${med(rs.map((x) => x.startMs)).toFixed(0)} | ${mib(med(rs.map((x) => x.idleKiB)))} | ${mib(best(rs, 'peakKiB', true))} | ${mib(med(rs.map((x) => x.peakKiB)))} |`) } lines.push('') const withBeside = names.filter((name) => results.impls[name].rounds.some((x) => x.beside)) if (withBeside.length) { lines.push('Not in the table above: what a server needs running beside it, its RSS sampled apart (the sum over its container\'s processes, from `docker top`, as fast as that answers; for PostgreSQL, whose processes share its buffers, the sum of their PSS, which counts each shared page once).', '') lines.push('| impl | beside it | idle MiB (median) | peak MiB (best) | peak MiB (median) |', '|---|---|--:|--:|--:|') for (const name of withBeside) { const xs = results.impls[name].rounds.map((x) => x.beside) lines.push(`| ${name} | ${xs[0].name} | ${mib(med(xs.map((x) => x.idleKiB)))} | ${mib(best(xs, 'peakKiB', true))} | ${mib(med(xs.map((x) => x.peakKiB)))} |`) } lines.push('') } const stored = names.filter((name) => results.impls[name].storage?.rounds.length) if (stored.length) { const { backfillMs, quietMs, sampleMs, samplesMs } = results.storage const mb = (b) => (b / 2 ** 20).toFixed(1) lines.push('### Storage', '', `From each implementation's storage round${results.storage.rounds > 1 ? `s (${results.storage.rounds}, the median)` : ''}, apart from the rounds above: a fresh server started with a backfill window W of ${backfillMs / 1000} s where it has a setting for one, the same warm-up of the parts and the parts, whose speed is not kept (a short window changes what replay sends). What it keeps on disk: the sum of the sizes of the files on its container's volume (its log goes to \`docker logs\`, not there). Under load: once the parts end, after replay. At rest: after the parts, no request for 2 W + 1 s (${quietMs / 1000} s), then one createRecord to each account the round made, the warm-up's too, then the size sampled every ${sampleMs} ms for ${samplesMs / 1000} s, the least kept (spec/STORE.md, "Bounds"). Live state: of each of those accounts, its getRepo, the blobs listBlobs lists and the uploads no record references.`, '') lines.push('| impl | backfill window | data under load MiB (median) | data at rest MiB (median) | live state MiB (median) | at rest ÷ live (median) |', '|---|---|--:|--:|--:|--:|') const notes = [] for (const name of stored) { const r = results.impls[name] const ds = r.storage.rounds if (r.storage.noWindow) notes.push(`${name}: ${r.storage.noWindow}.`) lines.push(`| ${name} | ${r.storage.window ? `W = ${backfillMs / 1000} s` : 'its default: no setting'} | ${mb(med(ds.map((d) => d.underLoadBytes)))} | ${mb(med(ds.map((d) => d.atRestBytes)))} | ${mb(med(ds.map((d) => d.liveBytes)))} | ${med(ds.map((d) => d.atRestBytes / d.liveBytes)).toFixed(2)} |`) } lines.push('', ...notes.flatMap((n) => [n, ''])) const withBesideDisk = stored.filter((name) => results.impls[name].storage.rounds.some((d) => d.beside)) if (withBesideDisk.length) { lines.push('Not in the table above: what a server keeps in what runs beside it (vlpds: its bucket, the files on RustFS\'s volume; tranquil: its database, the files on PostgreSQL\'s volume), measured the same way at the same moments; the last column counts the server\'s own and this together.', '') lines.push('| impl | beside it | under load MiB (median) | at rest MiB (median) | both at rest ÷ live (median) |', '|---|---|--:|--:|--:|') for (const name of withBesideDisk) { const ds = results.impls[name].storage.rounds lines.push(`| ${name} | ${ds[0].beside.name} | ${mb(med(ds.map((d) => d.beside.underLoadBytes)))} | ${mb(med(ds.map((d) => d.beside.atRestBytes)))} | ${med(ds.map((d) => (d.atRestBytes + d.beside.atRestBytes) / d.liveBytes)).toFixed(2)} |`) } lines.push('') } } return `${lines.join('\n')}\n` }