chadtmiller.com/monolith
An atproto PDS
0
0
// The other implementations, as the benchmark (bench.mjs) and the
// compatibility runs (lib/adapters/, compat.mjs) both run them: their
// pins (bench/impls.json), how each is configured, and the Docker it runs
// in. Nothing here reads compare.mjs's arguments, so bench.mjs may import it.
import { execFile } from 'node:child_process'
import { randomBytes } from 'node:crypto'
import { readFileSync, writeSync } from 'node:fs'
import { createServer } from 'node:net'
import { join } from 'node:path'
export const root = new URL('../..', import.meta.url).pathname
/// What each implementation runs, pinned: bench/impls.json. Those built here
/// have a `build`; those run from a release image have an `image` by digest,
/// and the images they need beside them are in `images`, by digest too. Those
/// built from another repository's Dockerfile have a `repo` and `commit`,
/// cloned into bench/vendor/<name> (gitignored), and the `tag` of the image.
export const pins = JSON.parse(readFileSync(join(root, 'bench', 'impls.json'), 'utf8'))
/// A command's standard output, trimmed, or '' if it failed.
export const sh = (cmd, args, cwd) => new Promise((ok) => execFile(cmd, args, { cwd }, (err, out) => ok(err ? '' : out.trim())))
/// Like `sh`, but a failure throws.
export const shOk = (cmd, args) => new Promise((ok, fail) => execFile(cmd, args, (err, out, errOut) => (err ? fail(new Error(`${cmd} ${args.join(' ')}: ${errOut || err.message}`)) : ok(out.trim()))))
/// The ID of the local Docker image `image`, or '' if there is none.
/// Docker Desktop sometimes answers a tag it has as unknown for a moment
/// (while another tag is made), so it is asked a few times.
export async function imageId(image) {
for (let i = 0; i < 5; i++) {
const id = (await sh('docker', ['image', 'inspect', '--format', '{{.Id}}', image])) || (await sh('docker', ['image', 'ls', '--no-trunc', '-q', image])).split('\n')[0]
if (id) return id
await new Promise((ok) => setTimeout(ok, 1000))
}
return ''
}
/// A free port below the ephemeral range (49152 and up on macOS, 32768 and
/// up on Linux), where outgoing connections take theirs: a server in Docker
/// binds its port only when its container starts, and a port the OS hands
/// out as free is often an outgoing connection's by then.
export const dockerPort = () => new Promise((ok) => {
const s = createServer()
s.once('error', () => ok(dockerPort()))
s.listen(20000 + Math.floor(Math.random() * 12000), '127.0.0.1', () => {
const { port } = s.address()
s.close(() => ok(port))
})
})
/// The RSS of a container's processes (`docker top`, ps inside Docker's
/// VM), those named `comm` or all of them, in KiB.
export const containerRssKiB = async (name, comm) => {
const rows = (await sh('docker', ['top', name, '-o', 'pid,rss,comm'])).split('\n').slice(1).map((l) => l.trim().split(/\s+/))
return rows.filter(([, , c]) => !comm || c === comm).reduce((sum, [, rss]) => sum + (Number(rss) || 0), 0)
}
/// The PSS of a container's processes named `comm`, in KiB: each page they
/// share counted once, split among them, where the sum of their RSS counts
/// it in each (PostgreSQL's shared buffers, in every backend that touched
/// them). Read in the container from /proc/<pid>/smaps_rollup as `user`,
/// theirs: root there may not read another user's.
export const containerPssKiB = async (name, user, comm) => {
const out = await sh('docker', ['exec', '-u', user, name, 'sh', '-c', 'for p in /proc/[0-9]*; do [ "$(cat "$p/comm" 2>/dev/null)" = "$1" ] && grep "^Pss:" "$p/smaps_rollup" 2>/dev/null; done; true', 'sh', comm])
return out.split('\n').reduce((sum, l) => sum + (Number(l.split(/\s+/)[1]) || 0), 0)
}
/// A container's log, appended to the file `log` (a descriptor).
export const containerLogs = (name, log) => new Promise((ok) => execFile('docker', ['logs', name], { maxBuffer: 1 << 30 }, (_, out, err) => {
writeSync(log, `---- docker logs ${name}\n${out ?? ''}${err ?? ''}`)
ok()
}))
/// A URL on this machine as a container reaches it.
export const fromDocker = (url) => url.replace('127.0.0.1', 'host.docker.internal').replace('localhost', 'host.docker.internal')
// ---- vlpds ---------------------------------------------------------------
// vlpds as the release image's single-node `tiny` profile
// (deploy/ansible/roles/vlpds/defaults/main.yml at v1.3.0, the profile
// docs/operations/deploy.md runs on one node): its memory limit, one shard,
// a 60 s lease, a 60 s manifest poll, a 4 GiB disk cache, and the tiny
// firehose, backfill and export sizes. Threads stay vlpds's defaults (from
// the cores it sees).
export const VLPDS_MEM_MIB = 2560
export const vlpdsEnv = {
VLPDS_NODE_ID: 'single',
VLPDS_SHARDS: '1',
VLPDS_LEASE_TTL_MS: '60000',
VLPDS_SLATEDB_MANIFEST_POLL: '60s',
VLPDS_CACHE_DIR: '/var/lib/vlpds/cache',
VLPDS_DISK_CACHE_MB: '4096',
VLPDS_MAX_EXPORTS: '4',
VLPDS_FIREHOSE_MAX_BACKFILLS: '4',
VLPDS_LAZY_MST_NODE_CACHE_MB: '64',
VLPDS_FIREHOSE_MERGE_QUEUE_MB: '32',
VLPDS_FIREHOSE_MAX_LAG_MB: '64',
VLPDS_LIVE_RING_MB: '32',
VLPDS_BACKFILL_READAHEAD_MB: '16',
VLPDS_BACKFILL_CACHE_MB: '32',
VLPDS_FIREHOSE_THREADS: '1',
VLPDS_PLC_MODE: 'directory',
VLPDS_PREFIX: 'bench',
// The bench's own: rate limits off, as for the others.
VLPDS_NO_RATE_LIMITS: 'true',
}
// Tiny's firehose ring, and nothing sent off the machine: no relays asked
// to crawl, no AS lookups of firehose subscribers.
export const vlpdsArgs = ['--firehose-ring-mb', '64', '--crawlers=', '--asn-lookup', 'off']
/// A fresh RustFS (the S3 store that passed vlpds-bucket-probe,
/// bench/README.md) on the Docker network `network`, as `s3:9000` there,
/// with the bucket `vlpds` made: answers its container's name and keys.
export async function startRustfs(network, name) {
const { S3Client, CreateBucketCommand } = await import('@aws-sdk/client-s3')
const pin = pins.implementations.vlpds
const port = await dockerPort()
const access = `bench${randomBytes(6).toString('hex')}`
const secret = randomBytes(24).toString('hex')
// A fresh volume; its S3 port on the network as s3:9000, and on loopback
// for the harness to make the bucket.
await shOk('docker', ['run', '-d', '--name', name, '--network', network, '--network-alias', 's3', '-p', `127.0.0.1:${port}:9000`, '-v', '/data', '-e', `RUSTFS_ACCESS_KEY=${access}`, '-e', `RUSTFS_SECRET_KEY=${secret}`, pins.images[pin.needs[0]].image])
const s3 = new S3Client({ endpoint: `http://127.0.0.1:${port}`, region: 'us-east-1', forcePathStyle: true, credentials: { accessKeyId: access, secretAccessKey: secret } })
for (let t = performance.now(); ; ) {
try {
await s3.send(new CreateBucketCommand({ Bucket: 'vlpds' }))
break
} catch (e) {
if (performance.now() - t > 60000) throw new Error(`RustFS ${name}: no bucket within 60 s: ${e.message}`)
await new Promise((ok) => setTimeout(ok, 200))
}
}
s3.destroy()
return { name, access, secret }
}
/// vlpds's environment besides `vlpdsEnv`, for a container on the network of
/// `store` (startRustfs), listening for `port` on loopback, registering DIDs
/// with the directory at `plcUrl` (on this machine) with the secp256k1
/// rotation key `rotationHex`: fresh secrets each time (not --dev-mode).
export function vlpdsOwnEnv({ port, plcUrl, rotationHex, store }) {
const hex = () => randomBytes(32).toString('hex')
return {
VLPDS_PUBLIC_URL: `http://127.0.0.1:${port}`,
VLPDS_PLC_URL: fromDocker(plcUrl),
VLPDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX: rotationHex,
VLPDS_S3_ENDPOINT: 'http://s3:9000',
VLPDS_S3_BUCKET: 'vlpds',
VLPDS_S3_REGION: 'us-east-1',
VLPDS_S3_ACCESS_KEY: store.access,
VLPDS_S3_SECRET_KEY: store.secret,
VLPDS_JWT_SECRET: hex(),
VLPDS_ADMIN_TOKEN: hex(),
VLPDS_INTERNAL_TOKEN: hex(),
VLPDS_KEK: hex(),
}
}
/// `docker create`'s arguments for vlpds as `name` on `network`, its port
/// 2583 on loopback's `port`, with `env` and vlpds's `args`.
export const vlpdsCreateArgs = ({ name, network, port, env, args = vlpdsArgs }) => [
'create', '--name', name, '--network', network, '-p', `127.0.0.1:${port}:2583`, '--add-host', 'host.docker.internal:host-gateway',
'--memory', `${VLPDS_MEM_MIB}m`, '--memory-swap', `${VLPDS_MEM_MIB}m`, '--stop-timeout', '90', '-v', '/var/lib/vlpds',
...Object.entries(env).flatMap(([k, v]) => ['-e', `${k}=${v}`]), pins.implementations.vlpds.image, ...args,
]
// ---- zds -----------------------------------------------------------------
// zds as its deployment runs it (docs/deployment.md, fly.toml): blobs on
// the data volume beside the database; plus the bench's own: no relays
// asked to crawl, and its one limit the workload hits lifted (as rate
// limits are off for the others): full getRepo exports at once, 4 by
// default and 429 past that, raised to the 8 getRepo runs at once. zds has
// no general rate limits, and createAccount sends no mail.
export const zdsEnv = {
ZDS_BLOBSTORE_PATH: '/data/blobs',
ZDS_CRAWLERS: '',
ZDS_MAX_CONCURRENT_REPO_EXPORTS: '8',
}
/// zds's environment besides `zdsEnv`: listening for `port` on loopback,
/// registering DIDs with the directory at `plcUrl` (on this machine) with
/// the secp256k1 rotation key `rotationHex`, a fresh JWT secret and admin
/// token (`admin`, if given).
export const zdsOwnEnv = ({ port, plcUrl, rotationHex, admin = randomBytes(32).toString('hex') }) => ({
ZDS_PUBLIC_URL: `http://127.0.0.1:${port}`,
ZDS_PLC_DIRECTORY: fromDocker(plcUrl),
ZDS_PLC_ROTATION_KEY: rotationHex,
ZDS_JWT_SECRET: randomBytes(32).toString('hex'),
ZDS_ADMIN_TOKEN: admin,
})
/// `docker create`'s arguments for zds as `name`, its port 2583 on
/// loopback's `port`, with `env`, from `image` (by default its pinned tag).
export const zdsCreateArgs = ({ name, port, env, image = pins.implementations.zds.tag }) => [
'create', '--name', name, '-p', `127.0.0.1:${port}:2583`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/data',
...Object.entries(env).flatMap(([k, v]) => ['-e', `${k}=${v}`]), image,
]
// ---- tranquil ------------------------------------------------------------
// tranquil as its container deployment runs it (docs/2_INSTALL_CONTAINERS.md,
// docker-compose.prod.yaml): on PostgreSQL, its default repository backend,
// as the database `pds` of a postgres:18-alpine beside it; blobs as files
// under /var/lib/tranquil-pds/blobs; its in-process cache (ripple, one
// node). Plus the bench's own: rate limits off and no invite codes, as for
// the others; the gate that holds an account's logins and record writes
// until it verifies an email or other channel off (no mail leaves the
// machine to verify, and the others have no such gate); private addresses
// fetchable, so it reaches the bench's PLC directory through
// host.docker.internal; no relays asked to crawl; and its one limit the
// workload hits lifted, as zds's: full getRepo exports at once, 4 by default
// and refused past that, raised to the 8 getRepo runs at once. No mail is
// sent: with no sender address, its email is off.
export const tranquilEnv = {
PDS_HOSTNAME: 'tranquil.test',
INVITE_CODE_REQUIRED: 'false',
DISABLE_RATE_LIMITING: 'true',
DISABLE_ACCOUNT_VERIFICATION_GATE: 'true',
ALLOW_PRIVATE_FETCH: 'true',
// A list of none: an empty value would be taken as unset, its defaults.
CRAWLERS: ',',
MAX_CONCURRENT_REPO_EXPORTS: '8',
}
/// A fresh PostgreSQL for tranquil on the Docker network `network`, as
/// `db:5432` there, its data on a fresh volume, ready for connections over
/// TCP (its entrypoint's first server, which makes the database, listens on
/// none): answers its container's name and the URL tranquil reaches it at.
export async function startPostgres(network, name) {
const password = randomBytes(24).toString('hex')
await shOk('docker', ['run', '-d', '--name', name, '--network', network, '--network-alias', 'db', '-v', '/var/lib/postgresql', '-e', 'POSTGRES_USER=tranquil_pds', '-e', `POSTGRES_PASSWORD=${password}`, '-e', 'POSTGRES_DB=pds', pins.images[pins.implementations.tranquil.needs[0]].image])
for (let t = performance.now(); ; ) {
if ((await sh('docker', ['exec', name, 'sh', '-c', 'pg_isready -q -h 127.0.0.1 -U tranquil_pds -d pds && echo ready'])) === 'ready') break
if (performance.now() - t > 60000) throw new Error(`PostgreSQL ${name}: not ready within 60 s`)
await new Promise((ok) => setTimeout(ok, 200))
}
return { name, url: `postgres://tranquil_pds:${password}@db:5432/pds` }
}
/// tranquil's environment besides `tranquilEnv`: its database at `db`
/// (startPostgres), and fresh secrets, each 48 random bytes as base64, as
/// its docs make them. It signs PLC operations with each account's own key,
/// so it needs no rotation key.
export const tranquilOwnEnv = ({ plcUrl, db }) => ({
DATABASE_URL: db.url,
PLC_DIRECTORY_URL: fromDocker(plcUrl),
JWT_SECRET: randomBytes(48).toString('base64'),
DPOP_SECRET: randomBytes(48).toString('base64'),
MASTER_KEY: randomBytes(48).toString('base64'),
})
/// `docker create`'s arguments for tranquil as `name` on `network`, its
/// port 3000 (the image's SERVER_PORT, on SERVER_HOST [::]) on loopback's
/// `port`, its blobs on a fresh volume, with `env`, from `image` (by default
/// its pinned tag).
export const tranquilCreateArgs = ({ name, network, port, env, image = pins.implementations.tranquil.tag }) => [
'create', '--name', name, '--network', network, '-p', `127.0.0.1:${port}:3000`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/var/lib/tranquil-pds/blobs',
...Object.entries(env).flatMap(([k, v]) => ['-e', `${k}=${v}`]), image,
]
// ---- pds.js --------------------------------------------------------------
// pds.js as its container runs it (docs/deploy-docker.md, its Dockerfile):
// SQLite and blobs on the /data volume, listening on 3000; its optional
// features as they default (spaces off), and no relay told of it.
/// pds.js's environment: its hostname `hostname`, registering its account's
/// DID with the directory at `plcUrl` (on this machine), a fresh JWT secret,
/// and `password`, its one account's, which it requires.
export const pdsjsEnv = ({ hostname, plcUrl, password }) => ({
PDS_HOSTNAME: hostname,
PLC_URL: fromDocker(plcUrl),
JWT_SECRET: randomBytes(32).toString('hex'),
PDS_PASSWORD: password,
})
/// `docker create`'s arguments for pds.js as `name`, its port 3000 on
/// loopback's `port`, with `env`, from `image` (by default its pinned tag).
export const pdsjsCreateArgs = ({ name, port, env, image = pins.implementations['pds.js'].tag }) => [
'create', '--name', name, '-p', `127.0.0.1:${port}:3000`, '--add-host', 'host.docker.internal:host-gateway', '-v', '/data',
...Object.entries(env).flatMap(([k, v]) => ['-e', `${k}=${v}`]), image,
]