chadtmiller.com/monolith
An atproto PDS
0
0
// Backups (spec/DURABILITY.md, "Backups"): a copy of the data directory
// taken at one instant while the server writes, which a filesystem or
// volume snapshot takes. SIGSTOP freezes the server between two of its
// system calls, as a snapshot freezes its files; the copy is made then, and
// a second server started on it must hold every write answered before, and
// of the rest a prefix, each whole.
import { spawn } from 'node:child_process'
import { cpSync, mkdtempSync, openSync, readFileSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { command } from '../lib/args.mjs'
import { COLLECTIONS, below, goodRecord, pick } from '../lib/random.mjs'
import { cand, dataDir, freePort, ref, s3, server, wait } from '../lib/processes.mjs'
import { count, differ } from '../lib/report.mjs'
import { accounts, didOf, nextBatch, noteCommit, noteWrite, tokenOf } from '../lib/state.mjs'
import { xrpc, xrpcHttp } from '../lib/xrpc.mjs'
/// The puts in a run, one after another.
const PUTS = 8
/// The variables the copy's server is started without: it exports
/// nothing, serves no metrics, writes no mail, keeps its log in no
/// directory (one server writes to a `PDS_LOG_DIR` at a time,
/// spec/OPERATIONS.md, "Kept"), and its guard keeps it from the harness's
/// services (spec/PROXY.md); with no crawlers too, nothing it does reaches
/// what the run checks.
export const APART = ['PDS_METRICS_PORT', 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_HEADERS', 'PDS_DEV_MAIL_DIR', 'PDS_EMAIL_SMTP_URL', 'PDS_EMAIL_FROM', 'PDS_DISABLE_SSRF_PROTECTION', 'PDS_LOG_DIR']
const broke = (what, request, want, got) => differ(`a server started on a snapshot: ${what}`, request, want, got, 'spec/DURABILITY.md, "Backups"')
/// With blobs in a bucket, each copy's server gets a bucket of its own, as
/// one beside the server it was copied from must (spec/BLOBS.md).
let copies = 0
/// A second server on `dir`, on a port of its own; answers it once it
/// answers, with its `url` and `stop()`.
async function startOn(dir, errPath) {
const port = await freePort()
const env = { ...process.env, ...server.env, PDS_PORT: String(port), PDS_DATA_DIR: dir, PDS_CRAWLERS: '' }
for (const k of APART) delete env[k]
if (env.PDS_BLOBSTORE_S3_BUCKET) {
env.PDS_BLOBSTORE_S3_BUCKET = `copy-${++copies}`
s3.makeBucket(env.PDS_BLOBSTORE_S3_BUCKET)
}
const proc = spawn(command[0], command.slice(1), { env, stdio: ['ignore', 'ignore', openSync(errPath, 'w')] })
let exited = null
proc.on('exit', (code, signal) => (exited = { code, signal }))
const stop = () => new Promise((ok) => (exited ? ok() : (proc.once('exit', ok), proc.kill('SIGKILL'))))
const url = `http://127.0.0.1:${port}`
for (let i = 0; i < 300 && !exited; i++) {
if (await fetch(`${url}/xrpc/com.atproto.server.describeServer`).then((r) => r.ok, () => false)) return { url, stop }
await wait(100)
}
await stop()
return broke('start', { data: 'a copy taken while it was stopped' }, 'it serves', { exited, stderr: readFileSync(errPath, 'utf8').slice(-400) })
}
/// A run of puts to one account; the server stopped once some are
/// answered, its data directory copied, and let go on; then the copy held
/// to what was answered before the stop, and the puts made on the
/// reference too, so that the repositories still agree.
export async function snapshot() {
const acct = pick(accounts)
const did = didOf(cand, acct)
const collection = pick(COLLECTIONS)
const batch = nextBatch()
const puts = Array.from({ length: PUTS }, (_, i) => ({ rkey: `snap${batch}-${i}`, record: { ...goodRecord(collection), batch } }))
const head = (await xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did } })).body?.cid
const dir = mkdtempSync(join(tmpdir(), 'pds-snapshot-'))
const copy = join(dir, 'data')
let answered = 0
let before = null
const stop = () => {
process.kill(server.proc.pid, 'SIGSTOP')
before = answered
cpSync(dataDir, copy, { recursive: true })
process.kill(server.proc.pid, 'SIGCONT')
}
// Stopped once `stopAfter` are answered, with the next on its way.
const stopAfter = 1 + below(PUTS - 2)
let stopped = null
for (const p of puts) {
const res = xrpcHttp(cand, 'com.atproto.repo.putRecord', { body: { repo: did, collection, rkey: p.rkey, record: p.record }, token: tokenOf(cand, acct) })
if (answered === stopAfter) stopped = new Promise((ok) => setImmediate(ok)).then(stop)
p.res = await res
if (p.res.status !== 200) await differ('putRecord, in a run a snapshot is taken during', { rkey: p.rkey }, 'the record put', p.res.body)
answered += 1
}
await stopped
const request = { account: `<did:${acct.n}>`, puts: PUTS, answeredBeforeTheCopy: before }
const other = await startOn(copy, join(dir, 'server.err'))
const fail = async (...args) => {
await other.stop()
await broke(...args)
}
// Each put in the copy whole, as answered; those in it a prefix of the
// run; every one answered before the copy among them.
let held = 0
for (const [i, p] of puts.entries()) {
const got = await xrpcHttp({ url: other.url }, 'com.atproto.repo.getRecord', { params: { repo: did, collection, rkey: p.rkey } })
const there = got.status === 200
if (there && got.body.cid !== p.res.body.cid) await fail('getRecord', { ...request, rkey: p.rkey }, { cid: p.res.body.cid }, got.body)
if (there && held !== i) await fail('getRecord', { ...request, rkey: p.rkey }, `not there: put ${held} is not`, got.body)
if (!there && i < before) await fail('getRecord', { ...request, rkey: p.rkey }, 'the put answered before the copy', got.body)
if (there) held += 1
}
const latest = await xrpcHttp({ url: other.url }, 'com.atproto.sync.getLatestCommit', { params: { did } })
const want = held ? puts[held - 1].res.body.commit.cid : head
if (latest.body?.cid !== want) await fail('getLatestCommit', request, { cid: want, after: `${held} puts` }, latest.body)
const session = await xrpcHttp({ url: other.url }, 'com.atproto.server.getSession', { token: tokenOf(cand, acct) })
if (session.status !== 200) await fail('getSession with a token given before', request, 'the session', session.body)
await other.stop()
rmSync(dir, { recursive: true, force: true })
count('snapshot taken while writing', true)
// The run on the reference too, and noted, as every put is.
for (const p of puts) {
const a = await xrpc(ref, 'com.atproto.repo.putRecord', { body: { repo: acct.handle, collection, rkey: p.rkey, record: p.record }, token: acct })
if (a.status !== 200 || a.body.cid !== p.res.body.cid) await differ('putRecord, in a run a snapshot is taken during', { rkey: p.rkey }, { cid: a.body?.cid }, { cid: p.res.body.cid })
noteWrite(acct, collection, p.rkey, a)
noteCommit(ref, acct, a)
noteCommit(cand, acct, p.res)
}
}