chadtmiller.com/monolith
An atproto PDS
0
0
// Moving blobs between the data directory and a bucket (spec/BLOBS.md,
// "Moving them"), a run of its own, --migrate: the server keeps its blobs on
// disk, then is moved into a bucket of the S3 store the harness runs
// (lib/s3.mjs), then back, each move killed at random moments and run again,
// one turned back halfway, one stopped by an object gone. The reference keeps
// its blobs on disk throughout, and after each move every account's blobs
// must read as the reference's; the server's bucket and data directory are
// held to what each place holds. And: the data directory is held by one
// process at a time (spec/COMMANDS.md, "One binary").
import { spawn } from 'node:child_process'
import { isDeepStrictEqual } from 'node:util'
import { command, STEPS_GIVEN } from '../lib/args.mjs'
import { below, chance, pick } from '../lib/random.mjs'
import { cand, freePort, s3, server, waitReady } from '../lib/processes.mjs'
import { count, differ, run } from '../lib/report.mjs'
import { BUCKETS, S3_KEYS, S3_TIMEOUT_MS } from '../lib/s3.mjs'
import { accounts, didOf } from '../lib/state.mjs'
import { createFirstAccounts } from './accounts.mjs'
import { blobRecord, blobsSurvive, compareGetBlob, getBlob, listBlobs, model, referencedCids, uploadBlob } from './blobs.mjs'
import { bucketHolds, dataDirHoldsNoBlobs } from './blobstore.mjs'
const SPEC = 'spec/BLOBS.md, "Moving them"'
/// The blob steps of each phase, unless --steps says.
const PHASE_STEPS = STEPS_GIVEN ?? 60
const STEPS = [[uploadBlob, 4], [blobRecord, 6], [getBlob, 3], [listBlobs, 2]]
/// The bucket's variables, which the command is given whichever way it
/// moves, and the server only while its blobs are in the bucket.
const BUCKET_ENV = () => ({
PDS_BLOBSTORE_S3_BUCKET: BUCKETS.server,
PDS_BLOBSTORE_S3_ENDPOINT: s3.url,
PDS_BLOBSTORE_S3_FORCE_PATH_STYLE: 'true',
PDS_BLOBSTORE_S3_ACCESS_KEY_ID: S3_KEYS.accessKeyId,
PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY: S3_KEYS.secretAccessKey,
PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS: String(S3_TIMEOUT_MS),
PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS: String(S3_TIMEOUT_MS),
})
/// The server's environment for blobs kept `where`.
function serveEnv(where) {
for (const k of Object.keys(BUCKET_ENV())) delete server.env[k]
if (where === 'bucket') Object.assign(server.env, BUCKET_ENV())
return { ...process.env, ...server.env }
}
/// `pds <args>` with `env`, killed after `killAfter` ms if given, counted
/// from when `armed()` first holds (polled every few ms; at once without
/// it); and with `stopOnStart`, shut down (SIGTERM) 5 s after it logs its
/// `start` line, once what a start after a kill finds is durable
/// (spec/STORE.md, "Scale"), so that its data directory is left as a
/// shutdown leaves it: its exit, standard output and standard error.
function pds(args, env, killAfter, armed = () => true, { stopOnStart = false } = {}) {
return new Promise((ok) => {
const p = spawn(command[0], args, { env, stdio: ['ignore', 'pipe', 'pipe'] })
const out = []
const err = []
let stopping = null
p.stdout.on('data', (d) => {
out.push(d)
if (stopOnStart && !stopping && /"msg":"start"/.test(Buffer.concat(out).toString('utf8'))) stopping = setTimeout(() => p.kill('SIGTERM'), 5000)
})
p.stderr.on('data', (d) => err.push(d))
let timer = null
const poll = killAfter === undefined ? null : setInterval(() => {
if (!armed()) return
clearInterval(poll)
timer = setTimeout(() => p.kill('SIGKILL'), killAfter)
}, 2)
p.on('exit', (code, signal) => {
clearInterval(poll)
clearTimeout(timer)
clearTimeout(stopping)
ok({ code, signal, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') })
})
})
}
const migrateEnv = () => ({ ...process.env, ...server.env, ...BUCKET_ENV() })
/// The blobs the harness knows the accounts hold, and their bytes.
function held() {
let n = 0
let bytes = 0
for (const acct of accounts) for (const s of acct.blobs?.stored.values() ?? []) (n += 1), (bytes += s.size)
return { moved: n, bytes }
}
/// A `pds serve` on the data directory, apart from the run's server: it
/// must exit 1, with why on standard error, before it serves. With
/// `servesIf`, a serve that starts, shut down at once, is allowed where
/// `servesIf()` then holds, and answers false.
async function serveRefused(where, why, spec = SPEC, servesIf) {
const port = await freePort()
// Its log apart from the run's server's, which `_logs` is held to: one
// that serves writes lines there.
const env = { ...serveEnv(where), PDS_PORT: String(port), PDS_METRICS_PORT: '', PDS_LOG_DIR: '' }
const r = await pds(['serve'], env, 20_000, undefined, { stopOnStart: true })
serveEnv(where)
const refused = r.code === 1 && r.stderr.trim() && !r.stdout.trim()
const started = /"msg":"start"/.test(r.stdout) && r.code === 0
if (!refused && started && servesIf && (await servesIf())) return false
if (!refused) await differ(`pds serve ${why}`, { blobs: where }, 'exit 1, with why on standard error and nothing on standard output', { code: r.code, signal: r.signal, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 300) }, spec)
count(`pds serve ${why}, refused`, false)
return true
}
/// `pds migrate-blobs --to <to>`, killed now and then, run again until it
/// finishes. The store answers each request 100 ms late, and a kill lands
/// at a random moment within 600 ms of its first request of the run: after
/// the move has marked the data directory (spec/BLOBS.md, "Moving them", 4
/// before 5), while it copies, switches or cleans up; or, rarely, after it
/// marked the move finished and before it exited (7). Such a kill leaves
/// the move finished: the data directory is served from where the blobs
/// went, and the next run moves nothing. With `back`, the first time it is
/// killed the move is turned back: moved where the blobs were, served from
/// there, then moved on.
async function moveTo(to, { kills = 3, back } = {}) {
let killed = 0
let lastKilled = false
for (let attempt = 0; ; attempt++) {
if (attempt > 40) throw new Error(`pds migrate-blobs --to ${to} did not finish in 40 runs`)
const kill = killed < kills && chance(0.8) ? below(600) : undefined
s3.fail({ bucket: BUCKETS.server, mode: { delayMs: 100 } })
const from = s3.requests.length
const r = await pds(['migrate-blobs', '--to', to], migrateEnv(), kill, () => s3.requests.slice(from).some((q) => q.bucket === BUCKETS.server))
s3.clearFailures()
if (r.signal === 'SIGKILL') {
killed += 1
lastKilled = true
// Unfinished, the data directory is not served, whichever way; but a
// kill after the move was marked finished leaves it finished.
const where = killed === 1 ? pick(['bucket', 'disk']) : null
const finished = async () => where === to && (await nothingMoved(to))
if (where && !(await serveRefused(where, 'on a data directory whose blobs are moving', SPEC, finished))) {
return count(`pds migrate-blobs --to ${to}, killed once it had finished`, true)
}
if (back && killed === 1) {
await moveTo(back, { kills: 1 })
await serves(back, `moved back to ${back} after a move to ${to} was killed`)
await server.stop('SIGTERM', 15_000)
lastKilled = false
}
continue
}
if (r.code !== 0) await differ(`pds migrate-blobs --to ${to}`, { attempt: attempt + 1, killed }, 'exit 0', { code: r.code, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 500) }, SPEC)
let line
try {
line = JSON.parse(r.stdout.trim())
} catch {
await differ(`pds migrate-blobs --to ${to}`, { attempt: attempt + 1 }, 'one line of JSON on standard output', r.stdout.slice(0, 300), SPEC)
}
// The run killed before this one had finished, between marking the move
// finished and exiting: this one finds nothing to move.
if (lastKilled && isDeepStrictEqual(line, { moved: 0, bytes: 0 })) return count(`pds migrate-blobs --to ${to}, killed once it had finished`, true)
if (!isDeepStrictEqual(line, held())) await differ(`pds migrate-blobs --to ${to}`, { attempt: attempt + 1, killed }, held(), line, SPEC)
return count(`pds migrate-blobs --to ${to}, killed ${killed} times first`, true)
}
}
/// Whether `pds migrate-blobs --to <where>` moves nothing: the data
/// directory keeps its blobs there, with no move unfinished.
async function nothingMoved(where) {
const r = await pds(['migrate-blobs', '--to', where], migrateEnv(), 60_000)
return r.code === 0 && r.stdout.trim() === '{"moved":0,"bytes":0}'
}
/// `pds migrate-blobs --to <where>` on a data directory that keeps its
/// blobs there already: nothing moved.
async function nothingToMove(where) {
const r = await pds(['migrate-blobs', '--to', where], migrateEnv(), 60_000)
if (r.code !== 0 || r.stdout.trim() !== '{"moved":0,"bytes":0}') await differ(`pds migrate-blobs --to ${where}, its blobs there already`, {}, { code: 0, stdout: '{"moved":0,"bytes":0}' }, { code: r.code, signal: r.signal, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 300) }, SPEC)
count(`pds migrate-blobs --to ${where}, its blobs there already`, true)
}
/// The server started with its blobs `where`, and every account's blobs
/// read as the reference's: every blob a record references, listBlobs, and
/// a record referencing an upload no record had (checks/blobs.mjs).
async function serves(where, when) {
serveEnv(where)
server.start()
await waitReady(server, cand.url)
for (const acct of accounts) for (const cid of referencedCids(model(acct))) await compareGetBlob(`getBlob, ${when}`, (side) => ({ did: didOf(side, acct), cid }))
await blobsSurvive()
if (where === 'bucket') {
await bucketHolds(when)
await dataDirHoldsNoBlobs(when)
} else {
const left = [...s3.objects(BUCKETS.server).keys()].filter((k) => k.startsWith('blocks/'))
if (left.length) await differ(`the bucket, ${when}`, { bucket: BUCKETS.server }, 'no object of the accounts\' blobs', left.slice(0, 10), SPEC)
count(`the bucket holds no blob, ${when}`, true)
}
}
/// Random blob steps, compared as in any run.
async function blobSteps(n) {
const total = STEPS.reduce((s, [, w]) => s + w, 0)
for (let i = 0; i < n; i++, run.step++) {
let r = below(total)
for (const [fn, w] of STEPS) if ((r -= w) < 0) {
await fn(pick(accounts))
break
}
}
}
/// The data directory held by the running server: a second `pds serve` on
/// it, and `pds migrate-blobs`, are refused.
async function heldByServer(where) {
await serveRefused(where, 'beside a running server on its data directory', 'spec/COMMANDS.md, "One binary"')
const r = await pds(['migrate-blobs', '--to', where === 'bucket' ? 'disk' : 'bucket'], migrateEnv(), 20_000)
if (r.code !== 1 || !r.stderr.trim()) await differ('pds migrate-blobs beside a running server on its data directory', {}, 'exit 1, with why on standard error', { code: r.code, signal: r.signal, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 300) }, 'spec/COMMANDS.md, "One binary"')
count('pds migrate-blobs beside a running server, refused', false)
}
/// Arguments that do not parse: usage errors, exit 2, with nothing moved.
async function usage() {
for (const args of [['migrate-blobs'], ['migrate-blobs', '--to', 'cloud']]) {
const r = await pds(args, migrateEnv(), 20_000)
if (r.code !== 2) await differ(`pds ${args.join(' ')}`, {}, 'a usage error: exit 2', { code: r.code, stderr: r.stderr.slice(0, 300) }, SPEC)
count(`pds ${args.join(' ')}, a usage error`, false)
}
}
/// A move into the data directory stopped by a blob whose object is gone:
/// exit 1 naming it, the data directory not served either way; then the
/// object back, and the move run again to its end.
async function stoppedByObjectGone() {
const holders = accounts.filter((a) => a.blobs?.stored.size)
if (!holders.length) return
const acct = pick(holders)
const cid = pick([...acct.blobs.stored.keys()])
const key = `blocks/${didOf(cand, acct)}/${cid}`
const saved = s3.objects(BUCKETS.server).get(key)
if (!saved) return differ('the server\'s bucket before a move', { cid }, `an object ${key}`, 'none', SPEC)
s3.remove(BUCKETS.server, key)
const r = await pds(['migrate-blobs', '--to', 'disk'], migrateEnv(), 60_000)
if (r.code !== 1 || !r.stderr.includes(cid) || !r.stderr.includes(didOf(cand, acct))) await differ('pds migrate-blobs --to disk, a blob\'s object gone', { cid }, 'exit 1, naming the DID and CID on standard error', { code: r.code, signal: r.signal, stderr: r.stderr.slice(0, 500) }, SPEC)
count('pds migrate-blobs, stopped by an object gone', false)
await serveRefused('disk', 'after a move stopped by an object gone')
await serveRefused('bucket', 'after a move stopped by an object gone')
s3.put(BUCKETS.server, key, saved.bytes, saved.contentType)
}
/// The run: blobs on disk; moved into the bucket, the first move turned
/// back halfway; blobs in the bucket; a move out stopped by an object gone,
/// then moved out; blobs on disk again. Each place served between moves.
export async function migrateRun() {
await createFirstAccounts()
await blobSteps(PHASE_STEPS)
await heldByServer('disk')
await server.stop('SIGTERM', 15_000)
await usage()
await nothingToMove('disk')
await moveTo('bucket', { back: 'disk' })
await nothingToMove('bucket')
await serves('bucket', 'moved into the bucket')
await blobSteps(PHASE_STEPS)
await heldByServer('bucket')
await server.stop('SIGTERM', 15_000)
await stoppedByObjectGone()
await moveTo('disk')
await serves('disk', 'moved into the data directory')
await blobSteps(PHASE_STEPS / 2)
}