~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
126 lines · 6.8 KB Raw History
// Backups (spec/DURABILITY.md, "Backups"): a copy of the data directory
// taken at one instant while the server writes, which a filesystem or
// volume snapshot takes. SIGSTOP freezes the server between two of its
// system calls, as a snapshot freezes its files; the copy is made then, and
// a second server started on it must hold every write answered before, and
// of the rest a prefix, each whole.

import { spawn } from 'node:child_process'
import { cpSync, mkdtempSync, openSync, readFileSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { command } from '../lib/args.mjs'
import { COLLECTIONS, below, goodRecord, pick } from '../lib/random.mjs'
import { cand, dataDir, freePort, ref, s3, server, wait } from '../lib/processes.mjs'
import { count, differ } from '../lib/report.mjs'
import { accounts, didOf, nextBatch, noteCommit, noteWrite, tokenOf } from '../lib/state.mjs'
import { xrpc, xrpcHttp } from '../lib/xrpc.mjs'

/// The puts in a run, one after another.
const PUTS = 8

/// The variables the copy's server is started without: it exports
/// nothing, serves no metrics, writes no mail, keeps its log in no
/// directory (one server writes to a `PDS_LOG_DIR` at a time,
/// spec/OPERATIONS.md, "Kept"), and its guard keeps it from the harness's
/// services (spec/PROXY.md); with no crawlers too, nothing it does reaches
/// what the run checks.
export const APART = ['PDS_METRICS_PORT', 'OTEL_EXPORTER_OTLP_ENDPOINT', 'OTEL_EXPORTER_OTLP_HEADERS', 'PDS_DEV_MAIL_DIR', 'PDS_EMAIL_SMTP_URL', 'PDS_EMAIL_FROM', 'PDS_DISABLE_SSRF_PROTECTION', 'PDS_LOG_DIR']

const broke = (what, request, want, got) => differ(`a server started on a snapshot: ${what}`, request, want, got, 'spec/DURABILITY.md, "Backups"')

/// With blobs in a bucket, each copy's server gets a bucket of its own, as
/// one beside the server it was copied from must (spec/BLOBS.md).
let copies = 0

/// A second server on `dir`, on a port of its own; answers it once it
/// answers, with its `url` and `stop()`.
async function startOn(dir, errPath) {
  const port = await freePort()
  const env = { ...process.env, ...server.env, PDS_PORT: String(port), PDS_DATA_DIR: dir, PDS_CRAWLERS: '' }
  for (const k of APART) delete env[k]
  if (env.PDS_BLOBSTORE_S3_BUCKET) {
    env.PDS_BLOBSTORE_S3_BUCKET = `copy-${++copies}`
    s3.makeBucket(env.PDS_BLOBSTORE_S3_BUCKET)
  }
  const proc = spawn(command[0], command.slice(1), { env, stdio: ['ignore', 'ignore', openSync(errPath, 'w')] })
  let exited = null
  proc.on('exit', (code, signal) => (exited = { code, signal }))
  const stop = () => new Promise((ok) => (exited ? ok() : (proc.once('exit', ok), proc.kill('SIGKILL'))))
  const url = `http://127.0.0.1:${port}`
  for (let i = 0; i < 300 && !exited; i++) {
    if (await fetch(`${url}/xrpc/com.atproto.server.describeServer`).then((r) => r.ok, () => false)) return { url, stop }
    await wait(100)
  }
  await stop()
  return broke('start', { data: 'a copy taken while it was stopped' }, 'it serves', { exited, stderr: readFileSync(errPath, 'utf8').slice(-400) })
}

/// A run of puts to one account; the server stopped once some are
/// answered, its data directory copied, and let go on; then the copy held
/// to what was answered before the stop, and the puts made on the
/// reference too, so that the repositories still agree.
export async function snapshot() {
  const acct = pick(accounts)
  const did = didOf(cand, acct)
  const collection = pick(COLLECTIONS)
  const batch = nextBatch()
  const puts = Array.from({ length: PUTS }, (_, i) => ({ rkey: `snap${batch}-${i}`, record: { ...goodRecord(collection), batch } }))
  const head = (await xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did } })).body?.cid
  const dir = mkdtempSync(join(tmpdir(), 'pds-snapshot-'))
  const copy = join(dir, 'data')
  let answered = 0
  let before = null
  const stop = () => {
    process.kill(server.proc.pid, 'SIGSTOP')
    before = answered
    cpSync(dataDir, copy, { recursive: true })
    process.kill(server.proc.pid, 'SIGCONT')
  }
  // Stopped once `stopAfter` are answered, with the next on its way.
  const stopAfter = 1 + below(PUTS - 2)
  let stopped = null
  for (const p of puts) {
    const res = xrpcHttp(cand, 'com.atproto.repo.putRecord', { body: { repo: did, collection, rkey: p.rkey, record: p.record }, token: tokenOf(cand, acct) })
    if (answered === stopAfter) stopped = new Promise((ok) => setImmediate(ok)).then(stop)
    p.res = await res
    if (p.res.status !== 200) await differ('putRecord, in a run a snapshot is taken during', { rkey: p.rkey }, 'the record put', p.res.body)
    answered += 1
  }
  await stopped

  const request = { account: `<did:${acct.n}>`, puts: PUTS, answeredBeforeTheCopy: before }
  const other = await startOn(copy, join(dir, 'server.err'))
  const fail = async (...args) => {
    await other.stop()
    await broke(...args)
  }
  // Each put in the copy whole, as answered; those in it a prefix of the
  // run; every one answered before the copy among them.
  let held = 0
  for (const [i, p] of puts.entries()) {
    const got = await xrpcHttp({ url: other.url }, 'com.atproto.repo.getRecord', { params: { repo: did, collection, rkey: p.rkey } })
    const there = got.status === 200
    if (there && got.body.cid !== p.res.body.cid) await fail('getRecord', { ...request, rkey: p.rkey }, { cid: p.res.body.cid }, got.body)
    if (there && held !== i) await fail('getRecord', { ...request, rkey: p.rkey }, `not there: put ${held} is not`, got.body)
    if (!there && i < before) await fail('getRecord', { ...request, rkey: p.rkey }, 'the put answered before the copy', got.body)
    if (there) held += 1
  }
  const latest = await xrpcHttp({ url: other.url }, 'com.atproto.sync.getLatestCommit', { params: { did } })
  const want = held ? puts[held - 1].res.body.commit.cid : head
  if (latest.body?.cid !== want) await fail('getLatestCommit', request, { cid: want, after: `${held} puts` }, latest.body)
  const session = await xrpcHttp({ url: other.url }, 'com.atproto.server.getSession', { token: tokenOf(cand, acct) })
  if (session.status !== 200) await fail('getSession with a token given before', request, 'the session', session.body)
  await other.stop()
  rmSync(dir, { recursive: true, force: true })
  count('snapshot taken while writing', true)

  // The run on the reference too, and noted, as every put is.
  for (const p of puts) {
    const a = await xrpc(ref, 'com.atproto.repo.putRecord', { body: { repo: acct.handle, collection, rkey: p.rkey, record: p.record }, token: acct })
    if (a.status !== 200 || a.body.cid !== p.res.body.cid) await differ('putRecord, in a run a snapshot is taken during', { rkey: p.rkey }, { cid: a.body?.cid }, { cid: p.res.body.cid })
    noteWrite(acct, collection, p.rkey, a)
    noteCommit(ref, acct, a)
    noteCommit(cand, acct, p.res)
  }
}