~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
654 lines · 31.3 KB Raw History
// The store (spec/STORE.md): a run of its own. A few accounts' records
// rewritten again and again, with blobs and space records, the server killed
// while it reorganizes its data, every read compared with the reference as
// everywhere; then the data directory's size held to the settled bound, once
// across a start of the server and once without, and the starts to their
// time. How the server lays its data out is its own: only the directory's
// size, and its files being made, removed or shortened, are seen.

import { spawn } from 'node:child_process'
import { chmodSync, existsSync, mkdtempSync, openSync, readFileSync, readdirSync, rmSync, statSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { isDeepStrictEqual } from 'node:util'
import { BACKFILL_MS, CHECK_EVERY, CRASH_EVERY, FIREHOSE, ONLY, STEPS, command } from '../lib/args.mjs'
import { below, chance, pick, rand } from '../lib/random.mjs'
import { cand, dataDir, freePort, ref, server, wait, waitReady } from '../lib/processes.mjs'
import { ADMIN_PASSWORD } from '../lib/env.mjs'
import { fail } from '../lib/plc.mjs'
import { count, differ, run } from '../lib/report.mjs'
import { accounts, didOf, exists, nextBatch, noteCommit, noteWrite } from '../lib/state.mjs'
import { both, normalize, shown, xrpcHttp } from '../lib/xrpc.mjs'
import { createAccount, createFirstAccounts, createSession } from './accounts.mjs'
import { APART } from './backup.mjs'
import { blobRecord, getBlob, listBlobs, uploadBlob } from './blobs.mjs'
import { crash, crashes } from './crashes.mjs'
import { getBlocks, getGoneBlocks, syncGetRecord } from './federation.mjs'
import { resubscribe, startFirehose, windowReplay } from './firehose.mjs'
import { checkRepos } from './repository.mjs'
import { refreshSession } from './sessions.mjs'

/// How many accounts the run churns.
const ACCOUNTS = 4
/// The collection the run rewrites, and its keys.
const C = 'com.example.store'
const KEYS = Array.from({ length: 16 }, (_, i) => `k${i}`)
/// The key of a record of the collection kept for takedowns, which no other
/// step writes.
const TD = 'td'
/// The space type, and the collection and keys of each account's space repo.
const SPACE_TYPE = 'com.example.group'
const SC = 'com.example.store.space'
const SKEYS = Array.from({ length: 6 }, (_, i) => `s${i}`)
/// A space whose authority no PLC directory knows.
const AWAY = `at://did:plc:storeunreachableauthorit/space/${SPACE_TYPE}/away`
const SP = 'com.atproto.space.'
const MS = 'com.atproto.memberspace.'
/// The bounds (spec/STORE.md, "Bounds").
const PER_ACCOUNT = 131072
const SERVER_SLACK = 1048576
const SETTLE_MS = 10_000
const START_SLACK = 250
/// How many starts on an empty data directory are timed on each side of a
/// start of the server.
const EMPTY_STARTS = 2
/// How many times more a start over its bound is taken, on the same data.
const RETAKES = 2
/// How many writes a crash makes, waiting to see the server reorganize its
/// data, before it kills the server all the same.
const LEAD_WRITES = 200

/// A record of the collection, of 8 to 24 KB.
function bigRecord() {
  const batch = nextBatch()
  return { $type: C, batch, text: `${batch} `.padEnd(8192 + below(16384), 'lorem ipsum dolor sit amet ') }
}

/// putRecord of a key of the collection, compared.
async function rewrite(acct, key = pick(KEYS), record = bigRecord()) {
  const body = { repo: acct.handle, collection: C, rkey: key, record }
  const [a, b] = await both('putRecord, rewritten', 'com.atproto.repo.putRecord', { body, token: acct })
  noteWrite(acct, C, key, a)
  noteCommit(ref, acct, a)
  noteCommit(cand, acct, b)
}

/// applyWrites of a few keys of the collection, each once: a create of a key
/// not there, else an update or, rarely, a delete.
async function rewriteBatch(acct) {
  const keys = [...KEYS].sort(() => rand() - 0.5).slice(0, 2 + below(3))
  const writes = keys.map((k) => {
    const kind = !exists(acct, C, k) ? 'create' : chance(0.15) ? 'delete' : 'update'
    return { $type: `com.atproto.repo.applyWrites#${kind}`, collection: C, rkey: k, ...(kind === 'delete' ? {} : { value: bigRecord() }) }
  })
  const [a, b] = await both('applyWrites, rewritten', 'com.atproto.repo.applyWrites', { body: { repo: acct.handle, writes }, token: acct })
  if (a.status === 200) {
    writes.forEach((w, i) => (w.value ? noteWrite(acct, C, w.rkey, { status: 200, body: a.body.results[i] }) : acct.records.delete(`${C}/${w.rkey}`)))
  }
  noteCommit(ref, acct, a)
  noteCommit(cand, acct, b)
}

/// deleteRecord of a key of the collection, there or not.
async function deleteKey(acct, key = pick(KEYS)) {
  const [a, b] = await both('deleteRecord, rewritten', 'com.atproto.repo.deleteRecord', { body: { repo: acct.handle, collection: C, rkey: key }, token: acct })
  if (a.status === 200) acct.records.delete(`${C}/${key}`)
  noteCommit(ref, acct, a)
  noteCommit(cand, acct, b)
}

/// getRecord of a key of the collection, compared.
async function readBack(acct) {
  await both('getRecord, rewritten', 'com.atproto.repo.getRecord', { params: { repo: acct.handle, collection: C, rkey: pick(KEYS) } })
}

// Spaces: on the server alone, as the reference serves none, held to what
// the harness wrote (spec/SPACES-REPO.md, spec/SPACES-POLICIES.md).

/// A call to the server with the account's token that must answer 200.
async function onServer(acct, nsid, request, what) {
  const res = await xrpcHttp(cand, nsid, { ...request, token: acct.sides.server.token })
  if (res.status !== 200) await fail(cand, nsid, { ...request, repo: `<did:${acct.n}>` }, what, res.body)
  count(nsid, true)
  return res
}

/// A space repo as the harness wrote it: its records by path, its
/// operation log, its last rev, and the rev that last wrote each record.
const spaceRepo = (uri) => ({ uri, records: new Map(), ops: [], rev: '', written: new Map() })

/// Each account's memberspace, with its members, and its repo there; and
/// its repo in a space whose authority no PLC directory knows, so that
/// each write's notification fails for now and is kept to be sent again
/// (spec/SPACES-SYNC.md, where the server serves it).
async function makeSpace(acct) {
  const res = await onServer(acct, `${SP}createSpace`, { body: { spaceType: SPACE_TYPE, skey: 'store', policy: { $type: `${MS}policy` } } }, 'the space made')
  acct.space = { ...spaceRepo(res.body.space), members: new Map() }
  acct.away = spaceRepo(AWAY)
}

/// A record put or deleted in one of the account's space repos.
async function spaceRecord(acct, s, put = chance(0.75)) {
  const rkey = pick(SKEYS)
  const path = `${SC}/${rkey}`
  const prev = s.records.get(path)?.cid ?? null
  if (put) {
    const record = { $type: SC, batch: nextBatch(), text: 'space '.repeat(50 + below(200)) }
    const res = await onServer(acct, `${SP}putRecord`, { body: { space: s.uri, repo: didOf(cand, acct), collection: SC, rkey, record } }, 'the record put')
    s.records.set(path, { cid: res.body.cid, value: record })
    s.ops.push({ collection: SC, rkey, cid: res.body.cid, prev })
  } else {
    await onServer(acct, `${SP}deleteRecord`, { body: { space: s.uri, repo: didOf(cand, acct), collection: SC, rkey } }, 'the record deleted')
    if (prev) {
      s.records.delete(path)
      s.ops.push({ collection: SC, rkey, cid: null, prev })
    }
  }
}

/// A space record put or deleted, in the account's own space or the one
/// away, or a member put or taken off.
async function spaceWrite(acct) {
  const s = acct.space
  const r = rand()
  if (r < 0.25) await spaceRecord(acct, acct.away)
  else if (r < 0.85) await spaceRecord(acct, s)
  else {
    const did = didOf(cand, pick(accounts))
    if (chance(0.7)) {
      const access = { read: chance(0.7), write: chance(0.5) }
      await onServer(acct, `${MS}putMembers`, { body: { space: s.uri, members: [{ did, ...access }] } }, 'the member put')
      s.members.set(did, access)
    } else {
      await onServer(acct, `${MS}removeMembers`, { body: { space: s.uri, dids: [did] } }, 'the member taken off')
      s.members.delete(did)
    }
  }
}

/// Every page of a list, at limit 1000, and the bytes of their bodies.
async function pages(acct, nsid, params, items) {
  const all = []
  let bytes = 0
  let cursor
  for (;;) {
    const res = await xrpcHttp(cand, nsid, { params: { ...params, limit: 1000, cursor }, token: acct.sides.server.token })
    if (res.status !== 200) await fail(cand, nsid, { ...params, repo: `<did:${acct.n}>` }, 'a page', res.body)
    bytes += Buffer.byteLength(JSON.stringify(res.body))
    all.push(...items(res.body))
    if (!res.body.cursor || !items(res.body).length) return { all, bytes }
    cursor = res.body.cursor
  }
}

/// One of the account's space repos as the harness wrote it: listRecords,
/// listRepoOps from the start and getLatestCommit, its rev never earlier
/// than before. Notes the rev that last wrote each record. Answers the
/// bytes of the live state it makes (spec/STORE.md, "Bounds").
async function checkSpaceRepo(acct, s, why) {
  const did = didOf(cand, acct)
  const request = { space: s.uri, repo: `<did:${acct.n}>`, after: why }
  const recs = await pages(acct, `${SP}listRecords`, { space: s.uri, repo: did }, (b) => b.records ?? [])
  const got = Object.fromEntries(recs.all.map((x) => [`${x.collection}/${x.rkey}`, { cid: x.cid, value: x.value }]))
  const want = Object.fromEntries([...s.records].map(([p, x]) => [p, { cid: x.cid, value: x.value }]))
  if (!isDeepStrictEqual(got, want)) await fail(cand, 'space listRecords', request, want, got)
  // A repo the harness never wrote is none: listRepoOps answers it
  // `RepoNotFound` (spec/SPACES-REPO.md, "Each read").
  if (!s.ops.length) {
    const none = await xrpcHttp(cand, `${SP}listRepoOps`, { params: { space: s.uri, repo: did, excludeValues: true }, token: acct.sides.server.token })
    if (none.status !== 400 || none.body?.error !== 'RepoNotFound') await fail(cand, 'space listRepoOps of a repo never written', request, '400 RepoNotFound', { status: none.status, body: none.body })
    return recs.bytes
  }
  const ops = await pages(acct, `${SP}listRepoOps`, { space: s.uri, repo: did, excludeValues: true }, (b) => b.ops ?? [])
  const shape = (o) => ({ collection: o.collection, rkey: o.rkey, cid: o.cid ?? null, prev: o.prev ?? null })
  if (!isDeepStrictEqual(ops.all.map(shape), s.ops)) await fail(cand, 'space listRepoOps from the start', request, s.ops, ops.all.map(shape))
  const revs = ops.all.map((o) => o.rev)
  if (revs.some((r, i) => i && r < revs[i - 1])) await fail(cand, 'space listRepoOps from the start', request, 'revs in order', revs)
  s.written = new Map()
  for (const o of ops.all) if (o.cid) s.written.set(`${o.collection}/${o.rkey}`, o.rev)
  if (s.ops.length) {
    const latest = await onServer(acct, `${SP}getLatestCommit`, { params: { space: s.uri, repo: did } }, 'the commit')
    const rev = latest.body.commit?.rev
    if (!(rev >= s.rev) || rev !== revs.at(-1)) await fail(cand, 'space getLatestCommit', request, `the last operation's rev, ${revs.at(-1)}, not before ${s.rev}`, rev)
    s.rev = rev
  }
  return recs.bytes + ops.bytes
}

/// The account's space repos, and its space's member list and archive, as
/// the harness wrote them. Answers the bytes of the live state they make,
/// and the archive (spec/SPACES-POLICIES.md, "The member list").
async function checkSpace(acct, why) {
  const s = acct.space
  const bytes = (await checkSpaceRepo(acct, s, why)) + (await checkSpaceRepo(acct, acct.away, why))
  const request = { space: s.uri, after: why }
  const members = await pages(acct, `${MS}listMembers`, { space: s.uri }, (b) => b.members ?? [])
  const wantMembers = [...s.members].map(([d, a]) => ({ did: d, ...a })).sort((x, y) => (x.did < y.did ? -1 : 1))
  if (!isDeepStrictEqual(members.all, wantMembers)) await fail(cand, 'listMembers', request, wantMembers, members.all)
  const meta = await xrpcHttp(cand, `${SP}exportSpaceMeta`, { params: { space: s.uri }, token: acct.sides.server.token, raw: true })
  if (meta.status !== 200) await fail(cand, 'exportSpaceMeta', request, 'the archive', meta.body)
  count('space checked', true)
  return { bytes: bytes + meta.bytes.length, meta: meta.bytes }
}

// What the server serves of pages it may not serve yet: their checks run
// once it does (spec/STORE.md, "What survives").

/// Whether the server serves each: the admin's takedowns (LIFECYCLE.md) and
/// preferences (PROXY.md).
const serves = { takedowns: false, preferences: false }
const served = (res) => res.status !== 501 && res.status !== 404

const ADMIN = `Basic ${Buffer.from(`admin:${ADMIN_PASSWORD}`).toString('base64')}`

/// A call to one side with the admin's auth.
async function admin(side, nsid, { params, body } = {}) {
  const url = new URL(`${side.url}/xrpc/${nsid}`)
  for (const [k, v] of Object.entries(params ?? {})) url.searchParams.append(k, String(v))
  const headers = { authorization: ADMIN, ...(body ? { 'content-type': 'application/json' } : {}) }
  try {
    const res = await fetch(url, { method: body ? 'POST' : 'GET', headers, body: body ? JSON.stringify(body) : undefined })
    const text = await res.text()
    let parsed = null
    try {
      parsed = text ? JSON.parse(text) : null
    } catch {
      parsed = { unparsed: text.slice(0, 200) }
    }
    return { status: res.status, body: parsed }
  } catch (err) {
    return { status: 0, body: { error: `request failed: ${err.message}` } }
  }
}

async function probe() {
  serves.takedowns = served(await admin(cand, 'com.atproto.admin.getSubjectStatus', { params: { did: didOf(cand, accounts[0]) } }))
  serves.preferences = served(await xrpcHttp(cand, 'app.bsky.actor.getPreferences', { token: accounts[0].sides.server.token }))
  console.error(`serves takedowns ${serves.takedowns}, preferences ${serves.preferences}`)
}

/// The admin's call on both sides, compared.
async function bothAdmin(kind, nsid, request) {
  const [a, b] = await Promise.all([ref, cand].map((side) => admin(side, nsid, request(side))))
  const [na, nb] = [normalize(ref, a, nsid), normalize(cand, b, nsid)]
  if (!isDeepStrictEqual(na, nb)) await differ(nsid, shown(request(ref)), na, nb)
  count(kind, a.status < 400)
  return [a, b]
}

const tdUri = (side, acct) => `at://${didOf(side, acct)}/${C}/${TD}`

/// A takedown of the account's record kept for it, or of a blob a record
/// references, applied or lifted, on both sides; then its status read.
async function takedown(acct) {
  if (!serves.takedowns) return
  const ref_ = `store-${nextBatch()}`
  const applied = chance(0.75)
  const m = acct.blobs
  const blobs = m ? [...new Set([...m.refs.values()].flatMap((x) => [...x]))].filter((c) => m.stored.has(c)) : []
  if (!blobs.length || chance(0.5)) {
    const cid = acct.records.get(`${C}/${TD}`)
    const subject = (side) => ({ $type: 'com.atproto.repo.strongRef', uri: tdUri(side, acct), cid })
    const [a] = await bothAdmin('takedown of a record', 'com.atproto.admin.updateSubjectStatus', (side) => ({ body: { subject: subject(side), takedown: { applied, ref: ref_ } } }))
    if (a.status === 200) acct.recordTakedown = applied ? ref_ : null
  } else {
    const cid = pick(blobs)
    const subject = (side) => ({ $type: 'com.atproto.admin.defs#repoBlobRef', did: didOf(side, acct), cid })
    const [a] = await bothAdmin('takedown of a blob', 'com.atproto.admin.updateSubjectStatus', (side) => ({ body: { subject: subject(side), takedown: { applied, ref: ref_ } } }))
    if (a.status === 200) applied ? m.takedowns.set(cid, ref_) : m.takedowns.delete(cid)
  }
  await takedownsAgree(acct)
}

/// Each takedown the harness made, its status read on both sides.
async function takedownsAgree(acct) {
  if (!serves.takedowns) return
  if (acct.recordTakedown !== undefined) await bothAdmin('getSubjectStatus of a record', 'com.atproto.admin.getSubjectStatus', (side) => ({ params: { uri: tdUri(side, acct) } }))
  for (const [cid] of acct.blobs?.takedowns ?? []) await bothAdmin('getSubjectStatus of a blob', 'com.atproto.admin.getSubjectStatus', (side) => ({ params: { did: didOf(side, acct), blob: cid } }))
}

/// putPreferences on both sides, of a type taken as it is (spec/PROXY.md,
/// "putPreferences", "getPreferences"), compared; then getPreferences must
/// answer that list.
async function preferences(acct) {
  if (!serves.preferences) return
  const prefs = Array.from({ length: below(4) }, () => ({ $type: 'app.bsky.example.storePref', n: nextBatch() }))
  const [a] = await both('putPreferences', 'app.bsky.actor.putPreferences', { body: { preferences: prefs }, token: acct })
  if (a.status === 200) acct.storePrefs = prefs
}

/// What survives a restart of the harness's own checks: the space repos,
/// takedowns and preferences.
async function survive(why) {
  for (const acct of accounts) {
    await checkSpace(acct, why)
    await takedownsAgree(acct)
    if (serves.preferences && acct.storePrefs) {
      const [a, b] = await both('getPreferences after a restart', 'app.bsky.actor.getPreferences', { token: acct })
      if (!isDeepStrictEqual(b.body?.preferences, acct.storePrefs)) await fail(cand, 'getPreferences', { repo: `<did:${acct.n}>`, after: why }, acct.storePrefs, b.body?.preferences)
      if (!isDeepStrictEqual(a.body?.preferences, acct.storePrefs)) await fail(ref, 'getPreferences', { repo: `<did:${acct.n}>`, after: why }, acct.storePrefs, a.body?.preferences)
    }
  }
}

// The data directory, whose layout is the server's own: only its size, and
// whether its files change otherwise than by growing, are seen.

/// The server's log, which the harness keeps in the data directory.
const HARNESS_LOG = 'server.log'

/// Every file in the data directory but the harness's log, by path, and
/// its size.
function filesOf(dir = dataDir, within = '') {
  const out = new Map()
  if (!existsSync(dir)) return out
  for (const e of readdirSync(dir, { withFileTypes: true })) {
    const p = join(within, e.name)
    if (e.isDirectory()) for (const [q, n] of filesOf(join(dir, e.name), p)) out.set(q, n)
    else if (p !== HARNESS_LOG) {
      try {
        out.set(p, statSync(join(dir, e.name)).size)
      } catch {}
    }
  }
  return out
}

const sizeOf = (files) => [...files.values()].reduce((n, x) => n + x, 0)

/// Whether the data directory changed from `before` otherwise than by its
/// files growing: a file made, removed or made shorter.
function reorganized(before, after) {
  for (const [p, n] of after) if (!before.has(p) || n < before.get(p)) return true
  for (const p of before.keys()) if (!after.has(p)) return true
  return false
}

/// Rewrites of one account until a write's answer finds the server
/// reorganizing its data directory, then a few milliseconds; answers whether
/// it did.
async function toReorganizing() {
  const acct = pick(accounts)
  for (let i = 0; i < LEAD_WRITES; i++) {
    const before = filesOf()
    await rewrite(acct)
    if (reorganized(before, filesOf())) {
      await wait(below(4))
      return true
    }
  }
  return false
}

/// The live state of the account (spec/STORE.md, "Bounds"): its getRepo,
/// the blobs it holds, and its space repos and space.
async function live(acct) {
  const repo = await xrpcHttp(cand, 'com.atproto.sync.getRepo', { params: { did: didOf(cand, acct) }, raw: true })
  if (repo.status !== 200) await fail(cand, 'getRepo', { did: `<did:${acct.n}>` }, 'the repository', repo.body)
  const blobs = [...(acct.blobs?.stored.values() ?? [])].reduce((n, b) => n + b.size, 0)
  const space = await checkSpace(acct, 'the run')
  return { bytes: repo.bytes.length + blobs + space.bytes, repo: repo.bytes.length, blobs, space: space.bytes }
}

/// Where the servers started on empty data directories write their output,
/// apart from those directories, which must be empty: made at the first.
let emptyLogs = null

/// The server's command and environment started on an empty data directory
/// of its own, on a port of its own, apart from the run as a snapshot's copy
/// is (backup.mjs), and timed as the server's starts are; then killed, and
/// its directory removed. Answers how long it took to answer describeServer.
async function emptyStart() {
  if (!emptyLogs) {
    const logDir = mkdtempSync(join(tmpdir(), 'pds-empty-log-'))
    emptyLogs = { logPath: join(logDir, 'server.log'), errPath: join(logDir, 'server.err') }
    emptyLogs.out = openSync(emptyLogs.logPath, 'a')
    emptyLogs.err = openSync(emptyLogs.errPath, 'a')
  }
  const dir = mkdtempSync(join(tmpdir(), 'pds-empty-'))
  // Open to all, as the run's own was at its first start.
  chmodSync(dir, 0o755)
  const port = await freePort()
  const url = `http://127.0.0.1:${port}`
  const env = { ...process.env, ...server.env, PDS_PORT: String(port), PDS_PUBLIC_URL: url, PDS_DATA_DIR: dir, PDS_CRAWLERS: '' }
  for (const k of APART) delete env[k]
  const p = { name: 'server on an empty data directory', logPath: emptyLogs.logPath, errPath: emptyLogs.errPath, exited: null }
  const proc = spawn(command[0], command.slice(1), { env, stdio: ['ignore', emptyLogs.out, emptyLogs.err] })
  const gone = new Promise((ok) => proc.on('exit', (code, signal) => ok((p.exited = { code, signal }))))
  const ms = await waitReady(p, url)
  proc.kill('SIGKILL')
  await gone
  rmSync(dir, { recursive: true, force: true })
  return ms
}

const median = (xs) => {
  const s = [...xs].sort((x, y) => x - y)
  const m = s.length >> 1
  return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2
}

/// Each start of the server timed against its bound: its time, and the
/// median of the empty starts timed around it; of a start taken again, the
/// take nearest within its bound.
const starts = []

/// One take of a start, the server down: EMPTY_STARTS starts on empty data
/// directories, then the server's own, timed; the server killed again as
/// soon as it answers, EMPTY_STARTS more, and the server started once more
/// on its data. Answers the start's time, the empty starts', and its bound:
/// twice their median, plus START_SLACK.
async function take() {
  const empty = []
  for (let i = 0; i < EMPTY_STARTS; i++) empty.push(await emptyStart())
  server.start()
  const ms = await waitReady(server, cand.url)
  await server.kill()
  for (let i = 0; i < EMPTY_STARTS; i++) empty.push(await emptyStart())
  server.start()
  await waitReady(server, cand.url)
  return { ms, empty, bound: 2 * median(empty) + START_SLACK }
}

/// Kill the server and start it again, led to its reorganizing its data if
/// `lead` is given, its start timed among starts on empty data directories
/// so that the machine's load slows them alike (take). A start over its
/// bound is taken again, the server killed and started on the same data,
/// up to RETAKES times more, and fails only if every take is over its own
/// bound (spec/STORE.md, "Bounds", "Starting"); the space repos, takedowns
/// and preferences must have survived.
async function restart(lead) {
  const takes = []
  await crash({
    lead,
    accountsMade: false,
    async start() {
      takes.push(await take())
      while (takes.at(-1).ms > takes.at(-1).bound && takes.length <= RETAKES) {
        await server.kill()
        takes.push(await take())
      }
      return takes[0].ms
    },
  })
  const best = takes.reduce((a, t) => (t.ms - t.bound < a.ms - a.bound ? t : a))
  starts.push({ ms: best.ms, empty: median(best.empty) })
  const timed = takes.map((t) => `${t.ms} ms, on empty data directories ${t.empty.join(', ')} ms`).join('; taken again, ')
  console.error(`start ${crashes.count}: ${timed}${crashes.reorganizing ? `; ${crashes.reorganizing} of them while reorganizing` : ''}`)
  if (best.ms > best.bound) {
    const want = takes.map((t) => `describeServer within ${t.bound} ms: twice the median of the starts on an empty data directory timed around it, ${t.empty.join(', ')} ms, and ${START_SLACK} ms`)
    await differ('starting again', { after: `${run.step} steps`, crashes: crashes.count, takes: takes.length }, `in one take of ${takes.length}, on the same data, ${want.join('; or ')} (spec/STORE.md, "Bounds")`, takes.map((t) => `${t.ms} ms`).join(', '))
  }
  await survive('a restart')
}

const STEPS_BY_WEIGHT = [
  [rewrite, 30], [rewriteBatch, 8], [deleteKey, 4], [readBack, 3],
  [blobRecord, 6], [uploadBlob, 1], [getBlob, 2], [listBlobs, 2],
  [spaceWrite, 10],
  [getBlocks, 2], [getGoneBlocks, 3], [syncGetRecord, 2],
  [createSession, 1], [refreshSession, 2],
  [windowReplay, 2],
  // Of pages the server may not serve yet: each does nothing until it does.
  [takedown, 2], [preferences, 1],
].filter(([fn]) => !ONLY || ONLY.has(fn.name))

/// The run: what spec/STORE.md's "How it is checked" says.
export async function storeRun() {
  await startFirehose()
  await createFirstAccounts()
  while (accounts.length < ACCOUNTS) await createAccount({ withPassword: true })
  for (const acct of accounts) await makeSpace(acct)
  await probe()
  for (const acct of accounts) for (const key of [...KEYS, TD]) await rewrite(acct, key)
  for (const acct of accounts) for (let i = 0; i < 4; i++) await spaceWrite(acct)
  await checkRepos()
  // Started again, once the accounts hold what they will.
  for (let i = 0; i < 3; i++) await restart()

  const total = STEPS_BY_WEIGHT.reduce((n, [, w]) => n + w, 0)
  for (run.step = 1; run.step <= STEPS; run.step++) {
    let r = below(total)
    for (const [fn, w] of STEPS_BY_WEIGHT) {
      if ((r -= w) < 0) {
        await fn(pick(accounts))
        break
      }
    }
    if (CRASH_EVERY && run.step % CRASH_EVERY === 0) await restart(toReorganizing)
    else if (run.step % CHECK_EVERY === 0) await checkRepos()
    if (run.step % 100 === 0) console.error(`${run.step} steps agree`)
  }
  await checkRepos()

  // Settled across a start: a burst of rewrites, every key of every account
  // eight times, so that the events in the window are many times the live
  // state; a quiet 2 W, the server killed and started again, and then a
  // write to each account with nothing read before it, so that each
  // account's state is first read for that write. First, as the check
  // without a start (below) leaves nothing to settle.
  for (let round = 0; round < 8; round++) for (const acct of accounts) for (const key of KEYS) await rewrite(acct, key)
  await wait(2 * BACKFILL_MS + 1000)
  await server.kill()
  server.start()
  await waitReady(server, cand.url)
  if (FIREHOSE) resubscribe()
  await settled('after a start')
  // Settled: a quiet 2 W, then a write to each account.
  await wait(2 * BACKFILL_MS + 1000)
  const report = await settled('')
  await coldReads()
  // Out of the window now: all but the latest writes' events.
  await windowReplay()
  await restart()
  await checkRepos()
  await windowReplay()
  return { report, starts, cold }
}

/// spec/STORE.md, "Bounds", "Settled": a write to each account, after 2 W
/// with none, and within 10 s the data directory within three times the
/// live state and the constants. Answers the report's lines.
async function settled(why) {
  for (const acct of accounts) await rewrite(acct, 'flush', { $type: C, batch: nextBatch(), text: 'flush' })
  const lives = []
  for (const acct of accounts) lives.push(await live(acct))
  const liveBytes = lives.reduce((n, l) => n + l.bytes, 0)
  const bound = 3 * liveBytes + PER_ACCOUNT * accounts.length + SERVER_SLACK
  const started = Date.now()
  let files = filesOf()
  while (sizeOf(files) > bound && Date.now() - started < SETTLE_MS) {
    await wait(100)
    files = filesOf()
  }
  const size = sizeOf(files)
  const what = `data directory, settled${why ? ` ${why}` : ''}`
  if (size > bound) {
    const largest = Object.fromEntries([...files].sort((x, y) => y[1] - x[1]).slice(0, 10))
    await fail(cand, what, { live: liveBytes, accounts: accounts.length, largest }, `at most ${bound} bytes: three times the live state, ${liveBytes}, ${PER_ACCOUNT} for each account and ${SERVER_SLACK}, within ${SETTLE_MS} ms`, `${size} bytes`)
  }
  count(what, true)
  return [
    ...lives.map((l, i) => `<did:${accounts[i].n}> ${l.bytes} live (repo ${l.repo}, blobs ${l.blobs}, spaces ${l.space})`),
    `${what}: ${size} bytes, bound ${bound}: ${(size / liveBytes).toFixed(2)}x the live state`,
  ]
}

/// The bytes the server's process has read, by any read call, from the page
/// cache or the disk: Linux's /proc/<pid>/io `rchar`. Null where the machine
/// does not count them, or the server is no process of this machine.
function readBytes() {
  const pid = server.proc?.pid
  if (!pid || !existsSync(`/proc/${pid}/io`)) return null
  const m = /^rchar:\s*(\d+)$/m.exec(readFileSync(`/proc/${pid}/io`, 'utf8'))
  return m ? Number(m[1]) : null
}

/// What coldReads measured, for the run's summary: null where not counted.
let cold = null

/// The data directory as it is left: no file changing for a second.
async function settle() {
  for (let last = sizeOf(filesOf()), quiet = 0; quiet < 10; ) {
    await wait(100)
    const now = sizeOf(filesOf())
    quiet = now === last ? quiet + 1 : 0
    last = now
  }
}

/// spec/STORE.md, "Bounds", "The first request to a cold account": a cold
/// account's first use reads at most 6 MiB of its files besides what the
/// request reads, whatever it holds. A new account, with a space of its
/// own and nothing a start takes up again (the run's accounts have
/// writes to a space no directory knows), given 48 records of 250 KB
/// at keys of their own, a table of 12 MB or more; then `2 W` with no write
/// and one write, which folds its log (spec/STORE.md, "Settled"), so that
/// its log begins again; at once 16 rewrites, each of which the log holds
/// twice, as the record and its event: 8 MB, well within `2 W`, which a log
/// folded only once it outgrows its table keeps whole; the data directory
/// let settle, the server shut down and started again; then, nothing else
/// asked of it, that account's first getRecord must read no more than
/// 6 MiB. Its records deleted after, so that the checks after read little.
async function coldReads() {
  if (readBytes() === null) {
    count('a cold account\'s first use, its reads not counted on this machine', false)
    return
  }
  // An account of its own, with nothing the server takes up again as it starts.
  const acct = await createAccount({ withPassword: true })
  await makeSpace(acct)
  const record = () => ({ $type: C, batch: nextBatch(), text: 'e'.repeat(250_000) })
  const keys = Array.from({ length: 48 }, (_, i) => `cold${i}`)
  for (const k of keys) await rewrite(acct, k, record())
  await wait(2 * BACKFILL_MS + 1000)
  await rewrite(acct, keys[0], record())
  for (let n = 0; n < 16; n++) await rewrite(acct, pick(keys), record())
  await settle()
  // The spans of the requests before it exported, as a shutdown may not wait for all.
  await wait(2000)
  await server.stop('SIGTERM', 15_000)
  server.start()
  await waitReady(server, cand.url)
  // Whatever the start reads of its own, done with: no read for half a second.
  for (let last = readBytes(), quiet = 0; quiet < 5; ) {
    await wait(100)
    const now = readBytes()
    quiet = now === last ? quiet + 1 : 0
    last = now
  }
  const key = pick(keys)
  const from = readBytes()
  const got = await xrpcHttp(cand, 'com.atproto.repo.getRecord', { params: { repo: didOf(cand, acct), collection: C, rkey: key } })
  const read = readBytes() - from
  if (got.status !== 200) await fail(cand, 'getRecord, a cold account\'s first request', { did: `<did:${acct.n}>`, key }, 'the record', got.body)
  if (FIREHOSE) resubscribe()
  const most = 6 * 1048576
  if (read > most) {
    const state = await live(acct)
    await differ('a cold account\'s first request', { did: `<did:${acct.n}>`, live: state.bytes }, `reading at most ${most} bytes of its files: a log folded before it holds 4 MiB of writes (spec/STORE.md, "The first request to a cold account")`, `${read} bytes read`)
  }
  cold = { read, most }
  count('a cold account\'s first use, reading at most 6 MiB', true)
  for (const k of keys) await deleteKey(acct, k)
}