~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
568 lines · 29.9 KB Raw History
// The scale run (spec/STORE.md, "Scale"): the server alone, as the reference
// says nothing of its memory, with `PDS_CACHE_BYTES` small. Its resident
// memory and its starts are measured with few accounts, many, and fifty
// times as many again, with a small repository and a large one, and while a
// large getRepo and a firehose replay are read; each held to its rule. What
// it answers is held to what the harness wrote.

import { execFileSync } from 'node:child_process'
import { isDeepStrictEqual } from 'node:util'
import { readCarWithRoot } from '@atproto/repo'
import { cborDecode, cborDecodeMulti } from '@atproto/common'
import { CACHE_BYTES } from '../lib/args.mjs'
import { cand, metricsUrl, server, wait, waitReady } from '../lib/processes.mjs'
import { fail } from '../lib/plc.mjs'
import { count } from '../lib/report.mjs'
import { xrpcHttp } from '../lib/xrpc.mjs'

/// The accounts made: few, then many, then most, of which some have a
/// password.
const FEW = 100
const MANY = 2000
const MOST = 100000
const WITH_PASSWORDS = 10000
/// How many times over every session with a password is refreshed.
const REFRESHES = 4
/// Of most accounts, every one in this many is looked for after a start.
const ONE_IN = 50
/// How many repositories `listRepos` is asked for at once, and how many times.
const LIST_LIMIT = 100
const LIST_PAGES = 3
/// The large repository: batches of records of about 120 KB, each under
/// the 1000000-byte body (spec/LIMITS.md, "Request size"), to about 64 MiB.
const BIG_RECORD = 120_000
const BIG_BATCH = 7
const BIG_BATCHES = 80
/// What one account holds besides its repository: blobs, each referenced
/// by a record of its own, and a space repo's records, of about 1 KB.
const BLOBS = 4000
const BLOB_BYTES = 512
const SPACE_RECORDS = 40000
/// The bytes of each space record's text.
const SPACE_RECORD_BYTES = 1000
const SPACE_BATCH = 200
const SPACE_TYPE = 'com.example.group'
const SPACE_COLLECTION = 'com.example.scale.space'
/// The rules (spec/STORE.md, "Bounds", "Scale"): what requests under way
/// may take, what an account costs idle, what a just-started server may hold
/// besides, the start's slack, and a cold account's first request's.
const REQUEST_SLACK = 32 * 1048576
const PER_ACCOUNT = 4096
const IDLE_SLACK = 8 * 1048576
const START_SLACK = 250
const COLD_MS = 1000
/// How many requests go at once when making or touching accounts; and when
/// making most of them, which is most of the run's time.
const AT_ONCE = 8
/// How many accounts are first used after a shutdown and a start.
const FIRST_USES = 200
const MOST_AT_ONCE = 32

/// The server's resident memory, in bytes, as `ps` reports it.
function rss() {
  const kib = Number(execFileSync('ps', ['-o', 'rss=', '-p', String(server.proc.pid)]).toString().trim())
  return kib * 1024
}

/// Run `fn`, sampling the server's resident memory every 20 ms; answers its
/// result and the most memory seen.
async function sampled(fn) {
  let peak = rss()
  const timer = setInterval(() => {
    try {
      peak = Math.max(peak, rss())
    } catch {}
  }, 20)
  try {
    const result = await fn()
    peak = Math.max(peak, rss())
    return { result, peak }
  } finally {
    clearInterval(timer)
  }
}

const MiB = (n) => `${(n / 1048576).toFixed(1)} MiB`

/// The memory bounds, given the base: at every moment, and just started.
const state = { r0: 0, t0: 0, accounts: [] }
const bound = () => state.r0 + REQUEST_SLACK + CACHE_BYTES + PER_ACCOUNT * state.accounts.length
const idleBound = () => state.r0 + IDLE_SLACK + PER_ACCOUNT * state.accounts.length

async function holdPeak(what, peak) {
  if (peak > bound()) {
    await fail(cand, `resident memory, ${what}`, { accounts: state.accounts.length, cache: MiB(CACHE_BYTES), base: MiB(state.r0) }, `at most ${MiB(bound())}: the base, ${MiB(REQUEST_SLACK)} for requests, the cache and 4 KiB an account`, MiB(peak))
  }
  count(`memory: ${what}`, true)
  console.error(`  ${what}: peak ${MiB(peak)}, bound ${MiB(bound())}`)
}

/// Stop the server, killed or with the signal given, and start it again;
/// its start held to the rule, and its memory just started to the idle
/// bound. Answers the start's time.
async function restart(what, signal = 'SIGKILL') {
  await server.stop(signal)
  server.start()
  const ms = await waitReady(server, cand.url)
  await wait(300)
  const r = rss()
  console.error(`  started ${what}: ${ms} ms, ${MiB(r)}`)
  if (ms > 2 * state.t0 + START_SLACK) {
    await fail(cand, `start, ${what}`, { accounts: state.accounts.length }, `describeServer within ${2 * state.t0 + START_SLACK} ms: twice the start on an empty data directory, ${state.t0} ms, and ${START_SLACK} ms`, `${ms} ms`)
  }
  if (r > idleBound()) {
    await fail(cand, `resident memory just started, ${what}`, { accounts: state.accounts.length, base: MiB(state.r0) }, `at most ${MiB(idleBound())}: the empty start's, 8 MiB and 4 KiB an account`, MiB(r))
  }
  count('start', true)
  return ms
}

/// Run `fn` on each of `items`, `atOnce` at a time.
async function each(items, fn, atOnce = AT_ONCE) {
  let next = 0
  await Promise.all(Array.from({ length: atOnce }, async () => {
    while (next < items.length) await fn(items[next++])
  }))
}

/// Make accounts on the server alone until there are `n`, `atOnce` at a
/// time: without passwords, which costs no password hashing, or each with
/// one.
async function makeAccounts(n, { passwords = false, atOnce = AT_ONCE } = {}) {
  const fresh = Array.from({ length: n - state.accounts.length }, (_, i) => state.accounts.length + i)
  await each(fresh, async (i) => {
    const handle = `scale${i}.test`
    const email = `scale${i}@example.com`
    const password = passwords ? `scale-password-${i}` : undefined
    const res = await xrpcHttp(cand, 'com.atproto.server.createAccount', { body: { handle, email, password } })
    if (res.status !== 200) await fail(cand, 'createAccount', { handle, email, password: password && '<password>' }, 'the account', res.body)
    state.accounts[i] = { did: res.body.did, handle, email, password, token: res.body.accessJwt, refresh: res.body.refreshJwt }
  }, atOnce)
  count('accounts made', true)
}

/// The session of every account with a password refreshed, `REFRESHES`
/// times over, each with the token its last refresh answered: what the
/// server keeps of its sessions written many times over.
async function refreshAll() {
  const signedIn = state.accounts.filter((a) => a.password)
  for (let round = 0; round < REFRESHES; round++) {
    await each(signedIn, async (a) => {
      const res = await xrpcHttp(cand, 'com.atproto.server.refreshSession', { token: a.refresh, method: 'POST' })
      if (res.status !== 200 || typeof res.body?.refreshJwt !== 'string' || typeof res.body?.accessJwt !== 'string') await fail(cand, 'refreshSession', { did: a.did, round }, 'the session, refreshed', res.body)
      a.token = res.body.accessJwt
      a.refresh = res.body.refreshJwt
    })
  }
  count('sessions refreshed', true)
}

/// The ways an account is found, each a request and whether its answer is
/// the account's: by its DID; by its handle; by the session it had before
/// the start; and, with a password, at a login with its email and with its
/// handle.
function ways(a) {
  const login = (identifier) => () => xrpcHttp(cand, 'com.atproto.server.createSession', { body: { identifier, password: a.password } })
  return [
    ['getLatestCommit', () => xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did: a.did } }), (r) => r.status === 200],
    ['resolveHandle', () => xrpcHttp(cand, 'com.atproto.identity.resolveHandle', { params: { handle: a.handle } }), (r) => r.status === 200 && r.body?.did === a.did],
    ['getSession, with a token from before the start', () => xrpcHttp(cand, 'com.atproto.server.getSession', { token: a.token }), (r) => r.status === 200 && r.body?.did === a.did && r.body?.handle === a.handle],
    ...(a.password ? [['createSession, with its email', login(a.email), (r) => r.status === 200 && r.body?.did === a.did], ['createSession, with its handle', login(a.handle), (r) => r.status === 200 && r.body?.did === a.did]] : []),
  ]
}

/// One account in `ONE_IN`, found after a start in every way an account is
/// found (`ways`), each account by another way first, so that each way
/// finds accounts nothing has asked for since the start; the first within
/// a second more than the same request again. Then the handles and the
/// emails of accounts not asked for yet are taken: no account can be made
/// with one. And `listRepos`' first pages are accounts the harness made,
/// none twice.
async function findSome(what) {
  const some = state.accounts.filter((_, i) => i % ONE_IN === 0)
  const slow = []
  await each(some.map((a, k) => [a, k]), async ([a, k]) => {
    const request = { did: a.did, handle: a.handle, after: what }
    const all = ways(a)
    const turned = [...all.slice(k % all.length), ...all.slice(0, k % all.length)]
    const t = Date.now()
    for (const [name, ask, found] of turned) {
      const res = await ask()
      if (!found(res)) await fail(cand, name, { ...request, first: turned[0][0] }, { did: a.did }, res.body)
      if (name === turned[0][0]) {
        const cold = Date.now() - t
        const u = Date.now()
        await ask()
        if (cold - (Date.now() - u) > COLD_MS) slow.push({ did: a.did, first: name, cold })
      }
    }
  })
  if (slow.length) await fail(cand, `a cold account's first request, ${what}`, { accounts: state.accounts.length }, `within ${COLD_MS} ms more than the next`, slow.slice(0, 5))
  count('accounts found', true)

  // An account's email is one it signs in with: of those with a password.
  // Four accounts nothing has asked for since the start, two for each.
  const fresh = state.accounts.filter((a, i) => a.password && i % ONE_IN === 1)
  const other = { handle: 'scale-other.test', email: 'scale-other@example.com', password: 'scale-other-password' }
  for (const body of [{ ...other, handle: fresh[0].handle }, { ...other, email: fresh[1].email }, { ...other, handle: fresh.at(-1).handle }, { ...other, email: fresh.at(-2).email }]) {
    const res = await xrpcHttp(cand, 'com.atproto.server.createAccount', { body })
    if (res.status !== 400) await fail(cand, "createAccount, its handle or its email an account's", { ...body, password: '<password>', after: what }, 'refused, 400', { status: res.status, body: res.body })
  }
  count('taken handles and emails refused', true)

  const made = new Set(state.accounts.map((a) => a.did))
  const listed = new Set()
  let cursor
  for (let page = 0; page < LIST_PAGES; page++) {
    const res = await xrpcHttp(cand, 'com.atproto.sync.listRepos', { params: { limit: LIST_LIMIT, cursor } })
    const dids = (res.body?.repos ?? []).map((r) => r.did)
    if (res.status !== 200 || dids.length !== LIST_LIMIT || typeof res.body.cursor !== 'string' || dids.some((did) => !made.has(did) || listed.has(did))) {
      await fail(cand, 'listRepos', { limit: LIST_LIMIT, page, after: what }, `${LIST_LIMIT} accounts the harness made, none listed before, and a cursor`, res.body)
    }
    for (const did of dids) listed.add(did)
    cursor = res.body.cursor
  }
  count('listRepos', true)
}

/// Each account's latest commit, all of them touched; and for some, the
/// first request after a start against the second.
async function touchAll(what) {
  const slow = []
  await each(state.accounts, async (a) => {
    const t = Date.now()
    const first = await xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did: a.did } })
    const cold = Date.now() - t
    if (first.status !== 200) await fail(cand, 'getLatestCommit', { did: a.did }, 'the latest commit', first.body)
    if (cold > COLD_MS) {
      const u = Date.now()
      await xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did: a.did } })
      if (cold - (Date.now() - u) > COLD_MS) slow.push({ did: a.did, cold })
    }
  })
  if (slow.length) await fail(cand, `a cold account's first request, ${what}`, { accounts: state.accounts.length }, `within ${COLD_MS} ms more than the next`, slow.slice(0, 5))
  count('cold accounts touched', true)
}

/// The large repository: records of about 120 KB on the first account, by
/// `applyWrites`, each batch under the largest commit (spec/LIMITS.md,
/// "Commits").
async function writeLarge(big) {
  const values = new Map()
  for (let b = 0; b < BIG_BATCHES; b++) {
    const writes = Array.from({ length: BIG_BATCH }, (_, i) => {
      const rkey = `big${b * BIG_BATCH + i}`
      const value = { $type: 'com.example.scale', n: b * BIG_BATCH + i, text: `${rkey} `.padEnd(BIG_RECORD, 'lorem ipsum dolor sit amet ') }
      values.set(rkey, value)
      return { $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.scale', rkey, value }
    })
    const res = await xrpcHttp(cand, 'com.atproto.repo.applyWrites', { body: { repo: big.did, writes }, token: big.token })
    if (res.status !== 200) await fail(cand, 'applyWrites of the large repository', { batch: b }, 'the batch written', res.body)
  }
  count('large repository written', true)
  return values
}

/// Much besides a repository, on one account: `BLOBS` blobs uploaded and
/// each referenced by a record, and a space made with `SPACE_RECORDS`
/// records in the account's repo there. Answers the space's URI.
async function writeMuchBesides(rich) {
  const call = async (nsid, body, what) => {
    const res = await xrpcHttp(cand, nsid, { body, token: rich.token })
    if (res.status !== 200) await fail(cand, nsid, { did: rich.did, what }, what, res.body)
    return res.body
  }
  const refs = new Array(BLOBS)
  await each(Array.from(refs, (_, i) => i), async (i) => {
    const bytes = Buffer.alloc(BLOB_BYTES, `blob ${i} `)
    const res = await fetch(`${cand.url}/xrpc/com.atproto.repo.uploadBlob`, { method: 'POST', headers: { authorization: `Bearer ${rich.token}`, 'content-type': 'application/octet-stream' }, body: bytes })
    const body = await res.json()
    if (res.status !== 200) await fail(cand, 'uploadBlob', { did: rich.did, n: i }, 'the blob', body)
    refs[i] = body.blob
  })
  for (let i = 0; i < BLOBS; i += 100) {
    const writes = refs.slice(i, i + 100).map((blob, j) => ({ $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.scale.blob', rkey: `blob${i + j}`, value: { $type: 'com.example.scale.blob', blob } }))
    await call('com.atproto.repo.applyWrites', { repo: rich.did, writes }, 'records referencing the blobs')
  }
  count('blobs held', true)
  const space = (await call('com.atproto.space.createSpace', { spaceType: SPACE_TYPE, skey: 'scale', policy: { $type: 'com.atproto.memberspace.policy' } }, 'the space made')).space
  for (let i = 0; i < SPACE_RECORDS; i += SPACE_BATCH) {
    const writes = Array.from({ length: SPACE_BATCH }, (_, j) => ({ $type: 'com.atproto.space.applyWrites#create', collection: SPACE_COLLECTION, rkey: `r${i + j}`, value: { $type: SPACE_COLLECTION, n: i + j, text: `${i + j} `.padEnd(SPACE_RECORD_BYTES, 'space ') } }))
    await call('com.atproto.space.applyWrites', { space, repo: rich.did, writes }, 'space records')
  }
  count('space records written', true)
  return space
}

/// What the account holds besides its repository, read: a space record,
/// two pages of its space repo and of its blobs, and its space repo whole.
/// Answers how long the first of them took, how long the same request
/// again, and the bytes of the space repo whole.
async function readMuchBesides(rich, space) {
  const record = () => xrpcHttp(cand, 'com.atproto.space.getRecord', { params: { space, repo: rich.did, collection: SPACE_COLLECTION, rkey: `r${SPACE_RECORDS - 1}` }, token: rich.token })
  const t = Date.now()
  const first = await record()
  const cold = Date.now() - t
  if (first.status !== 200 || first.body?.value?.n !== SPACE_RECORDS - 1) await fail(cand, 'getRecord of the space repo', { did: rich.did }, { n: SPACE_RECORDS - 1 }, first.body)
  const u = Date.now()
  await record()
  const warm = Date.now() - u
  for (const [nsid, params, items] of [['com.atproto.space.listRecords', { space, repo: rich.did, collection: SPACE_COLLECTION, limit: 100 }, 'records'], ['com.atproto.sync.listBlobs', { did: rich.did, limit: 500 }, 'cids']]) {
    let cursor
    for (let page = 0; page < 2; page++) {
      const res = await xrpcHttp(cand, nsid, { params: { ...params, cursor }, token: rich.token })
      if (res.status !== 200 || res.body?.[items]?.length !== params.limit || typeof res.body.cursor !== 'string') await fail(cand, nsid, { did: rich.did, page }, `${params.limit} ${items} and a cursor`, res.body)
      cursor = res.body.cursor
    }
  }
  // The space repo whole, as it is sent: every record's bytes are in it.
  const url = new URL(`${cand.url}/xrpc/com.atproto.space.getRepo`)
  url.searchParams.set('space', space)
  url.searchParams.set('repo', rich.did)
  const res = await fetch(url, { headers: { authorization: `Bearer ${rich.token}` } })
  let whole = 0
  if (res.ok) for await (const part of res.body) whole += part.length
  if (!res.ok || whole < SPACE_RECORDS * SPACE_RECORD_BYTES) await fail(cand, 'getRepo of the space repo', { did: rich.did, records: SPACE_RECORDS }, `at least ${SPACE_RECORDS * SPACE_RECORD_BYTES} bytes`, { status: res.status, bytes: whole })
  count('much besides read', true)
  return { cold, warm, whole }
}

/// A body read as it comes, counting its bytes, and kept.
async function streamed(url) {
  const res = await fetch(url)
  if (!res.ok) return { status: res.status, bytes: null }
  const parts = []
  for await (const part of res.body) parts.push(part)
  return { status: res.status, bytes: Buffer.concat(parts) }
}

/// Every event from the start, read as fast as they come, until none comes
/// for a second.
function replay() {
  return new Promise((done) => {
    const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos?cursor=0`)
    ws.binaryType = 'arraybuffer'
    const seqs = []
    let bytes = 0
    let quiet
    const settle = () => {
      clearTimeout(quiet)
      quiet = setTimeout(() => (ws.close(), done({ seqs, bytes })), 1000)
    }
    ws.onmessage = (m) => {
      bytes += m.data.byteLength
      const [, body] = cborDecodeMulti(new Uint8Array(m.data))
      if (body.seq !== undefined) seqs.push(body.seq)
      settle()
    }
    ws.onerror = () => {}
    settle()
  })
}

/// How long the firehose's first message is waited for before it is given
/// up: long, so that a failure says how late it was.
const FIRST_PATIENCE = 600_000

/// The first message of the firehose from `cursor`: how long it took from
/// asking, and its seq; no time if none came in `FIRST_PATIENCE`.
function firstMessage(cursor) {
  return new Promise((done) => {
    const at = Date.now()
    const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos?cursor=${cursor}`)
    ws.binaryType = 'arraybuffer'
    const give = (answer) => {
      clearTimeout(late)
      ws.close()
      done(answer)
    }
    const late = setTimeout(() => give({ ms: null }), FIRST_PATIENCE)
    ws.onmessage = (m) => give({ ms: Date.now() - at, seq: cborDecodeMulti(new Uint8Array(m.data))[1]?.seq })
    ws.onerror = () => {}
  })
}

/// How many syncs to the disk the server has made, as its metrics count
/// them (spec/OPERATIONS.md, `pds_fsync_duration_seconds`).
async function syncs() {
  const text = await fetch(metricsUrl).then((r) => r.text())
  const m = text.match(/^pds_fsync_duration_seconds_count (\d+)$/m)
  if (!m) await fail(cand, 'metrics', { metric: 'pds_fsync_duration_seconds_count' }, 'a count of syncs', 'none')
  return Number(m[1])
}

/// A record of each of `accounts` listed, which nothing has asked for since
/// the start that followed `what`, so that each one's state is read from
/// disk: answers how many syncs the server counts meanwhile.
async function firstUses(accounts, what) {
  const before = await syncs()
  await each(accounts, async (a) => {
    const res = await xrpcHttp(cand, 'com.atproto.repo.listRecords', { params: { repo: a.did, collection: 'app.bsky.actor.profile', limit: 1 } })
    if (res.status !== 200) await fail(cand, `listRecords, the first request to an account after ${what} and a start`, { did: a.did }, 200, res.body)
  })
  const made = (await syncs()) - before
  console.error(`  ${accounts.length} accounts first used after ${what} and a start: ${made} syncs`)
  return made
}

/// The seq of the first event `write` makes: a subscription with no cursor
/// is open before it, and its first message is that event's. Answered once
/// the write is answered too: the event may come first, and what follows
/// may kill the server.
function seqOf(write) {
  return new Promise((done, failed) => {
    const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`)
    ws.binaryType = 'arraybuffer'
    let written
    // Once the event has come the socket is closed, and what it says of its
    // closing is nothing to the answer.
    let got = false
    ws.onopen = () => {
      written = write()
      written.catch(failed)
    }
    ws.onmessage = (m) => {
      if (got) return
      got = true
      ws.close()
      const seq = cborDecodeMulti(new Uint8Array(m.data))[1]?.seq
      written.then(() => done(seq), failed)
    }
    ws.onerror = () => {
      if (!got) failed(new Error('no live subscription'))
    }
  })
}

/// The run: what spec/STORE.md's "How it is checked" says of it.
export async function scaleRun() {
  // The base: just started on an empty data directory.
  let t0 = 0
  for (let i = 0; i < 3; i++) {
    await server.kill()
    server.start()
    t0 = Math.max(t0, await waitReady(server, cand.url))
  }
  await wait(300)
  state.t0 = t0
  state.r0 = rss()
  console.error(`  started empty: ${t0} ms, ${MiB(state.r0)}`)

  // Few accounts, then many: the start, idle memory, and every account touched.
  await makeAccounts(FEW)
  await restart(`with ${FEW} accounts`)
  await makeAccounts(MANY)
  await restart(`with ${MANY} accounts`)
  await holdPeak(`${MANY} accounts each touched once`, (await sampled(() => touchAll(`${MANY} accounts`))).peak)

  // A large repository: written, held, read whole, and replayed.
  const big = state.accounts[0]
  const { result: values, peak: writing } = await sampled(() => writeLarge(big))
  await holdPeak('writing a large repository', writing)
  await restart('with a large repository')
  const t = Date.now()
  const first = await xrpcHttp(cand, 'com.atproto.repo.getRecord', { params: { repo: big.did, collection: 'com.example.scale', rkey: 'big0' } })
  const cold = Date.now() - t
  const u = Date.now()
  await xrpcHttp(cand, 'com.atproto.repo.getRecord', { params: { repo: big.did, collection: 'com.example.scale', rkey: 'big1' } })
  if (cold - (Date.now() - u) > COLD_MS) await fail(cand, "the large account's first request after a start", { did: big.did }, `within ${COLD_MS} ms more than the next`, `${cold} ms`)
  if (!isDeepStrictEqual(first.body?.value, values.get('big0'))) await fail(cand, 'getRecord of the large repository', { rkey: 'big0' }, values.get('big0').text.slice(0, 40), first.body)

  const url = `${cand.url}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(big.did)}`
  const { result: repo, peak: reading } = await sampled(() => streamed(url))
  if (repo.status !== 200) await fail(cand, 'getRepo of the large repository', { did: big.did }, 'the repository', repo.status)
  await holdPeak(`getRepo of ${MiB(repo.bytes.length)}`, reading)
  const car = await readCarWithRoot(repo.bytes)
  let records = 0
  car.blocks.forEach((bytes) => {
    const v = cborDecode(bytes)
    if (v?.$type === 'com.example.scale') {
      records += 1
      if (!isDeepStrictEqual(v, values.get(`big${v.n}`))) records = -Infinity
    }
  })
  if (records !== values.size) await fail(cand, 'getRepo of the large repository', { did: big.did }, `every record written, ${values.size}, as written`, records)

  const { result: events, peak: replaying } = await sampled(() => replay())
  await holdPeak(`the firehose replayed from 0, ${events.seqs.length} events, ${MiB(events.bytes)}`, replaying)
  if (events.seqs.length < MANY * 4 + BIG_BATCHES || events.seqs.some((s, i) => i && s <= events.seqs[i - 1])) {
    await fail(cand, 'the firehose replayed from 0', {}, `every event, at least ${MANY * 4 + BIG_BATCHES}, in order`, { events: events.seqs.length })
  }
  count('firehose replayed', true)

  // Much besides a repository, on one account: written, held and read
  // within the memory bound, and its first request after a start within a
  // second of the next.
  const rich = state.accounts[2]
  const { result: space, peak: holding } = await sampled(() => writeMuchBesides(rich))
  await holdPeak(`writing ${BLOBS} blobs and ${SPACE_RECORDS} space records to one account`, holding)
  await restart('with an account that holds much besides')
  const { result: took, peak: readingBesides } = await sampled(() => readMuchBesides(rich, space))
  console.error(`  the first request to the account that holds much besides: ${took.cold} ms, and again ${took.warm} ms`)
  console.error(`  its space repo whole: ${MiB(took.whole)}`)
  if (took.cold - took.warm > COLD_MS) await fail(cand, "the first request after a start to an account that holds much besides its repository", { did: rich.did }, `within ${COLD_MS} ms more than the next`, `${took.cold} ms, the next ${took.warm} ms`)
  await holdPeak('reading what the account holds besides', readingBesides)

  // Most: fifty times as many accounts, some with a password. The start,
  // idle memory, and accounts found; then their sessions refreshed, and the
  // start and the accounts again.
  await makeAccounts(MANY + WITH_PASSWORDS, { passwords: true, atOnce: MOST_AT_ONCE })
  await makeAccounts(MOST, { atOnce: MOST_AT_ONCE })
  await restart(`with ${MOST} accounts`)
  await holdPeak(`one account in ${ONE_IN} of ${MOST} found`, (await sampled(() => findSome(`a start with ${MOST} accounts`))).peak)
  await holdPeak(`${WITH_PASSWORDS} accounts' sessions each refreshed ${REFRESHES} times`, (await sampled(refreshAll)).peak)
  await restart(`with ${MOST} accounts, sessions refreshed`)
  await holdPeak(`one account in ${ONE_IN} of ${MOST} found again`, (await sampled(() => findSome('sessions refreshed and a start'))).peak)

  // The firehose after a start, a kill's as a shutdown's: its first
  // message within a second, from the first event ever made and from the
  // last, a commit made just before. This start followed a kill.
  const afterKill = await firstMessage(0)
  console.error(`  the firehose from 0, after a kill and a start with ${MOST} accounts: its first message ${afterKill.ms === null ? `not in ${FIRST_PATIENCE / 1000} s` : `in ${afterKill.ms} ms`}`)
  if (afterKill.ms === null || afterKill.ms > COLD_MS || afterKill.seq !== events.seqs[0]) {
    await fail(cand, 'the firehose from 0, after a kill and a start', { accounts: state.accounts.length }, `its first message within ${COLD_MS} ms, the event of seq ${events.seqs[0]}`, afterKill)
  }
  const writer = state.accounts[1]
  const write = (text) => async () => {
    const res = await xrpcHttp(cand, 'com.atproto.repo.createRecord', { body: { repo: writer.did, collection: 'com.example.scale', record: { $type: 'com.example.scale', text } }, token: writer.token })
    if (res.status !== 200) await fail(cand, `createRecord, ${text}`, { did: writer.did }, 'the record', res.body)
  }
  const last = await seqOf(write('the last before a shutdown'))
  await restart(`with ${MOST} accounts, after a shutdown`, 'SIGTERM')
  for (const [what, cursor, seq] of [['0', 0, events.seqs[0]], ['the seq before the last commit', last - 1, last]]) {
    const first = await firstMessage(cursor)
    console.error(`  the firehose from ${what}, after a shutdown and a start with ${MOST} accounts: its first message ${first.ms === null ? `not in ${FIRST_PATIENCE / 1000} s` : `in ${first.ms} ms`}`)
    if (first.ms === null || first.ms > COLD_MS || first.seq !== seq) {
      await fail(cand, `the firehose from ${what}, after a shutdown and a start`, { accounts: state.accounts.length, cursor }, `its first message within ${COLD_MS} ms, the event of seq ${seq}`, { ms: first.ms, seq: first.seq })
    }
  }
  count('firehose after a shutdown', true)

  // An account's first use after a shutdown syncs nothing: accounts nothing
  // has asked for since this start, a record of each listed, which reads
  // its state from disk, and the server's count of syncs before and after.
  // The few allowed are its own housekeeping's.
  const clean = await firstUses(state.accounts.slice(-FIRST_USES - 100, -100), 'a shutdown')
  if (clean >= FIRST_USES / 10) await fail(cand, 'syncs as accounts are first used after a shutdown and a start', { accounts: FIRST_USES }, `fewer than ${FIRST_USES / 10}: none for an account's first use`, clean)
  count('first uses after a shutdown', true)

  // A kill loses the firehose nothing of a write: a record written since
  // the shutdown, then a kill and a start, and the firehose from the seq
  // before that record's commit, the first thing asked of that start, is
  // sent it within a second.
  const later = await seqOf(write('after the shutdown, before a kill'))
  await restart(`with ${MOST} accounts, after a write and a kill`)
  const sent = await firstMessage(later - 1)
  console.error(`  the firehose from the seq before a commit made after the shutdown, after a kill: its first message ${sent.ms === null ? `not in ${FIRST_PATIENCE / 1000} s` : `in ${sent.ms} ms`}`)
  if (sent.ms === null || sent.ms > COLD_MS || sent.seq !== later) {
    await fail(cand, 'the firehose after a shutdown, a start, a write and a kill', { accounts: state.accounts.length, cursor: later - 1 }, `its first message within ${COLD_MS} ms, the event of seq ${later}`, sent)
  }
  count('firehose after a kill', true)

  // After a kill an account's log is synced as it is first used, and the
  // server's count of syncs says so: other accounts nothing has asked for.
  const killed = await firstUses(state.accounts.slice(-2 * FIRST_USES - 100, -FIRST_USES - 100), 'a kill')
  if (killed < FIRST_USES) await fail(cand, 'syncs as accounts are first used after a kill and a start', { accounts: FIRST_USES }, `at least ${FIRST_USES}: each account's log synced before it is used`, killed)
  count('first uses after a kill', true)
  return { r0: state.r0, t0, accounts: state.accounts.length, repo: repo.bytes.length, events: events.bytes }
}