~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
1015 lines · 62.3 KB Raw History
// Operations (spec/OPERATIONS.md), at the end of every run: /xrpc/_health
// held to the reference; the routes' labels; the trace context of requests
// sent with no `traceparent`, a bad one or an unsampled one; the metrics
// held to the log and to what the harness sees; the admin's reads, `_logs`
// held to standard output and to the bytes `PDS_LOG_DIR` may take, and
// `_stats` to the metrics around it; every span held to the request it is
// of; a shutdown, a start at another level with the PLC directory failing,
// a start after a kill, with traces off, with no `PDS_LOG_DIR`, starts on
// other addresses (spec/PDS.md), and starts with variables that do not
// parse; and last the log (checks/logging.mjs). And, in the middle of a run
// of random steps, the OTLP receiver hanging and then gone, while the
// requests still answer.

import { createHash } from 'node:crypto'
import { createServer, request as httpRequest } from 'node:http'
import { mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'
import { networkInterfaces, tmpdir } from 'node:os'
import { join } from 'node:path'
import { cborDecodeMulti } from '@atproto/common'
import { LIMITS, ONLY, SEED, STEPS } from '../lib/args.mjs'
import { cand, dataDir, metricsUrl, otlp, ref, server, wait, waitReady } from '../lib/processes.mjs'
import { OTEL_ENV, OTLP_KEY } from '../lib/adapters/monolith.mjs'
import { basic } from '../lib/adapters/common.mjs'
import { ADMIN_ROUTES, LEVELS, adminRead, bytesKept, holdLines, keepOf, keptLines, metricsCounts, newestWithin, statsBetween, statsCounts, statsForm } from '../lib/admin.mjs'
import { ASSETS, OWN_XRPC, ROUTES, SERVED, httpRoute, noteSecret, now, sent, settled } from '../lib/observe.mjs'
import { parseMetrics } from '../lib/prometheus.mjs'
import { below, pick } from '../lib/random.mjs'
import { count } from '../lib/report.mjs'
import { can } from '../lib/server.mjs'
import { accounts } from '../lib/state.mjs'
import { broke, checkLog, readLog } from './logging.mjs'

// ---------------------------------------------------------------- outage

/// When the receiver hung, went, and came back.
const outageAt = { hang: null, down: null, up: null }

/// In a run of random steps, the receiver stops answering a third of the
/// way through, and is gone at half way, once an export could have timed
/// out while it hung (the run waits for that if it must); it is back at two
/// thirds (spec/OPERATIONS.md, "Export").
export async function outage(step) {
  if (STEPS < 6) return
  if (step === Math.floor(STEPS / 3)) {
    otlp.hang()
    outageAt.hang = now()
  } else if (step === Math.floor(STEPS / 2)) {
    const timedOut = outageAt.hang + Number(OTEL_ENV.OTEL_BSP_SCHEDULE_DELAY) + Number(OTEL_ENV.OTEL_BSP_EXPORT_TIMEOUT) + 1000
    if (now() < timedOut) await wait(timedOut - now())
    await otlp.down()
    outageAt.down = now()
  } else if (step === Math.floor((2 * STEPS) / 3)) {
    await otlp.up()
    outageAt.up = now()
  }
}

// ---------------------------------------------------------------- helpers

const HEALTH = '/xrpc/_health'
const LIST_REPOS = 'com.atproto.sync.listRepos'
const TRACE_ID = /^(?!0{32})[0-9a-f]{32}$/
const hash = (s) => createHash('sha256').update(`${s} ${SEED}`).digest('hex')

/// A request to the server with only the headers given, not through the
/// harness's fetch, so with no `traceparent` unless given. Answers its
/// status, headers and body.
function raw(method, path, headers = {}) {
  const u = new URL(cand.url)
  return new Promise((ok) => {
    const req = httpRequest({ host: u.hostname, port: u.port, method, path, headers }, (res) => {
      const chunks = []
      res.on('data', (c) => chunks.push(c))
      res.on('end', () => ok({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks).toString('utf8') }))
      res.on('error', () => ok({ status: 0, headers: {}, body: '' }))
    })
    req.on('error', () => ok({ status: 0, headers: {}, body: '' }))
    req.end()
  })
}

/// The `request` lines written while `fn` ran, with nothing else in flight.
async function linesOf(fn) {
  const before = statSync(server.logPath).size
  const out = await fn()
  await wait(50)
  const text = readFileSync(server.logPath).subarray(before).toString('utf8')
  const lines = text.split('\n').filter(Boolean).map((t) => {
    try {
      return JSON.parse(t)
    } catch {
      return { unparsed: t }
    }
  })
  return { out, lines: lines.filter((l) => l.msg === 'request' || l.unparsed) }
}

const scrapeRaw = async (url = metricsUrl) => {
  try {
    const r = await fetch(url)
    return { status: r.status, type: r.headers.get('content-type'), text: await r.text() }
  } catch (err) {
    return { status: 0, error: err.message }
  }
}

/// The metrics, read and held to the format.
async function scrape(why) {
  const r = await scrapeRaw()
  if (r.status !== 200) await broke('metrics', { url: metricsUrl, why }, '200', r)
  if (r.type !== 'text/plain; version=0.0.4; charset=utf-8') await broke('metrics', { url: metricsUrl }, 'Content-Type text/plain; version=0.0.4; charset=utf-8', r.type)
  const { families, errors } = parseMetrics(r.text)
  if (errors.length) await broke('metrics', { url: metricsUrl }, 'Prometheus\' text format, version 0.0.4', errors.slice(0, 5))
  return families
}

const REQUEST_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
const FSYNC_BUCKETS = [0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 1]
const FAMILIES = {
  pds_requests_total: ['counter', ['method', 'status']],
  pds_request_duration_seconds: ['histogram', ['method'], REQUEST_BUCKETS],
  pds_rate_limited_total: ['counter', ['method']],
  pds_firehose_seq: ['gauge', []],
  pds_firehose_subscribers: ['gauge', []],
  pds_accounts: ['gauge', []],
  pds_log_bytes: ['gauge', []],
  pds_fsync_duration_seconds: ['histogram', [], FSYNC_BUCKETS],
  pds_outbound_requests_total: ['counter', ['target', 'outcome']],
  pds_outbound_request_duration_seconds: ['histogram', ['target'], REQUEST_BUCKETS],
}
const LABEL_VALUES = {
  method: (v) => SERVED.has(v) || OWN_XRPC.includes(v) || v === '_unknown' || v === ASSETS || ROUTES.includes(v),
  status: (v) => /^[1-5]\d\d$/.test(v),
  target: (v) => ['plc', 'crawler', 'service', 'did-web', 'client', 'space', 'lexicon', 'handle', 'blobstore'].includes(v),
  outcome: (v) => v === 'ok' || v === 'error',
}

/// Hold every family to the spec's type, labels and buckets
/// (spec/OPERATIONS.md, "Metrics").
async function checkFamilies(families) {
  for (const [name, [type, labels, buckets]] of Object.entries(FAMILIES)) {
    const f = families[name]
    if (!f || f.type !== type || f.help === null) await broke('metrics', { metric: name }, `# HELP and # TYPE ${type}`, f ? { type: f.type, help: f.help } : 'missing')
    if (type === 'gauge' && f.samples.length !== 1) await broke('metrics', { metric: name }, 'one sample', f.samples)
    for (const s of f.samples) {
      const keys = Object.keys(s.labels).filter((k) => !(s.name.endsWith('_bucket') && k === 'le')).sort()
      if (keys.join() !== [...labels].sort().join()) await broke('metrics', { metric: name, sample: s }, `labels ${labels.join(', ') || 'none'}`, keys)
      for (const k of keys) if (!LABEL_VALUES[k](s.labels[k])) await broke('metrics', { metric: name, sample: s }, `a \`${k}\` the spec allows: a route served or _unknown`, s.labels[k])
      if (!(s.value >= 0)) await broke('metrics', { metric: name, sample: s }, 'a value of 0 or more', s.value)
    }
    if (type !== 'histogram') continue
    const series = new Map()
    for (const s of f.samples) {
      const { le, ...rest } = s.labels
      const key = JSON.stringify(rest)
      if (!series.has(key)) series.set(key, { buckets: [], sum: null, count: null })
      const h = series.get(key)
      if (s.name === `${name}_bucket`) h.buckets.push([le === '+Inf' ? Infinity : Number(le), s.value])
      else if (s.name === `${name}_sum`) h.sum = s.value
      else if (s.name === `${name}_count`) h.count = s.value
      else await broke('metrics', { metric: name, sample: s }, `${name}_bucket, _sum or _count`, s.name)
    }
    for (const [key, h] of series) {
      const bounds = h.buckets.map(([b]) => b)
      if (bounds.join() !== [...buckets, Infinity].join()) await broke('metrics', { metric: name, labels: key }, `buckets ${buckets.join(', ')}, +Inf`, bounds)
      if (h.buckets.some(([, v], i) => i && v < h.buckets[i - 1][1])) await broke('metrics', { metric: name, labels: key }, 'cumulative buckets', h.buckets)
      if (h.count === null || h.sum === null || h.buckets.at(-1)[1] !== h.count) await broke('metrics', { metric: name, labels: key }, '_sum, and _count equal to the +Inf bucket', h)
    }
  }
}

/// A family's samples of `name` (the family's own name by default), by
/// their labels' values in `keys`' order.
const valuesOf = (families, family, keys, name = family) => {
  const out = new Map()
  for (const s of families[family]?.samples ?? []) if (s.name === name) out.set(keys.map((k) => s.labels[k]).join(' '), s.value)
  return out
}
const gauge = (families, name) => families[name]?.samples[0]?.value

/// Each file under `dir`, at any depth, by its path: its inode and size. A
/// file or directory gone while it is read is one not there.
function filesUnder(dir) {
  for (;;) {
    try {
      const out = new Map()
      for (const f of readdirSync(dir, { recursive: true })) {
        const st = statSync(join(dir, String(f)), { bigint: true, throwIfNoEntry: false })
        if (st?.isFile()) out.set(String(f), { ino: st.ino, size: Number(st.size) })
      }
      return out
    } catch (err) {
      if (err.code !== 'ENOENT') throw err
    }
  }
}

/// Every file under `dir`, in bytes.
const bytesUnder = (dir) => [...filesUnder(dir).values()].reduce((n, f) => n + f.size, 0)

/// Whether a file of `after` was not in `before`, or another file there,
/// or shorter; or one of `before` is not in `after` (each as `filesUnder`
/// gives them): only a file that grew, or stayed as it was, is no change.
function changedFile(before, after) {
  for (const [path, was] of before) {
    const is = after.get(path)
    if (!is || is.ino !== was.ino || is.size < was.size) return true
  }
  return [...after.keys()].some((path) => !before.has(path))
}

/// A subscription from the start of the firehose: answers `{ws, seqs,
/// opened, close()}`.
function subscribe() {
  const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos?cursor=0`)
  ws.binaryType = 'arraybuffer'
  const sub = { ws, seqs: [], revs: new Set(), last: now() }
  ws.onmessage = (m) => {
    sub.last = now()
    try {
      const [, body] = cborDecodeMulti(new Uint8Array(m.data))
      if (typeof body?.seq === 'number') sub.seqs.push(body.seq)
      if (typeof body?.rev === 'string') sub.revs.add(`${body.repo}@${body.rev}`)
    } catch {}
  }
  ws.onerror = () => {}
  sub.opened = new Promise((ok) => {
    ws.onopen = () => ok(true)
    ws.onclose = () => ok(false)
    setTimeout(() => ok(false), 5000)
  })
  sub.close = () => ws.close()
  return sub
}

// ---------------------------------------------------------------- checks

/// What the server answers that the spec settles for itself, to hold the
/// log and spans to after.
const probes = []
let version = null

/// /xrpc/_health, GET, with a query, HEAD and POST, held to the reference:
/// the status, Content-Type, and the body's form; and past the global rate
/// limit, with --limits, still 200 with no RateLimit headers. /metrics on
/// the server's own port is no route, as on the reference
/// (spec/OPERATIONS.md, "Health", "Metrics").
async function checkHealth() {
  const ask = (side, method, path, headers = {}) =>
    fetch(`${side.url}${path}`, { method, headers }).then(async (r) => ({ status: r.status, type: r.headers.get('content-type'), body: await r.text(), limits: [...r.headers.keys()].filter((k) => k.startsWith('ratelimit')) }))
  for (const [method, path] of [['GET', HEALTH], ['GET', `${HEALTH}?x=1`], ['HEAD', HEALTH], ['POST', HEALTH], ['GET', '/metrics']]) {
    const [a, b] = await Promise.all([ref, cand].map((side) => ask(side, method, path)))
    const shape = (r) => {
      let body = r.body
      try {
        body = JSON.parse(r.body)
      } catch {}
      if (r.status >= 400) return { status: r.status, error: body?.error ?? null }
      return { status: r.status, type: r.type, keys: body && typeof body === 'object' ? Object.keys(body).sort() : body, version: typeof body?.version === 'string' && body.version.length > 0 }
    }
    const [na, nb] = [shape(a), shape(b)]
    if (path === '/metrics') {
      if (a.status !== 404 || b.status !== 404) await broke('/metrics on its own port', { method, path }, 'no route: 404, as on the reference', { reference: a.status, server: b.status })
      continue
    }
    if (JSON.stringify(na) !== JSON.stringify(nb)) await broke('_health', { method, path }, na, nb)
    if (b.status === 200 && b.limits.length) await broke('_health', { method, path }, 'no RateLimit headers', b.limits)
    if (method === 'GET' && b.status === 200) version = JSON.parse(b.body).version
    count(`_health ${method}${path.includes('?') ? ' with a query' : ''}`, b.status < 400)
  }
  if (LIMITS) {
    const ip = '198.51.100.7'
    for (const side of [ref, cand]) for (let i = 0; i < 3001; i += 50) await Promise.all(Array.from({ length: Math.min(50, 3001 - i) }, () => fetch(`${side.url}/xrpc/com.atproto.server.describeServer`, { headers: { 'x-forwarded-for': ip } }).then((r) => r.arrayBuffer())))
    const [a, b] = await Promise.all([ref, cand].map((side) => ask(side, 'GET', HEALTH, { 'x-forwarded-for': ip })))
    if (a.status !== 200 || a.limits.length) await broke('harness', {}, 'the reference answers _health past global with 200', a)
    if (b.status !== 200 || b.limits.length) await broke('_health past the global rate limit', { ip }, '200, with no RateLimit headers, as the reference', { status: b.status, limits: b.limits })
    count('_health past global', true)
  }
}

/// The text `GET /` answers, as spec/OPERATIONS.md's "The root" writes it,
/// read from that page, with the server's own origin.
function rootText() {
  const page = readFileSync(new URL('../../spec/OPERATIONS.md', import.meta.url), 'utf8')
  const block = /## The root\n[\s\S]*?```text\n([\s\S]*?)\n```/.exec(page)?.[1]
  if (block === undefined) throw new Error('spec/OPERATIONS.md has no "The root" text block')
  return `${block.replaceAll('<origin>', new URL(cand.publicUrl).origin)}\n`
}

/// `/`, GET, with a query, HEAD and POST, held to the reference: the status
/// and Content-Type; and the server's text held to the spec, with no
/// RateLimit headers (spec/OPERATIONS.md, "The root").
async function checkRoot() {
  const want = rootText()
  const ask = (side, method, path) =>
    fetch(`${side.url}${path}`, { method }).then(async (r) => ({ status: r.status, type: r.status < 400 ? r.headers.get('content-type') : null, body: await r.text(), limits: [...r.headers.keys()].filter((k) => k.startsWith('ratelimit')) }))
  for (const [method, path] of [['GET', '/'], ['GET', '/?x=1'], ['HEAD', '/'], ['POST', '/']]) {
    const [a, b] = await Promise.all([ref, cand].map((side) => ask(side, method, path)))
    const [na, nb] = [a, b].map(({ status, type }) => ({ status, type }))
    if (JSON.stringify(na) !== JSON.stringify(nb)) await broke('the root', { method, path }, na, nb)
    if (b.status === 200 && b.limits.length) await broke('the root', { method, path }, 'no RateLimit headers', b.limits)
    const body = method === 'HEAD' ? '' : want
    if (b.status === 200 && b.body !== body) await broke('the root', { method, path }, body, b.body)
    count(`the root ${method}${path.includes('?') ? ' with a query' : ''}`, b.status < 400)
  }
}

/// Routes the server does not serve, each to be `_unknown` in the log and
/// the metrics (spec/OPERATIONS.md, "Metrics").
async function unknownRoutes() {
  for (const path of [`/xrpc/com.example.ops.nothing${SEED}`, '/xrpc/com.example.ops.nothing.else', '/xrpc/not%20an%20nsid', `/ops/no/such/route/${SEED}`]) await fetch(`${cand.url}${path}`).then((r) => r.arrayBuffer())
  count('routes not served', true)
}

/// Requests with no `traceparent`, ones not valid, an unsampled one and one
/// of a later version, sent one at a time: each line's trace id is new for
/// the first, and the one sent for the others (spec/OPERATIONS.md, "The
/// request's trace").
async function traceContext() {
  const t = hash('ops trace')
  const cases = [
    ['no traceparent', {}, null],
    ['a traceparent that does not parse', { traceparent: 'garbage' }, null],
    ['a traceparent of trace id 0', { traceparent: `00-${'0'.repeat(32)}-${t.slice(0, 16)}-01` }, null],
    ['a traceparent of version ff', { traceparent: `ff-${t.slice(0, 32)}-${t.slice(32, 48)}-01` }, null],
    ['a traceparent in upper case', { traceparent: `00-${t.slice(0, 32).toUpperCase()}-${t.slice(32, 48)}-01` }, null],
    ['an unsampled traceparent', { traceparent: `00-${t.slice(1, 33)}-${t.slice(33, 49)}-00` }, { traceId: t.slice(1, 33), spanId: t.slice(33, 49), sampled: false }],
    ['a traceparent of version 01, with more after it', { traceparent: `01-${t.slice(2, 34)}-${t.slice(34, 50)}-01-more` }, { traceId: t.slice(2, 34), spanId: t.slice(34, 50), sampled: true }],
  ]
  const harnessTraces = new Set(sent.map((r) => r.traceId))
  for (const [kind, headers, want] of cases) {
    const { out, lines } = await linesOf(() => raw('GET', HEALTH, headers))
    const request = { method: 'GET', path: HEALTH, headers }
    if (lines.length !== 1 || lines[0].unparsed) await broke('log', { ...request, kind }, 'one `request` line for the request', lines)
    const [line] = lines
    if (want) {
      if (line.trace_id !== want.traceId) await broke('log', { ...request, kind }, { trace_id: want.traceId }, line)
      probes.push({ kind, ...want, status: out.status, at: now() })
    } else {
      if (typeof line.trace_id !== 'string' || !TRACE_ID.test(line.trace_id) || harnessTraces.has(line.trace_id) || t.includes(line.trace_id)) await broke('log', { ...request, kind }, 'a `trace_id` of a new trace', line)
      probes.push({ kind, traceId: line.trace_id, spanId: '', sampled: true, status: out.status, at: now() })
    }
    count(`trace context: ${kind}`, true)
  }
}

/// How many records the canary writes, at most, for one written while the
/// server let nothing of its logs go.
const LOG_TRIES = 20

/// An account of the run's own on the server, and records whose text the
/// log must not hold: answers the last one's commit, as its account's DID,
/// `@` and its rev, the last write before the metrics are read.
///
/// `pds_log_bytes` grows across each, unless the server let go of what its
/// logs held meanwhile, which it does only as it makes, deletes, replaces
/// or cuts short a file in the data directory (spec/OPERATIONS.md,
/// "Metrics"): read the directory before the metrics and after them,
/// around the write, and where a file there changed but by growing, write
/// another record, until one is written with none so changed.
async function canary() {
  const ip = '198.51.100.9'
  const front = `opscanary${SEED}`
  const post = (nsid, body, headers = {}) =>
    fetch(`${cand.url}/xrpc/${nsid}`, { method: 'POST', headers: { 'content-type': 'application/json', 'x-forwarded-for': ip, ...headers }, body: JSON.stringify(body) }).then(async (r) => ({ status: r.status, body: await r.json().catch(() => null) }))
  const described = await fetch(`${cand.url}/xrpc/com.atproto.server.describeServer`).then((r) => r.json())
  let inviteCode
  if (described.inviteCodeRequired) {
    const admin = `Basic ${Buffer.from(`admin:${server.env.PDS_ADMIN_PASSWORD}`).toString('base64')}`
    inviteCode = (await post('com.atproto.server.createInviteCode', { useCount: 1 }, { authorization: admin })).body?.code
  }
  const made = await post('com.atproto.server.createAccount', { handle: `${front}.test`, email: `${front}@example.com`, password: `pw-${hash('canary password').slice(0, 20)}`, ...(inviteCode ? { inviteCode } : {}) })
  if (made.status !== 200) await broke('createAccount', { handle: `${front}.test` }, 200, made)
  let rev
  let held = false
  for (let n = 0; n < LOG_TRIES && !held; n++) {
    const files = filesUnder(dataDir)
    const before = gauge(await scrape('before a canary record'), 'pds_log_bytes')
    const text = `canary-${hash(n ? `canary record ${n}` : 'canary record').slice(0, 24)}`
    noteSecret(text)
    const wrote = await post('com.atproto.repo.createRecord', { repo: made.body.did, collection: 'com.example.note', record: { $type: 'com.example.note', text } }, { authorization: `Bearer ${made.body.accessJwt}` })
    if (wrote.status !== 200) await broke('createRecord', { canary: true }, 200, wrote)
    rev = wrote.body?.commit?.rev
    const after = gauge(await scrape('after a canary record'), 'pds_log_bytes')
    if (changedFile(files, filesUnder(dataDir))) continue
    if (!(after > before)) await broke('metrics', { metric: 'pds_log_bytes', after: 'a createRecord', record: n + 1, meanwhile: 'no file in the data directory made, deleted, replaced or cut short' }, `more than ${before}`, after)
    held = true
  }
  count('metrics: pds_log_bytes across a write that let nothing go', held)
  return `${made.body.did}@${rev}`
}

/// The metrics: the format, each family as the spec gives it, and each held
/// to the log of this start and to what the harness sees
/// (spec/OPERATIONS.md, "Metrics").
async function checkMetrics(before, last) {
  // The firehose: one more subscriber while the harness has one open, and
  // the seq of the last event it is sent. It has been sent them all once it
  // has the last write's commit, `last`, and nothing for a second after: a
  // quiet second alone says nothing, as a replay from cursor 0 may pause for
  // longer between two accounts' events.
  const base = gauge(await scrape('before subscribing'), 'pds_firehose_subscribers')
  const sub = subscribe()
  if (!(await sub.opened)) await broke('firehose', { cursor: 0 }, 'a subscription', 'none')
  for (const until = now() + 120000; !sub.revs.has(last); await wait(100)) {
    if (now() > until) await broke('firehose', { cursor: 0, commit: last }, 'the commit of the last write, within 120 s', `${sub.seqs.length} events, the last of seq ${Math.max(0, ...sub.seqs)}`)
  }
  while (now() - sub.last < 1000) await wait(100)
  const during = await scrape('while subscribed')
  const seq = Math.max(0, ...sub.seqs)
  if (gauge(during, 'pds_firehose_subscribers') !== base + 1) await broke('metrics', { metric: 'pds_firehose_subscribers', before: base }, base + 1, gauge(during, 'pds_firehose_subscribers'))
  if (gauge(during, 'pds_firehose_seq') !== seq) await broke('metrics', { metric: 'pds_firehose_seq' }, `${seq}, the last event's seq`, gauge(during, 'pds_firehose_seq'))
  sub.close()
  for (let i = 0; gauge(await scrape('after unsubscribing'), 'pds_firehose_subscribers') !== base; i++) if (i > 30) await broke('metrics', { metric: 'pds_firehose_subscribers', after: 'the subscription closed' }, base, 'still more')
  else await wait(100)
  // The accounts, as listRepos lists them.
  let accounts = 0
  let cursor
  do {
    const r = await fetch(`${cand.url}/xrpc/${LIST_REPOS}?limit=1000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`).then((x) => x.json())
    accounts += r.repos?.length ?? 0
    cursor = r.cursor && r.repos?.length ? r.cursor : undefined
  } while (cursor)
  await settled()
  const families = await scrape('at the end')
  const lives = readLog()
  await checkFamilies(families)
  if (gauge(families, 'pds_accounts') !== accounts) await broke('metrics', { metric: 'pds_accounts' }, `${accounts}, as listRepos lists`, gauge(families, 'pds_accounts'))
  const disk = bytesUnder(dataDir)
  const logBytes = gauge(families, 'pds_log_bytes')
  if (!(logBytes > 0 && logBytes <= disk)) await broke('metrics', { metric: 'pds_log_bytes', dataDir: `${disk} bytes` }, 'more than 0, and no more than the data directory holds', logBytes)
  // Requests, by route and status, as this start's lines count them.
  const life = lives.at(-1)
  const lines = new Map()
  for (const { obj } of life.lines) if (obj?.msg === 'request') lines.set(`${obj.method} ${obj.status}`, (lines.get(`${obj.method} ${obj.status}`) ?? 0) + 1)
  const counted = valuesOf(families, 'pds_requests_total', ['method', 'status'])
  for (const key of new Set([...lines.keys(), ...counted.keys()])) if ((counted.get(key) ?? 0) !== (lines.get(key) ?? 0)) await broke('metrics', { metric: 'pds_requests_total', 'method status': key }, `${lines.get(key) ?? 0}, the \`request\` lines since the server started`, counted.get(key) ?? 0)
  const ours = new Map()
  for (const rec of sentSince(life)) ours.set(`${rec.route} ${rec.status}`, (ours.get(`${rec.route} ${rec.status}`) ?? 0) + 1)
  for (const [key, n] of ours) if ((counted.get(key) ?? 0) < n) await broke('metrics', { metric: 'pds_requests_total', 'method status': key }, `at least the ${n} requests the harness sent and had answered`, counted.get(key) ?? 0)
  const durations = valuesOf(families, 'pds_request_duration_seconds', ['method'], 'pds_request_duration_seconds_count')
  const byMethod = new Map()
  for (const [key, n] of counted) byMethod.set(key.split(' ')[0], (byMethod.get(key.split(' ')[0]) ?? 0) + n)
  for (const key of new Set([...byMethod.keys(), ...durations.keys()])) if ((durations.get(key) ?? 0) !== (byMethod.get(key) ?? 0)) await broke('metrics', { metric: 'pds_request_duration_seconds_count', method: key }, byMethod.get(key) ?? 0, durations.get(key) ?? 0)
  const limited = valuesOf(families, 'pds_rate_limited_total', ['method'])
  for (const key of new Set([...byMethod.keys(), ...limited.keys()])) if ((limited.get(key) ?? 0) !== (counted.get(`${key} 429`) ?? 0)) await broke('metrics', { metric: 'pds_rate_limited_total', method: key }, `${counted.get(`${key} 429`) ?? 0}, its 429s`, limited.get(key) ?? 0)
  // Syncs, and the requests to the PLC directory.
  const syncs = valuesOf(families, 'pds_fsync_duration_seconds', [], 'pds_fsync_duration_seconds_count').get('') ?? 0
  const commits = sentSince(life).filter((r) => r.commit && r.status < 300).length
  if (commits && !(syncs > (valuesOf(before, 'pds_fsync_duration_seconds', [], 'pds_fsync_duration_seconds_count').get('') ?? 0))) await broke('metrics', { metric: 'pds_fsync_duration_seconds_count', commits }, 'more after a commit', syncs)
  const created = sentSince(life).filter((r) => r.route === 'com.atproto.server.createAccount' && r.status === 200).length
  const plcOk = valuesOf(families, 'pds_outbound_requests_total', ['target', 'outcome']).get('plc ok') ?? 0
  if (plcOk < created) await broke('metrics', { metric: 'pds_outbound_requests_total{target="plc",outcome="ok"}' }, `at least the ${created} accounts made since the server started`, plcOk)
  const elsewhere = await scrapeRaw(metricsUrl.replace(/\/metrics$/, '/elsewhere'))
  if (elsewhere.status !== 404) await broke('metrics port', { path: '/elsewhere' }, 404, elsewhere.status)
  count('metrics', true)
}

// ---------------------------------------------------------------- the admin's reads

const JSON_TYPE = 'application/json; charset=utf-8'

/// Whether the server's log is kept in `PDS_LOG_DIR` for `_logs` to read
/// back, as this start was given it: never when its adapter says it cannot
/// be (lib/server.mjs).
const keeps = () => can('logDir') && !!server.env.PDS_LOG_DIR

let adminReads = 0

/// A `traceparent` of an admin read's own, sampled, and its trace id, from
/// the seed: its `request` line is found by it.
function adminTrace() {
  const t = hash(`admin read ${++adminReads}`)
  return { traceId: t.slice(0, 32), header: `00-${t.slice(0, 32)}-${t.slice(32, 48)}-01` }
}

/// An admin's read that must be refused with `status` and `error`
/// (spec/OPERATIONS.md, "The admin's reads"; spec/LIMITS.md, "The admin").
async function refused(route, opts, status, error, kind) {
  const r = await adminRead(route, opts)
  if (r.status !== status || r.body?.error !== error) await broke(route, { method: opts.method ?? 'GET', path: `/xrpc/${route}`, params: opts.params, kind }, { status, error }, { status: r.status, body: r.text.slice(0, 300) })
  count(`${route}: ${kind}`, false)
}

/// The files in `PDS_LOG_DIR`, at any depth, taking no more than
/// `PDS_LOG_KEEP_BYTES` together (spec/OPERATIONS.md, "Kept").
async function keptBounded(why) {
  const dir = server.env.PDS_LOG_DIR
  const [bytes, keep] = [bytesKept(dir), keepOf(server.env)]
  if (bytes > keep) await broke('kept log', { PDS_LOG_DIR: dir, why }, `files taking no more than PDS_LOG_KEEP_BYTES, ${keep} bytes, together`, `${bytes} bytes`)
}

/// `_logs` with `params`, as the admin: its lines the last of those that
/// match of the lines the starts that kept them in this `PDS_LOG_DIR` wrote
/// to standard output before the read, each the same JSON object, as many
/// as its `limit`, and at least those among the newest within half of
/// `PDS_LOG_KEEP_BYTES`; and the files still within it
/// (spec/OPERATIONS.md, "_logs", "Kept"). Answers the lines.
async function readLogs(params, why) {
  await settled()
  const dir = server.env.PDS_LOG_DIR
  const keep = keepOf(server.env)
  const from = keptLines(readLog(), dir, keep).length
  const t = adminTrace()
  const r = await adminRead('_logs', { params, headers: { authorization: cand.admin, traceparent: t.header } })
  const request = { method: 'GET', path: '/xrpc/_logs', params, why }
  if (r.status !== 200 || r.type !== JSON_TYPE) await broke('_logs', request, `200, with Content-Type ${JSON_TYPE}`, { status: r.status, type: r.type, body: r.text.slice(0, 300) })
  const lines = r.body?.lines
  if (Object.keys(r.body ?? {}).join() !== 'lines' || !Array.isArray(lines) || lines.some((l) => !l || typeof l !== 'object' || Array.isArray(l))) return broke('_logs', request, '{lines}, each line a JSON object', r.text.slice(0, 300))
  const after = keptLines(readLog(), dir, keep)
  const own = after.findIndex((l, i) => i >= from && l.obj.msg === 'request' && l.obj.trace_id === t.traceId)
  const limit = params.limit === undefined ? 100 : Number(params.limit)
  const want = holdLines(lines, after, { from, to: own === -1 ? after.length : own, params, limit, half: keep / 2 })
  if (want) {
    const first = want.findIndex((l, i) => JSON.stringify(l) !== JSON.stringify(lines[i]))
    await broke('_logs', request, { lines: want.length, from: first, want: want.slice(Math.max(0, first), first + 3), note: `the last ${limit} that match of the lines the starts that kept them wrote to standard output before the read, each as it is there, at least those within the newest ${keep / 2} bytes` }, { lines: lines.length, from: first, got: lines.slice(Math.max(0, first), first + 3) })
  }
  await keptBounded(why)
  return lines
}

/// At a check: the files in `PDS_LOG_DIR` within `PDS_LOG_KEEP_BYTES`, and
/// `_logs` reading back the newest lines of half that, of every kind, of
/// the starts, and at `warn` (spec/OPERATIONS.md, "Kept").
async function keptHolds(why) {
  await keptBounded(why)
  for (const params of [{ limit: '500' }, { msg: 'start', limit: '500' }, { level: 'warn', limit: '500' }]) await readLogs(params, why)
  count(`_logs: the newest lines kept, ${why}`, true)
}

/// `_logs` by every parameter, as the admin: with none, with a limit; by
/// the trace of each of the last requests the harness sent, that request's
/// line alone; and by `msg`, `method`, `level` and `minMs`, each and
/// together, drawn from the lines kept (spec/OPERATIONS.md, "_logs").
async function checkLogsRead() {
  const recent = sent.filter((r) => r.traceId && r.status > 0 && r.ended !== null).slice(-10)
  await keptHolds('at the end of the run')
  await readLogs({}, 'no parameter')
  await readLogs({ limit: String(1 + below(500)) }, 'a limit')
  for (const rec of recent) await readLogs({ trace: rec.traceId }, `the trace of a request the harness sent, ${rec.method} ${rec.path}`)
  const kept = keptLines(readLog(), server.env.PDS_LOG_DIR, keepOf(server.env))
  const newest = kept.slice(kept.length - newestWithin(kept, keepOf(server.env) / 2))
  const msg = pick([...new Set(newest.map((l) => l.obj.msg))])
  const method = pick([...new Set(newest.map((l) => l.obj.method).filter((m) => typeof m === 'string'))])
  const level = pick(LEVELS)
  const timed = pick(newest.filter((l) => typeof l.obj.ms === 'number'))
  const one = pick(newest.filter((l) => l.obj.msg === 'request' && typeof l.obj.method === 'string' && typeof l.obj.ms === 'number' && LEVELS.includes(l.obj.level)))
  const atMost = pick(LEVELS.slice(0, LEVELS.indexOf(one?.obj.level ?? 'debug') + 1))
  const together = one ? { msg: 'request', method: one.obj.method, level: atMost, minMs: String(Math.floor(one.obj.ms)) } : { msg, level }
  const reads = [
    ['msg', { msg }],
    ['method', { method }],
    ['level', { level }],
    ['minMs', { minMs: timed ? String(Math.floor(timed.obj.ms)) : '0' }],
    ['all of them', together],
    ['all of them, with a limit', { ...together, limit: String(1 + below(20)) }],
  ]
  for (const [kind, params] of reads) if (Object.values(params).every((v) => v !== undefined)) await readLogs(params, `by ${kind}`)
  count('_logs by each parameter', true)
}

/// With `PDS_LOG_DIR` unset: `_logs` is `404 LogNotKept` once its auth and
/// parameters are checked, and `_stats` answers (spec/OPERATIONS.md,
/// "_logs").
async function notKept(why) {
  await refused('_logs', {}, 401, 'AuthenticationRequired', `${why}, no Authorization`)
  await refused('_logs', { params: { limit: '0' }, headers: { authorization: cand.admin } }, 400, 'InvalidRequest', `${why}, a limit of 0`)
  await refused('_logs', { headers: { authorization: cand.admin } }, 404, 'LogNotKept', why)
  const s = await adminRead('_stats', { headers: { authorization: cand.admin } })
  const form = s.status === 200 && s.type === JSON_TYPE ? statsForm(s.body) : `200, with Content-Type ${JSON_TYPE}`
  if (form) await broke('_stats', { method: 'GET', path: '/xrpc/_stats', why }, form, { status: s.status, type: s.type, body: s.text.slice(0, 300) })
}

/// `_stats` between two scrapes of the metrics, nothing else in flight:
/// its form, its version, `started` the time of this start's `start` line,
/// each number between the two scrapes', and the second scrape counting the
/// `_stats` request, one more than the first (spec/OPERATIONS.md,
/// "_stats").
async function checkStats() {
  await settled()
  const lo = await scrape('before _stats')
  const r = await adminRead('_stats', { headers: { authorization: cand.admin } })
  const hi = await scrape('after _stats')
  const request = { method: 'GET', path: '/xrpc/_stats' }
  if (r.status !== 200 || r.type !== JSON_TYPE) await broke('_stats', request, `200, with Content-Type ${JSON_TYPE}`, { status: r.status, type: r.type, body: r.text.slice(0, 300) })
  const form = statsForm(r.body)
  if (form) return broke('_stats', request, form, r.body)
  if (r.body.version !== version) await broke('_stats', request, { version }, { version: r.body.version })
  const start = readLog().at(-1).lines.find((l) => l.obj?.msg === 'start')?.obj
  if (!start || r.body.started !== start.time) await broke('_stats', request, { started: start?.time ?? 'the time of this start\'s `start` line' }, { started: r.body.started })
  const off = statsBetween(metricsCounts(lo), statsCounts(r.body), metricsCounts(hi))
  if (off) await broke('_stats', request, `each number as the metrics count it, between two scrapes around it: ${off.field} from ${off.lo} to ${off.hi}`, off.mid)
  const counted = (families) => metricsCounts(families).routes.get('_stats')?.count ?? 0
  if (counted(hi) !== counted(lo) + 1) await broke('metrics', { metric: 'pds_request_duration_seconds_count{method="_stats"}', around: 'one _stats' }, counted(lo) + 1, counted(hi))
  count('_stats', true)
}

/// The admin's reads: refused without the admin's auth, for another method
/// than GET and for parameters out of range; `_logs` by each parameter, or
/// `404 LogNotKept` with no `PDS_LOG_DIR`; and `_stats` (spec/OPERATIONS.md,
/// "The admin's reads"; spec/LIMITS.md, "The admin").
async function checkAdmin() {
  const password = server.env.PDS_ADMIN_PASSWORD
  const token = accounts.map((a) => a.sides[cand.name]?.token).find(Boolean) ?? 'not-a-token-of-the-server'
  const wrong = [
    ['no Authorization', {}],
    ['a bearer token', { authorization: `Bearer ${token}` }],
    ['the user admin with another password', { authorization: basic('admin', `${password}-not`) }],
    ['another user', { authorization: basic('someone', password) }],
  ]
  for (const route of ADMIN_ROUTES) {
    for (const [kind, headers] of wrong) await refused(route, { headers }, 401, 'AuthenticationRequired', kind)
    await refused(route, { method: 'POST' }, 401, 'AuthenticationRequired', 'POST, with no Authorization')
    await refused(route, { method: 'POST', headers: { authorization: cand.admin } }, 400, 'InvalidRequest', 'POST')
  }
  await refused('_logs', { params: { limit: '0' } }, 401, 'AuthenticationRequired', 'a limit of 0, with no Authorization')
  for (const params of [{ level: 'loud' }, { minMs: '-1' }, { limit: '0' }, { limit: '501' }, { limit: 'x' }]) {
    const [k, v] = Object.entries(params)[0]
    await refused('_logs', { params, headers: { authorization: cand.admin } }, 400, 'InvalidRequest', `${k} ${v}`)
  }
  if (can('logDir')) {
    if (keeps()) await checkLogsRead()
    else await notKept('PDS_LOG_DIR unset')
  }
  await checkStats()
}

/// The harness's requests answered by `life`'s server.
const sentSince = (life) => sent.filter((r) => r.life === life.n && r.status > 0 && r.ended !== null)

/// Whether spans of a request that ended at `t` may be lost: in the
/// outage, until an export could have gone after it, or just before the
/// server was killed, from when it was sent SIGKILL, which a process in the
/// middle of a disk write may outlive by seconds on a loaded machine, to
/// when it exited.
function mayBeLost(t) {
  if (outageAt.hang !== null && t >= outageAt.hang - 1500 && t <= (outageAt.up ?? Infinity) + Number(OTEL_ENV.OTEL_BSP_EXPORT_TIMEOUT) + 1500) return true
  return server.lives.some((l) => l.how === 'SIGKILL' && l.ended !== null && t >= (l.signalled ?? l.ended) - 1500 && t <= l.ended)
}

/// Whether the spans of a request that `life`'s server took from `from`
/// to `to` may be in an export it dropped, as its log says (spec/
/// OPERATIONS.md, "Export", "Events"): each span is exported after it
/// ends, in an export that fails `OTEL_BSP_EXPORT_TIMEOUT` after it began
/// at the latest, and begins once the one before it ended, within the
/// schedule delay; and a failure is logged but once a minute while exports
/// keep failing, none taken. So: a `trace export failed` line of that life
/// written from `from` until the schedule delay, two timeouts and a
/// second and a half after `to`; or the last before `from`, within the
/// minute and the timeout before it, with no export taken by the receiver
/// between the two, so that the failures may have run on unlogged.
function exportDropped(from, to, life) {
  const delay = Number(OTEL_ENV.OTEL_BSP_SCHEDULE_DELAY)
  const timeout = Number(OTEL_ENV.OTEL_BSP_EXPORT_TIMEOUT)
  const failed = (life?.lines ?? []).filter(({ obj }) => obj?.msg === 'trace export failed').map(({ obj }) => Date.parse(obj.time))
  if (failed.some((f) => f >= from && f <= to + delay + 2 * timeout + 1500)) return true
  const last = Math.max(...failed.filter((f) => f < from))
  return last >= from - 60000 - timeout - delay && !otlp.exports.some((e) => e.at > last && e.at < from)
}

/// Every span held to its request: one SERVER span for each request
/// sampled, the child of the span its traceparent named, with the
/// attributes the spec gives; none for one not sampled; a sync under every
/// commit and a CLIENT span under every account made; a root span for each
/// request that came with no traceparent; and every export as the spec
/// says (spec/OPERATIONS.md, "Traces", "The request's trace", "Spans",
/// "Export").
async function checkSpans(upTo) {
  await settled()
  const lives = readLog()
  const traced = (life) => !!(life.env.OTEL_EXPORTER_OTLP_ENDPOINT || life.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT)
  const recs = sent.filter((r) => r.status > 0 && r.ended !== null && r.traceId && r.ended <= upTo && traced(lives[r.life]) && (!lives[r.life + 1] || traced(lives[r.life + 1])) && !mayBeLost(r.ended))
  const servers = () => {
    const m = new Map()
    for (const sp of otlp.spans.values()) if (sp.kind === 'SERVER') m.set(sp.traceId, [...(m.get(sp.traceId) ?? []), sp])
    return m
  }
  for (let i = 0; i < 50; i++) {
    const have = servers()
    if (recs.every((r) => !r.sampled || have.has(r.traceId)) && probes.every((p) => !p.sampled || have.has(p.traceId))) break
    await wait(100)
  }
  if (otlp.refused.length) await broke('trace exports', otlp.refused[0], 'POST /v1/traces of an ExportTraceServiceRequest in protobuf', otlp.refused[0].why)
  const unkeyed = otlp.exports.find((e) => e.headers['x-otlp-key'] !== OTLP_KEY)
  if (unkeyed) await broke('trace exports', { OTEL_EXPORTER_OTLP_HEADERS: 'x-otlp-key=…' }, 'the header with each export', unkeyed.headers)
  const byTrace = servers()
  const children = new Map()
  // Every span, of any kind, by its trace.
  const inTrace = new Map()
  for (const sp of otlp.spans.values()) {
    if (sp.parentSpanId) children.set(sp.parentSpanId, [...(children.get(sp.parentSpanId) ?? []), sp])
    inTrace.set(sp.traceId, [...(inTrace.get(sp.traceId) ?? []), sp])
  }
  let checked = 0
  for (const rec of recs) {
    const request = { method: rec.method, path: rec.path, traceparent: `00-${rec.traceId}-${rec.spanId}-${rec.sampled ? '01' : '00'}`, status: rec.status }
    const spans = byTrace.get(rec.traceId) ?? []
    if (!rec.sampled) {
      const any = inTrace.get(rec.traceId) ?? []
      if (any.length) await broke('spans', request, 'none: the traceparent is not sampled', any.slice(0, 2))
      continue
    }
    // A span missing from an export the server says it dropped.
    const dropped = () => exportDropped(rec.started, rec.ended, lives[rec.life])
    if (!spans.length && dropped()) continue
    if (spans.length !== 1) {
      // When it ended, among the server's lives, so that a span lost shows where.
      const at = (t) => (t === null || t === undefined ? null : new Date(t).toISOString())
      const when = { ended: at(rec.ended), life: rec.life, lives: server.lives.map((l) => ({ started: at(l.started), ended: at(l.ended), how: l.how })) }
      // What the exports were then, so that a span lost shows how: the
      // exports the receiver took within 10 s of it, the server's logged
      // export failures, and the requests of its life whose spans are lost
      // with it.
      const near = otlp.exports.filter((e) => Math.abs(e.at - rec.ended) <= 10000).map((e) => ({ at: at(e.at), spans: e.spans }))
      const failed = lives.flatMap((l) => l.lines).filter(({ obj }) => obj?.msg === 'trace export failed').map(({ obj }) => ({ time: obj.time, error: obj.error }))
      const lost = recs.filter((r) => r.life === rec.life && r.sampled && !byTrace.has(r.traceId)).map((r) => ({ path: r.path, ended: at(r.ended) }))
      await broke('spans', { ...request, ...when, exportsNear: near, exportFailures: failed, lostInLife: { count: lost.length, first: lost.slice(0, 3), last: lost.slice(-3) } }, 'one SERVER span in the request\'s trace', spans.length ? spans : `none, of the ${otlp.spans.size} spans the receiver has from ${otlp.exports.length} exports`)
    }
    const [sp] = spans
    await checkServerSpan(sp, rec, request, lives[rec.life])
    const under = children.get(sp.spanId) ?? []
    if (rec.commit && rec.status < 300 && !under.some((c) => c.name === 'fsync' && c.kind === 'INTERNAL' && c.traceId === rec.traceId) && !dropped()) await broke('spans', request, 'an INTERNAL `fsync` span under a request that made a commit', under)
    if (rec.route === 'com.atproto.server.createAccount' && rec.status === 200) {
      // An account made with a `did` posts no operation: its call reads the
      // DID's document (spec/MIGRATION.md), from the directory for a
      // did:plc, from its host for a did:web. Only that one carries a
      // bearer token, the service token.
      const verb = rec.token ? 'GET' : 'POST'
      const web = /^did:web:([^:]+)$/.exec(rec.did ?? '')
      const host = web ? decodeURIComponent(web[1]) : null
      const base = host ? `${host.startsWith('localhost') ? 'http' : 'https'}://${host}/.well-known/did.json` : lives[rec.life].env.PDS_PLC_URL
      const at = new URL(base)
      const port = BigInt(at.port || (at.protocol === 'https:' ? 443 : 80))
      const call = under.find((c) => c.kind === 'CLIENT' && c.name === verb && String(c.attributes['url.full'] ?? '').startsWith(base))
      if ((!call && !dropped()) || (call && (call.attributes['http.request.method'] !== verb || call.attributes['server.address'] !== at.hostname || call.attributes['server.port'] !== port || call.attributes['http.response.status_code'] !== 200n))) await broke('spans', request, `a CLIENT span \`${verb}\` under it, of the call to ${host ? 'the DID\'s host' : 'the PLC directory'} at ${base}`, call ?? under)
    }
    checked += 1
  }
  for (const p of probes) {
    if (p.at > upTo) continue
    const spans = byTrace.get(p.traceId) ?? []
    if (!p.sampled) {
      if ([...otlp.spans.values()].some((sp) => sp.traceId === p.traceId)) await broke('spans', { kind: p.kind }, 'none: the traceparent is not sampled', 'a span')
    } else if (spans.length !== 1 || spans[0].parentSpanId !== p.spanId) await broke('spans', { kind: p.kind }, p.spanId ? `one SERVER span, the child of ${p.spanId}` : 'one SERVER span, the root of a new trace', spans)
  }
  // Each request line from a client that sent no traceparent, as a
  // subscription or a CLI command: a root span of its trace.
  const ours = new Set(sent.map((r) => r.traceId))
  for (const p of probes) ours.add(p.traceId)
  for (const life of lives) {
    if (!traced(life)) continue
    for (const { obj } of life.lines) {
      if (obj?.msg !== 'request' || ours.has(obj.trace_id)) continue
      const t = Date.parse(obj.time)
      if (t > upTo - 1000 || mayBeLost(t)) continue
      const spans = byTrace.get(obj.trace_id) ?? []
      if (!spans.length && exportDropped(t - (obj.ms ?? 0), t, life)) continue
      if (spans.length !== 1 || spans[0].parentSpanId) await broke('spans', { line: obj }, 'one SERVER span, the root of its trace', spans.length ? spans : 'none')
    }
  }
  // Every SERVER span is of a request with a line.
  const logged = new Set(lives.flatMap((l) => l.lines.filter((x) => x.obj?.msg === 'request').map((x) => x.obj.trace_id)))
  for (const [traceId, spans] of byTrace) if (!logged.has(traceId)) await broke('spans', { traceId }, 'a SERVER span only of a request with a log line', spans[0])
  // While the receiver hung, no request waited on it.
  if (outageAt.hang !== null) {
    const times = sent.filter((r) => r.started >= outageAt.hang && r.started < outageAt.down && r.ended !== null).map((r) => r.ended - r.started).sort((x, y) => x - y)
    const median = times[times.length >> 1]
    if (times.length >= 3 && median >= 1000) await broke('requests while the OTLP receiver hung', { requests: times.length, OTEL_BSP_EXPORT_TIMEOUT: OTEL_ENV.OTEL_BSP_EXPORT_TIMEOUT }, 'no request waiting on an export: a median time well under the export timeout', `${median.toFixed(0)} ms`)
    count('requests while the OTLP receiver was out', true)
  }
  count('spans of the harness\'s requests', checked > 0)
}

/// After an outage, a `trace export failed` line, once an export could
/// have failed: by the export timeout after the receiver stopped answering
/// (spec/OPERATIONS.md, "Events", "Export").
async function outageLogged() {
  if (outageAt.hang === null) return
  const by = outageAt.hang + Number(OTEL_ENV.OTEL_BSP_EXPORT_TIMEOUT) + Number(OTEL_ENV.OTEL_BSP_SCHEDULE_DELAY) + 2000
  for (;;) {
    const failed = readLog().flatMap((l) => l.lines).filter(({ obj }) => obj?.msg === 'trace export failed' && Date.parse(obj.time) >= outageAt.hang)
    if (failed.length) return count('trace export failed, logged', true)
    if (now() > by) await broke('log', { outage: 'the OTLP receiver hung, then went, then came back' }, 'a `trace export failed` line', 'none')
    await wait(250)
  }
}

async function checkServerSpan(sp, rec, request, life) {
  const route = httpRoute(rec.route)
  const want = {
    name: route ? `${rec.method} ${route}` : rec.method,
    kind: 'SERVER',
    parentSpanId: rec.spanId,
    status: rec.status >= 500 ? 'ERROR' : 'UNSET',
    'http.request.method': rec.method,
    'url.path': rec.path,
    'url.scheme': 'http',
    'http.route': route ?? undefined,
    'http.response.status_code': BigInt(rec.status),
    'error.type': rec.status >= 500 ? String(rec.status) : undefined,
    'service.name': life.env.OTEL_SERVICE_NAME || 'unknown_service:pds',
    'service.version': version,
    scope: 'pds',
  }
  const got = {
    name: sp.name, kind: sp.kind, parentSpanId: sp.parentSpanId, status: sp.status.code,
    ...Object.fromEntries(['http.request.method', 'url.path', 'url.scheme', 'http.route', 'http.response.status_code', 'error.type'].map((k) => [k, sp.attributes[k]])),
    'service.name': sp.resource['service.name'], 'service.version': sp.resource['service.version'], scope: sp.scope,
  }
  for (const k of Object.keys(want)) if (want[k] !== got[k]) await broke('spans', request, Object.fromEntries(Object.entries(want).map(([x, v]) => [x, typeof v === 'bigint' ? Number(v) : v])), Object.fromEntries(Object.entries(got).map(([x, v]) => [x, typeof v === 'bigint' ? Number(v) : v])))
  if (!(sp.start >= rec.started - 1000 && sp.end >= sp.start && sp.end <= rec.ended + 1000)) await broke('spans', request, `a span within the request, ${new Date(rec.started).toISOString()} to ${new Date(rec.ended).toISOString()}`, { start: new Date(sp.start).toISOString(), end: new Date(sp.end).toISOString() })
}

/// A PLC directory that fails every call, and keeps each call's
/// `traceparent`.
async function failingPlc() {
  const calls = []
  const s = createServer((req, res) => {
    calls.push({ method: req.method, path: req.url, traceparent: req.headers.traceparent ?? null })
    req.resume()
    req.on('end', () => {
      res.writeHead(500, { 'content-type': 'application/json' })
      res.end('{"message":"a directory that fails"}')
    })
  })
  await new Promise((ok) => s.listen(0, '127.0.0.1', ok))
  return { url: `http://127.0.0.1:${s.address().port}`, calls, close: () => s.close() }
}

/// Start the server again with `changes` to its environment (undefined
/// takes a variable out) and wait until it answers.
async function restart(changes) {
  for (const [k, v] of Object.entries(changes)) {
    if (v === undefined) delete server.env[k]
    else server.env[k] = v
  }
  server.start()
  await waitReady(server, cand.url)
}

/// The machine's first IPv4 address on an interface other than the
/// loopback, or null; and whether it has IPv6's loopback.
function addresses() {
  const all = Object.values(networkInterfaces()).flat()
  return {
    outside: all.find((a) => a.family === 'IPv4' && !a.internal)?.address ?? null,
    v6Loopback: all.some((a) => a.family === 'IPv6' && a.internal && a.address === '::1'),
  }
}

/// Whether the server answers `_health` at `host` on its port, within two
/// seconds.
const answers = (host) =>
  fetch(`http://${host}:${server.env.PDS_PORT}${HEALTH}`, { signal: AbortSignal.timeout(2000) }).then(
    (r) => r.arrayBuffer().then(() => r.ok),
    () => false,
  )

/// The server started with `PDS_BIND_ADDRESS` `address` (spec/PDS.md, "What
/// it serves"), answering on each host of `on` once it answers on the first,
/// on none of `off`, then shut down.
async function bound(keep, address, on, off) {
  Object.assign(server.env, keep, { PDS_BIND_ADDRESS: address })
  server.start()
  let up = false
  for (let i = 0; i < 300 && !server.exited && !(up = await answers(on[0])); i++) await wait(100)
  const at = { PDS_BIND_ADDRESS: address }
  if (!up) await broke('listener', { ...at, host: on[0] }, 'an answer to _health', server.exited ?? 'none within 30 s')
  for (const host of on.slice(1)) if (!(await answers(host))) await broke('listener', { ...at, host }, 'an answer to _health', 'none')
  for (const host of off) if (await answers(host)) await broke('listener', { ...at, host }, 'no answer: it listens on that address alone', 'an answer')
  const stopped = await server.stop('SIGTERM', 15000)
  if (!stopped || stopped.code !== 0) await broke('shutdown', { signal: 'SIGTERM', ...at }, 'exit 0 within 15 s', stopped ?? 'still running')
  count(`listening on ${address}`, true)
}

/// A shutdown, then the server at `warn` with a failing PLC directory and
/// the default service name; killed, then started with traces off; at
/// each, the log it keeps within its bounds; then with no `PDS_LOG_DIR`;
/// then on other addresses; then starts with variables that do not parse
/// (spec/OPERATIONS.md, "Environment", "Events", "Kept", "_logs", "Traces",
/// "Spans").
async function lifecycle() {
  const keep = { ...server.env }
  const { outside, v6Loopback } = addresses()
  // As the run started it, with no PDS_BIND_ADDRESS: on the loopback alone.
  if (outside && (await answers(outside))) await broke('listener', { PDS_BIND_ADDRESS: 'unset', host: outside }, 'no answer: it listens on 127.0.0.1 alone', 'an answer')
  const ended = await server.stop('SIGTERM', 15000)
  if (!ended || ended.code !== 0) await broke('shutdown', { signal: 'SIGTERM' }, 'exit 0 within 15 s', ended ?? 'still running')
  await checkSpans(now())
  count('shutdown on SIGTERM', true)

  const plc = await failingPlc()
  await restart({ PDS_LOG_LEVEL: 'warn', PDS_PLC_URL: plc.url, OTEL_SERVICE_NAME: undefined })
  const t = hash('ops plc')
  const traceparent = `00-${t.slice(0, 32)}-${t.slice(32, 48)}-01`
  const front = `opsplc${SEED}`
  // With codes required, a code first, which asks nothing of the directory.
  const admin = `Basic ${Buffer.from(`admin:${server.env.PDS_ADMIN_PASSWORD}`).toString('base64')}`
  const inviteCode = server.env.PDS_INVITE_REQUIRED === 'true' ? (await fetch(`${cand.url}/xrpc/com.atproto.server.createInviteCode`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: admin }, body: '{"useCount":1}' }).then((r) => r.json())).code : undefined
  const before = statSync(server.logPath).size
  const res = await fetch(`${cand.url}/xrpc/com.atproto.server.createAccount`, { method: 'POST', headers: { 'content-type': 'application/json', traceparent, 'x-forwarded-for': '198.51.100.11' }, body: JSON.stringify({ handle: `${front}.test`, email: `${front}@example.com`, password: `pw-${t.slice(0, 16)}`, ...(inviteCode ? { inviteCode } : {}) }) })
  await res.arrayBuffer()
  if (res.status === 200) await broke('createAccount', { plc: 'a directory answering 500 to every call' }, 'refused', 200)
  const posts = plc.calls.filter((c) => c.method === 'POST')
  if (!posts.length) await broke('createAccount', { plc: 'a directory answering 500' }, 'a POST to the directory', plc.calls)
  const propagated = /^00-([0-9a-f]{32})-([0-9a-f]{16})-01$/.exec(posts[0].traceparent ?? '')
  if (!propagated || propagated[1] !== t.slice(0, 32)) await broke('call to the PLC directory', { traceparent }, 'a traceparent in the request\'s trace, sampled', posts[0].traceparent)
  await wait(1000)
  const lines = readFileSync(server.logPath).subarray(before).toString('utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l))
  const failure = lines.find((l) => l.msg === 'plc failure' && l.status === 500)
  if (!failure || typeof failure.did !== 'string') await broke('log', { plc: 'a directory answering 500' }, 'a `plc failure` line, of status 500, with the DID', lines)
  for (let i = 0; i < 30 && !otlp.spans.has(propagated[2]); i++) await wait(100)
  const call = otlp.spans.get(propagated[2])
  const parent = [...otlp.spans.values()].find((sp) => sp.kind === 'SERVER' && sp.traceId === t.slice(0, 32))
  if (!call || call.kind !== 'CLIENT' || call.traceId !== t.slice(0, 32) || !parent || call.parentSpanId !== parent.spanId) await broke('spans', { traceparent, plc: 'a directory answering 500' }, 'the CLIENT span the directory\'s traceparent names, under the request\'s', call ?? 'none')
  if (call.attributes['http.response.status_code'] !== 500n || call.attributes['error.type'] !== '500' || call.status.code !== 'ERROR') await broke('spans', { plc: 'a directory answering 500' }, { 'http.response.status_code': 500, 'error.type': '500', status: 'ERROR' }, { ...call.attributes, 'http.response.status_code': Number(call.attributes['http.response.status_code']), status: call.status.code })
  if (call.resource['service.name'] !== 'unknown_service:pds') await broke('spans', { OTEL_SERVICE_NAME: 'unset' }, { 'service.name': 'unknown_service:pds' }, call.resource)
  const failures = valuesOf(await scrape('with a failing directory'), 'pds_outbound_requests_total', ['target', 'outcome']).get('plc error') ?? 0
  if (failures < 1) await broke('metrics', { metric: 'pds_outbound_requests_total{target="plc",outcome="error"}' }, 'at least 1', failures)
  count('a PLC directory that fails', true)
  if (keeps()) await keptHolds('at warn')
  await server.kill()
  plc.close()

  Object.assign(server.env, keep)
  await restart({ OTEL_EXPORTER_OTLP_ENDPOINT: undefined, OTEL_EXPORTER_OTLP_HEADERS: undefined })
  const quiet = hash('ops quiet')
  await fetch(`${cand.url}${HEALTH}`, { headers: { traceparent: `00-${quiet.slice(0, 32)}-${quiet.slice(32, 48)}-01` } }).then((r) => r.arrayBuffer())
  await wait(1500)
  if ([...otlp.spans.values()].some((sp) => sp.traceId === quiet.slice(0, 32))) await broke('spans', { OTEL_EXPORTER_OTLP_ENDPOINT: 'unset' }, 'no spans exported', 'a span')
  count('traces off', true)
  if (keeps()) await keptHolds('after a kill and a start')
  const stopped = await server.stop('SIGTERM', 15000)
  if (!stopped || stopped.code !== 0) await broke('shutdown', { signal: 'SIGTERM', traces: 'off' }, 'exit 0 within 15 s', stopped ?? 'still running')

  // Started with no PDS_LOG_DIR (spec/OPERATIONS.md, "_logs").
  if (can('logDir')) {
    await restart({ PDS_LOG_DIR: undefined })
    await notKept('started with PDS_LOG_DIR unset')
    count('a start with no PDS_LOG_DIR', true)
    const unkept = await server.stop('SIGTERM', 15000)
    if (!unkept || unkept.code !== 0) await broke('shutdown', { signal: 'SIGTERM', PDS_LOG_DIR: 'unset' }, 'exit 0 within 15 s', unkept ?? 'still running')
  }

  if (outside) await bound(keep, '0.0.0.0', [outside, '127.0.0.1'], [])
  if (v6Loopback) await bound(keep, '::1', ['[::1]'], ['127.0.0.1'])

  // 203.0.113.1, of TEST-NET-3, is no interface's: a start that cannot listen
  // (spec/PDS.md, "What it serves").
  // A lexicon authority that is not a DID is refused at start too
  // (spec/OAUTH-PERMISSION-SETS.md, "Environment").
  // The blobstore's: each wrong with blobs in a bucket or not; and the other
  // place than the data directory's first start chose (spec/BLOBS.md).
  const S3_VARS = ['BUCKET', 'REGION', 'ENDPOINT', 'FORCE_PATH_STYLE', 'ACCESS_KEY_ID', 'SECRET_ACCESS_KEY', 'UPLOAD_TIMEOUT_MS', 'REQUEST_TIMEOUT_MS'].map((v) => `PDS_BLOBSTORE_S3_${v}`)
  const elsewhere = keep.PDS_BLOBSTORE_S3_BUCKET
    ? Object.fromEntries(S3_VARS.map((v) => [v, '']))
    : { PDS_BLOBSTORE_S3_BUCKET: 'elsewhere', PDS_BLOBSTORE_S3_ENDPOINT: 'http://127.0.0.1:9', PDS_BLOBSTORE_S3_FORCE_PATH_STYLE: 'true', PDS_BLOBSTORE_S3_ACCESS_KEY_ID: 'k', PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY: 's' }
  const blobstore = [
    { PDS_BLOBSTORE_S3_FORCE_PATH_STYLE: 'yes' },
    { PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS: 'soon' },
    { PDS_BLOBSTORE_S3_ENDPOINT: 'ftp://store.example' },
    { PDS_BLOBSTORE_S3_BUCKET: keep.PDS_BLOBSTORE_S3_BUCKET ?? 'b', PDS_BLOBSTORE_S3_ACCESS_KEY_ID: 'k', PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY: '' },
    elsewhere,
  ]
  // One of the two SMTP variables without the other, the other unset: exit
  // 1, with `pds: ` and why on standard error (spec/EMAIL.md, "SMTP").
  const smtpHalves = [{ PDS_EMAIL_SMTP_URL: 'smtp://127.0.0.1:2525', PDS_EMAIL_FROM: undefined }, { PDS_EMAIL_FROM: 'mailer@pds-harness.test', PDS_EMAIL_SMTP_URL: undefined }]
  // A keep below 65536 or not a number, and a PDS_LOG_DIR whose parent is
  // a regular file, which cannot be made (spec/OPERATIONS.md, "Environment").
  const aFile = join(mkdtempSync(join(tmpdir(), 'pds-ops-')), 'a-file')
  writeFileSync(aFile, 'not a directory\n')
  const logStarts = can('logDir') ? [{ PDS_LOG_KEEP_BYTES: '65535' }, { PDS_LOG_KEEP_BYTES: 'many' }, { PDS_LOG_DIR: join(aFile, 'log') }] : []
  for (const bad of [{ PDS_LOG_LEVEL: 'loud' }, { OTEL_EXPORTER_OTLP_PROTOCOL: 'grpc' }, { OTEL_EXPORTER_OTLP_PROTOCOL: 'http/json' }, { PDS_METRICS_PORT: 'many' }, { PDS_METRICS_PORT: keep.PDS_PORT }, ...logStarts, { PDS_BIND_ADDRESS: 'localhost' }, { PDS_BIND_ADDRESS: '[::1]' }, { PDS_BIND_ADDRESS: '203.0.113.1' }, { PDS_LEXICON_AUTHORITY_DID: 'not-a-did' }, ...blobstore, ...smtpHalves]) {
    // Each start with this one bad value alone, not the last one's too.
    for (const k of Object.keys(server.env)) delete server.env[k]
    Object.assign(server.env, keep, bad)
    server.start()
    for (let i = 0; i < 100 && !server.exited; i++) await wait(100)
    const exited = server.exited
    if (!exited) await server.kill()
    if (exited?.code !== 1) await broke('start', bad, 'exit 1, with why on standard error', exited ?? 'it served')
    const life = readLog().at(-1)
    if (!life.stderr.trim()) await broke('start', bad, 'why it exited, on standard error', 'nothing')
    if (smtpHalves.includes(bad) && !/^pds: \S/m.test(life.stderr)) await broke('start', { ...bad, unset: Object.keys(bad)[1] }, '`pds: ` and why, on standard error (spec/EMAIL.md, "SMTP")', life.stderr.slice(0, 300))
    const [name, value] = Object.entries(bad)[0]
    count(`a start with ${name} ${name === 'PDS_LOG_DIR' ? 'under a file' : value}`, false)
  }
  for (const k of Object.keys(server.env)) delete server.env[k]
  Object.assign(server.env, keep)
}

/// Everything at the end of a run: the server is stopped after it.
export async function checkOperations() {
  if (ops('health')) await checkHealth()
  // The version the spans name, when the health checks did not read it.
  else version = await fetch(`${cand.url}${HEALTH}`).then((r) => r.json()).then((b) => b?.version ?? null).catch(() => null)
  if (ops('root')) await checkRoot()
  if (ops('routes')) await unknownRoutes()
  if (ops('traceContext')) await traceContext()
  if (ops('metrics')) {
    const before = await scrape('before the canary record')
    const last = await canary()
    await checkMetrics(before, last)
  }
  if (ops('admin')) await checkAdmin()
  if (ops('log') || ops('lifecycle')) await outageLogged()
  if (ops('lifecycle')) await lifecycle()
  else if (ops('metrics') || ops('traceContext') || ops('admin')) await checkSpans(now())
  if (ops('log')) await checkLog()
}

/// The checks at the end of a run, by name: with `--only` naming some of
/// them, only those.
const OPS = ['health', 'root', 'routes', 'traceContext', 'metrics', 'admin', 'lifecycle', 'log']
const ops = (name) => !ONLY || !OPS.some((n) => ONLY.has(n)) || ONLY.has(name)