~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
685 lines · 34.6 KB Raw History
// Blobs: uploadBlob, getBlob and listBlobs, and blob refs in records
// (spec/BLOBS.md). Records with blob refs go in a collection of their own,
// which only these steps write, or are records of known types the
// validation steps write (checks/validation.mjs), which note their refs
// here; so the harness knows which blobs each account holds, which records
// reference them, and the type each was stored with: what it needs to send
// at most one ref that may be wrong in a write, and to hold the reference's
// listBlobs to it, which checks the check.

import { isDeepStrictEqual } from 'node:util'
import { fileTypeFromBuffer } from 'file-type'
import { CID } from 'multiformats/cid'
import { sha256 } from 'multiformats/hashes/sha2'
import { COMMANDS } from '../lib/args.mjs'
import { RKEYS, below, chance, pick, rand } from '../lib/random.mjs'
import { cand, ref } from '../lib/processes.mjs'
import { count, differ } from '../lib/report.mjs'
import { own } from '../lib/server.mjs'
import { accounts, cids, didOf, exists, nextBatch, noteCommit, noteWrite, tokenOf } from '../lib/state.mjs'
import { runCli, viaXrpc } from '../lib/via.mjs'
import { both, normalize, shown, xrpc } from '../lib/xrpc.mjs'

/// The collection the records with blob refs are in.
export const BLOB_COLLECTION = 'com.example.blob'
const C = BLOB_COLLECTION
const UPLOAD = 'com.atproto.repo.uploadBlob'
const GET = 'com.atproto.sync.getBlob'
const LIST = 'com.atproto.sync.listBlobs'
const LIMIT = 5242880
/// A JWT no server here gave out.
const FORGED = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln'
/// A DID no server here holds.
const NOBODY = 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa'
const RAW = 0x55

/// What the harness knows of an account's blobs: every upload both sides
/// answered, `{cid, bytes, type}`; the blobs it holds, by CID, with the
/// `mimeType` and `size` each was stored with; the CIDs each record of the
/// collection references, by key; each side's rev of every commit the
/// collection's writes made, for `listBlobs since`; the unreferenced
/// blobs a write refused for a ref named, which the reference may serve
/// from then until a record references them (spec/BLOBS.md); the blobs
/// held that are taken down (checks/store.mjs), by CID, with the
/// takedown's reference, each gone with its blob once the account stops
/// holding it. A ref to one of those is refused, `BlobNotFound`
/// (spec/LIFECYCLE.md), so it is a write's one ref that may be wrong; and
/// the blobs it held and holds no more, until it holds them again, whose
/// objects a bucket must not keep (checks/blobstore.mjs).
export function model(acct) {
  acct.blobs ??= { uploads: [], stored: new Map(), refs: new Map(), revs: [], raced: new Set(), takedowns: new Map(), dropped: new Set() }
  return acct.blobs
}

/// The account holds `cid` again, stored as `blob` answered, unless it held it already.
export function hold(m, cid, blob) {
  if (!m.stored.has(cid)) m.stored.set(cid, { mimeType: blob.mimeType, size: blob.size })
  m.dropped.delete(cid)
}

export const referenced = (m, cid) => [...m.refs.values()].some((s) => s.has(cid))
export const referencedCids = (m) => [...new Set([...m.refs.values()].flatMap((s) => [...s]))].sort()

export const rawCid = async (bytes) => CID.create(1, RAW, await sha256.digest(bytes)).toString()

/// Whether `v` is a blob ref, in its JSON form (spec/BLOBS.md, "Blob refs in
/// records").
function isBlobRef(v) {
  if (!v || typeof v !== 'object' || Array.isArray(v)) return false
  const keys = Object.keys(v).sort()
  if (!isDeepStrictEqual(keys, ['$type', 'mimeType', 'ref', 'size'])) return false
  return v.$type === 'blob' && typeof v.ref?.$link === 'string' && Object.keys(v.ref).length === 1 &&
    typeof v.mimeType === 'string' && v.mimeType.includes('/') && Number.isSafeInteger(v.size) && v.size >= -1
}

/// The CIDs the blob refs in a record name.
function refsOf(value) {
  const out = new Set()
  const walk = (v) => {
    if (isBlobRef(v)) return out.add(v.ref.$link)
    if (Array.isArray(v)) v.forEach(walk)
    else if (v && typeof v === 'object') Object.values(v).forEach(walk)
  }
  walk(value)
  return out
}

/// After a write landed: the records it put or took away, each
/// `{rkey, value}` with a null value for a delete. A blob the last record
/// referencing it no longer references is no longer held, nor taken down.
function applyRefs(m, changes) {
  const before = new Set()
  for (const { rkey } of changes) for (const c of m.refs.get(rkey) ?? []) before.add(c)
  for (const { rkey, value } of changes) value ? m.refs.set(rkey, refsOf(value)) : m.refs.delete(rkey)
  for (const c of before) {
    if (referenced(m, c)) continue
    m.stored.delete(c)
    m.dropped.add(c)
    m.takedowns.delete(c)
  }
  for (const c of [...m.raced]) if (referenced(m, c)) m.raced.delete(c)
}

const REF_ERRORS = ['BlobNotFound', 'InvalidMimeType', 'InvalidSize']

/// After a write refused for a ref, answered `a`: each blob the account
/// holds, unreferenced, that one of its records' refs named.
function noteRaced(m, a, values) {
  if (!REF_ERRORS.includes(a.body?.error)) return
  for (const v of values) for (const c of refsOf(v)) if (m.stored.has(c) && !referenced(m, c)) m.raced.add(c)
}

/// A ref the write cannot refuse: to a blob the account holds, not taken
/// down, that a record references already, or that it matches.
function sureGood(m, r) {
  if (!isBlobRef(r)) return true
  const s = m.stored.get(r.ref.$link)
  if (!s || m.takedowns.has(r.ref.$link)) return false
  return referenced(m, r.ref.$link) || (r.mimeType === s.mimeType && r.size === s.size)
}

/// The blobs the account holds that a ref to, as stored, is sure to be
/// taken: all but those taken down.
const takable = (m) => [...m.stored].filter(([cid]) => !m.takedowns.has(cid))

/// For the validation steps: a ref to a blob the account holds, not taken
/// down, as it was stored, of a type `accept` takes (any, without `accept`)
/// and at most `maxSize` bytes; or null if it holds none.
export function heldRef(acct, accept, maxSize) {
  const takes = (t) => !accept || accept.some((a) => (a.endsWith('/*') ? t.startsWith(a.slice(0, -1)) : t === a))
  const fit = takable(model(acct)).filter(([, s]) => takes(s.mimeType) && (maxSize === undefined || s.size <= maxSize))
  if (!fit.length) return null
  const [cid, s] = pick(fit)
  return { $type: 'blob', ref: { $link: cid }, mimeType: s.mimeType, size: s.size }
}

/// Note a write of the validation steps, answered `a` by the reference:
/// landed, the records it changed, `[{rkey, value}]`, each `rkey` as
/// `collection/rkey` and `value` null for a delete; refused, the values it
/// sent.
export function noteRefs(acct, a, changes, values) {
  const m = model(acct)
  if (a.status < 400) applyRefs(m, changes)
  else noteRaced(m, a, values)
}

// ---- Bodies --------------------------------------------------------------

const PNG = Buffer.from('89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d49444154789c6360000002000154a24f5d0000000049454e44ae426082', 'hex')
/// The bodies whose first bytes show a type, each a function of a tail
/// that makes it the run's own, with the type the reference sees in it.
const SNIFFED = [
  [(t) => Buffer.concat([PNG, t]), 'image/png'],
  [(t) => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), t]), 'image/jpeg'],
  [(t) => Buffer.concat([Buffer.from('GIF89a\x01\x00\x01\x00', 'latin1'), t]), 'image/gif'],
  [(t) => Buffer.concat([Buffer.from('RIFF\x10\x00\x00\x00WEBPVP8 ', 'latin1'), t]), 'image/webp'],
  [(t) => Buffer.concat([Buffer.from('%PDF-1.4\n', 'latin1'), t]), 'application/pdf'],
  [(t) => Buffer.concat([Buffer.from('\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2', 'latin1'), t]), 'video/mp4'],
]
const TYPES = ['text/plain', 'image/png', 'image/jpeg', 'application/octet-stream', 'TEXT/Plain', 'image/PNG', 'text/plain; charset=utf-8', 'text/plain ;q=1', 'a/b/c', '*/*', 'video/mp4', 'x/y;', 'application/json']

let serial = 0
const randomBytes = (n) => Buffer.from(Array.from({ length: n }, () => below(256)))
/// A body that begins `spec-pds blob `, the run's own.
export const plainBody = (what = '') => Buffer.concat([Buffer.from(`spec-pds blob ${what}${++serial} `), randomBytes(below(300))])
/// A body of `n` bytes that begins `spec-pds blob `.
function bigBody(n) {
  const b = Buffer.alloc(n, 0x61)
  b.write(`spec-pds blob big ${++serial} `)
  return b
}

/// The type the reference's sniffing gives `bytes`, which must be what the
/// harness meant it to be: the check of the table spec/BLOBS.md gives.
async function sniffs(bytes, want) {
  const got = (await fileTypeFromBuffer(bytes))?.mime
  if (got !== want) throw new Error(`the harness's body sniffs as ${got}, not ${want}: ${bytes.subarray(0, 24).toString('hex')}`)
}

// ---- Requests ------------------------------------------------------------

/// uploadBlob on one side: `bytes` with `type` as Content-Type (none if
/// undefined), `token` an account or a string, sent chunked if asked. With
/// --commands, some of the server's go through `pds repo upload-blob`
/// (spec/COMMANDS.md, "Answers"), or through `pds xrpc` with the bytes as
/// `-` and a `Content-Type:` header (spec/COMMANDS.md, "Any call").
export async function uploadOn(side, { bytes, type, token, chunked }) {
  const t = token && typeof token === 'object' ? tokenOf(side, token) : token
  if (COMMANDS && side === cand && type && !chunked && token && typeof token === 'object' && bytes.length < 100_000 && chance(0.3)) return chance(0.5) ? uploadByCommand(type, t, bytes) : viaXrpc(UPLOAD, { bytes, contentType: type, token: t })
  const headers = {}
  if (type !== undefined) headers['content-type'] = type
  if (t) headers.authorization = `Bearer ${t}`
  const body = chunked ? new ReadableStream({ start: (c) => { c.enqueue(bytes); c.close() } }) : bytes
  try {
    const r = await fetch(`${side.url}/xrpc/${UPLOAD}`, { method: 'POST', headers, body, duplex: 'half' })
    const text = await r.text()
    let parsed = null
    try {
      parsed = text ? JSON.parse(text) : null
    } catch {
      parsed = { unparsed: text.slice(0, 200) }
    }
    return { status: r.status, body: parsed }
  } catch (err) {
    return { status: 0, body: { error: `request failed: ${err.message}` } }
  }
}

async function uploadByCommand(type, token, bytes) {
  count('via command repo upload-blob', true)
  const argv = ['repo', 'upload-blob', '--encoding', type]
  const r = await runCli(argv, { token, stdin: bytes })
  const request = { argv: ['pds', ...argv], stdin: `${bytes.length} bytes` }
  if (r.code === 3) return { status: 0, body: { error: 'unreachable' } }
  if (r.code !== 0 && r.code !== 1) await differ('command pds repo upload-blob', request, 'exit status 0 or 1', { code: r.code, stderr: r.stderr.slice(0, 500) })
  const lines = r.stdout.toString('utf8').split('\n').filter(Boolean)
  let parsed
  try {
    if (lines.length !== 1) throw new Error()
    parsed = JSON.parse(lines[0])
  } catch {
    await differ('command pds repo upload-blob', request, 'one line of JSON on standard output', r.stdout.toString('utf8').slice(0, 500))
  }
  if (r.code === 0) return { status: 200, body: parsed }
  const { status, error, message } = parsed
  return { status, body: { error, message } }
}

/// getBlob on one side, asking for no compression: its status, the headers
/// spec/BLOBS.md names, and the bytes, or the error (spec/BLOBS.md,
/// "getBlob").
async function getBlobOn(side, params, token) {
  const t = token && typeof token === 'object' ? tokenOf(side, token) : token
  const url = new URL(`${side.url}/xrpc/${GET}`)
  for (const [k, v] of Object.entries(params)) if (v !== undefined) url.searchParams.set(k, String(v))
  const headers = { 'accept-encoding': 'identity' }
  if (t) headers.authorization = `Bearer ${t}`
  try {
    const r = await fetch(url, { headers })
    const bytes = Buffer.from(await r.arrayBuffer())
    if (r.status !== 200) {
      let body = null
      try {
        body = JSON.parse(bytes.toString('utf8'))
      } catch {}
      return { status: r.status, error: body?.error ?? null }
    }
    const h = r.headers
    return {
      status: 200,
      headers: {
        'content-type': (h.get('content-type') ?? '').split(';')[0].trim(),
        'content-length': h.get('content-length'),
        'x-content-type-options': h.get('x-content-type-options'),
        'content-disposition': h.get('content-disposition'),
        'content-security-policy': h.get('content-security-policy'),
      },
      bytes,
    }
  } catch (err) {
    return { status: 0, error: `request failed: ${err.message}` }
  }
}

/// getBlob on both, compared: the status and error, or the headers and
/// the bytes. With --commands, sometimes also the server's through
/// `pds sync get-blob`, whose bytes must be the same. A blob a refused
/// write raced on (`raced`), which the reference may or may not serve, is
/// held to spec/BLOBS.md instead: the server must not serve it.
export async function compareGetBlob(kind, params, token, raced = false) {
  const on = (side) => (typeof params === 'function' ? params(side) : params)
  const [a, b] = await Promise.all([getBlobOn(ref, on(ref), token), getBlobOn(cand, on(cand), token)])
  const request = shown({ params: on(ref), token })
  if (raced && a.status === 200) {
    if (b.status !== 400 || b.error !== 'InvalidRequest') await differ(GET, { ...request, blob: 'unreferenced, named by a write refused for a ref' }, { status: 400, error: 'InvalidRequest' }, b.status === 200 ? { status: 200, headers: b.headers } : b, 'a refused write changes nothing (the reference may serve it, by a race of its own)')
    return count(`${kind}, of a blob a refused write named`, false)
  }
  if (a.status !== b.status || (a.status !== 200 && a.error !== b.error)) await differ(GET, request, a.status === 200 ? { status: 200, headers: a.headers } : a, b.status === 200 ? { status: 200, headers: b.headers } : b)
  if (a.status === 200) {
    if (!isDeepStrictEqual(a.headers, b.headers)) await differ(`${GET} headers`, request, a.headers, b.headers)
    if (!a.bytes.equals(b.bytes)) await differ(`${GET} bytes`, request, `${a.bytes.length} bytes: ${a.bytes.subarray(0, 40).toString('hex')}`, `${b.bytes.length} bytes: ${b.bytes.subarray(0, 40).toString('hex')}`)
    if (COMMANDS && token === undefined && chance(0.3)) {
      const c = await xrpc(cand, GET, { params: on(cand), raw: true })
      if (c.status !== 200 || !c.bytes || !Buffer.from(c.bytes).equals(a.bytes)) await differ('command pds sync get-blob', request, `${a.bytes.length} bytes`, c.bytes ? `${c.bytes.length} bytes` : c)
    }
  }
  count(kind, a.status === 200)
  return a
}

/// `xs` in a random order.
function shuffled(xs) {
  const out = [...xs]
  for (let i = out.length - 1; i > 0; i--) {
    const j = below(i + 1)
    ;[out[i], out[j]] = [out[j], out[i]]
  }
  return out
}

// ---- Steps ---------------------------------------------------------------

/// uploadBlob of `bytes` on both sides, compared whole; the reference's
/// `ref` and `size` held to the body. Notes an upload that landed, and
/// answers whether it did; `keep` notes its bytes, to send again.
export async function sendUpload(acct, m, { bytes, type, token, chunked = false, sniffed, keep = true }) {
  const request = { bytes: `${bytes.length} bytes${sniffed ? `, a ${sniffed} body` : ''}`, type, token, chunked }
  const [a, b] = await Promise.all([ref, cand].map((side) => uploadOn(side, { bytes, type, token, chunked })))
  const [na, nb] = [normalize(ref, a, UPLOAD), normalize(cand, b, UPLOAD)]
  if (!isDeepStrictEqual(na, nb)) await differ(UPLOAD, shown(request), na, nb)
  count('uploadBlob', a.status === 200)
  if (a.status !== 200) return false
  const blob = a.body.blob
  const want = await rawCid(bytes)
  if (blob.ref.$link !== want || blob.size !== bytes.length) throw new Error(`the reference's uploadBlob answers ${JSON.stringify(blob)} for a body whose raw CID is ${want}, of ${bytes.length} bytes`)
  hold(m, want, blob)
  if (keep) m.uploads.push({ cid: want, bytes, type, answer: blob })
  return true
}

/// uploadBlob: bodies the run makes, some whose first bytes show a type,
/// some sent before (by this account with another type, or by another), empty
/// ones and rarely ones at the limit or past it; Content-Types of each kind
/// spec/BLOBS.md names, or none; the account's token, none, or a JWT never
/// given out. Answers compared whole; the reference's `ref` and `size` held
/// to the body (spec/BLOBS.md, "uploadBlob"). The limit's exact number is
/// Monolith's own: another implementation is sent no body at it
/// (spec/COMPAT.md).
export async function uploadBlob(acct) {
  const m = model(acct)
  const r = rand()
  let bytes, sniffed, big = false
  if (r < 0.45) {
    bytes = plainBody()
    await sniffs(bytes, undefined)
  } else if (r < 0.65) {
    const prior = accounts.flatMap((a) => a.blobs?.uploads ?? [])
    if (prior.length) bytes = pick(prior).bytes
    else bytes = plainBody()
  } else if (r < 0.88) {
    const [make, type] = pick(SNIFFED)
    bytes = make(Buffer.from(`spec-pds ${++serial}`))
    sniffed = type
    await sniffs(bytes, type)
  } else if (r < 0.97 || !own()) {
    bytes = Buffer.alloc(0)
  } else {
    bytes = bigBody(LIMIT + below(2))
    big = true
  }
  const t = rand()
  const type = t < 0.05 ? undefined : t < 0.07 ? '' : pick(TYPES)
  const k = rand()
  const token = k < 0.92 ? acct : k < 0.96 ? undefined : FORGED
  const chunked = chance(big ? 0.5 : 0.1)
  const ok = await sendUpload(acct, m, { bytes, type, token, chunked, sniffed, keep: !big })
  if (!ok) return
  if (COMMANDS && chance(0.05)) {
    const u = await runCli(['repo', 'upload-blob'], { token: tokenOf(cand, acct), stdin: Buffer.from('x') })
    if (u.code !== 2) await differ('pds repo upload-blob without --encoding', {}, 'a usage error: exit 2', { code: u.code, stdout: u.stdout.toString().slice(0, 200) })
  }
  if (own() && chance(0.05)) {
    const [da, db] = await Promise.all([ref, cand].map((side) => xrpc(side, 'com.atproto.server.describeServer')))
    if (da.body?.blobUploadLimit !== LIMIT) throw new Error(`the reference's blobUploadLimit is ${da.body?.blobUploadLimit}, not ${LIMIT}`)
    if (db.body?.blobUploadLimit !== LIMIT) await differ('describeServer blobUploadLimit', {}, LIMIT, db.body?.blobUploadLimit ?? null)
    count('describeServer blobUploadLimit', true)
  }
}

/// A raw CID no account here has uploaded.
const neverUploaded = () => rawCid(Buffer.from(`spec-pds never uploaded ${++serial}`))

/// A map shaped almost as a blob ref, which is none, so references nothing.
function notARef(m) {
  const base = m.stored.size ? pick([...m.stored]) : null
  const [cid, s] = base ?? ['bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku', { mimeType: 'text/plain', size: 0 }]
  const ok = { $type: 'blob', ref: { $link: cid }, mimeType: s.mimeType, size: s.size }
  const { $type, ref, mimeType, size } = ok
  return pick([
    { ...ok, alt: 'a fifth key' },
    { ...ok, size: pick([-2, '16', null]) },
    { ...ok, mimeType: pick(['textplain', '', 5]) },
    { ...ok, ref: pick([cid, { $link: cid, x: 1 }, { $bytes: 'AAAA' }, { a: 1 }]) },
    { ...ok, $type: 'Blob' },
    pick([{ ref, mimeType, size }, { $type, mimeType, size }, { $type, ref, size }, { $type, ref, mimeType }]),
    { cid: 'not a cid', mimeType: 'text/plain' },
  ])
}

/// A ref that may be wrong: to a blob taken down, as it was stored; as an
/// upload was answered (which may name another type than the blob was
/// stored with), of another type or size, to another account's blob, to a
/// CID never uploaded or a record's, or in the legacy shape.
async function mayBeWrong(acct, m) {
  if (m.takedowns.size && chance(0.3)) {
    const cid = pick([...m.takedowns.keys()])
    const s = m.stored.get(cid)
    return { $type: 'blob', ref: { $link: cid }, mimeType: s.mimeType, size: s.size }
  }
  const stored = [...m.stored]
  const r = rand()
  if (r < 0.25 && m.uploads.length) return pick(m.uploads).answer
  if (r < 0.45 && stored.length) {
    const [cid, s] = pick(stored)
    return { $type: 'blob', ref: { $link: cid }, mimeType: pick(['image/png', 'text/plain', s.mimeType.toUpperCase(), 'a/']), size: s.size }
  }
  if (r < 0.6 && stored.length) {
    const [cid, s] = pick(stored)
    return { $type: 'blob', ref: { $link: cid }, mimeType: s.mimeType, size: pick([s.size + 1, -1, 0, Math.max(0, s.size - 1)]) }
  }
  const others = accounts.filter((a) => a !== acct).flatMap((a) => a.blobs?.uploads ?? [])
  if (r < 0.72 && others.length) return pick(others).answer
  if (r < 0.82) return { $type: 'blob', ref: { $link: await neverUploaded() }, mimeType: 'text/plain', size: 10 }
  if (r < 0.9 && cids.length) return { $type: 'blob', ref: { $link: pick(cids) }, mimeType: 'text/plain', size: 10 }
  const cid = stored.length ? pick(stored)[0] : await neverUploaded()
  return { cid, mimeType: 'text/plain' }
}

/// A record of the collection: up to three blob refs, at different depths,
/// of which at most `budget.n` may be wrong, the rest sure to be taken or
/// no blob refs at all.
async function blobValue(acct, m, budget) {
  const refs = []
  for (let i = below(4); i > 0; i--) {
    if (budget.n > 0 && chance(0.45)) {
      const r = await mayBeWrong(acct, m)
      if (!sureGood(m, r) || r.cid !== undefined) budget.n -= 1
      refs.push(r)
    } else if (takable(m).length && chance(0.8)) {
      const [cid, s] = pick(takable(m))
      refs.push({ $type: 'blob', ref: { $link: cid }, mimeType: s.mimeType, size: s.size })
    } else refs.push(notARef(m))
  }
  const value = { $type: C, n: below(100), batch: nextBatch() }
  if (refs[0]) value.img = refs[0]
  if (refs[1]) value.more = [refs[1], ...(refs[2] ? [{ deep: refs[2] }] : [])]
  return value
}

/// Note a write of the collection that landed, answered `a` and `b`.
export function landed(acct, m, a, b, changes) {
  noteCommit(ref, acct, a)
  noteCommit(cand, acct, b)
  if (a.body?.commit?.rev && b.body?.commit?.rev) m.revs.push({ reference: a.body.commit.rev, server: b.body.commit.rev })
  applyRefs(m, changes)
}

/// Records with blob refs, by createRecord, putRecord, deleteRecord and
/// applyWrites, in the collection of their own; at most one ref in a write
/// may be wrong. Compared as any write; sometimes the record is read back
/// (spec/BLOBS.md, "Blob refs in records").
export async function blobRecord(acct) {
  const m = model(acct)
  const budget = { n: 1 }
  // An account that holds no blob, or sometimes one that does, uploads one
  // first, for the record to reference.
  if (!m.stored.size || chance(0.25)) {
    const [make, sniffed] = chance(0.5) ? pick(SNIFFED) : [() => plainBody(), undefined]
    const bytes = make(Buffer.from(`spec-pds ${++serial}`))
    await sniffs(bytes, sniffed)
    await sendUpload(acct, m, { bytes, type: pick(TYPES), token: acct, sniffed })
  }
  const r = rand()
  const fresh = RKEYS.filter((k) => !exists(acct, C, k))
  const held = RKEYS.filter((k) => exists(acct, C, k))
  let rkey
  if (r < 0.75) {
    const nsid = r < 0.4 ? 'com.atproto.repo.createRecord' : 'com.atproto.repo.putRecord'
    rkey = nsid.endsWith('createRecord') && fresh.length && chance(0.85) ? pick(fresh) : pick(RKEYS)
    const value = await blobValue(acct, m, budget)
    const body = { repo: acct.handle, collection: C, rkey, record: value }
    const [a, b] = await both(`${nsid.split('.').pop()} with blob refs`, nsid, { body, token: acct })
    noteWrite(acct, C, rkey, a)
    if (a.status < 400) landed(acct, m, a, b, [{ rkey, value }])
    else noteRaced(m, a, [value])
  } else if (r < 0.87) {
    rkey = held.length && chance(0.9) ? pick(held) : pick(RKEYS)
    const body = { repo: acct.handle, collection: C, rkey }
    const [a, b] = await both('deleteRecord of a record with blob refs', 'com.atproto.repo.deleteRecord', { body, token: acct })
    if (a.status < 400) {
      acct.records.delete(`${C}/${rkey}`)
      landed(acct, m, a, b, [{ rkey, value: null }])
    }
  } else {
    // Each key once: the reference's index miscounts the blobs of a batch
    // that writes one key twice (spec/BLOBS.md).
    const keys = shuffled(RKEYS).slice(0, 1 + below(3))
    const writes = []
    for (const k of keys) {
      const kind = exists(acct, C, k) ? pick(['update', 'delete']) : 'create'
      const w = { $type: `com.atproto.repo.applyWrites#${kind}`, collection: C, rkey: k }
      if (kind !== 'delete') w.value = await blobValue(acct, m, budget)
      writes.push(w)
    }
    const [a, b] = await both('applyWrites with blob refs', 'com.atproto.repo.applyWrites', { body: { repo: acct.handle, writes }, token: acct })
    if (a.status < 400) {
      writes.forEach((w, i) => {
        const key = `${C}/${w.rkey}`
        if (!w.value) acct.records.delete(key)
        else if (a.body.results?.[i]?.cid) {
          acct.records.set(key, a.body.results[i].cid)
          cids.push(a.body.results[i].cid)
        }
      })
      landed(acct, m, a, b, writes.map((w) => ({ rkey: w.rkey, value: w.value ?? null })))
    } else noteRaced(m, a, writes.map((w) => w.value).filter(Boolean))
    rkey = pick(keys)
  }
  if (chance(0.25)) await both('getRecord of a record with blob refs', 'com.atproto.repo.getRecord', { params: { repo: acct.handle, collection: C, rkey } })
}

/// getBlob of blobs records reference, of this account or, when it has
/// none, another; of the account's uploads, held or not; another account's
/// upload; a CID never uploaded; a record's; arguments that do not parse;
/// and a DID no one holds. With no token, the account's, another's, or one
/// never given out (spec/BLOBS.md, "getBlob").
export async function getBlob(acct) {
  const r = rand()
  const holders = accounts.filter((a) => a.blobs && referencedCids(a.blobs).length)
  if (r < 0.4 && holders.length && !(acct.blobs && referencedCids(acct.blobs).length)) acct = pick(holders)
  const m = model(acct)
  const others = accounts.filter((a) => a !== acct && a.blobs?.uploads.length)
  const known = referencedCids(m)
  let cid
  if (r < 0.4 && known.length) cid = pick(known)
  else if (r < 0.55 && (m.stored.size || m.uploads.length)) cid = m.stored.size && (!m.uploads.length || chance(0.5)) ? pick([...m.stored.keys()]) : pick(m.uploads).cid
  else if (r < 0.67 && others.length) cid = pick(pick(others).blobs.uploads).cid
  else if (r < 0.77) cid = await neverUploaded()
  else if (r < 0.84 && cids.length) cid = pick(cids)
  else cid = pick(['not a cid', await neverUploaded()])
  let did = (side) => didOf(side, acct)
  const d = rand()
  if (d < 0.05) did = () => NOBODY
  else if (d < 0.08) {
    const bad = pick([acct.handle, 'not a did'])
    did = () => bad
  }
  const t = rand()
  const token = t < 0.7 ? undefined : t < 0.8 ? acct : t < 0.9 ? pick(accounts) : FORGED
  await compareGetBlob('getBlob', (side) => ({ did: did(side), cid }), token, d >= 0.08 && m.raced.has(cid))
}

/// listBlobs of the account, paged to the end with random limits, from a
/// random cursor, since the revs of the collection's writes (each side its
/// own, from the same write), a TID before every rev or one after; and
/// arguments that do not parse. Compared whole, cursors too, which are
/// CIDs. Listed from the start without `since`, the reference must list
/// exactly the blobs the harness knows to be referenced (spec/BLOBS.md,
/// "listBlobs").
export async function listBlobs(acct) {
  const m = model(acct)
  const params = {}
  if (chance(0.6)) params.limit = 1 + below(4)
  const c = rand()
  const known = referencedCids(m)
  if (c < 0.15) params.cursor = pick(['b', '', 'bafkrei', 'zzz'])
  else if (c < 0.3 && known.length) params.cursor = pick(known)
  const s = rand()
  let since = null
  if (s < 0.25 && m.revs.length) since = pick(m.revs)
  else if (s < 0.32) since = { reference: '2222222222222', server: '2222222222222' }
  else if (s < 0.37) since = { reference: 'jzzzzzzzzzzzz', server: 'jzzzzzzzzzzzz' }
  let nobody = false
  const bad = rand()
  if (bad < 0.04) params.limit = pick([0, 1001])
  else if (bad < 0.07) since = { reference: 'notatid', server: 'notatid' }
  else if (bad < 0.1) nobody = true
  const on = (side, cursor) => ({ params: { did: nobody ? NOBODY : didOf(side, acct), ...params, ...(since ? { since: since[side.name] } : {}), ...(cursor !== undefined ? { cursor } : {}) } })
  const sinceShown = since && (m.revs.includes(since) ? 'the rev of one of the collection\'s writes' : since.reference)
  const listed = []
  let cursor = params.cursor
  let complete = false
  for (let page = 0; page < 40 && !complete; page++) {
    const [a, b] = await Promise.all([ref, cand].map((side) => xrpc(side, LIST, on(side, cursor))))
    const request = { ...shown(on(ref, cursor)), since: sinceShown ?? undefined, page }
    if (a.status !== 200 || b.status !== 200) {
      const [na, nb] = [normalize(ref, a, LIST), normalize(cand, b, LIST)]
      if (!isDeepStrictEqual(na, nb)) await differ(LIST, request, na, nb)
      return count('listBlobs', false)
    }
    if (!isDeepStrictEqual(a.body, b.body)) await differ(LIST, request, a.body, b.body)
    if (a.body.cursor !== a.body.cids.at(-1)) throw new Error(`the reference's listBlobs cursor is ${a.body.cursor}, not its page's last CID`)
    listed.push(...a.body.cids)
    count('listBlobs', true)
    complete = !a.body.cursor
    cursor = a.body.cursor
  }
  if (since || !complete) return
  const want = known.filter((x) => !params.cursor || x > params.cursor)
  if (!isDeepStrictEqual(listed, want)) throw new Error(`the reference's listBlobs of <did:${acct.n}> lists ${JSON.stringify(listed)}, where the harness knows ${JSON.stringify(want)} to be referenced`)
}

// ---- Crashes -------------------------------------------------------------

/// An upload on its way to the server when it is killed, and already
/// answered by the reference. Its body is the run's own and never sent
/// again, so whatever the server kept of it touches nothing else
/// (spec/BLOBS.md, "Durability"; spec/DURABILITY.md, "Blobs").
export function uploadInFlight(acct) {
  const bytes = plainBody('in flight ')
  const type = 'text/plain'
  const sent = Promise.all([ref, cand].map((side) => uploadOn(side, { bytes, type, token: acct })))
  return {
    /// Once the server is back: the body uploaded to it again, which must
    /// be answered as the reference answered it, then referenced by a
    /// record on both sides, and read back whole.
    async settle() {
      const [a] = await sent
      if (a.status !== 200) throw new Error(`the reference refused an upload: ${JSON.stringify(a.body)}`)
      const b = await uploadOn(cand, { bytes, type, token: acct })
      const [na, nb] = [normalize(ref, a, UPLOAD), normalize(cand, b, UPLOAD)]
      if (!isDeepStrictEqual(na, nb)) await differ('uploadBlob again of an upload the server was killed during', { bytes: `${bytes.length} bytes`, type }, na, nb)
      const m = model(acct)
      const blob = a.body.blob
      hold(m, blob.ref.$link, blob)
      await referenceAgain(acct, m, blob.ref.$link, 'from before a restart', 'after a restart')
    },
  }
}

/// A record on both sides referencing the account's blob `cid`, as stored,
/// then the blob read back; `upload` and `read` say when, in what is shown.
/// Answers the record's key.
export async function referenceAgain(acct, m, cid, upload, read) {
  const s = m.stored.get(cid)
  const rkey = pick(RKEYS)
  const value = { $type: C, n: below(100), batch: nextBatch(), img: { $type: 'blob', ref: { $link: cid }, mimeType: s.mimeType, size: s.size } }
  const body = { repo: acct.handle, collection: C, rkey, record: value }
  const [a, b] = await both(`putRecord referencing an upload ${upload}`, 'com.atproto.repo.putRecord', { body, token: acct })
  noteWrite(acct, C, rkey, a)
  if (a.status >= 400) throw new Error(`the reference refused a ref to a blob it holds: ${JSON.stringify(a.body)}`)
  landed(acct, m, a, b, [{ rkey, value }])
  await compareGetBlob(`getBlob ${read}`, (side) => ({ did: didOf(side, acct), cid }))
  return rkey
}

/// The records of the collection that alone reference a blob whose bytes
/// the harness kept, each `[rkey, cid]`; or, when the account has none, one
/// made: a body of the run's own uploaded and referenced by one record.
export async function soleRefs(acct) {
  const m = model(acct)
  const kept = new Set(m.uploads.map((u) => u.cid))
  const sole = (rkey, c) => kept.has(c) && !m.takedowns.has(c) && [...m.refs].every(([k, s]) => k === rkey || !s.has(c))
  const found = [...m.refs].filter(([rkey]) => !rkey.includes('/')).flatMap(([rkey, cs]) => [...cs].filter((c) => sole(rkey, c)).map((c) => [rkey, c]))
  if (found.length) return found
  const bytes = plainBody('referenced ')
  if (!(await sendUpload(acct, m, { bytes, type: 'text/plain', token: acct }))) throw new Error('the reference refused an upload of a body of the run\'s own')
  const cid = await rawCid(bytes)
  const rkey = await referenceAgain(acct, m, cid, 'just made', 'just referenced')
  return sole(rkey, cid) ? [[rkey, cid]] : []
}

/// A blob the account stops holding, by deleting the one record of the
/// collection that references it, uploaded again at once and referenced
/// again, then read back: what deleting its object, or its file, once the
/// write is on disk must not take from the new upload (spec/BLOBS.md, "The
/// objects").
export async function reuploadAtOnce(acct) {
  const m = model(acct)
  const records = await soleRefs(acct)
  if (!records.length) return
  const [rkey, cid] = pick(records)
  const up = m.uploads.find((u) => u.cid === cid)
  const [a, b] = await both('deleteRecord of the one record referencing a blob', 'com.atproto.repo.deleteRecord', { body: { repo: acct.handle, collection: C, rkey }, token: acct })
  if (a.status >= 400) return
  acct.records.delete(`${C}/${rkey}`)
  landed(acct, m, a, b, [{ rkey, value: null }])
  if (await sendUpload(acct, m, { bytes: up.bytes, type: up.type, token: acct, keep: false })) await referenceAgain(acct, m, cid, 'uploaded again as the blob stopped being held', 'uploaded again as it stopped being held')
}

/// After a restart: every account's blobs as the reference holds them. Its
/// listBlobs, getBlob of some of its referenced blobs, and a record
/// referencing an upload no record referenced before the restart: an
/// upload answered survives a kill (spec/BLOBS.md, "Durability";
/// spec/DURABILITY.md, "Blobs").
export async function blobsSurvive() {
  for (const acct of accounts) {
    const m = acct.blobs
    if (!m) continue
    const [a, b] = await Promise.all([ref, cand].map((side) => xrpc(side, LIST, { params: { did: didOf(side, acct), limit: 1000 } })))
    if (!isDeepStrictEqual(normalize(ref, a, LIST), normalize(cand, b, LIST)) || a.body?.cursor !== b.body?.cursor) await differ('listBlobs after a restart', { did: `<did:${acct.n}>` }, a.body, b.body)
    for (const cid of shuffled(referencedCids(m)).slice(0, 3)) await compareGetBlob('getBlob after a restart', (side) => ({ did: didOf(side, acct), cid }))
    const untethered = [...m.stored.keys()].filter((c) => !referenced(m, c))
    if (untethered.length && chance(0.5)) await referenceAgain(acct, m, pick(untethered), 'from before a restart', 'after a restart')
  }
}