~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
889 lines · 62.8 KB Raw History
// Spaces' write notifications, writer set and syncers (spec/SPACES-SYNC.md),
// in the spaces run. Parties are played here, each answering as a check
// needs and recording what it is sent: two syncers, HTTP servers that confirm
// the tokens the harness gave them, per space, and take notifySpaceUpdate; a
// writer whose notifications the harness signs, and a space host for an
// authority of its own, each a did:plc in the directory with #atproto_pds at
// an HTTP server of the harness.
//
//   1. the server as space host: the authority on each side (the first alpha's
//      or the server's), writers on the second alpha PDS, on the same side,
//      and the harness's writer; registrations and their confirmation,
//      notifications taken, removals, listWriters, the updates sent to each
//      registration, a refusal dropping a registration, and deletion notices;
//   2. the server as repo host: a writer on each side writes into a space on
//      the second alpha PDS, and into the harness space host's spaces;
//   3. retries across a crash: failures arranged here, before the run's
//      restart of the server, and judged after it (`afterRestart`).
//
// Requests go to both sides alike; answers are compared with each side's
// accounts named alike, and each side is held to the page, the reference
// included, which checks the check. Every token and key name is drawn from
// the run's `rand`.

import crypto from 'node:crypto'
import { createServer } from 'node:http'
import { once } from 'node:events'
import { isDeepStrictEqual } from 'node:util'
import * as plc from '@did-plc/lib'
import { Secp256k1Keypair, verifySignature } from '@atproto/crypto'
import * as S from './lib/spec.mjs'

const T = 'com.example.group'
const NRU = 'com.atproto.space.notifyRepoUpdate'
const NSU = 'com.atproto.space.notifySpaceUpdate'
const CONFIRM = 'com.atproto.space.confirmRegistration'
const AUTHORIZE = 'com.atproto.space.authorizeSpaceSync'
const LW = 'com.atproto.space.listWriters'
const REPO_WRITE = 'com.atproto.space.defs#repoWrite'
const REPO_REMOVED = 'com.atproto.space.defs#repoRemoved'
const SPACE_DELETED = `${NSU}#spaceDeleted`
const MEMBERSPACE = 'com.atproto.memberspace.policy'
const TID = /^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/
/// A confirmation's challenge: 16 bytes in lower-case base32, unpadded.
const CHALLENGE = /^[a-z2-7]{26}$/
const EMPTY_DATA = 'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad'
/// The order of secp256k1, to make a high-S signature of a low-S one.
const N = BigInt('0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141')
const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
const now = () => Math.floor(Date.now() / 1000)
const b64 = (o) => Buffer.from(JSON.stringify(o)).toString('base64url')
const unb64 = (s) => JSON.parse(Buffer.from(s, 'base64url').toString('utf8'))
const hexOf = (bytes) => (bytes?.$bytes ? Buffer.from(bytes.$bytes, 'base64').toString('hex') : null)
const keysAre = (o, keys) => isDeepStrictEqual(Object.keys(o ?? {}).sort(), [...keys].sort())
/// The TID of a time in milliseconds, with a clock id.
function tid(ms, clock = 0) {
  let n = (BigInt(Math.floor(ms)) * 1000n << 10n) | BigInt(clock)
  let s = ''
  for (let i = 0; i < 13; i++) { s = '234567abcdefghijklmnopqrstuvwxyz'[Number(n & 31n)] + s; n >>= 5n }
  return s
}

export async function checkSpaceSync(h) {
  const { net, ref, cand, accounts, createAccount, normalize, differ, fail, count, rand, front, afterRestart } = h
  const sides = [ref, cand]
  const peer = net.extraPdses[0]
  const plcUrl = net.plc.url
  const skey = (p) => `${p}${Math.floor(rand() * 1e6)}`
  /// A syncer's token: `n` bytes of the run's seed, base64url.
  const token = (n = 32) => Buffer.from(Array.from({ length: n }, () => Math.floor(rand() * 256))).toString('base64url')

  // ---- requests ------------------------------------------------------------------

  async function call(base, nsid, { params, body, token, headers = {} } = {}) {
    const url = new URL(`${base}/xrpc/${nsid}`)
    for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v)
    const hs = { ...headers }
    if (token) hs.authorization = `Bearer ${token}`
    if (body !== undefined) hs['content-type'] = 'application/json'
    const res = await fetch(url, { method: body === undefined ? 'GET' : 'POST', headers: hs, body: body === undefined ? undefined : JSON.stringify(body) })
    const buf = Buffer.from(await res.arrayBuffer())
    let parsed = null
    if (buf.length) try { parsed = JSON.parse(buf.toString('utf8')) } catch { parsed = { unparsed: buf.toString('utf8').slice(0, 200) } }
    return { status: res.status, type: res.headers.get('content-type'), buf, body: parsed }
  }
  function alias(side, s) {
    for (const a of accounts) if (a.sides[side.name].did) s = s.split(a.sides[side.name].did).join(`<did:${a.n}>`)
    return s
  }
  const named = (side, v) => JSON.parse(alias(side, JSON.stringify(v ?? null)))
  const describe = (side, req) => named(side, JSON.parse(JSON.stringify(req, (k, v) => (k === 'token' || k === 'delegationToken' || k === 'authorization' || k === 'signature' ? '<…>' : v))))
  function shape(side, res) {
    if (res.status >= 400) return normalize(side, res)
    if (res.status === 200 && !res.buf.length) return { status: 200, empty: true }
    return normalize(side, res)
  }
  const err = (error, status = 400) => ({ status, error })
  const EMPTY = { status: 200, empty: true }
  /// The same request to both sides, compared as `view` shows them; each side
  /// held to `want` (an object, or a function answering what is wrong).
  async function both(kind, nsid, make, want, view = (g) => g, base = (side) => side.url) {
    const res = await Promise.all(sides.map((side) => call(base(side), nsid, make(side))))
    const got = sides.map((side, i) => shape(side, res[i]))
    for (const [i, side] of sides.entries()) {
      const wrong = want === undefined ? null : typeof want === 'function' ? want(got[i], res[i], side) : isDeepStrictEqual(got[i], want) ? null : want
      if (wrong) await fail(side, `${kind}: ${nsid}`, describe(side, make(side)), typeof wrong === 'string' ? wrong : named(side, wrong), got[i])
    }
    const [a, b] = got.map(view)
    if (!isDeepStrictEqual(a, b)) await differ(`${kind}: ${nsid}`, describe(cand, make(cand)), a, b)
    count(kind)
    return res
  }
  const ok200 = (g) => (g.status === 200 ? null : 'answered 200')

  // ---- the harness's parties -----------------------------------------------------------

  /// An HTTP server of the harness: records each request; answers as
  /// `answer`: `{status, body}` as JSON, `{status, raw, type?}`, or 'close'.
  async function party() {
    const p = { seen: [], answer: () => ({ status: 200, body: {} }) }
    const srv = createServer((req, res) => {
      const chunks = []
      req.on('data', (c) => chunks.push(c))
      req.on('end', async () => {
        const u = new URL(req.url, 'http://x')
        let body = null
        try { body = JSON.parse(Buffer.concat(chunks).toString('utf8') || 'null') } catch { body = '<not JSON>' }
        const r = { at: Date.now(), method: req.method, path: u.pathname, type: req.headers['content-type'] ?? null, authorization: req.headers.authorization ?? null, body }
        const a = p.answer(r)
        r.answered = a === 'close' ? 'close' : a.status
        p.seen.push(r)
        if (a === 'close') return req.socket.destroy()
        if ('raw' in a) return res.writeHead(a.status, a.type ? { 'content-type': a.type } : {}).end(a.raw)
        res.writeHead(a.status, { 'content-type': 'application/json' }).end(JSON.stringify(a.body))
      })
    })
    srv.listen(0, '127.0.0.1')
    await once(srv, 'listening')
    p.port = srv.address().port
    p.url = `http://localhost:${p.port}`
    p.close = () => srv.close()
    return p
  }
  /// A syncer: confirms a registration of the tokens it gave, per space, as
  /// `gave` says ('echo', or a way to fail), refusing any other; answers
  /// notifySpaceUpdate 200 with no body, or as `failing` says.
  async function syncerParty() {
    const s = await party()
    s.gave = new Map()
    s.failing = new Map()
    s.failOnce = new Map()
    s.answer = (r) => {
      const k = `${r.body?.space} ${r.body?.token}`
      if (r.path === `/xrpc/${CONFIRM}`) {
        const how = s.gave.get(k)
        const challenge = r.body?.challenge
        if (how === undefined) return { status: 400, body: { error: 'InvalidRequest', message: 'not a token this syncer gave for the space' } }
        if (how === 'echo') return { status: 200, body: { challenge } }
        if (how === 'extra') return { status: 200, body: { challenge, also: true } }
        if (how === 'wrong') return { status: 200, body: { challenge: `${challenge}x` } }
        if (how === 'text') return { status: 200, type: 'text/plain', raw: JSON.stringify({ challenge }) }
        if (how === 'empty') return { status: 200, raw: '' }
        if (how === 'close') return 'close'
        return { status: how, body: { error: 'InternalServerError' } }
      }
      if (r.path === `/xrpc/${NSU}`) {
        const once = s.failOnce.get(k)
        if (once) { s.failOnce.delete(k); return once }
        return s.failing.get(k) ?? { status: 200, raw: '' }
      }
      return { status: 404, body: { error: 'MethodNotImplemented' } }
    }
    /// What it was asked for a space and token, but updates.
    s.asked = (space, tok) => s.seen.filter((r) => r.path !== `/xrpc/${NSU}` && r.body?.space === space && r.body?.token === tok)
    s.updates = (space, tok) => s.seen.filter((r) => r.path === `/xrpc/${NSU}` && r.body?.space === space && r.body?.token === tok)
    return s
  }
  async function plcDid(services) {
    const key = await Secp256k1Keypair.create({ exportable: true })
    const op = await plc.signOperation({ type: 'plc_operation', rotationKeys: [key.did()], alsoKnownAs: [], verificationMethods: { atproto: key.did() }, services, prev: null }, key)
    const did = await plc.didForCreateOp(op)
    await new plc.Client(plcUrl).sendOperation(did, op)
    return { did, key }
  }
  const keyOf = async (did) => (await (await fetch(`${plcUrl}/${did}/data`)).json()).verificationMethods?.atproto
  /// A request's service auth as the page says it must be: what is wrong, or null.
  async function checkAuth(r, { iss, aud, lxm }) {
    const parts = (r.authorization ?? '').replace(/^Bearer /, '').split('.')
    let header, claims
    try { [header, claims] = parts.slice(0, 2).map(unb64) } catch { return 'service auth that is a JWT' }
    if (!isDeepStrictEqual(header, { typ: 'JWT', alg: 'ES256K' })) return { header: { typ: 'JWT', alg: 'ES256K' } }
    if (!keysAre(claims, ['aud', 'exp', 'iat', 'iss', 'jti', 'lxm']) || claims.iss !== iss || claims.aud !== aud || claims.lxm !== lxm || claims.exp - claims.iat !== 60 || typeof claims.jti !== 'string' || !claims.jti)
      return { claims: { iss, aud, lxm, iat: '<n>', exp: '<iat + 60>', jti: '<random>' } }
    const key = await keyOf(iss)
    const signed = parts.length === 3 && (await verifySignature(key, Buffer.from(`${parts[0]}.${parts[1]}`), Buffer.from(parts[2], 'base64url')).catch(() => false))
    return signed ? null : `a signature by ${iss}'s #atproto key, low-S`
  }
  const syncer = await syncerParty()
  const syncer2 = await syncerParty()
  const host = await party()
  const writerH = await plcDid({ atproto_pds: { type: 'AtprotoPersonalDataServer', endpoint: host.url } })
  const fake = await plcDid({ atproto_pds: { type: 'AtprotoPersonalDataServer', endpoint: host.url } })

  // ---- accounts --------------------------------------------------------------------

  const peerDomain = (await (await fetch(`${peer.url}/xrpc/com.atproto.server.describeServer`)).json()).availableUserDomains[0]
  async function peerAccount() {
    const f = front()
    const r = await call(peer.url, 'com.atproto.server.createAccount', { body: { handle: `${f}${peerDomain}`, email: `${f}@example.com`, password: `pw-${f}` } })
    if (r.status !== 200) await differ('a peer account on the second alpha PDS', { handle: `${f}${peerDomain}` }, 'an account', r.body, 'the harness could not make a peer')
    return { did: r.body.did, token: r.body.accessJwt }
  }
  const [bob, carol, dave] = [await peerAccount(), await peerAccount(), await peerAccount()]
  const A = await createAccount()
  const W = await createAccount()
  const of = (acct) => (side) => acct.sides[side.name]
  const auth = of(A)
  const sp = (side, key) => `at://${auth(side).did}/space/${T}/${key}`
  /// A peer account's delegation token for a space, from the second alpha PDS.
  async function delegation(user, space) {
    const r = await call(peer.url, 'com.atproto.space.getDelegationToken', { token: user.token, params: { space } })
    if (r.status !== 200) await differ('the second alpha PDS\'s getDelegationToken', { space }, 'a token', r.body, 'the harness could not get a delegation token')
    return r.body.token
  }

  /// A space credential for each side's space, from its authority, asked for
  /// with no delegation token (the space reads publicly) unless `extra` has
  /// one, with its holder's key.
  async function credential(side, space, authorityHost = side.url, extra = {}) {
    const key = S.newP256()
    const res = await call(authorityHost, AUTHORIZE, { body: { space, credentialKey: key.did, ...extra } })
    if (res.status !== 200) await fail(side, 'authorizeSpaceSync for a credential', { space: alias(side, space) }, 'a credential', normalize(side, res))
    return { key, credential: res.body.credential, spaceRev: res.body.spaceRev }
  }
  const signed = (c, audience) => S.signCredential(c.key, c.credential, audience)
  const create = (kind, key, policy) => both(kind, 'com.atproto.space.createSpace', (side) => ({ token: auth(side).token, body: { spaceType: T, skey: key, policy } }), ok200)
  /// putMembers of `[did, read, write]`s, a `did` a DID or a DID of the side.
  const putMembers = (kind, key, members) => both(kind, 'com.atproto.memberspace.putMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, key), members: members.map(([did, read, write]) => ({ did: typeof did === 'function' ? did(side) : did, read, write })) } }), EMPTY)

  /// A writer's latest commit in a space, from its own host.
  async function commitOf(hostUrl, token, space, repo) {
    const r = await call(hostUrl, 'com.atproto.space.getLatestCommit', { token, params: { space, repo } })
    return r.status === 200 ? { repoRev: r.body.commit.rev, data: hexOf(r.body.commit.data) } : null
  }
  async function write(hostUrl, token, space, repo, rkey) {
    return call(hostUrl, 'com.atproto.space.createRecord', { token, body: { space, repo, collection: 'com.example.post', rkey, record: { $type: 'com.example.post', text: rkey, createdAt: '2026-11-08T00:00:00.000Z' } } })
  }
  /// The same record written by `wr` on each side, into the side's space
  /// `key`, or into `space(side)`, so that both sides' repos hold alike.
  async function writeBoth(wr, key = sa, kind = 'a write', space = (side) => sp(side, key)) {
    const rkey = `r${Math.floor(rand() * 1e9)}`
    for (const side of sides) {
      const r = await write(wr.host(side), wr.token(side), space(side), wr.did(side), rkey)
      if (r.status !== 200) await fail(side, kind, { space: alias(side, space(side)) }, 'written', normalize(side, r))
    }
  }

  // ==== 1. the server as space host ==========================================================

  const sa = skey('sync')
  await create('createSpace for notifications', sa, { $type: MEMBERSPACE, publicRead: true })
  await putMembers('putMembers of writers and a reader', sa, [[bob.did, true, true], [writerH.did, true, true], [(side) => W.sides[side.name].did, true, true], [carol.did, true, false]])
  const cred = {}
  for (const side of sides) cred[side.name] = await credential(side, sp(side, sa))
  for (const side of sides) if (cred[side.name].spaceRev !== undefined) await fail(side, 'authorizeSpaceSync of a space no one wrote to', { space: alias(side, sp(side, sa)) }, 'no spaceRev', cred[side.name].spaceRev)
  const withCred = (side, extra = {}, aud = auth(side).did, c = cred[side.name]) => ({ headers: signed(c, aud), ...extra })

  /// listWriters of a space, held to the page per side: entries in
  /// ascending spaceRev, each a #repoWrite or a #repoRemoved
  /// (spec/SPACES-SYNC.md, "The writer set: listWriters").
  async function listAll(side, space, c = cred[side.name]) {
    const r = await call(side.url, LW, { headers: signed(c, auth(side).did), params: { space } })
    if (r.status !== 200) await fail(side, 'listWriters', { space: alias(side, space) }, 'the writer set', normalize(side, r))
    const repos = r.body.repos
    for (const [i, e] of repos.entries()) {
      const ok = e?.$type === REPO_WRITE ? keysAre(e, ['$type', 'data', 'repo', 'repoRev', 'spaceRev']) && TID.test(e.repoRev) && hexOf(e.data) !== null : e?.$type === REPO_REMOVED && keysAre(e, ['$type', 'repo', 'spaceRev'])
      if (!ok || !TID.test(e.spaceRev) || (i && repos[i - 1].spaceRev >= e.spaceRev)) await fail(side, 'listWriters entries, in ascending spaceRev', { space: alias(side, space) }, `{$type: "${REPO_WRITE}", repo, repoRev, data, spaceRev} or {$type: "${REPO_REMOVED}", repo, spaceRev}, spaceRev ascending`, named(side, repos))
    }
    if ((repos.length ? r.body.cursor !== repos.at(-1).spaceRev : 'cursor' in r.body)) await fail(side, 'listWriters cursor', { space: alias(side, space) }, repos.length ? repos.at(-1).spaceRev : 'none', r.body.cursor ?? 'none')
    return repos
  }
  /// A writer's entry as the checks compare it.
  const entryView = (e) => (!e ? null : e.$type === REPO_REMOVED ? { removed: true } : { repoRev: e.repoRev, data: hexOf(e.data) })
  /// Wait until each side's writer set shows `did(side)` as `want(side)`.
  async function tracked(kind, space, did, want, c = (side) => cred[side.name]) {
    const started = Date.now()
    const done = new Set()
    const last = {}
    for (;;) {
      for (const side of sides) {
        if (done.has(side.name)) continue
        last[side.name] = entryView((await listAll(side, space(side), c(side))).find((e) => e.repo === did(side)))
        if (isDeepStrictEqual(last[side.name], await want(side))) done.add(side.name)
      }
      if (done.size === 2) break
      if (Date.now() - started > 15000) { const side = done.has('reference') ? cand : ref; await fail(side, `${kind}: the writer set`, { space: alias(side, space(side)), writer: alias(side, did(side)) }, await want(side), last[side.name]) }
      await sleep(100)
    }
    count(kind)
  }
  await both('listWriters of no writers', LW, (side) => withCred(side, { params: { space: sp(side, sa) } }), { status: 200, body: { repos: [] } })

  // Registrations: each new one confirmed once, at the endpoint's origin,
  // with the token in the body and no Authorization; the same pair again
  // renewed (spec/SPACES-SYNC.md, "Syncers", "Registering";
  // spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync").
  const regKey = S.newP256()
  const challenges = new Set()
  const registered = (g, r) => (r.status === 200 && Object.keys(r.body ?? {}).every((k) => k === 'credential' || k === 'spaceRev') && typeof r.body.credential === 'string' && (r.body.spaceRev === undefined || TID.test(r.body.spaceRev)) ? null : '{credential, spaceRev?}')
  const asAuthorized = (g) => (g.status === 200 ? { status: 200, credential: typeof g.body?.credential === 'string', spaceRev: g.body?.spaceRev === undefined ? 'absent' : '<tid>' } : g)
  /// What is wrong with a confirmation the side sent, or null.
  function confirmWrong(r) {
    if (r.method !== 'POST' || r.path !== `/xrpc/${CONFIRM}` || !(r.type ?? '').startsWith('application/json') || r.authorization) return `a POST of JSON to /xrpc/${CONFIRM}, with no Authorization`
    if (!keysAre(r.body, ['challenge', 'space', 'token'])) return 'a body of exactly {space, token, challenge}'
    if (!CHALLENGE.test(r.body.challenge) || challenges.has(r.body.challenge)) return 'a challenge of 16 random bytes in base32, lower case, unpadded, new each time'
    challenges.add(r.body.challenge)
    return null
  }
  /// authorizeSpaceSync with `notify` on both sides, the syncer `p` having
  /// given `tok` for each side's space as `how` (null: as it was); the
  /// answer held to `want`, and `confirms` new requests at `p` for the pair,
  /// each a confirmation as the page says.
  async function register(kind, space, endpoint, tok, { p = syncer, how = 'echo', want = registered, confirms = 1, extra = () => ({}) } = {}) {
    if (how !== null) for (const side of sides) p.gave.set(`${space(side)} ${tok}`, how)
    const before = Object.fromEntries(sides.map((side) => [side.name, p.asked(space(side), tok).length]))
    const res = await both(kind, AUTHORIZE, (side) => ({ body: { space: space(side), credentialKey: regKey.did, notify: { endpoint: typeof endpoint === 'function' ? endpoint(side) : endpoint, token: tok }, ...extra(side) } }), want, asAuthorized)
    for (const side of sides) {
      const got = p.asked(space(side), tok).slice(before[side.name])
      const wrong = got.length !== confirms ? `${confirms} confirmation${confirms === 1 ? '' : 's'} at the syncer` : got.map(confirmWrong).find(Boolean)
      if (wrong) await fail(side, `${kind}: confirmRegistration`, { space: alias(side, space(side)) }, wrong, named(side, got.map((r) => ({ method: r.method, path: r.path, type: r.type, authorization: r.authorization, body: { ...r.body, token: '<…>' } }))))
    }
    count('registrations and their confirmation')
    return res
  }
  const inSa = (side) => sp(side, sa)
  const [t1, t2, t4, t5] = [token(), token(), token(), token(384)]
  await register('a registration', inSa, syncer.url, t1)
  await register('the same pair again, renewed', inSa, syncer.url, t1, { confirms: 0 })
  // Its endpoint has a path and a query, which are not used.
  await register('a registration at an endpoint with a path and a query', inSa, `${syncer2.url}/base/path?x=`, t2, { p: syncer2 })
  // The same token, the endpoint written otherwise: another registration.
  await register('the same token at the endpoint with a / after it', inSa, `${syncer.url}/`, t1, { how: null })
  await register('a registration answered with another member besides the challenge', inSa, syncer.url, t4, { how: 'extra' })
  await register('a registration with a 512-byte token', inSa, syncer2.url, t5, { p: syncer2 })
  const NOT_CONFIRMED = err('RegistrationNotConfirmed')
  const refusedTokens = []
  for (const [what, how] of [['a wrong challenge', 'wrong'], ['500', 500], ['text/plain', 'text'], ['an empty 200', 'empty'], ['a closed connection', 'close'], ['a token the syncer did not give', undefined]]) {
    const tok = token()
    refusedTokens.push(tok)
    if (how === undefined) for (const side of sides) syncer.gave.delete(`${inSa(side)} ${tok}`)
    await register(`a registration answered with ${what}`, inSa, syncer.url, tok, { how: how ?? null, want: NOT_CONFIRMED })
  }
  for (const [what, endpoint] of [['ftp://', `ftp://localhost:${syncer.port}`], ['with a user and password', `http://u:p@localhost:${syncer.port}`], ['a DID', `did:plc:${'a'.repeat(24)}`]]) {
    const tok = token()
    refusedTokens.push(tok)
    await register(`a registration at an endpoint ${what}`, inSa, endpoint, tok, { want: NOT_CONFIRMED, confirms: 0 })
  }
  for (const [what, notify] of [['an endpoint that is no URI', { endpoint: 'not a uri', token: token() }], ['a 513-byte token', { endpoint: syncer.url, token: `${token(384)}x` }], ['no token', { endpoint: syncer.url }]])
    await both(`a registration with ${what}`, AUTHORIZE, (side) => ({ body: { space: inSa(side), credentialKey: regKey.did, notify } }), err('InvalidRequest'))
  // Refused before step 10: nothing is confirmed (spec/SPACES-CREDENTIALS.md,
  // "authorizeSpaceSync").
  const sm = skey('members')
  await create('createSpace, members only', sm, { $type: MEMBERSPACE })
  await putMembers('putMembers of the peer writer', sm, [[bob.did, true, true]])
  const inSm = (side) => sp(side, sm)
  const away = skey('away')
  const peerSpace = `at://${bob.did}/space/${T}/${away}`
  await call(peer.url, 'com.atproto.space.createSpace', { token: bob.token, body: { spaceType: T, skey: away, policy: { $type: MEMBERSPACE, publicRead: true } } })
  for (const [what, space, want, extra] of [
    ['of a members-only space with no token', inSm, err('SpaceNotFound')],
    ['of a space never made', (side) => sp(side, skey('none')), err('SpaceNotFound')],
    ['of a space whose authority is elsewhere', () => peerSpace, err('SpaceNotFound')],
    ['with a credentialKey that is no key', inSa, err('InvalidRequest'), () => ({ credentialKey: 'nope' })],
  ]) {
    const tok = token()
    refusedTokens.push(tok)
    await register(`a registration ${what}`, space, syncer.url, tok, { want, confirms: 0, extra })
  }
  const regsA = [[syncer, t1, 'the registration and its twin, at the endpoint with a /', 2], [syncer2, t2, 'the endpoint with a path'], [syncer, t4, 'the answer with another member'], [syncer2, t5, 'the 512-byte token']]

  // Writes tracked, and sent: a peer's notified over HTTP, the same side's
  // taken directly (spec/SPACES-SYNC.md, "The writer set", "Telling the space
  // host: notifyRepoUpdate, sent", "For other slices").
  const writers = {
    peer: { did: () => bob.did, host: () => peer.url, token: () => bob.token },
    local: { did: (side) => W.sides[side.name].did, host: (side) => side.url, token: (side) => W.sides[side.name].token },
    authority: { did: (side) => auth(side).did, host: (side) => side.url, token: (side) => auth(side).token },
  }
  const inSaOf = (wr) => (side) => commitOf(wr.host(side), wr.token(side), sp(side, sa), wr.did(side))
  for (const [name, wr] of Object.entries(writers)) {
    await writeBoth(wr, sa, `a write by the ${name} writer`)
    await tracked(`a write by the ${name} writer, tracked`, inSa, wr.did, inSaOf(wr))
  }
  const reader = { did: () => carol.did, host: () => peer.url, token: () => carol.token }
  await writeBoth(reader)
  await writeBoth(writers.peer)
  await tracked('a later write, tracked', inSa, writers.peer.did, inSaOf(writers.peer))
  await sleep(300)
  /// The writer set compared across sides, by writer, data and removal.
  async function setIs(kind, key, wantDids) {
    const views = []
    for (const side of sides) {
      const repos = await listAll(side, sp(side, key))
      const dids = repos.filter((e) => e.$type === REPO_WRITE).map((e) => e.repo).sort()
      const want = wantDids.map((d) => d(side)).sort()
      if (!isDeepStrictEqual(dids, want)) await fail(side, `${kind}: who is in the writer set`, { space: alias(side, sp(side, key)) }, named(side, want), named(side, dids))
      views.push(named(side, repos.map((e) => ({ repo: e.repo, ...entryView(e) }))).map(({ repoRev, ...v }) => v).sort((x, y) => (x.repo < y.repo ? -1 : 1)))
    }
    if (!isDeepStrictEqual(views[0], views[1])) await differ(`${kind}: ${LW}`, { space: '<space>' }, views[0], views[1])
    count(kind)
  }
  await setIs('the writer set, a reader not in it', sa, [writers.peer.did, writers.local.did, writers.authority.did])

  // The harness writer's notifications, signed here (spec/SPACES-SYNC.md,
  // "Taking a notification: notifyRepoUpdate, served").
  const sentCount = (side) => syncer.updates(sp(side, sa), t4).length
  let revAt = Date.now()
  const nextRev = () => tid((revAt = Math.max(revAt + 1000, Date.now())))
  const notifyJwt = async (side, claims = {}, header = { typ: 'JWT', alg: 'ES256K' }, { key = writerH.key, highS = false } = {}) => {
    const c = { iss: writerH.did, aud: `${auth(side).did}#atproto_pds`, lxm: NRU, iat: now(), exp: now() + 60, jti: crypto.randomBytes(16).toString('hex'), ...claims }
    for (const k of Object.keys(c)) if (c[k] === undefined) delete c[k]
    const input = `${b64(header)}.${b64(c)}`
    let sig = Buffer.from(await key.sign(new Uint8Array(Buffer.from(input))))
    if (highS) { const s = BigInt(`0x${sig.subarray(32).toString('hex')}`); sig = Buffer.concat([sig.subarray(0, 32), Buffer.from((N - s).toString(16).padStart(64, '0'), 'hex')]) }
    return `${input}.${sig.toString('base64url')}`
  }
  const data = (n) => ({ $bytes: Buffer.alloc(32, n).toString('base64') })
  /// A notification from the harness writer to each side: `body(side)` merged
  /// over the usual, `claims(side)` over the usual service auth.
  async function notifyEach(kind, want, { body = () => ({}), claims = () => ({}), header, key, highS, authorization } = {}) {
    const rev = nextRev()
    const jwt = {}
    for (const side of sides) jwt[side.name] = await notifyJwt(side, claims(side), header, { key, highS })
    return both(kind, NRU, (side) => ({ headers: { authorization: authorization ?? `Bearer ${jwt[side.name]}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: rev, data: data(1), ...body(side) } }), want)
  }
  const hEntry = async (side) => entryView((await listAll(side, sp(side, sa))).find((x) => x.repo === writerH.did))
  const hState = {}
  /// A notification the page says is taken: the harness writer's entry then
  /// holds the `data` sent, as hex `wantHex`.
  const takes = async (kind, opts, dataN, wantHex = Buffer.alloc(32, dataN).toString('hex')) => {
    await notifyEach(kind, EMPTY, { ...opts, body: (side) => ({ data: data(dataN), ...(opts.body?.(side) ?? {}) }) })
    for (const side of sides) { const e = await hEntry(side); if (e?.data !== wantHex) await fail(side, `${kind}: the writer set`, { writer: 'the harness writer' }, { data: wantHex }, e); hState[side.name] = e }
  }
  await takes('notifyRepoUpdate taken', {}, 1)
  await takes('notifyRepoUpdate with a bare aud', { claims: (side) => ({ aud: auth(side).did }) }, 2)
  await takes('notifyRepoUpdate signed high-S', { highS: true }, 3)
  await takes('notifyRepoUpdate with spaceRev and prevSpaceRev sent', { body: () => ({ spaceRev: tid(Date.now()), prevSpaceRev: tid(Date.now() - 1000) }) }, 4)
  for (const [what, opts, want] of [
    ['to another fragment', { claims: (side) => ({ aud: `${auth(side).did}#other` }) }, err('Forbidden', 403)],
    ['to another DID', { claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('Forbidden', 403)],
    ['from another issuer than repo', { body: () => ({ repo: bob.did }) }, err('Forbidden', 403)],
    ['from an issuer with a fragment', { claims: () => ({ iss: `${writerH.did}#x` }) }, err('Forbidden', 403)],
    ['for another method', { claims: () => ({ lxm: LW }) }, err('BadJwtLexiconMethod', 401)],
    ['with no lxm', { claims: () => ({ lxm: undefined }) }, err('BadJwtLexiconMethod', 401)],
    ['expired', { claims: () => ({ exp: now() - 10 }) }, err('JwtExpired', 401)],
    ['signed by another key', { key: await Secp256k1Keypair.create() }, err('BadJwtSignature', 401)],
    ['of type at+jwt', { header: { typ: 'at+jwt', alg: 'ES256K' } }, err('BadJwtType', 401)],
    ['with no token', { authorization: '' }, err('MissingJwt', 401)],
    ['with a bad repoRev', { body: () => ({ repoRev: 'nope' }) }, err('InvalidRequest')],
    ['6 minutes ahead', { body: () => ({ repoRev: tid(Date.now() + 6 * 60000) }) }, err('FutureRev')],
    ['of a space never created', { body: (side) => ({ space: sp(side, skey('none')) }) }, err('SpaceNotFound')],
    ['of a space never created, 6 minutes ahead', { body: (side) => ({ space: sp(side, skey('none')), repoRev: tid(Date.now() + 6 * 60000) }) }, err('FutureRev')],
    ['of a space whose authority is elsewhere', { body: () => ({ space: peerSpace }), claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('SpaceNotFound')],
    ['of a space whose authority is elsewhere, 6 minutes ahead', { body: () => ({ space: peerSpace, repoRev: tid(Date.now() + 6 * 60000) }), claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('SpaceNotFound')],
  ]) await notifyEach(`notifyRepoUpdate ${what}`, want, opts)
  for (const side of sides) if (!isDeepStrictEqual(await hEntry(side), hState[side.name])) await fail(side, 'refused notifications change nothing', { writer: 'the harness writer' }, hState[side.name], await hEntry(side))
  // Stale and equal revs: taken, nothing changes, nothing sent.
  const before = Object.fromEntries(sides.map((side) => [side.name, sentCount(side)]))
  for (const side of sides) side.jwt = await notifyJwt(side)
  await both('notifyRepoUpdate of the same rev', NRU, (side) => ({ headers: { authorization: `Bearer ${side.jwt}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: hState[side.name].repoRev, data: data(9) } }), EMPTY)
  for (const side of sides) side.jwt = await notifyJwt(side)
  await both('notifyRepoUpdate of an earlier rev', NRU, (side) => ({ headers: { authorization: `Bearer ${side.jwt}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: tid(Date.now() - 60000), data: data(9) } }), EMPTY)
  for (const side of sides) delete side.jwt
  await writeBoth(writers.peer)
  await tracked('a write after stale notifications, tracked', inSa, writers.peer.did, inSaOf(writers.peer))
  await sleep(300)
  for (const side of sides) {
    if (!isDeepStrictEqual(await hEntry(side), hState[side.name])) await fail(side, 'a stale notification changes nothing', { writer: 'the harness writer' }, hState[side.name], await hEntry(side))
    if (sentCount(side) !== before[side.name] + 1) await fail(side, 'a stale notification is not sent', { space: alias(side, sp(side, sa)) }, `${before[side.name] + 1} updates (the peer's write alone)`, sentCount(side))
  }
  // 4 minutes ahead is taken; odd data too.
  revAt = Date.now() + 4 * 60000
  await takes('notifyRepoUpdate 4 minutes ahead', {}, 5)
  await takes('notifyRepoUpdate of 31 bytes', { body: () => ({ data: { $bytes: Buffer.alloc(31, 6).toString('base64') } }) }, 6, Buffer.alloc(31, 6).toString('hex'))

  // Removals, each sequenced and sent; nothing for a writer not in the set
  // (spec/SPACES-SYNC.md, "Removing a writer").
  const removed = async () => ({ removed: true })
  const quiet = async (kind, run) => {
    const at = Object.fromEntries(sides.map((side) => [side.name, sentCount(side)]))
    await run()
    await sleep(500)
    for (const side of sides) if (sentCount(side) !== at[side.name]) await fail(side, `${kind}: nothing sent`, { space: alias(side, sp(side, sa)) }, `${at[side.name]} updates`, sentCount(side))
  }
  await both('removeWriter of the peer writer', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: bob.did } }), EMPTY)
  await tracked('removeWriter, shown removed', inSa, writers.peer.did, removed)
  await quiet('removeWriter of it again', () => both('removeWriter of it again', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: bob.did } }), EMPTY))
  await quiet('removeWriter of an account never in the set', () => both('removeWriter of an account never in the set', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: carol.did } }), EMPTY))
  await writeBoth(writers.peer)
  await tracked('a removed writer\'s next write, tracked again', inSa, writers.peer.did, inSaOf(writers.peer))
  await putMembers('putMembers taking the peer writer\'s write away', sa, [[bob.did, true, false]])
  await tracked('putMembers taking a writer\'s write away, shown removed', inSa, writers.peer.did, removed)
  await both('removeMembers of the writer on the same side', 'com.atproto.memberspace.removeMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), dids: [W.sides[side.name].did] } }), EMPTY)
  await tracked('removeMembers of a writer, shown removed', inSa, writers.local.did, removed)
  // An unauthorized writer: refused, after the aud (spec/SPACES-SYNC.md,
  // "Taking a notification: notifyRepoUpdate, served", "For other slices").
  await both('removeMembers of the harness writer', 'com.atproto.memberspace.removeMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), dids: [writerH.did] } }), EMPTY)
  await tracked('removeMembers of the harness writer, shown removed', inSa, () => writerH.did, removed)
  await notifyEach('notifyRepoUpdate by a writer not authorized', err('Forbidden', 403))
  await notifyEach('notifyRepoUpdate by a writer not authorized, to another fragment', err('Forbidden', 403), { claims: (side) => ({ aud: `${auth(side).did}#other` }) })
  await quiet('a write by a writer whose write was taken away', async () => { await writeBoth(writers.peer); await sleep(500) })

  /// The updates a registration had for a side's space, each held to the page
  /// (spec/SPACES-SYNC.md, "Sending updates", "Deletion").
  async function updatesOf(kind, side, p, space, tok) {
    const got = p.updates(space, tok)
    for (const r of got) {
      const u = r.body?.update
      const asked = r.method === 'POST' && (r.type ?? '').startsWith('application/json') && !r.authorization && keysAre(r.body, ['space', 'token', 'update'])
      const has = (keys) => keysAre(u, keys) || keysAre(u, [...keys, 'prevSpaceRev'])
      const ok = u?.$type === REPO_WRITE ? has(['$type', 'data', 'repo', 'repoRev', 'spaceRev']) && TID.test(u.repoRev) && hexOf(u.data) !== null : u?.$type === REPO_REMOVED ? has(['$type', 'repo', 'spaceRev']) : u?.$type === SPACE_DELETED && keysAre(u, ['$type'])
      if (!asked || !ok || (u.spaceRev !== undefined && !TID.test(u.spaceRev)) || (u.prevSpaceRev !== undefined && !TID.test(u.prevSpaceRev)))
        await fail(side, `${kind}: notifySpaceUpdate`, { space: alias(side, space) }, `a POST of JSON to /xrpc/${NSU}, no Authorization, body exactly {space, token, update}, update a #repoWrite {repo, repoRev, data, spaceRev, prevSpaceRev?}, a #repoRemoved {repo, spaceRev, prevSpaceRev?} or a #spaceDeleted {}`, named(side, { method: r.method, type: r.type, authorization: r.authorization, body: { ...r.body, token: '<…>' } }))
    }
    return got.map((r) => r.body.update)
  }
  /// Every change of each side's writer set sent to each registration made
  /// before the first, `n` times (its twins), in spaceRev order, chained by
  /// prevSpaceRev from none; each listWriters entry sent at its spaceRev;
  /// the same updates to every registration, and on both sides
  /// (spec/SPACES-SYNC.md, "Sending updates").
  async function allSent(kind, key, regs) {
    const seqs = []
    for (const side of sides) {
      const space = sp(side, key)
      const entries = await listAll(side, space)
      const top = entries.reduce((m, e) => (e.spaceRev > m ? e.spaceRev : m), '')
      for (const started = Date.now(); Date.now() - started < 5000; await sleep(100))
        if (regs.every(([p, tok, , n = 1]) => p.updates(space, tok).filter((r) => r.body?.update?.spaceRev === top).length >= n)) break
      let first
      for (const [p, tok, name, n = 1] of regs) {
        const got = (await updatesOf(kind, side, p, space, tok)).filter((u) => u.$type !== SPACE_DELETED)
        const once = [...new Map(got.map((u) => [u.spaceRev, u])).values()].sort((x, y) => (x.spaceRev < y.spaceRev ? -1 : 1))
        const times = once.find((u) => got.filter((x) => x.spaceRev === u.spaceRev).length !== n)
        const chain = once.findIndex((u, i) => (i === 0 ? 'prevSpaceRev' in u : u.prevSpaceRev !== once[i - 1].spaceRev))
        if (times || chain !== -1 || got.some((u) => !isDeepStrictEqual(u, once.find((x) => x.spaceRev === u.spaceRev))))
          await fail(side, `${kind}: each change sent ${n === 1 ? 'once' : `${n} times, once to each registration`}, chained by prevSpaceRev from none`, { registration: name }, 'each prevSpaceRev the spaceRev before, absent on the first', named(side, got))
        for (const e of entries) {
          const u = once.find((x) => x.spaceRev === e.spaceRev)
          const view = (x) => x && { ...x, data: hexOf(x.data) ?? undefined, prevSpaceRev: undefined }
          if (!isDeepStrictEqual(view(u), view(e))) await fail(side, `${kind}: a writer set entry sent at its spaceRev`, { registration: name, writer: alias(side, e.repo) }, named(side, e), named(side, u ?? null))
        }
        if (once.some((u) => u.repo === carol.did)) await fail(side, `${kind}: a write not tracked, not sent`, { registration: name }, 'no update of the reader', named(side, once.filter((u) => u.repo === carol.did)))
        const seq = once.map((u) => `${u.$type.split('#')[1]} ${alias(side, u.repo)}`)
        if (first === undefined) first = seq
        else if (!isDeepStrictEqual(seq, first)) await fail(side, `${kind}: every registration sent the same updates`, { registration: name }, first, seq)
      }
      seqs.push(first)
    }
    if (!isDeepStrictEqual(seqs[0], seqs[1])) await differ(`${kind}: the updates sent, in order`, { space: '<space>' }, seqs[0], seqs[1])
    count(kind)
  }
  await allSent('updates of the writer set', sa, regsA)
  for (const side of sides) {
    const told = refusedTokens.filter((tok) => syncer.updates(sp(side, sa), tok).length)
    if (told.length) await fail(side, 'a registration refused is sent nothing', { space: alias(side, sp(side, sa)) }, 'no update', `updates to ${told.length} refused registrations`)
  }
  // authorizeSpaceSync answers the writer set's greatest spaceRev
  // (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync").
  for (const side of sides) {
    const entries = await listAll(side, sp(side, sa))
    const top = entries.reduce((m, e) => (e.spaceRev > m ? e.spaceRev : m), '')
    const c = await credential(side, sp(side, sa))
    if (c.spaceRev !== top) await fail(side, 'authorizeSpaceSync\'s spaceRev', { space: alias(side, sp(side, sa)) }, top, c.spaceRev ?? 'none')
  }
  count('authorizeSpaceSync\'s spaceRev, the writer set\'s')

  // listWriters' refusals (spec/SPACES-SYNC.md, "The writer set: listWriters",
  // "Commands").
  const sb = skey('other')
  await create('createSpace for another credential', sb, { $type: MEMBERSPACE, publicRead: true })
  const credB = {}
  for (const side of sides) credB[side.name] = await credential(side, sp(side, sb))
  const credAway = await credential(ref, peerSpace, peer.url)
  for (const [what, make, want] of [
    ['with no credential', () => ({}), err('MissingJwt', 401)],
    ['with a bearer token', (side) => ({ token: auth(side).token }), err('MissingJwt', 401)],
    ['to another audience', (side) => withCred(side, {}, bob.did), err('BadSpaceAudience', 401)],
    ['with another space\'s credential', (side) => withCred(side, {}, auth(side).did, credB[side.name]), err('InvalidCredential')],
  ]) await both(`listWriters ${what}`, LW, (side) => ({ ...make(side), params: { space: sp(side, sa) } }), want)
  await both('listWriters of a space whose authority is elsewhere', LW, () => ({ headers: signed(credAway, bob.did), params: { space: peerSpace } }), err('SpaceNotFound'))
  await both('listWriters of a malformed space', LW, (side) => withCred(side, { params: { space: 'nope' } }), err('InvalidRequest'))
  await both('listWriters with no credential and a malformed space', LW, () => ({ params: { space: 'nope' } }), err('InvalidRequest'))
  await both('listWriters with limit 0', LW, (side) => withCred(side, { params: { space: sp(side, sa), limit: 0 } }), err('InvalidRequest'))

  // listWriters' pages (spec/SPACES-SYNC.md, "The writer set: listWriters").
  for (const side of sides) {
    const all = await listAll(side, sp(side, sa))
    const paged = []
    let cursor
    for (let i = 0; i < 20; i++) {
      const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), limit: 1, ...(cursor ? { cursor } : {}) } })
      const want = all.filter((e) => cursor === undefined || e.spaceRev > cursor).slice(0, 1)
      const body = want.length ? { cursor: want[0].spaceRev, repos: want } : { repos: [] }
      if (r.status !== 200 || !isDeepStrictEqual(r.body, body)) await fail(side, 'listWriters page of 1', { cursor: cursor ?? null }, named(side, body), named(side, r.body))
      if (!want.length) break
      paged.push(...want)
      cursor = want[0].spaceRev
    }
    if (!isDeepStrictEqual(paged, all)) await fail(side, 'listWriters pages, together', {}, named(side, all), named(side, paged))
    for (const c of ['a', 'zzzz', all.at(-1).spaceRev]) {
      const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), cursor: c } })
      if (r.status !== 200 || !isDeepStrictEqual(r.body, { repos: [] })) await fail(side, 'listWriters after every spaceRev', { cursor: c }, { repos: [] }, named(side, r.body))
    }
    const first = all[0].spaceRev
    const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), cursor: first } })
    if (r.status !== 200 || !isDeepStrictEqual(r.body?.repos, all.slice(1))) await fail(side, 'listWriters after a spaceRev, exclusive', { cursor: first }, named(side, all.slice(1)), named(side, r.body))
    count('listWriters pages and cursors')
  }

  // A refusal drops the registration of its pair; an earlier refusal leaves
  // it (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync"; spec/SPACES-SYNC.md,
  // "Registering").
  {
    const t7 = token()
    const tokens = async (user) => Object.fromEntries(await Promise.all(sides.map(async (side) => [side.name, await delegation(user, inSm(side))])))
    const credM = {}
    for (const side of sides) credM[side.name] = await credential(side, inSm(side), side.url, { delegationToken: await delegation(bob, inSm(side)) })
    const writeSm = async (kind) => {
      await writeBoth(writers.authority, sm, kind)
      await tracked(kind, inSm, writers.authority.did, (side) => commitOf(side.url, auth(side).token, inSm(side), auth(side).did), (side) => credM[side.name])
      await sleep(500)
    }
    const told = async (kind, n) => {
      for (const side of sides) {
        const got = await updatesOf(kind, side, syncer, inSm(side), t7)
        if (got.length !== n) await fail(side, kind, { space: alias(side, inSm(side)) }, `${n} updates`, named(side, got))
      }
      count('a refusal drops a registration')
    }
    let dt = await tokens(bob)
    await register('a member\'s registration, members only', inSm, syncer.url, t7, { extra: (side) => ({ delegationToken: dt[side.name] }) })
    dt = await tokens(dave)
    await register('the same pair with a non-member\'s token', inSm, syncer.url, t7, { how: null, want: err('SpaceNotFound'), confirms: 0, extra: (side) => ({ delegationToken: dt[side.name] }) })
    await writeSm('a write after the registration is dropped')
    await told('a registration dropped, sent nothing', 0)
    dt = await tokens(bob)
    await register('the member\'s registration again, confirmed anew', inSm, syncer.url, t7, { how: null, extra: (side) => ({ delegationToken: dt[side.name] }) })
    await register('the same pair with a token that is no JWT', inSm, syncer.url, t7, { how: null, want: err('InvalidDelegationToken'), confirms: 0, extra: () => ({ delegationToken: 'abc' }) })
    await writeSm('a write after a refusal that leaves the registration')
    await told('a registration left by an earlier refusal, sent the write', 1)
  }

  // Deletion notices; a space created again has no registrations and no
  // writers (spec/SPACES-SYNC.md, "Deletion", "Notices", "For other slices").
  const sd = skey('del')
  const inSd = (side) => sp(side, sd)
  await create('createSpace to delete', sd, { $type: MEMBERSPACE, publicRead: true })
  const credD = {}
  for (const side of sides) credD[side.name] = await credential(side, inSd(side))
  const [t8, t9] = [token(), token()]
  await register('a registration to be told of deletion', inSd, syncer.url, t8)
  await register('another, at the second syncer', inSd, syncer2.url, t9, { p: syncer2 })
  await both('deleteSpace with registrations', 'com.atproto.space.deleteSpace', (side) => ({ token: auth(side).token, body: { space: inSd(side) } }), EMPTY)
  const notices = async (kind) => {
    for (const side of sides) {
      for (const [p, tok] of [[syncer, t8], [syncer2, t9]]) {
        for (const started = Date.now(); !p.updates(inSd(side), tok).length && Date.now() - started < 10000;) await sleep(100)
        const got = await updatesOf(kind, side, p, inSd(side), tok)
        if (got.length !== 1 || got[0]?.$type !== SPACE_DELETED) await fail(side, `${kind}: one deletion notice to each registration`, { space: alias(side, inSd(side)) }, [{ $type: SPACE_DELETED }], named(side, got))
      }
    }
    count(kind)
  }
  await notices('deletion notices')
  await both('deleteSpace again', 'com.atproto.space.deleteSpace', (side) => ({ token: auth(side).token, body: { space: inSd(side) } }), EMPTY)
  await sleep(500)
  await notices('nothing more for a second deleteSpace')
  await register('a registration of a deleted space', inSd, syncer.url, t8, { how: null, want: err('SpaceDeleted'), confirms: 0 })
  await both('listWriters of a deleted space', LW, (side) => ({ headers: signed(credD[side.name], auth(side).did), params: { space: inSd(side) } }), err('SpaceNotFound'))
  await notifyEach('notifyRepoUpdate of a deleted space', err('SpaceNotFound'), { body: (side) => ({ space: inSd(side) }) })
  await create('createSpace of the deleted space again', sd, { $type: MEMBERSPACE, publicRead: true, publicWrite: true })
  for (const side of sides) {
    const c = await credential(side, inSd(side))
    if (c.spaceRev !== undefined) await fail(side, 'authorizeSpaceSync of a space created again', { space: alias(side, inSd(side)) }, 'no spaceRev', c.spaceRev)
  }
  await writeBoth(writers.peer, sd)
  await tracked('a space created again, tracked with the old credential', inSd, writers.peer.did, (side) => commitOf(peer.url, bob.token, inSd(side), bob.did), (side) => credD[side.name])
  await sleep(500)
  await notices('a space created again sends to no one')
  await register('a registration of the space created again, confirmed anew', inSd, syncer.url, t8, { how: null })

  // The limit on registrations, held to the page alone (the reference has
  // none): 64 active a space; one registered already is always renewed; a
  // refusal's drop frees one (spec/SPACES-SYNC.md, "Registering", "Where the
  // reference departs from this page").
  {
    const caps = skey('caps')
    const space = sp(cand, caps)
    const made = await call(cand.url, 'com.atproto.space.createSpace', { token: auth(cand).token, body: { spaceType: T, skey: caps, policy: { $type: MEMBERSPACE } } })
    if (made.status !== 200) await fail(cand, 'createSpace for the registration limit', {}, 'the space created', normalize(cand, made))
    const added = await call(cand.url, 'com.atproto.memberspace.putMembers', { token: auth(cand).token, body: { space, members: [{ did: bob.did, read: true, write: false }] } })
    if (added.status !== 200) await fail(cand, 'putMembers for the registration limit', {}, 'the member added', normalize(cand, added))
    const toks = Array.from({ length: 66 }, () => token())
    const reg = async (what, i, want, confirms, user = bob) => {
      syncer.gave.set(`${space} ${toks[i]}`, 'echo')
      const asked = syncer.asked(space, toks[i]).length
      const r = await call(cand.url, AUTHORIZE, { body: { space, credentialKey: regKey.did, delegationToken: await delegation(user, space), notify: { endpoint: syncer.url, token: toks[i] } } })
      const g = shape(cand, r)
      const wrong = want === 200 ? registered(g, r) : isDeepStrictEqual(g, want) ? null : want
      if (wrong) await fail(cand, `authorizeSpaceSync ${what}`, { registration: i + 1 }, wrong, g)
      const n = syncer.asked(space, toks[i]).length - asked
      if (n !== confirms) await fail(cand, `authorizeSpaceSync ${what}: confirmRegistration`, { registration: i + 1 }, `${confirms} confirmations`, n)
      count('the limit on registrations (the page alone)')
    }
    for (let i = 0; i < 64; i++) await reg(`, the space's registration ${i + 1}`, i, 200, 1)
    await reg(', a 65th', 64, err('TooManyRegistrations'), 0)
    await reg(', the first again, renewed', 0, 200, 0)
    await reg(', a non-member with the second\'s pair', 1, err('SpaceNotFound'), 0, dave)
    await reg(', a 65th once one is dropped', 64, 200, 1)
  }

  // ==== 2. the server as repo host ==========================================================

  // Into a memberspace on the second alpha PDS, clearing too
  // (spec/SPACES-SYNC.md, "Telling the space host: notifyRepoUpdate, sent";
  // spec/SPACES-REPO.md, "For other parts").
  const ex = skey('ext')
  const extSpace = `at://${bob.did}/space/${T}/${ex}`
  await call(peer.url, 'com.atproto.space.createSpace', { token: bob.token, body: { spaceType: T, skey: ex, policy: { $type: MEMBERSPACE, publicRead: true } } })
  await call(peer.url, 'com.atproto.memberspace.putMembers', { token: bob.token, body: { space: extSpace, members: sides.map((side) => ({ did: W.sides[side.name].did, read: true, write: true })) } })
  const credExt = await credential(ref, extSpace, peer.url)
  const extEntry = async (side) => {
    const r = await call(peer.url, LW, { headers: signed(credExt, bob.did), params: { space: extSpace } })
    return entryView(r.body?.repos?.find((x) => x.repo === W.sides[side.name].did))
  }
  async function extTracked(kind, wantData) {
    const started = Date.now()
    const done = new Set()
    for (;;) {
      for (const side of sides) {
        if (done.has(side.name)) continue
        const want = await commitOf(side.url, W.sides[side.name].token, extSpace, W.sides[side.name].did)
        if (wantData && want?.data !== wantData) await fail(side, `${kind}: the commit`, {}, { data: wantData }, want)
        if (isDeepStrictEqual(await extEntry(side), want)) done.add(side.name)
      }
      if (done.size === 2) break
      if (Date.now() - started > 15000) { const side = done.has('reference') ? cand : ref; await fail(side, `${kind}: tracked by the space host on the second alpha PDS`, { space: '<the peer\'s space>' }, await commitOf(side.url, W.sides[side.name].token, extSpace, W.sides[side.name].did), await extEntry(side)) }
      await sleep(100)
    }
    const [a, b] = await Promise.all(sides.map(async (side) => (await extEntry(side))?.data))
    if (a !== b) await differ(`${kind}: the data the space host on the second alpha PDS holds`, {}, a, b)
    count(kind)
  }
  const localW = writers.local
  const extWrite = async (kind, nsid, body) => both(kind, nsid, (side) => ({ token: localW.token(side), body: { space: extSpace, repo: localW.did(side), ...body } }), ok200, (g) => ({ status: g.status }))
  await extWrite('createRecord into a space elsewhere', 'com.atproto.space.createRecord', { collection: 'com.example.post', rkey: 'a', record: { $type: 'com.example.post', text: 'a', createdAt: '2026-11-08T00:00:00.000Z' } })
  await extTracked('a write into a space elsewhere')
  await extWrite('createRecord into a space elsewhere again', 'com.atproto.space.createRecord', { collection: 'com.example.post', rkey: 'b', record: { $type: 'com.example.post', text: 'b', createdAt: '2026-11-08T00:00:00.000Z' } })
  await extWrite('deleteRecord in a space elsewhere', 'com.atproto.space.deleteRecord', { collection: 'com.example.post', rkey: 'a' })
  await extTracked('a delete in a space elsewhere')
  await extWrite('clearRecords in a space elsewhere', 'com.atproto.space.clearRecords', {})
  await extTracked('clearRecords in a space elsewhere', EMPTY_DATA)

  // Into the harness space host's spaces: each notification exactly as the page says
  // (spec/SPACES-SYNC.md, "Telling the space host: notifyRepoUpdate, sent").
  const fsp = (key) => `at://${fake.did}/space/${T}/${key}`
  host.answer = () => ({ status: 200, body: {} })
  const sent = (side, key) => host.seen.filter((r) => r.body?.space === fsp(key) && r.body?.repo === W.sides[side.name].did)
  async function sentExactly(kind, key) {
    const started = Date.now()
    const out = {}
    for (const side of sides) {
      for (;;) {
        const want = await commitOf(side.url, W.sides[side.name].token, fsp(key), W.sides[side.name].did)
        const r = sent(side, key).find((x) => x.body?.repoRev === want?.repoRev)
        if (r) {
          const wrongAuth = await checkAuth(r, { iss: W.sides[side.name].did, aud: `${fake.did}#atproto_pds`, lxm: NRU })
          const body = { space: fsp(key), repo: W.sides[side.name].did, repoRev: want.repoRev, data: want.data }
          const gotBody = r.body && typeof r.body === 'object' ? { ...r.body, data: hexOf(r.body.data) ?? r.body.data } : r.body
          if (wrongAuth || r.method !== 'POST' || r.path !== `/xrpc/${NRU}` || !(r.type ?? '').startsWith('application/json') || !isDeepStrictEqual(gotBody, body)) await fail(side, `${kind}: the notification sent`, { space: '<the harness space host\'s space>' }, named(side, { method: 'POST', path: `/xrpc/${NRU}`, body: { ...body, data: `${body.data} (as bytes)` }, auth: wrongAuth ?? 'as the page says' }), named(side, { method: r.method, path: r.path, type: r.type, body: gotBody, auth: wrongAuth ?? 'as the page says' }))
          out[side.name] = r
          break
        }
        if (Date.now() - started > 15000) await fail(side, `${kind}: a notification to the space host`, { space: '<the harness space host\'s space>' }, named(side, want), named(side, sent(side, key).map((x) => x.body)))
        await sleep(100)
      }
    }
    count(kind)
    return out
  }
  const o1 = skey('o')
  await writeBoth(localW, null, 'a write into the harness space host\'s space', () => fsp(o1))
  await sentExactly('notifyRepoUpdate sent to a space host elsewhere', o1)

  // ==== 3. retries across a crash ===========================================================

  // Notifications that fail for now, kept and retried across the restart;
  // the refused, not sent again; deletion notices retried, but not of a space
  // created again (spec/SPACES-SYNC.md, "Delivery and retries", "Deletion",
  // "Notices").
  const keys = { r503: skey('r503-'), r429: skey('r429-'), close: skey('close-'), r400: skey('r400-'), r401: skey('r401-'), r403: skey('r403-') }
  const failing = { [fsp(keys.r503)]: { status: 503, body: { error: 'Unavailable' } }, [fsp(keys.r429)]: { status: 429, body: { error: 'RateLimitExceeded' } }, [fsp(keys.close)]: 'close', [fsp(keys.r400)]: { status: 400, body: { error: 'InvalidRequest' } }, [fsp(keys.r401)]: { status: 401, body: { error: 'AuthenticationRequired' } }, [fsp(keys.r403)]: { status: 403, body: { error: 'Forbidden' } } }
  let healed = false
  host.answer = (r) => (!healed && failing[r.body?.space]) || { status: 200, body: {} }
  // Deletion notices that fail for now: one owed, one of a space created again.
  const [s3, s4] = [skey('d3-'), skey('d4-')]
  const t3 = token()
  for (const key of [s3, s4]) {
    await create('createSpace for a deletion notice retried', key, { $type: MEMBERSPACE, publicRead: true })
    for (const side of sides) {
      syncer.gave.set(`${sp(side, key)} ${t3}`, 'echo')
      // Its first attempt alone fails: the server is killed before the
      // retry is due, so the retry comes from what it kept.
      syncer.failOnce.set(`${sp(side, key)} ${t3}`, { status: 503, body: { error: 'Unavailable' } })
      const r = await call(side.url, AUTHORIZE, { body: { space: sp(side, key), credentialKey: regKey.did, notify: { endpoint: syncer.url, token: t3 } } })
      if (r.status !== 200) await fail(side, 'a registration for a deletion notice retried', {}, 'registered', normalize(side, r))
    }
  }
  // An update to a registration that errs: one attempt (spec/SPACES-SYNC.md,
  // "Sending updates"). Renewed first, so that it is active.
  await register('the registration at the endpoint with a path, renewed', inSa, `${syncer2.url}/base/path?x=`, t2, { p: syncer2, how: null, confirms: 0 })
  for (const side of sides) syncer2.failing.set(`${inSa(side)} ${t2}`, { status: 500, body: { error: 'InternalServerError' } })
  await writeBoth(writers.authority)
  const failedAt = {}
  const firstAttempts = async (what, find) => {
    const started = Date.now()
    for (const side of sides) {
      for (;;) {
        const r = find(side)
        if (r.length) { failedAt[`${side.name} ${what}`] = r.at(-1).at; break }
        if (Date.now() - started > 15000) await fail(side, `${what}: its first attempt`, {}, 'an attempt', 'none')
        await sleep(100)
      }
    }
  }
  for (const [name, key] of Object.entries(keys)) {
    const n = name === 'r503' ? 3 : 1
    for (let i = 0; i < n; i++) await writeBoth(localW, null, 'a write into the harness space host\'s space', () => fsp(key))
    await firstAttempts(name, (side) => sent(side, key).filter((r, _, all) => all.length >= n))
  }
  const latest503 = {}
  for (const side of sides) latest503[side.name] = (await commitOf(side.url, W.sides[side.name].token, fsp(keys.r503), W.sides[side.name].did)).repoRev
  for (const side of sides) for (const key of [s3, s4]) {
    const r = await call(side.url, 'com.atproto.space.deleteSpace', { token: auth(side).token, body: { space: sp(side, key) } })
    if (r.status !== 200) await fail(side, 'deleteSpace for a deletion notice retried', {}, 'deleted', normalize(side, r))
  }
  const deletionNotices = (key) => (side) => syncer.updates(sp(side, key), t3).filter((r) => r.body?.update?.$type === SPACE_DELETED)
  await firstAttempts('s3', deletionNotices(s3))
  await firstAttempts('s4', deletionNotices(s4))
  await firstAttempts('update', (side) => syncer2.updates(inSa(side), t2).filter((r) => r.answered === 500))
  await create('createSpace of a space owed a notice, again', s4, { $type: MEMBERSPACE })

  // After the run restarts the server: everything answers 200, and each
  // retry is judged against the time of its failure.
  afterRestart.push(async () => {
    healed = true
    syncer.failing.clear()
    syncer2.failing.clear()
    const since = (find, name) => (side) => find(side).filter((r) => r.at > failedAt[`${side.name} ${name}`])
    const toHost = (key, name) => since((side) => host.seen.filter((r) => r.body?.space === fsp(key) && r.body?.repo === W.sides[side.name].did), name)
    const expectations = [
      ['the 503 space\'s latest notification', toHost(keys.r503, 'r503'), 1, 'r503', (r, side) => r.body.repoRev === latest503[side.name]],
      ['the 429 notification', toHost(keys.r429, 'r429'), 1, 'r429'],
      ['the closed notification', toHost(keys.close, 'close'), 1, 'close'],
      ['the 400 notification', toHost(keys.r400, 'r400'), 0, 'r400'],
      ['the 401 notification', toHost(keys.r401, 'r401'), 0, 'r401'],
      ['the 403 notification', toHost(keys.r403, 'r403'), 0, 'r403'],
      ['the owed deletion notice', since(deletionNotices(s3), 's3'), 1, 's3'],
      ['the deletion notice of a space created again', since(deletionNotices(s4), 's4'), 0, 's4'],
      ['the update the registration refused', since((side) => syncer2.updates(inSa(side), t2), 'update'), 0, 'update'],
    ]
    const last = Math.max(...Object.values(failedAt))
    for (;;) {
      const pending = sides.some((side) => expectations.some(([, find, n]) => find(side).length < n))
      if ((!pending && Date.now() - last > 75000) || Date.now() - last > 100000) break
      await sleep(500)
    }
    for (const side of sides) {
      for (const [what, find, n, name, ok] of expectations) {
        const got = find(side)
        const failed = failedAt[`${side.name} ${name}`]
        const wrong = got.length !== n ? `${n} after the failure` : got.find((r) => r.answered !== 200 || r.at - failed < 25000 || (ok && !ok(r, side)))
        if (wrong) await fail(side, `a retry: ${what}`, { within: '100 seconds of the failure, across the server\'s restart' }, typeof wrong === 'string' ? wrong : 'a retry at least 25 seconds after the failure, of the latest state', named(side, got.map((r) => ({ after: `${Math.round((r.at - failed) / 1000)} s`, body: { ...r.body, token: r.body?.token && '<…>' }, answered: r.answered }))))
      }
      count('retries across a crash')
    }
    syncer.close(); syncer2.close(); host.close()
  })
}