chadtmiller.com/monolith
An atproto PDS
0
0
// Spaces' write notifications, writer set and syncers (spec/SPACES-SYNC.md),
// in the spaces run. Parties are played here, each answering as a check
// needs and recording what it is sent: two syncers, HTTP servers that confirm
// the tokens the harness gave them, per space, and take notifySpaceUpdate; a
// writer whose notifications the harness signs, and a space host for an
// authority of its own, each a did:plc in the directory with #atproto_pds at
// an HTTP server of the harness.
//
// 1. the server as space host: the authority on each side (the first alpha's
// or the server's), writers on the second alpha PDS, on the same side,
// and the harness's writer; registrations and their confirmation,
// notifications taken, removals, listWriters, the updates sent to each
// registration, a refusal dropping a registration, and deletion notices;
// 2. the server as repo host: a writer on each side writes into a space on
// the second alpha PDS, and into the harness space host's spaces;
// 3. retries across a crash: failures arranged here, before the run's
// restart of the server, and judged after it (`afterRestart`).
//
// Requests go to both sides alike; answers are compared with each side's
// accounts named alike, and each side is held to the page, the reference
// included, which checks the check. Every token and key name is drawn from
// the run's `rand`.
import crypto from 'node:crypto'
import { createServer } from 'node:http'
import { once } from 'node:events'
import { isDeepStrictEqual } from 'node:util'
import * as plc from '@did-plc/lib'
import { Secp256k1Keypair, verifySignature } from '@atproto/crypto'
import * as S from './lib/spec.mjs'
const T = 'com.example.group'
const NRU = 'com.atproto.space.notifyRepoUpdate'
const NSU = 'com.atproto.space.notifySpaceUpdate'
const CONFIRM = 'com.atproto.space.confirmRegistration'
const AUTHORIZE = 'com.atproto.space.authorizeSpaceSync'
const LW = 'com.atproto.space.listWriters'
const REPO_WRITE = 'com.atproto.space.defs#repoWrite'
const REPO_REMOVED = 'com.atproto.space.defs#repoRemoved'
const SPACE_DELETED = `${NSU}#spaceDeleted`
const MEMBERSPACE = 'com.atproto.memberspace.policy'
const TID = /^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/
/// A confirmation's challenge: 16 bytes in lower-case base32, unpadded.
const CHALLENGE = /^[a-z2-7]{26}$/
const EMPTY_DATA = 'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad'
/// The order of secp256k1, to make a high-S signature of a low-S one.
const N = BigInt('0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141')
const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
const now = () => Math.floor(Date.now() / 1000)
const b64 = (o) => Buffer.from(JSON.stringify(o)).toString('base64url')
const unb64 = (s) => JSON.parse(Buffer.from(s, 'base64url').toString('utf8'))
const hexOf = (bytes) => (bytes?.$bytes ? Buffer.from(bytes.$bytes, 'base64').toString('hex') : null)
const keysAre = (o, keys) => isDeepStrictEqual(Object.keys(o ?? {}).sort(), [...keys].sort())
/// The TID of a time in milliseconds, with a clock id.
function tid(ms, clock = 0) {
let n = (BigInt(Math.floor(ms)) * 1000n << 10n) | BigInt(clock)
let s = ''
for (let i = 0; i < 13; i++) { s = '234567abcdefghijklmnopqrstuvwxyz'[Number(n & 31n)] + s; n >>= 5n }
return s
}
export async function checkSpaceSync(h) {
const { net, ref, cand, accounts, createAccount, normalize, differ, fail, count, rand, front, afterRestart } = h
const sides = [ref, cand]
const peer = net.extraPdses[0]
const plcUrl = net.plc.url
const skey = (p) => `${p}${Math.floor(rand() * 1e6)}`
/// A syncer's token: `n` bytes of the run's seed, base64url.
const token = (n = 32) => Buffer.from(Array.from({ length: n }, () => Math.floor(rand() * 256))).toString('base64url')
// ---- requests ------------------------------------------------------------------
async function call(base, nsid, { params, body, token, headers = {} } = {}) {
const url = new URL(`${base}/xrpc/${nsid}`)
for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v)
const hs = { ...headers }
if (token) hs.authorization = `Bearer ${token}`
if (body !== undefined) hs['content-type'] = 'application/json'
const res = await fetch(url, { method: body === undefined ? 'GET' : 'POST', headers: hs, body: body === undefined ? undefined : JSON.stringify(body) })
const buf = Buffer.from(await res.arrayBuffer())
let parsed = null
if (buf.length) try { parsed = JSON.parse(buf.toString('utf8')) } catch { parsed = { unparsed: buf.toString('utf8').slice(0, 200) } }
return { status: res.status, type: res.headers.get('content-type'), buf, body: parsed }
}
function alias(side, s) {
for (const a of accounts) if (a.sides[side.name].did) s = s.split(a.sides[side.name].did).join(`<did:${a.n}>`)
return s
}
const named = (side, v) => JSON.parse(alias(side, JSON.stringify(v ?? null)))
const describe = (side, req) => named(side, JSON.parse(JSON.stringify(req, (k, v) => (k === 'token' || k === 'delegationToken' || k === 'authorization' || k === 'signature' ? '<…>' : v))))
function shape(side, res) {
if (res.status >= 400) return normalize(side, res)
if (res.status === 200 && !res.buf.length) return { status: 200, empty: true }
return normalize(side, res)
}
const err = (error, status = 400) => ({ status, error })
const EMPTY = { status: 200, empty: true }
/// The same request to both sides, compared as `view` shows them; each side
/// held to `want` (an object, or a function answering what is wrong).
async function both(kind, nsid, make, want, view = (g) => g, base = (side) => side.url) {
const res = await Promise.all(sides.map((side) => call(base(side), nsid, make(side))))
const got = sides.map((side, i) => shape(side, res[i]))
for (const [i, side] of sides.entries()) {
const wrong = want === undefined ? null : typeof want === 'function' ? want(got[i], res[i], side) : isDeepStrictEqual(got[i], want) ? null : want
if (wrong) await fail(side, `${kind}: ${nsid}`, describe(side, make(side)), typeof wrong === 'string' ? wrong : named(side, wrong), got[i])
}
const [a, b] = got.map(view)
if (!isDeepStrictEqual(a, b)) await differ(`${kind}: ${nsid}`, describe(cand, make(cand)), a, b)
count(kind)
return res
}
const ok200 = (g) => (g.status === 200 ? null : 'answered 200')
// ---- the harness's parties -----------------------------------------------------------
/// An HTTP server of the harness: records each request; answers as
/// `answer`: `{status, body}` as JSON, `{status, raw, type?}`, or 'close'.
async function party() {
const p = { seen: [], answer: () => ({ status: 200, body: {} }) }
const srv = createServer((req, res) => {
const chunks = []
req.on('data', (c) => chunks.push(c))
req.on('end', async () => {
const u = new URL(req.url, 'http://x')
let body = null
try { body = JSON.parse(Buffer.concat(chunks).toString('utf8') || 'null') } catch { body = '<not JSON>' }
const r = { at: Date.now(), method: req.method, path: u.pathname, type: req.headers['content-type'] ?? null, authorization: req.headers.authorization ?? null, body }
const a = p.answer(r)
r.answered = a === 'close' ? 'close' : a.status
p.seen.push(r)
if (a === 'close') return req.socket.destroy()
if ('raw' in a) return res.writeHead(a.status, a.type ? { 'content-type': a.type } : {}).end(a.raw)
res.writeHead(a.status, { 'content-type': 'application/json' }).end(JSON.stringify(a.body))
})
})
srv.listen(0, '127.0.0.1')
await once(srv, 'listening')
p.port = srv.address().port
p.url = `http://localhost:${p.port}`
p.close = () => srv.close()
return p
}
/// A syncer: confirms a registration of the tokens it gave, per space, as
/// `gave` says ('echo', or a way to fail), refusing any other; answers
/// notifySpaceUpdate 200 with no body, or as `failing` says.
async function syncerParty() {
const s = await party()
s.gave = new Map()
s.failing = new Map()
s.failOnce = new Map()
s.answer = (r) => {
const k = `${r.body?.space} ${r.body?.token}`
if (r.path === `/xrpc/${CONFIRM}`) {
const how = s.gave.get(k)
const challenge = r.body?.challenge
if (how === undefined) return { status: 400, body: { error: 'InvalidRequest', message: 'not a token this syncer gave for the space' } }
if (how === 'echo') return { status: 200, body: { challenge } }
if (how === 'extra') return { status: 200, body: { challenge, also: true } }
if (how === 'wrong') return { status: 200, body: { challenge: `${challenge}x` } }
if (how === 'text') return { status: 200, type: 'text/plain', raw: JSON.stringify({ challenge }) }
if (how === 'empty') return { status: 200, raw: '' }
if (how === 'close') return 'close'
return { status: how, body: { error: 'InternalServerError' } }
}
if (r.path === `/xrpc/${NSU}`) {
const once = s.failOnce.get(k)
if (once) { s.failOnce.delete(k); return once }
return s.failing.get(k) ?? { status: 200, raw: '' }
}
return { status: 404, body: { error: 'MethodNotImplemented' } }
}
/// What it was asked for a space and token, but updates.
s.asked = (space, tok) => s.seen.filter((r) => r.path !== `/xrpc/${NSU}` && r.body?.space === space && r.body?.token === tok)
s.updates = (space, tok) => s.seen.filter((r) => r.path === `/xrpc/${NSU}` && r.body?.space === space && r.body?.token === tok)
return s
}
async function plcDid(services) {
const key = await Secp256k1Keypair.create({ exportable: true })
const op = await plc.signOperation({ type: 'plc_operation', rotationKeys: [key.did()], alsoKnownAs: [], verificationMethods: { atproto: key.did() }, services, prev: null }, key)
const did = await plc.didForCreateOp(op)
await new plc.Client(plcUrl).sendOperation(did, op)
return { did, key }
}
const keyOf = async (did) => (await (await fetch(`${plcUrl}/${did}/data`)).json()).verificationMethods?.atproto
/// A request's service auth as the page says it must be: what is wrong, or null.
async function checkAuth(r, { iss, aud, lxm }) {
const parts = (r.authorization ?? '').replace(/^Bearer /, '').split('.')
let header, claims
try { [header, claims] = parts.slice(0, 2).map(unb64) } catch { return 'service auth that is a JWT' }
if (!isDeepStrictEqual(header, { typ: 'JWT', alg: 'ES256K' })) return { header: { typ: 'JWT', alg: 'ES256K' } }
if (!keysAre(claims, ['aud', 'exp', 'iat', 'iss', 'jti', 'lxm']) || claims.iss !== iss || claims.aud !== aud || claims.lxm !== lxm || claims.exp - claims.iat !== 60 || typeof claims.jti !== 'string' || !claims.jti)
return { claims: { iss, aud, lxm, iat: '<n>', exp: '<iat + 60>', jti: '<random>' } }
const key = await keyOf(iss)
const signed = parts.length === 3 && (await verifySignature(key, Buffer.from(`${parts[0]}.${parts[1]}`), Buffer.from(parts[2], 'base64url')).catch(() => false))
return signed ? null : `a signature by ${iss}'s #atproto key, low-S`
}
const syncer = await syncerParty()
const syncer2 = await syncerParty()
const host = await party()
const writerH = await plcDid({ atproto_pds: { type: 'AtprotoPersonalDataServer', endpoint: host.url } })
const fake = await plcDid({ atproto_pds: { type: 'AtprotoPersonalDataServer', endpoint: host.url } })
// ---- accounts --------------------------------------------------------------------
const peerDomain = (await (await fetch(`${peer.url}/xrpc/com.atproto.server.describeServer`)).json()).availableUserDomains[0]
async function peerAccount() {
const f = front()
const r = await call(peer.url, 'com.atproto.server.createAccount', { body: { handle: `${f}${peerDomain}`, email: `${f}@example.com`, password: `pw-${f}` } })
if (r.status !== 200) await differ('a peer account on the second alpha PDS', { handle: `${f}${peerDomain}` }, 'an account', r.body, 'the harness could not make a peer')
return { did: r.body.did, token: r.body.accessJwt }
}
const [bob, carol, dave] = [await peerAccount(), await peerAccount(), await peerAccount()]
const A = await createAccount()
const W = await createAccount()
const of = (acct) => (side) => acct.sides[side.name]
const auth = of(A)
const sp = (side, key) => `at://${auth(side).did}/space/${T}/${key}`
/// A peer account's delegation token for a space, from the second alpha PDS.
async function delegation(user, space) {
const r = await call(peer.url, 'com.atproto.space.getDelegationToken', { token: user.token, params: { space } })
if (r.status !== 200) await differ('the second alpha PDS\'s getDelegationToken', { space }, 'a token', r.body, 'the harness could not get a delegation token')
return r.body.token
}
/// A space credential for each side's space, from its authority, asked for
/// with no delegation token (the space reads publicly) unless `extra` has
/// one, with its holder's key.
async function credential(side, space, authorityHost = side.url, extra = {}) {
const key = S.newP256()
const res = await call(authorityHost, AUTHORIZE, { body: { space, credentialKey: key.did, ...extra } })
if (res.status !== 200) await fail(side, 'authorizeSpaceSync for a credential', { space: alias(side, space) }, 'a credential', normalize(side, res))
return { key, credential: res.body.credential, spaceRev: res.body.spaceRev }
}
const signed = (c, audience) => S.signCredential(c.key, c.credential, audience)
const create = (kind, key, policy) => both(kind, 'com.atproto.space.createSpace', (side) => ({ token: auth(side).token, body: { spaceType: T, skey: key, policy } }), ok200)
/// putMembers of `[did, read, write]`s, a `did` a DID or a DID of the side.
const putMembers = (kind, key, members) => both(kind, 'com.atproto.memberspace.putMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, key), members: members.map(([did, read, write]) => ({ did: typeof did === 'function' ? did(side) : did, read, write })) } }), EMPTY)
/// A writer's latest commit in a space, from its own host.
async function commitOf(hostUrl, token, space, repo) {
const r = await call(hostUrl, 'com.atproto.space.getLatestCommit', { token, params: { space, repo } })
return r.status === 200 ? { repoRev: r.body.commit.rev, data: hexOf(r.body.commit.data) } : null
}
async function write(hostUrl, token, space, repo, rkey) {
return call(hostUrl, 'com.atproto.space.createRecord', { token, body: { space, repo, collection: 'com.example.post', rkey, record: { $type: 'com.example.post', text: rkey, createdAt: '2026-11-08T00:00:00.000Z' } } })
}
/// The same record written by `wr` on each side, into the side's space
/// `key`, or into `space(side)`, so that both sides' repos hold alike.
async function writeBoth(wr, key = sa, kind = 'a write', space = (side) => sp(side, key)) {
const rkey = `r${Math.floor(rand() * 1e9)}`
for (const side of sides) {
const r = await write(wr.host(side), wr.token(side), space(side), wr.did(side), rkey)
if (r.status !== 200) await fail(side, kind, { space: alias(side, space(side)) }, 'written', normalize(side, r))
}
}
// ==== 1. the server as space host ==========================================================
const sa = skey('sync')
await create('createSpace for notifications', sa, { $type: MEMBERSPACE, publicRead: true })
await putMembers('putMembers of writers and a reader', sa, [[bob.did, true, true], [writerH.did, true, true], [(side) => W.sides[side.name].did, true, true], [carol.did, true, false]])
const cred = {}
for (const side of sides) cred[side.name] = await credential(side, sp(side, sa))
for (const side of sides) if (cred[side.name].spaceRev !== undefined) await fail(side, 'authorizeSpaceSync of a space no one wrote to', { space: alias(side, sp(side, sa)) }, 'no spaceRev', cred[side.name].spaceRev)
const withCred = (side, extra = {}, aud = auth(side).did, c = cred[side.name]) => ({ headers: signed(c, aud), ...extra })
/// listWriters of a space, held to the page per side: entries in
/// ascending spaceRev, each a #repoWrite or a #repoRemoved
/// (spec/SPACES-SYNC.md, "The writer set: listWriters").
async function listAll(side, space, c = cred[side.name]) {
const r = await call(side.url, LW, { headers: signed(c, auth(side).did), params: { space } })
if (r.status !== 200) await fail(side, 'listWriters', { space: alias(side, space) }, 'the writer set', normalize(side, r))
const repos = r.body.repos
for (const [i, e] of repos.entries()) {
const ok = e?.$type === REPO_WRITE ? keysAre(e, ['$type', 'data', 'repo', 'repoRev', 'spaceRev']) && TID.test(e.repoRev) && hexOf(e.data) !== null : e?.$type === REPO_REMOVED && keysAre(e, ['$type', 'repo', 'spaceRev'])
if (!ok || !TID.test(e.spaceRev) || (i && repos[i - 1].spaceRev >= e.spaceRev)) await fail(side, 'listWriters entries, in ascending spaceRev', { space: alias(side, space) }, `{$type: "${REPO_WRITE}", repo, repoRev, data, spaceRev} or {$type: "${REPO_REMOVED}", repo, spaceRev}, spaceRev ascending`, named(side, repos))
}
if ((repos.length ? r.body.cursor !== repos.at(-1).spaceRev : 'cursor' in r.body)) await fail(side, 'listWriters cursor', { space: alias(side, space) }, repos.length ? repos.at(-1).spaceRev : 'none', r.body.cursor ?? 'none')
return repos
}
/// A writer's entry as the checks compare it.
const entryView = (e) => (!e ? null : e.$type === REPO_REMOVED ? { removed: true } : { repoRev: e.repoRev, data: hexOf(e.data) })
/// Wait until each side's writer set shows `did(side)` as `want(side)`.
async function tracked(kind, space, did, want, c = (side) => cred[side.name]) {
const started = Date.now()
const done = new Set()
const last = {}
for (;;) {
for (const side of sides) {
if (done.has(side.name)) continue
last[side.name] = entryView((await listAll(side, space(side), c(side))).find((e) => e.repo === did(side)))
if (isDeepStrictEqual(last[side.name], await want(side))) done.add(side.name)
}
if (done.size === 2) break
if (Date.now() - started > 15000) { const side = done.has('reference') ? cand : ref; await fail(side, `${kind}: the writer set`, { space: alias(side, space(side)), writer: alias(side, did(side)) }, await want(side), last[side.name]) }
await sleep(100)
}
count(kind)
}
await both('listWriters of no writers', LW, (side) => withCred(side, { params: { space: sp(side, sa) } }), { status: 200, body: { repos: [] } })
// Registrations: each new one confirmed once, at the endpoint's origin,
// with the token in the body and no Authorization; the same pair again
// renewed (spec/SPACES-SYNC.md, "Syncers", "Registering";
// spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync").
const regKey = S.newP256()
const challenges = new Set()
const registered = (g, r) => (r.status === 200 && Object.keys(r.body ?? {}).every((k) => k === 'credential' || k === 'spaceRev') && typeof r.body.credential === 'string' && (r.body.spaceRev === undefined || TID.test(r.body.spaceRev)) ? null : '{credential, spaceRev?}')
const asAuthorized = (g) => (g.status === 200 ? { status: 200, credential: typeof g.body?.credential === 'string', spaceRev: g.body?.spaceRev === undefined ? 'absent' : '<tid>' } : g)
/// What is wrong with a confirmation the side sent, or null.
function confirmWrong(r) {
if (r.method !== 'POST' || r.path !== `/xrpc/${CONFIRM}` || !(r.type ?? '').startsWith('application/json') || r.authorization) return `a POST of JSON to /xrpc/${CONFIRM}, with no Authorization`
if (!keysAre(r.body, ['challenge', 'space', 'token'])) return 'a body of exactly {space, token, challenge}'
if (!CHALLENGE.test(r.body.challenge) || challenges.has(r.body.challenge)) return 'a challenge of 16 random bytes in base32, lower case, unpadded, new each time'
challenges.add(r.body.challenge)
return null
}
/// authorizeSpaceSync with `notify` on both sides, the syncer `p` having
/// given `tok` for each side's space as `how` (null: as it was); the
/// answer held to `want`, and `confirms` new requests at `p` for the pair,
/// each a confirmation as the page says.
async function register(kind, space, endpoint, tok, { p = syncer, how = 'echo', want = registered, confirms = 1, extra = () => ({}) } = {}) {
if (how !== null) for (const side of sides) p.gave.set(`${space(side)} ${tok}`, how)
const before = Object.fromEntries(sides.map((side) => [side.name, p.asked(space(side), tok).length]))
const res = await both(kind, AUTHORIZE, (side) => ({ body: { space: space(side), credentialKey: regKey.did, notify: { endpoint: typeof endpoint === 'function' ? endpoint(side) : endpoint, token: tok }, ...extra(side) } }), want, asAuthorized)
for (const side of sides) {
const got = p.asked(space(side), tok).slice(before[side.name])
const wrong = got.length !== confirms ? `${confirms} confirmation${confirms === 1 ? '' : 's'} at the syncer` : got.map(confirmWrong).find(Boolean)
if (wrong) await fail(side, `${kind}: confirmRegistration`, { space: alias(side, space(side)) }, wrong, named(side, got.map((r) => ({ method: r.method, path: r.path, type: r.type, authorization: r.authorization, body: { ...r.body, token: '<…>' } }))))
}
count('registrations and their confirmation')
return res
}
const inSa = (side) => sp(side, sa)
const [t1, t2, t4, t5] = [token(), token(), token(), token(384)]
await register('a registration', inSa, syncer.url, t1)
await register('the same pair again, renewed', inSa, syncer.url, t1, { confirms: 0 })
// Its endpoint has a path and a query, which are not used.
await register('a registration at an endpoint with a path and a query', inSa, `${syncer2.url}/base/path?x=`, t2, { p: syncer2 })
// The same token, the endpoint written otherwise: another registration.
await register('the same token at the endpoint with a / after it', inSa, `${syncer.url}/`, t1, { how: null })
await register('a registration answered with another member besides the challenge', inSa, syncer.url, t4, { how: 'extra' })
await register('a registration with a 512-byte token', inSa, syncer2.url, t5, { p: syncer2 })
const NOT_CONFIRMED = err('RegistrationNotConfirmed')
const refusedTokens = []
for (const [what, how] of [['a wrong challenge', 'wrong'], ['500', 500], ['text/plain', 'text'], ['an empty 200', 'empty'], ['a closed connection', 'close'], ['a token the syncer did not give', undefined]]) {
const tok = token()
refusedTokens.push(tok)
if (how === undefined) for (const side of sides) syncer.gave.delete(`${inSa(side)} ${tok}`)
await register(`a registration answered with ${what}`, inSa, syncer.url, tok, { how: how ?? null, want: NOT_CONFIRMED })
}
for (const [what, endpoint] of [['ftp://', `ftp://localhost:${syncer.port}`], ['with a user and password', `http://u:p@localhost:${syncer.port}`], ['a DID', `did:plc:${'a'.repeat(24)}`]]) {
const tok = token()
refusedTokens.push(tok)
await register(`a registration at an endpoint ${what}`, inSa, endpoint, tok, { want: NOT_CONFIRMED, confirms: 0 })
}
for (const [what, notify] of [['an endpoint that is no URI', { endpoint: 'not a uri', token: token() }], ['a 513-byte token', { endpoint: syncer.url, token: `${token(384)}x` }], ['no token', { endpoint: syncer.url }]])
await both(`a registration with ${what}`, AUTHORIZE, (side) => ({ body: { space: inSa(side), credentialKey: regKey.did, notify } }), err('InvalidRequest'))
// Refused before step 10: nothing is confirmed (spec/SPACES-CREDENTIALS.md,
// "authorizeSpaceSync").
const sm = skey('members')
await create('createSpace, members only', sm, { $type: MEMBERSPACE })
await putMembers('putMembers of the peer writer', sm, [[bob.did, true, true]])
const inSm = (side) => sp(side, sm)
const away = skey('away')
const peerSpace = `at://${bob.did}/space/${T}/${away}`
await call(peer.url, 'com.atproto.space.createSpace', { token: bob.token, body: { spaceType: T, skey: away, policy: { $type: MEMBERSPACE, publicRead: true } } })
for (const [what, space, want, extra] of [
['of a members-only space with no token', inSm, err('SpaceNotFound')],
['of a space never made', (side) => sp(side, skey('none')), err('SpaceNotFound')],
['of a space whose authority is elsewhere', () => peerSpace, err('SpaceNotFound')],
['with a credentialKey that is no key', inSa, err('InvalidRequest'), () => ({ credentialKey: 'nope' })],
]) {
const tok = token()
refusedTokens.push(tok)
await register(`a registration ${what}`, space, syncer.url, tok, { want, confirms: 0, extra })
}
const regsA = [[syncer, t1, 'the registration and its twin, at the endpoint with a /', 2], [syncer2, t2, 'the endpoint with a path'], [syncer, t4, 'the answer with another member'], [syncer2, t5, 'the 512-byte token']]
// Writes tracked, and sent: a peer's notified over HTTP, the same side's
// taken directly (spec/SPACES-SYNC.md, "The writer set", "Telling the space
// host: notifyRepoUpdate, sent", "For other slices").
const writers = {
peer: { did: () => bob.did, host: () => peer.url, token: () => bob.token },
local: { did: (side) => W.sides[side.name].did, host: (side) => side.url, token: (side) => W.sides[side.name].token },
authority: { did: (side) => auth(side).did, host: (side) => side.url, token: (side) => auth(side).token },
}
const inSaOf = (wr) => (side) => commitOf(wr.host(side), wr.token(side), sp(side, sa), wr.did(side))
for (const [name, wr] of Object.entries(writers)) {
await writeBoth(wr, sa, `a write by the ${name} writer`)
await tracked(`a write by the ${name} writer, tracked`, inSa, wr.did, inSaOf(wr))
}
const reader = { did: () => carol.did, host: () => peer.url, token: () => carol.token }
await writeBoth(reader)
await writeBoth(writers.peer)
await tracked('a later write, tracked', inSa, writers.peer.did, inSaOf(writers.peer))
await sleep(300)
/// The writer set compared across sides, by writer, data and removal.
async function setIs(kind, key, wantDids) {
const views = []
for (const side of sides) {
const repos = await listAll(side, sp(side, key))
const dids = repos.filter((e) => e.$type === REPO_WRITE).map((e) => e.repo).sort()
const want = wantDids.map((d) => d(side)).sort()
if (!isDeepStrictEqual(dids, want)) await fail(side, `${kind}: who is in the writer set`, { space: alias(side, sp(side, key)) }, named(side, want), named(side, dids))
views.push(named(side, repos.map((e) => ({ repo: e.repo, ...entryView(e) }))).map(({ repoRev, ...v }) => v).sort((x, y) => (x.repo < y.repo ? -1 : 1)))
}
if (!isDeepStrictEqual(views[0], views[1])) await differ(`${kind}: ${LW}`, { space: '<space>' }, views[0], views[1])
count(kind)
}
await setIs('the writer set, a reader not in it', sa, [writers.peer.did, writers.local.did, writers.authority.did])
// The harness writer's notifications, signed here (spec/SPACES-SYNC.md,
// "Taking a notification: notifyRepoUpdate, served").
const sentCount = (side) => syncer.updates(sp(side, sa), t4).length
let revAt = Date.now()
const nextRev = () => tid((revAt = Math.max(revAt + 1000, Date.now())))
const notifyJwt = async (side, claims = {}, header = { typ: 'JWT', alg: 'ES256K' }, { key = writerH.key, highS = false } = {}) => {
const c = { iss: writerH.did, aud: `${auth(side).did}#atproto_pds`, lxm: NRU, iat: now(), exp: now() + 60, jti: crypto.randomBytes(16).toString('hex'), ...claims }
for (const k of Object.keys(c)) if (c[k] === undefined) delete c[k]
const input = `${b64(header)}.${b64(c)}`
let sig = Buffer.from(await key.sign(new Uint8Array(Buffer.from(input))))
if (highS) { const s = BigInt(`0x${sig.subarray(32).toString('hex')}`); sig = Buffer.concat([sig.subarray(0, 32), Buffer.from((N - s).toString(16).padStart(64, '0'), 'hex')]) }
return `${input}.${sig.toString('base64url')}`
}
const data = (n) => ({ $bytes: Buffer.alloc(32, n).toString('base64') })
/// A notification from the harness writer to each side: `body(side)` merged
/// over the usual, `claims(side)` over the usual service auth.
async function notifyEach(kind, want, { body = () => ({}), claims = () => ({}), header, key, highS, authorization } = {}) {
const rev = nextRev()
const jwt = {}
for (const side of sides) jwt[side.name] = await notifyJwt(side, claims(side), header, { key, highS })
return both(kind, NRU, (side) => ({ headers: { authorization: authorization ?? `Bearer ${jwt[side.name]}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: rev, data: data(1), ...body(side) } }), want)
}
const hEntry = async (side) => entryView((await listAll(side, sp(side, sa))).find((x) => x.repo === writerH.did))
const hState = {}
/// A notification the page says is taken: the harness writer's entry then
/// holds the `data` sent, as hex `wantHex`.
const takes = async (kind, opts, dataN, wantHex = Buffer.alloc(32, dataN).toString('hex')) => {
await notifyEach(kind, EMPTY, { ...opts, body: (side) => ({ data: data(dataN), ...(opts.body?.(side) ?? {}) }) })
for (const side of sides) { const e = await hEntry(side); if (e?.data !== wantHex) await fail(side, `${kind}: the writer set`, { writer: 'the harness writer' }, { data: wantHex }, e); hState[side.name] = e }
}
await takes('notifyRepoUpdate taken', {}, 1)
await takes('notifyRepoUpdate with a bare aud', { claims: (side) => ({ aud: auth(side).did }) }, 2)
await takes('notifyRepoUpdate signed high-S', { highS: true }, 3)
await takes('notifyRepoUpdate with spaceRev and prevSpaceRev sent', { body: () => ({ spaceRev: tid(Date.now()), prevSpaceRev: tid(Date.now() - 1000) }) }, 4)
for (const [what, opts, want] of [
['to another fragment', { claims: (side) => ({ aud: `${auth(side).did}#other` }) }, err('Forbidden', 403)],
['to another DID', { claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('Forbidden', 403)],
['from another issuer than repo', { body: () => ({ repo: bob.did }) }, err('Forbidden', 403)],
['from an issuer with a fragment', { claims: () => ({ iss: `${writerH.did}#x` }) }, err('Forbidden', 403)],
['for another method', { claims: () => ({ lxm: LW }) }, err('BadJwtLexiconMethod', 401)],
['with no lxm', { claims: () => ({ lxm: undefined }) }, err('BadJwtLexiconMethod', 401)],
['expired', { claims: () => ({ exp: now() - 10 }) }, err('JwtExpired', 401)],
['signed by another key', { key: await Secp256k1Keypair.create() }, err('BadJwtSignature', 401)],
['of type at+jwt', { header: { typ: 'at+jwt', alg: 'ES256K' } }, err('BadJwtType', 401)],
['with no token', { authorization: '' }, err('MissingJwt', 401)],
['with a bad repoRev', { body: () => ({ repoRev: 'nope' }) }, err('InvalidRequest')],
['6 minutes ahead', { body: () => ({ repoRev: tid(Date.now() + 6 * 60000) }) }, err('FutureRev')],
['of a space never created', { body: (side) => ({ space: sp(side, skey('none')) }) }, err('SpaceNotFound')],
['of a space never created, 6 minutes ahead', { body: (side) => ({ space: sp(side, skey('none')), repoRev: tid(Date.now() + 6 * 60000) }) }, err('FutureRev')],
['of a space whose authority is elsewhere', { body: () => ({ space: peerSpace }), claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('SpaceNotFound')],
['of a space whose authority is elsewhere, 6 minutes ahead', { body: () => ({ space: peerSpace, repoRev: tid(Date.now() + 6 * 60000) }), claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('SpaceNotFound')],
]) await notifyEach(`notifyRepoUpdate ${what}`, want, opts)
for (const side of sides) if (!isDeepStrictEqual(await hEntry(side), hState[side.name])) await fail(side, 'refused notifications change nothing', { writer: 'the harness writer' }, hState[side.name], await hEntry(side))
// Stale and equal revs: taken, nothing changes, nothing sent.
const before = Object.fromEntries(sides.map((side) => [side.name, sentCount(side)]))
for (const side of sides) side.jwt = await notifyJwt(side)
await both('notifyRepoUpdate of the same rev', NRU, (side) => ({ headers: { authorization: `Bearer ${side.jwt}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: hState[side.name].repoRev, data: data(9) } }), EMPTY)
for (const side of sides) side.jwt = await notifyJwt(side)
await both('notifyRepoUpdate of an earlier rev', NRU, (side) => ({ headers: { authorization: `Bearer ${side.jwt}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: tid(Date.now() - 60000), data: data(9) } }), EMPTY)
for (const side of sides) delete side.jwt
await writeBoth(writers.peer)
await tracked('a write after stale notifications, tracked', inSa, writers.peer.did, inSaOf(writers.peer))
await sleep(300)
for (const side of sides) {
if (!isDeepStrictEqual(await hEntry(side), hState[side.name])) await fail(side, 'a stale notification changes nothing', { writer: 'the harness writer' }, hState[side.name], await hEntry(side))
if (sentCount(side) !== before[side.name] + 1) await fail(side, 'a stale notification is not sent', { space: alias(side, sp(side, sa)) }, `${before[side.name] + 1} updates (the peer's write alone)`, sentCount(side))
}
// 4 minutes ahead is taken; odd data too.
revAt = Date.now() + 4 * 60000
await takes('notifyRepoUpdate 4 minutes ahead', {}, 5)
await takes('notifyRepoUpdate of 31 bytes', { body: () => ({ data: { $bytes: Buffer.alloc(31, 6).toString('base64') } }) }, 6, Buffer.alloc(31, 6).toString('hex'))
// Removals, each sequenced and sent; nothing for a writer not in the set
// (spec/SPACES-SYNC.md, "Removing a writer").
const removed = async () => ({ removed: true })
const quiet = async (kind, run) => {
const at = Object.fromEntries(sides.map((side) => [side.name, sentCount(side)]))
await run()
await sleep(500)
for (const side of sides) if (sentCount(side) !== at[side.name]) await fail(side, `${kind}: nothing sent`, { space: alias(side, sp(side, sa)) }, `${at[side.name]} updates`, sentCount(side))
}
await both('removeWriter of the peer writer', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: bob.did } }), EMPTY)
await tracked('removeWriter, shown removed', inSa, writers.peer.did, removed)
await quiet('removeWriter of it again', () => both('removeWriter of it again', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: bob.did } }), EMPTY))
await quiet('removeWriter of an account never in the set', () => both('removeWriter of an account never in the set', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: carol.did } }), EMPTY))
await writeBoth(writers.peer)
await tracked('a removed writer\'s next write, tracked again', inSa, writers.peer.did, inSaOf(writers.peer))
await putMembers('putMembers taking the peer writer\'s write away', sa, [[bob.did, true, false]])
await tracked('putMembers taking a writer\'s write away, shown removed', inSa, writers.peer.did, removed)
await both('removeMembers of the writer on the same side', 'com.atproto.memberspace.removeMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), dids: [W.sides[side.name].did] } }), EMPTY)
await tracked('removeMembers of a writer, shown removed', inSa, writers.local.did, removed)
// An unauthorized writer: refused, after the aud (spec/SPACES-SYNC.md,
// "Taking a notification: notifyRepoUpdate, served", "For other slices").
await both('removeMembers of the harness writer', 'com.atproto.memberspace.removeMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), dids: [writerH.did] } }), EMPTY)
await tracked('removeMembers of the harness writer, shown removed', inSa, () => writerH.did, removed)
await notifyEach('notifyRepoUpdate by a writer not authorized', err('Forbidden', 403))
await notifyEach('notifyRepoUpdate by a writer not authorized, to another fragment', err('Forbidden', 403), { claims: (side) => ({ aud: `${auth(side).did}#other` }) })
await quiet('a write by a writer whose write was taken away', async () => { await writeBoth(writers.peer); await sleep(500) })
/// The updates a registration had for a side's space, each held to the page
/// (spec/SPACES-SYNC.md, "Sending updates", "Deletion").
async function updatesOf(kind, side, p, space, tok) {
const got = p.updates(space, tok)
for (const r of got) {
const u = r.body?.update
const asked = r.method === 'POST' && (r.type ?? '').startsWith('application/json') && !r.authorization && keysAre(r.body, ['space', 'token', 'update'])
const has = (keys) => keysAre(u, keys) || keysAre(u, [...keys, 'prevSpaceRev'])
const ok = u?.$type === REPO_WRITE ? has(['$type', 'data', 'repo', 'repoRev', 'spaceRev']) && TID.test(u.repoRev) && hexOf(u.data) !== null : u?.$type === REPO_REMOVED ? has(['$type', 'repo', 'spaceRev']) : u?.$type === SPACE_DELETED && keysAre(u, ['$type'])
if (!asked || !ok || (u.spaceRev !== undefined && !TID.test(u.spaceRev)) || (u.prevSpaceRev !== undefined && !TID.test(u.prevSpaceRev)))
await fail(side, `${kind}: notifySpaceUpdate`, { space: alias(side, space) }, `a POST of JSON to /xrpc/${NSU}, no Authorization, body exactly {space, token, update}, update a #repoWrite {repo, repoRev, data, spaceRev, prevSpaceRev?}, a #repoRemoved {repo, spaceRev, prevSpaceRev?} or a #spaceDeleted {}`, named(side, { method: r.method, type: r.type, authorization: r.authorization, body: { ...r.body, token: '<…>' } }))
}
return got.map((r) => r.body.update)
}
/// Every change of each side's writer set sent to each registration made
/// before the first, `n` times (its twins), in spaceRev order, chained by
/// prevSpaceRev from none; each listWriters entry sent at its spaceRev;
/// the same updates to every registration, and on both sides
/// (spec/SPACES-SYNC.md, "Sending updates").
async function allSent(kind, key, regs) {
const seqs = []
for (const side of sides) {
const space = sp(side, key)
const entries = await listAll(side, space)
const top = entries.reduce((m, e) => (e.spaceRev > m ? e.spaceRev : m), '')
for (const started = Date.now(); Date.now() - started < 5000; await sleep(100))
if (regs.every(([p, tok, , n = 1]) => p.updates(space, tok).filter((r) => r.body?.update?.spaceRev === top).length >= n)) break
let first
for (const [p, tok, name, n = 1] of regs) {
const got = (await updatesOf(kind, side, p, space, tok)).filter((u) => u.$type !== SPACE_DELETED)
const once = [...new Map(got.map((u) => [u.spaceRev, u])).values()].sort((x, y) => (x.spaceRev < y.spaceRev ? -1 : 1))
const times = once.find((u) => got.filter((x) => x.spaceRev === u.spaceRev).length !== n)
const chain = once.findIndex((u, i) => (i === 0 ? 'prevSpaceRev' in u : u.prevSpaceRev !== once[i - 1].spaceRev))
if (times || chain !== -1 || got.some((u) => !isDeepStrictEqual(u, once.find((x) => x.spaceRev === u.spaceRev))))
await fail(side, `${kind}: each change sent ${n === 1 ? 'once' : `${n} times, once to each registration`}, chained by prevSpaceRev from none`, { registration: name }, 'each prevSpaceRev the spaceRev before, absent on the first', named(side, got))
for (const e of entries) {
const u = once.find((x) => x.spaceRev === e.spaceRev)
const view = (x) => x && { ...x, data: hexOf(x.data) ?? undefined, prevSpaceRev: undefined }
if (!isDeepStrictEqual(view(u), view(e))) await fail(side, `${kind}: a writer set entry sent at its spaceRev`, { registration: name, writer: alias(side, e.repo) }, named(side, e), named(side, u ?? null))
}
if (once.some((u) => u.repo === carol.did)) await fail(side, `${kind}: a write not tracked, not sent`, { registration: name }, 'no update of the reader', named(side, once.filter((u) => u.repo === carol.did)))
const seq = once.map((u) => `${u.$type.split('#')[1]} ${alias(side, u.repo)}`)
if (first === undefined) first = seq
else if (!isDeepStrictEqual(seq, first)) await fail(side, `${kind}: every registration sent the same updates`, { registration: name }, first, seq)
}
seqs.push(first)
}
if (!isDeepStrictEqual(seqs[0], seqs[1])) await differ(`${kind}: the updates sent, in order`, { space: '<space>' }, seqs[0], seqs[1])
count(kind)
}
await allSent('updates of the writer set', sa, regsA)
for (const side of sides) {
const told = refusedTokens.filter((tok) => syncer.updates(sp(side, sa), tok).length)
if (told.length) await fail(side, 'a registration refused is sent nothing', { space: alias(side, sp(side, sa)) }, 'no update', `updates to ${told.length} refused registrations`)
}
// authorizeSpaceSync answers the writer set's greatest spaceRev
// (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync").
for (const side of sides) {
const entries = await listAll(side, sp(side, sa))
const top = entries.reduce((m, e) => (e.spaceRev > m ? e.spaceRev : m), '')
const c = await credential(side, sp(side, sa))
if (c.spaceRev !== top) await fail(side, 'authorizeSpaceSync\'s spaceRev', { space: alias(side, sp(side, sa)) }, top, c.spaceRev ?? 'none')
}
count('authorizeSpaceSync\'s spaceRev, the writer set\'s')
// listWriters' refusals (spec/SPACES-SYNC.md, "The writer set: listWriters",
// "Commands").
const sb = skey('other')
await create('createSpace for another credential', sb, { $type: MEMBERSPACE, publicRead: true })
const credB = {}
for (const side of sides) credB[side.name] = await credential(side, sp(side, sb))
const credAway = await credential(ref, peerSpace, peer.url)
for (const [what, make, want] of [
['with no credential', () => ({}), err('MissingJwt', 401)],
['with a bearer token', (side) => ({ token: auth(side).token }), err('MissingJwt', 401)],
['to another audience', (side) => withCred(side, {}, bob.did), err('BadSpaceAudience', 401)],
['with another space\'s credential', (side) => withCred(side, {}, auth(side).did, credB[side.name]), err('InvalidCredential')],
]) await both(`listWriters ${what}`, LW, (side) => ({ ...make(side), params: { space: sp(side, sa) } }), want)
await both('listWriters of a space whose authority is elsewhere', LW, () => ({ headers: signed(credAway, bob.did), params: { space: peerSpace } }), err('SpaceNotFound'))
await both('listWriters of a malformed space', LW, (side) => withCred(side, { params: { space: 'nope' } }), err('InvalidRequest'))
await both('listWriters with no credential and a malformed space', LW, () => ({ params: { space: 'nope' } }), err('InvalidRequest'))
await both('listWriters with limit 0', LW, (side) => withCred(side, { params: { space: sp(side, sa), limit: 0 } }), err('InvalidRequest'))
// listWriters' pages (spec/SPACES-SYNC.md, "The writer set: listWriters").
for (const side of sides) {
const all = await listAll(side, sp(side, sa))
const paged = []
let cursor
for (let i = 0; i < 20; i++) {
const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), limit: 1, ...(cursor ? { cursor } : {}) } })
const want = all.filter((e) => cursor === undefined || e.spaceRev > cursor).slice(0, 1)
const body = want.length ? { cursor: want[0].spaceRev, repos: want } : { repos: [] }
if (r.status !== 200 || !isDeepStrictEqual(r.body, body)) await fail(side, 'listWriters page of 1', { cursor: cursor ?? null }, named(side, body), named(side, r.body))
if (!want.length) break
paged.push(...want)
cursor = want[0].spaceRev
}
if (!isDeepStrictEqual(paged, all)) await fail(side, 'listWriters pages, together', {}, named(side, all), named(side, paged))
for (const c of ['a', 'zzzz', all.at(-1).spaceRev]) {
const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), cursor: c } })
if (r.status !== 200 || !isDeepStrictEqual(r.body, { repos: [] })) await fail(side, 'listWriters after every spaceRev', { cursor: c }, { repos: [] }, named(side, r.body))
}
const first = all[0].spaceRev
const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), cursor: first } })
if (r.status !== 200 || !isDeepStrictEqual(r.body?.repos, all.slice(1))) await fail(side, 'listWriters after a spaceRev, exclusive', { cursor: first }, named(side, all.slice(1)), named(side, r.body))
count('listWriters pages and cursors')
}
// A refusal drops the registration of its pair; an earlier refusal leaves
// it (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync"; spec/SPACES-SYNC.md,
// "Registering").
{
const t7 = token()
const tokens = async (user) => Object.fromEntries(await Promise.all(sides.map(async (side) => [side.name, await delegation(user, inSm(side))])))
const credM = {}
for (const side of sides) credM[side.name] = await credential(side, inSm(side), side.url, { delegationToken: await delegation(bob, inSm(side)) })
const writeSm = async (kind) => {
await writeBoth(writers.authority, sm, kind)
await tracked(kind, inSm, writers.authority.did, (side) => commitOf(side.url, auth(side).token, inSm(side), auth(side).did), (side) => credM[side.name])
await sleep(500)
}
const told = async (kind, n) => {
for (const side of sides) {
const got = await updatesOf(kind, side, syncer, inSm(side), t7)
if (got.length !== n) await fail(side, kind, { space: alias(side, inSm(side)) }, `${n} updates`, named(side, got))
}
count('a refusal drops a registration')
}
let dt = await tokens(bob)
await register('a member\'s registration, members only', inSm, syncer.url, t7, { extra: (side) => ({ delegationToken: dt[side.name] }) })
dt = await tokens(dave)
await register('the same pair with a non-member\'s token', inSm, syncer.url, t7, { how: null, want: err('SpaceNotFound'), confirms: 0, extra: (side) => ({ delegationToken: dt[side.name] }) })
await writeSm('a write after the registration is dropped')
await told('a registration dropped, sent nothing', 0)
dt = await tokens(bob)
await register('the member\'s registration again, confirmed anew', inSm, syncer.url, t7, { how: null, extra: (side) => ({ delegationToken: dt[side.name] }) })
await register('the same pair with a token that is no JWT', inSm, syncer.url, t7, { how: null, want: err('InvalidDelegationToken'), confirms: 0, extra: () => ({ delegationToken: 'abc' }) })
await writeSm('a write after a refusal that leaves the registration')
await told('a registration left by an earlier refusal, sent the write', 1)
}
// Deletion notices; a space created again has no registrations and no
// writers (spec/SPACES-SYNC.md, "Deletion", "Notices", "For other slices").
const sd = skey('del')
const inSd = (side) => sp(side, sd)
await create('createSpace to delete', sd, { $type: MEMBERSPACE, publicRead: true })
const credD = {}
for (const side of sides) credD[side.name] = await credential(side, inSd(side))
const [t8, t9] = [token(), token()]
await register('a registration to be told of deletion', inSd, syncer.url, t8)
await register('another, at the second syncer', inSd, syncer2.url, t9, { p: syncer2 })
await both('deleteSpace with registrations', 'com.atproto.space.deleteSpace', (side) => ({ token: auth(side).token, body: { space: inSd(side) } }), EMPTY)
const notices = async (kind) => {
for (const side of sides) {
for (const [p, tok] of [[syncer, t8], [syncer2, t9]]) {
for (const started = Date.now(); !p.updates(inSd(side), tok).length && Date.now() - started < 10000;) await sleep(100)
const got = await updatesOf(kind, side, p, inSd(side), tok)
if (got.length !== 1 || got[0]?.$type !== SPACE_DELETED) await fail(side, `${kind}: one deletion notice to each registration`, { space: alias(side, inSd(side)) }, [{ $type: SPACE_DELETED }], named(side, got))
}
}
count(kind)
}
await notices('deletion notices')
await both('deleteSpace again', 'com.atproto.space.deleteSpace', (side) => ({ token: auth(side).token, body: { space: inSd(side) } }), EMPTY)
await sleep(500)
await notices('nothing more for a second deleteSpace')
await register('a registration of a deleted space', inSd, syncer.url, t8, { how: null, want: err('SpaceDeleted'), confirms: 0 })
await both('listWriters of a deleted space', LW, (side) => ({ headers: signed(credD[side.name], auth(side).did), params: { space: inSd(side) } }), err('SpaceNotFound'))
await notifyEach('notifyRepoUpdate of a deleted space', err('SpaceNotFound'), { body: (side) => ({ space: inSd(side) }) })
await create('createSpace of the deleted space again', sd, { $type: MEMBERSPACE, publicRead: true, publicWrite: true })
for (const side of sides) {
const c = await credential(side, inSd(side))
if (c.spaceRev !== undefined) await fail(side, 'authorizeSpaceSync of a space created again', { space: alias(side, inSd(side)) }, 'no spaceRev', c.spaceRev)
}
await writeBoth(writers.peer, sd)
await tracked('a space created again, tracked with the old credential', inSd, writers.peer.did, (side) => commitOf(peer.url, bob.token, inSd(side), bob.did), (side) => credD[side.name])
await sleep(500)
await notices('a space created again sends to no one')
await register('a registration of the space created again, confirmed anew', inSd, syncer.url, t8, { how: null })
// The limit on registrations, held to the page alone (the reference has
// none): 64 active a space; one registered already is always renewed; a
// refusal's drop frees one (spec/SPACES-SYNC.md, "Registering", "Where the
// reference departs from this page").
{
const caps = skey('caps')
const space = sp(cand, caps)
const made = await call(cand.url, 'com.atproto.space.createSpace', { token: auth(cand).token, body: { spaceType: T, skey: caps, policy: { $type: MEMBERSPACE } } })
if (made.status !== 200) await fail(cand, 'createSpace for the registration limit', {}, 'the space created', normalize(cand, made))
const added = await call(cand.url, 'com.atproto.memberspace.putMembers', { token: auth(cand).token, body: { space, members: [{ did: bob.did, read: true, write: false }] } })
if (added.status !== 200) await fail(cand, 'putMembers for the registration limit', {}, 'the member added', normalize(cand, added))
const toks = Array.from({ length: 66 }, () => token())
const reg = async (what, i, want, confirms, user = bob) => {
syncer.gave.set(`${space} ${toks[i]}`, 'echo')
const asked = syncer.asked(space, toks[i]).length
const r = await call(cand.url, AUTHORIZE, { body: { space, credentialKey: regKey.did, delegationToken: await delegation(user, space), notify: { endpoint: syncer.url, token: toks[i] } } })
const g = shape(cand, r)
const wrong = want === 200 ? registered(g, r) : isDeepStrictEqual(g, want) ? null : want
if (wrong) await fail(cand, `authorizeSpaceSync ${what}`, { registration: i + 1 }, wrong, g)
const n = syncer.asked(space, toks[i]).length - asked
if (n !== confirms) await fail(cand, `authorizeSpaceSync ${what}: confirmRegistration`, { registration: i + 1 }, `${confirms} confirmations`, n)
count('the limit on registrations (the page alone)')
}
for (let i = 0; i < 64; i++) await reg(`, the space's registration ${i + 1}`, i, 200, 1)
await reg(', a 65th', 64, err('TooManyRegistrations'), 0)
await reg(', the first again, renewed', 0, 200, 0)
await reg(', a non-member with the second\'s pair', 1, err('SpaceNotFound'), 0, dave)
await reg(', a 65th once one is dropped', 64, 200, 1)
}
// ==== 2. the server as repo host ==========================================================
// Into a memberspace on the second alpha PDS, clearing too
// (spec/SPACES-SYNC.md, "Telling the space host: notifyRepoUpdate, sent";
// spec/SPACES-REPO.md, "For other parts").
const ex = skey('ext')
const extSpace = `at://${bob.did}/space/${T}/${ex}`
await call(peer.url, 'com.atproto.space.createSpace', { token: bob.token, body: { spaceType: T, skey: ex, policy: { $type: MEMBERSPACE, publicRead: true } } })
await call(peer.url, 'com.atproto.memberspace.putMembers', { token: bob.token, body: { space: extSpace, members: sides.map((side) => ({ did: W.sides[side.name].did, read: true, write: true })) } })
const credExt = await credential(ref, extSpace, peer.url)
const extEntry = async (side) => {
const r = await call(peer.url, LW, { headers: signed(credExt, bob.did), params: { space: extSpace } })
return entryView(r.body?.repos?.find((x) => x.repo === W.sides[side.name].did))
}
async function extTracked(kind, wantData) {
const started = Date.now()
const done = new Set()
for (;;) {
for (const side of sides) {
if (done.has(side.name)) continue
const want = await commitOf(side.url, W.sides[side.name].token, extSpace, W.sides[side.name].did)
if (wantData && want?.data !== wantData) await fail(side, `${kind}: the commit`, {}, { data: wantData }, want)
if (isDeepStrictEqual(await extEntry(side), want)) done.add(side.name)
}
if (done.size === 2) break
if (Date.now() - started > 15000) { const side = done.has('reference') ? cand : ref; await fail(side, `${kind}: tracked by the space host on the second alpha PDS`, { space: '<the peer\'s space>' }, await commitOf(side.url, W.sides[side.name].token, extSpace, W.sides[side.name].did), await extEntry(side)) }
await sleep(100)
}
const [a, b] = await Promise.all(sides.map(async (side) => (await extEntry(side))?.data))
if (a !== b) await differ(`${kind}: the data the space host on the second alpha PDS holds`, {}, a, b)
count(kind)
}
const localW = writers.local
const extWrite = async (kind, nsid, body) => both(kind, nsid, (side) => ({ token: localW.token(side), body: { space: extSpace, repo: localW.did(side), ...body } }), ok200, (g) => ({ status: g.status }))
await extWrite('createRecord into a space elsewhere', 'com.atproto.space.createRecord', { collection: 'com.example.post', rkey: 'a', record: { $type: 'com.example.post', text: 'a', createdAt: '2026-11-08T00:00:00.000Z' } })
await extTracked('a write into a space elsewhere')
await extWrite('createRecord into a space elsewhere again', 'com.atproto.space.createRecord', { collection: 'com.example.post', rkey: 'b', record: { $type: 'com.example.post', text: 'b', createdAt: '2026-11-08T00:00:00.000Z' } })
await extWrite('deleteRecord in a space elsewhere', 'com.atproto.space.deleteRecord', { collection: 'com.example.post', rkey: 'a' })
await extTracked('a delete in a space elsewhere')
await extWrite('clearRecords in a space elsewhere', 'com.atproto.space.clearRecords', {})
await extTracked('clearRecords in a space elsewhere', EMPTY_DATA)
// Into the harness space host's spaces: each notification exactly as the page says
// (spec/SPACES-SYNC.md, "Telling the space host: notifyRepoUpdate, sent").
const fsp = (key) => `at://${fake.did}/space/${T}/${key}`
host.answer = () => ({ status: 200, body: {} })
const sent = (side, key) => host.seen.filter((r) => r.body?.space === fsp(key) && r.body?.repo === W.sides[side.name].did)
async function sentExactly(kind, key) {
const started = Date.now()
const out = {}
for (const side of sides) {
for (;;) {
const want = await commitOf(side.url, W.sides[side.name].token, fsp(key), W.sides[side.name].did)
const r = sent(side, key).find((x) => x.body?.repoRev === want?.repoRev)
if (r) {
const wrongAuth = await checkAuth(r, { iss: W.sides[side.name].did, aud: `${fake.did}#atproto_pds`, lxm: NRU })
const body = { space: fsp(key), repo: W.sides[side.name].did, repoRev: want.repoRev, data: want.data }
const gotBody = r.body && typeof r.body === 'object' ? { ...r.body, data: hexOf(r.body.data) ?? r.body.data } : r.body
if (wrongAuth || r.method !== 'POST' || r.path !== `/xrpc/${NRU}` || !(r.type ?? '').startsWith('application/json') || !isDeepStrictEqual(gotBody, body)) await fail(side, `${kind}: the notification sent`, { space: '<the harness space host\'s space>' }, named(side, { method: 'POST', path: `/xrpc/${NRU}`, body: { ...body, data: `${body.data} (as bytes)` }, auth: wrongAuth ?? 'as the page says' }), named(side, { method: r.method, path: r.path, type: r.type, body: gotBody, auth: wrongAuth ?? 'as the page says' }))
out[side.name] = r
break
}
if (Date.now() - started > 15000) await fail(side, `${kind}: a notification to the space host`, { space: '<the harness space host\'s space>' }, named(side, want), named(side, sent(side, key).map((x) => x.body)))
await sleep(100)
}
}
count(kind)
return out
}
const o1 = skey('o')
await writeBoth(localW, null, 'a write into the harness space host\'s space', () => fsp(o1))
await sentExactly('notifyRepoUpdate sent to a space host elsewhere', o1)
// ==== 3. retries across a crash ===========================================================
// Notifications that fail for now, kept and retried across the restart;
// the refused, not sent again; deletion notices retried, but not of a space
// created again (spec/SPACES-SYNC.md, "Delivery and retries", "Deletion",
// "Notices").
const keys = { r503: skey('r503-'), r429: skey('r429-'), close: skey('close-'), r400: skey('r400-'), r401: skey('r401-'), r403: skey('r403-') }
const failing = { [fsp(keys.r503)]: { status: 503, body: { error: 'Unavailable' } }, [fsp(keys.r429)]: { status: 429, body: { error: 'RateLimitExceeded' } }, [fsp(keys.close)]: 'close', [fsp(keys.r400)]: { status: 400, body: { error: 'InvalidRequest' } }, [fsp(keys.r401)]: { status: 401, body: { error: 'AuthenticationRequired' } }, [fsp(keys.r403)]: { status: 403, body: { error: 'Forbidden' } } }
let healed = false
host.answer = (r) => (!healed && failing[r.body?.space]) || { status: 200, body: {} }
// Deletion notices that fail for now: one owed, one of a space created again.
const [s3, s4] = [skey('d3-'), skey('d4-')]
const t3 = token()
for (const key of [s3, s4]) {
await create('createSpace for a deletion notice retried', key, { $type: MEMBERSPACE, publicRead: true })
for (const side of sides) {
syncer.gave.set(`${sp(side, key)} ${t3}`, 'echo')
// Its first attempt alone fails: the server is killed before the
// retry is due, so the retry comes from what it kept.
syncer.failOnce.set(`${sp(side, key)} ${t3}`, { status: 503, body: { error: 'Unavailable' } })
const r = await call(side.url, AUTHORIZE, { body: { space: sp(side, key), credentialKey: regKey.did, notify: { endpoint: syncer.url, token: t3 } } })
if (r.status !== 200) await fail(side, 'a registration for a deletion notice retried', {}, 'registered', normalize(side, r))
}
}
// An update to a registration that errs: one attempt (spec/SPACES-SYNC.md,
// "Sending updates"). Renewed first, so that it is active.
await register('the registration at the endpoint with a path, renewed', inSa, `${syncer2.url}/base/path?x=`, t2, { p: syncer2, how: null, confirms: 0 })
for (const side of sides) syncer2.failing.set(`${inSa(side)} ${t2}`, { status: 500, body: { error: 'InternalServerError' } })
await writeBoth(writers.authority)
const failedAt = {}
const firstAttempts = async (what, find) => {
const started = Date.now()
for (const side of sides) {
for (;;) {
const r = find(side)
if (r.length) { failedAt[`${side.name} ${what}`] = r.at(-1).at; break }
if (Date.now() - started > 15000) await fail(side, `${what}: its first attempt`, {}, 'an attempt', 'none')
await sleep(100)
}
}
}
for (const [name, key] of Object.entries(keys)) {
const n = name === 'r503' ? 3 : 1
for (let i = 0; i < n; i++) await writeBoth(localW, null, 'a write into the harness space host\'s space', () => fsp(key))
await firstAttempts(name, (side) => sent(side, key).filter((r, _, all) => all.length >= n))
}
const latest503 = {}
for (const side of sides) latest503[side.name] = (await commitOf(side.url, W.sides[side.name].token, fsp(keys.r503), W.sides[side.name].did)).repoRev
for (const side of sides) for (const key of [s3, s4]) {
const r = await call(side.url, 'com.atproto.space.deleteSpace', { token: auth(side).token, body: { space: sp(side, key) } })
if (r.status !== 200) await fail(side, 'deleteSpace for a deletion notice retried', {}, 'deleted', normalize(side, r))
}
const deletionNotices = (key) => (side) => syncer.updates(sp(side, key), t3).filter((r) => r.body?.update?.$type === SPACE_DELETED)
await firstAttempts('s3', deletionNotices(s3))
await firstAttempts('s4', deletionNotices(s4))
await firstAttempts('update', (side) => syncer2.updates(inSa(side), t2).filter((r) => r.answered === 500))
await create('createSpace of a space owed a notice, again', s4, { $type: MEMBERSPACE })
// After the run restarts the server: everything answers 200, and each
// retry is judged against the time of its failure.
afterRestart.push(async () => {
healed = true
syncer.failing.clear()
syncer2.failing.clear()
const since = (find, name) => (side) => find(side).filter((r) => r.at > failedAt[`${side.name} ${name}`])
const toHost = (key, name) => since((side) => host.seen.filter((r) => r.body?.space === fsp(key) && r.body?.repo === W.sides[side.name].did), name)
const expectations = [
['the 503 space\'s latest notification', toHost(keys.r503, 'r503'), 1, 'r503', (r, side) => r.body.repoRev === latest503[side.name]],
['the 429 notification', toHost(keys.r429, 'r429'), 1, 'r429'],
['the closed notification', toHost(keys.close, 'close'), 1, 'close'],
['the 400 notification', toHost(keys.r400, 'r400'), 0, 'r400'],
['the 401 notification', toHost(keys.r401, 'r401'), 0, 'r401'],
['the 403 notification', toHost(keys.r403, 'r403'), 0, 'r403'],
['the owed deletion notice', since(deletionNotices(s3), 's3'), 1, 's3'],
['the deletion notice of a space created again', since(deletionNotices(s4), 's4'), 0, 's4'],
['the update the registration refused', since((side) => syncer2.updates(inSa(side), t2), 'update'), 0, 'update'],
]
const last = Math.max(...Object.values(failedAt))
for (;;) {
const pending = sides.some((side) => expectations.some(([, find, n]) => find(side).length < n))
if ((!pending && Date.now() - last > 75000) || Date.now() - last > 100000) break
await sleep(500)
}
for (const side of sides) {
for (const [what, find, n, name, ok] of expectations) {
const got = find(side)
const failed = failedAt[`${side.name} ${name}`]
const wrong = got.length !== n ? `${n} after the failure` : got.find((r) => r.answered !== 200 || r.at - failed < 25000 || (ok && !ok(r, side)))
if (wrong) await fail(side, `a retry: ${what}`, { within: '100 seconds of the failure, across the server\'s restart' }, typeof wrong === 'string' ? wrong : 'a retry at least 25 seconds after the failure, of the latest state', named(side, got.map((r) => ({ after: `${Math.round((r.at - failed) / 1000)} s`, body: { ...r.body, token: r.body?.token && '<…>' }, answered: r.answered }))))
}
count('retries across a crash')
}
syncer.close(); syncer2.close(); host.close()
})
}