// Spaces' write notifications, writer set and syncers (spec/SPACES-SYNC.md), // in the spaces run. Parties are played here, each answering as a check // needs and recording what it is sent: two syncers, HTTP servers that confirm // the tokens the harness gave them, per space, and take notifySpaceUpdate; a // writer whose notifications the harness signs, and a space host for an // authority of its own, each a did:plc in the directory with #atproto_pds at // an HTTP server of the harness. // // 1. the server as space host: the authority on each side (the first alpha's // or the server's), writers on the second alpha PDS, on the same side, // and the harness's writer; registrations and their confirmation, // notifications taken, removals, listWriters, the updates sent to each // registration, a refusal dropping a registration, and deletion notices; // 2. the server as repo host: a writer on each side writes into a space on // the second alpha PDS, and into the harness space host's spaces; // 3. retries across a crash: failures arranged here, before the run's // restart of the server, and judged after it (`afterRestart`). // // Requests go to both sides alike; answers are compared with each side's // accounts named alike, and each side is held to the page, the reference // included, which checks the check. Every token and key name is drawn from // the run's `rand`. import crypto from 'node:crypto' import { createServer } from 'node:http' import { once } from 'node:events' import { isDeepStrictEqual } from 'node:util' import * as plc from '@did-plc/lib' import { Secp256k1Keypair, verifySignature } from '@atproto/crypto' import * as S from './lib/spec.mjs' const T = 'com.example.group' const NRU = 'com.atproto.space.notifyRepoUpdate' const NSU = 'com.atproto.space.notifySpaceUpdate' const CONFIRM = 'com.atproto.space.confirmRegistration' const AUTHORIZE = 'com.atproto.space.authorizeSpaceSync' const LW = 'com.atproto.space.listWriters' const REPO_WRITE = 'com.atproto.space.defs#repoWrite' const REPO_REMOVED = 'com.atproto.space.defs#repoRemoved' const SPACE_DELETED = `${NSU}#spaceDeleted` const MEMBERSPACE = 'com.atproto.memberspace.policy' const TID = /^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/ /// A confirmation's challenge: 16 bytes in lower-case base32, unpadded. const CHALLENGE = /^[a-z2-7]{26}$/ const EMPTY_DATA = 'e5a00aa9991ac8a5ee3109844d84a55583bd20572ad3ffcd42792f3c36b183ad' /// The order of secp256k1, to make a high-S signature of a low-S one. const N = BigInt('0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141') const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms)) const now = () => Math.floor(Date.now() / 1000) const b64 = (o) => Buffer.from(JSON.stringify(o)).toString('base64url') const unb64 = (s) => JSON.parse(Buffer.from(s, 'base64url').toString('utf8')) const hexOf = (bytes) => (bytes?.$bytes ? Buffer.from(bytes.$bytes, 'base64').toString('hex') : null) const keysAre = (o, keys) => isDeepStrictEqual(Object.keys(o ?? {}).sort(), [...keys].sort()) /// The TID of a time in milliseconds, with a clock id. function tid(ms, clock = 0) { let n = (BigInt(Math.floor(ms)) * 1000n << 10n) | BigInt(clock) let s = '' for (let i = 0; i < 13; i++) { s = '234567abcdefghijklmnopqrstuvwxyz'[Number(n & 31n)] + s; n >>= 5n } return s } export async function checkSpaceSync(h) { const { net, ref, cand, accounts, createAccount, normalize, differ, fail, count, rand, front, afterRestart } = h const sides = [ref, cand] const peer = net.extraPdses[0] const plcUrl = net.plc.url const skey = (p) => `${p}${Math.floor(rand() * 1e6)}` /// A syncer's token: `n` bytes of the run's seed, base64url. const token = (n = 32) => Buffer.from(Array.from({ length: n }, () => Math.floor(rand() * 256))).toString('base64url') // ---- requests ------------------------------------------------------------------ async function call(base, nsid, { params, body, token, headers = {} } = {}) { const url = new URL(`${base}/xrpc/${nsid}`) for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v) const hs = { ...headers } if (token) hs.authorization = `Bearer ${token}` if (body !== undefined) hs['content-type'] = 'application/json' const res = await fetch(url, { method: body === undefined ? 'GET' : 'POST', headers: hs, body: body === undefined ? undefined : JSON.stringify(body) }) const buf = Buffer.from(await res.arrayBuffer()) let parsed = null if (buf.length) try { parsed = JSON.parse(buf.toString('utf8')) } catch { parsed = { unparsed: buf.toString('utf8').slice(0, 200) } } return { status: res.status, type: res.headers.get('content-type'), buf, body: parsed } } function alias(side, s) { for (const a of accounts) if (a.sides[side.name].did) s = s.split(a.sides[side.name].did).join(``) return s } const named = (side, v) => JSON.parse(alias(side, JSON.stringify(v ?? null))) const describe = (side, req) => named(side, JSON.parse(JSON.stringify(req, (k, v) => (k === 'token' || k === 'delegationToken' || k === 'authorization' || k === 'signature' ? '<…>' : v)))) function shape(side, res) { if (res.status >= 400) return normalize(side, res) if (res.status === 200 && !res.buf.length) return { status: 200, empty: true } return normalize(side, res) } const err = (error, status = 400) => ({ status, error }) const EMPTY = { status: 200, empty: true } /// The same request to both sides, compared as `view` shows them; each side /// held to `want` (an object, or a function answering what is wrong). async function both(kind, nsid, make, want, view = (g) => g, base = (side) => side.url) { const res = await Promise.all(sides.map((side) => call(base(side), nsid, make(side)))) const got = sides.map((side, i) => shape(side, res[i])) for (const [i, side] of sides.entries()) { const wrong = want === undefined ? null : typeof want === 'function' ? want(got[i], res[i], side) : isDeepStrictEqual(got[i], want) ? null : want if (wrong) await fail(side, `${kind}: ${nsid}`, describe(side, make(side)), typeof wrong === 'string' ? wrong : named(side, wrong), got[i]) } const [a, b] = got.map(view) if (!isDeepStrictEqual(a, b)) await differ(`${kind}: ${nsid}`, describe(cand, make(cand)), a, b) count(kind) return res } const ok200 = (g) => (g.status === 200 ? null : 'answered 200') // ---- the harness's parties ----------------------------------------------------------- /// An HTTP server of the harness: records each request; answers as /// `answer`: `{status, body}` as JSON, `{status, raw, type?}`, or 'close'. async function party() { const p = { seen: [], answer: () => ({ status: 200, body: {} }) } const srv = createServer((req, res) => { const chunks = [] req.on('data', (c) => chunks.push(c)) req.on('end', async () => { const u = new URL(req.url, 'http://x') let body = null try { body = JSON.parse(Buffer.concat(chunks).toString('utf8') || 'null') } catch { body = '' } const r = { at: Date.now(), method: req.method, path: u.pathname, type: req.headers['content-type'] ?? null, authorization: req.headers.authorization ?? null, body } const a = p.answer(r) r.answered = a === 'close' ? 'close' : a.status p.seen.push(r) if (a === 'close') return req.socket.destroy() if ('raw' in a) return res.writeHead(a.status, a.type ? { 'content-type': a.type } : {}).end(a.raw) res.writeHead(a.status, { 'content-type': 'application/json' }).end(JSON.stringify(a.body)) }) }) srv.listen(0, '127.0.0.1') await once(srv, 'listening') p.port = srv.address().port p.url = `http://localhost:${p.port}` p.close = () => srv.close() return p } /// A syncer: confirms a registration of the tokens it gave, per space, as /// `gave` says ('echo', or a way to fail), refusing any other; answers /// notifySpaceUpdate 200 with no body, or as `failing` says. async function syncerParty() { const s = await party() s.gave = new Map() s.failing = new Map() s.failOnce = new Map() s.answer = (r) => { const k = `${r.body?.space} ${r.body?.token}` if (r.path === `/xrpc/${CONFIRM}`) { const how = s.gave.get(k) const challenge = r.body?.challenge if (how === undefined) return { status: 400, body: { error: 'InvalidRequest', message: 'not a token this syncer gave for the space' } } if (how === 'echo') return { status: 200, body: { challenge } } if (how === 'extra') return { status: 200, body: { challenge, also: true } } if (how === 'wrong') return { status: 200, body: { challenge: `${challenge}x` } } if (how === 'text') return { status: 200, type: 'text/plain', raw: JSON.stringify({ challenge }) } if (how === 'empty') return { status: 200, raw: '' } if (how === 'close') return 'close' return { status: how, body: { error: 'InternalServerError' } } } if (r.path === `/xrpc/${NSU}`) { const once = s.failOnce.get(k) if (once) { s.failOnce.delete(k); return once } return s.failing.get(k) ?? { status: 200, raw: '' } } return { status: 404, body: { error: 'MethodNotImplemented' } } } /// What it was asked for a space and token, but updates. s.asked = (space, tok) => s.seen.filter((r) => r.path !== `/xrpc/${NSU}` && r.body?.space === space && r.body?.token === tok) s.updates = (space, tok) => s.seen.filter((r) => r.path === `/xrpc/${NSU}` && r.body?.space === space && r.body?.token === tok) return s } async function plcDid(services) { const key = await Secp256k1Keypair.create({ exportable: true }) const op = await plc.signOperation({ type: 'plc_operation', rotationKeys: [key.did()], alsoKnownAs: [], verificationMethods: { atproto: key.did() }, services, prev: null }, key) const did = await plc.didForCreateOp(op) await new plc.Client(plcUrl).sendOperation(did, op) return { did, key } } const keyOf = async (did) => (await (await fetch(`${plcUrl}/${did}/data`)).json()).verificationMethods?.atproto /// A request's service auth as the page says it must be: what is wrong, or null. async function checkAuth(r, { iss, aud, lxm }) { const parts = (r.authorization ?? '').replace(/^Bearer /, '').split('.') let header, claims try { [header, claims] = parts.slice(0, 2).map(unb64) } catch { return 'service auth that is a JWT' } if (!isDeepStrictEqual(header, { typ: 'JWT', alg: 'ES256K' })) return { header: { typ: 'JWT', alg: 'ES256K' } } if (!keysAre(claims, ['aud', 'exp', 'iat', 'iss', 'jti', 'lxm']) || claims.iss !== iss || claims.aud !== aud || claims.lxm !== lxm || claims.exp - claims.iat !== 60 || typeof claims.jti !== 'string' || !claims.jti) return { claims: { iss, aud, lxm, iat: '', exp: '', jti: '' } } const key = await keyOf(iss) const signed = parts.length === 3 && (await verifySignature(key, Buffer.from(`${parts[0]}.${parts[1]}`), Buffer.from(parts[2], 'base64url')).catch(() => false)) return signed ? null : `a signature by ${iss}'s #atproto key, low-S` } const syncer = await syncerParty() const syncer2 = await syncerParty() const host = await party() const writerH = await plcDid({ atproto_pds: { type: 'AtprotoPersonalDataServer', endpoint: host.url } }) const fake = await plcDid({ atproto_pds: { type: 'AtprotoPersonalDataServer', endpoint: host.url } }) // ---- accounts -------------------------------------------------------------------- const peerDomain = (await (await fetch(`${peer.url}/xrpc/com.atproto.server.describeServer`)).json()).availableUserDomains[0] async function peerAccount() { const f = front() const r = await call(peer.url, 'com.atproto.server.createAccount', { body: { handle: `${f}${peerDomain}`, email: `${f}@example.com`, password: `pw-${f}` } }) if (r.status !== 200) await differ('a peer account on the second alpha PDS', { handle: `${f}${peerDomain}` }, 'an account', r.body, 'the harness could not make a peer') return { did: r.body.did, token: r.body.accessJwt } } const [bob, carol, dave] = [await peerAccount(), await peerAccount(), await peerAccount()] const A = await createAccount() const W = await createAccount() const of = (acct) => (side) => acct.sides[side.name] const auth = of(A) const sp = (side, key) => `at://${auth(side).did}/space/${T}/${key}` /// A peer account's delegation token for a space, from the second alpha PDS. async function delegation(user, space) { const r = await call(peer.url, 'com.atproto.space.getDelegationToken', { token: user.token, params: { space } }) if (r.status !== 200) await differ('the second alpha PDS\'s getDelegationToken', { space }, 'a token', r.body, 'the harness could not get a delegation token') return r.body.token } /// A space credential for each side's space, from its authority, asked for /// with no delegation token (the space reads publicly) unless `extra` has /// one, with its holder's key. async function credential(side, space, authorityHost = side.url, extra = {}) { const key = S.newP256() const res = await call(authorityHost, AUTHORIZE, { body: { space, credentialKey: key.did, ...extra } }) if (res.status !== 200) await fail(side, 'authorizeSpaceSync for a credential', { space: alias(side, space) }, 'a credential', normalize(side, res)) return { key, credential: res.body.credential, spaceRev: res.body.spaceRev } } const signed = (c, audience) => S.signCredential(c.key, c.credential, audience) const create = (kind, key, policy) => both(kind, 'com.atproto.space.createSpace', (side) => ({ token: auth(side).token, body: { spaceType: T, skey: key, policy } }), ok200) /// putMembers of `[did, read, write]`s, a `did` a DID or a DID of the side. const putMembers = (kind, key, members) => both(kind, 'com.atproto.memberspace.putMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, key), members: members.map(([did, read, write]) => ({ did: typeof did === 'function' ? did(side) : did, read, write })) } }), EMPTY) /// A writer's latest commit in a space, from its own host. async function commitOf(hostUrl, token, space, repo) { const r = await call(hostUrl, 'com.atproto.space.getLatestCommit', { token, params: { space, repo } }) return r.status === 200 ? { repoRev: r.body.commit.rev, data: hexOf(r.body.commit.data) } : null } async function write(hostUrl, token, space, repo, rkey) { return call(hostUrl, 'com.atproto.space.createRecord', { token, body: { space, repo, collection: 'com.example.post', rkey, record: { $type: 'com.example.post', text: rkey, createdAt: '2026-11-08T00:00:00.000Z' } } }) } /// The same record written by `wr` on each side, into the side's space /// `key`, or into `space(side)`, so that both sides' repos hold alike. async function writeBoth(wr, key = sa, kind = 'a write', space = (side) => sp(side, key)) { const rkey = `r${Math.floor(rand() * 1e9)}` for (const side of sides) { const r = await write(wr.host(side), wr.token(side), space(side), wr.did(side), rkey) if (r.status !== 200) await fail(side, kind, { space: alias(side, space(side)) }, 'written', normalize(side, r)) } } // ==== 1. the server as space host ========================================================== const sa = skey('sync') await create('createSpace for notifications', sa, { $type: MEMBERSPACE, publicRead: true }) await putMembers('putMembers of writers and a reader', sa, [[bob.did, true, true], [writerH.did, true, true], [(side) => W.sides[side.name].did, true, true], [carol.did, true, false]]) const cred = {} for (const side of sides) cred[side.name] = await credential(side, sp(side, sa)) for (const side of sides) if (cred[side.name].spaceRev !== undefined) await fail(side, 'authorizeSpaceSync of a space no one wrote to', { space: alias(side, sp(side, sa)) }, 'no spaceRev', cred[side.name].spaceRev) const withCred = (side, extra = {}, aud = auth(side).did, c = cred[side.name]) => ({ headers: signed(c, aud), ...extra }) /// listWriters of a space, held to the page per side: entries in /// ascending spaceRev, each a #repoWrite or a #repoRemoved /// (spec/SPACES-SYNC.md, "The writer set: listWriters"). async function listAll(side, space, c = cred[side.name]) { const r = await call(side.url, LW, { headers: signed(c, auth(side).did), params: { space } }) if (r.status !== 200) await fail(side, 'listWriters', { space: alias(side, space) }, 'the writer set', normalize(side, r)) const repos = r.body.repos for (const [i, e] of repos.entries()) { const ok = e?.$type === REPO_WRITE ? keysAre(e, ['$type', 'data', 'repo', 'repoRev', 'spaceRev']) && TID.test(e.repoRev) && hexOf(e.data) !== null : e?.$type === REPO_REMOVED && keysAre(e, ['$type', 'repo', 'spaceRev']) if (!ok || !TID.test(e.spaceRev) || (i && repos[i - 1].spaceRev >= e.spaceRev)) await fail(side, 'listWriters entries, in ascending spaceRev', { space: alias(side, space) }, `{$type: "${REPO_WRITE}", repo, repoRev, data, spaceRev} or {$type: "${REPO_REMOVED}", repo, spaceRev}, spaceRev ascending`, named(side, repos)) } if ((repos.length ? r.body.cursor !== repos.at(-1).spaceRev : 'cursor' in r.body)) await fail(side, 'listWriters cursor', { space: alias(side, space) }, repos.length ? repos.at(-1).spaceRev : 'none', r.body.cursor ?? 'none') return repos } /// A writer's entry as the checks compare it. const entryView = (e) => (!e ? null : e.$type === REPO_REMOVED ? { removed: true } : { repoRev: e.repoRev, data: hexOf(e.data) }) /// Wait until each side's writer set shows `did(side)` as `want(side)`. async function tracked(kind, space, did, want, c = (side) => cred[side.name]) { const started = Date.now() const done = new Set() const last = {} for (;;) { for (const side of sides) { if (done.has(side.name)) continue last[side.name] = entryView((await listAll(side, space(side), c(side))).find((e) => e.repo === did(side))) if (isDeepStrictEqual(last[side.name], await want(side))) done.add(side.name) } if (done.size === 2) break if (Date.now() - started > 15000) { const side = done.has('reference') ? cand : ref; await fail(side, `${kind}: the writer set`, { space: alias(side, space(side)), writer: alias(side, did(side)) }, await want(side), last[side.name]) } await sleep(100) } count(kind) } await both('listWriters of no writers', LW, (side) => withCred(side, { params: { space: sp(side, sa) } }), { status: 200, body: { repos: [] } }) // Registrations: each new one confirmed once, at the endpoint's origin, // with the token in the body and no Authorization; the same pair again // renewed (spec/SPACES-SYNC.md, "Syncers", "Registering"; // spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync"). const regKey = S.newP256() const challenges = new Set() const registered = (g, r) => (r.status === 200 && Object.keys(r.body ?? {}).every((k) => k === 'credential' || k === 'spaceRev') && typeof r.body.credential === 'string' && (r.body.spaceRev === undefined || TID.test(r.body.spaceRev)) ? null : '{credential, spaceRev?}') const asAuthorized = (g) => (g.status === 200 ? { status: 200, credential: typeof g.body?.credential === 'string', spaceRev: g.body?.spaceRev === undefined ? 'absent' : '' } : g) /// What is wrong with a confirmation the side sent, or null. function confirmWrong(r) { if (r.method !== 'POST' || r.path !== `/xrpc/${CONFIRM}` || !(r.type ?? '').startsWith('application/json') || r.authorization) return `a POST of JSON to /xrpc/${CONFIRM}, with no Authorization` if (!keysAre(r.body, ['challenge', 'space', 'token'])) return 'a body of exactly {space, token, challenge}' if (!CHALLENGE.test(r.body.challenge) || challenges.has(r.body.challenge)) return 'a challenge of 16 random bytes in base32, lower case, unpadded, new each time' challenges.add(r.body.challenge) return null } /// authorizeSpaceSync with `notify` on both sides, the syncer `p` having /// given `tok` for each side's space as `how` (null: as it was); the /// answer held to `want`, and `confirms` new requests at `p` for the pair, /// each a confirmation as the page says. async function register(kind, space, endpoint, tok, { p = syncer, how = 'echo', want = registered, confirms = 1, extra = () => ({}) } = {}) { if (how !== null) for (const side of sides) p.gave.set(`${space(side)} ${tok}`, how) const before = Object.fromEntries(sides.map((side) => [side.name, p.asked(space(side), tok).length])) const res = await both(kind, AUTHORIZE, (side) => ({ body: { space: space(side), credentialKey: regKey.did, notify: { endpoint: typeof endpoint === 'function' ? endpoint(side) : endpoint, token: tok }, ...extra(side) } }), want, asAuthorized) for (const side of sides) { const got = p.asked(space(side), tok).slice(before[side.name]) const wrong = got.length !== confirms ? `${confirms} confirmation${confirms === 1 ? '' : 's'} at the syncer` : got.map(confirmWrong).find(Boolean) if (wrong) await fail(side, `${kind}: confirmRegistration`, { space: alias(side, space(side)) }, wrong, named(side, got.map((r) => ({ method: r.method, path: r.path, type: r.type, authorization: r.authorization, body: { ...r.body, token: '<…>' } })))) } count('registrations and their confirmation') return res } const inSa = (side) => sp(side, sa) const [t1, t2, t4, t5] = [token(), token(), token(), token(384)] await register('a registration', inSa, syncer.url, t1) await register('the same pair again, renewed', inSa, syncer.url, t1, { confirms: 0 }) // Its endpoint has a path and a query, which are not used. await register('a registration at an endpoint with a path and a query', inSa, `${syncer2.url}/base/path?x=`, t2, { p: syncer2 }) // The same token, the endpoint written otherwise: another registration. await register('the same token at the endpoint with a / after it', inSa, `${syncer.url}/`, t1, { how: null }) await register('a registration answered with another member besides the challenge', inSa, syncer.url, t4, { how: 'extra' }) await register('a registration with a 512-byte token', inSa, syncer2.url, t5, { p: syncer2 }) const NOT_CONFIRMED = err('RegistrationNotConfirmed') const refusedTokens = [] for (const [what, how] of [['a wrong challenge', 'wrong'], ['500', 500], ['text/plain', 'text'], ['an empty 200', 'empty'], ['a closed connection', 'close'], ['a token the syncer did not give', undefined]]) { const tok = token() refusedTokens.push(tok) if (how === undefined) for (const side of sides) syncer.gave.delete(`${inSa(side)} ${tok}`) await register(`a registration answered with ${what}`, inSa, syncer.url, tok, { how: how ?? null, want: NOT_CONFIRMED }) } for (const [what, endpoint] of [['ftp://', `ftp://localhost:${syncer.port}`], ['with a user and password', `http://u:p@localhost:${syncer.port}`], ['a DID', `did:plc:${'a'.repeat(24)}`]]) { const tok = token() refusedTokens.push(tok) await register(`a registration at an endpoint ${what}`, inSa, endpoint, tok, { want: NOT_CONFIRMED, confirms: 0 }) } for (const [what, notify] of [['an endpoint that is no URI', { endpoint: 'not a uri', token: token() }], ['a 513-byte token', { endpoint: syncer.url, token: `${token(384)}x` }], ['no token', { endpoint: syncer.url }]]) await both(`a registration with ${what}`, AUTHORIZE, (side) => ({ body: { space: inSa(side), credentialKey: regKey.did, notify } }), err('InvalidRequest')) // Refused before step 10: nothing is confirmed (spec/SPACES-CREDENTIALS.md, // "authorizeSpaceSync"). const sm = skey('members') await create('createSpace, members only', sm, { $type: MEMBERSPACE }) await putMembers('putMembers of the peer writer', sm, [[bob.did, true, true]]) const inSm = (side) => sp(side, sm) const away = skey('away') const peerSpace = `at://${bob.did}/space/${T}/${away}` await call(peer.url, 'com.atproto.space.createSpace', { token: bob.token, body: { spaceType: T, skey: away, policy: { $type: MEMBERSPACE, publicRead: true } } }) for (const [what, space, want, extra] of [ ['of a members-only space with no token', inSm, err('SpaceNotFound')], ['of a space never made', (side) => sp(side, skey('none')), err('SpaceNotFound')], ['of a space whose authority is elsewhere', () => peerSpace, err('SpaceNotFound')], ['with a credentialKey that is no key', inSa, err('InvalidRequest'), () => ({ credentialKey: 'nope' })], ]) { const tok = token() refusedTokens.push(tok) await register(`a registration ${what}`, space, syncer.url, tok, { want, confirms: 0, extra }) } const regsA = [[syncer, t1, 'the registration and its twin, at the endpoint with a /', 2], [syncer2, t2, 'the endpoint with a path'], [syncer, t4, 'the answer with another member'], [syncer2, t5, 'the 512-byte token']] // Writes tracked, and sent: a peer's notified over HTTP, the same side's // taken directly (spec/SPACES-SYNC.md, "The writer set", "Telling the space // host: notifyRepoUpdate, sent", "For other slices"). const writers = { peer: { did: () => bob.did, host: () => peer.url, token: () => bob.token }, local: { did: (side) => W.sides[side.name].did, host: (side) => side.url, token: (side) => W.sides[side.name].token }, authority: { did: (side) => auth(side).did, host: (side) => side.url, token: (side) => auth(side).token }, } const inSaOf = (wr) => (side) => commitOf(wr.host(side), wr.token(side), sp(side, sa), wr.did(side)) for (const [name, wr] of Object.entries(writers)) { await writeBoth(wr, sa, `a write by the ${name} writer`) await tracked(`a write by the ${name} writer, tracked`, inSa, wr.did, inSaOf(wr)) } const reader = { did: () => carol.did, host: () => peer.url, token: () => carol.token } await writeBoth(reader) await writeBoth(writers.peer) await tracked('a later write, tracked', inSa, writers.peer.did, inSaOf(writers.peer)) await sleep(300) /// The writer set compared across sides, by writer, data and removal. async function setIs(kind, key, wantDids) { const views = [] for (const side of sides) { const repos = await listAll(side, sp(side, key)) const dids = repos.filter((e) => e.$type === REPO_WRITE).map((e) => e.repo).sort() const want = wantDids.map((d) => d(side)).sort() if (!isDeepStrictEqual(dids, want)) await fail(side, `${kind}: who is in the writer set`, { space: alias(side, sp(side, key)) }, named(side, want), named(side, dids)) views.push(named(side, repos.map((e) => ({ repo: e.repo, ...entryView(e) }))).map(({ repoRev, ...v }) => v).sort((x, y) => (x.repo < y.repo ? -1 : 1))) } if (!isDeepStrictEqual(views[0], views[1])) await differ(`${kind}: ${LW}`, { space: '' }, views[0], views[1]) count(kind) } await setIs('the writer set, a reader not in it', sa, [writers.peer.did, writers.local.did, writers.authority.did]) // The harness writer's notifications, signed here (spec/SPACES-SYNC.md, // "Taking a notification: notifyRepoUpdate, served"). const sentCount = (side) => syncer.updates(sp(side, sa), t4).length let revAt = Date.now() const nextRev = () => tid((revAt = Math.max(revAt + 1000, Date.now()))) const notifyJwt = async (side, claims = {}, header = { typ: 'JWT', alg: 'ES256K' }, { key = writerH.key, highS = false } = {}) => { const c = { iss: writerH.did, aud: `${auth(side).did}#atproto_pds`, lxm: NRU, iat: now(), exp: now() + 60, jti: crypto.randomBytes(16).toString('hex'), ...claims } for (const k of Object.keys(c)) if (c[k] === undefined) delete c[k] const input = `${b64(header)}.${b64(c)}` let sig = Buffer.from(await key.sign(new Uint8Array(Buffer.from(input)))) if (highS) { const s = BigInt(`0x${sig.subarray(32).toString('hex')}`); sig = Buffer.concat([sig.subarray(0, 32), Buffer.from((N - s).toString(16).padStart(64, '0'), 'hex')]) } return `${input}.${sig.toString('base64url')}` } const data = (n) => ({ $bytes: Buffer.alloc(32, n).toString('base64') }) /// A notification from the harness writer to each side: `body(side)` merged /// over the usual, `claims(side)` over the usual service auth. async function notifyEach(kind, want, { body = () => ({}), claims = () => ({}), header, key, highS, authorization } = {}) { const rev = nextRev() const jwt = {} for (const side of sides) jwt[side.name] = await notifyJwt(side, claims(side), header, { key, highS }) return both(kind, NRU, (side) => ({ headers: { authorization: authorization ?? `Bearer ${jwt[side.name]}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: rev, data: data(1), ...body(side) } }), want) } const hEntry = async (side) => entryView((await listAll(side, sp(side, sa))).find((x) => x.repo === writerH.did)) const hState = {} /// A notification the page says is taken: the harness writer's entry then /// holds the `data` sent, as hex `wantHex`. const takes = async (kind, opts, dataN, wantHex = Buffer.alloc(32, dataN).toString('hex')) => { await notifyEach(kind, EMPTY, { ...opts, body: (side) => ({ data: data(dataN), ...(opts.body?.(side) ?? {}) }) }) for (const side of sides) { const e = await hEntry(side); if (e?.data !== wantHex) await fail(side, `${kind}: the writer set`, { writer: 'the harness writer' }, { data: wantHex }, e); hState[side.name] = e } } await takes('notifyRepoUpdate taken', {}, 1) await takes('notifyRepoUpdate with a bare aud', { claims: (side) => ({ aud: auth(side).did }) }, 2) await takes('notifyRepoUpdate signed high-S', { highS: true }, 3) await takes('notifyRepoUpdate with spaceRev and prevSpaceRev sent', { body: () => ({ spaceRev: tid(Date.now()), prevSpaceRev: tid(Date.now() - 1000) }) }, 4) for (const [what, opts, want] of [ ['to another fragment', { claims: (side) => ({ aud: `${auth(side).did}#other` }) }, err('Forbidden', 403)], ['to another DID', { claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('Forbidden', 403)], ['from another issuer than repo', { body: () => ({ repo: bob.did }) }, err('Forbidden', 403)], ['from an issuer with a fragment', { claims: () => ({ iss: `${writerH.did}#x` }) }, err('Forbidden', 403)], ['for another method', { claims: () => ({ lxm: LW }) }, err('BadJwtLexiconMethod', 401)], ['with no lxm', { claims: () => ({ lxm: undefined }) }, err('BadJwtLexiconMethod', 401)], ['expired', { claims: () => ({ exp: now() - 10 }) }, err('JwtExpired', 401)], ['signed by another key', { key: await Secp256k1Keypair.create() }, err('BadJwtSignature', 401)], ['of type at+jwt', { header: { typ: 'at+jwt', alg: 'ES256K' } }, err('BadJwtType', 401)], ['with no token', { authorization: '' }, err('MissingJwt', 401)], ['with a bad repoRev', { body: () => ({ repoRev: 'nope' }) }, err('InvalidRequest')], ['6 minutes ahead', { body: () => ({ repoRev: tid(Date.now() + 6 * 60000) }) }, err('FutureRev')], ['of a space never created', { body: (side) => ({ space: sp(side, skey('none')) }) }, err('SpaceNotFound')], ['of a space never created, 6 minutes ahead', { body: (side) => ({ space: sp(side, skey('none')), repoRev: tid(Date.now() + 6 * 60000) }) }, err('FutureRev')], ['of a space whose authority is elsewhere', { body: () => ({ space: peerSpace }), claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('SpaceNotFound')], ['of a space whose authority is elsewhere, 6 minutes ahead', { body: () => ({ space: peerSpace, repoRev: tid(Date.now() + 6 * 60000) }), claims: () => ({ aud: `${bob.did}#atproto_pds` }) }, err('SpaceNotFound')], ]) await notifyEach(`notifyRepoUpdate ${what}`, want, opts) for (const side of sides) if (!isDeepStrictEqual(await hEntry(side), hState[side.name])) await fail(side, 'refused notifications change nothing', { writer: 'the harness writer' }, hState[side.name], await hEntry(side)) // Stale and equal revs: taken, nothing changes, nothing sent. const before = Object.fromEntries(sides.map((side) => [side.name, sentCount(side)])) for (const side of sides) side.jwt = await notifyJwt(side) await both('notifyRepoUpdate of the same rev', NRU, (side) => ({ headers: { authorization: `Bearer ${side.jwt}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: hState[side.name].repoRev, data: data(9) } }), EMPTY) for (const side of sides) side.jwt = await notifyJwt(side) await both('notifyRepoUpdate of an earlier rev', NRU, (side) => ({ headers: { authorization: `Bearer ${side.jwt}` }, body: { space: sp(side, sa), repo: writerH.did, repoRev: tid(Date.now() - 60000), data: data(9) } }), EMPTY) for (const side of sides) delete side.jwt await writeBoth(writers.peer) await tracked('a write after stale notifications, tracked', inSa, writers.peer.did, inSaOf(writers.peer)) await sleep(300) for (const side of sides) { if (!isDeepStrictEqual(await hEntry(side), hState[side.name])) await fail(side, 'a stale notification changes nothing', { writer: 'the harness writer' }, hState[side.name], await hEntry(side)) if (sentCount(side) !== before[side.name] + 1) await fail(side, 'a stale notification is not sent', { space: alias(side, sp(side, sa)) }, `${before[side.name] + 1} updates (the peer's write alone)`, sentCount(side)) } // 4 minutes ahead is taken; odd data too. revAt = Date.now() + 4 * 60000 await takes('notifyRepoUpdate 4 minutes ahead', {}, 5) await takes('notifyRepoUpdate of 31 bytes', { body: () => ({ data: { $bytes: Buffer.alloc(31, 6).toString('base64') } }) }, 6, Buffer.alloc(31, 6).toString('hex')) // Removals, each sequenced and sent; nothing for a writer not in the set // (spec/SPACES-SYNC.md, "Removing a writer"). const removed = async () => ({ removed: true }) const quiet = async (kind, run) => { const at = Object.fromEntries(sides.map((side) => [side.name, sentCount(side)])) await run() await sleep(500) for (const side of sides) if (sentCount(side) !== at[side.name]) await fail(side, `${kind}: nothing sent`, { space: alias(side, sp(side, sa)) }, `${at[side.name]} updates`, sentCount(side)) } await both('removeWriter of the peer writer', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: bob.did } }), EMPTY) await tracked('removeWriter, shown removed', inSa, writers.peer.did, removed) await quiet('removeWriter of it again', () => both('removeWriter of it again', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: bob.did } }), EMPTY)) await quiet('removeWriter of an account never in the set', () => both('removeWriter of an account never in the set', 'com.atproto.space.removeWriter', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), repo: carol.did } }), EMPTY)) await writeBoth(writers.peer) await tracked('a removed writer\'s next write, tracked again', inSa, writers.peer.did, inSaOf(writers.peer)) await putMembers('putMembers taking the peer writer\'s write away', sa, [[bob.did, true, false]]) await tracked('putMembers taking a writer\'s write away, shown removed', inSa, writers.peer.did, removed) await both('removeMembers of the writer on the same side', 'com.atproto.memberspace.removeMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), dids: [W.sides[side.name].did] } }), EMPTY) await tracked('removeMembers of a writer, shown removed', inSa, writers.local.did, removed) // An unauthorized writer: refused, after the aud (spec/SPACES-SYNC.md, // "Taking a notification: notifyRepoUpdate, served", "For other slices"). await both('removeMembers of the harness writer', 'com.atproto.memberspace.removeMembers', (side) => ({ token: auth(side).token, body: { space: sp(side, sa), dids: [writerH.did] } }), EMPTY) await tracked('removeMembers of the harness writer, shown removed', inSa, () => writerH.did, removed) await notifyEach('notifyRepoUpdate by a writer not authorized', err('Forbidden', 403)) await notifyEach('notifyRepoUpdate by a writer not authorized, to another fragment', err('Forbidden', 403), { claims: (side) => ({ aud: `${auth(side).did}#other` }) }) await quiet('a write by a writer whose write was taken away', async () => { await writeBoth(writers.peer); await sleep(500) }) /// The updates a registration had for a side's space, each held to the page /// (spec/SPACES-SYNC.md, "Sending updates", "Deletion"). async function updatesOf(kind, side, p, space, tok) { const got = p.updates(space, tok) for (const r of got) { const u = r.body?.update const asked = r.method === 'POST' && (r.type ?? '').startsWith('application/json') && !r.authorization && keysAre(r.body, ['space', 'token', 'update']) const has = (keys) => keysAre(u, keys) || keysAre(u, [...keys, 'prevSpaceRev']) const ok = u?.$type === REPO_WRITE ? has(['$type', 'data', 'repo', 'repoRev', 'spaceRev']) && TID.test(u.repoRev) && hexOf(u.data) !== null : u?.$type === REPO_REMOVED ? has(['$type', 'repo', 'spaceRev']) : u?.$type === SPACE_DELETED && keysAre(u, ['$type']) if (!asked || !ok || (u.spaceRev !== undefined && !TID.test(u.spaceRev)) || (u.prevSpaceRev !== undefined && !TID.test(u.prevSpaceRev))) await fail(side, `${kind}: notifySpaceUpdate`, { space: alias(side, space) }, `a POST of JSON to /xrpc/${NSU}, no Authorization, body exactly {space, token, update}, update a #repoWrite {repo, repoRev, data, spaceRev, prevSpaceRev?}, a #repoRemoved {repo, spaceRev, prevSpaceRev?} or a #spaceDeleted {}`, named(side, { method: r.method, type: r.type, authorization: r.authorization, body: { ...r.body, token: '<…>' } })) } return got.map((r) => r.body.update) } /// Every change of each side's writer set sent to each registration made /// before the first, `n` times (its twins), in spaceRev order, chained by /// prevSpaceRev from none; each listWriters entry sent at its spaceRev; /// the same updates to every registration, and on both sides /// (spec/SPACES-SYNC.md, "Sending updates"). async function allSent(kind, key, regs) { const seqs = [] for (const side of sides) { const space = sp(side, key) const entries = await listAll(side, space) const top = entries.reduce((m, e) => (e.spaceRev > m ? e.spaceRev : m), '') for (const started = Date.now(); Date.now() - started < 5000; await sleep(100)) if (regs.every(([p, tok, , n = 1]) => p.updates(space, tok).filter((r) => r.body?.update?.spaceRev === top).length >= n)) break let first for (const [p, tok, name, n = 1] of regs) { const got = (await updatesOf(kind, side, p, space, tok)).filter((u) => u.$type !== SPACE_DELETED) const once = [...new Map(got.map((u) => [u.spaceRev, u])).values()].sort((x, y) => (x.spaceRev < y.spaceRev ? -1 : 1)) const times = once.find((u) => got.filter((x) => x.spaceRev === u.spaceRev).length !== n) const chain = once.findIndex((u, i) => (i === 0 ? 'prevSpaceRev' in u : u.prevSpaceRev !== once[i - 1].spaceRev)) if (times || chain !== -1 || got.some((u) => !isDeepStrictEqual(u, once.find((x) => x.spaceRev === u.spaceRev)))) await fail(side, `${kind}: each change sent ${n === 1 ? 'once' : `${n} times, once to each registration`}, chained by prevSpaceRev from none`, { registration: name }, 'each prevSpaceRev the spaceRev before, absent on the first', named(side, got)) for (const e of entries) { const u = once.find((x) => x.spaceRev === e.spaceRev) const view = (x) => x && { ...x, data: hexOf(x.data) ?? undefined, prevSpaceRev: undefined } if (!isDeepStrictEqual(view(u), view(e))) await fail(side, `${kind}: a writer set entry sent at its spaceRev`, { registration: name, writer: alias(side, e.repo) }, named(side, e), named(side, u ?? null)) } if (once.some((u) => u.repo === carol.did)) await fail(side, `${kind}: a write not tracked, not sent`, { registration: name }, 'no update of the reader', named(side, once.filter((u) => u.repo === carol.did))) const seq = once.map((u) => `${u.$type.split('#')[1]} ${alias(side, u.repo)}`) if (first === undefined) first = seq else if (!isDeepStrictEqual(seq, first)) await fail(side, `${kind}: every registration sent the same updates`, { registration: name }, first, seq) } seqs.push(first) } if (!isDeepStrictEqual(seqs[0], seqs[1])) await differ(`${kind}: the updates sent, in order`, { space: '' }, seqs[0], seqs[1]) count(kind) } await allSent('updates of the writer set', sa, regsA) for (const side of sides) { const told = refusedTokens.filter((tok) => syncer.updates(sp(side, sa), tok).length) if (told.length) await fail(side, 'a registration refused is sent nothing', { space: alias(side, sp(side, sa)) }, 'no update', `updates to ${told.length} refused registrations`) } // authorizeSpaceSync answers the writer set's greatest spaceRev // (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync"). for (const side of sides) { const entries = await listAll(side, sp(side, sa)) const top = entries.reduce((m, e) => (e.spaceRev > m ? e.spaceRev : m), '') const c = await credential(side, sp(side, sa)) if (c.spaceRev !== top) await fail(side, 'authorizeSpaceSync\'s spaceRev', { space: alias(side, sp(side, sa)) }, top, c.spaceRev ?? 'none') } count('authorizeSpaceSync\'s spaceRev, the writer set\'s') // listWriters' refusals (spec/SPACES-SYNC.md, "The writer set: listWriters", // "Commands"). const sb = skey('other') await create('createSpace for another credential', sb, { $type: MEMBERSPACE, publicRead: true }) const credB = {} for (const side of sides) credB[side.name] = await credential(side, sp(side, sb)) const credAway = await credential(ref, peerSpace, peer.url) for (const [what, make, want] of [ ['with no credential', () => ({}), err('MissingJwt', 401)], ['with a bearer token', (side) => ({ token: auth(side).token }), err('MissingJwt', 401)], ['to another audience', (side) => withCred(side, {}, bob.did), err('BadSpaceAudience', 401)], ['with another space\'s credential', (side) => withCred(side, {}, auth(side).did, credB[side.name]), err('InvalidCredential')], ]) await both(`listWriters ${what}`, LW, (side) => ({ ...make(side), params: { space: sp(side, sa) } }), want) await both('listWriters of a space whose authority is elsewhere', LW, () => ({ headers: signed(credAway, bob.did), params: { space: peerSpace } }), err('SpaceNotFound')) await both('listWriters of a malformed space', LW, (side) => withCred(side, { params: { space: 'nope' } }), err('InvalidRequest')) await both('listWriters with no credential and a malformed space', LW, () => ({ params: { space: 'nope' } }), err('InvalidRequest')) await both('listWriters with limit 0', LW, (side) => withCred(side, { params: { space: sp(side, sa), limit: 0 } }), err('InvalidRequest')) // listWriters' pages (spec/SPACES-SYNC.md, "The writer set: listWriters"). for (const side of sides) { const all = await listAll(side, sp(side, sa)) const paged = [] let cursor for (let i = 0; i < 20; i++) { const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), limit: 1, ...(cursor ? { cursor } : {}) } }) const want = all.filter((e) => cursor === undefined || e.spaceRev > cursor).slice(0, 1) const body = want.length ? { cursor: want[0].spaceRev, repos: want } : { repos: [] } if (r.status !== 200 || !isDeepStrictEqual(r.body, body)) await fail(side, 'listWriters page of 1', { cursor: cursor ?? null }, named(side, body), named(side, r.body)) if (!want.length) break paged.push(...want) cursor = want[0].spaceRev } if (!isDeepStrictEqual(paged, all)) await fail(side, 'listWriters pages, together', {}, named(side, all), named(side, paged)) for (const c of ['a', 'zzzz', all.at(-1).spaceRev]) { const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), cursor: c } }) if (r.status !== 200 || !isDeepStrictEqual(r.body, { repos: [] })) await fail(side, 'listWriters after every spaceRev', { cursor: c }, { repos: [] }, named(side, r.body)) } const first = all[0].spaceRev const r = await call(side.url, LW, { headers: signed(cred[side.name], auth(side).did), params: { space: sp(side, sa), cursor: first } }) if (r.status !== 200 || !isDeepStrictEqual(r.body?.repos, all.slice(1))) await fail(side, 'listWriters after a spaceRev, exclusive', { cursor: first }, named(side, all.slice(1)), named(side, r.body)) count('listWriters pages and cursors') } // A refusal drops the registration of its pair; an earlier refusal leaves // it (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync"; spec/SPACES-SYNC.md, // "Registering"). { const t7 = token() const tokens = async (user) => Object.fromEntries(await Promise.all(sides.map(async (side) => [side.name, await delegation(user, inSm(side))]))) const credM = {} for (const side of sides) credM[side.name] = await credential(side, inSm(side), side.url, { delegationToken: await delegation(bob, inSm(side)) }) const writeSm = async (kind) => { await writeBoth(writers.authority, sm, kind) await tracked(kind, inSm, writers.authority.did, (side) => commitOf(side.url, auth(side).token, inSm(side), auth(side).did), (side) => credM[side.name]) await sleep(500) } const told = async (kind, n) => { for (const side of sides) { const got = await updatesOf(kind, side, syncer, inSm(side), t7) if (got.length !== n) await fail(side, kind, { space: alias(side, inSm(side)) }, `${n} updates`, named(side, got)) } count('a refusal drops a registration') } let dt = await tokens(bob) await register('a member\'s registration, members only', inSm, syncer.url, t7, { extra: (side) => ({ delegationToken: dt[side.name] }) }) dt = await tokens(dave) await register('the same pair with a non-member\'s token', inSm, syncer.url, t7, { how: null, want: err('SpaceNotFound'), confirms: 0, extra: (side) => ({ delegationToken: dt[side.name] }) }) await writeSm('a write after the registration is dropped') await told('a registration dropped, sent nothing', 0) dt = await tokens(bob) await register('the member\'s registration again, confirmed anew', inSm, syncer.url, t7, { how: null, extra: (side) => ({ delegationToken: dt[side.name] }) }) await register('the same pair with a token that is no JWT', inSm, syncer.url, t7, { how: null, want: err('InvalidDelegationToken'), confirms: 0, extra: () => ({ delegationToken: 'abc' }) }) await writeSm('a write after a refusal that leaves the registration') await told('a registration left by an earlier refusal, sent the write', 1) } // Deletion notices; a space created again has no registrations and no // writers (spec/SPACES-SYNC.md, "Deletion", "Notices", "For other slices"). const sd = skey('del') const inSd = (side) => sp(side, sd) await create('createSpace to delete', sd, { $type: MEMBERSPACE, publicRead: true }) const credD = {} for (const side of sides) credD[side.name] = await credential(side, inSd(side)) const [t8, t9] = [token(), token()] await register('a registration to be told of deletion', inSd, syncer.url, t8) await register('another, at the second syncer', inSd, syncer2.url, t9, { p: syncer2 }) await both('deleteSpace with registrations', 'com.atproto.space.deleteSpace', (side) => ({ token: auth(side).token, body: { space: inSd(side) } }), EMPTY) const notices = async (kind) => { for (const side of sides) { for (const [p, tok] of [[syncer, t8], [syncer2, t9]]) { for (const started = Date.now(); !p.updates(inSd(side), tok).length && Date.now() - started < 10000;) await sleep(100) const got = await updatesOf(kind, side, p, inSd(side), tok) if (got.length !== 1 || got[0]?.$type !== SPACE_DELETED) await fail(side, `${kind}: one deletion notice to each registration`, { space: alias(side, inSd(side)) }, [{ $type: SPACE_DELETED }], named(side, got)) } } count(kind) } await notices('deletion notices') await both('deleteSpace again', 'com.atproto.space.deleteSpace', (side) => ({ token: auth(side).token, body: { space: inSd(side) } }), EMPTY) await sleep(500) await notices('nothing more for a second deleteSpace') await register('a registration of a deleted space', inSd, syncer.url, t8, { how: null, want: err('SpaceDeleted'), confirms: 0 }) await both('listWriters of a deleted space', LW, (side) => ({ headers: signed(credD[side.name], auth(side).did), params: { space: inSd(side) } }), err('SpaceNotFound')) await notifyEach('notifyRepoUpdate of a deleted space', err('SpaceNotFound'), { body: (side) => ({ space: inSd(side) }) }) await create('createSpace of the deleted space again', sd, { $type: MEMBERSPACE, publicRead: true, publicWrite: true }) for (const side of sides) { const c = await credential(side, inSd(side)) if (c.spaceRev !== undefined) await fail(side, 'authorizeSpaceSync of a space created again', { space: alias(side, inSd(side)) }, 'no spaceRev', c.spaceRev) } await writeBoth(writers.peer, sd) await tracked('a space created again, tracked with the old credential', inSd, writers.peer.did, (side) => commitOf(peer.url, bob.token, inSd(side), bob.did), (side) => credD[side.name]) await sleep(500) await notices('a space created again sends to no one') await register('a registration of the space created again, confirmed anew', inSd, syncer.url, t8, { how: null }) // The limit on registrations, held to the page alone (the reference has // none): 64 active a space; one registered already is always renewed; a // refusal's drop frees one (spec/SPACES-SYNC.md, "Registering", "Where the // reference departs from this page"). { const caps = skey('caps') const space = sp(cand, caps) const made = await call(cand.url, 'com.atproto.space.createSpace', { token: auth(cand).token, body: { spaceType: T, skey: caps, policy: { $type: MEMBERSPACE } } }) if (made.status !== 200) await fail(cand, 'createSpace for the registration limit', {}, 'the space created', normalize(cand, made)) const added = await call(cand.url, 'com.atproto.memberspace.putMembers', { token: auth(cand).token, body: { space, members: [{ did: bob.did, read: true, write: false }] } }) if (added.status !== 200) await fail(cand, 'putMembers for the registration limit', {}, 'the member added', normalize(cand, added)) const toks = Array.from({ length: 66 }, () => token()) const reg = async (what, i, want, confirms, user = bob) => { syncer.gave.set(`${space} ${toks[i]}`, 'echo') const asked = syncer.asked(space, toks[i]).length const r = await call(cand.url, AUTHORIZE, { body: { space, credentialKey: regKey.did, delegationToken: await delegation(user, space), notify: { endpoint: syncer.url, token: toks[i] } } }) const g = shape(cand, r) const wrong = want === 200 ? registered(g, r) : isDeepStrictEqual(g, want) ? null : want if (wrong) await fail(cand, `authorizeSpaceSync ${what}`, { registration: i + 1 }, wrong, g) const n = syncer.asked(space, toks[i]).length - asked if (n !== confirms) await fail(cand, `authorizeSpaceSync ${what}: confirmRegistration`, { registration: i + 1 }, `${confirms} confirmations`, n) count('the limit on registrations (the page alone)') } for (let i = 0; i < 64; i++) await reg(`, the space's registration ${i + 1}`, i, 200, 1) await reg(', a 65th', 64, err('TooManyRegistrations'), 0) await reg(', the first again, renewed', 0, 200, 0) await reg(', a non-member with the second\'s pair', 1, err('SpaceNotFound'), 0, dave) await reg(', a 65th once one is dropped', 64, 200, 1) } // ==== 2. the server as repo host ========================================================== // Into a memberspace on the second alpha PDS, clearing too // (spec/SPACES-SYNC.md, "Telling the space host: notifyRepoUpdate, sent"; // spec/SPACES-REPO.md, "For other parts"). const ex = skey('ext') const extSpace = `at://${bob.did}/space/${T}/${ex}` await call(peer.url, 'com.atproto.space.createSpace', { token: bob.token, body: { spaceType: T, skey: ex, policy: { $type: MEMBERSPACE, publicRead: true } } }) await call(peer.url, 'com.atproto.memberspace.putMembers', { token: bob.token, body: { space: extSpace, members: sides.map((side) => ({ did: W.sides[side.name].did, read: true, write: true })) } }) const credExt = await credential(ref, extSpace, peer.url) const extEntry = async (side) => { const r = await call(peer.url, LW, { headers: signed(credExt, bob.did), params: { space: extSpace } }) return entryView(r.body?.repos?.find((x) => x.repo === W.sides[side.name].did)) } async function extTracked(kind, wantData) { const started = Date.now() const done = new Set() for (;;) { for (const side of sides) { if (done.has(side.name)) continue const want = await commitOf(side.url, W.sides[side.name].token, extSpace, W.sides[side.name].did) if (wantData && want?.data !== wantData) await fail(side, `${kind}: the commit`, {}, { data: wantData }, want) if (isDeepStrictEqual(await extEntry(side), want)) done.add(side.name) } if (done.size === 2) break if (Date.now() - started > 15000) { const side = done.has('reference') ? cand : ref; await fail(side, `${kind}: tracked by the space host on the second alpha PDS`, { space: '' }, await commitOf(side.url, W.sides[side.name].token, extSpace, W.sides[side.name].did), await extEntry(side)) } await sleep(100) } const [a, b] = await Promise.all(sides.map(async (side) => (await extEntry(side))?.data)) if (a !== b) await differ(`${kind}: the data the space host on the second alpha PDS holds`, {}, a, b) count(kind) } const localW = writers.local const extWrite = async (kind, nsid, body) => both(kind, nsid, (side) => ({ token: localW.token(side), body: { space: extSpace, repo: localW.did(side), ...body } }), ok200, (g) => ({ status: g.status })) await extWrite('createRecord into a space elsewhere', 'com.atproto.space.createRecord', { collection: 'com.example.post', rkey: 'a', record: { $type: 'com.example.post', text: 'a', createdAt: '2026-11-08T00:00:00.000Z' } }) await extTracked('a write into a space elsewhere') await extWrite('createRecord into a space elsewhere again', 'com.atproto.space.createRecord', { collection: 'com.example.post', rkey: 'b', record: { $type: 'com.example.post', text: 'b', createdAt: '2026-11-08T00:00:00.000Z' } }) await extWrite('deleteRecord in a space elsewhere', 'com.atproto.space.deleteRecord', { collection: 'com.example.post', rkey: 'a' }) await extTracked('a delete in a space elsewhere') await extWrite('clearRecords in a space elsewhere', 'com.atproto.space.clearRecords', {}) await extTracked('clearRecords in a space elsewhere', EMPTY_DATA) // Into the harness space host's spaces: each notification exactly as the page says // (spec/SPACES-SYNC.md, "Telling the space host: notifyRepoUpdate, sent"). const fsp = (key) => `at://${fake.did}/space/${T}/${key}` host.answer = () => ({ status: 200, body: {} }) const sent = (side, key) => host.seen.filter((r) => r.body?.space === fsp(key) && r.body?.repo === W.sides[side.name].did) async function sentExactly(kind, key) { const started = Date.now() const out = {} for (const side of sides) { for (;;) { const want = await commitOf(side.url, W.sides[side.name].token, fsp(key), W.sides[side.name].did) const r = sent(side, key).find((x) => x.body?.repoRev === want?.repoRev) if (r) { const wrongAuth = await checkAuth(r, { iss: W.sides[side.name].did, aud: `${fake.did}#atproto_pds`, lxm: NRU }) const body = { space: fsp(key), repo: W.sides[side.name].did, repoRev: want.repoRev, data: want.data } const gotBody = r.body && typeof r.body === 'object' ? { ...r.body, data: hexOf(r.body.data) ?? r.body.data } : r.body if (wrongAuth || r.method !== 'POST' || r.path !== `/xrpc/${NRU}` || !(r.type ?? '').startsWith('application/json') || !isDeepStrictEqual(gotBody, body)) await fail(side, `${kind}: the notification sent`, { space: '' }, named(side, { method: 'POST', path: `/xrpc/${NRU}`, body: { ...body, data: `${body.data} (as bytes)` }, auth: wrongAuth ?? 'as the page says' }), named(side, { method: r.method, path: r.path, type: r.type, body: gotBody, auth: wrongAuth ?? 'as the page says' })) out[side.name] = r break } if (Date.now() - started > 15000) await fail(side, `${kind}: a notification to the space host`, { space: '' }, named(side, want), named(side, sent(side, key).map((x) => x.body))) await sleep(100) } } count(kind) return out } const o1 = skey('o') await writeBoth(localW, null, 'a write into the harness space host\'s space', () => fsp(o1)) await sentExactly('notifyRepoUpdate sent to a space host elsewhere', o1) // ==== 3. retries across a crash =========================================================== // Notifications that fail for now, kept and retried across the restart; // the refused, not sent again; deletion notices retried, but not of a space // created again (spec/SPACES-SYNC.md, "Delivery and retries", "Deletion", // "Notices"). const keys = { r503: skey('r503-'), r429: skey('r429-'), close: skey('close-'), r400: skey('r400-'), r401: skey('r401-'), r403: skey('r403-') } const failing = { [fsp(keys.r503)]: { status: 503, body: { error: 'Unavailable' } }, [fsp(keys.r429)]: { status: 429, body: { error: 'RateLimitExceeded' } }, [fsp(keys.close)]: 'close', [fsp(keys.r400)]: { status: 400, body: { error: 'InvalidRequest' } }, [fsp(keys.r401)]: { status: 401, body: { error: 'AuthenticationRequired' } }, [fsp(keys.r403)]: { status: 403, body: { error: 'Forbidden' } } } let healed = false host.answer = (r) => (!healed && failing[r.body?.space]) || { status: 200, body: {} } // Deletion notices that fail for now: one owed, one of a space created again. const [s3, s4] = [skey('d3-'), skey('d4-')] const t3 = token() for (const key of [s3, s4]) { await create('createSpace for a deletion notice retried', key, { $type: MEMBERSPACE, publicRead: true }) for (const side of sides) { syncer.gave.set(`${sp(side, key)} ${t3}`, 'echo') // Its first attempt alone fails: the server is killed before the // retry is due, so the retry comes from what it kept. syncer.failOnce.set(`${sp(side, key)} ${t3}`, { status: 503, body: { error: 'Unavailable' } }) const r = await call(side.url, AUTHORIZE, { body: { space: sp(side, key), credentialKey: regKey.did, notify: { endpoint: syncer.url, token: t3 } } }) if (r.status !== 200) await fail(side, 'a registration for a deletion notice retried', {}, 'registered', normalize(side, r)) } } // An update to a registration that errs: one attempt (spec/SPACES-SYNC.md, // "Sending updates"). Renewed first, so that it is active. await register('the registration at the endpoint with a path, renewed', inSa, `${syncer2.url}/base/path?x=`, t2, { p: syncer2, how: null, confirms: 0 }) for (const side of sides) syncer2.failing.set(`${inSa(side)} ${t2}`, { status: 500, body: { error: 'InternalServerError' } }) await writeBoth(writers.authority) const failedAt = {} const firstAttempts = async (what, find) => { const started = Date.now() for (const side of sides) { for (;;) { const r = find(side) if (r.length) { failedAt[`${side.name} ${what}`] = r.at(-1).at; break } if (Date.now() - started > 15000) await fail(side, `${what}: its first attempt`, {}, 'an attempt', 'none') await sleep(100) } } } for (const [name, key] of Object.entries(keys)) { const n = name === 'r503' ? 3 : 1 for (let i = 0; i < n; i++) await writeBoth(localW, null, 'a write into the harness space host\'s space', () => fsp(key)) await firstAttempts(name, (side) => sent(side, key).filter((r, _, all) => all.length >= n)) } const latest503 = {} for (const side of sides) latest503[side.name] = (await commitOf(side.url, W.sides[side.name].token, fsp(keys.r503), W.sides[side.name].did)).repoRev for (const side of sides) for (const key of [s3, s4]) { const r = await call(side.url, 'com.atproto.space.deleteSpace', { token: auth(side).token, body: { space: sp(side, key) } }) if (r.status !== 200) await fail(side, 'deleteSpace for a deletion notice retried', {}, 'deleted', normalize(side, r)) } const deletionNotices = (key) => (side) => syncer.updates(sp(side, key), t3).filter((r) => r.body?.update?.$type === SPACE_DELETED) await firstAttempts('s3', deletionNotices(s3)) await firstAttempts('s4', deletionNotices(s4)) await firstAttempts('update', (side) => syncer2.updates(inSa(side), t2).filter((r) => r.answered === 500)) await create('createSpace of a space owed a notice, again', s4, { $type: MEMBERSPACE }) // After the run restarts the server: everything answers 200, and each // retry is judged against the time of its failure. afterRestart.push(async () => { healed = true syncer.failing.clear() syncer2.failing.clear() const since = (find, name) => (side) => find(side).filter((r) => r.at > failedAt[`${side.name} ${name}`]) const toHost = (key, name) => since((side) => host.seen.filter((r) => r.body?.space === fsp(key) && r.body?.repo === W.sides[side.name].did), name) const expectations = [ ['the 503 space\'s latest notification', toHost(keys.r503, 'r503'), 1, 'r503', (r, side) => r.body.repoRev === latest503[side.name]], ['the 429 notification', toHost(keys.r429, 'r429'), 1, 'r429'], ['the closed notification', toHost(keys.close, 'close'), 1, 'close'], ['the 400 notification', toHost(keys.r400, 'r400'), 0, 'r400'], ['the 401 notification', toHost(keys.r401, 'r401'), 0, 'r401'], ['the 403 notification', toHost(keys.r403, 'r403'), 0, 'r403'], ['the owed deletion notice', since(deletionNotices(s3), 's3'), 1, 's3'], ['the deletion notice of a space created again', since(deletionNotices(s4), 's4'), 0, 's4'], ['the update the registration refused', since((side) => syncer2.updates(inSa(side), t2), 'update'), 0, 'update'], ] const last = Math.max(...Object.values(failedAt)) for (;;) { const pending = sides.some((side) => expectations.some(([, find, n]) => find(side).length < n)) if ((!pending && Date.now() - last > 75000) || Date.now() - last > 100000) break await sleep(500) } for (const side of sides) { for (const [what, find, n, name, ok] of expectations) { const got = find(side) const failed = failedAt[`${side.name} ${name}`] const wrong = got.length !== n ? `${n} after the failure` : got.find((r) => r.answered !== 200 || r.at - failed < 25000 || (ok && !ok(r, side))) if (wrong) await fail(side, `a retry: ${what}`, { within: '100 seconds of the failure, across the server\'s restart' }, typeof wrong === 'string' ? wrong : 'a retry at least 25 seconds after the failure, of the latest state', named(side, got.map((r) => ({ after: `${Math.round((r.at - failed) / 1000)} s`, body: { ...r.body, token: r.body?.token && '<…>' }, answered: r.answered })))) } count('retries across a crash') } syncer.close(); syncer2.close(); host.close() }) }