// A space repo whole, its blobs, and migration (spec/SPACES-EXPORT.md), in // the spaces run beside the space repo's steps (repo.mjs), whose model of // every repo it reads and adds to: the server and the alpha's first PDS each // the repo host of their side's accounts, the authorities' spaces on the // alpha's second PDS, the peer, which issues the harness, as a syncer, // credentials for them. Each step goes to both sides and the answers are // compared; on each side, the reference's too, every archive must verify // with lib/spec.mjs and be what its writer makes of the model's commit and // records, every blob be the model's, and every MAC the harness's. import crypto from 'node:crypto' import http from 'node:http' import { isDeepStrictEqual } from 'node:util' import { COMMIT_PROFILE, LtHash, RECORD_PROFILE, cidBytes, cidString, encapsulate, encode, formatMacRequest, headerNamesCommit, headerNamesRecord, newP256, readRepoStar, readStar, signCredential, writeStar } from './lib/spec.mjs' const BLOB_COLL = 'com.example.blob' const MAX_IMPORT = 5242880 /// The media type of a space repo's STAR archive (spec/SPACES-EXPORT.md, /// "getRepo", "importRepo"). const STAR = 'application/vnd.dasl.star' /// The MAC headers, which a read that takes no MAC never answers. const MAC_HEADERS = ['atproto-space-commit', 'atproto-space-record', 'signature-input', 'signature'] /// The checks, given the run (compare.mjs) and the space repo's internals. export function spaceExport(h, R) { const { ref, cand, accounts } = h const sides = [ref, cand] const { SP, COLLECTIONS, RKEYS, call, both, norm, shown, fail, tokenOf, sd, didOf, repoOf, digestOf, put, after, latestCommit, pickSpace, keyOn, cidOf, drisl, typed, below, chance, pick, checkMacHeaders, holdInvalidMac, checkCommit, commitOf, byBytes, macAsk, askHeaders, askName } = R const peer = h.net.extraPdses[0] const pairOf = (a, b) => [[ref, a], [cand, b]] // ---- credentials, from the peer --------------------------------------------------------- const kid = newP256() const creds = new Map() let peerSpaces = [] /// A credential for a space the peer hosts, fresh enough for a step, from /// its `authorizeSpaceSync`, as a syncer gets one /// (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync"). async function credentialFor(space) { const c = creds.get(space) if (c && Date.now() - c.at < 300_000) return c.credential const res = await call(peer, `${SP}authorizeSpaceSync`, { body: { space, credentialKey: kid.did } }) if (res.status !== 200) await h.differ('authorizeSpaceSync at the peer', { space }, 'a credential: the space is publicRead', res.body, 'the harness cannot read as a syncer') creds.set(space, { credential: res.body.credential, at: Date.now() }) return res.body.credential } const credHeaders = (credential, audience) => signCredential(kid, credential, audience) // ---- blobs ------------------------------------------------------------------------------ let bodies = 0 const tainted = new Set() const storeOf = (acct, side) => (sd(acct, side).store ??= new Map()) const publicRefs = (acct, side) => (sd(acct, side).publicRefs ??= new Set()) const rawCid = (bytes) => cidString(Buffer.concat([Buffer.from([0x01, 0x55, 0x12, 0x20]), crypto.createHash('sha256').update(bytes).digest()])) const isLink = (v) => v && typeof v === 'object' && !Array.isArray(v) && Object.keys(v).length === 1 && typeof v.$link === 'string' /// The CIDs a record's blob refs name (spec/BLOBS.md, "Blob refs in records"). function refsOf(v, out = new Set()) { if (Array.isArray(v)) { v.forEach((x) => refsOf(x, out)); return out } if (!v || typeof v !== 'object') return out const keys = Object.keys(v).sort() if (isDeepStrictEqual(keys, ['$type', 'mimeType', 'ref', 'size']) && v.$type === 'blob' && isLink(v.ref) && typeof v.mimeType === 'string' && v.mimeType.includes('/') && Number.isInteger(v.size) && v.size >= -1) { out.add(v.ref.$link); return out } for (const x of Object.values(v)) refsOf(x, out) return out } const repoRefs = (m) => { const out = new Set(); for (const { value } of m.records.values()) refsOf(value, out); return out } function referencedAnywhere(acct, side) { const out = new Set(publicRefs(acct, side)) for (const m of (sd(acct, side).repos ?? new Map()).values()) for (const c of repoRefs(m)) out.add(c) return out } /// A blob the account holds the bytes of: an upload no record references /// yet, or one a record references (spec/SPACES-EXPORT.md, "Blobs in space /// records"). function alive(acct, side, cid) { const e = storeOf(acct, side).get(cid) return e && !e.dead && (!e.perm || referencedAnywhere(acct, side).has(cid)) ? e : null } /// After a write on `side` that took refs: each blob it names is held for good. function settle(acct, side, record) { for (const cid of refsOf(record)) { const e = storeOf(acct, side).get(cid); if (e) e.perm = true } } /// The blobs a repo's records reference that no longer are, after its /// records were replaced by an import: the reference keeps them held, the /// page drops them, so the harness never names them again. function taint(acct, side, before) { const now = referencedAnywhere(acct, side) for (const cid of before) if (!now.has(cid) && storeOf(acct, side).get(cid)?.perm) tainted.add(cid) } async function uploadBlob(acct, body) { const known = [...storeOf(acct, ref).values()].filter((e) => !tainted.has(e.cid)) const others = accounts.filter((a) => a !== acct).flatMap((a) => [...storeOf(a, ref).values()]).filter((e) => !tainted.has(e.cid)) const r = rand() const bytes = body ?? (r < 0.6 || (!known.length && !others.length) ? Buffer.from(`spec-pds blob ${++bodies} ${crypto.randomBytes(4).toString('hex')}`) : r < 0.85 && known.length ? pick(known).bytes : pick(others.length ? others : known).bytes) if (tainted.has(rawCid(bytes))) return // Text, as the bytes are: the alpha refuses a blob whose bytes are not of // the type it is sent as, which BLOBS.md, held to the `latest` // reference, does not ask. const mime = 'text/plain' const mk = (side) => ({ token: tokenOf(acct, side), raw: bytes, contentType: mime }) const [a, b] = await both('uploadBlob (for space records)', acct, 'com.atproto.repo.uploadBlob', mk) if (a.status !== 200) return for (const side of sides) { const cid = rawCid(bytes) if (!alive(acct, side, cid)) storeOf(acct, side).set(cid, { cid, bytes, mime, perm: false }) if (referencedAnywhere(acct, side).has(cid)) storeOf(acct, side).get(cid).perm = true } return rawCid(bytes) } const rand = h.rand /// A blob ref for a write by `acct`: to one of its blobs as stored, or one /// that may be wrong (spec/SPACES-EXPORT.md, "Blobs in space records"). async function blobRef(acct) { let held = [...storeOf(acct, ref).values()].filter((e) => alive(acct, ref, e.cid) && !tainted.has(e.cid)) if (!held.length) { await uploadBlob(acct); held = [...storeOf(acct, ref).values()].filter((e) => alive(acct, ref, e.cid) && !tainted.has(e.cid)) } const e = held.length ? pick(held) : null const good = (x) => ({ $type: 'blob', ref: { $link: x.cid }, mimeType: x.mime, size: x.bytes.length }) const r = rand() if (!e || r < 0.05) return { $type: 'blob', ref: { $link: rawCid(crypto.randomBytes(16)) }, mimeType: 'text/plain', size: 16 } if (r < 0.1) return { ...good(e), mimeType: 'image/png' } if (r < 0.15) return { ...good(e), size: pick([e.bytes.length + 1, -1]) } if (r < 0.2) { const other = accounts.filter((a) => a !== acct).flatMap((a) => [...storeOf(a, ref).values()]).filter((x) => !tainted.has(x.cid)) if (other.length) return good(pick(other)) } return good(e) } /// A space write of a record with blob refs, in a collection of its own. async function blobWrite(acct) { const sp = pickSpace({ malformed: 0, ownP: 0.03 }) const key = pick(RKEYS) const rec = { $type: BLOB_COLL, img: await blobRef(acct), n: below(100) } if (chance(0.3)) { const more = [...storeOf(acct, ref).values()].filter((e) => alive(acct, ref, e.cid) && !tainted.has(e.cid)) if (more.length) { const e = pick(more); rec.list = [{ $type: 'blob', ref: { $link: e.cid }, mimeType: e.mime, size: e.bytes.length }] } } const nsid = chance(0.5) ? 'createRecord' : 'putRecord' const mk = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(acct, side), repo: didOf(acct, side), collection: BLOB_COLL, rkey: key, record: rec } }) const [a, b] = await both(`${nsid} with blob refs`, acct, `${SP}${nsid}`, mk) await after(nsid, acct, sp, a, b, async (side, res) => { await put(side, acct, sp, BLOB_COLL, key, rec, res.body.cid, shown(nsid, mk(cand)), `${nsid} with blob refs`) settle(acct, side, rec) return true }) } async function blobDelete(acct) { const sp = pickSpace({ malformed: 0, ownP: 0 }) const keys = [...repoOf(acct, ref, sp).records.keys()].filter((p) => p.startsWith(`${BLOB_COLL}/`)) const key = keys.length ? pick(keys).slice(BLOB_COLL.length + 1) : pick(RKEYS) const mk = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(acct, side), repo: didOf(acct, side), collection: BLOB_COLL, rkey: key } }) const [a, b] = await both('deleteRecord of a record with blob refs', acct, `${SP}deleteRecord`, mk) await after('deleteRecord', acct, sp, a, b, async (side) => repoOf(acct, side, sp).records.delete(`${BLOB_COLL}/${key}`)) } // ---- who reads ------------------------------------------------------------------------------ /// How a read authenticates, drawn for a step: the account's session, or a /// credential for its own repo, another's, or one with something wrong /// (spec/SPACES-EXPORT.md, "Who may read"). async function reader(acct) { const r = rand() const sp = peerSpaces.length && r >= 0.4 ? pick(peerSpaces) : null if (!sp) return { kind: chance(0.15) ? 'app' : 'session', sp: pickSpace({ malformed: 0.02, ownP: 0.03 }), target: acct } const others = accounts.filter((a) => a !== acct) const v = rand() const kind = v < 0.55 ? 'credential' : v < 0.7 ? 'credential, another account' : v < 0.76 ? 'credential, no account here' : v < 0.82 ? 'credential, wrong audience' : v < 0.88 ? 'credential of another space' : v < 0.94 ? 'credential, the session read another account' : 'credential, unknown DID' const credential = await credentialFor(kind === 'credential of another space' ? pick(peerSpaces.filter((s) => s !== sp).concat([sp])).uri() : sp.uri()) const target = kind === 'credential, another account' && others.length ? pick(others) : acct return { kind, sp, target, credential, authority: sp.uri().split('/')[2] } } /// The request's auth and `repo` on a side. function authOn(rd, acct, side) { const repo = rd.kind === 'credential, no account here' ? rd.authority : rd.kind === 'credential, unknown DID' ? 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' : didOf(rd.target, side) if (rd.kind === 'session' || rd.kind === 'app') return { repo, token: tokenOf(acct, side, rd.kind === 'app' ? 'app' : 'access') } if (rd.kind === 'credential, the session read another account') return { repo: didOf(accounts.find((a) => a !== acct) ?? acct, side), token: tokenOf(acct, side) } return { repo, headers: credHeaders(rd.credential, rd.kind === 'credential, wrong audience' ? didOf(acct, side) === repo ? 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' : didOf(acct, side) : repo) } } /// Whether a read on `side` is of the model's repo, whose answer the /// harness can say. const modelled = (rd) => ['session', 'app', 'credential', 'credential, another account'].includes(rd.kind) // ---- getBlob and listBlobs ------------------------------------------------------------------------ /// getBlob, compared, and against the blobs the model holds and the repo /// references (spec/SPACES-EXPORT.md, "getBlob"). async function getBlob(acct) { const rd = await reader(acct) const m = repoOf(rd.target, ref, rd.sp) const refs = rd.sp.malformed ? [] : [...repoRefs(m)].filter((c) => !tainted.has(c)) const stored = [...storeOf(rd.target, ref).keys()].filter((c) => !tainted.has(c)) const r = rand() const cid = refs.length && r < 0.6 ? pick(refs) : stored.length && r < 0.8 ? pick(stored) : r < 0.88 ? rawCid(crypto.randomBytes(8)) : r < 0.94 && R.knownCids.length ? pick(R.knownCids) : r < 0.97 ? 'nope' : pick(stored.length ? stored : ['nope']) const mk = (side) => { const { repo, ...auth } = authOn(rd, acct, side); return { ...auth, params: { space: rd.sp.uri(acct, side), repo, cid } } } const [a, b] = await Promise.all(sides.map((side) => call(side, `${SP}getBlob`, mk(side)))) const view = (side, res) => res.status >= 400 ? norm(rd.target, side, res) : { status: res.status, bytes: res.buf.toString('base64'), headers: blobHeaders(res) } const [na, nb] = [view(ref, a), view(cand, b)] if (!isDeepStrictEqual(na, nb)) await h.differ(`${SP}getBlob`, shown(`getBlob (${rd.kind})`, mk(cand)), na, nb) h.count(`getBlob (${rd.kind})`) if (!modelled(rd) || rd.sp.malformed) return for (const [side, res] of pairOf(a, b)) { const mm = repoOf(rd.target, side, rd.sp) if (cid === 'nope') continue const e = alive(rd.target, side, cid) const want = !repoRefs(mm).has(cid) ? { status: 400, error: 'BlobNotFound' } : e?.perm ? { status: 200, bytes: e.bytes.toString('base64'), headers: { 'content-type': e.mime, 'content-length': String(e.bytes.length), 'x-content-type-options': 'nosniff', 'content-disposition': `attachment; filename="${cid}"`, 'content-security-policy': "default-src 'none'; sandbox" } } : { status: 400, error: 'InvalidRequest' } const got = res.status >= 400 ? { status: res.status, error: res.body?.error ?? null } : { status: res.status, bytes: res.buf.toString('base64'), headers: blobHeaders(res) } if (!isDeepStrictEqual(got, want)) await fail(side, 'getBlob, against the blobs the harness uploaded and the records that reference them', shown('getBlob', mk(side)), want, got) } } function blobHeaders(res) { const out = {} for (const k of ['content-type', 'content-length', 'x-content-type-options', 'content-disposition', 'content-security-policy']) out[k] = res.headers.get(k) out['content-type'] = out['content-type']?.split(';')[0].trim() ?? null return out } /// listBlobs, paged with each side's cursors, against the blobs the repo /// references; a `since` sent is ignored (spec/SPACES-EXPORT.md, /// "listBlobs"). async function listBlobs(acct) { const rd = await reader(acct) const limit = chance(0.5) ? 1 + below(4) : undefined const revs = rd.sp.malformed ? [] : repoOf(rd.target, ref, rd.sp).revs const sinceAt = chance(0.3) ? (revs.length && chance(0.8) ? below(revs.length) : -1) : undefined const sinceOn = (side) => (sinceAt === undefined ? undefined : sinceAt === -1 ? 'not-a-tid' : repoOf(rd.target, side, rd.sp).revs[sinceAt]) const cursors = {} const got = { reference: [], server: [] } const mk = (side) => { const { repo, ...auth } = authOn(rd, acct, side); return { ...auth, params: { space: rd.sp.uri(acct, side), repo, limit, cursor: cursors[side.name], since: sinceOn(side) } } } for (let page = 0; ; page++) { const [a, b] = await both(`listBlobs (${rd.kind})`, rd.target, `${SP}listBlobs`, mk) if (a.status !== 200) return for (const [side, res] of pairOf(a, b)) { const cids = res.body.cids ?? [] got[side.name].push(...cids) if (Boolean(res.body.cursor) !== (cids.length === (limit ?? 500)) || (res.body.cursor && res.body.cursor !== cids.at(-1))) await fail(side, 'listBlobs cursor', shown('listBlobs', mk(side)), cids.length === (limit ?? 500) ? 'the page\'s last CID, as the page is full' : 'none, as the page is not full', res.body.cursor ?? null) cursors[side.name] = res.body.cursor } if (!a.body.cursor || page > 500) break } if (!modelled(rd) || rd.sp.malformed) return for (const side of sides) { const want = [...repoRefs(repoOf(rd.target, side, rd.sp))].sort() if (!isDeepStrictEqual(got[side.name], want)) await fail(side, 'listBlobs, every page, against the blobs the repo\'s records reference', shown('listBlobs', mk(side)), want, got[side.name]) } } // ---- getRepo ------------------------------------------------------------------------------------- /// An archive as compared across the sides: the commit and the pairs, the /// side's names taken out. function archiveView(acct, side, res) { let parsed try { parsed = readStar(res.buf, { meta: 65536, key: 4096, value: 1048576 }) } catch (e) { return { status: res.status, unreadable: e.message } } const { meta, pairs } = parsed const body = { meta: { ...meta, data: { $bytes: Buffer.from(meta.data ?? []).toString('base64') } }, pairs: pairs.map(({ key, value }) => ({ key: key.toString('utf8'), value: value.toString('hex') })) } return { ...norm(acct, side, { status: res.status, body, headers: res.headers }), type: res.headers.get('content-type') } } /// getRepo, compared, and on each side read by lib/spec.mjs, its MAC /// verified, against the model, and the bytes its writer makes; with a /// credential, a MAC asked for or `InvalidSpaceMac` (spec/SPACES-EXPORT.md, /// "getRepo", "Who may read"; spec/SPACES-REPO.md, "MACs on request", "For /// other parts"). async function getRepo(acct, rd0, why = '') { const rd = rd0 ?? await reader(acct) const credential = !['session', 'app'].includes(rd.kind) const w = credential && chance(0.8) ? macAsk(COMMIT_PROFILE, { none: 0, good: 1 }) : macAsk(COMMIT_PROFILE) const shares = {} const shareOf = (side) => (shares[side.name] ??= encapsulate(sd(rd.target, side).kem)) const mk = (side) => { const { repo, ...auth } = authOn(rd, acct, side) return { ...auth, params: { space: rd.sp.uri(acct, side), repo }, headers: { ...auth.headers, ...askHeaders(w, shareOf(side)) } } } const kind = `getRepo (${rd.kind})${askName(w)}${why}` const [a, b] = await Promise.all(sides.map((side) => call(side, `${SP}getRepo`, mk(side)))) const view = (side, res) => (res.status >= 400 ? norm(rd.target, side, res) : archiveView(rd.target, side, res)) const [na, nb] = [view(ref, a), view(cand, b)] if (!isDeepStrictEqual(na, nb)) await h.differ(`${SP}getRepo`, shown(kind, mk(cand)), na, nb) h.count(kind) if (!modelled(rd) || rd.sp.malformed) return if (w.refused || (credential && !w.asks)) return holdInvalidMac(a, b, shown(kind, mk(cand)), 'getRepo') if (a.status !== 200) return for (const [side, res] of pairOf(a, b)) { const request = shown('getRepo', mk(side)) const m = repoOf(rd.target, side, rd.sp) if (res.headers.get('content-type')?.split(';')[0].trim() !== STAR) await fail(side, 'getRepo Content-Type', request, STAR, res.headers.get('content-type')) const expect = { space: rd.sp.uri(acct, side), repo: didOf(rd.target, side) } let got try { got = readRepoStar(res.buf, w.asks ? { expect, ...shareOf(side), headers: Object.fromEntries(res.headers) } : { expect }) } catch (e) { await fail(side, 'getRepo: the archive, read by lib/spec.mjs with its MAC', request, 'an atproto-space-repo archive of the repo', e.message); continue } const c = { ver: got.commit.ver, repo: got.commit.repo, space: got.commit.space, rev: got.commit.rev, data: { $bytes: Buffer.from(got.commit.data).toString('base64').replace(/=+$/, '') } } await checkCommit(side, rd.target, rd.sp, c, request, 'getRepo') if (c.rev !== m.rev) await fail(side, 'getRepo: the commit\'s rev, against getLatestCommit\'s', request, m.rev, c.rev) const want = [...m.records].sort(([x], [y]) => byBytes(x, y)).map(([path, r]) => ({ path, cid: r.cid })) const records = got.records.map(({ path, cid }) => ({ path, cid })) if (!isDeepStrictEqual(records, want)) await fail(side, 'getRepo: the records, against those the harness wrote', request, want, records) const { meta, pairs } = readStar(res.buf, { meta: 65536, key: 4096, value: 1048576 }) if (!writeStar(meta, pairs).equals(res.buf)) await fail(side, 'getRepo: the archive\'s bytes, against what lib/spec.mjs writes of its commit and records', request, writeStar(meta, pairs).toString('hex').slice(0, 400), res.buf.toString('hex').slice(0, 400)) await checkMacHeaders(side, res, request, w, shareOf(side), (d) => headerNamesCommit(d, got.commit)) } } // ---- the space repo's reads with a credential ------------------------------------------------------------ /// The space repo's reads with a credential from the peer: a MAC asked /// for, or `InvalidSpaceMac`, where the method answers one, and any /// `Accept-Signature` ignored by listRecords (spec/SPACES-EXPORT.md, "Who /// may read"; spec/SPACES-REPO.md, "MACs on request", "For other parts"). async function credentialRead(acct) { if (!peerSpaces.length) return let rd = await reader(acct) if (['session', 'app'].includes(rd.kind)) rd = { ...rd, kind: 'credential', sp: pick(peerSpaces), credential: await credentialFor(pick(peerSpaces).uri()) } if (rd.kind === 'credential' || rd.kind === 'credential, another account') rd.credential = await credentialFor(rd.sp.uri()) const [nsid, profile] = pick([['getRecord', RECORD_PROFILE], ['listRecords', null], ['getLatestCommit', COMMIT_PROFILE], ['listRepoOps', COMMIT_PROFILE]]) const m = repoOf(rd.target, ref, rd.sp) const paths = [...m.records.keys()] const path = paths.length && chance(0.8) ? pick(paths) : `${pick(COLLECTIONS)}/${pick(RKEYS)}` const [coll, key] = [path.slice(0, path.indexOf('/')), path.slice(path.indexOf('/') + 1)] const w = profile && chance(0.8) ? macAsk(profile, { none: 0, good: 1 }) : macAsk(profile ?? COMMIT_PROFILE) const shares = {} const shareOf = (side) => (shares[side.name] ??= encapsulate(sd(rd.target, side).kem)) const mk = (side) => { const { repo, ...auth } = authOn(rd, acct, side) const params = { space: rd.sp.uri(acct, side), repo, ...(nsid === 'getRecord' ? { collection: coll, rkey: keyOn(rd.target, side, key) } : {}) } return { ...auth, params, headers: { ...auth.headers, ...askHeaders(w, shareOf(side)) } } } const kind = `${nsid} (${rd.kind})${askName(w)}` const [a, b] = await both(kind, rd.target, `${SP}${nsid}`, mk) if (!modelled(rd)) return if (profile && (w.refused || !w.asks)) return holdInvalidMac(a, b, shown(kind, mk(cand)), nsid) if (profile && a.status !== 200) return for (const [side, res] of pairOf(a, b)) { const request = shown(nsid, mk(side)) if (!profile) { const present = MAC_HEADERS.filter((k) => res.headers.has(k)) if (present.length) await fail(side, 'listRecords: MAC headers, as it takes no MAC', request, 'none', present) continue } const names = nsid === 'getRecord' ? (d) => headerNamesRecord(d, res.body.uri, res.body.cid) : (d) => Boolean(res.body.commit) && headerNamesCommit(d, commitOf(res.body.commit)) await checkMacHeaders(side, res, request, w, shareOf(side), names) } } // ---- importRepo -------------------------------------------------------------------------------- /// An archive of `records` ([path, JSON value]) for the account on a side, /// as lib/spec.mjs writes one; `o` changes its commit. function archiveOf(acct, side, space, records, o = {}) { const lt = new LtHash() const pairs = records.map(([path, v]) => { const value = encode(drisl(v)); lt.add(`${path}/${cidString(cidBytes(value))}`); return { key: Buffer.from(path), value } }).sort((x, y) => Buffer.compare(x.key, y.key)) return writeStar({ $type: 'atproto-space-repo', ver: 1, repo: didOf(acct, side), space, rev: '3mzzzzzzzzzz2', data: lt.digest(), ...o }, pairs) } /// One record to import: good, with blob refs held, uploaded only, or /// never uploaded, or of a published collection. function importRecord(acct, used) { const r = rand() const coll = r < 0.05 ? pick(['com.atproto.space.policy', 'com.atproto.memberspace.member']) : r < 0.45 ? BLOB_COLL : pick(COLLECTIONS) const free = RKEYS.filter((k) => !used.has(`${coll}/${k}`)) if (!free.length) return null const key = pick(free) used.add(`${coll}/${key}`) const v = { $type: coll, n: below(1000) } if (coll === BLOB_COLL) { const pool = [...storeOf(acct, ref).values()].filter((e) => !tainted.has(e.cid)) const e = pool.length && chance(0.8) ? pick(pool) : null v.img = e ? { $type: 'blob', ref: { $link: e.cid }, mimeType: e.mime, size: e.bytes.length } : { $type: 'blob', ref: { $link: rawCid(crypto.randomBytes(16)) }, mimeType: 'text/plain', size: 16 } } return [`${coll}/${key}`, v] } const otherDid = (acct, side) => (didOf(accounts.find((a) => a !== acct) ?? acct, side) === didOf(acct, side) ? 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' : didOf(accounts.find((a) => a !== acct), side)) /// The ways an archive can be wrong, each with the error the page says it /// gets, in the order of the checks (spec/SPACES-EXPORT.md, "importRepo"). const FAULTS = { 'another account\'s repo': ['RepoMismatch', (acct, side, space) => archiveOf(acct, side, space, [], { repo: otherDid(acct, side) })], 'another account\'s repo, data not the records\' LtHash': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { repo: otherDid(acct, side), data: Buffer.alloc(32) })], 'data not the records\' LtHash': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { data: Buffer.alloc(32) })], 'keys out of order': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/b', encode({ a: 1 })], ['com.example.note/a', encode({ a: 1 })]])], 'a key repeated': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/a', encode({ a: 1 })], ['com.example.note/a', encode({ a: 2 })]])], 'a value not canonical DRISL': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/a', Buffer.from('a2616202616101', 'hex')]])], 'a value not canonical DRISL, in a space of its own never created': ['SpaceNotFound', (acct, side) => writeRaw(acct, side, R.own.uri(acct, side), [['com.example.note/a', Buffer.from('a2616202616101', 'hex')]])], 'a value not a map': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/a', encode([1])]])], 'a key not a record path': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['notnsid/a', encode({ a: 1 })]])], 'ver 2': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { ver: 2 })], 'another $type': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { $type: 'atproto-space-meta' })], 'a rev that is no TID': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { rev: 'hello' })], 'a space that is no space-ref': ['InvalidRepoFile', (acct, side) => archiveOf(acct, side, 'at://did:web:example.com/spaces/com.example.group/k', [])], 'a repo that is no DID': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { repo: 'handle.test' })], 'bad magic': ['InvalidRepoFile', () => Buffer.from('hello')], 'no bytes': ['InvalidRepoFile', () => Buffer.alloc(0)], 'a byte after the end': ['InvalidRepoFile', (acct, side, space) => Buffer.concat([archiveOf(acct, side, space, []), Buffer.from([0])])], } /// The ways the request of a good archive can be wrong, and their errors. const REQUEST_FAULTS = { 'an app password': 'InvalidToken', 'a refresh token': 'InvalidToken', 'Content-Type JSON': 'InvalidRequest', 'Content-Type application/octet-stream': 'InvalidRequest', 'no Content-Type': 'InvalidRequest' } function writeRaw(acct, side, space, kv) { const lt = new LtHash() for (const [p, v] of kv) lt.add(`${p}/${cidString(cidBytes(v))}`) return writeStar({ $type: 'atproto-space-repo', ver: 1, repo: didOf(acct, side), space, rev: '3mzzzzzzzzzz2', data: lt.digest() }, kv.map(([p, v]) => ({ key: Buffer.from(p), value: v }))) } /// importRepo of the side's own archive, of one the harness writes, or of /// one with a fault, then the repo read whole; a fault is the error the /// page names on each side (spec/SPACES-EXPORT.md, "importRepo"). async function importRepo(acct) { const r = rand() const existing = R.spaces.filter((sp) => repoOf(acct, ref, sp).exists) const reimport = r >= 0.05 && r < 0.35 && existing.length > 0 const sp = r < 0.05 ? R.own : reimport ? pick(existing) : pick(R.spaces) const mode = reimport ? 'its own archive' : r < 0.8 ? 'an archive the harness writes' : 'fault' const fault = mode === 'fault' ? pick([...Object.keys(FAULTS), ...Object.keys(REQUEST_FAULTS)]) : null let records = null const bodyOf = {} for (const side of sides) { const space = sp.uri(acct, side) if (mode === 'its own archive') { const g = await call(side, `${SP}getRepo`, { token: tokenOf(acct, side), params: { space, repo: didOf(acct, side) } }) if (g.status !== 200) await fail(side, 'getRepo of a repo to import back', { space }, 'the archive', norm(acct, side, g)) bodyOf[side.name] = g.buf } else if (mode === 'an archive the harness writes') { if (!records) { const used = new Set(); records = []; for (let i = below(5); i > 0; i--) { const rec = importRecord(acct, used); if (rec) records.push(rec) } } bodyOf[side.name] = archiveOf(acct, side, space, records) } else bodyOf[side.name] = FAULTS[fault] ? FAULTS[fault][1](acct, side, space) : archiveOf(acct, side, space, []) } if (FAULTS[fault] === undefined && mode === 'fault') records = [] const BAD_TYPES = { 'Content-Type JSON': 'application/json', 'Content-Type application/octet-stream': 'application/octet-stream', 'no Content-Type': null } const contentType = fault in BAD_TYPES ? BAD_TYPES[fault] : chance(0.2) ? `${STAR}; charset=binary` : STAR const mk = (side) => ({ token: tokenOf(acct, side, fault === 'an app password' ? 'app' : fault === 'a refresh token' ? 'refresh' : 'access'), raw: bodyOf[side.name], contentType }) const before = {} for (const side of sides) before[side.name] = referencedAnywhere(acct, side) const [a, b] = await both(`importRepo (${fault ?? mode})`, acct, `${SP}importRepo`, mk) const error = FAULTS[fault]?.[0] ?? REQUEST_FAULTS[fault] if (error) for (const [side, res] of pairOf(a, b)) if (res.status !== 400 || res.body?.error !== error) await fail(side, `importRepo (${fault})`, shown('importRepo', { ...mk(side), raw: undefined }), { status: 400, error }, { status: res.status, error: res.body?.error ?? null }) await after('importRepo', acct, sp, a, b, async (side) => { // A blob no longer held stays so though the import references it // again, until it is uploaded again (spec/SPACES-EXPORT.md, // "importRepo"). for (const e of storeOf(acct, side).values()) if (!alive(acct, side, e.cid)) e.dead = true const m = repoOf(acct, side, sp) if (records) { m.records = new Map(records.map(([path, v]) => [path, { cid: cidOf(v, path.slice(0, path.indexOf('/'))), value: v }])) } else if (FAULTS[fault]) await fail(side, `importRepo of an archive with ${fault}`, {}, 'refused', 'taken') m.base = new Map(m.records) m.exists = true taint(acct, side, before[side.name]) return true }) if (a.status === 200 && !sp.malformed) { await getRepo(acct, { kind: 'session', sp, target: acct }, ' after importRepo') if (records?.some(([, v]) => refsOf(v).size)) { // A blob an imported record references, uploaded again: held, and served. const pool = records.flatMap(([, v]) => [...refsOf(v)]).map((cid) => storeOf(acct, ref).get(cid)).filter(Boolean) if (pool.length) await uploadBlob(acct, pick(pool).bytes) } } } // ---- fixed checks --------------------------------------------------------------------------------- /// The importRepo body limit, at its edge. The reference may answer either /// before it has read the whole body: the one at the limit `400`, having /// read only its archive, and the one past it `413`. It then either resets /// the connection, which may lose the answer, or leaves the rest of the /// body unread on a connection it keeps open, where the next request is /// never read and is reset when its keep-alive timeout (90 s) ends. So each /// is sent, on either side alike, on a connection of its own, which nothing /// is sent on after, and one reset before its answer is sent again, up to /// three times (spec/SPACES-EXPORT.md, "importRepo"). async function importSize(acct) { const alone = (side, { raw, token, contentType }) => new Promise((ok, no) => { const agent = new http.Agent({ keepAlive: true }) const headers = { authorization: `Bearer ${token}`, 'content-type': contentType, 'content-length': String(raw.length) } const req = http.request(new URL(`${side.url}/xrpc/${SP}importRepo`), { method: 'POST', agent, headers }, (res) => { const chunks = [] res.on('data', (c) => chunks.push(c)) res.on('error', no) res.on('end', () => { agent.destroy() const buf = Buffer.concat(chunks) const text = buf.toString('utf8') const hs = new Headers() for (let i = 0; i < res.rawHeaders.length; i += 2) hs.append(res.rawHeaders[i], res.rawHeaders[i + 1]) let parsed = null if ((hs.get('content-type') ?? '').startsWith('application/json') || res.statusCode >= 400) { try { parsed = text ? JSON.parse(text) : null } catch { parsed = { unparsed: text.slice(0, 200) } } } ok({ status: res.statusCode, body: parsed, buf, headers: hs }) }) }) // Once answered, what the connection does after is not read. req.on('error', (e) => { agent.destroy(); no(e) }) req.end(raw) }) const sent = async (side, opts) => { for (let i = 0; ; i++) { try { return await alone(side, opts) } catch (e) { if (i >= 2 || !['ECONNRESET', 'EPIPE'].includes(e?.code)) throw e } } } for (const n of [MAX_IMPORT, MAX_IMPORT + 1]) { const kind = `importRepo of a ${n}-byte body` const mk = (side) => { const raw = Buffer.alloc(n); archiveOf(acct, side, R.spaces[0].uri(), []).copy(raw); return { token: tokenOf(acct, side), raw, contentType: STAR } } const a = await sent(ref, mk(ref)) const b = await sent(cand, mk(cand)) const [na, nb] = [norm(acct, ref, a), norm(acct, cand, b)] if (!isDeepStrictEqual(na, nb)) await h.differ(`${SP}importRepo`, shown(kind, mk(cand)), na, nb) h.count(kind) } } /// Blobs public and space records reference: the public methods serve and /// list only those of public records (spec/SPACES-EXPORT.md, "Blobs in /// space records"). async function publicBlobs() { const acct = await h.createAccount() for (const side of sides) { sd(acct, side).tids = []; sd(acct, side).appToken = sd(acct, side).token } const sp = R.spaces[0] const cid = await uploadBlob(acct, Buffer.from(`spec-pds blob public ${crypto.randomBytes(4).toString('hex')}`)) const e = storeOf(acct, ref).get(cid) const rec = { $type: BLOB_COLL, img: { $type: 'blob', ref: { $link: cid }, mimeType: e.mime, size: e.bytes.length } } const mkSpace = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(), repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'a', record: rec } }) const [a, b] = await both('createRecord with a blob ref, for the public methods', acct, `${SP}createRecord`, mkSpace) await after('createRecord', acct, sp, a, b, async (side, res) => { await put(side, acct, sp, BLOB_COLL, 'a', rec, res.body.cid, {}, 'createRecord'); settle(acct, side, rec); return true }) const publicReads = async (want) => { for (const side of sides) { const g = await call(side, 'com.atproto.sync.getBlob', { params: { did: didOf(acct, side), cid } }) if ((g.status === 200) !== want) await fail(side, `com.atproto.sync.getBlob of a blob ${want ? 'a public record references' : 'only space records reference'}`, { cid }, want ? 200 : 'a 400', { status: g.status, error: g.body?.error }) const l = await call(side, 'com.atproto.sync.listBlobs', { params: { did: didOf(acct, side) } }) if (l.body?.cids?.includes(cid) !== want) await fail(side, `com.atproto.sync.listBlobs, a blob ${want ? 'a public record references' : 'only space records reference'}`, {}, want ? 'listed' : 'not listed', l.body) } h.count(`public reads of a blob ${want ? 'public and space records reference' : 'only space records reference'}`) } await publicReads(false) const [pa, pb] = await Promise.all(sides.map((side) => call(side, 'com.atproto.repo.createRecord', { token: tokenOf(acct, side), body: { repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'pub', record: rec } }))) if (pa.status !== 200 || pb.status !== 200) await h.differ('com.atproto.repo.createRecord with a blob ref', {}, pa.status, pb.status) for (const side of sides) publicRefs(acct, side).add(cid) const mkDel = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(), repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'a' } }) const [da, db] = await both('deleteRecord of the space record', acct, `${SP}deleteRecord`, mkDel) await after('deleteRecord', acct, sp, da, db, async (side) => repoOf(acct, side, sp).records.delete(`${BLOB_COLL}/a`)) await publicReads(true) for (const side of sides) sd(acct, side).publicCommit = (await call(side, 'com.atproto.sync.getLatestCommit', { params: { did: didOf(acct, side) } })).body } // ---- takedown and deactivation -------------------------------------------------------------------------- /// An account deactivated, then taken down: each side's reads of it, the /// reference's alone until the server can be put in either state /// (spec/SPACES-EXPORT.md, "Who may read", "importRepo", "Takedown and /// deactivation", "For other parts"). async function statuses() { const acct = await h.createAccount() for (const side of sides) sd(acct, side).tids = [] const sp = peerSpaces[0] ?? R.spaces[0] const cid = await uploadBlob(acct, Buffer.from(`spec-pds blob status ${crypto.randomBytes(4).toString('hex')}`)) const e = storeOf(acct, ref).get(cid) const rec = { $type: BLOB_COLL, img: { $type: 'blob', ref: { $link: cid }, mimeType: e.mime, size: e.bytes.length } } await both('createRecord, for an account to be deactivated', acct, `${SP}createRecord`, (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(), repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'a', record: rec } })) const credential = await credentialFor(sp.uri()) const shares = {} const reads = (side) => { shares[side.name] ??= encapsulate(sd(acct, side).kem) const p = { space: sp.uri(), repo: didOf(acct, side) } const repoMac = { 'accept-signature': formatMacRequest(COMMIT_PROFILE, shares[side.name].keyShare) } const recMac = { 'accept-signature': formatMacRequest(RECORD_PROFILE, shares[side.name].keyShare) } return [ ['getRepo', p, repoMac], ['listBlobs', p, {}], ['getBlob', { ...p, cid }, {}], ['getRecord', { ...p, collection: BLOB_COLL, rkey: 'a' }, recMac], ['listRecords', p, {}], ['getLatestCommit', p, repoMac], ['listRepoOps', p, repoMac], ['listSpaces', {}, {}], ] } const shape = (res) => ({ status: res.status, error: res.status >= 400 ? res.body?.error ?? null : undefined }) async function hold(state, inState, how, wantOf) { for (const side of inState) { for (const [name, params, mac] of reads(side)) { const opts = how(side) const res = await call(side, `${SP}${name}`, { ...opts, params, headers: { ...opts.headers, ...mac } }) const want = wantOf(name, side) if (want === null) continue if (!isDeepStrictEqual(shape(res), want)) await fail(side, `${name} of an account ${state}, ${how.label}`, { name }, want, shape(res)) } } h.count(`${state}, ${how.label}: ${inState.includes(cand) ? 'compared' : 'the reference alone'}`) } const ok = { status: 200, error: undefined } const sessionOf = (side) => ({ token: tokenOf(acct, side) }) sessionOf.label = 'its session' const credOf = (side) => ({ headers: credHeaders(credential, didOf(acct, side)) }) credOf.label = 'a credential' const err = (error) => ({ status: 400, error }) const deactivated = [] for (const side of sides) if ((await call(side, 'com.atproto.server.deactivateAccount', { token: tokenOf(acct, side), body: {} })).status === 200) deactivated.push(side) if (!deactivated.includes(ref)) await fail(ref, 'deactivateAccount', {}, 200, 'refused') await hold('deactivated', deactivated, sessionOf, () => ok) await hold('deactivated', deactivated, credOf, (name) => (name === 'listSpaces' ? err('InvalidToken') : err('RepoDeactivated'))) for (const side of deactivated) { const res = await call(side, `${SP}importRepo`, { token: tokenOf(acct, side), raw: archiveOf(acct, side, 'at://did:web:example.com/space/com.example.group/deactivated', []), contentType: STAR }) if (res.status !== 200) await fail(side, 'importRepo by an account deactivated', {}, 200, shape(res)) } for (const side of deactivated) await call(side, 'com.atproto.server.activateAccount', { token: tokenOf(acct, side), body: {} }) const admin = (side) => (side === ref ? h.refPds.adminAuth() : h.env.PDS_ADMIN_PASSWORD ? `Basic ${Buffer.from(`admin:${h.env.PDS_ADMIN_PASSWORD}`).toString('base64')}` : null) const down = [] for (const side of sides) { const a = admin(side) if (!a) continue const res = await fetch(`${side.url}/xrpc/com.atproto.admin.updateSubjectStatus`, { method: 'POST', headers: { authorization: a, 'content-type': 'application/json' }, body: JSON.stringify({ subject: { $type: 'com.atproto.admin.defs#repoRef', did: didOf(acct, side) }, takedown: { applied: true, ref: 'spaces' } }) }) if (res.status === 200) down.push(side) } if (!down.includes(ref)) await fail(ref, 'a takedown', {}, 200, 'refused') const tdSession = {} for (const side of down) { const s = await call(side, 'com.atproto.server.createSession', { body: { identifier: didOf(acct, side), password: acct.password, allowTakendown: true } }) if (s.status !== 200) await fail(side, 'createSession with allowTakendown', {}, 200, shape(s)) tdSession[side.name] = s.body?.accessJwt } const allowOf = (side) => ({ token: tdSession[side.name] }) allowOf.label = 'its allowTakendown session' await hold('taken down', down, allowOf, (name) => (['getRepo', 'listBlobs', 'listSpaces'].includes(name) ? ok : name === 'getBlob' ? null : err('InvalidToken'))) await hold('taken down', down, credOf, (name) => (name === 'listSpaces' ? err('InvalidToken') : err('RepoTakendown'))) for (const side of down) { const res = await call(side, `${SP}importRepo`, { token: tdSession[side.name], raw: archiveOf(acct, side, 'at://did:web:example.com/space/com.example.group/down', []), contentType: STAR }) if (!isDeepStrictEqual(shape(res), err('InvalidToken'))) await fail(side, 'importRepo with an allowTakendown session', {}, err('InvalidToken'), shape(res)) } } // ---- the part, for repo.mjs --------------------------------------------------------------------------- return { steps: [ [4, uploadBlob], [6, blobWrite], [2, blobDelete], [4, getBlob], [3, listBlobs], [5, (a) => getRepo(a)], [4, credentialRead], [3, importRepo], ], async setup() { for (const sp of R.spaces) { const res = await call(peer, `${SP}authorizeSpaceSync`, { body: { space: sp.uri(), credentialKey: kid.did } }) if (res.status === 200) { peerSpaces.push(sp); creds.set(sp.uri(), { credential: res.body.credential, at: Date.now() }) } } if (!peerSpaces.length) await h.differ('authorizeSpaceSync at the peer', {}, 'credentials for the authorities\' spaces, made publicRead', 'none', 'the harness cannot read as a syncer') }, async fixed() { await importSize(accounts[0]) await publicBlobs() }, /// A repo read whole: its archive, by session and by credential, and its /// blobs, before a restart and after (spec/SPACES-EXPORT.md, /// "Durability"). async whole(acct, sp) { await getRepo(acct, { kind: 'session', sp, target: acct }, ' (whole)') if (peerSpaces.includes(sp)) await getRepo(acct, { kind: 'credential', sp, target: acct, credential: await credentialFor(sp.uri()) }, ' (whole)') const refs = [...repoRefs(repoOf(acct, ref, sp))].filter((c) => !tainted.has(c)) for (const cid of refs) { const mk = (side) => ({ token: tokenOf(acct, side), params: { space: sp.uri(acct, side), repo: didOf(acct, side), cid } }) const [a, b] = await Promise.all(sides.map((side) => call(side, `${SP}getBlob`, mk(side)))) const view = (side, res) => (res.status >= 400 ? norm(acct, side, res) : { status: 200, bytes: res.buf.toString('base64'), headers: blobHeaders(res) }) if (!isDeepStrictEqual(view(ref, a), view(cand, b))) await h.differ(`${SP}getBlob`, shown('getBlob (whole)', mk(cand)), view(ref, a), view(cand, b)) h.count('getBlob (whole)') } }, statuses, } }