chadtmiller.com/monolith
An atproto PDS
0
0
// A space repo whole, its blobs, and migration (spec/SPACES-EXPORT.md), in
// the spaces run beside the space repo's steps (repo.mjs), whose model of
// every repo it reads and adds to: the server and the alpha's first PDS each
// the repo host of their side's accounts, the authorities' spaces on the
// alpha's second PDS, the peer, which issues the harness, as a syncer,
// credentials for them. Each step goes to both sides and the answers are
// compared; on each side, the reference's too, every archive must verify
// with lib/spec.mjs and be what its writer makes of the model's commit and
// records, every blob be the model's, and every MAC the harness's.
import crypto from 'node:crypto'
import http from 'node:http'
import { isDeepStrictEqual } from 'node:util'
import { COMMIT_PROFILE, LtHash, RECORD_PROFILE, cidBytes, cidString, encapsulate, encode, formatMacRequest, headerNamesCommit, headerNamesRecord, newP256, readRepoStar, readStar, signCredential, writeStar } from './lib/spec.mjs'
const BLOB_COLL = 'com.example.blob'
const MAX_IMPORT = 5242880
/// The media type of a space repo's STAR archive (spec/SPACES-EXPORT.md,
/// "getRepo", "importRepo").
const STAR = 'application/vnd.dasl.star'
/// The MAC headers, which a read that takes no MAC never answers.
const MAC_HEADERS = ['atproto-space-commit', 'atproto-space-record', 'signature-input', 'signature']
/// The checks, given the run (compare.mjs) and the space repo's internals.
export function spaceExport(h, R) {
const { ref, cand, accounts } = h
const sides = [ref, cand]
const { SP, COLLECTIONS, RKEYS, call, both, norm, shown, fail, tokenOf, sd, didOf, repoOf, digestOf, put, after, latestCommit, pickSpace, keyOn, cidOf, drisl, typed, below, chance, pick, checkMacHeaders, holdInvalidMac, checkCommit, commitOf, byBytes, macAsk, askHeaders, askName } = R
const peer = h.net.extraPdses[0]
const pairOf = (a, b) => [[ref, a], [cand, b]]
// ---- credentials, from the peer ---------------------------------------------------------
const kid = newP256()
const creds = new Map()
let peerSpaces = []
/// A credential for a space the peer hosts, fresh enough for a step, from
/// its `authorizeSpaceSync`, as a syncer gets one
/// (spec/SPACES-CREDENTIALS.md, "authorizeSpaceSync").
async function credentialFor(space) {
const c = creds.get(space)
if (c && Date.now() - c.at < 300_000) return c.credential
const res = await call(peer, `${SP}authorizeSpaceSync`, { body: { space, credentialKey: kid.did } })
if (res.status !== 200) await h.differ('authorizeSpaceSync at the peer', { space }, 'a credential: the space is publicRead', res.body, 'the harness cannot read as a syncer')
creds.set(space, { credential: res.body.credential, at: Date.now() })
return res.body.credential
}
const credHeaders = (credential, audience) => signCredential(kid, credential, audience)
// ---- blobs ------------------------------------------------------------------------------
let bodies = 0
const tainted = new Set()
const storeOf = (acct, side) => (sd(acct, side).store ??= new Map())
const publicRefs = (acct, side) => (sd(acct, side).publicRefs ??= new Set())
const rawCid = (bytes) => cidString(Buffer.concat([Buffer.from([0x01, 0x55, 0x12, 0x20]), crypto.createHash('sha256').update(bytes).digest()]))
const isLink = (v) => v && typeof v === 'object' && !Array.isArray(v) && Object.keys(v).length === 1 && typeof v.$link === 'string'
/// The CIDs a record's blob refs name (spec/BLOBS.md, "Blob refs in records").
function refsOf(v, out = new Set()) {
if (Array.isArray(v)) { v.forEach((x) => refsOf(x, out)); return out }
if (!v || typeof v !== 'object') return out
const keys = Object.keys(v).sort()
if (isDeepStrictEqual(keys, ['$type', 'mimeType', 'ref', 'size']) && v.$type === 'blob' && isLink(v.ref) && typeof v.mimeType === 'string' && v.mimeType.includes('/') && Number.isInteger(v.size) && v.size >= -1) { out.add(v.ref.$link); return out }
for (const x of Object.values(v)) refsOf(x, out)
return out
}
const repoRefs = (m) => { const out = new Set(); for (const { value } of m.records.values()) refsOf(value, out); return out }
function referencedAnywhere(acct, side) {
const out = new Set(publicRefs(acct, side))
for (const m of (sd(acct, side).repos ?? new Map()).values()) for (const c of repoRefs(m)) out.add(c)
return out
}
/// A blob the account holds the bytes of: an upload no record references
/// yet, or one a record references (spec/SPACES-EXPORT.md, "Blobs in space
/// records").
function alive(acct, side, cid) {
const e = storeOf(acct, side).get(cid)
return e && !e.dead && (!e.perm || referencedAnywhere(acct, side).has(cid)) ? e : null
}
/// After a write on `side` that took refs: each blob it names is held for good.
function settle(acct, side, record) {
for (const cid of refsOf(record)) { const e = storeOf(acct, side).get(cid); if (e) e.perm = true }
}
/// The blobs a repo's records reference that no longer are, after its
/// records were replaced by an import: the reference keeps them held, the
/// page drops them, so the harness never names them again.
function taint(acct, side, before) {
const now = referencedAnywhere(acct, side)
for (const cid of before) if (!now.has(cid) && storeOf(acct, side).get(cid)?.perm) tainted.add(cid)
}
async function uploadBlob(acct, body) {
const known = [...storeOf(acct, ref).values()].filter((e) => !tainted.has(e.cid))
const others = accounts.filter((a) => a !== acct).flatMap((a) => [...storeOf(a, ref).values()]).filter((e) => !tainted.has(e.cid))
const r = rand()
const bytes = body ?? (r < 0.6 || (!known.length && !others.length) ? Buffer.from(`spec-pds blob ${++bodies} ${crypto.randomBytes(4).toString('hex')}`) : r < 0.85 && known.length ? pick(known).bytes : pick(others.length ? others : known).bytes)
if (tainted.has(rawCid(bytes))) return
// Text, as the bytes are: the alpha refuses a blob whose bytes are not of
// the type it is sent as, which BLOBS.md, held to the `latest`
// reference, does not ask.
const mime = 'text/plain'
const mk = (side) => ({ token: tokenOf(acct, side), raw: bytes, contentType: mime })
const [a, b] = await both('uploadBlob (for space records)', acct, 'com.atproto.repo.uploadBlob', mk)
if (a.status !== 200) return
for (const side of sides) {
const cid = rawCid(bytes)
if (!alive(acct, side, cid)) storeOf(acct, side).set(cid, { cid, bytes, mime, perm: false })
if (referencedAnywhere(acct, side).has(cid)) storeOf(acct, side).get(cid).perm = true
}
return rawCid(bytes)
}
const rand = h.rand
/// A blob ref for a write by `acct`: to one of its blobs as stored, or one
/// that may be wrong (spec/SPACES-EXPORT.md, "Blobs in space records").
async function blobRef(acct) {
let held = [...storeOf(acct, ref).values()].filter((e) => alive(acct, ref, e.cid) && !tainted.has(e.cid))
if (!held.length) { await uploadBlob(acct); held = [...storeOf(acct, ref).values()].filter((e) => alive(acct, ref, e.cid) && !tainted.has(e.cid)) }
const e = held.length ? pick(held) : null
const good = (x) => ({ $type: 'blob', ref: { $link: x.cid }, mimeType: x.mime, size: x.bytes.length })
const r = rand()
if (!e || r < 0.05) return { $type: 'blob', ref: { $link: rawCid(crypto.randomBytes(16)) }, mimeType: 'text/plain', size: 16 }
if (r < 0.1) return { ...good(e), mimeType: 'image/png' }
if (r < 0.15) return { ...good(e), size: pick([e.bytes.length + 1, -1]) }
if (r < 0.2) {
const other = accounts.filter((a) => a !== acct).flatMap((a) => [...storeOf(a, ref).values()]).filter((x) => !tainted.has(x.cid))
if (other.length) return good(pick(other))
}
return good(e)
}
/// A space write of a record with blob refs, in a collection of its own.
async function blobWrite(acct) {
const sp = pickSpace({ malformed: 0, ownP: 0.03 })
const key = pick(RKEYS)
const rec = { $type: BLOB_COLL, img: await blobRef(acct), n: below(100) }
if (chance(0.3)) {
const more = [...storeOf(acct, ref).values()].filter((e) => alive(acct, ref, e.cid) && !tainted.has(e.cid))
if (more.length) { const e = pick(more); rec.list = [{ $type: 'blob', ref: { $link: e.cid }, mimeType: e.mime, size: e.bytes.length }] }
}
const nsid = chance(0.5) ? 'createRecord' : 'putRecord'
const mk = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(acct, side), repo: didOf(acct, side), collection: BLOB_COLL, rkey: key, record: rec } })
const [a, b] = await both(`${nsid} with blob refs`, acct, `${SP}${nsid}`, mk)
await after(nsid, acct, sp, a, b, async (side, res) => {
await put(side, acct, sp, BLOB_COLL, key, rec, res.body.cid, shown(nsid, mk(cand)), `${nsid} with blob refs`)
settle(acct, side, rec)
return true
})
}
async function blobDelete(acct) {
const sp = pickSpace({ malformed: 0, ownP: 0 })
const keys = [...repoOf(acct, ref, sp).records.keys()].filter((p) => p.startsWith(`${BLOB_COLL}/`))
const key = keys.length ? pick(keys).slice(BLOB_COLL.length + 1) : pick(RKEYS)
const mk = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(acct, side), repo: didOf(acct, side), collection: BLOB_COLL, rkey: key } })
const [a, b] = await both('deleteRecord of a record with blob refs', acct, `${SP}deleteRecord`, mk)
await after('deleteRecord', acct, sp, a, b, async (side) => repoOf(acct, side, sp).records.delete(`${BLOB_COLL}/${key}`))
}
// ---- who reads ------------------------------------------------------------------------------
/// How a read authenticates, drawn for a step: the account's session, or a
/// credential for its own repo, another's, or one with something wrong
/// (spec/SPACES-EXPORT.md, "Who may read").
async function reader(acct) {
const r = rand()
const sp = peerSpaces.length && r >= 0.4 ? pick(peerSpaces) : null
if (!sp) return { kind: chance(0.15) ? 'app' : 'session', sp: pickSpace({ malformed: 0.02, ownP: 0.03 }), target: acct }
const others = accounts.filter((a) => a !== acct)
const v = rand()
const kind = v < 0.55 ? 'credential' : v < 0.7 ? 'credential, another account' : v < 0.76 ? 'credential, no account here' : v < 0.82 ? 'credential, wrong audience' : v < 0.88 ? 'credential of another space' : v < 0.94 ? 'credential, the session read another account' : 'credential, unknown DID'
const credential = await credentialFor(kind === 'credential of another space' ? pick(peerSpaces.filter((s) => s !== sp).concat([sp])).uri() : sp.uri())
const target = kind === 'credential, another account' && others.length ? pick(others) : acct
return { kind, sp, target, credential, authority: sp.uri().split('/')[2] }
}
/// The request's auth and `repo` on a side.
function authOn(rd, acct, side) {
const repo = rd.kind === 'credential, no account here' ? rd.authority : rd.kind === 'credential, unknown DID' ? 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' : didOf(rd.target, side)
if (rd.kind === 'session' || rd.kind === 'app') return { repo, token: tokenOf(acct, side, rd.kind === 'app' ? 'app' : 'access') }
if (rd.kind === 'credential, the session read another account') return { repo: didOf(accounts.find((a) => a !== acct) ?? acct, side), token: tokenOf(acct, side) }
return { repo, headers: credHeaders(rd.credential, rd.kind === 'credential, wrong audience' ? didOf(acct, side) === repo ? 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' : didOf(acct, side) : repo) }
}
/// Whether a read on `side` is of the model's repo, whose answer the
/// harness can say.
const modelled = (rd) => ['session', 'app', 'credential', 'credential, another account'].includes(rd.kind)
// ---- getBlob and listBlobs ------------------------------------------------------------------------
/// getBlob, compared, and against the blobs the model holds and the repo
/// references (spec/SPACES-EXPORT.md, "getBlob").
async function getBlob(acct) {
const rd = await reader(acct)
const m = repoOf(rd.target, ref, rd.sp)
const refs = rd.sp.malformed ? [] : [...repoRefs(m)].filter((c) => !tainted.has(c))
const stored = [...storeOf(rd.target, ref).keys()].filter((c) => !tainted.has(c))
const r = rand()
const cid = refs.length && r < 0.6 ? pick(refs) : stored.length && r < 0.8 ? pick(stored) : r < 0.88 ? rawCid(crypto.randomBytes(8)) : r < 0.94 && R.knownCids.length ? pick(R.knownCids) : r < 0.97 ? 'nope' : pick(stored.length ? stored : ['nope'])
const mk = (side) => { const { repo, ...auth } = authOn(rd, acct, side); return { ...auth, params: { space: rd.sp.uri(acct, side), repo, cid } } }
const [a, b] = await Promise.all(sides.map((side) => call(side, `${SP}getBlob`, mk(side))))
const view = (side, res) => res.status >= 400 ? norm(rd.target, side, res) : { status: res.status, bytes: res.buf.toString('base64'), headers: blobHeaders(res) }
const [na, nb] = [view(ref, a), view(cand, b)]
if (!isDeepStrictEqual(na, nb)) await h.differ(`${SP}getBlob`, shown(`getBlob (${rd.kind})`, mk(cand)), na, nb)
h.count(`getBlob (${rd.kind})`)
if (!modelled(rd) || rd.sp.malformed) return
for (const [side, res] of pairOf(a, b)) {
const mm = repoOf(rd.target, side, rd.sp)
if (cid === 'nope') continue
const e = alive(rd.target, side, cid)
const want = !repoRefs(mm).has(cid) ? { status: 400, error: 'BlobNotFound' } : e?.perm ? { status: 200, bytes: e.bytes.toString('base64'), headers: { 'content-type': e.mime, 'content-length': String(e.bytes.length), 'x-content-type-options': 'nosniff', 'content-disposition': `attachment; filename="${cid}"`, 'content-security-policy': "default-src 'none'; sandbox" } } : { status: 400, error: 'InvalidRequest' }
const got = res.status >= 400 ? { status: res.status, error: res.body?.error ?? null } : { status: res.status, bytes: res.buf.toString('base64'), headers: blobHeaders(res) }
if (!isDeepStrictEqual(got, want)) await fail(side, 'getBlob, against the blobs the harness uploaded and the records that reference them', shown('getBlob', mk(side)), want, got)
}
}
function blobHeaders(res) {
const out = {}
for (const k of ['content-type', 'content-length', 'x-content-type-options', 'content-disposition', 'content-security-policy']) out[k] = res.headers.get(k)
out['content-type'] = out['content-type']?.split(';')[0].trim() ?? null
return out
}
/// listBlobs, paged with each side's cursors, against the blobs the repo
/// references; a `since` sent is ignored (spec/SPACES-EXPORT.md,
/// "listBlobs").
async function listBlobs(acct) {
const rd = await reader(acct)
const limit = chance(0.5) ? 1 + below(4) : undefined
const revs = rd.sp.malformed ? [] : repoOf(rd.target, ref, rd.sp).revs
const sinceAt = chance(0.3) ? (revs.length && chance(0.8) ? below(revs.length) : -1) : undefined
const sinceOn = (side) => (sinceAt === undefined ? undefined : sinceAt === -1 ? 'not-a-tid' : repoOf(rd.target, side, rd.sp).revs[sinceAt])
const cursors = {}
const got = { reference: [], server: [] }
const mk = (side) => { const { repo, ...auth } = authOn(rd, acct, side); return { ...auth, params: { space: rd.sp.uri(acct, side), repo, limit, cursor: cursors[side.name], since: sinceOn(side) } } }
for (let page = 0; ; page++) {
const [a, b] = await both(`listBlobs (${rd.kind})`, rd.target, `${SP}listBlobs`, mk)
if (a.status !== 200) return
for (const [side, res] of pairOf(a, b)) {
const cids = res.body.cids ?? []
got[side.name].push(...cids)
if (Boolean(res.body.cursor) !== (cids.length === (limit ?? 500)) || (res.body.cursor && res.body.cursor !== cids.at(-1))) await fail(side, 'listBlobs cursor', shown('listBlobs', mk(side)), cids.length === (limit ?? 500) ? 'the page\'s last CID, as the page is full' : 'none, as the page is not full', res.body.cursor ?? null)
cursors[side.name] = res.body.cursor
}
if (!a.body.cursor || page > 500) break
}
if (!modelled(rd) || rd.sp.malformed) return
for (const side of sides) {
const want = [...repoRefs(repoOf(rd.target, side, rd.sp))].sort()
if (!isDeepStrictEqual(got[side.name], want)) await fail(side, 'listBlobs, every page, against the blobs the repo\'s records reference', shown('listBlobs', mk(side)), want, got[side.name])
}
}
// ---- getRepo -------------------------------------------------------------------------------------
/// An archive as compared across the sides: the commit and the pairs, the
/// side's names taken out.
function archiveView(acct, side, res) {
let parsed
try { parsed = readStar(res.buf, { meta: 65536, key: 4096, value: 1048576 }) } catch (e) { return { status: res.status, unreadable: e.message } }
const { meta, pairs } = parsed
const body = { meta: { ...meta, data: { $bytes: Buffer.from(meta.data ?? []).toString('base64') } }, pairs: pairs.map(({ key, value }) => ({ key: key.toString('utf8'), value: value.toString('hex') })) }
return { ...norm(acct, side, { status: res.status, body, headers: res.headers }), type: res.headers.get('content-type') }
}
/// getRepo, compared, and on each side read by lib/spec.mjs, its MAC
/// verified, against the model, and the bytes its writer makes; with a
/// credential, a MAC asked for or `InvalidSpaceMac` (spec/SPACES-EXPORT.md,
/// "getRepo", "Who may read"; spec/SPACES-REPO.md, "MACs on request", "For
/// other parts").
async function getRepo(acct, rd0, why = '') {
const rd = rd0 ?? await reader(acct)
const credential = !['session', 'app'].includes(rd.kind)
const w = credential && chance(0.8) ? macAsk(COMMIT_PROFILE, { none: 0, good: 1 }) : macAsk(COMMIT_PROFILE)
const shares = {}
const shareOf = (side) => (shares[side.name] ??= encapsulate(sd(rd.target, side).kem))
const mk = (side) => {
const { repo, ...auth } = authOn(rd, acct, side)
return { ...auth, params: { space: rd.sp.uri(acct, side), repo }, headers: { ...auth.headers, ...askHeaders(w, shareOf(side)) } }
}
const kind = `getRepo (${rd.kind})${askName(w)}${why}`
const [a, b] = await Promise.all(sides.map((side) => call(side, `${SP}getRepo`, mk(side))))
const view = (side, res) => (res.status >= 400 ? norm(rd.target, side, res) : archiveView(rd.target, side, res))
const [na, nb] = [view(ref, a), view(cand, b)]
if (!isDeepStrictEqual(na, nb)) await h.differ(`${SP}getRepo`, shown(kind, mk(cand)), na, nb)
h.count(kind)
if (!modelled(rd) || rd.sp.malformed) return
if (w.refused || (credential && !w.asks)) return holdInvalidMac(a, b, shown(kind, mk(cand)), 'getRepo')
if (a.status !== 200) return
for (const [side, res] of pairOf(a, b)) {
const request = shown('getRepo', mk(side))
const m = repoOf(rd.target, side, rd.sp)
if (res.headers.get('content-type')?.split(';')[0].trim() !== STAR) await fail(side, 'getRepo Content-Type', request, STAR, res.headers.get('content-type'))
const expect = { space: rd.sp.uri(acct, side), repo: didOf(rd.target, side) }
let got
try { got = readRepoStar(res.buf, w.asks ? { expect, ...shareOf(side), headers: Object.fromEntries(res.headers) } : { expect }) } catch (e) { await fail(side, 'getRepo: the archive, read by lib/spec.mjs with its MAC', request, 'an atproto-space-repo archive of the repo', e.message); continue }
const c = { ver: got.commit.ver, repo: got.commit.repo, space: got.commit.space, rev: got.commit.rev, data: { $bytes: Buffer.from(got.commit.data).toString('base64').replace(/=+$/, '') } }
await checkCommit(side, rd.target, rd.sp, c, request, 'getRepo')
if (c.rev !== m.rev) await fail(side, 'getRepo: the commit\'s rev, against getLatestCommit\'s', request, m.rev, c.rev)
const want = [...m.records].sort(([x], [y]) => byBytes(x, y)).map(([path, r]) => ({ path, cid: r.cid }))
const records = got.records.map(({ path, cid }) => ({ path, cid }))
if (!isDeepStrictEqual(records, want)) await fail(side, 'getRepo: the records, against those the harness wrote', request, want, records)
const { meta, pairs } = readStar(res.buf, { meta: 65536, key: 4096, value: 1048576 })
if (!writeStar(meta, pairs).equals(res.buf)) await fail(side, 'getRepo: the archive\'s bytes, against what lib/spec.mjs writes of its commit and records', request, writeStar(meta, pairs).toString('hex').slice(0, 400), res.buf.toString('hex').slice(0, 400))
await checkMacHeaders(side, res, request, w, shareOf(side), (d) => headerNamesCommit(d, got.commit))
}
}
// ---- the space repo's reads with a credential ------------------------------------------------------------
/// The space repo's reads with a credential from the peer: a MAC asked
/// for, or `InvalidSpaceMac`, where the method answers one, and any
/// `Accept-Signature` ignored by listRecords (spec/SPACES-EXPORT.md, "Who
/// may read"; spec/SPACES-REPO.md, "MACs on request", "For other parts").
async function credentialRead(acct) {
if (!peerSpaces.length) return
let rd = await reader(acct)
if (['session', 'app'].includes(rd.kind)) rd = { ...rd, kind: 'credential', sp: pick(peerSpaces), credential: await credentialFor(pick(peerSpaces).uri()) }
if (rd.kind === 'credential' || rd.kind === 'credential, another account') rd.credential = await credentialFor(rd.sp.uri())
const [nsid, profile] = pick([['getRecord', RECORD_PROFILE], ['listRecords', null], ['getLatestCommit', COMMIT_PROFILE], ['listRepoOps', COMMIT_PROFILE]])
const m = repoOf(rd.target, ref, rd.sp)
const paths = [...m.records.keys()]
const path = paths.length && chance(0.8) ? pick(paths) : `${pick(COLLECTIONS)}/${pick(RKEYS)}`
const [coll, key] = [path.slice(0, path.indexOf('/')), path.slice(path.indexOf('/') + 1)]
const w = profile && chance(0.8) ? macAsk(profile, { none: 0, good: 1 }) : macAsk(profile ?? COMMIT_PROFILE)
const shares = {}
const shareOf = (side) => (shares[side.name] ??= encapsulate(sd(rd.target, side).kem))
const mk = (side) => {
const { repo, ...auth } = authOn(rd, acct, side)
const params = { space: rd.sp.uri(acct, side), repo, ...(nsid === 'getRecord' ? { collection: coll, rkey: keyOn(rd.target, side, key) } : {}) }
return { ...auth, params, headers: { ...auth.headers, ...askHeaders(w, shareOf(side)) } }
}
const kind = `${nsid} (${rd.kind})${askName(w)}`
const [a, b] = await both(kind, rd.target, `${SP}${nsid}`, mk)
if (!modelled(rd)) return
if (profile && (w.refused || !w.asks)) return holdInvalidMac(a, b, shown(kind, mk(cand)), nsid)
if (profile && a.status !== 200) return
for (const [side, res] of pairOf(a, b)) {
const request = shown(nsid, mk(side))
if (!profile) {
const present = MAC_HEADERS.filter((k) => res.headers.has(k))
if (present.length) await fail(side, 'listRecords: MAC headers, as it takes no MAC', request, 'none', present)
continue
}
const names = nsid === 'getRecord' ? (d) => headerNamesRecord(d, res.body.uri, res.body.cid) : (d) => Boolean(res.body.commit) && headerNamesCommit(d, commitOf(res.body.commit))
await checkMacHeaders(side, res, request, w, shareOf(side), names)
}
}
// ---- importRepo --------------------------------------------------------------------------------
/// An archive of `records` ([path, JSON value]) for the account on a side,
/// as lib/spec.mjs writes one; `o` changes its commit.
function archiveOf(acct, side, space, records, o = {}) {
const lt = new LtHash()
const pairs = records.map(([path, v]) => { const value = encode(drisl(v)); lt.add(`${path}/${cidString(cidBytes(value))}`); return { key: Buffer.from(path), value } }).sort((x, y) => Buffer.compare(x.key, y.key))
return writeStar({ $type: 'atproto-space-repo', ver: 1, repo: didOf(acct, side), space, rev: '3mzzzzzzzzzz2', data: lt.digest(), ...o }, pairs)
}
/// One record to import: good, with blob refs held, uploaded only, or
/// never uploaded, or of a published collection.
function importRecord(acct, used) {
const r = rand()
const coll = r < 0.05 ? pick(['com.atproto.space.policy', 'com.atproto.memberspace.member']) : r < 0.45 ? BLOB_COLL : pick(COLLECTIONS)
const free = RKEYS.filter((k) => !used.has(`${coll}/${k}`))
if (!free.length) return null
const key = pick(free)
used.add(`${coll}/${key}`)
const v = { $type: coll, n: below(1000) }
if (coll === BLOB_COLL) {
const pool = [...storeOf(acct, ref).values()].filter((e) => !tainted.has(e.cid))
const e = pool.length && chance(0.8) ? pick(pool) : null
v.img = e ? { $type: 'blob', ref: { $link: e.cid }, mimeType: e.mime, size: e.bytes.length } : { $type: 'blob', ref: { $link: rawCid(crypto.randomBytes(16)) }, mimeType: 'text/plain', size: 16 }
}
return [`${coll}/${key}`, v]
}
const otherDid = (acct, side) => (didOf(accounts.find((a) => a !== acct) ?? acct, side) === didOf(acct, side) ? 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa' : didOf(accounts.find((a) => a !== acct), side))
/// The ways an archive can be wrong, each with the error the page says it
/// gets, in the order of the checks (spec/SPACES-EXPORT.md, "importRepo").
const FAULTS = {
'another account\'s repo': ['RepoMismatch', (acct, side, space) => archiveOf(acct, side, space, [], { repo: otherDid(acct, side) })],
'another account\'s repo, data not the records\' LtHash': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { repo: otherDid(acct, side), data: Buffer.alloc(32) })],
'data not the records\' LtHash': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { data: Buffer.alloc(32) })],
'keys out of order': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/b', encode({ a: 1 })], ['com.example.note/a', encode({ a: 1 })]])],
'a key repeated': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/a', encode({ a: 1 })], ['com.example.note/a', encode({ a: 2 })]])],
'a value not canonical DRISL': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/a', Buffer.from('a2616202616101', 'hex')]])],
'a value not canonical DRISL, in a space of its own never created': ['SpaceNotFound', (acct, side) => writeRaw(acct, side, R.own.uri(acct, side), [['com.example.note/a', Buffer.from('a2616202616101', 'hex')]])],
'a value not a map': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['com.example.note/a', encode([1])]])],
'a key not a record path': ['InvalidRepoFile', (acct, side, space) => writeRaw(acct, side, space, [['notnsid/a', encode({ a: 1 })]])],
'ver 2': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { ver: 2 })],
'another $type': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { $type: 'atproto-space-meta' })],
'a rev that is no TID': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { rev: 'hello' })],
'a space that is no space-ref': ['InvalidRepoFile', (acct, side) => archiveOf(acct, side, 'at://did:web:example.com/spaces/com.example.group/k', [])],
'a repo that is no DID': ['InvalidRepoFile', (acct, side, space) => archiveOf(acct, side, space, [], { repo: 'handle.test' })],
'bad magic': ['InvalidRepoFile', () => Buffer.from('hello')],
'no bytes': ['InvalidRepoFile', () => Buffer.alloc(0)],
'a byte after the end': ['InvalidRepoFile', (acct, side, space) => Buffer.concat([archiveOf(acct, side, space, []), Buffer.from([0])])],
}
/// The ways the request of a good archive can be wrong, and their errors.
const REQUEST_FAULTS = { 'an app password': 'InvalidToken', 'a refresh token': 'InvalidToken', 'Content-Type JSON': 'InvalidRequest', 'Content-Type application/octet-stream': 'InvalidRequest', 'no Content-Type': 'InvalidRequest' }
function writeRaw(acct, side, space, kv) {
const lt = new LtHash()
for (const [p, v] of kv) lt.add(`${p}/${cidString(cidBytes(v))}`)
return writeStar({ $type: 'atproto-space-repo', ver: 1, repo: didOf(acct, side), space, rev: '3mzzzzzzzzzz2', data: lt.digest() }, kv.map(([p, v]) => ({ key: Buffer.from(p), value: v })))
}
/// importRepo of the side's own archive, of one the harness writes, or of
/// one with a fault, then the repo read whole; a fault is the error the
/// page names on each side (spec/SPACES-EXPORT.md, "importRepo").
async function importRepo(acct) {
const r = rand()
const existing = R.spaces.filter((sp) => repoOf(acct, ref, sp).exists)
const reimport = r >= 0.05 && r < 0.35 && existing.length > 0
const sp = r < 0.05 ? R.own : reimport ? pick(existing) : pick(R.spaces)
const mode = reimport ? 'its own archive' : r < 0.8 ? 'an archive the harness writes' : 'fault'
const fault = mode === 'fault' ? pick([...Object.keys(FAULTS), ...Object.keys(REQUEST_FAULTS)]) : null
let records = null
const bodyOf = {}
for (const side of sides) {
const space = sp.uri(acct, side)
if (mode === 'its own archive') {
const g = await call(side, `${SP}getRepo`, { token: tokenOf(acct, side), params: { space, repo: didOf(acct, side) } })
if (g.status !== 200) await fail(side, 'getRepo of a repo to import back', { space }, 'the archive', norm(acct, side, g))
bodyOf[side.name] = g.buf
} else if (mode === 'an archive the harness writes') {
if (!records) { const used = new Set(); records = []; for (let i = below(5); i > 0; i--) { const rec = importRecord(acct, used); if (rec) records.push(rec) } }
bodyOf[side.name] = archiveOf(acct, side, space, records)
} else bodyOf[side.name] = FAULTS[fault] ? FAULTS[fault][1](acct, side, space) : archiveOf(acct, side, space, [])
}
if (FAULTS[fault] === undefined && mode === 'fault') records = []
const BAD_TYPES = { 'Content-Type JSON': 'application/json', 'Content-Type application/octet-stream': 'application/octet-stream', 'no Content-Type': null }
const contentType = fault in BAD_TYPES ? BAD_TYPES[fault] : chance(0.2) ? `${STAR}; charset=binary` : STAR
const mk = (side) => ({ token: tokenOf(acct, side, fault === 'an app password' ? 'app' : fault === 'a refresh token' ? 'refresh' : 'access'), raw: bodyOf[side.name], contentType })
const before = {}
for (const side of sides) before[side.name] = referencedAnywhere(acct, side)
const [a, b] = await both(`importRepo (${fault ?? mode})`, acct, `${SP}importRepo`, mk)
const error = FAULTS[fault]?.[0] ?? REQUEST_FAULTS[fault]
if (error) for (const [side, res] of pairOf(a, b)) if (res.status !== 400 || res.body?.error !== error) await fail(side, `importRepo (${fault})`, shown('importRepo', { ...mk(side), raw: undefined }), { status: 400, error }, { status: res.status, error: res.body?.error ?? null })
await after('importRepo', acct, sp, a, b, async (side) => {
// A blob no longer held stays so though the import references it
// again, until it is uploaded again (spec/SPACES-EXPORT.md,
// "importRepo").
for (const e of storeOf(acct, side).values()) if (!alive(acct, side, e.cid)) e.dead = true
const m = repoOf(acct, side, sp)
if (records) {
m.records = new Map(records.map(([path, v]) => [path, { cid: cidOf(v, path.slice(0, path.indexOf('/'))), value: v }]))
} else if (FAULTS[fault]) await fail(side, `importRepo of an archive with ${fault}`, {}, 'refused', 'taken')
m.base = new Map(m.records)
m.exists = true
taint(acct, side, before[side.name])
return true
})
if (a.status === 200 && !sp.malformed) {
await getRepo(acct, { kind: 'session', sp, target: acct }, ' after importRepo')
if (records?.some(([, v]) => refsOf(v).size)) {
// A blob an imported record references, uploaded again: held, and served.
const pool = records.flatMap(([, v]) => [...refsOf(v)]).map((cid) => storeOf(acct, ref).get(cid)).filter(Boolean)
if (pool.length) await uploadBlob(acct, pick(pool).bytes)
}
}
}
// ---- fixed checks ---------------------------------------------------------------------------------
/// The importRepo body limit, at its edge. The reference may answer either
/// before it has read the whole body: the one at the limit `400`, having
/// read only its archive, and the one past it `413`. It then either resets
/// the connection, which may lose the answer, or leaves the rest of the
/// body unread on a connection it keeps open, where the next request is
/// never read and is reset when its keep-alive timeout (90 s) ends. So each
/// is sent, on either side alike, on a connection of its own, which nothing
/// is sent on after, and one reset before its answer is sent again, up to
/// three times (spec/SPACES-EXPORT.md, "importRepo").
async function importSize(acct) {
const alone = (side, { raw, token, contentType }) => new Promise((ok, no) => {
const agent = new http.Agent({ keepAlive: true })
const headers = { authorization: `Bearer ${token}`, 'content-type': contentType, 'content-length': String(raw.length) }
const req = http.request(new URL(`${side.url}/xrpc/${SP}importRepo`), { method: 'POST', agent, headers }, (res) => {
const chunks = []
res.on('data', (c) => chunks.push(c))
res.on('error', no)
res.on('end', () => {
agent.destroy()
const buf = Buffer.concat(chunks)
const text = buf.toString('utf8')
const hs = new Headers()
for (let i = 0; i < res.rawHeaders.length; i += 2) hs.append(res.rawHeaders[i], res.rawHeaders[i + 1])
let parsed = null
if ((hs.get('content-type') ?? '').startsWith('application/json') || res.statusCode >= 400) { try { parsed = text ? JSON.parse(text) : null } catch { parsed = { unparsed: text.slice(0, 200) } } }
ok({ status: res.statusCode, body: parsed, buf, headers: hs })
})
})
// Once answered, what the connection does after is not read.
req.on('error', (e) => { agent.destroy(); no(e) })
req.end(raw)
})
const sent = async (side, opts) => {
for (let i = 0; ; i++) {
try { return await alone(side, opts) } catch (e) { if (i >= 2 || !['ECONNRESET', 'EPIPE'].includes(e?.code)) throw e }
}
}
for (const n of [MAX_IMPORT, MAX_IMPORT + 1]) {
const kind = `importRepo of a ${n}-byte body`
const mk = (side) => { const raw = Buffer.alloc(n); archiveOf(acct, side, R.spaces[0].uri(), []).copy(raw); return { token: tokenOf(acct, side), raw, contentType: STAR } }
const a = await sent(ref, mk(ref))
const b = await sent(cand, mk(cand))
const [na, nb] = [norm(acct, ref, a), norm(acct, cand, b)]
if (!isDeepStrictEqual(na, nb)) await h.differ(`${SP}importRepo`, shown(kind, mk(cand)), na, nb)
h.count(kind)
}
}
/// Blobs public and space records reference: the public methods serve and
/// list only those of public records (spec/SPACES-EXPORT.md, "Blobs in
/// space records").
async function publicBlobs() {
const acct = await h.createAccount()
for (const side of sides) { sd(acct, side).tids = []; sd(acct, side).appToken = sd(acct, side).token }
const sp = R.spaces[0]
const cid = await uploadBlob(acct, Buffer.from(`spec-pds blob public ${crypto.randomBytes(4).toString('hex')}`))
const e = storeOf(acct, ref).get(cid)
const rec = { $type: BLOB_COLL, img: { $type: 'blob', ref: { $link: cid }, mimeType: e.mime, size: e.bytes.length } }
const mkSpace = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(), repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'a', record: rec } })
const [a, b] = await both('createRecord with a blob ref, for the public methods', acct, `${SP}createRecord`, mkSpace)
await after('createRecord', acct, sp, a, b, async (side, res) => { await put(side, acct, sp, BLOB_COLL, 'a', rec, res.body.cid, {}, 'createRecord'); settle(acct, side, rec); return true })
const publicReads = async (want) => {
for (const side of sides) {
const g = await call(side, 'com.atproto.sync.getBlob', { params: { did: didOf(acct, side), cid } })
if ((g.status === 200) !== want) await fail(side, `com.atproto.sync.getBlob of a blob ${want ? 'a public record references' : 'only space records reference'}`, { cid }, want ? 200 : 'a 400', { status: g.status, error: g.body?.error })
const l = await call(side, 'com.atproto.sync.listBlobs', { params: { did: didOf(acct, side) } })
if (l.body?.cids?.includes(cid) !== want) await fail(side, `com.atproto.sync.listBlobs, a blob ${want ? 'a public record references' : 'only space records reference'}`, {}, want ? 'listed' : 'not listed', l.body)
}
h.count(`public reads of a blob ${want ? 'public and space records reference' : 'only space records reference'}`)
}
await publicReads(false)
const [pa, pb] = await Promise.all(sides.map((side) => call(side, 'com.atproto.repo.createRecord', { token: tokenOf(acct, side), body: { repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'pub', record: rec } })))
if (pa.status !== 200 || pb.status !== 200) await h.differ('com.atproto.repo.createRecord with a blob ref', {}, pa.status, pb.status)
for (const side of sides) publicRefs(acct, side).add(cid)
const mkDel = (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(), repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'a' } })
const [da, db] = await both('deleteRecord of the space record', acct, `${SP}deleteRecord`, mkDel)
await after('deleteRecord', acct, sp, da, db, async (side) => repoOf(acct, side, sp).records.delete(`${BLOB_COLL}/a`))
await publicReads(true)
for (const side of sides) sd(acct, side).publicCommit = (await call(side, 'com.atproto.sync.getLatestCommit', { params: { did: didOf(acct, side) } })).body
}
// ---- takedown and deactivation --------------------------------------------------------------------------
/// An account deactivated, then taken down: each side's reads of it, the
/// reference's alone until the server can be put in either state
/// (spec/SPACES-EXPORT.md, "Who may read", "importRepo", "Takedown and
/// deactivation", "For other parts").
async function statuses() {
const acct = await h.createAccount()
for (const side of sides) sd(acct, side).tids = []
const sp = peerSpaces[0] ?? R.spaces[0]
const cid = await uploadBlob(acct, Buffer.from(`spec-pds blob status ${crypto.randomBytes(4).toString('hex')}`))
const e = storeOf(acct, ref).get(cid)
const rec = { $type: BLOB_COLL, img: { $type: 'blob', ref: { $link: cid }, mimeType: e.mime, size: e.bytes.length } }
await both('createRecord, for an account to be deactivated', acct, `${SP}createRecord`, (side) => ({ token: tokenOf(acct, side), body: { space: sp.uri(), repo: didOf(acct, side), collection: BLOB_COLL, rkey: 'a', record: rec } }))
const credential = await credentialFor(sp.uri())
const shares = {}
const reads = (side) => {
shares[side.name] ??= encapsulate(sd(acct, side).kem)
const p = { space: sp.uri(), repo: didOf(acct, side) }
const repoMac = { 'accept-signature': formatMacRequest(COMMIT_PROFILE, shares[side.name].keyShare) }
const recMac = { 'accept-signature': formatMacRequest(RECORD_PROFILE, shares[side.name].keyShare) }
return [
['getRepo', p, repoMac], ['listBlobs', p, {}], ['getBlob', { ...p, cid }, {}], ['getRecord', { ...p, collection: BLOB_COLL, rkey: 'a' }, recMac],
['listRecords', p, {}], ['getLatestCommit', p, repoMac], ['listRepoOps', p, repoMac], ['listSpaces', {}, {}],
]
}
const shape = (res) => ({ status: res.status, error: res.status >= 400 ? res.body?.error ?? null : undefined })
async function hold(state, inState, how, wantOf) {
for (const side of inState) {
for (const [name, params, mac] of reads(side)) {
const opts = how(side)
const res = await call(side, `${SP}${name}`, { ...opts, params, headers: { ...opts.headers, ...mac } })
const want = wantOf(name, side)
if (want === null) continue
if (!isDeepStrictEqual(shape(res), want)) await fail(side, `${name} of an account ${state}, ${how.label}`, { name }, want, shape(res))
}
}
h.count(`${state}, ${how.label}: ${inState.includes(cand) ? 'compared' : 'the reference alone'}`)
}
const ok = { status: 200, error: undefined }
const sessionOf = (side) => ({ token: tokenOf(acct, side) })
sessionOf.label = 'its session'
const credOf = (side) => ({ headers: credHeaders(credential, didOf(acct, side)) })
credOf.label = 'a credential'
const err = (error) => ({ status: 400, error })
const deactivated = []
for (const side of sides) if ((await call(side, 'com.atproto.server.deactivateAccount', { token: tokenOf(acct, side), body: {} })).status === 200) deactivated.push(side)
if (!deactivated.includes(ref)) await fail(ref, 'deactivateAccount', {}, 200, 'refused')
await hold('deactivated', deactivated, sessionOf, () => ok)
await hold('deactivated', deactivated, credOf, (name) => (name === 'listSpaces' ? err('InvalidToken') : err('RepoDeactivated')))
for (const side of deactivated) {
const res = await call(side, `${SP}importRepo`, { token: tokenOf(acct, side), raw: archiveOf(acct, side, 'at://did:web:example.com/space/com.example.group/deactivated', []), contentType: STAR })
if (res.status !== 200) await fail(side, 'importRepo by an account deactivated', {}, 200, shape(res))
}
for (const side of deactivated) await call(side, 'com.atproto.server.activateAccount', { token: tokenOf(acct, side), body: {} })
const admin = (side) => (side === ref ? h.refPds.adminAuth() : h.env.PDS_ADMIN_PASSWORD ? `Basic ${Buffer.from(`admin:${h.env.PDS_ADMIN_PASSWORD}`).toString('base64')}` : null)
const down = []
for (const side of sides) {
const a = admin(side)
if (!a) continue
const res = await fetch(`${side.url}/xrpc/com.atproto.admin.updateSubjectStatus`, { method: 'POST', headers: { authorization: a, 'content-type': 'application/json' }, body: JSON.stringify({ subject: { $type: 'com.atproto.admin.defs#repoRef', did: didOf(acct, side) }, takedown: { applied: true, ref: 'spaces' } }) })
if (res.status === 200) down.push(side)
}
if (!down.includes(ref)) await fail(ref, 'a takedown', {}, 200, 'refused')
const tdSession = {}
for (const side of down) {
const s = await call(side, 'com.atproto.server.createSession', { body: { identifier: didOf(acct, side), password: acct.password, allowTakendown: true } })
if (s.status !== 200) await fail(side, 'createSession with allowTakendown', {}, 200, shape(s))
tdSession[side.name] = s.body?.accessJwt
}
const allowOf = (side) => ({ token: tdSession[side.name] })
allowOf.label = 'its allowTakendown session'
await hold('taken down', down, allowOf, (name) => (['getRepo', 'listBlobs', 'listSpaces'].includes(name) ? ok : name === 'getBlob' ? null : err('InvalidToken')))
await hold('taken down', down, credOf, (name) => (name === 'listSpaces' ? err('InvalidToken') : err('RepoTakendown')))
for (const side of down) {
const res = await call(side, `${SP}importRepo`, { token: tdSession[side.name], raw: archiveOf(acct, side, 'at://did:web:example.com/space/com.example.group/down', []), contentType: STAR })
if (!isDeepStrictEqual(shape(res), err('InvalidToken'))) await fail(side, 'importRepo with an allowTakendown session', {}, err('InvalidToken'), shape(res))
}
}
// ---- the part, for repo.mjs ---------------------------------------------------------------------------
return {
steps: [
[4, uploadBlob], [6, blobWrite], [2, blobDelete], [4, getBlob], [3, listBlobs], [5, (a) => getRepo(a)], [4, credentialRead], [3, importRepo],
],
async setup() {
for (const sp of R.spaces) {
const res = await call(peer, `${SP}authorizeSpaceSync`, { body: { space: sp.uri(), credentialKey: kid.did } })
if (res.status === 200) { peerSpaces.push(sp); creds.set(sp.uri(), { credential: res.body.credential, at: Date.now() }) }
}
if (!peerSpaces.length) await h.differ('authorizeSpaceSync at the peer', {}, 'credentials for the authorities\' spaces, made publicRead', 'none', 'the harness cannot read as a syncer')
},
async fixed() {
await importSize(accounts[0])
await publicBlobs()
},
/// A repo read whole: its archive, by session and by credential, and its
/// blobs, before a restart and after (spec/SPACES-EXPORT.md,
/// "Durability").
async whole(acct, sp) {
await getRepo(acct, { kind: 'session', sp, target: acct }, ' (whole)')
if (peerSpaces.includes(sp)) await getRepo(acct, { kind: 'credential', sp, target: acct, credential: await credentialFor(sp.uri()) }, ' (whole)')
const refs = [...repoRefs(repoOf(acct, ref, sp))].filter((c) => !tainted.has(c))
for (const cid of refs) {
const mk = (side) => ({ token: tokenOf(acct, side), params: { space: sp.uri(acct, side), repo: didOf(acct, side), cid } })
const [a, b] = await Promise.all(sides.map((side) => call(side, `${SP}getBlob`, mk(side))))
const view = (side, res) => (res.status >= 400 ? norm(acct, side, res) : { status: 200, bytes: res.buf.toString('base64'), headers: blobHeaders(res) })
if (!isDeepStrictEqual(view(ref, a), view(cand, b))) await h.differ(`${SP}getBlob`, shown('getBlob (whole)', mk(cand)), view(ref, a), view(cand, b))
h.count('getBlob (whole)')
}
},
statuses,
}
}