chadtmiller.com/monolith
An atproto PDS
0
0
// What a relay reads: listRepos, getRepoStatus, getBlocks and the sync
// getRecord, each held to the reference, and the requestCrawl each side sends
// its crawlers (spec/FEDERATION.md).
import { isDeepStrictEqual } from 'node:util'
import { readCarStream, readCarWithRoot, verifyProofs } from '@atproto/repo'
import { CID } from 'multiformats/cid'
import * as Digest from 'multiformats/hashes/digest'
import { COLLECTIONS, RKEYS, below, chance, pick, rand } from '../lib/random.mjs'
import { LISTENING, crawls, sinceStart } from '../lib/crawlers.mjs'
import { cand, ref, wait } from '../lib/processes.mjs'
import { fail } from '../lib/plc.mjs'
import { count, differ } from '../lib/report.mjs'
import { accountNamed, accounts, didOf } from '../lib/state.mjs'
import { normalize, xrpc, xrpcHttp } from '../lib/xrpc.mjs'
const LIST = 'com.atproto.sync.listRepos'
const STATUS = 'com.atproto.sync.getRepoStatus'
const BLOCKS = 'com.atproto.sync.getBlocks'
const RECORD = 'com.atproto.sync.getRecord'
/// A DID no server here holds.
const NOBODY = 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa'
const BAD_RKEYS = ['.', '..', 'a/b', 'a b', '#x']
/// The account's latest commit on `side`, `{cid, rev}`, as getLatestCommit
/// answers it, which the repository checks hold to getRepo.
async function latest(side, acct) {
const r = await xrpcHttp(side, 'com.atproto.sync.getLatestCommit', { params: { did: didOf(side, acct) } })
if (r.status !== 200) await fail(side, 'getLatestCommit', { did: `<did:${acct.n}>` }, 'the latest commit', r.body)
return r.body
}
/// Send the request to both sides, `request` a function of the side; `judge`
/// checks each side's answer on its own terms and answers it with what
/// differs between two servers taken out, and those are compared. Errors are
/// compared by status and name.
async function compare(kind, nsid, request, shownAs, judge) {
const [a, b] = await Promise.all([ref, cand].map((side) => xrpc(side, nsid, request(side))))
const out = async (side, res) => (res.status === 200 ? judge(side, res) : normalize(side, res, nsid))
const [na, nb] = [await out(ref, a), await out(cand, b)]
if (!isDeepStrictEqual(na, nb)) await differ(nsid, shownAs, na, nb)
count(kind, a.status < 400)
return [a, b]
}
// listRepos
/// A listRepos page on `side`, normalized: each repo's `head` and `rev` must
/// be its account's latest commit on that side, and are then taken out.
async function page(side, res, shownAs) {
for (const repo of Array.isArray(res.body?.repos) ? res.body.repos : []) {
const acct = accountNamed(side, repo?.did)
if (!acct) continue
const { cid, rev } = await latest(side, acct)
if (repo.head !== cid || repo.rev !== rev) await fail(side, 'listRepos', shownAs, { did: `<did:${acct.n}>`, head: cid, rev }, { head: repo.head, rev: repo.rev })
}
const n = normalize(side, res, LIST)
if (Array.isArray(n.body?.repos)) n.body.repos = n.body.repos.map((r) => ({ ...r, head: typeof r?.head === 'string' ? '<head>' : r?.head, rev: typeof r?.rev === 'string' ? '<rev>' : r?.rev }))
return n
}
/// Page through both sides' listRepos at `limit`, from each side's own
/// `cursor`, comparing each page. Answers the pages, as the reference's, and
/// each side's cursor of the last page with repos.
async function pageThrough(kind, limit, cursor = {}) {
const pages = []
const last = { ...cursor }
for (;;) {
const shownAs = { limit, cursor: cursor.reference === undefined ? undefined : `the cursor each side gave, page ${pages.length + 1}` }
const answers = await Promise.all([ref, cand].map((side) => xrpc(side, LIST, { params: { limit, cursor: cursor[side.name] } })))
const [na, nb] = [await page(ref, answers[0], shownAs), await page(cand, answers[1], shownAs)]
if (!isDeepStrictEqual(na, nb)) await differ(LIST, shownAs, na, nb)
count(kind, answers[0].status < 400)
pages.push(na)
if (answers[0].status !== 200 || !answers[0].body.cursor || pages.length > accounts.length + 2) break
cursor = { reference: answers[0].body.cursor, server: answers[1].body.cursor }
if (answers[0].body.repos.length) Object.assign(last, cursor)
}
return { pages, last }
}
/// The accounts as the harness names them, from the `n`th on.
const named = (n = 0) => accounts.slice(n).map((a) => `<did:${a.n}>`)
const listedIn = (pages) => pages.flatMap((p) => p.body?.repos?.map((r) => r.did) ?? [])
/// Each side's cursor of the last page with repos at the last listRepos
/// step, and how many accounts there were then.
let kept = null
/// listRepos: the cursor kept from the last time, which must list the
/// accounts made since; then every account, paged through at a random
/// limit; or a limit out of range, or an empty cursor (spec/FEDERATION.md,
/// "listRepos").
export async function listRepos() {
if (kept) {
const { pages } = await pageThrough('listRepos from an old cursor', undefined, kept.cursor)
// The reference holds to it too, which checks the check.
if (!isDeepStrictEqual(listedIn(pages), named(kept.n))) await fail(ref, 'listRepos from an old cursor', { cursor: 'the last page’s, kept' }, named(kept.n), listedIn(pages))
}
const r = rand()
if (r < 0.15) {
const params = chance(0.6) ? { limit: pick([0, -1, 1001, 1.5, 'x']) } : { cursor: '' }
const [a] = await compare('listRepos, odd arguments', LIST, () => ({ params }), params, (side, res) => page(side, res, params))
if (a.status === 200 && !isDeepStrictEqual(listedIn([normalize(ref, a, LIST)]), named())) await fail(ref, 'listRepos', params, named(), listedIn([normalize(ref, a, LIST)]))
return
}
const limit = pick([undefined, 1, 2, 3, 1000])
const { pages, last } = await pageThrough('listRepos', limit)
if (!isDeepStrictEqual(listedIn(pages), named())) await fail(ref, 'listRepos', { limit }, named(), listedIn(pages))
kept = { cursor: last, n: accounts.length }
}
// getRepoStatus
/// getRepoStatus of `did` on both sides, `rev` held to the side's latest.
async function repoStatus(kind, did, acct, shownAs) {
await compare(kind, STATUS, (side) => ({ params: { did: did(side) } }), shownAs, async (side, res) => {
if (acct) {
const { rev } = await latest(side, acct)
if (res.body?.rev !== rev) await fail(side, 'getRepoStatus', shownAs, { rev }, { rev: res.body?.rev })
}
const n = normalize(side, res, STATUS)
if (typeof n.body?.rev === 'string') n.body.rev = '<rev>'
return n
})
}
/// getRepoStatus of the account, of a DID no one has, of a handle, of
/// something that is no DID, or of nothing (spec/FEDERATION.md,
/// "getRepoStatus").
export async function getRepoStatus(acct) {
const r = rand()
if (r < 0.6) return repoStatus('getRepoStatus', (side) => didOf(side, acct), acct, { did: `<did:${acct.n}>` })
const [what, did] = pick([
['a DID no one has', NOBODY],
['a handle', acct.handle],
['no DID', 'not a did'],
['no did', undefined],
])
await repoStatus(`getRepoStatus of ${what}`, () => did, null, { did })
}
// getBlocks
/// A CID of no block anyone has, with `codec`.
const madeUp = (codec) => CID.createV1(codec, Digest.create(0x12, Uint8Array.from({ length: 32 }, () => below(256)))).toString()
/// Every block of a CAR file in the order it holds them, each checked against
/// its CID, and its roots.
async function blocksOf(bytes) {
const { roots, blocks } = await readCarStream((async function* () { yield bytes })())
const cids = []
for await (const b of blocks) cids.push(b.cid.toString())
return { roots: roots.map(String), cids }
}
/// getBlocks of blocks of the account's repository: MST nodes and records,
/// read from the reference's getRepo and the same on both sides, and
/// sometimes each side's own latest commit, sometimes one named twice;
/// sometimes with a CID the repository never held; or a DID no one has, a
/// CID that does not parse, or none (spec/FEDERATION.md, "getBlocks").
export async function getBlocks(acct) {
const res = await xrpcHttp(ref, 'com.atproto.sync.getRepo', { params: { did: didOf(ref, acct) }, raw: true })
const car = await readCarWithRoot(res.bytes)
const tree = []
car.blocks.forEach((_, c) => c.equals(car.root) || tree.push(c.toString()))
const heads = { reference: (await latest(ref, acct)).cid, server: (await latest(cand, acct)).cid }
const asked = Array.from({ length: 1 + below(Math.min(6, tree.length)) }, () => pick(tree))
if (chance(0.2)) asked.push(pick(asked))
const withHead = chance(0.3)
const at = below(asked.length + 1)
const ownHead = (side) => (withHead ? [...asked.slice(0, at), heads[side.name], ...asked.slice(at)] : asked)
const r = rand()
let kind, did = (side) => didOf(side, acct), cids = ownHead, shownAs
if (r < 0.6) {
kind = 'getBlocks'
shownAs = { did: `<did:${acct.n}>`, cids: withHead ? [...asked, 'the side’s latest commit'] : asked }
} else if (r < 0.85) {
const other = accounts.filter((a) => a !== acct)
const which = other.length && chance(0.4) ? 'another account’s latest commit' : pick(['a dag-cbor CID of no block', 'a raw CID of no block'])
const whose = which.startsWith('another') ? pick(other) : null
const extra = {}
for (const side of [ref, cand]) extra[side.name] = whose ? (await latest(side, whose)).cid : null
const none = which.includes('raw') ? madeUp(0x55) : madeUp(0x71)
const where = below(asked.length + 1)
kind = 'getBlocks of a block the repository does not hold'
cids = (side) => {
const list = [...ownHead(side)]
list.splice(where, 0, extra[side.name] ?? none)
return list
}
shownAs = { did: `<did:${acct.n}>`, cids: asked, and: which }
} else {
const [what, d, c] = pick([
['a DID no one has', NOBODY, asked],
['a CID that does not parse', null, [...asked, 'notacid']],
['no cids', null, undefined],
['a handle', acct.handle, asked],
['no such DID, and a bad CID', NOBODY, ['notacid']],
])
kind = `getBlocks with ${what}`
if (d) did = () => d
cids = () => c
shownAs = { did: d ?? `<did:${acct.n}>`, cids: c }
}
await compare(kind, BLOCKS, (side) => ({ params: { did: did(side), cids: cids(side) }, raw: true }), shownAs, async (side, got) => {
let car
try {
car = await blocksOf(got.bytes)
} catch (err) {
return fail(side, 'getBlocks', shownAs, 'a CAR file whose blocks match their CIDs', String(err.message ?? err))
}
const want = [...new Set(cids(side))].sort()
const theirs = [...car.cids].sort()
if (car.roots.length !== 0) await fail(side, 'getBlocks', shownAs, 'a CAR header with no roots', car.roots)
if (!isDeepStrictEqual(theirs, want)) await fail(side, 'getBlocks', shownAs, { blocks: want, each: 'once' }, { blocks: theirs })
return { status: 200, body: 'the blocks asked' }
})
}
/// getBlocks of blocks the account's repository held and holds no longer:
/// MST nodes and records the reference's tree had at an earlier such step
/// and has not now, and each side's own earlier commit, sometimes with a
/// block it holds now. Both sides answer them as blocks the repository does
/// not hold, `400 InvalidRequest` (spec/FEDERATION.md, "getBlocks";
/// spec/STORE.md).
export async function getGoneBlocks(acct) {
const res = await xrpcHttp(ref, 'com.atproto.sync.getRepo', { params: { did: didOf(ref, acct) }, raw: true })
const car = await readCarWithRoot(res.bytes)
const now = new Set()
car.blocks.forEach((_, c) => c.equals(car.root) || now.add(c.toString()))
const before = acct.treeSeen ?? new Set()
acct.treeSeen = new Set([...before, ...now])
const gone = [...before].filter((c) => !now.has(c))
const heads = {}
for (const side of [ref, cand]) heads[side.name] = (await latest(side, acct)).cid
const earlier = (side) => [...new Set(acct.heads[side.name])].filter((c) => c !== heads[side.name])
const withHead = earlier(ref).length > 0 && earlier(cand).length > 0 && (!gone.length || chance(0.4))
if (!gone.length && !withHead) return
const asked = Array.from({ length: gone.length ? 1 + below(Math.min(3, gone.length)) : 0 }, () => pick(gone))
const at = rand()
const head = (side) => (withHead ? [earlier(side)[Math.floor(at * earlier(side).length)]] : [])
const held = now.size && chance(0.3) ? [pick([...now])] : []
const shownAs = { did: `<did:${acct.n}>`, cids: [...asked, ...(withHead ? ['an earlier commit of the side’s own'] : []), ...held] }
await compare('getBlocks of blocks the repository no longer holds', BLOCKS, (side) => ({ params: { did: didOf(side, acct), cids: [...asked, ...head(side), ...held] }, raw: true }), shownAs, (side) =>
fail(side, 'getBlocks', shownAs, '400 InvalidRequest, for blocks the repository no longer holds', 'a CAR file'),
)
}
// getRecord
/// The sync getRecord of a key of the account's repository, there or not,
/// or of a bad collection or record key, a handle, or a DID no one has
/// (spec/FEDERATION.md, "getRecord").
export async function syncGetRecord(acct) {
const held = [...acct.records.keys()]
let [collection, rkey] = held.length && chance(0.5) ? pick(held).split('/') : [pick(COLLECTIONS), pick(RKEYS)]
let did = (side) => didOf(side, acct)
let kind = 'sync getRecord'
const r = rand()
if (r < 0.2) {
const [what, apply] = pick([
['a bad collection', () => (collection = 'not an nsid')],
['a bad record key', () => (rkey = pick(BAD_RKEYS))],
['a DID no one has', () => (did = () => NOBODY)],
['a handle', () => (did = () => acct.handle)],
])
apply()
kind = `sync getRecord of ${what}`
}
const shownAs = { did: did(ref) === didOf(ref, acct) ? `<did:${acct.n}>` : did(ref), collection, rkey }
const cid = acct.records.get(`${collection}/${rkey}`) ?? null
await compare(kind, RECORD, (side) => ({ params: { did: did(side), collection, rkey }, raw: true }), shownAs, async (side, got) => {
let car
try {
car = await readCarWithRoot(got.bytes)
} catch (err) {
return fail(side, 'sync getRecord', shownAs, 'a CAR file with one root, whose blocks match their CIDs', String(err.message ?? err))
}
const head = await latest(side, acct)
if (car.root.toString() !== head.cid) await fail(side, 'sync getRecord', shownAs, { root: head.cid }, { root: car.root.toString() })
const claim = { collection, rkey, cid: cid ? CID.parse(cid) : null }
let proved
try {
proved = await verifyProofs(got.bytes, [claim], didOf(side, acct), acct.sides[side.name].key)
} catch (err) {
return fail(side, 'sync getRecord', shownAs, `a proof the reference library verifies`, String(err.message ?? err))
}
if (proved.verified.length !== 1) await fail(side, 'sync getRecord', shownAs, cid ? `a proof of the record ${cid}` : 'a proof that there is no record', 'a proof of something else')
const blocks = []
car.blocks.forEach((_, c) => c.equals(car.root) || blocks.push(c.toString()))
return { status: 200, body: { root: '<the latest commit>', blocks: blocks.sort() } }
})
}
// After a restart
/// After a restart, once the repositories are checked: listRepos lists every
/// account in the order made, and each account's getRepoStatus, each with
/// the head and rev the repository check found (spec/FEDERATION.md,
/// "listRepos", "getRepoStatus").
export async function checkListing() {
const { pages } = await pageThrough('listRepos after a restart', 1000)
if (!isDeepStrictEqual(listedIn(pages), named())) await fail(ref, 'listRepos', { after: 'a restart' }, named(), listedIn(pages))
for (const acct of accounts) await repoStatus('getRepoStatus after a restart', (side) => didOf(side, acct), acct, { did: `<did:${acct.n}>`, after: 'a restart' })
}
// requestCrawl
/// The request every crawler gets from either side: both sides' hostname is
/// `localhost` (lib/env.mjs, lib/processes.mjs).
const CRAWL = { method: 'POST', path: '/xrpc/com.atproto.sync.requestCrawl', type: 'application/json', body: { hostname: 'localhost' } }
const TWENTY_MINUTES = 20 * 60e3
/// Before the first account, no side has told its crawlers anything
/// (spec/FEDERATION.md, "requestCrawl").
export async function noCrawlsYet() {
await wait(300)
for (const side of [ref, cand]) if (crawls[side.name].length) await fail(side, 'requestCrawl', { before: 'any account was made' }, 'no request', crawls[side.name])
}
/// Each crawler that listens has had one request from each side since the
/// first event, the same, within 5 seconds of being checked; and no more
/// than one in each 20 minutes the run has lasted, besides, from the server,
/// one after each of its `restarts`. The crawlers are those each side's
/// `PDS_CRAWLERS` names (spec/FEDERATION.md, "Environment", "requestCrawl").
export async function checkCrawls(restarts) {
const started = Date.now()
for (const side of [ref, cand]) {
const of = (name) => crawls[side.name].filter((c) => c.crawler === name)
while (Object.keys(LISTENING).some((name) => !of(name).length) && Date.now() - started < 5000) await wait(20)
const allowed = 1 + Math.floor(sinceStart() / TWENTY_MINUTES) + (side === cand ? restarts : 0)
for (const name of Object.keys(LISTENING)) {
const got = of(name)
const shownAs = { crawler: name, restarts }
if (got.length < 1 || got.length > allowed) await fail(side, 'requestCrawl', shownAs, `1 to ${allowed} requests`, got)
for (const c of got) {
const { crawler, at, ...shape } = c
if (!isDeepStrictEqual(shape, CRAWL)) await fail(side, 'requestCrawl', shownAs, CRAWL, shape)
}
}
}
count('requestCrawl check', true)
}