~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
344 lines · 17.3 KB Raw History
// What a relay reads: listRepos, getRepoStatus, getBlocks and the sync
// getRecord, each held to the reference, and the requestCrawl each side sends
// its crawlers (spec/FEDERATION.md).

import { isDeepStrictEqual } from 'node:util'
import { readCarStream, readCarWithRoot, verifyProofs } from '@atproto/repo'
import { CID } from 'multiformats/cid'
import * as Digest from 'multiformats/hashes/digest'
import { COLLECTIONS, RKEYS, below, chance, pick, rand } from '../lib/random.mjs'
import { LISTENING, crawls, sinceStart } from '../lib/crawlers.mjs'
import { cand, ref, wait } from '../lib/processes.mjs'
import { fail } from '../lib/plc.mjs'
import { count, differ } from '../lib/report.mjs'
import { accountNamed, accounts, didOf } from '../lib/state.mjs'
import { normalize, xrpc, xrpcHttp } from '../lib/xrpc.mjs'

const LIST = 'com.atproto.sync.listRepos'
const STATUS = 'com.atproto.sync.getRepoStatus'
const BLOCKS = 'com.atproto.sync.getBlocks'
const RECORD = 'com.atproto.sync.getRecord'
/// A DID no server here holds.
const NOBODY = 'did:plc:aaaaaaaaaaaaaaaaaaaaaaaa'
const BAD_RKEYS = ['.', '..', 'a/b', 'a b', '#x']

/// The account's latest commit on `side`, `{cid, rev}`, as getLatestCommit
/// answers it, which the repository checks hold to getRepo.
async function latest(side, acct) {
  const r = await xrpcHttp(side, 'com.atproto.sync.getLatestCommit', { params: { did: didOf(side, acct) } })
  if (r.status !== 200) await fail(side, 'getLatestCommit', { did: `<did:${acct.n}>` }, 'the latest commit', r.body)
  return r.body
}

/// Send the request to both sides, `request` a function of the side; `judge`
/// checks each side's answer on its own terms and answers it with what
/// differs between two servers taken out, and those are compared. Errors are
/// compared by status and name.
async function compare(kind, nsid, request, shownAs, judge) {
  const [a, b] = await Promise.all([ref, cand].map((side) => xrpc(side, nsid, request(side))))
  const out = async (side, res) => (res.status === 200 ? judge(side, res) : normalize(side, res, nsid))
  const [na, nb] = [await out(ref, a), await out(cand, b)]
  if (!isDeepStrictEqual(na, nb)) await differ(nsid, shownAs, na, nb)
  count(kind, a.status < 400)
  return [a, b]
}

// listRepos

/// A listRepos page on `side`, normalized: each repo's `head` and `rev` must
/// be its account's latest commit on that side, and are then taken out.
async function page(side, res, shownAs) {
  for (const repo of Array.isArray(res.body?.repos) ? res.body.repos : []) {
    const acct = accountNamed(side, repo?.did)
    if (!acct) continue
    const { cid, rev } = await latest(side, acct)
    if (repo.head !== cid || repo.rev !== rev) await fail(side, 'listRepos', shownAs, { did: `<did:${acct.n}>`, head: cid, rev }, { head: repo.head, rev: repo.rev })
  }
  const n = normalize(side, res, LIST)
  if (Array.isArray(n.body?.repos)) n.body.repos = n.body.repos.map((r) => ({ ...r, head: typeof r?.head === 'string' ? '<head>' : r?.head, rev: typeof r?.rev === 'string' ? '<rev>' : r?.rev }))
  return n
}

/// Page through both sides' listRepos at `limit`, from each side's own
/// `cursor`, comparing each page. Answers the pages, as the reference's, and
/// each side's cursor of the last page with repos.
async function pageThrough(kind, limit, cursor = {}) {
  const pages = []
  const last = { ...cursor }
  for (;;) {
    const shownAs = { limit, cursor: cursor.reference === undefined ? undefined : `the cursor each side gave, page ${pages.length + 1}` }
    const answers = await Promise.all([ref, cand].map((side) => xrpc(side, LIST, { params: { limit, cursor: cursor[side.name] } })))
    const [na, nb] = [await page(ref, answers[0], shownAs), await page(cand, answers[1], shownAs)]
    if (!isDeepStrictEqual(na, nb)) await differ(LIST, shownAs, na, nb)
    count(kind, answers[0].status < 400)
    pages.push(na)
    if (answers[0].status !== 200 || !answers[0].body.cursor || pages.length > accounts.length + 2) break
    cursor = { reference: answers[0].body.cursor, server: answers[1].body.cursor }
    if (answers[0].body.repos.length) Object.assign(last, cursor)
  }
  return { pages, last }
}

/// The accounts as the harness names them, from the `n`th on.
const named = (n = 0) => accounts.slice(n).map((a) => `<did:${a.n}>`)
const listedIn = (pages) => pages.flatMap((p) => p.body?.repos?.map((r) => r.did) ?? [])

/// Each side's cursor of the last page with repos at the last listRepos
/// step, and how many accounts there were then.
let kept = null

/// listRepos: the cursor kept from the last time, which must list the
/// accounts made since; then every account, paged through at a random
/// limit; or a limit out of range, or an empty cursor (spec/FEDERATION.md,
/// "listRepos").
export async function listRepos() {
  if (kept) {
    const { pages } = await pageThrough('listRepos from an old cursor', undefined, kept.cursor)
    // The reference holds to it too, which checks the check.
    if (!isDeepStrictEqual(listedIn(pages), named(kept.n))) await fail(ref, 'listRepos from an old cursor', { cursor: 'the last page’s, kept' }, named(kept.n), listedIn(pages))
  }
  const r = rand()
  if (r < 0.15) {
    const params = chance(0.6) ? { limit: pick([0, -1, 1001, 1.5, 'x']) } : { cursor: '' }
    const [a] = await compare('listRepos, odd arguments', LIST, () => ({ params }), params, (side, res) => page(side, res, params))
    if (a.status === 200 && !isDeepStrictEqual(listedIn([normalize(ref, a, LIST)]), named())) await fail(ref, 'listRepos', params, named(), listedIn([normalize(ref, a, LIST)]))
    return
  }
  const limit = pick([undefined, 1, 2, 3, 1000])
  const { pages, last } = await pageThrough('listRepos', limit)
  if (!isDeepStrictEqual(listedIn(pages), named())) await fail(ref, 'listRepos', { limit }, named(), listedIn(pages))
  kept = { cursor: last, n: accounts.length }
}

// getRepoStatus

/// getRepoStatus of `did` on both sides, `rev` held to the side's latest.
async function repoStatus(kind, did, acct, shownAs) {
  await compare(kind, STATUS, (side) => ({ params: { did: did(side) } }), shownAs, async (side, res) => {
    if (acct) {
      const { rev } = await latest(side, acct)
      if (res.body?.rev !== rev) await fail(side, 'getRepoStatus', shownAs, { rev }, { rev: res.body?.rev })
    }
    const n = normalize(side, res, STATUS)
    if (typeof n.body?.rev === 'string') n.body.rev = '<rev>'
    return n
  })
}

/// getRepoStatus of the account, of a DID no one has, of a handle, of
/// something that is no DID, or of nothing (spec/FEDERATION.md,
/// "getRepoStatus").
export async function getRepoStatus(acct) {
  const r = rand()
  if (r < 0.6) return repoStatus('getRepoStatus', (side) => didOf(side, acct), acct, { did: `<did:${acct.n}>` })
  const [what, did] = pick([
    ['a DID no one has', NOBODY],
    ['a handle', acct.handle],
    ['no DID', 'not a did'],
    ['no did', undefined],
  ])
  await repoStatus(`getRepoStatus of ${what}`, () => did, null, { did })
}

// getBlocks

/// A CID of no block anyone has, with `codec`.
const madeUp = (codec) => CID.createV1(codec, Digest.create(0x12, Uint8Array.from({ length: 32 }, () => below(256)))).toString()

/// Every block of a CAR file in the order it holds them, each checked against
/// its CID, and its roots.
async function blocksOf(bytes) {
  const { roots, blocks } = await readCarStream((async function* () { yield bytes })())
  const cids = []
  for await (const b of blocks) cids.push(b.cid.toString())
  return { roots: roots.map(String), cids }
}

/// getBlocks of blocks of the account's repository: MST nodes and records,
/// read from the reference's getRepo and the same on both sides, and
/// sometimes each side's own latest commit, sometimes one named twice;
/// sometimes with a CID the repository never held; or a DID no one has, a
/// CID that does not parse, or none (spec/FEDERATION.md, "getBlocks").
export async function getBlocks(acct) {
  const res = await xrpcHttp(ref, 'com.atproto.sync.getRepo', { params: { did: didOf(ref, acct) }, raw: true })
  const car = await readCarWithRoot(res.bytes)
  const tree = []
  car.blocks.forEach((_, c) => c.equals(car.root) || tree.push(c.toString()))
  const heads = { reference: (await latest(ref, acct)).cid, server: (await latest(cand, acct)).cid }
  const asked = Array.from({ length: 1 + below(Math.min(6, tree.length)) }, () => pick(tree))
  if (chance(0.2)) asked.push(pick(asked))
  const withHead = chance(0.3)
  const at = below(asked.length + 1)
  const ownHead = (side) => (withHead ? [...asked.slice(0, at), heads[side.name], ...asked.slice(at)] : asked)
  const r = rand()
  let kind, did = (side) => didOf(side, acct), cids = ownHead, shownAs
  if (r < 0.6) {
    kind = 'getBlocks'
    shownAs = { did: `<did:${acct.n}>`, cids: withHead ? [...asked, 'the side’s latest commit'] : asked }
  } else if (r < 0.85) {
    const other = accounts.filter((a) => a !== acct)
    const which = other.length && chance(0.4) ? 'another account’s latest commit' : pick(['a dag-cbor CID of no block', 'a raw CID of no block'])
    const whose = which.startsWith('another') ? pick(other) : null
    const extra = {}
    for (const side of [ref, cand]) extra[side.name] = whose ? (await latest(side, whose)).cid : null
    const none = which.includes('raw') ? madeUp(0x55) : madeUp(0x71)
    const where = below(asked.length + 1)
    kind = 'getBlocks of a block the repository does not hold'
    cids = (side) => {
      const list = [...ownHead(side)]
      list.splice(where, 0, extra[side.name] ?? none)
      return list
    }
    shownAs = { did: `<did:${acct.n}>`, cids: asked, and: which }
  } else {
    const [what, d, c] = pick([
      ['a DID no one has', NOBODY, asked],
      ['a CID that does not parse', null, [...asked, 'notacid']],
      ['no cids', null, undefined],
      ['a handle', acct.handle, asked],
      ['no such DID, and a bad CID', NOBODY, ['notacid']],
    ])
    kind = `getBlocks with ${what}`
    if (d) did = () => d
    cids = () => c
    shownAs = { did: d ?? `<did:${acct.n}>`, cids: c }
  }
  await compare(kind, BLOCKS, (side) => ({ params: { did: did(side), cids: cids(side) }, raw: true }), shownAs, async (side, got) => {
    let car
    try {
      car = await blocksOf(got.bytes)
    } catch (err) {
      return fail(side, 'getBlocks', shownAs, 'a CAR file whose blocks match their CIDs', String(err.message ?? err))
    }
    const want = [...new Set(cids(side))].sort()
    const theirs = [...car.cids].sort()
    if (car.roots.length !== 0) await fail(side, 'getBlocks', shownAs, 'a CAR header with no roots', car.roots)
    if (!isDeepStrictEqual(theirs, want)) await fail(side, 'getBlocks', shownAs, { blocks: want, each: 'once' }, { blocks: theirs })
    return { status: 200, body: 'the blocks asked' }
  })
}

/// getBlocks of blocks the account's repository held and holds no longer:
/// MST nodes and records the reference's tree had at an earlier such step
/// and has not now, and each side's own earlier commit, sometimes with a
/// block it holds now. Both sides answer them as blocks the repository does
/// not hold, `400 InvalidRequest` (spec/FEDERATION.md, "getBlocks";
/// spec/STORE.md).
export async function getGoneBlocks(acct) {
  const res = await xrpcHttp(ref, 'com.atproto.sync.getRepo', { params: { did: didOf(ref, acct) }, raw: true })
  const car = await readCarWithRoot(res.bytes)
  const now = new Set()
  car.blocks.forEach((_, c) => c.equals(car.root) || now.add(c.toString()))
  const before = acct.treeSeen ?? new Set()
  acct.treeSeen = new Set([...before, ...now])
  const gone = [...before].filter((c) => !now.has(c))
  const heads = {}
  for (const side of [ref, cand]) heads[side.name] = (await latest(side, acct)).cid
  const earlier = (side) => [...new Set(acct.heads[side.name])].filter((c) => c !== heads[side.name])
  const withHead = earlier(ref).length > 0 && earlier(cand).length > 0 && (!gone.length || chance(0.4))
  if (!gone.length && !withHead) return
  const asked = Array.from({ length: gone.length ? 1 + below(Math.min(3, gone.length)) : 0 }, () => pick(gone))
  const at = rand()
  const head = (side) => (withHead ? [earlier(side)[Math.floor(at * earlier(side).length)]] : [])
  const held = now.size && chance(0.3) ? [pick([...now])] : []
  const shownAs = { did: `<did:${acct.n}>`, cids: [...asked, ...(withHead ? ['an earlier commit of the side’s own'] : []), ...held] }
  await compare('getBlocks of blocks the repository no longer holds', BLOCKS, (side) => ({ params: { did: didOf(side, acct), cids: [...asked, ...head(side), ...held] }, raw: true }), shownAs, (side) =>
    fail(side, 'getBlocks', shownAs, '400 InvalidRequest, for blocks the repository no longer holds', 'a CAR file'),
  )
}

// getRecord

/// The sync getRecord of a key of the account's repository, there or not,
/// or of a bad collection or record key, a handle, or a DID no one has
/// (spec/FEDERATION.md, "getRecord").
export async function syncGetRecord(acct) {
  const held = [...acct.records.keys()]
  let [collection, rkey] = held.length && chance(0.5) ? pick(held).split('/') : [pick(COLLECTIONS), pick(RKEYS)]
  let did = (side) => didOf(side, acct)
  let kind = 'sync getRecord'
  const r = rand()
  if (r < 0.2) {
    const [what, apply] = pick([
      ['a bad collection', () => (collection = 'not an nsid')],
      ['a bad record key', () => (rkey = pick(BAD_RKEYS))],
      ['a DID no one has', () => (did = () => NOBODY)],
      ['a handle', () => (did = () => acct.handle)],
    ])
    apply()
    kind = `sync getRecord of ${what}`
  }
  const shownAs = { did: did(ref) === didOf(ref, acct) ? `<did:${acct.n}>` : did(ref), collection, rkey }
  const cid = acct.records.get(`${collection}/${rkey}`) ?? null
  await compare(kind, RECORD, (side) => ({ params: { did: did(side), collection, rkey }, raw: true }), shownAs, async (side, got) => {
    let car
    try {
      car = await readCarWithRoot(got.bytes)
    } catch (err) {
      return fail(side, 'sync getRecord', shownAs, 'a CAR file with one root, whose blocks match their CIDs', String(err.message ?? err))
    }
    const head = await latest(side, acct)
    if (car.root.toString() !== head.cid) await fail(side, 'sync getRecord', shownAs, { root: head.cid }, { root: car.root.toString() })
    const claim = { collection, rkey, cid: cid ? CID.parse(cid) : null }
    let proved
    try {
      proved = await verifyProofs(got.bytes, [claim], didOf(side, acct), acct.sides[side.name].key)
    } catch (err) {
      return fail(side, 'sync getRecord', shownAs, `a proof the reference library verifies`, String(err.message ?? err))
    }
    if (proved.verified.length !== 1) await fail(side, 'sync getRecord', shownAs, cid ? `a proof of the record ${cid}` : 'a proof that there is no record', 'a proof of something else')
    const blocks = []
    car.blocks.forEach((_, c) => c.equals(car.root) || blocks.push(c.toString()))
    return { status: 200, body: { root: '<the latest commit>', blocks: blocks.sort() } }
  })
}

// After a restart

/// After a restart, once the repositories are checked: listRepos lists every
/// account in the order made, and each account's getRepoStatus, each with
/// the head and rev the repository check found (spec/FEDERATION.md,
/// "listRepos", "getRepoStatus").
export async function checkListing() {
  const { pages } = await pageThrough('listRepos after a restart', 1000)
  if (!isDeepStrictEqual(listedIn(pages), named())) await fail(ref, 'listRepos', { after: 'a restart' }, named(), listedIn(pages))
  for (const acct of accounts) await repoStatus('getRepoStatus after a restart', (side) => didOf(side, acct), acct, { did: `<did:${acct.n}>`, after: 'a restart' })
}

// requestCrawl

/// The request every crawler gets from either side: both sides' hostname is
/// `localhost` (lib/env.mjs, lib/processes.mjs).
const CRAWL = { method: 'POST', path: '/xrpc/com.atproto.sync.requestCrawl', type: 'application/json', body: { hostname: 'localhost' } }
const TWENTY_MINUTES = 20 * 60e3

/// Before the first account, no side has told its crawlers anything
/// (spec/FEDERATION.md, "requestCrawl").
export async function noCrawlsYet() {
  await wait(300)
  for (const side of [ref, cand]) if (crawls[side.name].length) await fail(side, 'requestCrawl', { before: 'any account was made' }, 'no request', crawls[side.name])
}

/// Each crawler that listens has had one request from each side since the
/// first event, the same, within 5 seconds of being checked; and no more
/// than one in each 20 minutes the run has lasted, besides, from the server,
/// one after each of its `restarts`. The crawlers are those each side's
/// `PDS_CRAWLERS` names (spec/FEDERATION.md, "Environment", "requestCrawl").
export async function checkCrawls(restarts) {
  const started = Date.now()
  for (const side of [ref, cand]) {
    const of = (name) => crawls[side.name].filter((c) => c.crawler === name)
    while (Object.keys(LISTENING).some((name) => !of(name).length) && Date.now() - started < 5000) await wait(20)
    const allowed = 1 + Math.floor(sinceStart() / TWENTY_MINUTES) + (side === cand ? restarts : 0)
    for (const name of Object.keys(LISTENING)) {
      const got = of(name)
      const shownAs = { crawler: name, restarts }
      if (got.length < 1 || got.length > allowed) await fail(side, 'requestCrawl', shownAs, `1 to ${allowed} requests`, got)
      for (const c of got) {
        const { crawler, at, ...shape } = c
        if (!isDeepStrictEqual(shape, CRAWL)) await fail(side, 'requestCrawl', shownAs, CRAWL, shape)
      }
    }
  }
  count('requestCrawl check', true)
}