~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
289 lines · 16.5 KB Raw History
// A compatibility run (compare.mjs --area, which compat.mjs runs): the core
// checks of one area, compared with the reference as compare.mjs compares
// them, but going on past a difference. Each area is a run of its own, on a
// fresh pair of servers, so that what differs in one does not spill into
// another: its own steps, drawn by weight as compare.mjs draws its steps,
// and the steps it needs others for (writes, for the repository, sync and
// the firehose), whose differences are kept apart. A check is one step;
// it goes on past what differs (lib/report.mjs keeps it), is counted
// failed if anything did, and the run goes on. What it found is written to
// `--result` as JSON.
//
// Only the core is here (compat/README.md, the map): the email flows,
// invites, rate limits, the admin lifecycle and proxying are not, nor
// Monolith's own rules (its commands, logs, traces and metrics, store and
// scale, the exact numbers of its limits, its timing floors, and the
// reference's page API).

import { readFileSync, writeFileSync } from 'node:fs'
import { AREA, CHECK_EVERY, FIREHOSE, IMPL, RESULT, SEED, STEPS } from './args.mjs'
import { below, chance, pick } from './random.mjs'
import { fail, plcLog } from './plc.mjs'
import { adapter, cand, ref, shutdown } from './processes.mjs'
import { allowed, count, counts, differ, found, run } from './report.mjs'
import { can, passwordFor } from './server.mjs'
import { accounts, didOf } from './state.mjs'
import { both, shown, xrpc } from './xrpc.mjs'
import { accountRace, createAccount, createAccountRefused, createSession, crossWrites, made, newDetails } from '../checks/accounts.mjs'
import { blobRecord, getBlob, listBlobs, uploadBlob } from '../checks/blobs.mjs'
import { checkCrawls, getBlocks, getGoneBlocks, getRepoStatus, listRepos, noCrawlsYet, syncGetRecord } from '../checks/federation.mjs'
import { checkFirehose, checkReplay, startFirehose } from '../checks/firehose.mjs'
import { handleRace, recommendedCredentials, resolveHandle, updateHandle, updateHandleNoToken, updateHandleRefused, wellKnownHandle } from '../checks/identity.mjs'
import { oddRequests } from '../checks/odd.mjs'
import { checkOAuthMetadata, oauthClientLibrary, oauthDpopRefused, oauthGetSession, oauthLogin, oauthParRefused, oauthRefresh, oauthRevoke, oauthRevokeRefused, oauthTokenRefused } from '../checks/oauth.mjs'
import { oauthScopedCall, oauthScopedEdges, oauthScopedRefused } from '../checks/oauth-scopes.mjs'
import { oauthSetCall, oauthSetLogin, oauthSetPar } from '../checks/permission-sets.mjs'
import { oauthAssertionRefused, oauthClientCache, oauthClientDocument, oauthClientLogin, oauthConfidentialSession } from '../checks/oauth-clients.mjs'
import { applyWrites, concurrent, createRecord, createWithTid, deleteRecord, describeRepo, getRecord, listRecords, putRecord, syncHeads } from '../checks/records.mjs'
import { checkRepo, checkRepos } from '../checks/repository.mjs'
import { describeServer, getSession } from '../checks/session.mjs'
import { appPasswords, deleteSession, refreshSession, sessionRace, wrongTokens } from '../checks/sessions.mjs'
import { duplicates, validatedBatch, validatedElsewhere, validatedWrite } from '../checks/validation.mjs'

/// The record writes the repository, sync and firehose areas need.
const WRITES = [[createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8]]
const newAccount = Object.defineProperty(() => createAccount(), 'name', { value: 'createAccount' })
const firehoseChecks = Object.defineProperty(async () => {
  await checkFirehose()
  if (chance(0.3)) await checkReplay()
}, 'name', { value: 'checkFirehose' })
const crawlsSoFar = Object.defineProperty(() => checkCrawls(0), 'name', { value: 'checkCrawls' })

/// The one account of a server that hosts one (spec/COMPAT.md, "One
/// account"): the reference makes it with createAccount, the server through
/// its adapter's `claim`, with the same handle and email and the password
/// the server was started with. Their answers are not compared, nor is the
/// server's genesis operation; the account's signing keys, and the server's
/// rotation key, are read from the PLC directory. Then it is checked as a
/// new account is, from getSession on. Answers the account, or null.
async function claimAccount() {
  const CREATE = 'com.atproto.server.createAccount'
  const { handle, email } = newDetails(true)
  const body = { handle, email, password: cand.password }
  const [a, b] = await Promise.all([xrpc(ref, CREATE, { body }), cand.claim({ handle, email })])
  count('createAccount (the server\'s one account)', a.status === 200 && b.status === 200)
  if (a.status !== 200) await differ(CREATE, body, 'an account', a.body, 'the reference refuses an account the harness means to make')
  if (b.status !== 200) await differ('the server\'s one account, made by its adapter', { handle, email }, 'an account', b.body, 'the adapter fails its own step')
  if (a.status !== 200 || b.status !== 200) return null
  const acct = made(body, a, b)
  for (const side of [ref, cand]) {
    const log = await plcLog(didOf(side, acct))
    const op = Array.isArray(log.body) ? log.body[0] : null
    if (!op) {
      await fail(side, 'PLC log', shown({ body }), 'a genesis operation', log)
      continue
    }
    acct.sides[side.name].key = op.verificationMethods?.atproto
    if (side === cand) {
      acct.sides[side.name].kem = op.verificationMethods?.atproto_kem
      cand.rotationKey ??= op.rotationKeys?.[0]
    }
  }
  await both('getSession of a new account', 'com.atproto.server.getSession', { token: acct })
  await syncHeads(acct)
  await checkRepo(acct)
  return acct
}

/// The areas, in the order the report lists them. Each has `about`, what it
/// checks; `steps`, `[check, weight, needs]`, drawn as compare.mjs draws
/// its steps, `needs` the settings (lib/adapters/, `unsupported`) a
/// check depends on; `support`, `[step, weight, needs]` drawn among them,
/// whose differences are kept apart; `pre`, checks of the area before the
/// steps (and after the first accounts unless `beforeAccounts`); and
/// `every`, checks of the area every `--check-every` steps and at the end.
/// `accounts` is the need of a check that makes an account or needs a
/// second one (spec/COMPAT.md, "One account").
export const AREAS = {
  oauth: {
    about: 'OAuth, the protocol: the server\'s metadata, PAR, the code exchange with PKCE, DPoP and its nonces, refresh and its rotation, revocation, DPoP-bound calls to XRPC, and a client app\'s own library',
    pre: [[checkOAuthMetadata]],
    steps: [[oauthLogin, 4], [oauthParRefused, 3], [oauthDpopRefused, 3], [oauthTokenRefused, 2], [oauthRevokeRefused, 1], [oauthRefresh, 3], [oauthRevoke, 2], [oauthGetSession, 2], [oauthClientLibrary, 1, ['address']]],
  },
  'oauth scopes': {
    about: 'OAuth scopes, enforced on every method: calls with an OAuth session\'s token, and with one thing wrong in its proof, token or scheme',
    steps: [[oauthScopedCall, 8], [oauthScopedRefused, 4], [oauthScopedEdges, 1]],
  },
  'oauth permission sets': {
    about: 'OAuth permission sets (`include:` scopes), resolved from the harness\'s lexicon authority: PAR with sets that resolve and do not, the page\'s data, consent keeping or dropping each, the token\'s expanded scope, and calls the sets\' grants allow or refuse',
    steps: [[oauthSetPar, 2, ['lexiconAuthority']], [oauthSetLogin, 2, ['lexiconAuthority']], [oauthSetCall, 3, ['lexiconAuthority']]],
  },
  'oauth clients': {
    about: 'OAuth clients\' metadata documents: fetched from the harness\'s web, kept and refused as the reference does; confidential clients and their assertions',
    steps: [[oauthClientDocument, 4, ['clientFetch']], [oauthClientCache, 1, ['clientFetch']], [oauthClientLogin, 3, ['clientFetch']], [oauthAssertionRefused, 3, ['clientFetch']], [oauthConfidentialSession, 2, ['clientFetch']]],
  },
  sessions: {
    about: 'Sessions: createSession, getSession, refreshSession, app passwords and their scopes, deleteSession, tokens of the wrong kind, and describeServer',
    steps: [[createSession, 3], [getSession, 1], [describeServer, 1, ['handleDomains']], [refreshSession, 3], [deleteSession, 2], [wrongTokens, 2], [appPasswords, 4], [sessionRace, 1]],
  },
  accounts: {
    about: 'Accounts: createAccount, the DID it registers with the PLC directory and its genesis operation, and what it refuses',
    steps: [[newAccount, 4, ['accounts']], [createAccountRefused, 4, ['accounts']], [accountRace, 1, ['accounts']]],
  },
  records: {
    about: 'Records: createRecord, putRecord, deleteRecord, applyWrites, swapCommit and swapRecord, getRecord, listRecords, describeRepo, writes to another account\'s repository, and requests the lexicons refuse',
    steps: [[createRecord, 14], [putRecord, 14], [deleteRecord, 8], [applyWrites, 8], [getRecord, 10], [listRecords, 6], [describeRepo, 2], [createWithTid, 2], [concurrent, 6], [oddRequests, 3], [crossWrites, 3, ['accounts']]],
  },
  validation: {
    about: 'Records checked against their lexicons: known types valid and not, `validate`, duplicates replaced, in applyWrites too',
    steps: [[validatedWrite, 10], [duplicates, 5], [validatedBatch, 4], [validatedElsewhere, 1]],
  },
  repository: {
    about: 'The repository: each account\'s whole repository from getRepo, verified (signature, MST, every record) by the reference\'s library against the DID\'s key, its MST root the reference\'s, getRepo since, getLatestCommit',
    support: WRITES,
    pre: [[checkRepos]],
    every: [[checkRepos]],
  },
  sync: {
    about: 'Sync: listRepos, getRepoStatus, getBlocks, sync.getRecord with its proof, and requestCrawl',
    pre: [[noCrawlsYet, ['hostname'], true], [crawlsSoFar, ['hostname']]],
    steps: [[listRepos, 2], [getRepoStatus, 2], [getBlocks, 3], [getGoneBlocks, 2], [syncGetRecord, 3]],
    support: WRITES,
    every: [[crawlsSoFar, ['hostname']]],
  },
  firehose: {
    about: 'The firehose: each side\'s subscribeRepos from the start, every event checked as a consumer would, in order, with its blocks, and replays from a cursor',
    support: [...WRITES, [newAccount, 1, ['accounts']], [updateHandle, 2]],
    every: [[firehoseChecks]],
  },
  identity: {
    about: 'Identity: handles, their changes and the PLC operations they make, resolveHandle, /.well-known/atproto-did, getRecommendedDidCredentials, and DID documents',
    steps: [[updateHandle, 3], [updateHandleRefused, 3], [updateHandleNoToken, 1], [resolveHandle, 3], [wellKnownHandle, 2], [recommendedCredentials, 1, ['recoveryKey']], [describeRepo, 2], [handleRace, 1, ['accounts']]],
  },
  blobs: {
    about: 'Blobs: uploadBlob, records that reference them, getBlob, listBlobs',
    steps: [[uploadBlob, 4], [blobRecord, 5], [getBlob, 3], [listBlobs, 2]],
  },
}

/// Why `needs` cannot be met by the server, or null.
const missing = (needs = []) => needs.map((n) => adapter.unsupported[n] && `${n}: ${adapter.unsupported[n]}`).filter(Boolean)

/// What the run found, of its own checks and of its supporting steps: how
/// many ran, differed, and could not go on; the first difference; and by
/// check, the same, with each one's first difference.
const own = { checks: 0, failed: 0, errors: 0, first: null, byCheck: {} }
const support = { checks: 0, failed: 0, errors: 0, first: null, byCheck: {} }
const skipped = {}

/// An exception a check did not expect, as an answer of a shape it cannot
/// read, kept as a difference is.
const exception = (e, name) => ({ name, step: run.step, what: `the check could not go on: ${String(e?.message ?? e).split('\n')[0]}`, note: 'an exception, not a comparison: what the server answered was of a shape the check cannot read, or the harness is wrong', request: null, reference: null, server: String(e?.stack ?? e).split('\n').slice(0, 6).join('\n') })

/// Run `fn` as one check of the area (or a supporting step, `mine` false).
/// It goes on past what differs (lib/report.mjs keeps it): one with a
/// difference is counted failed, and its first difference kept; one that
/// throws, or does not end within 3 minutes, with none, is counted as not
/// going on. Then the run goes on.
async function check(fn, args, mine) {
  const name = fn.name || 'anonymous'
  const tally = mine ? own : support
  const by = (tally.byCheck[name] ??= { checks: 0, failed: 0, errors: 0, first: null, whats: {} })
  tally.checks++
  by.checks++
  found.length = 0
  let timer, error = null
  try {
    await Promise.race([fn(...args), new Promise((_, no) => (timer = setTimeout(() => no(new Error('no answer within 3 minutes')), 180_000)))])
  } catch (e) {
    error = e
  } finally {
    clearTimeout(timer)
  }
  const diffs = found.splice(0)
  if (!diffs.length && !error) return
  // Each thing that differed, how often, the exception it ended in after
  // them too.
  const whats = [...diffs.map((d) => d.what), ...(error && diffs.length ? [`then could not go on: ${String(error?.message ?? error).split('\n')[0]}`] : [])]
  for (const w of whats) by.whats[w] = (by.whats[w] ?? 0) + 1
  const f = diffs.length ? { name, ...diffs[0], ...(diffs.length > 1 ? { more: diffs.length - 1 } : {}) } : exception(error, name)
  const kind = diffs.length ? 'failed' : 'errors'
  tally[kind]++
  by[kind]++
  tally.first ??= f
  by.first ??= f
  console.error(`step ${run.step}: ${name}: ${f.what}${f.note ? ` (${f.note})` : ''}`)
}

/// Each of `list`, `[check, needs, beforeAccounts]`, run unless the server
/// cannot meet its needs, which is noted.
async function runAll(list, mine, when) {
  for (const [fn, needs, before] of list ?? []) {
    if (Boolean(before) !== when) continue
    const why = missing(needs)
    if (why.length) skipped[fn.name] = why
    else await check(fn, [], mine)
  }
}

/// The area's run, and its result written; then the run ends.
export async function areaRun() {
  const area = AREAS[AREA]
  if (!area) {
    console.error(`no area ${AREA}; there are ${Object.keys(AREAS).join(', ')}`)
    await shutdown(2)
  }
  const drawn = []
  const drawable = (list, mine) => {
    for (const [fn, w, needs] of list ?? []) {
      const why = missing(needs)
      if (why.length) skipped[fn.name] = why
      else drawn.push([fn, w, mine])
    }
  }
  drawable(area.steps, true)
  drawable(area.support, false)
  if (FIREHOSE) await check(startFirehose, [], true)
  await runAll(area.pre, true, true)
  // The first accounts, as compare.mjs makes them, each a supporting step;
  // as many more as it takes (up to ten tries), when the server refuses
  // one the reference makes, which is then not kept. A server that hosts
  // one account has that one, made once, through its adapter.
  if (!can('accounts')) {
    await check(Object.defineProperty(() => claimAccount(), 'name', { value: 'the server\'s one account' }), [], false)
  } else {
    const first = Object.defineProperty(() => createAccount({ withPassword: true, password: passwordFor }), 'name', { value: 'createAccount (a first account)' })
    const want = 2 + below(2)
    for (let tries = 0; accounts.length < want && tries < 10; tries++) await check(first, [], false)
  }
  for (const acct of accounts) await check(describeRepo, [acct], false)
  await runAll(area.pre, true, false)
  const total = drawn.reduce((n, [, w]) => n + w, 0)
  for (run.step = 1; run.step <= STEPS; run.step++) {
    if (drawn.length && accounts.length) {
      let r = below(total)
      for (const [fn, w, mine] of drawn) {
        if ((r -= w) < 0) {
          await check(fn, [pick(accounts)], mine)
          break
        }
      }
    }
    if (run.step % CHECK_EVERY === 0) await runAll(area.every, true, false)
    if (run.step % 100 === 0) console.error(`${run.step} steps`)
  }
  run.step = STEPS
  await runAll(area.every, true, false)
  const pkg = (name) => JSON.parse(readFileSync(new URL(`../node_modules/${name}/package.json`, import.meta.url), 'utf8')).version
  const result = {
    impl: IMPL,
    area: AREA,
    about: area.about,
    seed: SEED,
    steps: STEPS,
    reference: `@atproto/pds ${pkg('@atproto/pds')} (through @atproto/dev-env ${pkg('@atproto/dev-env')})`,
    server: await adapter.info(),
    own,
    support,
    skipped,
    unsupported: adapter.unsupported,
    allowed,
    accounts: accounts.length,
    requests: Object.values(counts).reduce((n, { ok, error }) => n + ok + error, 0),
  }
  // A difference may hold what a firehose frame decodes to: BigInts and bytes.
  const plain = (k, v) => (typeof v === 'bigint' ? `${v}` : v instanceof Uint8Array ? `<${v.length} bytes>` : v)
  writeFileSync(RESULT, `${JSON.stringify(result, plain, 2)}\n`)
  console.error(`\n${AREA}: ${own.checks - own.failed - own.errors} of ${own.checks} checks agree, ${own.failed} differ, ${own.errors} could not go on; supporting steps: ${support.failed + support.errors} of ${support.checks} failed`)
  await shutdown(0)
}