~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
114 lines · 5.2 KB Raw History
// Crashes: the server killed with SIGKILL and started again on its data
// (spec/DURABILITY.md).

import { FIREHOSE, S3 } from '../lib/args.mjs'
import { COLLECTIONS, RKEYS, below, chance, goodRecord, pick, rand } from '../lib/random.mjs'
import { cand, ref, server, wait, waitReady } from '../lib/processes.mjs'
import { differ } from '../lib/report.mjs'
import { accounts, nextBatch, noteCommit, noteWrite } from '../lib/state.mjs'
import { xrpc } from '../lib/xrpc.mjs'
import { createAccount, sessionsSurvive } from './accounts.mjs'
import { checkListing } from './federation.mjs'
import { sessionsAfterCrash } from './sessions.mjs'
import { resubscribe } from './firehose.mjs'
import { inFlightHandle } from './identity.mjs'
import { syncHeads } from './records.mjs'
import { checkRepos } from './repository.mjs'
import { blobsSurvive, uploadInFlight } from './blobs.mjs'
import { afterRestart, orphanUpload } from './blobstore.mjs'
import { oauthAfterCrash, oauthBeforeCrash } from './oauth.mjs'
import { oauthClientsAfterCrash, oauthClientsBeforeCrash } from './oauth-clients.mjs'
import { mcpAfterCrash, mcpBeforeCrash } from './oauth-mcp.mjs'
import { oauthAccountAfterCrash, oauthAccountBeforeCrash } from './oauth-account.mjs'
import { passkeysAfterCrash, passkeysBeforeCrash } from './passkeys.mjs'
import { invitesSurvive } from './limits.mjs'
import { preferencesAfterCrash } from './proxy.mjs'

export const crashes = { count: 0, inFlight: 0, landed: 0, restartMs: [], reorganizing: 0 }

/// Kill the server with SIGKILL, sometimes with a put or a handle change on
/// its way, and start it again on the same data. Every account, its
/// sessions, and every write it answered must still be there, and what it
/// was killed during must have happened entirely or not at all. Creating
/// accounts carries on after it, unless `accountsMade` is false. `lead`, if
/// given, runs first, and answers whether the server is to be killed at
/// once, as when it is seen reorganizing its data (spec/STORE.md), with
/// nothing on its way. `start`, if given, starts the server again in place
/// of starting it and waiting for it to answer, and answers how long its
/// start took. Answers how long the server took to start again.
export async function crash({ lead, accountsMade = true, start = startAgain } = {}) {
  await oauthBeforeCrash()
  await oauthClientsBeforeCrash()
  await mcpBeforeCrash()
  await oauthAccountBeforeCrash()
  await passkeysBeforeCrash()
  const now = lead ? await lead() : false
  const op = !now && rand() < 0.6 ? await (chance(0.25) ? inFlightHandle : inFlightPut)(pick(accounts)) : null
  if (op) await wait(below(8))
  const blobOp = !now && !op && chance(0.5) ? uploadInFlight(pick(accounts)) : null
  if (blobOp) await wait(below(8))
  // With blobs in a bucket, sometimes an upload whose object the store
  // kept and never answered (spec/BLOBS.md, "The objects").
  const orphan = S3 && !now && chance(0.5) ? await orphanUpload() : null
  await server.kill()
  crashes.count += 1
  if (now) crashes.reorganizing += 1
  const ms = await start()
  crashes.restartMs.push(ms)
  if (FIREHOSE) resubscribe()
  if (op) {
    crashes.inFlight += 1
    // A request that had not reached the process before it died may reach
    // the new one, so whether it landed is known only once it is answered.
    op.answer = await op.sent
    await op.settle()
  }
  if (blobOp) await blobOp.settle()
  if (orphan) await orphan.settle()
  for (const acct of accounts) await syncHeads(acct)
  await sessionsSurvive()
  await sessionsAfterCrash()
  await blobsSurvive()
  await oauthAfterCrash()
  await oauthClientsAfterCrash()
  await mcpAfterCrash()
  await oauthAccountAfterCrash()
  await passkeysAfterCrash()
  await invitesSurvive()
  await preferencesAfterCrash()
  if (accountsMade && chance(0.3)) await createAccount()
  await checkRepos()
  await checkListing()
  if (S3) await afterRestart()
  return ms
}

/// The server started again, and how long it took to answer.
function startAgain() {
  server.start()
  return waitReady(server, cand.url)
}

/// A put to the account's repository on its way.
function inFlightPut(acct) {
  const collection = pick(COLLECTIONS)
  const key = pick(RKEYS)
  const body = { repo: acct.handle, collection, rkey: key, record: { ...goodRecord(collection), batch: nextBatch() } }
  const before = acct.records.get(`${collection}/${key}`) ?? null
  return {
    sent: xrpc(cand, 'com.atproto.repo.putRecord', { body, token: acct }),
    async settle() {
      const got = await xrpc(cand, 'com.atproto.repo.getRecord', { params: { repo: acct.handle, collection, rkey: key } })
      const nowCid = got.status === 200 ? got.body.cid : null
      if (nowCid === before) return
      // It landed, so make it on the reference too; the repository check
      // then says whether it landed whole.
      crashes.landed += 1
      const a = await xrpc(ref, 'com.atproto.repo.putRecord', { body, token: acct })
      if (a.status >= 400) await differ('a write the process was killed during', body, 'nothing, or the whole write', got.body, 'the reference refuses it')
      if (a.body.cid !== nowCid) await differ('a write the process was killed during', body, { cid: a.body.cid }, { cid: nowCid }, 'it landed only in part')
      noteWrite(acct, collection, key, a)
      noteCommit(ref, acct, a)
    },
  }
}