~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
265 lines · 13.5 KB Raw History
// Moving blobs between the data directory and a bucket (spec/BLOBS.md,
// "Moving them"), a run of its own, --migrate: the server keeps its blobs on
// disk, then is moved into a bucket of the S3 store the harness runs
// (lib/s3.mjs), then back, each move killed at random moments and run again,
// one turned back halfway, one stopped by an object gone. The reference keeps
// its blobs on disk throughout, and after each move every account's blobs
// must read as the reference's; the server's bucket and data directory are
// held to what each place holds. And: the data directory is held by one
// process at a time (spec/COMMANDS.md, "One binary").

import { spawn } from 'node:child_process'
import { isDeepStrictEqual } from 'node:util'
import { command, STEPS_GIVEN } from '../lib/args.mjs'
import { below, chance, pick } from '../lib/random.mjs'
import { cand, freePort, s3, server, waitReady } from '../lib/processes.mjs'
import { count, differ, run } from '../lib/report.mjs'
import { BUCKETS, S3_KEYS, S3_TIMEOUT_MS } from '../lib/s3.mjs'
import { accounts, didOf } from '../lib/state.mjs'
import { createFirstAccounts } from './accounts.mjs'
import { blobRecord, blobsSurvive, compareGetBlob, getBlob, listBlobs, model, referencedCids, uploadBlob } from './blobs.mjs'
import { bucketHolds, dataDirHoldsNoBlobs } from './blobstore.mjs'

const SPEC = 'spec/BLOBS.md, "Moving them"'
/// The blob steps of each phase, unless --steps says.
const PHASE_STEPS = STEPS_GIVEN ?? 60
const STEPS = [[uploadBlob, 4], [blobRecord, 6], [getBlob, 3], [listBlobs, 2]]

/// The bucket's variables, which the command is given whichever way it
/// moves, and the server only while its blobs are in the bucket.
const BUCKET_ENV = () => ({
  PDS_BLOBSTORE_S3_BUCKET: BUCKETS.server,
  PDS_BLOBSTORE_S3_ENDPOINT: s3.url,
  PDS_BLOBSTORE_S3_FORCE_PATH_STYLE: 'true',
  PDS_BLOBSTORE_S3_ACCESS_KEY_ID: S3_KEYS.accessKeyId,
  PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY: S3_KEYS.secretAccessKey,
  PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS: String(S3_TIMEOUT_MS),
  PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS: String(S3_TIMEOUT_MS),
})

/// The server's environment for blobs kept `where`.
function serveEnv(where) {
  for (const k of Object.keys(BUCKET_ENV())) delete server.env[k]
  if (where === 'bucket') Object.assign(server.env, BUCKET_ENV())
  return { ...process.env, ...server.env }
}

/// `pds <args>` with `env`, killed after `killAfter` ms if given, counted
/// from when `armed()` first holds (polled every few ms; at once without
/// it); and with `stopOnStart`, shut down (SIGTERM) 5 s after it logs its
/// `start` line, once what a start after a kill finds is durable
/// (spec/STORE.md, "Scale"), so that its data directory is left as a
/// shutdown leaves it: its exit, standard output and standard error.
function pds(args, env, killAfter, armed = () => true, { stopOnStart = false } = {}) {
  return new Promise((ok) => {
    const p = spawn(command[0], args, { env, stdio: ['ignore', 'pipe', 'pipe'] })
    const out = []
    const err = []
    let stopping = null
    p.stdout.on('data', (d) => {
      out.push(d)
      if (stopOnStart && !stopping && /"msg":"start"/.test(Buffer.concat(out).toString('utf8'))) stopping = setTimeout(() => p.kill('SIGTERM'), 5000)
    })
    p.stderr.on('data', (d) => err.push(d))
    let timer = null
    const poll = killAfter === undefined ? null : setInterval(() => {
      if (!armed()) return
      clearInterval(poll)
      timer = setTimeout(() => p.kill('SIGKILL'), killAfter)
    }, 2)
    p.on('exit', (code, signal) => {
      clearInterval(poll)
      clearTimeout(timer)
      clearTimeout(stopping)
      ok({ code, signal, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') })
    })
  })
}

const migrateEnv = () => ({ ...process.env, ...server.env, ...BUCKET_ENV() })

/// The blobs the harness knows the accounts hold, and their bytes.
function held() {
  let n = 0
  let bytes = 0
  for (const acct of accounts) for (const s of acct.blobs?.stored.values() ?? []) (n += 1), (bytes += s.size)
  return { moved: n, bytes }
}

/// A `pds serve` on the data directory, apart from the run's server: it
/// must exit 1, with why on standard error, before it serves. With
/// `servesIf`, a serve that starts, shut down at once, is allowed where
/// `servesIf()` then holds, and answers false.
async function serveRefused(where, why, spec = SPEC, servesIf) {
  const port = await freePort()
  // Its log apart from the run's server's, which `_logs` is held to: one
  // that serves writes lines there.
  const env = { ...serveEnv(where), PDS_PORT: String(port), PDS_METRICS_PORT: '', PDS_LOG_DIR: '' }
  const r = await pds(['serve'], env, 20_000, undefined, { stopOnStart: true })
  serveEnv(where)
  const refused = r.code === 1 && r.stderr.trim() && !r.stdout.trim()
  const started = /"msg":"start"/.test(r.stdout) && r.code === 0
  if (!refused && started && servesIf && (await servesIf())) return false
  if (!refused) await differ(`pds serve ${why}`, { blobs: where }, 'exit 1, with why on standard error and nothing on standard output', { code: r.code, signal: r.signal, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 300) }, spec)
  count(`pds serve ${why}, refused`, false)
  return true
}

/// `pds migrate-blobs --to <to>`, killed now and then, run again until it
/// finishes. The store answers each request 100 ms late, and a kill lands
/// at a random moment within 600 ms of its first request of the run: after
/// the move has marked the data directory (spec/BLOBS.md, "Moving them", 4
/// before 5), while it copies, switches or cleans up; or, rarely, after it
/// marked the move finished and before it exited (7). Such a kill leaves
/// the move finished: the data directory is served from where the blobs
/// went, and the next run moves nothing. With `back`, the first time it is
/// killed the move is turned back: moved where the blobs were, served from
/// there, then moved on.
async function moveTo(to, { kills = 3, back } = {}) {
  let killed = 0
  let lastKilled = false
  for (let attempt = 0; ; attempt++) {
    if (attempt > 40) throw new Error(`pds migrate-blobs --to ${to} did not finish in 40 runs`)
    const kill = killed < kills && chance(0.8) ? below(600) : undefined
    s3.fail({ bucket: BUCKETS.server, mode: { delayMs: 100 } })
    const from = s3.requests.length
    const r = await pds(['migrate-blobs', '--to', to], migrateEnv(), kill, () => s3.requests.slice(from).some((q) => q.bucket === BUCKETS.server))
    s3.clearFailures()
    if (r.signal === 'SIGKILL') {
      killed += 1
      lastKilled = true
      // Unfinished, the data directory is not served, whichever way; but a
      // kill after the move was marked finished leaves it finished.
      const where = killed === 1 ? pick(['bucket', 'disk']) : null
      const finished = async () => where === to && (await nothingMoved(to))
      if (where && !(await serveRefused(where, 'on a data directory whose blobs are moving', SPEC, finished))) {
        return count(`pds migrate-blobs --to ${to}, killed once it had finished`, true)
      }
      if (back && killed === 1) {
        await moveTo(back, { kills: 1 })
        await serves(back, `moved back to ${back} after a move to ${to} was killed`)
        await server.stop('SIGTERM', 15_000)
        lastKilled = false
      }
      continue
    }
    if (r.code !== 0) await differ(`pds migrate-blobs --to ${to}`, { attempt: attempt + 1, killed }, 'exit 0', { code: r.code, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 500) }, SPEC)
    let line
    try {
      line = JSON.parse(r.stdout.trim())
    } catch {
      await differ(`pds migrate-blobs --to ${to}`, { attempt: attempt + 1 }, 'one line of JSON on standard output', r.stdout.slice(0, 300), SPEC)
    }
    // The run killed before this one had finished, between marking the move
    // finished and exiting: this one finds nothing to move.
    if (lastKilled && isDeepStrictEqual(line, { moved: 0, bytes: 0 })) return count(`pds migrate-blobs --to ${to}, killed once it had finished`, true)
    if (!isDeepStrictEqual(line, held())) await differ(`pds migrate-blobs --to ${to}`, { attempt: attempt + 1, killed }, held(), line, SPEC)
    return count(`pds migrate-blobs --to ${to}, killed ${killed} times first`, true)
  }
}

/// Whether `pds migrate-blobs --to <where>` moves nothing: the data
/// directory keeps its blobs there, with no move unfinished.
async function nothingMoved(where) {
  const r = await pds(['migrate-blobs', '--to', where], migrateEnv(), 60_000)
  return r.code === 0 && r.stdout.trim() === '{"moved":0,"bytes":0}'
}

/// `pds migrate-blobs --to <where>` on a data directory that keeps its
/// blobs there already: nothing moved.
async function nothingToMove(where) {
  const r = await pds(['migrate-blobs', '--to', where], migrateEnv(), 60_000)
  if (r.code !== 0 || r.stdout.trim() !== '{"moved":0,"bytes":0}') await differ(`pds migrate-blobs --to ${where}, its blobs there already`, {}, { code: 0, stdout: '{"moved":0,"bytes":0}' }, { code: r.code, signal: r.signal, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 300) }, SPEC)
  count(`pds migrate-blobs --to ${where}, its blobs there already`, true)
}

/// The server started with its blobs `where`, and every account's blobs
/// read as the reference's: every blob a record references, listBlobs, and
/// a record referencing an upload no record had (checks/blobs.mjs).
async function serves(where, when) {
  serveEnv(where)
  server.start()
  await waitReady(server, cand.url)
  for (const acct of accounts) for (const cid of referencedCids(model(acct))) await compareGetBlob(`getBlob, ${when}`, (side) => ({ did: didOf(side, acct), cid }))
  await blobsSurvive()
  if (where === 'bucket') {
    await bucketHolds(when)
    await dataDirHoldsNoBlobs(when)
  } else {
    const left = [...s3.objects(BUCKETS.server).keys()].filter((k) => k.startsWith('blocks/'))
    if (left.length) await differ(`the bucket, ${when}`, { bucket: BUCKETS.server }, 'no object of the accounts\' blobs', left.slice(0, 10), SPEC)
    count(`the bucket holds no blob, ${when}`, true)
  }
}

/// Random blob steps, compared as in any run.
async function blobSteps(n) {
  const total = STEPS.reduce((s, [, w]) => s + w, 0)
  for (let i = 0; i < n; i++, run.step++) {
    let r = below(total)
    for (const [fn, w] of STEPS) if ((r -= w) < 0) {
      await fn(pick(accounts))
      break
    }
  }
}

/// The data directory held by the running server: a second `pds serve` on
/// it, and `pds migrate-blobs`, are refused.
async function heldByServer(where) {
  await serveRefused(where, 'beside a running server on its data directory', 'spec/COMMANDS.md, "One binary"')
  const r = await pds(['migrate-blobs', '--to', where === 'bucket' ? 'disk' : 'bucket'], migrateEnv(), 20_000)
  if (r.code !== 1 || !r.stderr.trim()) await differ('pds migrate-blobs beside a running server on its data directory', {}, 'exit 1, with why on standard error', { code: r.code, signal: r.signal, stdout: r.stdout.slice(0, 300), stderr: r.stderr.slice(0, 300) }, 'spec/COMMANDS.md, "One binary"')
  count('pds migrate-blobs beside a running server, refused', false)
}

/// Arguments that do not parse: usage errors, exit 2, with nothing moved.
async function usage() {
  for (const args of [['migrate-blobs'], ['migrate-blobs', '--to', 'cloud']]) {
    const r = await pds(args, migrateEnv(), 20_000)
    if (r.code !== 2) await differ(`pds ${args.join(' ')}`, {}, 'a usage error: exit 2', { code: r.code, stderr: r.stderr.slice(0, 300) }, SPEC)
    count(`pds ${args.join(' ')}, a usage error`, false)
  }
}

/// A move into the data directory stopped by a blob whose object is gone:
/// exit 1 naming it, the data directory not served either way; then the
/// object back, and the move run again to its end.
async function stoppedByObjectGone() {
  const holders = accounts.filter((a) => a.blobs?.stored.size)
  if (!holders.length) return
  const acct = pick(holders)
  const cid = pick([...acct.blobs.stored.keys()])
  const key = `blocks/${didOf(cand, acct)}/${cid}`
  const saved = s3.objects(BUCKETS.server).get(key)
  if (!saved) return differ('the server\'s bucket before a move', { cid }, `an object ${key}`, 'none', SPEC)
  s3.remove(BUCKETS.server, key)
  const r = await pds(['migrate-blobs', '--to', 'disk'], migrateEnv(), 60_000)
  if (r.code !== 1 || !r.stderr.includes(cid) || !r.stderr.includes(didOf(cand, acct))) await differ('pds migrate-blobs --to disk, a blob\'s object gone', { cid }, 'exit 1, naming the DID and CID on standard error', { code: r.code, signal: r.signal, stderr: r.stderr.slice(0, 500) }, SPEC)
  count('pds migrate-blobs, stopped by an object gone', false)
  await serveRefused('disk', 'after a move stopped by an object gone')
  await serveRefused('bucket', 'after a move stopped by an object gone')
  s3.put(BUCKETS.server, key, saved.bytes, saved.contentType)
}

/// The run: blobs on disk; moved into the bucket, the first move turned
/// back halfway; blobs in the bucket; a move out stopped by an object gone,
/// then moved out; blobs on disk again. Each place served between moves.
export async function migrateRun() {
  await createFirstAccounts()
  await blobSteps(PHASE_STEPS)
  await heldByServer('disk')
  await server.stop('SIGTERM', 15_000)
  await usage()
  await nothingToMove('disk')
  await moveTo('bucket', { back: 'disk' })
  await nothingToMove('bucket')
  await serves('bucket', 'moved into the bucket')
  await blobSteps(PHASE_STEPS)
  await heldByServer('bucket')
  await server.stop('SIGTERM', 15_000)
  await stoppedByObjectGone()
  await moveTo('disk')
  await serves('disk', 'moved into the data directory')
  await blobSteps(PHASE_STEPS / 2)
}