~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
234 lines · 13.5 KB Raw History
// The tables of RESULTS.md, from a run's files (README.md, "Results"):
//
//   node bench/soak/analyze.mjs <run dir> [--split <ISO time>]... [--from <ISO>] [--to <ISO>]
//
// The run's directory holds what soak.mjs wrote (summary-*.json, events-*.jsonl)
// and, where they were kept, a sampler's samples.jsonl and kills.jsonl, and
// Monolith's request lines as server-log.jsonl (its log, PDS_LOG_DIR, cut to
// them); each may be gzipped. --split cuts the time into periods, as before
// and after a change; --from and --to bound everything read by time.
// Markdown on stdout: only the sections the files allow.
//
// A percentile q of n values sorted is the value at floor(q × n), from 0.

import { existsSync, readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import { gunzipSync } from 'node:zlib'

const args = process.argv.slice(2)
const dir = args.find((a, i) => !a.startsWith('--') && !args[i - 1]?.startsWith('--'))
if (!dir) {
  console.error('usage: node bench/soak/analyze.mjs <run dir> [--split <ISO>]... [--from <ISO>] [--to <ISO>]')
  process.exit(2)
}
const opt = (name) => args.flatMap((a, i) => (a === name ? [args[i + 1]] : []))
const FROM = opt('--from')[0] ?? ''
const TO = opt('--to')[0] ?? '9'
const SPLITS = opt('--split').sort()
const inRange = (t) => t >= FROM && t < TO

// ---------------------------------------------------------------- reading

/// A file of the run, gzipped or not: its text, or null.
function text(name) {
  for (const [f, gz] of [[name, false], [`${name}.gz`, true]]) {
    const p = join(dir, f)
    if (existsSync(p)) return gz ? gunzipSync(readFileSync(p)).toString('utf8') : readFileSync(p, 'utf8')
  }
  return null
}
const lines = (name) => (text(name) ?? '').split('\n').filter(Boolean).map((l) => JSON.parse(l))
const files = (re) => readdirSync(dir).filter((f) => re.test(f))

const pct = (sorted, q) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] : null)
const num = (x, d = 1) => (x == null ? '–' : Number(x).toFixed(d))
const int = (x) => (x == null ? '–' : Math.round(x).toLocaleString('en'))
const mib = (kb) => (kb == null ? '–' : Math.floor(kb / 1024).toLocaleString('en'))
const median = (xs) => pct([...xs].sort((a, b) => a - b), 0.5)
function corr(xs, ys) {
  const n = xs.length
  if (n < 3) return null
  const mx = xs.reduce((a, b) => a + b, 0) / n, my = ys.reduce((a, b) => a + b, 0) / n
  let sxy = 0, sxx = 0, syy = 0
  for (let i = 0; i < n; i++) { sxy += (xs[i] - mx) * (ys[i] - my); sxx += (xs[i] - mx) ** 2; syy += (ys[i] - my) ** 2 }
  return sxx && syy ? sxy / Math.sqrt(sxx * syy) : null
}
/// The periods --split makes of [FROM, TO): each [from, to).
const periods = () => {
  const cuts = [FROM || '0', ...SPLITS, TO]
  return cuts.slice(0, -1).map((from, i) => [from, cuts[i + 1]])
}
const label = ([a, b]) => `${a === '0' || a === '' ? 'start' : a.slice(11, 16)}–${b === '9' ? 'end' : b.slice(11, 16)}`
const out = []
const table = (head, rows) => {
  out.push(`| ${head.join(' | ')} |`, `| ${head.map((_, i) => (i ? '--:' : '---')).join(' | ')} |`)
  for (const r of rows) out.push(`| ${r.join(' | ')} |`)
  out.push('')
}

out.push(`# ${dir}`, '')

// ---------------------------------------------------------------- the soak's totals

const summaries = files(/^summary-.*\.json$/).map((f) => JSON.parse(readFileSync(join(dir, f), 'utf8')))
if (summaries.length) {
  out.push('## Totals (summary-*.json)', '')
  const t = (s) => s.totals
  table(['', ...summaries.map((s) => s.side)], [
    ['from, to', ...summaries.map((s) => `${s.started?.slice(11, 16)}–${s.updated?.slice(11, 16)}`)],
    ['accounts, records', ...summaries.map((s) => `${s.accounts}, ${int(s.records)}`)],
    ['operations', ...summaries.map((s) => int(t(s).ops))],
    ['5xx', ...summaries.map((s) => int(t(s).e5xx))],
    ['rate limited (429)', ...summaries.map((s) => int(t(s).limited))],
    ['lost', ...summaries.map((s) => int(t(s).lost))],
    ['mismatches', ...summaries.map((s) => int(t(s).mismatches))],
    ['commits not on the firehose', ...summaries.map((s) => int(t(s).missingCommits))],
    ['outages, their time', ...summaries.map((s) => `${t(s).outages}, ${num((t(s).outageMs ?? 0) / 1000)} s`)],
  ])
}

// ---------------------------------------------------------------- the soak's windows and verdicts

const eventsBySide = Object.fromEntries(files(/^events-.*\.jsonl(\.gz)?$/).map((f) => [f.replace(/^events-|\.jsonl(\.gz)?$/g, ''), lines(f.replace(/\.gz$/, ''))]))
const sides = Object.keys(eventsBySide)
if (sides.length) {
  const windows = (s) => eventsBySide[s].filter((e) => e.kind === 'window' && inRange(e.from))
  const of = (s, op, q) => windows(s).map((w) => w.ops?.[op]?.[q]).filter((x) => x != null)
  out.push("## The soak's times, by five-minute window (events-*.jsonl)", '', 'Each window has its own p50 and p99 per operation; the median of them, and the worst p99. Times are the soak\'s, from request to answer.', '')
  const ops = ['post', 'like', 'applyWrites', 'delete', 'getRecord', 'listRecords', 'getLatestCommit', 'getRepo']
  table(['operation', ...sides.flatMap((s) => [`${s} p50`, `${s} p99`, `${s} worst p99`])], ops.map((op) => [op, ...sides.flatMap((s) => [num(median(of(s, op, 'p50'))), num(median(of(s, op, 'p99'))), num(Math.max(...of(s, op, 'p99')) === -Infinity ? null : Math.max(...of(s, op, 'p99')))])]))
  const kinds = (s) => {
    const c = {}
    for (const e of eventsBySide[s]) if (e.kind !== 'window' && inRange(e.time)) c[e.kind] = (c[e.kind] || 0) + 1
    return c
  }
  out.push('### Events by kind', '')
  const all = [...new Set(sides.flatMap((s) => Object.keys(kinds(s))))].sort()
  table(['kind', ...sides], all.map((k) => [k, ...sides.map((s) => int(kinds(s)[k] ?? 0))]))
  const settled = sides.flatMap((s) => eventsBySide[s].filter((e) => e.kind === 'settled'))
  if (settled.length) {
    out.push('### The settled check', '')
    table(['', ...settled.map((v) => `${v.side}, ${v.time.slice(11, 16)}`)], [
      ['live state: repositories + blobs', ...settled.map((v) => `${v.liveMiB} MiB (${v.reposMiB} + ${v.blobsMiB})`)],
      ['when the load stopped', ...settled.map((v) => (v.loadEndMiB == null ? '–' : `${v.loadEndMiB} MiB`))],
      ['after 2 W with no write', ...settled.map((v) => `${v.beforeMiB} MiB`)],
      ['at rest, after a write to each', ...settled.map((v) => `${v.leastMiB} MiB, ${(v.leastMiB / v.liveMiB).toFixed(2)} × live`)],
      ['the settled bound', ...settled.map((v) => `${v.boundMiB} MiB: ${v.pass ? 'within' : 'over'}`)],
    ])
  }
}

// ---------------------------------------------------------------- kills

const kills = lines('kills.jsonl').filter((k) => k.killed && inRange(k.time))
if (kills.length) {
  out.push('## Kills (kills.jsonl)', '')
  const by = {}
  for (const k of kills) (by[k.killed] ??= []).push(k.restarted ?? k.started)
  table(['container', 'kills', 'serving again after'], Object.entries(by).map(([c, ts]) => [c, ts.length, `${Math.min(...ts)}–${Math.max(...ts)} s`]))
}

// ---------------------------------------------------------------- samples

const samples = lines('samples.jsonl').filter((s) => inRange(s.time))
if (samples.length) {
  const containers = Object.keys(samples[0]).filter((k) => samples[0][k]?.rssKb !== undefined)
  out.push('## Memory and data over time (samples.jsonl), MiB', '')
  const v = (c, k) => samples.map((s) => s[c]?.[k]).filter((x) => x != null && x > 0)
  table(['container', 'RSS first, most, last', 'data first, most, last'], containers.map((c) => [c, `${mib(v(c, 'rssKb')[0])}, ${mib(Math.max(...v(c, 'rssKb')))}, ${mib(v(c, 'rssKb').at(-1))}`, `${mib(v(c, 'dataKb')[0])}, ${mib(Math.max(...v(c, 'dataKb')))}, ${mib(v(c, 'dataKb').at(-1))}`]))
  const hours = {}
  for (const s of samples) hours[s.time.slice(0, 13)] = s
  table(['hour (last sample)', ...containers.flatMap((c) => [`${c} RSS`, `${c} data`]), 'free GiB'], Object.values(hours).map((s) => [s.time.slice(11, 16), ...containers.flatMap((c) => [mib(s[c]?.rssKb), mib(s[c]?.dataKb)]), Math.floor(s.freeKb / 1048576)]))

  const synced = samples.filter((s) => s.specFsync && s.specFsync['+Inf'] != null)
  if (synced.length > 1) {
    out.push("## Monolith's syncs (samples.jsonl: its fsync histogram, the disk's counters)", '')
    // Counts between consecutive samples of the same start (a start sets the histogram to 0).
    const steps = []
    for (let i = 1; i < synced.length; i++) {
      const a = synced[i - 1], b = synced[i]
      const d = (le) => b.specFsync[le] - a.specFsync[le]
      if (d('+Inf') < 0) continue
      steps.push({ time: b.time, syncs: d('+Inf'), over25: d('+Inf') - d('0.025'), over100: d('+Inf') - d('0.1'), flushMs: b.disk && a.disk ? b.disk.flushMs - a.disk.flushMs : null })
    }
    table(['period', 'syncs', 'over 25 ms', 'over 100 ms', 'slow syncs ~ disk flush time (r, a sample apart)'], periods().map((p) => {
      const ss = steps.filter((s) => s.time >= p[0] && s.time < p[1])
      const withFlush = ss.filter((s) => s.flushMs != null)
      const r = corr(withFlush.map((s) => s.flushMs), withFlush.map((s) => s.over100))
      return [label(p), int(ss.reduce((n, s) => n + s.syncs, 0)), int(ss.reduce((n, s) => n + s.over25, 0)), int(ss.reduce((n, s) => n + s.over100, 0)), num(r, 2)]
    }))
  }
}

// ---------------------------------------------------------------- Monolith's log

const log = lines('server-log.jsonl').filter((l) => inRange(l.time))
if (log.length) {
  const reqs = log.filter((l) => l.msg === 'request')
  const starts = log.filter((l) => l.msg === 'start').map((l) => Date.parse(l.time))
  const AFTER_START_MS = 60_000
  const afterStart = (t) => starts.some((s) => t >= s && t < s + AFTER_START_MS)
  const WRITES = /\.(createRecord|deleteRecord|applyWrites|putRecord)$/
  const READS = /\.(getRecord|listRecords|getLatestCommit|describeServer)$/
  const stats = (ms) => {
    const s = [...ms].sort((a, b) => a - b)
    return [int(s.length), num(pct(s, 0.5)), num(pct(s, 0.99)), num(pct(s, 0.999)), num(s.at(-1))]
  }
  out.push("## Monolith's requests, server side (server-log.jsonl)", '', `Its \`request\` lines' \`ms\`. Starts in the log: ${starts.length}.`, '')
  const by = {}
  for (const r of reqs) (by[r.method] ??= []).push(r.ms)
  table(['method', 'n', 'p50 ms', 'p99 ms', 'p99.9 ms', 'worst ms'], Object.entries(by).filter(([, v]) => v.length >= 500).sort((a, b) => b[1].length - a[1].length).map(([m, v]) => [m.replace('com.atproto.', ''), ...stats(v)]))

  out.push(`### createRecord by period, and the ${AFTER_START_MS / 1000} s after each start apart`, '')
  const cr = reqs.filter((r) => r.method === 'com.atproto.repo.createRecord')
  const rows = periods().flatMap((p) => {
    const inP = cr.filter((r) => r.time >= p[0] && r.time < p[1])
    const steady = inP.filter((r) => !afterStart(Date.parse(r.time)))
    const after = inP.filter((r) => afterStart(Date.parse(r.time)))
    return [[`${label(p)}, but after a start`, ...stats(steady.map((r) => r.ms))], ...(after.length ? [[`${label(p)}, after a start`, ...stats(after.map((r) => r.ms))]] : [])]
  })
  table(['', 'n', 'p50 ms', 'p99 ms', 'p99.9 ms', 'worst ms'], rows)

  // What a slow write held up: the requests under way while it was.
  out.push('### What a slow write (over 200 ms) held up', '', 'Requests under way during each slow write, but it, slow being over 100 ms.', '')
  const span = reqs.map((r) => ({ end: Date.parse(r.time), start: Date.parse(r.time) - r.ms, ms: r.ms, method: r.method, did: r.did })).sort((a, b) => a.start - b.start)
  const starts0 = span.map((s) => s.start)
  const lower = (x) => { let lo = 0, hi = starts0.length; while (lo < hi) { const m = (lo + hi) >> 1; if (starts0[m] < x) lo = m + 1; else hi = m } return lo }
  const c = { other: [0, 0], same: [0, 0], reads: [0, 0] }
  let slowN = 0
  for (const w of span) {
    if (w.ms <= 200 || !WRITES.test(w.method)) continue
    slowN++
    for (let i = lower(w.start - 5000); i < span.length && span[i].start < w.end; i++) {
      const o = span[i]
      if (o === w || o.end <= w.start) continue
      const k = o.ms > 100 ? 1 : 0
      if (WRITES.test(o.method)) c[o.did === w.did ? 'same' : 'other'][k]++
      else if (READS.test(o.method)) c.reads[k]++
    }
  }
  table(['under way', 'fast', 'slow'], [['writes to other accounts', int(c.other[0]), int(c.other[1])], ['writes to the same account', int(c.same[0]), int(c.same[1])], ['reads', int(c.reads[0]), int(c.reads[1])]])
  out.push(`Slow writes: ${int(slowN)}.`, '')

  if (samples.length) {
    // Slow writes against the disk's flush time and Monolith's slow syncs, a sample apart.
    const steps = []
    for (let i = 1; i < samples.length; i++) {
      const a = samples[i - 1], b = samples[i]
      if (!a.disk || !b.disk) continue
      const slow = reqs.filter((r) => r.time > a.time && r.time <= b.time && WRITES.test(r.method) && r.ms > 200).length
      const syncs = a.specFsync?.['+Inf'] != null && b.specFsync?.['+Inf'] != null && b.specFsync['+Inf'] >= a.specFsync['+Inf'] ? (b.specFsync['+Inf'] - b.specFsync['0.1']) - (a.specFsync['+Inf'] - a.specFsync['0.1']) : null
      steps.push({ time: b.time, slow, flushMs: b.disk.flushMs - a.disk.flushMs, syncs })
    }
    if (steps.length > 2) {
      out.push('### Slow writes against the disk, a sample apart', '')
      table(['period', 'samples', 'r: slow writes ~ disk flush time', 'r: slow syncs ~ disk flush time'], periods().map((p) => {
        const ss = steps.filter((s) => s.time >= p[0] && s.time < p[1])
        const sy = ss.filter((s) => s.syncs != null)
        return [label(p), ss.length, num(corr(ss.map((s) => s.flushMs), ss.map((s) => s.slow)), 2), num(corr(sy.map((s) => s.flushMs), sy.map((s) => s.syncs)), 2)]
      }))
    }
  }
}

console.log(out.join('\n'))