// The tables of RESULTS.md, from a run's files (README.md, "Results"): // // node bench/soak/analyze.mjs [--split ]... [--from ] [--to ] // // The run's directory holds what soak.mjs wrote (summary-*.json, events-*.jsonl) // and, where they were kept, a sampler's samples.jsonl and kills.jsonl, and // Monolith's request lines as server-log.jsonl (its log, PDS_LOG_DIR, cut to // them); each may be gzipped. --split cuts the time into periods, as before // and after a change; --from and --to bound everything read by time. // Markdown on stdout: only the sections the files allow. // // A percentile q of n values sorted is the value at floor(q × n), from 0. import { existsSync, readFileSync, readdirSync } from 'node:fs' import { join } from 'node:path' import { gunzipSync } from 'node:zlib' const args = process.argv.slice(2) const dir = args.find((a, i) => !a.startsWith('--') && !args[i - 1]?.startsWith('--')) if (!dir) { console.error('usage: node bench/soak/analyze.mjs [--split ]... [--from ] [--to ]') process.exit(2) } const opt = (name) => args.flatMap((a, i) => (a === name ? [args[i + 1]] : [])) const FROM = opt('--from')[0] ?? '' const TO = opt('--to')[0] ?? '9' const SPLITS = opt('--split').sort() const inRange = (t) => t >= FROM && t < TO // ---------------------------------------------------------------- reading /// A file of the run, gzipped or not: its text, or null. function text(name) { for (const [f, gz] of [[name, false], [`${name}.gz`, true]]) { const p = join(dir, f) if (existsSync(p)) return gz ? gunzipSync(readFileSync(p)).toString('utf8') : readFileSync(p, 'utf8') } return null } const lines = (name) => (text(name) ?? '').split('\n').filter(Boolean).map((l) => JSON.parse(l)) const files = (re) => readdirSync(dir).filter((f) => re.test(f)) const pct = (sorted, q) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] : null) const num = (x, d = 1) => (x == null ? '–' : Number(x).toFixed(d)) const int = (x) => (x == null ? '–' : Math.round(x).toLocaleString('en')) const mib = (kb) => (kb == null ? '–' : Math.floor(kb / 1024).toLocaleString('en')) const median = (xs) => pct([...xs].sort((a, b) => a - b), 0.5) function corr(xs, ys) { const n = xs.length if (n < 3) return null const mx = xs.reduce((a, b) => a + b, 0) / n, my = ys.reduce((a, b) => a + b, 0) / n let sxy = 0, sxx = 0, syy = 0 for (let i = 0; i < n; i++) { sxy += (xs[i] - mx) * (ys[i] - my); sxx += (xs[i] - mx) ** 2; syy += (ys[i] - my) ** 2 } return sxx && syy ? sxy / Math.sqrt(sxx * syy) : null } /// The periods --split makes of [FROM, TO): each [from, to). const periods = () => { const cuts = [FROM || '0', ...SPLITS, TO] return cuts.slice(0, -1).map((from, i) => [from, cuts[i + 1]]) } const label = ([a, b]) => `${a === '0' || a === '' ? 'start' : a.slice(11, 16)}–${b === '9' ? 'end' : b.slice(11, 16)}` const out = [] const table = (head, rows) => { out.push(`| ${head.join(' | ')} |`, `| ${head.map((_, i) => (i ? '--:' : '---')).join(' | ')} |`) for (const r of rows) out.push(`| ${r.join(' | ')} |`) out.push('') } out.push(`# ${dir}`, '') // ---------------------------------------------------------------- the soak's totals const summaries = files(/^summary-.*\.json$/).map((f) => JSON.parse(readFileSync(join(dir, f), 'utf8'))) if (summaries.length) { out.push('## Totals (summary-*.json)', '') const t = (s) => s.totals table(['', ...summaries.map((s) => s.side)], [ ['from, to', ...summaries.map((s) => `${s.started?.slice(11, 16)}–${s.updated?.slice(11, 16)}`)], ['accounts, records', ...summaries.map((s) => `${s.accounts}, ${int(s.records)}`)], ['operations', ...summaries.map((s) => int(t(s).ops))], ['5xx', ...summaries.map((s) => int(t(s).e5xx))], ['rate limited (429)', ...summaries.map((s) => int(t(s).limited))], ['lost', ...summaries.map((s) => int(t(s).lost))], ['mismatches', ...summaries.map((s) => int(t(s).mismatches))], ['commits not on the firehose', ...summaries.map((s) => int(t(s).missingCommits))], ['outages, their time', ...summaries.map((s) => `${t(s).outages}, ${num((t(s).outageMs ?? 0) / 1000)} s`)], ]) } // ---------------------------------------------------------------- the soak's windows and verdicts const eventsBySide = Object.fromEntries(files(/^events-.*\.jsonl(\.gz)?$/).map((f) => [f.replace(/^events-|\.jsonl(\.gz)?$/g, ''), lines(f.replace(/\.gz$/, ''))])) const sides = Object.keys(eventsBySide) if (sides.length) { const windows = (s) => eventsBySide[s].filter((e) => e.kind === 'window' && inRange(e.from)) const of = (s, op, q) => windows(s).map((w) => w.ops?.[op]?.[q]).filter((x) => x != null) out.push("## The soak's times, by five-minute window (events-*.jsonl)", '', 'Each window has its own p50 and p99 per operation; the median of them, and the worst p99. Times are the soak\'s, from request to answer.', '') const ops = ['post', 'like', 'applyWrites', 'delete', 'getRecord', 'listRecords', 'getLatestCommit', 'getRepo'] table(['operation', ...sides.flatMap((s) => [`${s} p50`, `${s} p99`, `${s} worst p99`])], ops.map((op) => [op, ...sides.flatMap((s) => [num(median(of(s, op, 'p50'))), num(median(of(s, op, 'p99'))), num(Math.max(...of(s, op, 'p99')) === -Infinity ? null : Math.max(...of(s, op, 'p99')))])])) const kinds = (s) => { const c = {} for (const e of eventsBySide[s]) if (e.kind !== 'window' && inRange(e.time)) c[e.kind] = (c[e.kind] || 0) + 1 return c } out.push('### Events by kind', '') const all = [...new Set(sides.flatMap((s) => Object.keys(kinds(s))))].sort() table(['kind', ...sides], all.map((k) => [k, ...sides.map((s) => int(kinds(s)[k] ?? 0))])) const settled = sides.flatMap((s) => eventsBySide[s].filter((e) => e.kind === 'settled')) if (settled.length) { out.push('### The settled check', '') table(['', ...settled.map((v) => `${v.side}, ${v.time.slice(11, 16)}`)], [ ['live state: repositories + blobs', ...settled.map((v) => `${v.liveMiB} MiB (${v.reposMiB} + ${v.blobsMiB})`)], ['when the load stopped', ...settled.map((v) => (v.loadEndMiB == null ? '–' : `${v.loadEndMiB} MiB`))], ['after 2 W with no write', ...settled.map((v) => `${v.beforeMiB} MiB`)], ['at rest, after a write to each', ...settled.map((v) => `${v.leastMiB} MiB, ${(v.leastMiB / v.liveMiB).toFixed(2)} × live`)], ['the settled bound', ...settled.map((v) => `${v.boundMiB} MiB: ${v.pass ? 'within' : 'over'}`)], ]) } } // ---------------------------------------------------------------- kills const kills = lines('kills.jsonl').filter((k) => k.killed && inRange(k.time)) if (kills.length) { out.push('## Kills (kills.jsonl)', '') const by = {} for (const k of kills) (by[k.killed] ??= []).push(k.restarted ?? k.started) table(['container', 'kills', 'serving again after'], Object.entries(by).map(([c, ts]) => [c, ts.length, `${Math.min(...ts)}–${Math.max(...ts)} s`])) } // ---------------------------------------------------------------- samples const samples = lines('samples.jsonl').filter((s) => inRange(s.time)) if (samples.length) { const containers = Object.keys(samples[0]).filter((k) => samples[0][k]?.rssKb !== undefined) out.push('## Memory and data over time (samples.jsonl), MiB', '') const v = (c, k) => samples.map((s) => s[c]?.[k]).filter((x) => x != null && x > 0) table(['container', 'RSS first, most, last', 'data first, most, last'], containers.map((c) => [c, `${mib(v(c, 'rssKb')[0])}, ${mib(Math.max(...v(c, 'rssKb')))}, ${mib(v(c, 'rssKb').at(-1))}`, `${mib(v(c, 'dataKb')[0])}, ${mib(Math.max(...v(c, 'dataKb')))}, ${mib(v(c, 'dataKb').at(-1))}`])) const hours = {} for (const s of samples) hours[s.time.slice(0, 13)] = s table(['hour (last sample)', ...containers.flatMap((c) => [`${c} RSS`, `${c} data`]), 'free GiB'], Object.values(hours).map((s) => [s.time.slice(11, 16), ...containers.flatMap((c) => [mib(s[c]?.rssKb), mib(s[c]?.dataKb)]), Math.floor(s.freeKb / 1048576)])) const synced = samples.filter((s) => s.specFsync && s.specFsync['+Inf'] != null) if (synced.length > 1) { out.push("## Monolith's syncs (samples.jsonl: its fsync histogram, the disk's counters)", '') // Counts between consecutive samples of the same start (a start sets the histogram to 0). const steps = [] for (let i = 1; i < synced.length; i++) { const a = synced[i - 1], b = synced[i] const d = (le) => b.specFsync[le] - a.specFsync[le] if (d('+Inf') < 0) continue steps.push({ time: b.time, syncs: d('+Inf'), over25: d('+Inf') - d('0.025'), over100: d('+Inf') - d('0.1'), flushMs: b.disk && a.disk ? b.disk.flushMs - a.disk.flushMs : null }) } table(['period', 'syncs', 'over 25 ms', 'over 100 ms', 'slow syncs ~ disk flush time (r, a sample apart)'], periods().map((p) => { const ss = steps.filter((s) => s.time >= p[0] && s.time < p[1]) const withFlush = ss.filter((s) => s.flushMs != null) const r = corr(withFlush.map((s) => s.flushMs), withFlush.map((s) => s.over100)) return [label(p), int(ss.reduce((n, s) => n + s.syncs, 0)), int(ss.reduce((n, s) => n + s.over25, 0)), int(ss.reduce((n, s) => n + s.over100, 0)), num(r, 2)] })) } } // ---------------------------------------------------------------- Monolith's log const log = lines('server-log.jsonl').filter((l) => inRange(l.time)) if (log.length) { const reqs = log.filter((l) => l.msg === 'request') const starts = log.filter((l) => l.msg === 'start').map((l) => Date.parse(l.time)) const AFTER_START_MS = 60_000 const afterStart = (t) => starts.some((s) => t >= s && t < s + AFTER_START_MS) const WRITES = /\.(createRecord|deleteRecord|applyWrites|putRecord)$/ const READS = /\.(getRecord|listRecords|getLatestCommit|describeServer)$/ const stats = (ms) => { const s = [...ms].sort((a, b) => a - b) return [int(s.length), num(pct(s, 0.5)), num(pct(s, 0.99)), num(pct(s, 0.999)), num(s.at(-1))] } out.push("## Monolith's requests, server side (server-log.jsonl)", '', `Its \`request\` lines' \`ms\`. Starts in the log: ${starts.length}.`, '') const by = {} for (const r of reqs) (by[r.method] ??= []).push(r.ms) table(['method', 'n', 'p50 ms', 'p99 ms', 'p99.9 ms', 'worst ms'], Object.entries(by).filter(([, v]) => v.length >= 500).sort((a, b) => b[1].length - a[1].length).map(([m, v]) => [m.replace('com.atproto.', ''), ...stats(v)])) out.push(`### createRecord by period, and the ${AFTER_START_MS / 1000} s after each start apart`, '') const cr = reqs.filter((r) => r.method === 'com.atproto.repo.createRecord') const rows = periods().flatMap((p) => { const inP = cr.filter((r) => r.time >= p[0] && r.time < p[1]) const steady = inP.filter((r) => !afterStart(Date.parse(r.time))) const after = inP.filter((r) => afterStart(Date.parse(r.time))) return [[`${label(p)}, but after a start`, ...stats(steady.map((r) => r.ms))], ...(after.length ? [[`${label(p)}, after a start`, ...stats(after.map((r) => r.ms))]] : [])] }) table(['', 'n', 'p50 ms', 'p99 ms', 'p99.9 ms', 'worst ms'], rows) // What a slow write held up: the requests under way while it was. out.push('### What a slow write (over 200 ms) held up', '', 'Requests under way during each slow write, but it, slow being over 100 ms.', '') const span = reqs.map((r) => ({ end: Date.parse(r.time), start: Date.parse(r.time) - r.ms, ms: r.ms, method: r.method, did: r.did })).sort((a, b) => a.start - b.start) const starts0 = span.map((s) => s.start) const lower = (x) => { let lo = 0, hi = starts0.length; while (lo < hi) { const m = (lo + hi) >> 1; if (starts0[m] < x) lo = m + 1; else hi = m } return lo } const c = { other: [0, 0], same: [0, 0], reads: [0, 0] } let slowN = 0 for (const w of span) { if (w.ms <= 200 || !WRITES.test(w.method)) continue slowN++ for (let i = lower(w.start - 5000); i < span.length && span[i].start < w.end; i++) { const o = span[i] if (o === w || o.end <= w.start) continue const k = o.ms > 100 ? 1 : 0 if (WRITES.test(o.method)) c[o.did === w.did ? 'same' : 'other'][k]++ else if (READS.test(o.method)) c.reads[k]++ } } table(['under way', 'fast', 'slow'], [['writes to other accounts', int(c.other[0]), int(c.other[1])], ['writes to the same account', int(c.same[0]), int(c.same[1])], ['reads', int(c.reads[0]), int(c.reads[1])]]) out.push(`Slow writes: ${int(slowN)}.`, '') if (samples.length) { // Slow writes against the disk's flush time and Monolith's slow syncs, a sample apart. const steps = [] for (let i = 1; i < samples.length; i++) { const a = samples[i - 1], b = samples[i] if (!a.disk || !b.disk) continue const slow = reqs.filter((r) => r.time > a.time && r.time <= b.time && WRITES.test(r.method) && r.ms > 200).length const syncs = a.specFsync?.['+Inf'] != null && b.specFsync?.['+Inf'] != null && b.specFsync['+Inf'] >= a.specFsync['+Inf'] ? (b.specFsync['+Inf'] - b.specFsync['0.1']) - (a.specFsync['+Inf'] - a.specFsync['0.1']) : null steps.push({ time: b.time, slow, flushMs: b.disk.flushMs - a.disk.flushMs, syncs }) } if (steps.length > 2) { out.push('### Slow writes against the disk, a sample apart', '') table(['period', 'samples', 'r: slow writes ~ disk flush time', 'r: slow syncs ~ disk flush time'], periods().map((p) => { const ss = steps.filter((s) => s.time >= p[0] && s.time < p[1]) const sy = ss.filter((s) => s.syncs != null) return [label(p), ss.length, num(corr(ss.map((s) => s.flushMs), ss.map((s) => s.slow)), 2), num(corr(sy.map((s) => s.flushMs), sy.map((s) => s.syncs)), 2)] })) } } } console.log(out.join('\n'))