chadtmiller.com/monolith
An atproto PDS
0
0
// The tables of RESULTS.md, from a run's files (README.md, "Results"):
//
// node bench/soak/analyze.mjs <run dir> [--split <ISO time>]... [--from <ISO>] [--to <ISO>]
//
// The run's directory holds what soak.mjs wrote (summary-*.json, events-*.jsonl)
// and, where they were kept, a sampler's samples.jsonl and kills.jsonl, and
// Monolith's request lines as server-log.jsonl (its log, PDS_LOG_DIR, cut to
// them); each may be gzipped. --split cuts the time into periods, as before
// and after a change; --from and --to bound everything read by time.
// Markdown on stdout: only the sections the files allow.
//
// A percentile q of n values sorted is the value at floor(q × n), from 0.
import { existsSync, readFileSync, readdirSync } from 'node:fs'
import { join } from 'node:path'
import { gunzipSync } from 'node:zlib'
const args = process.argv.slice(2)
const dir = args.find((a, i) => !a.startsWith('--') && !args[i - 1]?.startsWith('--'))
if (!dir) {
console.error('usage: node bench/soak/analyze.mjs <run dir> [--split <ISO>]... [--from <ISO>] [--to <ISO>]')
process.exit(2)
}
const opt = (name) => args.flatMap((a, i) => (a === name ? [args[i + 1]] : []))
const FROM = opt('--from')[0] ?? ''
const TO = opt('--to')[0] ?? '9'
const SPLITS = opt('--split').sort()
const inRange = (t) => t >= FROM && t < TO
// ---------------------------------------------------------------- reading
/// A file of the run, gzipped or not: its text, or null.
function text(name) {
for (const [f, gz] of [[name, false], [`${name}.gz`, true]]) {
const p = join(dir, f)
if (existsSync(p)) return gz ? gunzipSync(readFileSync(p)).toString('utf8') : readFileSync(p, 'utf8')
}
return null
}
const lines = (name) => (text(name) ?? '').split('\n').filter(Boolean).map((l) => JSON.parse(l))
const files = (re) => readdirSync(dir).filter((f) => re.test(f))
const pct = (sorted, q) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] : null)
const num = (x, d = 1) => (x == null ? '–' : Number(x).toFixed(d))
const int = (x) => (x == null ? '–' : Math.round(x).toLocaleString('en'))
const mib = (kb) => (kb == null ? '–' : Math.floor(kb / 1024).toLocaleString('en'))
const median = (xs) => pct([...xs].sort((a, b) => a - b), 0.5)
function corr(xs, ys) {
const n = xs.length
if (n < 3) return null
const mx = xs.reduce((a, b) => a + b, 0) / n, my = ys.reduce((a, b) => a + b, 0) / n
let sxy = 0, sxx = 0, syy = 0
for (let i = 0; i < n; i++) { sxy += (xs[i] - mx) * (ys[i] - my); sxx += (xs[i] - mx) ** 2; syy += (ys[i] - my) ** 2 }
return sxx && syy ? sxy / Math.sqrt(sxx * syy) : null
}
/// The periods --split makes of [FROM, TO): each [from, to).
const periods = () => {
const cuts = [FROM || '0', ...SPLITS, TO]
return cuts.slice(0, -1).map((from, i) => [from, cuts[i + 1]])
}
const label = ([a, b]) => `${a === '0' || a === '' ? 'start' : a.slice(11, 16)}–${b === '9' ? 'end' : b.slice(11, 16)}`
const out = []
const table = (head, rows) => {
out.push(`| ${head.join(' | ')} |`, `| ${head.map((_, i) => (i ? '--:' : '---')).join(' | ')} |`)
for (const r of rows) out.push(`| ${r.join(' | ')} |`)
out.push('')
}
out.push(`# ${dir}`, '')
// ---------------------------------------------------------------- the soak's totals
const summaries = files(/^summary-.*\.json$/).map((f) => JSON.parse(readFileSync(join(dir, f), 'utf8')))
if (summaries.length) {
out.push('## Totals (summary-*.json)', '')
const t = (s) => s.totals
table(['', ...summaries.map((s) => s.side)], [
['from, to', ...summaries.map((s) => `${s.started?.slice(11, 16)}–${s.updated?.slice(11, 16)}`)],
['accounts, records', ...summaries.map((s) => `${s.accounts}, ${int(s.records)}`)],
['operations', ...summaries.map((s) => int(t(s).ops))],
['5xx', ...summaries.map((s) => int(t(s).e5xx))],
['rate limited (429)', ...summaries.map((s) => int(t(s).limited))],
['lost', ...summaries.map((s) => int(t(s).lost))],
['mismatches', ...summaries.map((s) => int(t(s).mismatches))],
['commits not on the firehose', ...summaries.map((s) => int(t(s).missingCommits))],
['outages, their time', ...summaries.map((s) => `${t(s).outages}, ${num((t(s).outageMs ?? 0) / 1000)} s`)],
])
}
// ---------------------------------------------------------------- the soak's windows and verdicts
const eventsBySide = Object.fromEntries(files(/^events-.*\.jsonl(\.gz)?$/).map((f) => [f.replace(/^events-|\.jsonl(\.gz)?$/g, ''), lines(f.replace(/\.gz$/, ''))]))
const sides = Object.keys(eventsBySide)
if (sides.length) {
const windows = (s) => eventsBySide[s].filter((e) => e.kind === 'window' && inRange(e.from))
const of = (s, op, q) => windows(s).map((w) => w.ops?.[op]?.[q]).filter((x) => x != null)
out.push("## The soak's times, by five-minute window (events-*.jsonl)", '', 'Each window has its own p50 and p99 per operation; the median of them, and the worst p99. Times are the soak\'s, from request to answer.', '')
const ops = ['post', 'like', 'applyWrites', 'delete', 'getRecord', 'listRecords', 'getLatestCommit', 'getRepo']
table(['operation', ...sides.flatMap((s) => [`${s} p50`, `${s} p99`, `${s} worst p99`])], ops.map((op) => [op, ...sides.flatMap((s) => [num(median(of(s, op, 'p50'))), num(median(of(s, op, 'p99'))), num(Math.max(...of(s, op, 'p99')) === -Infinity ? null : Math.max(...of(s, op, 'p99')))])]))
const kinds = (s) => {
const c = {}
for (const e of eventsBySide[s]) if (e.kind !== 'window' && inRange(e.time)) c[e.kind] = (c[e.kind] || 0) + 1
return c
}
out.push('### Events by kind', '')
const all = [...new Set(sides.flatMap((s) => Object.keys(kinds(s))))].sort()
table(['kind', ...sides], all.map((k) => [k, ...sides.map((s) => int(kinds(s)[k] ?? 0))]))
const settled = sides.flatMap((s) => eventsBySide[s].filter((e) => e.kind === 'settled'))
if (settled.length) {
out.push('### The settled check', '')
table(['', ...settled.map((v) => `${v.side}, ${v.time.slice(11, 16)}`)], [
['live state: repositories + blobs', ...settled.map((v) => `${v.liveMiB} MiB (${v.reposMiB} + ${v.blobsMiB})`)],
['when the load stopped', ...settled.map((v) => (v.loadEndMiB == null ? '–' : `${v.loadEndMiB} MiB`))],
['after 2 W with no write', ...settled.map((v) => `${v.beforeMiB} MiB`)],
['at rest, after a write to each', ...settled.map((v) => `${v.leastMiB} MiB, ${(v.leastMiB / v.liveMiB).toFixed(2)} × live`)],
['the settled bound', ...settled.map((v) => `${v.boundMiB} MiB: ${v.pass ? 'within' : 'over'}`)],
])
}
}
// ---------------------------------------------------------------- kills
const kills = lines('kills.jsonl').filter((k) => k.killed && inRange(k.time))
if (kills.length) {
out.push('## Kills (kills.jsonl)', '')
const by = {}
for (const k of kills) (by[k.killed] ??= []).push(k.restarted ?? k.started)
table(['container', 'kills', 'serving again after'], Object.entries(by).map(([c, ts]) => [c, ts.length, `${Math.min(...ts)}–${Math.max(...ts)} s`]))
}
// ---------------------------------------------------------------- samples
const samples = lines('samples.jsonl').filter((s) => inRange(s.time))
if (samples.length) {
const containers = Object.keys(samples[0]).filter((k) => samples[0][k]?.rssKb !== undefined)
out.push('## Memory and data over time (samples.jsonl), MiB', '')
const v = (c, k) => samples.map((s) => s[c]?.[k]).filter((x) => x != null && x > 0)
table(['container', 'RSS first, most, last', 'data first, most, last'], containers.map((c) => [c, `${mib(v(c, 'rssKb')[0])}, ${mib(Math.max(...v(c, 'rssKb')))}, ${mib(v(c, 'rssKb').at(-1))}`, `${mib(v(c, 'dataKb')[0])}, ${mib(Math.max(...v(c, 'dataKb')))}, ${mib(v(c, 'dataKb').at(-1))}`]))
const hours = {}
for (const s of samples) hours[s.time.slice(0, 13)] = s
table(['hour (last sample)', ...containers.flatMap((c) => [`${c} RSS`, `${c} data`]), 'free GiB'], Object.values(hours).map((s) => [s.time.slice(11, 16), ...containers.flatMap((c) => [mib(s[c]?.rssKb), mib(s[c]?.dataKb)]), Math.floor(s.freeKb / 1048576)]))
const synced = samples.filter((s) => s.specFsync && s.specFsync['+Inf'] != null)
if (synced.length > 1) {
out.push("## Monolith's syncs (samples.jsonl: its fsync histogram, the disk's counters)", '')
// Counts between consecutive samples of the same start (a start sets the histogram to 0).
const steps = []
for (let i = 1; i < synced.length; i++) {
const a = synced[i - 1], b = synced[i]
const d = (le) => b.specFsync[le] - a.specFsync[le]
if (d('+Inf') < 0) continue
steps.push({ time: b.time, syncs: d('+Inf'), over25: d('+Inf') - d('0.025'), over100: d('+Inf') - d('0.1'), flushMs: b.disk && a.disk ? b.disk.flushMs - a.disk.flushMs : null })
}
table(['period', 'syncs', 'over 25 ms', 'over 100 ms', 'slow syncs ~ disk flush time (r, a sample apart)'], periods().map((p) => {
const ss = steps.filter((s) => s.time >= p[0] && s.time < p[1])
const withFlush = ss.filter((s) => s.flushMs != null)
const r = corr(withFlush.map((s) => s.flushMs), withFlush.map((s) => s.over100))
return [label(p), int(ss.reduce((n, s) => n + s.syncs, 0)), int(ss.reduce((n, s) => n + s.over25, 0)), int(ss.reduce((n, s) => n + s.over100, 0)), num(r, 2)]
}))
}
}
// ---------------------------------------------------------------- Monolith's log
const log = lines('server-log.jsonl').filter((l) => inRange(l.time))
if (log.length) {
const reqs = log.filter((l) => l.msg === 'request')
const starts = log.filter((l) => l.msg === 'start').map((l) => Date.parse(l.time))
const AFTER_START_MS = 60_000
const afterStart = (t) => starts.some((s) => t >= s && t < s + AFTER_START_MS)
const WRITES = /\.(createRecord|deleteRecord|applyWrites|putRecord)$/
const READS = /\.(getRecord|listRecords|getLatestCommit|describeServer)$/
const stats = (ms) => {
const s = [...ms].sort((a, b) => a - b)
return [int(s.length), num(pct(s, 0.5)), num(pct(s, 0.99)), num(pct(s, 0.999)), num(s.at(-1))]
}
out.push("## Monolith's requests, server side (server-log.jsonl)", '', `Its \`request\` lines' \`ms\`. Starts in the log: ${starts.length}.`, '')
const by = {}
for (const r of reqs) (by[r.method] ??= []).push(r.ms)
table(['method', 'n', 'p50 ms', 'p99 ms', 'p99.9 ms', 'worst ms'], Object.entries(by).filter(([, v]) => v.length >= 500).sort((a, b) => b[1].length - a[1].length).map(([m, v]) => [m.replace('com.atproto.', ''), ...stats(v)]))
out.push(`### createRecord by period, and the ${AFTER_START_MS / 1000} s after each start apart`, '')
const cr = reqs.filter((r) => r.method === 'com.atproto.repo.createRecord')
const rows = periods().flatMap((p) => {
const inP = cr.filter((r) => r.time >= p[0] && r.time < p[1])
const steady = inP.filter((r) => !afterStart(Date.parse(r.time)))
const after = inP.filter((r) => afterStart(Date.parse(r.time)))
return [[`${label(p)}, but after a start`, ...stats(steady.map((r) => r.ms))], ...(after.length ? [[`${label(p)}, after a start`, ...stats(after.map((r) => r.ms))]] : [])]
})
table(['', 'n', 'p50 ms', 'p99 ms', 'p99.9 ms', 'worst ms'], rows)
// What a slow write held up: the requests under way while it was.
out.push('### What a slow write (over 200 ms) held up', '', 'Requests under way during each slow write, but it, slow being over 100 ms.', '')
const span = reqs.map((r) => ({ end: Date.parse(r.time), start: Date.parse(r.time) - r.ms, ms: r.ms, method: r.method, did: r.did })).sort((a, b) => a.start - b.start)
const starts0 = span.map((s) => s.start)
const lower = (x) => { let lo = 0, hi = starts0.length; while (lo < hi) { const m = (lo + hi) >> 1; if (starts0[m] < x) lo = m + 1; else hi = m } return lo }
const c = { other: [0, 0], same: [0, 0], reads: [0, 0] }
let slowN = 0
for (const w of span) {
if (w.ms <= 200 || !WRITES.test(w.method)) continue
slowN++
for (let i = lower(w.start - 5000); i < span.length && span[i].start < w.end; i++) {
const o = span[i]
if (o === w || o.end <= w.start) continue
const k = o.ms > 100 ? 1 : 0
if (WRITES.test(o.method)) c[o.did === w.did ? 'same' : 'other'][k]++
else if (READS.test(o.method)) c.reads[k]++
}
}
table(['under way', 'fast', 'slow'], [['writes to other accounts', int(c.other[0]), int(c.other[1])], ['writes to the same account', int(c.same[0]), int(c.same[1])], ['reads', int(c.reads[0]), int(c.reads[1])]])
out.push(`Slow writes: ${int(slowN)}.`, '')
if (samples.length) {
// Slow writes against the disk's flush time and Monolith's slow syncs, a sample apart.
const steps = []
for (let i = 1; i < samples.length; i++) {
const a = samples[i - 1], b = samples[i]
if (!a.disk || !b.disk) continue
const slow = reqs.filter((r) => r.time > a.time && r.time <= b.time && WRITES.test(r.method) && r.ms > 200).length
const syncs = a.specFsync?.['+Inf'] != null && b.specFsync?.['+Inf'] != null && b.specFsync['+Inf'] >= a.specFsync['+Inf'] ? (b.specFsync['+Inf'] - b.specFsync['0.1']) - (a.specFsync['+Inf'] - a.specFsync['0.1']) : null
steps.push({ time: b.time, slow, flushMs: b.disk.flushMs - a.disk.flushMs, syncs })
}
if (steps.length > 2) {
out.push('### Slow writes against the disk, a sample apart', '')
table(['period', 'samples', 'r: slow writes ~ disk flush time', 'r: slow syncs ~ disk flush time'], periods().map((p) => {
const ss = steps.filter((s) => s.time >= p[0] && s.time < p[1])
const sy = ss.filter((s) => s.syncs != null)
return [label(p), ss.length, num(corr(ss.map((s) => s.flushMs), ss.map((s) => s.slow)), 2), num(corr(sy.map((s) => s.flushMs), sy.map((s) => s.syncs)), 2)]
}))
}
}
}
console.log(out.join('\n'))