// The admin's reads, `/xrpc/_logs` and `/xrpc/_stats` (spec/OPERATIONS.md, // "The admin's reads"), for the checks of them over XRPC // (checks/operations.mjs) and through their MCP tools and CLI commands // (checks/commands.mjs): asking for them; the lines a `_logs` answer may // hold, of those the server wrote to standard output; and a `_stats` // answer's form, and its numbers held between two counts of the same, by // the metrics or by other answers. import { readFileSync, readdirSync, statSync } from 'node:fs' import { join } from 'node:path' import { isDeepStrictEqual } from 'node:util' import { cand } from './processes.mjs' /// The log's levels, least first (spec/OPERATIONS.md, "Every line"). export const LEVELS = ['debug', 'info', 'warn', 'error'] /// `PDS_LOG_KEEP_BYTES` when unset (spec/OPERATIONS.md, "Environment"). export const KEEP_DEFAULT = 67108864 /// The admin's reads, by route. export const ADMIN_ROUTES = ['_logs', '_stats'] /// The address the admin's reads come from: one of their own, so that with /// the rate limits on they spend no other address's points of `global`. const FROM = '198.51.100.13' /// `method` (GET by default) `/xrpc/` on the server, with `params` as /// its query and `headers` as given. Answers its status, `Content-Type`, /// text, and body as JSON (null if it is not). export async function adminRead(route, { params = {}, headers = {}, method = 'GET' } = {}) { const query = new URLSearchParams(params).toString() const res = await fetch(`${cand.url}/xrpc/${route}${query ? `?${query}` : ''}`, { method, headers: { 'x-forwarded-for': FROM, ...headers } }) const text = await res.text() let body = null try { body = JSON.parse(text) } catch {} return { status: res.status, type: res.headers.get('content-type'), text, body } } /// Every file under `dir`, any depth, in bytes; 0 for a directory not there. /// A file let go of while it is counted counts 0. export function bytesKept(dir) { let names try { names = readdirSync(dir, { recursive: true }) } catch { return 0 } let n = 0 for (const f of names) { try { const st = statSync(join(dir, String(f))) if (st.isFile()) n += st.size } catch {} } return n } /// The files under `dir`, any depth, each `{path, text}`. export function keptFiles(dir) { let names try { names = readdirSync(dir, { recursive: true }) } catch { return [] } const out = [] for (const f of names) { const path = join(dir, String(f)) try { if (statSync(path).isFile()) out.push({ path, text: readFileSync(path, 'utf8') }) } catch {} } return out } /// The keep of a start, `PDS_LOG_KEEP_BYTES` or its default. export const keepOf = (env) => Number(env.PDS_LOG_KEEP_BYTES || KEEP_DEFAULT) /// The lines a `_logs` read could find in `dir`, oldest first: those the /// starts of `lives` (logging.mjs's readLog) that kept them in `dir` wrote /// to standard output, each a JSON object, but a line whose bytes, with its /// `\n`, are more than half of `keep`, which is not kept at all /// (spec/OPERATIONS.md, "Kept"). Each `{obj, bytes, lone}`, `bytes` with /// its `\n`; `lone` for the last line of a start killed with SIGKILL, which /// the server writes to standard output before the file, so that a kill /// between the two leaves it on standard output alone. export function keptLines(lives, dir, keep) { const out = [] for (const life of lives) { if (life.env.PDS_LOG_DIR !== dir) continue const killed = life.how === 'SIGKILL' || life.exited?.signal === 'SIGKILL' const whole = life.lines.filter((l) => l.obj && !l.unended) for (const [i, { text, obj }] of whole.entries()) { const bytes = Buffer.byteLength(text) + 1 if (bytes > keep / 2) continue out.push({ obj, bytes, lone: killed && i === whole.length - 1 }) } } return out } /// How many of the newest of `lines` add up, with their `\n`, to no more /// than `half` bytes: those the server always keeps (spec/OPERATIONS.md, /// "Kept"). export function newestWithin(lines, half) { let n = 0 for (let i = lines.length - 1, sum = 0; i >= 0 && (sum += lines[i].bytes) <= half; i--) n += 1 return n } /// Whether a line matches every parameter of a `_logs` read /// (spec/OPERATIONS.md, "_logs"). export function matches(params, obj) { if (params.trace !== undefined && obj.trace_id !== params.trace) return false if (params.msg !== undefined && obj.msg !== params.msg) return false if (params.method !== undefined && obj.method !== params.method) return false if (params.level !== undefined && LEVELS.indexOf(obj.level) < LEVELS.indexOf(params.level)) return false if (params.minMs !== undefined && !(typeof obj.ms === 'number' && obj.ms >= Number(params.minMs))) return false return true } /// Whether `got`, the lines of a `_logs` read with `params`, are what it /// may answer of `lines` (keptLines after the read): the last of those /// matching that were kept, kept being the newest, as they were when the /// read began, which is after the first `from` of them were written and /// before the `to`th, its own `request` line. As many as `limit`, or all /// kept if fewer, and at least those of the newest within `half`; a `lone` /// line may be missing, and nothing else. Answers null if they are, or the /// lines it should have answered had it begun at `from`, all kept. export function holdLines(got, lines, { from, to, params, limit, half }) { for (let end = to; end >= from; end--) { const before = lines.slice(0, end) const matching = before.filter((l) => matches(params, l.obj)) const sure = before.slice(before.length - newestWithin(before, half)).filter((l) => !l.lone && matches(params, l.obj)).length const n = got.length if (n > limit || n > matching.length || n < Math.min(limit, sure)) continue // The newest matching lines, from the last back, each `got`'s in turn, // but a `lone` one that is not there. let i = matching.length - 1 let j = n - 1 while (j >= 0 && i >= 0) { if (isDeepStrictEqual(matching[i].obj, got[j])) j -= 1 else if (!matching[i].lone) break i -= 1 } if (j < 0) return null } return lines.slice(0, from).filter((l) => matches(params, l.obj)).slice(-limit).map((l) => l.obj) } // ---------------------------------------------------------------- _stats const TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ const isCount = (v) => Number.isInteger(v) && v >= 0 const isMean = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0 const keysAre = (o, keys) => o && typeof o === 'object' && !Array.isArray(o) && isDeepStrictEqual(Object.keys(o).sort(), [...keys].sort()) /// What is wrong with the form of a `_stats` answer, or null: its fields, /// their types, and `routes` and `outbound` in their order /// (spec/OPERATIONS.md, "_stats"). export function statsForm(b) { if (!keysAre(b, ['version', 'started', 'accounts', 'logBytes', 'firehose', 'routes', 'outbound', 'fsync'])) return 'exactly version, started, accounts, logBytes, firehose, routes, outbound and fsync' if (typeof b.version !== 'string' || !b.version) return '`version` a string' if (typeof b.started !== 'string' || !TIME.test(b.started)) return '`started` RFC 3339 in UTC with milliseconds' if (!isCount(b.accounts) || !isCount(b.logBytes)) return '`accounts` and `logBytes` integers of 0 or more' if (!keysAre(b.firehose, ['seq', 'subscribers']) || !isCount(b.firehose.seq) || !isCount(b.firehose.subscribers)) return '`firehose` {seq, subscribers}, integers' if (!Array.isArray(b.routes)) return '`routes` an array' for (const r of b.routes) { if (!keysAre(r, ['method', 'count', 'statuses', 'rateLimited', 'meanMs']) || typeof r.method !== 'string' || !isCount(r.count) || r.count < 1 || !isCount(r.rateLimited) || !isMean(r.meanMs)) return 'each route {method, count of 1 or more, statuses, rateLimited, meanMs}' if (!r.statuses || typeof r.statuses !== 'object' || Array.isArray(r.statuses) || Object.entries(r.statuses).some(([s, n]) => !/^[1-5]\d\d$/.test(s) || !isCount(n) || n < 1)) return 'each route\'s `statuses` an object of HTTP statuses, as strings, to counts of 1 or more' } for (let i = 1; i < b.routes.length; i++) { const [x, y] = [b.routes[i - 1], b.routes[i]] if (!(x.count > y.count || (x.count === y.count && x.method < y.method))) return '`routes` the most `count` first, then by `method`' } if (!Array.isArray(b.outbound)) return '`outbound` an array' for (const o of b.outbound) if (!keysAre(o, ['target', 'ok', 'error', 'meanMs']) || typeof o.target !== 'string' || !isCount(o.ok) || !isCount(o.error) || o.ok + o.error < 1 || !isMean(o.meanMs)) return 'each outbound {target, ok, error, meanMs}, with a request made' for (let i = 1; i < b.outbound.length; i++) if (!(b.outbound[i - 1].target < b.outbound[i].target)) return '`outbound` by `target`' if (!keysAre(b.fsync, ['count', 'meanMs']) || !isCount(b.fsync.count) || !isMean(b.fsync.meanMs) || (b.fsync.count === 0 && b.fsync.meanMs !== 0)) return '`fsync` {count, meanMs}, meanMs 0 with no sync' return null } /// A `_stats` answer as counts: each mean as the sum it is of, in ms. export function statsCounts(b) { return { accounts: b.accounts, logBytes: b.logBytes, seq: b.firehose.seq, subscribers: b.firehose.subscribers, routes: new Map(b.routes.map((r) => [r.method, { count: r.count, statuses: new Map(Object.entries(r.statuses)), rateLimited: r.rateLimited, sum: r.meanMs * r.count }])), outbound: new Map(b.outbound.map((o) => [o.target, { ok: o.ok, error: o.error, sum: o.meanMs * (o.ok + o.error) }])), fsync: { count: b.fsync.count, sum: b.fsync.meanMs * b.fsync.count }, } } /// The metrics (lib/prometheus.mjs's families) as `statsCounts` gives a /// `_stats` answer (spec/OPERATIONS.md, "_stats"). export function metricsCounts(families) { const samples = (family, name = family) => (families[family]?.samples ?? []).filter((s) => s.name === name) const gauge = (name) => families[name]?.samples[0]?.value const routes = new Map() const route = (m) => routes.get(m) ?? routes.set(m, { count: 0, statuses: new Map(), rateLimited: 0, sum: 0 }).get(m) for (const s of samples('pds_request_duration_seconds', 'pds_request_duration_seconds_count')) route(s.labels.method).count = s.value for (const s of samples('pds_request_duration_seconds', 'pds_request_duration_seconds_sum')) route(s.labels.method).sum = s.value * 1000 for (const s of samples('pds_requests_total')) route(s.labels.method).statuses.set(s.labels.status, s.value) for (const s of samples('pds_rate_limited_total')) route(s.labels.method).rateLimited = s.value const outbound = new Map() const target = (t) => outbound.get(t) ?? outbound.set(t, { ok: 0, error: 0, sum: 0 }).get(t) for (const s of samples('pds_outbound_requests_total')) target(s.labels.target)[s.labels.outcome === 'ok' ? 'ok' : 'error'] = s.value for (const s of samples('pds_outbound_request_duration_seconds', 'pds_outbound_request_duration_seconds_sum')) target(s.labels.target).sum = s.value * 1000 return { accounts: gauge('pds_accounts'), logBytes: gauge('pds_log_bytes'), seq: gauge('pds_firehose_seq'), subscribers: gauge('pds_firehose_subscribers'), routes, outbound, fsync: { count: samples('pds_fsync_duration_seconds', 'pds_fsync_duration_seconds_count')[0]?.value ?? 0, sum: (samples('pds_fsync_duration_seconds', 'pds_fsync_duration_seconds_sum')[0]?.value ?? 0) * 1000 }, } } /// The first number of `mid` not between `lo`'s and `hi`'s (each as /// `statsCounts` gives them), as `{field, lo, mid, hi}`, or null. A sum of /// times may differ by a microsecond, or a millionth of it, for how the /// metrics print it. export function statsBetween(lo, mid, hi) { const out = (field, a, b, c, slack = 0) => (b >= Math.min(a, c) - slack && b <= Math.max(a, c) + slack ? null : { field, lo: a, mid: b, hi: c }) const sum = (field, a, b, c) => out(field, a, b, c, 1e-3 + 1e-6 * Math.max(Math.abs(a), Math.abs(c))) for (const k of ['accounts', 'logBytes', 'seq', 'subscribers']) { const bad = out(k === 'seq' || k === 'subscribers' ? `firehose.${k}` : k, lo[k], mid[k], hi[k]) if (bad) return bad } for (const m of new Set([...lo.routes.keys(), ...mid.routes.keys(), ...hi.routes.keys()])) { const none = { count: 0, statuses: new Map(), rateLimited: 0, sum: 0 } const [a, b, c] = [lo, mid, hi].map((x) => x.routes.get(m) ?? none) const bad = out(`routes ${m} count`, a.count, b.count, c.count) ?? out(`routes ${m} rateLimited`, a.rateLimited, b.rateLimited, c.rateLimited) ?? sum(`routes ${m} meanMs × count`, a.sum, b.sum, c.sum) if (bad) return bad for (const s of new Set([...a.statuses.keys(), ...b.statuses.keys(), ...c.statuses.keys()])) { const st = out(`routes ${m} statuses ${s}`, a.statuses.get(s) ?? 0, b.statuses.get(s) ?? 0, c.statuses.get(s) ?? 0) if (st) return st } } for (const t of new Set([...lo.outbound.keys(), ...mid.outbound.keys(), ...hi.outbound.keys()])) { const none = { ok: 0, error: 0, sum: 0 } const [a, b, c] = [lo, mid, hi].map((x) => x.outbound.get(t) ?? none) const bad = out(`outbound ${t} ok`, a.ok, b.ok, c.ok) ?? out(`outbound ${t} error`, a.error, b.error, c.error) ?? sum(`outbound ${t} meanMs × count`, a.sum, b.sum, c.sum) if (bad) return bad } return out('fsync count', lo.fsync.count, mid.fsync.count, hi.fsync.count) ?? sum('fsync meanMs × count', lo.fsync.sum, mid.fsync.sum, hi.fsync.sum) }