~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
252 lines · 13.3 KB Raw History
// The admin's reads, `/xrpc/_logs` and `/xrpc/_stats` (spec/OPERATIONS.md,
// "The admin's reads"), for the checks of them over XRPC
// (checks/operations.mjs) and through their MCP tools and CLI commands
// (checks/commands.mjs): asking for them; the lines a `_logs` answer may
// hold, of those the server wrote to standard output; and a `_stats`
// answer's form, and its numbers held between two counts of the same, by
// the metrics or by other answers.

import { readFileSync, readdirSync, statSync } from 'node:fs'
import { join } from 'node:path'
import { isDeepStrictEqual } from 'node:util'
import { cand } from './processes.mjs'

/// The log's levels, least first (spec/OPERATIONS.md, "Every line").
export const LEVELS = ['debug', 'info', 'warn', 'error']

/// `PDS_LOG_KEEP_BYTES` when unset (spec/OPERATIONS.md, "Environment").
export const KEEP_DEFAULT = 67108864

/// The admin's reads, by route.
export const ADMIN_ROUTES = ['_logs', '_stats']

/// The address the admin's reads come from: one of their own, so that with
/// the rate limits on they spend no other address's points of `global`.
const FROM = '198.51.100.13'

/// `method` (GET by default) `/xrpc/<route>` on the server, with `params` as
/// its query and `headers` as given. Answers its status, `Content-Type`,
/// text, and body as JSON (null if it is not).
export async function adminRead(route, { params = {}, headers = {}, method = 'GET' } = {}) {
  const query = new URLSearchParams(params).toString()
  const res = await fetch(`${cand.url}/xrpc/${route}${query ? `?${query}` : ''}`, { method, headers: { 'x-forwarded-for': FROM, ...headers } })
  const text = await res.text()
  let body = null
  try {
    body = JSON.parse(text)
  } catch {}
  return { status: res.status, type: res.headers.get('content-type'), text, body }
}

/// Every file under `dir`, any depth, in bytes; 0 for a directory not there.
/// A file let go of while it is counted counts 0.
export function bytesKept(dir) {
  let names
  try {
    names = readdirSync(dir, { recursive: true })
  } catch {
    return 0
  }
  let n = 0
  for (const f of names) {
    try {
      const st = statSync(join(dir, String(f)))
      if (st.isFile()) n += st.size
    } catch {}
  }
  return n
}

/// The files under `dir`, any depth, each `{path, text}`.
export function keptFiles(dir) {
  let names
  try {
    names = readdirSync(dir, { recursive: true })
  } catch {
    return []
  }
  const out = []
  for (const f of names) {
    const path = join(dir, String(f))
    try {
      if (statSync(path).isFile()) out.push({ path, text: readFileSync(path, 'utf8') })
    } catch {}
  }
  return out
}

/// The keep of a start, `PDS_LOG_KEEP_BYTES` or its default.
export const keepOf = (env) => Number(env.PDS_LOG_KEEP_BYTES || KEEP_DEFAULT)

/// The lines a `_logs` read could find in `dir`, oldest first: those the
/// starts of `lives` (logging.mjs's readLog) that kept them in `dir` wrote
/// to standard output, each a JSON object, but a line whose bytes, with its
/// `\n`, are more than half of `keep`, which is not kept at all
/// (spec/OPERATIONS.md, "Kept"). Each `{obj, bytes, lone}`, `bytes` with
/// its `\n`; `lone` for the last line of a start killed with SIGKILL, which
/// the server writes to standard output before the file, so that a kill
/// between the two leaves it on standard output alone.
export function keptLines(lives, dir, keep) {
  const out = []
  for (const life of lives) {
    if (life.env.PDS_LOG_DIR !== dir) continue
    const killed = life.how === 'SIGKILL' || life.exited?.signal === 'SIGKILL'
    const whole = life.lines.filter((l) => l.obj && !l.unended)
    for (const [i, { text, obj }] of whole.entries()) {
      const bytes = Buffer.byteLength(text) + 1
      if (bytes > keep / 2) continue
      out.push({ obj, bytes, lone: killed && i === whole.length - 1 })
    }
  }
  return out
}

/// How many of the newest of `lines` add up, with their `\n`, to no more
/// than `half` bytes: those the server always keeps (spec/OPERATIONS.md,
/// "Kept").
export function newestWithin(lines, half) {
  let n = 0
  for (let i = lines.length - 1, sum = 0; i >= 0 && (sum += lines[i].bytes) <= half; i--) n += 1
  return n
}

/// Whether a line matches every parameter of a `_logs` read
/// (spec/OPERATIONS.md, "_logs").
export function matches(params, obj) {
  if (params.trace !== undefined && obj.trace_id !== params.trace) return false
  if (params.msg !== undefined && obj.msg !== params.msg) return false
  if (params.method !== undefined && obj.method !== params.method) return false
  if (params.level !== undefined && LEVELS.indexOf(obj.level) < LEVELS.indexOf(params.level)) return false
  if (params.minMs !== undefined && !(typeof obj.ms === 'number' && obj.ms >= Number(params.minMs))) return false
  return true
}

/// Whether `got`, the lines of a `_logs` read with `params`, are what it
/// may answer of `lines` (keptLines after the read): the last of those
/// matching that were kept, kept being the newest, as they were when the
/// read began, which is after the first `from` of them were written and
/// before the `to`th, its own `request` line. As many as `limit`, or all
/// kept if fewer, and at least those of the newest within `half`; a `lone`
/// line may be missing, and nothing else. Answers null if they are, or the
/// lines it should have answered had it begun at `from`, all kept.
export function holdLines(got, lines, { from, to, params, limit, half }) {
  for (let end = to; end >= from; end--) {
    const before = lines.slice(0, end)
    const matching = before.filter((l) => matches(params, l.obj))
    const sure = before.slice(before.length - newestWithin(before, half)).filter((l) => !l.lone && matches(params, l.obj)).length
    const n = got.length
    if (n > limit || n > matching.length || n < Math.min(limit, sure)) continue
    // The newest matching lines, from the last back, each `got`'s in turn,
    // but a `lone` one that is not there.
    let i = matching.length - 1
    let j = n - 1
    while (j >= 0 && i >= 0) {
      if (isDeepStrictEqual(matching[i].obj, got[j])) j -= 1
      else if (!matching[i].lone) break
      i -= 1
    }
    if (j < 0) return null
  }
  return lines.slice(0, from).filter((l) => matches(params, l.obj)).slice(-limit).map((l) => l.obj)
}

// ---------------------------------------------------------------- _stats

const TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
const isCount = (v) => Number.isInteger(v) && v >= 0
const isMean = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0
const keysAre = (o, keys) => o && typeof o === 'object' && !Array.isArray(o) && isDeepStrictEqual(Object.keys(o).sort(), [...keys].sort())

/// What is wrong with the form of a `_stats` answer, or null: its fields,
/// their types, and `routes` and `outbound` in their order
/// (spec/OPERATIONS.md, "_stats").
export function statsForm(b) {
  if (!keysAre(b, ['version', 'started', 'accounts', 'logBytes', 'firehose', 'routes', 'outbound', 'fsync'])) return 'exactly version, started, accounts, logBytes, firehose, routes, outbound and fsync'
  if (typeof b.version !== 'string' || !b.version) return '`version` a string'
  if (typeof b.started !== 'string' || !TIME.test(b.started)) return '`started` RFC 3339 in UTC with milliseconds'
  if (!isCount(b.accounts) || !isCount(b.logBytes)) return '`accounts` and `logBytes` integers of 0 or more'
  if (!keysAre(b.firehose, ['seq', 'subscribers']) || !isCount(b.firehose.seq) || !isCount(b.firehose.subscribers)) return '`firehose` {seq, subscribers}, integers'
  if (!Array.isArray(b.routes)) return '`routes` an array'
  for (const r of b.routes) {
    if (!keysAre(r, ['method', 'count', 'statuses', 'rateLimited', 'meanMs']) || typeof r.method !== 'string' || !isCount(r.count) || r.count < 1 || !isCount(r.rateLimited) || !isMean(r.meanMs)) return 'each route {method, count of 1 or more, statuses, rateLimited, meanMs}'
    if (!r.statuses || typeof r.statuses !== 'object' || Array.isArray(r.statuses) || Object.entries(r.statuses).some(([s, n]) => !/^[1-5]\d\d$/.test(s) || !isCount(n) || n < 1)) return 'each route\'s `statuses` an object of HTTP statuses, as strings, to counts of 1 or more'
  }
  for (let i = 1; i < b.routes.length; i++) {
    const [x, y] = [b.routes[i - 1], b.routes[i]]
    if (!(x.count > y.count || (x.count === y.count && x.method < y.method))) return '`routes` the most `count` first, then by `method`'
  }
  if (!Array.isArray(b.outbound)) return '`outbound` an array'
  for (const o of b.outbound) if (!keysAre(o, ['target', 'ok', 'error', 'meanMs']) || typeof o.target !== 'string' || !isCount(o.ok) || !isCount(o.error) || o.ok + o.error < 1 || !isMean(o.meanMs)) return 'each outbound {target, ok, error, meanMs}, with a request made'
  for (let i = 1; i < b.outbound.length; i++) if (!(b.outbound[i - 1].target < b.outbound[i].target)) return '`outbound` by `target`'
  if (!keysAre(b.fsync, ['count', 'meanMs']) || !isCount(b.fsync.count) || !isMean(b.fsync.meanMs) || (b.fsync.count === 0 && b.fsync.meanMs !== 0)) return '`fsync` {count, meanMs}, meanMs 0 with no sync'
  return null
}

/// A `_stats` answer as counts: each mean as the sum it is of, in ms.
export function statsCounts(b) {
  return {
    accounts: b.accounts,
    logBytes: b.logBytes,
    seq: b.firehose.seq,
    subscribers: b.firehose.subscribers,
    routes: new Map(b.routes.map((r) => [r.method, { count: r.count, statuses: new Map(Object.entries(r.statuses)), rateLimited: r.rateLimited, sum: r.meanMs * r.count }])),
    outbound: new Map(b.outbound.map((o) => [o.target, { ok: o.ok, error: o.error, sum: o.meanMs * (o.ok + o.error) }])),
    fsync: { count: b.fsync.count, sum: b.fsync.meanMs * b.fsync.count },
  }
}

/// The metrics (lib/prometheus.mjs's families) as `statsCounts` gives a
/// `_stats` answer (spec/OPERATIONS.md, "_stats").
export function metricsCounts(families) {
  const samples = (family, name = family) => (families[family]?.samples ?? []).filter((s) => s.name === name)
  const gauge = (name) => families[name]?.samples[0]?.value
  const routes = new Map()
  const route = (m) => routes.get(m) ?? routes.set(m, { count: 0, statuses: new Map(), rateLimited: 0, sum: 0 }).get(m)
  for (const s of samples('pds_request_duration_seconds', 'pds_request_duration_seconds_count')) route(s.labels.method).count = s.value
  for (const s of samples('pds_request_duration_seconds', 'pds_request_duration_seconds_sum')) route(s.labels.method).sum = s.value * 1000
  for (const s of samples('pds_requests_total')) route(s.labels.method).statuses.set(s.labels.status, s.value)
  for (const s of samples('pds_rate_limited_total')) route(s.labels.method).rateLimited = s.value
  const outbound = new Map()
  const target = (t) => outbound.get(t) ?? outbound.set(t, { ok: 0, error: 0, sum: 0 }).get(t)
  for (const s of samples('pds_outbound_requests_total')) target(s.labels.target)[s.labels.outcome === 'ok' ? 'ok' : 'error'] = s.value
  for (const s of samples('pds_outbound_request_duration_seconds', 'pds_outbound_request_duration_seconds_sum')) target(s.labels.target).sum = s.value * 1000
  return {
    accounts: gauge('pds_accounts'),
    logBytes: gauge('pds_log_bytes'),
    seq: gauge('pds_firehose_seq'),
    subscribers: gauge('pds_firehose_subscribers'),
    routes,
    outbound,
    fsync: { count: samples('pds_fsync_duration_seconds', 'pds_fsync_duration_seconds_count')[0]?.value ?? 0, sum: (samples('pds_fsync_duration_seconds', 'pds_fsync_duration_seconds_sum')[0]?.value ?? 0) * 1000 },
  }
}

/// The first number of `mid` not between `lo`'s and `hi`'s (each as
/// `statsCounts` gives them), as `{field, lo, mid, hi}`, or null. A sum of
/// times may differ by a microsecond, or a millionth of it, for how the
/// metrics print it.
export function statsBetween(lo, mid, hi) {
  const out = (field, a, b, c, slack = 0) => (b >= Math.min(a, c) - slack && b <= Math.max(a, c) + slack ? null : { field, lo: a, mid: b, hi: c })
  const sum = (field, a, b, c) => out(field, a, b, c, 1e-3 + 1e-6 * Math.max(Math.abs(a), Math.abs(c)))
  for (const k of ['accounts', 'logBytes', 'seq', 'subscribers']) {
    const bad = out(k === 'seq' || k === 'subscribers' ? `firehose.${k}` : k, lo[k], mid[k], hi[k])
    if (bad) return bad
  }
  for (const m of new Set([...lo.routes.keys(), ...mid.routes.keys(), ...hi.routes.keys()])) {
    const none = { count: 0, statuses: new Map(), rateLimited: 0, sum: 0 }
    const [a, b, c] = [lo, mid, hi].map((x) => x.routes.get(m) ?? none)
    const bad = out(`routes ${m} count`, a.count, b.count, c.count) ?? out(`routes ${m} rateLimited`, a.rateLimited, b.rateLimited, c.rateLimited) ?? sum(`routes ${m} meanMs × count`, a.sum, b.sum, c.sum)
    if (bad) return bad
    for (const s of new Set([...a.statuses.keys(), ...b.statuses.keys(), ...c.statuses.keys()])) {
      const st = out(`routes ${m} statuses ${s}`, a.statuses.get(s) ?? 0, b.statuses.get(s) ?? 0, c.statuses.get(s) ?? 0)
      if (st) return st
    }
  }
  for (const t of new Set([...lo.outbound.keys(), ...mid.outbound.keys(), ...hi.outbound.keys()])) {
    const none = { ok: 0, error: 0, sum: 0 }
    const [a, b, c] = [lo, mid, hi].map((x) => x.outbound.get(t) ?? none)
    const bad = out(`outbound ${t} ok`, a.ok, b.ok, c.ok) ?? out(`outbound ${t} error`, a.error, b.error, c.error) ?? sum(`outbound ${t} meanMs × count`, a.sum, b.sum, c.sum)
    if (bad) return bad
  }
  return out('fsync count', lo.fsync.count, mid.fsync.count, hi.fsync.count) ?? sum('fsync meanMs × count', lo.fsync.sum, mid.fsync.sum, hi.fsync.sum)
}