#!/usr/bin/env node // Capture the watched collections from several Jetstreams and relays at once, deduplicated, as NDJSON the page // replays: one { src, recv, frame } line per frame that delivered a new op. // node capture.mjs [--out capture.ndjson] [--seconds N] [--rewind-minutes N] [--all] [url ...] // --all keeps duplicate frames too (the page dedups on load either way). Stops on SIGINT/SIGTERM or after --seconds. import { appendFileSync, writeFileSync } from "node:fs"; import { runSource } from "./source.js"; import { Watch, PREFIXES, fetchedFrame } from "./model.js"; import { pdsOf } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/index.js"; import { DEFAULT_SOURCES } from "./sources.js"; const args = process.argv.slice(2); const flag = (name, fallback) => { const i = args.indexOf(name); return i < 0 ? fallback : args.splice(i, 2)[1]; }; const out = flag("--out", "capture.ndjson"); const seconds = Number(flag("--seconds", 0)); const rewindMinutes = Number(flag("--rewind-minutes", 0)); const all = args.includes("--all") ? (args.splice(args.indexOf("--all"), 1), true) : false; const urls = args.length ? args : DEFAULT_SOURCES.filter((s) => s.on).map((s) => s.url); writeFileSync(out, ""); // older commits are a relay re-emitting history (backfill): not written (--all writes them) const watch = new Watch(PREFIXES, Date.now() - (rewindMinutes + 1) * 60_000); const stop = new AbortController(); let lines = 0; const states = new Map(); for (const url of urls) runSource(url, { rewindMinutes, signal: stop.signal, emit: (frame, recv) => { const fresh = watch.ingest(url, frame, recv); if (fresh.length || all) appendFileSync(out, `${JSON.stringify({ src: url, recv, frame })}\n`), lines++; for (const op of fresh) if (!op.record && op.action !== "delete") setTimeout(() => fill(op), 4000); }, status: (s) => states.get(url) !== s.state && (states.set(url, s.state), console.error(`${new Date().toISOString()} ${url} ${s.state}${s.error ? `: ${s.error}` : ""}`)), }); // a body-less op (the JSON relay sends none) that no other source filled within 4 s: getRecord at its PDS, written // as a "getRecord" line at the op's rev, so the capture replays with the body (an R2's repo is gone after the run) const ctx = { http: { fetch: (u, i = {}) => fetch(u, { ...i, signal: AbortSignal.timeout(10_000) }) } }; const pds = new Map(); async function fill(op) { const uri = `at://${op.did}/${op.collection}/${op.rkey}`; if (watch.records.get(uri)?.record && !watch.records.get(uri).stale) return; try { if (!pds.has(op.did)) pds.set(op.did, pdsOf(ctx, op.did).catch(() => null)); const base = await pds.get(op.did); if (!base) return; const r = await ctx.http.fetch(`${base.replace(/\/$/, "")}/xrpc/com.atproto.repo.getRecord?${new URLSearchParams({ repo: op.did, collection: op.collection, rkey: op.rkey })}`); if (!r.ok) return; const { cid, value } = await r.json(); const frame = fetchedFrame(op.did, op.collection, op.rkey, op.rev, cid, value); watch.ingest("getRecord", frame); appendFileSync(out, `${JSON.stringify({ src: "getRecord", recv: Date.now(), frame })}\n`), lines++; } catch {} } const report = () => console.error(`${new Date().toISOString()} ${out}: ${lines} lines; ${[...watch.sources].map(([u, s]) => `${URL.canParse(u) ? new URL(u).host : u} ${s.fresh}/${s.ops}`).join(", ")}`); const timer = setInterval(report, 60_000); const end = () => (stop.abort(), clearInterval(timer), report(), setTimeout(() => process.exit(0), 200)); process.on("SIGINT", end); process.on("SIGTERM", end); if (seconds > 0) setTimeout(end, seconds * 1000);