johnandersen777.bsky.social/socialweb-computer-watch
The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay
0
0
#!/usr/bin/env node
// Capture the watched collections from several Jetstreams and relays at once, deduplicated, as NDJSON the page
// replays: one { src, recv, frame } line per frame that delivered a new op.
// node capture.mjs [--out capture.ndjson] [--seconds N] [--rewind-minutes N] [--all] [url ...]
// --all keeps duplicate frames too (the page dedups on load either way). Stops on SIGINT/SIGTERM or after --seconds.
import { appendFileSync, writeFileSync } from "node:fs";
import { runSource } from "./source.js";
import { Watch, PREFIXES, fetchedFrame } from "./model.js";
import { pdsOf } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/index.js";
import { DEFAULT_SOURCES } from "./sources.js";
const args = process.argv.slice(2);
const flag = (name, fallback) => {
const i = args.indexOf(name);
return i < 0 ? fallback : args.splice(i, 2)[1];
};
const out = flag("--out", "capture.ndjson");
const seconds = Number(flag("--seconds", 0));
const rewindMinutes = Number(flag("--rewind-minutes", 0));
const all = args.includes("--all") ? (args.splice(args.indexOf("--all"), 1), true) : false;
const urls = args.length ? args : DEFAULT_SOURCES.filter((s) => s.on).map((s) => s.url);
writeFileSync(out, "");
// older commits are a relay re-emitting history (backfill): not written (--all writes them)
const watch = new Watch(PREFIXES, Date.now() - (rewindMinutes + 1) * 60_000);
const stop = new AbortController();
let lines = 0;
const states = new Map();
for (const url of urls)
runSource(url, {
rewindMinutes,
signal: stop.signal,
emit: (frame, recv) => {
const fresh = watch.ingest(url, frame, recv);
if (fresh.length || all) appendFileSync(out, `${JSON.stringify({ src: url, recv, frame })}\n`), lines++;
for (const op of fresh) if (!op.record && op.action !== "delete") setTimeout(() => fill(op), 4000);
},
status: (s) => states.get(url) !== s.state && (states.set(url, s.state), console.error(`${new Date().toISOString()} ${url} ${s.state}${s.error ? `: ${s.error}` : ""}`)),
});
// a body-less op (the JSON relay sends none) that no other source filled within 4 s: getRecord at its PDS, written
// as a "getRecord" line at the op's rev, so the capture replays with the body (an R2's repo is gone after the run)
const ctx = { http: { fetch: (u, i = {}) => fetch(u, { ...i, signal: AbortSignal.timeout(10_000) }) } };
const pds = new Map();
async function fill(op) {
const uri = `at://${op.did}/${op.collection}/${op.rkey}`;
if (watch.records.get(uri)?.record && !watch.records.get(uri).stale) return;
try {
if (!pds.has(op.did)) pds.set(op.did, pdsOf(ctx, op.did).catch(() => null));
const base = await pds.get(op.did);
if (!base) return;
const r = await ctx.http.fetch(`${base.replace(/\/$/, "")}/xrpc/com.atproto.repo.getRecord?${new URLSearchParams({ repo: op.did, collection: op.collection, rkey: op.rkey })}`);
if (!r.ok) return;
const { cid, value } = await r.json();
const frame = fetchedFrame(op.did, op.collection, op.rkey, op.rev, cid, value);
watch.ingest("getRecord", frame);
appendFileSync(out, `${JSON.stringify({ src: "getRecord", recv: Date.now(), frame })}\n`), lines++;
} catch {}
}
const report = () => console.error(`${new Date().toISOString()} ${out}: ${lines} lines; ${[...watch.sources].map(([u, s]) => `${URL.canParse(u) ? new URL(u).host : u} ${s.fresh}/${s.ops}`).join(", ")}`);
const timer = setInterval(report, 60_000);
const end = () => (stop.abort(), clearInterval(timer), report(), setTimeout(() => process.exit(0), 200));
process.on("SIGINT", end);
process.on("SIGTERM", end);
if (seconds > 0) setTimeout(end, seconds * 1000);