~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-watch

The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay

0
0
main
64 lines · 3.5 KB Raw History
#!/usr/bin/env node
// Capture the watched collections from several Jetstreams and relays at once, deduplicated, as NDJSON the page
// replays: one { src, recv, frame } line per frame that delivered a new op.
//   node capture.mjs [--out capture.ndjson] [--seconds N] [--rewind-minutes N] [--all] [url ...]
// --all keeps duplicate frames too (the page dedups on load either way). Stops on SIGINT/SIGTERM or after --seconds.
import { appendFileSync, writeFileSync } from "node:fs";
import { runSource } from "./source.js";
import { Watch, PREFIXES, fetchedFrame } from "./model.js";
import { pdsOf } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/index.js";
import { DEFAULT_SOURCES } from "./sources.js";

const args = process.argv.slice(2);
const flag = (name, fallback) => {
	const i = args.indexOf(name);
	return i < 0 ? fallback : args.splice(i, 2)[1];
};
const out = flag("--out", "capture.ndjson");
const seconds = Number(flag("--seconds", 0));
const rewindMinutes = Number(flag("--rewind-minutes", 0));
const all = args.includes("--all") ? (args.splice(args.indexOf("--all"), 1), true) : false;
const urls = args.length ? args : DEFAULT_SOURCES.filter((s) => s.on).map((s) => s.url);

writeFileSync(out, "");
// older commits are a relay re-emitting history (backfill): not written (--all writes them)
const watch = new Watch(PREFIXES, Date.now() - (rewindMinutes + 1) * 60_000);
const stop = new AbortController();
let lines = 0;
const states = new Map();
for (const url of urls)
	runSource(url, {
		rewindMinutes,
		signal: stop.signal,
		emit: (frame, recv) => {
			const fresh = watch.ingest(url, frame, recv);
			if (fresh.length || all) appendFileSync(out, `${JSON.stringify({ src: url, recv, frame })}\n`), lines++;
			for (const op of fresh) if (!op.record && op.action !== "delete") setTimeout(() => fill(op), 4000);
		},
		status: (s) => states.get(url) !== s.state && (states.set(url, s.state), console.error(`${new Date().toISOString()} ${url} ${s.state}${s.error ? `: ${s.error}` : ""}`)),
	});
// a body-less op (the JSON relay sends none) that no other source filled within 4 s: getRecord at its PDS, written
// as a "getRecord" line at the op's rev, so the capture replays with the body (an R2's repo is gone after the run)
const ctx = { http: { fetch: (u, i = {}) => fetch(u, { ...i, signal: AbortSignal.timeout(10_000) }) } };
const pds = new Map();
async function fill(op) {
	const uri = `at://${op.did}/${op.collection}/${op.rkey}`;
	if (watch.records.get(uri)?.record && !watch.records.get(uri).stale) return;
	try {
		if (!pds.has(op.did)) pds.set(op.did, pdsOf(ctx, op.did).catch(() => null));
		const base = await pds.get(op.did);
		if (!base) return;
		const r = await ctx.http.fetch(`${base.replace(/\/$/, "")}/xrpc/com.atproto.repo.getRecord?${new URLSearchParams({ repo: op.did, collection: op.collection, rkey: op.rkey })}`);
		if (!r.ok) return;
		const { cid, value } = await r.json();
		const frame = fetchedFrame(op.did, op.collection, op.rkey, op.rev, cid, value);
		watch.ingest("getRecord", frame);
		appendFileSync(out, `${JSON.stringify({ src: "getRecord", recv: Date.now(), frame })}\n`), lines++;
	} catch {}
}
const report = () => console.error(`${new Date().toISOString()} ${out}: ${lines} lines; ${[...watch.sources].map(([u, s]) => `${URL.canParse(u) ? new URL(u).host : u} ${s.fresh}/${s.ops}`).join(", ")}`);
const timer = setInterval(report, 60_000);
const end = () => (stop.abort(), clearInterval(timer), report(), setTimeout(() => process.exit(0), 200));
process.on("SIGINT", end);
process.on("SIGTERM", end);
if (seconds > 0) setTimeout(end, seconds * 1000);