~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-watch

The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay

0
0
main
73 lines · 3.4 KB Raw History
// One stream source: a Jetstream (…/subscribe, JSON) or a relay (com.atproto.sync.subscribeRepos, CBOR or JSON).
// Shared by the page's worker (worker.js) and the CLI capture (capture.mjs). Emits only commits with an op in a
// watched collection; a relay commit is trimmed to those ops, records decoded from its CAR inline.
import { subscribeXrpc } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js";
import { commitFrame, wanted, PREFIXES } from "./model.js";

const SUBSCRIBE_REPOS = "com.atproto.sync.subscribeRepos";

export const isJetstream = (url) => new URL(url).pathname.endsWith("/subscribe");

/** The URL to open: Jetstream with wantedCollections (and a cursor), a relay with its xrpc path (and a cursor). */
export function streamUrl(url, { prefixes = PREFIXES, cursor } = {}) {
	const u = new URL(url);
	if (isJetstream(url)) {
		u.searchParams.delete("wantedCollections");
		for (const p of prefixes) u.searchParams.append("wantedCollections", p.endsWith(".") ? `${p}*` : p);
	} else if (!u.pathname.startsWith("/xrpc/")) u.pathname = `/xrpc/${SUBSCRIBE_REPOS}`;
	if (cursor !== undefined) u.searchParams.set("cursor", String(cursor));
	return u.href;
}

/**
 * Run a source until signal aborts, reconnecting with backoff from the last cursor it saw.
 * emit(frame, recvMs) per kept frame; status({ state, error?, frames, kept, cursor }) on changes and every second.
 * rewindMinutes: a Jetstream starts that far back (time_us cursor); relays start live.
 */
export async function runSource(url, { prefixes = PREFIXES, rewindMinutes = 0, signal, emit, status = () => {} }) {
	const jet = isJetstream(url);
	let cursor = jet && rewindMinutes > 0 ? (Date.now() - rewindMinutes * 60_000) * 1000 : undefined;
	const st = { state: "connecting", frames: 0, kept: 0, cursor, error: undefined };
	const tick = setInterval(() => status({ ...st }), 1000);
	let backoff = 1000;
	try {
		while (!signal?.aborted) {
			st.state = "connecting";
			status({ ...st });
			try {
				const live = () => ((st.state = "live"), (st.error = undefined), (backoff = 1000), status({ ...st }));
				// subscribeXrpc takes its own onopen; this one only says the socket is up before the first frame
				const WS = class extends globalThis.WebSocket {
					constructor(u) {
						super(u);
						this.addEventListener("open", live);
					}
				};
				const sub = subscribeXrpc(streamUrl(url, { prefixes, cursor }), { nsid: jet ? undefined : SUBSCRIBE_REPOS, signal, WebSocket: WS });
				for await (const frame of sub) {
					st.frames++;
					if (frame?.error) throw new Error(`${frame.error}${frame.message ? `: ${frame.message}` : ""}`);
					if (jet) cursor = frame.time_us ?? cursor;
					else if (typeof frame.seq === "number") cursor = frame.seq;
					st.cursor = cursor;
					const kept = jet ? (frame.kind === "commit" && wanted(frame.commit?.collection, prefixes) ? frame : null) : commitFrame(frame, prefixes);
					if (kept) st.kept++, emit(kept, Date.now());
				}
				if (signal?.aborted) break;
				throw new Error("closed");
			} catch (error) {
				if (signal?.aborted) break;
				if (/FutureCursor|OutdatedCursor/.test(String(error?.message))) cursor = undefined;
				st.state = "retrying";
				st.error = String(error?.message ?? error);
				status({ ...st });
				await new Promise((r) => setTimeout(r, backoff));
				backoff = Math.min(backoff * 2, 30_000);
			}
		}
	} finally {
		clearInterval(tick);
		st.state = "stopped";
		status({ ...st });
	}
}