// One stream source: a Jetstream (…/subscribe, JSON) or a relay (com.atproto.sync.subscribeRepos, CBOR or JSON). // Shared by the page's worker (worker.js) and the CLI capture (capture.mjs). Emits only commits with an op in a // watched collection; a relay commit is trimmed to those ops, records decoded from its CAR inline. import { subscribeXrpc } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js"; import { commitFrame, wanted, PREFIXES } from "./model.js"; const SUBSCRIBE_REPOS = "com.atproto.sync.subscribeRepos"; export const isJetstream = (url) => new URL(url).pathname.endsWith("/subscribe"); /** The URL to open: Jetstream with wantedCollections (and a cursor), a relay with its xrpc path (and a cursor). */ export function streamUrl(url, { prefixes = PREFIXES, cursor } = {}) { const u = new URL(url); if (isJetstream(url)) { u.searchParams.delete("wantedCollections"); for (const p of prefixes) u.searchParams.append("wantedCollections", p.endsWith(".") ? `${p}*` : p); } else if (!u.pathname.startsWith("/xrpc/")) u.pathname = `/xrpc/${SUBSCRIBE_REPOS}`; if (cursor !== undefined) u.searchParams.set("cursor", String(cursor)); return u.href; } /** * Run a source until signal aborts, reconnecting with backoff from the last cursor it saw. * emit(frame, recvMs) per kept frame; status({ state, error?, frames, kept, cursor }) on changes and every second. * rewindMinutes: a Jetstream starts that far back (time_us cursor); relays start live. */ export async function runSource(url, { prefixes = PREFIXES, rewindMinutes = 0, signal, emit, status = () => {} }) { const jet = isJetstream(url); let cursor = jet && rewindMinutes > 0 ? (Date.now() - rewindMinutes * 60_000) * 1000 : undefined; const st = { state: "connecting", frames: 0, kept: 0, cursor, error: undefined }; const tick = setInterval(() => status({ ...st }), 1000); let backoff = 1000; try { while (!signal?.aborted) { st.state = "connecting"; status({ ...st }); try { const live = () => ((st.state = "live"), (st.error = undefined), (backoff = 1000), status({ ...st })); // subscribeXrpc takes its own onopen; this one only says the socket is up before the first frame const WS = class extends globalThis.WebSocket { constructor(u) { super(u); this.addEventListener("open", live); } }; const sub = subscribeXrpc(streamUrl(url, { prefixes, cursor }), { nsid: jet ? undefined : SUBSCRIBE_REPOS, signal, WebSocket: WS }); for await (const frame of sub) { st.frames++; if (frame?.error) throw new Error(`${frame.error}${frame.message ? `: ${frame.message}` : ""}`); if (jet) cursor = frame.time_us ?? cursor; else if (typeof frame.seq === "number") cursor = frame.seq; st.cursor = cursor; const kept = jet ? (frame.kind === "commit" && wanted(frame.commit?.collection, prefixes) ? frame : null) : commitFrame(frame, prefixes); if (kept) st.kept++, emit(kept, Date.now()); } if (signal?.aborted) break; throw new Error("closed"); } catch (error) { if (signal?.aborted) break; if (/FutureCursor|OutdatedCursor/.test(String(error?.message))) cursor = undefined; st.state = "retrying"; st.error = String(error?.message ?? error); status({ ...st }); await new Promise((r) => setTimeout(r, backoff)); backoff = Math.min(backoff * 2, 30_000); } } } finally { clearInterval(tick); st.state = "stopped"; status({ ...st }); } }