johnandersen777.bsky.social/socialweb-computer-watch
The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay
0
0
// One stream source: a Jetstream (…/subscribe, JSON) or a relay (com.atproto.sync.subscribeRepos, CBOR or JSON).
// Shared by the page's worker (worker.js) and the CLI capture (capture.mjs). Emits only commits with an op in a
// watched collection; a relay commit is trimmed to those ops, records decoded from its CAR inline.
import { subscribeXrpc } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js";
import { commitFrame, wanted, PREFIXES } from "./model.js";
const SUBSCRIBE_REPOS = "com.atproto.sync.subscribeRepos";
export const isJetstream = (url) => new URL(url).pathname.endsWith("/subscribe");
/** The URL to open: Jetstream with wantedCollections (and a cursor), a relay with its xrpc path (and a cursor). */
export function streamUrl(url, { prefixes = PREFIXES, cursor } = {}) {
const u = new URL(url);
if (isJetstream(url)) {
u.searchParams.delete("wantedCollections");
for (const p of prefixes) u.searchParams.append("wantedCollections", p.endsWith(".") ? `${p}*` : p);
} else if (!u.pathname.startsWith("/xrpc/")) u.pathname = `/xrpc/${SUBSCRIBE_REPOS}`;
if (cursor !== undefined) u.searchParams.set("cursor", String(cursor));
return u.href;
}
/**
* Run a source until signal aborts, reconnecting with backoff from the last cursor it saw.
* emit(frame, recvMs) per kept frame; status({ state, error?, frames, kept, cursor }) on changes and every second.
* rewindMinutes: a Jetstream starts that far back (time_us cursor); relays start live.
*/
export async function runSource(url, { prefixes = PREFIXES, rewindMinutes = 0, signal, emit, status = () => {} }) {
const jet = isJetstream(url);
let cursor = jet && rewindMinutes > 0 ? (Date.now() - rewindMinutes * 60_000) * 1000 : undefined;
const st = { state: "connecting", frames: 0, kept: 0, cursor, error: undefined };
const tick = setInterval(() => status({ ...st }), 1000);
let backoff = 1000;
try {
while (!signal?.aborted) {
st.state = "connecting";
status({ ...st });
try {
const live = () => ((st.state = "live"), (st.error = undefined), (backoff = 1000), status({ ...st }));
// subscribeXrpc takes its own onopen; this one only says the socket is up before the first frame
const WS = class extends globalThis.WebSocket {
constructor(u) {
super(u);
this.addEventListener("open", live);
}
};
const sub = subscribeXrpc(streamUrl(url, { prefixes, cursor }), { nsid: jet ? undefined : SUBSCRIBE_REPOS, signal, WebSocket: WS });
for await (const frame of sub) {
st.frames++;
if (frame?.error) throw new Error(`${frame.error}${frame.message ? `: ${frame.message}` : ""}`);
if (jet) cursor = frame.time_us ?? cursor;
else if (typeof frame.seq === "number") cursor = frame.seq;
st.cursor = cursor;
const kept = jet ? (frame.kind === "commit" && wanted(frame.commit?.collection, prefixes) ? frame : null) : commitFrame(frame, prefixes);
if (kept) st.kept++, emit(kept, Date.now());
}
if (signal?.aborted) break;
throw new Error("closed");
} catch (error) {
if (signal?.aborted) break;
if (/FutureCursor|OutdatedCursor/.test(String(error?.message))) cursor = undefined;
st.state = "retrying";
st.error = String(error?.message ?? error);
status({ ...st });
await new Promise((r) => setTimeout(r, backoff));
backoff = Math.min(backoff * 2, 30_000);
}
}
} finally {
clearInterval(tick);
st.state = "stopped";
status({ ...st });
}
}