// The page: sources in workers, one Watch (dedup + records), the topology drawn with d3, replay of a capture file. // Nothing persists: a reload starts from nothing. import { Watch, topology, filterTopology, captureLine, short, PREFIXES, fetchedFrame } from "./model.js"; // a live Watch takes commits from the rewind window on (a minute's slack); a replay from its first frame's window const liveWatch = () => new Watch(PREFIXES, Date.now() - ((Number($("rewind").value) || 0) + 1) * 60_000); import { DEFAULT_SOURCES } from "./sources.js"; import { pdsOf, resolveDid } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/index.js"; const $ = (id) => document.getElementById(id); const params = new URLSearchParams(location.search); // every lookup gives up after 10 s: one hung PDS must not stall the rest const fetch10 = (url, init = {}) => fetch(url, { ...init, signal: AbortSignal.timeout(10_000) }); const ctx = { http: { fetch: fetch10 } }; const MAX_CAPTURE = 300_000; let watch = new Watch(PREFIXES, Date.now()); let mode = "live"; let workers = new Map(); let workerStatus = new Map(); let captured = []; let log = []; let drawn = ""; let selected = null; const docs = new Map(); let docsVersion = 0; const tried = new Set(); const pdsCache = new Map(); // ---- sources ----------------------------------------------------------------------------------------------------- $("sources").value = (params.get("sources")?.split(",").map((url) => ({ url, on: true })) ?? DEFAULT_SOURCES).map((s) => `${s.on ? "" : "# "}${s.url}`).join("\n"); $("rewind").value = params.get("rewind") ?? "30"; $("filter").value = params.get("filter") ?? ""; function sourceList() { return $("sources") .value.split("\n") .map((l) => l.trim()) .filter((l) => l && !l.startsWith("#")); } function stopSources() { for (const w of workers.values()) w.terminate(); workers = new Map(); workerStatus = new Map(); } function startSources() { stopSources(); const rewindMinutes = Number($("rewind").value) || 0; for (const url of sourceList()) { try { new URL(url); } catch { workerStatus.set(url, { state: "bad url" }); continue; } const w = new Worker(new URL("./worker.js", import.meta.url), { type: "module" }); w.onmessage = ({ data }) => { if (data.status) return void workerStatus.set(url, data.status); ingest(url, data.frame, data.recv); if (captured.length < MAX_CAPTURE) captured.push(JSON.stringify({ src: url, recv: data.recv, frame: data.frame })); }; w.onerror = (e) => workerStatus.set(url, { state: "error", error: e.message }); w.postMessage({ url, prefixes: PREFIXES, rewindMinutes }); workers.set(url, w); } } function ingest(src, frame, recv) { // heartbeats count in the source table, never in the feed for (const op of watch.ingest(src, frame, recv)) if (!op.collection.endsWith(".market.heartbeat")) log.push(op); if (log.length > 400) log = log.slice(-200); } function goLive() { player.stop(); mode = "live"; watch = liveWatch(); log = []; captured = []; tried.clear(); startSources(); draw(true); } $("apply").onclick = goLive; $("live").onclick = goLive; $("save").onclick = () => { const blob = new Blob([captured.join("\n") + "\n"], { type: "application/x-ndjson" }); const a = Object.assign(document.createElement("a"), { href: URL.createObjectURL(blob), download: `swc-watch-${new Date().toISOString().replace(/[:.]/g, "")}.ndjson` }); a.click(); URL.revokeObjectURL(a.href); }; // ---- identity and missing records (getRecord at the repo's PDS, resolved with atlibs) ------------------------------ async function pool(items, n, fn) { const queue = [...items]; await Promise.all(Array.from({ length: n }, async () => { while (queue.length) await fn(queue.shift()).catch(() => {}); })); } let resolving = false; async function resolveMore(nodes, wants) { // not while a Jetstream replays its rewind: a DID seen there may be tombstoned by now, its release still to come const behind = [...workerStatus.values()].some((w) => w.cursor > 1e15 && w.cursor / 1000 < Date.now() - 60_000); if (resolving || (mode === "live" && behind)) return; resolving = true; try { // a released guest's DID is tombstoned: resolving it is only a 404/410 in the console // a replay's guests and requesters may be long gone (tombstoned): only the lasting accounts resolve; it fetches no records either (the capture is what happened) const dids = nodes.filter((n) => n.type === "actor" && !n.gone && !docs.has(n.did) && (mode === "live" || ["bidder", "runner", "principal"].includes(n.role))).map((n) => n.did).slice(0, 24); await pool(dids, 4, async (did) => { docs.set(did, { handle: short(did) }); const doc = await resolveDid(ctx, did); const handle = doc?.alsoKnownAs?.find((a) => a.startsWith("at://"))?.slice(5); const agentOf = doc?.alsoKnownAs?.find((a) => a.startsWith("https://ident.link/agent-of/"))?.split("/").pop(); docs.set(did, { handle: handle ?? `${short(did)}${doc ? "" : " (gone)"}`, agentOf, gone: !doc }); docsVersion++; }); const gone = new Set(nodes.filter((n) => n.gone || docs.get(n.did)?.gone).map((n) => n.did)); // a body-less op (the JSON relay sends none) waits 4 s for another source to bring it // tried per version (uri@cid): a record updated again is fetched again const tag = (u) => `${u}@${watch.records.get(u)?.cid ?? ""}`; const uris = mode !== "live" ? [] : wants.filter((u) => !tried.has(tag(u)) && !gone.has(u.split("/")[2]) && !(watch.records.get(u)?.seenAt > Date.now() - 4000)).slice(0, 24); uris.forEach((u) => tried.add(tag(u))); const w = watch; await pool(uris, 4, async (uri) => { const [, , did, collection, rkey] = uri.split("/"); if (!pdsCache.has(did)) pdsCache.set(did, pdsOf(ctx, did)); const pds = await pdsCache.get(did); const r = await fetch10(`${pds.replace(/\/$/, "")}/xrpc/com.atproto.repo.getRecord?${new URLSearchParams({ repo: did, collection, rkey })}`); if (!r.ok) return; const body = await r.json(); w.put(uri, body.cid, body.value); // the capture keeps it, at the rev the stream named, so a replay has the body (the repo may be gone by then) const rec = w.records.get(uri); if (mode === "live" && captured.length < MAX_CAPTURE) captured.push(JSON.stringify({ src: "getRecord", recv: Date.now(), frame: fetchedFrame(did, collection, rkey, rec.rev, body.cid, body.value) })); }); } finally { resolving = false; } } // ---- graph --------------------------------------------------------------------------------------------------------- const svg = d3.select("#graph"); const root = svg.append("g"); const groupLayer = root.append("g").attr("class", "groups"); const linkLayer = root.append("g").attr("class", "links"); const nodeLayer = root.append("g").attr("class", "nodes"); svg.append("defs").append("marker").attr("id", "arrow").attr("viewBox", "0 -4 8 8").attr("refX", 22).attr("markerWidth", 7).attr("markerHeight", 7).attr("orient", "auto").append("path").attr("d", "M0,-4L8,0L0,4").attr("class", "arrowhead"); const zoom = d3.zoom().scaleExtent([0.1, 4]).on("zoom", (e) => root.attr("transform", e.transform)); svg.call(zoom); const COL = 230; const sim = d3 .forceSimulation() .force("link", d3.forceLink().id((d) => d.id).distance(110).strength(0.15)) .force("charge", d3.forceManyBody().strength(-260)) .force("collide", d3.forceCollide(34)) .force("x", d3.forceX().strength(0.5)) .force("y", d3.forceY(0).strength(0.04)) .force("group", groupForce) .on("tick", tick); const simNodes = new Map(); // one staging network's members drift together (y only: x is the column) function groupForce(alpha) { const sums = new Map(); for (const d of sim?.nodes() ?? []) if (d.group) sums.set(d.group, (sums.get(d.group) ?? []).concat(d)); for (const [id, members] of sums) { const y = d3.mean(members, (d) => d.y); for (const d of members) d.vy += (y - d.y) * alpha * 0.15; // and everything else stays out of its box: pushed to the nearer top or bottom edge const [x0, x1, y0, y1] = [d3.min(members, (d) => d.x) - 90, d3.max(members, (d) => d.x) + 90, d3.min(members, (d) => d.y) - 60, d3.max(members, (d) => d.y) + 70]; for (const d of sim.nodes()) if (d.group !== id && d.x > x0 && d.x < x1 && d.y > y0 && d.y < y1) d.vy += ((d.y - y0 < y1 - d.y ? y0 : y1) - d.y) * alpha * 0.3; } } let fitted = false; function shape(sel) { sel.each(function (d) { const g = d3.select(this); g.selectAll("circle,rect,path.shape").remove(); if (d.type === "actor") g.insert("circle", "text").attr("r", d.role === "bidder" ? 16 : 13); else if (d.type === "contract") g.insert("rect", "text").attr("x", -16).attr("y", -11).attr("width", 32).attr("height", 22).attr("rx", 5); else if (d.type === "infra") g.insert("path", "text").attr("class", "shape").attr("d", "M-14,-14H14V14H-14Z"); else if (d.type === "roster") g.insert("path", "text").attr("class", "shape").attr("d", "M-13,0L-6.5,-12L6.5,-12L13,0L6.5,12L-6.5,12Z"); else g.insert("path", "text").attr("class", "shape").attr("d", "M0,-15L15,0L0,15L-15,0Z"); }); } function draw(force = false) { const now = mode === "replay" ? player.clock : Date.now(); watch.id ??= Math.random(); const key = `${watch.id}:${watch.version}:${docsVersion}`; if (!force && drawn === key) return renderSide(); drawn = key; const full = topology(watch.records, docs, now); resolveMore(full.nodes, full.wants); const { nodes, links } = filterTopology(full, $("filter").value); const maxDepth = Math.max(1, ...nodes.filter((n) => n.role !== "bidder").map((n) => n.depth)); const byId = new Map(); for (const n of nodes) { let o = simNodes.get(n.id); if (!o) { const from = links.find((l) => l.target === n.id && simNodes.has(l.source)); const s = from && simNodes.get(from.source); o = { x: s ? s.x + 40 : (n.depth - 2) * COL, y: s ? s.y + (Math.random() - 0.5) * 60 : (Math.random() - 0.5) * 300 }; } Object.assign(o, n, { col: (n.role === "bidder" ? maxDepth + 1 : n.depth) * COL }); byId.set(n.id, o); } simNodes.clear(); for (const [id, o] of byId) simNodes.set(id, o); const data = [...byId.values()]; const edges = links.map((l) => ({ ...l })); sim.nodes(data); sim.force("link").links(edges); sim.force("x").x((d) => d.col); sim.alpha(Math.max(sim.alpha(), 0.3)).restart(); linkLayer .selectAll("g.link") .data(edges, (d) => d.id) .join((enter) => { const g = enter.append("g").attr("class", "link"); g.append("line").attr("marker-end", "url(#arrow)"); g.append("text"); return g; }) .attr("data-kind", (d) => d.kind) .classed("dashed", (d) => d.inferred || d.revoked || d.kind === "bid" || d.declared) .select("text") .text((d) => d.kind); nodeLayer .selectAll("g.node") .data(data, (d) => d.id) .join((enter) => { const g = enter.append("g").attr("class", "node").on("click", (_, d) => ((selected = d.id), renderSide(true))); g.call(d3.drag().on("start", (e, d) => (sim.alphaTarget(0.2).restart(), (d.fx = d.x), (d.fy = d.y))).on("drag", (e, d) => ((d.fx = e.x), (d.fy = e.y))).on("end", (e, d) => (sim.alphaTarget(0), (d.fx = null), (d.fy = null)))); g.append("text").attr("class", "label").attr("y", 28); g.append("text").attr("class", "sub").attr("y", 41); return g; }) .attr("data-type", (d) => d.type) .attr("data-status", (d) => d.status) .attr("data-role", (d) => d.role ?? "") .attr("data-memory", (d) => d.memory || null) .classed("selected", (d) => d.id === selected) .call(shape) .call((g) => g.select("text.label").text((d) => d.label)) .call((g) => g.select("text.sub").text((d) => d.sub)); $("empty").hidden = data.length > 0; if (!fitted && data.length > 3) setTimeout(fit, 1500), (fitted = true); renderSide(); } function tick() { linkLayer.selectAll("g.link").each(function (d) { const g = d3.select(this); g.select("line").attr("x1", d.source.x).attr("y1", d.source.y).attr("x2", d.target.x).attr("y2", d.target.y); g.select("text").attr("x", (d.source.x + d.target.x) / 2).attr("y", (d.source.y + d.target.y) / 2 - 3); }); nodeLayer.selectAll("g.node").attr("transform", (d) => `translate(${d.x},${d.y})`); const groups = d3.groups(sim.nodes().filter((d) => d.group), (d) => d.group).map(([id, m]) => ({ id, x0: d3.min(m, (d) => d.x) - 90, x1: d3.max(m, (d) => d.x) + 90, y0: d3.min(m, (d) => d.y) - 38, y1: d3.max(m, (d) => d.y) + 54 })); groupLayer .selectAll("g.group") .data(groups, (d) => d.id) .join((enter) => { const g = enter.append("g").attr("class", "group"); g.append("rect").attr("rx", 10); g.append("text"); return g; }) .call((g) => g.select("rect").attr("x", (d) => d.x0).attr("y", (d) => d.y0).attr("width", (d) => d.x1 - d.x0).attr("height", (d) => d.y1 - d.y0)) .call((g) => g.select("text").attr("x", (d) => d.x0 + 8).attr("y", (d) => d.y0 + 14).text((d) => simNodes.get(d.id)?.groupLabel ?? `prod VMs carrying a staging network · ${docs.get(d.id)?.handle ?? short(d.id)}`)); } function fit() { const box = root.node().getBBox(); const { width, height } = svg.node().getBoundingClientRect(); if (!box.width || !width) return; const k = Math.min(2, 0.9 * Math.min(width / box.width, height / box.height)); svg.transition().duration(500).call(zoom.transform, d3.zoomIdentity.translate(width / 2 - k * (box.x + box.width / 2), height / 2 - k * (box.y + box.height / 2)).scale(k)); } $("fit").onclick = fit; $("filter").oninput = () => draw(true); // ---- side panels --------------------------------------------------------------------------------------------------- const esc = (s) => String(s ?? "").replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ })[c]); const ago = (ms) => (ms ? `${Math.max(0, Math.round(((mode === "replay" ? player.clock : Date.now()) - ms) / 1000))} s` : ""); const host = (url) => { try { return new URL(url).host; } catch { return url; } }; function renderSide(force = false) { const s = [...watch.sources.values()].reduce((a, b) => ({ ops: a.ops + b.ops, dups: a.dups + b.dups, frames: a.frames + b.frames }), { ops: 0, dups: 0, frames: 0 }); $("totals").textContent = `${mode} · ${watch.seen.size} unique ops · ${s.dups} duplicates dropped · ${watch.records.size} records · ${simNodes.size} nodes`; const urls = new Set([...workerStatus.keys(), ...watch.sources.keys()]); $("stats").innerHTML = [...urls] .map((url) => { const w = workerStatus.get(url) ?? {}; const d = watch.sources.get(url) ?? { ops: 0, fresh: 0, dups: 0, backfill: 0, last: 0 }; return `
Click a node.
`); const recs = (n.uris ?? [n.did]).filter(Boolean); $("detail").innerHTML = `${esc(n.sub)}
${esc(n.id)}
${n.events.map((e) => esc(e.kind)).join(" → ")}
` : ""}${n.bids?.size ? `bids from ${[...n.bids].map((d) => esc(docs.get(d)?.handle ?? short(d)) + (d === (n.bidder ?? n.winner) ? " (won)" : "")).join(", ")}
` : ""}${recs .map((u) => { const r = watch.records.get(u); return `${esc(JSON.stringify(r?.record ?? null, (k, v) => (k === "user_data" || k === "signatures" ? "…" : v), 1))}