johnandersen777.bsky.social/socialweb-computer-watch
The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay
0
0
// The page: sources in workers, one Watch (dedup + records), the topology drawn with d3, replay of a capture file.
// Nothing persists: a reload starts from nothing.
import { Watch, topology, filterTopology, captureLine, short, PREFIXES, fetchedFrame } from "./model.js";
// a live Watch takes commits from the rewind window on (a minute's slack); a replay from its first frame's window
const liveWatch = () => new Watch(PREFIXES, Date.now() - ((Number($("rewind").value) || 0) + 1) * 60_000);
import { DEFAULT_SOURCES } from "./sources.js";
import { pdsOf, resolveDid } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/index.js";
const $ = (id) => document.getElementById(id);
const params = new URLSearchParams(location.search);
// every lookup gives up after 10 s: one hung PDS must not stall the rest
const fetch10 = (url, init = {}) => fetch(url, { ...init, signal: AbortSignal.timeout(10_000) });
const ctx = { http: { fetch: fetch10 } };
const MAX_CAPTURE = 300_000;
let watch = new Watch(PREFIXES, Date.now());
let mode = "live";
let workers = new Map();
let workerStatus = new Map();
let captured = [];
let log = [];
let drawn = "";
let selected = null;
const docs = new Map();
let docsVersion = 0;
const tried = new Set();
const pdsCache = new Map();
// ---- sources -----------------------------------------------------------------------------------------------------
$("sources").value = (params.get("sources")?.split(",").map((url) => ({ url, on: true })) ?? DEFAULT_SOURCES).map((s) => `${s.on ? "" : "# "}${s.url}`).join("\n");
$("rewind").value = params.get("rewind") ?? "30";
$("filter").value = params.get("filter") ?? "";
function sourceList() {
return $("sources")
.value.split("\n")
.map((l) => l.trim())
.filter((l) => l && !l.startsWith("#"));
}
function stopSources() {
for (const w of workers.values()) w.terminate();
workers = new Map();
workerStatus = new Map();
}
function startSources() {
stopSources();
const rewindMinutes = Number($("rewind").value) || 0;
for (const url of sourceList()) {
try {
new URL(url);
} catch {
workerStatus.set(url, { state: "bad url" });
continue;
}
const w = new Worker(new URL("./worker.js", import.meta.url), { type: "module" });
w.onmessage = ({ data }) => {
if (data.status) return void workerStatus.set(url, data.status);
ingest(url, data.frame, data.recv);
if (captured.length < MAX_CAPTURE) captured.push(JSON.stringify({ src: url, recv: data.recv, frame: data.frame }));
};
w.onerror = (e) => workerStatus.set(url, { state: "error", error: e.message });
w.postMessage({ url, prefixes: PREFIXES, rewindMinutes });
workers.set(url, w);
}
}
function ingest(src, frame, recv) {
// heartbeats count in the source table, never in the feed
for (const op of watch.ingest(src, frame, recv)) if (!op.collection.endsWith(".market.heartbeat")) log.push(op);
if (log.length > 400) log = log.slice(-200);
}
function goLive() {
player.stop();
mode = "live";
watch = liveWatch();
log = [];
captured = [];
tried.clear();
startSources();
draw(true);
}
$("apply").onclick = goLive;
$("live").onclick = goLive;
$("save").onclick = () => {
const blob = new Blob([captured.join("\n") + "\n"], { type: "application/x-ndjson" });
const a = Object.assign(document.createElement("a"), { href: URL.createObjectURL(blob), download: `swc-watch-${new Date().toISOString().replace(/[:.]/g, "")}.ndjson` });
a.click();
URL.revokeObjectURL(a.href);
};
// ---- identity and missing records (getRecord at the repo's PDS, resolved with atlibs) ------------------------------
async function pool(items, n, fn) {
const queue = [...items];
await Promise.all(Array.from({ length: n }, async () => {
while (queue.length) await fn(queue.shift()).catch(() => {});
}));
}
let resolving = false;
async function resolveMore(nodes, wants) {
// not while a Jetstream replays its rewind: a DID seen there may be tombstoned by now, its release still to come
const behind = [...workerStatus.values()].some((w) => w.cursor > 1e15 && w.cursor / 1000 < Date.now() - 60_000);
if (resolving || (mode === "live" && behind)) return;
resolving = true;
try {
// a released guest's DID is tombstoned: resolving it is only a 404/410 in the console
// a replay's guests and requesters may be long gone (tombstoned): only the lasting accounts resolve; it fetches no records either (the capture is what happened)
const dids = nodes.filter((n) => n.type === "actor" && !n.gone && !docs.has(n.did) && (mode === "live" || ["bidder", "runner", "principal"].includes(n.role))).map((n) => n.did).slice(0, 24);
await pool(dids, 4, async (did) => {
docs.set(did, { handle: short(did) });
const doc = await resolveDid(ctx, did);
const handle = doc?.alsoKnownAs?.find((a) => a.startsWith("at://"))?.slice(5);
const agentOf = doc?.alsoKnownAs?.find((a) => a.startsWith("https://ident.link/agent-of/"))?.split("/").pop();
docs.set(did, { handle: handle ?? `${short(did)}${doc ? "" : " (gone)"}`, agentOf, gone: !doc });
docsVersion++;
});
const gone = new Set(nodes.filter((n) => n.gone || docs.get(n.did)?.gone).map((n) => n.did));
// a body-less op (the JSON relay sends none) waits 4 s for another source to bring it
// tried per version (uri@cid): a record updated again is fetched again
const tag = (u) => `${u}@${watch.records.get(u)?.cid ?? ""}`;
const uris = mode !== "live" ? [] : wants.filter((u) => !tried.has(tag(u)) && !gone.has(u.split("/")[2]) && !(watch.records.get(u)?.seenAt > Date.now() - 4000)).slice(0, 24);
uris.forEach((u) => tried.add(tag(u)));
const w = watch;
await pool(uris, 4, async (uri) => {
const [, , did, collection, rkey] = uri.split("/");
if (!pdsCache.has(did)) pdsCache.set(did, pdsOf(ctx, did));
const pds = await pdsCache.get(did);
const r = await fetch10(`${pds.replace(/\/$/, "")}/xrpc/com.atproto.repo.getRecord?${new URLSearchParams({ repo: did, collection, rkey })}`);
if (!r.ok) return;
const body = await r.json();
w.put(uri, body.cid, body.value);
// the capture keeps it, at the rev the stream named, so a replay has the body (the repo may be gone by then)
const rec = w.records.get(uri);
if (mode === "live" && captured.length < MAX_CAPTURE) captured.push(JSON.stringify({ src: "getRecord", recv: Date.now(), frame: fetchedFrame(did, collection, rkey, rec.rev, body.cid, body.value) }));
});
} finally {
resolving = false;
}
}
// ---- graph ---------------------------------------------------------------------------------------------------------
const svg = d3.select("#graph");
const root = svg.append("g");
const groupLayer = root.append("g").attr("class", "groups");
const linkLayer = root.append("g").attr("class", "links");
const nodeLayer = root.append("g").attr("class", "nodes");
svg.append("defs").append("marker").attr("id", "arrow").attr("viewBox", "0 -4 8 8").attr("refX", 22).attr("markerWidth", 7).attr("markerHeight", 7).attr("orient", "auto").append("path").attr("d", "M0,-4L8,0L0,4").attr("class", "arrowhead");
const zoom = d3.zoom().scaleExtent([0.1, 4]).on("zoom", (e) => root.attr("transform", e.transform));
svg.call(zoom);
const COL = 230;
const sim = d3
.forceSimulation()
.force("link", d3.forceLink().id((d) => d.id).distance(110).strength(0.15))
.force("charge", d3.forceManyBody().strength(-260))
.force("collide", d3.forceCollide(34))
.force("x", d3.forceX().strength(0.5))
.force("y", d3.forceY(0).strength(0.04))
.force("group", groupForce)
.on("tick", tick);
const simNodes = new Map();
// one staging network's members drift together (y only: x is the column)
function groupForce(alpha) {
const sums = new Map();
for (const d of sim?.nodes() ?? []) if (d.group) sums.set(d.group, (sums.get(d.group) ?? []).concat(d));
for (const [id, members] of sums) {
const y = d3.mean(members, (d) => d.y);
for (const d of members) d.vy += (y - d.y) * alpha * 0.15;
// and everything else stays out of its box: pushed to the nearer top or bottom edge
const [x0, x1, y0, y1] = [d3.min(members, (d) => d.x) - 90, d3.max(members, (d) => d.x) + 90, d3.min(members, (d) => d.y) - 60, d3.max(members, (d) => d.y) + 70];
for (const d of sim.nodes())
if (d.group !== id && d.x > x0 && d.x < x1 && d.y > y0 && d.y < y1) d.vy += ((d.y - y0 < y1 - d.y ? y0 : y1) - d.y) * alpha * 0.3;
}
}
let fitted = false;
function shape(sel) {
sel.each(function (d) {
const g = d3.select(this);
g.selectAll("circle,rect,path.shape").remove();
if (d.type === "actor") g.insert("circle", "text").attr("r", d.role === "bidder" ? 16 : 13);
else if (d.type === "contract") g.insert("rect", "text").attr("x", -16).attr("y", -11).attr("width", 32).attr("height", 22).attr("rx", 5);
else if (d.type === "infra") g.insert("path", "text").attr("class", "shape").attr("d", "M-14,-14H14V14H-14Z");
else if (d.type === "roster") g.insert("path", "text").attr("class", "shape").attr("d", "M-13,0L-6.5,-12L6.5,-12L13,0L6.5,12L-6.5,12Z");
else g.insert("path", "text").attr("class", "shape").attr("d", "M0,-15L15,0L0,15L-15,0Z");
});
}
function draw(force = false) {
const now = mode === "replay" ? player.clock : Date.now();
watch.id ??= Math.random();
const key = `${watch.id}:${watch.version}:${docsVersion}`;
if (!force && drawn === key) return renderSide();
drawn = key;
const full = topology(watch.records, docs, now);
resolveMore(full.nodes, full.wants);
const { nodes, links } = filterTopology(full, $("filter").value);
const maxDepth = Math.max(1, ...nodes.filter((n) => n.role !== "bidder").map((n) => n.depth));
const byId = new Map();
for (const n of nodes) {
let o = simNodes.get(n.id);
if (!o) {
const from = links.find((l) => l.target === n.id && simNodes.has(l.source));
const s = from && simNodes.get(from.source);
o = { x: s ? s.x + 40 : (n.depth - 2) * COL, y: s ? s.y + (Math.random() - 0.5) * 60 : (Math.random() - 0.5) * 300 };
}
Object.assign(o, n, { col: (n.role === "bidder" ? maxDepth + 1 : n.depth) * COL });
byId.set(n.id, o);
}
simNodes.clear();
for (const [id, o] of byId) simNodes.set(id, o);
const data = [...byId.values()];
const edges = links.map((l) => ({ ...l }));
sim.nodes(data);
sim.force("link").links(edges);
sim.force("x").x((d) => d.col);
sim.alpha(Math.max(sim.alpha(), 0.3)).restart();
linkLayer
.selectAll("g.link")
.data(edges, (d) => d.id)
.join((enter) => {
const g = enter.append("g").attr("class", "link");
g.append("line").attr("marker-end", "url(#arrow)");
g.append("text");
return g;
})
.attr("data-kind", (d) => d.kind)
.classed("dashed", (d) => d.inferred || d.revoked || d.kind === "bid" || d.declared)
.select("text")
.text((d) => d.kind);
nodeLayer
.selectAll("g.node")
.data(data, (d) => d.id)
.join((enter) => {
const g = enter.append("g").attr("class", "node").on("click", (_, d) => ((selected = d.id), renderSide(true)));
g.call(d3.drag().on("start", (e, d) => (sim.alphaTarget(0.2).restart(), (d.fx = d.x), (d.fy = d.y))).on("drag", (e, d) => ((d.fx = e.x), (d.fy = e.y))).on("end", (e, d) => (sim.alphaTarget(0), (d.fx = null), (d.fy = null))));
g.append("text").attr("class", "label").attr("y", 28);
g.append("text").attr("class", "sub").attr("y", 41);
return g;
})
.attr("data-type", (d) => d.type)
.attr("data-status", (d) => d.status)
.attr("data-role", (d) => d.role ?? "")
.attr("data-memory", (d) => d.memory || null)
.classed("selected", (d) => d.id === selected)
.call(shape)
.call((g) => g.select("text.label").text((d) => d.label))
.call((g) => g.select("text.sub").text((d) => d.sub));
$("empty").hidden = data.length > 0;
if (!fitted && data.length > 3) setTimeout(fit, 1500), (fitted = true);
renderSide();
}
function tick() {
linkLayer.selectAll("g.link").each(function (d) {
const g = d3.select(this);
g.select("line").attr("x1", d.source.x).attr("y1", d.source.y).attr("x2", d.target.x).attr("y2", d.target.y);
g.select("text").attr("x", (d.source.x + d.target.x) / 2).attr("y", (d.source.y + d.target.y) / 2 - 3);
});
nodeLayer.selectAll("g.node").attr("transform", (d) => `translate(${d.x},${d.y})`);
const groups = d3.groups(sim.nodes().filter((d) => d.group), (d) => d.group).map(([id, m]) => ({ id, x0: d3.min(m, (d) => d.x) - 90, x1: d3.max(m, (d) => d.x) + 90, y0: d3.min(m, (d) => d.y) - 38, y1: d3.max(m, (d) => d.y) + 54 }));
groupLayer
.selectAll("g.group")
.data(groups, (d) => d.id)
.join((enter) => {
const g = enter.append("g").attr("class", "group");
g.append("rect").attr("rx", 10);
g.append("text");
return g;
})
.call((g) => g.select("rect").attr("x", (d) => d.x0).attr("y", (d) => d.y0).attr("width", (d) => d.x1 - d.x0).attr("height", (d) => d.y1 - d.y0))
.call((g) => g.select("text").attr("x", (d) => d.x0 + 8).attr("y", (d) => d.y0 + 14).text((d) => simNodes.get(d.id)?.groupLabel ?? `prod VMs carrying a staging network · ${docs.get(d.id)?.handle ?? short(d.id)}`));
}
function fit() {
const box = root.node().getBBox();
const { width, height } = svg.node().getBoundingClientRect();
if (!box.width || !width) return;
const k = Math.min(2, 0.9 * Math.min(width / box.width, height / box.height));
svg.transition().duration(500).call(zoom.transform, d3.zoomIdentity.translate(width / 2 - k * (box.x + box.width / 2), height / 2 - k * (box.y + box.height / 2)).scale(k));
}
$("fit").onclick = fit;
$("filter").oninput = () => draw(true);
// ---- side panels ---------------------------------------------------------------------------------------------------
const esc = (s) => String(s ?? "").replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ })[c]);
const ago = (ms) => (ms ? `${Math.max(0, Math.round(((mode === "replay" ? player.clock : Date.now()) - ms) / 1000))} s` : "");
const host = (url) => {
try {
return new URL(url).host;
} catch {
return url;
}
};
function renderSide(force = false) {
const s = [...watch.sources.values()].reduce((a, b) => ({ ops: a.ops + b.ops, dups: a.dups + b.dups, frames: a.frames + b.frames }), { ops: 0, dups: 0, frames: 0 });
$("totals").textContent = `${mode} · ${watch.seen.size} unique ops · ${s.dups} duplicates dropped · ${watch.records.size} records · ${simNodes.size} nodes`;
const urls = new Set([...workerStatus.keys(), ...watch.sources.keys()]);
$("stats").innerHTML = [...urls]
.map((url) => {
const w = workerStatus.get(url) ?? {};
const d = watch.sources.get(url) ?? { ops: 0, fresh: 0, dups: 0, backfill: 0, last: 0 };
return `<tr title="${esc(w.error ?? url)}"><td><span class="dot" data-state="${esc(mode === "replay" ? "replay" : w.state)}"></span>${esc(host(url))}</td><td>${d.ops}</td><td>${d.fresh}</td><td>${d.dups}</td><td>${d.backfill ?? 0}</td><td>${ago(d.last)}</td></tr>`;
})
.join("");
$("save").textContent = `Save capture (${captured.length})`;
$("save").disabled = mode !== "live" || !captured.length;
$("log").innerHTML = log
.slice(-120)
.reverse()
.map((e) => `<li title="${esc(e.key)} via ${esc([...e.srcs].map(host).join(", "))}"><time>${new Date(e.time).toISOString().slice(11, 19)}</time> <b>${esc(e.action)}</b> ${esc(e.collection.replace("com.publicdomainrelay.temp.", ""))} <span class="muted">${esc(docs.get(e.did)?.handle ?? short(e.did))} ×${e.srcs.size}</span></li>`)
.join("");
if (!force && document.activeElement?.closest?.("#detail")) return;
const n = simNodes.get(selected);
if (!n) return void ($("detail").innerHTML = `<p class="muted">Click a node.</p>`);
const recs = (n.uris ?? [n.did]).filter(Boolean);
$("detail").innerHTML = `<h3>${esc(n.label)}</h3><p class="muted">${esc(n.sub)}</p><p><code>${esc(n.id)}</code></p>${n.events?.length ? `<p>${n.events.map((e) => esc(e.kind)).join(" → ")}</p>` : ""}${n.bids?.size ? `<p>bids from ${[...n.bids].map((d) => esc(docs.get(d)?.handle ?? short(d)) + (d === (n.bidder ?? n.winner) ? " (won)" : "")).join(", ")}</p>` : ""}${recs
.map((u) => {
const r = watch.records.get(u);
return `<details><summary>${esc(u.split("/").slice(3).join("/") || u)}${r?.deleted ? " (deleted)" : ""}</summary><pre>${esc(JSON.stringify(r?.record ?? null, (k, v) => (k === "user_data" || k === "signatures" ? "…" : v), 1))}</pre></details>`;
})
.join("")}`;
}
// ---- replay --------------------------------------------------------------------------------------------------------
const player = {
lines: [],
i: 0,
t0: 0,
t1: 0,
clock: 0,
timer: null,
stop() {
clearInterval(this.timer);
this.timer = null;
$("play").textContent = "Play";
},
seek(t) {
if (t < this.clock || !this.i) (watch = new Watch(PREFIXES, this.since)), (log = []), (this.i = 0);
this.clock = t;
while (this.i < this.lines.length && this.lines[this.i].t <= t) {
const l = this.lines[this.i++];
ingest(l.src, l.frame, l.t);
}
$("scrub").value = String(t - this.t0);
$("clock").textContent = `${new Date(t).toISOString().replace("T", " ").slice(0, 19)}Z · ${this.i}/${this.lines.length}`;
if (this.i >= this.lines.length) this.stop();
},
play() {
if (this.timer) return this.stop();
if (this.i >= this.lines.length) this.seek(this.t0 - 1);
$("play").textContent = "Pause";
this.timer = setInterval(() => {
const speed = $("speed").value;
this.seek(speed === "max" ? this.t1 : this.clock + 200 * Number(speed));
}, 200);
},
};
$("file").onchange = async () => {
const file = $("file").files[0];
if (!file) return;
const lines = [];
let bad = 0;
for (const text of (await file.text()).split("\n")) {
if (!text.trim()) continue;
try {
const l = captureLine(text, file.name);
if (Number.isFinite(l.t)) lines.push(l);
else bad++;
} catch {
bad++;
}
}
lines.sort((a, b) => a.t - b.t || a.recv - b.recv);
stopSources();
mode = "replay";
captured = [];
tried.clear();
const first = lines.reduce((m, l) => Math.min(m, l.recv || l.t), Infinity);
Object.assign(player, { lines, i: 0, t0: lines[0]?.t ?? 0, t1: lines.at(-1)?.t ?? 0, since: first - ((Number($("rewind").value) || 0) + 1) * 60_000 });
$("scrub").max = String(player.t1 - player.t0);
$("replay-info").textContent = `${file.name}: ${lines.length} frames${bad ? `, ${bad} lines skipped` : ""}, ${Math.round((player.t1 - player.t0) / 1000)} s`;
$("replaying").hidden = false;
player.stop();
player.seek(player.t0 - 1);
player.seek(player.t1);
draw(true);
fitted = false;
};
$("play").onclick = () => player.play();
$("scrub").oninput = () => (player.stop(), player.seek(player.t0 + Number($("scrub").value)));
// ---- go ------------------------------------------------------------------------------------------------------------
setInterval(() => draw(), 700);
goLive();