~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-watch

The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay

0
0
main
410 lines · 18.3 KB Raw History
// The page: sources in workers, one Watch (dedup + records), the topology drawn with d3, replay of a capture file.
// Nothing persists: a reload starts from nothing.
import { Watch, topology, filterTopology, captureLine, short, PREFIXES, fetchedFrame } from "./model.js";

// a live Watch takes commits from the rewind window on (a minute's slack); a replay from its first frame's window
const liveWatch = () => new Watch(PREFIXES, Date.now() - ((Number($("rewind").value) || 0) + 1) * 60_000);
import { DEFAULT_SOURCES } from "./sources.js";
import { pdsOf, resolveDid } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/index.js";

const $ = (id) => document.getElementById(id);
const params = new URLSearchParams(location.search);
// every lookup gives up after 10 s: one hung PDS must not stall the rest
const fetch10 = (url, init = {}) => fetch(url, { ...init, signal: AbortSignal.timeout(10_000) });
const ctx = { http: { fetch: fetch10 } };
const MAX_CAPTURE = 300_000;

let watch = new Watch(PREFIXES, Date.now());
let mode = "live";
let workers = new Map();
let workerStatus = new Map();
let captured = [];
let log = [];
let drawn = "";
let selected = null;
const docs = new Map();
let docsVersion = 0;
const tried = new Set();
const pdsCache = new Map();

// ---- sources -----------------------------------------------------------------------------------------------------

$("sources").value = (params.get("sources")?.split(",").map((url) => ({ url, on: true })) ?? DEFAULT_SOURCES).map((s) => `${s.on ? "" : "# "}${s.url}`).join("\n");
$("rewind").value = params.get("rewind") ?? "30";
$("filter").value = params.get("filter") ?? "";

function sourceList() {
	return $("sources")
		.value.split("\n")
		.map((l) => l.trim())
		.filter((l) => l && !l.startsWith("#"));
}

function stopSources() {
	for (const w of workers.values()) w.terminate();
	workers = new Map();
	workerStatus = new Map();
}

function startSources() {
	stopSources();
	const rewindMinutes = Number($("rewind").value) || 0;
	for (const url of sourceList()) {
		try {
			new URL(url);
		} catch {
			workerStatus.set(url, { state: "bad url" });
			continue;
		}
		const w = new Worker(new URL("./worker.js", import.meta.url), { type: "module" });
		w.onmessage = ({ data }) => {
			if (data.status) return void workerStatus.set(url, data.status);
			ingest(url, data.frame, data.recv);
			if (captured.length < MAX_CAPTURE) captured.push(JSON.stringify({ src: url, recv: data.recv, frame: data.frame }));
		};
		w.onerror = (e) => workerStatus.set(url, { state: "error", error: e.message });
		w.postMessage({ url, prefixes: PREFIXES, rewindMinutes });
		workers.set(url, w);
	}
}

function ingest(src, frame, recv) {
	// heartbeats count in the source table, never in the feed
	for (const op of watch.ingest(src, frame, recv)) if (!op.collection.endsWith(".market.heartbeat")) log.push(op);
	if (log.length > 400) log = log.slice(-200);
}

function goLive() {
	player.stop();
	mode = "live";
	watch = liveWatch();
	log = [];
	captured = [];
	tried.clear();
	startSources();
	draw(true);
}

$("apply").onclick = goLive;
$("live").onclick = goLive;
$("save").onclick = () => {
	const blob = new Blob([captured.join("\n") + "\n"], { type: "application/x-ndjson" });
	const a = Object.assign(document.createElement("a"), { href: URL.createObjectURL(blob), download: `swc-watch-${new Date().toISOString().replace(/[:.]/g, "")}.ndjson` });
	a.click();
	URL.revokeObjectURL(a.href);
};

// ---- identity and missing records (getRecord at the repo's PDS, resolved with atlibs) ------------------------------

async function pool(items, n, fn) {
	const queue = [...items];
	await Promise.all(Array.from({ length: n }, async () => {
		while (queue.length) await fn(queue.shift()).catch(() => {});
	}));
}

let resolving = false;
async function resolveMore(nodes, wants) {
	// not while a Jetstream replays its rewind: a DID seen there may be tombstoned by now, its release still to come
	const behind = [...workerStatus.values()].some((w) => w.cursor > 1e15 && w.cursor / 1000 < Date.now() - 60_000);
	if (resolving || (mode === "live" && behind)) return;
	resolving = true;
	try {
				// a released guest's DID is tombstoned: resolving it is only a 404/410 in the console
		// a replay's guests and requesters may be long gone (tombstoned): only the lasting accounts resolve; it fetches no records either (the capture is what happened)
		const dids = nodes.filter((n) => n.type === "actor" && !n.gone && !docs.has(n.did) && (mode === "live" || ["bidder", "runner", "principal"].includes(n.role))).map((n) => n.did).slice(0, 24);
		await pool(dids, 4, async (did) => {
			docs.set(did, { handle: short(did) });
			const doc = await resolveDid(ctx, did);
			const handle = doc?.alsoKnownAs?.find((a) => a.startsWith("at://"))?.slice(5);
			const agentOf = doc?.alsoKnownAs?.find((a) => a.startsWith("https://ident.link/agent-of/"))?.split("/").pop();
			docs.set(did, { handle: handle ?? `${short(did)}${doc ? "" : " (gone)"}`, agentOf, gone: !doc });
			docsVersion++;
		});
		const gone = new Set(nodes.filter((n) => n.gone || docs.get(n.did)?.gone).map((n) => n.did));
		// a body-less op (the JSON relay sends none) waits 4 s for another source to bring it
		// tried per version (uri@cid): a record updated again is fetched again
		const tag = (u) => `${u}@${watch.records.get(u)?.cid ?? ""}`;
		const uris = mode !== "live" ? [] : wants.filter((u) => !tried.has(tag(u)) && !gone.has(u.split("/")[2]) && !(watch.records.get(u)?.seenAt > Date.now() - 4000)).slice(0, 24);
		uris.forEach((u) => tried.add(tag(u)));
		const w = watch;
		await pool(uris, 4, async (uri) => {
			const [, , did, collection, rkey] = uri.split("/");
			if (!pdsCache.has(did)) pdsCache.set(did, pdsOf(ctx, did));
			const pds = await pdsCache.get(did);
			const r = await fetch10(`${pds.replace(/\/$/, "")}/xrpc/com.atproto.repo.getRecord?${new URLSearchParams({ repo: did, collection, rkey })}`);
			if (!r.ok) return;
			const body = await r.json();
			w.put(uri, body.cid, body.value);
			// the capture keeps it, at the rev the stream named, so a replay has the body (the repo may be gone by then)
			const rec = w.records.get(uri);
			if (mode === "live" && captured.length < MAX_CAPTURE) captured.push(JSON.stringify({ src: "getRecord", recv: Date.now(), frame: fetchedFrame(did, collection, rkey, rec.rev, body.cid, body.value) }));
		});
	} finally {
		resolving = false;
	}
}

// ---- graph ---------------------------------------------------------------------------------------------------------

const svg = d3.select("#graph");
const root = svg.append("g");
const groupLayer = root.append("g").attr("class", "groups");
const linkLayer = root.append("g").attr("class", "links");
const nodeLayer = root.append("g").attr("class", "nodes");
svg.append("defs").append("marker").attr("id", "arrow").attr("viewBox", "0 -4 8 8").attr("refX", 22).attr("markerWidth", 7).attr("markerHeight", 7).attr("orient", "auto").append("path").attr("d", "M0,-4L8,0L0,4").attr("class", "arrowhead");
const zoom = d3.zoom().scaleExtent([0.1, 4]).on("zoom", (e) => root.attr("transform", e.transform));
svg.call(zoom);
const COL = 230;
const sim = d3
	.forceSimulation()
	.force("link", d3.forceLink().id((d) => d.id).distance(110).strength(0.15))
	.force("charge", d3.forceManyBody().strength(-260))
	.force("collide", d3.forceCollide(34))
	.force("x", d3.forceX().strength(0.5))
	.force("y", d3.forceY(0).strength(0.04))
	.force("group", groupForce)
	.on("tick", tick);
const simNodes = new Map();

// one staging network's members drift together (y only: x is the column)
function groupForce(alpha) {
	const sums = new Map();
	for (const d of sim?.nodes() ?? []) if (d.group) sums.set(d.group, (sums.get(d.group) ?? []).concat(d));
	for (const [id, members] of sums) {
		const y = d3.mean(members, (d) => d.y);
		for (const d of members) d.vy += (y - d.y) * alpha * 0.15;
		// and everything else stays out of its box: pushed to the nearer top or bottom edge
		const [x0, x1, y0, y1] = [d3.min(members, (d) => d.x) - 90, d3.max(members, (d) => d.x) + 90, d3.min(members, (d) => d.y) - 60, d3.max(members, (d) => d.y) + 70];
		for (const d of sim.nodes())
			if (d.group !== id && d.x > x0 && d.x < x1 && d.y > y0 && d.y < y1) d.vy += ((d.y - y0 < y1 - d.y ? y0 : y1) - d.y) * alpha * 0.3;
	}
}
let fitted = false;

function shape(sel) {
	sel.each(function (d) {
		const g = d3.select(this);
		g.selectAll("circle,rect,path.shape").remove();
		if (d.type === "actor") g.insert("circle", "text").attr("r", d.role === "bidder" ? 16 : 13);
		else if (d.type === "contract") g.insert("rect", "text").attr("x", -16).attr("y", -11).attr("width", 32).attr("height", 22).attr("rx", 5);
		else if (d.type === "infra") g.insert("path", "text").attr("class", "shape").attr("d", "M-14,-14H14V14H-14Z");
		else if (d.type === "roster") g.insert("path", "text").attr("class", "shape").attr("d", "M-13,0L-6.5,-12L6.5,-12L13,0L6.5,12L-6.5,12Z");
		else g.insert("path", "text").attr("class", "shape").attr("d", "M0,-15L15,0L0,15L-15,0Z");
	});
}

function draw(force = false) {
	const now = mode === "replay" ? player.clock : Date.now();
	watch.id ??= Math.random();
	const key = `${watch.id}:${watch.version}:${docsVersion}`;
	if (!force && drawn === key) return renderSide();
	drawn = key;
	const full = topology(watch.records, docs, now);
	resolveMore(full.nodes, full.wants);
	const { nodes, links } = filterTopology(full, $("filter").value);
	const maxDepth = Math.max(1, ...nodes.filter((n) => n.role !== "bidder").map((n) => n.depth));
	const byId = new Map();
	for (const n of nodes) {
		let o = simNodes.get(n.id);
		if (!o) {
			const from = links.find((l) => l.target === n.id && simNodes.has(l.source));
			const s = from && simNodes.get(from.source);
			o = { x: s ? s.x + 40 : (n.depth - 2) * COL, y: s ? s.y + (Math.random() - 0.5) * 60 : (Math.random() - 0.5) * 300 };
		}
		Object.assign(o, n, { col: (n.role === "bidder" ? maxDepth + 1 : n.depth) * COL });
		byId.set(n.id, o);
	}
	simNodes.clear();
	for (const [id, o] of byId) simNodes.set(id, o);
	const data = [...byId.values()];
	const edges = links.map((l) => ({ ...l }));
	sim.nodes(data);
	sim.force("link").links(edges);
	sim.force("x").x((d) => d.col);
	sim.alpha(Math.max(sim.alpha(), 0.3)).restart();

	linkLayer
		.selectAll("g.link")
		.data(edges, (d) => d.id)
		.join((enter) => {
			const g = enter.append("g").attr("class", "link");
			g.append("line").attr("marker-end", "url(#arrow)");
			g.append("text");
			return g;
		})
		.attr("data-kind", (d) => d.kind)
		.classed("dashed", (d) => d.inferred || d.revoked || d.kind === "bid" || d.declared)
		.select("text")
		.text((d) => d.kind);

	nodeLayer
		.selectAll("g.node")
		.data(data, (d) => d.id)
		.join((enter) => {
			const g = enter.append("g").attr("class", "node").on("click", (_, d) => ((selected = d.id), renderSide(true)));
			g.call(d3.drag().on("start", (e, d) => (sim.alphaTarget(0.2).restart(), (d.fx = d.x), (d.fy = d.y))).on("drag", (e, d) => ((d.fx = e.x), (d.fy = e.y))).on("end", (e, d) => (sim.alphaTarget(0), (d.fx = null), (d.fy = null))));
			g.append("text").attr("class", "label").attr("y", 28);
			g.append("text").attr("class", "sub").attr("y", 41);
			return g;
		})
		.attr("data-type", (d) => d.type)
		.attr("data-status", (d) => d.status)
		.attr("data-role", (d) => d.role ?? "")
		.attr("data-memory", (d) => d.memory || null)
		.classed("selected", (d) => d.id === selected)
		.call(shape)
		.call((g) => g.select("text.label").text((d) => d.label))
		.call((g) => g.select("text.sub").text((d) => d.sub));
	$("empty").hidden = data.length > 0;
	if (!fitted && data.length > 3) setTimeout(fit, 1500), (fitted = true);
	renderSide();
}

function tick() {
	linkLayer.selectAll("g.link").each(function (d) {
		const g = d3.select(this);
		g.select("line").attr("x1", d.source.x).attr("y1", d.source.y).attr("x2", d.target.x).attr("y2", d.target.y);
		g.select("text").attr("x", (d.source.x + d.target.x) / 2).attr("y", (d.source.y + d.target.y) / 2 - 3);
	});
	nodeLayer.selectAll("g.node").attr("transform", (d) => `translate(${d.x},${d.y})`);
	const groups = d3.groups(sim.nodes().filter((d) => d.group), (d) => d.group).map(([id, m]) => ({ id, x0: d3.min(m, (d) => d.x) - 90, x1: d3.max(m, (d) => d.x) + 90, y0: d3.min(m, (d) => d.y) - 38, y1: d3.max(m, (d) => d.y) + 54 }));
	groupLayer
		.selectAll("g.group")
		.data(groups, (d) => d.id)
		.join((enter) => {
			const g = enter.append("g").attr("class", "group");
			g.append("rect").attr("rx", 10);
			g.append("text");
			return g;
		})
		.call((g) => g.select("rect").attr("x", (d) => d.x0).attr("y", (d) => d.y0).attr("width", (d) => d.x1 - d.x0).attr("height", (d) => d.y1 - d.y0))
		.call((g) => g.select("text").attr("x", (d) => d.x0 + 8).attr("y", (d) => d.y0 + 14).text((d) => simNodes.get(d.id)?.groupLabel ?? `prod VMs carrying a staging network · ${docs.get(d.id)?.handle ?? short(d.id)}`));
}

function fit() {
	const box = root.node().getBBox();
	const { width, height } = svg.node().getBoundingClientRect();
	if (!box.width || !width) return;
	const k = Math.min(2, 0.9 * Math.min(width / box.width, height / box.height));
	svg.transition().duration(500).call(zoom.transform, d3.zoomIdentity.translate(width / 2 - k * (box.x + box.width / 2), height / 2 - k * (box.y + box.height / 2)).scale(k));
}
$("fit").onclick = fit;
$("filter").oninput = () => draw(true);

// ---- side panels ---------------------------------------------------------------------------------------------------

const esc = (s) => String(s ?? "").replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]);
const ago = (ms) => (ms ? `${Math.max(0, Math.round(((mode === "replay" ? player.clock : Date.now()) - ms) / 1000))} s` : "");
const host = (url) => {
	try {
		return new URL(url).host;
	} catch {
		return url;
	}
};

function renderSide(force = false) {
	const s = [...watch.sources.values()].reduce((a, b) => ({ ops: a.ops + b.ops, dups: a.dups + b.dups, frames: a.frames + b.frames }), { ops: 0, dups: 0, frames: 0 });
	$("totals").textContent = `${mode} · ${watch.seen.size} unique ops · ${s.dups} duplicates dropped · ${watch.records.size} records · ${simNodes.size} nodes`;
	const urls = new Set([...workerStatus.keys(), ...watch.sources.keys()]);
	$("stats").innerHTML = [...urls]
		.map((url) => {
			const w = workerStatus.get(url) ?? {};
			const d = watch.sources.get(url) ?? { ops: 0, fresh: 0, dups: 0, backfill: 0, last: 0 };
			return `<tr title="${esc(w.error ?? url)}"><td><span class="dot" data-state="${esc(mode === "replay" ? "replay" : w.state)}"></span>${esc(host(url))}</td><td>${d.ops}</td><td>${d.fresh}</td><td>${d.dups}</td><td>${d.backfill ?? 0}</td><td>${ago(d.last)}</td></tr>`;
		})
		.join("");
	$("save").textContent = `Save capture (${captured.length})`;
	$("save").disabled = mode !== "live" || !captured.length;
	$("log").innerHTML = log
		.slice(-120)
		.reverse()
		.map((e) => `<li title="${esc(e.key)} via ${esc([...e.srcs].map(host).join(", "))}"><time>${new Date(e.time).toISOString().slice(11, 19)}</time> <b>${esc(e.action)}</b> ${esc(e.collection.replace("com.publicdomainrelay.temp.", ""))} <span class="muted">${esc(docs.get(e.did)?.handle ?? short(e.did))} ×${e.srcs.size}</span></li>`)
		.join("");
	if (!force && document.activeElement?.closest?.("#detail")) return;
	const n = simNodes.get(selected);
	if (!n) return void ($("detail").innerHTML = `<p class="muted">Click a node.</p>`);
	const recs = (n.uris ?? [n.did]).filter(Boolean);
	$("detail").innerHTML = `<h3>${esc(n.label)}</h3><p class="muted">${esc(n.sub)}</p><p><code>${esc(n.id)}</code></p>${n.events?.length ? `<p>${n.events.map((e) => esc(e.kind)).join(" → ")}</p>` : ""}${n.bids?.size ? `<p>bids from ${[...n.bids].map((d) => esc(docs.get(d)?.handle ?? short(d)) + (d === (n.bidder ?? n.winner) ? " (won)" : "")).join(", ")}</p>` : ""}${recs
		.map((u) => {
			const r = watch.records.get(u);
			return `<details><summary>${esc(u.split("/").slice(3).join("/") || u)}${r?.deleted ? " (deleted)" : ""}</summary><pre>${esc(JSON.stringify(r?.record ?? null, (k, v) => (k === "user_data" || k === "signatures" ? "…" : v), 1))}</pre></details>`;
		})
		.join("")}`;
}

// ---- replay --------------------------------------------------------------------------------------------------------

const player = {
	lines: [],
	i: 0,
	t0: 0,
	t1: 0,
	clock: 0,
	timer: null,
	stop() {
		clearInterval(this.timer);
		this.timer = null;
		$("play").textContent = "Play";
	},
	seek(t) {
		if (t < this.clock || !this.i) (watch = new Watch(PREFIXES, this.since)), (log = []), (this.i = 0);
		this.clock = t;
		while (this.i < this.lines.length && this.lines[this.i].t <= t) {
			const l = this.lines[this.i++];
			ingest(l.src, l.frame, l.t);
		}
		$("scrub").value = String(t - this.t0);
		$("clock").textContent = `${new Date(t).toISOString().replace("T", " ").slice(0, 19)}Z · ${this.i}/${this.lines.length}`;
		if (this.i >= this.lines.length) this.stop();
	},
	play() {
		if (this.timer) return this.stop();
		if (this.i >= this.lines.length) this.seek(this.t0 - 1);
		$("play").textContent = "Pause";
		this.timer = setInterval(() => {
			const speed = $("speed").value;
			this.seek(speed === "max" ? this.t1 : this.clock + 200 * Number(speed));
		}, 200);
	},
};

$("file").onchange = async () => {
	const file = $("file").files[0];
	if (!file) return;
	const lines = [];
	let bad = 0;
	for (const text of (await file.text()).split("\n")) {
		if (!text.trim()) continue;
		try {
			const l = captureLine(text, file.name);
			if (Number.isFinite(l.t)) lines.push(l);
			else bad++;
		} catch {
			bad++;
		}
	}
	lines.sort((a, b) => a.t - b.t || a.recv - b.recv);
	stopSources();
	mode = "replay";
	captured = [];
	tried.clear();
	const first = lines.reduce((m, l) => Math.min(m, l.recv || l.t), Infinity);
	Object.assign(player, { lines, i: 0, t0: lines[0]?.t ?? 0, t1: lines.at(-1)?.t ?? 0, since: first - ((Number($("rewind").value) || 0) + 1) * 60_000 });
	$("scrub").max = String(player.t1 - player.t0);
	$("replay-info").textContent = `${file.name}: ${lines.length} frames${bad ? `, ${bad} lines skipped` : ""}, ${Math.round((player.t1 - player.t0) / 1000)} s`;
	$("replaying").hidden = false;
	player.stop();
	player.seek(player.t0 - 1);
	player.seek(player.t1);
	draw(true);
	fitted = false;
};
$("play").onclick = () => player.play();
$("scrub").oninput = () => (player.stop(), player.seek(player.t0 + Number($("scrub").value)));

// ---- go ------------------------------------------------------------------------------------------------------------

setInterval(() => draw(), 700);
goLive();