// Frames -> deduplicated ops -> records -> the market topology. No DOM, no network: the page, the capture CLI and // the test share it. import { decodeCbor } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js"; import { toBase32 } from "./node_modules/@publicdomainrelay/atlibs/src/encoding/index.js"; const T = "com.publicdomainrelay.temp."; export const PREFIXES = [T, "computer.socialweb.", "link.ident.", "dev.knotless."]; export const wanted = (collection, prefixes = PREFIXES) => typeof collection === "string" && prefixes.some((p) => collection.startsWith(p)); const fromBase64 = (text) => Uint8Array.from(atob(text.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0)); function varint(bytes, at) { let n = 0; for (let shift = 0; ; shift += 7) { const b = bytes[at++]; n += (b & 127) * 2 ** shift; if (!(b & 128)) return [n, at]; } } /** A CAR v1's blocks by CID (CIDv1, base32), as atproto commits carry them. */ export function carBlocks(bytes) { const blocks = new Map(); let [len, at] = varint(bytes, 0); at += len; while (at < bytes.length) { [len, at] = varint(bytes, at); const end = at + len; let p = varint(bytes, varint(bytes, varint(bytes, at)[1])[1])[1]; // version, codec, multihash code const [digest, q] = varint(bytes, p); p = q + digest; blocks.set(`b${toBase32(bytes.subarray(at, p))}`, bytes.subarray(p, end)); at = end; } return blocks; } /** * Any frame shape -> { did, rev, seq?, time, ops: [{ collection, rkey, action, cid, record }] }, or null. * Jetstream commits; relay commits as atlibs decodeFrame gives them (blocks { $bytes } CAR), as goat firehose prints * them ({ type, payload }), as a JSON-framed relay sends them (blocks {}), or trimmed (ops[].record). */ export function commitOf(frame) { if (!frame || typeof frame !== "object") return null; if (frame.kind === "commit" && frame.commit) { const c = frame.commit; return { did: frame.did, rev: c.rev, time: frame.time_us / 1000, ops: [{ collection: c.collection, rkey: c.rkey, action: c.operation, cid: c.cid ?? null, record: c.record ?? null }] }; } if (frame.type && frame.payload) frame = { $type: `#${frame.type}`, ...frame.payload }; if (!String(frame.$type ?? "").endsWith("#commit") || !Array.isArray(frame.ops) || !frame.repo) return null; let blocks; const ops = frame.ops.map((op) => { const slash = op.path.indexOf("/"); const cid = op.cid?.$link ?? op.cid ?? null; let record = op.record ?? null; if (!record && cid && frame.blocks?.$bytes) { blocks ??= carBlocks(fromBase64(frame.blocks.$bytes)); const block = blocks.get(cid); if (block) record = decodeCbor(block)[0]; } return { collection: op.path.slice(0, slash), rkey: op.path.slice(slash + 1), action: op.action, cid, record }; }); return { did: frame.repo, rev: frame.rev, seq: frame.seq, time: Date.parse(frame.time), ops }; } /** A relay commit trimmed to its watched ops, records inline and no CAR; null when it has none. */ export function commitFrame(frame, prefixes = PREFIXES) { if (!String(frame?.$type ?? "").endsWith("#commit") || !frame.ops?.some((op) => wanted(op.path?.split("/")[0], prefixes))) return null; const c = commitOf(frame); const ops = c.ops.filter((op) => wanted(op.collection, prefixes)).map((op) => ({ action: op.action, path: `${op.collection}/${op.rkey}`, cid: op.cid, ...(op.record ? { record: op.record } : {}) })); return { $type: frame.$type, seq: frame.seq, repo: frame.repo, rev: frame.rev, time: frame.time, ops }; } /** A record fetched with getRecord as a Jetstream-shaped frame at the op's rev: a capture line that fills its body. */ export const fetchedFrame = (did, collection, rkey, rev, cid, record, now = Date.now()) => ({ did, time_us: now * 1000, kind: "commit", commit: { rev, operation: "update", collection, rkey, cid, record } }); /** A capture line ({ src, recv?, frame } or a bare frame) -> { src, t, recv, frame }. */ export function captureLine(line, fallbackSrc) { const o = JSON.parse(line); const frame = o && o.frame && typeof o.frame === "object" ? o.frame : o; const src = (o.frame && o.src) || fallbackSrc; const c = commitOf(frame); // replay on the event's own time (a Jetstream rewind arrives all at once); recv orders copies of one op return { src, t: c?.time || o.recv || NaN, recv: o.recv ?? 0, frame }; } const S32 = "234567abcdefghijklmnopqrstuvwxyz"; /** A TID's time in ms (its top 53 bits are microseconds), or NaN. */ export function tidTime(tid) { if (typeof tid !== "string" || !/^[2-7a-z]{13}$/.test(tid)) return NaN; let n = 0n; for (const c of tid) n = (n << 5n) | BigInt(S32.indexOf(c)); return Number(n >> 10n) / 1000; } /** * Ops deduplicated across sources by repo/collection/rkey@rev, and the latest record at each URI. * since (ms): an op whose commit (rev TID, else record createdAt) is older is backfill (a relay re-emitting a repo's * history after a restart): counted per source, kept out of the records and the feed. */ export class Watch { constructor(prefixes = PREFIXES, since = -Infinity) { this.prefixes = prefixes; this.since = since; this.seen = new Map(); this.records = new Map(); this.sources = new Map(); this.version = 0; } stat(src) { let s = this.sources.get(src); if (!s) this.sources.set(src, (s = { frames: 0, ops: 0, fresh: 0, dups: 0, backfill: 0, last: 0 })); return s; } /** Returns the ops this frame was first to deliver. */ ingest(src, frame, recv = Date.now()) { const s = this.stat(src); s.frames++; s.last = recv; const c = commitOf(frame); if (!c?.did) return []; const fresh = []; for (const op of c.ops) { if (!wanted(op.collection, this.prefixes)) continue; const at = tidTime(c.rev) || Date.parse(op.record?.createdAt); if (at < this.since) { s.backfill++; continue; } s.ops++; const key = `${c.did}/${op.collection}/${op.rkey}@${c.rev ?? op.cid}`; const prior = this.seen.get(key); if (prior) { s.dups++; prior.srcs.add(src); if (!prior.record && op.record) (prior.record = op.record), this.apply(prior); continue; } s.fresh++; const e = { key, did: c.did, ...op, rev: c.rev ?? "", time: c.time || recv, recv, srcs: new Set([src]) }; this.seen.set(key, e); this.apply(e); fresh.push(e); } return fresh; } apply(e) { const uri = `at://${e.did}/${e.collection}/${e.rkey}`; const cur = this.records.get(uri); if (cur && cur.rev > e.rev) return; const deleted = e.action === "delete"; // an update without a body (the JSON relay) keeps the previous body, marked stale until fetched or delivered const fresh = deleted || e.record || cur?.cid === e.cid; const record = e.record ?? cur?.record ?? null; this.records.set(uri, { uri, did: e.did, collection: e.collection, rkey: e.rkey, rev: e.rev, cid: e.cid ?? cur?.cid ?? null, time: e.time, first: cur?.first ?? e.time, seenAt: Date.now(), deleted, record, stale: !fresh || (cur?.stale && !e.record) }); this.version++; } /** A record fetched by getRecord, for an op whose source carried no body (or a strongRef target). */ put(uri, cid, record) { const cur = this.records.get(uri); if (cur?.record && !cur.stale) return; const [, , did, collection, rkey] = uri.split("/"); this.records.set(uri, { uri, did, collection, rkey, rev: cur?.rev ?? "", cid, time: cur?.time ?? (Date.parse(record?.createdAt) || Date.now()), first: cur?.first ?? Date.now(), deleted: cur?.deleted ?? false, fetched: true, record, stale: false }); this.version++; } } const didOf = (uri) => (typeof uri === "string" ? uri.split("/")[2] : undefined); const collOf = (uri) => (typeof uri === "string" ? uri.split("/")[3] : undefined); const timeOf = (r) => Date.parse(r?.record?.createdAt) || r?.time || 0; const STATUS = ["bidding", "accepted", "held", "running", "released"]; const RELEASE = new Set(["vm.delete", "vm.terminated", "guest.r2Released"]); // a VM's memory events (the guest agent's, the host cgroup's): an OOM kill marks it red, pressure amber const MEMORY_KILL = new Set(["guest.oomKill", "vm.oomKilled"]); const MEMORY_PRESSURE = new Set(["guest.memoryPressure", "vm.memoryPressure"]); const RUNNING = new Set(["vm.started", "vm.onNetwork", "vm.hostKeys", "guest.publicKey", "guest.r2Start", "guest.requesterDid", "guest.agentStart"]); const TOPO = new Set(["market.rfp", "market.bid", "market.accept", "market.receipt", "market.event", "market.association", "market.associationRequest", "compute.vm", "compute.events.guest.requesterDid"].map((c) => T + c).concat(["link.ident.endorsement", "dev.knotless.check", "computer.socialweb.infra", "computer.socialweb.ephemeralInfra", "computer.socialweb.mesh.roster"])); export const short = (did) => (did ? did.replace(/^did:plc:/, "").slice(0, 8) : "?"); /** * The market topology from the records: CI runs (dev.knotless.check) -> requesters -> contracts (rented VMs) -> * guest requesters (R2, named by guest.requesterDid) -> their contracts, every contract -> its bidder; principal -> * agent (market.association, link.ident.endorsement, alsoKnownAs agent-of). `wants`: URIs worth a getRecord. * docs: did -> { handle, agentOf? }. */ export function topology(records, docs = new Map(), now = Date.now()) { const nodes = new Map(); const links = new Map(); const wants = new Set(); const name = (did) => docs.get(did)?.handle ?? short(did); const actor = (did, role) => { let n = nodes.get(did); if (!n) nodes.set(did, (n = { id: did, type: "actor", did, roles: new Set(), uris: [] })); if (role) n.roles.add(role); return n; }; const link = (source, target, kind, extra = {}) => { const id = `${source}|${target}|${kind}`; links.set(id, { id, source, target, kind, ...links.get(id), ...extra }); }; const contracts = new Map(); const contract = (rfp) => { let c = contracts.get(rfp); if (!c) contracts.set(rfp, (c = { id: rfp, type: "contract", requester: didOf(rfp), bids: new Set(), events: [], uris: [] })); return c; }; const byReceipt = new Map(); const beats = new Map(); for (const r of records.values()) { if ((!r.record || r.stale) && !r.deleted && TOPO.has(r.collection)) wants.add(r.uri); const v = r.record ?? {}; switch (r.collection) { case `${T}market.rfp`: { const c = contract(r.uri); c.rfp = r; c.vm = v.payload?.uri; c.uris.push(r.uri); break; } // a bid is a count on its RFP: only the winner gets a node and an edge case `${T}market.bid`: if (v.rfp?.uri) contract(v.rfp.uri).bids.add(r.did), contract(v.rfp.uri).uris.push(r.uri); break; case `${T}market.accept`: if (v.rfp?.uri) { const c = contract(v.rfp.uri); c.accept = r; c.winner = didOf(v.bid?.uri); c.uris.push(r.uri); } break; case `${T}market.receipt`: if (v.rfp?.uri) { const c = contract(v.rfp.uri); c.receipt = r; c.bidder = r.did; c.vm ??= v.payload?.uri; c.uris.push(r.uri); byReceipt.set(r.uri, c); } break; case `${T}market.offering`: case `${T}market.heartbeat`: if (r.collection.endsWith("heartbeat") && !r.deleted) beats.set(r.did, Math.max(beats.get(r.did) ?? 0, timeOf(r))); break; case `${T}market.association`: if (v.agent) actor(r.did, "principal"), actor(v.agent, "agent"), link(r.did, v.agent, "agent-of", { grant: !r.deleted, revoked: r.deleted, expiresAt: v.expiresAt }); break; case "link.ident.endorsement": if (v.agent) actor(r.did, "principal"), actor(v.agent, "agent"), link(r.did, v.agent, "agent-of", { endorsed: !r.deleted }); break; case "computer.socialweb.peer": if (!r.deleted) actor(r.did).peer = Object.keys(v).filter((k) => ["iroh", "webrtc"].includes(k) && v[k]); break; case "computer.socialweb.mesh.roster": actor(r.did, "orchestrator"); nodes.set(r.uri, { id: r.uri, type: "roster", did: r.did, label: `mesh ${v.zone ?? r.rkey}`, peers: (v.peers ?? []).map((p) => p.name).filter(Boolean), status: r.deleted ? "released" : "running", group: r.did, uris: [r.uri] }); link(r.did, r.uri, "orchestrates"); break; case "dev.knotless.check": { const run = actor(r.did, "runner"); const repo = v.repo?.uri; nodes.set(r.uri, { id: r.uri, type: "check", did: r.did, runner: run.id, repo, owner: didOf(repo), repoName: repo?.split("/").pop(), workflow: v.workflow, commit: v.commit, status: v.status ?? "?", startedAt: Date.parse(v.startedAt), finishedAt: Date.parse(v.finishedAt), deleted: r.deleted, uris: [r.uri] }); break; } } } for (const [did, doc] of docs) if (doc.agentOf && nodes.has(did)) actor(doc.agentOf, "principal"), link(doc.agentOf, did, "agent-of", { declared: true }); for (const r of records.values()) { if (r.collection !== `${T}market.event`) continue; const c = byReceipt.get(r.record?.receipt?.uri); if (!c) continue; const pay = r.record.payload?.uri; const kind = collOf(pay)?.replace(`${T}compute.events.`, ""); c.events.push({ kind, time: timeOf(r) }); c.uris.push(r.uri); if (kind === "vm.terminated") { const p = records.get(pay); if (p?.record?.reason === "oom") c.oomTerminated = true; else if (!p?.record) wants.add(pay); } if (kind === "guest.requesterDid") { const did = records.get(pay)?.record?.did; if (did) c.guest = did; else wants.add(pay); } } for (const c of contracts.values()) { const vm = records.get(c.vm)?.record; const kinds = new Set(c.events.map((e) => e.kind)); const gone = [c.rfp, c.accept, c.receipt].some((r) => r?.deleted); c.memory = c.oomTerminated || [...kinds].some((k) => MEMORY_KILL.has(k)) ? "kill" : [...kinds].some((k) => MEMORY_PRESSURE.has(k)) ? "pressure" : ""; c.status = gone || [...kinds].some((k) => RELEASE.has(k)) ? "released" : [...kinds].some((k) => RUNNING.has(k)) ? "running" : c.receipt ? "held" : c.accept ? "accepted" : "bidding"; // a released contract's compute.vm is deleted with it if (c.vm && !vm && c.status !== "released") wants.add(c.vm); c.created = timeOf(c.rfp) || timeOf(c.accept) || timeOf(c.receipt); c.vmRole = vm?.role ?? ""; // a stg- VM is a prod market VM that carries one layer of a staging network (its insides are not on prod) c.label = c.vmRole.startsWith("stg-") ? `prod VM hosting ${c.vmRole}` : (c.vmRole || c.id.split("/").pop()); c.size = vm ? `${vm.cpus ?? "?"} vCPU / ${vm.mem ?? "?"}` : ""; c.uris.push(c.vm); nodes.set(c.id, c); actor(c.requester, "requester"); link(c.requester, c.id, "rents"); const bidder = c.bidder ?? c.winner; if (bidder) (actor(bidder, "bidder").heartbeat = beats.get(bidder)), link(c.id, bidder, "held by", { held: !!c.bidder }); if (c.guest) actor(c.guest, "guest"), link(c.id, c.guest, "runs requester"); // a staging stand-up names its layers stg-atproto, stg-bidder-N, stg-gateway: one staging network per requester if (c.vmRole.startsWith("stg-")) c.group = c.requester; } // a kcp Infra / EphemeralInfra (computer.socialweb.infra, .ephemeralInfra): a box owned by its repo's DID around // its contracts, matched by status.contracts[].receipt, else by name or vmName = the VM's role for (const r of records.values()) { if (r.collection !== "computer.socialweb.infra" && r.collection !== "computer.socialweb.ephemeralInfra") continue; const v = r.record ?? {}; const specs = v.contracts ?? []; const st = v.status ?? {}; if (!specs.length && !st.contracts?.length) continue; // infra/self is only an index const phase = r.deleted ? "deleted" : (st.phase ?? "?"); const eph = r.collection.endsWith("ephemeralInfra"); const ttl = eph && v.ttlSeconds ? `ttl ${v.ttlSeconds >= 3600 ? `${Math.round(v.ttlSeconds / 3600)} h` : `${Math.round(v.ttlSeconds / 60)} min`}` : ""; const expires = st.expiresAt ? `expires ${st.expiresAt.slice(5, 16).replace("T", " ")}Z` : ""; nodes.set(r.uri, { id: r.uri, type: "infra", did: r.did, label: `${eph ? "ephemeralInfra" : "infra"} ${r.rkey}`, status: /released|ended|deleted|failed/i.test(phase) ? "released" : "running", phase, sub: [phase, ttl, v.endWhen ? `end when ${v.endWhen}` : "", expires, st.endReason ?? ""].filter(Boolean).join(" · "), group: r.uri, groupLabel: `${eph ? "ephemeralInfra" : "infra"} ${r.rkey}: prod VMs carrying a staging network · ${phase}${ttl ? ` · ${ttl}` : ""}${v.endWhen ? ` · end when ${v.endWhen}` : ""}${expires ? ` · ${expires}` : ""}`, uris: [r.uri] }); link(r.did, r.uri, "owns"); for (const spec of specs.length ? specs : st.contracts) { const cs = st.contracts?.find((x) => x.name === spec.name) ?? {}; // by name: the newest of this repo's contracts with that role made since the record (a role repeats per run) const since = (Date.parse(v.createdAt) || 0) - 60_000; const c = byReceipt.get(cs.receipt) ?? [...contracts.values()].filter((c) => c.requester === r.did && c.created >= since && [spec.name, spec.vmName].includes(c.vmRole)).sort((a, b) => b.created - a.created)[0]; if (!c) continue; c.group = r.uri; c.infraPhase = cs.phase; link(r.uri, c.id, "contract"); } } for (const m of nodes.values()) if (m.type === "roster") for (const c of contracts.values()) if (c.requester === m.did && m.peers.includes(c.vmRole)) link(m.id, c.id, "mesh peer"); // likely gone (tombstoned): a guest of released contracts only, an agent whose grants are all revoked for (const n of nodes.values()) { if (n.type !== "actor") continue; const hosts = [...contracts.values()].filter((c) => c.guest === n.did); const grants = [...links.values()].filter((l) => l.kind === "agent-of" && l.target === n.did); n.gone = (hosts.length > 0 && hosts.every((c) => c.status === "released")) || (grants.length > 0 && grants.every((l) => l.revoked)); } // a CI run rents as its pusher, here the repo's owner: a contract made while runs went belongs to the one that // started last before it (inferred; workflows of one push run side by side) const checks = [...nodes.values()].filter((n) => n.type === "check"); for (const c of contracts.values()) { const run = checks .filter((n) => c.requester === n.owner && c.created >= n.startedAt && c.created <= (Number.isFinite(n.finishedAt) ? n.finishedAt : now)) .sort((a, b) => b.startedAt - a.startedAt)[0]; if (run) link(run.id, c.id, "rents (time)", { inferred: true }); } // depth: checks 0, requesters 1, a contract one past its requester, a guest one past its host contract, an agent // one past its principal; bidders get the last column at render time // a requester none of whose contracts a CI run made is a hand run: the root of its own tree const fromCheck = new Set([...links.values()].filter((l) => l.kind === "rents (time)").map((l) => l.target)); for (const n of nodes.values()) { n.depth = n.type === "check" ? 0 : 1; if (n.type !== "actor" || !n.roles.has("requester") || ["guest", "agent", "bidder"].some((r) => n.roles.has(r))) continue; const own = [...contracts.values()].filter((c) => c.requester === n.did); if (own.length && !own.some((c) => fromCheck.has(c.id))) (n.depth = 0), (n.handRun = true); } for (let i = 0; i < 6; i++) for (const l of links.values()) { const s = nodes.get(l.source); const t = nodes.get(l.target); if (!s || !t || t.roles?.has("bidder")) continue; if (l.kind === "rents" || l.kind === "orchestrates" || l.kind === "owns" || l.kind === "contract" || l.kind === "runs requester" || (l.kind === "agent-of" && !t.roles.has("requester"))) t.depth = Math.max(t.depth, s.depth + 1); if (l.kind === "agent-of" && t.roles.has("guest")) t.depth = Math.max(t.depth, s.depth + 1); } for (const n of nodes.values()) { if (n.type === "actor") { n.label = name(n.did); n.role = n.roles.has("guest") ? "guest" : n.roles.has("bidder") ? "bidder" : n.roles.has("runner") ? "runner" : n.roles.has("requester") ? "requester" : n.roles.has("agent") ? "agent" : "principal"; n.sub = [n.handRun ? "hand run" : "", ...n.roles, n.peer?.length ? `mesh peer (${n.peer.join(", ")})` : ""].filter(Boolean).join(", "); n.status = n.role === "bidder" ? (n.heartbeat && now - n.heartbeat < 300_000 ? "fresh" : "stale") : n.role; } else if (n.type === "check") { n.label = `${n.repoName}: ${n.workflow}`; n.sub = `${n.commit?.slice(0, 8) ?? ""} ${n.status}${Number.isFinite(n.finishedAt) ? "" : " (running)"}`; } else if (n.type === "roster") n.sub = `${n.peers.length} peers${n.status === "released" ? " · torn down" : ""}`; else if (n.type === "infra") continue; else n.sub = [n.size, n.status, n.memory === "kill" ? "OOM killed" : n.memory === "pressure" ? "memory pressure" : "", n.vmRole?.startsWith("stg-") && (n.bidder ?? n.winner) ? `on ${name(n.bidder ?? n.winner)}` : "", n.infraPhase && n.infraPhase !== n.status ? `infra: ${n.infraPhase}` : "", n.bids.size ? `${n.bids.size} bid${n.bids.size > 1 ? "s" : ""}` : ""].filter(Boolean).join(" · "); } return { nodes: [...nodes.values()], links: [...links.values()].filter((l) => nodes.has(l.source) && nodes.has(l.target)), wants: [...wants] }; } /** The nodes a filter keeps: matches, everything downstream of them and their ancestors (not through bidders). */ export function filterTopology({ nodes, links }, text) { const terms = text.toLowerCase().split(/[\s,]+/).filter(Boolean); if (!terms.length) return { nodes, links }; const hay = (n) => [n.id, n.label, n.repo, n.commit, n.repoName, n.workflow].filter(Boolean).join(" ").toLowerCase(); const keep = new Set(nodes.filter((n) => terms.some((t) => hay(n).includes(t))).map((n) => n.id)); const byId = new Map(nodes.map((n) => [n.id, n])); const walk = (from, to) => { const queue = [...keep]; const seen = new Set(queue); while (queue.length) { const id = queue.shift(); if (byId.get(id)?.role === "bidder" && !keep.has(id)) continue; for (const l of links) if (l[from] === id && !seen.has(l[to])) seen.add(l[to]), queue.push(l[to]); } return seen; }; const all = new Set([...walk("source", "target"), ...walk("target", "source")]); return { nodes: nodes.filter((n) => all.has(n.id)), links: links.filter((l) => all.has(l.source) && all.has(l.target)) }; } export { STATUS };