johnandersen777.bsky.social/socialweb-computer-watch
The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay
0
0
// Frames -> deduplicated ops -> records -> the market topology. No DOM, no network: the page, the capture CLI and
// the test share it.
import { decodeCbor } from "./node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js";
import { toBase32 } from "./node_modules/@publicdomainrelay/atlibs/src/encoding/index.js";
const T = "com.publicdomainrelay.temp.";
export const PREFIXES = [T, "computer.socialweb.", "link.ident.", "dev.knotless."];
export const wanted = (collection, prefixes = PREFIXES) => typeof collection === "string" && prefixes.some((p) => collection.startsWith(p));
const fromBase64 = (text) => Uint8Array.from(atob(text.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0));
function varint(bytes, at) {
let n = 0;
for (let shift = 0; ; shift += 7) {
const b = bytes[at++];
n += (b & 127) * 2 ** shift;
if (!(b & 128)) return [n, at];
}
}
/** A CAR v1's blocks by CID (CIDv1, base32), as atproto commits carry them. */
export function carBlocks(bytes) {
const blocks = new Map();
let [len, at] = varint(bytes, 0);
at += len;
while (at < bytes.length) {
[len, at] = varint(bytes, at);
const end = at + len;
let p = varint(bytes, varint(bytes, varint(bytes, at)[1])[1])[1]; // version, codec, multihash code
const [digest, q] = varint(bytes, p);
p = q + digest;
blocks.set(`b${toBase32(bytes.subarray(at, p))}`, bytes.subarray(p, end));
at = end;
}
return blocks;
}
/**
* Any frame shape -> { did, rev, seq?, time, ops: [{ collection, rkey, action, cid, record }] }, or null.
* Jetstream commits; relay commits as atlibs decodeFrame gives them (blocks { $bytes } CAR), as goat firehose prints
* them ({ type, payload }), as a JSON-framed relay sends them (blocks {}), or trimmed (ops[].record).
*/
export function commitOf(frame) {
if (!frame || typeof frame !== "object") return null;
if (frame.kind === "commit" && frame.commit) {
const c = frame.commit;
return { did: frame.did, rev: c.rev, time: frame.time_us / 1000, ops: [{ collection: c.collection, rkey: c.rkey, action: c.operation, cid: c.cid ?? null, record: c.record ?? null }] };
}
if (frame.type && frame.payload) frame = { $type: `#${frame.type}`, ...frame.payload };
if (!String(frame.$type ?? "").endsWith("#commit") || !Array.isArray(frame.ops) || !frame.repo) return null;
let blocks;
const ops = frame.ops.map((op) => {
const slash = op.path.indexOf("/");
const cid = op.cid?.$link ?? op.cid ?? null;
let record = op.record ?? null;
if (!record && cid && frame.blocks?.$bytes) {
blocks ??= carBlocks(fromBase64(frame.blocks.$bytes));
const block = blocks.get(cid);
if (block) record = decodeCbor(block)[0];
}
return { collection: op.path.slice(0, slash), rkey: op.path.slice(slash + 1), action: op.action, cid, record };
});
return { did: frame.repo, rev: frame.rev, seq: frame.seq, time: Date.parse(frame.time), ops };
}
/** A relay commit trimmed to its watched ops, records inline and no CAR; null when it has none. */
export function commitFrame(frame, prefixes = PREFIXES) {
if (!String(frame?.$type ?? "").endsWith("#commit") || !frame.ops?.some((op) => wanted(op.path?.split("/")[0], prefixes))) return null;
const c = commitOf(frame);
const ops = c.ops.filter((op) => wanted(op.collection, prefixes)).map((op) => ({ action: op.action, path: `${op.collection}/${op.rkey}`, cid: op.cid, ...(op.record ? { record: op.record } : {}) }));
return { $type: frame.$type, seq: frame.seq, repo: frame.repo, rev: frame.rev, time: frame.time, ops };
}
/** A record fetched with getRecord as a Jetstream-shaped frame at the op's rev: a capture line that fills its body. */
export const fetchedFrame = (did, collection, rkey, rev, cid, record, now = Date.now()) => ({ did, time_us: now * 1000, kind: "commit", commit: { rev, operation: "update", collection, rkey, cid, record } });
/** A capture line ({ src, recv?, frame } or a bare frame) -> { src, t, recv, frame }. */
export function captureLine(line, fallbackSrc) {
const o = JSON.parse(line);
const frame = o && o.frame && typeof o.frame === "object" ? o.frame : o;
const src = (o.frame && o.src) || fallbackSrc;
const c = commitOf(frame);
// replay on the event's own time (a Jetstream rewind arrives all at once); recv orders copies of one op
return { src, t: c?.time || o.recv || NaN, recv: o.recv ?? 0, frame };
}
const S32 = "234567abcdefghijklmnopqrstuvwxyz";
/** A TID's time in ms (its top 53 bits are microseconds), or NaN. */
export function tidTime(tid) {
if (typeof tid !== "string" || !/^[2-7a-z]{13}$/.test(tid)) return NaN;
let n = 0n;
for (const c of tid) n = (n << 5n) | BigInt(S32.indexOf(c));
return Number(n >> 10n) / 1000;
}
/**
* Ops deduplicated across sources by repo/collection/rkey@rev, and the latest record at each URI.
* since (ms): an op whose commit (rev TID, else record createdAt) is older is backfill (a relay re-emitting a repo's
* history after a restart): counted per source, kept out of the records and the feed.
*/
export class Watch {
constructor(prefixes = PREFIXES, since = -Infinity) {
this.prefixes = prefixes;
this.since = since;
this.seen = new Map();
this.records = new Map();
this.sources = new Map();
this.version = 0;
}
stat(src) {
let s = this.sources.get(src);
if (!s) this.sources.set(src, (s = { frames: 0, ops: 0, fresh: 0, dups: 0, backfill: 0, last: 0 }));
return s;
}
/** Returns the ops this frame was first to deliver. */
ingest(src, frame, recv = Date.now()) {
const s = this.stat(src);
s.frames++;
s.last = recv;
const c = commitOf(frame);
if (!c?.did) return [];
const fresh = [];
for (const op of c.ops) {
if (!wanted(op.collection, this.prefixes)) continue;
const at = tidTime(c.rev) || Date.parse(op.record?.createdAt);
if (at < this.since) {
s.backfill++;
continue;
}
s.ops++;
const key = `${c.did}/${op.collection}/${op.rkey}@${c.rev ?? op.cid}`;
const prior = this.seen.get(key);
if (prior) {
s.dups++;
prior.srcs.add(src);
if (!prior.record && op.record) (prior.record = op.record), this.apply(prior);
continue;
}
s.fresh++;
const e = { key, did: c.did, ...op, rev: c.rev ?? "", time: c.time || recv, recv, srcs: new Set([src]) };
this.seen.set(key, e);
this.apply(e);
fresh.push(e);
}
return fresh;
}
apply(e) {
const uri = `at://${e.did}/${e.collection}/${e.rkey}`;
const cur = this.records.get(uri);
if (cur && cur.rev > e.rev) return;
const deleted = e.action === "delete";
// an update without a body (the JSON relay) keeps the previous body, marked stale until fetched or delivered
const fresh = deleted || e.record || cur?.cid === e.cid;
const record = e.record ?? cur?.record ?? null;
this.records.set(uri, { uri, did: e.did, collection: e.collection, rkey: e.rkey, rev: e.rev, cid: e.cid ?? cur?.cid ?? null, time: e.time, first: cur?.first ?? e.time, seenAt: Date.now(), deleted, record, stale: !fresh || (cur?.stale && !e.record) });
this.version++;
}
/** A record fetched by getRecord, for an op whose source carried no body (or a strongRef target). */
put(uri, cid, record) {
const cur = this.records.get(uri);
if (cur?.record && !cur.stale) return;
const [, , did, collection, rkey] = uri.split("/");
this.records.set(uri, { uri, did, collection, rkey, rev: cur?.rev ?? "", cid, time: cur?.time ?? (Date.parse(record?.createdAt) || Date.now()), first: cur?.first ?? Date.now(), deleted: cur?.deleted ?? false, fetched: true, record, stale: false });
this.version++;
}
}
const didOf = (uri) => (typeof uri === "string" ? uri.split("/")[2] : undefined);
const collOf = (uri) => (typeof uri === "string" ? uri.split("/")[3] : undefined);
const timeOf = (r) => Date.parse(r?.record?.createdAt) || r?.time || 0;
const STATUS = ["bidding", "accepted", "held", "running", "released"];
const RELEASE = new Set(["vm.delete", "vm.terminated", "guest.r2Released"]);
// a VM's memory events (the guest agent's, the host cgroup's): an OOM kill marks it red, pressure amber
const MEMORY_KILL = new Set(["guest.oomKill", "vm.oomKilled"]);
const MEMORY_PRESSURE = new Set(["guest.memoryPressure", "vm.memoryPressure"]);
const RUNNING = new Set(["vm.started", "vm.onNetwork", "vm.hostKeys", "guest.publicKey", "guest.r2Start", "guest.requesterDid", "guest.agentStart"]);
const TOPO = new Set(["market.rfp", "market.bid", "market.accept", "market.receipt", "market.event", "market.association", "market.associationRequest", "compute.vm", "compute.events.guest.requesterDid"].map((c) => T + c).concat(["link.ident.endorsement", "dev.knotless.check", "computer.socialweb.infra", "computer.socialweb.ephemeralInfra", "computer.socialweb.mesh.roster"]));
export const short = (did) => (did ? did.replace(/^did:plc:/, "").slice(0, 8) : "?");
/**
* The market topology from the records: CI runs (dev.knotless.check) -> requesters -> contracts (rented VMs) ->
* guest requesters (R2, named by guest.requesterDid) -> their contracts, every contract -> its bidder; principal ->
* agent (market.association, link.ident.endorsement, alsoKnownAs agent-of). `wants`: URIs worth a getRecord.
* docs: did -> { handle, agentOf? }.
*/
export function topology(records, docs = new Map(), now = Date.now()) {
const nodes = new Map();
const links = new Map();
const wants = new Set();
const name = (did) => docs.get(did)?.handle ?? short(did);
const actor = (did, role) => {
let n = nodes.get(did);
if (!n) nodes.set(did, (n = { id: did, type: "actor", did, roles: new Set(), uris: [] }));
if (role) n.roles.add(role);
return n;
};
const link = (source, target, kind, extra = {}) => {
const id = `${source}|${target}|${kind}`;
links.set(id, { id, source, target, kind, ...links.get(id), ...extra });
};
const contracts = new Map();
const contract = (rfp) => {
let c = contracts.get(rfp);
if (!c) contracts.set(rfp, (c = { id: rfp, type: "contract", requester: didOf(rfp), bids: new Set(), events: [], uris: [] }));
return c;
};
const byReceipt = new Map();
const beats = new Map();
for (const r of records.values()) {
if ((!r.record || r.stale) && !r.deleted && TOPO.has(r.collection)) wants.add(r.uri);
const v = r.record ?? {};
switch (r.collection) {
case `${T}market.rfp`: {
const c = contract(r.uri);
c.rfp = r;
c.vm = v.payload?.uri;
c.uris.push(r.uri);
break;
}
// a bid is a count on its RFP: only the winner gets a node and an edge
case `${T}market.bid`:
if (v.rfp?.uri) contract(v.rfp.uri).bids.add(r.did), contract(v.rfp.uri).uris.push(r.uri);
break;
case `${T}market.accept`:
if (v.rfp?.uri) {
const c = contract(v.rfp.uri);
c.accept = r;
c.winner = didOf(v.bid?.uri);
c.uris.push(r.uri);
}
break;
case `${T}market.receipt`:
if (v.rfp?.uri) {
const c = contract(v.rfp.uri);
c.receipt = r;
c.bidder = r.did;
c.vm ??= v.payload?.uri;
c.uris.push(r.uri);
byReceipt.set(r.uri, c);
}
break;
case `${T}market.offering`:
case `${T}market.heartbeat`:
if (r.collection.endsWith("heartbeat") && !r.deleted) beats.set(r.did, Math.max(beats.get(r.did) ?? 0, timeOf(r)));
break;
case `${T}market.association`:
if (v.agent) actor(r.did, "principal"), actor(v.agent, "agent"), link(r.did, v.agent, "agent-of", { grant: !r.deleted, revoked: r.deleted, expiresAt: v.expiresAt });
break;
case "link.ident.endorsement":
if (v.agent) actor(r.did, "principal"), actor(v.agent, "agent"), link(r.did, v.agent, "agent-of", { endorsed: !r.deleted });
break;
case "computer.socialweb.peer":
if (!r.deleted) actor(r.did).peer = Object.keys(v).filter((k) => ["iroh", "webrtc"].includes(k) && v[k]);
break;
case "computer.socialweb.mesh.roster":
actor(r.did, "orchestrator");
nodes.set(r.uri, { id: r.uri, type: "roster", did: r.did, label: `mesh ${v.zone ?? r.rkey}`, peers: (v.peers ?? []).map((p) => p.name).filter(Boolean), status: r.deleted ? "released" : "running", group: r.did, uris: [r.uri] });
link(r.did, r.uri, "orchestrates");
break;
case "dev.knotless.check": {
const run = actor(r.did, "runner");
const repo = v.repo?.uri;
nodes.set(r.uri, { id: r.uri, type: "check", did: r.did, runner: run.id, repo, owner: didOf(repo), repoName: repo?.split("/").pop(), workflow: v.workflow, commit: v.commit, status: v.status ?? "?", startedAt: Date.parse(v.startedAt), finishedAt: Date.parse(v.finishedAt), deleted: r.deleted, uris: [r.uri] });
break;
}
}
}
for (const [did, doc] of docs) if (doc.agentOf && nodes.has(did)) actor(doc.agentOf, "principal"), link(doc.agentOf, did, "agent-of", { declared: true });
for (const r of records.values()) {
if (r.collection !== `${T}market.event`) continue;
const c = byReceipt.get(r.record?.receipt?.uri);
if (!c) continue;
const pay = r.record.payload?.uri;
const kind = collOf(pay)?.replace(`${T}compute.events.`, "");
c.events.push({ kind, time: timeOf(r) });
c.uris.push(r.uri);
if (kind === "vm.terminated") {
const p = records.get(pay);
if (p?.record?.reason === "oom") c.oomTerminated = true;
else if (!p?.record) wants.add(pay);
}
if (kind === "guest.requesterDid") {
const did = records.get(pay)?.record?.did;
if (did) c.guest = did;
else wants.add(pay);
}
}
for (const c of contracts.values()) {
const vm = records.get(c.vm)?.record;
const kinds = new Set(c.events.map((e) => e.kind));
const gone = [c.rfp, c.accept, c.receipt].some((r) => r?.deleted);
c.memory = c.oomTerminated || [...kinds].some((k) => MEMORY_KILL.has(k)) ? "kill" : [...kinds].some((k) => MEMORY_PRESSURE.has(k)) ? "pressure" : "";
c.status = gone || [...kinds].some((k) => RELEASE.has(k)) ? "released" : [...kinds].some((k) => RUNNING.has(k)) ? "running" : c.receipt ? "held" : c.accept ? "accepted" : "bidding";
// a released contract's compute.vm is deleted with it
if (c.vm && !vm && c.status !== "released") wants.add(c.vm);
c.created = timeOf(c.rfp) || timeOf(c.accept) || timeOf(c.receipt);
c.vmRole = vm?.role ?? "";
// a stg- VM is a prod market VM that carries one layer of a staging network (its insides are not on prod)
c.label = c.vmRole.startsWith("stg-") ? `prod VM hosting ${c.vmRole}` : (c.vmRole || c.id.split("/").pop());
c.size = vm ? `${vm.cpus ?? "?"} vCPU / ${vm.mem ?? "?"}` : "";
c.uris.push(c.vm);
nodes.set(c.id, c);
actor(c.requester, "requester");
link(c.requester, c.id, "rents");
const bidder = c.bidder ?? c.winner;
if (bidder) (actor(bidder, "bidder").heartbeat = beats.get(bidder)), link(c.id, bidder, "held by", { held: !!c.bidder });
if (c.guest) actor(c.guest, "guest"), link(c.id, c.guest, "runs requester");
// a staging stand-up names its layers stg-atproto, stg-bidder-N, stg-gateway: one staging network per requester
if (c.vmRole.startsWith("stg-")) c.group = c.requester;
}
// a kcp Infra / EphemeralInfra (computer.socialweb.infra, .ephemeralInfra): a box owned by its repo's DID around
// its contracts, matched by status.contracts[].receipt, else by name or vmName = the VM's role
for (const r of records.values()) {
if (r.collection !== "computer.socialweb.infra" && r.collection !== "computer.socialweb.ephemeralInfra") continue;
const v = r.record ?? {};
const specs = v.contracts ?? [];
const st = v.status ?? {};
if (!specs.length && !st.contracts?.length) continue; // infra/self is only an index
const phase = r.deleted ? "deleted" : (st.phase ?? "?");
const eph = r.collection.endsWith("ephemeralInfra");
const ttl = eph && v.ttlSeconds ? `ttl ${v.ttlSeconds >= 3600 ? `${Math.round(v.ttlSeconds / 3600)} h` : `${Math.round(v.ttlSeconds / 60)} min`}` : "";
const expires = st.expiresAt ? `expires ${st.expiresAt.slice(5, 16).replace("T", " ")}Z` : "";
nodes.set(r.uri, { id: r.uri, type: "infra", did: r.did, label: `${eph ? "ephemeralInfra" : "infra"} ${r.rkey}`, status: /released|ended|deleted|failed/i.test(phase) ? "released" : "running", phase, sub: [phase, ttl, v.endWhen ? `end when ${v.endWhen}` : "", expires, st.endReason ?? ""].filter(Boolean).join(" · "), group: r.uri, groupLabel: `${eph ? "ephemeralInfra" : "infra"} ${r.rkey}: prod VMs carrying a staging network · ${phase}${ttl ? ` · ${ttl}` : ""}${v.endWhen ? ` · end when ${v.endWhen}` : ""}${expires ? ` · ${expires}` : ""}`, uris: [r.uri] });
link(r.did, r.uri, "owns");
for (const spec of specs.length ? specs : st.contracts) {
const cs = st.contracts?.find((x) => x.name === spec.name) ?? {};
// by name: the newest of this repo's contracts with that role made since the record (a role repeats per run)
const since = (Date.parse(v.createdAt) || 0) - 60_000;
const c = byReceipt.get(cs.receipt) ?? [...contracts.values()].filter((c) => c.requester === r.did && c.created >= since && [spec.name, spec.vmName].includes(c.vmRole)).sort((a, b) => b.created - a.created)[0];
if (!c) continue;
c.group = r.uri;
c.infraPhase = cs.phase;
link(r.uri, c.id, "contract");
}
}
for (const m of nodes.values())
if (m.type === "roster") for (const c of contracts.values()) if (c.requester === m.did && m.peers.includes(c.vmRole)) link(m.id, c.id, "mesh peer");
// likely gone (tombstoned): a guest of released contracts only, an agent whose grants are all revoked
for (const n of nodes.values()) {
if (n.type !== "actor") continue;
const hosts = [...contracts.values()].filter((c) => c.guest === n.did);
const grants = [...links.values()].filter((l) => l.kind === "agent-of" && l.target === n.did);
n.gone = (hosts.length > 0 && hosts.every((c) => c.status === "released")) || (grants.length > 0 && grants.every((l) => l.revoked));
}
// a CI run rents as its pusher, here the repo's owner: a contract made while runs went belongs to the one that
// started last before it (inferred; workflows of one push run side by side)
const checks = [...nodes.values()].filter((n) => n.type === "check");
for (const c of contracts.values()) {
const run = checks
.filter((n) => c.requester === n.owner && c.created >= n.startedAt && c.created <= (Number.isFinite(n.finishedAt) ? n.finishedAt : now))
.sort((a, b) => b.startedAt - a.startedAt)[0];
if (run) link(run.id, c.id, "rents (time)", { inferred: true });
}
// depth: checks 0, requesters 1, a contract one past its requester, a guest one past its host contract, an agent
// one past its principal; bidders get the last column at render time
// a requester none of whose contracts a CI run made is a hand run: the root of its own tree
const fromCheck = new Set([...links.values()].filter((l) => l.kind === "rents (time)").map((l) => l.target));
for (const n of nodes.values()) {
n.depth = n.type === "check" ? 0 : 1;
if (n.type !== "actor" || !n.roles.has("requester") || ["guest", "agent", "bidder"].some((r) => n.roles.has(r))) continue;
const own = [...contracts.values()].filter((c) => c.requester === n.did);
if (own.length && !own.some((c) => fromCheck.has(c.id))) (n.depth = 0), (n.handRun = true);
}
for (let i = 0; i < 6; i++)
for (const l of links.values()) {
const s = nodes.get(l.source);
const t = nodes.get(l.target);
if (!s || !t || t.roles?.has("bidder")) continue;
if (l.kind === "rents" || l.kind === "orchestrates" || l.kind === "owns" || l.kind === "contract" || l.kind === "runs requester" || (l.kind === "agent-of" && !t.roles.has("requester"))) t.depth = Math.max(t.depth, s.depth + 1);
if (l.kind === "agent-of" && t.roles.has("guest")) t.depth = Math.max(t.depth, s.depth + 1);
}
for (const n of nodes.values()) {
if (n.type === "actor") {
n.label = name(n.did);
n.role = n.roles.has("guest") ? "guest" : n.roles.has("bidder") ? "bidder" : n.roles.has("runner") ? "runner" : n.roles.has("requester") ? "requester" : n.roles.has("agent") ? "agent" : "principal";
n.sub = [n.handRun ? "hand run" : "", ...n.roles, n.peer?.length ? `mesh peer (${n.peer.join(", ")})` : ""].filter(Boolean).join(", ");
n.status = n.role === "bidder" ? (n.heartbeat && now - n.heartbeat < 300_000 ? "fresh" : "stale") : n.role;
} else if (n.type === "check") {
n.label = `${n.repoName}: ${n.workflow}`;
n.sub = `${n.commit?.slice(0, 8) ?? ""} ${n.status}${Number.isFinite(n.finishedAt) ? "" : " (running)"}`;
} else if (n.type === "roster") n.sub = `${n.peers.length} peers${n.status === "released" ? " · torn down" : ""}`;
else if (n.type === "infra") continue;
else n.sub = [n.size, n.status, n.memory === "kill" ? "OOM killed" : n.memory === "pressure" ? "memory pressure" : "", n.vmRole?.startsWith("stg-") && (n.bidder ?? n.winner) ? `on ${name(n.bidder ?? n.winner)}` : "", n.infraPhase && n.infraPhase !== n.status ? `infra: ${n.infraPhase}` : "", n.bids.size ? `${n.bids.size} bid${n.bids.size > 1 ? "s" : ""}` : ""].filter(Boolean).join(" · ");
}
return { nodes: [...nodes.values()], links: [...links.values()].filter((l) => nodes.has(l.source) && nodes.has(l.target)), wants: [...wants] };
}
/** The nodes a filter keeps: matches, everything downstream of them and their ancestors (not through bidders). */
export function filterTopology({ nodes, links }, text) {
const terms = text.toLowerCase().split(/[\s,]+/).filter(Boolean);
if (!terms.length) return { nodes, links };
const hay = (n) => [n.id, n.label, n.repo, n.commit, n.repoName, n.workflow].filter(Boolean).join(" ").toLowerCase();
const keep = new Set(nodes.filter((n) => terms.some((t) => hay(n).includes(t))).map((n) => n.id));
const byId = new Map(nodes.map((n) => [n.id, n]));
const walk = (from, to) => {
const queue = [...keep];
const seen = new Set(queue);
while (queue.length) {
const id = queue.shift();
if (byId.get(id)?.role === "bidder" && !keep.has(id)) continue;
for (const l of links) if (l[from] === id && !seen.has(l[to])) seen.add(l[to]), queue.push(l[to]);
}
return seen;
};
const all = new Set([...walk("source", "target"), ...walk("target", "source")]);
return { nodes: nodes.filter((n) => all.has(n.id)), links: links.filter((l) => all.has(l.source) && all.has(l.target)) };
}
export { STATUS };