johnandersen777.bsky.social/socialweb-computer-watch
The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay
0
0
// node --test test/model.test.mjs
import { test } from "node:test";
import assert from "node:assert/strict";
import { Watch, topology, filterTopology, captureLine, tidTime, fetchedFrame } from "../model.js";
import { encodeCbor } from "../node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js";
import { toBase32 } from "../node_modules/@publicdomainrelay/atlibs/src/encoding/index.js";
const T = "com.publicdomainrelay.temp.";
const R = "did:plc:requester";
const B = "did:plc:bidder";
const G = "did:plc:guest";
const uri = (did, c, rkey) => `at://${did}/${T}${c}/${rkey}`;
const jet = (did, collection, rkey, rev, record, operation = "create") => ({ did, time_us: 1_700_000_000_000_000 + Number(rev) * 1000, kind: "commit", commit: { rev, operation, collection: T + collection, rkey, cid: `bafy${rkey}`, record } });
// a relay commit whose record is only in its CAR, as atlibs decodeFrame returns it
function relayFrame(did, collection, rkey, rev, record) {
const data = encodeCbor(record);
const cid = Uint8Array.from([1, 0x71, 0x12, 0x20, ...new Uint8Array(32).fill(7)]);
const header = encodeCbor({ version: 1, roots: [] });
const car = Uint8Array.from([header.length, ...header, cid.length + data.length, ...cid, ...data]);
return { $type: "com.atproto.sync.subscribeRepos#commit", seq: 1, repo: did, rev, time: new Date(1_700_000_000_000).toISOString(), blocks: { $bytes: Buffer.from(car).toString("base64") }, ops: [{ action: "create", path: `${T}${collection}/${rkey}`, cid: { $link: `b${toBase32(cid)}` } }] };
}
test("one op from three sources is one op; a body-less first delivery is filled by a later one", () => {
const w = new Watch();
const hb = { $type: `${T}market.heartbeat`, offering: "x", createdAt: "2026-11-13T00:00:00Z" };
const mini = { $type: "com.atproto.sync.subscribeRepos#commit", seq: 9, repo: B, rev: "3", time: "2026-11-13T00:00:00Z", blocks: {}, ops: [{ action: "update", path: `${T}market.heartbeat/self`, cid: { $link: "bafyself" } }] };
assert.equal(w.ingest("mini", mini).length, 1);
assert.equal(w.records.get(uri(B, "market.heartbeat", "self")).record, null);
assert.equal(w.ingest("relay", relayFrame(B, "market.heartbeat", "self", "3", hb)).length, 0);
assert.equal(w.ingest("jet", jet(B, "market.heartbeat", "self", "3", hb, "update")).length, 0);
assert.deepEqual(w.records.get(uri(B, "market.heartbeat", "self")).record, hb);
assert.deepEqual([...w.sources].map(([s, v]) => [s, v.fresh, v.dups]), [["mini", 1, 0], ["relay", 0, 1], ["jet", 0, 1]]);
// the next rev is new; an older rev arriving late does not roll the record back
assert.equal(w.ingest("jet", jet(B, "market.heartbeat", "self", "4", { ...hb, createdAt: "later" }, "update")).length, 1);
w.ingest("x", jet(B, "market.heartbeat", "self", "2", { ...hb, createdAt: "earlier" }, "update"));
assert.equal(w.records.get(uri(B, "market.heartbeat", "self")).record.createdAt, "later");
});
test("a contract builds out: rfp, bid, accept, receipt, guest requester, release", () => {
const w = new Watch();
const ref = (u) => ({ uri: u, cid: "c" });
const rfp = uri(R, "market.rfp", "r1");
const bid = uri(B, "market.bid", "bid-1");
const acc = uri(R, "market.accept", "a1");
const rec = uri(B, "market.receipt", "receipt-1");
const frames = [
jet(R, "compute.vm", "vm1", "1", { role: "test-vm", cpus: 4, mem: "8G" }),
jet(R, "market.rfp", "r1", "2", { payload: ref(uri(R, "compute.vm", "vm1")), createdAt: "2026-11-13T00:00:10Z" }),
jet(B, "market.bid", "bid-1", "3", { rfp: ref(rfp) }),
jet(R, "market.accept", "a1", "4", { rfp: ref(rfp), bid: ref(bid) }),
jet(B, "market.receipt", "receipt-1", "5", { rfp: ref(rfp), bid: ref(bid), accept: ref(acc) }),
jet(B, "compute.events.guest.requesterDid", "p1", "6", { did: G }),
jet(B, "market.event", "e1", "7", { receipt: ref(rec), payload: ref(uri(B, "compute.events.guest.requesterDid", "p1")) }),
jet(R, "market.association", G, "8", { agent: G }),
{ did: "did:plc:runner", time_us: 0, kind: "commit", commit: { rev: "9", operation: "create", collection: "dev.knotless.check", rkey: "k1", record: { repo: { uri: `at://${R}/dev.knotless.repo/kcp` }, workflow: "test", commit: "abcdef0123", status: "running", startedAt: "2026-11-13T00:00:00Z" } } },
];
const status = () => topology(w.records, new Map(), Date.parse("2026-11-13T00:05:00Z")).nodes.find((n) => n.id === rfp).status;
frames.slice(0, 2).forEach((f) => w.ingest("j", f));
assert.equal(status(), "bidding");
frames.slice(2, 5).forEach((f) => w.ingest("j", f));
assert.equal(status(), "held");
frames.slice(5).forEach((f) => w.ingest("j", f));
const t = topology(w.records, new Map(), Date.parse("2026-11-13T00:05:00Z"));
const c = t.nodes.find((n) => n.id === rfp);
assert.equal(c.status, "running");
assert.equal(c.label, "test-vm");
assert.equal(t.nodes.find((n) => n.id === G).role, "guest");
assert.equal(t.nodes.find((n) => n.id === G).depth, 3);
const links = new Set(t.links.map((l) => `${l.source} ${l.kind} ${l.target}`));
for (const l of [`${R} rents ${rfp}`, `${rfp} held by ${B}`, `${rfp} runs requester ${G}`, `${R} agent-of ${G}`]) assert.ok(links.has(l), l);
assert.ok(t.links.some((l) => l.source.endsWith("dev.knotless.check/k1") && l.target === rfp), "the CI run links to the contract its repo owner rented");
const memEvent = (rkey, rev, kind, record) => [jet(B, `compute.events.${kind}`, `m${rkey}`, rev, record), jet(B, "market.event", `em${rkey}`, `${rev}1`, { receipt: ref(rec), payload: ref(uri(B, `compute.events.${kind}`, `m${rkey}`)) })];
const memory = () => topology(w.records).nodes.find((n) => n.id === rfp);
assert.equal(memory().memory, "");
memEvent(1, "91", "guest.memoryPressure", { kind: "swap", threshold: 1000 }).forEach((f) => w.ingest("j", f));
assert.equal(memory().memory, "pressure");
assert.match(memory().sub, /memory pressure/);
memEvent(2, "92", "guest.oomKill", { kills: 1, total: 1 }).forEach((f) => w.ingest("j", f));
assert.equal(memory().memory, "kill");
w.ingest("j", jet(B, "compute.events.vm.terminated", "p2", "10", { reason: "released" }));
w.ingest("j", jet(B, "market.event", "e2", "11", { receipt: ref(rec), payload: ref(uri(B, "compute.events.vm.terminated", "p2")) }));
assert.equal(status(), "released");
// a filter on the CI commit keeps the run, its contract, the guest and the bidder
const f = filterTopology(topology(w.records), "abcdef01");
assert.deepEqual(new Set(f.nodes.map((n) => n.type + ":" + (n.role ?? n.status))), new Set(["check:running", "contract:released", "actor:guest", "actor:bidder"]));
});
test("capture lines: wrapped, bare jetstream, goat firehose", () => {
assert.equal(captureLine(JSON.stringify({ src: "a", recv: 5, frame: jet(R, "market.rfp", "r", "1", {}) })).t, 1_700_000_000_001);
assert.equal(captureLine(JSON.stringify({ src: "a", recv: 5, frame: { ...jet(R, "market.rfp", "r", "1", {}), time_us: 0 } })).t, 5);
assert.equal(captureLine(JSON.stringify(jet(R, "market.rfp", "r", "1", {})), "file").src, "file");
const goat = { type: "commit", payload: { repo: R, rev: "1", time: "2026-11-13T00:00:00Z", ops: [{ action: "create", path: `${T}market.rfp/r`, cid: { $link: "x" } }], blocks: null } };
assert.equal(captureLine(JSON.stringify(goat), "g").t, Date.parse("2026-11-13T00:00:00Z"));
});
test("a hand run is a root; its stg- VMs are one staging network with its mesh roster", () => {
const w = new Watch();
const ref = (u) => ({ uri: u, cid: "c" });
for (const [i, role] of ["stg-atproto", "stg-bidder-1", "other"].entries()) {
w.ingest("j", jet(R, "compute.vm", `vm${i}`, `${i}1`, { role }));
w.ingest("j", jet(R, "market.rfp", `r${i}`, `${i}2`, { payload: ref(uri(R, "compute.vm", `vm${i}`)) }));
}
w.ingest("j", { did: R, time_us: 1, kind: "commit", commit: { rev: "9", operation: "create", collection: "computer.socialweb.mesh.roster", rkey: "stg", record: { zone: "stg.mesh", peers: [{ name: "stg-atproto" }, { name: "stg-bidder-1" }] } } });
const t = topology(w.records);
const req = t.nodes.find((n) => n.id === R);
assert.equal(req.depth, 0);
assert.ok(req.sub.includes("hand run"));
assert.deepEqual(t.nodes.filter((n) => n.group === R).map((n) => n.label).sort(), ["mesh stg.mesh", "prod VM hosting stg-atproto", "prod VM hosting stg-bidder-1"]);
assert.equal(t.links.filter((l) => l.kind === "mesh peer").length, 2);
});
test("an ephemeralInfra is a box around its contracts, by receipt or by name", () => {
const w = new Watch();
const ref = (u) => ({ uri: u, cid: "c" });
w.ingest("j", jet(R, "compute.vm", "vm1", "1", { role: "stg-atproto" }));
w.ingest("j", jet(R, "market.rfp", "r1", "2", { payload: ref(uri(R, "compute.vm", "vm1")) }));
w.ingest("j", jet(R, "compute.vm", "vm2", "3", { role: "stg-gateway" }));
w.ingest("j", jet(R, "market.rfp", "r2", "4", { payload: ref(uri(R, "compute.vm", "vm2")) }));
w.ingest("j", jet(B, "market.receipt", "rc2", "5", { rfp: ref(uri(R, "market.rfp", "r2")) }));
const infra = { contracts: [{ name: "stg-atproto" }, { name: "gw" }], ttlSeconds: 7200, endWhen: "deadline", status: { phase: "Ready", expiresAt: "2026-11-13T20:00:00Z", contracts: [{ name: "gw", phase: "Held", receipt: uri(B, "market.receipt", "rc2") }] } };
w.ingest("j", { did: R, time_us: 1, kind: "commit", commit: { rev: "6", operation: "create", collection: "computer.socialweb.ephemeralInfra", rkey: "stg-1", record: infra } });
const t = topology(w.records);
const box = `at://${R}/computer.socialweb.ephemeralInfra/stg-1`;
assert.deepEqual(t.nodes.filter((n) => n.group === box).map((n) => n.label).sort(), ["ephemeralInfra stg-1", "prod VM hosting stg-atproto", "prod VM hosting stg-gateway"]);
assert.ok(t.nodes.find((n) => n.id === box).groupLabel.includes("ttl 2 h · end when deadline"));
assert.ok(t.links.some((l) => l.source === R && l.target === box && l.kind === "owns"));
});
test("losing bids are a count, not nodes or edges", () => {
const w = new Watch();
const ref = (u) => ({ uri: u, cid: "c" });
const rfp = uri(R, "market.rfp", "r1");
w.ingest("j", jet(R, "market.rfp", "r1", "1", {}));
for (const [i, b] of ["did:plc:b1", "did:plc:b2", B].entries()) w.ingest("j", jet(b, "market.bid", `bid-${i}`, `${i}2`, { rfp: ref(rfp) }));
let t = topology(w.records);
assert.equal(t.nodes.filter((n) => n.role === "bidder").length, 0);
assert.ok(t.nodes.find((n) => n.id === rfp).sub.includes("3 bids"));
w.ingest("j", jet(B, "market.receipt", "rc", "9", { rfp: ref(rfp) }));
t = topology(w.records);
assert.deepEqual(t.nodes.filter((n) => n.role === "bidder").map((n) => n.id), [B]);
assert.deepEqual(t.links.filter((l) => l.source === rfp).map((l) => l.kind), ["held by"]);
});
// a TID for a time: 53 bits of microseconds, 10 bits of clock id
const tid = (ms) => {
let n = BigInt(ms * 1000) << 10n;
let out = "";
for (let i = 0; i < 13; i++) (out = "234567abcdefghijklmnopqrstuvwxyz"[Number(n & 31n)] + out), (n >>= 5n);
return out;
};
test("a relay re-emitting a repo's history is backfill: counted, not shown", () => {
const now = Date.parse("2026-11-13T18:43:00Z");
assert.equal(tidTime(tid(now)), now);
assert.equal(tidTime("3n27fhny2yh22") > Date.parse("2026-11-13T00:00:00Z"), true);
const w = new Watch(undefined, now - 31 * 60_000);
const ref = (u) => ({ uri: u, cid: "c" });
// 500 old commits (Oct 23 on), as hat-9's restart replayed them, then one live one
for (let i = 0; i < 500; i++) {
const rev = tid(Date.parse("2026-10-23T00:00:00Z") + i * 3_600_000);
w.ingest("mini", { $type: "com.atproto.sync.subscribeRepos#commit", seq: 70000 + i, repo: B, rev, time: new Date(now).toISOString(), blocks: {}, ops: [{ action: "create", path: `${T}market.bid/bid-${i}`, cid: { $link: `b${i}` } }] });
}
const live = w.ingest("jet", jet(B, "market.receipt", "rc", tid(now), { rfp: ref(uri(R, "market.rfp", "r")), createdAt: new Date(now).toISOString() }));
assert.equal(live.length, 1);
assert.deepEqual([w.sources.get("mini").backfill, w.sources.get("mini").ops], [500, 0]);
assert.equal(w.records.size, 1);
});
test("an update without a body keeps the last body, stale until fetched", () => {
const w = new Watch();
const u = `at://${R}/computer.socialweb.ephemeralInfra/stg-1`;
const mini = (rev, cid) => ({ $type: "com.atproto.sync.subscribeRepos#commit", seq: 1, repo: R, rev, time: "2026-11-13T20:00:00Z", blocks: {}, ops: [{ action: "update", path: "computer.socialweb.ephemeralInfra/stg-1", cid: { $link: cid } }] });
w.ingest("mini", mini("1", "c1"));
w.put(u, "c1", { status: { phase: "deploying" } });
w.ingest("mini", mini("2", "c2"));
assert.equal(w.records.get(u).record.status.phase, "deploying");
assert.ok(topology(w.records).wants.includes(u));
w.put(u, "c2", { status: { phase: "deployed" } });
assert.equal(w.records.get(u).record.status.phase, "deployed");
assert.ok(!topology(w.records).wants.includes(u));
});
test("a getRecord line at the op's rev fills a body-less op in a replay", () => {
const w = new Watch();
const mini = { $type: "com.atproto.sync.subscribeRepos#commit", seq: 1, repo: R, rev: "5", time: "2026-11-13T20:00:00Z", blocks: {}, ops: [{ action: "create", path: "computer.socialweb.ephemeralInfra/stg-1", cid: { $link: "c5" } }] };
w.ingest("mini", mini);
assert.equal(w.ingest("getRecord", fetchedFrame(R, "computer.socialweb.ephemeralInfra", "stg-1", "5", "c5", { contracts: [] })).length, 0);
assert.deepEqual(w.records.get(`at://${R}/computer.socialweb.ephemeralInfra/stg-1`).record, { contracts: [] });
});