~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/socialweb-computer-watch

The socialweb.computer market's live topology from Jetstreams and relays, deduplicated; capture and replay

0
0
main
190 lines · 13.2 KB Raw History
// node --test test/model.test.mjs
import { test } from "node:test";
import assert from "node:assert/strict";
import { Watch, topology, filterTopology, captureLine, tidTime, fetchedFrame } from "../model.js";
import { encodeCbor } from "../node_modules/@publicdomainrelay/atlibs/src/run-module/subscribe.js";
import { toBase32 } from "../node_modules/@publicdomainrelay/atlibs/src/encoding/index.js";

const T = "com.publicdomainrelay.temp.";
const R = "did:plc:requester";
const B = "did:plc:bidder";
const G = "did:plc:guest";
const uri = (did, c, rkey) => `at://${did}/${T}${c}/${rkey}`;
const jet = (did, collection, rkey, rev, record, operation = "create") => ({ did, time_us: 1_700_000_000_000_000 + Number(rev) * 1000, kind: "commit", commit: { rev, operation, collection: T + collection, rkey, cid: `bafy${rkey}`, record } });

// a relay commit whose record is only in its CAR, as atlibs decodeFrame returns it
function relayFrame(did, collection, rkey, rev, record) {
	const data = encodeCbor(record);
	const cid = Uint8Array.from([1, 0x71, 0x12, 0x20, ...new Uint8Array(32).fill(7)]);
	const header = encodeCbor({ version: 1, roots: [] });
	const car = Uint8Array.from([header.length, ...header, cid.length + data.length, ...cid, ...data]);
	return { $type: "com.atproto.sync.subscribeRepos#commit", seq: 1, repo: did, rev, time: new Date(1_700_000_000_000).toISOString(), blocks: { $bytes: Buffer.from(car).toString("base64") }, ops: [{ action: "create", path: `${T}${collection}/${rkey}`, cid: { $link: `b${toBase32(cid)}` } }] };
}

test("one op from three sources is one op; a body-less first delivery is filled by a later one", () => {
	const w = new Watch();
	const hb = { $type: `${T}market.heartbeat`, offering: "x", createdAt: "2026-11-13T00:00:00Z" };
	const mini = { $type: "com.atproto.sync.subscribeRepos#commit", seq: 9, repo: B, rev: "3", time: "2026-11-13T00:00:00Z", blocks: {}, ops: [{ action: "update", path: `${T}market.heartbeat/self`, cid: { $link: "bafyself" } }] };
	assert.equal(w.ingest("mini", mini).length, 1);
	assert.equal(w.records.get(uri(B, "market.heartbeat", "self")).record, null);
	assert.equal(w.ingest("relay", relayFrame(B, "market.heartbeat", "self", "3", hb)).length, 0);
	assert.equal(w.ingest("jet", jet(B, "market.heartbeat", "self", "3", hb, "update")).length, 0);
	assert.deepEqual(w.records.get(uri(B, "market.heartbeat", "self")).record, hb);
	assert.deepEqual([...w.sources].map(([s, v]) => [s, v.fresh, v.dups]), [["mini", 1, 0], ["relay", 0, 1], ["jet", 0, 1]]);
	// the next rev is new; an older rev arriving late does not roll the record back
	assert.equal(w.ingest("jet", jet(B, "market.heartbeat", "self", "4", { ...hb, createdAt: "later" }, "update")).length, 1);
	w.ingest("x", jet(B, "market.heartbeat", "self", "2", { ...hb, createdAt: "earlier" }, "update"));
	assert.equal(w.records.get(uri(B, "market.heartbeat", "self")).record.createdAt, "later");
});

test("a contract builds out: rfp, bid, accept, receipt, guest requester, release", () => {
	const w = new Watch();
	const ref = (u) => ({ uri: u, cid: "c" });
	const rfp = uri(R, "market.rfp", "r1");
	const bid = uri(B, "market.bid", "bid-1");
	const acc = uri(R, "market.accept", "a1");
	const rec = uri(B, "market.receipt", "receipt-1");
	const frames = [
		jet(R, "compute.vm", "vm1", "1", { role: "test-vm", cpus: 4, mem: "8G" }),
		jet(R, "market.rfp", "r1", "2", { payload: ref(uri(R, "compute.vm", "vm1")), createdAt: "2026-11-13T00:00:10Z" }),
		jet(B, "market.bid", "bid-1", "3", { rfp: ref(rfp) }),
		jet(R, "market.accept", "a1", "4", { rfp: ref(rfp), bid: ref(bid) }),
		jet(B, "market.receipt", "receipt-1", "5", { rfp: ref(rfp), bid: ref(bid), accept: ref(acc) }),
		jet(B, "compute.events.guest.requesterDid", "p1", "6", { did: G }),
		jet(B, "market.event", "e1", "7", { receipt: ref(rec), payload: ref(uri(B, "compute.events.guest.requesterDid", "p1")) }),
		jet(R, "market.association", G, "8", { agent: G }),
		{ did: "did:plc:runner", time_us: 0, kind: "commit", commit: { rev: "9", operation: "create", collection: "dev.knotless.check", rkey: "k1", record: { repo: { uri: `at://${R}/dev.knotless.repo/kcp` }, workflow: "test", commit: "abcdef0123", status: "running", startedAt: "2026-11-13T00:00:00Z" } } },
	];
	const status = () => topology(w.records, new Map(), Date.parse("2026-11-13T00:05:00Z")).nodes.find((n) => n.id === rfp).status;
	frames.slice(0, 2).forEach((f) => w.ingest("j", f));
	assert.equal(status(), "bidding");
	frames.slice(2, 5).forEach((f) => w.ingest("j", f));
	assert.equal(status(), "held");
	frames.slice(5).forEach((f) => w.ingest("j", f));
	const t = topology(w.records, new Map(), Date.parse("2026-11-13T00:05:00Z"));
	const c = t.nodes.find((n) => n.id === rfp);
	assert.equal(c.status, "running");
	assert.equal(c.label, "test-vm");
	assert.equal(t.nodes.find((n) => n.id === G).role, "guest");
	assert.equal(t.nodes.find((n) => n.id === G).depth, 3);
	const links = new Set(t.links.map((l) => `${l.source} ${l.kind} ${l.target}`));
	for (const l of [`${R} rents ${rfp}`, `${rfp} held by ${B}`, `${rfp} runs requester ${G}`, `${R} agent-of ${G}`]) assert.ok(links.has(l), l);
	assert.ok(t.links.some((l) => l.source.endsWith("dev.knotless.check/k1") && l.target === rfp), "the CI run links to the contract its repo owner rented");
	const memEvent = (rkey, rev, kind, record) => [jet(B, `compute.events.${kind}`, `m${rkey}`, rev, record), jet(B, "market.event", `em${rkey}`, `${rev}1`, { receipt: ref(rec), payload: ref(uri(B, `compute.events.${kind}`, `m${rkey}`)) })];
	const memory = () => topology(w.records).nodes.find((n) => n.id === rfp);
	assert.equal(memory().memory, "");
	memEvent(1, "91", "guest.memoryPressure", { kind: "swap", threshold: 1000 }).forEach((f) => w.ingest("j", f));
	assert.equal(memory().memory, "pressure");
	assert.match(memory().sub, /memory pressure/);
	memEvent(2, "92", "guest.oomKill", { kills: 1, total: 1 }).forEach((f) => w.ingest("j", f));
	assert.equal(memory().memory, "kill");
	w.ingest("j", jet(B, "compute.events.vm.terminated", "p2", "10", { reason: "released" }));
	w.ingest("j", jet(B, "market.event", "e2", "11", { receipt: ref(rec), payload: ref(uri(B, "compute.events.vm.terminated", "p2")) }));
	assert.equal(status(), "released");
	// a filter on the CI commit keeps the run, its contract, the guest and the bidder
	const f = filterTopology(topology(w.records), "abcdef01");
	assert.deepEqual(new Set(f.nodes.map((n) => n.type + ":" + (n.role ?? n.status))), new Set(["check:running", "contract:released", "actor:guest", "actor:bidder"]));
});

test("capture lines: wrapped, bare jetstream, goat firehose", () => {
	assert.equal(captureLine(JSON.stringify({ src: "a", recv: 5, frame: jet(R, "market.rfp", "r", "1", {}) })).t, 1_700_000_000_001);
	assert.equal(captureLine(JSON.stringify({ src: "a", recv: 5, frame: { ...jet(R, "market.rfp", "r", "1", {}), time_us: 0 } })).t, 5);
	assert.equal(captureLine(JSON.stringify(jet(R, "market.rfp", "r", "1", {})), "file").src, "file");
	const goat = { type: "commit", payload: { repo: R, rev: "1", time: "2026-11-13T00:00:00Z", ops: [{ action: "create", path: `${T}market.rfp/r`, cid: { $link: "x" } }], blocks: null } };
	assert.equal(captureLine(JSON.stringify(goat), "g").t, Date.parse("2026-11-13T00:00:00Z"));
});

test("a hand run is a root; its stg- VMs are one staging network with its mesh roster", () => {
	const w = new Watch();
	const ref = (u) => ({ uri: u, cid: "c" });
	for (const [i, role] of ["stg-atproto", "stg-bidder-1", "other"].entries()) {
		w.ingest("j", jet(R, "compute.vm", `vm${i}`, `${i}1`, { role }));
		w.ingest("j", jet(R, "market.rfp", `r${i}`, `${i}2`, { payload: ref(uri(R, "compute.vm", `vm${i}`)) }));
	}
	w.ingest("j", { did: R, time_us: 1, kind: "commit", commit: { rev: "9", operation: "create", collection: "computer.socialweb.mesh.roster", rkey: "stg", record: { zone: "stg.mesh", peers: [{ name: "stg-atproto" }, { name: "stg-bidder-1" }] } } });
	const t = topology(w.records);
	const req = t.nodes.find((n) => n.id === R);
	assert.equal(req.depth, 0);
	assert.ok(req.sub.includes("hand run"));
	assert.deepEqual(t.nodes.filter((n) => n.group === R).map((n) => n.label).sort(), ["mesh stg.mesh", "prod VM hosting stg-atproto", "prod VM hosting stg-bidder-1"]);
	assert.equal(t.links.filter((l) => l.kind === "mesh peer").length, 2);
});

test("an ephemeralInfra is a box around its contracts, by receipt or by name", () => {
	const w = new Watch();
	const ref = (u) => ({ uri: u, cid: "c" });
	w.ingest("j", jet(R, "compute.vm", "vm1", "1", { role: "stg-atproto" }));
	w.ingest("j", jet(R, "market.rfp", "r1", "2", { payload: ref(uri(R, "compute.vm", "vm1")) }));
	w.ingest("j", jet(R, "compute.vm", "vm2", "3", { role: "stg-gateway" }));
	w.ingest("j", jet(R, "market.rfp", "r2", "4", { payload: ref(uri(R, "compute.vm", "vm2")) }));
	w.ingest("j", jet(B, "market.receipt", "rc2", "5", { rfp: ref(uri(R, "market.rfp", "r2")) }));
	const infra = { contracts: [{ name: "stg-atproto" }, { name: "gw" }], ttlSeconds: 7200, endWhen: "deadline", status: { phase: "Ready", expiresAt: "2026-11-13T20:00:00Z", contracts: [{ name: "gw", phase: "Held", receipt: uri(B, "market.receipt", "rc2") }] } };
	w.ingest("j", { did: R, time_us: 1, kind: "commit", commit: { rev: "6", operation: "create", collection: "computer.socialweb.ephemeralInfra", rkey: "stg-1", record: infra } });
	const t = topology(w.records);
	const box = `at://${R}/computer.socialweb.ephemeralInfra/stg-1`;
	assert.deepEqual(t.nodes.filter((n) => n.group === box).map((n) => n.label).sort(), ["ephemeralInfra stg-1", "prod VM hosting stg-atproto", "prod VM hosting stg-gateway"]);
	assert.ok(t.nodes.find((n) => n.id === box).groupLabel.includes("ttl 2 h · end when deadline"));
	assert.ok(t.links.some((l) => l.source === R && l.target === box && l.kind === "owns"));
});

test("losing bids are a count, not nodes or edges", () => {
	const w = new Watch();
	const ref = (u) => ({ uri: u, cid: "c" });
	const rfp = uri(R, "market.rfp", "r1");
	w.ingest("j", jet(R, "market.rfp", "r1", "1", {}));
	for (const [i, b] of ["did:plc:b1", "did:plc:b2", B].entries()) w.ingest("j", jet(b, "market.bid", `bid-${i}`, `${i}2`, { rfp: ref(rfp) }));
	let t = topology(w.records);
	assert.equal(t.nodes.filter((n) => n.role === "bidder").length, 0);
	assert.ok(t.nodes.find((n) => n.id === rfp).sub.includes("3 bids"));
	w.ingest("j", jet(B, "market.receipt", "rc", "9", { rfp: ref(rfp) }));
	t = topology(w.records);
	assert.deepEqual(t.nodes.filter((n) => n.role === "bidder").map((n) => n.id), [B]);
	assert.deepEqual(t.links.filter((l) => l.source === rfp).map((l) => l.kind), ["held by"]);
});

// a TID for a time: 53 bits of microseconds, 10 bits of clock id
const tid = (ms) => {
	let n = BigInt(ms * 1000) << 10n;
	let out = "";
	for (let i = 0; i < 13; i++) (out = "234567abcdefghijklmnopqrstuvwxyz"[Number(n & 31n)] + out), (n >>= 5n);
	return out;
};

test("a relay re-emitting a repo's history is backfill: counted, not shown", () => {
	const now = Date.parse("2026-11-13T18:43:00Z");
	assert.equal(tidTime(tid(now)), now);
	assert.equal(tidTime("3n27fhny2yh22") > Date.parse("2026-11-13T00:00:00Z"), true);
	const w = new Watch(undefined, now - 31 * 60_000);
	const ref = (u) => ({ uri: u, cid: "c" });
	// 500 old commits (Oct 23 on), as hat-9's restart replayed them, then one live one
	for (let i = 0; i < 500; i++) {
		const rev = tid(Date.parse("2026-10-23T00:00:00Z") + i * 3_600_000);
		w.ingest("mini", { $type: "com.atproto.sync.subscribeRepos#commit", seq: 70000 + i, repo: B, rev, time: new Date(now).toISOString(), blocks: {}, ops: [{ action: "create", path: `${T}market.bid/bid-${i}`, cid: { $link: `b${i}` } }] });
	}
	const live = w.ingest("jet", jet(B, "market.receipt", "rc", tid(now), { rfp: ref(uri(R, "market.rfp", "r")), createdAt: new Date(now).toISOString() }));
	assert.equal(live.length, 1);
	assert.deepEqual([w.sources.get("mini").backfill, w.sources.get("mini").ops], [500, 0]);
	assert.equal(w.records.size, 1);
});

test("an update without a body keeps the last body, stale until fetched", () => {
	const w = new Watch();
	const u = `at://${R}/computer.socialweb.ephemeralInfra/stg-1`;
	const mini = (rev, cid) => ({ $type: "com.atproto.sync.subscribeRepos#commit", seq: 1, repo: R, rev, time: "2026-11-13T20:00:00Z", blocks: {}, ops: [{ action: "update", path: "computer.socialweb.ephemeralInfra/stg-1", cid: { $link: cid } }] });
	w.ingest("mini", mini("1", "c1"));
	w.put(u, "c1", { status: { phase: "deploying" } });
	w.ingest("mini", mini("2", "c2"));
	assert.equal(w.records.get(u).record.status.phase, "deploying");
	assert.ok(topology(w.records).wants.includes(u));
	w.put(u, "c2", { status: { phase: "deployed" } });
	assert.equal(w.records.get(u).record.status.phase, "deployed");
	assert.ok(!topology(w.records).wants.includes(u));
});

test("a getRecord line at the op's rev fills a body-less op in a replay", () => {
	const w = new Watch();
	const mini = { $type: "com.atproto.sync.subscribeRepos#commit", seq: 1, repo: R, rev: "5", time: "2026-11-13T20:00:00Z", blocks: {}, ops: [{ action: "create", path: "computer.socialweb.ephemeralInfra/stg-1", cid: { $link: "c5" } }] };
	w.ingest("mini", mini);
	assert.equal(w.ingest("getRecord", fetchedFrame(R, "computer.socialweb.ephemeralInfra", "stg-1", "5", "c5", { contracts: [] })).length, 0);
	assert.deepEqual(w.records.get(`at://${R}/computer.socialweb.ephemeralInfra/stg-1`).record, { contracts: [] });
});