// node --test modules/knotless/test.mjs (after `cargo wasm`; needs git) // The module against an in-memory PDS whose packs real git made, and whose commits real git reads back. import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { argumentsOf } from "../../../atlibs/javascript/src/mcp/index.js"; import { did, load, now } from "../../../atlibs/rust/src/harness.mjs"; // The host loop, held to module.json's calls and anyRecords (the ref records it moves). const { module, run, handle } = await load(new URL("../../target/wasm32-unknown-unknown/release/knotless.wasm", import.meta.url), new URL("./module.json", import.meta.url)); const start = (fields) => ({ did, now, ...fields, state: null, event: { type: "start" } }); const KEY = "3mzaaaaaaaaaa"; const MAIN = `${KEY}.7eq32bpgrmbxidcfbzlf4dtbop`; // refs/heads/main's rkey under KEY const b64 = (bytes) => Buffer.from(bytes).toString("base64").replace(/=+$/, ""); // A git repository to make packs in and read commits back from. function gitRepo() { const dir = mkdtempSync(join(tmpdir(), "knotless-test-")); const git = (args, input) => execFileSync("git", ["-C", dir, "-c", "user.name=T", "-c", "user.email=t@example.com", ...args], { input, maxBuffer: 1 << 28, env: { ...process.env, GIT_AUTHOR_DATE: "1700000000 +0000", GIT_COMMITTER_DATE: "1700000000 +0000" } }); git(["init", "-q", "-b", "main"]); const commit = (files, message) => { for (const [path, text] of Object.entries(files)) { execFileSync("mkdir", ["-p", join(dir, path, "..")]); writeFileSync(join(dir, path), text, { mode: path.endsWith(".sh") ? 0o755 : 0o644 }); } git(["add", "-A"]); git(["commit", "-q", "-m", message]); return git(["rev-parse", "HEAD"]).toString().trim(); }; // What a knotless push stores: everything reachable from tips and not from bases, no thin deltas. const pack = (tips, bases = []) => git(["pack-objects", "--revs", "--stdout", "--delta-base-offset"], [...tips, ...bases.map((b) => `^${b}`)].join("\n") + "\n"); return { dir, git, commit, pack }; } // git's own account of a pack: its objects, and its delta chains. function verify(bytes) { const dir = mkdtempSync(join(tmpdir(), "knotless-pack-")); writeFileSync(join(dir, "p.pack"), bytes); execFileSync("git", ["index-pack", join(dir, "p.pack")]); return execFileSync("git", ["verify-pack", "-v", join(dir, "p.idx")]).toString(); } // One account's records, as the PDS lists them: highest rkey first, below the cursor. function pds(records) { const writes = []; const blobs = {}; let refuse = 0; const respond = ({ nsid, params, input, encoding }) => { if (nsid === "com.atproto.repo.uploadBlob") { const cid = `bafkrei${Object.keys(blobs).length}`; blobs[cid] = { bytes: Buffer.from(input.$bytes, "base64"), encoding }; return { ok: true, output: { blob: { $type: "blob", ref: { $link: cid }, mimeType: encoding, size: blobs[cid].bytes.length } } }; } if (nsid === "com.atproto.sync.getBlob") { const blob = params.did === did && blobs[params.cid]; return blob ? { ok: true, output: { $bytes: b64(blob.bytes) } } : { ok: false, error: "com.atproto.sync.getBlob failed with BlobNotFound" }; } if (nsid === "com.atproto.repo.getRecord") { const value = records[`${params.collection}/${params.rkey}`]; return value ? { ok: true, output: { uri: `at://${did}/${params.collection}/${params.rkey}`, value } } : { ok: false, error: "com.atproto.repo.getRecord failed with RecordNotFound: Could not locate record" }; } if (nsid === "com.atproto.repo.listRecords") { const rkeys = Object.keys(records).filter((k) => k.startsWith(`${params.collection}/`)).map((k) => k.slice(params.collection.length + 1)).filter((r) => !params.cursor || r < params.cursor).sort().reverse().slice(0, Number(params.limit)); const page = rkeys.map((rkey) => ({ uri: `at://${did}/${params.collection}/${rkey}`, value: records[`${params.collection}/${rkey}`] })); return { ok: true, output: { records: page, ...(page.length && { cursor: rkeys.at(-1) }) } }; } if (nsid === "com.atproto.repo.createRecord") { writes.push(input); records[`${input.collection}/${input.rkey}`] = input.record; return { ok: true, output: { uri: `at://${did}/${input.collection}/${input.rkey}`, cid: "bafy" } }; } if (nsid === "com.atproto.repo.applyWrites") { writes.push(input); if (refuse > 0) return refuse--, { ok: false, error: "com.atproto.repo.applyWrites failed with InternalServerError" }; for (const w of input.writes) { const k = `${w.collection}/${w.rkey}`; if (w.$type.endsWith("#create") && records[k]) return { ok: false, error: "applyWrites failed with InvalidRequest: exists" }; records[k] = w.value; } return { ok: true, output: {} }; } return { ok: false, error: `unexpected ${nsid}` }; }; return { records, writes, blobs, respond, refuseNext: (n = 1) => (refuse = n) }; } const packRecord = (bytes, tips, bases) => ({ $type: "dev.knotless.pack", repo: KEY, tips, ...(bases.length && { bases }), size: bytes.length, objects: bytes.readUInt32BE(8), data: { $bytes: b64(bytes) }, createdAt: now }); const refRecord = (target, pack, generation = 1700000000000) => ({ $type: "dev.knotless.ref", repo: KEY, name: "refs/heads/main", target, generation, updatedAt: now, createdAt: now, pack }); const repoRecord = { $type: "dev.knotless.repo", key: KEY, name: "demo", defaultBranch: "main", createdAt: now }; // Index the module's pack (inline, or its blob parts in order) into the git repo and read what it made. function absorb(repo, write, blobs = {}) { const pack = write.writes.find((w) => w.collection === "dev.knotless.pack").value; const bytes = pack.data ? Buffer.from(pack.data.$bytes, "base64") : Buffer.concat(pack.parts.map((p) => blobs[p.ref.$link].bytes)); assert.equal(pack.size, bytes.length); repo.git(["index-pack", "--stdin", "--strict"], bytes); return pack; } test("module has no imports and exports the ABI", () => { assert.equal(WebAssembly.Module.imports(module).length, 0); const names = WebAssembly.Module.exports(module).map((e) => e.name); for (const name of ["memory", "alloc", "create_repo", "commit"]) assert.ok(names.includes(name), name); }); test("create_repo writes the repo record under its name, once", async () => { const fake = pds({}); const ran = await handle("create_repo", { name: "demo", description: "a demo" }, { respond: fake.respond }); assert.equal(ran.error, undefined, ran.error); const [write] = fake.writes; assert.equal(write.collection, "dev.knotless.repo"); assert.equal(write.rkey, "demo"); assert.match(write.record.key, /^[2-7a-z]{13}$/); assert.deepEqual({ ...write.record, key: null }, { $type: "dev.knotless.repo", key: null, name: "demo", description: "a demo", defaultBranch: "main", createdAt: now }); assert.deepEqual(ran.result, { uri: `at://${did}/dev.knotless.repo/demo`, url: `knotless://${did}/demo`, name: "demo", key: write.record.key }); const again = await handle("create_repo", { name: "demo" }, { respond: fake.respond }); assert.match(again.error, /already exists/); assert.equal(fake.writes.length, 1); }); test("create_repo refuses bad input before any call", () => { for (const name of ["", ".", "..", "a/b", "a b", "x".repeat(101)]) assert.match(run("create_repo", start({ name })).error, /not a repository name/, name); assert.match(run("create_repo", start({ name: "ok", private: true })).error, /space/); assert.match(run("create_repo", start({ name: "ok", defaultBranch: "a..b" })).error, /not a branch name/); }); test("commit reads trees across packs, applies puts, patches and deletes, and moves the ref with a claim", async () => { const repo = gitRepo(); const lines = Array.from({ length: 40 }, (_, i) => `line ${i}\n`).join(""); const c1 = repo.commit({ "a.txt": lines, "a2.txt": lines.replace("line 9\n", "line nine\n"), "dir/b.txt": "b\n", "dir/deep/c.txt": "c\n", "run.sh": "#!/bin/sh\n" }, "one"); const c2 = repo.commit({ "other.txt": lines.replace("line 3\n", "line three\n") }, "two"); const p1 = repo.pack([c1]); const p2 = repo.pack([c2], [c1]); assert.match(verify(p1), /chain length = 1/, "the first pack holds a delta"); const fake = pds({ "dev.knotless.repo/demo": repoRecord, [`dev.knotless.pack/${KEY}.3mzaaaaaaaaa2`]: packRecord(p1, [c1], []), [`dev.knotless.pack/${KEY}.3mzaaaaaaaaa3`]: packRecord(p2, [c2], [c1]), // Another repository's pack, which the walk must stop before. ["dev.knotless.pack/3mz2222222222.3mzaaaaaaaaa9"]: { repo: "3mz2222222222", tips: [], size: 0, objects: 1, createdAt: now }, [`dev.knotless.ref/${MAIN}`]: refRecord(c2, `${KEY}.3mzaaaaaaaaa3`), }); const changes = [ { path: "a.txt", patch: "--- a/a.txt\n+++ b/a.txt\n@@ -20,3 +20,3 @@\n line 19\n-line 20\n+line twenty\n line 21\n" }, { path: "dir/b.txt", delete: true }, { path: "new/dir/n.txt", content: "new\n" }, { path: "run.sh", content: "#!/bin/sh\necho hi\n" }, { path: "tool", content: "x", executable: true }, ]; const ran = await handle("commit", { name: "demo", branch: "main", message: "three", author: "Ada ", changes }, { respond: fake.respond }); assert.equal(ran.error, undefined, ran.error); const write = fake.writes.at(-1); const pack = absorb(repo, write); const sha = ran.result.sha; assert.deepEqual(ran.result, { sha, parent: c2, changed: true, branch: "main", pack: write.writes[0].rkey, url: `knotless://${did}/demo` }); assert.deepEqual(pack.tips, [sha]); assert.deepEqual(pack.bases, [c2]); assert.equal(repo.git(["show", `${sha}:a.txt`]).toString(), lines.replace("line 20\n", "line twenty\n")); assert.equal(repo.git(["show", `${sha}:new/dir/n.txt`]).toString(), "new\n"); const tree = repo.git(["ls-tree", "-r", sha]).toString(); assert.match(tree, /^100755 blob \S+\trun\.sh$/m, "an existing file keeps its mode"); assert.match(tree, /^100755 blob \S+\ttool$/m); assert.doesNotMatch(tree, /dir\/b\.txt/); assert.match(tree, /dir\/deep\/c\.txt/); const commit = repo.git(["cat-file", "-p", sha]).toString(); assert.match(commit, new RegExp(`^parent ${c2}$`, "m")); assert.match(commit, /^author Ada 1793968496 \+0000$/m); assert.match(commit, /\n\nthree\n$/); repo.git(["update-ref", "refs/heads/main", sha]); repo.git(["fsck", "--strict", "--no-dangling"]); // The claim takes the generation read; the ref moves to the next one and names the new pack. assert.deepEqual(write.writes.map((w) => [w.$type.split("#")[1], w.collection]), [["create", "dev.knotless.pack"], ["create", "dev.knotless.claim"], ["update", "dev.knotless.ref"]]); assert.equal(write.writes[1].rkey, `${MAIN}.1700000000000`); assert.deepEqual(write.writes[1].value, { $type: "dev.knotless.claim", repo: KEY, name: "refs/heads/main", generation: 1700000000000, from: c2, to: sha, pack: write.writes[0].rkey, createdAt: now }); assert.deepEqual(write.writes[2].value, { ...refRecord(sha, write.writes[0].rkey, 1700000000001) }); // The same changes again leave the tree as it is: nothing is written. const same = await handle("commit", { name: "demo", branch: "main", message: "again", changes: JSON.stringify([{ path: "new/dir/n.txt", content: "new\n" }]) }, { respond: fake.respond }); assert.deepEqual(same.result, { sha, changed: false, branch: "main", url: `knotless://${did}/demo` }); // A new branch starts from the default branch, its ref a create with no claim. const branched = await handle("commit", { name: "demo", branch: "topic", message: "topic", changes: [{ path: "t.txt", content: "t" }] }, { respond: fake.respond }); assert.equal(branched.error, undefined, branched.error); const bw = fake.writes.at(-1); absorb(repo, bw); assert.equal(branched.result.parent, sha); assert.deepEqual(bw.writes.map((w) => [w.$type.split("#")[1], w.collection]), [["create", "dev.knotless.pack"], ["create", "dev.knotless.ref"]]); assert.equal(bw.writes[1].value.name, "refs/heads/topic"); assert.equal(bw.writes[1].value.generation, 1793968496789); assert.equal(repo.git(["show", `${branched.result.sha}:t.txt`]).toString(), "t"); assert.match(repo.git(["cat-file", "-p", branched.result.sha]).toString(), new RegExp(`^author ${did} <${did}@knotless.invalid>`, "m")); }); test("a commit that loses to a move of its branch builds again on the new tip", async () => { const repo = gitRepo(); const c1 = repo.commit({ "f.txt": "1\n2\n3\n" }, "one"); const p1 = repo.pack([c1]); const fake = pds({ "dev.knotless.repo/demo": repoRecord, [`dev.knotless.pack/${KEY}.3mzaaaaaaaaa2`]: packRecord(p1, [c1], []), [`dev.knotless.ref/${MAIN}`]: refRecord(c1, `${KEY}.3mzaaaaaaaaa2`), }); // Someone else lands c2 between our read and our write. const c2 = repo.commit({ "g.txt": "g\n" }, "two"); const p2 = repo.pack([c2], [c1]); const original = fake.respond; let raced = false; const respond = (req) => { if (req.nsid === "com.atproto.repo.applyWrites" && !raced) { raced = true; fake.records[`dev.knotless.pack/${KEY}.3mzaaaaaaaaa3`] = packRecord(p2, [c2], [c1]); fake.records[`dev.knotless.claim/${MAIN}.1700000000000`] = { generation: 1700000000000 }; fake.records[`dev.knotless.ref/${MAIN}`] = refRecord(c2, `${KEY}.3mzaaaaaaaaa3`, 1700000000001); } return original(req); }; const ran = await handle("commit", { name: "demo", branch: "main", message: "mine", changes: [{ path: "f.txt", patch: "@@ -2 +2 @@\n-2\n+two\n" }] }, { respond }); assert.equal(ran.error, undefined, ran.error); assert.equal(ran.result.parent, c2); absorb(repo, fake.writes.at(-1)); assert.equal(repo.git(["show", `${ran.result.sha}:f.txt`]).toString(), "1\ntwo\n3\n"); assert.equal(repo.git(["show", `${ran.result.sha}:g.txt`]).toString(), "g\n"); }); test("commit to an empty repository makes a root commit; errors come back named", async () => { const repo = gitRepo(); const fake = pds({ "dev.knotless.repo/demo": repoRecord }); const ran = await handle("commit", { name: "demo", branch: "main", message: "first", changes: [{ path: "README.md", content: "# demo\n" }] }, { respond: fake.respond }); assert.equal(ran.error, undefined, ran.error); absorb(repo, fake.writes.at(-1)); assert.equal(ran.result.parent, null); assert.doesNotMatch(repo.git(["cat-file", "-p", ran.result.sha]).toString(), /^parent/m); assert.equal(fake.writes.at(-1).writes[0].value.bases, undefined); const failed = await handle("commit", { name: "demo", branch: "main", message: "x", changes: [{ path: "README.md", patch: "@@ -1 +1 @@\n-nope\n+yes\n" }] }, { respond: fake.respond }); assert.match(failed.error, /README\.md: hunk 1 .* does not apply/); fake.refuseNext(1); const refused = await handle("commit", { name: "demo", branch: "main", message: "x", changes: [{ path: "b", content: "b" }] }, { respond: fake.respond }); assert.match(refused.error, /InternalServerError/, "a write refused with the ref unmoved is the run's error"); assert.match((await handle("commit", { name: "nope", branch: "main", message: "x", changes: [{ path: "a", content: "a" }] }, { respond: fake.respond })).error, /does not exist/); assert.match((await handle("commit", { name: "demo", branch: "new", parent: "gone", message: "x", changes: [{ path: "a", content: "a" }] }, { respond: fake.respond })).error, /branch gone does not exist/); }); test("commit refuses bad changes before any call", () => { const base = { name: "demo", branch: "main", message: "m" }; for (const path of ["", "/a", "a//b", "a/../b", ".git/config", "x/.GIT/y", "a\nb"]) assert.match(run("commit", start({ ...base, changes: [{ path, content: "" }] })).error, /not a path/, path); assert.match(run("commit", start({ ...base, changes: [{ path: "a", content: "x", delete: true }] })).error, /exactly one/); assert.match(run("commit", start({ ...base, changes: [] })).error, /empty/); assert.match(run("commit", start({ ...base, changes: "[" })).error, /not JSON/); assert.match(run("commit", start({ ...base, branch: "a b", changes: [{ path: "a", content: "" }] })).error, /not a branch name/); assert.match(run("commit", start({ ...base, author: "no email", changes: [{ path: "a", content: "" }] })).error, /Name /); }); test("a pack past 192 KiB goes over 4 MB blob parts, and a later commit reads its trees back from them", async () => { const repo = gitRepo(); const fake = pds({ "dev.knotless.repo/demo": repoRecord }); // ~6 MB of incompressible text: a pack of two parts const big = randomBytes(4_500_000).toString("base64"); const ran = await handle("commit", { name: "demo", branch: "main", message: "big", changes: [{ path: "big.txt", content: big }, { path: "dir/small.txt", content: "s\n" }] }, { respond: fake.respond, budget: { ioBytes: 16 << 20 } }); assert.equal(ran.error, undefined, ran.error); const write = fake.writes.at(-1); const pack = absorb(repo, write, fake.blobs); assert.equal(pack.data, undefined); assert.equal(pack.parts.length, 2); assert.ok(Object.values(fake.blobs).every((b) => b.encoding === "application/x-git-packed-objects" && b.bytes.length <= 4_000_000)); assert.equal(repo.git(["show", `${ran.result.sha}:big.txt`]).toString(), big); // The next commit finds the parent's trees only in that blob-backed pack. const next = await handle("commit", { name: "demo", branch: "main", message: "small", changes: [{ path: "dir/small.txt", patch: "@@ -1 +1 @@\n-s\n+S\n" }] }, { respond: fake.respond, budget: { ioBytes: 16 << 20 } }); assert.equal(next.error, undefined, next.error); assert.equal(next.result.parent, ran.result.sha); assert.deepEqual(next.calls.filter((c) => c.nsid === "com.atproto.sync.getBlob").length, 2); absorb(repo, fake.writes.at(-1), fake.blobs); assert.equal(repo.git(["show", `${next.result.sha}:dir/small.txt`]).toString(), "S\n"); assert.equal(repo.git(["show", `${next.result.sha}:big.txt`]).toString(), big); repo.git(["update-ref", "refs/heads/main", next.result.sha]); repo.git(["fsck", "--strict", "--no-dangling"]); }); // The MCP adapter types arguments by module.json: changes is a list (or its JSON text), not a string. test("module.json: commit's changes is an array field", async () => { const { default: spec } = await import("./module.json", { with: { type: "json" } }); const fields = spec.actions.find((a) => a.entry === "commit").fields; const changes = [{ path: "a", content: "a" }]; assert.deepEqual(argumentsOf(fields, { name: "demo", changes }).args.changes, changes); assert.deepEqual(argumentsOf(fields, { name: "demo", changes: JSON.stringify(changes) }).args.changes, changes); assert.deepEqual(argumentsOf(fields, { changes: "nope" }).problems, ["changes must be an array"]); });