1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
// node --test modules/knotless/test.mjs (after `cargo wasm`; needs git)
// The module against an in-memory PDS whose packs real git made, and whose commits real git reads back.
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { randomBytes } from "node:crypto";
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { argumentsOf } from "../../../atlibs/javascript/src/mcp/index.js";
import { did, load, now } from "../../../atlibs/rust/src/harness.mjs";
// The host loop, held to module.json's calls and anyRecords (the ref records it moves).
const { module, run, handle } = await load(new URL("../../target/wasm32-unknown-unknown/release/knotless.wasm", import.meta.url), new URL("./module.json", import.meta.url));
const start = (fields) => ({ did, now, ...fields, state: null, event: { type: "start" } });
const KEY = "3mzaaaaaaaaaa";
const MAIN = `${KEY}.7eq32bpgrmbxidcfbzlf4dtbop`; // refs/heads/main's rkey under KEY
const b64 = (bytes) => Buffer.from(bytes).toString("base64").replace(/=+$/, "");
// A git repository to make packs in and read commits back from.
function gitRepo() {
const dir = mkdtempSync(join(tmpdir(), "knotless-test-"));
const git = (args, input) =>
execFileSync("git", ["-C", dir, "-c", "user.name=T", "-c", "user.email=t@example.com", ...args], { input, maxBuffer: 1 << 28, env: { ...process.env, GIT_AUTHOR_DATE: "1700000000 +0000", GIT_COMMITTER_DATE: "1700000000 +0000" } });
git(["init", "-q", "-b", "main"]);
const commit = (files, message) => {
for (const [path, text] of Object.entries(files)) {
execFileSync("mkdir", ["-p", join(dir, path, "..")]);
writeFileSync(join(dir, path), text, { mode: path.endsWith(".sh") ? 0o755 : 0o644 });
}
git(["add", "-A"]);
git(["commit", "-q", "-m", message]);
return git(["rev-parse", "HEAD"]).toString().trim();
};
// What a knotless push stores: everything reachable from tips and not from bases, no thin deltas.
const pack = (tips, bases = []) => git(["pack-objects", "--revs", "--stdout", "--delta-base-offset"], [...tips, ...bases.map((b) => `^${b}`)].join("\n") + "\n");
return { dir, git, commit, pack };
}
// git's own account of a pack: its objects, and its delta chains.
function verify(bytes) {
const dir = mkdtempSync(join(tmpdir(), "knotless-pack-"));
writeFileSync(join(dir, "p.pack"), bytes);
execFileSync("git", ["index-pack", join(dir, "p.pack")]);
return execFileSync("git", ["verify-pack", "-v", join(dir, "p.idx")]).toString();
}
// One account's records, as the PDS lists them: highest rkey first, below the cursor.
function pds(records) {
const writes = [];
const blobs = {};
let refuse = 0;
const respond = ({ nsid, params, input, encoding }) => {
if (nsid === "com.atproto.repo.uploadBlob") {
const cid = `bafkrei${Object.keys(blobs).length}`;
blobs[cid] = { bytes: Buffer.from(input.$bytes, "base64"), encoding };
return { ok: true, output: { blob: { $type: "blob", ref: { $link: cid }, mimeType: encoding, size: blobs[cid].bytes.length } } };
}
if (nsid === "com.atproto.sync.getBlob") {
const blob = params.did === did && blobs[params.cid];
return blob ? { ok: true, output: { $bytes: b64(blob.bytes) } } : { ok: false, error: "com.atproto.sync.getBlob failed with BlobNotFound" };
}
if (nsid === "com.atproto.repo.getRecord") {
const value = records[`${params.collection}/${params.rkey}`];
return value ? { ok: true, output: { uri: `at://${did}/${params.collection}/${params.rkey}`, value } } : { ok: false, error: "com.atproto.repo.getRecord failed with RecordNotFound: Could not locate record" };
}
if (nsid === "com.atproto.repo.listRecords") {
const rkeys = Object.keys(records).filter((k) => k.startsWith(`${params.collection}/`)).map((k) => k.slice(params.collection.length + 1)).filter((r) => !params.cursor || r < params.cursor).sort().reverse().slice(0, Number(params.limit));
const page = rkeys.map((rkey) => ({ uri: `at://${did}/${params.collection}/${rkey}`, value: records[`${params.collection}/${rkey}`] }));
return { ok: true, output: { records: page, ...(page.length && { cursor: rkeys.at(-1) }) } };
}
if (nsid === "com.atproto.repo.createRecord") {
writes.push(input);
records[`${input.collection}/${input.rkey}`] = input.record;
return { ok: true, output: { uri: `at://${did}/${input.collection}/${input.rkey}`, cid: "bafy" } };
}
if (nsid === "com.atproto.repo.applyWrites") {
writes.push(input);
if (refuse > 0) return refuse--, { ok: false, error: "com.atproto.repo.applyWrites failed with InternalServerError" };
for (const w of input.writes) {
const k = `${w.collection}/${w.rkey}`;
if (w.$type.endsWith("#create") && records[k]) return { ok: false, error: "applyWrites failed with InvalidRequest: exists" };
records[k] = w.value;
}
return { ok: true, output: {} };
}
return { ok: false, error: `unexpected ${nsid}` };
};
return { records, writes, blobs, respond, refuseNext: (n = 1) => (refuse = n) };
}
const packRecord = (bytes, tips, bases) => ({ $type: "dev.knotless.pack", repo: KEY, tips, ...(bases.length && { bases }), size: bytes.length, objects: bytes.readUInt32BE(8), data: { $bytes: b64(bytes) }, createdAt: now });
const refRecord = (target, pack, generation = 1700000000000) => ({ $type: "dev.knotless.ref", repo: KEY, name: "refs/heads/main", target, generation, updatedAt: now, createdAt: now, pack });
const repoRecord = { $type: "dev.knotless.repo", key: KEY, name: "demo", defaultBranch: "main", createdAt: now };
// Index the module's pack (inline, or its blob parts in order) into the git repo and read what it made.
function absorb(repo, write, blobs = {}) {
const pack = write.writes.find((w) => w.collection === "dev.knotless.pack").value;
const bytes = pack.data ? Buffer.from(pack.data.$bytes, "base64") : Buffer.concat(pack.parts.map((p) => blobs[p.ref.$link].bytes));
assert.equal(pack.size, bytes.length);
repo.git(["index-pack", "--stdin", "--strict"], bytes);
return pack;
}
test("module has no imports and exports the ABI", () => {
assert.equal(WebAssembly.Module.imports(module).length, 0);
const names = WebAssembly.Module.exports(module).map((e) => e.name);
for (const name of ["memory", "alloc", "create_repo", "commit"]) assert.ok(names.includes(name), name);
});
test("create_repo writes the repo record under its name, once", async () => {
const fake = pds({});
const ran = await handle("create_repo", { name: "demo", description: "a demo" }, { respond: fake.respond });
assert.equal(ran.error, undefined, ran.error);
const [write] = fake.writes;
assert.equal(write.collection, "dev.knotless.repo");
assert.equal(write.rkey, "demo");
assert.match(write.record.key, /^[2-7a-z]{13}$/);
assert.deepEqual({ ...write.record, key: null }, { $type: "dev.knotless.repo", key: null, name: "demo", description: "a demo", defaultBranch: "main", createdAt: now });
assert.deepEqual(ran.result, { uri: `at://${did}/dev.knotless.repo/demo`, url: `knotless://${did}/demo`, name: "demo", key: write.record.key });
const again = await handle("create_repo", { name: "demo" }, { respond: fake.respond });
assert.match(again.error, /already exists/);
assert.equal(fake.writes.length, 1);
});
test("create_repo refuses bad input before any call", () => {
for (const name of ["", ".", "..", "a/b", "a b", "x".repeat(101)]) assert.match(run("create_repo", start({ name })).error, /not a repository name/, name);
assert.match(run("create_repo", start({ name: "ok", private: true })).error, /space/);
assert.match(run("create_repo", start({ name: "ok", defaultBranch: "a..b" })).error, /not a branch name/);
});
test("commit reads trees across packs, applies puts, patches and deletes, and moves the ref with a claim", async () => {
const repo = gitRepo();
const lines = Array.from({ length: 40 }, (_, i) => `line ${i}\n`).join("");
const c1 = repo.commit({ "a.txt": lines, "a2.txt": lines.replace("line 9\n", "line nine\n"), "dir/b.txt": "b\n", "dir/deep/c.txt": "c\n", "run.sh": "#!/bin/sh\n" }, "one");
const c2 = repo.commit({ "other.txt": lines.replace("line 3\n", "line three\n") }, "two");
const p1 = repo.pack([c1]);
const p2 = repo.pack([c2], [c1]);
assert.match(verify(p1), /chain length = 1/, "the first pack holds a delta");
const fake = pds({
"dev.knotless.repo/demo": repoRecord,
[`dev.knotless.pack/${KEY}.3mzaaaaaaaaa2`]: packRecord(p1, [c1], []),
[`dev.knotless.pack/${KEY}.3mzaaaaaaaaa3`]: packRecord(p2, [c2], [c1]),
// Another repository's pack, which the walk must stop before.
["dev.knotless.pack/3mz2222222222.3mzaaaaaaaaa9"]: { repo: "3mz2222222222", tips: [], size: 0, objects: 1, createdAt: now },
[`dev.knotless.ref/${MAIN}`]: refRecord(c2, `${KEY}.3mzaaaaaaaaa3`),
});
const changes = [
{ path: "a.txt", patch: "--- a/a.txt\n+++ b/a.txt\n@@ -20,3 +20,3 @@\n line 19\n-line 20\n+line twenty\n line 21\n" },
{ path: "dir/b.txt", delete: true },
{ path: "new/dir/n.txt", content: "new\n" },
{ path: "run.sh", content: "#!/bin/sh\necho hi\n" },
{ path: "tool", content: "x", executable: true },
];
const ran = await handle("commit", { name: "demo", branch: "main", message: "three", author: "Ada <ada@example.com>", changes }, { respond: fake.respond });
assert.equal(ran.error, undefined, ran.error);
const write = fake.writes.at(-1);
const pack = absorb(repo, write);
const sha = ran.result.sha;
assert.deepEqual(ran.result, { sha, parent: c2, changed: true, branch: "main", pack: write.writes[0].rkey, url: `knotless://${did}/demo` });
assert.deepEqual(pack.tips, [sha]);
assert.deepEqual(pack.bases, [c2]);
assert.equal(repo.git(["show", `${sha}:a.txt`]).toString(), lines.replace("line 20\n", "line twenty\n"));
assert.equal(repo.git(["show", `${sha}:new/dir/n.txt`]).toString(), "new\n");
const tree = repo.git(["ls-tree", "-r", sha]).toString();
assert.match(tree, /^100755 blob \S+\trun\.sh$/m, "an existing file keeps its mode");
assert.match(tree, /^100755 blob \S+\ttool$/m);
assert.doesNotMatch(tree, /dir\/b\.txt/);
assert.match(tree, /dir\/deep\/c\.txt/);
const commit = repo.git(["cat-file", "-p", sha]).toString();
assert.match(commit, new RegExp(`^parent ${c2}$`, "m"));
assert.match(commit, /^author Ada <ada@example\.com> 1793968496 \+0000$/m);
assert.match(commit, /\n\nthree\n$/);
repo.git(["update-ref", "refs/heads/main", sha]);
repo.git(["fsck", "--strict", "--no-dangling"]);
// The claim takes the generation read; the ref moves to the next one and names the new pack.
assert.deepEqual(write.writes.map((w) => [w.$type.split("#")[1], w.collection]), [["create", "dev.knotless.pack"], ["create", "dev.knotless.claim"], ["update", "dev.knotless.ref"]]);
assert.equal(write.writes[1].rkey, `${MAIN}.1700000000000`);
assert.deepEqual(write.writes[1].value, { $type: "dev.knotless.claim", repo: KEY, name: "refs/heads/main", generation: 1700000000000, from: c2, to: sha, pack: write.writes[0].rkey, createdAt: now });
assert.deepEqual(write.writes[2].value, { ...refRecord(sha, write.writes[0].rkey, 1700000000001) });
// The same changes again leave the tree as it is: nothing is written.
const same = await handle("commit", { name: "demo", branch: "main", message: "again", changes: JSON.stringify([{ path: "new/dir/n.txt", content: "new\n" }]) }, { respond: fake.respond });
assert.deepEqual(same.result, { sha, changed: false, branch: "main", url: `knotless://${did}/demo` });
// A new branch starts from the default branch, its ref a create with no claim.
const branched = await handle("commit", { name: "demo", branch: "topic", message: "topic", changes: [{ path: "t.txt", content: "t" }] }, { respond: fake.respond });
assert.equal(branched.error, undefined, branched.error);
const bw = fake.writes.at(-1);
absorb(repo, bw);
assert.equal(branched.result.parent, sha);
assert.deepEqual(bw.writes.map((w) => [w.$type.split("#")[1], w.collection]), [["create", "dev.knotless.pack"], ["create", "dev.knotless.ref"]]);
assert.equal(bw.writes[1].value.name, "refs/heads/topic");
assert.equal(bw.writes[1].value.generation, 1793968496789);
assert.equal(repo.git(["show", `${branched.result.sha}:t.txt`]).toString(), "t");
assert.match(repo.git(["cat-file", "-p", branched.result.sha]).toString(), new RegExp(`^author ${did} <${did}@knotless.invalid>`, "m"));
});
test("a commit that loses to a move of its branch builds again on the new tip", async () => {
const repo = gitRepo();
const c1 = repo.commit({ "f.txt": "1\n2\n3\n" }, "one");
const p1 = repo.pack([c1]);
const fake = pds({
"dev.knotless.repo/demo": repoRecord,
[`dev.knotless.pack/${KEY}.3mzaaaaaaaaa2`]: packRecord(p1, [c1], []),
[`dev.knotless.ref/${MAIN}`]: refRecord(c1, `${KEY}.3mzaaaaaaaaa2`),
});
// Someone else lands c2 between our read and our write.
const c2 = repo.commit({ "g.txt": "g\n" }, "two");
const p2 = repo.pack([c2], [c1]);
const original = fake.respond;
let raced = false;
const respond = (req) => {
if (req.nsid === "com.atproto.repo.applyWrites" && !raced) {
raced = true;
fake.records[`dev.knotless.pack/${KEY}.3mzaaaaaaaaa3`] = packRecord(p2, [c2], [c1]);
fake.records[`dev.knotless.claim/${MAIN}.1700000000000`] = { generation: 1700000000000 };
fake.records[`dev.knotless.ref/${MAIN}`] = refRecord(c2, `${KEY}.3mzaaaaaaaaa3`, 1700000000001);
}
return original(req);
};
const ran = await handle("commit", { name: "demo", branch: "main", message: "mine", changes: [{ path: "f.txt", patch: "@@ -2 +2 @@\n-2\n+two\n" }] }, { respond });
assert.equal(ran.error, undefined, ran.error);
assert.equal(ran.result.parent, c2);
absorb(repo, fake.writes.at(-1));
assert.equal(repo.git(["show", `${ran.result.sha}:f.txt`]).toString(), "1\ntwo\n3\n");
assert.equal(repo.git(["show", `${ran.result.sha}:g.txt`]).toString(), "g\n");
});
test("commit to an empty repository makes a root commit; errors come back named", async () => {
const repo = gitRepo();
const fake = pds({ "dev.knotless.repo/demo": repoRecord });
const ran = await handle("commit", { name: "demo", branch: "main", message: "first", changes: [{ path: "README.md", content: "# demo\n" }] }, { respond: fake.respond });
assert.equal(ran.error, undefined, ran.error);
absorb(repo, fake.writes.at(-1));
assert.equal(ran.result.parent, null);
assert.doesNotMatch(repo.git(["cat-file", "-p", ran.result.sha]).toString(), /^parent/m);
assert.equal(fake.writes.at(-1).writes[0].value.bases, undefined);
const failed = await handle("commit", { name: "demo", branch: "main", message: "x", changes: [{ path: "README.md", patch: "@@ -1 +1 @@\n-nope\n+yes\n" }] }, { respond: fake.respond });
assert.match(failed.error, /README\.md: hunk 1 .* does not apply/);
fake.refuseNext(1);
const refused = await handle("commit", { name: "demo", branch: "main", message: "x", changes: [{ path: "b", content: "b" }] }, { respond: fake.respond });
assert.match(refused.error, /InternalServerError/, "a write refused with the ref unmoved is the run's error");
assert.match((await handle("commit", { name: "nope", branch: "main", message: "x", changes: [{ path: "a", content: "a" }] }, { respond: fake.respond })).error, /does not exist/);
assert.match((await handle("commit", { name: "demo", branch: "new", parent: "gone", message: "x", changes: [{ path: "a", content: "a" }] }, { respond: fake.respond })).error, /branch gone does not exist/);
});
test("commit refuses bad changes before any call", () => {
const base = { name: "demo", branch: "main", message: "m" };
for (const path of ["", "/a", "a//b", "a/../b", ".git/config", "x/.GIT/y", "a\nb"]) assert.match(run("commit", start({ ...base, changes: [{ path, content: "" }] })).error, /not a path/, path);
assert.match(run("commit", start({ ...base, changes: [{ path: "a", content: "x", delete: true }] })).error, /exactly one/);
assert.match(run("commit", start({ ...base, changes: [] })).error, /empty/);
assert.match(run("commit", start({ ...base, changes: "[" })).error, /not JSON/);
assert.match(run("commit", start({ ...base, branch: "a b", changes: [{ path: "a", content: "" }] })).error, /not a branch name/);
assert.match(run("commit", start({ ...base, author: "no email", changes: [{ path: "a", content: "" }] })).error, /Name <email>/);
});
test("a pack past 192 KiB goes over 4 MB blob parts, and a later commit reads its trees back from them", async () => {
const repo = gitRepo();
const fake = pds({ "dev.knotless.repo/demo": repoRecord });
// ~6 MB of incompressible text: a pack of two parts
const big = randomBytes(4_500_000).toString("base64");
const ran = await handle("commit", { name: "demo", branch: "main", message: "big", changes: [{ path: "big.txt", content: big }, { path: "dir/small.txt", content: "s\n" }] }, { respond: fake.respond, budget: { ioBytes: 16 << 20 } });
assert.equal(ran.error, undefined, ran.error);
const write = fake.writes.at(-1);
const pack = absorb(repo, write, fake.blobs);
assert.equal(pack.data, undefined);
assert.equal(pack.parts.length, 2);
assert.ok(Object.values(fake.blobs).every((b) => b.encoding === "application/x-git-packed-objects" && b.bytes.length <= 4_000_000));
assert.equal(repo.git(["show", `${ran.result.sha}:big.txt`]).toString(), big);
// The next commit finds the parent's trees only in that blob-backed pack.
const next = await handle("commit", { name: "demo", branch: "main", message: "small", changes: [{ path: "dir/small.txt", patch: "@@ -1 +1 @@\n-s\n+S\n" }] }, { respond: fake.respond, budget: { ioBytes: 16 << 20 } });
assert.equal(next.error, undefined, next.error);
assert.equal(next.result.parent, ran.result.sha);
assert.deepEqual(next.calls.filter((c) => c.nsid === "com.atproto.sync.getBlob").length, 2);
absorb(repo, fake.writes.at(-1), fake.blobs);
assert.equal(repo.git(["show", `${next.result.sha}:dir/small.txt`]).toString(), "S\n");
assert.equal(repo.git(["show", `${next.result.sha}:big.txt`]).toString(), big);
repo.git(["update-ref", "refs/heads/main", next.result.sha]);
repo.git(["fsck", "--strict", "--no-dangling"]);
});
// The MCP adapter types arguments by module.json: changes is a list (or its JSON text), not a string.
test("module.json: commit's changes is an array field", async () => {
const { default: spec } = await import("./module.json", { with: { type: "json" } });
const fields = spec.actions.find((a) => a.entry === "commit").fields;
const changes = [{ path: "a", content: "a" }];
assert.deepEqual(argumentsOf(fields, { name: "demo", changes }).args.changes, changes);
assert.deepEqual(argumentsOf(fields, { name: "demo", changes: JSON.stringify(changes) }).args.changes, changes);
assert.deepEqual(argumentsOf(fields, { changes: "nope" }).problems, ["changes must be an array"]);
});
