chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
/**
* The pack object reader and browser, checked against real git: bundles made
* by the git CLI are indexed in JS and the resulting trees, listings, and
* file contents must match what git itself reports.
*/
import { execFile } from 'node:child_process';
import { randomBytes } from 'node:crypto';
import {
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { promisify } from 'node:util';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createGitBrowser } from '../src/browser.js';
import { parseBundle } from '../src/bundle.js';
import { diffLines } from '../src/diff.js';
import {
commitTree,
indexPacks,
objectId,
parseTree,
resolveToTree,
} from '../src/objects.js';
import { buildBundleIndex, encodePackIndex } from '../src/pack-index.js';
import { splitIntoChunks } from '../src/record.js';
const execFileAsync = promisify(execFile);
const PNG_MAGIC = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
/**
* A file that reads as a PNG. The IHDR length prefix carries the NUL bytes
* the binary detector looks for, as it does in a real PNG.
* @param {number} tail - how many bytes follow the header
*/
function fakePng(tail) {
return Buffer.concat([
Buffer.from(PNG_MAGIC),
Buffer.from([0, 0, 0, 13]),
Buffer.from('IHDR'),
randomBytes(tail),
]);
}
/** @type {string} */
let dir = '';
/** @type {string} */
let repo = '';
/** @type {string} */
let headSha = '';
/** @type {Uint8Array[]} */
let packs = [];
const env = {
.../** @type {Record<string, string>} */ (process.env),
GIT_AUTHOR_NAME: 'Test',
GIT_AUTHOR_EMAIL: 'test@example.com',
GIT_COMMITTER_NAME: 'Test',
GIT_COMMITTER_EMAIL: 'test@example.com',
};
/**
* @param {string} cwd
* @param {string[]} args
*/
async function runGit(cwd, args) {
const { stdout } = await execFileAsync('git', args, { cwd, env });
return stdout.trim();
}
beforeAll(async () => {
dir = mkdtempSync(join(tmpdir(), 'pdsjs-git-browser-'));
repo = join(dir, 'repo');
await runGit(dir, ['init', '-b', 'main', repo]);
mkdirSync(join(repo, 'src'));
mkdirSync(join(repo, 'src', 'deep'));
writeFileSync(join(repo, 'README.md'), '# browser test\n');
writeFileSync(join(repo, 'src', 'index.js'), 'export const one = 1;\n');
writeFileSync(join(repo, 'src', 'deep', 'nested.txt'), 'nested content\n');
// Deterministically binary: the detector looks for NUL bytes
writeFileSync(
join(repo, 'binary.bin'),
Buffer.concat([Buffer.from([0, 1, 2, 3]), randomBytes(508)]),
);
writeFileSync(join(repo, 'logo.png'), fakePng(48));
await runGit(repo, ['add', '.']);
await runGit(repo, ['commit', '-m', 'one']);
const first = await runGit(repo, ['rev-parse', 'HEAD']);
await runGit(repo, ['bundle', 'create', join(dir, 'b1.bundle'), 'main']);
// A second commit REWRITING an existing file makes the incremental bundle
// carry deltas whose bases live in the first pack.
writeFileSync(
join(repo, 'src', 'index.js'),
'export const one = 1;\nexport const two = 2;\n',
);
await runGit(repo, ['add', '.']);
await runGit(repo, ['commit', '-m', 'two']);
await runGit(repo, ['tag', '-a', 'v1', '-m', 'release one']);
headSha = await runGit(repo, ['rev-parse', 'HEAD']);
await runGit(repo, [
'bundle',
'create',
join(dir, 'b2.bundle'),
`^${first}`,
'main',
'v1',
]);
packs = [
parseBundle(readFileSync(join(dir, 'b1.bundle'))).pack,
parseBundle(readFileSync(join(dir, 'b2.bundle'))).pack,
];
});
afterAll(() => {
rmSync(dir, { recursive: true, force: true });
});
describe('pack object reader', () => {
it('indexes every object with ids git agrees with', async () => {
const objects = await indexPacks(packs);
const commit = objects.get(headSha);
expect(commit).toBeDefined();
expect(commit?.type).toBe('commit');
// Every object id git can list must be present and correctly typed
const listed = (
await runGit(repo, ['rev-list', '--objects', 'main'])
).split('\n');
for (const line of listed) {
const sha = line.split(' ')[0];
const object = objects.get(sha);
expect(object, sha).toBeDefined();
expect(
objectId(
/** @type {{type: string}} */ (object).type,
/** @type {{data: Uint8Array}} */ (object).data,
),
).toBe(sha);
}
});
it('walks commit to tree to entries like git ls-tree', async () => {
const objects = await indexPacks(packs);
const tree = commitTree(
/** @type {{data: Uint8Array}} */ (objects.get(headSha)).data,
);
const entries = parseTree(
/** @type {{data: Uint8Array}} */ (objects.get(tree)).data,
);
const expected = (await runGit(repo, ['ls-tree', 'main']))
.split('\n')
.map((line) => line.split('\t')[1]);
expect(entries.map((e) => e.name)).toEqual(expected);
});
it('peels annotated tags to the commit tree', async () => {
const objects = await indexPacks(packs);
const tagSha = await runGit(repo, ['rev-parse', 'v1']);
const treeFromTag = resolveToTree(objects, tagSha);
const treeFromCommit = resolveToTree(objects, headSha);
expect(treeFromTag).toBe(treeFromCommit);
});
});
/**
* @param {Uint8Array} bytes
* @param {string} cid
*/
function blobRef(bytes, cid) {
return {
$type: 'blob',
ref: { $link: cid },
mimeType: 'application/octet-stream',
size: bytes.length,
};
}
/**
* A fake PDS over a blob map. Range requests answer 206 with the slice,
* the way the real getBlob endpoint does, and every response's bytes are
* counted so a test can bound what a reader downloaded.
* @param {Record<string, unknown>} record
* @param {Map<string, Uint8Array>} blobs
* @param {{bytes: number}} stats
* @param {{ranges?: boolean}} [opts] - ranges: false ignores Range headers
* and answers 200 whole, like a server from before the range support
*/
function fakePds(record, blobs, stats, opts = {}) {
return async (/** @type {Request} */ request) => {
const url = new URL(request.url);
if (url.pathname.endsWith('getRecord')) {
return Response.json({
cid: 'bafyrecordcid',
value: record,
});
}
if (url.pathname.endsWith('getBlob')) {
const blob = blobs.get(url.searchParams.get('cid') || '');
if (!blob) return Response.json({ error: 'NotFound' }, { status: 404 });
const match = /^bytes=(\d+)-(\d+)$/.exec(
request.headers.get('range') || '',
);
if (match && opts.ranges !== false) {
const start = Number(match[1]);
const end = Math.min(Number(match[2]), blob.length - 1);
const body = blob.subarray(start, end + 1);
stats.bytes += body.length;
return new Response(/** @type {BodyInit} */ (body), { status: 206 });
}
stats.bytes += blob.length;
return new Response(/** @type {BodyInit} */ (blob));
}
return Response.json({ error: 'NotFound' }, { status: 404 });
};
}
/**
* Record and blob map for a bundle chain. Lazy mode chunks each bundle
* small enough that ranged reads span chunk boundaries, and adds each
* bundle's object index; chunked mode splits that index across blobs the
* way a large repository does; eager mode serves whole single-part bundles
* with no index, like a record from before the index existed.
* @param {'eager'|'lazy'|'chunked'} mode
* @param {Uint8Array[]} bundleFiles
* @param {string} head
* @returns {Promise<{record: Record<string, unknown>, blobs: Map<string, Uint8Array>}>}
*/
async function chainFixture(mode, bundleFiles, head) {
/** @type {Map<string, Uint8Array>} */
const blobs = new Map();
/** @type {Array<Record<string, unknown>>} */
const entries = [];
/** @type {Map<string, import('../src/objects.js').GitObject>} */
const prior = new Map();
/** The widest index in the chain, which the chunked mode asserts on. */
let chunkedIndexes = 0;
for (let i = 0; i < bundleFiles.length; i++) {
const bytes = bundleFiles[i];
if (mode === 'eager') {
const cid = `bafkreibundle${i}`;
blobs.set(cid, bytes);
entries.push({ heads: [head], parts: [blobRef(bytes, cid)] });
} else {
const parts = splitIntoChunks(bytes, 1024).map((chunk, c) => {
const cid = `bafkreibundle${i}chunk${c}`;
blobs.set(cid, chunk);
return blobRef(chunk, cid);
});
const encoded = encodePackIndex(
await buildBundleIndex(bytes, async (sha) => prior.get(sha) ?? null),
);
if (mode === 'chunked') {
// 64 bytes is not a multiple of the 33-byte row, so entries straddle
// the blob boundaries a reader has to join.
const indexParts = splitIntoChunks(encoded, 64).map((chunk, c) => {
const cid = `bafkreiindex${i}chunk${c}`;
blobs.set(cid, chunk);
return blobRef(chunk, cid);
});
chunkedIndexes = Math.max(chunkedIndexes, indexParts.length);
entries.push({ heads: [head], parts, indexParts });
} else {
const cid = `bafkreiindex${i}`;
blobs.set(cid, encoded);
entries.push({
heads: [head],
parts,
index: blobRef(encoded, cid),
});
}
}
// Later bundles may delta against this one's objects.
for (const [sha, object] of await indexPacks(
[parseBundle(bytes).pack],
undefined,
async (sha) => prior.get(sha) ?? null,
)) {
prior.set(sha, object);
}
}
if (mode === 'chunked') expect(chunkedIndexes).toBeGreaterThan(1);
const record = {
name: 'testrepo',
defaultBranch: 'main',
refs: [{ name: 'refs/heads/main', sha: head }],
bundles: entries,
createdAt: '2026-08-01T00:00:00.000Z',
};
return { record, blobs };
}
describe.each(/** @type {const} */ (['eager', 'lazy', 'chunked']))(
'git browser (%s)',
(mode) => {
const DID = 'did:plc:browsertest';
/** @type {Record<string, unknown>} */
let record = {};
/** @type {Map<string, Uint8Array>} */
let blobs = new Map();
beforeAll(async () => {
const fixture = await chainFixture(
mode,
[
new Uint8Array(readFileSync(join(dir, 'b1.bundle'))),
new Uint8Array(readFileSync(join(dir, 'b2.bundle'))),
],
headSha,
);
record = fixture.record;
blobs = fixture.blobs;
});
function fakeXrpc() {
return fakePds(record, blobs, { bytes: 0 });
}
it('keeps the repositories it opened, the oldest out first', async () => {
// Three repositories over one chain, told apart by name. An open
// reads whole blobs (the chain, or the index blobs); a later read of
// an open repository ranges into them or hits the store.
let opens = 0;
const stats = { bytes: 0 };
const serve = fakePds(record, blobs, stats);
const serveOther = fakePds({ ...record, name: 'other' }, blobs, stats);
const serveThird = fakePds({ ...record, name: 'third' }, blobs, stats);
const xrpc = (/** @type {Request} */ request) => {
const url = new URL(request.url);
if (url.pathname.endsWith('getBlob') && !request.headers.get('range')) {
opens++;
}
const rkey = url.searchParams.get('rkey');
if (rkey === 'other') return serveOther(request);
if (rkey === 'third') return serveThird(request);
return serve(request);
};
const browser = createGitBrowser({ xrpc, cacheSize: 2 });
await browser.listTree(DID, 'testrepo');
await browser.listTree(DID, 'other');
const held = opens;
await browser.listTree(DID, 'testrepo');
await browser.listTree(DID, 'other');
expect(opens).toBe(held);
// The third opens in the place of the oldest untouched, testrepo.
await browser.listTree(DID, 'third');
const three = opens;
expect(three).toBeGreaterThan(held);
await browser.listTree(DID, 'other');
expect(opens).toBe(three);
await browser.listTree(DID, 'testrepo');
expect(opens).toBeGreaterThan(three);
});
it('reads at a commit id the way it reads at a ref', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const listing = await browser.listTree(DID, 'testrepo', headSha);
expect(listing?.ref).toBe(headSha);
expect(listing?.entries.map((e) => e.name)).toContain('README.md');
await expect(
browser.listTree(DID, 'testrepo', 'f'.repeat(40)),
).rejects.toThrow(/not found in packs/);
});
it('refuses to download a chain past the eager limit', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc(), maxEagerBytes: 1 });
if (mode === 'eager') {
await expect(browser.listTree(DID, 'testrepo')).rejects.toMatchObject({
code: 'RepositoryTooLarge',
});
} else {
// Indexed bundles read ranges and are never downloaded whole.
const listing = await browser.listTree(DID, 'testrepo');
expect(listing?.entries.length).toBeGreaterThan(0);
}
});
it('lists the root with directories first and file sizes', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const listing = await browser.listTree(DID, 'testrepo');
expect(listing?.ref).toBe('refs/heads/main');
expect(listing?.entries.map((e) => e.name)).toEqual([
'src',
'binary.bin',
'logo.png',
'README.md',
]);
const readme = listing?.entries.find((e) => e.name === 'README.md');
expect(readme?.size).toBe('# browser test\n'.length);
});
it('lists a nested directory', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const listing = await browser.listTree(DID, 'testrepo', 'main', 'src');
expect(listing?.entries.map((e) => e.name)).toEqual(['deep', 'index.js']);
});
it('lists every file at a ref with its size', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const listing = await browser.listFiles(DID, 'testrepo');
expect(listing?.truncated).toBe(false);
expect(listing?.files.map((f) => f.path).sort()).toEqual([
'README.md',
'binary.bin',
'logo.png',
'src/deep/nested.txt',
'src/index.js',
]);
const readme = listing?.files.find((f) => f.path === 'README.md');
expect(readme?.size).toBe('# browser test\n'.length);
});
it('reads a text file, including delta-updated content', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const file = await browser.readFile(
DID,
'testrepo',
'main',
'src/index.js',
);
expect(file?.binary).toBe(false);
expect(file?.text).toBe('export const one = 1;\nexport const two = 2;\n');
});
it('flags binary files without inlining them', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const file = await browser.readFile(
DID,
'testrepo',
'main',
'binary.bin',
);
expect(file?.binary).toBe(true);
expect(file?.text).toBeNull();
expect(file?.size).toBe(512);
});
it('names the media type of an image instead of its content', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const file = await browser.readFile(DID, 'testrepo', 'main', 'logo.png');
expect(file?.mediaType).toBe('image/png');
expect(file?.binary).toBe(true);
expect(file?.text).toBeNull();
});
it('reads an image as bytes', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const image = await browser.readImage(
DID,
'testrepo',
'main',
'logo.png',
);
expect(image?.mediaType).toBe('image/png');
expect(image?.size).toBe(64);
expect(Array.from(image?.bytes.subarray(0, 8) ?? [])).toEqual(PNG_MAGIC);
});
it('reads an image only for an image extension', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
const file = await browser.readFile(DID, 'testrepo', 'main', 'README.md');
expect(file?.mediaType).toBeNull();
await expect(
browser.readImage(DID, 'testrepo', 'main', 'README.md'),
).rejects.toThrow(/not an image/);
});
it('rejects unknown refs and paths', async () => {
const browser = createGitBrowser({ xrpc: fakeXrpc() });
await expect(browser.listTree(DID, 'testrepo', 'nope')).rejects.toThrow(
/no such ref/,
);
await expect(
browser.readFile(DID, 'testrepo', 'main', 'no/such/file'),
).rejects.toThrow(/no such path/);
});
it('answers an empty repository as holding no commits', async () => {
const empty = { name: 'empty', refs: [], bundles: [], createdAt: '' };
const browser = createGitBrowser({
xrpc: fakePds(empty, new Map(), { bytes: 0 }),
});
expect(await browser.listCommits(DID, 'empty', 'main')).toEqual({
ref: 'main',
commits: [],
truncated: false,
});
expect(await browser.listRange(DID, 'empty', '', 'main')).toEqual({
commits: [],
truncated: false,
});
expect(await browser.lastCommit(DID, 'empty', 'main')).toBeNull();
});
},
);
/**
* Render hunks the way `git diff -U3` prints them, so the two can be compared
* as text. git omits the count when a side spans one line.
* @param {Array<{oldStart: number, oldLines: number, newStart: number, newLines: number, lines: Array<{type: string, text: string}>}>} hunks
*/
function formatHunks(hunks) {
const out = [];
for (const hunk of hunks) {
const before =
hunk.oldLines === 1
? `${hunk.oldStart}`
: `${hunk.oldStart},${hunk.oldLines}`;
const after =
hunk.newLines === 1
? `${hunk.newStart}`
: `${hunk.newStart},${hunk.newLines}`;
out.push(`@@ -${before} +${after} @@`);
for (const line of hunk.lines) {
const sign =
line.type === 'add' ? '+' : line.type === 'delete' ? '-' : ' ';
out.push(sign + line.text);
}
}
return out.join('\n');
}
/**
* The hunk region of a `git show` for one path, without the file headers.
* git appends the enclosing section to a hunk header, chosen by a
* language-aware heuristic. The browser prints the range alone, so the
* comparison drops that suffix.
* @param {string} output
*/
function hunksFromGit(output) {
const start = output.indexOf('@@');
if (start === -1) return '';
let body = output.slice(start);
// Drop the trailing newline only. Trimming whitespace would eat the space
// that marks a blank context line at the end of a hunk.
if (body.endsWith('\n')) body = body.slice(0, -1);
return body
.split('\n')
.map((line) =>
line.startsWith('@@') ? line.replace(/^(@@ [^@]* @@).*$/, '$1') : line,
)
.join('\n');
}
/**
* Rebuild the new text from the old text plus the hunks, the way `git apply`
* would. A diff that cannot do this is wrong however it is grouped.
* @param {string} beforeText
* @param {Array<{oldStart: number, oldLines: number, lines: Array<{type: string, text: string}>}>} hunks
*/
function applyHunks(beforeText, hunks) {
const before =
beforeText === '' ? [] : beforeText.replace(/\n$/, '').split('\n');
const out = [];
let cursor = 0;
for (const hunk of hunks) {
const start = hunk.oldLines === 0 ? hunk.oldStart : hunk.oldStart - 1;
while (cursor < start) out.push(before[cursor++]);
for (const line of hunk.lines) {
if (line.type === 'add') out.push(line.text);
else if (line.type === 'context') out.push(before[cursor++]);
else cursor++;
}
}
while (cursor < before.length) out.push(before[cursor++]);
return out.join('\n');
}
describe.each(/** @type {const} */ (['eager', 'lazy', 'chunked']))(
'commit diffs (%s)',
(mode) => {
const DID = 'did:plc:difftest';
/** @type {string} */
let diffDir = '';
/** @type {string} */
let diffRepo = '';
/** @type {string[]} */
let shas = [];
/** @type {Record<string, unknown>} */
let diffRecord = {};
/** @type {Map<string, Uint8Array>} */
let diffBlobs = new Map();
beforeAll(async () => {
diffDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-diff-'));
diffRepo = join(diffDir, 'repo');
await runGit(diffDir, ['init', '-b', 'main', diffRepo]);
mkdirSync(join(diffRepo, 'dir'));
// Twenty lines, so edits far apart produce separate hunks and edits close
// together are merged into one.
const base = Array.from({ length: 20 }, (_, i) => `line ${i + 1}`).join(
'\n',
);
writeFileSync(join(diffRepo, 'a.txt'), `${base}\n`);
writeFileSync(join(diffRepo, 'dir', 'b.txt'), 'to be deleted\n');
writeFileSync(
join(diffRepo, 'bin.dat'),
Buffer.concat([Buffer.from([0, 1, 2]), randomBytes(64)]),
);
await runGit(diffRepo, ['add', '.']);
await runGit(diffRepo, ['commit', '-m', 'first']);
// One edit near the top, one near the bottom, two adjacent in the middle.
const edited = base.split('\n');
edited[1] = 'line 2 changed';
edited[9] = 'line 10 changed';
edited.splice(10, 0, 'inserted after line 10');
edited[18] = 'line 18 changed';
writeFileSync(join(diffRepo, 'a.txt'), `${edited.join('\n')}\n`);
rmSync(join(diffRepo, 'dir', 'b.txt'));
writeFileSync(join(diffRepo, 'c.txt'), 'brand new\nsecond line\n');
writeFileSync(
join(diffRepo, 'doc.js'),
[
'/**',
' * The first helper.',
' */',
'export function first() {',
' return 1;',
'}',
'',
'/**',
' * The last helper.',
' */',
'export function last() {',
' return 2;',
'}',
'',
].join('\n'),
);
writeFileSync(
join(diffRepo, 'bin.dat'),
Buffer.concat([Buffer.from([0, 9, 9]), randomBytes(64)]),
);
writeFileSync(join(diffRepo, 'pic.png'), fakePng(48));
await runGit(diffRepo, ['add', '-A']);
await runGit(diffRepo, ['commit', '-m', 'second']);
// Insert a whole doc-commented function between two others. The inserted
// block starts with the same lines it is surrounded by, so the diff can
// place it at several positions for the same cost.
const doc = readFileSync(join(diffRepo, 'doc.js'), 'utf8').split('\n');
doc.splice(
7,
0,
'/**',
' * The middle helper.',
' */',
'export function middle() {',
' return 3;',
'}',
'',
);
writeFileSync(join(diffRepo, 'doc.js'), doc.join('\n'));
writeFileSync(join(diffRepo, 'pic.png'), fakePng(96));
await runGit(diffRepo, ['add', '-A']);
await runGit(diffRepo, ['commit', '-m', 'third']);
await runGit(diffRepo, [
'bundle',
'create',
join(diffDir, 'all.bundle'),
'--all',
]);
shas = (await runGit(diffRepo, ['rev-list', '--reverse', 'main'])).split(
'\n',
);
const fixture = await chainFixture(
mode,
[new Uint8Array(readFileSync(join(diffDir, 'all.bundle')))],
shas[shas.length - 1],
);
fixture.record.name = 'diffrepo';
diffRecord = fixture.record;
diffBlobs = fixture.blobs;
});
afterAll(() => {
rmSync(diffDir, { recursive: true, force: true });
});
/** A fake PDS serving this fixture's bundle chain. */
function diffXrpc() {
return fakePds(diffRecord, diffBlobs, { bytes: 0 });
}
it('reports the same statuses as git for each changed path', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
const fromGit = (
await runGit(diffRepo, ['show', '--name-status', '--format=', shas[1]])
)
.split('\n')
.map((line) => line.split('\t'))
.sort((a, b) => a[1].localeCompare(b[1]))
.map(([status, path]) => `${status} ${path}`);
expect(
commit?.files.map((f) => `${f.status[0].toUpperCase()} ${f.path}`),
).toEqual(fromGit);
});
it('produces the hunks git produces for a modified file', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
const file = commit?.files.find((f) => f.path === 'a.txt');
const expected = hunksFromGit(
await runGit(diffRepo, [
'show',
'-U3',
'--format=',
shas[1],
'--',
'a.txt',
]),
);
expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected);
});
it('produces the hunks git produces for an added file', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
const file = commit?.files.find((f) => f.path === 'c.txt');
const expected = hunksFromGit(
await runGit(diffRepo, [
'show',
'-U3',
'--format=',
shas[1],
'--',
'c.txt',
]),
);
expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected);
expect(file?.additions).toBe(2);
expect(file?.deletions).toBe(0);
});
it('produces the hunks git produces for a deleted file', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
const file = commit?.files.find((f) => f.path === 'dir/b.txt');
const expected = hunksFromGit(
await runGit(diffRepo, [
'show',
'-U3',
'--format=',
shas[1],
'--',
'dir/b.txt',
]),
);
expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected);
expect(file?.status).toBe('deleted');
});
it('reports a binary change without hunks', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
const file = commit?.files.find((f) => f.path === 'bin.dat');
expect(file?.binary).toBe(true);
expect(file?.hunks).toBeNull();
});
it('carries the bytes of a changed image on each side', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const second = await browser.readCommit(DID, 'diffrepo', shas[1]);
const added = second?.files.find((f) => f.path === 'pic.png');
expect(added?.mediaType).toBe('image/png');
expect(added?.hunks).toBeNull();
expect(added?.oldImage).toBeNull();
expect(Array.from(added?.newImage?.subarray(0, 8) ?? [])).toEqual(
PNG_MAGIC,
);
const third = await browser.readCommit(DID, 'diffrepo', shas[2]);
const changed = third?.files.find((f) => f.path === 'pic.png');
expect(changed?.oldImage).toEqual(added?.newImage);
expect(changed?.newImage?.length).toBe(fakePng(96).length);
const binary = second?.files.find((f) => f.path === 'bin.dat');
expect(binary?.mediaType).toBeNull();
expect(binary?.newImage).toBeNull();
});
it('carries both sides of a text file whole', async () => {
// The page colours a whole side, not a hunk, so each side rides along
// with the change.
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
const file = commit?.files.find((f) => f.path === 'a.txt');
// runGit trims what git prints, so the texts are compared without
// their last newline.
expect(file?.oldText?.trimEnd()).toBe(
await runGit(diffRepo, ['show', `${shas[0]}:a.txt`]),
);
expect(file?.newText?.trimEnd()).toBe(
await runGit(diffRepo, ['show', `${shas[1]}:a.txt`]),
);
const binary = commit?.files.find((f) => f.path === 'bin.dat');
expect(binary?.oldText).toBeNull();
expect(binary?.newText).toBeNull();
});
it('treats a root commit as every file added', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[0]);
expect(commit?.parents).toEqual([]);
expect(commit?.files.map((f) => f.path)).toEqual([
'a.txt',
'bin.dat',
'dir/b.txt',
]);
expect(commit?.files.every((f) => f.status === 'added')).toBe(true);
});
it('carries the commit metadata alongside the files', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[1]);
expect(commit?.sha).toBe(shas[1]);
expect(commit?.parents).toEqual([shas[0]]);
expect(commit?.message.trim()).toBe('second');
expect(commit?.author).toBe('Test <test@example.com>');
});
it('places an ambiguous insertion where git places it', async () => {
// The inserted block opens with the same lines that already surround it,
// so Myers alone can anchor it mid-comment. Compaction slides it onto the
// comment's first line, which is where git shows it.
const browser = createGitBrowser({ xrpc: diffXrpc() });
const commit = await browser.readCommit(DID, 'diffrepo', shas[2]);
const file = commit?.files.find((f) => f.path === 'doc.js');
const expected = hunksFromGit(
await runGit(diffRepo, [
'show',
'-U3',
'--format=',
shas[2],
'--',
'doc.js',
]),
);
expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected);
expect(file?.hunks?.[0].lines.find((l) => l.type === 'add')?.text).toBe(
'/**',
);
});
it('keeps every changed line when a slide merges two groups', () => {
// One line becomes a repeated pair. The inserted lines slide onto the
// single unchanged line between them, joining two groups into one. A
// slide that does not take the joined lines with it drops one, and the
// diff no longer rebuilds its own result.
const hunks = diffLines('b\n', 'a\nb\na\nb\n');
expect(applyHunks('b\n', /** @type {any} */ (hunks))).toBe('a\nb\na\nb');
});
it('produces diffs that rebuild the files they describe', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
for (const sha of shas) {
const commit = await browser.readCommit(DID, 'diffrepo', sha);
for (const file of commit?.files ?? []) {
if (!file.hunks) continue;
const before =
file.status === 'added'
? ''
: await runGit(diffRepo, ['show', `${sha}^:${file.path}`]);
const after =
file.status === 'deleted'
? ''
: await runGit(diffRepo, ['show', `${sha}:${file.path}`]);
expect(applyHunks(before, file.hunks), `${sha} ${file.path}`).toBe(
after.replace(/\n$/, ''),
);
}
}
});
it('rejects an id that is not a commit', async () => {
const browser = createGitBrowser({ xrpc: diffXrpc() });
await expect(
browser.readCommit(DID, 'diffrepo', 'f'.repeat(40)),
).rejects.toThrow(/no such commit/);
});
},
);
describe('lazy reader efficiency', () => {
const DID = 'did:plc:lazytest';
const SPACE = 'at://did:plc:lazytest/com.atproto.simplespace.space/team';
/** @type {string} */
let bigDir = '';
/** @type {string} */
let bigRepo = '';
/** @type {string} */
let head = '';
/** @type {Uint8Array} */
let bundleBytes = new Uint8Array();
/** @type {Record<string, unknown>} */
let record = {};
/** @type {Map<string, Uint8Array>} */
let blobs = new Map();
beforeAll(async () => {
bigDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-lazy-'));
bigRepo = join(bigDir, 'repo');
await runGit(bigDir, ['init', '-b', 'main', bigRepo]);
writeFileSync(join(bigRepo, 'README.md'), '# lazy test\n');
// Incompressible bulk, so the chain's size dwarfs what a tree page
// needs.
writeFileSync(join(bigRepo, 'big.bin'), randomBytes(300 * 1024));
await runGit(bigRepo, ['add', '.']);
await runGit(bigRepo, ['commit', '-m', 'one']);
head = (await runGit(bigRepo, ['rev-parse', 'HEAD'])).trim();
await runGit(bigRepo, [
'bundle',
'create',
join(bigDir, 'all.bundle'),
'main',
]);
bundleBytes = new Uint8Array(readFileSync(join(bigDir, 'all.bundle')));
const fixture = await chainFixture('lazy', [bundleBytes], head);
record = fixture.record;
blobs = fixture.blobs;
});
afterAll(() => {
rmSync(bigDir, { recursive: true, force: true });
});
it('serves a tree listing and a small file for a fraction of the chain', async () => {
const stats = { bytes: 0 };
const browser = createGitBrowser({ xrpc: fakePds(record, blobs, stats) });
const listing = await browser.listTree(DID, 'testrepo');
expect(listing?.entries.map((e) => e.name)).toEqual([
'big.bin',
'README.md',
]);
// Sizes come from the index, with no blob bytes behind them.
expect(listing?.entries.find((e) => e.name === 'big.bin')?.size).toBe(
300 * 1024,
);
const file = await browser.readFile(DID, 'testrepo', 'main', 'README.md');
expect(file?.text).toBe('# lazy test\n');
expect(stats.bytes).toBeLessThan(bundleBytes.length / 4);
});
it('answers the same through a server that ignores Range', async () => {
const stats = { bytes: 0 };
const browser = createGitBrowser({
xrpc: fakePds(record, blobs, stats, { ranges: false }),
});
const file = await browser.readFile(DID, 'testrepo', 'main', 'README.md');
expect(file?.text).toBe('# lazy test\n');
// Whole chunks instead of ranges, still nowhere near the whole chain.
expect(stats.bytes).toBeLessThan(bundleBytes.length / 4);
});
/**
* The account page reads a space repository straight from storage, so
* these stand in for the platform's own blob access.
* @param {{bytes: number}} stats
* @param {{ranges?: boolean}} [opts] - ranges: false is a reader that
* answers whole blobs only
* @returns {import('../src/browser.js').SpaceReader}
*/
function fakeSpaceReader(stats, opts = {}) {
return {
getRecord: async () => ({ cid: 'bafyrecordcid', value: record }),
getBlob: async (cid) => {
const blob = blobs.get(cid) ?? null;
if (blob) stats.bytes += blob.length;
return blob;
},
getBlobRange:
opts.ranges === false
? undefined
: async (cid, start, end) => {
const blob = blobs.get(cid);
if (!blob) return null;
const slice = blob.subarray(start, Math.min(end, blob.length));
stats.bytes += slice.length;
return slice;
},
};
}
it('reads a space repository for a fraction of the chain', async () => {
const stats = { bytes: 0 };
const browser = createGitBrowser({
xrpc: fakePds(record, blobs, { bytes: 0 }),
spaceReader: fakeSpaceReader(stats),
});
const file = await browser.readFile(
DID,
'testrepo',
'main',
'README.md',
SPACE,
);
expect(file?.text).toBe('# lazy test\n');
expect(stats.bytes).toBeLessThan(bundleBytes.length / 4);
});
it('reads a space repository whole when its reader has no ranges', async () => {
const stats = { bytes: 0 };
const browser = createGitBrowser({
xrpc: fakePds(record, blobs, { bytes: 0 }),
spaceReader: fakeSpaceReader(stats, { ranges: false }),
});
const file = await browser.readFile(
DID,
'testrepo',
'main',
'README.md',
SPACE,
);
expect(file?.text).toBe('# lazy test\n');
expect(stats.bytes).toBeGreaterThanOrEqual(bundleBytes.length);
});
it('still reads the large file whole when asked', async () => {
const browser = createGitBrowser({
xrpc: fakePds(record, blobs, { bytes: 0 }),
});
const bytes = await browser.readBytes(DID, 'testrepo', 'main', 'big.bin');
expect(bytes?.size).toBe(300 * 1024);
expect(bytes?.binary).toBe(true);
});
});
describe('the commit that last changed a path', () => {
const DID = 'did:plc:lastcommit';
/** @type {string} */
let workDir = '';
/** @type {string} */
let workRepo = '';
/** @type {{record: Record<string, unknown>, blobs: Map<string, Uint8Array>}} */
let fixture = { record: {}, blobs: new Map() };
beforeAll(async () => {
workDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-lastcommit-'));
workRepo = join(workDir, 'repo');
await runGit(workDir, ['init', '-b', 'main', workRepo]);
mkdirSync(join(workRepo, 'dir'));
writeFileSync(join(workRepo, 'a.txt'), 'a one\n');
writeFileSync(join(workRepo, 'b.txt'), 'b one\n');
writeFileSync(join(workRepo, 'dir', 'c.txt'), 'c one\n');
await runGit(workRepo, ['add', '.']);
await runGit(workRepo, ['commit', '-m', 'base']);
// A branch changes one file while the trunk changes another, so the
// merge that follows differs from each parent in a different path.
await runGit(workRepo, ['checkout', '-b', 'topic']);
writeFileSync(join(workRepo, 'b.txt'), 'b from topic\n');
await runGit(workRepo, ['commit', '-am', 'change b on topic']);
await runGit(workRepo, ['checkout', 'main']);
writeFileSync(join(workRepo, 'a.txt'), 'a from main\n');
await runGit(workRepo, ['commit', '-am', 'change a on main']);
await runGit(workRepo, [
'merge',
'--no-ff',
'topic',
'-m',
'merge topic into main',
]);
const head = await runGit(workRepo, ['rev-parse', 'HEAD']);
await runGit(workRepo, [
'bundle',
'create',
join(workDir, 'merge.bundle'),
'main',
]);
fixture = await chainFixture(
'lazy',
[new Uint8Array(readFileSync(join(workDir, 'merge.bundle')))],
head,
);
});
afterAll(() => {
rmSync(workDir, { recursive: true, force: true });
});
const browserOf = () =>
createGitBrowser({
xrpc: fakePds(fixture.record, fixture.blobs, { bytes: 0 }),
});
it('names the commit git names, for every path', async () => {
const browser = browserOf();
for (const path of ['a.txt', 'b.txt', 'dir/c.txt']) {
const wanted = await runGit(workRepo, [
'log',
'-1',
'--format=%H',
'--',
path,
]);
const answer = await browser.lastCommit(DID, 'testrepo', 'main', path);
expect(answer?.sha, path).toBe(wanted);
}
});
it('lists the commits that changed a path, the way git log does', async () => {
const browser = browserOf();
for (const path of ['a.txt', 'b.txt', 'dir/c.txt', 'dir']) {
const wanted = (
await runGit(workRepo, ['log', '--format=%H', '--', path])
)
.split('\n')
.filter(Boolean);
const answer = await browser.listCommits(
DID,
'testrepo',
'main',
50,
undefined,
path,
);
expect(
answer?.commits.map((one) => one.sha),
path,
).toEqual(wanted);
expect(answer?.truncated).toBe(false);
}
const one = await browser.listCommits(
DID,
'testrepo',
'main',
1,
undefined,
'a.txt',
);
expect(one?.commits).toHaveLength(1);
});
it('credits the branch a merge took a change from, not the merge', async () => {
const browser = browserOf();
const merge = await runGit(workRepo, ['rev-parse', 'HEAD']);
const answer = await browser.lastCommit(DID, 'testrepo', 'main', 'b.txt');
expect(answer?.sha).not.toBe(merge);
expect(answer?.message).toContain('change b on topic');
});
it('answers null for a path no commit holds', async () => {
const browser = browserOf();
const answer = await browser.lastCommit(
DID,
'testrepo',
'main',
'gone.txt',
);
expect(answer).toBeNull();
});
});
describe('a path the walk has to go deep to find', () => {
const DID = 'did:plc:deepwalk';
/** The walk starts at 64 commits and doubles, so this crosses one run. */
const CHURN = 150;
/** @type {string} */
let workDir = '';
/** @type {string} */
let workRepo = '';
/** @type {{record: Record<string, unknown>, blobs: Map<string, Uint8Array>}} */
let fixture = { record: {}, blobs: new Map() };
beforeAll(async () => {
workDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-deepwalk-'));
workRepo = join(workDir, 'repo');
await runGit(workDir, ['init', '-b', 'main', workRepo]);
// One shell keeps the churn to a single process; 150 spawns of git do
// not fit the suite's budget.
await execFileAsync(
'sh',
[
'-c',
`echo base > old.txt && mkdir -p dir && echo base > dir/deep.txt &&
git add . && git commit -qm base &&
for i in $(seq 1 ${CHURN}); do
echo $i > churn.txt && git add churn.txt && git commit -qm "churn $i"
done`,
],
{ cwd: workRepo, env },
);
const head = await runGit(workRepo, ['rev-parse', 'HEAD']);
await runGit(workRepo, [
'bundle',
'create',
join(workDir, 'deep.bundle'),
'main',
]);
fixture = await chainFixture(
'lazy',
[new Uint8Array(readFileSync(join(workDir, 'deep.bundle')))],
head,
);
});
afterAll(() => {
rmSync(workDir, { recursive: true, force: true });
});
it('names the commit git names, across run boundaries', async () => {
const browser = createGitBrowser({
xrpc: fakePds(fixture.record, fixture.blobs, { bytes: 0 }),
});
for (const path of ['old.txt', 'dir/deep.txt', 'churn.txt']) {
const wanted = await runGit(workRepo, [
'log',
'-1',
'--format=%H',
'--',
path,
]);
const answer = await browser.lastCommit(DID, 'testrepo', 'main', path);
expect(answer?.sha, path).toBe(wanted);
}
});
it('answers null for a path no commit in the deep history holds', async () => {
const browser = createGitBrowser({
xrpc: fakePds(fixture.record, fixture.blobs, { bytes: 0 }),
});
expect(
await browser.lastCommit(DID, 'testrepo', 'main', 'dir/absent.txt'),
).toBeNull();
});
});