/** * The pack object reader and browser, checked against real git: bundles made * by the git CLI are indexed in JS and the resulting trees, listings, and * file contents must match what git itself reports. */ import { execFile } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { createGitBrowser } from '../src/browser.js'; import { parseBundle } from '../src/bundle.js'; import { diffLines } from '../src/diff.js'; import { commitTree, indexPacks, objectId, parseTree, resolveToTree, } from '../src/objects.js'; import { buildBundleIndex, encodePackIndex } from '../src/pack-index.js'; import { splitIntoChunks } from '../src/record.js'; const execFileAsync = promisify(execFile); const PNG_MAGIC = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; /** * A file that reads as a PNG. The IHDR length prefix carries the NUL bytes * the binary detector looks for, as it does in a real PNG. * @param {number} tail - how many bytes follow the header */ function fakePng(tail) { return Buffer.concat([ Buffer.from(PNG_MAGIC), Buffer.from([0, 0, 0, 13]), Buffer.from('IHDR'), randomBytes(tail), ]); } /** @type {string} */ let dir = ''; /** @type {string} */ let repo = ''; /** @type {string} */ let headSha = ''; /** @type {Uint8Array[]} */ let packs = []; const env = { .../** @type {Record} */ (process.env), GIT_AUTHOR_NAME: 'Test', GIT_AUTHOR_EMAIL: 'test@example.com', GIT_COMMITTER_NAME: 'Test', GIT_COMMITTER_EMAIL: 'test@example.com', }; /** * @param {string} cwd * @param {string[]} args */ async function runGit(cwd, args) { const { stdout } = await execFileAsync('git', args, { cwd, env }); return stdout.trim(); } beforeAll(async () => { dir = mkdtempSync(join(tmpdir(), 'pdsjs-git-browser-')); repo = join(dir, 'repo'); await runGit(dir, ['init', '-b', 'main', repo]); mkdirSync(join(repo, 'src')); mkdirSync(join(repo, 'src', 'deep')); writeFileSync(join(repo, 'README.md'), '# browser test\n'); writeFileSync(join(repo, 'src', 'index.js'), 'export const one = 1;\n'); writeFileSync(join(repo, 'src', 'deep', 'nested.txt'), 'nested content\n'); // Deterministically binary: the detector looks for NUL bytes writeFileSync( join(repo, 'binary.bin'), Buffer.concat([Buffer.from([0, 1, 2, 3]), randomBytes(508)]), ); writeFileSync(join(repo, 'logo.png'), fakePng(48)); await runGit(repo, ['add', '.']); await runGit(repo, ['commit', '-m', 'one']); const first = await runGit(repo, ['rev-parse', 'HEAD']); await runGit(repo, ['bundle', 'create', join(dir, 'b1.bundle'), 'main']); // A second commit REWRITING an existing file makes the incremental bundle // carry deltas whose bases live in the first pack. writeFileSync( join(repo, 'src', 'index.js'), 'export const one = 1;\nexport const two = 2;\n', ); await runGit(repo, ['add', '.']); await runGit(repo, ['commit', '-m', 'two']); await runGit(repo, ['tag', '-a', 'v1', '-m', 'release one']); headSha = await runGit(repo, ['rev-parse', 'HEAD']); await runGit(repo, [ 'bundle', 'create', join(dir, 'b2.bundle'), `^${first}`, 'main', 'v1', ]); packs = [ parseBundle(readFileSync(join(dir, 'b1.bundle'))).pack, parseBundle(readFileSync(join(dir, 'b2.bundle'))).pack, ]; }); afterAll(() => { rmSync(dir, { recursive: true, force: true }); }); describe('pack object reader', () => { it('indexes every object with ids git agrees with', async () => { const objects = await indexPacks(packs); const commit = objects.get(headSha); expect(commit).toBeDefined(); expect(commit?.type).toBe('commit'); // Every object id git can list must be present and correctly typed const listed = ( await runGit(repo, ['rev-list', '--objects', 'main']) ).split('\n'); for (const line of listed) { const sha = line.split(' ')[0]; const object = objects.get(sha); expect(object, sha).toBeDefined(); expect( objectId( /** @type {{type: string}} */ (object).type, /** @type {{data: Uint8Array}} */ (object).data, ), ).toBe(sha); } }); it('walks commit to tree to entries like git ls-tree', async () => { const objects = await indexPacks(packs); const tree = commitTree( /** @type {{data: Uint8Array}} */ (objects.get(headSha)).data, ); const entries = parseTree( /** @type {{data: Uint8Array}} */ (objects.get(tree)).data, ); const expected = (await runGit(repo, ['ls-tree', 'main'])) .split('\n') .map((line) => line.split('\t')[1]); expect(entries.map((e) => e.name)).toEqual(expected); }); it('peels annotated tags to the commit tree', async () => { const objects = await indexPacks(packs); const tagSha = await runGit(repo, ['rev-parse', 'v1']); const treeFromTag = resolveToTree(objects, tagSha); const treeFromCommit = resolveToTree(objects, headSha); expect(treeFromTag).toBe(treeFromCommit); }); }); /** * @param {Uint8Array} bytes * @param {string} cid */ function blobRef(bytes, cid) { return { $type: 'blob', ref: { $link: cid }, mimeType: 'application/octet-stream', size: bytes.length, }; } /** * A fake PDS over a blob map. Range requests answer 206 with the slice, * the way the real getBlob endpoint does, and every response's bytes are * counted so a test can bound what a reader downloaded. * @param {Record} record * @param {Map} blobs * @param {{bytes: number}} stats * @param {{ranges?: boolean}} [opts] - ranges: false ignores Range headers * and answers 200 whole, like a server from before the range support */ function fakePds(record, blobs, stats, opts = {}) { return async (/** @type {Request} */ request) => { const url = new URL(request.url); if (url.pathname.endsWith('getRecord')) { return Response.json({ cid: 'bafyrecordcid', value: record, }); } if (url.pathname.endsWith('getBlob')) { const blob = blobs.get(url.searchParams.get('cid') || ''); if (!blob) return Response.json({ error: 'NotFound' }, { status: 404 }); const match = /^bytes=(\d+)-(\d+)$/.exec( request.headers.get('range') || '', ); if (match && opts.ranges !== false) { const start = Number(match[1]); const end = Math.min(Number(match[2]), blob.length - 1); const body = blob.subarray(start, end + 1); stats.bytes += body.length; return new Response(/** @type {BodyInit} */ (body), { status: 206 }); } stats.bytes += blob.length; return new Response(/** @type {BodyInit} */ (blob)); } return Response.json({ error: 'NotFound' }, { status: 404 }); }; } /** * Record and blob map for a bundle chain. Lazy mode chunks each bundle * small enough that ranged reads span chunk boundaries, and adds each * bundle's object index; chunked mode splits that index across blobs the * way a large repository does; eager mode serves whole single-part bundles * with no index, like a record from before the index existed. * @param {'eager'|'lazy'|'chunked'} mode * @param {Uint8Array[]} bundleFiles * @param {string} head * @returns {Promise<{record: Record, blobs: Map}>} */ async function chainFixture(mode, bundleFiles, head) { /** @type {Map} */ const blobs = new Map(); /** @type {Array>} */ const entries = []; /** @type {Map} */ const prior = new Map(); /** The widest index in the chain, which the chunked mode asserts on. */ let chunkedIndexes = 0; for (let i = 0; i < bundleFiles.length; i++) { const bytes = bundleFiles[i]; if (mode === 'eager') { const cid = `bafkreibundle${i}`; blobs.set(cid, bytes); entries.push({ heads: [head], parts: [blobRef(bytes, cid)] }); } else { const parts = splitIntoChunks(bytes, 1024).map((chunk, c) => { const cid = `bafkreibundle${i}chunk${c}`; blobs.set(cid, chunk); return blobRef(chunk, cid); }); const encoded = encodePackIndex( await buildBundleIndex(bytes, async (sha) => prior.get(sha) ?? null), ); if (mode === 'chunked') { // 64 bytes is not a multiple of the 33-byte row, so entries straddle // the blob boundaries a reader has to join. const indexParts = splitIntoChunks(encoded, 64).map((chunk, c) => { const cid = `bafkreiindex${i}chunk${c}`; blobs.set(cid, chunk); return blobRef(chunk, cid); }); chunkedIndexes = Math.max(chunkedIndexes, indexParts.length); entries.push({ heads: [head], parts, indexParts }); } else { const cid = `bafkreiindex${i}`; blobs.set(cid, encoded); entries.push({ heads: [head], parts, index: blobRef(encoded, cid), }); } } // Later bundles may delta against this one's objects. for (const [sha, object] of await indexPacks( [parseBundle(bytes).pack], undefined, async (sha) => prior.get(sha) ?? null, )) { prior.set(sha, object); } } if (mode === 'chunked') expect(chunkedIndexes).toBeGreaterThan(1); const record = { name: 'testrepo', defaultBranch: 'main', refs: [{ name: 'refs/heads/main', sha: head }], bundles: entries, createdAt: '2026-08-01T00:00:00.000Z', }; return { record, blobs }; } describe.each(/** @type {const} */ (['eager', 'lazy', 'chunked']))( 'git browser (%s)', (mode) => { const DID = 'did:plc:browsertest'; /** @type {Record} */ let record = {}; /** @type {Map} */ let blobs = new Map(); beforeAll(async () => { const fixture = await chainFixture( mode, [ new Uint8Array(readFileSync(join(dir, 'b1.bundle'))), new Uint8Array(readFileSync(join(dir, 'b2.bundle'))), ], headSha, ); record = fixture.record; blobs = fixture.blobs; }); function fakeXrpc() { return fakePds(record, blobs, { bytes: 0 }); } it('keeps the repositories it opened, the oldest out first', async () => { // Three repositories over one chain, told apart by name. An open // reads whole blobs (the chain, or the index blobs); a later read of // an open repository ranges into them or hits the store. let opens = 0; const stats = { bytes: 0 }; const serve = fakePds(record, blobs, stats); const serveOther = fakePds({ ...record, name: 'other' }, blobs, stats); const serveThird = fakePds({ ...record, name: 'third' }, blobs, stats); const xrpc = (/** @type {Request} */ request) => { const url = new URL(request.url); if (url.pathname.endsWith('getBlob') && !request.headers.get('range')) { opens++; } const rkey = url.searchParams.get('rkey'); if (rkey === 'other') return serveOther(request); if (rkey === 'third') return serveThird(request); return serve(request); }; const browser = createGitBrowser({ xrpc, cacheSize: 2 }); await browser.listTree(DID, 'testrepo'); await browser.listTree(DID, 'other'); const held = opens; await browser.listTree(DID, 'testrepo'); await browser.listTree(DID, 'other'); expect(opens).toBe(held); // The third opens in the place of the oldest untouched, testrepo. await browser.listTree(DID, 'third'); const three = opens; expect(three).toBeGreaterThan(held); await browser.listTree(DID, 'other'); expect(opens).toBe(three); await browser.listTree(DID, 'testrepo'); expect(opens).toBeGreaterThan(three); }); it('reads at a commit id the way it reads at a ref', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const listing = await browser.listTree(DID, 'testrepo', headSha); expect(listing?.ref).toBe(headSha); expect(listing?.entries.map((e) => e.name)).toContain('README.md'); await expect( browser.listTree(DID, 'testrepo', 'f'.repeat(40)), ).rejects.toThrow(/not found in packs/); }); it('refuses to download a chain past the eager limit', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc(), maxEagerBytes: 1 }); if (mode === 'eager') { await expect(browser.listTree(DID, 'testrepo')).rejects.toMatchObject({ code: 'RepositoryTooLarge', }); } else { // Indexed bundles read ranges and are never downloaded whole. const listing = await browser.listTree(DID, 'testrepo'); expect(listing?.entries.length).toBeGreaterThan(0); } }); it('lists the root with directories first and file sizes', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const listing = await browser.listTree(DID, 'testrepo'); expect(listing?.ref).toBe('refs/heads/main'); expect(listing?.entries.map((e) => e.name)).toEqual([ 'src', 'binary.bin', 'logo.png', 'README.md', ]); const readme = listing?.entries.find((e) => e.name === 'README.md'); expect(readme?.size).toBe('# browser test\n'.length); }); it('lists a nested directory', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const listing = await browser.listTree(DID, 'testrepo', 'main', 'src'); expect(listing?.entries.map((e) => e.name)).toEqual(['deep', 'index.js']); }); it('lists every file at a ref with its size', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const listing = await browser.listFiles(DID, 'testrepo'); expect(listing?.truncated).toBe(false); expect(listing?.files.map((f) => f.path).sort()).toEqual([ 'README.md', 'binary.bin', 'logo.png', 'src/deep/nested.txt', 'src/index.js', ]); const readme = listing?.files.find((f) => f.path === 'README.md'); expect(readme?.size).toBe('# browser test\n'.length); }); it('reads a text file, including delta-updated content', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const file = await browser.readFile( DID, 'testrepo', 'main', 'src/index.js', ); expect(file?.binary).toBe(false); expect(file?.text).toBe('export const one = 1;\nexport const two = 2;\n'); }); it('flags binary files without inlining them', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const file = await browser.readFile( DID, 'testrepo', 'main', 'binary.bin', ); expect(file?.binary).toBe(true); expect(file?.text).toBeNull(); expect(file?.size).toBe(512); }); it('names the media type of an image instead of its content', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const file = await browser.readFile(DID, 'testrepo', 'main', 'logo.png'); expect(file?.mediaType).toBe('image/png'); expect(file?.binary).toBe(true); expect(file?.text).toBeNull(); }); it('reads an image as bytes', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const image = await browser.readImage( DID, 'testrepo', 'main', 'logo.png', ); expect(image?.mediaType).toBe('image/png'); expect(image?.size).toBe(64); expect(Array.from(image?.bytes.subarray(0, 8) ?? [])).toEqual(PNG_MAGIC); }); it('reads an image only for an image extension', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); const file = await browser.readFile(DID, 'testrepo', 'main', 'README.md'); expect(file?.mediaType).toBeNull(); await expect( browser.readImage(DID, 'testrepo', 'main', 'README.md'), ).rejects.toThrow(/not an image/); }); it('rejects unknown refs and paths', async () => { const browser = createGitBrowser({ xrpc: fakeXrpc() }); await expect(browser.listTree(DID, 'testrepo', 'nope')).rejects.toThrow( /no such ref/, ); await expect( browser.readFile(DID, 'testrepo', 'main', 'no/such/file'), ).rejects.toThrow(/no such path/); }); it('answers an empty repository as holding no commits', async () => { const empty = { name: 'empty', refs: [], bundles: [], createdAt: '' }; const browser = createGitBrowser({ xrpc: fakePds(empty, new Map(), { bytes: 0 }), }); expect(await browser.listCommits(DID, 'empty', 'main')).toEqual({ ref: 'main', commits: [], truncated: false, }); expect(await browser.listRange(DID, 'empty', '', 'main')).toEqual({ commits: [], truncated: false, }); expect(await browser.lastCommit(DID, 'empty', 'main')).toBeNull(); }); }, ); /** * Render hunks the way `git diff -U3` prints them, so the two can be compared * as text. git omits the count when a side spans one line. * @param {Array<{oldStart: number, oldLines: number, newStart: number, newLines: number, lines: Array<{type: string, text: string}>}>} hunks */ function formatHunks(hunks) { const out = []; for (const hunk of hunks) { const before = hunk.oldLines === 1 ? `${hunk.oldStart}` : `${hunk.oldStart},${hunk.oldLines}`; const after = hunk.newLines === 1 ? `${hunk.newStart}` : `${hunk.newStart},${hunk.newLines}`; out.push(`@@ -${before} +${after} @@`); for (const line of hunk.lines) { const sign = line.type === 'add' ? '+' : line.type === 'delete' ? '-' : ' '; out.push(sign + line.text); } } return out.join('\n'); } /** * The hunk region of a `git show` for one path, without the file headers. * git appends the enclosing section to a hunk header, chosen by a * language-aware heuristic. The browser prints the range alone, so the * comparison drops that suffix. * @param {string} output */ function hunksFromGit(output) { const start = output.indexOf('@@'); if (start === -1) return ''; let body = output.slice(start); // Drop the trailing newline only. Trimming whitespace would eat the space // that marks a blank context line at the end of a hunk. if (body.endsWith('\n')) body = body.slice(0, -1); return body .split('\n') .map((line) => line.startsWith('@@') ? line.replace(/^(@@ [^@]* @@).*$/, '$1') : line, ) .join('\n'); } /** * Rebuild the new text from the old text plus the hunks, the way `git apply` * would. A diff that cannot do this is wrong however it is grouped. * @param {string} beforeText * @param {Array<{oldStart: number, oldLines: number, lines: Array<{type: string, text: string}>}>} hunks */ function applyHunks(beforeText, hunks) { const before = beforeText === '' ? [] : beforeText.replace(/\n$/, '').split('\n'); const out = []; let cursor = 0; for (const hunk of hunks) { const start = hunk.oldLines === 0 ? hunk.oldStart : hunk.oldStart - 1; while (cursor < start) out.push(before[cursor++]); for (const line of hunk.lines) { if (line.type === 'add') out.push(line.text); else if (line.type === 'context') out.push(before[cursor++]); else cursor++; } } while (cursor < before.length) out.push(before[cursor++]); return out.join('\n'); } describe.each(/** @type {const} */ (['eager', 'lazy', 'chunked']))( 'commit diffs (%s)', (mode) => { const DID = 'did:plc:difftest'; /** @type {string} */ let diffDir = ''; /** @type {string} */ let diffRepo = ''; /** @type {string[]} */ let shas = []; /** @type {Record} */ let diffRecord = {}; /** @type {Map} */ let diffBlobs = new Map(); beforeAll(async () => { diffDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-diff-')); diffRepo = join(diffDir, 'repo'); await runGit(diffDir, ['init', '-b', 'main', diffRepo]); mkdirSync(join(diffRepo, 'dir')); // Twenty lines, so edits far apart produce separate hunks and edits close // together are merged into one. const base = Array.from({ length: 20 }, (_, i) => `line ${i + 1}`).join( '\n', ); writeFileSync(join(diffRepo, 'a.txt'), `${base}\n`); writeFileSync(join(diffRepo, 'dir', 'b.txt'), 'to be deleted\n'); writeFileSync( join(diffRepo, 'bin.dat'), Buffer.concat([Buffer.from([0, 1, 2]), randomBytes(64)]), ); await runGit(diffRepo, ['add', '.']); await runGit(diffRepo, ['commit', '-m', 'first']); // One edit near the top, one near the bottom, two adjacent in the middle. const edited = base.split('\n'); edited[1] = 'line 2 changed'; edited[9] = 'line 10 changed'; edited.splice(10, 0, 'inserted after line 10'); edited[18] = 'line 18 changed'; writeFileSync(join(diffRepo, 'a.txt'), `${edited.join('\n')}\n`); rmSync(join(diffRepo, 'dir', 'b.txt')); writeFileSync(join(diffRepo, 'c.txt'), 'brand new\nsecond line\n'); writeFileSync( join(diffRepo, 'doc.js'), [ '/**', ' * The first helper.', ' */', 'export function first() {', ' return 1;', '}', '', '/**', ' * The last helper.', ' */', 'export function last() {', ' return 2;', '}', '', ].join('\n'), ); writeFileSync( join(diffRepo, 'bin.dat'), Buffer.concat([Buffer.from([0, 9, 9]), randomBytes(64)]), ); writeFileSync(join(diffRepo, 'pic.png'), fakePng(48)); await runGit(diffRepo, ['add', '-A']); await runGit(diffRepo, ['commit', '-m', 'second']); // Insert a whole doc-commented function between two others. The inserted // block starts with the same lines it is surrounded by, so the diff can // place it at several positions for the same cost. const doc = readFileSync(join(diffRepo, 'doc.js'), 'utf8').split('\n'); doc.splice( 7, 0, '/**', ' * The middle helper.', ' */', 'export function middle() {', ' return 3;', '}', '', ); writeFileSync(join(diffRepo, 'doc.js'), doc.join('\n')); writeFileSync(join(diffRepo, 'pic.png'), fakePng(96)); await runGit(diffRepo, ['add', '-A']); await runGit(diffRepo, ['commit', '-m', 'third']); await runGit(diffRepo, [ 'bundle', 'create', join(diffDir, 'all.bundle'), '--all', ]); shas = (await runGit(diffRepo, ['rev-list', '--reverse', 'main'])).split( '\n', ); const fixture = await chainFixture( mode, [new Uint8Array(readFileSync(join(diffDir, 'all.bundle')))], shas[shas.length - 1], ); fixture.record.name = 'diffrepo'; diffRecord = fixture.record; diffBlobs = fixture.blobs; }); afterAll(() => { rmSync(diffDir, { recursive: true, force: true }); }); /** A fake PDS serving this fixture's bundle chain. */ function diffXrpc() { return fakePds(diffRecord, diffBlobs, { bytes: 0 }); } it('reports the same statuses as git for each changed path', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); const fromGit = ( await runGit(diffRepo, ['show', '--name-status', '--format=', shas[1]]) ) .split('\n') .map((line) => line.split('\t')) .sort((a, b) => a[1].localeCompare(b[1])) .map(([status, path]) => `${status} ${path}`); expect( commit?.files.map((f) => `${f.status[0].toUpperCase()} ${f.path}`), ).toEqual(fromGit); }); it('produces the hunks git produces for a modified file', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); const file = commit?.files.find((f) => f.path === 'a.txt'); const expected = hunksFromGit( await runGit(diffRepo, [ 'show', '-U3', '--format=', shas[1], '--', 'a.txt', ]), ); expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected); }); it('produces the hunks git produces for an added file', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); const file = commit?.files.find((f) => f.path === 'c.txt'); const expected = hunksFromGit( await runGit(diffRepo, [ 'show', '-U3', '--format=', shas[1], '--', 'c.txt', ]), ); expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected); expect(file?.additions).toBe(2); expect(file?.deletions).toBe(0); }); it('produces the hunks git produces for a deleted file', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); const file = commit?.files.find((f) => f.path === 'dir/b.txt'); const expected = hunksFromGit( await runGit(diffRepo, [ 'show', '-U3', '--format=', shas[1], '--', 'dir/b.txt', ]), ); expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected); expect(file?.status).toBe('deleted'); }); it('reports a binary change without hunks', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); const file = commit?.files.find((f) => f.path === 'bin.dat'); expect(file?.binary).toBe(true); expect(file?.hunks).toBeNull(); }); it('carries the bytes of a changed image on each side', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const second = await browser.readCommit(DID, 'diffrepo', shas[1]); const added = second?.files.find((f) => f.path === 'pic.png'); expect(added?.mediaType).toBe('image/png'); expect(added?.hunks).toBeNull(); expect(added?.oldImage).toBeNull(); expect(Array.from(added?.newImage?.subarray(0, 8) ?? [])).toEqual( PNG_MAGIC, ); const third = await browser.readCommit(DID, 'diffrepo', shas[2]); const changed = third?.files.find((f) => f.path === 'pic.png'); expect(changed?.oldImage).toEqual(added?.newImage); expect(changed?.newImage?.length).toBe(fakePng(96).length); const binary = second?.files.find((f) => f.path === 'bin.dat'); expect(binary?.mediaType).toBeNull(); expect(binary?.newImage).toBeNull(); }); it('carries both sides of a text file whole', async () => { // The page colours a whole side, not a hunk, so each side rides along // with the change. const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); const file = commit?.files.find((f) => f.path === 'a.txt'); // runGit trims what git prints, so the texts are compared without // their last newline. expect(file?.oldText?.trimEnd()).toBe( await runGit(diffRepo, ['show', `${shas[0]}:a.txt`]), ); expect(file?.newText?.trimEnd()).toBe( await runGit(diffRepo, ['show', `${shas[1]}:a.txt`]), ); const binary = commit?.files.find((f) => f.path === 'bin.dat'); expect(binary?.oldText).toBeNull(); expect(binary?.newText).toBeNull(); }); it('treats a root commit as every file added', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[0]); expect(commit?.parents).toEqual([]); expect(commit?.files.map((f) => f.path)).toEqual([ 'a.txt', 'bin.dat', 'dir/b.txt', ]); expect(commit?.files.every((f) => f.status === 'added')).toBe(true); }); it('carries the commit metadata alongside the files', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[1]); expect(commit?.sha).toBe(shas[1]); expect(commit?.parents).toEqual([shas[0]]); expect(commit?.message.trim()).toBe('second'); expect(commit?.author).toBe('Test '); }); it('places an ambiguous insertion where git places it', async () => { // The inserted block opens with the same lines that already surround it, // so Myers alone can anchor it mid-comment. Compaction slides it onto the // comment's first line, which is where git shows it. const browser = createGitBrowser({ xrpc: diffXrpc() }); const commit = await browser.readCommit(DID, 'diffrepo', shas[2]); const file = commit?.files.find((f) => f.path === 'doc.js'); const expected = hunksFromGit( await runGit(diffRepo, [ 'show', '-U3', '--format=', shas[2], '--', 'doc.js', ]), ); expect(formatHunks(/** @type {any} */ (file?.hunks))).toBe(expected); expect(file?.hunks?.[0].lines.find((l) => l.type === 'add')?.text).toBe( '/**', ); }); it('keeps every changed line when a slide merges two groups', () => { // One line becomes a repeated pair. The inserted lines slide onto the // single unchanged line between them, joining two groups into one. A // slide that does not take the joined lines with it drops one, and the // diff no longer rebuilds its own result. const hunks = diffLines('b\n', 'a\nb\na\nb\n'); expect(applyHunks('b\n', /** @type {any} */ (hunks))).toBe('a\nb\na\nb'); }); it('produces diffs that rebuild the files they describe', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); for (const sha of shas) { const commit = await browser.readCommit(DID, 'diffrepo', sha); for (const file of commit?.files ?? []) { if (!file.hunks) continue; const before = file.status === 'added' ? '' : await runGit(diffRepo, ['show', `${sha}^:${file.path}`]); const after = file.status === 'deleted' ? '' : await runGit(diffRepo, ['show', `${sha}:${file.path}`]); expect(applyHunks(before, file.hunks), `${sha} ${file.path}`).toBe( after.replace(/\n$/, ''), ); } } }); it('rejects an id that is not a commit', async () => { const browser = createGitBrowser({ xrpc: diffXrpc() }); await expect( browser.readCommit(DID, 'diffrepo', 'f'.repeat(40)), ).rejects.toThrow(/no such commit/); }); }, ); describe('lazy reader efficiency', () => { const DID = 'did:plc:lazytest'; const SPACE = 'at://did:plc:lazytest/com.atproto.simplespace.space/team'; /** @type {string} */ let bigDir = ''; /** @type {string} */ let bigRepo = ''; /** @type {string} */ let head = ''; /** @type {Uint8Array} */ let bundleBytes = new Uint8Array(); /** @type {Record} */ let record = {}; /** @type {Map} */ let blobs = new Map(); beforeAll(async () => { bigDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-lazy-')); bigRepo = join(bigDir, 'repo'); await runGit(bigDir, ['init', '-b', 'main', bigRepo]); writeFileSync(join(bigRepo, 'README.md'), '# lazy test\n'); // Incompressible bulk, so the chain's size dwarfs what a tree page // needs. writeFileSync(join(bigRepo, 'big.bin'), randomBytes(300 * 1024)); await runGit(bigRepo, ['add', '.']); await runGit(bigRepo, ['commit', '-m', 'one']); head = (await runGit(bigRepo, ['rev-parse', 'HEAD'])).trim(); await runGit(bigRepo, [ 'bundle', 'create', join(bigDir, 'all.bundle'), 'main', ]); bundleBytes = new Uint8Array(readFileSync(join(bigDir, 'all.bundle'))); const fixture = await chainFixture('lazy', [bundleBytes], head); record = fixture.record; blobs = fixture.blobs; }); afterAll(() => { rmSync(bigDir, { recursive: true, force: true }); }); it('serves a tree listing and a small file for a fraction of the chain', async () => { const stats = { bytes: 0 }; const browser = createGitBrowser({ xrpc: fakePds(record, blobs, stats) }); const listing = await browser.listTree(DID, 'testrepo'); expect(listing?.entries.map((e) => e.name)).toEqual([ 'big.bin', 'README.md', ]); // Sizes come from the index, with no blob bytes behind them. expect(listing?.entries.find((e) => e.name === 'big.bin')?.size).toBe( 300 * 1024, ); const file = await browser.readFile(DID, 'testrepo', 'main', 'README.md'); expect(file?.text).toBe('# lazy test\n'); expect(stats.bytes).toBeLessThan(bundleBytes.length / 4); }); it('answers the same through a server that ignores Range', async () => { const stats = { bytes: 0 }; const browser = createGitBrowser({ xrpc: fakePds(record, blobs, stats, { ranges: false }), }); const file = await browser.readFile(DID, 'testrepo', 'main', 'README.md'); expect(file?.text).toBe('# lazy test\n'); // Whole chunks instead of ranges, still nowhere near the whole chain. expect(stats.bytes).toBeLessThan(bundleBytes.length / 4); }); /** * The account page reads a space repository straight from storage, so * these stand in for the platform's own blob access. * @param {{bytes: number}} stats * @param {{ranges?: boolean}} [opts] - ranges: false is a reader that * answers whole blobs only * @returns {import('../src/browser.js').SpaceReader} */ function fakeSpaceReader(stats, opts = {}) { return { getRecord: async () => ({ cid: 'bafyrecordcid', value: record }), getBlob: async (cid) => { const blob = blobs.get(cid) ?? null; if (blob) stats.bytes += blob.length; return blob; }, getBlobRange: opts.ranges === false ? undefined : async (cid, start, end) => { const blob = blobs.get(cid); if (!blob) return null; const slice = blob.subarray(start, Math.min(end, blob.length)); stats.bytes += slice.length; return slice; }, }; } it('reads a space repository for a fraction of the chain', async () => { const stats = { bytes: 0 }; const browser = createGitBrowser({ xrpc: fakePds(record, blobs, { bytes: 0 }), spaceReader: fakeSpaceReader(stats), }); const file = await browser.readFile( DID, 'testrepo', 'main', 'README.md', SPACE, ); expect(file?.text).toBe('# lazy test\n'); expect(stats.bytes).toBeLessThan(bundleBytes.length / 4); }); it('reads a space repository whole when its reader has no ranges', async () => { const stats = { bytes: 0 }; const browser = createGitBrowser({ xrpc: fakePds(record, blobs, { bytes: 0 }), spaceReader: fakeSpaceReader(stats, { ranges: false }), }); const file = await browser.readFile( DID, 'testrepo', 'main', 'README.md', SPACE, ); expect(file?.text).toBe('# lazy test\n'); expect(stats.bytes).toBeGreaterThanOrEqual(bundleBytes.length); }); it('still reads the large file whole when asked', async () => { const browser = createGitBrowser({ xrpc: fakePds(record, blobs, { bytes: 0 }), }); const bytes = await browser.readBytes(DID, 'testrepo', 'main', 'big.bin'); expect(bytes?.size).toBe(300 * 1024); expect(bytes?.binary).toBe(true); }); }); describe('the commit that last changed a path', () => { const DID = 'did:plc:lastcommit'; /** @type {string} */ let workDir = ''; /** @type {string} */ let workRepo = ''; /** @type {{record: Record, blobs: Map}} */ let fixture = { record: {}, blobs: new Map() }; beforeAll(async () => { workDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-lastcommit-')); workRepo = join(workDir, 'repo'); await runGit(workDir, ['init', '-b', 'main', workRepo]); mkdirSync(join(workRepo, 'dir')); writeFileSync(join(workRepo, 'a.txt'), 'a one\n'); writeFileSync(join(workRepo, 'b.txt'), 'b one\n'); writeFileSync(join(workRepo, 'dir', 'c.txt'), 'c one\n'); await runGit(workRepo, ['add', '.']); await runGit(workRepo, ['commit', '-m', 'base']); // A branch changes one file while the trunk changes another, so the // merge that follows differs from each parent in a different path. await runGit(workRepo, ['checkout', '-b', 'topic']); writeFileSync(join(workRepo, 'b.txt'), 'b from topic\n'); await runGit(workRepo, ['commit', '-am', 'change b on topic']); await runGit(workRepo, ['checkout', 'main']); writeFileSync(join(workRepo, 'a.txt'), 'a from main\n'); await runGit(workRepo, ['commit', '-am', 'change a on main']); await runGit(workRepo, [ 'merge', '--no-ff', 'topic', '-m', 'merge topic into main', ]); const head = await runGit(workRepo, ['rev-parse', 'HEAD']); await runGit(workRepo, [ 'bundle', 'create', join(workDir, 'merge.bundle'), 'main', ]); fixture = await chainFixture( 'lazy', [new Uint8Array(readFileSync(join(workDir, 'merge.bundle')))], head, ); }); afterAll(() => { rmSync(workDir, { recursive: true, force: true }); }); const browserOf = () => createGitBrowser({ xrpc: fakePds(fixture.record, fixture.blobs, { bytes: 0 }), }); it('names the commit git names, for every path', async () => { const browser = browserOf(); for (const path of ['a.txt', 'b.txt', 'dir/c.txt']) { const wanted = await runGit(workRepo, [ 'log', '-1', '--format=%H', '--', path, ]); const answer = await browser.lastCommit(DID, 'testrepo', 'main', path); expect(answer?.sha, path).toBe(wanted); } }); it('lists the commits that changed a path, the way git log does', async () => { const browser = browserOf(); for (const path of ['a.txt', 'b.txt', 'dir/c.txt', 'dir']) { const wanted = ( await runGit(workRepo, ['log', '--format=%H', '--', path]) ) .split('\n') .filter(Boolean); const answer = await browser.listCommits( DID, 'testrepo', 'main', 50, undefined, path, ); expect( answer?.commits.map((one) => one.sha), path, ).toEqual(wanted); expect(answer?.truncated).toBe(false); } const one = await browser.listCommits( DID, 'testrepo', 'main', 1, undefined, 'a.txt', ); expect(one?.commits).toHaveLength(1); }); it('credits the branch a merge took a change from, not the merge', async () => { const browser = browserOf(); const merge = await runGit(workRepo, ['rev-parse', 'HEAD']); const answer = await browser.lastCommit(DID, 'testrepo', 'main', 'b.txt'); expect(answer?.sha).not.toBe(merge); expect(answer?.message).toContain('change b on topic'); }); it('answers null for a path no commit holds', async () => { const browser = browserOf(); const answer = await browser.lastCommit( DID, 'testrepo', 'main', 'gone.txt', ); expect(answer).toBeNull(); }); }); describe('a path the walk has to go deep to find', () => { const DID = 'did:plc:deepwalk'; /** The walk starts at 64 commits and doubles, so this crosses one run. */ const CHURN = 150; /** @type {string} */ let workDir = ''; /** @type {string} */ let workRepo = ''; /** @type {{record: Record, blobs: Map}} */ let fixture = { record: {}, blobs: new Map() }; beforeAll(async () => { workDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-deepwalk-')); workRepo = join(workDir, 'repo'); await runGit(workDir, ['init', '-b', 'main', workRepo]); // One shell keeps the churn to a single process; 150 spawns of git do // not fit the suite's budget. await execFileAsync( 'sh', [ '-c', `echo base > old.txt && mkdir -p dir && echo base > dir/deep.txt && git add . && git commit -qm base && for i in $(seq 1 ${CHURN}); do echo $i > churn.txt && git add churn.txt && git commit -qm "churn $i" done`, ], { cwd: workRepo, env }, ); const head = await runGit(workRepo, ['rev-parse', 'HEAD']); await runGit(workRepo, [ 'bundle', 'create', join(workDir, 'deep.bundle'), 'main', ]); fixture = await chainFixture( 'lazy', [new Uint8Array(readFileSync(join(workDir, 'deep.bundle')))], head, ); }); afterAll(() => { rmSync(workDir, { recursive: true, force: true }); }); it('names the commit git names, across run boundaries', async () => { const browser = createGitBrowser({ xrpc: fakePds(fixture.record, fixture.blobs, { bytes: 0 }), }); for (const path of ['old.txt', 'dir/deep.txt', 'churn.txt']) { const wanted = await runGit(workRepo, [ 'log', '-1', '--format=%H', '--', path, ]); const answer = await browser.lastCommit(DID, 'testrepo', 'main', path); expect(answer?.sha, path).toBe(wanted); } }); it('answers null for a path no commit in the deep history holds', async () => { const browser = createGitBrowser({ xrpc: fakePds(fixture.record, fixture.blobs, { bytes: 0 }), }); expect( await browser.lastCommit(DID, 'testrepo', 'main', 'dir/absent.txt'), ).toBeNull(); }); });