/** * End-to-end test of the read-only git smart HTTP layer: repositories are * pushed with the remote helper, then cloned and pulled back with stock git * over plain HTTP, no helper on the PATH needed for the reading side. * * The repack threshold is set high so the chain accumulates bundles and the * clone exercises multi-pack merging. */ import { execFile } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { defineLexicon } from '@bigmoves/lexicon'; import { LexiconResolver } from '@pdsjs/lexicon-resolver'; import { createServer } from '@pdsjs/node'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { createGitHttp } from '../src/http.js'; import { gitRepoLexicon } from '../src/index.js'; import { concatBytes, FLUSH, parsePkts, pktLine, pktText } from '../src/pkt.js'; const execFileAsync = promisify(execFile); const PORT = 2475; const BASE = `http://127.0.0.1:${PORT}`; const PASSWORD = 'test-password'; const HANDLE = 'githttp.test'; /** @param {number} length */ function randomBase32(length) { const chars = 'abcdefghijklmnopqrstuvwxyz234567'; return Array.from(randomBytes(length)) .map((b) => chars[b % 32]) .join(''); } const DID = `did:plc:${randomBase32(24)}`; const HTTP_URL = `${BASE}/git/${DID}/webrepo`; /** @type {Awaited>|null} */ let server = null; /** @type {string} */ let workDir = ''; /** @type {string} */ let srcDir = ''; /** @type {string} */ let cloneDir = ''; /** @type {Record} */ let gitEnv = {}; /** * @param {string} cwd * @param {string[]} args * @returns {Promise} stdout */ async function runGit(cwd, args) { const { stdout } = await execFileAsync('git', args, { cwd, env: gitEnv, encoding: 'utf8', }); return stdout; } beforeAll(async () => { workDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-http-')); const lexiconSource = /** @type {Parameters[0]} */ ( /** @type {unknown} */ (gitRepoLexicon) ); server = await createServer({ port: PORT, dbPath: join(workDir, 'pds.db'), blobsDir: join(workDir, 'blobs'), jwtSecret: 'test-secret-for-git-http', hostname: `127.0.0.1:${PORT}`, password: PASSWORD, lexiconResolver: new LexiconResolver({ schemas: [defineLexicon(lexiconSource)], }), experimental: { git: { http: true } }, }); await server.listen(); const init = await fetch(`${BASE}/init?did=${DID}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ did: DID, privateKey: randomBytes(32).toString('hex'), handle: HANDLE, password: PASSWORD, }), }); expect(init.ok).toBe(true); const binDir = join(workDir, 'bin'); mkdirSync(binDir); const cliPath = join( dirname(fileURLToPath(import.meta.url)), '..', 'src', 'cli.js', ); const shim = join(binDir, 'git-remote-atproto'); writeFileSync(shim, `#!/bin/sh\nexec node ${cliPath} "$@"\n`); chmodSync(shim, 0o755); gitEnv = { .../** @type {Record} */ (process.env), PATH: `${binDir}:${process.env.PATH}`, ATPROTO_GIT_SERVICE: BASE, ATPROTO_GIT_IDENTIFIER: DID, ATPROTO_GIT_PASSWORD: PASSWORD, ATPROTO_GIT_CHUNK_SIZE: '1024', // High threshold and no tail merges: the chain must stay unpacked so // HTTP clones merge ATPROTO_GIT_REPACK_THRESHOLD: '100', ATPROTO_GIT_COMPACT_FACTOR: '0', GIT_TERMINAL_PROMPT: '0', GIT_AUTHOR_NAME: 'Test', GIT_AUTHOR_EMAIL: 'test@example.com', GIT_COMMITTER_NAME: 'Test', GIT_COMMITTER_EMAIL: 'test@example.com', }; // Three pushes, three bundles: the clone must merge a chain srcDir = join(workDir, 'src-repo'); mkdirSync(srcDir); await runGit(srcDir, ['init', '-b', 'main']); writeFileSync(join(srcDir, 'README.md'), '# served over smart http\n'); writeFileSync(join(srcDir, 'data.bin'), randomBytes(4096)); mkdirSync(join(srcDir, 'docs')); writeFileSync(join(srcDir, 'docs', 'guide.md'), 'a nested file\n'); await runGit(srcDir, ['add', '.']); await runGit(srcDir, ['commit', '-m', 'one']); await runGit(srcDir, ['remote', 'add', 'origin', `atproto://${DID}/webrepo`]); await runGit(srcDir, ['push', 'origin', 'main']); writeFileSync(join(srcDir, 'second.txt'), 'two\n'); await runGit(srcDir, ['add', '.']); await runGit(srcDir, ['commit', '-m', 'two']); await runGit(srcDir, ['push', 'origin', 'main']); writeFileSync(join(srcDir, 'third.txt'), 'three\n'); await runGit(srcDir, ['add', '.']); await runGit(srcDir, ['commit', '-m', 'three']); await runGit(srcDir, ['tag', 'v1']); await runGit(srcDir, ['push', 'origin', 'main', 'v1']); }, 120_000); afterAll(async () => { if (server) await server.close(); rmSync(workDir, { recursive: true, force: true }); }); describe('git smart HTTP e2e', () => { it('clones a multi-bundle repository with stock git', async () => { cloneDir = join(workDir, 'http-clone'); await runGit(workDir, ['clone', HTTP_URL, cloneDir]); expect((await runGit(cloneDir, ['branch', '--show-current'])).trim()).toBe( 'main', ); const log = await runGit(cloneDir, ['log', '--format=%s', 'main']); expect(log.trim().split('\n')).toEqual(['three', 'two', 'one']); expect(await runGit(cloneDir, ['show', 'HEAD:README.md'])).toContain( 'served over smart http', ); expect((await runGit(cloneDir, ['tag', '-l'])).trim()).toBe('v1'); // fsck proves full connectivity of the merged pack await runGit(cloneDir, ['fsck', '--strict']); }); it('clones through the handle path too', async () => { const handleClone = join(workDir, 'handle-clone'); await runGit(workDir, [ 'clone', `${BASE}/git/${HANDLE}/webrepo`, handleClone, ]); const log = await runGit(handleClone, ['log', '--format=%s', 'main']); expect(log.trim().split('\n')).toEqual(['three', 'two', 'one']); }); it('pulls an incremental push', async () => { writeFileSync(join(srcDir, 'fourth.txt'), 'four\n'); await runGit(srcDir, ['add', '.']); await runGit(srcDir, ['commit', '-m', 'four']); await runGit(srcDir, ['push', 'origin', 'main']); await runGit(cloneDir, ['pull', 'origin', 'main']); const log = await runGit(cloneDir, ['log', '--format=%s', 'HEAD']); expect(log.trim().split('\n')[0]).toBe('four'); }); it('fetches a refs-only update (new branch on existing objects)', async () => { await runGit(srcDir, ['branch', 'feature', 'main~1']); await runGit(srcDir, ['push', 'origin', 'feature']); await runGit(cloneDir, ['fetch', 'origin']); const sha = ( await runGit(cloneDir, ['rev-parse', 'origin/feature']) ).trim(); expect(sha).toBe((await runGit(srcDir, ['rev-parse', 'main~1'])).trim()); }); it('refuses pushes with a helpful error', async () => { await expect(runGit(cloneDir, ['push', HTTP_URL, 'main'])).rejects.toThrow( /read-only over HTTP|git-remote-atproto/, ); }); it('404s an unknown repository', async () => { await expect( runGit(workDir, [ 'clone', `${BASE}/git/${DID}/nosuchrepo`, join(workDir, 'nope'), ]), ).rejects.toThrow(); }); }); describe('upload-pack failures', () => { it('reports a blob lost mid-stream on the error band', async () => { const head = (await runGit(srcDir, ['rev-parse', 'main'])).trim(); // Every blob is served once and is gone after that, the way orphan // cleanup can take a chunk between the header read and the body. /** @type {Set} */ const served = new Set(); const git = createGitHttp({ xrpc: async (request) => { const url = new URL(request.url); const cid = url.searchParams.get('cid'); if (url.pathname.endsWith('getBlob') && cid) { if (served.has(cid)) return new Response('gone', { status: 404 }); served.add(cid); } return fetch(`${BASE}${url.pathname}${url.search}`, { headers: request.headers, }); }, }); const res = await git.handle( new Request(`http://pds.internal/git/${DID}/webrepo/git-upload-pack`, { method: 'POST', body: /** @type {BodyInit} */ ( concatBytes([ pktLine(`want ${head} side-band-64k\n`), FLUSH, pktLine('done\n'), ]) ), }), ); // The header is out before the read fails, so the failure rides the // error band rather than a status code. expect(res?.status).toBe(200); const pkts = parsePkts( new Uint8Array(await /** @type {Response} */ (res).arrayBuffer()), ); expect(pktText(/** @type {{data: Uint8Array}} */ (pkts[0]).data)).toBe( 'NAK', ); const last = /** @type {{data: Uint8Array}} */ (pkts[pkts.length - 1]).data; expect(last[0]).toBe(3); expect(pktText(last.subarray(1))).toMatch(/unavailable/); }); }); describe('raw files', () => { /** @param {string} path */ const raw = (path) => fetch(`${HTTP_URL}/raw/${path}`); it('serves a file byte for byte', async () => { const res = await raw('main/README.md'); expect(res.status).toBe(200); expect(await res.text()).toBe('# served over smart http\n'); }); it('serves text as text/plain, whatever the extension', async () => { // A repository can hold a page; serving it as HTML would run it on this // origin. const res = await raw('main/README.md'); expect(res.headers.get('Content-Type')).toBe('text/plain; charset=utf-8'); expect(res.headers.get('X-Content-Type-Options')).toBe('nosniff'); expect(res.headers.get('Content-Security-Policy')).toContain("'none'"); }); it('serves a binary file as bytes', async () => { const res = await raw('main/data.bin'); expect(res.status).toBe(200); expect(res.headers.get('Content-Type')).toBe('application/octet-stream'); expect((await res.arrayBuffer()).byteLength).toBe(4096); }); it('lists a ref root so a crawler can walk in', async () => { const names = (await (await raw('main/')).text()) .split('\n') .filter(Boolean); expect(names).toContain('README.md'); expect(names).toContain('data.bin'); }); it('lists the default branch with no ref at all', async () => { const withRef = await (await raw('main/')).text(); const bare = await (await fetch(`${HTTP_URL}/raw`)).text(); expect(bare).toBe(withRef); }); it('marks directories with a trailing slash', async () => { // A crawler tells a directory from a file by the slash alone. const names = (await (await raw('main/')).text()) .split('\n') .filter(Boolean); for (const name of names) { const res = await raw(`main/${name}`); expect(res.status, name).toBe(200); if (name.endsWith('/')) { expect(res.headers.get('Content-Type')).toBe( 'text/plain; charset=utf-8', ); } } }); it('reads a ref root without the trailing slash', async () => { expect((await raw('main')).status).toBe(200); }); it('answers cross-origin, so a page can read it too', async () => { const res = await raw('main/README.md'); expect(res.headers.get('Access-Control-Allow-Origin')).toBe('*'); }); it('404s an unknown repository', async () => { const res = await fetch(`${BASE}/git/${DID}/nope/raw/main/README.md`); expect(res.status).toBe(404); }); it('404s a path that is not in the tree', async () => { const res = await raw('main/no/such/file.txt'); expect(res.status).toBe(404); }); it('404s an unknown ref', async () => { expect((await raw('nope/README.md')).status).toBe(404); }); it('reads a nested file', async () => { expect(await (await raw('main/docs/guide.md')).text()).toBe( 'a nested file\n', ); }); it('lists a directory asked for without its trailing slash', async () => { const withSlash = await (await raw('main/docs/')).text(); const without = await (await raw('main/docs')).text(); expect(without).toBe(withSlash); expect(without.trim()).toBe('guide.md'); }); it('marks a nested directory in the root listing', async () => { const names = (await (await raw('main/')).text()).split('\n'); expect(names).toContain('docs/'); }); it('refuses to write', async () => { const res = await fetch(`${HTTP_URL}/raw/main/README.md`, { method: 'POST', }); expect(res.status).toBe(405); }); }); describe('etag revalidation', () => { // The record cid validates every read: a push writes a new cid. const REFS_URL = `${HTTP_URL}/info/refs?service=git-upload-pack`; // A page on the account's site is a different origin from its PDS, so the // browser reads this only with the open origin on it. it('advertises refs to a browser on another origin', async () => { const res = await fetch(REFS_URL); expect(res.headers.get('Access-Control-Allow-Origin')).toBe('*'); }); it('advertises refs with the record cid as the etag', async () => { const res = await fetch(REFS_URL); const etag = res.headers.get('ETag'); expect(etag).toMatch(/^".+"$/); const again = await fetch(REFS_URL, { headers: { 'If-None-Match': /** @type {string} */ (etag) }, }); expect(again.status).toBe(304); expect(again.headers.get('ETag')).toBe(etag); }); it('revalidates a raw read before any blob is fetched', async () => { const res = await fetch(`${HTTP_URL}/raw/main/README.md`); const etag = /** @type {string} */ (res.headers.get('ETag')); expect(etag).toMatch(/^".+"$/); const again = await fetch(`${HTTP_URL}/raw/main/README.md`, { headers: { 'If-None-Match': etag }, }); expect(again.status).toBe(304); expect(again.headers.get('Access-Control-Allow-Origin')).toBe('*'); expect(await again.text()).toBe(''); }); it('serves fresh content once a push changes the record', async () => { const before = (await fetch(REFS_URL)).headers.get('ETag'); writeFileSync(join(srcDir, 'fifth.txt'), 'five\n'); await runGit(srcDir, ['add', '.']); await runGit(srcDir, ['commit', '-m', 'five']); await runGit(srcDir, ['push', 'origin', 'main']); const after = await fetch(REFS_URL, { headers: { 'If-None-Match': /** @type {string} */ (before) }, }); expect(after.status).toBe(200); expect(after.headers.get('ETag')).not.toBe(before); }); });