chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
/**
* End-to-end test of the read-only git smart HTTP layer: repositories are
* pushed with the remote helper, then cloned and pulled back with stock git
* over plain HTTP, no helper on the PATH needed for the reading side.
*
* The repack threshold is set high so the chain accumulates bundles and the
* clone exercises multi-pack merging.
*/
import { execFile } from 'node:child_process';
import { randomBytes } from 'node:crypto';
import {
chmodSync,
mkdirSync,
mkdtempSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { defineLexicon } from '@bigmoves/lexicon';
import { LexiconResolver } from '@pdsjs/lexicon-resolver';
import { createServer } from '@pdsjs/node';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createGitHttp } from '../src/http.js';
import { gitRepoLexicon } from '../src/index.js';
import { concatBytes, FLUSH, parsePkts, pktLine, pktText } from '../src/pkt.js';
const execFileAsync = promisify(execFile);
const PORT = 2475;
const BASE = `http://127.0.0.1:${PORT}`;
const PASSWORD = 'test-password';
const HANDLE = 'githttp.test';
/** @param {number} length */
function randomBase32(length) {
const chars = 'abcdefghijklmnopqrstuvwxyz234567';
return Array.from(randomBytes(length))
.map((b) => chars[b % 32])
.join('');
}
const DID = `did:plc:${randomBase32(24)}`;
const HTTP_URL = `${BASE}/git/${DID}/webrepo`;
/** @type {Awaited<ReturnType<typeof createServer>>|null} */
let server = null;
/** @type {string} */
let workDir = '';
/** @type {string} */
let srcDir = '';
/** @type {string} */
let cloneDir = '';
/** @type {Record<string, string>} */
let gitEnv = {};
/**
* @param {string} cwd
* @param {string[]} args
* @returns {Promise<string>} stdout
*/
async function runGit(cwd, args) {
const { stdout } = await execFileAsync('git', args, {
cwd,
env: gitEnv,
encoding: 'utf8',
});
return stdout;
}
beforeAll(async () => {
workDir = mkdtempSync(join(tmpdir(), 'pdsjs-git-http-'));
const lexiconSource = /** @type {Parameters<typeof defineLexicon>[0]} */ (
/** @type {unknown} */ (gitRepoLexicon)
);
server = await createServer({
port: PORT,
dbPath: join(workDir, 'pds.db'),
blobsDir: join(workDir, 'blobs'),
jwtSecret: 'test-secret-for-git-http',
hostname: `127.0.0.1:${PORT}`,
password: PASSWORD,
lexiconResolver: new LexiconResolver({
schemas: [defineLexicon(lexiconSource)],
}),
experimental: { git: { http: true } },
});
await server.listen();
const init = await fetch(`${BASE}/init?did=${DID}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
did: DID,
privateKey: randomBytes(32).toString('hex'),
handle: HANDLE,
password: PASSWORD,
}),
});
expect(init.ok).toBe(true);
const binDir = join(workDir, 'bin');
mkdirSync(binDir);
const cliPath = join(
dirname(fileURLToPath(import.meta.url)),
'..',
'src',
'cli.js',
);
const shim = join(binDir, 'git-remote-atproto');
writeFileSync(shim, `#!/bin/sh\nexec node ${cliPath} "$@"\n`);
chmodSync(shim, 0o755);
gitEnv = {
.../** @type {Record<string, string>} */ (process.env),
PATH: `${binDir}:${process.env.PATH}`,
ATPROTO_GIT_SERVICE: BASE,
ATPROTO_GIT_IDENTIFIER: DID,
ATPROTO_GIT_PASSWORD: PASSWORD,
ATPROTO_GIT_CHUNK_SIZE: '1024',
// High threshold and no tail merges: the chain must stay unpacked so
// HTTP clones merge
ATPROTO_GIT_REPACK_THRESHOLD: '100',
ATPROTO_GIT_COMPACT_FACTOR: '0',
GIT_TERMINAL_PROMPT: '0',
GIT_AUTHOR_NAME: 'Test',
GIT_AUTHOR_EMAIL: 'test@example.com',
GIT_COMMITTER_NAME: 'Test',
GIT_COMMITTER_EMAIL: 'test@example.com',
};
// Three pushes, three bundles: the clone must merge a chain
srcDir = join(workDir, 'src-repo');
mkdirSync(srcDir);
await runGit(srcDir, ['init', '-b', 'main']);
writeFileSync(join(srcDir, 'README.md'), '# served over smart http\n');
writeFileSync(join(srcDir, 'data.bin'), randomBytes(4096));
mkdirSync(join(srcDir, 'docs'));
writeFileSync(join(srcDir, 'docs', 'guide.md'), 'a nested file\n');
await runGit(srcDir, ['add', '.']);
await runGit(srcDir, ['commit', '-m', 'one']);
await runGit(srcDir, ['remote', 'add', 'origin', `atproto://${DID}/webrepo`]);
await runGit(srcDir, ['push', 'origin', 'main']);
writeFileSync(join(srcDir, 'second.txt'), 'two\n');
await runGit(srcDir, ['add', '.']);
await runGit(srcDir, ['commit', '-m', 'two']);
await runGit(srcDir, ['push', 'origin', 'main']);
writeFileSync(join(srcDir, 'third.txt'), 'three\n');
await runGit(srcDir, ['add', '.']);
await runGit(srcDir, ['commit', '-m', 'three']);
await runGit(srcDir, ['tag', 'v1']);
await runGit(srcDir, ['push', 'origin', 'main', 'v1']);
}, 120_000);
afterAll(async () => {
if (server) await server.close();
rmSync(workDir, { recursive: true, force: true });
});
describe('git smart HTTP e2e', () => {
it('clones a multi-bundle repository with stock git', async () => {
cloneDir = join(workDir, 'http-clone');
await runGit(workDir, ['clone', HTTP_URL, cloneDir]);
expect((await runGit(cloneDir, ['branch', '--show-current'])).trim()).toBe(
'main',
);
const log = await runGit(cloneDir, ['log', '--format=%s', 'main']);
expect(log.trim().split('\n')).toEqual(['three', 'two', 'one']);
expect(await runGit(cloneDir, ['show', 'HEAD:README.md'])).toContain(
'served over smart http',
);
expect((await runGit(cloneDir, ['tag', '-l'])).trim()).toBe('v1');
// fsck proves full connectivity of the merged pack
await runGit(cloneDir, ['fsck', '--strict']);
});
it('clones through the handle path too', async () => {
const handleClone = join(workDir, 'handle-clone');
await runGit(workDir, [
'clone',
`${BASE}/git/${HANDLE}/webrepo`,
handleClone,
]);
const log = await runGit(handleClone, ['log', '--format=%s', 'main']);
expect(log.trim().split('\n')).toEqual(['three', 'two', 'one']);
});
it('pulls an incremental push', async () => {
writeFileSync(join(srcDir, 'fourth.txt'), 'four\n');
await runGit(srcDir, ['add', '.']);
await runGit(srcDir, ['commit', '-m', 'four']);
await runGit(srcDir, ['push', 'origin', 'main']);
await runGit(cloneDir, ['pull', 'origin', 'main']);
const log = await runGit(cloneDir, ['log', '--format=%s', 'HEAD']);
expect(log.trim().split('\n')[0]).toBe('four');
});
it('fetches a refs-only update (new branch on existing objects)', async () => {
await runGit(srcDir, ['branch', 'feature', 'main~1']);
await runGit(srcDir, ['push', 'origin', 'feature']);
await runGit(cloneDir, ['fetch', 'origin']);
const sha = (
await runGit(cloneDir, ['rev-parse', 'origin/feature'])
).trim();
expect(sha).toBe((await runGit(srcDir, ['rev-parse', 'main~1'])).trim());
});
it('refuses pushes with a helpful error', async () => {
await expect(runGit(cloneDir, ['push', HTTP_URL, 'main'])).rejects.toThrow(
/read-only over HTTP|git-remote-atproto/,
);
});
it('404s an unknown repository', async () => {
await expect(
runGit(workDir, [
'clone',
`${BASE}/git/${DID}/nosuchrepo`,
join(workDir, 'nope'),
]),
).rejects.toThrow();
});
});
describe('upload-pack failures', () => {
it('reports a blob lost mid-stream on the error band', async () => {
const head = (await runGit(srcDir, ['rev-parse', 'main'])).trim();
// Every blob is served once and is gone after that, the way orphan
// cleanup can take a chunk between the header read and the body.
/** @type {Set<string>} */
const served = new Set();
const git = createGitHttp({
xrpc: async (request) => {
const url = new URL(request.url);
const cid = url.searchParams.get('cid');
if (url.pathname.endsWith('getBlob') && cid) {
if (served.has(cid)) return new Response('gone', { status: 404 });
served.add(cid);
}
return fetch(`${BASE}${url.pathname}${url.search}`, {
headers: request.headers,
});
},
});
const res = await git.handle(
new Request(`http://pds.internal/git/${DID}/webrepo/git-upload-pack`, {
method: 'POST',
body: /** @type {BodyInit} */ (
concatBytes([
pktLine(`want ${head} side-band-64k\n`),
FLUSH,
pktLine('done\n'),
])
),
}),
);
// The header is out before the read fails, so the failure rides the
// error band rather than a status code.
expect(res?.status).toBe(200);
const pkts = parsePkts(
new Uint8Array(await /** @type {Response} */ (res).arrayBuffer()),
);
expect(pktText(/** @type {{data: Uint8Array}} */ (pkts[0]).data)).toBe(
'NAK',
);
const last = /** @type {{data: Uint8Array}} */ (pkts[pkts.length - 1]).data;
expect(last[0]).toBe(3);
expect(pktText(last.subarray(1))).toMatch(/unavailable/);
});
});
describe('raw files', () => {
/** @param {string} path */
const raw = (path) => fetch(`${HTTP_URL}/raw/${path}`);
it('serves a file byte for byte', async () => {
const res = await raw('main/README.md');
expect(res.status).toBe(200);
expect(await res.text()).toBe('# served over smart http\n');
});
it('serves text as text/plain, whatever the extension', async () => {
// A repository can hold a page; serving it as HTML would run it on this
// origin.
const res = await raw('main/README.md');
expect(res.headers.get('Content-Type')).toBe('text/plain; charset=utf-8');
expect(res.headers.get('X-Content-Type-Options')).toBe('nosniff');
expect(res.headers.get('Content-Security-Policy')).toContain("'none'");
});
it('serves a binary file as bytes', async () => {
const res = await raw('main/data.bin');
expect(res.status).toBe(200);
expect(res.headers.get('Content-Type')).toBe('application/octet-stream');
expect((await res.arrayBuffer()).byteLength).toBe(4096);
});
it('lists a ref root so a crawler can walk in', async () => {
const names = (await (await raw('main/')).text())
.split('\n')
.filter(Boolean);
expect(names).toContain('README.md');
expect(names).toContain('data.bin');
});
it('lists the default branch with no ref at all', async () => {
const withRef = await (await raw('main/')).text();
const bare = await (await fetch(`${HTTP_URL}/raw`)).text();
expect(bare).toBe(withRef);
});
it('marks directories with a trailing slash', async () => {
// A crawler tells a directory from a file by the slash alone.
const names = (await (await raw('main/')).text())
.split('\n')
.filter(Boolean);
for (const name of names) {
const res = await raw(`main/${name}`);
expect(res.status, name).toBe(200);
if (name.endsWith('/')) {
expect(res.headers.get('Content-Type')).toBe(
'text/plain; charset=utf-8',
);
}
}
});
it('reads a ref root without the trailing slash', async () => {
expect((await raw('main')).status).toBe(200);
});
it('answers cross-origin, so a page can read it too', async () => {
const res = await raw('main/README.md');
expect(res.headers.get('Access-Control-Allow-Origin')).toBe('*');
});
it('404s an unknown repository', async () => {
const res = await fetch(`${BASE}/git/${DID}/nope/raw/main/README.md`);
expect(res.status).toBe(404);
});
it('404s a path that is not in the tree', async () => {
const res = await raw('main/no/such/file.txt');
expect(res.status).toBe(404);
});
it('404s an unknown ref', async () => {
expect((await raw('nope/README.md')).status).toBe(404);
});
it('reads a nested file', async () => {
expect(await (await raw('main/docs/guide.md')).text()).toBe(
'a nested file\n',
);
});
it('lists a directory asked for without its trailing slash', async () => {
const withSlash = await (await raw('main/docs/')).text();
const without = await (await raw('main/docs')).text();
expect(without).toBe(withSlash);
expect(without.trim()).toBe('guide.md');
});
it('marks a nested directory in the root listing', async () => {
const names = (await (await raw('main/')).text()).split('\n');
expect(names).toContain('docs/');
});
it('refuses to write', async () => {
const res = await fetch(`${HTTP_URL}/raw/main/README.md`, {
method: 'POST',
});
expect(res.status).toBe(405);
});
});
describe('etag revalidation', () => {
// The record cid validates every read: a push writes a new cid.
const REFS_URL = `${HTTP_URL}/info/refs?service=git-upload-pack`;
// A page on the account's site is a different origin from its PDS, so the
// browser reads this only with the open origin on it.
it('advertises refs to a browser on another origin', async () => {
const res = await fetch(REFS_URL);
expect(res.headers.get('Access-Control-Allow-Origin')).toBe('*');
});
it('advertises refs with the record cid as the etag', async () => {
const res = await fetch(REFS_URL);
const etag = res.headers.get('ETag');
expect(etag).toMatch(/^".+"$/);
const again = await fetch(REFS_URL, {
headers: { 'If-None-Match': /** @type {string} */ (etag) },
});
expect(again.status).toBe(304);
expect(again.headers.get('ETag')).toBe(etag);
});
it('revalidates a raw read before any blob is fetched', async () => {
const res = await fetch(`${HTTP_URL}/raw/main/README.md`);
const etag = /** @type {string} */ (res.headers.get('ETag'));
expect(etag).toMatch(/^".+"$/);
const again = await fetch(`${HTTP_URL}/raw/main/README.md`, {
headers: { 'If-None-Match': etag },
});
expect(again.status).toBe(304);
expect(again.headers.get('Access-Control-Allow-Origin')).toBe('*');
expect(await again.text()).toBe('');
});
it('serves fresh content once a push changes the record', async () => {
const before = (await fetch(REFS_URL)).headers.get('ETag');
writeFileSync(join(srcDir, 'fifth.txt'), 'five\n');
await runGit(srcDir, ['add', '.']);
await runGit(srcDir, ['commit', '-m', 'five']);
await runGit(srcDir, ['push', 'origin', 'main']);
const after = await fetch(REFS_URL, {
headers: { 'If-None-Match': /** @type {string} */ (before) },
});
expect(after.status).toBe(200);
expect(after.headers.get('ETag')).not.toBe(before);
});
});