chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
/**
* End-to-end test of the OCI registry over a real node server: blobs are
* pushed with the upload session protocol, manifests land in the record,
* and pulls read everything back the way a docker client would.
*/
import { createHash, randomBytes } from 'node:crypto';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createServer } from '@pdsjs/node';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { MEDIA_TYPES } from '../src/manifest.js';
const PORT = 2485;
const BASE = `http://127.0.0.1:${PORT}`;
const PASSWORD = 'registry-test-password';
const HANDLE = 'registry.test';
/** @param {number} length */
function randomBase32(length) {
const chars = 'abcdefghijklmnopqrstuvwxyz234567';
return Array.from(randomBytes(length))
.map((b) => chars[b % 32])
.join('');
}
const DID = `did:plc:${randomBase32(24)}`;
const AUTH = `Basic ${Buffer.from(`${DID}:${PASSWORD}`).toString('base64')}`;
/** @type {Awaited<ReturnType<typeof createServer>>|null} */
let server = null;
/** @type {string} */
let workDir = '';
/** @param {Uint8Array|string} bytes */
function digestOf(bytes) {
return `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
}
/**
* Push one blob through the session protocol: POST, PATCH, PUT.
* @param {string} name
* @param {Uint8Array} bytes
* @returns {Promise<string>} the blob's digest
*/
async function pushBlob(name, bytes) {
const digest = digestOf(bytes);
const started = await fetch(`${BASE}/v2/${name}/blobs/uploads/`, {
method: 'POST',
headers: { Authorization: AUTH },
});
expect(started.status).toBe(202);
const location = started.headers.get('location');
expect(location).toBeTruthy();
expect(started.headers.get('docker-upload-uuid')).toBeTruthy();
const patched = await fetch(`${BASE}${location}`, {
method: 'PATCH',
headers: { Authorization: AUTH },
body: /** @type {BodyInit} */ (bytes),
});
expect(patched.status).toBe(202);
expect(patched.headers.get('range')).toBe(`0-${bytes.length - 1}`);
const put = await fetch(
`${BASE}${patched.headers.get('location')}?digest=${digest}`,
{ method: 'PUT', headers: { Authorization: AUTH } },
);
expect(put.status).toBe(201);
expect(put.headers.get('docker-content-digest')).toBe(digest);
return digest;
}
/**
* @param {string} name
* @param {string} reference
* @param {Uint8Array} bytes
* @param {string} mediaType
*/
async function pushManifest(name, reference, bytes, mediaType) {
const res = await fetch(`${BASE}/v2/${name}/manifests/${reference}`, {
method: 'PUT',
headers: { Authorization: AUTH, 'Content-Type': mediaType },
body: /** @type {BodyInit} */ (bytes),
});
return res;
}
/**
* @param {{config: Uint8Array, layers: Uint8Array[]}} parts
* @returns {Uint8Array}
*/
function imageManifest({ config, layers }) {
return new TextEncoder().encode(
JSON.stringify({
schemaVersion: 2,
mediaType: MEDIA_TYPES.ociManifest,
config: {
mediaType: 'application/vnd.oci.image.config.v1+json',
digest: digestOf(config),
size: config.length,
},
layers: layers.map((layer) => ({
mediaType: 'application/vnd.oci.image.layer.v1.tar+gzip',
digest: digestOf(layer),
size: layer.length,
})),
}),
);
}
const layerOne = new Uint8Array(randomBytes(256 * 1024));
const layerTwo = new Uint8Array(randomBytes(64 * 1024));
const configBytes = new TextEncoder().encode(
JSON.stringify({ architecture: 'amd64', os: 'linux' }),
);
const manifestBytes = imageManifest({
config: configBytes,
layers: [layerOne, layerTwo],
});
const manifestDigest = digestOf(manifestBytes);
beforeAll(async () => {
workDir = mkdtempSync(join(tmpdir(), 'pdsjs-registry-'));
server = await createServer({
port: PORT,
dbPath: join(workDir, 'pds.db'),
blobsDir: join(workDir, 'blobs'),
jwtSecret: 'test-secret-for-registry',
hostname: `127.0.0.1:${PORT}`,
password: PASSWORD,
experimental: { oci: true },
});
await server.listen();
const init = await fetch(`${BASE}/init?did=${DID}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
did: DID,
privateKey: randomBytes(32).toString('hex'),
handle: HANDLE,
password: PASSWORD,
}),
});
expect(init.ok).toBe(true);
}, 60_000);
afterAll(async () => {
if (server) await server.close();
rmSync(workDir, { recursive: true, force: true });
});
describe('OCI registry e2e', () => {
it('challenges the anonymous version check and accepts credentials', async () => {
// The ping carries the Bearer challenge; docker reads its auth scheme
// from this response.
const anonymous = await fetch(`${BASE}/v2/`);
expect(anonymous.status).toBe(401);
expect(anonymous.headers.get('www-authenticate')).toMatch(
/^Bearer realm=".*\/v2\/token"/,
);
const authed = await fetch(`${BASE}/v2/`, {
headers: { Authorization: AUTH },
});
expect(authed.status).toBe(200);
expect(authed.headers.get('docker-distribution-api-version')).toBe(
'registry/2.0',
);
});
it('challenges an unauthenticated push with Bearer', async () => {
const res = await fetch(`${BASE}/v2/app/blobs/uploads/`, {
method: 'POST',
});
expect(res.status).toBe(401);
expect(res.headers.get('www-authenticate')).toMatch(/^Bearer realm=/);
const body = await res.json();
expect(body.errors[0].code).toBe('UNAUTHORIZED');
});
it('walks the token flow the way docker does', async () => {
// Anonymous realm request: a pull-only token.
const anonToken = await fetch(
`${BASE}/v2/token?service=x&scope=repository:app:pull`,
);
expect(anonToken.status).toBe(200);
const { token: pullToken } = await anonToken.json();
expect(pullToken).toBeTruthy();
const ping = await fetch(`${BASE}/v2/`, {
headers: { Authorization: `Bearer ${pullToken}` },
});
expect(ping.status).toBe(200);
// The pull-only token cannot push.
const denied = await fetch(`${BASE}/v2/app/blobs/uploads/`, {
method: 'POST',
headers: { Authorization: `Bearer ${pullToken}` },
});
expect(denied.status).toBe(403);
// Basic credentials at the realm: a push token.
const authToken = await fetch(
`${BASE}/v2/token?service=x&scope=repository:app:push,pull`,
{
headers: { Authorization: AUTH },
},
);
const { token: pushToken } = await authToken.json();
const started = await fetch(`${BASE}/v2/app/blobs/uploads/`, {
method: 'POST',
headers: { Authorization: `Bearer ${pushToken}` },
});
expect(started.status).toBe(202);
await fetch(`${BASE}${started.headers.get('location')}`, {
method: 'DELETE',
headers: { Authorization: `Bearer ${pushToken}` },
});
// Bad credentials at the realm are refused, not downgraded, so docker
// login fails loudly.
const bad = await fetch(`${BASE}/v2/token`, {
headers: {
Authorization: `Basic ${Buffer.from(`${DID}:wrong`).toString('base64')}`,
},
});
expect(bad.status).toBe(401);
});
it('refuses bad credentials', async () => {
const res = await fetch(`${BASE}/v2/app/blobs/uploads/`, {
method: 'POST',
headers: {
Authorization: `Basic ${Buffer.from(`${DID}:wrong`).toString('base64')}`,
},
});
expect(res.status).toBe(401);
});
it('pushes a complete image', async () => {
await pushBlob('app', layerOne);
await pushBlob('app', configBytes);
// The second layer goes up monolithically: one POST with the digest.
const posted = await fetch(
`${BASE}/v2/app/blobs/uploads/?digest=${digestOf(layerTwo)}`,
{ method: 'POST', headers: { Authorization: AUTH }, body: layerTwo },
);
expect(posted.status).toBe(201);
const put = await pushManifest(
'app',
'latest',
manifestBytes,
MEDIA_TYPES.ociManifest,
);
expect(put.status).toBe(201);
expect(put.headers.get('docker-content-digest')).toBe(manifestDigest);
expect(put.headers.get('location')).toBe(
`/v2/app/manifests/${manifestDigest}`,
);
});
it('answers HEAD for a pushed blob', async () => {
const res = await fetch(`${BASE}/v2/app/blobs/${digestOf(layerOne)}`, {
method: 'HEAD',
});
expect(res.status).toBe(200);
expect(res.headers.get('content-length')).toBe(String(layerOne.length));
expect(res.headers.get('docker-content-digest')).toBe(digestOf(layerOne));
});
it('serves blob bytes, whole and ranged', async () => {
const whole = await fetch(`${BASE}/v2/app/blobs/${digestOf(layerOne)}`);
expect(whole.status).toBe(200);
const bytes = new Uint8Array(await whole.arrayBuffer());
expect(bytes).toEqual(layerOne);
const ranged = await fetch(`${BASE}/v2/app/blobs/${digestOf(layerOne)}`, {
headers: { Range: 'bytes=16-31' },
});
expect(ranged.status).toBe(206);
expect(new Uint8Array(await ranged.arrayBuffer())).toEqual(
layerOne.subarray(16, 32),
);
});
it('serves the manifest by tag and by digest, byte for byte', async () => {
for (const reference of ['latest', manifestDigest]) {
const res = await fetch(`${BASE}/v2/app/manifests/${reference}`);
expect(res.status).toBe(200);
expect(res.headers.get('content-type')).toBe(MEDIA_TYPES.ociManifest);
expect(res.headers.get('docker-content-digest')).toBe(manifestDigest);
expect(new Uint8Array(await res.arrayBuffer())).toEqual(manifestBytes);
}
const head = await fetch(`${BASE}/v2/app/manifests/latest`, {
method: 'HEAD',
});
expect(head.status).toBe(200);
expect(head.headers.get('content-length')).toBe(
String(manifestBytes.length),
);
});
it('holds every pushed blob in the repository record', async () => {
const params = new URLSearchParams({
repo: DID,
collection: 'dev.pdsjs.oci.repo',
rkey: 'app',
});
const res = await fetch(
`${BASE}/xrpc/com.atproto.repo.getRecord?${params}`,
);
expect(res.ok).toBe(true);
const body = await res.json();
expect(body.value.name).toBe('app');
expect(body.value.tags).toEqual([
{ tag: 'latest', digest: manifestDigest },
]);
// One manifest, plus config and two layers: four blob references keep
// the bytes out of orphan cleanup.
expect(body.value.manifests).toHaveLength(1);
expect(body.value.layers).toHaveLength(3);
});
it('lists tags and the catalog', async () => {
const tags = await fetch(`${BASE}/v2/app/tags/list`);
expect(await tags.json()).toEqual({ name: 'app', tags: ['latest'] });
const catalog = await fetch(`${BASE}/v2/_catalog`);
expect(await catalog.json()).toEqual({ repositories: ['app'] });
});
it('mounts an existing blob without a new upload', async () => {
const res = await fetch(
`${BASE}/v2/other/blobs/uploads/?mount=${digestOf(layerOne)}&from=app`,
{ method: 'POST', headers: { Authorization: AUTH } },
);
expect(res.status).toBe(201);
expect(res.headers.get('docker-content-digest')).toBe(digestOf(layerOne));
});
it('accepts an upload in several PATCH chunks', async () => {
const bytes = new Uint8Array(randomBytes(96 * 1024));
const digest = digestOf(bytes);
const started = await fetch(`${BASE}/v2/chunky/blobs/uploads/`, {
method: 'POST',
headers: { Authorization: AUTH },
});
let location = started.headers.get('location');
const half = 48 * 1024;
const first = await fetch(`${BASE}${location}`, {
method: 'PATCH',
headers: { Authorization: AUTH, 'Content-Range': `0-${half - 1}` },
body: bytes.subarray(0, half),
});
expect(first.status).toBe(202);
location = first.headers.get('location');
const stale = await fetch(`${BASE}${location}`, {
method: 'PATCH',
headers: { Authorization: AUTH, 'Content-Range': `0-${half - 1}` },
body: bytes.subarray(0, half),
});
expect(stale.status).toBe(416);
const second = await fetch(`${BASE}${location}`, {
method: 'PATCH',
headers: {
Authorization: AUTH,
'Content-Range': `${half}-${bytes.length - 1}`,
},
body: bytes.subarray(half),
});
expect(second.status).toBe(202);
const put = await fetch(
`${BASE}${second.headers.get('location')}?digest=${digest}`,
{ method: 'PUT', headers: { Authorization: AUTH } },
);
expect(put.status).toBe(201);
const head = await fetch(`${BASE}/v2/chunky/blobs/${digest}`, {
method: 'HEAD',
});
expect(head.status).toBe(200);
});
it('refuses a digest the bytes do not hash to', async () => {
const started = await fetch(`${BASE}/v2/app/blobs/uploads/`, {
method: 'POST',
headers: { Authorization: AUTH },
});
const location = started.headers.get('location');
const wrong = `sha256:${'0'.repeat(64)}`;
const put = await fetch(`${BASE}${location}?digest=${wrong}`, {
method: 'PUT',
headers: { Authorization: AUTH },
body: new Uint8Array(randomBytes(64)),
});
expect(put.status).toBe(400);
const body = await put.json();
expect(body.errors[0].code).toBe('DIGEST_INVALID');
});
it('refuses a manifest whose blobs are missing', async () => {
const orphan = imageManifest({
config: new Uint8Array(randomBytes(16)),
layers: [new Uint8Array(randomBytes(16))],
});
const res = await pushManifest(
'app',
'broken',
orphan,
MEDIA_TYPES.ociManifest,
);
expect(res.status).toBe(404);
const body = await res.json();
expect(body.errors[0].code).toBe('MANIFEST_BLOB_UNKNOWN');
});
it('pushes and serves a multi-arch index', async () => {
// The child goes up by digest first, the way buildx pushes.
const child = await pushManifest(
'multi',
manifestDigest,
manifestBytes,
MEDIA_TYPES.ociManifest,
);
expect(child.status).toBe(201);
const indexBytes = new TextEncoder().encode(
JSON.stringify({
schemaVersion: 2,
mediaType: MEDIA_TYPES.ociIndex,
manifests: [
{
mediaType: MEDIA_TYPES.ociManifest,
digest: manifestDigest,
size: manifestBytes.length,
platform: { architecture: 'amd64', os: 'linux' },
},
],
}),
);
const put = await pushManifest(
'multi',
'latest',
indexBytes,
MEDIA_TYPES.ociIndex,
);
expect(put.status).toBe(201);
const res = await fetch(`${BASE}/v2/multi/manifests/latest`);
expect(res.headers.get('content-type')).toBe(MEDIA_TYPES.ociIndex);
expect(new Uint8Array(await res.arrayBuffer())).toEqual(indexBytes);
});
it('reports upload status and honours upload DELETE', async () => {
const started = await fetch(`${BASE}/v2/app/blobs/uploads/`, {
method: 'POST',
headers: { Authorization: AUTH },
});
const location = started.headers.get('location');
const status = await fetch(`${BASE}${location}`, {
headers: { Authorization: AUTH },
});
expect(status.status).toBe(204);
expect(status.headers.get('range')).toBe('0-0');
const cancel = await fetch(`${BASE}${location}`, {
method: 'DELETE',
headers: { Authorization: AUTH },
});
expect(cancel.status).toBe(204);
const gone = await fetch(`${BASE}${location}`, {
method: 'PATCH',
headers: { Authorization: AUTH },
body: new Uint8Array(8),
});
expect(gone.status).toBe(404);
});
it('deletes a tag, keeping the manifest', async () => {
await pushManifest('multi', 'temp', manifestBytes, MEDIA_TYPES.ociManifest);
const res = await fetch(`${BASE}/v2/multi/manifests/temp`, {
method: 'DELETE',
headers: { Authorization: AUTH },
});
expect(res.status).toBe(202);
const tags = await fetch(`${BASE}/v2/multi/tags/list`);
expect((await tags.json()).tags).toEqual(['latest']);
const manifest = await fetch(
`${BASE}/v2/multi/manifests/${manifestDigest}`,
);
expect(manifest.status).toBe(200);
});
it('summarizes hosted images for the account page', async () => {
const signIn = await fetch(`${BASE}/account/sign-in`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({ username: DID, password: PASSWORD }),
redirect: 'manual',
});
const cookie = (signIn.headers.get('set-cookie') || '').split(';')[0];
expect(cookie).toContain('pdsjs_account=');
const overview = await fetch(`${BASE}/account/api/overview`, {
headers: { Cookie: cookie },
});
const account = await overview.json();
expect(account.features.oci).toBe(true);
expect(account.stats.ociRepos).toBeGreaterThanOrEqual(1);
const res = await fetch(`${BASE}/account/api/oci/repos`, {
headers: { Cookie: cookie },
});
expect(res.ok).toBe(true);
const body = await res.json();
expect(body.enabled).toBe(true);
const app = body.repos.find(
(/** @type {{name: string}} */ r) => r.name === 'app',
);
expect(app.tags).toEqual([{ tag: 'latest', digest: manifestDigest }]);
expect(app.manifests).toBe(1);
expect(app.layers).toBe(3);
expect(app.size).toBeGreaterThan(
layerOne.length + layerTwo.length + configBytes.length,
);
expect(app.pullRef).toBe(`127.0.0.1:${PORT}/app`);
const anonymous = await fetch(`${BASE}/account/api/oci/repos`);
expect(anonymous.status).toBe(401);
});
it('answers 404 for unknown names and manifests', async () => {
const name = await fetch(`${BASE}/v2/absent/manifests/latest`);
expect(name.status).toBe(404);
expect((await name.json()).errors[0].code).toBe('NAME_UNKNOWN');
const manifest = await fetch(`${BASE}/v2/app/manifests/nope`);
expect(manifest.status).toBe(404);
expect((await manifest.json()).errors[0].code).toBe('MANIFEST_UNKNOWN');
const blob = await fetch(`${BASE}/v2/app/blobs/sha256:${'f'.repeat(64)}`);
expect(blob.status).toBe(404);
expect((await blob.json()).errors[0].code).toBe('BLOB_UNKNOWN');
});
it('refuses a name outside the grammar', async () => {
const res = await fetch(`${BASE}/v2/Upper/manifests/latest`);
expect(res.status).toBe(400);
expect((await res.json()).errors[0].code).toBe('NAME_INVALID');
});
});