chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
#!/usr/bin/env node
// Publish the built worker artifact as site content on a PDS.
//
// The artifact from scripts/build-worker-bundle.mjs becomes files in a
// dev.pdsjs.site.deploy record: latest/ points at this release and v<n>/
// keeps every release before it, so an update page can install any of them.
// On the canary channel the artifact lands at canary/ alone, one moving
// pointer with no history and no entry in releases.json. Blobs are
// content-addressed; a re-publish of identical bytes reuses them.
//
// Environment:
// PDS_URL e.g. https://pds.pdsjs.dev
// PDS_IDENTIFIER handle or DID of the publishing account
// PDS_PASSWORD account password or an app password
// SITE_NAME record key and subdomain (default: artifacts)
// ARTIFACT_DIR default: dist/worker
// CHANNEL stable (default) or canary
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const artifactDir = resolve(
process.env.ARTIFACT_DIR ?? join(repoRoot, 'dist', 'worker'),
);
const siteName = process.env.SITE_NAME ?? 'artifacts';
const pdsUrl = process.env.PDS_URL;
const identifier = process.env.PDS_IDENTIFIER;
const password = process.env.PDS_PASSWORD;
if (!pdsUrl || !identifier || !password) {
console.error('Set PDS_URL, PDS_IDENTIFIER and PDS_PASSWORD.');
process.exit(1);
}
const channel = process.env.CHANNEL ?? 'stable';
if (!['stable', 'canary'].includes(channel)) {
console.error(`CHANNEL must be stable or canary, not "${channel}".`);
process.exit(1);
}
const COLLECTION = 'dev.pdsjs.site.deploy';
// The lexicon caps a site blob at 5 MB.
const BLOB_LIMIT = 5 * 1024 * 1024;
const manifestBytes = readFileSync(join(artifactDir, 'manifest.json'));
const moduleBytes = readFileSync(join(artifactDir, 'index.js'));
const { version } = JSON.parse(manifestBytes.toString());
if (!version) throw new Error('manifest.json names no version');
// The suffix marks a canary build; the channels may not cross. A suffixed
// stable publish would mint v<n>/ history and a releases.json entry for a
// build no tag names, and an unsuffixed canary publish would offer a
// version the update check cannot tell from the release it shadows.
if (channel === 'canary' && !version.includes('-')) {
throw new Error(`a canary publish needs a suffixed version, not ${version}`);
}
if (channel === 'stable' && version.includes('-')) {
throw new Error(`a stable publish needs a release version, not ${version}`);
}
if (moduleBytes.length > BLOB_LIMIT) {
throw new Error(
`index.js is ${moduleBytes.length} bytes; the site lexicon caps a blob at ${BLOB_LIMIT}`,
);
}
/** @param {string} path @param {RequestInit} init */
async function xrpc(path, init = {}) {
const response = await fetch(`${pdsUrl}/xrpc/${path}`, init);
const body = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(
`${path}: ${response.status} ${body.message ?? body.error ?? ''}`,
);
}
return body;
}
const session = await xrpc('com.atproto.server.createSession', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ identifier, password }),
});
const auth = { Authorization: `Bearer ${session.accessJwt}` };
/** @param {Uint8Array} bytes @param {string} contentType */
async function uploadBlob(bytes, contentType) {
const { blob } = await xrpc('com.atproto.repo.uploadBlob', {
method: 'POST',
headers: { ...auth, 'Content-Type': contentType },
body: bytes,
});
return blob;
}
const moduleBlob = await uploadBlob(
moduleBytes,
'text/javascript; charset=utf-8',
);
const manifestBlob = await uploadBlob(manifestBytes, 'application/json');
/**
* The commit the canary channel currently serves, read from the record this
* publish is about to replace. The record is the source the site renders, and
* reading it needs no guess at the domain the site answers on. Null when
* nothing is published yet, or when the blob cannot be read: neither is a
* reason to refuse a publish.
* @param {Array<{path: string, blob?: {ref?: {$link?: string}}}>} files
* @returns {Promise<string|null>}
*/
async function publishedCanaryCommit(files) {
const cid = files.find((f) => f.path === 'canary/manifest.json')?.blob?.ref
?.$link;
if (!cid) return null;
try {
const response = await fetch(
`${pdsUrl}/xrpc/com.atproto.sync.getBlob` +
`?did=${encodeURIComponent(session.did)}&cid=${encodeURIComponent(cid)}`,
);
if (!response.ok) return null;
const { version: live } = await response.json();
return typeof live === 'string'
? (live.split('-canary.')[1] ?? null)
: null;
} catch {
return null;
}
}
/**
* Whether `older` is an ancestor of `newer` in this checkout's history.
*
* git answers 1 for "no", and anything else means it could not answer at all:
* a commit this checkout does not have, or a repository it refuses to read.
* Both let the publish through, so the guard never blocks a release it cannot
* reason about. The second says the guard is not working and is worth saying
* out loud, because a guard that silently never fires reads as a guard.
* @param {string} older @param {string} newer
* @returns {boolean}
*/
function isAncestor(older, newer) {
try {
execFileSync('git', ['merge-base', '--is-ancestor', older, newer], {
cwd: repoRoot,
stdio: 'ignore',
});
return true;
} catch (error) {
const status = /** @type {{status?: number}} */ (error).status;
if (status !== 1) {
console.warn(
`Could not compare ${older} with ${newer}: git exited ${status}.`,
);
console.warn('Publishing without the ordering check.');
}
return false;
}
}
// The current record carries every prior release; this publish keeps them.
let existingFiles = [];
try {
const current = await xrpc(
`com.atproto.repo.getRecord?repo=${encodeURIComponent(session.did)}` +
`&collection=${COLLECTION}&rkey=${encodeURIComponent(siteName)}`,
);
existingFiles = current.value?.files ?? [];
} catch {
// No record yet: the first publish creates it.
}
// The canary channel is one moving pointer, and every push to main races for
// it: a slow run for an older commit finishing last would move the channel
// backwards, and the update check reads any difference as an update, so a
// server ahead of the pointer is offered a downgrade. Git decides which of
// two commits is older, so a publish that is an ancestor of what is already
// there stops here. FORCE=1 is the way back after a revert the history does
// not describe.
if (channel === 'canary') {
const published = await publishedCanaryCommit(existingFiles);
const incoming = version.split('-canary.')[1];
if (published && incoming && published !== incoming) {
if (isAncestor(incoming, published)) {
if (process.env.FORCE === '1') {
console.log(
`canary/ holds ${published}, newer than ${incoming}; FORCE=1 publishes anyway.`,
);
} else {
console.log(
`canary/ already holds ${published}, which has ${incoming} behind it.`,
);
console.log('Publishing would move the channel backwards. Skipping.');
console.log('Set FORCE=1 to publish this commit regardless.');
process.exit(0);
}
}
}
}
const IMMUTABLE = 'public, max-age=31536000, immutable';
const SHORT = 'public, max-age=300';
/** @type {{path: string, blob: unknown, cacheControl: string}[]} */
let entries;
if (channel === 'canary') {
entries = [
{ path: 'canary/index.js', blob: moduleBlob, cacheControl: SHORT },
{ path: 'canary/manifest.json', blob: manifestBlob, cacheControl: SHORT },
];
} else {
// releases.json accumulates what the record cannot say: when each release
// was published. The directory page renders from it.
/** @type {{version: string, publishedAt: string, moduleSize: number}[]} */
let releases = [];
const releasesEntry = existingFiles.find((f) => f.path === 'releases.json');
const releasesCid = releasesEntry?.blob?.ref?.$link;
if (releasesCid) {
try {
const response = await fetch(
`${pdsUrl}/xrpc/com.atproto.sync.getBlob` +
`?did=${encodeURIComponent(session.did)}&cid=${encodeURIComponent(releasesCid)}`,
);
if (response.ok) releases = await response.json();
} catch {
// A fresh list rebuilds from this release forward.
}
}
releases = [
...releases.filter((r) => r.version !== version),
{
version,
publishedAt: new Date().toISOString(),
moduleSize: moduleBytes.length,
},
];
// Newest first, numerically by dotted parts.
releases.sort((a, b) => {
const pa = a.version.split('.').map(Number);
const pb = b.version.split('.').map(Number);
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
if ((pa[i] || 0) !== (pb[i] || 0)) return (pb[i] || 0) - (pa[i] || 0);
}
return 0;
});
/** @param {number} bytes */
const formatSize = (bytes) => `${(bytes / 1024 / 1024).toFixed(1)} MB`;
// The project mark, inlined for the header and published as a file for the
// favicon. The letters are outlines, so the mark does not depend on the
// viewer's fonts.
const logoSvg = readFileSync(
join(repoRoot, 'scripts', 'pds-logo.svg'),
'utf8',
);
const logoInline = logoSvg.replace(
'width="630" height="630"',
'width="48" height="48"',
);
const hasCanary = existingFiles.some(
(f) => f.path === 'canary/manifest.json',
);
const indexHtml = `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>pds.js worker artifacts</title>
<link rel="icon" type="image/svg+xml" href="/logo.svg">
<style>
body { font-family: ui-monospace, monospace; max-width: 44rem;
margin: 3rem auto; padding: 0 1.5rem; color: #1a1a18; background: #fdfdfc; }
h1 { font-size: 1.1rem; margin: 1.5rem 0; }
table { border-collapse: collapse; width: 100%; }
td, th { text-align: left; padding: .4rem .8rem .4rem 0; font-weight: normal;
border-bottom: 1px solid #e4e4df; font-size: .9rem; }
th { color: #6b6b66; }
a { color: inherit; }
p { font-size: .85rem; color: #6b6b66; line-height: 1.5; }
</style>
</head>
<body>
${logoInline}
<h1>Cloudflare worker artifacts</h1>
<table>
<tr><th>version</th><th>published</th><th>size</th><th></th><th></th></tr>
${releases
.map(
(r) =>
`<tr><td>${r.version}</td>` +
`<td>${r.publishedAt.slice(0, 10)}</td>` +
`<td>${formatSize(r.moduleSize)}</td>` +
`<td><a href="/v${r.version}/index.js">module</a></td>` +
`<td><a href="/v${r.version}/manifest.json">manifest</a></td></tr>`,
)
.join('\n')}
</table>
<p><a href="/latest/manifest.json">latest/</a> points at ${releases[0].version}.
${hasCanary ? '<a href="/canary/manifest.json">canary/</a> tracks the newest build of the main branch.\n' : ''}Each release is a self-contained Cloudflare worker module and a manifest
naming its compatibility settings and bindings.
<a href="https://start.pdsjs.dev">start.pdsjs.dev</a> installs it into your
own account. Built by the release pipeline from
<a href="https://tangled.org/chadtmiller.com/pds.js">pds.js</a>.</p>
</body>
</html>
`;
const indexBlob = await uploadBlob(
new TextEncoder().encode(indexHtml),
'text/html; charset=utf-8',
);
const releasesBlob = await uploadBlob(
new TextEncoder().encode(JSON.stringify(releases, null, 2)),
'application/json',
);
const logoBlob = await uploadBlob(
new TextEncoder().encode(logoSvg),
'image/svg+xml',
);
// latest/ names the highest release, which is what releases[0] is and what
// the directory page claims. An older tag publishing again is a backfill of
// its own v<n>/ directory, so it leaves the pointer alone: writing it would
// offer every server ahead of that version a downgrade. A runner that lost
// its state file reruns the tag workflow for every tag it sees, so this is
// reachable without anyone asking for it. FORCE=1 moves the pointer anyway.
const newest = releases[0]?.version === version;
if (!newest && process.env.FORCE !== '1') {
console.log(
`latest/ holds ${releases[0]?.version}, which is above ${version}.`,
);
console.log(`Publishing v${version}/ alone. Set FORCE=1 to move latest/.`);
}
entries = [
{ path: `v${version}/index.js`, blob: moduleBlob, cacheControl: IMMUTABLE },
{
path: `v${version}/manifest.json`,
blob: manifestBlob,
cacheControl: IMMUTABLE,
},
...(newest || process.env.FORCE === '1'
? [
{ path: 'latest/index.js', blob: moduleBlob, cacheControl: SHORT },
{
path: 'latest/manifest.json',
blob: manifestBlob,
cacheControl: SHORT,
},
]
: []),
{ path: 'index.html', blob: indexBlob, cacheControl: SHORT },
{ path: 'releases.json', blob: releasesBlob, cacheControl: SHORT },
{ path: 'logo.svg', blob: logoBlob, cacheControl: SHORT },
];
}
const replaced = new Set(entries.map((e) => e.path));
const files = [
...existingFiles.filter((f) => !replaced.has(f.path)),
...entries,
];
await xrpc('com.atproto.repo.putRecord', {
method: 'POST',
headers: { ...auth, 'Content-Type': 'application/json' },
body: JSON.stringify({
repo: session.did,
collection: COLLECTION,
rkey: siteName,
record: {
$type: COLLECTION,
files,
createdAt: new Date().toISOString(),
},
}),
});
console.log(
`Published pds.js ${version} to the "${siteName}" site's ${channel} ` +
`channel (${files.length} files in the record, module ` +
`${moduleBytes.length} bytes).`,
);