chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
/**
* Read-only browsing of a hosted git repository for the account page: refs,
* directory listings, and file contents, served straight from the bundle
* chain in blob storage via the pack object reader. Workers-clean; all PDS
* access goes through an injected fetch, typically the server's own fetch.
*
* A chain whose bundles all carry an object index is read lazily: the
* reader downloads the small index blobs, then fetches single objects with
* ranged blob reads. A chain without indexes downloads whole and unpacks in
* memory, which is also the fallback when a server ignores Range requests'
* semantics entirely (a 200 answer still resolves, sliced locally).
*/
import { parseBundle } from './bundle.js';
import { diffLines } from './diff.js';
import { GIT_REPO_COLLECTION } from './lexicon.js';
import { commitTree, indexPacks, parseCommit, parseTree } from './objects.js';
import { concatChunks, indexPartsOf, parseRepoRecord } from './record.js';
import { createLazyStore } from './store.js';
// A page that shows a file from a repository needs to know a pointer from a
// picture, and this is the module such a page already reads.
export { lfsPointer } from './lfs.js';
/** @typedef {import('./lfs.js').LfsPointer} LfsPointer */
/** @typedef {import('./store.js').ObjectStore} ObjectStore */
/** Text preview cap; larger files report size only. */
const MAX_PREVIEW_BYTES = 512 * 1024;
/**
* How far a whole-tree listing walks. The walk runs over the pack index
* rather than file bodies, so this is a guard against a pathological
* repository rather than a budget.
*/
const MAX_LISTED_FILES = 50000;
/**
* The widest look-ahead a path walk grows to. A run this size resolves the
* path for every commit in it, so the cap bounds what one round trip reads.
*/
const MAX_RUN = 4096;
/** How far back a path's history is walked before it is called truncated. */
const PATH_WALK_LIMIT = 4000;
/**
* Image media types the browser serves inline, by lowercase extension. SVG is
* absent on purpose: it can carry script, and the account page serves these
* bytes from its own origin.
*/
const IMAGE_MEDIA_TYPES = {
apng: 'image/apng',
avif: 'image/avif',
bmp: 'image/bmp',
gif: 'image/gif',
ico: 'image/x-icon',
jpeg: 'image/jpeg',
jpg: 'image/jpeg',
png: 'image/png',
webp: 'image/webp',
};
/**
* Direct storage access for repositories held in a permissioned space. The
* account page is the owner's own surface, so the platform reads space
* records and blobs from storage rather than through the authenticated
* space endpoints.
* @typedef {Object} SpaceReader
* @property {(space: string, rkey: string) => Promise<{cid: string, value: unknown}|null>} getRecord
* @property {(cid: string) => Promise<Uint8Array|null>} getBlob
* @property {(cid: string, start: number, end: number) => Promise<Uint8Array|null>} [getBlobRange] -
* Optional: one byte range of a blob, end exclusive. A reader that
* answers ranges lets a space repository read lazily, as a public one
* does. Absent, the chain is read whole.
*/
/**
* @typedef {Object} GitBrowserContext
* @property {(request: Request) => Promise<Response>} xrpc
* @property {SpaceReader} [spaceReader]
* @property {string} [collection]
* @property {(progress: {loaded: number, total: number}) => void} [onProgress] -
* called as an open's downloads arrive, so a caller can show a repository
* being read. The total comes from the record and is known before the
* first byte: the whole chain for an eager open, the index blobs for a
* lazy one.
* @property {number} [cacheSize] - how many opened repositories to keep (default 1). A server answering for many repositories keeps a few.
* @property {number} [maxEagerBytes] - the largest bundle chain to download whole. A repository without indexes past it throws an error with code RepositoryTooLarge rather than opening. Absent, any size opens.
*/
/**
* @param {Uint8Array} bytes
* @returns {boolean} whether the content looks binary (NUL in the first 8KB)
*/
function looksBinary(bytes) {
const probe = bytes.subarray(0, 8192);
return probe.includes(0);
}
/**
* @param {Map<string, import('./objects.js').GitObject>} objects
* @returns {ObjectStore}
*/
function createEagerStore(objects) {
return {
meta(sha) {
const object = objects.get(sha);
return object ? { type: object.type, size: object.data.length } : null;
},
peek(sha) {
return objects.get(sha) ?? null;
},
async get(sha) {
return objects.get(sha) ?? null;
},
async getMany() {},
async prefetchCommits() {
/** @type {string[]} */
const shas = [];
for (const [sha, object] of objects) {
if (object.type === 'commit' || object.type === 'tag') shas.push(sha);
}
return shas;
},
};
}
/**
* A tag chain peeled down to its commit id.
* @param {ObjectStore} store
* @param {string} sha
* @returns {Promise<string>}
*/
async function peelToCommit(store, sha) {
let current = sha;
for (let hops = 0; hops < 10; hops++) {
const meta = store.meta(current);
if (!meta) throw new Error(`object ${current} not found in packs`);
if (meta.type === 'commit') return current;
if (meta.type !== 'tag') {
throw new Error(`ref points at a ${meta.type}, not a commit`);
}
const tag = await store.get(current);
if (!tag) throw new Error(`object ${current} not found in packs`);
const text = new TextDecoder().decode(tag.data);
const match = /^object ([0-9a-f]{40})/m.exec(text);
if (!match) throw new Error('tag has no object line');
current = match[1];
}
throw new Error('tag chain too deep');
}
/**
* A ref target peeled down to the tree it names.
* @param {ObjectStore} store
* @param {string} sha
* @returns {Promise<string>}
*/
async function peelToTree(store, sha) {
if (store.meta(sha)?.type === 'tree') return sha;
const commitSha = await peelToCommit(store, sha);
const commit = await store.get(commitSha);
if (!commit) throw new Error(`object ${commitSha} not found in packs`);
return commitTree(commit.data);
}
/**
* One side of a file change, read out of the object store. An absent entry
* reads as empty text with no size, which is how an added or deleted file
* gets a diff against nothing. A submodule has no blob in the pack, so its
* text is null and it reports no hunks. A blob past the preview cap keeps
* its size and skips its bytes; without them the binary probe answers
* false, unless the bytes are already in memory.
*
* @typedef {Object} BlobSide
* @property {string|null} text
* @property {number|null} size
* @property {boolean} binary
* @property {Uint8Array|null} bytes
*
* @param {ObjectStore} store
* @param {import('./objects.js').TreeEntry|null} entry
* @returns {BlobSide}
*/
function blobContent(store, entry) {
if (!entry) return { text: '', size: null, binary: false, bytes: null };
if (entry.type === 'submodule') {
return { text: null, size: null, binary: false, bytes: null };
}
const meta = store.meta(entry.sha);
if (meta?.type !== 'blob') {
return { text: null, size: null, binary: false, bytes: null };
}
const object = store.peek(entry.sha);
if (!object) {
return { text: null, size: meta.size, binary: false, bytes: null };
}
const binary = looksBinary(object.data);
return {
text: binary ? null : new TextDecoder().decode(object.data),
size: object.data.length,
binary,
bytes: object.data,
};
}
/**
* The entries of a tree by name. A missing or non-tree id reads as empty,
* which is how the root commit compares against nothing.
* @param {ObjectStore} store
* @param {string|null} treeSha
* @returns {Promise<Map<string, import('./objects.js').TreeEntry>>}
*/
async function entriesByName(store, treeSha) {
/** @type {Map<string, import('./objects.js').TreeEntry>} */
const entries = new Map();
if (!treeSha) return entries;
const tree = await store.get(treeSha);
if (tree?.type !== 'tree') return entries;
for (const entry of parseTree(tree.data)) entries.set(entry.name, entry);
return entries;
}
/**
* Record one side of a path's change, keeping whatever the other side put
* there.
* @param {Map<string, {from: import('./objects.js').TreeEntry|null, to: import('./objects.js').TreeEntry|null}>} changes
* @param {string} path
* @param {'from'|'to'} side
* @param {import('./objects.js').TreeEntry} entry
*/
function recordSide(changes, path, side, entry) {
const change = changes.get(path) ?? { from: null, to: null };
change[side] = entry;
changes.set(path, change);
}
/**
* Collect the paths that differ between two trees. Two subtrees with the
* same object id hold the same bytes, so the walk skips them without
* reading either one. The walk goes level by level, each level's trees
* fetched in one batch. A pair with one side null lists everything under
* the other side, which serves added and deleted directories.
* @param {ObjectStore} store
* @param {string|null} oldTree
* @param {string|null} newTree
* @param {Map<string, {from: import('./objects.js').TreeEntry|null, to: import('./objects.js').TreeEntry|null}>} changes
*/
async function walkTreeDiff(store, oldTree, newTree, changes) {
let level = [{ from: oldTree, to: newTree, prefix: '' }];
while (level.length > 0) {
/** @type {string[]} */
const treeShas = [];
for (const pair of level) {
if (pair.from) treeShas.push(pair.from);
if (pair.to) treeShas.push(pair.to);
}
await store.getMany(treeShas);
/** @type {typeof level} */
const next = [];
for (const { from: oldSha, to: newSha, prefix } of level) {
if (oldSha === newSha) continue;
const before = await entriesByName(store, oldSha);
const after = await entriesByName(store, newSha);
for (const name of new Set([...before.keys(), ...after.keys()])) {
const from = before.get(name) ?? null;
const to = after.get(name) ?? null;
const path = prefix ? `${prefix}/${name}` : name;
if (from && to && from.sha === to.sha && from.mode === to.mode) {
continue;
}
const fromDir = from?.type === 'dir';
const toDir = to?.type === 'dir';
if (fromDir || toDir) {
// A name that is a directory on one side and a file on the other
// becomes every file under the directory plus the file itself.
next.push({
from: fromDir ? /** @type {string} */ (from?.sha) : null,
to: toDir ? /** @type {string} */ (to?.sha) : null,
prefix: path,
});
}
if (from && !fromDir) recordSide(changes, path, 'from', from);
if (to && !toDir) recordSide(changes, path, 'to', to);
}
}
level = next;
}
}
/**
* @param {string} path
* @returns {string|null} the image media type for the file extension, or null
* for any other file
*/
function imageMediaType(path) {
const name = path.slice(path.lastIndexOf('/') + 1).toLowerCase();
const dot = name.lastIndexOf('.');
if (dot <= 0) return null;
return (
/** @type {Record<string, string>} */ (IMAGE_MEDIA_TYPES)[
name.slice(dot + 1)
] ?? null
);
}
/**
* Turn a tree diff into the file list a diff view renders: statuses, modes,
* sizes and unified hunks. A file past the preview cap, a binary file and a
* submodule report null hunks rather than their contents.
* @param {ObjectStore} store
* @param {Map<string, {from: import('./objects.js').TreeEntry|null, to: import('./objects.js').TreeEntry|null}>} changes
*/
async function buildFiles(store, changes) {
// Every blob the diff shows, in one batch. A side past the preview cap
// diffs as null hunks, so its bytes stay unfetched.
/** @type {string[]} */
const blobShas = [];
for (const { from, to } of changes.values()) {
for (const entry of [from, to]) {
if (!entry || entry.type === 'submodule') continue;
const meta = store.meta(entry.sha);
if (meta?.type === 'blob' && meta.size <= MAX_PREVIEW_BYTES) {
blobShas.push(entry.sha);
}
}
}
await store.getMany(blobShas);
return [...changes.entries()]
.sort((a, b) => a[0].localeCompare(b[0]))
.map(([path, { from, to }]) => {
const before = blobContent(store, from);
const after = blobContent(store, to);
const oversized =
(before.size ?? 0) > MAX_PREVIEW_BYTES ||
(after.size ?? 0) > MAX_PREVIEW_BYTES;
const hunks =
before.text === null || after.text === null || oversized
? null
: diffLines(before.text, after.text);
let additions = 0;
let deletions = 0;
for (const hunk of hunks ?? []) {
for (const line of hunk.lines) {
if (line.type === 'add') additions++;
else if (line.type === 'delete') deletions++;
}
}
const mediaType = imageMediaType(path);
return {
path,
status: /** @type {'added'|'deleted'|'modified'} */ (
!from ? 'added' : !to ? 'deleted' : 'modified'
),
oldMode: from?.mode ?? null,
newMode: to?.mode ?? null,
oldSize: before.size,
newSize: after.size,
binary: before.binary || after.binary,
mediaType,
oldImage: mediaType ? before.bytes : null,
newImage: mediaType ? after.bytes : null,
hunks,
// The page colours each side whole. A hunk is a fragment, and a
// grammar that reads ahead, as highlight.js does for JSX, prints
// a tag it cannot see the end of plain.
oldText: hunks ? before.text : null,
newText: hunks ? after.text : null,
additions,
deletions,
};
});
}
/**
* How far back a base's ancestry is marked before a range walk gives up on
* excluding it. A change deeper than this is outside personal scale, and the
* walk then reports what it reached rather than nothing.
*/
const MAX_EXCLUDE = 10_000;
/** How many of a change's commits a range read counts before it stops. */
const MAX_RANGE_COMMITS = 1_000;
/**
* The commits one change carries: reachable from the tip, not from the base.
*
* Both walks run over commits already in memory, so the cost is the prefetch
* and nothing after it.
* @param {ObjectStore} store
* @param {string|null} base
* @param {string} tip
* @param {number} limit
*/
async function walkRange(store, base, tip, limit) {
await store.prefetchCommits();
/** @param {string} sha */
const info = (sha) => {
const object = store.peek(sha);
if (object?.type !== 'commit') return null;
return parseCommit(object.data);
};
// Everything the base reaches, which the change by definition does not.
const excluded = new Set();
if (base) {
const frontier = [base];
while (frontier.length > 0 && excluded.size < MAX_EXCLUDE) {
const sha = /** @type {string} */ (frontier.pop());
if (excluded.has(sha)) continue;
excluded.add(sha);
for (const parent of info(sha)?.parents ?? []) frontier.push(parent);
}
}
/** @type {string[]} */
const frontier = [tip];
const seen = new Set(frontier);
const commits = [];
while (frontier.length > 0 && commits.length < limit) {
// Newest first, the way a log reads.
let next = 0;
for (let i = 1; i < frontier.length; i++) {
const here = info(frontier[i]);
const best = info(frontier[next]);
if ((here?.committedAt ?? 0) > (best?.committedAt ?? 0)) next = i;
}
const sha = frontier.splice(next, 1)[0];
if (excluded.has(sha)) continue;
const commit = info(sha);
if (!commit) continue;
commits.push({
sha,
parents: commit.parents,
author: commit.author,
authoredAt: commit.authoredAt,
committedAt: commit.committedAt,
message: commit.message,
});
for (const parent of commit.parents) {
if (!seen.has(parent)) {
seen.add(parent);
frontier.push(parent);
}
}
}
return { commits, truncated: frontier.some((sha) => !excluded.has(sha)) };
}
/**
* The tree diff between two commits. A null base compares against the empty
* tree, which lists everything the tip holds.
* @param {ObjectStore} store
* @param {string|null} base
* @param {string} tip
*/
async function rangeChanges(store, base, tip) {
/** @param {string} sha */
const treeOf = async (sha) => {
const object = await store.get(sha);
if (object?.type !== 'commit') throw new Error(`no such commit: ${sha}`);
return commitTree(object.data);
};
/** @type {Map<string, {from: import('./objects.js').TreeEntry|null, to: import('./objects.js').TreeEntry|null}>} */
const changes = new Map();
await walkTreeDiff(
store,
base ? await treeOf(base) : null,
await treeOf(tip),
changes,
);
return changes;
}
/**
* The commits that changed one path, walking back from a commit, newest
* first. A commit changed the path when what stands there differs from what
* stood there in every one of its parents. A commit that matches one of them
* changed nothing, and the walk carries on down that parent, which is what
* credits a merge's change to the branch it came from. After a change the
* walk carries on down the first parent, for the change before it.
*
* The walk reads trees rather than diffs, and remembers what it finds: a
* path untouched for a hundred commits resolves to the same object in all of
* them, so a change costs a handful of reads however far back it was made.
* @param {import('./store.js').ObjectStore} store
* @param {string} start - the commit the walk begins at
* @param {string} path
* @param {number} limit - commits to walk before giving up
* @param {number} want - changes to collect before stopping
* @returns {Promise<{commits: Array<import('./objects.js').CommitInfo & {sha: string}>, truncated: boolean}>}
* truncated where the walk stopped with history still to read
*/
async function pathHistory(store, start, path, limit, want) {
const segments = path.split('/').filter(Boolean);
/** What stands at the path in one commit, by the commit's own id. */
/** @type {Map<string, string|null>} */
const standing = new Map();
/** Entries by tree, so a tree shared by many commits is read once. */
/** @type {Map<string, import('./objects.js').TreeEntry[]>} */
const parsed = new Map();
const entriesOf = (/** @type {string} */ treeSha) => {
let entries = parsed.get(treeSha);
if (!entries) {
const object = store.peek(treeSha);
entries = object?.type === 'tree' ? parseTree(object.data) : [];
parsed.set(treeSha, entries);
}
return entries;
};
/**
* Resolve the path for a run of commits at once, a level of the path at a
* time. Every commit's root tree is read in one coalesced batch, then every
* directory below it, so the walk costs a read per level rather than a
* read per commit per level.
* @param {string[]} commitShas
*/
const resolve = async (commitShas) => {
const wanted = commitShas.filter((sha) => !standing.has(sha));
if (wanted.length === 0) return;
await store.getMany(wanted);
/** Where each commit has got to, as the walk descends. */
/** @type {Map<string, string>} */
let here = new Map();
for (const sha of wanted) {
const object = store.peek(sha);
if (object?.type === 'commit') here.set(sha, commitTree(object.data));
else standing.set(sha, null);
}
for (const segment of segments) {
await store.getMany(new Set(here.values()));
/** @type {Map<string, string>} */
const next = new Map();
for (const [sha, treeSha] of here) {
const found = entriesOf(treeSha).find((e) => e.name === segment);
if (found) next.set(sha, found.sha);
else standing.set(sha, null);
}
here = next;
}
for (const [sha, found] of here) standing.set(sha, found);
};
/** @type {Array<import('./objects.js').CommitInfo & {sha: string}>} */
const found = [];
const done = (/** @type {boolean} */ truncated) => ({
commits: found,
truncated,
});
/**
* How many commits to look ahead down the first parents at a time. Each
* run costs one round trip per level of the path, so the run doubles as
* the walk goes deeper. A path changed near the tip pays one small run; a
* path changed at the root costs a round trip per doubling rather than one
* per 64 commits.
*/
let runSize = 64;
let current = start;
let walked = 0;
while (current && walked < limit) {
// The commits are all in memory, so the run costs no reads to collect;
// only resolving the path for it does.
/** @type {string[]} */
const run = [];
let cursor = current;
while (cursor && run.length < runSize) {
run.push(cursor);
const object = store.peek(cursor) ?? (await store.get(cursor));
if (object?.type !== 'commit') break;
cursor = parseCommit(object.data).parents[0] ?? '';
}
await resolve(cursor ? [...run, cursor] : run);
let redirected = false;
for (let index = 0; index < run.length && walked < limit; index++) {
const sha = run[index];
walked++;
const object = store.peek(sha);
if (object?.type !== 'commit') return done(false);
const commit = parseCommit(object.data);
const here = standing.get(sha) ?? null;
// The path is absent here, so nothing older can have changed it.
if (here === null) return done(false);
// A commit that leaves the path as one of its parents left it did not
// change it, and the walk carries on down that parent. Only a commit
// that differs from all of them made the change. That is what credits
// a merge's change to the commit on the branch it came from rather
// than to the merge.
let unchanged = null;
for (const parent of commit.parents) {
if (!standing.has(parent)) await resolve([parent]);
if (standing.get(parent) === here) {
unchanged = parent;
break;
}
}
if (unchanged === null) {
found.push({ ...commit, sha });
const before = commit.parents[0] ?? '';
if (found.length >= want) return done(Boolean(before));
if (!before) return done(false);
current = before;
} else {
current = unchanged;
}
// A merge sent the walk down a parent this run does not hold, so the
// next run starts from there.
if (current !== run[index + 1]) {
redirected = true;
break;
}
}
if (!redirected && run.length < runSize) return done(false);
runSize = Math.min(runSize * 2, MAX_RUN);
}
return done(walked >= limit);
}
/**
* @param {GitBrowserContext} ctx
*/
export function createGitBrowser(ctx) {
if (!ctx || typeof ctx.xrpc !== 'function') {
throw new Error('createGitBrowser requires an xrpc fetch function');
}
const collection = ctx.collection ?? GIT_REPO_COLLECTION;
/** @param {string} path */
const xrpcGet = (path) =>
ctx.xrpc(new Request(`https://pds.internal${path}`));
/**
* @typedef {{cid: string, repo: import('./record.js').GitRepoRecord, store: ObjectStore}} Opened
*/
// Opened repositories by name and space, each keyed by record cid so a
// push invalidates it. Insertion order is the eviction order: a hit
// re-inserts, so the oldest untouched entry leaves first.
/** @type {Map<string, Opened>} */
const cache = new Map();
const cacheSize = Math.max(1, ctx.cacheSize ?? 1);
/**
* @param {string} did
* @param {string} cid
* @returns {Promise<Uint8Array>}
*/
async function fetchBlob(did, cid) {
const params = new URLSearchParams({ did, cid });
const res = await xrpcGet(`/xrpc/com.atproto.sync.getBlob?${params}`);
if (!res.ok) throw new Error(`bundle blob ${cid} unavailable`);
return new Uint8Array(await res.arrayBuffer());
}
/**
* One byte range of a blob. A 206 answers the range itself; a server
* that ignores Range answers 200 with the whole blob, sliced here so
* both behave the same.
* @param {string} did
* @param {string} cid
* @param {number} start
* @param {number} end - exclusive
* @returns {Promise<Uint8Array>}
*/
async function fetchBlobRange(did, cid, start, end) {
const params = new URLSearchParams({ did, cid });
const res = await ctx.xrpc(
new Request(
`https://pds.internal/xrpc/com.atproto.sync.getBlob?${params}`,
{
headers: { range: `bytes=${start}-${end - 1}` },
},
),
);
if (!res.ok) throw new Error(`bundle blob ${cid} unavailable`);
const bytes = new Uint8Array(await res.arrayBuffer());
if (res.status === 206) return bytes;
return bytes.subarray(start, end);
}
/**
* Blob reads for one open. A public repository reads through the sync
* endpoints; a space repository reads its own storage through the space
* reader, which answers ranges only when the platform gave it a
* getBlobRange.
* @param {string} did
* @param {string|undefined} space
* @returns {{read: (cid: string) => Promise<Uint8Array>, readRange: ((cid: string, start: number, end: number) => Promise<Uint8Array>)|null}}
*/
function blobReaders(did, space) {
const reader = ctx.spaceReader;
if (!space || !reader) {
return {
read: (cid) => fetchBlob(did, cid),
readRange: (cid, start, end) => fetchBlobRange(did, cid, start, end),
};
}
const getBlobRange = reader.getBlobRange;
return {
read: async (cid) => {
const bytes = await reader.getBlob(cid);
if (!bytes) throw new Error(`bundle blob ${cid} unavailable`);
return bytes;
},
readRange: getBlobRange
? async (cid, start, end) => {
const bytes = await getBlobRange(cid, start, end);
if (!bytes) throw new Error(`bundle blob ${cid} unavailable`);
return bytes;
}
: null,
};
}
/**
* Download the whole chain and unpack it in memory.
* @param {import('./record.js').GitRepoRecord} repo
* @param {(cid: string) => Promise<Uint8Array>} fetchPart
* @returns {Promise<ObjectStore>}
*/
async function openEager(repo, fetchPart) {
// Every chunk of every bundle at once. They are independent reads, and
// asking for them one after another spends a round trip on each.
const total = repo.bundles.reduce(
(sum, bundle) =>
sum + bundle.parts.reduce((n, part) => n + (part.size ?? 0), 0),
0,
);
let loaded = 0;
ctx.onProgress?.({ loaded, total });
const fetched = await Promise.all(
repo.bundles.map((bundle) =>
Promise.all(
bundle.parts.map(async (part) => {
const bytes = await fetchPart(part.ref.$link);
loaded += bytes.length;
ctx.onProgress?.({ loaded, total });
return bytes;
}),
),
),
);
const packs = fetched.map(
(chunks) => parseBundle(concatChunks(chunks)).pack,
);
return createEagerStore(await indexPacks(packs));
}
/**
* @param {string} did
* @param {string} rkey
* @param {string} [space] - space AT-URI; the record and blobs then come
* from the space reader rather than the public endpoints
* @returns {Promise<{repo: import('./record.js').GitRepoRecord, store: ObjectStore}|null>}
*/
async function open(did, rkey, space) {
/** @type {{cid: string, value: unknown}|null} */
let body;
if (space) {
if (!ctx.spaceReader) return null;
body = await ctx.spaceReader.getRecord(space, rkey);
} else {
const params = new URLSearchParams({
repo: did,
collection,
rkey: rkey,
});
const res = await xrpcGet(`/xrpc/com.atproto.repo.getRecord?${params}`);
body = res.ok
? /** @type {{cid: string, value: unknown}} */ (await res.json())
: null;
}
if (!body) return null;
const repo = parseRepoRecord(body.value);
const key = `${space ?? ''}\u0000${rkey}`;
const held = cache.get(key);
if (held && held.cid === body.cid) {
cache.delete(key);
cache.set(key, held);
return held;
}
const { read: readBlob, readRange } = blobReaders(did, space);
// Ranged reads need an index for every bundle, and a reader that
// answers ranges.
const lazy =
readRange !== null &&
repo.bundles.length > 0 &&
repo.bundles.every((bundle) => indexPartsOf(bundle).length > 0);
/** @type {ObjectStore|null} */
let store = null;
if (lazy) {
try {
const total = repo.bundles.reduce(
(sum, bundle) =>
sum +
indexPartsOf(bundle).reduce((bytes, part) => bytes + part.size, 0),
0,
);
let loaded = 0;
ctx.onProgress?.({ loaded, total });
store = await createLazyStore(
repo,
async (cid) => {
const bytes = await readBlob(cid);
loaded += bytes.length;
ctx.onProgress?.({ loaded, total });
return bytes;
},
readRange,
);
} catch {
// An index blob that is missing or unreadable costs laziness, not
// the repository.
store = null;
}
}
if (!store) {
const chainBytes = repo.bundles.reduce(
(sum, bundle) =>
sum + bundle.parts.reduce((n, part) => n + (part.size ?? 0), 0),
0,
);
if (ctx.maxEagerBytes !== undefined && chainBytes > ctx.maxEagerBytes) {
throw Object.assign(
new Error(
`repository ${repo.name} has no object indexes and its ${chainBytes} bytes of bundles exceed the ${ctx.maxEagerBytes} this reader downloads whole`,
),
{ code: 'RepositoryTooLarge' },
);
}
store = await openEager(repo, readBlob);
}
/** @type {Opened} */
const opened = { cid: body.cid, repo, store };
cache.delete(key);
cache.set(key, opened);
while (cache.size > cacheSize) {
const oldest = cache.keys().next().value;
if (oldest === undefined) break;
cache.delete(oldest);
}
return opened;
}
/**
* @param {import('./record.js').GitRepoRecord} repo
* @param {string|undefined} ref - short or full ref name; default branch
* when omitted
* @returns {{name: string, sha: string}|null}
*/
function findRef(repo, ref) {
const wanted = ref || repo.defaultBranch || 'main';
return (
repo.refs.find((r) => r.name === wanted) ??
repo.refs.find((r) => r.name === `refs/heads/${wanted}`) ??
repo.refs.find((r) => r.name === `refs/tags/${wanted}`) ??
// An object id names a commit no ref may point at, the way git
// checks one out.
(/^[0-9a-f]{40}$/.test(wanted) ? { name: wanted, sha: wanted } : null)
);
}
/**
* Walk from the ref's root tree down the given directory path.
* @param {ObjectStore} store
* @param {string} rootTree
* @param {string} path - slash-separated, '' for the root
* @returns {Promise<{sha: string, entry: import('./objects.js').TreeEntry|null}>} the
* object at the path plus its tree entry (null at the root)
*/
async function walkPath(store, rootTree, path) {
let sha = rootTree;
/** @type {import('./objects.js').TreeEntry|null} */
let entry = null;
for (const segment of path.split('/').filter(Boolean)) {
const tree = await store.get(sha);
if (tree?.type !== 'tree') {
throw new Error(`not a directory: ${segment}`);
}
const found = parseTree(tree.data).find((e) => e.name === segment);
if (!found) throw new Error(`no such path: ${path}`);
entry = found;
sha = found.sha;
}
return { sha, entry };
}
/**
* Resolve a path to its blob object at a ref.
* @param {{repo: import('./record.js').GitRepoRecord, store: ObjectStore}} opened
* @param {string|undefined} ref
* @param {string} path
*/
async function blobAt(opened, ref, path) {
const refEntry = findRef(opened.repo, ref);
if (!refEntry) throw new Error(`no such ref: ${ref}`);
const rootTree = await peelToTree(opened.store, refEntry.sha);
const { sha, entry } = await walkPath(opened.store, rootTree, path);
const object = await opened.store.get(sha);
if (object?.type !== 'blob') {
throw new Error(`not a file: ${path}`);
}
return { refEntry, entry, object };
}
return {
/**
* List a directory. Directories sort first, then files, both by name.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {string} [path]
* @param {string} [space]
*/
async listTree(did, rkey, ref, path = '', space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const refEntry = findRef(opened.repo, ref);
if (!refEntry) throw new Error(`no such ref: ${ref}`);
const rootTree = await peelToTree(opened.store, refEntry.sha);
const { sha } = await walkPath(opened.store, rootTree, path);
const tree = await opened.store.get(sha);
if (tree?.type !== 'tree') {
throw new Error(`not a directory: ${path}`);
}
const entries = parseTree(tree.data)
.map((entry) => ({
...entry,
size:
entry.type === 'file'
? (opened.store.meta(entry.sha)?.size ?? null)
: null,
}))
.sort((a, b) => {
if ((a.type === 'dir') !== (b.type === 'dir')) {
return a.type === 'dir' ? -1 : 1;
}
return a.name.localeCompare(b.name);
});
return { ref: refEntry.name, sha: refEntry.sha, path, entries };
},
/**
* Every file at a ref, with the size of each. The walk reads trees only:
* a blob's size comes from the pack index, so no file body is fetched.
* One depth of directories is read per batch, which costs one set of
* coalesced range reads per level rather than one per directory.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {string} [space]
* @returns {Promise<{ref: string, files: Array<{path: string, size: number}>, truncated: boolean}|null>}
*/
async listFiles(did, rkey, ref, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const refEntry = findRef(opened.repo, ref);
if (!refEntry) throw new Error(`no such ref: ${ref}`);
const rootTree = await peelToTree(opened.store, refEntry.sha);
/** @type {Array<{path: string, size: number}>} */
const files = [];
/** @type {Array<{sha: string, path: string}>} */
let level = [{ sha: rootTree, path: '' }];
let truncated = false;
while (level.length > 0 && !truncated) {
await opened.store.getMany(level.map((dir) => dir.sha));
/** @type {Array<{sha: string, path: string}>} */
const next = [];
for (const dir of level) {
const tree = opened.store.peek(dir.sha);
if (tree?.type !== 'tree') continue;
for (const entry of parseTree(tree.data)) {
const path = dir.path ? `${dir.path}/${entry.name}` : entry.name;
if (entry.type === 'dir') {
next.push({ sha: entry.sha, path });
continue;
}
// A symlink names a path and a submodule names another
// repository. Neither holds bytes of this one.
if (entry.type !== 'file') continue;
const size = opened.store.meta(entry.sha)?.size;
if (size === undefined) continue;
files.push({ path, size });
if (files.length >= MAX_LISTED_FILES) {
truncated = true;
break;
}
}
if (truncated) break;
}
level = next;
}
return { ref: refEntry.name, files, truncated };
},
/**
* Read a file at a path. Text comes back inline up to the preview cap;
* binary or oversized files report metadata only. An image reports its
* media type, which the caller fetches as bytes from readImage.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {string} [path]
* @param {string} [space]
*/
async readFile(did, rkey, ref, path = '', space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const { refEntry, entry, object } = await blobAt(opened, ref, path);
const binary = looksBinary(object.data);
const truncated = object.data.length > MAX_PREVIEW_BYTES;
return {
ref: refEntry.name,
path,
size: object.data.length,
mode: entry?.mode ?? '100644',
mediaType: imageMediaType(path),
binary,
truncated,
text:
binary || truncated ? null : new TextDecoder().decode(object.data),
};
},
/**
* The commit history reachable from a ref, newest first the way git log
* orders it: among the commits whose children are all listed, the one
* with the latest committer date comes next.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {number} [limit]
* @param {string} [space]
* @param {string} [path] - a path, for the commits that changed it
*/
async listCommits(did, rkey, ref, limit = 50, space, path = '') {
const opened = await open(did, rkey, space);
if (!opened) return null;
// A repository nothing has been pushed to holds no refs, and a ref a
// record does not name is answered the same as the list of a repo that
// has none: empty rather than an error.
if (opened.repo.refs.length === 0) {
return { ref, commits: [], truncated: false };
}
const refEntry = findRef(opened.repo, ref);
if (!refEntry) throw new Error(`no such ref: ${ref}`);
// One batched fetch for the whole graph; the walk below then reads
// from memory in both modes.
await opened.store.prefetchCommits();
// The history of one path is the commits that changed it, which is a
// different walk from the graph's, one that reads trees.
if (path) {
const start = await peelToCommit(opened.store, refEntry.sha);
const { commits, truncated } = await pathHistory(
opened.store,
start,
path,
PATH_WALK_LIMIT,
limit,
);
return {
ref: refEntry.name,
commits: commits.map(({ sha, ...commit }) => ({
sha,
parents: commit.parents,
author: commit.author,
authoredAt: commit.authoredAt,
committedAt: commit.committedAt,
message: commit.message,
})),
truncated,
};
}
/** @type {Map<string, import('./objects.js').CommitInfo>} */
const parsed = new Map();
const info = (/** @type {string} */ sha) => {
let commit = parsed.get(sha);
if (!commit) {
const object = opened.store.peek(sha);
if (object?.type !== 'commit') {
throw new Error(`object ${sha} is not a commit`);
}
commit = parseCommit(object.data);
parsed.set(sha, commit);
}
return commit;
};
/** @type {string[]} */
const frontier = [await peelToCommit(opened.store, refEntry.sha)];
const seen = new Set(frontier);
const commits = [];
while (frontier.length > 0 && commits.length < limit) {
let next = 0;
for (let i = 1; i < frontier.length; i++) {
if (
info(frontier[i]).committedAt > info(frontier[next]).committedAt
) {
next = i;
}
}
const sha = frontier.splice(next, 1)[0];
const commit = info(sha);
commits.push({
sha,
parents: commit.parents,
author: commit.author,
authoredAt: commit.authoredAt,
committedAt: commit.committedAt,
message: commit.message,
});
for (const parent of commit.parents) {
if (!seen.has(parent)) {
seen.add(parent);
frontier.push(parent);
}
}
}
return { ref: refEntry.name, commits, truncated: frontier.length > 0 };
},
/**
* The last commit to change one path, walking back from a ref. See
* pathHistory for what counts as a change.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {string} [path]
* @param {number} [limit] - commits to walk before giving up
* @param {string} [space]
* @returns {Promise<import('./objects.js').CommitInfo & {sha: string}|null>}
* null where nothing in the walk touched the path
*/
async lastCommit(did, rkey, ref, path = '', limit = 4000, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
// An empty repository holds no commit, which is the same answer as one
// whose walk found nothing touching the path.
if (opened.repo.refs.length === 0) return null;
const refEntry = findRef(opened.repo, ref);
if (!refEntry) throw new Error(`no such ref: ${ref}`);
await opened.store.prefetchCommits();
const start = await peelToCommit(opened.store, refEntry.sha);
const { commits } = await pathHistory(
opened.store,
start,
path,
limit,
1,
);
return commits[0] ?? null;
},
/**
* One commit: its metadata and the files it changed, each with unified
* diff hunks. The comparison is against the first parent, which is how
* `git show` presents a merge. A root commit compares against nothing,
* so every file in it reads as added.
*
* A binary file, a submodule, or a change too large to walk reports its
* status with null hunks. An image also names its media type and carries
* each side's bytes, for the page to show where hunks would go.
* @param {string} did
* @param {string} rkey
* @param {string} sha - full object id of the commit
* @param {string} [space]
*/
async readCommit(did, rkey, sha, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const object = await opened.store.get(sha);
if (object?.type !== 'commit') throw new Error(`no such commit: ${sha}`);
const commit = parseCommit(object.data);
/** @type {string|null} */
let parentTree = null;
if (commit.parents.length > 0) {
const parent = await opened.store.get(commit.parents[0]);
if (parent?.type !== 'commit') {
throw new Error(`parent ${commit.parents[0]} is not in the bundles`);
}
parentTree = commitTree(parent.data);
}
/** @type {Map<string, {from: import('./objects.js').TreeEntry|null, to: import('./objects.js').TreeEntry|null}>} */
const changes = new Map();
await walkTreeDiff(opened.store, parentTree, commit.tree, changes);
const files = await buildFiles(opened.store, changes);
return {
sha,
parents: commit.parents,
author: commit.author,
authoredAt: commit.authoredAt,
committedAt: commit.committedAt,
message: commit.message,
files,
};
},
/**
* The paths that differ between two commits, with nothing about their
* contents. Two subtrees holding the same object id are skipped whole, so
* the walk reads the trees along the changed paths and no others. What a
* caller placing one branch against another needs, where readCommit would
* fetch every blob on both sides to build hunks it never shows.
* @param {string} did
* @param {string} rkey
* @param {string} base - full object id of the commit to compare against
* @param {string} tip - full object id of the commit
* @param {string} [space]
* @returns {Promise<string[]|null>} the paths, sorted
*/
async changedPaths(did, rkey, base, tip, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const changes = await rangeChanges(opened.store, base, tip);
return [...changes.keys()].sort();
},
/**
* Everything that differs between two commits, as a diff view renders it.
* A change written over a dozen commits is one thing to read, and reading
* it a commit at a time shows a file as many times as it was touched.
* @param {string} did
* @param {string} rkey
* @param {string} base - full object id of the commit to compare against
* @param {string} tip - full object id of the commit
* @param {string} [space]
*/
async readRange(did, rkey, base, tip, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const object = await opened.store.get(tip);
if (object?.type !== 'commit') throw new Error(`no such commit: ${tip}`);
const commit = parseCommit(object.data);
const changes = await rangeChanges(opened.store, base, tip);
const log = await walkRange(opened.store, base, tip, MAX_RANGE_COMMITS);
return {
base,
sha: tip,
author: commit.author,
authoredAt: commit.authoredAt,
committedAt: commit.committedAt,
message: commit.message,
commits: log.commits.length,
files: await buildFiles(opened.store, changes),
};
},
/**
* The commits one change carries, newest first. Its own log, rather than
* the branch's: everything the tip reaches that the base does not.
* @param {string} did
* @param {string} rkey
* @param {string|null} base
* @param {string} tip
* @param {number} [limit]
* @param {string} [space]
*/
async listRange(did, rkey, base, tip, limit = 100, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
if (opened.repo.refs.length === 0) {
return { commits: [], truncated: false };
}
return walkRange(opened.store, base, tip, limit);
},
/**
* Read an image file as bytes, for inline display. Only the extensions in
* IMAGE_MEDIA_TYPES resolve; any other path is rejected.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {string} [path]
* @param {string} [space]
*/
async readImage(did, rkey, ref, path = '', space) {
const mediaType = imageMediaType(path);
if (!mediaType) throw new Error(`not an image: ${path}`);
const opened = await open(did, rkey, space);
if (!opened) return null;
const { refEntry, object } = await blobAt(opened, ref, path);
return {
ref: refEntry.name,
path,
size: object.data.length,
mediaType,
bytes: object.data,
};
},
/**
* Read a file as bytes, whole and unaltered. readFile caps and decodes
* for display; this is the file itself, for a caller that serves or
* stores it. An image reports its media type, so a caller can serve it
* as one.
* @param {string} did
* @param {string} rkey
* @param {string} [ref]
* @param {string} [path]
* @param {string} [space]
*/
async readBytes(did, rkey, ref, path = '', space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
const { refEntry, entry, object } = await blobAt(opened, ref, path);
return {
ref: refEntry.name,
path,
size: object.data.length,
mode: entry?.mode ?? '100644',
mediaType: imageMediaType(path),
binary: looksBinary(object.data),
bytes: object.data,
};
},
/**
* The refs a repository carries, for a caller that has to tell a ref name
* from the path after it in a URL. Both may hold slashes, so the only
* reliable split is against these names.
* @param {string} did
* @param {string} rkey
* @param {string} [space]
* @returns {Promise<string[]|null>} full ref names, longest first
*/
async refNames(did, rkey, space) {
const opened = await open(did, rkey, space);
if (!opened) return null;
return opened.repo.refs
.map((entry) => entry.name)
.sort((a, b) => b.length - a.length);
},
};
}