/** * The pull request list, read on the repository's own server. * * A pull request is a branch in one account's copy, and a copy is on its * author's server, so reading the list means reading several servers. The * server does that once, for every visitor, where the page did it once per * visitor. What the object graph says is computed here and never stored: * open and merged are where a tip stands against canonical main. * * Two kinds of copy are read. A collaborator the config names is read whole, * every branch. Anybody else is found through the discovery port, which * answers with the pull request records about this repository, and only the * branch each record names is read: the rest of a stranger's copy is not * this repository's business. * * Workers-clean: no node imports. */ import { createGitBrowser } from './browser.js'; import { mergeText } from './merge3.js'; import { byPull, fixesOf, mergedRun, place, pullKey, pullRkey, runsForCopies, shortRef, subjectOf, } from './pull-requests.js'; import { repoNameOf, rkeyOf } from './record.js'; import { collaboratorsOf, reviewersOf, runnerOf } from './rules.js'; /** @typedef {import('./pull-requests.js').Contributor} Contributor */ /** @typedef {import('./pull-requests.js').PullRequest} PullRequest */ /** @typedef {import('./discovery.js').DiscoveryPort} DiscoveryPort */ /** @typedef {ReturnType} GitBrowser */ /** @typedef {Pick} CopyReader */ const REPO_COLLECTION = 'dev.pdsjs.git.repo'; const REVIEW_COLLECTION = 'dev.pdsjs.git.review'; const PULL_COLLECTION = 'dev.pdsjs.git.pull'; const CHECK_COLLECTION = 'dev.pdsjs.git.check'; /** * How far back main is read when placing a branch against it. A branch taken * further back than this reads as unrelated rather than as behind, which is * the honest answer from a bounded walk. */ const MAIN_DEPTH = 500; /** How far back a copy's branch is read before it counts as unrelated. */ const BRANCH_DEPTH = 100; /** Records read per listing, the most a PDS answers in one page. */ const LIST_LIMIT = 100; /** Pages of the runner's runs read before the rest are left. */ const CHECK_PAGES = 10; /** Records followed per discovery answer, so one noisy index stays bounded. */ const MAX_DISCOVERED = 100; /** * Drifted paths a single pull request has its conflicts read on, and the most * across one listing. A three-way merge of one path is three file reads, so * an unbounded list of drifting pull requests is an unbounded read. Past the * cap a pull request's conflicts are left unread rather than half-read, and * the page says the merge itself will tell. */ const MAX_CONFLICT_PATHS = 25; const CONFLICT_BUDGET = 200; /** Readers over contributors' servers kept open, oldest dropped past this. */ const OPEN_COPIES = 32; const READ_TIMEOUT_MS = 5000; /** How long a copy's record key stands before its holder is asked again. */ const COPY_KEY_TTL_MS = 5 * 60 * 1000; /** * @typedef {Object} PullRequestReaderContext * @property {(did: string) => Promise} resolveContributor - * where one account's records and blobs are read from. A read names an * account by DID, the server resolves the DID document, and reads its * serviceEndpoint. A did:web the caller controls names any endpoint, so * this is a request the server makes to a URL a request chose: bounded to * GET of com.atproto.repo.* and dev.pdsjs.git.* paths whose answer is * parsed as records, but a deployment behind a network where an internal * host answers on those paths should resolve DIDs through a directory it * trusts and hold the endpoint to public hosts. * @property {(did: string, collection: string, rkey: string) => Promise} readRecord - * this account's own records * @property {Pick} browser - the * server's own reader, which holds the canonical copy * @property {DiscoveryPort} discovery * @property {(message: string) => void} [onNotice] - where a bound the read hit is reported, a silent cap otherwise * @property {typeof fetch} [fetch] - what carries a request to another server; the global by default * @property {(who: Contributor) => CopyReader} [openCopy] - a reader over one * contributor's server. By default a browser over `fetch`, kept per * contributor. * @property {number} [maxEagerBytes] - the largest copy read whole from * another server. A copy past it costs its branches, not the list. */ /** * @typedef {Object} PullRequestList * @property {PullRequest[]} open * @property {PullRequest[]} merged * @property {Contributor[]} collaborators * @property {Array<[string, Array>]>} reviews - by * pull request id * @property {Array<[string, Array>]>} checks - by * pull request id * @property {boolean} configured - whether the config names any collaborator */ /** * @param {PullRequestReaderContext} ctx */ export function createPullRequestReader(ctx) { const send = ctx.fetch ?? ((input, init) => globalThis.fetch(input, init)); /** @type {Map} */ const copies = new Map(); /** * A reader over one contributor's server. Their objects are there and * nowhere else, so every call crosses the network. * @param {Contributor} who * @returns {CopyReader} */ function openCopy(who) { if (ctx.openCopy) return ctx.openCopy(who); let reader = copies.get(who.did); if (!reader) { reader = createGitBrowser({ xrpc: (request) => { const url = new URL(request.url); return send(`${who.pds}${url.pathname}${url.search}`, { headers: request.headers, signal: AbortSignal.timeout(READ_TIMEOUT_MS), }); }, maxEagerBytes: ctx.maxEagerBytes, }); if (copies.size >= OPEN_COPIES) { const oldest = copies.keys().next().value; if (oldest !== undefined) copies.delete(oldest); } copies.set(who.did, reader); } return reader; } /** * The server's own copy, read through its own browser. The refs come from * the record rather than a walk, since the record is in hand. * @param {string} did * @returns {CopyReader} */ function ownCopy(did) { return { refNames: async (_did, repo) => { const record = /** @type {{refs?: unknown}|null} */ ( await ctx.readRecord(did, REPO_COLLECTION, repo) ); const refs = Array.isArray(record?.refs) ? record.refs : []; return refs .map((entry) => /** @type {{name?: unknown}|null} */ (entry)?.name) .filter((name) => typeof name === 'string') .sort((a, b) => b.length - a.length); }, listCommits: (_did, repo, ref, limit) => ctx.browser.listCommits(did, repo, ref, limit), changedPaths: (_did, repo, base, tip) => ctx.browser.changedPaths(did, repo, base, tip), readFile: (_did, repo, ref, path) => ctx.browser.readFile(did, repo, ref, path), }; } /** * The key each account's copy of a repository is at, by account and name. * @type {Map} */ const copyKeys = new Map(); /** * The record key one account's copy of a repository is at, on their own * server. A copy is its holder's own record and carries their own key, so * the name is what the two copies share and what finds it. * * The answer is kept, because a key never moves once minted; the hold is * short so that a copy deleted and pushed again is found. * @param {Contributor} who * @param {string} name * @returns {Promise} */ async function copyKeyOf(who, name) { const at = `${who.did}\u0000${name}`; const held = copyKeys.get(at); if (held && Date.now() - held.at < COPY_KEY_TTL_MS) return held.rkey; const found = (await listRecords(who, REPO_COLLECTION)).find( (row) => repoNameOf(row.value, rkeyOf(row.uri)) === name, ); const rkey = found ? rkeyOf(found.uri) : null; copyKeys.set(at, { at: Date.now(), rkey }); return rkey; } /** * One page of an account's records in a collection, from their server. * @param {Contributor} who * @param {string} collection * @returns {Promise}>>} */ async function listRecords(who, collection) { const params = new URLSearchParams({ repo: who.did, collection, limit: String(LIST_LIMIT), }); try { const res = await send( `${who.pds}/xrpc/com.atproto.repo.listRecords?${params}`, { signal: AbortSignal.timeout(READ_TIMEOUT_MS) }, ); if (!res.ok) return []; const body = /** @type {{records?: unknown[]}} */ (await res.json()); return (body.records ?? []).flatMap((entry) => { const row = /** @type {Record} */ (entry ?? {}); return typeof row.uri === 'string' && typeof row.cid === 'string' ? [ { uri: row.uri, cid: row.cid, value: /** @type {Record} */ (row.value ?? {}), }, ] : []; }); } catch { // An unreachable account costs its records, not the list. return []; } } /** * One record from its author's server. * @param {Contributor} who * @param {string} collection * @param {string} rkey * @returns {Promise<{uri: string, cid: string, value: Record}|null>} */ async function getRecord(who, collection, rkey) { const params = new URLSearchParams({ repo: who.did, collection, rkey }); try { const res = await send( `${who.pds}/xrpc/com.atproto.repo.getRecord?${params}`, { signal: AbortSignal.timeout(READ_TIMEOUT_MS) }, ); if (!res.ok) return null; const body = /** @type {Record} */ (await res.json()); return typeof body.uri === 'string' && typeof body.cid === 'string' ? { uri: body.uri, cid: body.cid, value: /** @type {Record} */ (body.value ?? {}), } : null; } catch { return null; } } /** * The records the discovery port names about the subject, each read from * its author. Accounts already read directly are skipped, so a listed * voice's word is read once. * @param {string} subjectUri * @param {string} collection * @param {Set} except - DIDs read directly * @param {string} [path] - where the record names the subject; `subject.uri` * unless given * @returns {Promise}>>} */ async function discovered( subjectUri, collection, except, path = 'subject.uri', ) { const all = ( await ctx.discovery.about(subjectUri, collection, path) ).filter((entry) => !except.has(entry.did)); if (all.length > MAX_DISCOVERED && ctx.onNotice) { ctx.onNotice( `discovery named ${all.length} ${collection} records; reading the first ${MAX_DISCOVERED}`, ); } const entries = all.slice(0, MAX_DISCOVERED); const found = await Promise.all( entries.map(async (entry) => { const who = await ctx.resolveContributor(entry.did); if (!who) return null; const record = await getRecord(who, collection, entry.rkey); if (!record) return null; // The index said the record names this repository. The record is // what says so. const named = path === 'upstream' ? record.value.upstream : /** @type {{uri?: unknown}} */ (record.value.subject ?? {}).uri; if (named !== subjectUri) return null; return { who, ...record }; }), ); return found.filter((one) => one !== null); } /** * The pull requests one account's copy carries. * @param {Contributor} who * @param {string} repo * @param {{shas: string[], commits: Array<{sha: string, parents: string[]}>}} main * @param {string} canonicalBranch * @param {CopyReader} reader * @param {string[]} [only] - short branch names to read, where the caller * wants some of them * @returns {Promise} */ async function pullsFrom(who, repo, main, canonicalBranch, reader, only) { /** @type {string[]|null} */ let refs; try { refs = await reader.refNames(who.did, repo); } catch { return []; } if (!refs) return []; const branches = ( only ?? refs .filter((name) => name.startsWith('refs/heads/')) .map((name) => name.slice('refs/heads/'.length)) ).filter((name) => name !== canonicalBranch); const found = await Promise.all( branches.map(async (branch) => { try { const log = await reader.listCommits( who.did, repo, branch, BRANCH_DEPTH, ); if (!log || log.commits.length === 0) return null; const placed = place(log.commits, main.shas); const { ahead, base } = placed.merged ? mergedRun(log.commits, main.commits) : placed; const { behind, merged, rooted } = placed; const tip = log.commits[0]; // One tree walk from the base to the tip, whatever the pull // request's length. const files = rooted && base ? ((await reader.changedPaths(who.did, repo, base, tip.sha)) ?? []) : []; const fixes = [ ...new Set( log.commits .slice(0, Math.max(ahead, 1)) .flatMap((entry) => fixesOf(entry.message)), ), ]; /** @type {PullRequest} */ const pull = { id: pullKey(who.did, branch), author: who, branch, title: subjectOf(tip.message), sha: tip.sha, when: tip.committedAt, ahead, base, behind, merged, rooted, files, fixes, }; return pull; } catch { // An unreachable copy costs its branches, not the list. return null; } }), ); return found.filter((pull) => pull !== null); } /** * A file at one commit, as text, or null where the path is not there or * cannot be read as text. What a three-way merge of one path needs. * @param {CopyReader} reader * @param {string} did @param {string} repo @param {string} sha * @param {string} path * @returns {Promise<{text: string|null}|null>} */ async function fileAt(reader, did, repo, sha, path) { try { const file = await reader.readFile(did, repo, sha, path); return file ? { text: file.binary ? null : file.text } : null; } catch { return null; } } /** * Whether main and the branch can be put together at one path, or * disagree there. The rules are the tree merge's: a path only one side * changed takes that side, both deleting it agree, text both changed * merges line by line, and anything else both changed is a conflict. * @param {PullRequest} pull * @param {string} mainTip * @param {string} did @param {string} repo * @param {CopyReader} theirs - the copy holding the branch * @param {string} path */ async function conflictsAt(pull, mainTip, did, repo, theirs, path) { const base = /** @type {string} */ (pull.base); const [b, o, t] = await Promise.all([ fileAt(ownCopy(did), did, repo, base, path), fileAt(ownCopy(did), did, repo, mainTip, path), fileAt(theirs, pull.author.did, repo, pull.sha, path), ]); const same = ( /** @type {{text: string|null}|null} */ x, /** @type {{text: string|null}|null} */ y, ) => (x === null && y === null) || (x !== null && y !== null && x.text === y.text); if (same(o, t) || same(o, b) || same(t, b)) return false; if (!o || !t || o.text === null || t.text === null) return true; return mergeText(b?.text ?? '', o.text, t.text).conflicts > 0; } /** * What each pull request has drifted from: the files main changed since it * branched off, that the pull request touches too. One tree walk each, * from its own merge base to the tip of main. Then, at each of those * paths, whether the two sides can be put together, which is what a * reader wants to know before pressing merge. * @param {PullRequest[]} pulls * @param {string|undefined} mainTip * @param {string} did * @param {string} repo * @param {(pull: PullRequest) => CopyReader} copyOf - the reader over a * pull request's copy */ async function fillDrift(pulls, mainTip, did, repo, copyOf) { /** @type {Map>} */ const walks = new Map(); let budget = CONFLICT_BUDGET; for (const pull of pulls) { if (!pull.base || pull.behind === 0 || !mainTip) { pull.drift = []; pull.conflicts = []; continue; } // Branches taken from the same commit ask the same question. let walk = walks.get(pull.base); if (!walk) { walk = ctx.browser .changedPaths(did, repo, pull.base, mainTip) .catch(() => null); walks.set(pull.base, walk); } const since = new Set((await walk) ?? []); pull.drift = pull.files.filter((path) => since.has(path)); const check = pull.drift.slice( 0, Math.max(0, Math.min(MAX_CONFLICT_PATHS, budget)), ); budget -= check.length; const theirs = copyOf(pull); const clashes = await Promise.all( check.map((path) => conflictsAt(pull, mainTip, did, repo, theirs, path), ), ); pull.conflicts = check.filter((_path, at) => clashes[at]); // A conflict anywhere in the subset settles it. None in a subset short // of the whole drift leaves it unknown: the merge itself reads every // path, and the page says so rather than reporting a clean it did not // confirm. pull.conflictsUnknown = check.length < pull.drift.length && pull.conflicts.length === 0; } } /** * The pull requests from accounts the config does not name, read from their * own copies. Two things find them. A copy whose record names this * repository as its upstream is a fork, and every branch of it is read. A * pull request record names one branch, and only that branch is read: the * rest of that account's copy is not this repository's business. * @param {string} subjectUri * @param {{shas: string[], commits: Array<{sha: string, parents: string[]}>}} main * @param {string} canonicalBranch * @param {Set} collaborators - DIDs read whole already * @returns {Promise} */ async function forkedPulls(subjectUri, main, canonicalBranch, collaborators) { const canonicalKey = rkeyOf(subjectUri); const [records, forks] = await Promise.all([ discovered(subjectUri, PULL_COLLECTION, collaborators), discovered(subjectUri, REPO_COLLECTION, collaborators, 'upstream'), ]); /** @type {Map|null, words: Map}>} */ const byDid = new Map(); // A declared fork is read whole: null names every branch. Discovery // named the fork's own record, so its key is in hand. for (const { who, uri } of forks) { byDid.set(who.did, { who, copy: rkeyOf(uri), refs: null, words: new Map(), }); } // A pull request lives at the key its branch derives, and that record is // the current word. Discovery may name several records for one branch; the // one at the derived key wins, and where none is, the last read does. for (const { who, uri, value } of records) { const { ref, note, status, title } = /** @type {Record} */ (value); if (typeof ref !== 'string') continue; const short = shortRef(ref); // The pull request record names the copy holding the branch, which is // the only thing that says where a fork under another name is. const copy = typeof (/** @type {{repo?: unknown}} */ (value).repo) === 'string' ? /** @type {{repo: string}} */ (value).repo : ''; const group = byDid.get(who.did) ?? { who, copy, refs: new Set(), words: new Map(), }; group.refs?.add(short); const rkey = rkeyOf(String(uri)); if (note || status || title) { const canonical = rkey === pullRkey(canonicalKey, ref); const current = group.words.get(short); if (!current || canonical || !current.canonical) { group.words.set(short, { note, status, title, canonical }); } } byDid.set(who.did, group); } const found = await Promise.all( [...byDid.values()].map(async ({ who, copy, refs, words }) => { if (!copy) return []; const pulls = await pullsFrom( who, copy, main, canonicalBranch, openCopy(who), refs ? [...refs] : undefined, ); return pulls.map((pull) => ({ ...pull, fork: true, title: words.get(pull.branch)?.title || pull.title, note: words.get(pull.branch)?.note ?? '', status: words.get(pull.branch)?.status, })); }), ); return found.flat(); } /** * Every statement about this repository in one collection: the listed * voices' own listings, and whoever else the discovery port names. * @param {Contributor[]} voices * @param {string} subjectUri * @param {string} collection * @param {string[]} [copies] - the other repository records a statement * may name: the collaborators' copies, where the branches are * @returns {Promise>>} */ async function statements(voices, subjectUri, collection, copies) { // A statement may name the canonical record or the copy holding the // branch it speaks to; a review of a collaborator's branch names theirs. const named = new Set([subjectUri, ...(copies ?? [])]); const listed = await Promise.all( voices.map(async (who) => (await listRecords(who, collection)) .filter((row) => { const subject = /** @type {{uri?: unknown}} */ ( row.value.subject ?? {} ); return typeof subject.uri === 'string' && named.has(subject.uri); }) .map((row) => ({ ...row.value, uri: row.uri, cid: row.cid, by: who, })), ), ); const strangers = await discovered( subjectUri, collection, new Set(voices.map((who) => who.did)), ); return [ ...listed.flat(), ...strangers.map(({ who, uri, cid, value }) => ({ ...value, uri, cid, by: who, indexed: true, })), ]; } /** * The runner's runs about any copy of this repository. * @param {unknown} config * @param {string[]} copyUris - the copies that count */ async function runs(config, copyUris) { const runner = runnerOf(config); if (!runner) return []; const who = await ctx.resolveContributor(runner); if (!who) return []; /** @type {unknown[]} */ const records = []; let cursor = ''; for (let page = 0; page < CHECK_PAGES; page++) { const params = new URLSearchParams({ repo: who.did, collection: CHECK_COLLECTION, limit: String(LIST_LIMIT), }); if (cursor) params.set('cursor', cursor); try { const res = await send( `${who.pds}/xrpc/com.atproto.repo.listRecords?${params}`, { signal: AbortSignal.timeout(READ_TIMEOUT_MS) }, ); if (!res.ok) break; const body = /** @type {{records?: unknown[], cursor?: unknown}} */ ( await res.json() ); records.push(...(body.records ?? [])); cursor = typeof body.cursor === 'string' ? body.cursor : ''; if (!cursor) break; } catch { break; } } return runsForCopies(records, copyUris); } return { /** * A browser over one account's copy, for reading what is in it: a * commit, a range, a tree. Null for an account with no server. * @param {string} did * @returns {Promise} */ async copy(did) { const who = await ctx.resolveContributor(did); return who ? /** @type {GitBrowser} */ (openCopy(who)) : null; }, /** * The record key one account's copy of a repository is at. * @param {string} did * @param {string} name * @returns {Promise} */ async copyKey(did, name) { const who = await ctx.resolveContributor(did); return who ? copyKeyOf(who, name) : null; }, /** * Every pull request on one repository, from every copy. * @param {string} did - the repository owner * @param {string} rkey - the repository's record key * @returns {Promise} null for no such repository */ async listPulls(did, rkey) { const record = /** @type {{defaultBranch?: unknown, name?: unknown}|null} */ ( await ctx.readRecord(did, REPO_COLLECTION, rkey) ); if (!record) return null; const name = repoNameOf(record, rkey); const canonicalBranch = typeof record.defaultBranch === 'string' && record.defaultBranch ? record.defaultBranch : 'main'; const config = await ctx.readRecord(did, 'dev.pdsjs.git.config', rkey); const subjectUri = `at://${did}/${REPO_COLLECTION}/${rkey}`; const mainLog = await ctx.browser .listCommits(did, rkey, canonicalBranch, MAIN_DEPTH) .catch(() => null); const main = { commits: mainLog?.commits ?? [], shas: (mainLog?.commits ?? []).map((entry) => entry.sha), }; const dids = [did, ...collaboratorsOf(config)]; const collaborators = ( await Promise.all(dids.map((one) => ctx.resolveContributor(one))) ).filter((who) => who !== null); const voices = ( await Promise.all( [...new Set([...dids, ...reviewersOf(config)])].map((one) => ctx.resolveContributor(one), ), ) ).filter((who) => who !== null); // Which key each collaborator's copy is at. A copy is its holder's own // record and carries their own key; the name is what the copies share. /** @type {Map} */ const keys = new Map([[did, rkey]]); await Promise.all( collaborators .filter((who) => who.did !== did) .map(async (who) => { const theirs = await copyKeyOf(who, name); if (theirs) keys.set(who.did, theirs); }), ); const holding = collaborators.filter((who) => keys.has(who.did)); const copyUris = [...keys].map( ([one, key]) => `at://${one}/${REPO_COLLECTION}/${key}`, ); const [found, reviews, checks, forked] = await Promise.all([ Promise.all( holding.map((who) => pullsFrom( who, /** @type {string} */ (keys.get(who.did)), main, canonicalBranch, // This server holds its own copy; every other copy is read // from the server that holds it. who.did === did ? ownCopy(did) : openCopy(who), ), ), ), statements(voices, subjectUri, REVIEW_COLLECTION, copyUris), runs(config, copyUris), forkedPulls(subjectUri, main, canonicalBranch, new Set(dids)), ]); const mine = found.flat(); await Promise.all( mine.map(async (pull) => { const who = pull.author; const record = await getRecord( who, PULL_COLLECTION, pullRkey(rkey, `refs/heads/${pull.branch}`), ); if (!record) return; const value = /** @type {Record} */ (record.value); if (typeof value.title === 'string') pull.title = value.title; if (typeof value.status === 'string') pull.status = value.status; if (typeof value.note === 'string') pull.note = value.note; if (typeof value.createdAt === 'string') pull.noteAt = value.createdAt; }), ); const pulls = [...mine, ...forked].sort((a, b) => b.when - a.when); const open = pulls.filter((pull) => !pull.merged); await fillDrift(open, main.shas[0], did, rkey, (pull) => pull.author.did === did ? ownCopy(did) : openCopy(pull.author), ); return { open, merged: pulls.filter((pull) => pull.merged), collaborators, reviews: [...byPull(reviews)], checks: [...byPull(checks)], configured: collaboratorsOf(config).length > 0, }; }, }; }