//! A repository and what it holds, as XRPC queries: `dev.knotless.getRepo`, //! `getTree`, `listPaths`, `getLastCommits`, `getFile`, `getBlame`, `listCommits`, `getCommit`, //! `listBranches` and `listTags`. use super::*; /// The parameters of the queries of a repository: each takes what it /// declares. #[derive(Deserialize)] pub struct RepoParams { repo: String, #[serde(rename = "ref")] refname: Option, path: Option, limit: Option, cursor: Option, #[serde(rename = "subjectOnly", default)] subject_only: bool, #[serde(default)] commits: bool, } /// The most commits a page of `listCommits` holds, and how many if not /// asked. const COMMITS_MAX: usize = 100; const COMMITS_PER_PAGE: usize = 50; /// The same of branches and tags, for `listBranches` and `listTags`. const BRANCHES_MAX: usize = 1000; const BRANCHES_PER_PAGE: usize = 100; pub(super) fn invalid(message: String) -> Shown { Shown(StatusCode::BAD_REQUEST, message) } /// The repository whose record `uri` names, its authority a DID or a /// handle, or its record in its space, under the name it has now. pub(super) async fn found(app: &App, uri: &str) -> Result { named(app, uri, false).await } /// [`found`] for a view of its code, which may be of a mirror a change has /// made stale while it syncs. async fn found_for_reader(app: &App, uri: &str) -> Result { named(app, uri, true).await } async fn named(app: &App, uri: &str, reader: bool) -> Result { match knotless::aturi::AtUri::parse(uri) { Ok(at) if at.collection == nsid::REPO && reader => { Repo::find_for_reader(app, &at.author, &at.rkey).await } Ok(at) if at.collection == nsid::REPO => Repo::find(app, &at.author, &at.rkey).await, _ => Err(invalid(format!( "{uri} is not at:///{}/", nsid::REPO ))), } } /// How many of a list the pages before held, as a cursor says. pub(super) fn skipped(cursor: Option<&str>) -> Result { match cursor.map(str::parse::) { None => Ok(0), Some(Ok(skip)) => Ok(skip), Some(Err(_)) => Err(invalid("cursor is no count".into())), } } /// The revision and the path `p` names: the default branch's root, or /// `ref` and `path` together, split where a branch or a tag ends. fn located(repo: &Repo, p: &RepoParams) -> Result<(String, String)> { let path = p.path.as_deref().unwrap_or_default(); let Some(r) = &p.refname else { return Ok((repo.head.clone(), path.to_string())); }; let rest = match path { "" => r.clone(), path => format!("{r}/{path}"), }; let (rev, path) = repo.locate(&rest)?; Ok((rev, path.to_string())) } /// `dev.knotless.getRepo`. pub async fn get_repo( State(app): State>, axum::Extension(viewer): axum::Extension, Query(p): Query, ) -> Result { let repo = found_for_reader(&app, &p.repo).await?; Ok(answer(repo.head(&app, viewer.0.as_deref()).await)) } /// One entry of a directory. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct Entry { name: String, /// `tree` or `blob`. kind: String, #[serde(skip_serializing_if = "Option::is_none")] size: Option, } /// A README, as Markdown made into HTML or as text. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct Readme { name: String, #[serde(skip_serializing_if = "Option::is_none")] html: Option, #[serde(skip_serializing_if = "Option::is_none")] text: Option, } /// `dev.knotless.getTree`'s answer: a directory at a revision, its latest /// commit, its entries, folders first, and its README. Nothing at all, for /// a repository never pushed. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct Tree { #[serde(rename = "ref")] refname: String, path: String, #[serde(skip_serializing_if = "std::ops::Not::not")] empty: bool, #[serde(skip_serializing_if = "Option::is_none")] last_commit: Option, entries: Vec, #[serde(skip_serializing_if = "Option::is_none")] readme: Option, /// What the repository's runners report of the commit shown. checks: Vec, /// What the repository is written in, most first: shown at the top /// of its default branch, and nowhere else. languages: Vec, } /// `dev.knotless.getTree`. pub async fn get_tree( State(app): State>, Query(p): Query, ) -> Result { let repo = found_for_reader(&app, &p.repo).await?; if unpushed(&repo, &p)? { return Ok(answer(Tree { refname: repo.head.clone(), path: String::new(), empty: true, last_commit: None, entries: Vec::new(), readme: None, checks: Vec::new(), languages: Vec::new(), })); } let (rev, path) = located(&repo, &p)?; Ok(answer(tree(&app, &repo, rev, path).await?)) } /// Whether `p` asks for the root of the default branch of a repository /// never pushed, which is nothing rather than not found: the pages link /// there before the branch exists. fn unpushed(repo: &Repo, p: &RepoParams) -> Result { let root = p.path.as_deref().unwrap_or_default().is_empty(); let head = p.refname.as_deref().is_none_or(|r| r == repo.head); Ok(root && head && repo.objects()?.tree("HEAD", "").is_err()) } async fn tree(app: &App, repo: &Repo, rev: String, path: String) -> Result { let o = repo.objects()?; let mut found = o.tree(&rev, &path).map_err(not_found)?; found.sort_by_key(|e| (e.kind != "tree", e.name.to_lowercase())); let last = o.log(&rev, &path, 0, 1).map_err(not_found)?; let tip = o.resolve(&rev).map_err(not_found)?; let prefix = if path.is_empty() { String::new() } else { format!("{path}/") }; let readme = match found .iter() .find(|e| e.kind == "blob" && e.name.to_lowercase().starts_with("readme")) { Some(readme) => { let bytes = o .blob(&rev, &format!("{prefix}{}", readme.name)) .map_err(not_found)?; let text = String::from_utf8_lossy(&bytes).into_owned(); let html = is_markdown(&readme.name).then(|| { markdown( &text, &repo.files("blob", &rev), &repo.files("raw", &rev), &path, Some((&repo.files("tree", &rev), &|dir| o.is_dir(&rev, dir))), ) }); Some(Readme { name: readme.name.clone(), text: html.is_none().then_some(text), html, }) } None => None, }; let entries = found .into_iter() .map(|e| Entry { name: e.name, kind: e.kind, size: e.size, }) .collect(); // The checks of the commit the directory shows as its last, which is // the tip at the root but may be older below it. let shown_at = last.first().map_or(tip, |c| c.sha.clone()); let at = CheckFilter { commit: Some(&shown_at), ..CheckFilter::default() }; let basic = repo.basic(app).await; let languages = if path.is_empty() && rev == repo.head { repo.languages() .into_iter() .map(LanguageView::from) .collect() } else { Vec::new() }; Ok(Tree { last_commit: shown(app, repo, last).await.pop(), checks: newest(checks(app, &repo.reference(), &basic, at, 100)), refname: rev, path, empty: false, entries, readme, languages, }) } /// The most paths `listPaths` gives: what a page can filter as it is /// typed into, and more than nearly every repository holds. const PATHS_MAX: usize = 50_000; /// One path of a tree, as `listPaths` gives it. #[derive(Serialize)] pub struct PathView { path: String, kind: &'static str, } /// `dev.knotless.listPaths`'s answer. #[derive(Serialize)] pub struct Paths { #[serde(rename = "ref")] refname: String, paths: Vec, #[serde(skip_serializing_if = "std::ops::Not::not")] truncated: bool, } /// `dev.knotless.listPaths`: every path at a revision, for a page to /// find a file by as it is typed. pub async fn list_paths( State(app): State>, Query(p): Query, ) -> Result { let repo = found(&app, &p.repo).await?; let (rev, _) = located(&repo, &p)?; let limit = p.limit.unwrap_or(PATHS_MAX).clamp(1, PATHS_MAX); let (paths, truncated) = repo.objects()?.paths(&rev, limit).map_err(not_found)?; Ok(answer(Paths { refname: rev, paths: paths .into_iter() .map(|(path, kind)| PathView { path, kind }) .collect(), truncated, })) } /// One entry of a directory, and the commit that last changed it. #[derive(Serialize)] pub struct LastCommit { name: String, sha: String, } /// `dev.knotless.getLastCommits`'s answer: the newest commit to change /// each entry of a directory, and each of those commits once. #[derive(Serialize)] pub struct LastCommits { #[serde(rename = "ref")] refname: String, path: String, entries: Vec, commits: Vec, /// How many commits the ref's history holds: at the root only. #[serde(rename = "commitCount", skip_serializing_if = "Option::is_none")] commit_count: Option, } /// `dev.knotless.getLastCommits`: found off the server's async threads, as /// a long history takes a while to walk the first time. pub async fn get_last_commits( State(app): State>, Query(p): Query, ) -> Result { let repo = found_for_reader(&app, &p.repo).await?; let (rev, path) = located(&repo, &p)?; let (dir, at, from) = (repo.git.dir.clone(), rev.clone(), path.clone()); let walked = tokio::task::spawn_blocking(move || walk(&dir, &at, &from)); let walked = walked.await.expect("a walk does not panic"); let ((entries, commits), commit_count) = walked.map_err(not_found)?; Ok(answer(LastCommits { entries: entries .into_iter() .map(|(name, sha)| LastCommit { name, sha }) .collect(), commits: shown(&app, &repo, commits).await, commit_count, refname: rev, path, })) } /// The last changes of the directory `path` at `rev`, in the mirror at /// `dir`, and at the root, how many commits its history holds. fn walk( dir: &std::path::Path, rev: &str, path: &str, ) -> anyhow::Result<(crate::objects::LastChanges, Option)> { let objects = crate::objects::open(dir)?; let changes = objects.last_changes(rev, path)?; let count = match path { "" => Some(objects.count(rev)?), _ => None, }; Ok((changes, count)) } /// `dev.knotless.getFile`'s answer: one file at a revision, an image, a /// binary, too large to show, or text with the ranges its highlighting /// paints, and for Markdown, made into HTML too. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct Blob { #[serde(rename = "ref")] refname: String, path: String, /// `image`, `binary`, `large` or `text`. kind: String, size: u64, lines: usize, #[serde(skip_serializing_if = "Option::is_none")] text: Option, #[serde(skip_serializing_if = "Option::is_none")] ranges: Option, /// A Markdown file made into HTML, shown in place of its text unless the /// page is asked for it plain. #[serde(skip_serializing_if = "Option::is_none")] html: Option, } /// `dev.knotless.getFile`. pub async fn get_file( State(app): State>, Query(p): Query, ) -> Result { let repo = found(&app, &p.repo).await?; let (rev, path) = located(&repo, &p)?; let o = repo.objects()?; let bytes = o.blob(&rev, &path).map_err(not_found)?; let text = String::from_utf8_lossy(&bytes); let kind = if is_image(&path).is_some() { "image" } else if is_binary(&bytes) { "binary" } else if bytes.len() > INLINE { "large" } else { "text" }; let ranges = (kind == "text").then(|| { highlight::code(&path, &text) .map(|r| r.encode()) .unwrap_or_default() }); let html = (kind == "text" && is_markdown(&path)).then(|| { let dir = path.rsplit_once('/').map_or("", |(dir, _)| dir); markdown( &text, &repo.files("blob", &rev), &repo.files("raw", &rev), dir, Some((&repo.files("tree", &rev), &|dir| o.is_dir(&rev, dir))), ) }); Ok(answer(Blob { kind: kind.to_string(), size: bytes.len() as u64, lines: text.lines().count().max(1), text: ranges.is_some().then(|| text.into_owned()), ranges, html, refname: rev, path, })) } /// The largest file blamed. Blame reads every version of a file back to /// where each of its lines began, so its cost grows with the file as well /// as its history. pub const BLAME_MAX: usize = 256 * 1024; /// How many blames run at once: the rest wait their turn. static BLAMING: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); /// The blames kept, in each of the cache's two generations. const BLAMES_KEPT: usize = 256; /// Each file's blame, by the commit and the path: a blame at a commit /// never changes, so it is kept, for any repository that asks. type Blames = objects::Kept<(String, String), Arc>>; static BLAMES: LazyLock> = LazyLock::new(|| Mutex::new(Blames::new(BLAMES_KEPT))); /// A run of a file's lines that one commit last changed. #[derive(Serialize)] pub struct BlameHunk { sha: String, start: usize, count: usize, #[serde(skip_serializing_if = "Option::is_none")] previous: Option, } /// `dev.knotless.getBlame`'s answer: a text file at a revision, the ranges /// its highlighting paints, who last changed each run of its lines, and /// each of those commits once. #[derive(Serialize)] pub struct Blame { #[serde(rename = "ref")] refname: String, path: String, lines: usize, text: String, ranges: String, hunks: Vec, commits: Vec, } /// `dev.knotless.getBlame`: refused for a binary or a file over /// [`BLAME_MAX`]. pub async fn get_blame( State(app): State>, Query(p): Query, ) -> Result { let repo = found(&app, &p.repo).await?; let (rev, path) = located(&repo, &p)?; let (bytes, sha) = { let o = repo.objects()?; let bytes = o.blob(&rev, &path).map_err(not_found)?; (bytes, o.resolve(&rev).map_err(not_found)?) }; if is_binary(&bytes) { return Err(invalid(format!("{path} is binary: only text is blamed"))); } if bytes.len() > BLAME_MAX { let most = BLAME_MAX / 1024; return Err(invalid(format!( "{path} is over {most} KiB, too large to blame" ))); } let hunks = blamed(&repo.git, &sha, &path).await?; let mut seen = HashSet::new(); let shas = hunks .iter() .map(|h| h.sha.as_str()) .filter(|s| seen.insert(*s)); let commits = { let o = repo.objects()?; let read: anyhow::Result> = shas.map(|sha| o.tip(sha)).collect(); read.map_err(AppError::internal)? }; let text = String::from_utf8_lossy(&bytes).into_owned(); let ranges = highlight::code(&path, &text).map(|r| r.encode()); let hunks = hunks.iter().map(|h| BlameHunk { sha: h.sha.clone(), start: h.start, count: h.count, previous: h.previous.clone(), }); Ok(answer(Blame { hunks: hunks.collect(), commits: shown(&app, &repo, commits).await, lines: text.lines().count().max(1), ranges: ranges.unwrap_or_default(), text, refname: rev, path, })) } /// Who last changed each run of `path`'s lines at the commit `sha`, as /// kept, or else as git finds it, among as many others as may run at once. async fn blamed(git: &Git, sha: &str, path: &str) -> Result>> { let key = (sha.to_string(), path.to_string()); let kept = BLAMES.lock().unwrap().get(&key); if let Some(kept) = kept { return Ok(kept); } let _turn = BLAMING.acquire().await.expect("never closed"); let found = read::blame(git, sha, path, None).await; let found = Arc::new(found.map_err(AppError::internal)?); BLAMES.lock().unwrap().put(key, found.clone()); Ok(found) } /// `dev.knotless.listCommits`'s answer: history, a page at a time, newest /// first. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct History { #[serde(rename = "ref")] refname: String, path: String, commits: Vec, /// The newest run of each workflow at each of `commits`. checks: Vec, /// Where the next page starts; none on the last. #[serde(skip_serializing_if = "Option::is_none")] cursor: Option, } /// The most runs read for a page of `listCommits`' checks: the newest of /// each workflow at each commit are among them unless a page's commits /// were run that many times. const COMMIT_CHECKS_MAX: usize = 1000; /// The newest run of each workflow at each of `commits`, of `repo`. async fn commit_checks(app: &App, repo: &Repo, commits: &[CommitView]) -> Vec { let shas: Vec = commits.iter().map(|c| c.sha.clone()).collect(); let of = CheckFilter { commits: Some(&shas), ..CheckFilter::default() }; let basic = repo.basic(app).await; let all = checks(app, &repo.reference(), &basic, of, COMMIT_CHECKS_MAX); // The runs come newest first, so the first of each is the one kept. let mut newest = BTreeMap::new(); for c in all { newest .entry((c.workflow.clone(), c.commit.clone())) .or_insert(c); } newest.into_values().collect() } /// `dev.knotless.listCommits`. pub async fn list_commits( State(app): State>, Query(p): Query, ) -> Result { let limit = p.limit.unwrap_or(COMMITS_PER_PAGE).clamp(1, COMMITS_MAX); let skip = skipped(p.cursor.as_deref())?; let repo = found(&app, &p.repo).await?; if unpushed(&repo, &p)? { return Ok(answer(History { commits: Vec::new(), checks: Vec::new(), refname: repo.head.clone(), path: String::new(), cursor: None, })); } let (rev, path) = located(&repo, &p)?; let log = repo.objects()?.log(&rev, &path, skip, limit); let log = log.map_err(not_found)?; let cursor = (log.len() == limit).then(|| (skip + limit).to_string()); let mut commits = shown(&app, &repo, log).await; if p.subject_only { for c in &mut commits { let end = c.message.find('\n').unwrap_or(c.message.len()); c.message.truncate(end); } } let checks = commit_checks(&app, &repo, &commits).await; Ok(answer(History { commits, checks, refname: rev, path, cursor, })) } /// One branch or tag of a repository, and, when asked, the commit it /// names and that commit's newest run of each workflow. #[derive(Serialize)] pub struct RefView { name: String, #[serde(skip_serializing_if = "Option::is_none")] commit: Option, #[serde(skip_serializing_if = "Option::is_none")] checks: Option>, } /// Each of `names`, branches or tags, as a view: with the commit it /// names, among `tips` in the same order, and its checks when given. async fn ref_views( app: &App, repo: &Repo, names: Vec, tips: Option>, ) -> Vec { let Some(tips) = tips else { let bare = |name| RefView { name, commit: None, checks: None, }; return names.into_iter().map(bare).collect(); }; let commits = shown(app, repo, tips).await; let mut at: HashMap> = HashMap::new(); for c in commit_checks(app, repo, &commits).await { at.entry(c.commit.clone()).or_default().push(c); } let view = |(name, commit): (String, CommitView)| RefView { name, checks: Some(at.get(&commit.sha).cloned().unwrap_or_default()), commit: Some(commit), }; names.into_iter().zip(commits).map(view).collect() } /// `dev.knotless.listBranches`'s answer: a page of a repository's /// branches, the default first. #[derive(Serialize)] pub struct Branches { branches: Vec, /// Where the next page starts; none on the last. #[serde(skip_serializing_if = "Option::is_none")] cursor: Option, } /// `dev.knotless.listBranches`. pub async fn list_branches( State(app): State>, Query(p): Query, ) -> Result { let limit = p.limit.unwrap_or(BRANCHES_PER_PAGE).clamp(1, BRANCHES_MAX); let skip = skipped(p.cursor.as_deref())?; let repo = found_for_reader(&app, &p.repo).await?; let all = repo.branches()?; let cursor = (all.len() > skip + limit).then(|| (skip + limit).to_string()); let page: Vec = all.into_iter().skip(skip).take(limit).collect(); // Each tip read only when asked: the branch menu asks for a thousand. let tips = if p.commits { let o = repo.objects()?; let read: anyhow::Result> = page.iter().map(|name| o.tip(name)).collect(); Some(read.map_err(AppError::internal)?) } else { None }; Ok(answer(Branches { branches: ref_views(&app, &repo, page, tips).await, cursor, })) } /// `dev.knotless.listTags`'s answer: a page of a repository's tags, the /// newest commit first. #[derive(Serialize)] pub struct Tags { tags: Vec, /// Where the next page starts; none on the last. #[serde(skip_serializing_if = "Option::is_none")] cursor: Option, } /// `dev.knotless.listTags`. pub async fn list_tags( State(app): State>, Query(p): Query, ) -> Result { let limit = p.limit.unwrap_or(BRANCHES_PER_PAGE).clamp(1, BRANCHES_MAX); let skip = skipped(p.cursor.as_deref())?; let repo = found_for_reader(&app, &p.repo).await?; let all = repo.objects()?.tags().map_err(AppError::internal)?; let cursor = (all.len() > skip + limit).then(|| (skip + limit).to_string()); let (names, tips) = all.into_iter().skip(skip).take(limit).unzip(); Ok(answer(Tags { tags: ref_views(&app, &repo, names, p.commits.then_some(tips)).await, cursor, })) } /// `dev.knotless.getCommit`'s answer: a commit and what it changed. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct Change { commit: CommitView, files: Vec, } /// `dev.knotless.getCommit`. A merge's diff is against its first parent: /// what it brought into the branch it was made on, where git's own would /// show only what differs from both, nothing at all for a clean merge. /// With `firstRows`, only the first files come with their rows, as the /// commit page draws them at once; with `paths`, only those files, rows /// and all, for the rest as a reader comes to them: as a pull's files /// come from `getThread` and `getPullDiff`. Its parameters are pairs, as /// `paths` repeats. pub async fn get_commit( State(app): State>, Query(pairs): Query>, ) -> Result { let one = |name| super::items::named(&pairs, name).first().copied(); let (Some(repo), Some(sha)) = (one("repo"), one("sha")) else { return Err(invalid("repo and sha are required".into())); }; let paths = super::items::named(&pairs, "paths"); let max = super::items::DIFF_PATHS_MAX; if paths.len() > max { return Err(invalid(format!("paths names at most {max} files"))); } let rows = match (paths.is_empty(), one("firstRows")) { (true, Some("true")) => Rows::First, _ => Rows::All, }; let repo = found(&app, repo).await?; let rev = read::rev(Some(sha)).map_err(not_found)?; let log = repo.objects()?.log(&rev, "", 0, 1).map_err(not_found)?; let c = shown(&app, &repo, log).await.remove(0); let files = files( &repo.git, &["show", "--format=", "--patch", "--diff-merges=first-parent"], &c.sha, &paths, rows, ) .await?; Ok(answer(Change { commit: c, files })) }