//! What the web pages show, as `dev.knotless.*` XRPC queries, each with //! its lexicon under `lexicons/`. The pages themselves are a SvelteKit app in `web/`: this server //! hands every request it does not answer itself to it, and the app reads //! back here for what to show. //! //! Writing is for git, the CLI and agents over MCP, and for the account //! logged in to the pages, through the same code: a star, a follow, and //! what is said on a pull or an issue ([`threads`]), and opening one ([`opening`]). The app's types are made from the lexicons, and the tests hold every answer //! they receive to its lexicon (harness/answers.mjs). use crate::card::{self, Card}; use crate::diff::{self, Kind, Status}; use crate::index::{Assembled, CheckFilter, Index}; use crate::languages::{self, Language}; use crate::objects::{self, Objects}; use crate::{App, AppError, highlight, repo_name}; use axum::Router; use axum::extract::{Path, Query, Request, State}; use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use futures::FutureExt; use knotless::closing; use knotless::forge::{self, Item, Thread}; use knotless::git::Git; use knotless::profile; use knotless::read; use knotless::records::{ Attachment, CheckOutput, CheckRecord, CheckStep, Freeing, ImageAttachment, MinutesRecord, ProfileRecord, PullHead, RepoRecord, RepoRef, SiteRecord, nsid, }; use knotless::remote::Remote; use knotless::session; use knotless::store; use knotless::telemetry::{fill, looked}; use knotless::url::RepoUrl; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::Write; use std::sync::atomic::Ordering; use std::sync::{Arc, LazyLock, Mutex}; use std::time::{Duration, SystemTime}; mod account_data; mod account_settings; mod actor; mod archive; mod badges; mod cargo; mod controls; mod delegation; mod feed; mod feeds; mod follows; mod items; mod landing; mod live; mod notifications; mod opening; mod orgs; mod pages; mod preview; pub mod push; mod queue; mod reactions; mod releases; mod repo; mod repos; mod runs; mod search; mod settings; pub(crate) use settings::Counts; mod stars; mod steps; mod subscriptions; mod themes; mod threads; mod views; use actor::{ActorView, RepoViewBasic, repo_basic}; use queue::InQueue; use views::*; pub use actor::{actor, get_actor_activity, get_actor_repos, get_actor_stars, list_actor_items}; pub use feed::get_feed; use follows::{follow_actor, get_followers, get_follows, unfollow_actor}; use items::{get_pull_diff, get_thread, list_items}; use notifications::{get_unread_count, list_notifications, update_seen}; pub use pages::pages; use push::{register_push, unregister_push}; use reactions::{add_reaction, get_reactions, remove_reaction}; use repo::{ get_blame, get_commit, get_file, get_last_commits, get_repo, get_tree, list_branches, list_commits, list_paths, list_tags, }; use stars::{get_viewer, star_repo, unstar_repo}; pub use themes::{cookie as palette_cookie, palette}; /// The queries the pages ask. pub fn queries() -> Router> { Router::new() .route("/xrpc/dev.knotless.getRepo", get(get_repo)) .route("/xrpc/dev.knotless.searchCode", get(search::search_code)) .route( "/xrpc/dev.knotless.listReleases", get(releases::list_releases), ) .route("/xrpc/dev.knotless.getViewer", get(get_viewer)) .route("/xrpc/dev.knotless.getOrg", get(orgs::get_org)) .route("/_live/repo", get(live::follow_repo)) .route("/_live/check", get(live::follow_check)) .route("/xrpc/dev.knotless.getActorStars", get(get_actor_stars)) .route("/xrpc/dev.knotless.listActorItems", get(list_actor_items)) .route("/xrpc/dev.knotless.starRepo", post(star_repo)) .route("/xrpc/dev.knotless.unstarRepo", post(unstar_repo)) .route("/xrpc/dev.knotless.createRepo", post(repos::create_repo)) .route("/xrpc/dev.knotless.forkRepo", post(repos::fork_repo)) .route( "/xrpc/dev.knotless.getRepoSettings", get(settings::get_repo_settings), ) .route( "/xrpc/dev.knotless.getRepoStorage", get(settings::get_repo_storage), ) .route("/xrpc/dev.knotless.updateRepo", post(settings::update_repo)) .route("/xrpc/dev.knotless.renameRepo", post(settings::rename_repo)) .route( "/xrpc/dev.knotless.compactRepo", post(settings::compact_repo), ) .route( "/xrpc/dev.knotless.transferRepo", post(settings::transfer_repo), ) .route( "/xrpc/dev.knotless.archiveRepo", post(settings::archive_repo), ) .route( "/xrpc/dev.knotless.acceptTransfer", post(settings::accept_transfer), ) .route("/xrpc/dev.knotless.deleteRepo", post(settings::delete_repo)) .route( "/xrpc/dev.knotless.createBranch", post(controls::create_branch), ) .route( "/xrpc/dev.knotless.deleteBranch", post(controls::delete_branch), ) .route( "/xrpc/dev.knotless.renameBranch", post(controls::rename_branch), ) .route( "/xrpc/dev.knotless.protectBranch", post(controls::protect_branch), ) .route( "/xrpc/dev.knotless.unprotectBranch", post(controls::unprotect_branch), ) .route( "/xrpc/dev.knotless.defineLabel", post(controls::define_label), ) .route( "/xrpc/dev.knotless.removeLabel", post(controls::remove_label), ) .route("/xrpc/dev.knotless.shareRepo", post(controls::share_repo)) .route( "/xrpc/dev.knotless.unshareRepo", post(controls::unshare_repo), ) .route("/xrpc/dev.knotless.setTriager", post(controls::set_triager)) .route("/xrpc/dev.knotless.addAgent", post(controls::add_agent)) .route( "/xrpc/dev.knotless.removeAgent", post(controls::remove_agent), ) .route( "/xrpc/dev.knotless.removeTriager", post(controls::remove_triager), ) .route("/xrpc/dev.knotless.listThemes", get(themes::list_themes)) .route("/xrpc/dev.knotless.listSteps", get(steps::list_steps)) .route("/xrpc/dev.knotless.getStep", get(steps::get_step)) .route("/xrpc/dev.knotless.createTheme", post(themes::create_theme)) .route("/xrpc/dev.knotless.deleteTheme", post(themes::delete_theme)) .route("/xrpc/dev.knotless.followActor", post(follow_actor)) .route("/xrpc/dev.knotless.unfollowActor", post(unfollow_actor)) .route("/xrpc/dev.knotless.getFollowers", get(get_followers)) .route("/xrpc/dev.knotless.getFollows", get(get_follows)) .route("/xrpc/dev.knotless.addReaction", post(add_reaction)) .route("/xrpc/dev.knotless.removeReaction", post(remove_reaction)) .route("/xrpc/dev.knotless.getReactions", get(get_reactions)) .route( "/xrpc/dev.knotless.listNotifications", get(list_notifications), ) .route("/xrpc/dev.knotless.getUnreadCount", get(get_unread_count)) .route("/xrpc/dev.knotless.updateSeen", post(update_seen)) .route("/xrpc/dev.knotless.registerPush", post(register_push)) .route("/xrpc/dev.knotless.unregisterPush", post(unregister_push)) .route("/xrpc/dev.knotless.getTree", get(get_tree)) .route("/xrpc/dev.knotless.getLastCommits", get(get_last_commits)) .route("/xrpc/dev.knotless.listPaths", get(list_paths)) .route("/xrpc/dev.knotless.getFile", get(get_file)) .route("/xrpc/dev.knotless.getBlame", get(get_blame)) .route("/xrpc/dev.knotless.listCommits", get(list_commits)) .route("/xrpc/dev.knotless.getCommit", get(get_commit)) .route("/xrpc/dev.knotless.listBranches", get(list_branches)) .route("/xrpc/dev.knotless.listTags", get(list_tags)) .route("/xrpc/dev.knotless.listItems", get(list_items)) .route("/xrpc/dev.knotless.getThread", get(get_thread)) .route("/xrpc/dev.knotless.getPullDiff", get(get_pull_diff)) .route("/xrpc/dev.knotless.getQueue", get(queue::get_queue)) .route("/xrpc/dev.knotless.listRuns", get(runs::list_runs)) .route("/xrpc/dev.knotless.getCheck", get(get_check)) .route("/xrpc/dev.knotless.getAttachment", get(get_attachment)) .route("/xrpc/dev.knotless.getCheckImage", get(get_check_image)) .route("/xrpc/dev.knotless.getAvatar", get(get_avatar)) .merge(landing_routes()) .merge(account_settings::routes()) .merge(account_data::routes()) .merge(threads::routes()) .merge(subscriptions::routes()) .merge(opening::routes()) .merge(preview::routes()) .merge(delegation::routes()) .merge(orgs::routes()) .layer(middleware::from_fn(renamed)) .layer(tower_http::compression::CompressionLayer::new()) } /// Landing pulls and running checks, from the pages ([`landing`]). fn landing_routes() -> Router> { use landing::*; Router::new() .route("/xrpc/dev.knotless.getMergeStatus", get(get_merge_status)) .route("/xrpc/dev.knotless.mergePull", post(merge_pull)) .route("/xrpc/dev.knotless.revertPull", post(revert_pull)) .route("/xrpc/dev.knotless.queuePull", post(queue_pull)) .route("/xrpc/dev.knotless.setAutoMerge", post(set_auto_merge)) .route("/xrpc/dev.knotless.leaveQueue", post(leave_queue)) .route("/xrpc/dev.knotless.retargetPull", post(retarget_pull)) .route("/xrpc/dev.knotless.requestRerun", post(request_rerun)) .route("/xrpc/dev.knotless.listWorkflows", get(list_workflows)) .route("/xrpc/dev.knotless.listRunners", get(list_runners)) .route("/xrpc/dev.knotless.addRunner", post(add_runner)) .route("/xrpc/dev.knotless.removeRunner", post(remove_runner)) } pub fn routes() -> Router> { let compressed = Router::new() .route("/{authority}/{repo}/raw/{*rest}", get(raw)) .route("/{authority}/{repo}/commits.atom", get(feeds::commits)) .route("/{authority}/{repo}/tags.atom", get(feeds::tags)) .route("/{authority}/{repo}/releases.atom", get(feeds::releases)) .route("/card.png", get(home_card)) .route("/{authority}/{repo}/card.png", get(repo_card)) .route( "/{authority}/{repo}/{kind}/{did}/{rkey}/card.png", get(item_card_png), ) .layer(tower_http::compression::CompressionLayer::new()); // A release's files are sent as they are kept: most are archives, // compressed already, and their length is known. A revision's archive // is compressed as it is written. Router::new() .route( "/{authority}/{repo}/releases/download/{*rest}", get(releases::download), ) .route("/cargo/{did}/{key}/config.json", get(cargo::config)) .route("/cargo/{did}/{key}/me", get(cargo::me)) .route("/cargo/{did}/{key}/api/v1/crates", get(cargo::search)) .route( "/cargo/{did}/{key}/api/v1/crates/new", axum::routing::put(cargo::publish), ) .route( "/cargo/{did}/{key}/api/v1/crates/{name}/{version}/yank", axum::routing::delete(cargo::yank), ) .route( "/cargo/{did}/{key}/api/v1/crates/{name}/{version}/unyank", axum::routing::put(cargo::unyank), ) .route( "/cargo/{did}/{key}/dl/{name}/{version}/download", get(cargo::download), ) .route("/cargo/{did}/{key}/{*path}", get(cargo::index)) .route("/{authority}/{repo}/archive/{*rest}", get(archive::archive)) .merge(compressed) .layer(middleware::from_fn(renamed)) } /// Where a repository is now, on the answer to a request that used a name /// it had before: its name, and its owner if it moved to another account. #[derive(Clone)] struct Renamed(String); /// Points a request made with a repository's old name at the same path /// under its new one. The redirect is temporary: the old name can be given /// to another repository. async fn renamed(req: Request, next: Next) -> Response { let path = req.uri().path().to_string(); let query = req .uri() .query() .map(|q| format!("?{q}")) .unwrap_or_default(); let mut res = next.run(req).await; if let Some(Renamed(now)) = res.extensions_mut().remove::() { // "", the authority, the name, and the rest. let mut parts: Vec<&str> = path.splitn(4, '/').collect(); match now.split_once('/') { Some((owner, name)) => (parts[1], parts[2]) = (owner, name), None => parts[2] = &now, } let to = format!("{}{query}", parts.join("/")); res.headers_mut() .insert(header::LOCATION, HeaderValue::from_str(&to).unwrap()); } res } /// Text inlined above, past this, is a link to the raw file instead. const INLINE: usize = 1024 * 1024; /// A file's diff with more rows than this is folded away. const DIFF_ROWS: usize = 2000; /// The first files of a pull drawn with their rows, as its files page /// draws them at once: so many at most, of so many lines in all, and none /// changing more lines than [`FIRST_HELD`], which waits to be asked for. const FIRST_FILES: usize = 10; const FIRST_LINES: usize = 1500; const FIRST_HELD: usize = 400; /// Which files of a diff come with their rows. #[derive(Clone, Copy, PartialEq, Debug)] pub(crate) enum Rows { None, All, /// The first, as a pull's files page draws them at once. First, } /// A path for a URL: each segment percent-encoded, the slashes kept. fn url_path(path: &str) -> String { let mut out = String::new(); for b in path.bytes() { match b { b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' | b'/' | b':' | b'@' => out.push(b as char), b => write!(out, "%{b:02X}").unwrap(), } } out } /// Caching for a file at `rev`: forever when it names a commit by hash, /// else checked again each time, as a branch can move. fn cache_for(rev: &str) -> &'static str { if read::is_pinned(rev) { "public, max-age=31536000, immutable" } else { "no-cache" } } /// `value` as the answer, checked again each time: even at a commit hash /// it shows the repository as it is now, its authors' accounts, and a shape /// a later release may change. fn answer(value: T) -> Response { let mut res = axum::Json(value).into_response(); res.headers_mut() .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")); res } /// A failure, as the pages show it: a status and why. #[derive(Debug)] pub struct Shown(StatusCode, String); impl From for Shown { fn from(e: AppError) -> Shown { Shown(e.0, e.1) } } /// The account logged in to the pages that sent a write, as their own /// writes take it, for a method that is not only the pages'. pub(crate) fn logged_in( app: &App, viewer: crate::viewer::Viewer, headers: &HeaderMap, act: &str, ) -> std::result::Result { stars::logged_in(app, viewer, headers, act).map_err(|Shown(code, why)| AppError(code, why)) } /// A failure, as XRPC has one: its kind, by name, and why. #[derive(Serialize)] pub struct Failure { error: String, message: String, } /// The name XRPC gives a failure with `status`. fn failure_name(status: StatusCode) -> &'static str { match status { StatusCode::BAD_REQUEST => "InvalidRequest", StatusCode::NOT_FOUND => "NotFound", StatusCode::UNAUTHORIZED => "AuthenticationRequired", StatusCode::FORBIDDEN => "Forbidden", StatusCode::CONFLICT => "Conflict", StatusCode::BAD_GATEWAY => "UpstreamFailure", _ => "InternalServerError", } } impl IntoResponse for Shown { fn into_response(self) -> Response { if self.0 == StatusCode::TEMPORARY_REDIRECT { let mut res = self.0.into_response(); res.extensions_mut().insert(Renamed(self.1)); return res; } let Shown(status, message) = behind(self); let failure = Failure { error: failure_name(status).into(), message, }; let cause = crate::telemetry::Cause(failure.message.clone()); let mut res = (status, axum::Json(failure)).into_response(); res.extensions_mut().insert(cause); res.headers_mut() .insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); res } } /// `shown`, or for a write refused for want of scope, why in a sentence /// that says how to grant it: the gateway's, when it was the owner's grant /// to it that fell short, else the login's to the pages. fn behind(shown: Shown) -> Shown { let said = &shown.1; if said.contains(knotless::oauth::GRANT_BEHIND) { return Shown(StatusCode::FORBIDDEN, knotless::oauth::GRANT_BEHIND.into()); } match knotless::oauth::out_of_scope(said) { true => Shown(StatusCode::FORBIDDEN, knotless::oauth::LOGIN_BEHIND.into()), false => shown, } } fn not_found(e: anyhow::Error) -> Shown { Shown(StatusCode::NOT_FOUND, crate::shown(&e)) } type Result = std::result::Result; static HANDLES: LazyLock>> = LazyLock::new(Mutex::default); /// How long a handle the index keeps is shown before it is looked up again, /// behind the page showing it. An identity event looks it up at once. const HANDLE_KEPT_FOR: Duration = Duration::from_secs(24 * 60 * 60); /// How long an account's having no handle is kept before it is looked up /// again: a handle's DNS can go live just after the account claims it. const NO_HANDLE_KEPT_FOR: Duration = Duration::from_secs(600); /// Seconds since the epoch. fn now_secs() -> i64 { let since = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH); since.unwrap_or_default().as_secs() as i64 } /// The handle `did` claims, if it resolves back to `did`; else the DID. /// An appview keeps them in its index, where they outlast a restart and /// search finds them; without one, they are kept in memory. pub(crate) async fn handle(app: &App, did: &str) -> String { if let Some(index) = app.index.clone() { let kept = looked("handles", index.handle(did).ok().flatten()); let Some((handle, checked)) = kept else { let found = rename(app.ctx.clone(), index, did.to_string()).await; return found.unwrap_or(did.to_string()); }; let kept_for = match handle { Some(_) => HANDLE_KEPT_FOR, None => NO_HANDLE_KEPT_FOR, }; if now_secs() - checked >= kept_for.as_secs() as i64 { // Marked as looked up now, so the pages showing it meanwhile // look it up once between them. let _ = index.set_handle(did, handle.as_deref(), now_secs()); tokio::spawn(rename(app.ctx.clone(), index, did.to_string())); } return handle.unwrap_or(did.to_string()); } if let Some(known) = looked("handles", HANDLES.lock().unwrap().get(did)) { return known.clone(); } let shown = knotless::identity::handle(&app.ctx, did) .await .unwrap_or(did.to_string()); HANDLES .lock() .unwrap() .insert(did.to_string(), shown.clone()); shown } /// Looks `did`'s handle up now and keeps what was found in `index`: none, /// when the handle it claims does not resolve back to it. A DID document /// that cannot be read says nothing of its handle, and changes nothing. pub(crate) async fn rename( ctx: knotless::env::Ctx, index: Arc, did: String, ) -> Option { let found = knotless::identity::verified_handle(&ctx, &did).await; let found = found.ok()?; let _ = index.set_handle(&did, found.as_deref(), now_secs()); found } /// How long what an account says of itself is kept before it is looked up /// again: its picture, and the addresses it commits as. const KEPT_FOR: Duration = Duration::from_secs(600); /// How long a page waits for something about an account never looked up /// before, before it is shown without it; the lookup goes on, for the next. const LOOKUP_WAIT: Duration = Duration::from_secs(5); /// A lookup going on, shared by every page waiting for it. type Looking = futures::future::Shared>>; /// Values looked up over the network, by key. One older than its `ttl` is /// still shown, and looked up again behind it, so a page waits only for a /// value never found, and for `wait` at most; one lookup at a time a key, /// however many pages ask. A lookup that fails keeps what was found /// before. They are kept in the cache directory too, so the server a /// deploy starts has them at once, rather than asking every account's PDS /// again just as relays may be fetching whole repositories from it. struct Kept { /// The file they are kept in, in the cache directory. name: &'static str, ttl: Duration, wait: Duration, values: Mutex>, /// The file, once read. file: std::sync::OnceLock, looking: Mutex>>, } impl Kept where V: Clone + Serialize + serde::de::DeserializeOwned + Send + Sync + 'static, { fn new(name: &'static str, ttl: Duration, wait: Duration) -> Kept { Kept { name, ttl, wait, values: Mutex::default(), file: std::sync::OnceLock::new(), looking: Mutex::default(), } } /// What is kept in the cache directory `dir`, read the first time. fn load(&self, dir: &std::path::Path) { self.file.get_or_init(|| { let file = dir.join(self.name); let read: HashMap = store::read(&file).unwrap_or_default(); let mut values = self.values.lock().unwrap(); for (key, kept) in read { values.entry(key).or_insert(kept); } file }); } /// The value kept for `key`, if any, and whether it is still fresh. fn get(&self, key: &str) -> Option<(V, bool)> { let values = self.values.lock().unwrap(); let (at, value) = values.get(key)?; let age = at.elapsed().unwrap_or_default(); Some((value.clone(), age < self.ttl)) } fn put(&self, key: &str, value: V) { let mut values = self.values.lock().unwrap(); values.insert(key.to_string(), (SystemTime::now(), value)); let Some(file) = self.file.get() else { return; }; // A cache: one not written is looked up again. let _ = store::write(file, &*values, false); } /// The value for `key`, as `look` finds it if none fresh is kept: none /// if it never found one, or not within `wait`. async fn through(&'static self, dir: &std::path::Path, key: &str, look: F) -> Option where F: Future> + Send + 'static, { self.load(dir); let kept = self.get(key); let cache = self.name.trim_end_matches(".json"); let _ = looked(cache, kept.as_ref().filter(|k| k.1)); if let Some((value, true)) = kept { return Some(value); } let looking = self .looking .lock() .unwrap() .entry(key.to_string()) .or_insert_with(|| { let key = key.to_string(); let found = tokio::spawn(async move { let found = look.await; fill( cache, &key, &found.as_ref().ok_or("not read: its PDS did not answer"), ); if let Some(value) = &found { self.put(&key, value.clone()); } self.looking.lock().unwrap().remove(&key); found }); found.map(|ended| ended.ok().flatten()).boxed().shared() }) .clone(); match kept { Some((value, _)) => Some(value), None => tokio::time::timeout(self.wait, looking) .await .ok() .flatten(), } } } static AVATARS: LazyLock>> = LazyLock::new(|| Kept::new("avatars.json", KEPT_FOR, LOOKUP_WAIT)); static CLAIMS: LazyLock>> = LazyLock::new(|| Kept::new("claims.json", KEPT_FOR, LOOKUP_WAIT)); /// The accounts `author:` filters named by handle, by handle: none for a /// handle that names none. static AUTHORS: LazyLock>> = LazyLock::new(|| Kept::new("authors.json", KEPT_FOR, LOOKUP_WAIT)); /// Where Bluesky's CDN serves profile pictures, by DID and blob CID. const AVATAR_CDN: &str = "https://cdn.bsky.app/img/avatar/plain"; /// An account's profile records, by collection and record key. type Records = Vec<(String, String, serde_json::Value)>; /// `remote`'s knotless profile, from [`profile_records`]: as the index /// keeps it while a feed keeps it current, else as its PDS has it. An error /// if the PDS did not answer. async fn own_profile(app: &App, remote: &Remote) -> Result> { let watching = app.watching.load(Ordering::Relaxed); let records = profile_records(app.index.as_deref(), watching, remote).await; let unanswered = format!("{}: its PDS did not answer", remote.did()); let records = records.ok_or(AppError(StatusCode::BAD_GATEWAY, unanswered))?; let own = records .into_iter() .find(|(c, k, _)| c == nsid::PROFILE && k == profile::RKEY); Ok(own.and_then(|(_, _, value)| serde_json::from_value(value).ok())) } /// The records `remote`'s profile is made of: as `index` keeps them, while /// the firehose keeps them current (`watching`), else as its PDS has them, /// and then kept. Nothing if the PDS did not answer for both profiles. async fn profile_records( index: Option<&Index>, watching: bool, remote: &Remote, ) -> Option { let did = remote.did(); let kept = index .filter(|_| watching) .and_then(|i| i.profile(did).ok().flatten()); if kept.is_some() { return kept; } // The Bluesky links are listed with the profiles, all at once, though // they are shown only when knotless's own profile names none: asked // after the Bluesky profile, which names them, they would be a second // round of asking the PDS. let links = remote.list_all::(nsid::BSKY_LINK); let bsky = remote.get_record::(nsid::BSKY_PROFILE, profile::RKEY); let own = remote.get_record::(nsid::PROFILE, profile::RKEY); let (bsky, own, links) = futures::join!(bsky, own, links); let (bsky, own) = (bsky.ok()?, own.ok()?); let whole = links.is_ok(); let mut records: Records = Vec::new(); records.extend(bsky.map(|r| (nsid::BSKY_PROFILE.to_string(), r.rkey, r.value))); records.extend(own.map(|r| (nsid::PROFILE.to_string(), r.rkey, r.value))); let links = links.unwrap_or_default().into_iter(); records.extend(links.map(|r| (nsid::BSKY_LINK.to_string(), r.rkey, r.value))); // Kept only whole: links that could not be listed are asked for again. if let Some(index) = index.filter(|_| whole) { let _ = index.keep_profile(did, &records); } Some(records) } /// A link on an account's profile: always an http or https address. #[derive(Serialize, Clone, Debug, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ProfileLink { url: String, #[serde(skip_serializing_if = "Option::is_none")] title: Option, } /// The links a repository's record names that its pages can show: each an /// http or https address, and no more than a repository shows. fn repo_links(named: Vec) -> Vec { let web = named.into_iter().filter(|l| profile::is_web(&l.url)); let shown = web.take(knotless::manage::MAX_LINKS).map(|l| ProfileLink { url: l.url, title: l.title, }); shown.collect() } /// Who an account says it is: its knotless profile, field by field over /// its Bluesky one. #[derive(Default)] struct About { name: Option, bio: Option, pronouns: Option, links: Vec, avatar: Option, /// Whether its knotless profile says it is a bot. bot: bool, /// Whether both profiles were read: an account whose PDS did not /// answer shows as having none, but is not kept so. answered: bool, } /// The links `did`'s Bluesky profile `bsky` shows, in order: the /// `app.bsky.actor.link` records it names in its own repo, of those /// `listed`, by record key. fn bsky_links( did: &str, bsky: &serde_json::Value, mut listed: HashMap, ) -> Vec { let mine = format!("at://{did}/{}/", nsid::BSKY_LINK); let named = bsky["links"].as_array().into_iter().flatten(); let rkeys = named.filter_map(|l| l["uri"].as_str()?.strip_prefix(&mine)); let found = rkeys.filter_map(|rkey| listed.remove(rkey)); let links = found.take(profile::MAX_LINKS).map(|r| ProfileLink { url: r["url"].as_str().unwrap_or_default().to_string(), title: r["title"].as_str().map(str::to_string), }); links.collect() } /// Who `did` says it is, from its profile `records`, if they were read. fn about(did: &str, records: Option) -> About { let answered = records.is_some(); let records = records.unwrap_or_default(); let one = |collection: &str| { let found = records .iter() .find(|(c, k, _)| c == collection && k == profile::RKEY); found.map(|(_, _, value)| value.clone()) }; let bsky = one(nsid::BSKY_PROFILE).unwrap_or_default(); let own = one(nsid::PROFILE).and_then(|v| serde_json::from_value::(v).ok()); let own = own.unwrap_or_default(); let listed = records.into_iter().filter(|(c, _, _)| c == nsid::BSKY_LINK); let listed = listed.map(|(_, rkey, value)| (rkey, value)).collect(); let text = |field: &str| { let said = bsky[field].as_str().map(str::to_string); said.filter(|t| !t.trim().is_empty()) }; let mut links: Vec = own .links .into_iter() .map(|l| ProfileLink { url: l.url, title: l.title, }) .collect(); if links.is_empty() { links = bsky_links(did, &bsky, listed); } if links.is_empty() { links.extend(text("website").map(|url| ProfileLink { url, title: None })); } links.retain(|l| profile::is_web(&l.url)); let avatar = match own.avatar { Some(own) => Some(own_avatar_path(did, &own.link.cid)), None => bsky["avatar"]["ref"]["$link"] .as_str() .map(|cid| format!("{AVATAR_CDN}/{did}/{cid}@jpeg")), }; About { name: own.display_name.or_else(|| text("displayName")), bio: own.description.or_else(|| text("description")), pronouns: own.pronouns.or_else(|| text("pronouns")), links, avatar, bot: own.bot, answered, } } #[derive(Deserialize)] pub struct ProfileParams { actor: String, } /// `dev.knotless.getProfile#profileView`. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct ProfileView { did: String, #[serde(skip_serializing_if = "Option::is_none")] handle: Option, #[serde(skip_serializing_if = "Option::is_none")] display_name: Option, #[serde(skip_serializing_if = "Option::is_none")] description: Option, #[serde(skip_serializing_if = "Option::is_none")] pronouns: Option, #[serde(skip_serializing_if = "Vec::is_empty")] links: Vec, #[serde(skip_serializing_if = "Option::is_none")] avatar: Option, #[serde(skip_serializing_if = "Option::is_none")] followers_count: Option, #[serde(skip_serializing_if = "Option::is_none")] follows_count: Option, #[serde(skip_serializing_if = "Option::is_none")] viewer: Option, #[serde(skip_serializing_if = "std::ops::Not::not")] organization: bool, #[serde(skip_serializing_if = "std::ops::Not::not")] bot: bool, #[serde(skip_serializing_if = "Vec::is_empty")] orgs: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] badges: Vec, } /// `dev.knotless.defs#actorViewerState`. #[derive(Serialize)] pub struct ActorViewerState { #[serde(skip_serializing_if = "Option::is_none")] following: Option, #[serde(skip_serializing_if = "Option::is_none")] blocking: Option, } /// `dev.knotless.getProfile`'s failure for an account not found, as `e` /// says. fn no_profile(e: anyhow::Error) -> Response { let failure = Failure { error: "ProfileNotFound".into(), message: format!("{e:#}"), }; (StatusCode::BAD_REQUEST, axum::Json(failure)).into_response() } tokio::task_local! { /// Where the request being answered reached this server. static ORIGIN: String; } /// Answers the request with its origin known, for the addresses answers /// give in full: as the one asking reached this server, through the pages, /// which ask for the site they serve, or directly. pub async fn origin(req: Request, next: Next) -> Response { let origin = origin_of(req.headers()); ORIGIN.scope(origin, next.run(req)).await } /// A failure an XRPC method answered as text, as XRPC has one instead: /// its kind, by name, and why. Its extensions go with it, the cause its /// request's log line gives among them. pub async fn xrpc_failures(req: Request, next: Next) -> Response { let xrpc = req.uri().path().starts_with("/xrpc/"); let res = next.run(req).await; let text = res.headers().get(header::CONTENT_TYPE); let text = text.is_some_and(|t| t.as_bytes().starts_with(b"text/plain")); if !xrpc || !text || res.status().is_success() { return res; } let (parts, body) = res.into_parts(); let said = axum::body::to_bytes(body, 64 * 1024) .await .unwrap_or_default(); let failure = Failure { error: failure_name(parts.status).into(), message: String::from_utf8_lossy(&said).into_owned(), }; let mut answer = (parts.status, axum::Json(failure)).into_response(); *answer.extensions_mut() = parts.extensions; answer } fn origin_of(headers: &HeaderMap) -> String { let header = |name: &str| headers.get(name).and_then(|v| v.to_str().ok()); let proto = header("x-forwarded-proto").unwrap_or("http"); let host = header("x-forwarded-host").or(header(header::HOST.as_str())); format!("{proto}://{}", host.unwrap_or_default()) } /// `path`, an address on this server, in full, at the request's origin. fn absolute(path: String) -> String { match ORIGIN.try_with(String::clone) { Ok(origin) if path.starts_with('/') => format!("{origin}{path}"), _ => path, } } /// `dev.knotless.getProfile`: how an account shows on knotless, with its /// picture's address in full, and its follows as the index counts them. pub async fn get_profile( State(app): State>, axum::Extension(viewer): axum::Extension, Query(p): Query, ) -> Response { let at = RepoUrl { authority: p.actor, name: "_".into(), }; let remote = match Remote::connect(&app.ctx, &at).await { Ok(remote) => remote, Err(e) => return no_profile(e), }; let watching = app.watching.load(Ordering::Relaxed); let records = profile_records(app.index.as_deref(), watching, &remote).await; let about = about(remote.did(), records); if about.answered { AVATARS.load(&app.ctx.env.cache_dir); AVATARS.put(remote.did(), about.avatar.clone()); } let avatar = about.avatar.map(absolute); let index = app.index.as_ref(); let counts = index.and_then(|i| i.follow_counts(remote.did()).ok()); let following = |v: &String| index.and_then(|i| i.follow_of(v, remote.did()).ok()); let blocking = |v: &String| index.and_then(|i| i.block_of(v, remote.did()).ok()); let state = |v: &String| ActorViewerState { following: following(v).flatten(), blocking: blocking(v).flatten(), }; let view = ProfileView { did: remote.did().to_string(), handle: Some(handle(&app, remote.did()).await), display_name: about.name, description: about.bio, pronouns: about.pronouns, links: about.links, avatar, followers_count: counts.map(|c| c.0), follows_count: counts.map(|c| c.1), viewer: viewer.0.as_ref().map(state), organization: orgs::is_org(&app, remote.did()), bot: about.bot, orgs: orgs::orgs_of(&app, remote.did(), viewer.0.as_deref()).await, badges: match index { Some(i) => { let home = badges::home(&app, i.clone(), &remote).await; // Counted on a thread of its own: an account of many pulls // takes a while, which would hold up every request this // worker answers meanwhile. let (i, did) = (i.clone(), remote.did().to_string()); let count = move || badges::badges_of(&i, &did, home); tokio::task::spawn_blocking(count).await.unwrap_or_default() } None => Vec::new(), }, }; axum::Json(view).into_response() } /// `did`'s profile picture, if it has one: looked up now and then, and /// again whenever its profile page is shown. async fn avatar(app: &App, did: &str) -> Option { let (ctx, at) = (app.ctx.clone(), did.to_string()); let (index, watching) = (app.index.clone(), app.watching.load(Ordering::Relaxed)); let look = async move { let at = RepoUrl { authority: at, name: "_".into(), }; let remote = Remote::connect(&ctx, &at).await.ok()?; // A PDS that did not answer, as one busy after a deploy may not, // leaves the picture found before, rather than none. let records = profile_records(index.as_deref(), watching, &remote).await; let about = about(remote.did(), records); about.answered.then_some(about.avatar) }; let dir = &app.ctx.env.cache_dir; AVATARS.through(dir, did, look).await.flatten() } /// How long a card waits for the picture it shows before it is drawn /// without it. const PICTURE_WAIT: Duration = Duration::from_secs(2); /// The most bytes of a picture a card takes. const PICTURE_MAX: usize = 1 << 20; /// The pictures cards show, by address, the newest so many. An address /// names one picture for good: it carries the picture's CID. static PICTURES: LazyLock>>> = LazyLock::new(|| card::Newest::new(128)); /// The picture at `url`, an address `about` gives, for a card: from the /// account's PDS for one its knotless profile names, else Bluesky's /// CDN's small form of it. Nothing if it is not read in `PICTURE_WAIT`, /// or is larger than `PICTURE_MAX`; and a picture not read is asked for /// again next time. async fn card_picture(ctx: &knotless::env::Ctx, url: &str) -> Option>> { if let Some(known) = looked("pictures", PICTURES.get(url)) { return Some(known); } let read = tokio::time::timeout(PICTURE_WAIT, read_picture(ctx, url)).await; let read = read.unwrap_or(Err(anyhow::anyhow!("not read in {PICTURE_WAIT:?}"))); fill("pictures", url, &read); let bytes = read.ok().filter(|b| b.len() <= PICTURE_MAX)?; let bytes = Arc::new(bytes); PICTURES.put(url, bytes.clone()); Some(bytes) } async fn read_picture(ctx: &knotless::env::Ctx, url: &str) -> anyhow::Result> { if url.starts_with(OWN_AVATAR) { let asked = reqwest::Url::parse(&format!("http://x{url}"))?; let param = |name: &str| asked.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v); let (did, cid) = ( param("actor").unwrap_or_default(), param("cid").unwrap_or_default(), ); let at = RepoUrl { authority: did.to_string(), name: "_".into(), }; let remote = Remote::connect(ctx, &at).await?; return remote.get_blob(&cid).await; } let small = url.replacen("/img/avatar/", "/img/avatar_thumbnail/", 1); let sent = knotless::telemetry::send(ctx.http.get(small), 0).await?; let mut res = sent.error_for_status()?; let mut bytes = Vec::new(); while let Some(chunk) = res.chunk().await? { bytes.extend_from_slice(&chunk); anyhow::ensure!( bytes.len() <= PICTURE_MAX, "a picture larger than a card takes" ); } Ok(bytes) } /// The addresses `did` claims to commit as, hashed (`knotless::emails`). async fn claims(app: &App, did: &str) -> Vec { let (ctx, of) = (app.ctx.clone(), did.to_string()); // A PDS that did not answer leaves the claims read before, rather than // its account's commits shown as no one's. let look = async move { knotless::emails::claimed(&ctx, &of).await.ok() }; let dir = &app.ctx.env.cache_dir; CLAIMS.through(dir, did, look).await.unwrap_or_default() } /// The path of a page's card, under the version of what it shows. fn card_path(page: &str, card: &Card) -> String { format!("{page}/card.png?v={}", card::version(&card.svg())) } /// A repository's DID, and its name now if the index knows it. fn now_named<'a>(app: &App, repo: &'a RepoRef) -> Option<(&'a str, String)> { let (did, _, named_then) = forge::parse_uri(&repo.uri).ok()?; let now = app .index .as_ref() .and_then(|i| i.name(did, &repo.key).ok().flatten()); Some((did, now.as_deref().unwrap_or(named_then).to_string())) } /// One whose claims a repository's pages believe: its owner and its /// collaborators everywhere in it, and the author of a pull merged into it /// only in the commits the pull brought in, which the merge vouched for. struct Vouched { did: String, /// The addresses it claims, hashed (`knotless::emails`). hashes: Vec, /// The commits it is believed in, if not all. only: Option>, } /// Which of `people` `email` is in `sha`, if any. fn vouched_as<'a>(people: &'a [Vouched], sha: &str, email: &str) -> Option<&'a str> { let by = people.iter().find(|p| { let here = p.only.as_ref().is_none_or(|only| only.contains(sha)); here && knotless::emails::is(&p.did, email, &p.hashes) }); by.map(|p| p.did.as_str()) } /// `name` at `email`, by account if `email` is one of `people`'s in `sha`, /// or else the one account other repositories with `sha` vouch for as it /// there ([`walk_authors`]): none when they disagree. async fn author( app: &App, (repo, people): (&Repo, &[Vouched]), sha: &str, name: String, email: &str, ) -> CommitAuthorView { let by = match vouched_as(people, sha, email) { Some(did) => Some(did.to_string()), None => elsewhere(app, repo, sha, email), }; let actor = match by { Some(did) => Some(actor(app, &did).await), None => None, }; let agent = email .eq_ignore_ascii_case("noreply@anthropic.com") .then_some("claude"); CommitAuthorView { name, actor, agent } } /// The co-authors a commit message's closing paragraph names, as /// `Co-authored-by: Name ` trailers: names and addresses. fn coauthors(message: &str) -> Vec<(String, String)> { let trailers = message .trim_end() .rsplit_once("\n\n") .map_or("", |(_, t)| t); trailers.lines().filter_map(coauthor).collect() } fn coauthor(line: &str) -> Option<(String, String)> { let (key, value) = line.split_once(':')?; let named = key.trim().eq_ignore_ascii_case("co-authored-by"); let (name, email) = value.split_once('<').filter(|_| named)?; let email = email.trim_end().strip_suffix('>')?; Some((name.trim().to_string(), email.trim().to_string())) } /// What a merge as `merged` brought in, as the ends of a range of /// history: from where the base was before it, or, for a merge recorded /// without that, a merge commit's second parent less its first, which /// names nothing for any other kind of merge. fn brought(merged: &str, from: Option<&str>) -> (String, String) { match from { Some(from) => (from.to_string(), merged.to_string()), None => (format!("{merged}^1"), format!("{merged}^2")), } } type Range = (String, String); static BROUGHT: LazyLock>>> = LazyLock::new(Mutex::default); /// The commits each of `ranges` names in the mirror at `dir`, kept once /// found: a merge never changes what it brought in. Read in this process, /// all with the one repository opened, as a repository's merged pulls are /// many and a git process for each, all at once, took seconds. fn brought_in(dir: &std::path::Path, ranges: Vec>) -> Vec>> { let objects = crate::objects::open(dir).ok(); let find = |(from, to): Range| { if let Some(known) = BROUGHT.lock().unwrap().get(&(from.clone(), to.clone())) { return known.clone(); } // Not found, as a commit the mirror has yet to fetch, it is looked // for again next time. let found = objects.as_ref().map(|o| o.between(&from, &to)); let found = found.and_then(|f| f.ok()); if let Some(shas) = &found { BROUGHT.lock().unwrap().insert((from, to), shas.clone()); } found.unwrap_or_default() }; ranges.into_iter().map(|r| r.map(find)).collect() } /// Whose claims `repo`'s pages believe, and where: its owner, its /// collaborators and the members of the organization its owner is who may /// push to it first, so no pull's author can take their addresses. And /// the merged pull that brought each commit in, from what its merge record /// says it brought: a fast-forward's commits, a merge's and a squash's. async fn vouched( app: &App, repo: &Repo, record: RepoRecord, items: &[Item], ) -> (Vec, HashMap) { let merged: Vec<(&Item, Range)> = items .iter() .filter_map(|i| Some((i, brought(i.merged.as_deref()?, i.merged_from.as_deref())))) .collect(); let writers: Vec = record.writers().map(String::from).collect(); let everywhere = std::iter::once(repo.did.clone()).chain(writers); let people: Vec<(String, Option)> = everywhere .map(|did| (did, None)) .chain( merged .iter() .map(|(i, r)| (i.author.clone(), Some(r.clone()))), ) .collect(); // Each account's claims are read once, however many of its pulls // merged: one page asks its PDS once, not once a pull. let dids: HashSet<&String> = people.iter().map(|(did, _)| did).collect(); let read = dids.into_iter().map(|did| async move { let hashes = claims(app, did).await; (did.clone(), hashes) }); let claimed: HashMap> = futures::future::join_all(read).await.into_iter().collect(); let (dids, ranges): (Vec, Vec>) = people.into_iter().unzip(); let dir = repo.git.dir.clone(); let found = tokio::task::spawn_blocking(move || brought_in(&dir, ranges)); let found = found.await.expect("a walk does not panic"); // A commit two merges brought in, as a stack pushed at once brings // each pull its own and those under it, is the one bringing least's. let ranges = found[found.len() - merged.len()..].iter().flatten(); let mut brought: Vec<_> = merged.iter().zip(ranges).collect(); brought.sort_by_key(|(_, shas)| shas.len()); let mut pulls = HashMap::new(); for ((item, _), shas) in brought { let view = ItemViewBasic { uri: item.uri.clone(), title: item.title.clone(), }; for sha in shas { pulls.entry(sha.clone()).or_insert_with(|| view.clone()); } } let people = dids .into_iter() .zip(found) .map(|(did, only)| { let hashes = claimed.get(&did).cloned().unwrap_or_default(); Vouched { did, hashes, only } }) .collect(); (people, pulls) } /// The one account other repositories with `sha` vouch for as `email` /// in it, if they agree on one. fn elsewhere(app: &App, repo: &Repo, sha: &str, email: &str) -> Option { let index = app.index.as_ref()?; let address = crate::index::address(email); let found = index.authors_elsewhere(sha, &address, (&repo.did, &repo.key)); let mut found = found.ok()?; found.pop().filter(|_| found.is_empty()) } /// The repositories whose authors are being walked now. static WALKING: LazyLock>> = LazyLock::new(Mutex::default); /// Keeps in the index the account `people` vouch for as each address of /// each commit on `repo`'s default branch, so that a page of another /// repository with those commits, as a copy of it, shows them as theirs. /// Each walk goes on from the tip the last stopped at, and starts over /// when who `repo` believes changes, as one it no longer believes vouches /// for nothing. A private repository vouches for nothing elsewhere. fn walk_authors(app: &App, repo: &Repo, people: Arc>) { let index = app.index.clone().filter(|_| repo.in_space.is_none()); let id = (repo.did.clone(), repo.key.clone()); // One walk of a repository at a time. let index = index.filter(|_| WALKING.lock().unwrap().insert(id.clone())); if let Some(index) = index { let (dir, head) = (repo.git.dir.clone(), repo.head.clone()); tokio::task::spawn_blocking(move || { // One with no tip yet, as an empty repository, has none to walk. let _ = vouch_walked(&index, &id, &dir, &head, &people); WALKING.lock().unwrap().remove(&id); }); } } /// [`walk_authors`]'s walk. fn vouch_walked( index: &Index, (did, key): &(String, String), dir: &std::path::Path, head: &str, people: &[Vouched], ) -> anyhow::Result<()> { let believed = believed(people); let objects = objects::open(dir)?; let tip = objects.resolve(head)?; let walked = index.authors_walked(did, key)?; let from = match &walked { Some((at, was)) if *at == tip && *was == believed => return Ok(()), Some((at, was)) if *was == believed => Some(at.as_str()), _ => None, }; let known: Vec<&str> = from.into_iter().collect(); let mut rows = Vec::new(); for c in objects.brought(&tip, &known, usize::MAX)? { let helped = coauthors(&c.message).into_iter().map(|(_, email)| email); let emails = [c.author.email, c.committer.email] .into_iter() .chain(helped); for email in emails { if let Some(by) = vouched_as(people, &c.sha, &email) { rows.push((c.sha.clone(), crate::index::address(&email), by.to_string())); } } } index.vouch_authors((did, key), (&tip, &believed), &rows, from.is_none())?; Ok(()) } /// Who `people` are, as a walk of the authors they vouch for keeps it: /// each account and the addresses it claims, but not the pulls merged, /// as one merged since brought in only commits a walk on from its tip /// finds. fn believed(people: &[Vouched]) -> String { let mut each: Vec = people .iter() .map(|p| { let mut hashes = p.hashes.clone(); hashes.sort(); let everywhere = if p.only.is_none() { "*" } else { "" }; format!("{}{everywhere}:{}", p.did, hashes.join(",")) }) .collect(); each.sort(); each.dedup(); crate::index::address(&each.join("\n")) } /// `commits`, as the pages show them, each with the accounts that wrote it /// where `repo`'s pages believe their claims (`vouched`). async fn shown(app: &App, repo: &Repo, commits: Vec) -> Vec { let record = repo.record(app).unwrap_or_default(); let made = assembled(app, repo).await.unwrap_or_default(); let items = &made.items; let (people, pulls) = vouched(app, repo, record, items).await; let people = Arc::new(people); walk_authors(app, repo, people.clone()); let people = (repo, people.as_slice()); let mut out = Vec::new(); for c in commits { let (pull, message) = match from_pull(items, &c.message) { Some((pull, message)) => (Some(pull), message), None => (pulls.get(&c.sha).cloned(), c.message), }; let mut helped = Vec::new(); for (name, email) in coauthors(&message) { helped.push(author(app, people, &c.sha, name, &email).await); } let (a, m) = (&c.author, &c.committer); let by = author(app, people, &c.sha, a.name.clone(), &a.email).await; let made = author(app, people, &c.sha, m.name.clone(), &m.email).await; let same = match (&by.actor, &made.actor) { (Some(x), Some(y)) => x.did == y.did, _ => (&a.name, &a.email) == (&m.name, &m.email), }; out.push(CommitView { sha: c.sha, parents: c .parents .into_iter() .map(|sha| CommitRefView { sha }) .collect(), author: by, coauthors: helped, committer: (!same).then_some(made), date: c.author.date, committed_at: c.committer.date, message, pull, }); } out } /// The pull `message`'s last line names, when it is one of `items`, and /// the message without that line. fn from_pull(items: &[Item], message: &str) -> Option<(ItemViewBasic, String)> { let message = message.trim_end(); let (rest, last) = message.rsplit_once('\n').unwrap_or(("", message)); let pull = items .iter() .find(|i| i.kind == "pull" && i.uri == last.trim())?; let view = ItemViewBasic { uri: pull.uri.clone(), title: pull.title.clone(), }; Some((view, rest.trim_end().to_string())) } /// The most of those who may push to a repository its pages name. const COLLABORATORS_SHOWN: usize = 20; /// A repository being shown. struct Repo { authority: String, name: String, did: String, key: String, git: Git, /// The default branch. head: String, /// Its record as the PDS has it, for a private repository the index /// never sees. space: Option, /// The space it is in, if it is private. in_space: Option, /// Its record, as the PDS had it when its mirror last synced, which a /// write from the pages has it sync again for. synced: RepoRecord, /// What reads its records: for a private repository, the viewer it /// was found for, never this server ([`crate::private`]). reader: knotless::env::Ctx, } impl Repo { async fn open(app: &App, authority: &str, name: &str) -> Result { let name = repo_name(name); let found = app.located(authority, name).await?; if found.moved { let now = format!("{}/{}", found.did, found.name); return Err(Shown(StatusCode::TEMPORARY_REDIRECT, now)); } if found.name != name { return Err(Shown(StatusCode::TEMPORARY_REDIRECT, found.name)); } Repo::of(authority.to_string(), found) } /// The repository `authority/name` names, under the name it has now, /// and its owner's handle if it moved. async fn find(app: &App, authority: &str, name: &str) -> Result { let found = app.located(authority, repo_name(name)).await?; Repo::moved(app, authority, found).await } /// [`Repo::find`] for a view of its code that answers a request, which /// may be of a mirror a change has made stale while it syncs /// ([`App::located_for_reader`]). async fn find_for_reader(app: &App, authority: &str, name: &str) -> Result { let found = app.located_for_reader(authority, repo_name(name)).await?; Repo::moved(app, authority, found).await } async fn moved(app: &App, authority: &str, found: crate::Located) -> Result { let authority = match found.moved { true => handle(app, &found.did).await, false => authority.to_string(), }; Repo::of(authority, found) } fn of(authority: String, found: crate::Located) -> Result { let git = found.git; let head = objects::open(&git.dir) .map_err(AppError::internal)? .head() .unwrap_or_default(); Ok(Repo { authority, name: found.name, did: found.did, key: found.key, git, head, space: found.space.is_some().then_some(found.record.clone()), in_space: found.space, synced: found.record, reader: found.reader, }) } /// Its objects, read in process. Not held across an await: a reader /// belongs to one thread. fn objects(&self) -> Result { Ok(objects::open(&self.git.dir).map_err(AppError::internal)?) } fn href(&self, tail: &str) -> String { format!( "/{}/{}{tail}", url_path(&self.authority), url_path(&self.name) ) } /// Where its files at `rev` are, as `view` (`blob` or `raw`) shows /// them: what links in Markdown found there resolve under. fn files(&self, view: &str, rev: &str) -> String { self.href(&format!("/{view}/{}", url_path(rev))) } /// Its record's URI: in full, in its space, for a private one. fn uri(&self) -> String { self.named(&forge::repo_uri(&self.did, &self.name)) } /// `uri`, a record of this repository's in either form, as its records /// are named here ([`forge::Target::named`]). fn named(&self, uri: &str) -> String { let space = self.in_space.as_deref(); let space = space.and_then(|s| knotless::aturi::SpaceRef::parse(s).ok()); knotless::aturi::resolve(uri, space.as_ref()) } /// The repository as records name it: a private one's record by its /// URI in its space. fn reference(&self) -> RepoRef { RepoRef { uri: self.uri(), key: self.key.clone(), } } /// The record, as the index has it: for a private one, this server's /// index of its space, else its mirror's. fn record(&self, app: &App) -> Option { let kept = indexed(app, self).and_then(|index| index.repo(&self.uri()).ok().flatten()); let kept = kept.and_then(|value| serde_json::from_value(value).ok()); let record = kept.or(self.space.clone()); // With who may push to it as a member of its owner's organization. record.map(|r| app.with_members(&self.did, self.in_space.as_deref(), r)) } /// Its open pulls and issues. async fn open_items(&self, app: &App) -> Option<(usize, usize)> { let made = assembled(app, self).await.ok()?; let open = |kind: &str| { made.items .iter() .filter(|i| i.kind == kind && i.state == "open") .count() }; Some((open("pull"), open("issue"))) } /// Whether its pages lead to its merge queue: a rule of `record` has a /// branch take pulls through it, or a pull waits in it still, its rule /// dropped since. async fn queues(&self, app: &App, record: Option<&RepoRecord>) -> bool { let ruled = record.is_some_and(|r| r.rules.iter().any(|rule| rule.queue)); if ruled { return true; } let made = assembled(app, self).await.ok(); let waits = |i: &Item| { let queued = i.queue.as_ref().is_some_and(|q| q.state == "queued"); i.state == "open" && queued }; made.is_some_and(|made| made.items.iter().any(waits)) } /// What its pages show above themselves, to `viewer` if someone asks. async fn head(&self, app: &App, viewer: Option<&str>) -> RepoViewDetailed { let record = self.record(app); let fork_of = match record.as_ref().and_then(|r| r.fork_of.as_ref()) { Some(from) => repo_basic(app, from).await, None => None, }; let open = self.open_items(app).await.unwrap_or_default(); let merge_queue = self.queues(app, record.as_ref()).await; let offer = record.as_ref().and_then(|r| r.transfer.clone()); let viewer = match viewer { Some(v) => Some( self.viewer_state(app, v, record.as_ref(), offer.as_ref()) .await, ), None => None, }; let avatar = avatar(app, &self.did).await; let card = self.card(record.as_ref(), open, avatar); let pushers = self.pushers(record.as_ref()); let shown = pushers .iter() .take(COLLABORATORS_SHOWN) .map(|did| actor(app, did)); let collaborators = futures::future::join_all(shown).await; let owner = actor(app, &self.did).await; let site = self.site(app, &owner.handle, record.as_ref()); RepoViewDetailed { uri: self.uri(), owner, site, name: self.name.clone(), key: record.as_ref().map(|r| r.key.clone()), description: record.as_ref().and_then(|r| r.description.clone()), archived_at: record.as_ref().and_then(|r| r.archived_at.clone()), topics: record .as_ref() .map(|r| r.topics.clone()) .unwrap_or_default(), collaborator_count: Some(pushers.len()).filter(|n| *n > 0), collaborators, merge_queue, links: repo_links(record.map(|r| r.links).unwrap_or_default()), fork_of, default_branch: self.head.clone(), open_pulls: open.0, open_issues: open.1, card: card_path(&self.href(""), &card), star_count: self.star_count(app), fork_count: self.fork_count(app), releases: releases::summary(app, self), viewer, private: self.in_space.is_some(), } } /// Its website, as the index last saw its record, where this server /// serves sites: its address, by its owner's `handle` or plc id, and /// what it was deployed from. A check counts only from a runner its /// `record` names. fn site(&self, app: &App, handle: &str, record: Option<&RepoRecord>) -> Option { let domain = app.pages_domain.get()?; let index = indexed(app, self)?; let value = match self.in_space.as_deref() { Some(space) => index.space_site(space, &self.did, &self.key), None => index.site(&self.did, &self.key), }; let site: SiteRecord = serde_json::from_str(&value.ok()??).ok()?; let url = match self.in_space { Some(_) => { let label = knotless::site::private_label(&self.did, &self.key)?; format!("https://{label}.{}.{domain}/", knotless::site::PRIVATE) } None => { let label = knotless::site::owner_label(handle, &self.did)?; format!("https://{label}.{domain}/{}/", self.name) } }; let runners = record.map(|r| r.runners.as_slice()).unwrap_or_default(); let by_runner = |uri: &String| { let runner = knotless::aturi::AtUri::parse(uri).ok().map(|at| at.author); runners.iter().any(|r| Some(&r.did) == runner.as_ref()) }; Some(SiteView { url, commit: site.commit, check: site.check.filter(by_runner), deployed_at: site.created_at, }) } /// Who `record` names among its collaborators, in its order, besides /// its owner: not its organization's members, whom its page names. fn pushers(&self, record: Option<&RepoRecord>) -> Vec { let named = record.into_iter().flat_map(|r| &r.collaborators); named .filter(|c| c.did != self.did) .map(|c| c.did.clone()) .collect() } /// How many starred it: none for a private repository, which cannot /// be starred, or without an index. fn star_count(&self, app: &App) -> Option { let index = app.index.as_ref().filter(|_| self.in_space.is_none())?; index.star_count(&self.did, &self.key).ok() } /// How many forks of it there are: none for a private repository, /// whose forks are private too, or without an index. fn fork_count(&self, app: &App) -> Option { let index = app.index.as_ref().filter(|_| self.in_space.is_none())?; index.fork_count(&self.did, &self.key).ok() } /// What `viewer` has to do with it, as its `record` says, `offer` /// naming where its owner offered to move it, if anywhere. async fn viewer_state( &self, app: &App, viewer: &str, record: Option<&RepoRecord>, offer: Option<&RepoRef>, ) -> RepoViewerState { let star = app .index .as_ref() .map(|i| i.star_of(viewer, &self.did, &self.key)); let transfer = match offer.filter(|to| to.did() == viewer) { Some(to) => Some(RepoViewBasic { did: viewer.to_string(), handle: handle(app, viewer).await, name: to.name().to_string(), }), None => None, }; let labels = record.into_iter().flat_map(|r| { let may = r .labels .iter() .filter(|l| r.may_label(&self.did, viewer, l)); may.map(label_view) }); let subscription = app .index .as_ref() .and_then(|i| i.watching(viewer, &self.did, &self.key).ok().flatten()); RepoViewerState { star: star.and_then(|s| s.ok().flatten()), transfer, labels: labels.collect(), subscription, admin: self.administered(app, viewer), } } /// Whether `viewer` may change its settings: as its owner, or as an /// admin who joined the organization that owns it, when that delegated /// to this server's gateway, which keeps its session to write as it. fn administered(&self, app: &App, viewer: &str) -> bool { let kept = || app.gateway.is_some() && session::load(&app.ctx, &self.did).is_some(); viewer == self.did || (app.org_admits(&self.did, viewer) && kept()) } /// The repository as views name it, by its owner's handle. async fn basic(&self, app: &App) -> RepoViewBasic { RepoViewBasic { did: self.did.clone(), handle: handle(app, &self.did).await, name: self.name.clone(), } } /// Its card, as `record` and the counts of open pulls and issues say, /// with its owner's picture at `avatar`. fn card( &self, record: Option<&RepoRecord>, (pulls, issues): (usize, usize), avatar: Option, ) -> Card { Card::Repo { owner: self.authority.clone(), avatar, name: self.name.clone(), description: record.and_then(|r| r.description.clone()), pulls, issues, languages: self.languages(), } } /// What its default branch is written in: nothing, if it has none. fn languages(&self) -> Vec { let found = self.objects().ok(); found .and_then(|o| languages::of(&o, "HEAD").ok()) .unwrap_or_default() } /// Splits `rest` into a revision and a path: the longest branch or tag /// that starts it, else its first segment. fn locate<'a>(&self, rest: &'a str) -> Result<(String, &'a str)> { let names = self.objects()?.names().map_err(AppError::internal)?; let longest = names .iter() .filter(|n| rest == n.as_str() || rest.starts_with(&format!("{n}/"))) .map(String::len) .max(); let cut = longest.unwrap_or(rest.find('/').unwrap_or(rest.len())); let rev = read::rev(Some(&rest[..cut])).map_err(not_found)?; Ok((rev, rest[cut..].trim_start_matches('/'))) } /// Its branches, the default first, then the rest by name. fn branches(&self) -> Result> { let mut all = self.objects()?.branches().map_err(AppError::internal)?; all.sort_by_key(|b| *b != self.head); Ok(all) } } type RepoPath = Path<(String, String)>; type RestPath = Path<(String, String, String)>; /// The newest `limit` checks the repository `repo`, which views name as /// `shown`, has had of what `filter` names, by the runners its record /// names, newest first: of a pull's runs if it names one, else of refs /// only. None from a server with no index. fn checks( app: &App, repo: &RepoRef, shown: &RepoViewBasic, filter: CheckFilter, limit: usize, ) -> Vec { let found = checks_index(app, &repo.uri).and_then(|(index, private)| { let (did, _, _) = forge::parse_uri(&repo.uri).ok()?; let record: RepoRecord = serde_json::from_value(index.repo(&repo.uri).ok()??).ok()?; // A private repository's, such as a pull's head, only to a viewer // shown it as a member, and then theirs alone. let viewer = crate::private::viewer().unwrap_or_default(); let shown = !private || app.shown_private(&viewer, did, &record.name); if private && shown { crate::private::touched(); } let runners: Vec = record.runners.into_iter().map(|r| r.did).collect(); let found = index .checks(did, &repo.key, &runners, filter, limit, None) .ok() .filter(|_| shown); if let Some((entries, _)) = found.as_ref().filter(|_| !private) { app.recheck(index, entries); } found }); found .map(|(entries, _)| entries) .unwrap_or_default() .into_iter() .filter_map(|e| Some((e.uri, serde_json::from_value(e.value).ok()?))) .filter(|(_, c): &(String, CheckRecord)| filter.pull.is_some() || c.pull.is_none()) .filter_map(|(uri, c)| check_view(shown, uri, c)) .collect() } /// `dev.knotless.defs#minutesView`: the minutes a runner counts for a /// repository's owner. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct MinutesView { runner: ActorView, minutes: u64, #[serde(skip_serializing_if = "Option::is_none")] allowance: Option, #[serde(skip_serializing_if = "Vec::is_empty")] freeing: Vec, updated_at: String, } /// The minutes `runners`, those the repository `uri` of `owner` names, /// count for `owner`, in the order they are named: none for a private /// repository, whose minutes no runner publishes, or from a server with no /// index. async fn minutes(app: &App, uri: &str, owner: &str, runners: &[String]) -> Vec { let index = checks_index(app, uri).filter(|(_, private)| !private); let found = index.and_then(|(index, _)| index.minutes(owner, runners).ok()); let counted = found.unwrap_or_default().into_iter().filter_map(|e| { let m: MinutesRecord = serde_json::from_value(e.value).ok()?; Some((e.author, m)) }); let views = counted.map(|(runner, m)| async move { MinutesView { runner: actor(app, &runner).await, minutes: m.minutes, allowance: m.allowance, freeing: m.freeing, updated_at: m.updated_at, } }); futures::future::join_all(views).await } /// The index the checks of the repository `uri` are read from, and /// whether it is this server's own of the private ones, synced from their /// spaces, which no relay rechecks: else the appview's. fn checks_index<'a>(app: &'a App, uri: &str) -> Option<(&'a Arc, bool)> { let private = app .spaces .as_ref() .filter(|i| matches!(i.repo(uri), Ok(Some(_)))); match private { Some(index) => Some((index, true)), None => app.index.as_ref().map(|index| (index, false)), } } /// The check at `uri` of the repository `repo`, as `c` reports it. fn check_view(repo: &RepoViewBasic, uri: String, c: CheckRecord) -> Option { forge::parse_uri(&uri).ok()?; Some(CheckView { images: run_images(repo, &uri, c.outputs), repo: repo.clone(), workflow: c.workflow, status: c.status, commit: c.commit, started_at: c.started_at, finished_at: c.finished_at, error: c.error, skipped: c.skipped, uri, }) } /// The first of each workflow in `checks`, by workflow name: the order of /// the files in `.knotless/workflows`, rather than which one filed last. fn newest(checks: Vec) -> Vec { let mut seen = HashSet::new(); let mut newest: Vec<_> = checks .into_iter() .filter(|c| seen.insert(c.workflow.clone())) .collect(); newest.sort_by(|a, b| a.workflow.cmp(&b.workflow)); newest } /// The pictures among the outputs of the check at `uri`, of `repo`, each /// served by `dev.knotless.getCheckImage`. fn run_images(repo: &RepoViewBasic, uri: &str, outputs: Vec) -> Vec { let repo = format!("at://{}/{}/{}", repo.did, nsid::REPO, repo.name); let image = |o| match o { CheckOutput::Image(i) => Some(CheckImageView { alt: i.alt, url: query_path( "getCheckImage", &[("repo", &repo), ("uri", uri), ("cid", &i.image.link.cid)], ), }), CheckOutput::Other => None, }; outputs.into_iter().filter_map(image).collect() } /// The path of the query `nsid` of `dev.knotless`'s, given `params`. fn query_path(nsid: &str, params: &[(&str, &str)]) -> String { let url = reqwest::Url::parse_with_params("http://x/", params).expect("a URL"); format!( "/xrpc/dev.knotless.{nsid}?{}", url.query().unwrap_or_default() ) } /// `dev.knotless.getCheck`'s answer: a check of one of a repository's /// workflows, as its runner reported it: each step, and the end of what /// they printed. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct CheckRun { check: CheckView, /// The ref it ran at, in full. #[serde(rename = "ref")] refname: String, /// The pull it ran for, if it ran for one. #[serde(skip_serializing_if = "Option::is_none")] pull: Option, runner: ActorView, steps: Vec, /// The end of what its steps printed, each step's after a line /// `==> `. #[serde(skip_serializing_if = "Option::is_none")] log: Option, } /// `dev.knotless.getCheck#stepView`: one of a check's steps. #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct RunStep { name: String, /// `success`, `failure` or `skipped`. status: String, /// None when it was killed for its time, or skipped. #[serde(skip_serializing_if = "Option::is_none")] exit_code: Option, duration_ms: u64, } /// How much of a check's log its page shows: the end, where a failure is. const LOG_SHOWN: usize = 256 * 1024; /// The last [`LOG_SHOWN`] bytes of `log`, from a character on. fn log_tail(log: &[u8]) -> String { let text = String::from_utf8_lossy(log); let cut = text.len().saturating_sub(LOG_SHOWN); let from = text.char_indices().map(|(i, _)| i).find(|&i| i >= cut); text[from.unwrap_or(text.len())..].to_string() } #[derive(Deserialize)] pub struct CheckParams { repo: String, uri: String, cid: Option, } /// The check at `p.uri`, if it is of the repository `p.repo` names and by /// a runner that names: the only checks shown, as everywhere. async fn trusted_check(app: &App, p: &CheckParams) -> Result<(Repo, Remote, CheckRecord)> { let (repo, c) = named_check(app, p).await?; let (runner, _, _) = forge::parse_uri(&p.uri).expect("a check named"); // A private repository's runner writes its checks into the space. let remote = writer(app, &repo, runner).await?; Ok((repo, remote, c)) } /// Where `did` writes about `repo`: their own repo, or for a private /// repository their repo in its space. async fn writer(app: &App, repo: &Repo, did: &str) -> Result { let at = RepoUrl { authority: did.to_string(), name: "_".into(), }; let remote = match &repo.in_space { Some(space) => in_space(&repo.reader, space, did).await, None => Remote::connect(&app.ctx, &at).await, }; remote.map_err(not_found) } /// [`trusted_check`], as the index has it, with no word from its runner. async fn named_check(app: &App, p: &CheckParams) -> Result<(Repo, CheckRecord)> { let repo = repo::found(app, &p.repo).await?; let name = &repo.name; let runner = match forge::parse_uri(&p.uri) { Ok((runner, nsid::CHECK, _)) => runner, _ => return Err(repo::invalid(format!("{} is no check", p.uri))), }; // Only a runner the repository names is trusted, as everywhere checks show. let runners = repo.record(app).unwrap_or_default().runners; if !runners.iter().any(|r| r.did == runner) { return Err(not_found(anyhow::anyhow!( "{runner} runs no checks of {name}" ))); } // As the index has it: the appview's, or this server's own of the space. let uri = repo.named(&p.uri); let kept = indexed(app, &repo).and_then(|index| index.check_record(&uri).ok().flatten()); let found = kept.and_then(|value| serde_json::from_value::(value).ok()); let of_this = |c: &CheckRecord| { let owner = forge::parse_uri(&c.repo.uri).map(|(did, _, _)| did == repo.did); c.repo.key == repo.key && owner.unwrap_or(false) }; let Some(c) = found.filter(of_this) else { return Err(not_found(anyhow::anyhow!("no such check of {name}"))); }; Ok((repo, c)) } /// `did`'s repo in the space `uri`, as `reader` reads it. async fn in_space(reader: &knotless::env::Ctx, uri: &str, did: &str) -> anyhow::Result { let joined = knotless::space::joined(reader, uri).await?; crate::spaces::remote(reader, &joined, did).await } /// Pictures check runs made and blobs attached, by kind, author and CID. /// A blob is the same for good, so it is read once; one in a space is /// read each time, as whoever asks may read it. static RUN_BLOBS: LazyLock>>> = LazyLock::new(|| card::Newest::new(64)); /// The tails of check logs, kept as [`RUN_BLOBS`] are: a few kilobytes /// most, and at most [`LOG_SHOWN`], so many more of them, up to /// [`LOG_TAILS_KEPT`] bytes in all. static LOG_TAILS: LazyLock>>> = LazyLock::new(|| card::Newest::weighed(LOG_TAILS_KEPT, |tail| tail.len())); /// What the kept tails of check logs weigh at most. const LOG_TAILS_KEPT: usize = 64 * 1024 * 1024; /// The blob `cid` of the runner `remote`, as `shown` makes it of `kind`. async fn run_blob( remote: &Remote, kind: &str, cid: &str, shown: fn(Vec) -> Vec, ) -> anyhow::Result>> { let public = remote.space.is_none(); let key = format!("{kind}/{}/{cid}", remote.did()); let cache = [&RUN_BLOBS, &LOG_TAILS][usize::from(kind == "log")]; let kept = public.then(|| cache.get(&key)).flatten(); if let Some(kept) = looked("run blobs", kept) { return Ok(kept); } let blob = Arc::new(shown(remote.get_blob(cid).await?)); if public { cache.put(&key, blob.clone()); } Ok(blob) } /// A log as a check's page shows it: its tail. fn shown_tail(log: Vec) -> Vec { log_tail(&log).into_bytes() } /// How many check logs are read ahead at once, as their checks come. static WARMING: LazyLock = LazyLock::new(|| tokio::sync::Semaphore::new(4)); /// The CID of the log of `did`'s check `rkey` as the index has it, when /// its repository names `did` among its runners, so that its page shows it. fn log_to_warm(index: &Index, did: &str, rkey: &str) -> Option { let uri = format!("at://{did}/{}/{rkey}", nsid::CHECK); let value = index.check_record(&uri).ok().flatten()?; let check: CheckRecord = serde_json::from_value(value).ok()?; let cid = check.log?.link.cid; let repo = index.repo(&check.repo.uri).ok().flatten()?; let repo: RepoRecord = serde_json::from_value(repo).ok()?; repo.is_runner(did).then_some(cid) } /// Reads ahead the tail of the log of `did`'s check `rkey`, as the check /// comes from the network, so its page's first look need not wait on the /// runner's PDS. A few at once, and none past them: a burst of checks /// leaves the rest to be read as they are looked at. pub(crate) async fn warm_check_log(app: Arc, did: String, rkey: String) { let index = app.index.as_ref(); let cid = index.and_then(|index| log_to_warm(index, &did, &rkey)); let permit = cid.as_ref().and_then(|_| WARMING.try_acquire().ok()); let (Some(cid), Some(_permit)) = (cid, permit) else { return; }; let at = RepoUrl { authority: did, name: "_".into(), }; let read = async { let remote = Remote::connect(&app.ctx, &at).await?; run_blob(&remote, "log", &cid, shown_tail).await }; fill( "check logs", &format!("{}/{cid}", at.authority), &read.await, ); } /// `dev.knotless.getCheck`. async fn get_check(State(app): State>, Query(p): Query) -> Result { let (repo, remote, mut c) = trusted_check(&app, &p).await?; let read = c .log .as_ref() .map(|b| run_blob(&remote, "log", &b.link.cid, shown_tail)); let blob = futures::future::OptionFuture::from(read).await; let log = blob.and_then(|b| b.ok()); let log = log.map(|b| String::from_utf8_lossy(&b).into_owned()); let runner = actor(&app, remote.did()).await; let steps = std::mem::take(&mut c.steps).into_iter(); let steps = steps.map(|s: CheckStep| RunStep { name: s.name, status: s.status, exit_code: s.exit_code, duration_ms: s.duration_ms, }); let (refname, pull) = (std::mem::take(&mut c.ref_name), c.pull.take()); let shown = repo.basic(&app).await; let check = check_view(&shown, p.uri, c).expect("a check's own uri parses"); Ok(answer(CheckRun { check, refname, pull, runner, steps: steps.collect(), log, })) } /// `dev.knotless.getCheckImage`: a picture a check made, by its CID: only /// one the check names, from its runner's PDS. Never run as a page, and /// the same for good. async fn get_check_image( State(app): State>, Query(p): Query, ) -> Result { let (_, remote, c) = trusted_check(&app, &p).await?; let cid = p.cid.unwrap_or_default(); let named = c.outputs.into_iter().find_map(|o| match o { CheckOutput::Image(i) if i.image.link.cid == cid => Some(i.image), _ => None, }); let Some(image) = named else { return Err(not_found(anyhow::anyhow!( "{cid} is no picture of this check" ))); }; let whole = |b: Vec| b; let bytes = run_blob(&remote, "image", &cid, whole).await; let bytes = bytes.map_err(not_found)?; Ok(picture(&image.mime_type, bytes.to_vec())) } #[derive(Deserialize)] pub struct AttachmentParams { repo: String, uri: String, cid: String, } /// `dev.knotless.getAttachment`: a blob a pull, issue or comment attaches, /// by its CID: only one its record names, from its author's PDS, or for a /// private repository from their repo in its space. Never run as a page, /// and the same for good. async fn get_attachment( State(app): State>, Query(p): Query, ) -> Result { let repo = repo::found(&app, &p.repo).await?; let made = assembled(&app, &repo).await?; let uri = repo.named(&p.uri); let said = made.entries.iter().find(|e| e.uri == uri).filter(|e| { let collection = forge::parse_uri(&e.uri).map(|(_, c, _)| c); matches!(collection, Ok(nsid::PULL | nsid::ISSUE | nsid::COMMENT)) }); let attachments = said.map(|e| e.value["attachments"].clone()); let attachments = attachments.and_then(|a| serde_json::from_value::>(a).ok()); let named = attachments .unwrap_or_default() .into_iter() .find_map(|a| match a { Attachment::Image(i) if i.image.link.cid == p.cid => Some(i.image), _ => None, }); let (Some(image), Some(said)) = (named, said) else { let why = format!("{} attaches no {}", p.uri, p.cid); return Err(not_found(anyhow::anyhow!(why))); }; let remote = writer(&app, &repo, &said.author).await?; let whole = |b: Vec| b; let bytes = run_blob(&remote, "picture", &p.cid, whole).await; let bytes = bytes.map_err(not_found)?; Ok(picture(&image.mime_type, bytes.to_vec())) } /// The pictures a pull, issue or comment attaches, to show from here /// rather than from where its body links to them. #[derive(Clone, Copy)] pub(super) struct Attached<'a> { /// The repository's record, as `getAttachment` takes it. pub repo: &'a str, /// The pull's, issue's or comment's record. pub uri: &'a str, pub images: &'a [ImageAttachment], } impl Attached<'_> { /// Where `dest`, a link to a blob its record attaches on its author's /// PDS, is shown from here. fn served(&self, dest: &str) -> Option { let url = reqwest::Url::parse(dest).ok()?; let read = url.path().ends_with("/xrpc/com.atproto.sync.getBlob"); let asked: HashMap<_, _> = url.query_pairs().collect(); let (did, cid) = (asked.get("did")?, asked.get("cid")?); let (author, _, _) = forge::parse_uri(self.uri).ok()?; let attached = self.images.iter().any(|i| &i.image.link.cid == cid); let ours = read && did == author && attached; ours.then(|| self.path(cid)) } /// Where its picture `cid` is shown from here. fn path(&self, cid: &str) -> String { let at = [("repo", self.repo), ("uri", self.uri), ("cid", cid)]; query_path("getAttachment", &at) } /// `html` with each of its pictures shown from here drawn at the size /// its record gives, as a page then makes room for it before it loads. fn sized(&self, html: String) -> String { let sizes = self.images.iter().filter_map(|i| { let size = i.width.zip(i.height)?; Some((self.path(&i.image.link.cid).replace('&', "&"), size)) }); sizes.fold(html, |html, (src, (w, h))| { let img = format!(r#" String { query_path("getAvatar", &[("actor", did), ("cid", cid)]) } #[derive(Deserialize)] pub struct AvatarParams { actor: String, cid: String, } /// `dev.knotless.getAvatar`: an account's picture, by its CID: only the /// one its knotless profile names, from its PDS. Never run as a page, and /// the same for good. async fn get_avatar( State(app): State>, Query(AvatarParams { actor: did, cid }): Query, ) -> Result { let at = RepoUrl { authority: did, name: "_".into(), }; let remote = Remote::connect(&app.ctx, &at).await.map_err(not_found)?; let own = own_profile(&app, &remote).await?; let named = own.and_then(|p| p.avatar).filter(|a| a.link.cid == cid); let Some(avatar) = named else { return Err(not_found(anyhow::anyhow!( "{cid} is no picture of this account's" ))); }; let bytes = remote.get_blob(&cid).await.map_err(not_found)?; Ok(picture(&avatar.mime_type, bytes)) } /// A picture of `kind`, as an answer: one of a kind a page could be run as /// is sent as bytes alone. fn picture(kind: &str, bytes: Vec) -> Response { let kind = match kind { kind @ ("image/png" | "image/jpeg" | "image/webp" | "image/gif") => kind, _ => "application/octet-stream", }; ( [ (header::CONTENT_TYPE, kind), (header::CACHE_CONTROL, "public, max-age=31536000, immutable"), ( header::CONTENT_SECURITY_POLICY, "sandbox; default-src 'none'", ), (header::X_CONTENT_TYPE_OPTIONS, "nosniff"), ], bytes, ) .into_response() } /// Whether `path` names a Markdown file, by its extension. fn is_markdown(path: &str) -> bool { let ext = path.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase()); matches!(ext.as_deref(), Some("md" | "markdown" | "mdown" | "mkd")) } /// Where a repository's directories are shown, and which paths are one: /// Markdown's links to a directory go there rather than to a file. type Trees<'a> = Option<(&'a str, &'a dyn Fn(&str) -> bool)>; /// Markdown, with raw HTML shown as text but for the folds [`folds`] /// keeps, and links made relative to the directory `dir` it was found /// in, among files shown under `blobs`, directories under `trees` and, /// for images, `raws`. fn markdown(text: &str, blobs: &str, raws: &str, dir: &str, trees: Trees) -> String { drawn(text, blobs, raws, dir, trees, None, None) } /// What a pull, issue or comment says, as [`markdown`] draws it under /// `base`, its pictures shown as `attached` says, and each account it /// mentions linked to: to the account `mentioned` says, by DID, which its /// facets give ([`knotless::mentions::mentioned`]), and otherwise to the /// handle. pub(super) fn written( text: &str, base: &str, attached: Option, mentioned: &HashMap, ) -> String { written_naming(text, base, attached, mentioned, &Named::new()) } /// The pulls and issues of a repository, by their `at://` URIs: each its /// page's path and its title. pub(super) type Named = HashMap; /// [`written`], with each pull or issue `named` holds linked to its page /// where the text names it by its `at://` URI, bare or as a link's target. /// A bare one shows as its title. pub(super) fn written_naming( text: &str, base: &str, attached: Option, mentioned: &HashMap, named: &Named, ) -> String { let said = Said { mentioned, named }; drawn(text, base, base, "", None, attached, Some(said)) } /// Whom and what a pull, issue or comment names, to link to. #[derive(Clone, Copy)] struct Said<'a> { mentioned: &'a HashMap, named: &'a Named, } /// [`markdown`], with the pictures `attached` shown from here, and with /// `said`, its mentions and references linked as [`written_naming`] links /// them. fn drawn( text: &str, blobs: &str, raws: &str, dir: &str, trees: Trees, attached: Option, said: Option, ) -> String { let dir = url_path(dir); use pulldown_cmark::{Event, Options, Parser, Tag}; let options = Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH | Options::ENABLE_TASKLISTS; let parsed = pulldown_cmark::TextMergeStream::new(Parser::new_ext(text, options)); let page = |dest: &str| { said.and_then(|s| s.named.get(dest)) .map(|(path, _)| path.clone()) }; // Folds first, so what a summary says is linked as the rest is. let events = autolinks(folds(parsed, options).into_iter()).map(|event| match event { Event::Start(Tag::Link { link_type, dest_url, title, id, }) => Event::Start(Tag::Link { link_type, dest_url: page(&dest_url) .unwrap_or_else(|| link(&dest_url, blobs, &dir, trees)) .into(), title, id, }), Event::Start(Tag::Image { link_type, dest_url, title, id, }) => Event::Start(Tag::Image { link_type, dest_url: attached .and_then(|a| a.served(&dest_url)) .unwrap_or_else(|| link(&dest_url, raws, &dir, None)) .into(), title, id, }), other => other, }); let events: Vec = match said { Some(said) => mentions(references(events, said.named), said.mentioned).collect(), None => events.collect(), }; let mut out = String::new(); pulldown_cmark::html::push_html(&mut out, fenced(events).into_iter()); match attached { Some(attached) => attached.sized(out), None => out, } } /// A part of Markdown folded away: `
`, its `` said /// beside the arrow that opens it. #[derive(Clone, Copy, PartialEq)] enum Fold { Details, Summary, } impl Fold { fn closer(self) -> &'static str { match self { Fold::Details => "
", Fold::Summary => "", } } } /// Raw HTML shown as text, but for the `
` and `` that /// fold a part away, as GitHub's Markdown keeps them: each written anew, /// `open` the only attribute kept, and only on lines of their own (an /// HTML block), so never inside a paragraph. They nest as the blocks /// around them do: a closing tag counts only where it closes what was /// opened beside it, a block closes what it left open as it ends, and so /// does the end of the text, so nothing written can close the page's own /// elements. What a summary says is Markdown on a line, as [`summary`] /// reads it; raw HTML's text is written escaped, for no later step to /// link. fn folds<'a>( events: impl Iterator>, options: pulldown_cmark::Options, ) -> Vec> { use pulldown_cmark::{Event, Tag, TagEnd}; let mut out = Vec::new(); // What is open, each with how many blocks deep it was opened. let mut open: Vec<(Fold, usize)> = Vec::new(); let mut depth = 0usize; for event in events { match event { Event::Start(Tag::HtmlBlock) | Event::End(TagEnd::HtmlBlock) => out.push(event), Event::Start(_) => { depth += 1; out.push(event); } Event::End(_) => { depth -= 1; let kept = open.iter().take_while(|(_, d)| *d <= depth).count(); shut(&mut out, &mut open, kept); out.push(event); } Event::InlineHtml(html) => out.push(Event::Html(escaped(&html).into())), Event::Html(html) => unfolded(&html, depth, options, &mut open, &mut out), other => out.push(other), } } shut(&mut out, &mut open, 0); out } /// `text` escaped as HTML's text is, as pulldown-cmark escapes it. fn escaped(text: &str) -> String { let mut out = String::with_capacity(text.len()); for c in text.chars() { match c { '&' => out.push_str("&"), '<' => out.push_str("<"), '>' => out.push_str(">"), c => out.push(c), } } out } /// What a `` says, `text`, as Markdown on a line: its emphasis, /// code and links. Written as more than that (a heading, a list, a /// picture), it is shown as text. fn summary(text: &str, options: pulldown_cmark::Options) -> Vec> { use pulldown_cmark::{Event, Parser, Tag, TagEnd}; let events: Vec = Parser::new_ext(text, options).collect(); let inline = |e: &Event| { matches!( e, Event::Start(Tag::Emphasis | Tag::Strong | Tag::Strikethrough | Tag::Link { .. }) | Event::End( TagEnd::Emphasis | TagEnd::Strong | TagEnd::Strikethrough | TagEnd::Link ) | Event::Text(_) | Event::Code(_) | Event::SoftBreak | Event::HardBreak | Event::InlineHtml(_) ) }; match events.as_slice() { [ Event::Start(Tag::Paragraph), inner @ .., Event::End(TagEnd::Paragraph), ] if inner.iter().all(inline) => { let said = inner.iter().cloned().map(|e| match e { Event::InlineHtml(html) => Event::Html(escaped(&html).into()), e => e.into_static(), }); said.collect() } [] => Vec::new(), _ => vec![Event::Html(escaped(text).into())], } } /// The folds `open` holds past its first `kept`, closed. fn shut(out: &mut Vec>, open: &mut Vec<(Fold, usize)>, kept: usize) { for (fold, _) in open.drain(kept..).rev() { out.push(pulldown_cmark::Event::Html(fold.closer().into())); } } /// A line of an HTML block, `depth` blocks deep, as [`folds`] shows it. fn unfolded<'a>( html: &str, depth: usize, options: pulldown_cmark::Options, open: &mut Vec<(Fold, usize)>, out: &mut Vec>, ) { use pulldown_cmark::Event; // The text since the last tag, said in a summary or not. let said = |text: String, summarised: bool, out: &mut Vec>| { if text.is_empty() { return; } if summarised { let words: Vec> = summary(&text, options); out.extend(words); } else { out.push(Event::Html(escaped(&text).into())); } }; let mut text = String::new(); let mut rest = html; while let Some(at) = rest.find('<') { text.push_str(&rest[..at]); rest = &rest[at..]; let top = open.last().copied(); let under = open.len().checked_sub(2).map(|i| open[i]); let before = std::mem::take(&mut text); let kept: Option<(&'static str, usize)> = match fold_tag(rest) { Some((FoldTag::Opens(Fold::Details, shown), len)) => { open.push((Fold::Details, depth)); Some((if shown { "
" } else { "
" }, len)) } Some((FoldTag::Opens(Fold::Summary, _), len)) if top == Some((Fold::Details, depth)) => { open.push((Fold::Summary, depth)); Some(("", len)) } Some((FoldTag::Closes(Fold::Summary), len)) if top == Some((Fold::Summary, depth)) => { open.pop(); Some(("", len)) } Some((FoldTag::Closes(Fold::Details), len)) if top == Some((Fold::Details, depth)) => { open.pop(); Some(("
", len)) } // A summary left open closes with its fold. Some((FoldTag::Closes(Fold::Details), len)) if top == Some((Fold::Summary, depth)) && under == Some((Fold::Details, depth)) => { open.truncate(open.len() - 2); Some(("
", len)) } _ => None, }; match kept { Some((tag, len)) => { said(before, top == Some((Fold::Summary, depth)), out); out.push(Event::Html(tag.into())); rest = &rest[len..]; } None => { text = before; text.push('<'); rest = &rest[1..]; } } } text.push_str(rest); said(text, open.last() == Some(&(Fold::Summary, depth)), out); } /// A `
` or `` tag [`folds`] keeps. enum FoldTag { /// Opening one, and for `
`, whether it is drawn open. Opens(Fold, bool), Closes(Fold), } /// The fold tag `html` starts with, and its length: `
`, /// `
`, `` or the closing tag of either, in any /// case, with any space before its `>`. fn fold_tag(html: &str) -> Option<(FoldTag, usize)> { let end = html.find('>')?; let inner = &html[1..end]; let (closes, inner) = match inner.strip_prefix('/') { Some(inner) => (true, inner), None => (false, inner), }; if !inner.starts_with(|c: char| c.is_ascii_alphabetic()) { return None; } let mut words = inner.split_ascii_whitespace(); let fold = match words.next()?.to_ascii_lowercase().as_str() { "details" => Fold::Details, "summary" => Fold::Summary, _ => return None, }; let attributes: Vec<&str> = words.collect(); let tag = match (closes, fold, attributes.as_slice()) { (true, _, []) => FoldTag::Closes(fold), (false, _, []) => FoldTag::Opens(fold, false), (false, Fold::Details, [open]) if open.eq_ignore_ascii_case("open") => { FoldTag::Opens(Fold::Details, true) } _ => return None, }; Some((tag, end + 1)) } /// Fenced code in a language highlighted here, its block opened with the /// ranges [`highlight::fence`] gives, which the page paints as it paints a /// file's. fn fenced(mut events: Vec>) -> Vec> { use pulldown_cmark::{CodeBlockKind, Event, Tag}; for i in 1..events.len() { let (Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(info))), Event::Text(text)) = (&events[i - 1], &events[i]) else { continue; }; if let Some(ranges) = highlight::fence(info, text) { let open = format!("
", ranges.encode());
            events[i - 1] = Event::Html(open.into());
        }
    }
    events
}

/// Text's mentions, `@handle`, as links to the account's profile: to its
/// DID where `mentioned` names it, else to the handle. Not in code, nor in
/// what is already a link.
fn mentions<'a>(
    events: impl Iterator>,
    mentioned: &HashMap,
) -> impl Iterator> {
    use pulldown_cmark::{Event, LinkType, Tag, TagEnd};
    let mut inside = 0usize;
    events.flat_map(move |event| match event {
        Event::Text(text) if inside == 0 => {
            let found = knotless::mentions::in_text(&text);
            if found.is_empty() {
                return vec![Event::Text(text)];
            }
            let mut events = Vec::new();
            let mut done = 0;
            for (at, len, handle) in found {
                events.push(Event::Text(text[done..at].to_string().into()));
                let whom = mentioned.get(&handle).unwrap_or(&handle);
                events.push(Event::Start(Tag::Link {
                    link_type: LinkType::Inline,
                    dest_url: format!("/{}", url_path(whom)).into(),
                    title: "".into(),
                    id: "".into(),
                }));
                events.push(Event::Text(text[at..at + len].to_string().into()));
                events.push(Event::End(TagEnd::Link));
                done = at + len;
            }
            events.push(Event::Text(text[done..].to_string().into()));
            events
        }
        event => {
            match &event {
                Event::Start(Tag::Link { .. } | Tag::Image { .. } | Tag::CodeBlock(_)) => {
                    inside += 1
                }
                Event::End(TagEnd::Link | TagEnd::Image | TagEnd::CodeBlock) => inside -= 1,
                _ => {}
            }
            vec![event]
        }
    })
}

/// Text's `at://` URIs of the pulls and issues `named` holds, as links to
/// their pages, each shown as its title. Not in code, nor in what is
/// already a link.
fn references<'a>(
    events: impl Iterator>,
    named: &Named,
) -> impl Iterator> {
    use pulldown_cmark::{Event, Tag, TagEnd};
    let mut inside = 0usize;
    events.flat_map(move |event| match event {
        Event::Text(text) if inside == 0 => referenced(text, named),
        event => {
            match &event {
                Event::Start(Tag::Link { .. } | Tag::Image { .. } | Tag::CodeBlock(_)) => {
                    inside += 1
                }
                Event::End(TagEnd::Link | TagEnd::Image | TagEnd::CodeBlock) => inside -= 1,
                _ => {}
            }
            vec![event]
        }
    })
}

/// `text` as text, with each `at://` URI in it that `named` holds made a
/// link to its page, showing its title.
fn referenced<'a>(
    text: pulldown_cmark::CowStr<'a>,
    named: &Named,
) -> Vec> {
    use pulldown_cmark::{Event, LinkType, Tag, TagEnd};
    let mut events = Vec::new();
    let (mut done, mut from) = (0, 0);
    while let Some(found) = text[from..].find("at://") {
        let at = from + found;
        from = at + "at://".len();
        let before = text[..at].chars().next_back();
        if before.is_some_and(char::is_alphanumeric) {
            continue;
        }
        let rest = &text[at..];
        let end = rest.find(|c: char| c.is_whitespace() || "<>()[]\"'`".contains(c));
        let uri =
            rest[..end.unwrap_or(rest.len())].trim_end_matches(['.', ',', ';', ':', '!', '?']);
        let Some((path, title)) = named.get(uri) else {
            continue;
        };
        if done < at {
            events.push(Event::Text(text[done..at].to_string().into()));
        }
        events.push(Event::Start(Tag::Link {
            link_type: LinkType::Inline,
            dest_url: path.clone().into(),
            title: "".into(),
            id: "".into(),
        }));
        events.push(Event::Text(title.clone().into()));
        events.push(Event::End(TagEnd::Link));
        done = at + uri.len();
        from = done;
    }
    if done == 0 {
        return vec![Event::Text(text)];
    }
    if done < text.len() {
        events.push(Event::Text(text[done..].to_string().into()));
    }
    events
}

/// Text's bare web addresses as links, as GitHub makes them, but not in
/// code or in what is already a link.
fn autolinks<'a>(
    events: impl Iterator>,
) -> impl Iterator> {
    use pulldown_cmark::{Event, Tag, TagEnd};
    let mut inside = 0usize;
    events.flat_map(move |event| match event {
        Event::Text(text) if inside == 0 => linked(text),
        event => {
            match &event {
                Event::Start(Tag::Link { .. } | Tag::Image { .. } | Tag::CodeBlock(_)) => {
                    inside += 1
                }
                Event::End(TagEnd::Link | TagEnd::Image | TagEnd::CodeBlock) => inside -= 1,
                _ => {}
            }
            vec![event]
        }
    })
}

/// `text` as text, with each web address in it made a link.
fn linked(text: pulldown_cmark::CowStr<'_>) -> Vec> {
    use pulldown_cmark::{Event, LinkType, Tag, TagEnd};
    let mut events = Vec::new();
    let mut done = 0;
    let mut at = 0;
    while at < text.len() {
        let after = text[..at].chars().next_back();
        let starts = after.is_none_or(|c| c.is_whitespace() || "(*_~".contains(c));
        let found = starts.then(|| web_address(&text[at..])).flatten();
        let Some((len, dest)) = found else {
            at += text[at..].chars().next().map_or(1, char::len_utf8);
            continue;
        };
        if done < at {
            events.push(Event::Text(text[done..at].to_string().into()));
        }
        events.push(Event::Start(Tag::Link {
            link_type: LinkType::Autolink,
            dest_url: dest.into(),
            title: "".into(),
            id: "".into(),
        }));
        events.push(Event::Text(text[at..at + len].to_string().into()));
        events.push(Event::End(TagEnd::Link));
        at += len;
        done = at;
    }
    if done == 0 {
        return vec![Event::Text(text)];
    }
    if done < text.len() {
        events.push(Event::Text(text[done..].to_string().into()));
    }
    events
}

/// The web address `text` starts with, by GFM's extended autolinks: its
/// length, less trailing punctuation and unmatched `)`, and where it goes.
fn web_address(text: &str) -> Option<(usize, String)> {
    let scheme = ["https://", "http://", "www."].into_iter().find(|s| {
        let start = text.as_bytes().get(..s.len());
        start.is_some_and(|b| b.eq_ignore_ascii_case(s.as_bytes()))
    })?;
    let end = text.find(|c: char| c.is_whitespace() || c == '<');
    let mut url = &text[..end.unwrap_or(text.len())];
    while let Some(last) = url.chars().last() {
        let opens = url.matches('(').count();
        let unmatched = last == ')' && opens < url.matches(')').count();
        if !unmatched && !".,:;!?*_~'\"".contains(last) {
            break;
        }
        url = &url[..url.len() - 1];
    }
    let rest = if scheme == "www." {
        url
    } else {
        &url[scheme.len()..]
    };
    let domain = &rest[..rest
        .find(|c: char| !(c.is_alphanumeric() || "-_.".contains(c)))
        .unwrap_or(rest.len())];
    let segments: Vec<&str> = domain.split('.').collect();
    let last_two = &segments[segments.len().saturating_sub(2)..];
    let valid = segments.len() > 1 && last_two.iter().all(|s| !s.is_empty() && !s.contains('_'));
    let dest = if scheme == "www." {
        format!("http://{url}")
    } else {
        url.to_string()
    };
    valid.then_some((url.len(), dest))
}

/// A link from Markdown found in `dir`: web and mail links as they are,
/// other schemes nowhere, fragments in place, and paths resolved inside the
/// repository, whose files are under `root` and directories under `trees`,
/// never above it.
fn link(dest: &str, root: &str, dir: &str, trees: Trees) -> String {
    let scheme = dest
        .split_once(':')
        .map(|(s, _)| s)
        .filter(|s| !s.contains(['/', '#', '?']));
    match scheme.map(str::to_ascii_lowercase).as_deref() {
        Some("http" | "https" | "mailto") => dest.to_string(),
        Some(_) => "#".to_string(),
        None if dest.starts_with('#') => dest.to_string(),
        None => {
            let (path, rest) = dest.split_at(dest.find(['?', '#']).unwrap_or(dest.len()));
            let from = if path.starts_with('/') { "" } else { dir };
            let mut parts = Vec::new();
            for part in from.split('/').chain(path.split('/')) {
                match part {
                    "" | "." => {}
                    ".." => {
                        parts.pop();
                    }
                    part => parts.push(part),
                }
            }
            let path = parts.join("/");
            let root = match trees {
                Some((trees, is_dir)) if is_dir(&unescaped(&path)) => trees,
                _ => root,
            };
            format!("{root}/{path}{rest}")
        }
    }
}

/// A URL's path as the tree names it, its `%XX` escapes decoded.
fn unescaped(path: &str) -> String {
    let bytes = path.as_bytes();
    let mut out = Vec::new();
    let mut at = 0;
    while at < bytes.len() {
        let hex = bytes
            .get(at + 1..at + 3)
            .filter(|h| h.iter().all(u8::is_ascii_hexdigit));
        let byte = hex.and_then(|h| u8::from_str_radix(std::str::from_utf8(h).ok()?, 16).ok());
        match (bytes[at], byte) {
            (b'%', Some(byte)) => {
                out.push(byte);
                at += 3;
            }
            (b, _) => {
                out.push(b);
                at += 1;
            }
        }
    }
    String::from_utf8_lossy(&out).into_owned()
}

fn is_image(path: &str) -> Option<&'static str> {
    let ext = path.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase())?;
    Some(match ext.as_str() {
        "png" => "image/png",
        "jpg" | "jpeg" => "image/jpeg",
        "gif" => "image/gif",
        "webp" => "image/webp",
        "avif" => "image/avif",
        "svg" => "image/svg+xml",
        "ico" => "image/x-icon",
        _ => return None,
    })
}

fn is_binary(bytes: &[u8]) -> bool {
    bytes[..bytes.len().min(8000)].contains(&0) || std::str::from_utf8(bytes).is_err()
}

/// The file itself. Never run as a page: sandboxed, and typed only as text
/// or an image. A private repository's, to a member logged in to the pages
/// alone, and kept by no cache.
async fn raw(State(app): State>, headers: HeaderMap, path: RestPath) -> Result {
    let viewer = crate::login::viewer(&app, &headers);
    let (file, touched) = crate::private::reading(viewer, None, raw_file(&app, path)).await;
    let mut res = file?;
    if touched {
        res = crate::private::unshared(res);
    }
    let vary = HeaderValue::from_static("cookie");
    res.headers_mut().append(header::VARY, vary);
    Ok(res)
}

async fn raw_file(app: &App, Path((authority, name, rest)): RestPath) -> Result {
    let repo = Repo::open(app, &authority, &name).await?;
    let (rev, path) = repo.locate(&rest)?;
    let bytes = repo.objects()?.blob(&rev, path).map_err(not_found)?;
    let kind = match is_image(path) {
        Some(image) => image,
        None if is_binary(&bytes) => "application/octet-stream",
        None => "text/plain; charset=utf-8",
    };
    Ok((
        [
            (header::CONTENT_TYPE, kind),
            (header::CACHE_CONTROL, cache_for(&rev)),
            (
                header::CONTENT_SECURITY_POLICY,
                "sandbox; default-src 'none'; style-src 'unsafe-inline'",
            ),
            (header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
        ],
        bytes,
    )
        .into_response())
}

/// Why a file's rows are not shown, if they are not.
fn note(f: &diff::File) -> Option<&'static str> {
    if f.binary {
        Some("A binary file, not shown.")
    } else if f.rows.is_empty() {
        Some(if f.status == Status::Renamed {
            "Moved without changes."
        } else {
            "No changes to the content."
        })
    } else if f.rows.len() > DIFF_ROWS {
        Some("A large diff, not shown.")
    } else {
        None
    }
}

/// The text of the blob `oid` names, if `texts` has it.
fn text_of<'a>(texts: &'a HashMap, oid: &Option) -> Option<&'a str> {
    texts.get(oid.as_deref()?).map(String::as_str)
}

/// One file of a diff without its rows, as a list of what changed shows
/// it: why its rows would not be shown, or how many they are.
fn listed(f: &diff::File) -> FileDiffView {
    let status = match f.status {
        Status::Added => "added",
        Status::Deleted => "deleted",
        Status::Renamed => "renamed",
        Status::Modified => "modified",
    };
    let note = note(f);
    let shown = note.is_none();
    let hunks = f.rows.iter().filter(|r| r.kind == Kind::Hunk).count();
    FileDiffView {
        old: f.old.clone(),
        new: f.new.clone(),
        status: status.to_string(),
        added: f.added,
        removed: f.removed,
        note: note.map(str::to_string),
        ranges: None,
        rows: Vec::new(),
        row_count: shown.then_some(f.rows.len()),
        hunk_count: shown.then_some(hunks),
    }
}

/// One file of a diff, its rows highlighted in the whole of each side
/// `texts` has, by blob.
fn file_diff(f: diff::File, texts: &HashMap) -> FileDiffView {
    let note = note(&f);
    let path = f.path();
    let sides = highlight::Sides {
        old: text_of(texts, &f.old_blob),
        new: text_of(texts, &f.new_blob),
    };
    let ranges = note
        .is_none()
        .then(|| highlight::rows(path, &f.rows, sides).encode());
    let listed = listed(&f);
    let rows = match note {
        Some(_) => Vec::new(),
        None => f.rows.into_iter().map(row).collect(),
    };
    FileDiffView {
        ranges,
        rows,
        row_count: None,
        hunk_count: None,
        ..listed
    }
}

fn row(r: diff::Row) -> DiffRowView {
    let kind = match r.kind {
        Kind::Hunk => "h",
        Kind::Note => "x",
        Kind::Add => "a",
        Kind::Del => "d",
        Kind::Context => "",
    };
    DiffRowView {
        kind: kind.to_string(),
        line: r.kind.is_line(),
        old: r.old,
        new: r.new,
        text: r.text,
    }
}

/// What `git  ` prints as a patch of `paths` (all, if none),
/// read into files in the order an editor's file tree lists them, with
/// their rows as `rows` says: a file without is neither highlighted nor
/// has its blobs read.
async fn files(
    git: &Git,
    verb: &[&str],
    rev: &str,
    paths: &[&str],
    rows: Rows,
) -> Result> {
    let mut args = vec!["-c", "core.quotePath=false"];
    args.extend(verb);
    args.extend([
        "-M",
        "--full-index",
        "--no-color",
        "--no-ext-diff",
        rev,
        "--",
    ]);
    args.extend(paths);
    let patch = git.text(&args, None).await.map_err(not_found)?;
    let mut parsed = diff::parse(&patch);
    diff::sort(&mut parsed);
    let with = match rows {
        Rows::None => return Ok(parsed.iter().map(listed).collect()),
        Rows::All => vec![true; parsed.len()],
        Rows::First => first(&parsed),
    };
    let shown: Vec<&str> = parsed
        .iter()
        .zip(&with)
        .filter(|(f, with)| **with && note(f).is_none())
        .flat_map(|(f, _)| [&f.old_blob, &f.new_blob])
        .filter_map(|oid| oid.as_deref())
        .collect();
    let texts = texts(git, &shown).await;
    // Each listed, and those with their rows drawn in its place.
    let mut out: Vec = parsed.iter().map(listed).collect();
    let picked = parsed.into_iter().zip(with).enumerate();
    for (i, (f, _)) in picked.filter(|(_, (_, with))| *with) {
        out[i] = file_diff(f, &texts);
    }
    Ok(out)
}

/// Which of `files` a pull's files page draws with their rows at once:
/// from the first, those whose rows would show, as many as
/// [`FIRST_FILES`] and [`FIRST_LINES`] allow, passing over any that
/// change more than [`FIRST_HELD`] lines.
fn first(files: &[diff::File]) -> Vec {
    let mut with = vec![false; files.len()];
    let (mut count, mut lines) = (0, 0);
    for (i, f) in files.iter().enumerate() {
        let changed = f.added + f.removed;
        if note(f).is_some() || changed > FIRST_HELD {
            continue;
        }
        if count == FIRST_FILES || lines + changed > FIRST_LINES {
            break;
        }
        with[i] = true;
        count += 1;
        lines += changed;
    }
    with
}

/// The text of each blob of `oids` small enough to highlight, by name. One
/// git cannot read is left out, and its rows highlighted on their own.
async fn texts(git: &Git, oids: &[&str]) -> HashMap {
    let names: String = oids.iter().map(|o| format!("{o}\n")).collect();
    let check = ["cat-file", "--batch-check=%(objectname) %(objectsize)"];
    let sizes = git.text(&check, Some(names.as_bytes())).await;
    let small: String = sizes
        .unwrap_or_default()
        .lines()
        .filter_map(|l| l.split_once(' '))
        .filter(|(_, size)| size.parse::().is_ok_and(|n| n <= highlight::LIMIT))
        .map(|(oid, _)| format!("{oid}\n"))
        .collect();
    let batch = ["cat-file", "--batch=%(objectname) %(objectsize)"];
    let out = git.run(&batch, Some(small.as_bytes())).await;
    let out = out.unwrap_or_default();
    let mut texts = HashMap::new();
    let mut rest = &out[..];
    while let Some((oid, text, after)) = next_object(rest) {
        texts.insert(oid, String::from_utf8_lossy(text).into_owned());
        rest = after;
    }
    texts
}

/// The first object `cat-file --batch` prints in `out`: its name, its
/// content, and what follows it.
fn next_object(out: &[u8]) -> Option<(String, &[u8], &[u8])> {
    let nl = out.iter().position(|&b| b == b'\n')?;
    let head = std::str::from_utf8(&out[..nl]).ok()?;
    let (oid, size) = head.split_once(' ')?;
    let end = nl + 1 + size.parse::().ok()?;
    let content = out.get(nl + 1..end)?;
    Some((
        oid.to_string(),
        content,
        out.get(end + 1..).unwrap_or_default(),
    ))
}

/// The most commits a pull shows.
const PULL_COMMITS: usize = 250;

/// What a pull brings in: for a merged one, what its merge brought into the
/// base; otherwise what its branch has since it left the base, read with
/// the objects of the repository it comes from. Its files carry their
/// rows as `rows` says.
async fn pull_changes(
    app: &App,
    repo: &Repo,
    item: &Item,
    base: &str,
    head: &PullHead,
    rows: Rows,
) -> Result {
    let (git, start, tip, at) = pull_range(app, repo, item, base, head).await?;
    let range = format!("{start}..{tip}");
    let log = read::log(&git, &range, "", PULL_COMMITS).await;
    let mut commits = shown(app, repo, log.map_err(not_found)?).await;
    commits.reverse();
    let files = files(&git, &["diff"], &format!("{start}...{tip}"), &[], rows).await?;
    let head = item.merged.is_none().then_some(tip);
    Ok(ChangesView {
        repo: at,
        commits,
        files,
        head,
    })
}

/// What a pull's diff runs between: the git to read it with, the commit
/// it starts from and the one it ends at, and the repository whose pages
/// show its commits.
async fn pull_range(
    app: &App,
    repo: &Repo,
    item: &Item,
    base: &str,
    head: &PullHead,
) -> Result<(Git, String, String, RepoViewBasic)> {
    Ok(match &item.merged {
        Some(m) => {
            let before = item.merged_from.clone().unwrap_or(format!("{m}^"));
            (repo.git.clone(), before, m.clone(), repo.basic(app).await)
        }
        None => {
            let (did, name) = now_named(app, &head.repo).ok_or(Shown(
                StatusCode::NOT_FOUND,
                format!("no repository {}", head.repo.uri),
            ))?;
            let from = app.located(did, &name).await?.git;
            let tip = format!("refs/heads/{}", head.branch);
            let tip = from.rev_parse(&tip).await.map_err(not_found)?;
            let start = format!("refs/heads/{base}");
            let start = repo.git.rev_parse(&start).await.map_err(not_found)?;
            let mut git = repo.git.clone();
            let objects = from.dir.join("objects").to_string_lossy().into_owned();
            git.envs
                .push(("GIT_ALTERNATE_OBJECT_DIRECTORIES".into(), objects));
            let at = RepoViewBasic {
                did: did.to_string(),
                handle: handle(app, did).await,
                name,
            };
            let since = forge::since_of(&git, item, &start, &tip).await;
            let since = since.map_err(not_found)?;
            (git, since.unwrap_or(start), tip, at)
        }
    })
}

/// Everything known about the repository's pulls and issues, and the
/// items they make: from the index, or for a private one from this
/// server's own of its space, which keeps them until they change.
async fn assembled(app: &App, repo: &Repo) -> Result> {
    let index = indexed(app, repo).ok_or(Shown(
        StatusCode::NOT_FOUND,
        "This server keeps no index, so it knows no pulls or issues.".into(),
    ))?;
    let record = repo.record(app).unwrap_or_default();
    let made = index
        .assembled(&repo.did, &repo.key, &record)
        .map_err(anyhow::Error::from)
        .map_err(AppError::internal)?;
    Ok(made)
}

/// The index `repo`'s pulls, issues and reactions are read from: this
/// server's own of its space, which reading it synced
/// ([`App::keep`]), if it is private; else the appview's.
fn indexed<'a>(app: &'a App, repo: &Repo) -> Option<&'a Arc> {
    match repo.in_space {
        Some(_) => app.spaces.as_ref(),
        None => app.index.as_ref(),
    }
}

type ItemPath = Path<(String, String, String, String, String)>;

/// A pull or an issue, by the kind its path names, who opened it, and
/// the repository's pulls and issues.
async fn thread(
    app: &App,
    repo: &Repo,
    kind: &str,
    did: &str,
    rkey: &str,
) -> Result<(Thread, String, Arc)> {
    let collection = match kind {
        "pull" => knotless::records::nsid::PULL,
        "issue" => knotless::records::nsid::ISSUE,
        _ => {
            return Err(Shown(
                StatusCode::NOT_FOUND,
                format!("no such page: {kind}"),
            ));
        }
    };
    let made = assembled(app, repo).await?;
    let record = repo.record(app).unwrap_or_default();
    let uri = repo.named(&format!("at://{did}/{collection}/{rkey}"));
    let (entries, items) = (&made.entries, &made.items);
    let thread = forge::thread_in(&record, &repo.did, entries, items, &uri);
    let thread = thread.map_err(not_found)?;
    let author = handle(app, &thread.item.author).await;
    Ok((thread, author, made))
}

/// A pull's or an issue's card, its repository's owner's picture
/// looked up.
async fn item_card(app: &App, repo: &Repo, kind: &str, thread: &Thread, author: &str) -> Card {
    Card::Item {
        owner: repo.authority.clone(),
        avatar: avatar(app, &repo.did).await,
        name: repo.name.clone(),
        pull: kind == "pull",
        title: thread.item.title.clone(),
        author: author.to_string(),
        opened: Some(thread.item.created_at.clone()),
        comments: thread.comments.len(),
        reviews: thread.item.reviews.len(),
        size: match kind {
            "pull" => pull_size(app, repo, &thread.item).await,
            _ => None,
        },
    }
}

/// What a pull changes, for its card: its commits, the files it changes
/// and the lines it adds and removes; none when its range cannot be read.
async fn pull_size(app: &App, repo: &Repo, item: &Item) -> Option {
    let (base, head) = item.base.as_ref().zip(item.head.as_ref())?;
    let range = pull_range(app, repo, item, base, head).await;
    let (git, start, tip, _) = range.ok()?;
    let commits = format!("{start}..{tip}");
    let commits = git.text(&["rev-list", "--count", &commits], None).await;
    let commits = commits.ok()?.parse().ok()?;
    let changed = format!("{start}...{tip}");
    let stat = git.text(&["diff", "--shortstat", &changed], None).await;
    let (files, added, removed) = shortstat(&stat.ok()?);
    Some(card::Size {
        commits,
        files,
        added,
        removed,
    })
}

/// The files, lines added and lines removed `git diff --shortstat` counts,
/// as ` 4 files changed, 181 insertions(+), 18 deletions(-)`, each left
/// out when none.
fn shortstat(stat: &str) -> (usize, usize, usize) {
    let mut counted = (0, 0, 0);
    for part in stat.split(',') {
        let mut words = part.split_whitespace();
        let n = words.next().and_then(|n| n.parse().ok()).unwrap_or(0);
        match words.next().unwrap_or_default() {
            w if w.starts_with("file") => counted.0 = n,
            w if w.starts_with("insertion") => counted.1 = n,
            w if w.starts_with("deletion") => counted.2 = n,
            _ => {}
        }
    }
    counted
}

static CARDS: LazyLock = LazyLock::new(|| card::Cache::new(512));

#[derive(Deserialize, Default)]
pub struct CardQuery {
    v: Option,
}

/// A card as a PNG: cached for good under the version its page names, and
/// checked again under any other, as it may have changed since. One drawn
/// without the picture it names, which could not be read, is neither kept
/// nor tagged, so that the next ask draws it again.
async fn card_png(
    ctx: &knotless::env::Ctx,
    card: Card,
    asked: Option,
    headers: &HeaderMap,
) -> Response {
    let svg = card.svg();
    let version = card::version(&svg);
    let tag = format!("\"{version}\"");
    let cache = if asked.as_deref() == Some(version.as_str()) {
        "public, max-age=31536000, immutable"
    } else {
        "no-cache"
    };
    // Only a whole card is tagged, so one asked for by its tag is whole.
    let (mut res, whole) = if headers
        .get(header::IF_NONE_MATCH)
        .is_some_and(|v| v.as_bytes() == tag.as_bytes())
    {
        (StatusCode::NOT_MODIFIED.into_response(), true)
    } else {
        let (png, whole) = match looked("cards", CARDS.drawn.get(&version)) {
            Some(png) => (png, true),
            None => {
                let wanted = card.picture().map(|url| card_picture(ctx, url));
                let picture = futures::future::OptionFuture::from(wanted).await;
                let whole = picture.as_ref().is_none_or(Option::is_some);
                let png = CARDS.draw(&version, svg, picture.flatten(), whole).await;
                (png, whole)
            }
        };
        let mut res = axum::body::Body::from(axum::body::Bytes::from_owner(png)).into_response();
        res.headers_mut()
            .insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png"));
        (res, whole)
    };
    let h = res.headers_mut();
    if !whole {
        h.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
        return res;
    }
    h.insert(header::CACHE_CONTROL, HeaderValue::from_static(cache));
    h.insert(header::ETAG, HeaderValue::from_str(&tag).unwrap());
    res
}

async fn home_card(
    State(app): State>,
    headers: HeaderMap,
    Query(q): Query,
) -> Response {
    card_png(&app.ctx, Card::Home, q.v, &headers).await
}

async fn repo_card(
    State(app): State>,
    Path((authority, name)): RepoPath,
    headers: HeaderMap,
    Query(q): Query,
) -> Result {
    let repo = Repo::open(&app, &authority, &name).await?;
    let open = repo.open_items(&app).await.unwrap_or_default();
    let avatar = avatar(&app, &repo.did).await;
    let card = repo.card(repo.record(&app).as_ref(), open, avatar);
    Ok(card_png(&app.ctx, card, q.v, &headers).await)
}

async fn item_card_png(
    State(app): State>,
    Path((authority, name, kind, did, rkey)): ItemPath,
    headers: HeaderMap,
    Query(q): Query,
) -> Result {
    let repo = Repo::open(&app, &authority, &name).await?;
    let (thread, author, _) = self::thread(&app, &repo, &kind, &did, &rkey).await?;
    let card = item_card(&app, &repo, &kind, &thread, &author).await;
    Ok(card_png(&app.ctx, card, q.v, &headers).await)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn only_a_named_runners_logs_are_read_ahead() {
        let dir = tempfile::tempdir().unwrap();
        let index = Index::open(&dir.path().join("i.db")).unwrap();
        let ran = serde_json::json!({"name": "r", "key": "k", "runners": [{"did": "did:ci"}],
            "createdAt": "2026-01-01T00:00:00Z"});
        index.put("did:a", nsid::REPO, "r", &ran).unwrap();
        let repo = serde_json::json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
        let check = |by: &str, rkey: &str, log: bool| {
            let mut run = serde_json::json!({"repo": repo, "ref": "refs/heads/main",
                "commit": "c", "workflow": "test", "status": "success",
                "startedAt": "2026-01-02T00:00:00Z"});
            if log {
                run["log"] = serde_json::json!({"$type": "blob", "ref": {"$link": "bafylog"},
                    "mimeType": "text/plain", "size": 3});
            }
            index.put(by, nsid::CHECK, rkey, &run).unwrap();
        };
        check("did:ci", "1", true);
        check("did:ci", "2", false);
        check("did:other", "3", true);
        assert_eq!(
            log_to_warm(&index, "did:ci", "1").as_deref(),
            Some("bafylog")
        );
        // No log yet, a runner the repository does not name, or no check.
        assert_eq!(log_to_warm(&index, "did:ci", "2"), None);
        assert_eq!(log_to_warm(&index, "did:other", "3"), None);
        assert_eq!(log_to_warm(&index, "did:ci", "none"), None);
    }

    #[test]
    fn first_files_drawn_stop_at_ten_or_the_lines() {
        let file = |changed: usize, binary: bool| diff::File {
            binary,
            added: changed,
            rows: vec![diff::Row {
                kind: Kind::Add,
                old: None,
                new: Some(1),
                text: "x".into(),
            }],
            ..diff::File::default()
        };
        // A binary file and one too large to draw at once are passed over.
        let mut files = vec![file(1, true), file(FIRST_HELD + 1, false)];
        files.extend((0..12).map(|_| file(10, false)));
        let with = first(&files);
        assert_eq!(with.iter().filter(|w| **w).count(), FIRST_FILES);
        assert_eq!(&with[..3], [false, false, true]);
        // Five of 300 lines fill the budget, and the sixth waits.
        let files: Vec = (0..6).map(|_| file(300, false)).collect();
        assert_eq!(first(&files), [true, true, true, true, true, false]);
    }

    #[test]
    fn refusals_for_want_of_scope_say_how_to_grant_it() {
        let gateway = format!(
            "Conflict (409): Forbidden (403): {}",
            knotless::oauth::GRANT_BEHIND
        );
        let shown = behind(Shown(StatusCode::BAD_REQUEST, gateway));
        assert_eq!(
            (shown.0, shown.1.as_str()),
            (StatusCode::FORBIDDEN, knotless::oauth::GRANT_BEHIND)
        );
        let pds =
            "InvalidToken (403): Scope does not permit create on at://did:plc:a/x".to_string();
        let shown = behind(Shown(StatusCode::BAD_REQUEST, pds));
        assert_eq!(
            (shown.0, shown.1.as_str()),
            (StatusCode::FORBIDDEN, knotless::oauth::LOGIN_BEHIND)
        );
        let other = behind(Shown(StatusCode::NOT_FOUND, "no such repository".into()));
        assert_eq!(
            (other.0, other.1.as_str()),
            (StatusCode::NOT_FOUND, "no such repository")
        );
    }

    #[test]
    fn a_log_shown_from_its_tail() {
        assert_eq!(log_tail(b"==> fmt\nok\n"), "==> fmt\nok\n");
        assert_eq!(log_tail(b""), "");
        // Cut a byte into a three-byte character: shown from the next one.
        let mut long = vec![b'a'; LOG_SHOWN - 1];
        long.splice(0..0, "\u{2713}".bytes());
        let shown = log_tail(&long);
        assert_eq!(shown.len(), LOG_SHOWN - 1);
        assert!(shown.bytes().all(|b| b == b'a'));
    }

    #[test]
    fn shortstats() {
        let stat = " 4 files changed, 181 insertions(+), 18 deletions(-)\n";
        assert_eq!(shortstat(stat), (4, 181, 18));
        assert_eq!(shortstat(" 1 file changed, 1 deletion(-)"), (1, 0, 1));
        assert_eq!(shortstat(""), (0, 0, 0));
    }

    #[test]
    fn what_merges_brought() {
        let range = |from: &str, to: &str| (from.to_string(), to.to_string());
        assert_eq!(brought("m", Some("f")), range("f", "m"));
        assert_eq!(brought("m", None), range("m^1", "m^2"));
    }

    #[test]
    fn coauthor_trailers() {
        let message = "Fix it\n\nCo-authored-by: Claude \nSigned-off-by: A \nco-authored-by:  B  < b@x > \nCo-authored-by: nobody\n";
        let named = |n: &str, e: &str| (n.to_string(), e.to_string());
        assert_eq!(
            coauthors(message),
            [named("Claude", "noreply@anthropic.com"), named("B", "b@x")]
        );
        assert!(coauthors("Co-authored-by: A ").is_empty());
        assert!(coauthors("Subject\n\nCo-authored-by: A  (String, Arc) {
        use std::sync::atomic::{AtomicUsize, Ordering};
        let asked = Arc::new(AtomicUsize::new(0));
        let counted = asked.clone();
        let mut pixmap = resvg::tiny_skia::Pixmap::new(8, 8).unwrap();
        pixmap.fill(resvg::tiny_skia::Color::from_rgba8(200, 10, 20, 255));
        let face = card::encode(&pixmap);
        let blob = face.clone();
        let small = move || {
            counted.fetch_add(1, Ordering::SeqCst);
            std::future::ready(face.clone())
        };
        let router = Router::new()
            .route("/img/avatar_thumbnail/plain/did/face@jpeg", get(small))
            .route(
                "/xrpc/com.atproto.sync.getBlob",
                get(move || std::future::ready(blob.clone())),
            )
            .route("/big", get(|| async { vec![0u8; PICTURE_MAX + 1] }))
            .route("/slow", get(std::future::pending::<&'static str>));
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let base = format!("http://{}", listener.local_addr().unwrap());
        tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
        (base, asked)
    }

    #[tokio::test]
    async fn pictures_on_cards() {
        use std::sync::atomic::Ordering;
        let (base, asked) = pictures_served().await;
        let cache = tempfile::tempdir().unwrap();
        let cache_dir = cache.path().to_str().unwrap().to_string();
        // Every account's PDS is the server of pictures.
        let vars = |name: &str| match name {
            "KNOTLESS_SERVICE" => Some(base.clone()),
            "KNOTLESS_CACHE_DIR" => Some(cache_dir.clone()),
            _ => None,
        };
        let ctx = knotless::env::Ctx::new(knotless::env::Env::from_vars(vars));
        // A knotless profile's picture, from the account's PDS.
        let own = card_picture(&ctx, &own_avatar_path("did:plc:card", "bafyface")).await;
        assert_eq!(&own.unwrap()[..4], b"\x89PNG");
        // The small form of a picture on Bluesky's CDN, read once.
        let url = format!("{base}/img/avatar/plain/did/face@jpeg");
        let face = card_picture(&ctx, &url).await.unwrap();
        assert_eq!(&face[..4], b"\x89PNG");
        assert!(Arc::ptr_eq(&face, &card_picture(&ctx, &url).await.unwrap()));
        assert_eq!(asked.load(Ordering::SeqCst), 1);
        // None too large, none not there, none not sent in time; and none
        // kept, to be asked for again.
        for unread in ["big", "gone", "slow"] {
            let unread = format!("{base}/{unread}");
            assert!(card_picture(&ctx, &unread).await.is_none());
            assert!(PICTURES.get(&unread).is_none());
        }

        let card = |avatar: String| Card::Repo {
            owner: "alice.test".into(),
            avatar: Some(avatar),
            name: "party".into(),
            description: None,
            pulls: 0,
            issues: 0,
            languages: Vec::new(),
        };
        let named = |res: &Response, name: header::HeaderName| res.headers().get(name).cloned();
        let none = HeaderMap::new();
        // Drawn with its picture, a card is kept, and tagged.
        let whole = card(url);
        let version = card::version(&whole.svg());
        let tag = HeaderValue::from_str(&format!("\"{version}\"")).unwrap();
        let res = card_png(&ctx, whole.clone(), Some(version.clone()), &none).await;
        let forever = HeaderValue::from_static("public, max-age=31536000, immutable");
        assert_eq!(named(&res, header::CACHE_CONTROL), Some(forever));
        assert_eq!(named(&res, header::ETAG), Some(tag.clone()));
        let kept = CARDS.drawn.get(&version).unwrap();
        let res = card_png(&ctx, whole.clone(), None, &none).await;
        let body = axum::body::to_bytes(res.into_body(), usize::MAX).await;
        assert_eq!(body.unwrap().as_ref(), kept.as_ref());
        let mut tagged = HeaderMap::new();
        tagged.insert(header::IF_NONE_MATCH, tag);
        let res = card_png(&ctx, whole, Some(version), &tagged).await;
        assert_eq!(res.status(), StatusCode::NOT_MODIFIED);
        // Drawn without the picture it names, as that was not read, a card
        // is neither kept nor tagged, though asked for by its version: the
        // next ask draws it again, and may find the picture.
        let failed = card(format!("{base}/gone"));
        let version = card::version(&failed.svg());
        let res = card_png(&ctx, failed, Some(version.clone()), &none).await;
        assert_eq!(res.status(), StatusCode::OK);
        let again = HeaderValue::from_static("no-cache");
        assert_eq!(named(&res, header::CACHE_CONTROL), Some(again));
        assert_eq!(named(&res, header::ETAG), None);
        assert!(CARDS.drawn.get(&version).is_none());
    }

    /// A lookup for `Kept::through` that counts itself in `asked`, waits
    /// for `go` if given, and finds `found`.
    fn lookup(
        asked: &Arc,
        go: Option>,
        found: Option,
    ) -> impl Future> + Send + 'static {
        let asked = asked.clone();
        async move {
            asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            if let Some(go) = go {
                go.notified().await;
            }
            found
        }
    }

    #[tokio::test]
    async fn kept_values_outlive_their_server_and_their_source() {
        use std::sync::atomic::{AtomicUsize, Ordering};
        let dir = tempfile::tempdir().unwrap();
        let dir = dir.path();
        let asked = Arc::new(AtomicUsize::new(0));
        let kept = |ttl| -> &'static Kept {
            Box::leak(Box::new(Kept::new(
                "kept.json",
                ttl,
                Duration::from_millis(200),
            )))
        };
        // Put before its file is read, a value is kept, but not written.
        let fresh = kept(KEPT_FOR);
        fresh.put("early", 0);
        assert!(!dir.join("kept.json").exists());
        // Never found, it is looked up, once however many ask at once, and
        // then kept, in its file too.
        let go = Arc::new(tokio::sync::Notify::new());
        let first = fresh.through(dir, "k", lookup(&asked, Some(go.clone()), Some(1)));
        let second = fresh.through(dir, "k", lookup(&asked, None, Some(9)));
        let both = futures::future::join(first, second);
        let ((a, b), _) = futures::future::join(both, async { go.notify_one() }).await;
        assert_eq!((a, b), (Some(1), Some(1)));
        assert_eq!(asked.load(Ordering::SeqCst), 1);
        let again = fresh.through(dir, "k", lookup(&asked, None, Some(9)));
        assert_eq!(again.await, Some(1));
        assert_eq!(asked.load(Ordering::SeqCst), 1);

        // A server started after has it at once, from the file, though too
        // old now to be fresh: shown while it is looked up again behind it.
        let stale = kept(Duration::ZERO);
        let go = Arc::new(tokio::sync::Notify::new());
        let shown = stale.through(dir, "k", lookup(&asked, Some(go.clone()), Some(2)));
        assert_eq!(shown.await, Some(1));
        go.notify_one();
        wait_for(|| stale.get("k").map(|(v, _)| v) == Some(2)).await;
        // A lookup that finds nothing, as a PDS that does not answer, keeps
        // what was found before.
        assert_eq!(
            stale.through(dir, "k", lookup(&asked, None, None)).await,
            Some(2)
        );
        wait_for(|| stale.looking.lock().unwrap().is_empty()).await;
        assert_eq!(stale.get("k").map(|(v, _)| v), Some(2));

        // One never found that does not come within the wait is shown as
        // none, and kept once it comes.
        let go = Arc::new(tokio::sync::Notify::new());
        let slow = stale.through(dir, "new", lookup(&asked, Some(go.clone()), Some(3)));
        assert_eq!(slow.await, None);
        go.notify_one();
        wait_for(|| stale.get("new").is_some()).await;
        assert_eq!(fresh.get("early"), Some((0, true)));
    }

    /// Waits until `done`, a few seconds at most.
    async fn wait_for(done: impl Fn() -> bool) {
        let waited = async {
            while !done() {
                tokio::time::sleep(Duration::from_millis(10)).await;
            }
        };
        tokio::time::timeout(Duration::from_secs(5), waited)
            .await
            .unwrap();
    }

    #[test]
    fn bare_addresses_are_links() {
        let md = |text: &str| markdown(text, "/B", "/R", "", None);
        let a = |href: &str, shown: &str| format!(r#"{shown}"#);
        let x = "https://x.dev/a";
        assert_eq!(
            md("see https://x.dev/a"),
            format!("

see {}

\n", a(x, x)) ); let html = md("At https://x.dev/a. Or (https://x.dev/a), and https://x.dev/a?!\n"); assert_eq!(html.matches(&a(x, x)).count(), 3, "{html}"); assert!(html.contains("(),"), "{html}"); let wiki = "https://en.wikipedia.org/wiki/Foo_(bar)"; let html = md("(see https://en.wikipedia.org/wiki/Foo_(bar))"); assert!(html.contains(&format!("{})", a(wiki, wiki))), "{html}"); let html = md("`https://x.dev/a` and\n\n```\nhttps://x.dev/a\n```\n"); assert!(!html.contains(" ![https://x.dev/i](i.png)"); assert_eq!(html.matches("Fixes {}

\n", a(issue, issue))); let html = md("www.x.dev/é and HTTP://X.DEV, after é"); assert!( html.contains(&a("http://www.x.dev/%C3%A9", "www.x.dev/é")), "{html}" ); assert!(html.contains(&a("HTTP://X.DEV", "HTTP://X.DEV")), "{html}"); for plain in [ "xhttps://x.dev", "https://localhost/a", "https://x.dev_a", "www.", "https://a..b", ] { assert!(!md(plain).contains("x\n\n[a](b.md) ![i](i.png) t", "/B", "/R", "", None, ); assert!( html.contains("<script>") && html.contains("<b>"), "{html}" ); assert!( html.contains(r#"href="/B/b.md""#) && html.contains(r#"src="/R/i.png""#), "{html}" ); } /// Each tag's attributes in `html`, by name and value. fn attributes(html: &str) -> Vec<(String, String)> { let mut found = Vec::new(); for tag in html.split('<').skip(1) { let tag = tag.split('>').next().unwrap_or_default(); let mut rest = tag.split_once(' ').map_or("", |(_, r)| r); while let Some((name, after)) = rest.split_once("=\"") { let (value, after) = after.split_once('"').unwrap_or((after, "")); found.push((name.trim().to_ascii_lowercase(), value.to_string())); rest = after; } } found } #[test] fn attached_pictures_are_shown_from_here() { let blob_of = |cid: &str| knotless::records::BlobRef { kind: "blob".into(), link: knotless::records::Link { cid: cid.into() }, mime_type: "image/png".into(), size: 3, }; let images = [ImageAttachment::of(blob_of("bafa"))]; let attached = Attached { repo: "at://did:o/dev.knotless.repo/r", uri: "at://did:a/dev.knotless.comment/1", images: &images, }; let blob = |did: &str, cid: &str| { format!("https://pds.example/xrpc/com.atproto.sync.getBlob?did={did}&cid={cid}") }; let text = format!( "![shot]({}) ![b]({}) ![c]({}) ![d](https://pds.example/xrpc/com.atproto.sync.getRecord?did=did:a&cid=bafa) ![e](https://x.example/a.png?did=did:a) ![f](x.png)", blob("did:a", "bafa"), blob("did:a", "bafb"), blob("did:b", "bafa"), ); let html = written(&text, "/o/r/", Some(attached), &HashMap::new()); let shown = "/xrpc/dev.knotless.getAttachment?repo=at%3A%2F%2Fdid%3Ao%2Fdev.knotless.repo%2Fr&uri=at%3A%2F%2Fdid%3Aa%2Fdev.knotless.comment%2F1&cid=bafa"; assert_eq!(html.matches(shown).count(), 1, "{html}"); // Not attached, by someone else, or not a blob: as it was. assert!( html.contains("cid=bafb") && html.contains("did=did:b"), "{html}" ); assert!( html.contains("getRecord") && html.contains("x.example/a.png"), "{html}" ); let elsewhere = Attached { uri: "nonsense", ..attached }; let none = HashMap::new(); assert!(!written(&text, "/o/r/", Some(elsewhere), &none).contains("getAttachment")); assert!(!html.contains("width="), "{html}"); // One whose record gives its size is drawn at it, wherever it shows. let sized = [ImageAttachment { width: Some(195), height: Some(150), ..images[0].clone() }]; let attached = Attached { images: &sized, ..attached }; let twice = format!("{text} ![again]({})", blob("did:a", "bafa")); let html = written(&twice, "/o/r/", Some(attached), &none); let drawn = format!(r#"".to_string(), ), ), ( pull.to_string(), ("/o/r/pull/did:plc:b/2".to_string(), "Fix it".to_string()), ), ]); let text = format!( "See ({issue}), {pull}. Not `{issue}`, x{issue}, [here]({pull}) or at://did:plc:c/dev.knotless.issue/3; ask @bob.test." ); let none = HashMap::new(); let html = written_naming(&text, "/o/r", None, &none, &named); assert!( html.contains(r#"See (A <crash>), Fix it. Not "#), "{html}" ); assert!(html.contains(&format!("x{issue}")), "{html}"); assert!( html.contains(r#"here"#), "{html}" ); assert!( html.contains("or at://did:plc:c/dev.knotless.issue/3;"), "{html}" ); assert!( html.contains(r#"@bob.test"#), "{html}" ); // Without them, as it was. assert!(written(&text, "/o/r", None, &none).contains(&format!("({issue})"))); } #[test] fn mentions_link_to_whom_they_name() { let mentioned = HashMap::from([("alice.test".to_string(), "did:plc:a".to_string())]); let text = "Ask @Alice.test or @bob.test, not me@carol.test, `@dan.test` or https://x.test/@erin.test.\n\n```\n@fay.test\n```"; let html = written(text, "/o/r", None, &mentioned); assert!( html.contains(r#"Ask @Alice.test or @bob.test, not me@carol.test"#), "{html}" ); assert_eq!(html.matches("@dan.test") && html.contains("@fay.test\n
"), "{html}" ); // A file's Markdown links none. assert!(!markdown(text, "/B", "/R", "", None).contains("/bob.test")); } #[test] fn fenced_code_is_highlighted() { let html = markdown( "```rust\nfn main() {}\n```\n\n```nothing\n\n```\n\n```\n```\n", "/B", "/R", "", None, ); assert!( html.starts_with("
"), "{html}"); // Its text is escaped as ever, and a language without a grammar here // is left as it was. assert!( html.contains("
<b>"),
            "{html}"
        );
    }

    #[test]
    fn markdown_is_sanitized() {
        let html = markdown(
            concat!(
                "\n\n",
                "a \n\n",
                "\n\n",
                "[b](javascript:alert(1)) [c](JAVASCRIPT:x) [d](javascript:x)\n",
                "[e](vbscript:x \"\\\" onmouseover=\\\"x\") ![f](data:image/svg+xml,x)\n",
                "[g]() \n",
            ),
            "/B",
            "/R",
            "",
            None,
        );
        assert!(
            !html.contains("\nClaude Opus 5.5\n\n",
            "## What changes\n\n- [ ] a task\n\n
\n", )); assert!( html.contains("
\nClaude Opus 5.5\n

What changes

"), "{html}" ); assert!(html.ends_with("\n
\n"), "{html}"); assert_eq!( draw("
a <b>
"), "
a <b>
" ); // A summary is Markdown on a line, linked as the rest is; more // than that, or HTML in it, is text. let html = written( "
**Why** `x` [l](javascript:alert(1)) at https://knotless.dev", "/B", None, &HashMap::new(), ); assert!( html.starts_with( "
Why x l" ), "{html}" ); assert!(html.contains(""), "{html}"); for (said, shown) in [ ("# Not a heading", "# Not a heading"), ("![i](i.png)", "![i](i.png)"), ("a b", "a <b>b</b>"), (" ", ""), ] { assert_eq!( draw(&format!("
{said}
")), format!("
{shown}
") ); } // Raw HTML's text is never linked, and is escaped. assert_eq!( draw("
at https://knotless.dev & more"), "
at https://knotless.dev & more
" ); // Anything else about them is text: attributes, a tag in a // paragraph, one unfinished. for text in [ "
", "
", "
", "a
b", "", "", ] { let html = draw(text); assert!(!html.contains("\n\n
"), "</summary>\n<summary>\n</details>" ); assert_eq!( draw("
a
"), "
a
" ); assert_eq!( draw("
a"), "
a
" ); // A `<` that starts no tag is text beside them. assert_eq!( draw("
1 < 2 >\n"), "
1 < 2 >\n
" ); let html = draw(">
\n\n
"); assert!( html.contains("
\n
"), "{html}" ); assert!(html.ends_with("</details>"), "{html}"); } #[test] fn failures_by_name() { let named = [ StatusCode::BAD_REQUEST, StatusCode::NOT_FOUND, StatusCode::CONFLICT, StatusCode::BAD_GATEWAY, StatusCode::INTERNAL_SERVER_ERROR, ] .map(failure_name); assert_eq!( named, [ "InvalidRequest", "NotFound", "Conflict", "UpstreamFailure", "InternalServerError" ] ); } /// A failure answered as text is answered as XRPC's, and its request's /// log line still reads why it failed. #[tokio::test] async fn xrpc_failures_keep_their_cause() { let seen = Arc::new(Mutex::new(None)); let kept = seen.clone(); let failing = || async { AppError::internal(anyhow::anyhow!("disk on fire")) }; let router = Router::new() .route("/xrpc/x.y", get(failing)) .layer(middleware::from_fn(xrpc_failures)) .layer(middleware::from_fn(move |req: Request, next: Next| { let kept = kept.clone(); async move { let res = next.run(req).await; let cause = res.extensions().get::(); *kept.lock().unwrap() = cause.map(|c| c.0.clone()); res } })); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let at = format!("http://{}/xrpc/x.y", listener.local_addr().unwrap()); tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); let res = reqwest::get(at).await.unwrap(); assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR); let body: serde_json::Value = res.json().await.unwrap(); assert_eq!(body["error"], "InternalServerError"); let cause = seen.lock().unwrap().clone().expect("the cause is kept"); assert_eq!(body["message"], cause.as_str()); } #[tokio::test] async fn addresses_in_full() { let mut headers = HeaderMap::new(); headers.insert(header::HOST, HeaderValue::from_static("127.0.0.1:2590")); let here = origin_of(&headers); assert_eq!(here, "http://127.0.0.1:2590"); headers.insert("x-forwarded-host", HeaderValue::from_static("site.example")); headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); assert_eq!(origin_of(&headers), "https://site.example"); assert_eq!(origin_of(&HeaderMap::new()), "http://"); let cdn = "https://cdn.example/a".to_string(); let full = ORIGIN.scope(here, async { (absolute(cdn.clone()), absolute("/a".into())) }); assert_eq!(full.await, (cdn, "http://127.0.0.1:2590/a".to_string())); // Outside a request, a path stays one. assert_eq!(absolute("/a".into()), "/a"); } #[test] fn small_things() { assert_eq!(url_path("a b/c#d?é"), "a%20b/c%23d%3F%C3%A9"); assert_eq!(is_image("a/B.PNG"), Some("image/png")); assert_eq!(is_image("Makefile"), None); assert!(is_binary(&[0, 1]) && is_binary(&[0xff, 0xfe]) && !is_binary(b"text")); assert!(is_markdown("docs/A.MD") && is_markdown("notes.markdown")); assert!(!is_markdown("README") && !is_markdown("md") && !is_markdown("a.mdx")); } #[test] fn files_of_a_diff() { let file = |status, binary, rows: usize| diff::File { old: "a.rs".into(), new: "b.rs".into(), old_blob: None, new_blob: None, status, binary, rows: (0..rows) .map(|i| diff::Row { kind: Kind::Add, old: None, new: Some(i as u32), text: "x".into(), }) .collect(), added: rows, removed: 0, }; let none = HashMap::new(); let note = |f| file_diff(f, &none).note.unwrap_or_default(); assert_eq!( note(file(Status::Renamed, false, 0)), "Moved without changes." ); assert_eq!( note(file(Status::Modified, false, 0)), "No changes to the content." ); assert_eq!( note(file(Status::Added, true, 0)), "A binary file, not shown." ); let large = file_diff(file(Status::Deleted, false, DIFF_ROWS + 1), &none); assert_eq!(large.note.as_deref(), Some("A large diff, not shown.")); assert!(large.rows.is_empty() && large.ranges.is_none()); assert_eq!(large.status, "deleted"); let row = |kind, old, new, text: &str| diff::Row { kind, old, new, text: text.into(), }; let mut mixed = file(Status::Modified, false, 0); mixed.rows = vec![ row(Kind::Hunk, None, None, "@@ -1,2 +1,2 @@"), row(Kind::Context, Some(1), Some(1), "a"), row(Kind::Del, Some(2), None, "b"), row(Kind::Note, None, None, "\\ No newline at end of file"), row(Kind::Add, None, Some(2), "c"), ]; let shown = file_diff(mixed, &none); let kinds: Vec<(&str, bool)> = shown .rows .iter() .map(|r| (r.kind.as_str(), r.line)) .collect(); assert_eq!( kinds, [ ("h", false), ("", true), ("d", true), ("x", false), ("a", true) ] ); assert!(shown.note.is_none() && shown.ranges.is_some()); assert_eq!( file_diff(file(Status::Added, false, 1), &none).status, "added" ); assert_eq!( file_diff(file(Status::Renamed, false, 1), &none).status, "renamed" ); // `cat-file --batch`'s objects, each followed by a newline. let out = b"a 2\nhi\nb 0\n\nc x\n"; let (oid, text, rest) = next_object(out).unwrap(); assert_eq!((oid.as_str(), text), ("a", &b"hi"[..])); let (oid, text, rest) = next_object(rest).unwrap(); assert_eq!((oid.as_str(), text), ("b", &b""[..])); assert_eq!(next_object(rest), None); assert_eq!(next_object(b"a 9\nshort\n"), None); } }