johnandersen777.bsky.social/knotless
git that lives in your Atmosphere account
Forked from chadtmiller.com/knotless
0
0
//! What the web pages show, as `dev.knotless.*` XRPC queries, each with
//! its lexicon under `lexicons/`. The pages themselves are a SvelteKit app in `web/`: this server
//! hands every request it does not answer itself to it, and the app reads
//! back here for what to show.
//!
//! Writing is for git, the CLI and agents over MCP, and for the account
//! logged in to the pages, through the same code: a star, a follow, and
//! what is said on a pull or an issue ([`threads`]), and opening one ([`opening`]). The app's types are made from the lexicons, and the tests hold every answer
//! they receive to its lexicon (harness/answers.mjs).
use crate::card::{self, Card};
use crate::diff::{self, Kind, Status};
use crate::index::{Assembled, CheckFilter, Index};
use crate::languages::{self, Language};
use crate::objects::{self, Objects};
use crate::{App, AppError, highlight, repo_name};
use axum::Router;
use axum::extract::{Path, Query, Request, State};
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::middleware::{self, Next};
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use futures::FutureExt;
use knotless::closing;
use knotless::forge::{self, Item, Thread};
use knotless::git::Git;
use knotless::profile;
use knotless::read;
use knotless::records::{
Attachment, CheckOutput, CheckRecord, CheckStep, Freeing, ImageAttachment, MinutesRecord,
ProfileRecord, PullHead, RepoRecord, RepoRef, SiteRecord, nsid,
};
use knotless::remote::Remote;
use knotless::session;
use knotless::store;
use knotless::telemetry::{fill, looked};
use knotless::url::RepoUrl;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt::Write;
use std::sync::atomic::Ordering;
use std::sync::{Arc, LazyLock, Mutex};
use std::time::{Duration, SystemTime};
mod account_data;
mod account_settings;
mod actor;
mod archive;
mod badges;
mod cargo;
mod controls;
mod delegation;
mod feed;
mod feeds;
mod follows;
mod items;
mod landing;
mod live;
mod notifications;
mod opening;
mod orgs;
mod pages;
mod preview;
pub mod push;
mod queue;
mod reactions;
mod releases;
mod repo;
mod repos;
mod runs;
mod search;
mod settings;
pub(crate) use settings::Counts;
mod stars;
mod steps;
mod subscriptions;
mod themes;
mod threads;
mod views;
use actor::{ActorView, RepoViewBasic, repo_basic};
use queue::InQueue;
use views::*;
pub use actor::{actor, get_actor_activity, get_actor_repos, get_actor_stars, list_actor_items};
pub use feed::get_feed;
use follows::{follow_actor, get_followers, get_follows, unfollow_actor};
use items::{get_pull_diff, get_thread, list_items};
use notifications::{get_unread_count, list_notifications, update_seen};
pub use pages::pages;
use push::{register_push, unregister_push};
use reactions::{add_reaction, get_reactions, remove_reaction};
use repo::{
get_blame, get_commit, get_file, get_last_commits, get_repo, get_tree, list_branches,
list_commits, list_paths, list_tags,
};
use stars::{get_viewer, star_repo, unstar_repo};
pub use themes::{cookie as palette_cookie, palette};
/// The queries the pages ask.
pub fn queries() -> Router<Arc<App>> {
Router::new()
.route("/xrpc/dev.knotless.getRepo", get(get_repo))
.route("/xrpc/dev.knotless.searchCode", get(search::search_code))
.route(
"/xrpc/dev.knotless.listReleases",
get(releases::list_releases),
)
.route("/xrpc/dev.knotless.getViewer", get(get_viewer))
.route("/xrpc/dev.knotless.getOrg", get(orgs::get_org))
.route("/_live/repo", get(live::follow_repo))
.route("/_live/check", get(live::follow_check))
.route("/xrpc/dev.knotless.getActorStars", get(get_actor_stars))
.route("/xrpc/dev.knotless.listActorItems", get(list_actor_items))
.route("/xrpc/dev.knotless.starRepo", post(star_repo))
.route("/xrpc/dev.knotless.unstarRepo", post(unstar_repo))
.route("/xrpc/dev.knotless.createRepo", post(repos::create_repo))
.route("/xrpc/dev.knotless.forkRepo", post(repos::fork_repo))
.route(
"/xrpc/dev.knotless.getRepoSettings",
get(settings::get_repo_settings),
)
.route(
"/xrpc/dev.knotless.getRepoStorage",
get(settings::get_repo_storage),
)
.route("/xrpc/dev.knotless.updateRepo", post(settings::update_repo))
.route("/xrpc/dev.knotless.renameRepo", post(settings::rename_repo))
.route(
"/xrpc/dev.knotless.compactRepo",
post(settings::compact_repo),
)
.route(
"/xrpc/dev.knotless.transferRepo",
post(settings::transfer_repo),
)
.route(
"/xrpc/dev.knotless.archiveRepo",
post(settings::archive_repo),
)
.route(
"/xrpc/dev.knotless.acceptTransfer",
post(settings::accept_transfer),
)
.route("/xrpc/dev.knotless.deleteRepo", post(settings::delete_repo))
.route(
"/xrpc/dev.knotless.createBranch",
post(controls::create_branch),
)
.route(
"/xrpc/dev.knotless.deleteBranch",
post(controls::delete_branch),
)
.route(
"/xrpc/dev.knotless.renameBranch",
post(controls::rename_branch),
)
.route(
"/xrpc/dev.knotless.protectBranch",
post(controls::protect_branch),
)
.route(
"/xrpc/dev.knotless.unprotectBranch",
post(controls::unprotect_branch),
)
.route(
"/xrpc/dev.knotless.defineLabel",
post(controls::define_label),
)
.route(
"/xrpc/dev.knotless.removeLabel",
post(controls::remove_label),
)
.route("/xrpc/dev.knotless.shareRepo", post(controls::share_repo))
.route(
"/xrpc/dev.knotless.unshareRepo",
post(controls::unshare_repo),
)
.route("/xrpc/dev.knotless.setTriager", post(controls::set_triager))
.route("/xrpc/dev.knotless.addAgent", post(controls::add_agent))
.route(
"/xrpc/dev.knotless.removeAgent",
post(controls::remove_agent),
)
.route(
"/xrpc/dev.knotless.removeTriager",
post(controls::remove_triager),
)
.route("/xrpc/dev.knotless.listThemes", get(themes::list_themes))
.route("/xrpc/dev.knotless.listSteps", get(steps::list_steps))
.route("/xrpc/dev.knotless.getStep", get(steps::get_step))
.route("/xrpc/dev.knotless.createTheme", post(themes::create_theme))
.route("/xrpc/dev.knotless.deleteTheme", post(themes::delete_theme))
.route("/xrpc/dev.knotless.followActor", post(follow_actor))
.route("/xrpc/dev.knotless.unfollowActor", post(unfollow_actor))
.route("/xrpc/dev.knotless.getFollowers", get(get_followers))
.route("/xrpc/dev.knotless.getFollows", get(get_follows))
.route("/xrpc/dev.knotless.addReaction", post(add_reaction))
.route("/xrpc/dev.knotless.removeReaction", post(remove_reaction))
.route("/xrpc/dev.knotless.getReactions", get(get_reactions))
.route(
"/xrpc/dev.knotless.listNotifications",
get(list_notifications),
)
.route("/xrpc/dev.knotless.getUnreadCount", get(get_unread_count))
.route("/xrpc/dev.knotless.updateSeen", post(update_seen))
.route("/xrpc/dev.knotless.registerPush", post(register_push))
.route("/xrpc/dev.knotless.unregisterPush", post(unregister_push))
.route("/xrpc/dev.knotless.getTree", get(get_tree))
.route("/xrpc/dev.knotless.getLastCommits", get(get_last_commits))
.route("/xrpc/dev.knotless.listPaths", get(list_paths))
.route("/xrpc/dev.knotless.getFile", get(get_file))
.route("/xrpc/dev.knotless.getBlame", get(get_blame))
.route("/xrpc/dev.knotless.listCommits", get(list_commits))
.route("/xrpc/dev.knotless.getCommit", get(get_commit))
.route("/xrpc/dev.knotless.listBranches", get(list_branches))
.route("/xrpc/dev.knotless.listTags", get(list_tags))
.route("/xrpc/dev.knotless.listItems", get(list_items))
.route("/xrpc/dev.knotless.getThread", get(get_thread))
.route("/xrpc/dev.knotless.getPullDiff", get(get_pull_diff))
.route("/xrpc/dev.knotless.getQueue", get(queue::get_queue))
.route("/xrpc/dev.knotless.listRuns", get(runs::list_runs))
.route("/xrpc/dev.knotless.getCheck", get(get_check))
.route("/xrpc/dev.knotless.getAttachment", get(get_attachment))
.route("/xrpc/dev.knotless.getCheckImage", get(get_check_image))
.route("/xrpc/dev.knotless.getAvatar", get(get_avatar))
.merge(landing_routes())
.merge(account_settings::routes())
.merge(account_data::routes())
.merge(threads::routes())
.merge(subscriptions::routes())
.merge(opening::routes())
.merge(preview::routes())
.merge(delegation::routes())
.merge(orgs::routes())
.layer(middleware::from_fn(renamed))
.layer(tower_http::compression::CompressionLayer::new())
}
/// Landing pulls and running checks, from the pages ([`landing`]).
fn landing_routes() -> Router<Arc<App>> {
use landing::*;
Router::new()
.route("/xrpc/dev.knotless.getMergeStatus", get(get_merge_status))
.route("/xrpc/dev.knotless.mergePull", post(merge_pull))
.route("/xrpc/dev.knotless.revertPull", post(revert_pull))
.route("/xrpc/dev.knotless.queuePull", post(queue_pull))
.route("/xrpc/dev.knotless.setAutoMerge", post(set_auto_merge))
.route("/xrpc/dev.knotless.leaveQueue", post(leave_queue))
.route("/xrpc/dev.knotless.retargetPull", post(retarget_pull))
.route("/xrpc/dev.knotless.requestRerun", post(request_rerun))
.route("/xrpc/dev.knotless.listWorkflows", get(list_workflows))
.route("/xrpc/dev.knotless.listRunners", get(list_runners))
.route("/xrpc/dev.knotless.addRunner", post(add_runner))
.route("/xrpc/dev.knotless.removeRunner", post(remove_runner))
}
pub fn routes() -> Router<Arc<App>> {
let compressed = Router::new()
.route("/{authority}/{repo}/raw/{*rest}", get(raw))
.route("/{authority}/{repo}/commits.atom", get(feeds::commits))
.route("/{authority}/{repo}/tags.atom", get(feeds::tags))
.route("/{authority}/{repo}/releases.atom", get(feeds::releases))
.route("/card.png", get(home_card))
.route("/{authority}/{repo}/card.png", get(repo_card))
.route(
"/{authority}/{repo}/{kind}/{did}/{rkey}/card.png",
get(item_card_png),
)
.layer(tower_http::compression::CompressionLayer::new());
// A release's files are sent as they are kept: most are archives,
// compressed already, and their length is known. A revision's archive
// is compressed as it is written.
Router::new()
.route(
"/{authority}/{repo}/releases/download/{*rest}",
get(releases::download),
)
.route("/cargo/{did}/{key}/config.json", get(cargo::config))
.route("/cargo/{did}/{key}/me", get(cargo::me))
.route("/cargo/{did}/{key}/api/v1/crates", get(cargo::search))
.route(
"/cargo/{did}/{key}/api/v1/crates/new",
axum::routing::put(cargo::publish),
)
.route(
"/cargo/{did}/{key}/api/v1/crates/{name}/{version}/yank",
axum::routing::delete(cargo::yank),
)
.route(
"/cargo/{did}/{key}/api/v1/crates/{name}/{version}/unyank",
axum::routing::put(cargo::unyank),
)
.route(
"/cargo/{did}/{key}/dl/{name}/{version}/download",
get(cargo::download),
)
.route("/cargo/{did}/{key}/{*path}", get(cargo::index))
.route("/{authority}/{repo}/archive/{*rest}", get(archive::archive))
.merge(compressed)
.layer(middleware::from_fn(renamed))
}
/// Where a repository is now, on the answer to a request that used a name
/// it had before: its name, and its owner if it moved to another account.
#[derive(Clone)]
struct Renamed(String);
/// Points a request made with a repository's old name at the same path
/// under its new one. The redirect is temporary: the old name can be given
/// to another repository.
async fn renamed(req: Request, next: Next) -> Response {
let path = req.uri().path().to_string();
let query = req
.uri()
.query()
.map(|q| format!("?{q}"))
.unwrap_or_default();
let mut res = next.run(req).await;
if let Some(Renamed(now)) = res.extensions_mut().remove::<Renamed>() {
// "", the authority, the name, and the rest.
let mut parts: Vec<&str> = path.splitn(4, '/').collect();
match now.split_once('/') {
Some((owner, name)) => (parts[1], parts[2]) = (owner, name),
None => parts[2] = &now,
}
let to = format!("{}{query}", parts.join("/"));
res.headers_mut()
.insert(header::LOCATION, HeaderValue::from_str(&to).unwrap());
}
res
}
/// Text inlined above, past this, is a link to the raw file instead.
const INLINE: usize = 1024 * 1024;
/// A file's diff with more rows than this is folded away.
const DIFF_ROWS: usize = 2000;
/// The first files of a pull drawn with their rows, as its files page
/// draws them at once: so many at most, of so many lines in all, and none
/// changing more lines than [`FIRST_HELD`], which waits to be asked for.
const FIRST_FILES: usize = 10;
const FIRST_LINES: usize = 1500;
const FIRST_HELD: usize = 400;
/// Which files of a diff come with their rows.
#[derive(Clone, Copy, PartialEq, Debug)]
pub(crate) enum Rows {
None,
All,
/// The first, as a pull's files page draws them at once.
First,
}
/// A path for a URL: each segment percent-encoded, the slashes kept.
fn url_path(path: &str) -> String {
let mut out = String::new();
for b in path.bytes() {
match b {
b'a'..=b'z'
| b'A'..=b'Z'
| b'0'..=b'9'
| b'-'
| b'.'
| b'_'
| b'~'
| b'/'
| b':'
| b'@' => out.push(b as char),
b => write!(out, "%{b:02X}").unwrap(),
}
}
out
}
/// Caching for a file at `rev`: forever when it names a commit by hash,
/// else checked again each time, as a branch can move.
fn cache_for(rev: &str) -> &'static str {
if read::is_pinned(rev) {
"public, max-age=31536000, immutable"
} else {
"no-cache"
}
}
/// `value` as the answer, checked again each time: even at a commit hash
/// it shows the repository as it is now, its authors' accounts, and a shape
/// a later release may change.
fn answer<T: Serialize>(value: T) -> Response {
let mut res = axum::Json(value).into_response();
res.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
res
}
/// A failure, as the pages show it: a status and why.
#[derive(Debug)]
pub struct Shown(StatusCode, String);
impl From<AppError> for Shown {
fn from(e: AppError) -> Shown {
Shown(e.0, e.1)
}
}
/// The account logged in to the pages that sent a write, as their own
/// writes take it, for a method that is not only the pages'.
pub(crate) fn logged_in(
app: &App,
viewer: crate::viewer::Viewer,
headers: &HeaderMap,
act: &str,
) -> std::result::Result<String, AppError> {
stars::logged_in(app, viewer, headers, act).map_err(|Shown(code, why)| AppError(code, why))
}
/// A failure, as XRPC has one: its kind, by name, and why.
#[derive(Serialize)]
pub struct Failure {
error: String,
message: String,
}
/// The name XRPC gives a failure with `status`.
fn failure_name(status: StatusCode) -> &'static str {
match status {
StatusCode::BAD_REQUEST => "InvalidRequest",
StatusCode::NOT_FOUND => "NotFound",
StatusCode::UNAUTHORIZED => "AuthenticationRequired",
StatusCode::FORBIDDEN => "Forbidden",
StatusCode::CONFLICT => "Conflict",
StatusCode::BAD_GATEWAY => "UpstreamFailure",
_ => "InternalServerError",
}
}
impl IntoResponse for Shown {
fn into_response(self) -> Response {
if self.0 == StatusCode::TEMPORARY_REDIRECT {
let mut res = self.0.into_response();
res.extensions_mut().insert(Renamed(self.1));
return res;
}
let Shown(status, message) = behind(self);
let failure = Failure {
error: failure_name(status).into(),
message,
};
let cause = crate::telemetry::Cause(failure.message.clone());
let mut res = (status, axum::Json(failure)).into_response();
res.extensions_mut().insert(cause);
res.headers_mut()
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
res
}
}
/// `shown`, or for a write refused for want of scope, why in a sentence
/// that says how to grant it: the gateway's, when it was the owner's grant
/// to it that fell short, else the login's to the pages.
fn behind(shown: Shown) -> Shown {
let said = &shown.1;
if said.contains(knotless::oauth::GRANT_BEHIND) {
return Shown(StatusCode::FORBIDDEN, knotless::oauth::GRANT_BEHIND.into());
}
match knotless::oauth::out_of_scope(said) {
true => Shown(StatusCode::FORBIDDEN, knotless::oauth::LOGIN_BEHIND.into()),
false => shown,
}
}
fn not_found(e: anyhow::Error) -> Shown {
Shown(StatusCode::NOT_FOUND, crate::shown(&e))
}
type Result<T> = std::result::Result<T, Shown>;
static HANDLES: LazyLock<Mutex<HashMap<String, String>>> = LazyLock::new(Mutex::default);
/// How long a handle the index keeps is shown before it is looked up again,
/// behind the page showing it. An identity event looks it up at once.
const HANDLE_KEPT_FOR: Duration = Duration::from_secs(24 * 60 * 60);
/// How long an account's having no handle is kept before it is looked up
/// again: a handle's DNS can go live just after the account claims it.
const NO_HANDLE_KEPT_FOR: Duration = Duration::from_secs(600);
/// Seconds since the epoch.
fn now_secs() -> i64 {
let since = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH);
since.unwrap_or_default().as_secs() as i64
}
/// The handle `did` claims, if it resolves back to `did`; else the DID.
/// An appview keeps them in its index, where they outlast a restart and
/// search finds them; without one, they are kept in memory.
pub(crate) async fn handle(app: &App, did: &str) -> String {
if let Some(index) = app.index.clone() {
let kept = looked("handles", index.handle(did).ok().flatten());
let Some((handle, checked)) = kept else {
let found = rename(app.ctx.clone(), index, did.to_string()).await;
return found.unwrap_or(did.to_string());
};
let kept_for = match handle {
Some(_) => HANDLE_KEPT_FOR,
None => NO_HANDLE_KEPT_FOR,
};
if now_secs() - checked >= kept_for.as_secs() as i64 {
// Marked as looked up now, so the pages showing it meanwhile
// look it up once between them.
let _ = index.set_handle(did, handle.as_deref(), now_secs());
tokio::spawn(rename(app.ctx.clone(), index, did.to_string()));
}
return handle.unwrap_or(did.to_string());
}
if let Some(known) = looked("handles", HANDLES.lock().unwrap().get(did)) {
return known.clone();
}
let shown = knotless::identity::handle(&app.ctx, did)
.await
.unwrap_or(did.to_string());
HANDLES
.lock()
.unwrap()
.insert(did.to_string(), shown.clone());
shown
}
/// Looks `did`'s handle up now and keeps what was found in `index`: none,
/// when the handle it claims does not resolve back to it. A DID document
/// that cannot be read says nothing of its handle, and changes nothing.
pub(crate) async fn rename(
ctx: knotless::env::Ctx,
index: Arc<crate::index::Index>,
did: String,
) -> Option<String> {
let found = knotless::identity::verified_handle(&ctx, &did).await;
let found = found.ok()?;
let _ = index.set_handle(&did, found.as_deref(), now_secs());
found
}
/// How long what an account says of itself is kept before it is looked up
/// again: its picture, and the addresses it commits as.
const KEPT_FOR: Duration = Duration::from_secs(600);
/// How long a page waits for something about an account never looked up
/// before, before it is shown without it; the lookup goes on, for the next.
const LOOKUP_WAIT: Duration = Duration::from_secs(5);
/// A lookup going on, shared by every page waiting for it.
type Looking<V> = futures::future::Shared<futures::future::BoxFuture<'static, Option<V>>>;
/// Values looked up over the network, by key. One older than its `ttl` is
/// still shown, and looked up again behind it, so a page waits only for a
/// value never found, and for `wait` at most; one lookup at a time a key,
/// however many pages ask. A lookup that fails keeps what was found
/// before. They are kept in the cache directory too, so the server a
/// deploy starts has them at once, rather than asking every account's PDS
/// again just as relays may be fetching whole repositories from it.
struct Kept<V> {
/// The file they are kept in, in the cache directory.
name: &'static str,
ttl: Duration,
wait: Duration,
values: Mutex<HashMap<String, (SystemTime, V)>>,
/// The file, once read.
file: std::sync::OnceLock<std::path::PathBuf>,
looking: Mutex<HashMap<String, Looking<V>>>,
}
impl<V> Kept<V>
where
V: Clone + Serialize + serde::de::DeserializeOwned + Send + Sync + 'static,
{
fn new(name: &'static str, ttl: Duration, wait: Duration) -> Kept<V> {
Kept {
name,
ttl,
wait,
values: Mutex::default(),
file: std::sync::OnceLock::new(),
looking: Mutex::default(),
}
}
/// What is kept in the cache directory `dir`, read the first time.
fn load(&self, dir: &std::path::Path) {
self.file.get_or_init(|| {
let file = dir.join(self.name);
let read: HashMap<String, (SystemTime, V)> = store::read(&file).unwrap_or_default();
let mut values = self.values.lock().unwrap();
for (key, kept) in read {
values.entry(key).or_insert(kept);
}
file
});
}
/// The value kept for `key`, if any, and whether it is still fresh.
fn get(&self, key: &str) -> Option<(V, bool)> {
let values = self.values.lock().unwrap();
let (at, value) = values.get(key)?;
let age = at.elapsed().unwrap_or_default();
Some((value.clone(), age < self.ttl))
}
fn put(&self, key: &str, value: V) {
let mut values = self.values.lock().unwrap();
values.insert(key.to_string(), (SystemTime::now(), value));
let Some(file) = self.file.get() else {
return;
};
// A cache: one not written is looked up again.
let _ = store::write(file, &*values, false);
}
/// The value for `key`, as `look` finds it if none fresh is kept: none
/// if it never found one, or not within `wait`.
async fn through<F>(&'static self, dir: &std::path::Path, key: &str, look: F) -> Option<V>
where
F: Future<Output = Option<V>> + Send + 'static,
{
self.load(dir);
let kept = self.get(key);
let cache = self.name.trim_end_matches(".json");
let _ = looked(cache, kept.as_ref().filter(|k| k.1));
if let Some((value, true)) = kept {
return Some(value);
}
let looking = self
.looking
.lock()
.unwrap()
.entry(key.to_string())
.or_insert_with(|| {
let key = key.to_string();
let found = tokio::spawn(async move {
let found = look.await;
fill(
cache,
&key,
&found.as_ref().ok_or("not read: its PDS did not answer"),
);
if let Some(value) = &found {
self.put(&key, value.clone());
}
self.looking.lock().unwrap().remove(&key);
found
});
found.map(|ended| ended.ok().flatten()).boxed().shared()
})
.clone();
match kept {
Some((value, _)) => Some(value),
None => tokio::time::timeout(self.wait, looking)
.await
.ok()
.flatten(),
}
}
}
static AVATARS: LazyLock<Kept<Option<String>>> =
LazyLock::new(|| Kept::new("avatars.json", KEPT_FOR, LOOKUP_WAIT));
static CLAIMS: LazyLock<Kept<Vec<String>>> =
LazyLock::new(|| Kept::new("claims.json", KEPT_FOR, LOOKUP_WAIT));
/// The accounts `author:` filters named by handle, by handle: none for a
/// handle that names none.
static AUTHORS: LazyLock<Kept<Option<String>>> =
LazyLock::new(|| Kept::new("authors.json", KEPT_FOR, LOOKUP_WAIT));
/// Where Bluesky's CDN serves profile pictures, by DID and blob CID.
const AVATAR_CDN: &str = "https://cdn.bsky.app/img/avatar/plain";
/// An account's profile records, by collection and record key.
type Records = Vec<(String, String, serde_json::Value)>;
/// `remote`'s knotless profile, from [`profile_records`]: as the index
/// keeps it while a feed keeps it current, else as its PDS has it. An error
/// if the PDS did not answer.
async fn own_profile(app: &App, remote: &Remote) -> Result<Option<ProfileRecord>> {
let watching = app.watching.load(Ordering::Relaxed);
let records = profile_records(app.index.as_deref(), watching, remote).await;
let unanswered = format!("{}: its PDS did not answer", remote.did());
let records = records.ok_or(AppError(StatusCode::BAD_GATEWAY, unanswered))?;
let own = records
.into_iter()
.find(|(c, k, _)| c == nsid::PROFILE && k == profile::RKEY);
Ok(own.and_then(|(_, _, value)| serde_json::from_value(value).ok()))
}
/// The records `remote`'s profile is made of: as `index` keeps them, while
/// the firehose keeps them current (`watching`), else as its PDS has them,
/// and then kept. Nothing if the PDS did not answer for both profiles.
async fn profile_records(
index: Option<&Index>,
watching: bool,
remote: &Remote,
) -> Option<Records> {
let did = remote.did();
let kept = index
.filter(|_| watching)
.and_then(|i| i.profile(did).ok().flatten());
if kept.is_some() {
return kept;
}
// The Bluesky links are listed with the profiles, all at once, though
// they are shown only when knotless's own profile names none: asked
// after the Bluesky profile, which names them, they would be a second
// round of asking the PDS.
let links = remote.list_all::<serde_json::Value>(nsid::BSKY_LINK);
let bsky = remote.get_record::<serde_json::Value>(nsid::BSKY_PROFILE, profile::RKEY);
let own = remote.get_record::<serde_json::Value>(nsid::PROFILE, profile::RKEY);
let (bsky, own, links) = futures::join!(bsky, own, links);
let (bsky, own) = (bsky.ok()?, own.ok()?);
let whole = links.is_ok();
let mut records: Records = Vec::new();
records.extend(bsky.map(|r| (nsid::BSKY_PROFILE.to_string(), r.rkey, r.value)));
records.extend(own.map(|r| (nsid::PROFILE.to_string(), r.rkey, r.value)));
let links = links.unwrap_or_default().into_iter();
records.extend(links.map(|r| (nsid::BSKY_LINK.to_string(), r.rkey, r.value)));
// Kept only whole: links that could not be listed are asked for again.
if let Some(index) = index.filter(|_| whole) {
let _ = index.keep_profile(did, &records);
}
Some(records)
}
/// A link on an account's profile: always an http or https address.
#[derive(Serialize, Clone, Debug, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ProfileLink {
url: String,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<String>,
}
/// The links a repository's record names that its pages can show: each an
/// http or https address, and no more than a repository shows.
fn repo_links(named: Vec<knotless::records::ProfileLink>) -> Vec<ProfileLink> {
let web = named.into_iter().filter(|l| profile::is_web(&l.url));
let shown = web.take(knotless::manage::MAX_LINKS).map(|l| ProfileLink {
url: l.url,
title: l.title,
});
shown.collect()
}
/// Who an account says it is: its knotless profile, field by field over
/// its Bluesky one.
#[derive(Default)]
struct About {
name: Option<String>,
bio: Option<String>,
pronouns: Option<String>,
links: Vec<ProfileLink>,
avatar: Option<String>,
/// Whether its knotless profile says it is a bot.
bot: bool,
/// Whether both profiles were read: an account whose PDS did not
/// answer shows as having none, but is not kept so.
answered: bool,
}
/// The links `did`'s Bluesky profile `bsky` shows, in order: the
/// `app.bsky.actor.link` records it names in its own repo, of those
/// `listed`, by record key.
fn bsky_links(
did: &str,
bsky: &serde_json::Value,
mut listed: HashMap<String, serde_json::Value>,
) -> Vec<ProfileLink> {
let mine = format!("at://{did}/{}/", nsid::BSKY_LINK);
let named = bsky["links"].as_array().into_iter().flatten();
let rkeys = named.filter_map(|l| l["uri"].as_str()?.strip_prefix(&mine));
let found = rkeys.filter_map(|rkey| listed.remove(rkey));
let links = found.take(profile::MAX_LINKS).map(|r| ProfileLink {
url: r["url"].as_str().unwrap_or_default().to_string(),
title: r["title"].as_str().map(str::to_string),
});
links.collect()
}
/// Who `did` says it is, from its profile `records`, if they were read.
fn about(did: &str, records: Option<Records>) -> About {
let answered = records.is_some();
let records = records.unwrap_or_default();
let one = |collection: &str| {
let found = records
.iter()
.find(|(c, k, _)| c == collection && k == profile::RKEY);
found.map(|(_, _, value)| value.clone())
};
let bsky = one(nsid::BSKY_PROFILE).unwrap_or_default();
let own = one(nsid::PROFILE).and_then(|v| serde_json::from_value::<ProfileRecord>(v).ok());
let own = own.unwrap_or_default();
let listed = records.into_iter().filter(|(c, _, _)| c == nsid::BSKY_LINK);
let listed = listed.map(|(_, rkey, value)| (rkey, value)).collect();
let text = |field: &str| {
let said = bsky[field].as_str().map(str::to_string);
said.filter(|t| !t.trim().is_empty())
};
let mut links: Vec<ProfileLink> = own
.links
.into_iter()
.map(|l| ProfileLink {
url: l.url,
title: l.title,
})
.collect();
if links.is_empty() {
links = bsky_links(did, &bsky, listed);
}
if links.is_empty() {
links.extend(text("website").map(|url| ProfileLink { url, title: None }));
}
links.retain(|l| profile::is_web(&l.url));
let avatar = match own.avatar {
Some(own) => Some(own_avatar_path(did, &own.link.cid)),
None => bsky["avatar"]["ref"]["$link"]
.as_str()
.map(|cid| format!("{AVATAR_CDN}/{did}/{cid}@jpeg")),
};
About {
name: own.display_name.or_else(|| text("displayName")),
bio: own.description.or_else(|| text("description")),
pronouns: own.pronouns.or_else(|| text("pronouns")),
links,
avatar,
bot: own.bot,
answered,
}
}
#[derive(Deserialize)]
pub struct ProfileParams {
actor: String,
}
/// `dev.knotless.getProfile#profileView`.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ProfileView {
did: String,
#[serde(skip_serializing_if = "Option::is_none")]
handle: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
display_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pronouns: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
links: Vec<ProfileLink>,
#[serde(skip_serializing_if = "Option::is_none")]
avatar: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
followers_count: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
follows_count: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
viewer: Option<ActorViewerState>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
organization: bool,
#[serde(skip_serializing_if = "std::ops::Not::not")]
bot: bool,
#[serde(skip_serializing_if = "Vec::is_empty")]
orgs: Vec<orgs::OrgView>,
#[serde(skip_serializing_if = "Vec::is_empty")]
badges: Vec<badges::BadgeView>,
}
/// `dev.knotless.defs#actorViewerState`.
#[derive(Serialize)]
pub struct ActorViewerState {
#[serde(skip_serializing_if = "Option::is_none")]
following: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
blocking: Option<String>,
}
/// `dev.knotless.getProfile`'s failure for an account not found, as `e`
/// says.
fn no_profile(e: anyhow::Error) -> Response {
let failure = Failure {
error: "ProfileNotFound".into(),
message: format!("{e:#}"),
};
(StatusCode::BAD_REQUEST, axum::Json(failure)).into_response()
}
tokio::task_local! {
/// Where the request being answered reached this server.
static ORIGIN: String;
}
/// Answers the request with its origin known, for the addresses answers
/// give in full: as the one asking reached this server, through the pages,
/// which ask for the site they serve, or directly.
pub async fn origin(req: Request, next: Next) -> Response {
let origin = origin_of(req.headers());
ORIGIN.scope(origin, next.run(req)).await
}
/// A failure an XRPC method answered as text, as XRPC has one instead:
/// its kind, by name, and why. Its extensions go with it, the cause its
/// request's log line gives among them.
pub async fn xrpc_failures(req: Request, next: Next) -> Response {
let xrpc = req.uri().path().starts_with("/xrpc/");
let res = next.run(req).await;
let text = res.headers().get(header::CONTENT_TYPE);
let text = text.is_some_and(|t| t.as_bytes().starts_with(b"text/plain"));
if !xrpc || !text || res.status().is_success() {
return res;
}
let (parts, body) = res.into_parts();
let said = axum::body::to_bytes(body, 64 * 1024)
.await
.unwrap_or_default();
let failure = Failure {
error: failure_name(parts.status).into(),
message: String::from_utf8_lossy(&said).into_owned(),
};
let mut answer = (parts.status, axum::Json(failure)).into_response();
*answer.extensions_mut() = parts.extensions;
answer
}
fn origin_of(headers: &HeaderMap) -> String {
let header = |name: &str| headers.get(name).and_then(|v| v.to_str().ok());
let proto = header("x-forwarded-proto").unwrap_or("http");
let host = header("x-forwarded-host").or(header(header::HOST.as_str()));
format!("{proto}://{}", host.unwrap_or_default())
}
/// `path`, an address on this server, in full, at the request's origin.
fn absolute(path: String) -> String {
match ORIGIN.try_with(String::clone) {
Ok(origin) if path.starts_with('/') => format!("{origin}{path}"),
_ => path,
}
}
/// `dev.knotless.getProfile`: how an account shows on knotless, with its
/// picture's address in full, and its follows as the index counts them.
pub async fn get_profile(
State(app): State<Arc<App>>,
axum::Extension(viewer): axum::Extension<crate::viewer::Viewer>,
Query(p): Query<ProfileParams>,
) -> Response {
let at = RepoUrl {
authority: p.actor,
name: "_".into(),
};
let remote = match Remote::connect(&app.ctx, &at).await {
Ok(remote) => remote,
Err(e) => return no_profile(e),
};
let watching = app.watching.load(Ordering::Relaxed);
let records = profile_records(app.index.as_deref(), watching, &remote).await;
let about = about(remote.did(), records);
if about.answered {
AVATARS.load(&app.ctx.env.cache_dir);
AVATARS.put(remote.did(), about.avatar.clone());
}
let avatar = about.avatar.map(absolute);
let index = app.index.as_ref();
let counts = index.and_then(|i| i.follow_counts(remote.did()).ok());
let following = |v: &String| index.and_then(|i| i.follow_of(v, remote.did()).ok());
let blocking = |v: &String| index.and_then(|i| i.block_of(v, remote.did()).ok());
let state = |v: &String| ActorViewerState {
following: following(v).flatten(),
blocking: blocking(v).flatten(),
};
let view = ProfileView {
did: remote.did().to_string(),
handle: Some(handle(&app, remote.did()).await),
display_name: about.name,
description: about.bio,
pronouns: about.pronouns,
links: about.links,
avatar,
followers_count: counts.map(|c| c.0),
follows_count: counts.map(|c| c.1),
viewer: viewer.0.as_ref().map(state),
organization: orgs::is_org(&app, remote.did()),
bot: about.bot,
orgs: orgs::orgs_of(&app, remote.did(), viewer.0.as_deref()).await,
badges: match index {
Some(i) => {
let home = badges::home(&app, i.clone(), &remote).await;
// Counted on a thread of its own: an account of many pulls
// takes a while, which would hold up every request this
// worker answers meanwhile.
let (i, did) = (i.clone(), remote.did().to_string());
let count = move || badges::badges_of(&i, &did, home);
tokio::task::spawn_blocking(count).await.unwrap_or_default()
}
None => Vec::new(),
},
};
axum::Json(view).into_response()
}
/// `did`'s profile picture, if it has one: looked up now and then, and
/// again whenever its profile page is shown.
async fn avatar(app: &App, did: &str) -> Option<String> {
let (ctx, at) = (app.ctx.clone(), did.to_string());
let (index, watching) = (app.index.clone(), app.watching.load(Ordering::Relaxed));
let look = async move {
let at = RepoUrl {
authority: at,
name: "_".into(),
};
let remote = Remote::connect(&ctx, &at).await.ok()?;
// A PDS that did not answer, as one busy after a deploy may not,
// leaves the picture found before, rather than none.
let records = profile_records(index.as_deref(), watching, &remote).await;
let about = about(remote.did(), records);
about.answered.then_some(about.avatar)
};
let dir = &app.ctx.env.cache_dir;
AVATARS.through(dir, did, look).await.flatten()
}
/// How long a card waits for the picture it shows before it is drawn
/// without it.
const PICTURE_WAIT: Duration = Duration::from_secs(2);
/// The most bytes of a picture a card takes.
const PICTURE_MAX: usize = 1 << 20;
/// The pictures cards show, by address, the newest so many. An address
/// names one picture for good: it carries the picture's CID.
static PICTURES: LazyLock<card::Newest<Arc<Vec<u8>>>> = LazyLock::new(|| card::Newest::new(128));
/// The picture at `url`, an address `about` gives, for a card: from the
/// account's PDS for one its knotless profile names, else Bluesky's
/// CDN's small form of it. Nothing if it is not read in `PICTURE_WAIT`,
/// or is larger than `PICTURE_MAX`; and a picture not read is asked for
/// again next time.
async fn card_picture(ctx: &knotless::env::Ctx, url: &str) -> Option<Arc<Vec<u8>>> {
if let Some(known) = looked("pictures", PICTURES.get(url)) {
return Some(known);
}
let read = tokio::time::timeout(PICTURE_WAIT, read_picture(ctx, url)).await;
let read = read.unwrap_or(Err(anyhow::anyhow!("not read in {PICTURE_WAIT:?}")));
fill("pictures", url, &read);
let bytes = read.ok().filter(|b| b.len() <= PICTURE_MAX)?;
let bytes = Arc::new(bytes);
PICTURES.put(url, bytes.clone());
Some(bytes)
}
async fn read_picture(ctx: &knotless::env::Ctx, url: &str) -> anyhow::Result<Vec<u8>> {
if url.starts_with(OWN_AVATAR) {
let asked = reqwest::Url::parse(&format!("http://x{url}"))?;
let param = |name: &str| asked.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v);
let (did, cid) = (
param("actor").unwrap_or_default(),
param("cid").unwrap_or_default(),
);
let at = RepoUrl {
authority: did.to_string(),
name: "_".into(),
};
let remote = Remote::connect(ctx, &at).await?;
return remote.get_blob(&cid).await;
}
let small = url.replacen("/img/avatar/", "/img/avatar_thumbnail/", 1);
let sent = knotless::telemetry::send(ctx.http.get(small), 0).await?;
let mut res = sent.error_for_status()?;
let mut bytes = Vec::new();
while let Some(chunk) = res.chunk().await? {
bytes.extend_from_slice(&chunk);
anyhow::ensure!(
bytes.len() <= PICTURE_MAX,
"a picture larger than a card takes"
);
}
Ok(bytes)
}
/// The addresses `did` claims to commit as, hashed (`knotless::emails`).
async fn claims(app: &App, did: &str) -> Vec<String> {
let (ctx, of) = (app.ctx.clone(), did.to_string());
// A PDS that did not answer leaves the claims read before, rather than
// its account's commits shown as no one's.
let look = async move { knotless::emails::claimed(&ctx, &of).await.ok() };
let dir = &app.ctx.env.cache_dir;
CLAIMS.through(dir, did, look).await.unwrap_or_default()
}
/// The path of a page's card, under the version of what it shows.
fn card_path(page: &str, card: &Card) -> String {
format!("{page}/card.png?v={}", card::version(&card.svg()))
}
/// A repository's DID, and its name now if the index knows it.
fn now_named<'a>(app: &App, repo: &'a RepoRef) -> Option<(&'a str, String)> {
let (did, _, named_then) = forge::parse_uri(&repo.uri).ok()?;
let now = app
.index
.as_ref()
.and_then(|i| i.name(did, &repo.key).ok().flatten());
Some((did, now.as_deref().unwrap_or(named_then).to_string()))
}
/// One whose claims a repository's pages believe: its owner and its
/// collaborators everywhere in it, and the author of a pull merged into it
/// only in the commits the pull brought in, which the merge vouched for.
struct Vouched {
did: String,
/// The addresses it claims, hashed (`knotless::emails`).
hashes: Vec<String>,
/// The commits it is believed in, if not all.
only: Option<HashSet<String>>,
}
/// Which of `people` `email` is in `sha`, if any.
fn vouched_as<'a>(people: &'a [Vouched], sha: &str, email: &str) -> Option<&'a str> {
let by = people.iter().find(|p| {
let here = p.only.as_ref().is_none_or(|only| only.contains(sha));
here && knotless::emails::is(&p.did, email, &p.hashes)
});
by.map(|p| p.did.as_str())
}
/// `name` at `email`, by account if `email` is one of `people`'s in `sha`,
/// or else the one account other repositories with `sha` vouch for as it
/// there ([`walk_authors`]): none when they disagree.
async fn author(
app: &App,
(repo, people): (&Repo, &[Vouched]),
sha: &str,
name: String,
email: &str,
) -> CommitAuthorView {
let by = match vouched_as(people, sha, email) {
Some(did) => Some(did.to_string()),
None => elsewhere(app, repo, sha, email),
};
let actor = match by {
Some(did) => Some(actor(app, &did).await),
None => None,
};
let agent = email
.eq_ignore_ascii_case("noreply@anthropic.com")
.then_some("claude");
CommitAuthorView { name, actor, agent }
}
/// The co-authors a commit message's closing paragraph names, as
/// `Co-authored-by: Name <email>` trailers: names and addresses.
fn coauthors(message: &str) -> Vec<(String, String)> {
let trailers = message
.trim_end()
.rsplit_once("\n\n")
.map_or("", |(_, t)| t);
trailers.lines().filter_map(coauthor).collect()
}
fn coauthor(line: &str) -> Option<(String, String)> {
let (key, value) = line.split_once(':')?;
let named = key.trim().eq_ignore_ascii_case("co-authored-by");
let (name, email) = value.split_once('<').filter(|_| named)?;
let email = email.trim_end().strip_suffix('>')?;
Some((name.trim().to_string(), email.trim().to_string()))
}
/// What a merge as `merged` brought in, as the ends of a range of
/// history: from where the base was before it, or, for a merge recorded
/// without that, a merge commit's second parent less its first, which
/// names nothing for any other kind of merge.
fn brought(merged: &str, from: Option<&str>) -> (String, String) {
match from {
Some(from) => (from.to_string(), merged.to_string()),
None => (format!("{merged}^1"), format!("{merged}^2")),
}
}
type Range = (String, String);
static BROUGHT: LazyLock<Mutex<HashMap<Range, HashSet<String>>>> = LazyLock::new(Mutex::default);
/// The commits each of `ranges` names in the mirror at `dir`, kept once
/// found: a merge never changes what it brought in. Read in this process,
/// all with the one repository opened, as a repository's merged pulls are
/// many and a git process for each, all at once, took seconds.
fn brought_in(dir: &std::path::Path, ranges: Vec<Option<Range>>) -> Vec<Option<HashSet<String>>> {
let objects = crate::objects::open(dir).ok();
let find = |(from, to): Range| {
if let Some(known) = BROUGHT.lock().unwrap().get(&(from.clone(), to.clone())) {
return known.clone();
}
// Not found, as a commit the mirror has yet to fetch, it is looked
// for again next time.
let found = objects.as_ref().map(|o| o.between(&from, &to));
let found = found.and_then(|f| f.ok());
if let Some(shas) = &found {
BROUGHT.lock().unwrap().insert((from, to), shas.clone());
}
found.unwrap_or_default()
};
ranges.into_iter().map(|r| r.map(find)).collect()
}
/// Whose claims `repo`'s pages believe, and where: its owner, its
/// collaborators and the members of the organization its owner is who may
/// push to it first, so no pull's author can take their addresses. And
/// the merged pull that brought each commit in, from what its merge record
/// says it brought: a fast-forward's commits, a merge's and a squash's.
async fn vouched(
app: &App,
repo: &Repo,
record: RepoRecord,
items: &[Item],
) -> (Vec<Vouched>, HashMap<String, ItemViewBasic>) {
let merged: Vec<(&Item, Range)> = items
.iter()
.filter_map(|i| Some((i, brought(i.merged.as_deref()?, i.merged_from.as_deref()))))
.collect();
let writers: Vec<String> = record.writers().map(String::from).collect();
let everywhere = std::iter::once(repo.did.clone()).chain(writers);
let people: Vec<(String, Option<Range>)> = everywhere
.map(|did| (did, None))
.chain(
merged
.iter()
.map(|(i, r)| (i.author.clone(), Some(r.clone()))),
)
.collect();
// Each account's claims are read once, however many of its pulls
// merged: one page asks its PDS once, not once a pull.
let dids: HashSet<&String> = people.iter().map(|(did, _)| did).collect();
let read = dids.into_iter().map(|did| async move {
let hashes = claims(app, did).await;
(did.clone(), hashes)
});
let claimed: HashMap<String, Vec<String>> =
futures::future::join_all(read).await.into_iter().collect();
let (dids, ranges): (Vec<String>, Vec<Option<Range>>) = people.into_iter().unzip();
let dir = repo.git.dir.clone();
let found = tokio::task::spawn_blocking(move || brought_in(&dir, ranges));
let found = found.await.expect("a walk does not panic");
// A commit two merges brought in, as a stack pushed at once brings
// each pull its own and those under it, is the one bringing least's.
let ranges = found[found.len() - merged.len()..].iter().flatten();
let mut brought: Vec<_> = merged.iter().zip(ranges).collect();
brought.sort_by_key(|(_, shas)| shas.len());
let mut pulls = HashMap::new();
for ((item, _), shas) in brought {
let view = ItemViewBasic {
uri: item.uri.clone(),
title: item.title.clone(),
};
for sha in shas {
pulls.entry(sha.clone()).or_insert_with(|| view.clone());
}
}
let people = dids
.into_iter()
.zip(found)
.map(|(did, only)| {
let hashes = claimed.get(&did).cloned().unwrap_or_default();
Vouched { did, hashes, only }
})
.collect();
(people, pulls)
}
/// The one account other repositories with `sha` vouch for as `email`
/// in it, if they agree on one.
fn elsewhere(app: &App, repo: &Repo, sha: &str, email: &str) -> Option<String> {
let index = app.index.as_ref()?;
let address = crate::index::address(email);
let found = index.authors_elsewhere(sha, &address, (&repo.did, &repo.key));
let mut found = found.ok()?;
found.pop().filter(|_| found.is_empty())
}
/// The repositories whose authors are being walked now.
static WALKING: LazyLock<Mutex<HashSet<(String, String)>>> = LazyLock::new(Mutex::default);
/// Keeps in the index the account `people` vouch for as each address of
/// each commit on `repo`'s default branch, so that a page of another
/// repository with those commits, as a copy of it, shows them as theirs.
/// Each walk goes on from the tip the last stopped at, and starts over
/// when who `repo` believes changes, as one it no longer believes vouches
/// for nothing. A private repository vouches for nothing elsewhere.
fn walk_authors(app: &App, repo: &Repo, people: Arc<Vec<Vouched>>) {
let index = app.index.clone().filter(|_| repo.in_space.is_none());
let id = (repo.did.clone(), repo.key.clone());
// One walk of a repository at a time.
let index = index.filter(|_| WALKING.lock().unwrap().insert(id.clone()));
if let Some(index) = index {
let (dir, head) = (repo.git.dir.clone(), repo.head.clone());
tokio::task::spawn_blocking(move || {
// One with no tip yet, as an empty repository, has none to walk.
let _ = vouch_walked(&index, &id, &dir, &head, &people);
WALKING.lock().unwrap().remove(&id);
});
}
}
/// [`walk_authors`]'s walk.
fn vouch_walked(
index: &Index,
(did, key): &(String, String),
dir: &std::path::Path,
head: &str,
people: &[Vouched],
) -> anyhow::Result<()> {
let believed = believed(people);
let objects = objects::open(dir)?;
let tip = objects.resolve(head)?;
let walked = index.authors_walked(did, key)?;
let from = match &walked {
Some((at, was)) if *at == tip && *was == believed => return Ok(()),
Some((at, was)) if *was == believed => Some(at.as_str()),
_ => None,
};
let known: Vec<&str> = from.into_iter().collect();
let mut rows = Vec::new();
for c in objects.brought(&tip, &known, usize::MAX)? {
let helped = coauthors(&c.message).into_iter().map(|(_, email)| email);
let emails = [c.author.email, c.committer.email]
.into_iter()
.chain(helped);
for email in emails {
if let Some(by) = vouched_as(people, &c.sha, &email) {
rows.push((c.sha.clone(), crate::index::address(&email), by.to_string()));
}
}
}
index.vouch_authors((did, key), (&tip, &believed), &rows, from.is_none())?;
Ok(())
}
/// Who `people` are, as a walk of the authors they vouch for keeps it:
/// each account and the addresses it claims, but not the pulls merged,
/// as one merged since brought in only commits a walk on from its tip
/// finds.
fn believed(people: &[Vouched]) -> String {
let mut each: Vec<String> = people
.iter()
.map(|p| {
let mut hashes = p.hashes.clone();
hashes.sort();
let everywhere = if p.only.is_none() { "*" } else { "" };
format!("{}{everywhere}:{}", p.did, hashes.join(","))
})
.collect();
each.sort();
each.dedup();
crate::index::address(&each.join("\n"))
}
/// `commits`, as the pages show them, each with the accounts that wrote it
/// where `repo`'s pages believe their claims (`vouched`).
async fn shown(app: &App, repo: &Repo, commits: Vec<read::Commit>) -> Vec<CommitView> {
let record = repo.record(app).unwrap_or_default();
let made = assembled(app, repo).await.unwrap_or_default();
let items = &made.items;
let (people, pulls) = vouched(app, repo, record, items).await;
let people = Arc::new(people);
walk_authors(app, repo, people.clone());
let people = (repo, people.as_slice());
let mut out = Vec::new();
for c in commits {
let (pull, message) = match from_pull(items, &c.message) {
Some((pull, message)) => (Some(pull), message),
None => (pulls.get(&c.sha).cloned(), c.message),
};
let mut helped = Vec::new();
for (name, email) in coauthors(&message) {
helped.push(author(app, people, &c.sha, name, &email).await);
}
let (a, m) = (&c.author, &c.committer);
let by = author(app, people, &c.sha, a.name.clone(), &a.email).await;
let made = author(app, people, &c.sha, m.name.clone(), &m.email).await;
let same = match (&by.actor, &made.actor) {
(Some(x), Some(y)) => x.did == y.did,
_ => (&a.name, &a.email) == (&m.name, &m.email),
};
out.push(CommitView {
sha: c.sha,
parents: c
.parents
.into_iter()
.map(|sha| CommitRefView { sha })
.collect(),
author: by,
coauthors: helped,
committer: (!same).then_some(made),
date: c.author.date,
committed_at: c.committer.date,
message,
pull,
});
}
out
}
/// The pull `message`'s last line names, when it is one of `items`, and
/// the message without that line.
fn from_pull(items: &[Item], message: &str) -> Option<(ItemViewBasic, String)> {
let message = message.trim_end();
let (rest, last) = message.rsplit_once('\n').unwrap_or(("", message));
let pull = items
.iter()
.find(|i| i.kind == "pull" && i.uri == last.trim())?;
let view = ItemViewBasic {
uri: pull.uri.clone(),
title: pull.title.clone(),
};
Some((view, rest.trim_end().to_string()))
}
/// The most of those who may push to a repository its pages name.
const COLLABORATORS_SHOWN: usize = 20;
/// A repository being shown.
struct Repo {
authority: String,
name: String,
did: String,
key: String,
git: Git,
/// The default branch.
head: String,
/// Its record as the PDS has it, for a private repository the index
/// never sees.
space: Option<RepoRecord>,
/// The space it is in, if it is private.
in_space: Option<String>,
/// Its record, as the PDS had it when its mirror last synced, which a
/// write from the pages has it sync again for.
synced: RepoRecord,
/// What reads its records: for a private repository, the viewer it
/// was found for, never this server ([`crate::private`]).
reader: knotless::env::Ctx,
}
impl Repo {
async fn open(app: &App, authority: &str, name: &str) -> Result<Repo> {
let name = repo_name(name);
let found = app.located(authority, name).await?;
if found.moved {
let now = format!("{}/{}", found.did, found.name);
return Err(Shown(StatusCode::TEMPORARY_REDIRECT, now));
}
if found.name != name {
return Err(Shown(StatusCode::TEMPORARY_REDIRECT, found.name));
}
Repo::of(authority.to_string(), found)
}
/// The repository `authority/name` names, under the name it has now,
/// and its owner's handle if it moved.
async fn find(app: &App, authority: &str, name: &str) -> Result<Repo> {
let found = app.located(authority, repo_name(name)).await?;
Repo::moved(app, authority, found).await
}
/// [`Repo::find`] for a view of its code that answers a request, which
/// may be of a mirror a change has made stale while it syncs
/// ([`App::located_for_reader`]).
async fn find_for_reader(app: &App, authority: &str, name: &str) -> Result<Repo> {
let found = app.located_for_reader(authority, repo_name(name)).await?;
Repo::moved(app, authority, found).await
}
async fn moved(app: &App, authority: &str, found: crate::Located) -> Result<Repo> {
let authority = match found.moved {
true => handle(app, &found.did).await,
false => authority.to_string(),
};
Repo::of(authority, found)
}
fn of(authority: String, found: crate::Located) -> Result<Repo> {
let git = found.git;
let head = objects::open(&git.dir)
.map_err(AppError::internal)?
.head()
.unwrap_or_default();
Ok(Repo {
authority,
name: found.name,
did: found.did,
key: found.key,
git,
head,
space: found.space.is_some().then_some(found.record.clone()),
in_space: found.space,
synced: found.record,
reader: found.reader,
})
}
/// Its objects, read in process. Not held across an await: a reader
/// belongs to one thread.
fn objects(&self) -> Result<Objects> {
Ok(objects::open(&self.git.dir).map_err(AppError::internal)?)
}
fn href(&self, tail: &str) -> String {
format!(
"/{}/{}{tail}",
url_path(&self.authority),
url_path(&self.name)
)
}
/// Where its files at `rev` are, as `view` (`blob` or `raw`) shows
/// them: what links in Markdown found there resolve under.
fn files(&self, view: &str, rev: &str) -> String {
self.href(&format!("/{view}/{}", url_path(rev)))
}
/// Its record's URI: in full, in its space, for a private one.
fn uri(&self) -> String {
self.named(&forge::repo_uri(&self.did, &self.name))
}
/// `uri`, a record of this repository's in either form, as its records
/// are named here ([`forge::Target::named`]).
fn named(&self, uri: &str) -> String {
let space = self.in_space.as_deref();
let space = space.and_then(|s| knotless::aturi::SpaceRef::parse(s).ok());
knotless::aturi::resolve(uri, space.as_ref())
}
/// The repository as records name it: a private one's record by its
/// URI in its space.
fn reference(&self) -> RepoRef {
RepoRef {
uri: self.uri(),
key: self.key.clone(),
}
}
/// The record, as the index has it: for a private one, this server's
/// index of its space, else its mirror's.
fn record(&self, app: &App) -> Option<RepoRecord> {
let kept = indexed(app, self).and_then(|index| index.repo(&self.uri()).ok().flatten());
let kept = kept.and_then(|value| serde_json::from_value(value).ok());
let record = kept.or(self.space.clone());
// With who may push to it as a member of its owner's organization.
record.map(|r| app.with_members(&self.did, self.in_space.as_deref(), r))
}
/// Its open pulls and issues.
async fn open_items(&self, app: &App) -> Option<(usize, usize)> {
let made = assembled(app, self).await.ok()?;
let open = |kind: &str| {
made.items
.iter()
.filter(|i| i.kind == kind && i.state == "open")
.count()
};
Some((open("pull"), open("issue")))
}
/// Whether its pages lead to its merge queue: a rule of `record` has a
/// branch take pulls through it, or a pull waits in it still, its rule
/// dropped since.
async fn queues(&self, app: &App, record: Option<&RepoRecord>) -> bool {
let ruled = record.is_some_and(|r| r.rules.iter().any(|rule| rule.queue));
if ruled {
return true;
}
let made = assembled(app, self).await.ok();
let waits = |i: &Item| {
let queued = i.queue.as_ref().is_some_and(|q| q.state == "queued");
i.state == "open" && queued
};
made.is_some_and(|made| made.items.iter().any(waits))
}
/// What its pages show above themselves, to `viewer` if someone asks.
async fn head(&self, app: &App, viewer: Option<&str>) -> RepoViewDetailed {
let record = self.record(app);
let fork_of = match record.as_ref().and_then(|r| r.fork_of.as_ref()) {
Some(from) => repo_basic(app, from).await,
None => None,
};
let open = self.open_items(app).await.unwrap_or_default();
let merge_queue = self.queues(app, record.as_ref()).await;
let offer = record.as_ref().and_then(|r| r.transfer.clone());
let viewer = match viewer {
Some(v) => Some(
self.viewer_state(app, v, record.as_ref(), offer.as_ref())
.await,
),
None => None,
};
let avatar = avatar(app, &self.did).await;
let card = self.card(record.as_ref(), open, avatar);
let pushers = self.pushers(record.as_ref());
let shown = pushers
.iter()
.take(COLLABORATORS_SHOWN)
.map(|did| actor(app, did));
let collaborators = futures::future::join_all(shown).await;
let owner = actor(app, &self.did).await;
let site = self.site(app, &owner.handle, record.as_ref());
RepoViewDetailed {
uri: self.uri(),
owner,
site,
name: self.name.clone(),
key: record.as_ref().map(|r| r.key.clone()),
description: record.as_ref().and_then(|r| r.description.clone()),
archived_at: record.as_ref().and_then(|r| r.archived_at.clone()),
topics: record
.as_ref()
.map(|r| r.topics.clone())
.unwrap_or_default(),
collaborator_count: Some(pushers.len()).filter(|n| *n > 0),
collaborators,
merge_queue,
links: repo_links(record.map(|r| r.links).unwrap_or_default()),
fork_of,
default_branch: self.head.clone(),
open_pulls: open.0,
open_issues: open.1,
card: card_path(&self.href(""), &card),
star_count: self.star_count(app),
fork_count: self.fork_count(app),
releases: releases::summary(app, self),
viewer,
private: self.in_space.is_some(),
}
}
/// Its website, as the index last saw its record, where this server
/// serves sites: its address, by its owner's `handle` or plc id, and
/// what it was deployed from. A check counts only from a runner its
/// `record` names.
fn site(&self, app: &App, handle: &str, record: Option<&RepoRecord>) -> Option<SiteView> {
let domain = app.pages_domain.get()?;
let index = indexed(app, self)?;
let value = match self.in_space.as_deref() {
Some(space) => index.space_site(space, &self.did, &self.key),
None => index.site(&self.did, &self.key),
};
let site: SiteRecord = serde_json::from_str(&value.ok()??).ok()?;
let url = match self.in_space {
Some(_) => {
let label = knotless::site::private_label(&self.did, &self.key)?;
format!("https://{label}.{}.{domain}/", knotless::site::PRIVATE)
}
None => {
let label = knotless::site::owner_label(handle, &self.did)?;
format!("https://{label}.{domain}/{}/", self.name)
}
};
let runners = record.map(|r| r.runners.as_slice()).unwrap_or_default();
let by_runner = |uri: &String| {
let runner = knotless::aturi::AtUri::parse(uri).ok().map(|at| at.author);
runners.iter().any(|r| Some(&r.did) == runner.as_ref())
};
Some(SiteView {
url,
commit: site.commit,
check: site.check.filter(by_runner),
deployed_at: site.created_at,
})
}
/// Who `record` names among its collaborators, in its order, besides
/// its owner: not its organization's members, whom its page names.
fn pushers(&self, record: Option<&RepoRecord>) -> Vec<String> {
let named = record.into_iter().flat_map(|r| &r.collaborators);
named
.filter(|c| c.did != self.did)
.map(|c| c.did.clone())
.collect()
}
/// How many starred it: none for a private repository, which cannot
/// be starred, or without an index.
fn star_count(&self, app: &App) -> Option<u64> {
let index = app.index.as_ref().filter(|_| self.in_space.is_none())?;
index.star_count(&self.did, &self.key).ok()
}
/// How many forks of it there are: none for a private repository,
/// whose forks are private too, or without an index.
fn fork_count(&self, app: &App) -> Option<u64> {
let index = app.index.as_ref().filter(|_| self.in_space.is_none())?;
index.fork_count(&self.did, &self.key).ok()
}
/// What `viewer` has to do with it, as its `record` says, `offer`
/// naming where its owner offered to move it, if anywhere.
async fn viewer_state(
&self,
app: &App,
viewer: &str,
record: Option<&RepoRecord>,
offer: Option<&RepoRef>,
) -> RepoViewerState {
let star = app
.index
.as_ref()
.map(|i| i.star_of(viewer, &self.did, &self.key));
let transfer = match offer.filter(|to| to.did() == viewer) {
Some(to) => Some(RepoViewBasic {
did: viewer.to_string(),
handle: handle(app, viewer).await,
name: to.name().to_string(),
}),
None => None,
};
let labels = record.into_iter().flat_map(|r| {
let may = r
.labels
.iter()
.filter(|l| r.may_label(&self.did, viewer, l));
may.map(label_view)
});
let subscription = app
.index
.as_ref()
.and_then(|i| i.watching(viewer, &self.did, &self.key).ok().flatten());
RepoViewerState {
star: star.and_then(|s| s.ok().flatten()),
transfer,
labels: labels.collect(),
subscription,
admin: self.administered(app, viewer),
}
}
/// Whether `viewer` may change its settings: as its owner, or as an
/// admin who joined the organization that owns it, when that delegated
/// to this server's gateway, which keeps its session to write as it.
fn administered(&self, app: &App, viewer: &str) -> bool {
let kept = || app.gateway.is_some() && session::load(&app.ctx, &self.did).is_some();
viewer == self.did || (app.org_admits(&self.did, viewer) && kept())
}
/// The repository as views name it, by its owner's handle.
async fn basic(&self, app: &App) -> RepoViewBasic {
RepoViewBasic {
did: self.did.clone(),
handle: handle(app, &self.did).await,
name: self.name.clone(),
}
}
/// Its card, as `record` and the counts of open pulls and issues say,
/// with its owner's picture at `avatar`.
fn card(
&self,
record: Option<&RepoRecord>,
(pulls, issues): (usize, usize),
avatar: Option<String>,
) -> Card {
Card::Repo {
owner: self.authority.clone(),
avatar,
name: self.name.clone(),
description: record.and_then(|r| r.description.clone()),
pulls,
issues,
languages: self.languages(),
}
}
/// What its default branch is written in: nothing, if it has none.
fn languages(&self) -> Vec<Language> {
let found = self.objects().ok();
found
.and_then(|o| languages::of(&o, "HEAD").ok())
.unwrap_or_default()
}
/// Splits `rest` into a revision and a path: the longest branch or tag
/// that starts it, else its first segment.
fn locate<'a>(&self, rest: &'a str) -> Result<(String, &'a str)> {
let names = self.objects()?.names().map_err(AppError::internal)?;
let longest = names
.iter()
.filter(|n| rest == n.as_str() || rest.starts_with(&format!("{n}/")))
.map(String::len)
.max();
let cut = longest.unwrap_or(rest.find('/').unwrap_or(rest.len()));
let rev = read::rev(Some(&rest[..cut])).map_err(not_found)?;
Ok((rev, rest[cut..].trim_start_matches('/')))
}
/// Its branches, the default first, then the rest by name.
fn branches(&self) -> Result<Vec<String>> {
let mut all = self.objects()?.branches().map_err(AppError::internal)?;
all.sort_by_key(|b| *b != self.head);
Ok(all)
}
}
type RepoPath = Path<(String, String)>;
type RestPath = Path<(String, String, String)>;
/// The newest `limit` checks the repository `repo`, which views name as
/// `shown`, has had of what `filter` names, by the runners its record
/// names, newest first: of a pull's runs if it names one, else of refs
/// only. None from a server with no index.
fn checks(
app: &App,
repo: &RepoRef,
shown: &RepoViewBasic,
filter: CheckFilter,
limit: usize,
) -> Vec<CheckView> {
let found = checks_index(app, &repo.uri).and_then(|(index, private)| {
let (did, _, _) = forge::parse_uri(&repo.uri).ok()?;
let record: RepoRecord = serde_json::from_value(index.repo(&repo.uri).ok()??).ok()?;
// A private repository's, such as a pull's head, only to a viewer
// shown it as a member, and then theirs alone.
let viewer = crate::private::viewer().unwrap_or_default();
let shown = !private || app.shown_private(&viewer, did, &record.name);
if private && shown {
crate::private::touched();
}
let runners: Vec<String> = record.runners.into_iter().map(|r| r.did).collect();
let found = index
.checks(did, &repo.key, &runners, filter, limit, None)
.ok()
.filter(|_| shown);
if let Some((entries, _)) = found.as_ref().filter(|_| !private) {
app.recheck(index, entries);
}
found
});
found
.map(|(entries, _)| entries)
.unwrap_or_default()
.into_iter()
.filter_map(|e| Some((e.uri, serde_json::from_value(e.value).ok()?)))
.filter(|(_, c): &(String, CheckRecord)| filter.pull.is_some() || c.pull.is_none())
.filter_map(|(uri, c)| check_view(shown, uri, c))
.collect()
}
/// `dev.knotless.defs#minutesView`: the minutes a runner counts for a
/// repository's owner.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct MinutesView {
runner: ActorView,
minutes: u64,
#[serde(skip_serializing_if = "Option::is_none")]
allowance: Option<u64>,
#[serde(skip_serializing_if = "Vec::is_empty")]
freeing: Vec<Freeing>,
updated_at: String,
}
/// The minutes `runners`, those the repository `uri` of `owner` names,
/// count for `owner`, in the order they are named: none for a private
/// repository, whose minutes no runner publishes, or from a server with no
/// index.
async fn minutes(app: &App, uri: &str, owner: &str, runners: &[String]) -> Vec<MinutesView> {
let index = checks_index(app, uri).filter(|(_, private)| !private);
let found = index.and_then(|(index, _)| index.minutes(owner, runners).ok());
let counted = found.unwrap_or_default().into_iter().filter_map(|e| {
let m: MinutesRecord = serde_json::from_value(e.value).ok()?;
Some((e.author, m))
});
let views = counted.map(|(runner, m)| async move {
MinutesView {
runner: actor(app, &runner).await,
minutes: m.minutes,
allowance: m.allowance,
freeing: m.freeing,
updated_at: m.updated_at,
}
});
futures::future::join_all(views).await
}
/// The index the checks of the repository `uri` are read from, and
/// whether it is this server's own of the private ones, synced from their
/// spaces, which no relay rechecks: else the appview's.
fn checks_index<'a>(app: &'a App, uri: &str) -> Option<(&'a Arc<crate::index::Index>, bool)> {
let private = app
.spaces
.as_ref()
.filter(|i| matches!(i.repo(uri), Ok(Some(_))));
match private {
Some(index) => Some((index, true)),
None => app.index.as_ref().map(|index| (index, false)),
}
}
/// The check at `uri` of the repository `repo`, as `c` reports it.
fn check_view(repo: &RepoViewBasic, uri: String, c: CheckRecord) -> Option<CheckView> {
forge::parse_uri(&uri).ok()?;
Some(CheckView {
images: run_images(repo, &uri, c.outputs),
repo: repo.clone(),
workflow: c.workflow,
status: c.status,
commit: c.commit,
started_at: c.started_at,
finished_at: c.finished_at,
error: c.error,
skipped: c.skipped,
uri,
})
}
/// The first of each workflow in `checks`, by workflow name: the order of
/// the files in `.knotless/workflows`, rather than which one filed last.
fn newest(checks: Vec<CheckView>) -> Vec<CheckView> {
let mut seen = HashSet::new();
let mut newest: Vec<_> = checks
.into_iter()
.filter(|c| seen.insert(c.workflow.clone()))
.collect();
newest.sort_by(|a, b| a.workflow.cmp(&b.workflow));
newest
}
/// The pictures among the outputs of the check at `uri`, of `repo`, each
/// served by `dev.knotless.getCheckImage`.
fn run_images(repo: &RepoViewBasic, uri: &str, outputs: Vec<CheckOutput>) -> Vec<CheckImageView> {
let repo = format!("at://{}/{}/{}", repo.did, nsid::REPO, repo.name);
let image = |o| match o {
CheckOutput::Image(i) => Some(CheckImageView {
alt: i.alt,
url: query_path(
"getCheckImage",
&[("repo", &repo), ("uri", uri), ("cid", &i.image.link.cid)],
),
}),
CheckOutput::Other => None,
};
outputs.into_iter().filter_map(image).collect()
}
/// The path of the query `nsid` of `dev.knotless`'s, given `params`.
fn query_path(nsid: &str, params: &[(&str, &str)]) -> String {
let url = reqwest::Url::parse_with_params("http://x/", params).expect("a URL");
format!(
"/xrpc/dev.knotless.{nsid}?{}",
url.query().unwrap_or_default()
)
}
/// `dev.knotless.getCheck`'s answer: a check of one of a repository's
/// workflows, as its runner reported it: each step, and the end of what
/// they printed.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct CheckRun {
check: CheckView,
/// The ref it ran at, in full.
#[serde(rename = "ref")]
refname: String,
/// The pull it ran for, if it ran for one.
#[serde(skip_serializing_if = "Option::is_none")]
pull: Option<String>,
runner: ActorView,
steps: Vec<RunStep>,
/// The end of what its steps printed, each step's after a line
/// `==> <name>`.
#[serde(skip_serializing_if = "Option::is_none")]
log: Option<String>,
}
/// `dev.knotless.getCheck#stepView`: one of a check's steps.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct RunStep {
name: String,
/// `success`, `failure` or `skipped`.
status: String,
/// None when it was killed for its time, or skipped.
#[serde(skip_serializing_if = "Option::is_none")]
exit_code: Option<i32>,
duration_ms: u64,
}
/// How much of a check's log its page shows: the end, where a failure is.
const LOG_SHOWN: usize = 256 * 1024;
/// The last [`LOG_SHOWN`] bytes of `log`, from a character on.
fn log_tail(log: &[u8]) -> String {
let text = String::from_utf8_lossy(log);
let cut = text.len().saturating_sub(LOG_SHOWN);
let from = text.char_indices().map(|(i, _)| i).find(|&i| i >= cut);
text[from.unwrap_or(text.len())..].to_string()
}
#[derive(Deserialize)]
pub struct CheckParams {
repo: String,
uri: String,
cid: Option<String>,
}
/// The check at `p.uri`, if it is of the repository `p.repo` names and by
/// a runner that names: the only checks shown, as everywhere.
async fn trusted_check(app: &App, p: &CheckParams) -> Result<(Repo, Remote, CheckRecord)> {
let (repo, c) = named_check(app, p).await?;
let (runner, _, _) = forge::parse_uri(&p.uri).expect("a check named");
// A private repository's runner writes its checks into the space.
let remote = writer(app, &repo, runner).await?;
Ok((repo, remote, c))
}
/// Where `did` writes about `repo`: their own repo, or for a private
/// repository their repo in its space.
async fn writer(app: &App, repo: &Repo, did: &str) -> Result<Remote> {
let at = RepoUrl {
authority: did.to_string(),
name: "_".into(),
};
let remote = match &repo.in_space {
Some(space) => in_space(&repo.reader, space, did).await,
None => Remote::connect(&app.ctx, &at).await,
};
remote.map_err(not_found)
}
/// [`trusted_check`], as the index has it, with no word from its runner.
async fn named_check(app: &App, p: &CheckParams) -> Result<(Repo, CheckRecord)> {
let repo = repo::found(app, &p.repo).await?;
let name = &repo.name;
let runner = match forge::parse_uri(&p.uri) {
Ok((runner, nsid::CHECK, _)) => runner,
_ => return Err(repo::invalid(format!("{} is no check", p.uri))),
};
// Only a runner the repository names is trusted, as everywhere checks show.
let runners = repo.record(app).unwrap_or_default().runners;
if !runners.iter().any(|r| r.did == runner) {
return Err(not_found(anyhow::anyhow!(
"{runner} runs no checks of {name}"
)));
}
// As the index has it: the appview's, or this server's own of the space.
let uri = repo.named(&p.uri);
let kept = indexed(app, &repo).and_then(|index| index.check_record(&uri).ok().flatten());
let found = kept.and_then(|value| serde_json::from_value::<CheckRecord>(value).ok());
let of_this = |c: &CheckRecord| {
let owner = forge::parse_uri(&c.repo.uri).map(|(did, _, _)| did == repo.did);
c.repo.key == repo.key && owner.unwrap_or(false)
};
let Some(c) = found.filter(of_this) else {
return Err(not_found(anyhow::anyhow!("no such check of {name}")));
};
Ok((repo, c))
}
/// `did`'s repo in the space `uri`, as `reader` reads it.
async fn in_space(reader: &knotless::env::Ctx, uri: &str, did: &str) -> anyhow::Result<Remote> {
let joined = knotless::space::joined(reader, uri).await?;
crate::spaces::remote(reader, &joined, did).await
}
/// Pictures check runs made and blobs attached, by kind, author and CID.
/// A blob is the same for good, so it is read once; one in a space is
/// read each time, as whoever asks may read it.
static RUN_BLOBS: LazyLock<card::Newest<Arc<Vec<u8>>>> = LazyLock::new(|| card::Newest::new(64));
/// The tails of check logs, kept as [`RUN_BLOBS`] are: a few kilobytes
/// most, and at most [`LOG_SHOWN`], so many more of them, up to
/// [`LOG_TAILS_KEPT`] bytes in all.
static LOG_TAILS: LazyLock<card::Newest<Arc<Vec<u8>>>> =
LazyLock::new(|| card::Newest::weighed(LOG_TAILS_KEPT, |tail| tail.len()));
/// What the kept tails of check logs weigh at most.
const LOG_TAILS_KEPT: usize = 64 * 1024 * 1024;
/// The blob `cid` of the runner `remote`, as `shown` makes it of `kind`.
async fn run_blob(
remote: &Remote,
kind: &str,
cid: &str,
shown: fn(Vec<u8>) -> Vec<u8>,
) -> anyhow::Result<Arc<Vec<u8>>> {
let public = remote.space.is_none();
let key = format!("{kind}/{}/{cid}", remote.did());
let cache = [&RUN_BLOBS, &LOG_TAILS][usize::from(kind == "log")];
let kept = public.then(|| cache.get(&key)).flatten();
if let Some(kept) = looked("run blobs", kept) {
return Ok(kept);
}
let blob = Arc::new(shown(remote.get_blob(cid).await?));
if public {
cache.put(&key, blob.clone());
}
Ok(blob)
}
/// A log as a check's page shows it: its tail.
fn shown_tail(log: Vec<u8>) -> Vec<u8> {
log_tail(&log).into_bytes()
}
/// How many check logs are read ahead at once, as their checks come.
static WARMING: LazyLock<tokio::sync::Semaphore> = LazyLock::new(|| tokio::sync::Semaphore::new(4));
/// The CID of the log of `did`'s check `rkey` as the index has it, when
/// its repository names `did` among its runners, so that its page shows it.
fn log_to_warm(index: &Index, did: &str, rkey: &str) -> Option<String> {
let uri = format!("at://{did}/{}/{rkey}", nsid::CHECK);
let value = index.check_record(&uri).ok().flatten()?;
let check: CheckRecord = serde_json::from_value(value).ok()?;
let cid = check.log?.link.cid;
let repo = index.repo(&check.repo.uri).ok().flatten()?;
let repo: RepoRecord = serde_json::from_value(repo).ok()?;
repo.is_runner(did).then_some(cid)
}
/// Reads ahead the tail of the log of `did`'s check `rkey`, as the check
/// comes from the network, so its page's first look need not wait on the
/// runner's PDS. A few at once, and none past them: a burst of checks
/// leaves the rest to be read as they are looked at.
pub(crate) async fn warm_check_log(app: Arc<App>, did: String, rkey: String) {
let index = app.index.as_ref();
let cid = index.and_then(|index| log_to_warm(index, &did, &rkey));
let permit = cid.as_ref().and_then(|_| WARMING.try_acquire().ok());
let (Some(cid), Some(_permit)) = (cid, permit) else {
return;
};
let at = RepoUrl {
authority: did,
name: "_".into(),
};
let read = async {
let remote = Remote::connect(&app.ctx, &at).await?;
run_blob(&remote, "log", &cid, shown_tail).await
};
fill(
"check logs",
&format!("{}/{cid}", at.authority),
&read.await,
);
}
/// `dev.knotless.getCheck`.
async fn get_check(State(app): State<Arc<App>>, Query(p): Query<CheckParams>) -> Result<Response> {
let (repo, remote, mut c) = trusted_check(&app, &p).await?;
let read = c
.log
.as_ref()
.map(|b| run_blob(&remote, "log", &b.link.cid, shown_tail));
let blob = futures::future::OptionFuture::from(read).await;
let log = blob.and_then(|b| b.ok());
let log = log.map(|b| String::from_utf8_lossy(&b).into_owned());
let runner = actor(&app, remote.did()).await;
let steps = std::mem::take(&mut c.steps).into_iter();
let steps = steps.map(|s: CheckStep| RunStep {
name: s.name,
status: s.status,
exit_code: s.exit_code,
duration_ms: s.duration_ms,
});
let (refname, pull) = (std::mem::take(&mut c.ref_name), c.pull.take());
let shown = repo.basic(&app).await;
let check = check_view(&shown, p.uri, c).expect("a check's own uri parses");
Ok(answer(CheckRun {
check,
refname,
pull,
runner,
steps: steps.collect(),
log,
}))
}
/// `dev.knotless.getCheckImage`: a picture a check made, by its CID: only
/// one the check names, from its runner's PDS. Never run as a page, and
/// the same for good.
async fn get_check_image(
State(app): State<Arc<App>>,
Query(p): Query<CheckParams>,
) -> Result<Response> {
let (_, remote, c) = trusted_check(&app, &p).await?;
let cid = p.cid.unwrap_or_default();
let named = c.outputs.into_iter().find_map(|o| match o {
CheckOutput::Image(i) if i.image.link.cid == cid => Some(i.image),
_ => None,
});
let Some(image) = named else {
return Err(not_found(anyhow::anyhow!(
"{cid} is no picture of this check"
)));
};
let whole = |b: Vec<u8>| b;
let bytes = run_blob(&remote, "image", &cid, whole).await;
let bytes = bytes.map_err(not_found)?;
Ok(picture(&image.mime_type, bytes.to_vec()))
}
#[derive(Deserialize)]
pub struct AttachmentParams {
repo: String,
uri: String,
cid: String,
}
/// `dev.knotless.getAttachment`: a blob a pull, issue or comment attaches,
/// by its CID: only one its record names, from its author's PDS, or for a
/// private repository from their repo in its space. Never run as a page,
/// and the same for good.
async fn get_attachment(
State(app): State<Arc<App>>,
Query(p): Query<AttachmentParams>,
) -> Result<Response> {
let repo = repo::found(&app, &p.repo).await?;
let made = assembled(&app, &repo).await?;
let uri = repo.named(&p.uri);
let said = made.entries.iter().find(|e| e.uri == uri).filter(|e| {
let collection = forge::parse_uri(&e.uri).map(|(_, c, _)| c);
matches!(collection, Ok(nsid::PULL | nsid::ISSUE | nsid::COMMENT))
});
let attachments = said.map(|e| e.value["attachments"].clone());
let attachments = attachments.and_then(|a| serde_json::from_value::<Vec<_>>(a).ok());
let named = attachments
.unwrap_or_default()
.into_iter()
.find_map(|a| match a {
Attachment::Image(i) if i.image.link.cid == p.cid => Some(i.image),
_ => None,
});
let (Some(image), Some(said)) = (named, said) else {
let why = format!("{} attaches no {}", p.uri, p.cid);
return Err(not_found(anyhow::anyhow!(why)));
};
let remote = writer(&app, &repo, &said.author).await?;
let whole = |b: Vec<u8>| b;
let bytes = run_blob(&remote, "picture", &p.cid, whole).await;
let bytes = bytes.map_err(not_found)?;
Ok(picture(&image.mime_type, bytes.to_vec()))
}
/// The pictures a pull, issue or comment attaches, to show from here
/// rather than from where its body links to them.
#[derive(Clone, Copy)]
pub(super) struct Attached<'a> {
/// The repository's record, as `getAttachment` takes it.
pub repo: &'a str,
/// The pull's, issue's or comment's record.
pub uri: &'a str,
pub images: &'a [ImageAttachment],
}
impl Attached<'_> {
/// Where `dest`, a link to a blob its record attaches on its author's
/// PDS, is shown from here.
fn served(&self, dest: &str) -> Option<String> {
let url = reqwest::Url::parse(dest).ok()?;
let read = url.path().ends_with("/xrpc/com.atproto.sync.getBlob");
let asked: HashMap<_, _> = url.query_pairs().collect();
let (did, cid) = (asked.get("did")?, asked.get("cid")?);
let (author, _, _) = forge::parse_uri(self.uri).ok()?;
let attached = self.images.iter().any(|i| &i.image.link.cid == cid);
let ours = read && did == author && attached;
ours.then(|| self.path(cid))
}
/// Where its picture `cid` is shown from here.
fn path(&self, cid: &str) -> String {
let at = [("repo", self.repo), ("uri", self.uri), ("cid", cid)];
query_path("getAttachment", &at)
}
/// `html` with each of its pictures shown from here drawn at the size
/// its record gives, as a page then makes room for it before it loads.
fn sized(&self, html: String) -> String {
let sizes = self.images.iter().filter_map(|i| {
let size = i.width.zip(i.height)?;
Some((self.path(&i.image.link.cid).replace('&', "&"), size))
});
sizes.fold(html, |html, (src, (w, h))| {
let img = format!(r#"<img src="{src}""#);
html.replace(
&img,
&format!(r#"<img width="{w}" height="{h}" src="{src}""#),
)
})
}
}
/// Where an account's own pictures are served.
const OWN_AVATAR: &str = "/xrpc/dev.knotless.getAvatar?";
/// Where the account `did`'s own picture `cid` is served.
fn own_avatar_path(did: &str, cid: &str) -> String {
query_path("getAvatar", &[("actor", did), ("cid", cid)])
}
#[derive(Deserialize)]
pub struct AvatarParams {
actor: String,
cid: String,
}
/// `dev.knotless.getAvatar`: an account's picture, by its CID: only the
/// one its knotless profile names, from its PDS. Never run as a page, and
/// the same for good.
async fn get_avatar(
State(app): State<Arc<App>>,
Query(AvatarParams { actor: did, cid }): Query<AvatarParams>,
) -> Result<Response> {
let at = RepoUrl {
authority: did,
name: "_".into(),
};
let remote = Remote::connect(&app.ctx, &at).await.map_err(not_found)?;
let own = own_profile(&app, &remote).await?;
let named = own.and_then(|p| p.avatar).filter(|a| a.link.cid == cid);
let Some(avatar) = named else {
return Err(not_found(anyhow::anyhow!(
"{cid} is no picture of this account's"
)));
};
let bytes = remote.get_blob(&cid).await.map_err(not_found)?;
Ok(picture(&avatar.mime_type, bytes))
}
/// A picture of `kind`, as an answer: one of a kind a page could be run as
/// is sent as bytes alone.
fn picture(kind: &str, bytes: Vec<u8>) -> Response {
let kind = match kind {
kind @ ("image/png" | "image/jpeg" | "image/webp" | "image/gif") => kind,
_ => "application/octet-stream",
};
(
[
(header::CONTENT_TYPE, kind),
(header::CACHE_CONTROL, "public, max-age=31536000, immutable"),
(
header::CONTENT_SECURITY_POLICY,
"sandbox; default-src 'none'",
),
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
],
bytes,
)
.into_response()
}
/// Whether `path` names a Markdown file, by its extension.
fn is_markdown(path: &str) -> bool {
let ext = path.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase());
matches!(ext.as_deref(), Some("md" | "markdown" | "mdown" | "mkd"))
}
/// Where a repository's directories are shown, and which paths are one:
/// Markdown's links to a directory go there rather than to a file.
type Trees<'a> = Option<(&'a str, &'a dyn Fn(&str) -> bool)>;
/// Markdown, with raw HTML shown as text but for the folds [`folds`]
/// keeps, and links made relative to the directory `dir` it was found
/// in, among files shown under `blobs`, directories under `trees` and,
/// for images, `raws`.
fn markdown(text: &str, blobs: &str, raws: &str, dir: &str, trees: Trees) -> String {
drawn(text, blobs, raws, dir, trees, None, None)
}
/// What a pull, issue or comment says, as [`markdown`] draws it under
/// `base`, its pictures shown as `attached` says, and each account it
/// mentions linked to: to the account `mentioned` says, by DID, which its
/// facets give ([`knotless::mentions::mentioned`]), and otherwise to the
/// handle.
pub(super) fn written(
text: &str,
base: &str,
attached: Option<Attached>,
mentioned: &HashMap<String, String>,
) -> String {
written_naming(text, base, attached, mentioned, &Named::new())
}
/// The pulls and issues of a repository, by their `at://` URIs: each its
/// page's path and its title.
pub(super) type Named = HashMap<String, (String, String)>;
/// [`written`], with each pull or issue `named` holds linked to its page
/// where the text names it by its `at://` URI, bare or as a link's target.
/// A bare one shows as its title.
pub(super) fn written_naming(
text: &str,
base: &str,
attached: Option<Attached>,
mentioned: &HashMap<String, String>,
named: &Named,
) -> String {
let said = Said { mentioned, named };
drawn(text, base, base, "", None, attached, Some(said))
}
/// Whom and what a pull, issue or comment names, to link to.
#[derive(Clone, Copy)]
struct Said<'a> {
mentioned: &'a HashMap<String, String>,
named: &'a Named,
}
/// [`markdown`], with the pictures `attached` shown from here, and with
/// `said`, its mentions and references linked as [`written_naming`] links
/// them.
fn drawn(
text: &str,
blobs: &str,
raws: &str,
dir: &str,
trees: Trees,
attached: Option<Attached>,
said: Option<Said>,
) -> String {
let dir = url_path(dir);
use pulldown_cmark::{Event, Options, Parser, Tag};
let options =
Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH | Options::ENABLE_TASKLISTS;
let parsed = pulldown_cmark::TextMergeStream::new(Parser::new_ext(text, options));
let page = |dest: &str| {
said.and_then(|s| s.named.get(dest))
.map(|(path, _)| path.clone())
};
// Folds first, so what a summary says is linked as the rest is.
let events = autolinks(folds(parsed, options).into_iter()).map(|event| match event {
Event::Start(Tag::Link {
link_type,
dest_url,
title,
id,
}) => Event::Start(Tag::Link {
link_type,
dest_url: page(&dest_url)
.unwrap_or_else(|| link(&dest_url, blobs, &dir, trees))
.into(),
title,
id,
}),
Event::Start(Tag::Image {
link_type,
dest_url,
title,
id,
}) => Event::Start(Tag::Image {
link_type,
dest_url: attached
.and_then(|a| a.served(&dest_url))
.unwrap_or_else(|| link(&dest_url, raws, &dir, None))
.into(),
title,
id,
}),
other => other,
});
let events: Vec<Event> = match said {
Some(said) => mentions(references(events, said.named), said.mentioned).collect(),
None => events.collect(),
};
let mut out = String::new();
pulldown_cmark::html::push_html(&mut out, fenced(events).into_iter());
match attached {
Some(attached) => attached.sized(out),
None => out,
}
}
/// A part of Markdown folded away: `<details>`, its `<summary>` said
/// beside the arrow that opens it.
#[derive(Clone, Copy, PartialEq)]
enum Fold {
Details,
Summary,
}
impl Fold {
fn closer(self) -> &'static str {
match self {
Fold::Details => "</details>",
Fold::Summary => "</summary>",
}
}
}
/// Raw HTML shown as text, but for the `<details>` and `<summary>` that
/// fold a part away, as GitHub's Markdown keeps them: each written anew,
/// `open` the only attribute kept, and only on lines of their own (an
/// HTML block), so never inside a paragraph. They nest as the blocks
/// around them do: a closing tag counts only where it closes what was
/// opened beside it, a block closes what it left open as it ends, and so
/// does the end of the text, so nothing written can close the page's own
/// elements. What a summary says is Markdown on a line, as [`summary`]
/// reads it; raw HTML's text is written escaped, for no later step to
/// link.
fn folds<'a>(
events: impl Iterator<Item = pulldown_cmark::Event<'a>>,
options: pulldown_cmark::Options,
) -> Vec<pulldown_cmark::Event<'a>> {
use pulldown_cmark::{Event, Tag, TagEnd};
let mut out = Vec::new();
// What is open, each with how many blocks deep it was opened.
let mut open: Vec<(Fold, usize)> = Vec::new();
let mut depth = 0usize;
for event in events {
match event {
Event::Start(Tag::HtmlBlock) | Event::End(TagEnd::HtmlBlock) => out.push(event),
Event::Start(_) => {
depth += 1;
out.push(event);
}
Event::End(_) => {
depth -= 1;
let kept = open.iter().take_while(|(_, d)| *d <= depth).count();
shut(&mut out, &mut open, kept);
out.push(event);
}
Event::InlineHtml(html) => out.push(Event::Html(escaped(&html).into())),
Event::Html(html) => unfolded(&html, depth, options, &mut open, &mut out),
other => out.push(other),
}
}
shut(&mut out, &mut open, 0);
out
}
/// `text` escaped as HTML's text is, as pulldown-cmark escapes it.
fn escaped(text: &str) -> String {
let mut out = String::with_capacity(text.len());
for c in text.chars() {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
c => out.push(c),
}
}
out
}
/// What a `<summary>` says, `text`, as Markdown on a line: its emphasis,
/// code and links. Written as more than that (a heading, a list, a
/// picture), it is shown as text.
fn summary(text: &str, options: pulldown_cmark::Options) -> Vec<pulldown_cmark::Event<'static>> {
use pulldown_cmark::{Event, Parser, Tag, TagEnd};
let events: Vec<Event> = Parser::new_ext(text, options).collect();
let inline = |e: &Event| {
matches!(
e,
Event::Start(Tag::Emphasis | Tag::Strong | Tag::Strikethrough | Tag::Link { .. })
| Event::End(
TagEnd::Emphasis | TagEnd::Strong | TagEnd::Strikethrough | TagEnd::Link
)
| Event::Text(_)
| Event::Code(_)
| Event::SoftBreak
| Event::HardBreak
| Event::InlineHtml(_)
)
};
match events.as_slice() {
[
Event::Start(Tag::Paragraph),
inner @ ..,
Event::End(TagEnd::Paragraph),
] if inner.iter().all(inline) => {
let said = inner.iter().cloned().map(|e| match e {
Event::InlineHtml(html) => Event::Html(escaped(&html).into()),
e => e.into_static(),
});
said.collect()
}
[] => Vec::new(),
_ => vec![Event::Html(escaped(text).into())],
}
}
/// The folds `open` holds past its first `kept`, closed.
fn shut(out: &mut Vec<pulldown_cmark::Event<'_>>, open: &mut Vec<(Fold, usize)>, kept: usize) {
for (fold, _) in open.drain(kept..).rev() {
out.push(pulldown_cmark::Event::Html(fold.closer().into()));
}
}
/// A line of an HTML block, `depth` blocks deep, as [`folds`] shows it.
fn unfolded<'a>(
html: &str,
depth: usize,
options: pulldown_cmark::Options,
open: &mut Vec<(Fold, usize)>,
out: &mut Vec<pulldown_cmark::Event<'a>>,
) {
use pulldown_cmark::Event;
// The text since the last tag, said in a summary or not.
let said = |text: String, summarised: bool, out: &mut Vec<Event<'a>>| {
if text.is_empty() {
return;
}
if summarised {
let words: Vec<Event<'a>> = summary(&text, options);
out.extend(words);
} else {
out.push(Event::Html(escaped(&text).into()));
}
};
let mut text = String::new();
let mut rest = html;
while let Some(at) = rest.find('<') {
text.push_str(&rest[..at]);
rest = &rest[at..];
let top = open.last().copied();
let under = open.len().checked_sub(2).map(|i| open[i]);
let before = std::mem::take(&mut text);
let kept: Option<(&'static str, usize)> = match fold_tag(rest) {
Some((FoldTag::Opens(Fold::Details, shown), len)) => {
open.push((Fold::Details, depth));
Some((if shown { "<details open>" } else { "<details>" }, len))
}
Some((FoldTag::Opens(Fold::Summary, _), len))
if top == Some((Fold::Details, depth)) =>
{
open.push((Fold::Summary, depth));
Some(("<summary>", len))
}
Some((FoldTag::Closes(Fold::Summary), len)) if top == Some((Fold::Summary, depth)) => {
open.pop();
Some(("</summary>", len))
}
Some((FoldTag::Closes(Fold::Details), len)) if top == Some((Fold::Details, depth)) => {
open.pop();
Some(("</details>", len))
}
// A summary left open closes with its fold.
Some((FoldTag::Closes(Fold::Details), len))
if top == Some((Fold::Summary, depth)) && under == Some((Fold::Details, depth)) =>
{
open.truncate(open.len() - 2);
Some(("</summary></details>", len))
}
_ => None,
};
match kept {
Some((tag, len)) => {
said(before, top == Some((Fold::Summary, depth)), out);
out.push(Event::Html(tag.into()));
rest = &rest[len..];
}
None => {
text = before;
text.push('<');
rest = &rest[1..];
}
}
}
text.push_str(rest);
said(text, open.last() == Some(&(Fold::Summary, depth)), out);
}
/// A `<details>` or `<summary>` tag [`folds`] keeps.
enum FoldTag {
/// Opening one, and for `<details>`, whether it is drawn open.
Opens(Fold, bool),
Closes(Fold),
}
/// The fold tag `html` starts with, and its length: `<details>`,
/// `<details open>`, `<summary>` or the closing tag of either, in any
/// case, with any space before its `>`.
fn fold_tag(html: &str) -> Option<(FoldTag, usize)> {
let end = html.find('>')?;
let inner = &html[1..end];
let (closes, inner) = match inner.strip_prefix('/') {
Some(inner) => (true, inner),
None => (false, inner),
};
if !inner.starts_with(|c: char| c.is_ascii_alphabetic()) {
return None;
}
let mut words = inner.split_ascii_whitespace();
let fold = match words.next()?.to_ascii_lowercase().as_str() {
"details" => Fold::Details,
"summary" => Fold::Summary,
_ => return None,
};
let attributes: Vec<&str> = words.collect();
let tag = match (closes, fold, attributes.as_slice()) {
(true, _, []) => FoldTag::Closes(fold),
(false, _, []) => FoldTag::Opens(fold, false),
(false, Fold::Details, [open]) if open.eq_ignore_ascii_case("open") => {
FoldTag::Opens(Fold::Details, true)
}
_ => return None,
};
Some((tag, end + 1))
}
/// Fenced code in a language highlighted here, its block opened with the
/// ranges [`highlight::fence`] gives, which the page paints as it paints a
/// file's.
fn fenced(mut events: Vec<pulldown_cmark::Event<'_>>) -> Vec<pulldown_cmark::Event<'_>> {
use pulldown_cmark::{CodeBlockKind, Event, Tag};
for i in 1..events.len() {
let (Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(info))), Event::Text(text)) =
(&events[i - 1], &events[i])
else {
continue;
};
if let Some(ranges) = highlight::fence(info, text) {
let open = format!("<pre><code data-ranges=\"{}\">", ranges.encode());
events[i - 1] = Event::Html(open.into());
}
}
events
}
/// Text's mentions, `@handle`, as links to the account's profile: to its
/// DID where `mentioned` names it, else to the handle. Not in code, nor in
/// what is already a link.
fn mentions<'a>(
events: impl Iterator<Item = pulldown_cmark::Event<'a>>,
mentioned: &HashMap<String, String>,
) -> impl Iterator<Item = pulldown_cmark::Event<'a>> {
use pulldown_cmark::{Event, LinkType, Tag, TagEnd};
let mut inside = 0usize;
events.flat_map(move |event| match event {
Event::Text(text) if inside == 0 => {
let found = knotless::mentions::in_text(&text);
if found.is_empty() {
return vec![Event::Text(text)];
}
let mut events = Vec::new();
let mut done = 0;
for (at, len, handle) in found {
events.push(Event::Text(text[done..at].to_string().into()));
let whom = mentioned.get(&handle).unwrap_or(&handle);
events.push(Event::Start(Tag::Link {
link_type: LinkType::Inline,
dest_url: format!("/{}", url_path(whom)).into(),
title: "".into(),
id: "".into(),
}));
events.push(Event::Text(text[at..at + len].to_string().into()));
events.push(Event::End(TagEnd::Link));
done = at + len;
}
events.push(Event::Text(text[done..].to_string().into()));
events
}
event => {
match &event {
Event::Start(Tag::Link { .. } | Tag::Image { .. } | Tag::CodeBlock(_)) => {
inside += 1
}
Event::End(TagEnd::Link | TagEnd::Image | TagEnd::CodeBlock) => inside -= 1,
_ => {}
}
vec![event]
}
})
}
/// Text's `at://` URIs of the pulls and issues `named` holds, as links to
/// their pages, each shown as its title. Not in code, nor in what is
/// already a link.
fn references<'a>(
events: impl Iterator<Item = pulldown_cmark::Event<'a>>,
named: &Named,
) -> impl Iterator<Item = pulldown_cmark::Event<'a>> {
use pulldown_cmark::{Event, Tag, TagEnd};
let mut inside = 0usize;
events.flat_map(move |event| match event {
Event::Text(text) if inside == 0 => referenced(text, named),
event => {
match &event {
Event::Start(Tag::Link { .. } | Tag::Image { .. } | Tag::CodeBlock(_)) => {
inside += 1
}
Event::End(TagEnd::Link | TagEnd::Image | TagEnd::CodeBlock) => inside -= 1,
_ => {}
}
vec![event]
}
})
}
/// `text` as text, with each `at://` URI in it that `named` holds made a
/// link to its page, showing its title.
fn referenced<'a>(
text: pulldown_cmark::CowStr<'a>,
named: &Named,
) -> Vec<pulldown_cmark::Event<'a>> {
use pulldown_cmark::{Event, LinkType, Tag, TagEnd};
let mut events = Vec::new();
let (mut done, mut from) = (0, 0);
while let Some(found) = text[from..].find("at://") {
let at = from + found;
from = at + "at://".len();
let before = text[..at].chars().next_back();
if before.is_some_and(char::is_alphanumeric) {
continue;
}
let rest = &text[at..];
let end = rest.find(|c: char| c.is_whitespace() || "<>()[]\"'`".contains(c));
let uri =
rest[..end.unwrap_or(rest.len())].trim_end_matches(['.', ',', ';', ':', '!', '?']);
let Some((path, title)) = named.get(uri) else {
continue;
};
if done < at {
events.push(Event::Text(text[done..at].to_string().into()));
}
events.push(Event::Start(Tag::Link {
link_type: LinkType::Inline,
dest_url: path.clone().into(),
title: "".into(),
id: "".into(),
}));
events.push(Event::Text(title.clone().into()));
events.push(Event::End(TagEnd::Link));
done = at + uri.len();
from = done;
}
if done == 0 {
return vec![Event::Text(text)];
}
if done < text.len() {
events.push(Event::Text(text[done..].to_string().into()));
}
events
}
/// Text's bare web addresses as links, as GitHub makes them, but not in
/// code or in what is already a link.
fn autolinks<'a>(
events: impl Iterator<Item = pulldown_cmark::Event<'a>>,
) -> impl Iterator<Item = pulldown_cmark::Event<'a>> {
use pulldown_cmark::{Event, Tag, TagEnd};
let mut inside = 0usize;
events.flat_map(move |event| match event {
Event::Text(text) if inside == 0 => linked(text),
event => {
match &event {
Event::Start(Tag::Link { .. } | Tag::Image { .. } | Tag::CodeBlock(_)) => {
inside += 1
}
Event::End(TagEnd::Link | TagEnd::Image | TagEnd::CodeBlock) => inside -= 1,
_ => {}
}
vec![event]
}
})
}
/// `text` as text, with each web address in it made a link.
fn linked(text: pulldown_cmark::CowStr<'_>) -> Vec<pulldown_cmark::Event<'_>> {
use pulldown_cmark::{Event, LinkType, Tag, TagEnd};
let mut events = Vec::new();
let mut done = 0;
let mut at = 0;
while at < text.len() {
let after = text[..at].chars().next_back();
let starts = after.is_none_or(|c| c.is_whitespace() || "(*_~".contains(c));
let found = starts.then(|| web_address(&text[at..])).flatten();
let Some((len, dest)) = found else {
at += text[at..].chars().next().map_or(1, char::len_utf8);
continue;
};
if done < at {
events.push(Event::Text(text[done..at].to_string().into()));
}
events.push(Event::Start(Tag::Link {
link_type: LinkType::Autolink,
dest_url: dest.into(),
title: "".into(),
id: "".into(),
}));
events.push(Event::Text(text[at..at + len].to_string().into()));
events.push(Event::End(TagEnd::Link));
at += len;
done = at;
}
if done == 0 {
return vec![Event::Text(text)];
}
if done < text.len() {
events.push(Event::Text(text[done..].to_string().into()));
}
events
}
/// The web address `text` starts with, by GFM's extended autolinks: its
/// length, less trailing punctuation and unmatched `)`, and where it goes.
fn web_address(text: &str) -> Option<(usize, String)> {
let scheme = ["https://", "http://", "www."].into_iter().find(|s| {
let start = text.as_bytes().get(..s.len());
start.is_some_and(|b| b.eq_ignore_ascii_case(s.as_bytes()))
})?;
let end = text.find(|c: char| c.is_whitespace() || c == '<');
let mut url = &text[..end.unwrap_or(text.len())];
while let Some(last) = url.chars().last() {
let opens = url.matches('(').count();
let unmatched = last == ')' && opens < url.matches(')').count();
if !unmatched && !".,:;!?*_~'\"".contains(last) {
break;
}
url = &url[..url.len() - 1];
}
let rest = if scheme == "www." {
url
} else {
&url[scheme.len()..]
};
let domain = &rest[..rest
.find(|c: char| !(c.is_alphanumeric() || "-_.".contains(c)))
.unwrap_or(rest.len())];
let segments: Vec<&str> = domain.split('.').collect();
let last_two = &segments[segments.len().saturating_sub(2)..];
let valid = segments.len() > 1 && last_two.iter().all(|s| !s.is_empty() && !s.contains('_'));
let dest = if scheme == "www." {
format!("http://{url}")
} else {
url.to_string()
};
valid.then_some((url.len(), dest))
}
/// A link from Markdown found in `dir`: web and mail links as they are,
/// other schemes nowhere, fragments in place, and paths resolved inside the
/// repository, whose files are under `root` and directories under `trees`,
/// never above it.
fn link(dest: &str, root: &str, dir: &str, trees: Trees) -> String {
let scheme = dest
.split_once(':')
.map(|(s, _)| s)
.filter(|s| !s.contains(['/', '#', '?']));
match scheme.map(str::to_ascii_lowercase).as_deref() {
Some("http" | "https" | "mailto") => dest.to_string(),
Some(_) => "#".to_string(),
None if dest.starts_with('#') => dest.to_string(),
None => {
let (path, rest) = dest.split_at(dest.find(['?', '#']).unwrap_or(dest.len()));
let from = if path.starts_with('/') { "" } else { dir };
let mut parts = Vec::new();
for part in from.split('/').chain(path.split('/')) {
match part {
"" | "." => {}
".." => {
parts.pop();
}
part => parts.push(part),
}
}
let path = parts.join("/");
let root = match trees {
Some((trees, is_dir)) if is_dir(&unescaped(&path)) => trees,
_ => root,
};
format!("{root}/{path}{rest}")
}
}
}
/// A URL's path as the tree names it, its `%XX` escapes decoded.
fn unescaped(path: &str) -> String {
let bytes = path.as_bytes();
let mut out = Vec::new();
let mut at = 0;
while at < bytes.len() {
let hex = bytes
.get(at + 1..at + 3)
.filter(|h| h.iter().all(u8::is_ascii_hexdigit));
let byte = hex.and_then(|h| u8::from_str_radix(std::str::from_utf8(h).ok()?, 16).ok());
match (bytes[at], byte) {
(b'%', Some(byte)) => {
out.push(byte);
at += 3;
}
(b, _) => {
out.push(b);
at += 1;
}
}
}
String::from_utf8_lossy(&out).into_owned()
}
fn is_image(path: &str) -> Option<&'static str> {
let ext = path.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase())?;
Some(match ext.as_str() {
"png" => "image/png",
"jpg" | "jpeg" => "image/jpeg",
"gif" => "image/gif",
"webp" => "image/webp",
"avif" => "image/avif",
"svg" => "image/svg+xml",
"ico" => "image/x-icon",
_ => return None,
})
}
fn is_binary(bytes: &[u8]) -> bool {
bytes[..bytes.len().min(8000)].contains(&0) || std::str::from_utf8(bytes).is_err()
}
/// The file itself. Never run as a page: sandboxed, and typed only as text
/// or an image. A private repository's, to a member logged in to the pages
/// alone, and kept by no cache.
async fn raw(State(app): State<Arc<App>>, headers: HeaderMap, path: RestPath) -> Result<Response> {
let viewer = crate::login::viewer(&app, &headers);
let (file, touched) = crate::private::reading(viewer, None, raw_file(&app, path)).await;
let mut res = file?;
if touched {
res = crate::private::unshared(res);
}
let vary = HeaderValue::from_static("cookie");
res.headers_mut().append(header::VARY, vary);
Ok(res)
}
async fn raw_file(app: &App, Path((authority, name, rest)): RestPath) -> Result<Response> {
let repo = Repo::open(app, &authority, &name).await?;
let (rev, path) = repo.locate(&rest)?;
let bytes = repo.objects()?.blob(&rev, path).map_err(not_found)?;
let kind = match is_image(path) {
Some(image) => image,
None if is_binary(&bytes) => "application/octet-stream",
None => "text/plain; charset=utf-8",
};
Ok((
[
(header::CONTENT_TYPE, kind),
(header::CACHE_CONTROL, cache_for(&rev)),
(
header::CONTENT_SECURITY_POLICY,
"sandbox; default-src 'none'; style-src 'unsafe-inline'",
),
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
],
bytes,
)
.into_response())
}
/// Why a file's rows are not shown, if they are not.
fn note(f: &diff::File) -> Option<&'static str> {
if f.binary {
Some("A binary file, not shown.")
} else if f.rows.is_empty() {
Some(if f.status == Status::Renamed {
"Moved without changes."
} else {
"No changes to the content."
})
} else if f.rows.len() > DIFF_ROWS {
Some("A large diff, not shown.")
} else {
None
}
}
/// The text of the blob `oid` names, if `texts` has it.
fn text_of<'a>(texts: &'a HashMap<String, String>, oid: &Option<String>) -> Option<&'a str> {
texts.get(oid.as_deref()?).map(String::as_str)
}
/// One file of a diff without its rows, as a list of what changed shows
/// it: why its rows would not be shown, or how many they are.
fn listed(f: &diff::File) -> FileDiffView {
let status = match f.status {
Status::Added => "added",
Status::Deleted => "deleted",
Status::Renamed => "renamed",
Status::Modified => "modified",
};
let note = note(f);
let shown = note.is_none();
let hunks = f.rows.iter().filter(|r| r.kind == Kind::Hunk).count();
FileDiffView {
old: f.old.clone(),
new: f.new.clone(),
status: status.to_string(),
added: f.added,
removed: f.removed,
note: note.map(str::to_string),
ranges: None,
rows: Vec::new(),
row_count: shown.then_some(f.rows.len()),
hunk_count: shown.then_some(hunks),
}
}
/// One file of a diff, its rows highlighted in the whole of each side
/// `texts` has, by blob.
fn file_diff(f: diff::File, texts: &HashMap<String, String>) -> FileDiffView {
let note = note(&f);
let path = f.path();
let sides = highlight::Sides {
old: text_of(texts, &f.old_blob),
new: text_of(texts, &f.new_blob),
};
let ranges = note
.is_none()
.then(|| highlight::rows(path, &f.rows, sides).encode());
let listed = listed(&f);
let rows = match note {
Some(_) => Vec::new(),
None => f.rows.into_iter().map(row).collect(),
};
FileDiffView {
ranges,
rows,
row_count: None,
hunk_count: None,
..listed
}
}
fn row(r: diff::Row) -> DiffRowView {
let kind = match r.kind {
Kind::Hunk => "h",
Kind::Note => "x",
Kind::Add => "a",
Kind::Del => "d",
Kind::Context => "",
};
DiffRowView {
kind: kind.to_string(),
line: r.kind.is_line(),
old: r.old,
new: r.new,
text: r.text,
}
}
/// What `git <verb> <rev>` prints as a patch of `paths` (all, if none),
/// read into files in the order an editor's file tree lists them, with
/// their rows as `rows` says: a file without is neither highlighted nor
/// has its blobs read.
async fn files(
git: &Git,
verb: &[&str],
rev: &str,
paths: &[&str],
rows: Rows,
) -> Result<Vec<FileDiffView>> {
let mut args = vec!["-c", "core.quotePath=false"];
args.extend(verb);
args.extend([
"-M",
"--full-index",
"--no-color",
"--no-ext-diff",
rev,
"--",
]);
args.extend(paths);
let patch = git.text(&args, None).await.map_err(not_found)?;
let mut parsed = diff::parse(&patch);
diff::sort(&mut parsed);
let with = match rows {
Rows::None => return Ok(parsed.iter().map(listed).collect()),
Rows::All => vec![true; parsed.len()],
Rows::First => first(&parsed),
};
let shown: Vec<&str> = parsed
.iter()
.zip(&with)
.filter(|(f, with)| **with && note(f).is_none())
.flat_map(|(f, _)| [&f.old_blob, &f.new_blob])
.filter_map(|oid| oid.as_deref())
.collect();
let texts = texts(git, &shown).await;
// Each listed, and those with their rows drawn in its place.
let mut out: Vec<FileDiffView> = parsed.iter().map(listed).collect();
let picked = parsed.into_iter().zip(with).enumerate();
for (i, (f, _)) in picked.filter(|(_, (_, with))| *with) {
out[i] = file_diff(f, &texts);
}
Ok(out)
}
/// Which of `files` a pull's files page draws with their rows at once:
/// from the first, those whose rows would show, as many as
/// [`FIRST_FILES`] and [`FIRST_LINES`] allow, passing over any that
/// change more than [`FIRST_HELD`] lines.
fn first(files: &[diff::File]) -> Vec<bool> {
let mut with = vec![false; files.len()];
let (mut count, mut lines) = (0, 0);
for (i, f) in files.iter().enumerate() {
let changed = f.added + f.removed;
if note(f).is_some() || changed > FIRST_HELD {
continue;
}
if count == FIRST_FILES || lines + changed > FIRST_LINES {
break;
}
with[i] = true;
count += 1;
lines += changed;
}
with
}
/// The text of each blob of `oids` small enough to highlight, by name. One
/// git cannot read is left out, and its rows highlighted on their own.
async fn texts(git: &Git, oids: &[&str]) -> HashMap<String, String> {
let names: String = oids.iter().map(|o| format!("{o}\n")).collect();
let check = ["cat-file", "--batch-check=%(objectname) %(objectsize)"];
let sizes = git.text(&check, Some(names.as_bytes())).await;
let small: String = sizes
.unwrap_or_default()
.lines()
.filter_map(|l| l.split_once(' '))
.filter(|(_, size)| size.parse::<usize>().is_ok_and(|n| n <= highlight::LIMIT))
.map(|(oid, _)| format!("{oid}\n"))
.collect();
let batch = ["cat-file", "--batch=%(objectname) %(objectsize)"];
let out = git.run(&batch, Some(small.as_bytes())).await;
let out = out.unwrap_or_default();
let mut texts = HashMap::new();
let mut rest = &out[..];
while let Some((oid, text, after)) = next_object(rest) {
texts.insert(oid, String::from_utf8_lossy(text).into_owned());
rest = after;
}
texts
}
/// The first object `cat-file --batch` prints in `out`: its name, its
/// content, and what follows it.
fn next_object(out: &[u8]) -> Option<(String, &[u8], &[u8])> {
let nl = out.iter().position(|&b| b == b'\n')?;
let head = std::str::from_utf8(&out[..nl]).ok()?;
let (oid, size) = head.split_once(' ')?;
let end = nl + 1 + size.parse::<usize>().ok()?;
let content = out.get(nl + 1..end)?;
Some((
oid.to_string(),
content,
out.get(end + 1..).unwrap_or_default(),
))
}
/// The most commits a pull shows.
const PULL_COMMITS: usize = 250;
/// What a pull brings in: for a merged one, what its merge brought into the
/// base; otherwise what its branch has since it left the base, read with
/// the objects of the repository it comes from. Its files carry their
/// rows as `rows` says.
async fn pull_changes(
app: &App,
repo: &Repo,
item: &Item,
base: &str,
head: &PullHead,
rows: Rows,
) -> Result<ChangesView> {
let (git, start, tip, at) = pull_range(app, repo, item, base, head).await?;
let range = format!("{start}..{tip}");
let log = read::log(&git, &range, "", PULL_COMMITS).await;
let mut commits = shown(app, repo, log.map_err(not_found)?).await;
commits.reverse();
let files = files(&git, &["diff"], &format!("{start}...{tip}"), &[], rows).await?;
let head = item.merged.is_none().then_some(tip);
Ok(ChangesView {
repo: at,
commits,
files,
head,
})
}
/// What a pull's diff runs between: the git to read it with, the commit
/// it starts from and the one it ends at, and the repository whose pages
/// show its commits.
async fn pull_range(
app: &App,
repo: &Repo,
item: &Item,
base: &str,
head: &PullHead,
) -> Result<(Git, String, String, RepoViewBasic)> {
Ok(match &item.merged {
Some(m) => {
let before = item.merged_from.clone().unwrap_or(format!("{m}^"));
(repo.git.clone(), before, m.clone(), repo.basic(app).await)
}
None => {
let (did, name) = now_named(app, &head.repo).ok_or(Shown(
StatusCode::NOT_FOUND,
format!("no repository {}", head.repo.uri),
))?;
let from = app.located(did, &name).await?.git;
let tip = format!("refs/heads/{}", head.branch);
let tip = from.rev_parse(&tip).await.map_err(not_found)?;
let start = format!("refs/heads/{base}");
let start = repo.git.rev_parse(&start).await.map_err(not_found)?;
let mut git = repo.git.clone();
let objects = from.dir.join("objects").to_string_lossy().into_owned();
git.envs
.push(("GIT_ALTERNATE_OBJECT_DIRECTORIES".into(), objects));
let at = RepoViewBasic {
did: did.to_string(),
handle: handle(app, did).await,
name,
};
let since = forge::since_of(&git, item, &start, &tip).await;
let since = since.map_err(not_found)?;
(git, since.unwrap_or(start), tip, at)
}
})
}
/// Everything known about the repository's pulls and issues, and the
/// items they make: from the index, or for a private one from this
/// server's own of its space, which keeps them until they change.
async fn assembled(app: &App, repo: &Repo) -> Result<Arc<Assembled>> {
let index = indexed(app, repo).ok_or(Shown(
StatusCode::NOT_FOUND,
"This server keeps no index, so it knows no pulls or issues.".into(),
))?;
let record = repo.record(app).unwrap_or_default();
let made = index
.assembled(&repo.did, &repo.key, &record)
.map_err(anyhow::Error::from)
.map_err(AppError::internal)?;
Ok(made)
}
/// The index `repo`'s pulls, issues and reactions are read from: this
/// server's own of its space, which reading it synced
/// ([`App::keep`]), if it is private; else the appview's.
fn indexed<'a>(app: &'a App, repo: &Repo) -> Option<&'a Arc<Index>> {
match repo.in_space {
Some(_) => app.spaces.as_ref(),
None => app.index.as_ref(),
}
}
type ItemPath = Path<(String, String, String, String, String)>;
/// A pull or an issue, by the kind its path names, who opened it, and
/// the repository's pulls and issues.
async fn thread(
app: &App,
repo: &Repo,
kind: &str,
did: &str,
rkey: &str,
) -> Result<(Thread, String, Arc<Assembled>)> {
let collection = match kind {
"pull" => knotless::records::nsid::PULL,
"issue" => knotless::records::nsid::ISSUE,
_ => {
return Err(Shown(
StatusCode::NOT_FOUND,
format!("no such page: {kind}"),
));
}
};
let made = assembled(app, repo).await?;
let record = repo.record(app).unwrap_or_default();
let uri = repo.named(&format!("at://{did}/{collection}/{rkey}"));
let (entries, items) = (&made.entries, &made.items);
let thread = forge::thread_in(&record, &repo.did, entries, items, &uri);
let thread = thread.map_err(not_found)?;
let author = handle(app, &thread.item.author).await;
Ok((thread, author, made))
}
/// A pull's or an issue's card, its repository's owner's picture
/// looked up.
async fn item_card(app: &App, repo: &Repo, kind: &str, thread: &Thread, author: &str) -> Card {
Card::Item {
owner: repo.authority.clone(),
avatar: avatar(app, &repo.did).await,
name: repo.name.clone(),
pull: kind == "pull",
title: thread.item.title.clone(),
author: author.to_string(),
opened: Some(thread.item.created_at.clone()),
comments: thread.comments.len(),
reviews: thread.item.reviews.len(),
size: match kind {
"pull" => pull_size(app, repo, &thread.item).await,
_ => None,
},
}
}
/// What a pull changes, for its card: its commits, the files it changes
/// and the lines it adds and removes; none when its range cannot be read.
async fn pull_size(app: &App, repo: &Repo, item: &Item) -> Option<card::Size> {
let (base, head) = item.base.as_ref().zip(item.head.as_ref())?;
let range = pull_range(app, repo, item, base, head).await;
let (git, start, tip, _) = range.ok()?;
let commits = format!("{start}..{tip}");
let commits = git.text(&["rev-list", "--count", &commits], None).await;
let commits = commits.ok()?.parse().ok()?;
let changed = format!("{start}...{tip}");
let stat = git.text(&["diff", "--shortstat", &changed], None).await;
let (files, added, removed) = shortstat(&stat.ok()?);
Some(card::Size {
commits,
files,
added,
removed,
})
}
/// The files, lines added and lines removed `git diff --shortstat` counts,
/// as ` 4 files changed, 181 insertions(+), 18 deletions(-)`, each left
/// out when none.
fn shortstat(stat: &str) -> (usize, usize, usize) {
let mut counted = (0, 0, 0);
for part in stat.split(',') {
let mut words = part.split_whitespace();
let n = words.next().and_then(|n| n.parse().ok()).unwrap_or(0);
match words.next().unwrap_or_default() {
w if w.starts_with("file") => counted.0 = n,
w if w.starts_with("insertion") => counted.1 = n,
w if w.starts_with("deletion") => counted.2 = n,
_ => {}
}
}
counted
}
static CARDS: LazyLock<card::Cache> = LazyLock::new(|| card::Cache::new(512));
#[derive(Deserialize, Default)]
pub struct CardQuery {
v: Option<String>,
}
/// A card as a PNG: cached for good under the version its page names, and
/// checked again under any other, as it may have changed since. One drawn
/// without the picture it names, which could not be read, is neither kept
/// nor tagged, so that the next ask draws it again.
async fn card_png(
ctx: &knotless::env::Ctx,
card: Card,
asked: Option<String>,
headers: &HeaderMap,
) -> Response {
let svg = card.svg();
let version = card::version(&svg);
let tag = format!("\"{version}\"");
let cache = if asked.as_deref() == Some(version.as_str()) {
"public, max-age=31536000, immutable"
} else {
"no-cache"
};
// Only a whole card is tagged, so one asked for by its tag is whole.
let (mut res, whole) = if headers
.get(header::IF_NONE_MATCH)
.is_some_and(|v| v.as_bytes() == tag.as_bytes())
{
(StatusCode::NOT_MODIFIED.into_response(), true)
} else {
let (png, whole) = match looked("cards", CARDS.drawn.get(&version)) {
Some(png) => (png, true),
None => {
let wanted = card.picture().map(|url| card_picture(ctx, url));
let picture = futures::future::OptionFuture::from(wanted).await;
let whole = picture.as_ref().is_none_or(Option::is_some);
let png = CARDS.draw(&version, svg, picture.flatten(), whole).await;
(png, whole)
}
};
let mut res = axum::body::Body::from(axum::body::Bytes::from_owner(png)).into_response();
res.headers_mut()
.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png"));
(res, whole)
};
let h = res.headers_mut();
if !whole {
h.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
return res;
}
h.insert(header::CACHE_CONTROL, HeaderValue::from_static(cache));
h.insert(header::ETAG, HeaderValue::from_str(&tag).unwrap());
res
}
async fn home_card(
State(app): State<Arc<App>>,
headers: HeaderMap,
Query(q): Query<CardQuery>,
) -> Response {
card_png(&app.ctx, Card::Home, q.v, &headers).await
}
async fn repo_card(
State(app): State<Arc<App>>,
Path((authority, name)): RepoPath,
headers: HeaderMap,
Query(q): Query<CardQuery>,
) -> Result<Response> {
let repo = Repo::open(&app, &authority, &name).await?;
let open = repo.open_items(&app).await.unwrap_or_default();
let avatar = avatar(&app, &repo.did).await;
let card = repo.card(repo.record(&app).as_ref(), open, avatar);
Ok(card_png(&app.ctx, card, q.v, &headers).await)
}
async fn item_card_png(
State(app): State<Arc<App>>,
Path((authority, name, kind, did, rkey)): ItemPath,
headers: HeaderMap,
Query(q): Query<CardQuery>,
) -> Result<Response> {
let repo = Repo::open(&app, &authority, &name).await?;
let (thread, author, _) = self::thread(&app, &repo, &kind, &did, &rkey).await?;
let card = item_card(&app, &repo, &kind, &thread, &author).await;
Ok(card_png(&app.ctx, card, q.v, &headers).await)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn only_a_named_runners_logs_are_read_ahead() {
let dir = tempfile::tempdir().unwrap();
let index = Index::open(&dir.path().join("i.db")).unwrap();
let ran = serde_json::json!({"name": "r", "key": "k", "runners": [{"did": "did:ci"}],
"createdAt": "2026-01-01T00:00:00Z"});
index.put("did:a", nsid::REPO, "r", &ran).unwrap();
let repo = serde_json::json!({"uri": "at://did:a/dev.knotless.repo/r", "key": "k"});
let check = |by: &str, rkey: &str, log: bool| {
let mut run = serde_json::json!({"repo": repo, "ref": "refs/heads/main",
"commit": "c", "workflow": "test", "status": "success",
"startedAt": "2026-01-02T00:00:00Z"});
if log {
run["log"] = serde_json::json!({"$type": "blob", "ref": {"$link": "bafylog"},
"mimeType": "text/plain", "size": 3});
}
index.put(by, nsid::CHECK, rkey, &run).unwrap();
};
check("did:ci", "1", true);
check("did:ci", "2", false);
check("did:other", "3", true);
assert_eq!(
log_to_warm(&index, "did:ci", "1").as_deref(),
Some("bafylog")
);
// No log yet, a runner the repository does not name, or no check.
assert_eq!(log_to_warm(&index, "did:ci", "2"), None);
assert_eq!(log_to_warm(&index, "did:other", "3"), None);
assert_eq!(log_to_warm(&index, "did:ci", "none"), None);
}
#[test]
fn first_files_drawn_stop_at_ten_or_the_lines() {
let file = |changed: usize, binary: bool| diff::File {
binary,
added: changed,
rows: vec![diff::Row {
kind: Kind::Add,
old: None,
new: Some(1),
text: "x".into(),
}],
..diff::File::default()
};
// A binary file and one too large to draw at once are passed over.
let mut files = vec![file(1, true), file(FIRST_HELD + 1, false)];
files.extend((0..12).map(|_| file(10, false)));
let with = first(&files);
assert_eq!(with.iter().filter(|w| **w).count(), FIRST_FILES);
assert_eq!(&with[..3], [false, false, true]);
// Five of 300 lines fill the budget, and the sixth waits.
let files: Vec<diff::File> = (0..6).map(|_| file(300, false)).collect();
assert_eq!(first(&files), [true, true, true, true, true, false]);
}
#[test]
fn refusals_for_want_of_scope_say_how_to_grant_it() {
let gateway = format!(
"Conflict (409): Forbidden (403): {}",
knotless::oauth::GRANT_BEHIND
);
let shown = behind(Shown(StatusCode::BAD_REQUEST, gateway));
assert_eq!(
(shown.0, shown.1.as_str()),
(StatusCode::FORBIDDEN, knotless::oauth::GRANT_BEHIND)
);
let pds =
"InvalidToken (403): Scope does not permit create on at://did:plc:a/x".to_string();
let shown = behind(Shown(StatusCode::BAD_REQUEST, pds));
assert_eq!(
(shown.0, shown.1.as_str()),
(StatusCode::FORBIDDEN, knotless::oauth::LOGIN_BEHIND)
);
let other = behind(Shown(StatusCode::NOT_FOUND, "no such repository".into()));
assert_eq!(
(other.0, other.1.as_str()),
(StatusCode::NOT_FOUND, "no such repository")
);
}
#[test]
fn a_log_shown_from_its_tail() {
assert_eq!(log_tail(b"==> fmt\nok\n"), "==> fmt\nok\n");
assert_eq!(log_tail(b""), "");
// Cut a byte into a three-byte character: shown from the next one.
let mut long = vec![b'a'; LOG_SHOWN - 1];
long.splice(0..0, "\u{2713}".bytes());
let shown = log_tail(&long);
assert_eq!(shown.len(), LOG_SHOWN - 1);
assert!(shown.bytes().all(|b| b == b'a'));
}
#[test]
fn shortstats() {
let stat = " 4 files changed, 181 insertions(+), 18 deletions(-)\n";
assert_eq!(shortstat(stat), (4, 181, 18));
assert_eq!(shortstat(" 1 file changed, 1 deletion(-)"), (1, 0, 1));
assert_eq!(shortstat(""), (0, 0, 0));
}
#[test]
fn what_merges_brought() {
let range = |from: &str, to: &str| (from.to_string(), to.to_string());
assert_eq!(brought("m", Some("f")), range("f", "m"));
assert_eq!(brought("m", None), range("m^1", "m^2"));
}
#[test]
fn coauthor_trailers() {
let message = "Fix it\n\nCo-authored-by: Claude <noreply@anthropic.com>\nSigned-off-by: A <a@x>\nco-authored-by: B < b@x > \nCo-authored-by: nobody\n";
let named = |n: &str, e: &str| (n.to_string(), e.to_string());
assert_eq!(
coauthors(message),
[named("Claude", "noreply@anthropic.com"), named("B", "b@x")]
);
assert!(coauthors("Co-authored-by: A <a@x>").is_empty());
assert!(coauthors("Subject\n\nCo-authored-by: A <a@x").is_empty());
}
/// A server of pictures, as Bluesky's CDN serves them: the small form
/// of one, one too large, and one never sent, else none; and as a PDS
/// serves blobs, the one again; and how many times the CDN's was asked
/// for.
async fn pictures_served() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
use std::sync::atomic::{AtomicUsize, Ordering};
let asked = Arc::new(AtomicUsize::new(0));
let counted = asked.clone();
let mut pixmap = resvg::tiny_skia::Pixmap::new(8, 8).unwrap();
pixmap.fill(resvg::tiny_skia::Color::from_rgba8(200, 10, 20, 255));
let face = card::encode(&pixmap);
let blob = face.clone();
let small = move || {
counted.fetch_add(1, Ordering::SeqCst);
std::future::ready(face.clone())
};
let router = Router::new()
.route("/img/avatar_thumbnail/plain/did/face@jpeg", get(small))
.route(
"/xrpc/com.atproto.sync.getBlob",
get(move || std::future::ready(blob.clone())),
)
.route("/big", get(|| async { vec![0u8; PICTURE_MAX + 1] }))
.route("/slow", get(std::future::pending::<&'static str>));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
(base, asked)
}
#[tokio::test]
async fn pictures_on_cards() {
use std::sync::atomic::Ordering;
let (base, asked) = pictures_served().await;
let cache = tempfile::tempdir().unwrap();
let cache_dir = cache.path().to_str().unwrap().to_string();
// Every account's PDS is the server of pictures.
let vars = |name: &str| match name {
"KNOTLESS_SERVICE" => Some(base.clone()),
"KNOTLESS_CACHE_DIR" => Some(cache_dir.clone()),
_ => None,
};
let ctx = knotless::env::Ctx::new(knotless::env::Env::from_vars(vars));
// A knotless profile's picture, from the account's PDS.
let own = card_picture(&ctx, &own_avatar_path("did:plc:card", "bafyface")).await;
assert_eq!(&own.unwrap()[..4], b"\x89PNG");
// The small form of a picture on Bluesky's CDN, read once.
let url = format!("{base}/img/avatar/plain/did/face@jpeg");
let face = card_picture(&ctx, &url).await.unwrap();
assert_eq!(&face[..4], b"\x89PNG");
assert!(Arc::ptr_eq(&face, &card_picture(&ctx, &url).await.unwrap()));
assert_eq!(asked.load(Ordering::SeqCst), 1);
// None too large, none not there, none not sent in time; and none
// kept, to be asked for again.
for unread in ["big", "gone", "slow"] {
let unread = format!("{base}/{unread}");
assert!(card_picture(&ctx, &unread).await.is_none());
assert!(PICTURES.get(&unread).is_none());
}
let card = |avatar: String| Card::Repo {
owner: "alice.test".into(),
avatar: Some(avatar),
name: "party".into(),
description: None,
pulls: 0,
issues: 0,
languages: Vec::new(),
};
let named = |res: &Response, name: header::HeaderName| res.headers().get(name).cloned();
let none = HeaderMap::new();
// Drawn with its picture, a card is kept, and tagged.
let whole = card(url);
let version = card::version(&whole.svg());
let tag = HeaderValue::from_str(&format!("\"{version}\"")).unwrap();
let res = card_png(&ctx, whole.clone(), Some(version.clone()), &none).await;
let forever = HeaderValue::from_static("public, max-age=31536000, immutable");
assert_eq!(named(&res, header::CACHE_CONTROL), Some(forever));
assert_eq!(named(&res, header::ETAG), Some(tag.clone()));
let kept = CARDS.drawn.get(&version).unwrap();
let res = card_png(&ctx, whole.clone(), None, &none).await;
let body = axum::body::to_bytes(res.into_body(), usize::MAX).await;
assert_eq!(body.unwrap().as_ref(), kept.as_ref());
let mut tagged = HeaderMap::new();
tagged.insert(header::IF_NONE_MATCH, tag);
let res = card_png(&ctx, whole, Some(version), &tagged).await;
assert_eq!(res.status(), StatusCode::NOT_MODIFIED);
// Drawn without the picture it names, as that was not read, a card
// is neither kept nor tagged, though asked for by its version: the
// next ask draws it again, and may find the picture.
let failed = card(format!("{base}/gone"));
let version = card::version(&failed.svg());
let res = card_png(&ctx, failed, Some(version.clone()), &none).await;
assert_eq!(res.status(), StatusCode::OK);
let again = HeaderValue::from_static("no-cache");
assert_eq!(named(&res, header::CACHE_CONTROL), Some(again));
assert_eq!(named(&res, header::ETAG), None);
assert!(CARDS.drawn.get(&version).is_none());
}
/// A lookup for `Kept::through` that counts itself in `asked`, waits
/// for `go` if given, and finds `found`.
fn lookup(
asked: &Arc<std::sync::atomic::AtomicUsize>,
go: Option<Arc<tokio::sync::Notify>>,
found: Option<u32>,
) -> impl Future<Output = Option<u32>> + Send + 'static {
let asked = asked.clone();
async move {
asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if let Some(go) = go {
go.notified().await;
}
found
}
}
#[tokio::test]
async fn kept_values_outlive_their_server_and_their_source() {
use std::sync::atomic::{AtomicUsize, Ordering};
let dir = tempfile::tempdir().unwrap();
let dir = dir.path();
let asked = Arc::new(AtomicUsize::new(0));
let kept = |ttl| -> &'static Kept<u32> {
Box::leak(Box::new(Kept::new(
"kept.json",
ttl,
Duration::from_millis(200),
)))
};
// Put before its file is read, a value is kept, but not written.
let fresh = kept(KEPT_FOR);
fresh.put("early", 0);
assert!(!dir.join("kept.json").exists());
// Never found, it is looked up, once however many ask at once, and
// then kept, in its file too.
let go = Arc::new(tokio::sync::Notify::new());
let first = fresh.through(dir, "k", lookup(&asked, Some(go.clone()), Some(1)));
let second = fresh.through(dir, "k", lookup(&asked, None, Some(9)));
let both = futures::future::join(first, second);
let ((a, b), _) = futures::future::join(both, async { go.notify_one() }).await;
assert_eq!((a, b), (Some(1), Some(1)));
assert_eq!(asked.load(Ordering::SeqCst), 1);
let again = fresh.through(dir, "k", lookup(&asked, None, Some(9)));
assert_eq!(again.await, Some(1));
assert_eq!(asked.load(Ordering::SeqCst), 1);
// A server started after has it at once, from the file, though too
// old now to be fresh: shown while it is looked up again behind it.
let stale = kept(Duration::ZERO);
let go = Arc::new(tokio::sync::Notify::new());
let shown = stale.through(dir, "k", lookup(&asked, Some(go.clone()), Some(2)));
assert_eq!(shown.await, Some(1));
go.notify_one();
wait_for(|| stale.get("k").map(|(v, _)| v) == Some(2)).await;
// A lookup that finds nothing, as a PDS that does not answer, keeps
// what was found before.
assert_eq!(
stale.through(dir, "k", lookup(&asked, None, None)).await,
Some(2)
);
wait_for(|| stale.looking.lock().unwrap().is_empty()).await;
assert_eq!(stale.get("k").map(|(v, _)| v), Some(2));
// One never found that does not come within the wait is shown as
// none, and kept once it comes.
let go = Arc::new(tokio::sync::Notify::new());
let slow = stale.through(dir, "new", lookup(&asked, Some(go.clone()), Some(3)));
assert_eq!(slow.await, None);
go.notify_one();
wait_for(|| stale.get("new").is_some()).await;
assert_eq!(fresh.get("early"), Some((0, true)));
}
/// Waits until `done`, a few seconds at most.
async fn wait_for(done: impl Fn() -> bool) {
let waited = async {
while !done() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
};
tokio::time::timeout(Duration::from_secs(5), waited)
.await
.unwrap();
}
#[test]
fn bare_addresses_are_links() {
let md = |text: &str| markdown(text, "/B", "/R", "", None);
let a = |href: &str, shown: &str| format!(r#"<a href="{href}">{shown}</a>"#);
let x = "https://x.dev/a";
assert_eq!(
md("see https://x.dev/a"),
format!("<p>see {}</p>\n", a(x, x))
);
let html = md("At https://x.dev/a. Or (https://x.dev/a), and https://x.dev/a?!\n");
assert_eq!(html.matches(&a(x, x)).count(), 3, "{html}");
assert!(html.contains("(<a") && html.contains("</a>),"), "{html}");
let wiki = "https://en.wikipedia.org/wiki/Foo_(bar)";
let html = md("(see https://en.wikipedia.org/wiki/Foo_(bar))");
assert!(html.contains(&format!("{})", a(wiki, wiki))), "{html}");
let html = md("`https://x.dev/a` and\n\n```\nhttps://x.dev/a\n```\n");
assert!(!html.contains("<a"), "{html}");
let html =
md("[https://x.dev/a](https://x.dev/b) <https://x.dev/c> ");
assert_eq!(html.matches("<a").count(), 2, "{html}");
assert!(html.contains(&a("https://x.dev/b", x)), "{html}");
let under = "https://x.dev/a_b_c/d*e";
assert!(md("go https://x.dev/a_b_c/d*e_").contains(&a(under, under)));
let issue = "https://knotless.dev/chadtmiller.com/knotless/issue/did:plc:bcgltzqazw5tb6k2g3ttenbj/3mylzlbklrutw";
let html = md(&format!("Fixes {issue}"));
assert_eq!(html, format!("<p>Fixes {}</p>\n", a(issue, issue)));
let html = md("www.x.dev/é and HTTP://X.DEV, after é");
assert!(
html.contains(&a("http://www.x.dev/%C3%A9", "www.x.dev/é")),
"{html}"
);
assert!(html.contains(&a("HTTP://X.DEV", "HTTP://X.DEV")), "{html}");
for plain in [
"xhttps://x.dev",
"https://localhost/a",
"https://x.dev_a",
"www.",
"https://a..b",
] {
assert!(!md(plain).contains("<a"), "{plain}");
}
}
#[test]
fn readme_links() {
let at = |dest| link(dest, "/a/r/blob/main", "", None);
assert_eq!(at("https://x.dev"), "https://x.dev");
assert_eq!(at("MAILTO:a@b.c"), "MAILTO:a@b.c");
assert_eq!(at("javascript:alert(1)"), "#");
assert_eq!(at("JavaScript:alert(1)"), "#");
assert_eq!(at("data:text/html,x"), "#");
assert_eq!(at("#usage"), "#usage");
assert_eq!(at("/elsewhere"), "/a/r/blob/main/elsewhere");
assert_eq!(at("./docs/a.md"), "/a/r/blob/main/docs/a.md");
assert_eq!(at("docs/a.md?x=y:z"), "/a/r/blob/main/docs/a.md?x=y:z");
let deep = |dest| link(dest, "/a/r/blob/main", "docs/how%20to", None);
assert_eq!(deep("a.md#top"), "/a/r/blob/main/docs/how%20to/a.md#top");
assert_eq!(deep("../img/b.png"), "/a/r/blob/main/docs/img/b.png");
assert_eq!(deep("/README.md"), "/a/r/blob/main/README.md");
assert_eq!(deep("../../../../etc/x"), "/a/r/blob/main/etc/x");
// A directory is shown as one, wherever the link is from.
let is_dir = |path: &str| ["", "docs/how to", "src"].contains(&path);
let dirs = |dest| {
link(
dest,
"/a/r/blob/main",
"docs",
Some(("/a/r/tree/main", &is_dir)),
)
};
assert_eq!(dirs("how%20to"), "/a/r/tree/main/docs/how%20to");
assert_eq!(dirs("../src/#top"), "/a/r/tree/main/src#top");
assert_eq!(dirs(".."), "/a/r/tree/main/");
assert_eq!(dirs("how%20to/a.md"), "/a/r/blob/main/docs/how%20to/a.md");
assert_eq!(unescaped("a%20b%2Fc%zz%4"), "a b/c%zz%4");
let html = markdown(
"<script>x</script>\n\n[a](b.md)  <b>t</b>",
"/B",
"/R",
"",
None,
);
assert!(
html.contains("<script>") && html.contains("<b>"),
"{html}"
);
assert!(
html.contains(r#"href="/B/b.md""#) && html.contains(r#"src="/R/i.png""#),
"{html}"
);
}
/// Each tag's attributes in `html`, by name and value.
fn attributes(html: &str) -> Vec<(String, String)> {
let mut found = Vec::new();
for tag in html.split('<').skip(1) {
let tag = tag.split('>').next().unwrap_or_default();
let mut rest = tag.split_once(' ').map_or("", |(_, r)| r);
while let Some((name, after)) = rest.split_once("=\"") {
let (value, after) = after.split_once('"').unwrap_or((after, ""));
found.push((name.trim().to_ascii_lowercase(), value.to_string()));
rest = after;
}
}
found
}
#[test]
fn attached_pictures_are_shown_from_here() {
let blob_of = |cid: &str| knotless::records::BlobRef {
kind: "blob".into(),
link: knotless::records::Link { cid: cid.into() },
mime_type: "image/png".into(),
size: 3,
};
let images = [ImageAttachment::of(blob_of("bafa"))];
let attached = Attached {
repo: "at://did:o/dev.knotless.repo/r",
uri: "at://did:a/dev.knotless.comment/1",
images: &images,
};
let blob = |did: &str, cid: &str| {
format!("https://pds.example/xrpc/com.atproto.sync.getBlob?did={did}&cid={cid}")
};
let text = format!(
"     ",
blob("did:a", "bafa"),
blob("did:a", "bafb"),
blob("did:b", "bafa"),
);
let html = written(&text, "/o/r/", Some(attached), &HashMap::new());
let shown = "/xrpc/dev.knotless.getAttachment?repo=at%3A%2F%2Fdid%3Ao%2Fdev.knotless.repo%2Fr&uri=at%3A%2F%2Fdid%3Aa%2Fdev.knotless.comment%2F1&cid=bafa";
assert_eq!(html.matches(shown).count(), 1, "{html}");
// Not attached, by someone else, or not a blob: as it was.
assert!(
html.contains("cid=bafb") && html.contains("did=did:b"),
"{html}"
);
assert!(
html.contains("getRecord") && html.contains("x.example/a.png"),
"{html}"
);
let elsewhere = Attached {
uri: "nonsense",
..attached
};
let none = HashMap::new();
assert!(!written(&text, "/o/r/", Some(elsewhere), &none).contains("getAttachment"));
assert!(!html.contains("width="), "{html}");
// One whose record gives its size is drawn at it, wherever it shows.
let sized = [ImageAttachment {
width: Some(195),
height: Some(150),
..images[0].clone()
}];
let attached = Attached {
images: &sized,
..attached
};
let twice = format!("{text} ", blob("did:a", "bafa"));
let html = written(&twice, "/o/r/", Some(attached), &none);
let drawn = format!(r#"<img width="195" height="150" src="{shown}""#);
assert_eq!(html.matches(&drawn).count(), 2, "{html}");
assert_eq!(html.matches("width=").count(), 2, "{html}");
}
#[test]
fn references_link_to_the_pulls_and_issues_they_name() {
let issue = "at://did:plc:a/dev.knotless.issue/1";
let pull = "at://did:plc:b/dev.knotless.pull/2";
let named = Named::from([
(
issue.to_string(),
(
"/o/r/issue/did:plc:a/1".to_string(),
"A <crash>".to_string(),
),
),
(
pull.to_string(),
("/o/r/pull/did:plc:b/2".to_string(), "Fix it".to_string()),
),
]);
let text = format!(
"See ({issue}), {pull}. Not `{issue}`, x{issue}, [here]({pull}) or at://did:plc:c/dev.knotless.issue/3; ask @bob.test."
);
let none = HashMap::new();
let html = written_naming(&text, "/o/r", None, &none, &named);
assert!(
html.contains(r#"See (<a href="/o/r/issue/did:plc:a/1">A <crash></a>), <a href="/o/r/pull/did:plc:b/2">Fix it</a>. Not <code>"#),
"{html}"
);
assert!(html.contains(&format!("x{issue}")), "{html}");
assert!(
html.contains(r#"<a href="/o/r/pull/did:plc:b/2">here</a>"#),
"{html}"
);
assert!(
html.contains("or at://did:plc:c/dev.knotless.issue/3;"),
"{html}"
);
assert!(
html.contains(r#"<a href="/bob.test">@bob.test</a>"#),
"{html}"
);
// Without them, as it was.
assert!(written(&text, "/o/r", None, &none).contains(&format!("({issue})")));
}
#[test]
fn mentions_link_to_whom_they_name() {
let mentioned = HashMap::from([("alice.test".to_string(), "did:plc:a".to_string())]);
let text = "Ask @Alice.test or @bob.test, not me@carol.test, `@dan.test` or https://x.test/@erin.test.\n\n```\n@fay.test\n```";
let html = written(text, "/o/r", None, &mentioned);
assert!(
html.contains(r#"Ask <a href="/did:plc:a">@Alice.test</a> or <a href="/bob.test">@bob.test</a>, not me@carol.test"#),
"{html}"
);
assert_eq!(html.matches("<a ").count(), 3, "{html}");
assert!(
html.contains("<code>@dan.test</code>") && html.contains("@fay.test\n</code>"),
"{html}"
);
// A file's Markdown links none.
assert!(!markdown(text, "/B", "/R", "", None).contains("/bob.test"));
}
#[test]
fn fenced_code_is_highlighted() {
let html = markdown(
"```rust\nfn main() {}\n```\n\n```nothing\n<b>\n```\n\n```\n```\n",
"/B",
"/R",
"",
None,
);
assert!(
html.starts_with("<pre><code data-ranges=\"") && html.contains("keyword:0,2"),
"{html}"
);
assert!(html.contains("fn main() {}\n</code></pre>"), "{html}");
// Its text is escaped as ever, and a language without a grammar here
// is left as it was.
assert!(
html.contains("<pre><code class=\"language-nothing\"><b>"),
"{html}"
);
}
#[test]
fn markdown_is_sanitized() {
let html = markdown(
concat!(
"<script>alert(1)</script>\n\n",
"<a href=\"javascript:x\" onclick=\"y\">a</a> <img src=x onerror=y>\n\n",
"<iframe src=\"https://x.dev\"></iframe>\n\n",
"[b](javascript:alert(1)) [c](JAVASCRIPT:x) [d](javascript:x)\n",
"[e](vbscript:x \"\\\" onmouseover=\\\"x\") \n",
"[g](<java\tscript:x>) <javascript:x>\n",
),
"/B",
"/R",
"",
None,
);
assert!(
!html.contains("<script") && !html.contains("<iframe"),
"{html}"
);
assert!(!html.contains("<a href=\"javascript") && !html.contains("<img src=\"x"));
let found = attributes(&html);
assert!(found.iter().any(|(n, _)| n == "href"), "{html}");
for (name, value) in found {
assert!(!name.starts_with("on"), "{name} in {html}");
let value = value.to_ascii_lowercase();
assert!(
!value.contains("script:") && !value.starts_with("data:"),
"{value} in {html}"
);
}
}
#[test]
fn details_fold_a_part_away() {
let draw = |text: &str| markdown(text, "/B", "/R", "", None);
// As GitHub's: the summary beside the arrow, Markdown inside.
let html = draw(concat!(
"Short.\n\n<details>\n<summary>Claude Opus 5.5</summary>\n\n",
"## What changes\n\n- [ ] a task\n\n</details>\n",
));
assert!(
html.contains("<details>\n<summary>Claude Opus 5.5</summary>\n<h2>What changes</h2>"),
"{html}"
);
assert!(html.ends_with("</ul>\n</details>\n"), "{html}");
assert_eq!(
draw("<DETAILS Open><Summary >a <b></summary ></details>"),
"<details open><summary>a <b></summary></details>"
);
// A summary is Markdown on a line, linked as the rest is; more
// than that, or HTML in it, is text.
let html = written(
"<details><summary>**Why** `x` [l](javascript:alert(1)) at https://knotless.dev</summary>",
"/B",
None,
&HashMap::new(),
);
assert!(
html.starts_with(
"<details><summary><strong>Why</strong> <code>x</code> <a href=\"#\">l</a>"
),
"{html}"
);
assert!(html.contains("<a href=\"https://knotless.dev\">"), "{html}");
for (said, shown) in [
("# Not a heading", "# Not a heading"),
("", ""),
("a <b>b</b>", "a <b>b</b>"),
(" ", ""),
] {
assert_eq!(
draw(&format!("<details><summary>{said}</summary></details>")),
format!("<details><summary>{shown}</summary></details>")
);
}
// Raw HTML's text is never linked, and is escaped.
assert_eq!(
draw("<details>at https://knotless.dev & more"),
"<details>at https://knotless.dev & more</details>"
);
// Anything else about them is text: attributes, a tag in a
// paragraph, one unfinished.
for text in [
"<details onclick=\"x\">",
"<details open=x>",
"</details x>",
"a <details> b",
"<details\n",
"< details>",
"<detailsx>",
] {
let html = draw(text);
assert!(!html.contains("<details"), "{text}: {html}");
}
// A summary only in its fold; closers only of what is open beside
// them; what is left open closed as its block, or the text, ends.
assert_eq!(
draw("</summary>\n<summary>\n</details>"),
"</summary>\n<summary>\n</details>"
);
assert_eq!(
draw("<details><summary>a</details>"),
"<details><summary>a</summary></details>"
);
assert_eq!(
draw("<details><summary>a"),
"<details><summary>a</summary></details>"
);
// A `<` that starts no tag is text beside them.
assert_eq!(
draw("<details>1 < 2 >\n"),
"<details>1 < 2 >\n</details>"
);
let html = draw("> <details>\n\n</details>");
assert!(
html.contains("<details>\n</details></blockquote>"),
"{html}"
);
assert!(html.ends_with("</details>"), "{html}");
}
#[test]
fn failures_by_name() {
let named = [
StatusCode::BAD_REQUEST,
StatusCode::NOT_FOUND,
StatusCode::CONFLICT,
StatusCode::BAD_GATEWAY,
StatusCode::INTERNAL_SERVER_ERROR,
]
.map(failure_name);
assert_eq!(
named,
[
"InvalidRequest",
"NotFound",
"Conflict",
"UpstreamFailure",
"InternalServerError"
]
);
}
/// A failure answered as text is answered as XRPC's, and its request's
/// log line still reads why it failed.
#[tokio::test]
async fn xrpc_failures_keep_their_cause() {
let seen = Arc::new(Mutex::new(None));
let kept = seen.clone();
let failing = || async { AppError::internal(anyhow::anyhow!("disk on fire")) };
let router = Router::new()
.route("/xrpc/x.y", get(failing))
.layer(middleware::from_fn(xrpc_failures))
.layer(middleware::from_fn(move |req: Request, next: Next| {
let kept = kept.clone();
async move {
let res = next.run(req).await;
let cause = res.extensions().get::<crate::telemetry::Cause>();
*kept.lock().unwrap() = cause.map(|c| c.0.clone());
res
}
}));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let at = format!("http://{}/xrpc/x.y", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
let res = reqwest::get(at).await.unwrap();
assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body: serde_json::Value = res.json().await.unwrap();
assert_eq!(body["error"], "InternalServerError");
let cause = seen.lock().unwrap().clone().expect("the cause is kept");
assert_eq!(body["message"], cause.as_str());
}
#[tokio::test]
async fn addresses_in_full() {
let mut headers = HeaderMap::new();
headers.insert(header::HOST, HeaderValue::from_static("127.0.0.1:2590"));
let here = origin_of(&headers);
assert_eq!(here, "http://127.0.0.1:2590");
headers.insert("x-forwarded-host", HeaderValue::from_static("site.example"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(origin_of(&headers), "https://site.example");
assert_eq!(origin_of(&HeaderMap::new()), "http://");
let cdn = "https://cdn.example/a".to_string();
let full = ORIGIN.scope(here, async {
(absolute(cdn.clone()), absolute("/a".into()))
});
assert_eq!(full.await, (cdn, "http://127.0.0.1:2590/a".to_string()));
// Outside a request, a path stays one.
assert_eq!(absolute("/a".into()), "/a");
}
#[test]
fn small_things() {
assert_eq!(url_path("a b/c#d?é"), "a%20b/c%23d%3F%C3%A9");
assert_eq!(is_image("a/B.PNG"), Some("image/png"));
assert_eq!(is_image("Makefile"), None);
assert!(is_binary(&[0, 1]) && is_binary(&[0xff, 0xfe]) && !is_binary(b"text"));
assert!(is_markdown("docs/A.MD") && is_markdown("notes.markdown"));
assert!(!is_markdown("README") && !is_markdown("md") && !is_markdown("a.mdx"));
}
#[test]
fn files_of_a_diff() {
let file = |status, binary, rows: usize| diff::File {
old: "a.rs".into(),
new: "b.rs".into(),
old_blob: None,
new_blob: None,
status,
binary,
rows: (0..rows)
.map(|i| diff::Row {
kind: Kind::Add,
old: None,
new: Some(i as u32),
text: "x".into(),
})
.collect(),
added: rows,
removed: 0,
};
let none = HashMap::new();
let note = |f| file_diff(f, &none).note.unwrap_or_default();
assert_eq!(
note(file(Status::Renamed, false, 0)),
"Moved without changes."
);
assert_eq!(
note(file(Status::Modified, false, 0)),
"No changes to the content."
);
assert_eq!(
note(file(Status::Added, true, 0)),
"A binary file, not shown."
);
let large = file_diff(file(Status::Deleted, false, DIFF_ROWS + 1), &none);
assert_eq!(large.note.as_deref(), Some("A large diff, not shown."));
assert!(large.rows.is_empty() && large.ranges.is_none());
assert_eq!(large.status, "deleted");
let row = |kind, old, new, text: &str| diff::Row {
kind,
old,
new,
text: text.into(),
};
let mut mixed = file(Status::Modified, false, 0);
mixed.rows = vec![
row(Kind::Hunk, None, None, "@@ -1,2 +1,2 @@"),
row(Kind::Context, Some(1), Some(1), "a"),
row(Kind::Del, Some(2), None, "b"),
row(Kind::Note, None, None, "\\ No newline at end of file"),
row(Kind::Add, None, Some(2), "c"),
];
let shown = file_diff(mixed, &none);
let kinds: Vec<(&str, bool)> = shown
.rows
.iter()
.map(|r| (r.kind.as_str(), r.line))
.collect();
assert_eq!(
kinds,
[
("h", false),
("", true),
("d", true),
("x", false),
("a", true)
]
);
assert!(shown.note.is_none() && shown.ranges.is_some());
assert_eq!(
file_diff(file(Status::Added, false, 1), &none).status,
"added"
);
assert_eq!(
file_diff(file(Status::Renamed, false, 1), &none).status,
"renamed"
);
// `cat-file --batch`'s objects, each followed by a newline.
let out = b"a 2\nhi\nb 0\n\nc x\n";
let (oid, text, rest) = next_object(out).unwrap();
assert_eq!((oid.as_str(), text), ("a", &b"hi"[..]));
let (oid, text, rest) = next_object(rest).unwrap();
assert_eq!((oid.as_str(), text), ("b", &b""[..]));
assert_eq!(next_object(rest), None);
assert_eq!(next_object(b"a 9\nshort\n"), None);
}
}