chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
A revision's files download as one tar.gz or zip archive
crates/knotless-serve/src/web/archive.rsnew+125−0
| @@ -0,0 +1,125 @@ | ||
//! A repository's files at a revision as one download, `.tar.gz` or | ||
//! `.zip`, as `git archive` writes it from the mirror. | ||
| ||
use super::{Repo, RestPath, Result, cache_for, not_found}; | ||
use crate::{App, AppError}; | ||
use anyhow::Context; | ||
use axum::body::Body; | ||
use axum::extract::State; | ||
use axum::http::{HeaderMap, HeaderValue, header}; | ||
use axum::response::Response; | ||
use knotless::read; | ||
use std::process::Stdio; | ||
use std::sync::Arc; | ||
use tokio::process::{Child, Command}; | ||
use tokio_util::io::ReaderStream; | ||
| ||
/// How many archives are written at once: each reads the whole tree and | ||
/// compresses it, so the rest wait their turn. | ||
static ARCHIVING: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); | ||
| ||
/// How long an archive may take to write, its client reading it: past | ||
/// that, git is killed and the download ends cut short, so that clients | ||
/// that stall cannot keep the rest waiting. | ||
const WRITING: std::time::Duration = std::time::Duration::from_secs(10 * 60); | ||
| ||
/// The kinds of archive, by the ending that asks for each: `git archive`'s | ||
/// name for it, and its media type. | ||
const KINDS: [(&str, &str, &str); 2] = [ | ||
(".tar.gz", "tar.gz", "application/gzip"), | ||
(".zip", "zip", "application/zip"), | ||
]; | ||
| ||
/// `/{authority}/{repo}/archive/{rev}.tar.gz` (or `.zip`): the files at | ||
/// `rev`, a branch, tag or commit, in a folder named for the repository | ||
/// and the revision. | ||
pub async fn archive( | ||
State(app): State<Arc<App>>, | ||
headers: HeaderMap, | ||
path: RestPath, | ||
) -> Result<Response> { | ||
let viewer = crate::login::viewer(&app, &headers); | ||
let (made, touched) = crate::private::reading(viewer, None, archived(&app, path)).await; | ||
let mut res = made?; | ||
if touched { | ||
res = crate::private::unshared(res); | ||
} | ||
res.headers_mut() | ||
.append(header::VARY, HeaderValue::from_static("cookie")); | ||
Ok(res) | ||
} | ||
| ||
async fn archived( | ||
app: &App, | ||
axum::extract::Path((authority, name, rest)): RestPath, | ||
) -> Result<Response> { | ||
let repo = Repo::open(app, &authority, &name).await?; | ||
let Some((rev, format, kind)) = KINDS | ||
.iter() | ||
.find_map(|(end, format, kind)| Some((rest.strip_suffix(end)?, *format, *kind))) | ||
else { | ||
return Err(not_found(anyhow::anyhow!( | ||
"no archive {rest:?}: ask for one ending .tar.gz or .zip" | ||
))); | ||
}; | ||
let rev = read::rev(Some(rev)).map_err(not_found)?; | ||
let commit = repo.objects()?.resolve(&rev).map_err(not_found)?; | ||
let folder = filename(&format!("{}-{rev}", repo.name)); | ||
let ending = &rest[rev.len()..]; | ||
let permit = ARCHIVING.acquire().await.expect("never closed"); | ||
let mut child = spawn(&repo.git.dir, format, &folder, &commit).map_err(AppError::internal)?; | ||
let stdout = child.stdout.take().expect("stdout is piped"); | ||
// The permit is held until git is done writing, or killed at the | ||
// deadline; kill_on_drop covers a client that hangs up first. | ||
tokio::spawn(async move { | ||
let waited = tokio::time::timeout(WRITING, child.wait()).await; | ||
drop((waited, child, permit)); | ||
}); | ||
let disposition = format!("attachment; filename=\"{folder}{ending}\""); | ||
Ok(Response::builder() | ||
.header(header::CONTENT_TYPE, kind) | ||
.header(header::CONTENT_DISPOSITION, disposition) | ||
.header(header::CACHE_CONTROL, cache_for(&rev)) | ||
.header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") | ||
.body(Body::from_stream(ReaderStream::new(stdout))) | ||
.expect("the headers are valid")) | ||
} | ||
| ||
/// `git archive` writing `commit` from `dir` as `format`, its files in | ||
/// `folder`. | ||
fn spawn(dir: &std::path::Path, format: &str, folder: &str, commit: &str) -> anyhow::Result<Child> { | ||
let child = Command::new("git") | ||
.arg("--git-dir") | ||
.arg(dir) | ||
.args(["archive", &format!("--format={format}")]) | ||
.arg(format!("--prefix={folder}/")) | ||
.arg(commit) | ||
.stdin(Stdio::null()) | ||
.stdout(Stdio::piped()) | ||
.stderr(Stdio::null()) | ||
.kill_on_drop(true) | ||
.spawn(); | ||
child.context("running git archive") | ||
} | ||
| ||
/// `name` as a file's name: what is not a letter, a digit, `.`, `_` or | ||
/// `-` becomes `-`, so a branch's `/` and anything a header would not | ||
/// carry are left out. | ||
fn filename(name: &str) -> String { | ||
name.chars() | ||
.map(|c| match c { | ||
'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '_' | '-' => c, | ||
_ => '-', | ||
}) | ||
.collect() | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
#[test] | ||
fn filenames_keep_only_what_a_header_carries() { | ||
assert_eq!(super::filename("tally-web/mermaid"), "tally-web-mermaid"); | ||
assert_eq!(super::filename("a-v1.0_b"), "a-v1.0_b"); | ||
assert_eq!(super::filename("x-\"é\""), "x----"); | ||
} | ||
} | ||
crates/knotless-serve/src/web.rs+4−1
| @@ -45,6 +45,7 @@ use std::time::{Duration, SystemTime}; | ||
mod account_data; | ||
mod account_settings; | ||
mod actor; | ||
mod archive; | ||
mod badges; | ||
mod controls; | ||
mod delegation; | ||
| @@ -242,12 +243,14 @@ pub fn routes() -> Router<Arc<App>> { | ||
) | ||
.layer(tower_http::compression::CompressionLayer::new()); | ||
// A release's files are sent as they are kept: most are archives, | ||
// compressed already, and their length is known. | ||
// compressed already, and their length is known. A revision's archive | ||
// is compressed as it is written. | ||
Router::new() | ||
.route( | ||
"/{authority}/{repo}/releases/download/{*rest}", | ||
get(releases::download), | ||
) | ||
.route("/{authority}/{repo}/archive/{*rest}", get(archive::archive)) | ||
.merge(compressed) | ||
.layer(middleware::from_fn(renamed)) | ||
} | ||
crates/knotless-serve/tests/gateway.rs+10−0
| @@ -2783,6 +2783,16 @@ fn private_repositories_on_the_pages() { | ||
assert_eq!((file.0, file.1.as_str()), (200, "private, no-store")); | ||
assert!(file.2.contains("cookie"), "{}", file.2); | ||
assert_eq!(file.3, "# Notes\n"); | ||
// Its archive, as its files: to members alone, kept by no cache. | ||
let archive = format!( | ||
"{}/{}/hidden/archive/main.zip", | ||
server.base, owner.account.handle | ||
); | ||
let zip = looked(&archive, theirs); | ||
assert_eq!((zip.0, zip.1.as_str()), (200, "private, no-store")); | ||
assert!(zip.2.contains("cookie"), "{}", zip.2); | ||
assert!(zip.3.contains("hidden-main/notes.md")); | ||
assert_eq!(looked(&archive, None).0, 404); | ||
// A public repository, to the same member, is theirs alone to cache, | ||
// but may be kept. | ||
let open = owner.url("open"); | ||
crates/knotless-serve/tests/web.rs+43−0
| @@ -573,6 +573,49 @@ fn browsing_a_repository() { | ||
assert_eq!(cached(tip.trim()), "public, max-age=31536000, immutable"); | ||
assert_eq!(cached("main"), "no-cache"); | ||
| ||
// The files at a revision as one download, in a folder named for the | ||
// repository and the revision. | ||
let archive = |rest: &str| site.get(&format!("{repo}/archive/{rest}")); | ||
let header = |res: &Response, name: &str| res.headers()[name].to_str().unwrap().to_string(); | ||
let tarred = archive("main.tar.gz"); | ||
assert_eq!(tarred.status(), 200); | ||
assert_eq!(header(&tarred, "content-type"), "application/gzip"); | ||
assert_eq!(header(&tarred, "cache-control"), "no-cache"); | ||
assert_eq!( | ||
header(&tarred, "content-disposition"), | ||
"attachment; filename=\"browsed-main.tar.gz\"" | ||
); | ||
let mut tar = Vec::new(); | ||
std::io::Read::read_to_end( | ||
&mut flate2::read::GzDecoder::new(&tarred.bytes().unwrap()[..]), | ||
&mut tar, | ||
) | ||
.unwrap(); | ||
let holds = |bytes: &[u8], name: &str| bytes.windows(name.len()).any(|w| w == name.as_bytes()); | ||
assert!(holds(&tar, "browsed-main/src/main.rs")); | ||
assert!(holds(&tar, "println!(\"hi\");")); | ||
// A branch's `/` is left out of the names; a tag's archive, as a | ||
// commit's, never changes. | ||
let branch = archive("feature/x.zip"); | ||
assert_eq!( | ||
header(&branch, "content-disposition"), | ||
"attachment; filename=\"browsed-feature-x.zip\"" | ||
); | ||
let zip = branch.bytes().unwrap(); | ||
assert!(zip.starts_with(b"PK\x03\x04")); | ||
assert!(holds(&zip, "browsed-feature-x/src/main.rs")); | ||
let pinned = archive(&format!("{}.zip", tip.trim())); | ||
assert_eq!(header(&pinned, "content-type"), "application/zip"); | ||
assert_eq!( | ||
header(&pinned, "cache-control"), | ||
"public, max-age=31536000, immutable" | ||
); | ||
assert_eq!(archive("v1.tar.gz").status(), 200); | ||
// Only the two kinds, of a revision naming a commit. | ||
for missing in ["main.rar", "main", "nope.zip", "tree.zip", "-x.zip"] { | ||
assert_eq!(archive(missing).status(), 404, "{missing}"); | ||
} | ||
| ||
let log = site.data(&format!("{repo}/commits/main")); | ||
assert_eq!(log["commits"].as_array().unwrap().len(), 50); | ||
assert_eq!(log["cursor"], "50"); | ||
docs/design/access.md+1−1
| @@ -266,7 +266,7 @@ sequenceDiagram | ||
| ||
Only queries asked by someone read as them: the queries carrying the | ||
session cookie, a service token or a delegation token, and the raw file | ||
route. Cards, clones, | ||
and archive routes. Cards, clones, | ||
the object API, and refreshes the server starts on its own read as no one, | ||
so they only ever find public repositories. Anything private never reaches | ||
the appview index, so feeds, search, stars and activity never show it. | ||
docs/design/serve.md+8−0
| @@ -257,6 +257,14 @@ Details: | ||
memory by commit and path, the last 256 to 512 of them. The page draws it | ||
beside the highlighted text: each run of lines with its commit, and a link | ||
to blame the file at the commit's parent, as it was before. | ||
- A revision's files download as one archive, `.tar.gz` or `.zip`, from the | ||
Code menu at the revision shown and beside each tag. `git archive` writes | ||
it from the mirror, streamed as it goes, in a folder named for the | ||
repository and the revision. Each archive reads the whole tree and | ||
compresses it, so at most four are written at once, the rest waiting their | ||
turn. One still writing after ten minutes, as for a client that stalls, | ||
is cut short, so that slow clients cannot keep the rest waiting. One at | ||
a commit's hash never changes, so it is cached for good. | ||
| ||
## The appview index | ||
| ||
docs/reference/api.md+5−4
| @@ -38,7 +38,7 @@ A repository no one may read publicly is read as the asker, with the | ||
session they logged in to the pages with: a member of it reads it as a | ||
public one, and anyone else, asker or not, gets the 404 `NotFound` a | ||
repository never made gets ([access.md](../design/access.md#browsing)). | ||
`GET /<owner>/<name>/raw/<rev>/<path>` reads as the cookie's account too, | ||
`GET /<owner>/<name>/raw/<rev>/<path>` and `/archive/<rev>.zip` read as the cookie's account too, | ||
and varies by cookie; the cards, git and `/api/` routes read as no one. | ||
| ||
A query may instead prove its asker a member with a delegation token, | ||
| @@ -744,11 +744,12 @@ they are, never as a page. Served beside them, not as queries: | ||
| Route | | | ||
| --- | --- | | ||
| `/<authority>/<repo>/raw/<rev>/<path>` | a file's bytes | | ||
| `/<authority>/<repo>/archive/<rev>.tar.gz`, `.zip` | the files at a branch, tag or commit as one archive, as `git archive` writes it, in a folder `<repo>-<rev>` (each character but a letter, a digit, `.`, `_` and `-` made `-`), `Content-Disposition: attachment` with that name and the ending; `Cache-Control` immutable at a commit's full hash, else `no-cache`; not compressed again. Another ending, or a revision naming no commit, is 404 | | ||
| `/<authority>/<repo>/releases/download/<package>/<version>/<file>` | a file of a release the index has, `application/octet-stream`, `Content-Disposition: attachment` with its name, `Content-Length`, its SHA-256 as `ETag` (a matching `If-None-Match` is 304), read once from the owner's PDS, checked against its record and kept, as `/releases/` keeps its own ([packages.md](../design/packages.md#on-the-pages)); not compressed. One the index does not have, taken down, or of a private repository is 404, and one that cannot be read 502 | | ||
| `/card.png`, `/<authority>/<repo>/card.png`, `/<authority>/<repo>/<pull or issue>/<did>/<rkey>/card.png` | social cards, 1200×630 PNG | | ||
| ||
These routes but the release files are compressed (gzip or br). Answers | ||
are `Cache-Control: no-cache`, even at a commit hash, and a failure | ||
`no-store`. A raw file, release file or | ||
These routes but the release files and archives are compressed (gzip or br). Answers | ||
are `Cache-Control: no-cache`, but a raw file or archive at a commit's | ||
full hash is immutable, and a failure `no-store`. A raw file, archive, release file or | ||
card asked for by an old name answers 307 to the same path under the new | ||
name, and the new owner if it moved, keeping the query. | ||
docs/development.md+1−1
| @@ -71,7 +71,7 @@ knotless-serve. | ||
`http://127.0.0.1:2590`). | ||
- The app's `handleFetch` sends its own-origin `/xrpc` fetches straight to `KNOTLESS_SERVE`, since knotless-serve can share the | ||
site's origin, which Kit would answer itself. | ||
- `vite dev` proxies `/xrpc` and `/<authority>/<repo>/raw/` to | ||
- `vite dev` proxies `/xrpc`, `/<authority>/<repo>/raw/` and `/archive/` to | ||
`KNOTLESS_SERVE`. | ||
- An element is marked `data-drawing` until its diagrams are drawn, and | ||
the root gets `data-ready` once mounted. Screenshots wait for both. | ||
web/src/lib/molecules/CloneMenu.svelte+6−2
| @@ -1,8 +1,9 @@ | ||
<script lang="ts"> | ||
import CopyButton from '#lib/atoms/CopyButton.svelte'; | ||
import Icon from '#lib/atoms/Icon.svelte'; | ||
/** The Code button, opening to how to clone the repository `name` that `authority` owns. */ | ||
let { authority, name }: { authority: string; name: string } = $props(); | ||
import { archiveHref } from '#lib/format.ts'; | ||
/** The Code button, opening to how to clone the repository `name` that `authority` owns, and to download its files at `rev`. */ | ||
let { authority, name, rev }: { authority: string; name: string; rev: string } = $props(); | ||
const clone = $derived(`git clone knotless://${authority}/${name}`); | ||
let open = $state(false); | ||
function outside(e: MouseEvent) { | ||
| @@ -18,5 +19,8 @@ | ||
<div class="menu-head">Clone with git</div> | ||
<div class="clone"><code>{clone}</code><CopyButton text={clone} /></div> | ||
<p>Needs the knotless git helper. <a href="/start">Set it up</a></p> | ||
<div class="menu-head">Download</div> | ||
<a href={archiveHref(authority, name, rev, 'zip')} download><Icon name="download" />Download zip</a> | ||
<a href={archiveHref(authority, name, rev, 'tar.gz')} download><Icon name="download" />Download tar.gz</a> | ||
</div> | ||
</details> | ||
web/src/lib/organisms/RefList.svelte+4−4
| @@ -1,12 +1,12 @@ | ||
<script lang="ts"> | ||
import type { CheckView, CommitView } from '#lib/lexicons/dev/knotless/defs.ts'; | ||
import { repoHref, urlPath } from '#lib/format.ts'; | ||
import { archiveHref, repoHref, urlPath } from '#lib/format.ts'; | ||
import CheckMark from '#lib/molecules/CheckMark.svelte'; | ||
import Committed from '#lib/molecules/Committed.svelte'; | ||
import Sha from '#lib/atoms/Sha.svelte'; | ||
import Empty from '#lib/molecules/Empty.svelte'; | ||
/** A repository's branches or tags, under `heading`: each linked to its code, the default branch `head` marked, with the commit it names, who committed it and when, and how its checks stand; `empty` said when there are none. */ | ||
let { authority, name, heading, refs, head, empty }: { authority: string; name: string; heading: string; refs: { name: string; commit?: CommitView; checks?: CheckView[] }[]; head?: string; empty: string } = $props(); | ||
/** A repository's branches or tags, under `heading`: each linked to its code, the default branch `head` marked, with the commit it names, who committed it and when, and how its checks stand, and with `archives` its files to download; `empty` said when there are none. */ | ||
let { authority, name, heading, refs, head, empty, archives = false }: { authority: string; name: string; heading: string; refs: { name: string; commit?: CommitView; checks?: CheckView[] }[]; head?: string; empty: string; archives?: boolean } = $props(); | ||
const href = (tail: string) => repoHref(authority, name, tail); | ||
</script> | ||
| ||
| @@ -18,7 +18,7 @@ | ||
<li> | ||
<div class="grow"> | ||
<div class="ref-name"><a class="title clip" href={href(`/tree/${urlPath(r.name)}`)}>{r.name}</a>{#if r.name === head}<span class="count">Default</span>{/if}</div> | ||
{#if r.commit}<div class="meta"><Committed commit={r.commit} /><CheckMark checks={r.checks ?? []} /></div>{/if} | ||
{#if r.commit}<div class="meta"><Committed commit={r.commit} /><CheckMark checks={r.checks ?? []} />{#if archives}<a class="archive" href={archiveHref(authority, name, r.name, 'zip')} download>zip</a><a class="archive" href={archiveHref(authority, name, r.name, 'tar.gz')} download>tar.gz</a>{/if}</div>{/if} | ||
</div> | ||
{#if r.commit}<Sha sha={r.commit.sha} href={href(`/commit/${r.commit.sha}`)} />{/if} | ||
</li> | ||