//! A Cargo registry of the crates a repository publishes as releases //! (docs/design/packages.md), one registry for each repository: //! //! ```text //! index = "sparse+https://knotless.dev/cargo///" //! cargo login --registry # : //! cargo publish --registry //! cargo yank --registry --version 1.0.0 //! ``` //! //! Its address holds no name that can change, as Cargo keeps it in //! `Cargo.lock`, and in the index lines of crates of other repositories //! that depend on these. What it answers is what the index has, so a //! takedown holds, and a publish or a yank puts its record there too, as //! `cargo publish` waits to find its crate. //! //! A write is the owner's alone, by the app password its token holds, which //! logs in to their PDS for that request and is kept nowhere. // A handler's error is the answer it sends, made once per request. #![allow(clippy::result_large_err)] use super::releases::sending; use super::*; use crate::releases::Records; use axum::body::Body; use flate2::read::GzDecoder; use knotless::records::{CargoCrate, RELEASE_MIME, ReleaseFile, ReleasePart, ReleaseRecord, nsid}; use knotless::release::{self, Made, Target}; use knotless::remote::{self, Remote}; use knotless::{keys, push}; use serde_json::{Value, json}; use std::collections::{BTreeMap, HashSet}; use std::io::Read; /// The largest publish taken, its crate and what is said of it. const MOST_PUBLISH: usize = 64 << 20; /// The most of a crate read, unpacked, looking for its `Cargo.toml`. const MOST_UNPACKED: u64 = 1 << 30; /// The longest index line a release holds, as its lexicon has it. const MOST_LINE: usize = 100_000; /// The most of a crate's `Cargo.toml` read: what `cargo package` writes is /// a few KiB. const MOST_MANIFEST: u64 = 1 << 20; /// The most characters of a crate's description a release holds. const MOST_DESCRIPTION: usize = 1000; type Answer = std::result::Result; fn refusal(status: StatusCode, detail: impl std::fmt::Display) -> Response { let body = json!({"errors": [{"detail": detail.to_string()}]}); (status, axum::Json(body)).into_response() } fn refused(detail: impl std::fmt::Display) -> Response { refusal(StatusCode::BAD_REQUEST, detail) } fn upstream(detail: anyhow::Error) -> Response { refusal(StatusCode::BAD_GATEWAY, format!("{detail:#}")) } /// A crate as `cargo publish` describes it. #[derive(Deserialize, Debug)] struct Published { name: String, vers: String, #[serde(default)] deps: Vec, #[serde(default)] features: BTreeMap>, links: Option, rust_version: Option, description: Option, } /// A dependency as `cargo publish` describes it. #[derive(Deserialize, Debug)] struct Dep { name: String, version_req: String, #[serde(default)] features: Vec, optional: Option, default_features: Option, target: Option, kind: Option, registry: Option, explicit_name_in_toml: Option, } /// The line of Cargo's index for `crate_`, of SHA-256 `cksum`. Features a /// Cargo before namespaced features cannot read go in `features2`, as the /// index asks. Whether it is yanked is said as it is read. fn index_line(crate_: &Published, cksum: &str) -> String { let namespaced = |f: &String| f.starts_with("dep:") || f.contains("?/"); let (second, first): (BTreeMap<_, _>, BTreeMap<_, _>) = crate_ .features .iter() .partition(|(_, wants)| wants.iter().any(namespaced)); let deps = crate_.deps.iter().map(|d| { let renamed = d.explicit_name_in_toml.as_ref(); json!({ "name": renamed.unwrap_or(&d.name), "req": d.version_req, "features": d.features, "optional": d.optional.unwrap_or(false), "default_features": d.default_features.unwrap_or(true), "target": d.target, "kind": d.kind.as_deref().unwrap_or("normal"), "registry": d.registry, "package": renamed.map(|_| &d.name), }) }); let mut line = json!({ "name": crate_.name, "vers": crate_.vers, "deps": deps.collect::>(), "cksum": cksum, "features": first, "links": crate_.links, "rust_version": crate_.rust_version, "v": 1, }); if !second.is_empty() { line["v"] = json!(2); line["features2"] = json!(second); } line.to_string() } /// The JSON and the crate of a publish's body: each its length, four bytes /// little-endian, and then itself. fn framed(body: &[u8]) -> Option<(&[u8], &[u8])> { let (length, rest) = body.split_first_chunk::<4>()?; let (said, rest) = rest.split_at_checked(u32::from_le_bytes(*length) as usize)?; let (length, rest) = rest.split_first_chunk::<4>()?; (rest.len() == u32::from_le_bytes(*length) as usize).then_some((said, rest)) } /// The text of the file at `/Cargo.toml` in a gzipped tar, read /// as far as it, and so no further than [`MOST_UNPACKED`]. fn manifest_of(crate_: &[u8]) -> Option { let mut tar = GzDecoder::new(crate_).take(MOST_UNPACKED); let mut header = [0u8; 512]; loop { tar.read_exact(&mut header).ok()?; let field = |from: usize, to: usize| { let bytes = &header[from..to]; let end = bytes.iter().position(|b| *b == 0).unwrap_or(bytes.len()); String::from_utf8_lossy(&bytes[..end]).trim().to_string() }; let (name, size) = (field(0, 100), field(124, 136)); let size = u64::from_str_radix(&size, 8).ok()?; if name.split('/').count() == 2 && name.ends_with("/Cargo.toml") { if size > MOST_MANIFEST { return None; } let mut text = String::new(); tar.by_ref().take(size).read_to_string(&mut text).ok()?; return Some(text); } let padded = size.div_ceil(512) * 512; std::io::copy(&mut tar.by_ref().take(padded), &mut std::io::sink()).ok()?; } } /// The name and version `manifest`'s `[package]` says. fn package_of(manifest: &str) -> Option<(&str, &str)> { let (_, package) = manifest.split_once("[package]\n")?; let package = package.split("\n[").next()?; let said = |key: &str| { package.lines().find_map(|line| { let (k, v) = line.split_once('=')?; (k.trim() == key).then(|| v.trim().trim_matches('"')) }) }; Some((said("name")?, said("version")?)) } /// A version with no build metadata, as Cargo's index tells versions apart. fn base(version: &str) -> &str { version.split('+').next().unwrap_or(version) } /// `name`'s releases as crates, the oldest first, each version once however /// its build metadata differs: a record can always be made that a publish /// here would refuse, and the first is what was offered first. fn crates<'a>(releases: &'a [(String, ReleaseRecord)], name: &str) -> Vec<&'a ReleaseRecord> { let mut seen = HashSet::new(); let oldest = releases.iter().rev().map(|(_, r)| r); oldest .filter(|r| r.package == name && r.cargo.is_some()) .filter(|r| seen.insert(base(&r.version))) .collect() } /// `r`'s line of the index, saying whether it is yanked: none if the line /// is not of `r`. fn line_of(r: &ReleaseRecord) -> Option { let mut line: Value = serde_json::from_str(&r.cargo.as_ref()?.index).ok()?; let fields = line.as_object_mut()?; let name = fields.get("name")?.as_str()?; let version = fields.get("vers")?.as_str()?; if name != r.package || version != r.version { return None; } fields.insert("yanked".into(), json!(r.yanked)); Some(line.to_string()) } /// Where the index keeps `name`'s file. fn index_path(name: &str) -> String { match name.len() { 1 => format!("1/{name}"), 2 => format!("2/{name}"), 3 => format!("3/{}/{name}", &name[..1]), _ => format!("{}/{}/{name}", &name[..2], &name[2..4]), } } /// `did`'s repository `key` and its releases, by what the index has: else /// that there is none, its or its owner's taken down among them. fn opened(app: &App, did: &str, key: &str) -> std::result::Result { let none = || refusal(StatusCode::NOT_FOUND, "no such repository"); let index = app.index.as_ref().ok_or_else(none)?; let listed = index .listed(did, key) .map_err(anyhow::Error::from) .map_err(upstream)?; let listed = listed.ok_or_else(none)?; let url = RepoUrl { authority: did.into(), name: listed.name, }; let releases = index .releases(did, key) .map_err(anyhow::Error::from) .map_err(upstream)?; Ok(Opened { url, releases }) } struct Opened { url: RepoUrl, releases: Vec<(String, ReleaseRecord)>, } /// The owner `did`'s session, by the app password `headers` hold. async fn logged_in( app: &App, headers: &HeaderMap, did: &str, ) -> std::result::Result { let denied = |says: String| refusal(StatusCode::FORBIDDEN, says); let token = headers .get(header::AUTHORIZATION) .and_then(|v| v.to_str().ok()); let token = token.and_then(|t| t.split_once(':')); let Some((user, password)) = token else { return Err(denied( "log in: cargo login, with :".into(), )); }; let session = session::create(&app.ctx, user, password).await; let session = session.map_err(|e| denied(format!("{user} could not log in: {e:#}")))?; if session.did != did { return Err(denied(format!("only {did} writes to this registry"))); } Ok(session) } /// `config.json`: where crates are read from and published to. pub(super) async fn config( State(app): State>, headers: HeaderMap, Path((did, key)): Path<(String, String)>, ) -> Answer { opened(&app, &did, &key)?; let here = format!("{}/cargo/{did}/{key}", origin_of(&headers)); let body = json!({"dl": format!("{here}/dl/{{crate}}/{{version}}/download"), "api": here}); Ok(axum::Json(body).into_response()) } /// `me`, where `cargo login` sends its user for a token: what one is. pub(super) async fn me( State(app): State>, Path((did, key)): Path<(String, String)>, ) -> Answer { opened(&app, &did, &key)?; let body = format!( "The token for this registry is a handle and an app password, \ written :, such as alice.example.com:abcd-efgh-ijkl-mnop.\n\n\ Only {did}, the repository's owner, publishes and yanks here. \ Make an app password in your PDS's settings (in Bluesky, \ Settings, then Privacy and security, then App passwords), \ then give the token to cargo login.\n\n\ Reading the registry needs no token.\n" ); let kind = (header::CONTENT_TYPE, "text/plain; charset=utf-8"); Ok(([kind], body).into_response()) } /// A crate's file of the index, asked again each time, at the cost of a 304. pub(super) async fn index( State(app): State>, headers: HeaderMap, Path((did, key, path)): Path<(String, String, String)>, ) -> Answer { let opened = opened(&app, &did, &key)?; let name = path.rsplit('/').next().unwrap_or_default(); let found = crates(&opened.releases, name); let lines = found.into_iter().filter_map(line_of); let mut body = lines.collect::>().join("\n"); if body.is_empty() || index_path(name) != path { return Err(refusal(StatusCode::NOT_FOUND, format!("no crate {name}"))); } body.push('\n'); let etag = format!("\"{}\"", release::sha256(body.as_bytes())); let cache = (header::CACHE_CONTROL, "no-cache".to_string()); let tag = (header::ETAG, etag.clone()); let seen = headers.get(header::IF_NONE_MATCH); if seen.is_some_and(|v| v.as_bytes() == etag.as_bytes()) { return Ok((StatusCode::NOT_MODIFIED, [cache, tag]).into_response()); } Ok(([cache, tag], body).into_response()) } /// A crate, read from its owner's PDS once and kept, as a release's files are. pub(super) async fn download( State(app): State>, Path((did, key, name, version)): Path<(String, String, String, String)>, ) -> Answer { let opened = opened(&app, &did, &key)?; let found = crates(&opened.releases, &name); let found = found.into_iter().find(|r| r.version == version); let file = format!("{name}-{version}.crate"); let found = found.and_then(|r| r.files.iter().find(|f| f.name == file)); let Some(f) = found else { return Err(refusal(StatusCode::NOT_FOUND, format!("no {file}"))); }; let records = Records::new(app.ctx.clone(), opened.url, name); let sent = records .send(f, sending(f, format!("\"{}\"", f.sha256))) .await; sent.map_err(upstream) } /// `cargo publish`: the crate uploaded to its owner's PDS, its release made. pub(super) async fn publish( State(app): State>, headers: HeaderMap, Path((did, key)): Path<(String, String)>, body: Body, ) -> Answer { let session = logged_in(&app, &headers, &did).await?; let opened = opened(&app, &did, &key)?; let large = format!("a publish is at most {MOST_PUBLISH} bytes"); let body = axum::body::to_bytes(body, MOST_PUBLISH).await; let body = body.map_err(|_| refusal(StatusCode::PAYLOAD_TOO_LARGE, large))?; let Some((said, crate_)) = framed(&body) else { return Err(refused("this is no publish: lengths and sizes differ")); }; let said: Published = serde_json::from_slice(said).map_err(refused)?; let (name, version) = (said.name.as_str(), said.vers.as_str()); let file = format!("{name}-{version}.crate"); release::check_names(name, version, &[&file]).map_err(refused)?; let manifest = manifest_of(crate_) .ok_or_else(|| refused("the crate holds no Cargo.toml of at most 1 MiB"))?; if package_of(&manifest) != Some((name, version)) { let says = format!("what was said is {name} {version}, which its Cargo.toml is not"); return Err(refused(says)); } let cksum = release::sha256(crate_); let line = index_line(&said, &cksum); if line.len() > MOST_LINE { return Err(refused(format!( "{name} says more than {MOST_LINE} bytes of its index line" ))); } let target = Target::open(&app.ctx, &opened.url, name, version).await; let target = target.map_err(refused)?; let auth = session.auth(); let blobs = push::upload(&app.ctx, &target.remote, &auth, crate_, RELEASE_MIME).await; let parts = blobs.map_err(upstream)?; let files = vec![ReleaseFile { name: file, size: crate_.len() as u64, sha256: cksum, parts: parts.into_iter().map(|blob| ReleasePart { blob }).collect(), }]; let description = said.description.as_ref(); let cargo = CargoCrate { index: line, description: description.map(|d| d.chars().take(MOST_DESCRIPTION).collect()), }; let made = Made { cargo: Some(cargo), ..Made::default() }; target.create(&auth, files, made).await.map_err(upstream)?; // It is published now, so what follows only hastens what the feed // brings: a failure to read it back or index it is not the publish's, // and answering one would have a retry told it is published already. let rkey = keys::release_rkey(&key, name, version); let got = target .remote .get_record::(nsid::RELEASE, &rkey) .await; let index = app.index.as_ref().expect("opened with an index"); let got = got.ok().flatten(); let _ = got.map(|record| index.put(&did, nsid::RELEASE, &rkey, &record.value)); // Build metadata makes another record's key, so only now can a version // be seen to have been published already under another's. Should the // index not say, both stay, and it answers the oldest. let all = index.releases(&did, &key).unwrap_or_default(); let found = crates(&all, name); let first = found.iter().find(|r| base(&r.version) == base(version)); if let Some(first) = first.filter(|r| r.version != version) { let says = format!("{name} {} is published already", first.version); let gone = target .remote .apply(vec![remote::delete(nsid::RELEASE, &rkey)], &auth); gone.await.map_err(upstream)?; index .delete(&did, nsid::RELEASE, &rkey) .map_err(anyhow::Error::from) .map_err(upstream)?; return Err(refused(says)); } let warnings = json!({"invalid_categories": [], "invalid_badges": [], "other": []}); Ok(axum::Json(json!({"warnings": warnings})).into_response()) } async fn yanking( app: &App, headers: &HeaderMap, (did, key, name, version): (String, String, String, String), undo: bool, ) -> Answer { let session = logged_in(app, headers, &did).await?; let opened = opened(app, &did, &key)?; let remote = Remote::open(&app.ctx, &opened.url) .await .map_err(upstream)?; let yanked = release::yank_as(&remote, &session.auth(), &name, &version, undo).await; yanked.map_err(refused)?; let rkey = keys::release_rkey(&key, &name, &version); let index = app.index.as_ref().expect("opened with an index"); let said = if undo { index.delete(&did, nsid::RELEASE_YANK, &rkey) } else { let record = json!({"repo": key, "createdAt": knotless::records::now()}); index .put(&did, nsid::RELEASE_YANK, &rkey, &record) .map(|_| ()) }; said.map_err(anyhow::Error::from).map_err(upstream)?; Ok(axum::Json(json!({"ok": true})).into_response()) } /// `cargo yank`. pub(super) async fn yank( State(app): State>, headers: HeaderMap, Path(asked): Path<(String, String, String, String)>, ) -> Answer { yanking(&app, &headers, asked, false).await } /// `cargo yank --undo`. pub(super) async fn unyank( State(app): State>, headers: HeaderMap, Path(asked): Path<(String, String, String, String)>, ) -> Answer { yanking(&app, &headers, asked, true).await } #[derive(Deserialize)] pub(super) struct SearchParams { q: Option, per_page: Option, } /// `cargo search`: the crates whose names hold what was asked, by name, /// each at its latest. pub(super) async fn search( State(app): State>, Path((did, key)): Path<(String, String)>, Query(p): Query, ) -> Answer { let opened = opened(&app, &did, &key)?; let records: Vec = opened.releases.into_iter().map(|(_, r)| r).collect(); let q = p.q.unwrap_or_default(); let names: std::collections::BTreeSet<&str> = records .iter() .filter(|r| r.cargo.is_some() && r.package.contains(&q)) .map(|r| r.package.as_str()) .collect(); let found = names .into_iter() .filter_map(|name| release::latest(&records, name)); let found: Vec = found .map(|r| { let said = r.cargo.as_ref().and_then(|c| c.description.as_deref()); json!({"name": r.package, "max_version": r.version, "description": said}) }) .collect(); let total = found.len(); let shown: Vec = found .into_iter() .take(p.per_page.unwrap_or(10).clamp(1, 100)) .collect(); Ok(axum::Json(json!({"crates": shown, "meta": {"total": total}})).into_response()) } #[cfg(test)] mod tests { use super::*; use flate2::Compression; use flate2::write::GzEncoder; use std::io::Write; /// A gzipped tar of each named file, as `cargo package` makes one. fn tarball(files: &[(&str, &str)]) -> Vec { let mut gz = GzEncoder::new(Vec::new(), Compression::default()); for (name, text) in files { let mut header = [0u8; 512]; header[..name.len()].copy_from_slice(name.as_bytes()); let size = format!("{:011o}\0", text.len()); header[124..136].copy_from_slice(size.as_bytes()); gz.write_all(&header).unwrap(); gz.write_all(text.as_bytes()).unwrap(); let pad = text.len().div_ceil(512) * 512 - text.len(); gz.write_all(&vec![0; pad]).unwrap(); } gz.write_all(&[0; 1024]).unwrap(); gz.finish().unwrap() } #[test] fn a_crates_manifest_is_found_in_it() { let toml = "# made\n[package]\nname = \"a\"\nversion = \"1.0.0\"\n\n[lib]\nname = \"b\"\n"; let crate_ = tarball(&[ ("a-1.0.0/.cargo_vcs_info.json", "{}"), ("a-1.0.0/sub/Cargo.toml", "no"), ("a-1.0.0/Cargo.toml", toml), ]); let found = manifest_of(&crate_).unwrap(); assert_eq!(package_of(&found), Some(("a", "1.0.0"))); assert_eq!(manifest_of(&tarball(&[("a-1.0.0/x", "x")])), None); assert_eq!(manifest_of(b"not gzip"), None); let large = "#".repeat(MOST_MANIFEST as usize + 1); assert_eq!( manifest_of(&tarball(&[("a-1.0.0/Cargo.toml", &large)])), None ); let bad = tarball(&[("a-1.0.0/x", "x")]); let mut tar = Vec::new(); GzDecoder::new(&bad[..]).read_to_end(&mut tar).unwrap(); tar[124..136].copy_from_slice(b"not a size\0\0"); let mut gz = GzEncoder::new(Vec::new(), Compression::default()); gz.write_all(&tar).unwrap(); assert_eq!(manifest_of(&gz.finish().unwrap()), None); assert_eq!(package_of("no package"), None); assert_eq!(package_of("[package]\nname = \"a\"\n"), None); } #[test] fn a_publish_is_framed() { let body = [ &2u32.to_le_bytes()[..], b"{}", &3u32.to_le_bytes()[..], b"abc", ] .concat(); assert_eq!(framed(&body), Some((&b"{}"[..], &b"abc"[..]))); assert_eq!(framed(&body[..body.len() - 1]), None); assert_eq!(framed(&body[..3]), None); assert_eq!(framed(&body[..5]), None); assert_eq!(framed(&body[..7]), None); } #[test] fn a_crate_makes_its_index_line() { let said: Published = serde_json::from_value(json!({ "name": "a", "vers": "1.0.0", "links": "z", "rust_version": "1.80", "features": {"plain": ["other"], "named": ["dep:b"], "weak": ["b?/x"]}, "deps": [ {"name": "b", "version_req": "^1", "explicit_name_in_toml": "c", "optional": true, "default_features": false, "features": ["f"], "target": "cfg(unix)", "kind": "dev", "registry": "https://elsewhere"}, {"name": "d", "version_req": "^2"}, ], })) .unwrap(); let line: Value = serde_json::from_str(&index_line(&said, "ff")).unwrap(); assert_eq!(line["cksum"], "ff"); assert_eq!(line["v"], 2); assert_eq!(line["features"], json!({"plain": ["other"]})); assert_eq!( line["features2"], json!({"named": ["dep:b"], "weak": ["b?/x"]}) ); assert_eq!( line["deps"], json!([ {"name": "c", "req": "^1", "features": ["f"], "optional": true, "default_features": false, "target": "cfg(unix)", "kind": "dev", "registry": "https://elsewhere", "package": "b"}, {"name": "d", "req": "^2", "features": [], "optional": false, "default_features": true, "target": null, "kind": "normal", "registry": null, "package": null}, ]) ); let plain: Published = serde_json::from_value(json!({"name": "a", "vers": "1.0.0"})).unwrap(); let line: Value = serde_json::from_str(&index_line(&plain, "ff")).unwrap(); assert_eq!(line["v"], 1); assert!(line.get("features2").is_none()); } #[test] fn the_index_has_each_version_once_as_cargo_reads_them() { for (name, path) in [ ("a", "1/a"), ("ab", "2/ab"), ("abc", "3/a/abc"), ("abcd", "ab/cd/abcd"), ("abcde", "ab/cd/abcde"), ] { assert_eq!(index_path(name), path); } let release = |version: &str, line: Option, yanked| { let cargo = line.map(|index| CargoCrate { index, description: None, }); let record = ReleaseRecord { repo: "k".into(), package: "a".into(), version: version.into(), commit: None, check: None, notes: None, files: Vec::new(), created_at: String::new(), cargo, yanked, }; (String::new(), record) }; let of = |v: &str| Some(json!({"name": "a", "vers": v}).to_string()); // Newest first, as the index gives them: the oldest of a version wins. let all = [ release("2.0.0", None, false), release("1.1.0+later", of("1.1.0+later"), false), release("1.1.0", of("1.1.0"), true), release("1.0.0", of("1.0.0"), false), ]; let found: Vec<&str> = crates(&all, "a") .iter() .map(|r| r.version.as_str()) .collect(); assert_eq!(found, ["1.0.0", "1.1.0"]); assert!(crates(&all, "b").is_empty()); let line: Value = serde_json::from_str(&line_of(&all[2].1).unwrap()).unwrap(); assert_eq!(line["yanked"], true); // A line that is not of its release, or none, is passed by. for bad in [ of("9.9.9"), Some(json!({"name": "b", "vers": "1.0.0"}).to_string()), Some("[]".to_string()), Some("not json".to_string()), Some(json!({"vers": "1.0.0"}).to_string()), Some(json!({"name": "a"}).to_string()), None, ] { assert_eq!(line_of(&release("1.0.0", bad, false).1), None); } } #[test] fn a_version_is_told_apart_without_its_build() { assert_eq!(base("1.0.0+b.1"), "1.0.0"); assert_eq!(base("1.0.0-rc.1"), "1.0.0-rc.1"); } }