chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Deleting a ref deletes its record
.changeset/refs-deleted.mdnew+9−0
| @@ -0,0 +1,9 @@ | ||
--- | ||
default: minor | ||
--- | ||
| ||
# Deleting a branch deletes its record | ||
| ||
Deleting a branch or tag now deletes its ref record from your PDS, rather than leaving an empty one so that its count of moves never repeats. A new ref starts its count at the time instead. Compaction removes the empty records older clients left once they are a day old, so a repository that has had hundreds of branches reads only the ones it has now on every push, fetch and page. | ||
| ||
Upgrade every client that pushes: an older one making a branch again under a name deleted less than a day before can have its pushes to it refused until the next compaction. | ||
crates/knotless/src/cli.rs+5−2
| @@ -2328,8 +2328,11 @@ async fn command( | ||
} | ||
"compact" => { | ||
let c = compact::compact_repo(ctx, &args.url()?).await?; | ||
let (packs, folded, claims) = (c.before, c.removed, c.claims); | ||
let line = format!("{packs} packs, {folded} folded, {claims} old claims dropped"); | ||
let (packs, folded, claims, refs) = (c.before, c.removed, c.claims, c.refs); | ||
let line = format!( | ||
"{packs} packs, {folded} folded, {claims} old claims dropped, \ | ||
{refs} deleted refs' records dropped" | ||
); | ||
writeln!(out, "{line}")?; | ||
} | ||
"storage" => { | ||
crates/knotless/src/compact.rs+85−1
| @@ -8,7 +8,9 @@ | ||
//! deleted since one of them was pushed would leave a base no pack holds. Packs younger than | ||
//! the grace period stay, so a reader partway through a walk still finds | ||
//! what it listed; a reader that loses the race restarts its walk once. | ||
//! Claims a day old go too: every push adds one per ref it moves. | ||
//! Claims a day old go too: every push adds one per ref it moves. So do | ||
//! the records deleted refs left with no target, before a delete removed | ||
//! its ref's record, once a day old. | ||
| ||
use crate::env::Ctx; | ||
use crate::fetch; | ||
| @@ -30,6 +32,9 @@ pub struct Compacted { | ||
pub removed: usize, | ||
/// Claims dropped for being older than the claim grace period. | ||
pub claims: usize, | ||
/// Records of deleted refs dropped, each left with no target by a | ||
/// client from before a delete removed them. | ||
pub refs: usize, | ||
} | ||
| ||
/// Compacts the repository at `url`, as its owner logged in here. | ||
| @@ -88,13 +93,50 @@ pub async fn compact(ctx: &Ctx, remote: &Remote, session: &Session) -> Result<Co | ||
for chunk in writes.chunks(200) { | ||
remote.apply(chunk.to_vec(), &session.auth()).await?; | ||
} | ||
let dead = dead_refs(&snap, &key, since); | ||
// In pairs, each claim with the delete it guards. | ||
for chunk in dead.chunks(200) { | ||
remote.apply(chunk.to_vec(), &session.auth()).await?; | ||
crates/knotless/src/compact.rsline 99This apply, and the | ||
} | ||
Ok(Compacted { | ||
before: packs.len(), | ||
removed, | ||
claims: claims.len(), | ||
refs: dead.len() / 2, | ||
}) | ||
} | ||
| ||
/// The writes that drop the records deleted refs left with no target, of | ||
/// those that moved last before `since`. Each goes beside a claim on its | ||
/// generation, so a writer that read it, and would move the ref off that | ||
/// generation, cannot move one made again since; a ref made again starts | ||
/// past it ([`keys::first_generation`]). | ||
fn dead_refs(snap: &Snapshot, key: &str, since: SystemTime) -> Vec<serde_json::Value> { | ||
let now = records::now(); | ||
let old = |r: &&remote::Stored<records::RefRecord>| { | ||
let moved = humantime::parse_rfc3339_weak(&r.value.updated_at); | ||
r.value.target.is_none() && moved.is_ok_and(|t| t < since) | ||
}; | ||
let mut writes = Vec::new(); | ||
for r in snap.refs.iter().filter(old) { | ||
let claim = records::ClaimRecord { | ||
repo: key.to_string(), | ||
name: r.value.name.clone(), | ||
generation: r.value.generation, | ||
from: None, | ||
to: None, | ||
pack: None, | ||
created_at: now.clone(), | ||
pusher: None, | ||
}; | ||
let rkey = keys::claim_rkey(key, &r.value.name, r.value.generation); | ||
let claim = records::typed(nsid::CLAIM, &claim); | ||
writes.push(remote::create(nsid::CLAIM, &rkey, claim)); | ||
writes.push(remote::delete(nsid::REF, &r.rkey)); | ||
} | ||
writes | ||
} | ||
| ||
/// The writes that replace the `old` packs with one holding everything the | ||
/// live refs reach, and the `kept` bases the packs that stay name. | ||
async fn fold( | ||
| @@ -142,3 +184,45 @@ async fn fold( | ||
writes.extend(old.iter().map(|rkey| remote::delete(nsid::PACK, rkey))); | ||
Ok(writes) | ||
} | ||
| ||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use crate::records::RefRecord; | ||
use crate::remote::Stored; | ||
use std::time::Duration; | ||
| ||
#[test] | ||
fn only_deleted_refs_a_day_old_go() { | ||
let at = |name: &str, target: Option<&str>, moved: &str| Stored { | ||
rkey: format!("k.{name}"), | ||
cid: String::new(), | ||
value: RefRecord { | ||
repo: "k".into(), | ||
name: name.into(), | ||
target: target.map(str::to_string), | ||
generation: 3, | ||
updated_at: moved.into(), | ||
created_at: None, | ||
pusher: None, | ||
}, | ||
}; | ||
let snap = Snapshot { | ||
refs: vec![ | ||
at("old", None, "2026-01-01T00:00:00Z"), | ||
at("young", None, "2026-01-03T00:00:00Z"), | ||
at("live", Some("a1"), "2026-01-01T00:00:00Z"), | ||
], | ||
..Snapshot::default() | ||
}; | ||
let since = humantime::parse_rfc3339("2026-01-02T00:00:00Z").unwrap(); | ||
let writes = dead_refs(&snap, "k", since); | ||
// The old one's record goes, beside a claim on its generation. | ||
assert_eq!(writes.len(), 2); | ||
assert_eq!(writes[0]["collection"], nsid::CLAIM); | ||
assert_eq!(writes[0]["rkey"], keys::claim_rkey("k", "old", 3)); | ||
assert_eq!(writes[1]["collection"], nsid::REF); | ||
assert_eq!(writes[1]["rkey"], "k.old"); | ||
assert!(dead_refs(&snap, "k", since - Duration::from_secs(86_400 * 2)).is_empty()); | ||
} | ||
} | ||
crates/knotless/src/forge.rs+1−1
| @@ -2497,7 +2497,7 @@ async fn make_ref( | ||
let t = Target::open(ctx, url).await?; | ||
let dir = if kind == "tag" { "tags" } else { "heads" }; | ||
let refname = format!("refs/{dir}/{branch}"); | ||
// A deleted ref keeps its record, with no target. | ||
// A ref deleted by a client before 0.6 kept its record, with no target. | ||
let exists = t | ||
.snap | ||
.find_ref(&refname) | ||
crates/knotless/src/keys.rs+8−0
| @@ -66,6 +66,14 @@ pub fn label_rkey(key: &str, subject: &str) -> String { | ||
format!("{key}.{}", ref_hash(&crate::aturi::short(subject))) | ||
} | ||
| ||
/// The generation a new ref starts at: the time in milliseconds, past any | ||
/// generation an earlier ref of the same name reached, as one moves far | ||
/// less often than once a millisecond. Its claims, which compaction keeps a | ||
/// day, then never stand in the way of the new one's. | ||
pub fn first_generation() -> u64 { | ||
now_micros() / 1000 | ||
} | ||
| ||
/// The claim a push creates to move `name` off generation `generation`. | ||
pub fn claim_rkey(key: &str, name: &str, generation: u64) -> String { | ||
format!("{key}.{}.{generation}", ref_hash(name)) | ||
crates/knotless/src/push.rs+19−12
| @@ -213,25 +213,32 @@ pub async fn push_by( | ||
repo: key.clone(), | ||
name: a.name.clone(), | ||
target: a.to.clone(), | ||
generation: generation + 1, | ||
generation: match &a.previous { | ||
Some(_) => generation + 1, | ||
None => keys::first_generation(), | ||
}, | ||
updated_at: now.clone(), | ||
created_at: match &a.previous { | ||
Some(r) => r.value.created_at.clone(), | ||
None => Some(now.clone()), | ||
}, | ||
pusher: pusher.clone(), | ||
}; | ||
let rkey = keys::ref_rkey(&key, &a.name); | ||
let value = records::typed(nsid::REF, &record); | ||
let claim = remote::create( | ||
nsid::CLAIM, | ||
&keys::claim_rkey(&key, &a.name, generation), | ||
records::typed(nsid::CLAIM, &claim), | ||
); | ||
// A create fails on a key that exists, so a new ref's record guards | ||
// itself, and a claim beside it would only spend rate-limit points. | ||
ref_writes.push(if a.previous.is_some() { | ||
vec![ | ||
remote::create( | ||
nsid::CLAIM, | ||
&keys::claim_rkey(&key, &a.name, generation), | ||
records::typed(nsid::CLAIM, &claim), | ||
), | ||
remote::update(nsid::REF, &rkey, value), | ||
] | ||
} else { | ||
vec![remote::create(nsid::REF, &rkey, value)] | ||
// A deleted ref's record goes, its claim left to guard the | ||
// generation it was at. | ||
ref_writes.push(match (&a.previous, &a.to) { | ||
(None, _) => vec![remote::create(nsid::REF, &rkey, value)], | ||
(Some(_), None) => vec![claim, remote::delete(nsid::REF, &rkey)], | ||
(Some(_), Some(_)) => vec![claim, remote::update(nsid::REF, &rkey, value)], | ||
}); | ||
} | ||
| ||
crates/knotless/src/records.rs+9−2
| @@ -832,8 +832,11 @@ pub mod status { | ||
} | ||
} | ||
| ||
/// One ref. A deleted ref keeps its record with no target, so its generation | ||
/// never repeats and an old claim can never be replayed. | ||
/// One ref. Deleting a ref deletes its record, beside the claim on the | ||
/// generation it was at; a ref made again starts at a generation read from | ||
/// the clock, past any an earlier life of it reached, so a claim of that | ||
/// life can never be replayed against it. Records written before deletes | ||
/// went so remain with no target, until compaction removes them. | ||
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] | ||
#[serde(rename_all = "camelCase")] | ||
pub struct RefRecord { | ||
| @@ -843,6 +846,10 @@ pub struct RefRecord { | ||
pub target: Option<String>, | ||
pub generation: u64, | ||
pub updated_at: String, | ||
/// When the ref was made: equal to `updated_at` until it first moves. | ||
/// Absent from records written before it was kept. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
pub created_at: Option<String>, | ||
/// Who moved it last, when that was not the owner: a collaborator or a | ||
/// runner writing through the gateway. | ||
#[serde(default, skip_serializing_if = "Option::is_none")] | ||
crates/knotless/src/remote.rs+3−3
| @@ -522,9 +522,9 @@ impl Remote { | ||
Ok(all) | ||
} | ||
| ||
/// Every ref record of the repository `key`, the deleted refs' among | ||
/// them. A repository keeps one for every branch it ever had, which | ||
/// a busy one counts in hundreds: past the first page, the rest are | ||
/// Every ref record of the repository `key`, with any a client before | ||
/// 0.6 left of a deleted ref until compaction drops it, which a busy | ||
/// repository can count in hundreds: past the first page, the rest are | ||
/// read as ranges of their keys side by side, rather than as pages one | ||
/// after another. | ||
pub async fn refs(&self, key: &str) -> Result<Vec<Stored<RefRecord>>> { | ||
crates/knotless/tests/cli.rs+1−1
| @@ -324,7 +324,7 @@ fn repository_lifecycle() { | ||
assert!(std::path::Path::new(&out).join("HEAD").exists()); | ||
assert_eq!( | ||
s.cli(&["compact", &url], b"").1, | ||
"2 packs, 2 folded, 0 old claims dropped" | ||
"2 packs, 2 folded, 0 old claims dropped, 0 deleted refs' records dropped" | ||
); | ||
// Folded, its history is one pack, small enough to ride inline. | ||
let (_, storage) = s.cli(&["storage", &url], b""); | ||
crates/knotless/tests/git_flows.rs+1−1
| @@ -312,7 +312,7 @@ fn compaction_off_past_threshold() { | ||
s.git(&work, &["push", "-q", &url, "main"]); | ||
assert_eq!( | ||
s.cli(&["compact", &url], b"").1, | ||
"2 packs, 2 folded, 0 old claims dropped" | ||
"2 packs, 2 folded, 0 old claims dropped, 0 deleted refs' records dropped" | ||
); | ||
} | ||
| ||