chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
A mirror's sync reads its packs only when an object is missing
crates/knotless/src/mirror.rs+5−2
| @@ -124,8 +124,11 @@ impl Mirror { | ||
}; | ||
let live = snap.live_refs(); | ||
let wants: Vec<String> = live.iter().map(|(_, sha)| sha.clone()).collect(); | ||
// Indexing packs needs no lock: each lands under its own name. | ||
fetch::fetch(remote, key, &self.git, &wants, Some(snap.packs.clone())).await?; | ||
// Indexing packs needs no lock: each lands under its own name. A | ||
// snapshot read without its packs leaves the fetch to read them, and | ||
// only if an object is missing. | ||
let first = (!snap.packs.items.is_empty()).then(|| snap.packs.clone()); | ||
fetch::fetch(remote, key, &self.git, &wants, first).await?; | ||
let branch = snap.repo.as_ref(); | ||
let branch = branch.and_then(|r| r.value.default_branch.clone()); | ||
// Set in a task of its own, which holds the lock until git is done: | ||
crates/knotless/src/remote.rs+23−2
| @@ -548,6 +548,27 @@ impl Remote { | ||
/// cached this is one round trip of three parallel reads; the repository | ||
/// record still comes along to catch a name that now means another key. | ||
pub async fn snapshot(&self, ctx: &Ctx) -> Result<Snapshot> { | ||
self.records(ctx, true).await | ||
} | ||
| ||
/// The repository record and its refs, without the packs: what a mirror | ||
/// that already has its objects needs to follow them. The newest page of | ||
/// packs can be megabytes of inline packs, so a reader that syncs often | ||
/// leaves it to [`fetch`](crate::fetch::fetch), which reads it only when | ||
/// an object is missing. The snapshot's `packs` is then empty. | ||
pub async fn heads(&self, ctx: &Ctx) -> Result<Snapshot> { | ||
self.records(ctx, false).await | ||
} | ||
| ||
/// The newest page of `key`'s packs when `wanted`, else none. | ||
async fn packs_if(&self, key: &str, wanted: bool) -> Result<Page<PackRecord>> { | ||
if !wanted { | ||
return Ok(Page::default()); | ||
} | ||
self.packs(key, None).await | ||
} | ||
| ||
async fn records(&self, ctx: &Ctx, packs: bool) -> Result<Snapshot> { | ||
let cache_key = self.cache_key(); | ||
let path = key_cache_path(ctx); | ||
let mut cache: BTreeMap<String, String> = store::read(&path).unwrap_or_default(); | ||
| @@ -555,7 +576,7 @@ impl Remote { | ||
let repo = match cache.get(&cache_key) { | ||
Some(key) => { | ||
let (repo, refs, packs) = | ||
tokio::try_join!(self.repo(), self.refs(key), self.packs(key, None))?; | ||
tokio::try_join!(self.repo(), self.refs(key), self.packs_if(key, packs))?; | ||
if repo.as_ref().is_some_and(|r| &r.value.key == key) { | ||
return Ok(Snapshot { | ||
repo, | ||
| @@ -574,7 +595,7 @@ impl Remote { | ||
let key = repo.value.key.clone(); | ||
cache.insert(cache_key, key.clone()); | ||
let _ = store::write(&path, &cache, false); | ||
let (refs, packs) = tokio::try_join!(self.refs(&key), self.packs(&key, None))?; | ||
let (refs, packs) = tokio::try_join!(self.refs(&key), self.packs_if(&key, packs))?; | ||
Ok(Snapshot { | ||
repo: Some(repo), | ||
refs, | ||
crates/knotless/src/storage.rs+1−10
| @@ -2,7 +2,7 @@ | ||
| ||
use crate::env::Ctx; | ||
use crate::records::{PackRecord, nsid}; | ||
use crate::remote::{Remote, Snapshot, Stored}; | ||
use crate::remote::{Remote, Stored}; | ||
use crate::url::RepoUrl; | ||
use anyhow::{Result, bail}; | ||
use serde::Serialize; | ||
| @@ -49,12 +49,3 @@ pub fn counted(packs: &[Stored<PackRecord>]) -> Storage { | ||
} | ||
total | ||
} | ||
| ||
/// What the repository `snap` is of takes up, if `snap` holds all of its | ||
/// packs: it holds only their newest page. | ||
pub fn of_snapshot(snap: &Snapshot) -> Option<Storage> { | ||
snap.packs | ||
.next | ||
.is_none() | ||
.then_some(counted(&snap.packs.items)) | ||
} | ||
crates/knotless-serve/src/web/settings.rs+2−8
| @@ -110,14 +110,8 @@ pub async fn get_repo_settings( | ||
Some(to) => Some(basic(&app, to).await), | ||
None => None, | ||
}; | ||
// Counted afresh only past the one page of packs a sync reads. | ||
let counted = knotless::storage::storage(&reader, &url); | ||
let counted = repo.storage.is_none().then_some(counted); | ||
let counted = futures::future::OptionFuture::from(counted).await; | ||
let storage = repo | ||
.storage | ||
.clone() | ||
.or(counted.and_then(std::result::Result::ok)); | ||
// Counted from its pack records, which a sync no longer reads. | ||
let storage = knotless::storage::storage(&reader, &url).await.ok(); | ||
let mut readers = None; | ||
if repo.in_space.is_some() { | ||
// Only its owner may list them; anyone else is shown none. | ||
crates/knotless-serve/src/lib.rs+3−8
| @@ -157,8 +157,6 @@ struct Entry { | ||
record: RepoRecord, | ||
/// The space it is in, if it is private. | ||
space: Option<String>, | ||
/// What it takes up, if the last sync read all of its packs. | ||
storage: Option<knotless::storage::Storage>, | ||
synced: Option<Instant>, | ||
/// When its viewer was last found allowed to read it, for a private | ||
/// repository a viewer was shown. | ||
| @@ -184,9 +182,6 @@ pub struct Located { | ||
pub record: RepoRecord, | ||
/// The space it is in, if it is private. | ||
pub space: Option<String>, | ||
/// What it takes up, as the mirror last synced, if that read all of | ||
/// its packs. | ||
pub storage: Option<knotless::storage::Storage>, | ||
/// What reads it: this server, or for a private repository the viewer | ||
/// it was found for, so that its records are read as theirs. | ||
pub reader: Ctx, | ||
| @@ -578,7 +573,9 @@ impl App { | ||
entry: &mut Entry, | ||
) -> Result<(), AppError> { | ||
let started = Instant::now(); | ||
let snap = remote.snapshot(ctx).await.map_err(AppError::upstream)?; | ||
// Its record and refs, not its packs: those are read only when the | ||
// mirror lacks an object they hold. | ||
let snap = remote.heads(ctx).await.map_err(AppError::upstream)?; | ||
let (remote, snap) = remote.follow(ctx, snap).await.map_err(AppError::upstream)?; | ||
let records = started.elapsed(); | ||
let Some(repo) = &snap.repo else { | ||
| @@ -602,7 +599,6 @@ impl App { | ||
entry.found = Some((remote.did().to_string(), key.to_string(), repo.rkey.clone())); | ||
entry.record = repo.value.clone(); | ||
entry.space = remote.space.as_ref().map(|s| s.uri.clone()); | ||
entry.storage = knotless::storage::of_snapshot(&snap); | ||
entry.git = Some(mirror.git); | ||
entry.synced = Some(Instant::now()); | ||
// Every read of the repository waits on its sync: how long each | ||
| @@ -632,7 +628,6 @@ impl App { | ||
name, | ||
record, | ||
space: entry.space.clone(), | ||
storage: entry.storage.clone(), | ||
reader, | ||
} | ||
} | ||
crates/knotless-serve/src/web.rs+0−4
| @@ -1255,9 +1255,6 @@ struct Repo { | ||
/// Its record, as the PDS had it when its mirror last synced, which a | ||
/// write from the pages has it sync again for. | ||
synced: RepoRecord, | ||
/// What it takes up, as its mirror last synced, if that read all of its | ||
/// packs. | ||
storage: Option<knotless::storage::Storage>, | ||
/// What reads its records: for a private repository, the viewer it | ||
/// was found for, never this server ([`crate::private`]). | ||
reader: knotless::env::Ctx, | ||
| @@ -1304,7 +1301,6 @@ impl Repo { | ||
space: found.space.is_some().then_some(found.record.clone()), | ||
in_space: found.space, | ||
synced: found.record, | ||
storage: found.storage, | ||
reader: found.reader, | ||
}) | ||
} | ||
docs/design/serve.md+5−0
| @@ -84,6 +84,11 @@ owner's DID (an account's permanent ID) and the repository's key, so a rename | ||
or a transfer reuses it. Clones and the API follow a renamed or moved | ||
repository without a redirect, because syncing follows the move. | ||
| ||
A sync reads the repository record and its refs, not the newest page of packs | ||
a snapshot holds: with packs inline in their records, that page can be | ||
megabytes. The mirror reads packs only when a ref names an object it lacks, | ||
as after a push. | ||
| ||
Every object the server hands out is named by its hash, so a stale cache still | ||
cannot lie about an object. | ||