chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Fetches into one mirror take turns
.changeset/fetches-take-turns.mdnew+7−0
| @@ -0,0 +1,7 @@ | ||
--- | ||
default: patch | ||
--- | ||
| ||
# Commands reading one repository at once no longer fail on its history | ||
| ||
Two commands fetching the same repository at once could fail with `git commit-graph: Could not read`, when one found a commit whose parents the other had not fetched yet. Fetches now take turns. | ||
crates/knotless/src/fetch.rs+4−0
| @@ -3,6 +3,7 @@ | ||
//! ones whose tips are missing, until every wanted object is here. | ||
| ||
use crate::git::Git; | ||
use crate::mirror::Lock; | ||
use crate::records::{PackRecord, RefRecord}; | ||
use crate::remote::{Remote, Snapshot, Stored}; | ||
use anyhow::{Result, bail}; | ||
| @@ -26,6 +27,9 @@ pub async fn fetch( | ||
wants: &[String], | ||
snap: &Snapshot, | ||
) -> Result<usize> { | ||
// One fetch at a time here: the newest packs land first, so a commit | ||
// being here does not yet say its parents are, while another is at it. | ||
let _turn = Lock::acquire(git.dir.join("fetching")).await?; | ||
let mut missing: BTreeSet<String> = git.missing(wants).await?.into_iter().collect(); | ||
if missing.is_empty() { | ||
return Ok(0); | ||
crates/knotless/src/mirror.rs+3−3
| @@ -124,9 +124,9 @@ impl Mirror { | ||
}; | ||
let live = snap.live_refs(); | ||
let wants: Vec<String> = live.iter().map(|(_, sha)| sha.clone()).collect(); | ||
// Indexing packs needs no lock: each lands under its own name. The | ||
// fetch reads packs only if an object is missing, and then the ones | ||
// the refs name. | ||
// The fetch takes its own turn, apart from the lock on the refs, and | ||
// reads packs only if an object is missing, and then the ones the | ||
// refs name. | ||
fetch::fetch(remote, key, &self.git, &wants, snap).await?; | ||
let branch = snap.repo.as_ref(); | ||
let branch = branch.and_then(|r| r.value.default_branch.clone()); | ||
crates/knotless/tests/library.rs+38−0
| @@ -1111,6 +1111,44 @@ fn refs_name_the_packs_holding_their_targets() { | ||
}); | ||
} | ||
| ||
#[test] | ||
fn syncs_at_once_wait_for_a_fetch_under_way() { | ||
let rt = rt(); | ||
let s = Sandbox::new(); | ||
rt.block_on(async { | ||
let url = s.url("syncs"); | ||
let ctx = ctx(&s); | ||
let r = connect(&ctx, &url).await; | ||
// A pack for each commit, each on top of the last. | ||
for i in 0..6 { | ||
let change = put("a", &i.to_string()); | ||
knotless::commit::commit(&ctx, &r.url, &spec(vec![change], 0)) | ||
.await | ||
.unwrap(); | ||
} | ||
let snap = r.snapshot(&ctx).await.unwrap(); | ||
let (did, key) = (r.did(), snap.key().unwrap()); | ||
let dir = knotless::mirror::path(&ctx, did, key); | ||
// A fetch lands a commit before the packs holding its parents, and | ||
// a sync that finds it there must not take the history as whole. | ||
for _ in 0..30 { | ||
let _ = std::fs::remove_dir_all(&dir); | ||
let mirror = knotless::mirror::Mirror::open(&ctx, did, key) | ||
.await | ||
.unwrap(); | ||
let mirror = &mirror; | ||
let (r, snap) = (&r, &snap); | ||
let syncs = (0..12).map(|i| async move { | ||
tokio::time::sleep(std::time::Duration::from_millis(i * 4)).await; | ||
mirror.sync(r, snap).await | ||
}); | ||
for done in futures::future::join_all(syncs).await { | ||
done.unwrap(); | ||
} | ||
} | ||
}); | ||
} | ||
| ||
#[test] | ||
fn concurrent_commits_replay_onto_the_new_tip() { | ||
let s = Sandbox::new(); | ||
docs/design/storage.md+7−5
| @@ -490,11 +490,13 @@ cache, with each `:` of the DID written `_`, kept in step with the records. | ||
Compaction, checkout-less commits, branches, merges, the gateway and | ||
knotless-serve all work from one, shared by every process using the cache. A | ||
sync fetches what the snapshot's live refs need, then sets the mirror's refs | ||
exactly to the snapshot's in one `git update-ref --stdin` transaction. Packs | ||
are indexed without a lock, because each lands under its own name. The ref | ||
transaction runs under the operating system's lock on a file beside the | ||
mirror. The system lets go of it when its process ends, so a crashed process | ||
cannot block the mirror. The transaction runs in a task of its own that | ||
exactly to the snapshot's in one `git update-ref --stdin` transaction. Fetches | ||
into a mirror take turns, under the operating system's lock on a file inside | ||
it: the newest packs land first, so a commit being there does not say its | ||
parents are until the fetch that brought it has finished. The ref transaction | ||
runs under the same kind of lock on a file beside the mirror. The system lets | ||
go of either when its process ends, so a crashed process cannot block the | ||
mirror. The transaction runs in a task of its own that | ||
holds the lock until git is done. A caller that stops waiting, such as a | ||
request whose asker left, does not end it, so the next sync never finds | ||
git's own lock files on the refs still there. A new mirror is created under a temporary name and | ||