//! What the server holds in memory of its accounts (`spec/STORE.md`, //! "Scale"), within `PDS_CACHE_BYTES`. //! //! **Held once used**, for each account: its DID, handle, keys, state, mail //! and DID document (`Account`), with its sessions, app passwords and //! preferences, a few KiB, made of its row of the service checkpoint and //! its entries of the service log on its first use (`directory`), and held //! from then on. **Held while used**: its store (`Store`), loaded from its //! checkpoint and log on its first use, a read or a write (`Slot`), and let //! go when it has not been used for `IDLE`, or when the stores held count //! more than the cache's share of `PDS_CACHE_BYTES`, least recently used //! first. A store counts what it holds in memory (`Store::cost`): its //! table's index, where in the log each entry the log changed is, its //! blobs, and of each space repo its commit and hash, not its records, which //! are read from the table and the log as they are asked for. //! //! **A store being written** holds where each entry of its log after its //! table is (its memtable), which grows with the log. Past an eighth of //! the stores' share of the cache (`Cache::memtable`), its log is //! compacted, whatever the log's size, and what the table then holds is //! let go; past twice that while the compaction runs, a writer waits for //! it (`Slot::write_lock`): written faster than its log can be compacted, //! a store holds no more for it. So too once its log holds `store::BEHIND` //! past what compacts it by size (`store::ACCOUNT_CAP` at most, past its //! table): its first use after a start replays no more of it, however //! large its table. The compaction holds as much again of //! the logs it folds, with the account's state and a few blocks //! (`Cache::fold_blocks`), counted with the store while it runs //! (`Slot::folding`). A store is let go only with nothing of it in flight: no write //! waiting for its log to be on disk, no compaction of its log, no upload //! moving into place. Everything it holds is on disk by then, so letting it //! go is dropping it; its log gives back the room in its file as it goes //! (`pds_store`), and loading it again syncs nothing: its slot keeps that //! its log's frames are all on disk (`Slot::durable`), as every account's //! are from the start where the last server shut down. A log found with //! zeros after its frames all the same, as a kill of a run before left //! one it had open, is cut to its frames as it is loaded, with no sync. //! No store is loaded once a shutdown has closed the accounts' logs //! (`ops::settle`): what asks for one not held then is unwound //! (`ShuttingDown`), and the account's files stay as the shutdown's mark //! says they are. //! //! **A start** loads no account but those marked in the service log //! (`Eager`): an account whose handle change is in flight, that has write //! notifications or notices to syncers to send, or, with blobs in a bucket, //! an upload on its way or objects to delete (`blobstore`). Each is marked, //! durably, before the entry that needs it reaches its own log, and //! unmarked, with the handle it has then, once nothing needs it. So the //! service log holds each account's handle but while a mark says to look in //! its own log. use crate::events::Firehose; use crate::{Account, Locked, Store}; use k256::ecdsa::SigningKey; use pds_store::{Files, Policy, Wal}; use serde_json::{Value as Json, json}; use std::collections::{HashMap, HashSet}; use std::io; use std::ops::{Deref, DerefMut}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard, Weak}; use std::time::{Duration, Instant}; /// How long a store not used is held. pub(crate) const IDLE: Duration = Duration::from_secs(120); /// The service log's entry of an account's mark and handle. pub(crate) const DIR: &str = "dir"; /// The bytes a record taken down is counted as in the cache: its path and /// its takedown's reference. const TAKEDOWN_COST: u64 = 192; /// What every account's slot shares: where the files are, and what loading /// a store needs. pub(crate) struct Ctx { pub dir: PathBuf, /// The accounts' logs, of which at most a share are open at once /// (`Config::share`). pub files: Arc, pub policy: Policy, /// The backfill window, in milliseconds. pub window: u64, pub events: Arc, pub eager: Arc, pub cache: Cache, /// The blocks of the stores' tables and logs read from disk, and of /// the service checkpoint's (`directory`), shared, and the tables' /// files, of which at most a share are open at once. pub blocks: Arc, /// While the server starts: a store loaded then with no commit gets /// its first from the start, once it knows where seqs go on from. pub starting: std::sync::atomic::AtomicBool, /// Whether the last server shut down, in this very directory, with /// every frame of every log on disk (`ops::durable`): an account's log /// is then synced by no load of its store, its first in this run /// neither (`Slot::durable`). pub durable: bool, /// Whether every frame the logs held as this run started is on disk: /// from the start where `durable`, or where the directory held no data; /// else once the run has made durable what it found (`ops::sync_found`). /// Its shutdown leaves the word in its mark only then (`ops::mark`). It /// changes nothing for this run's own loads. pub found: AtomicBool, /// Whether a load of a store failed in this run. What it had mended /// of its log by then nothing has synced, and the server stops for it /// (`Slot::ensure`); a shutdown that got there first leaves no word /// in its mark (`ops::mark`). pub failed: AtomicBool, /// The bucket blobs are kept in, if they are not kept in the data /// directory (`blobstore`). pub bucket: Option>, } fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(PoisonError::into_inner) } /// Milliseconds since the server started: when a store was last used. fn clock() -> u64 { static START: std::sync::LazyLock = std::sync::LazyLock::new(Instant::now); START.elapsed().as_millis() as u64 + 1 } // ---- The marks ---------------------------------------------------------------- /// The accounts a start loads, as the service log marks them. pub(crate) struct Eager { log: Arc, marked: Mutex>, } /// The service log's entry: the account's mark, and its handle if given. pub(crate) fn dir_entry(did: &str, handle: Option<&str>, eager: bool) -> Json { let mut e = json!({ "t": DIR, "did": did, "eager": eager }); if let Some(h) = handle { e["handle"] = json!(h); } e } impl Eager { pub fn new(log: Arc, marked: HashSet) -> Eager { Eager { log, marked: Mutex::new(marked), } } pub fn is_marked(&self, did: &str) -> bool { lock(&self.marked).contains(did) } /// Mark `did`, on disk before this returns, unless it is already. pub fn mark(&self, did: &str) -> io::Result<()> { if self.is_marked(did) { return Ok(()); } let n = self.log.append(&[dir_entry(did, None, true)])?; self.log.wait(n)?; lock(&self.marked).insert(did.to_string()); Ok(()) } /// Nothing of `did` needs a start to load it now: unmark it, with the /// handle it has, `handle`, if given. Not waited for: a mark lost /// only makes a start load the account. pub fn relax(&self, did: &str, handle: Option<&str>) { let marked = lock(&self.marked).remove(did); if !marked && handle.is_none() { return; } if let Err(e) = self.log.append(&[dir_entry(did, handle, false)]) { crate::ops::failed_for( crate::ops::Level::Error, "unmarking an account failed", did, e, ); if marked { lock(&self.marked).insert(did.to_string()); } } } /// Note the handle `did` has now, its mark kept. Not waited for. pub fn relax_handle(&self, did: &str, handle: &str) { if let Err(e) = self.log.append(&[dir_entry(did, Some(handle), true)]) { crate::ops::failed_for( crate::ops::Level::Error, "noting an account's handle failed", did, e, ); } } /// Forget a deleted account's mark. pub fn forget(&self, did: &str) { lock(&self.marked).remove(did); } } impl Store { /// Whether a start must load it: a handle change in flight, sends due, /// or with blobs in a bucket, an upload on its way or a stray. pub(crate) fn needs_start(&self) -> bool { self.pending.is_some() || self.notify.next_due().is_some() || (self.ctx.bucket.is_some() && self.blobs.objects_unsettled()) } /// Whether nothing of it is in flight, so that it may be let go: every /// frame its log was given is on disk, with the logs a compaction /// sealed, and no compaction runs (`pds_store`, "An open that syncs /// nothing"). fn idle(&self) -> bool { Arc::strong_count(&self.wal) == 1 && self.wal.is_durable(self.wal.written()) && !self.wal.compacting() && self.blobs.settled() } /// The bytes it is counted as in the cache: what it holds in memory, /// not the blocks it reads, which the block cache counts, nor what a /// compaction of its log holds, which its slot counts with it. Its table's /// index and what the memtable keeps of the log after it, its blobs, /// each space repo's commit and hash, the spaces it is the authority /// of, its write notifications and its records taken down: each a /// total kept as it changes, but the records' blob refs, which are /// walked. pub(crate) fn cost(&self) -> u64 { 4096 + self.repo.head_bytes().len() as u64 + self.disk.cost() + self.blobs.cost() + self.spaces.cost() + self.authority.cost() + self.notify.cost() + self.takedowns.len() as u64 * TAKEDOWN_COST } } // ---- The cache -------------------------------------------------------------------- /// The stores held, and the bytes they count, within `budget`. pub(crate) struct Cache { budget: u64, held: Mutex, } /// The least a store's memtable may count before its log is compacted for /// what it counts, however small the cache. const MEMTABLE_FLOOR: u64 = 256 << 10; #[derive(Default)] struct Held { slots: HashMap>, bytes: u64, } impl Cache { pub fn new(budget: u64) -> Cache { Cache { budget, held: Mutex::default(), } } /// What one store's memtable may count before its log is compacted, /// whatever the log's size: an eighth of the stores' share of the cache. /// A store being written holds where each entry of its log after its /// table is, and its log grows to its table's size, or /// `store::ACCOUNT_CAP` if that is less, before it is compacted for that /// (`store::policy`): past this, it is compacted for what it holds in /// memory. pub fn memtable(&self) -> u64 { (self.budget / 8).max(MEMTABLE_FLOOR) } /// What one store's memtable may count while its log is compacted /// before a writer waits for the compaction: twice `memtable`. So a /// store written faster than its log is compacted holds no more than /// this and one write of its log, and the compaction as much again of /// the logs it folds: half the stores' share in all. pub fn memtable_most(&self) -> u64 { self.memtable() * 2 } /// The bytes a fold of an account's log may keep of what it reads to /// read it again: the places of its frames' items, and the table's /// blocks it looks records up in. pub fn fold_blocks(&self) -> u64 { (self.budget / 8).clamp(1 << 20, 4 << 20) } /// The bytes the stores held count. #[cfg(test)] pub fn bytes(&self) -> u64 { lock(&self.held).bytes } /// How many stores are held. #[cfg(test)] pub fn len(&self) -> usize { lock(&self.held).slots.len() } /// A store held now, or its cost changed by `delta`; let go of others, /// least recently used first, if the stores held count too much. fn count(&self, a: &Arc, delta: i64) { let over = { let mut h = lock(&self.held); h.slots .entry(a.did.clone()) .or_insert_with(|| Arc::downgrade(a)); h.bytes = h.bytes.saturating_add_signed(delta); h.bytes > self.budget }; if over { self.shrink(Some(&a.did), self.budget - self.budget / 8, 0); } } /// A store let go, which counted `cost`. fn forget(&self, did: &str, cost: u64) { let mut h = lock(&self.held); h.slots.remove(did); h.bytes = h.bytes.saturating_sub(cost); } /// Let go of stores, least recently used first, but `keep`'s, until they /// count at most `target`; and of every store not used since `before`. fn shrink(&self, keep: Option<&str>, target: u64, before: u64) { let mut held: Vec<(u64, Arc)> = { let mut h = lock(&self.held); h.slots.retain(|_, w| w.strong_count() > 0); h.slots .iter() .filter(|(did, _)| Some(did.as_str()) != keep) .filter_map(|(_, w)| w.upgrade()) .map(|a| (a.store.used.load(Ordering::Relaxed), a)) .collect() }; held.sort_by_key(|(used, _)| *used); for (used, a) in held { if used >= before && lock(&self.held).bytes <= target { break; } a.store.evict(); } } /// Let go of every store not used for `IDLE`. pub fn sweep(&self) { let before = clock().saturating_sub(IDLE.as_millis() as u64); self.shrink(None, u64::MAX, before); } } /// Let go of idle stores every so often while the server runs. pub(crate) fn sweep_every(svc: &crate::Shared) { let weak = Arc::downgrade(svc); tokio::spawn(async move { loop { tokio::time::sleep(IDLE / 4).await; let Some(svc) = weak.upgrade() else { return }; let _ = tokio::task::spawn_blocking(move || svc.ctx.cache.sweep()).await; } }); } // ---- An account's slot ------------------------------------------------------------- /// An account's signing key, made on first use. Making one derives its /// public key, which is most of what making an account of its entries /// would cost, and many an account is loaded to answer what signs nothing: /// until then it is the secret alone, checked to be one. pub(crate) struct Key { secret: k256::SecretKey, made: std::sync::OnceLock, } impl Key { /// The key of these bytes, if they are one: as `SigningKey::from_slice` /// takes them. pub fn from_slice(bytes: &[u8]) -> Option { let secret = k256::SecretKey::from_slice(bytes).ok()?; Some(Key { secret, made: std::sync::OnceLock::new(), }) } /// A key made already. pub fn made(key: SigningKey) -> Key { Key { secret: k256::SecretKey::from(&key), made: std::sync::OnceLock::from(key), } } pub fn get(&self) -> &SigningKey { self.made.get_or_init(|| SigningKey::from(&self.secret)) } /// The secret's bytes, as an account's entry keeps them. pub fn to_bytes(&self) -> k256::FieldBytes { self.secret.to_bytes() } } /// An account's store: loaded on first use, let go when the cache needs /// the room or it is idle. pub(crate) struct Slot { account: Weak, key: Key, ctx: Arc, store: RwLock>>, /// When it was last used (`clock`). used: AtomicU64, /// The bytes the store held counts, `0` if none is. cost: AtomicU64, /// The bytes a compaction of its log under way holds, counted with its /// store's: `0` with none. folding: AtomicU64, /// Whether every frame its log's files hold is on disk, so that the /// load that next opens the log syncs nothing unless it mends it /// (`store`, "Starting"): from the start, where the last server left /// them so (`Ctx::durable`); and from when its store is let go, which /// it is only with all it wrote on disk (`Store::idle`). Not so while /// its store is held or being loaded, nor once a load has failed. durable: AtomicBool, /// Whether its events were looked into for void ones in this run /// (`events::Firehose::loaded`): as its store is first loaded, or from /// the start for an account made in it. looked: AtomicBool, } /// An account's store, read. pub(crate) struct StoreRead<'a>(RwLockReadGuard<'a, Option>>); impl Deref for StoreRead<'_> { type Target = Store; fn deref(&self) -> &Store { self.0.as_ref().expect("a store loaded") } } /// An account's store, written: its cost counted again once let go. pub(crate) struct StoreWrite<'a> { guard: Option>>>, slot: &'a Slot, } impl Deref for StoreWrite<'_> { type Target = Store; fn deref(&self) -> &Store { self.guard .as_ref() .and_then(|g| g.as_ref()) .expect("a store loaded") } } impl DerefMut for StoreWrite<'_> { fn deref_mut(&mut self) -> &mut Store { self.guard .as_mut() .and_then(|g| g.as_mut()) .expect("a store loaded") } } impl Drop for StoreWrite<'_> { fn drop(&mut self) { let Some(g) = self.guard.take() else { return }; let cost = g.as_ref().map_or(0, |st| { // More of its log held in memory than its share of the cache: // compacted now, whatever the log's size. if st.disk.mem.cost() > self.slot.ctx.cache.memtable() { st.wal.compact_soon(); } st.cost() }); drop(g); self.slot.recount(cost); } } /// Data the server cannot recover is never served: the server stops. pub(crate) fn unrecoverable(dir: &std::path::Path, why: &str) -> ! { if cfg!(test) { panic!("cannot recover {}: {why}", dir.display()); } eprintln!("pds: cannot recover {}: {why}", dir.display()); std::process::exit(1) } /// What unwinds whatever asks for a store that is not held once the /// shutdown has closed the accounts' logs (`Slot::load`): a request is /// answered an internal error (`limits::serve`), and the server's own /// housekeeping ends there. It is no panic to the log, and no data is /// unrecoverable: the next start loads the store. pub(crate) struct ShuttingDown; /// What a load refused because the accounts' logs are closed answers. const SHUTTING_DOWN: &str = "the server is shutting down"; impl Slot { /// The slot of `account`, signing with `key`, holding `store` if given. pub fn new(account: Weak, key: Key, ctx: Arc, store: Option) -> Slot { let slot = Slot { account, key, store: RwLock::new(None), used: AtomicU64::new(clock()), cost: AtomicU64::new(0), folding: AtomicU64::new(0), durable: AtomicBool::new(ctx.durable), looked: AtomicBool::new(store.is_some()), ctx, }; if let Some(st) = store { slot.compacted(&st); *crate::write(&slot.store) = Some(Box::new(st)); } slot } /// The account's signing key, its store loaded or not. pub fn signing_key(&self) -> &SigningKey { self.key.get() } /// Compact its store's log from now on, the table put in place under its lock. fn compacted(&self, st: &Store) { let fold = crate::checkpoint::AccountFold { did: st.repo.did.clone(), key: self.key.get().clone(), window: self.ctx.window, blocks: self.ctx.cache.fold_blocks(), account: self.account.clone(), }; st.wal.set_fold(Arc::new(fold)); } /// A compaction of its log holds `bytes` in memory now: counted with /// its store's when that is next counted. pub fn folding(&self, bytes: u64) { self.folding.store(bytes, Ordering::Relaxed); } /// Count a store made with the account in the cache, once the account is in its `Arc`. pub fn counted(&self) { let cost = crate::read(&self.store).as_ref().map_or(0, |st| st.cost()); self.recount(cost); } /// Whether its store is held. pub fn is_loaded(&self) -> bool { crate::read(&self.store).is_some() } /// Its events, each one's seq and when it was made, in order of seq, /// the greatest seq any event of its has had, and the greatest of those /// its log's compaction let go: as its store holds them, or, while it is not loaded, as its files do, read without /// loading it, which would make its log durable first and count it in /// the cache (`checkpoint::events_on_disk`). Only where its events are /// is learnt so: each is sent as its store reads it. pub fn events(&self) -> (Vec<(u64, u64)>, u64, u64) { { // Held while its files are read: nothing loads it meanwhile, so // nothing appends to its log or compacts it. let g = crate::read(&self.store); if let Some(st) = g.as_deref() { return (st.event_seqs(), st.last_seq, st.let_go()); } let found = self .account .upgrade() .map(|a| crate::checkpoint::events_on_disk(&self.ctx, &a.did)); if let Some(Ok(found)) = found { return found; } } // Files a start must mend, or the server stop for: as a load finds them. let st = self.read_lock(); (st.event_seqs(), st.last_seq, st.let_go()) } /// Its store's log, if its store is held: nothing is loaded to answer. /// A store not held has nothing written that is not on disk (`idle`). pub fn loaded_wal(&self) -> Option> { crate::read(&self.store).as_ref().map(|st| st.wal.clone()) } /// The bytes of its logs and its checkpoint (`ops::metrics`): as its /// store's log counts them, or, while it is not loaded, as its files /// hold them (`directory::files_bytes`), read without loading it. pub fn log_bytes(&self) -> u64 { // Held while its files are read: nothing loads it meanwhile. let g = crate::read(&self.store); if let Some(st) = g.as_deref() { return st.wal.log_bytes() + st.wal.checkpoint_bytes(); } match self.account.upgrade() { Some(a) => crate::directory::files_bytes(&self.ctx.dir, &a.did), None => 0, } } /// Load its store if it is not held: an error if the data directory /// holds what cannot be recovered. A store loaded after the start that /// recovery cut bytes off says so: the start's `recovered` line counts /// only the logs the start read (`ops::recovered`). /// /// None is loaded once the shutdown has closed the accounts' logs /// (`ops::settle`): a load is a write where it finds something to /// mend, and nothing would sync it before the shutdown's mark says /// that every log is durable. It is refused under the slot's lock, /// before anything of the account's files is read or changed, and the /// shutdown closes each store held under that lock too, after it has /// closed the logs: so a load either began before, and its log is /// closed, and synced, with the others, or is refused. pub fn load(&self) -> Result<(), String> { self.used.store(clock(), Ordering::Relaxed); let mut g = crate::write(&self.store); if g.is_some() { return Ok(()); } if self.ctx.files.is_closed() { return Err(SHUTTING_DOWN.into()); } // Its log is this load's from here on, whatever becomes of it. let durable = self.durable.swap(false, Ordering::SeqCst); let st = self.open(durable).inspect_err(|_| { // Noted under the slot's lock, which a shutdown waits for. self.ctx.failed.store(true, Ordering::SeqCst); })?; let dropped = st.wal.dropped(); // First loaded in this run: its log may hold a write a power cut // kept whose event the index lost, whatever start this is, as a // start that followed a shutdown may follow a run that never used it. if !self.looked.swap(true, Ordering::SeqCst) { self.ctx.events.loaded(&self.account, || st.event_seqs()); } if dropped > 0 && !self.ctx.starting.load(Ordering::SeqCst) { crate::ops::warn( "recovered an account's log", &[ ("did", serde_json::json!(st.repo.did)), ("dropped_bytes", serde_json::json!(dropped)), ], ); } let cost = st.cost(); *g = Some(Box::new(st)); drop(g); self.recount(cost); Ok(()) } /// Its store as its checkpoint and log have it, its first commit made /// if it has none; `durable` if every frame its log's files hold is on /// disk already. fn open(&self, durable: bool) -> Result { let a = self.account.upgrade().ok_or("an account no longer held")?; let c = &self.ctx; let mut r = crate::log::open_repo(c, &a.did, self.key.get(), durable)?; self.compacted(&r.store); if r.fresh && !c.starting.load(Ordering::SeqCst) { // With no events if it was made deactivated, moving here. let quiet = crate::read(&a.state).deactivated; let landing = crate::log::first_commit(&c.events, &a.did, &a.handle(), &mut r.store, quiet) .map_err(|e| format!("{}: {e}", a.did))?; landing .as_ref() .map_or(Ok(()), crate::events::Landing::wait_entry) .and_then(|()| r.store.wal.wait(r.store.wal.written())) .map_err(|e| format!("{}: {e}", a.did))?; if let Some(l) = landing { c.events.land(l); } } Ok(r.store) } /// Load its store, for a caller with no error to give. A load refused /// because the server is shutting down unwinds the caller /// (`ShuttingDown`), with nothing of the account's files touched. Any /// other that fails stops the server: one the shutdown overtook too, /// which may have mended part of its log, so that the shutdown leaves /// no mark. fn ensure(&self) { if let Err(why) = self.load() { if why == SHUTTING_DOWN { std::panic::resume_unwind(Box::new(ShuttingDown)); } let did = self .account .upgrade() .map(|a| a.did.clone()) .unwrap_or_default(); unrecoverable(&self.ctx.dir.join("repos").join(did), &why); } } /// The store's cost now `cost`: counted in the cache, with what a /// compaction of its log holds. fn recount(&self, cost: u64) { let cost = match cost { 0 => 0, held => held + self.folding.load(Ordering::Relaxed), }; let before = self.cost.swap(cost, Ordering::Relaxed); if before == cost { return; } let Some(a) = self.account.upgrade() else { return; }; if cost == 0 { self.ctx.cache.forget(&a.did, before); } else { self.ctx.cache.count(&a, cost as i64 - before as i64); } } /// Let go of its store if nothing of it is in flight: whether it did. pub fn evict(&self) -> bool { let mut g = match self.store.try_write() { Ok(g) => g, Err(std::sync::TryLockError::Poisoned(p)) => p.into_inner(), Err(std::sync::TryLockError::WouldBlock) => return false, }; if !g.as_ref().is_some_and(|st| st.idle()) { return false; } *g = None; // Let go with all it wrote on disk (`idle`), under the lock a load takes. self.durable.store(true, Ordering::SeqCst); drop(g); self.recount(0); true } /// Put a compaction's table in place under the store's lock: `f` is /// given the store, if it is held, to read the table from then on. pub fn installing( &self, f: impl FnOnce(Option<&mut Store>) -> Result<(), String>, ) -> Result<(), String> { let mut g = crate::write(&self.store); let done = f(g.as_deref_mut()); let cost = g.as_ref().map_or(0, |st| st.cost()); drop(g); self.recount(cost); done } } impl Locked for Slot { type Read<'a> = StoreRead<'a>; type Write<'a> = StoreWrite<'a>; fn read_lock(&self) -> StoreRead<'_> { loop { self.used.store(clock(), Ordering::Relaxed); let g = crate::read(&self.store); if g.is_some() { return StoreRead(g); } drop(g); self.ensure(); } } /// A writer of a store whose log is compacted waits for the compaction, /// which lets go of what it folded, once the store's memtable counts /// more than `Cache::memtable_most`, or its log holds more than /// `store::BEHIND` past what compacts it by size (`store::policy`): what /// a store being written holds, and what its first use after a start /// replays of its log, do not follow how fast the writes come /// (`spec/STORE.md`, "Bounds", "The first request to a cold account"). /// The wait gives up its place among the runtime's workers, so that /// other requests go on meanwhile. fn write_lock(&self) -> StoreWrite<'_> { let most = self.ctx.cache.memtable_most(); let behind = |wal: &Wal| { wal.log_bytes() > self .ctx .policy .limit(wal.checkpoint_bytes()) .saturating_add(crate::store::BEHIND) }; loop { self.used.store(clock(), Ordering::Relaxed); let g = crate::write(&self.store); match g.as_deref() { Some(st) if st.wal.compacting() && (st.disk.mem.cost() > most || behind(&st.wal)) => { let wal = st.wal.clone(); drop(g); off_worker(|| wal.wait_compacted()); } Some(_) => { return StoreWrite { guard: Some(g), slot: self, }; } None => { drop(g); self.ensure(); } } } } } /// Run `wait`, which may block for long, without holding one of a /// multi-threaded runtime's workers through it. fn off_worker(wait: impl FnOnce()) { use tokio::runtime::{Handle, RuntimeFlavor}; match Handle::try_current() { Ok(h) if h.runtime_flavor() == RuntimeFlavor::MultiThread => { tokio::task::block_in_place(wait) } _ => wait(), } } #[cfg(test)] mod tests { use super::*; use crate::testing::*; use serde_json::json; impl Cache { /// Let go of every store nothing of which is in flight. fn empty(&self) { self.shrink(None, 0, u64::MAX); } } /// A server that loads each account on first use, with a cache of `bytes`. fn lazy(dir: &std::path::Path, plc: &str, bytes: u64) -> crate::Shared { open_with(dir, plc, |c| { c.load_all = false; c.cache_bytes = bytes; }) } fn loaded(s: &crate::Service) -> usize { crate::read(&s.directory) .by_did .values() .filter(|a| a.store.is_loaded()) .count() } async fn records(s: &crate::Shared, did: &str) -> Vec { let p = [ ("repo".to_string(), did.to_string()), ("collection".to_string(), "a.b.c".to_string()), ("limit".to_string(), "100".to_string()), ]; let got = call_with(s, "com.atproto.repo.listRecords", None, &p, Json::Null) .await .unwrap(); let mut v: Vec = got["records"] .as_array() .unwrap() .iter() .map(|r| r["value"].clone()) .collect(); v.sort_by_key(|r| r["n"].as_i64()); v } /// Stores let go while writes to them go on side by side, as often as /// the cache allows (it holds nothing): every write answered is there, /// read back, after a restart too, and on the firehose in order. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn evicted_under_concurrent_writes() { let plc = plc().await; let dir = scratch("evict-writes"); let s = lazy(&dir, &plc, 0); let mut accounts = Vec::new(); for i in 0..6 { accounts.push(create(&s, &format!("evict{i}")).await); } let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); let evicting = { let (s, stop) = (s.clone(), stop.clone()); std::thread::spawn(move || { let mut n = 0; while !stop.load(Ordering::SeqCst) { s.ctx.cache.empty(); n += 1; } n }) }; let mut tasks = Vec::new(); for (did, token) in accounts.clone() { let s = s.clone(); tasks.push(tokio::spawn(async move { for n in 0..15 { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": format!("r{n}"), "record": record(n) }); call(&s, "com.atproto.repo.putRecord", Some(&token), body).await.unwrap(); // Read back at once, loaded again if it went. let p = [("repo".to_string(), did.clone()), ("collection".to_string(), "a.b.c".to_string()), ("rkey".to_string(), format!("r{n}"))]; let got = call_with(&s, "com.atproto.repo.getRecord", None, &p, Json::Null).await.unwrap(); assert_eq!(got["value"]["n"], json!(n)); } })); } for t in tasks { t.await.unwrap(); } stop.store(true, Ordering::SeqCst); assert!(evicting.join().unwrap() > 0); s.ctx.cache.empty(); assert_eq!(loaded(&s), 0, "every store let go once idle"); let want: Vec = (0..15).map(record).collect(); for (did, _) in &accounts { assert_eq!(records(&s, did).await, want); } let events = s.events.get(&s, 0, u64::MAX - 1, 0); assert_eq!(events.len(), 6 * (4 + 15)); drop(s); let s = lazy(&dir, &plc, 0); assert_eq!(loaded(&s), 0, "a start loads no account"); for (did, _) in &accounts { assert_eq!(records(&s, did).await, want); } assert_eq!(s.events.get(&s, 0, u64::MAX - 1, 0), events); let _ = std::fs::remove_dir_all(&dir); } /// Killed with a write cut short in an account's log, the server starts /// without reading it; the account's first request reads it, as before /// the write; and the cache holds no more than it may. #[tokio::test] async fn cold_loads_after_a_crash() { let plc = plc().await; let dir = scratch("cold-crash"); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; for n in 0..5 { for (did, t) in [(&alice, &ta), (&bob, &tb)] { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": format!("r{n}"), "record": record(n) }); call(&s, "com.atproto.repo.putRecord", Some(t), body) .await .unwrap(); } } let before = records(&s, &alice).await; let body = json!({ "repo": alice, "collection": "a.b.c", "rkey": "late", "record": record(9) }); call(&s, "com.atproto.repo.putRecord", Some(&ta), body) .await .unwrap(); let events = s.events.get(&s, 0, u64::MAX - 1, 0); drop(s); redone(&dir); // The last write cut short, as a kill during it leaves it. let log = crate::log::repo_log(&dir, &alice); let bytes = crate::log::log_bytes(&log); let last = bytes.len() - last_frame_at(&bytes); std::fs::write(&log, &bytes[..bytes.len() - last / 2]).unwrap(); let s = lazy(&dir, &plc, 1 << 30); assert_eq!(loaded(&s), 0); assert_eq!(records(&s, &alice).await, before); assert_eq!(loaded(&s), 1); assert_eq!( records(&s, &bob).await, (0..5).map(record).collect::>() ); // The events before the start are found in the stores; the lost one is not among them. let now = s.events.get(&s, 0, u64::MAX - 1, 0); assert_eq!(now[..], events[..events.len() - 1]); // New events go on after every seq reserved. let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": record(7) }); call(&s, "com.atproto.repo.putRecord", Some(&tb), body) .await .unwrap(); assert_eq!(s.events.get(&s, 0, u64::MAX - 1, 0).len(), now.len() + 1); assert!(s.ctx.cache.bytes() > 0 && s.ctx.cache.len() == 2); let _ = std::fs::remove_dir_all(&dir); } /// An account's events are read from its files, no store loaded for /// it: those a table holds and those of the log after it, as the store /// answers them once loaded, the event of a write cut short left out. /// The firehose learns them from its index, no store loaded either, and /// sends each as its store reads it; the entry of the write cut short /// is let go as its frame is found missing, its seq the last numbered /// still, whatever seqs were reserved past it. #[tokio::test] async fn events_found_without_loading() { let plc = plc().await; let dir = scratch("cold-events"); let s = lazy(&dir, &plc, 1 << 30); let mut accounts = Vec::new(); for i in 0..12 { let (did, token) = create(&s, &format!("found{i}")).await; for n in 0..3 { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": format!("r{n}"), "record": record(n) }); call(&s, "com.atproto.repo.putRecord", Some(&token), body) .await .unwrap(); if (i, n) == (0, 1) { // The first has a table, and a log after it. let wal = crate::read(&s.account(&did).unwrap().store).wal.clone(); wal.wait_compacted(); wal.compact().unwrap(); } } accounts.push(did); } let sent = s.events.get(&s, 0, u64::MAX - 1, 0); assert_eq!(sent.len(), 12 * 7); let held: Vec> = accounts .iter() .map(|did| s.account(did).unwrap().store.events().0) .collect(); assert!(held.iter().all(|e| e.len() == 7)); drop(s); redone(&dir); // The last account's last write cut short, as a kill during it leaves it. let log = crate::log::repo_log(&dir, &accounts[11]); let bytes = crate::log::log_bytes(&log); std::fs::write(&log, &bytes[..bytes.len() - 100]).unwrap(); let s = lazy(&dir, &plc, 1 << 30); for (did, held) in accounts.iter().zip(&held) { let (found, last, _) = s.account(did).unwrap().store.events(); assert_eq!(found[..], held[..if *did == accounts[11] { 6 } else { 7 }]); assert_eq!(Some(last), found.last().map(|e| e.0)); } let (published, last) = (*s.events.subscribe().borrow(), held[11][5].0); assert!( published > last + 1, "seqs are reserved past the last event" ); // The index has the write cut short: numbered, never to be sent. s.events.find_old(); assert_eq!((loaded(&s), s.events.held_len()), (0, sent.len())); assert_eq!(s.events.last_made(published), held[11][6].0); assert_eq!( std::fs::read(&log).unwrap().len(), bytes.len() - 100, "only read" ); assert_eq!( s.events.get(&s, 0, u64::MAX - 1, 0)[..], sent[..sent.len() - 1] ); assert_eq!(loaded(&s), 12); assert_eq!(s.events.held_len(), sent.len() - 1); // Loaded, a store answers the same. for (did, held) in accounts.iter().zip(&held).take(11) { assert_eq!( s.account(did).unwrap().store.events(), (held.clone(), held[6].0, 0) ); } // An event numbered since the start is the last made. let (did, token) = create(&s, "found12").await; let published = *s.events.subscribe().borrow(); assert_eq!( ( s.events.last_made(published), s.account(&did).unwrap().store.events().1 ), (published, published) ); drop(token); let _ = std::fs::remove_dir_all(&dir); } /// `spec/STORE.md`, "The first request to a cold account": an account's /// log is compacted once it holds the cap (`Config::account_cap`) past /// its table, however large the table: written past it twice over since /// its last compaction, with a table several times it, it is left /// holding no more, and its first load after a start replays no more of /// it, and reads little of the table. At a small cap, so that its data /// directory stays within a couple of MB. #[tokio::test] async fn a_first_load_replays_a_capped_log() { const CAP: u64 = 32 << 10; let plc = plc().await; let dir = scratch("cold-capped"); let start = || { open_with(&dir, &plc, |c| { (c.load_all, c.cache_bytes) = (false, 1 << 30); (c.log_floor, c.account_cap) = (2048, CAP); }) }; let s = start(); let (alice, ta) = create(&s, "alice").await; let big = |n: usize| json!({ "$type": "a.b.c", "text": format!("{n}{}", "e".repeat(1_000)) }); let write = async |s: &crate::Shared, n: usize| { let body = json!({ "repo": alice, "collection": "a.b.c", "rkey": format!("r{n}"), "record": big(n) }); call(s, "com.atproto.repo.putRecord", Some(&ta), body) .await .unwrap(); }; // A table of several times the cap: the log compacted as it passes // the table's size, while that is less than the cap. let wal = crate::read(&s.account(&alice).unwrap().store).wal.clone(); let mut n = 0; while wal.checkpoint_bytes() <= 4 * CAP { write(&s, n).await; n += 1; } compact(&s, &alice); wal.wait_compacted(); let generation = wal.checkpoint_generation(); // Written past the cap twice over since: each write's frame holds // its record, of 1000 bytes and more. for n in n..n + (2 * CAP / 1000 + 1) as usize { write(&s, n).await; } // Once no compaction runs: one that ends begins the next if due. loop { wal.wait_compacted(); tokio::time::sleep(Duration::from_millis(20)).await; if !wal.compacting() { break; } } let log = wal.log_bytes(); assert!( wal.checkpoint_generation() > generation && log <= CAP, "a log of {log} bytes past a table of {}", wal.checkpoint_bytes() ); drop(wal); shut_down(s); let s = start(); s.events.find_old(); let a = s.account(&alice).unwrap(); let checkpoint = std::fs::metadata(crate::store::account_layout(&dir, &alice).checkpoint()) .unwrap() .len(); let replayed = crate::directory::files_bytes(&dir, &alice) - checkpoint; let before = s.ctx.blocks.read_bytes(); a.store.load().unwrap(); let read = s.ctx.blocks.read_bytes() - before; assert!( replayed <= CAP && read < checkpoint / 4, "its first load replayed {replayed} bytes of log, read {read} of a table of {checkpoint}" ); let _ = std::fs::remove_dir_all(&dir); } /// After a start with every event out of the backfill window, a /// subscriber at the last event made has missed none, though seqs are /// reserved past it; one before it has. #[tokio::test] async fn the_last_event_is_not_outdated() { let plc = plc().await; let dir = scratch("cold-window"); let open = || { open_with(&dir, &plc, |c| { c.load_all = false; c.backfill = std::time::Duration::from_millis(300); }) }; let s = open(); create(&s, "alice").await; let last = *s.events.subscribe().borrow(); drop(s); tokio::time::sleep(std::time::Duration::from_millis(400)).await; let s = open(); let published = *s.events.subscribe().borrow(); s.events.find_old(); assert!(published > last); assert_eq!( (s.events.last_made(published), s.events.held_len()), (last, 0) ); assert!(!s.events.outdated(last, published)); assert!(s.events.outdated(last - 1, published)); let _ = std::fs::remove_dir_all(&dir); } /// The events let go are real ones, not the seqs before the first kept: /// after a start, seqs are reserved past the last event, and a write /// since is numbered after them. A subscriber at the last event before /// that start has missed none; one before it has missed that event, /// which its log's compaction let go, and which only the table's state /// still tells of. #[tokio::test] async fn seqs_reserved_are_no_events_let_go() { let plc = plc().await; let dir = scratch("cold-holes"); let open = || { open_with(&dir, &plc, |c| { c.load_all = false; c.backfill = std::time::Duration::from_millis(1500); }) }; let s = open(); let (did, token) = create(&s, "alice").await; let old = *s.events.subscribe().borrow(); drop(s); // Out of the window, and of what a compaction keeps past it. tokio::time::sleep(std::time::Duration::from_millis(2700)).await; let s = open(); let body = json!({ "repo": did, "collection": "a.b.c", "rkey": "r", "record": record(1) }); call(&s, "com.atproto.repo.putRecord", Some(&token), body) .await .unwrap(); let new = *s.events.subscribe().borrow(); assert!(new > old + 2, "numbered after the seqs reserved"); let wal = crate::read(&s.account(&did).unwrap().store).wal.clone(); wal.wait_compacted(); wal.compact().unwrap(); assert_eq!( s.account(&did).unwrap().store.events(), ( vec![(new, s.account(&did).unwrap().store.events().0[0].1)], new, old ) ); drop(s); let s = open(); let published = *s.events.subscribe().borrow(); assert_eq!(s.account(&did).unwrap().store.events().2, old); s.events.find_old(); assert_eq!( ( loaded(&s), s.events.held_len(), s.events.last_made(published) ), (0, 1, new) ); assert!( !s.events.outdated(old, published), "the next event after it is in the window" ); assert!( s.events.outdated(old - 1, published), "the event after it was let go" ); let _ = std::fs::remove_dir_all(&dir); } /// An upload that is its account's first request after a start is /// stored: its store, loaded while the upload's bytes are on their way /// in, removes the uploads a process before left on theirs, not this one. #[tokio::test] async fn an_upload_loads_its_store() { let plc = plc().await; let dir = scratch("cold-upload"); let s = lazy(&dir, &plc, 1 << 30); let (did, token) = create(&s, "alice").await; drop(s); let blobs = crate::blobs::blob_dir(&dir, &did); std::fs::create_dir_all(&blobs).unwrap(); std::fs::write(blobs.join(".tmp-left"), b"half").unwrap(); let s = lazy(&dir, &plc, 1 << 30); assert_eq!(loaded(&s), 0); let body = b"an upload to an account not loaded".to_vec(); let got = crate::blobs::test_upload(&s, Some(&token), Some("text/plain"), body.clone(), false) .await .unwrap(); assert_eq!((loaded(&s), &got["blob"]["size"]), (1, &json!(body.len()))); let files: Vec = std::fs::read_dir(&blobs) .unwrap() .map(|e| e.unwrap().file_name().into_string().unwrap()) .collect(); assert_eq!(files, [got["blob"]["ref"]["$link"].as_str().unwrap()]); let _ = std::fs::remove_dir_all(&dir); } /// A start loads the accounts marked: one whose handle change is in /// flight is loaded, its change held and settled; one settled is not. #[tokio::test] async fn a_start_loads_the_accounts_marked() { let plc = crate::identity::stub::Plc::start().await; let dir = scratch("cold-marked"); let s = lazy(&dir, &plc.url, 1 << 30); let (alice, ta) = create(&s, "alice").await; create(&s, "bob").await; let body = json!({ "handle": "carol.test" }); call(&s, "com.atproto.identity.updateHandle", Some(&ta), body) .await .unwrap(); drop(s); let s = lazy(&dir, &plc.url, 1 << 30); assert_eq!(loaded(&s), 0, "a change settled leaves no mark"); assert_eq!(s.account("carol.test").unwrap().did, alice); drop(s); crate::identity::tests::stop_before_settling(&dir, &alice); let s = lazy(&dir, &plc.url, 1 << 30); assert_eq!(loaded(&s), 1); assert!( crate::read(&s.directory) .claimed_handles .contains("carol.test") ); crate::identity::settle_unfinished(&s); for _ in 0..100 { if s.account("carol.test").is_some() { break; } tokio::time::sleep(std::time::Duration::from_millis(20)).await; } assert_eq!(s.account("carol.test").unwrap().did, alice); drop(s); let s = lazy(&dir, &plc.url, 1 << 30); assert_eq!( (loaded(&s), s.account("carol.test").unwrap().did.clone()), (0, alice) ); let _ = std::fs::remove_dir_all(&dir); } /// The memtable of `did`'s store: what it counts, and whether its log /// is being compacted. fn memtable(s: &crate::Service, did: &str) -> (u64, bool) { store_of(s, did, |st| (st.disk.mem.cost(), st.wal.compacting())) } /// A batch of space records written to `did`'s repo in a space of /// another's, each of a key no other has. async fn space_batch(s: &crate::Shared, did: &str, token: &str, batch: usize) { let space = "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/space/com.example.forum/main"; let writes: Vec = (0..50) .map(|i| json!({ "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "rkey": format!("b{batch:04}r{i:02}"), "value": record(i) })) .collect(); let body = json!({ "space": space, "repo": did, "writes": writes }); call(s, "com.atproto.space.applyWrites", Some(token), body) .await .unwrap(); } /// A batch of records written to `did`'s repository, each of a key no /// other has. async fn repo_batch(s: &crate::Shared, did: &str, token: &str, batch: usize) { let writes: Vec = (0..50) .map(|i| json!({ "$type": "com.atproto.repo.applyWrites#create", "collection": "a.b.c", "rkey": format!("b{batch:04}r{i:02}"), "value": record(i) })) .collect(); let body = json!({ "repo": did, "writes": writes }); call(s, "com.atproto.repo.applyWrites", Some(token), body) .await .unwrap(); } /// A store's log is compacted for what its memtable counts, whatever /// the log's size: written without a pause, the memtable never counts /// more than `memtable_most` and one write, the store with a /// compaction's share no more than the cache gives a store being /// written, and every record is read back, after a start too. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_store_being_written_stays_within_its_share() { let plc = plc().await; let dir = scratch("memtable-share"); // Never compacted for its log's size. let opened = || { open_with(&dir, &plc, |c| { (c.load_all, c.cache_bytes, c.log_floor) = (false, 2 << 20, 1 << 30) }) }; let s = opened(); let (did, token) = create(&s, "alice").await; let (soft, most) = (s.ctx.cache.memtable(), s.ctx.cache.memtable_most()); assert_eq!((soft, most), (256 << 10, 512 << 10)); space_batch(&s, &did, &token, 0).await; let one = memtable(&s, &did).0; let (mut peak, mut compactions, mut last) = (0, 0, 0); for batch in 1..120 { space_batch(&s, &did, &token, batch).await; let (cost, _) = memtable(&s, &did); peak = peak.max(cost); assert!(cost <= most + one, "batch {batch}: {cost}"); let generation = store_of(&s, &did, |st| st.wal.checkpoint_generation()); compactions += u64::from(generation != last); last = generation; } assert!(peak > soft, "{peak}"); assert!(compactions >= 4, "{compactions}"); let wal = store_of(&s, &did, |st| st.wal.clone()); wal.wait_compacted(); assert!(wal.log_bytes() < 1 << 30); drop(wal); let listed = |s: &crate::Shared| { let (s, did, token) = (s.clone(), did.clone(), token.clone()); async move { let space = "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/space/com.example.forum/main"; let mut n = 0; let mut cursor: Option = None; loop { let mut p = vec![ ("space".to_string(), space.to_string()), ("repo".to_string(), did.clone()), ("limit".to_string(), "1000".to_string()), ]; p.extend(cursor.take().map(|c| ("cursor".to_string(), c))); let got = call_with( &s, "com.atproto.space.listRecords", Some(&token), &p, Json::Null, ) .await .unwrap(); n += got["records"].as_array().unwrap().len(); match got["cursor"].as_str() { Some(c) => cursor = Some(c.to_string()), None => return n, } } } }; assert_eq!(listed(&s).await, 120 * 50); crate::space_notify::tests::idle(&s).await; drop(s); assert_eq!(listed(&opened()).await, 120 * 50); let _ = std::fs::remove_dir_all(&dir); } /// An account's fold that waits for word before it folds, the first /// time; then folds as the account's does. struct Waits { fold: crate::checkpoint::AccountFold, word: Mutex>, } impl pds_store::Fold for Waits { fn fold_files( &self, checkpoint: Option<&std::path::Path>, logs: &[PathBuf], generation: u64, out: &mut dyn std::io::Write, ) -> Result<(), String> { let _ = lock(&self.word).recv(); self.fold.fold_files(checkpoint, logs, generation, out) } fn install( &self, generation: u64, rename: &mut dyn FnMut() -> Result<(), String>, ) -> Result<(), String> { self.fold.install(generation, rename) } } /// A compaction that cannot keep up: while it runs, writes go on until /// the memtable counts more than `memtable_most`; the next writer waits /// for the compaction, and writes once it has let go of what it folded. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_writer_waits_for_a_compaction_that_cannot_keep_up() { let plc = plc().await; let dir = scratch("memtable-waits"); let s = open_with(&dir, &plc, |c| { (c.load_all, c.cache_bytes, c.log_floor) = (false, 2 << 20, 1 << 30) }); let (did, token) = create(&s, "alice").await; let a = s.account(&did).unwrap(); let (go, word) = std::sync::mpsc::channel(); store_of(&s, &did, |st| { let fold = crate::checkpoint::AccountFold { did: did.clone(), key: a.store.key.get().clone(), window: s.ctx.window, blocks: s.ctx.cache.fold_blocks(), account: Arc::downgrade(&a), }; st.wal.set_fold(Arc::new(Waits { fold, word: Mutex::new(word), })); }); let most = s.ctx.cache.memtable_most(); // Past its share, a compaction starts, and waits; writes go on. let mut batch = 0; while memtable(&s, &did).0 <= most { repo_batch(&s, &did, &token, batch).await; batch += 1; } assert!(memtable(&s, &did).1, "no compaction started"); let held = memtable(&s, &did).0; // The next writer waits. let writing = { let (s, did, token) = (s.clone(), did.clone(), token.clone()); tokio::spawn(async move { repo_batch(&s, &did, &token, batch).await }) }; tokio::time::sleep(Duration::from_millis(300)).await; assert!(!writing.is_finished(), "a writer did not wait"); assert_eq!(memtable(&s, &did), (held, true)); // A read does not. let p = [("did".to_string(), did.clone())]; call_with(&s, "com.atproto.sync.getLatestCommit", None, &p, Json::Null) .await .unwrap(); go.send(()).unwrap(); drop(go); writing.await.unwrap(); // What the compaction folded is let go, and the write is there. assert!(memtable(&s, &did).0 < held); assert!(store_of(&s, &did, |st| st.wal.checkpoint_generation()) > 0); let _ = std::fs::remove_dir_all(&dir); } async fn put(s: &crate::Shared, did: &str, token: &str, rkey: &str, n: i64) { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) }); call(s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); } /// The syncs this thread makes loading the store of `did`, not held. fn load_syncs(s: &crate::Shared, did: &str) -> u64 { let a = s.account(did).unwrap(); assert!(!a.store.is_loaded()); let before = syncs(); a.store.load().unwrap(); syncs() - before } /// Compact the log of `did` now. fn compact(s: &crate::Shared, did: &str) { let wal = crate::read(&s.account(did).unwrap().store).wal.clone(); wal.wait_compacted(); wal.compact().unwrap(); } /// Let go of the store of `did`, once nothing of it is in flight. async fn let_go(s: &crate::Shared, did: &str) { for _ in 0..500 { if s.account(did).unwrap().store.evict() { return; } tokio::time::sleep(Duration::from_millis(10)).await; } panic!("the store of {did} is never idle"); } /// Where the last frame of a log's frames `bytes` begins. fn last_frame_at(bytes: &[u8]) -> usize { let mut last = 0; let end = pds_store::scan_each(bytes, |at, _| { last = at; Ok(()) }); assert_eq!(end, Ok(bytes.len())); last } fn numbered(n: std::ops::Range) -> Vec { n.map(record).collect() } /// A seal, as a compaction ends a log with. fn seal() -> Vec { pds_store::frame(&[json!({ "t": "seal" })]) } /// After a kill, an account's log is synced as its store is first /// loaded. After a shutdown of a run that had made durable what it /// found it is not, whether the account has a table or a log alone: /// what it reads is what was written, the firehose's events too, and /// its next write is kept. The shutdown's word does not outlive the run /// that read it: killed, the next start syncs every account's log /// again, one that run never used too. #[tokio::test] async fn after_a_shutdown_first_loads_sync_nothing() { let plc = plc().await; let dir = scratch("clean-first"); let start = || lazy(&dir, &plc, 1 << 30); let s = start(); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; let (carol, tc) = create(&s, "carol").await; for n in 0..3 { for (did, t) in [(&alice, &ta), (&bob, &tb), (&carol, &tc)] { put(&s, did, t, &format!("r{n}"), n).await; } } // One has a table, and a log after it. compact(&s, &alice); put(&s, &alice, &ta, "r3", 3).await; let events = s.events.get(&s, 0, u64::MAX - 1, 0); let all = [(&alice, 4), (&bob, 3), (&carol, 3)]; drop(s); // Killed: each log is synced as it is first used. assert!(crate::ops::unclean(&dir)); let s = start(); assert_eq!(loaded(&s), 0); for (did, n) in all { assert!(load_syncs(&s, did) >= 1, "{did} after a kill"); assert_eq!(records(&s, did).await, numbered(0..n)); } crate::ops::sync_found(&s).unwrap(); shut_down(s); // Shut down: none is, and the word is out of the mark. assert!(!crate::ops::unclean(&dir)); let s = start(); assert_eq!(std::fs::read(dir.join(crate::ops::CLEAN)).unwrap(), b""); assert_eq!(loaded(&s), 0); for (did, n) in [(&alice, 4), (&bob, 3)] { assert_eq!(load_syncs(&s, did), 0, "{did} after a shutdown"); assert_eq!(records(&s, did).await, numbered(0..n)); } // Carol's store is not loaded in this run. assert_eq!(loaded(&s), 2); put(&s, &alice, &ta, "r4", 4).await; put(&s, &bob, &tb, "r3", 3).await; assert_eq!(records(&s, &alice).await, numbered(0..5)); drop(s); // Killed since: every log is synced again, and the writes are there. let s = start(); for (did, n) in [(&alice, 5), (&bob, 4), (&carol, 3)] { assert!(load_syncs(&s, did) >= 1, "{did} after a kill once more"); assert_eq!(records(&s, did).await, numbered(0..n)); } assert_eq!(s.events.get(&s, 0, u64::MAX - 1, 0).len(), events.len() + 2); let _ = std::fs::remove_dir_all(&dir); } /// A store let go is loaded again with no sync, in a run that followed /// a kill too: it was let go with all it wrote on disk. Its first load /// of that run syncs, and what was written in between is read, and kept. #[tokio::test] async fn a_store_let_go_is_loaded_again_with_no_sync() { let plc = plc().await; let dir = scratch("clean-again"); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; for n in 0..3 { put(&s, &alice, &ta, &format!("r{n}"), n).await; } drop(s); let s = lazy(&dir, &plc, 1 << 30); assert!(load_syncs(&s, &alice) >= 1); let_go(&s, &alice).await; assert_eq!(load_syncs(&s, &alice), 0); assert_eq!(records(&s, &alice).await, numbered(0..3)); // Written, let go, loaded: and again once its log is compacted. put(&s, &alice, &ta, "r3", 3).await; let_go(&s, &alice).await; assert_eq!(load_syncs(&s, &alice), 0); assert_eq!(records(&s, &alice).await, numbered(0..4)); compact(&s, &alice); put(&s, &alice, &ta, "r4", 4).await; let_go(&s, &alice).await; assert_eq!(load_syncs(&s, &alice), 0); assert_eq!(records(&s, &alice).await, numbered(0..5)); // An account made in this run, its store never loaded from disk yet. let (bob, tb) = create(&s, "bob").await; put(&s, &bob, &tb, "r0", 0).await; let_go(&s, &bob).await; assert_eq!(load_syncs(&s, &bob), 0); assert_eq!(records(&s, &bob).await, numbered(0..1)); drop(s); let s = lazy(&dir, &plc, 1 << 30); assert!(load_syncs(&s, &alice) >= 1 && load_syncs(&s, &bob) >= 1); assert_eq!(records(&s, &alice).await, numbered(0..5)); assert_eq!(records(&s, &bob).await, numbered(0..1)); let _ = std::fs::remove_dir_all(&dir); } /// What a start that followed a shutdown finds to mend of an account's /// log, it mends, and syncs: a torn tail, the log a compaction cut /// short left under its table, the `.tmp` of one, a log that ends with /// its seal and none after it. Room is nothing to mend: a log with /// zeros after its frames, as a kill of a run before leaves one no run /// has loaded since, is cut to its frames with no sync, as an account /// with nothing to mend is loaded with none. #[tokio::test] async fn after_a_shutdown_what_is_mended_is_synced() { let plc = plc().await; let dir = scratch("clean-mend"); let s = lazy(&dir, &plc, 1 << 30); let mut made = Vec::new(); for name in ["alice", "bob", "carol", "dave", "erin", "frank"] { let (did, token) = create(&s, name).await; for n in 0..3 { put(&s, &did, &token, &format!("r{n}"), n).await; } made.push((did, token)); } let [alice, bob, carol, dave, erin, frank] = [0, 1, 2, 3, 4, 5].map(|i| made[i].0.clone()); let layout = |did: &str| crate::store::account_layout(&dir, did); // Bob's log folded into a table, and a write after it. let folded = std::fs::read(layout(&bob).log(0)).unwrap(); compact(&s, &bob); put(&s, &bob, &made[1].1, "r3", 3).await; assert!(!layout(&bob).log(0).exists()); shut_down(s); // Alice's last write cut short. let log = layout(&alice).log(0); let bytes = std::fs::read(&log).unwrap(); let last = bytes.len() - last_frame_at(&bytes); std::fs::write(&log, &bytes[..bytes.len() - last / 2]).unwrap(); // Bob's compaction cut short before it deleted the log it folded. std::fs::write(layout(&bob).log(0), [folded, seal()].concat()).unwrap(); // Carol's cut short while it wrote its table. std::fs::write(layout(&carol).tmp(), b"part of a table").unwrap(); // Erin's cut short once it had sealed her log. let sealed = [std::fs::read(layout(&erin).log(0)).unwrap(), seal()].concat(); std::fs::write(layout(&erin).log(0), sealed).unwrap(); // Frank's log with zeros after its frames: room, as a kill leaves it. let roomy = layout(&frank).log(0); let frames = std::fs::read(&roomy).unwrap(); assert_eq!(pds_store::log_len(&roomy).unwrap(), frames.len() as u64); std::fs::write(&roomy, [&frames[..], &[0u8; 300]].concat()).unwrap(); let s = lazy(&dir, &plc, 1 << 30); assert_eq!(load_syncs(&s, &dave), 0); assert!(load_syncs(&s, &alice) >= 1); assert_eq!( std::fs::metadata(&log).unwrap().len(), (bytes.len() - last) as u64 ); assert_eq!(records(&s, &alice).await, numbered(0..2)); assert!(load_syncs(&s, &bob) >= 2); assert!(!layout(&bob).log(0).exists()); assert_eq!(records(&s, &bob).await, numbered(0..4)); assert!(load_syncs(&s, &carol) >= 1); assert!(!layout(&carol).tmp().exists()); assert!(load_syncs(&s, &erin) >= 2); assert!(layout(&erin).log(1).exists()); assert_eq!(load_syncs(&s, &frank), 0); assert_eq!(std::fs::read(&roomy).unwrap(), frames); // Each goes on, and what it then holds is kept. for (i, (did, token)) in made.iter().enumerate() { put(&s, did, token, "late", 9).await; let before = match i { 0 => 2, 1 => 4, _ => 3, }; let mut want = numbered(0..before); want.push(record(9)); assert_eq!(records(&s, did).await, want); } drop(s); let s = lazy(&dir, &plc, 1 << 30); assert_eq!(records(&s, &alice).await.len(), 3); assert_eq!(records(&s, &bob).await.len(), 5); assert_eq!(records(&s, &erin).await.len(), 4); let _ = std::fs::remove_dir_all(&dir); } /// A shutdown that did not finish is as a kill to the next start, which /// syncs each log as it is first used: killed once its logs were synced, /// before its mark; one that could not sync a log, which leaves no /// mark; and a mark with no word, as a version before left. #[tokio::test] async fn a_shutdown_not_finished_is_as_a_kill() { let plc = plc().await; let dir = scratch("clean-unfinished"); let start = || lazy(&dir, &plc, 1 << 30); let mark = dir.join(crate::ops::CLEAN); let s = start(); let (alice, ta) = create(&s, "alice").await; put(&s, &alice, &ta, "r0", 0).await; let events = s.events.get(&s, 0, u64::MAX - 1, 0); // Killed before the mark. s.events.close(); crate::ops::settle(&s).unwrap(); drop(s); assert!(!mark.exists() && crate::ops::unclean(&dir)); let s = start(); s.events.find_old(); assert!(load_syncs(&s, &alice) >= 1); assert_eq!(s.events.get(&s, 0, u64::MAX - 1, 0), events); // A log that cannot be synced: it holds an upload's entry no one // waited for, and is to sync first a file that cannot be opened. crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), b"kept".to_vec(), false) .await .unwrap(); store_of(&s, &alice, |st| { assert!(!st.wal.is_durable(st.wal.written())); st.wal.sync_first([dir.join("jwt.key").join("x")]); }); s.events.close(); assert!(crate::ops::settle(&s).is_err()); drop(s); assert!(!mark.exists()); let s = start(); assert!(load_syncs(&s, &alice) >= 1); assert_eq!(records(&s, &alice).await, numbered(0..1)); drop(s); // The mark of a version before. std::fs::write(&mark, b"").unwrap(); assert!(!crate::ops::unclean(&dir)); let s = start(); assert!(load_syncs(&s, &alice) >= 1); assert_eq!(records(&s, &alice).await, numbered(0..1)); let _ = std::fs::remove_dir_all(&dir); } /// Once a shutdown has synced the logs none takes a write: one to a /// store held and one to the service log are each refused, and a store /// not held is not loaded; the logs are left as they were. So the /// start after it reads what the shutdown synced, and syncs nothing. #[tokio::test] async fn a_shutdown_closes_the_logs() { let plc = plc().await; let dir = scratch("clean-closed"); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; let (bob, _) = create(&s, "bob").await; put(&s, &alice, &ta, "r0", 0).await; // An upload's entry is not waited for: the shutdown syncs it. let upload = |token: &str, body: &[u8]| { let (s, token, body) = (s.clone(), token.to_string(), body.to_vec()); async move { crate::blobs::test_upload(&s, Some(&token), Some("text/plain"), body, false).await } }; upload(&ta, b"before").await.unwrap(); let_go(&s, &bob).await; s.events.close(); crate::ops::settle(&s).unwrap(); store_of(&s, &alice, |st| { assert!(st.wal.is_durable(st.wal.written())) }); let logs = || { [ crate::store::service_layout(&dir).log(0), crate::log::repo_log(&dir, &alice), crate::log::repo_log(&dir, &bob), ] .map(|p| std::fs::read(p).unwrap()) }; let left = logs(); assert!(upload(&ta, b"after, to a store held").await.is_err()); let b = s.account(&bob).unwrap(); assert!(!b.store.is_loaded() && b.store.load().is_err()); drop(b); let body = json!({ "identifier": "alice.test", "password": "alice-pw" }); assert!( call(&s, "com.atproto.server.createSession", None, body) .await .is_err() ); assert_eq!(logs(), left); crate::ops::mark(&s).unwrap(); drop(s); let s = lazy(&dir, &plc, 1 << 30); assert_eq!((load_syncs(&s, &alice), load_syncs(&s, &bob)), (0, 0)); assert_eq!(records(&s, &alice).await, numbered(0..1)); session(&s, "alice").await; let _ = std::fs::remove_dir_all(&dir); } /// Once a shutdown has closed the logs no store is loaded: a load of /// an account not held is refused, whatever asks for its store unwinds, /// a request for it is answered an internal error, and nothing of its /// files is made, mended or synced, though its log has something to /// mend, a seal and no log after it. A store held is read as before. /// The start after the shutdown mends the log, and syncs. #[tokio::test] async fn a_shutdown_loads_no_store() { let plc = plc().await; let dir = scratch("clean-no-load"); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; put(&s, &alice, &ta, "r0", 0).await; put(&s, &bob, &tb, "r0", 0).await; shut_down(s); // Bob's compaction was cut short once it had sealed his log. let layout = crate::store::account_layout(&dir, &bob); let sealed = [std::fs::read(layout.log(0)).unwrap(), seal()].concat(); std::fs::write(layout.log(0), sealed).unwrap(); let files = || { let mut files: Vec<(String, Vec)> = std::fs::read_dir(&layout.dir) .unwrap() .map(|e| e.unwrap()) .filter(|e| e.file_type().unwrap().is_file()) .map(|e| { ( e.file_name().into_string().unwrap(), std::fs::read(e.path()).unwrap(), ) }) .collect(); files.sort(); files }; let found = files(); let s = lazy(&dir, &plc, 1 << 30); assert_eq!(load_syncs(&s, &alice), 0); let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await .unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(crate::limits::serve( listener, crate::router(s.clone()), std::future::pending::<()>(), )); let latest = |did: &str| { let url = format!("http://{addr}/xrpc/com.atproto.sync.getLatestCommit?did={did}"); async move { reqwest::get(url).await.unwrap().status().as_u16() } }; assert_eq!(latest(&alice).await, 200); s.events.close(); crate::ops::settle(&s).unwrap(); let b = s.account(&bob).unwrap(); let before = syncs(); assert_eq!(b.store.load().unwrap_err(), SHUTTING_DOWN); let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let _ = crate::read(&b.store); })) .unwrap_err(); assert!(unwound.is::()); assert_eq!((latest(&bob).await, latest(&alice).await), (500, 200)); assert!(!b.store.is_loaded()); assert_eq!((syncs(), files()), (before, found)); drop(b); crate::ops::mark(&s).unwrap(); drop(s); let s = lazy(&dir, &plc, 1 << 30); assert!(load_syncs(&s, &bob) >= 2); assert!(layout.log(1).exists()); assert_eq!(records(&s, &bob).await, numbered(0..1)); let _ = std::fs::remove_dir_all(&dir); } /// A run in which a load of a store failed leaves no word at its /// shutdown: what that load had mended by then, nothing synced. The /// start after it syncs each log as it is first loaded. #[tokio::test] async fn a_load_that_failed_leaves_no_word() { let plc = plc().await; let dir = scratch("clean-failed-load"); let mark = || std::fs::read(dir.join(crate::ops::CLEAN)).unwrap(); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; put(&s, &alice, &ta, "r0", 0).await; put(&s, &bob, &tb, "r0", 0).await; shut_down(s); assert!(mark().starts_with(crate::ops::DURABLE)); // A log after one that is missing: nothing a load can mend. let stray = crate::store::account_layout(&dir, &alice).log(2); std::fs::write(&stray, b"").unwrap(); let s = lazy(&dir, &plc, 1 << 30); assert!(s.account(&alice).unwrap().store.load().is_err()); assert_eq!(load_syncs(&s, &bob), 0); shut_down(s); assert_eq!(mark(), b""); std::fs::remove_file(&stray).unwrap(); let s = lazy(&dir, &plc, 1 << 30); assert!(load_syncs(&s, &alice) >= 1 && load_syncs(&s, &bob) >= 1); assert_eq!(records(&s, &alice).await, numbered(0..1)); let _ = std::fs::remove_dir_all(&dir); } /// Copy every file of the directory `from` into `to`, made new. fn copy_all(from: &std::path::Path, to: &std::path::Path) { std::fs::create_dir_all(to).unwrap(); for e in std::fs::read_dir(from).unwrap() { let e = e.unwrap(); let to = to.join(e.file_name()); if e.file_type().unwrap().is_dir() { copy_all(&e.path(), &to); } else { std::fs::copy(e.path(), to).unwrap(); } } } /// The word is its directory's. A copy of a stopped server's directory, /// every file of it in a new one, holds the mark byte for byte, with /// its time, and not the word: a start on it syncs a log as it is /// first loaded, and makes durable what it found, as after a kill, /// though it is not said to have recovered. No word either in a mark /// written again since its shutdown wrote it, in one with a number /// altered, in the bare word, as nothing wrote it, or once `repos` has /// changed since the mark was left; and whatever a mark held is taken /// out of it. The directory the shutdown left has the word, and its /// first loads sync nothing. #[tokio::test] async fn the_word_is_its_directorys() { use crate::ops::{CLEAN, DURABLE, durable}; let plc = plc().await; let dir = scratch("clean-bound"); let copy = scratch("clean-bound-copy"); let found = |s: &crate::Shared| s.ctx.found.load(Ordering::SeqCst); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; put(&s, &alice, &ta, "r0", 0).await; shut_down(s); let mark = dir.join(CLEAN); let word = std::fs::read(&mark).unwrap(); let written = std::fs::metadata(&mark).unwrap().modified().unwrap(); let text = String::from_utf8(word.clone()).unwrap(); let lines: Vec<&str> = text.lines().collect(); assert_eq!(lines.len(), 4, "{text}"); assert!(word.starts_with(DURABLE) && text.ends_with('\n')); // Write a mark, as of `at` if given. let write = |mark: &std::path::Path, held: &[u8], at: Option| { std::fs::write(mark, held).unwrap(); if let Some(at) = at { let f = std::fs::File::options().write(true).open(mark).unwrap(); f.set_modified(at).unwrap(); } }; // A copy, which keeps the mark's time. copy_all(&dir, ©); write(©.join(CLEAN), &word, Some(written)); assert!(!crate::ops::unclean(©)); let c = lazy(©, &plc, 1 << 30); assert_eq!(std::fs::read(copy.join(CLEAN)).unwrap(), b""); assert!(!found(&c)); assert!(load_syncs(&c, &alice) >= 1, "a first load on a copy"); assert_eq!(records(&c, &alice).await, numbered(0..1)); crate::ops::sync_found(&c).unwrap(); shut_down(c); // Left by its own shutdown, the copy has a word of its own. let c = lazy(©, &plc, 1 << 30); assert!(found(&c)); assert_eq!(load_syncs(&c, &alice), 0); drop(c); // The mark written over, in place: the directory is as it was left. let ask = |held: &[u8], at: Option| { write(&mark, held, at); let word = durable(&dir).unwrap(); assert_eq!(std::fs::read(&mark).unwrap(), b"", "taken out"); word }; let asked = |held: &[u8]| ask(held, Some(written)); assert!(asked(&word), "the mark as it was left"); assert!(!ask(&word, None), "written since"); assert!(!asked(DURABLE), "the bare word"); assert!(!asked(b"")); assert!(!asked(&[&word[..], b"x"].concat())); assert!(!asked(&word[..word.len() - 1])); // Each number, altered. let mut altered = 0; for (l, line) in lines.iter().enumerate().skip(1) { let fields: Vec<&str> = line.split(' ').collect(); for f in 1..fields.len() { let mut other = fields.clone(); let last = if fields[f].ends_with('1') { "2" } else { "1" }; let changed = format!("{}{last}", &fields[f][..fields[f].len() - 1]); other[f] = &changed; let mut all = lines.clone(); let line = other.join(" "); all[l] = &line; assert!(!asked(format!("{}\n", all.join("\n")).as_bytes()), "{line}"); altered += 1; } } assert_eq!(altered, 7, "{text}"); assert!(asked(&word)); // `repos` changed since: its mode set, to what it was. tokio::time::sleep(Duration::from_millis(50)).await; let repos = dir.join("repos"); let mode = std::fs::metadata(&repos).unwrap().permissions(); std::fs::set_permissions(&repos, mode).unwrap(); assert!(!asked(&word)); // A start takes the word of its own directory, and no other's. let s = lazy(&dir, &plc, 1 << 30); assert!(!found(&s)); assert!(load_syncs(&s, &alice) >= 1); crate::ops::sync_found(&s).unwrap(); shut_down(s); let left = std::fs::read(&mark).unwrap(); let left_at = std::fs::metadata(&mark).unwrap().modified().unwrap(); assert!(left.starts_with(DURABLE) && left != word); // A start comes later than its shutdown by more than the grain of // the file system's times, a few milliseconds at most. tokio::time::sleep(Duration::from_millis(50)).await; let s = lazy(&dir, &plc, 1 << 30); assert_eq!(std::fs::read(&mark).unwrap(), b""); assert!(found(&s)); assert_eq!(load_syncs(&s, &alice), 0); assert_eq!(records(&s, &alice).await, numbered(0..1)); // The word a start took names the directory no more, put back. drop(s); assert!(!ask(&left, Some(left_at)) && !asked(&word)); for dir in [&dir, ©] { let _ = std::fs::remove_dir_all(dir); } } /// A log no run has loaded since a kill is synced by no load and by no /// shutdown. So a run that started after a kill leaves the word at its /// shutdown only once it has made durable what it found, in syncs /// counted as any are. Shut down before that, it leaves its mark /// without the word, and the next start syncs such a log as it is first /// loaded. Shut down after it, the next start syncs nothing for it, and /// reads what was written. That pass changes nothing for its own run's /// first loads, which sync as after a kill. #[tokio::test] async fn what_a_start_found_is_durable_before_the_word_is_left() { let plc = plc().await; let dir = scratch("clean-found"); let start = || lazy(&dir, &plc, 1 << 30); let mark = || std::fs::read(dir.join(crate::ops::CLEAN)).unwrap(); let found = |s: &crate::Shared| s.ctx.found.load(Ordering::SeqCst); let s = start(); assert!(found(&s), "a directory with no data"); let mut made = Vec::new(); for name in ["alice", "bob", "carol"] { let (did, token) = create(&s, name).await; put(&s, &did, &token, "r0", 0).await; made.push((did, token)); } let [alice, bob, carol] = [0, 1, 2].map(|i| made[i].0.clone()); // Bob has a table, and a log after it. compact(&s, &bob); put(&s, &bob, &made[1].1, "r1", 1).await; drop(s); // Killed. A run that loads alice and never bob, shut down before // it has made durable what it found, leaves no word. let s = start(); assert!(!found(&s)); assert!(load_syncs(&s, &alice) >= 1); shut_down(s); assert_eq!(mark(), b""); assert!(!crate::ops::unclean(&dir)); // So bob's log, which nothing has synced since the kill, is synced // as it is first loaded. let s = start(); assert!(!found(&s)); assert!(load_syncs(&s, &bob) >= 1); assert_eq!(records(&s, &bob).await, numbered(0..2)); // On a thread of its own, as a server that serves makes it; and // killed all the same, which leaves no word. crate::ops::sync_found_soon(&s); for _ in 0..500 { if found(&s) { break; } tokio::time::sleep(Duration::from_millis(10)).await; } assert!(found(&s)); drop(s); // Killed again. This run makes durable what it found, bob's log // among it, which it never loads. let s = start(); assert!(load_syncs(&s, &alice) >= 1); let before = syncs(); crate::ops::sync_found(&s).unwrap(); assert!(syncs() > before, "counted as any sync is"); assert!(found(&s)); let after = syncs(); crate::ops::sync_found(&s).unwrap(); assert_eq!(syncs(), after, "made durable once"); assert!(load_syncs(&s, &carol) >= 1, "its own run's first load"); shut_down(s); assert!(mark().starts_with(crate::ops::DURABLE)); let s = start(); assert_eq!(mark(), b""); assert!(found(&s)); assert_eq!(load_syncs(&s, &bob), 0, "not loaded since the kill"); assert_eq!(records(&s, &bob).await, numbered(0..2)); assert_eq!(load_syncs(&s, &alice), 0); assert_eq!(records(&s, &alice).await, numbered(0..1)); assert_eq!(records(&s, &carol).await, numbered(0..1)); let _ = std::fs::remove_dir_all(&dir); } /// File by file, what a start found is each account's logs, its table /// and its directory handed to the drive, and then one sync of `repos`: /// each counted, nothing loaded, on one thread or several. What is no /// account's directory is passed over, and a directory with no /// accounts needs none. #[tokio::test] async fn what_a_start_found_is_synced_file_by_file() { let plc = plc().await; let dir = scratch("clean-walk"); let s = lazy(&dir, &plc, 1 << 30); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; put(&s, &alice, &ta, "r0", 0).await; compact(&s, &bob); put(&s, &bob, &tb, "r0", 0).await; drop(s); std::fs::write(dir.join("repos").join("stray"), b"no account").unwrap(); let before = syncs(); crate::store::flush_found(&dir, 1).unwrap(); // Alice's log and directory; bob's log, table and directory; `repos`. assert_eq!(syncs() - before, 6); crate::store::flush_found(&dir, 4).unwrap(); let s = lazy(&dir, &plc, 1 << 30); assert_eq!(loaded(&s), 0); assert_eq!(records(&s, &bob).await, numbered(0..1)); let before = syncs(); crate::store::flush_found(&scratch("clean-walk-none"), 4).unwrap(); assert_eq!(syncs(), before); let _ = std::fs::remove_dir_all(&dir); } }