//! What the server keeps, and how it starts again from it, as //! `spec/DURABILITY.md` and `spec/STORE.md` say. In `PDS_DATA_DIR`, which is //! its owner's alone: //! //! - the service log, `accounts..log` from its checkpoint //! `accounts.checkpoint` on (`compact`): each account (DID, handle, email, //! password hash, signing key, KEM key, DID document), each change to the //! sessions it gave out and its app passwords (`sessions::ENTRIES`), each //! change to its email and password and the tokens mailed for them //! (`email::ENTRIES`), each change to what OAuth keeps (`oauth::ENTRIES`), //! each single-use value OAuth spent (`replay::ENTRY`), the invite codes //! and their uses (`invites::ENTRIES`), the single-use values the spaces //! took (`single_use`), the spaces' credentials revoked here //! (`space_credentials`), each account's state and its events //! (`lifecycle::ENTRIES`), each account's preferences as last put //! (`preferences::ENTRY`), and with blobs in a bucket, the objects of //! deleted accounts not yet deleted (`blobstore::ENTRIES`); //! - `jwt.key`, the secret its tokens are signed with, and `oauth.key`, the //! one OAuth seals its tokens, nonces and cookies with; //! - each account's own log, `repos//repo..log` from its checkpoint //! `repo.checkpoint` on (`checkpoint`): the mark that its spaces state is //! this version's from there (`checkpoint::unmarked`), first; its //! commits, each in one frame //! with its firehose events, the blobs it stored and stopped holding, its //! handle changes: each recorded before its operation goes to the PLC //! directory, then settled, in one frame with its `#identity` event if it //! landed; its space repos' commits and imports, each one frame //! (`space_repo`); each change to the spaces it is the authority of, one //! frame each (`space_authority::ENTRIES`), with the commits of the //! records it publishes and the repo it drops (`space_repo::DROP`); and its //! write notifications waiting to be sent again, and its spaces' syncers' //! registrations and the notices kept for them (`space_notify::ENTRIES`); //! - `repos//blobs/`, the bytes of each blob it holds (`blobs`), //! or with blobs in a bucket, none, and the mark `blobstore` (`blobstore`); //! - the firehose's index, `events..log` from its checkpoint //! `events.checkpoint` on: of each event, as it is numbered, its seq, //! account and time, so that a start learns the events made before it //! without reading the accounts' logs (`events::index`). Its entry of an //! event is appended before the frame that holds the event, and both are //! on disk before anything is made of it; //! - `clean-shutdown`, the mark a shutdown leaves and the next start takes //! away (`ops`): it holds the word that every log was closed and synced, //! for which the next start syncs no log as it opens it (`store`, "What //! a start syncs"). //! //! Each log is compacted into its checkpoint, generation by generation //! (`store`, `pds_store`). A start reads the service checkpoint's summary //! and its entries that are not one account's, and replays the service log //! after it, noting where each account's entries are without applying them //! (`directory`); each account, and then its own checkpoint and log, is //! read on its first use. A data directory with the layout before //! generations is moved to generation 0 first (`store::migrate`), and a //! service checkpoint of the format before a table is made one //! (`compact::convert`). //! //! An account's log, with its repository's first commit, is on disk before //! the account is in the service log: if the server stops between the two, //! the log is of no account, and goes once the server serves again //! (`directory::Accounts::sweep`). An account in the service log whose log //! has no commit, as one older servers left, gets its first commit when its //! store is loaded. Events are numbered in one sequence across all the //! accounts' logs (`events::Firehose`), each keeping its seq, the next //! after the greatest any log or checkpoint names. An account's handle is //! the one its last landed change gave it, or else the one the service log //! has. //! //! At most a share of accounts' logs are open at once (`Config::share`: a //! quarter of the file descriptors the process may have, as //! `pds_store::files::share` counts it), and at most a share of their //! tables' files, and they are recovered on at most as many threads as the //! machine has cores. use crate::blobs::blob_dir; use crate::cache::{Ctx, Slot}; use crate::compact::ServiceTable; use crate::directory::{Accounts, Directory}; use crate::email::{self, Mail}; use crate::events::{self, Firehose, Landing, Reserved}; use crate::sessions::Sessions; use crate::{Account, Config, Service, Store}; use axum::body::Bytes; use k256::ecdsa::SigningKey; use pds_repo::*; use pds_store::{Files, Holed, Wal}; use rustix::io::Errno; use serde_json::{Value as Json, json}; use std::collections::HashSet; use std::io; use std::path::Path as FsPath; #[cfg(test)] use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError, RwLock}; /// What error messages call the service log. pub(crate) const ACCOUNTS_LOG: &str = "the service log"; /// The service log of generation 0, by its name. #[cfg(test)] pub(crate) const ACCOUNTS_LOG_FILE: &str = "accounts.0.log"; /// The service log of generation 0, which a server that has not compacted it holds. #[cfg(test)] pub(crate) fn accounts_log(dir: &FsPath) -> PathBuf { crate::store::service_layout(dir).log(0) } /// An account's log of generation 0, which an account whose log has not /// been compacted holds. #[cfg(test)] pub(crate) fn repo_log(dir: &FsPath, did: &str) -> PathBuf { crate::store::account_layout(dir, did).log(0) } /// A copy of the data directory `dir` as it is now, beside it: what a /// server started again would find, for one to start on while the server /// on `dir`, which holds its logs, goes on. #[cfg(test)] pub(crate) fn copied(dir: &FsPath) -> PathBuf { let to = dir.with_extension("copy"); let _ = std::fs::remove_dir_all(&to); let mut todo = vec![dir.to_path_buf()]; while let Some(d) = todo.pop() { std::fs::create_dir_all(to.join(d.strip_prefix(dir).unwrap())).unwrap(); for e in std::fs::read_dir(&d).unwrap() { let p = e.unwrap().path(); if p.is_dir() { todo.push(p); } else { std::fs::copy(&p, to.join(p.strip_prefix(dir).unwrap())).unwrap(); } } } to } /// What the log at `path` holds: its file's bytes, without the room after /// its frames, which a server running or killed leaves there /// (`pds_store`). #[cfg(test)] pub(crate) fn log_bytes(path: &FsPath) -> Vec { without_room(std::fs::read(path).unwrap()) } /// A log's bytes without the room after its frames: the zeros they end with. #[cfg(test)] fn without_room(mut log: Vec) -> Vec { log.truncate(log.iter().rposition(|&b| b != 0).map_or(0, |last| last + 1)); log } /// The logs opened on each data directory, in tests alone, where one /// process starts a server on a directory again and again while tasks of /// the server before still hold it: a start closes that server's logs /// first, as the restart of a process ends its writers (`Wal::close`). #[cfg(test)] static OPENED: Mutex)>> = Mutex::new(Vec::new()); /// A log as the server holds it, opened on the data directory `dir`. pub(crate) fn held(dir: &FsPath, wal: Wal) -> Arc { let wal = Arc::new(wal); #[cfg(test)] OPENED .lock() .unwrap_or_else(PoisonError::into_inner) .push((dir.to_path_buf(), Arc::downgrade(&wal))); let _ = dir; wal } /// The journals opened on each data directory, in tests alone, for the /// same reason: a start lets the one before go, as a kill does. #[cfg(test)] static JOURNALS: Mutex)>> = Mutex::new(Vec::new()); /// Close every log a server before opened on `dir` and still holds: its /// files are let go whether or not what it had written could be synced. /// Then let its journal go, its segments left for this start to redo. #[cfg(test)] fn stop_writers(dir: &FsPath) { OPENED .lock() .unwrap_or_else(PoisonError::into_inner) .retain(|(of, wal)| match wal.upgrade() { Some(wal) if of == dir => { let _ = wal.close(); false } held => held.is_some(), }); JOURNALS .lock() .unwrap_or_else(PoisonError::into_inner) .retain(|(of, j)| match j.upgrade() { Some(j) if of == dir => { j.let_go(); false } held => held.is_some(), }); } /// An account as `accounts.log` has it. pub(crate) struct AccountEntry { pub did: String, pub handle: String, /// None for an account without a password. pub email: Option, pub password: Option, /// Made on first use: an account is loaded for much that signs nothing. pub key: crate::cache::Key, /// Its KEM key (`spec/SPACES.md`); None for an account made before /// accounts had one, which is given none. pub kem: Option, /// Its DID document, as JSON text: held so from the start, a few /// allocations rather than one for each of its fields. pub did_doc: Box, } impl AccountEntry { pub fn entry(&self) -> Json { json!({ "t": "account", "did": self.did, "handle": self.handle, "email": self.email, "password": self.password, "key": hex::encode(self.key.to_bytes()), "kemKey": self.kem.as_ref().map(|k| hex::encode(k.to_bytes())), "didDoc": serde_json::from_str::(&self.did_doc).unwrap_or(Json::Null), }) } pub(crate) fn parse(e: &Json) -> Result { let text = |k: &str| { e[k].as_str() .map(str::to_string) .ok_or_else(|| format!("an account with no {k}: {e}")) }; let key = hex::decode(text("key")?) .ok() .and_then(|b| crate::cache::Key::from_slice(&b)) .ok_or_else(|| format!("an account with a bad key: {e}"))?; let optional = |k: &str| match &e[k] { Json::Null => Ok(None), Json::String(p) => Ok(Some(p.clone())), _ => Err(format!("an account with a bad {k}: {e}")), }; let (password, email) = (optional("password")?, optional("email")?); let kem = match optional("kemKey")? { None => None, Some(k) => Some( hex::decode(k) .ok() .and_then(|b| pds_space::KemKey::from_bytes(&b).ok()) .ok_or_else(|| format!("an account with a bad kemKey: {e}"))?, ), }; if password.is_some() != email.is_some() { return Err(format!( "an account with a password and no email, or an email and no password: {e}" )); } if !e["didDoc"].is_object() { return Err(format!("an account with no didDoc: {e}")); } Ok(AccountEntry { did: text("did")?, handle: text("handle")?, email, password, key, kem, did_doc: e["didDoc"].to_string().into(), }) } /// The account, its store held if given, else loaded on first use; its /// DID document as fetched at `fetched`, or none if it may be out of /// date; its mail, if not as made, and its state. pub fn account( self, store: Option, fetched: Option, ctx: &Arc, mail: Option, state: crate::lifecycle::State, ) -> Arc { let mail = mail.unwrap_or_else(|| Mail::new(self.email, self.password)); let a = Arc::new_cyclic(|me| Account { did: self.did, handle: RwLock::new(self.handle), mail: Mutex::new(mail), kem: self.kem, did_doc: RwLock::new((self.did_doc, fetched)), fetching: tokio::sync::Mutex::new(()), store: Slot::new(me.clone(), self.key, ctx.clone(), store), state: RwLock::new(state), }); a.store.counted(); a } } /// A commit's entry: its rev and block, each record it put, with its CID, /// and each path it deleted; and what it changed of the tree: the nodes it /// made, with their bytes, those it took out, each record whose count of /// paths changed, and the counts after (`checkpoint`). Written as it is /// serialized, borrowing the commit: no copy of its records is made. Its /// block and its nodes' bytes are holes of its frame, written as they are /// (`pds_store::frame_with_holes`); CIDs are hex. struct CommitEntry<'h, 'a>(&'a Change, &'h Holed<'a>); impl serde::Serialize for CommitEntry<'_, '_> { fn serialize(&self, s: S) -> Result { use serde::ser::SerializeMap; let (c, w) = (self.0, self.1); let mut m = s.serialize_map(Some(10))?; m.serialize_entry("t", "commit")?; m.serialize_entry("rev", &c.rev)?; m.serialize_entry("head", &Hole(w, &c.head_bytes))?; m.serialize_entry( "puts", &c.puts .iter() .zip(&c.put_cids) .map(|((k, v), cid)| (k, v, Hex(cid))) .collect::>(), )?; m.serialize_entry("dels", &c.dels)?; m.serialize_entry( "tree", &c.tree .iter() .map(|(cid, b)| (Hex(cid), Hole(w, b))) .collect::>(), )?; m.serialize_entry("gone", &c.gone.iter().map(|g| Hex(g)).collect::>())?; m.serialize_entry( "refs", &c.refs .iter() .map(|(cid, n, rev)| (Hex(cid), n, rev)) .collect::>(), )?; m.serialize_entry("counts", &[c.counts.0, c.counts.1])?; m.end() } } /// An event's entry: its seq, when it was made, and its frame, a hole of /// the log's frame. struct EventEntry<'h, 'a>(u64, u64, &'a [u8], &'h Holed<'a>); impl serde::Serialize for EventEntry<'_, '_> { fn serialize(&self, s: S) -> Result { use serde::ser::SerializeMap; let mut m = s.serialize_map(Some(4))?; m.serialize_entry("t", "event")?; m.serialize_entry("seq", &self.0)?; m.serialize_entry("time", &self.1)?; m.serialize_entry("frame", &Hole(self.3, self.2))?; m.end() } } /// Bytes in a hole of the frame `Holed` writes: an empty string in its /// text, and the bytes, as they are, after it. struct Hole<'h, 'a>(&'h Holed<'a>, &'a [u8]); impl serde::Serialize for Hole<'_, '_> { fn serialize(&self, s: S) -> Result { let written = s.serialize_str("")?; self.0.hole(self.1); Ok(written) } } /// Bytes as hex, written as they are serialized, never held as a string. struct Hex<'a>(&'a [u8]); impl serde::Serialize for Hex<'_> { fn serialize(&self, s: S) -> Result { s.collect_str(self) } } impl std::fmt::Display for Hex<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut buf = [0u8; 2048]; for chunk in self.0.chunks(buf.len() / 2) { let out = &mut buf[..chunk.len() * 2]; hex::encode_to_slice(chunk, out).expect("twice the length"); f.write_str(std::str::from_utf8(out).expect("hex"))?; } Ok(()) } } /// A commit's frame, JSON with holes: its entry, if any, then `entries`, /// then the events numbered from `seq`, each with when it was made. fn commit_frame( commit: Option<&Change>, entries: &[Json], seq: u64, frames: &[Vec], times: &[u64], ) -> Vec { // The records, as JSON, are about the size of the events' blocks, and // so are the nodes and the block, besides the events themselves. let capacity = frames.iter().map(|f| f.len() * 3).sum::() + 4096; pds_store::frame_with_holes(capacity, |w| { use serde::ser::{SerializeSeq, Serializer}; let mut s = serde_json::Serializer::new(w); let mut seq_out = s.serialize_seq(None).expect("a frame serializes"); if let Some(c) = commit { seq_out .serialize_element(&CommitEntry(c, w)) .expect("a commit serializes"); } for e in entries { seq_out.serialize_element(e).expect("an entry serializes"); } for (i, f) in frames.iter().enumerate() { seq_out .serialize_element(&EventEntry(seq + i as u64, times[i], f, w)) .expect("an event serializes"); } seq_out.end().expect("a frame serializes"); }) } /// A handle change, recorded before its operation, by its CID, goes to the directory. pub(crate) fn handle_entry(handle: &str, cid: &str) -> Json { json!({ "t": "handle", "handle": handle, "cid": cid }) } /// Append `commit`'s entry, if given, and `entries`, with the events /// `numbered`, if any, numbered and made as their reservation says, the /// commit's change kept in the store's memtable: the events to land once /// the log, and their index entry, are on disk, or, if the append failed, /// none of them, their numbers left unused. With blobs in a bucket, a /// commit that stops the account holding a blob has it marked for a start /// to load first, so that the blob's object is deleted (`blobstore`). fn append_events( events: &Firehose, st: &mut Store, entries: &[Json], numbered: Option<(&Reserved, Vec>)>, commit: Option<&Change>, ) -> io::Result> { let (r, frames) = match numbered { Some((r, frames)) => (Some(r), frames), None => (None, Vec::new()), }; let (seq, at) = r.map_or((0, 0), |r| (r.seq, r.at)); let fail = |e: io::Error| { if !frames.is_empty() { events.fail(seq, frames.len()); } e }; if st.ctx.bucket.is_some() && st.blobs.dooms() && let Err(e) = st.ctx.eager.mark(&st.repo.did) { return Err(fail(e)); } let times = vec![at; frames.len()]; let bytes = commit_frame(commit, entries, seq, &frames, ×); let pos = match st.wal.append_frame_at(&bytes) { Ok((_, pos)) => pos, Err(e) => return Err(fail(e)), }; st.disk .indexed(&st.ctx.blocks, pos, &bytes[pds_store::HEADER..]); drop(bytes); if let Some(c) = commit { st.disk.mem.commit(pos, c); } for (i, t) in times.iter().enumerate() { st.disk.mem.event(seq + i as u64, *t, pos); } let Some(r) = r else { return Ok(None); }; st.last_seq = st.last_seq.max(seq + frames.len() as u64 - 1); Ok(Some( r.landing(frames.into_iter().map(Bytes::from).collect()), )) } /// Settle the account's handle change in flight, under its account's lock: /// it landed, with its `#identity` event in the same frame, or it was /// dropped. Answers the event to land once the log is on disk. pub(crate) fn log_settled( events: &Firehose, did: &str, st: &mut Store, cid: &str, landed: Option<&str>, ) -> io::Result> { let settled = json!({ "t": "handled", "cid": cid, "landed": landed.is_some() }); let Some(handle) = landed else { st.wal.append(&[settled])?; st.pending = None; return Ok(None); }; let r = events.reserve(did, 1)?; let frame = events::identity_event(did, handle, r.seq, &r.time()); let landing = append_events(events, st, &[settled], Some((&r, vec![frame])), None)?; st.pending = None; st.logged_handle = Some(handle.to_string()); Ok(landing) } /// Log a commit made under its account's lock, with its event; land the /// event once the log is on disk. pub(crate) fn log_commit( events: &Firehose, did: &str, st: &mut Store, c: &Change, ) -> io::Result { let r = events.reserve(did, 1)?; let frame = events::commit_event(did, r.seq, &r.time(), c); let landing = append_events(events, st, &[], Some((&r, vec![frame])), Some(c))?; Ok(landing.expect("a commit's event")) } /// Log a repository imported whole (`spec/MIGRATION.md`), its commit made /// under its account's lock: with its `#sync` event if the account is /// `active`, to land once the log is on disk, or with none. pub(crate) fn log_import( events: &Firehose, did: &str, st: &mut Store, c: &Change, active: bool, ) -> io::Result> { if !active { return append_events(events, st, &[], None, Some(c)); } let r = events.reserve(did, 1)?; let frame = events::sync_event(did, r.seq, &r.time(), &c.head_bytes, &c.rev); append_events(events, st, &[], Some((&r, vec![frame])), Some(c)) } /// Make a new repository's first commit, the empty repository, and log it /// with the events a new account's stream starts with; with `quiet`, an /// account moving here (`spec/MIGRATION.md`), with none. pub(crate) fn first_commit( events: &Firehose, did: &str, handle: &str, st: &mut Store, quiet: bool, ) -> io::Result> { let mut c = st.repo.init(); let landed = if quiet { append_events(events, st, &[], None, Some(&c)) } else { events.reserve(did, 4).and_then(|r| { let frames = events::initial_events(did, handle, r.seq, &r.time(), &c); append_events(events, st, &[], Some((&r, frames)), Some(&c)) }) }; if landed.is_err() && let Some(u) = c.undo.take() { st.repo.undo(u); } landed } /// An account's store as its checkpoint and log have it. pub(crate) struct Recovered { pub store: Store, /// Whether it has no commit yet. pub fresh: bool, } /// Try `f` again while it fails for want of a file descriptor, waiting for /// one to be closed, up to a point. pub(crate) fn retrying(mut f: impl FnMut() -> io::Result) -> io::Result { for _ in 0..200 { match f() { Err(e) if matches!(Errno::from_io_error(&e), Some(Errno::MFILE | Errno::NFILE)) => { std::thread::sleep(std::time::Duration::from_millis(50)) } done => return done, } } f() } /// Load an account's store: its checkpoint's table opened, the logs after /// it replayed, or begun; compacted from now on as `store::policy` says, /// once the account's slot sets its fold (`cache::Slot`). `durable` says /// that every frame its log's files hold is on disk, and nothing of them /// is then synced but what is mended (`checkpoint::load`). pub(crate) fn open_repo( ctx: &Arc, did: &str, key: &SigningKey, durable: bool, ) -> Result { let dir = &ctx.dir; let layout = crate::store::account_layout(dir, did); let at = |why: String| format!("{}: {why}", layout.dir.display()); crate::store::remove_legacy(dir, Some(did)).map_err(|e| at(e.to_string()))?; // Uploads on their way when the server stopped: not this process's own. if let Ok(entries) = std::fs::read_dir(blob_dir(dir, did)) { for e in entries.flatten() { let name = e.file_name(); let name = name.to_string_lossy(); if name.starts_with(".tmp-") && !name.starts_with(crate::blobs::temp_prefix()) { std::fs::remove_file(e.path()).map_err(|why| at(why.to_string()))?; } } } let crate::checkpoint::Loaded { repo, wal, disk, mut small, } = crate::checkpoint::load(ctx, did, key, durable)?; // Spaces state of no version but this one is let go, once: from its // mark on, the log is of this version's (`checkpoint::unmarked`). if small.unmarked.is_some() { wal.append(&[crate::checkpoint::mark()]) .map_err(|e| at(e.to_string()))?; small.mark(); } match &ctx.bucket { None => small.blobs.recover(&blob_dir(dir, did))?, Some(bucket) => { small.blobs.recover_kept(); // An upload the server stopped during, or an object not yet deleted. if small.blobs.strays().next().is_some() { bucket.wake(did); } } } let fresh = repo.rev.is_empty(); Ok(Recovered { store: Store { repo, wal: held(dir, wal), blobs: small.blobs, takedowns: small.takedowns, spaces: small.spaces, authority: small.authority, notify: small.notify, disk, last_seq: small.last_seq, logged_handle: small.handle, pending: small.pending, ctx: ctx.clone(), #[cfg(test)] thread: std::thread::current().id(), }, fresh, }) } /// Begin a new account's log, with its mark of spaces state of this /// version (`checkpoint::unmarked`): its store, with no commit yet. pub(crate) fn new_repo(ctx: &Arc, did: &str, key: &SigningKey) -> io::Result { let layout = crate::store::account_layout(&ctx.dir, did); let (wal, frames) = Wal::open_log(&layout, 0, 0, Some(ctx.files.clone()), ctx.policy)?; if !frames.is_empty() { return Err(io::Error::other("its log holds something already")); } wal.append(&[crate::checkpoint::mark()])?; Ok(Store { repo: Repo::new(did.to_string(), key.clone()), wal: held(&ctx.dir, wal), blobs: Default::default(), takedowns: Default::default(), spaces: Default::default(), authority: Default::default(), notify: Default::default(), disk: crate::checkpoint::Disk::empty(layout), last_seq: 0, logged_handle: None, pending: None, ctx: ctx.clone(), #[cfg(test)] thread: std::thread::current().id(), }) } /// Load the stores of `accounts`, side by side on at most as many threads /// as the machine has cores, this one among them: the first error, if one /// cannot be. fn load_all(accounts: &[Arc]) -> Result<(), String> { let next = AtomicUsize::new(0); let failed: Mutex> = Mutex::new(None); let work = || { while failed .lock() .unwrap_or_else(PoisonError::into_inner) .is_none() { let i = next.fetch_add(1, Ordering::Relaxed); let Some(a) = accounts.get(i) else { return }; if let Err(why) = a.store.load() { failed .lock() .unwrap_or_else(PoisonError::into_inner) .get_or_insert(why); } } }; let threads = std::thread::available_parallelism() .map_or(1, |n| n.get()) .min(accounts.len()) .max(1); std::thread::scope(|s| { for i in 1..threads { if std::thread::Builder::new() .name(format!("pds-recover-{i}")) .spawn_scoped(s, work) .is_err() { break; } } work(); }); failed .into_inner() .unwrap_or_else(PoisonError::into_inner) .map_or(Ok(()), Err) } impl Service { /// Start from the logs in `dir`: read the service checkpoint's summary /// and its entries that are not one account's, and replay the service /// log after it, or begin it. Of an account's entries in the log, only /// where they are is noted (`directory`): no account is loaded, nor its /// own log read, but those the service log marks (`cache`), or every one /// with `Config::load_all`, or on a data directory from before seqs /// were reserved (`events::SEQS`). pub(crate) fn open(dir: &FsPath, config: Config) -> Result { use std::os::unix::fs::PermissionsExt; #[cfg(test)] stop_writers(dir); pds_store::make_dirs(dir).map_err(|e| format!("{}: {e}", dir.display()))?; // Whether the last server left every log durable, in this very // directory, asked before anything of it is changed or written: // no log is then synced as it is opened, the service log now and // each account's on its first use. The directory's mode is set // after, which changes when it last changed: the word names that // too, and from here on names this directory no more. let durable = crate::ops::durable(dir)?; std::fs::set_permissions(dir, std::fs::Permissions::from_mode(pds_store::DIR_MODE)) .map_err(|e| format!("{}: {e}", dir.display()))?; // What the journal holds of writes answered before a kill or a // power cut is put back where it is kept, before anything else of // the directory is read or changed, whichever way this run syncs. // To sync together, the journal begins again, adopting the segments // where that start left them, for its first checkpoint to make // durable and let go; to sync each by itself, there is no journal // to adopt them, and the start makes them durable and deletes them. let journal = match config.sync { crate::Syncing::Together => { let redone = pds_store::journal::redo(dir, pds_store::journal::Redo::Adopting) .map_err(|e| e.to_string())?; Some(pds_store::Journal::open(dir, redone.adopted).map_err(|e| e.to_string())?) } crate::Syncing::Each => { pds_store::journal::redo(dir, pds_store::journal::Redo::Eager) .map_err(|e| e.to_string())?; None } }; #[cfg(test)] if let Some(j) = &journal { JOURNALS .lock() .unwrap_or_else(PoisonError::into_inner) .push((dir.to_path_buf(), Arc::downgrade(j))); } // What this run finds is durable already where that was so, or // where it finds no data; else once it has made it so // (`ops::sync_found`). let found = durable || !crate::store::holds_data(dir); crate::store::migrate(dir)?; crate::store::remove_legacy(dir, None).map_err(|e| format!("{}: {e}", dir.display()))?; let layout = crate::store::service_layout(dir); let window = config.backfill.as_millis() as u64; // A checkpoint of the format before a table is made one, once. let now_ms = events::now_ms(); crate::compact::convert(&layout, now_ms / 1000, |_, f| { crate::store::in_window(f, now_ms, window) })?; // Of `PDS_CACHE_BYTES`: half the stores held, a quarter the blocks // read, and the firehose's tail (`events::TAIL`) the rest. let blocks = Arc::new(pds_store::table::BlockCache::with_open( config.cache_bytes / 4, config.share, )); let table = ServiceTable::open(&layout.checkpoint(), blocks.clone())?; let (generation, size, service_seq, mut ceiling) = table.as_ref().map_or((0, 0, 0, None), |t| { ( t.summary.generation, t.size(), t.summary.last_seq, t.summary.ceiling, ) }); let new = table.is_none(); // No account loaded: what the log holds of each is noted in it. let mut accounts = Directory::new(layout.clone(), table); // The accounts marked. let mut marked: HashSet = HashSet::new(); // Keyed once the replay has said whether the data directory is new. let mut sessions = Sessions::unkeyed(&config.hostname); let oauth = crate::oauth::OAuth::open(dir, &config.public_url, config.oauth_client_fetch.clone())?; let replay = crate::replay::Replay::default(); let (single_use, space_credentials) = ( crate::single_use::Spent::default(), crate::space_credentials::Credentials::default(), ); let now = sessions.now(); let invites = crate::invites::Invites::default(); // The events this log holds, and the places of a deleted account's. let mut lifecycle = crate::lifecycle::Replayed::default(); // Deleted accounts' objects in a bucket not yet deleted. let mut doomed = crate::blobstore::Doomed::default(); // Replay an entry that is not one account's, given what is noted of // the accounts so far: answers the DID of an account it deletes. let mut apply = |e: &Json, accounts: &Directory| -> Result, String> { let at = |why: String| format!("{ACCOUNTS_LOG}: {why}"); match e["t"].as_str() { Some(crate::replay::ENTRY) => replay.replay(e, now).map_err(at)?, Some(t) if crate::oauth::ENTRIES.contains(&t) => { oauth.replay(e, &|did| accounts.may_be(did)).map_err(at)? } Some(crate::single_use::ENTRY) => single_use.replay(e, now).map_err(at)?, Some(crate::space_credentials::ENTRY) => { space_credentials.replay(e, now).map_err(at)? } Some(t) if crate::invites::ENTRIES.contains(&t) => invites.replay(e).map_err(at)?, Some(t) if crate::lifecycle::ENTRIES.contains(&t) => { if let Some(did) = lifecycle.replay(e).map_err(at)? { // Deleted: it is no account; its files, if any are // left, go once the server serves. marked.remove(&did); oauth.forget_account(&did); return Ok(Some(did)); } } Some(crate::cache::DIR) => { let (Some(did), Some(eager)) = (e["did"].as_str(), e["eager"].as_bool()) else { return Err(at(format!("a mark with no did or eager: {e}"))); }; // Nothing is kept of an account deleted. if accounts.is_noted_deleted(did) { return Ok(None); } if !accounts.may_be(did) { return Err(at(format!("a mark of {did:?}, which no account has"))); } if eager { marked.insert(did.to_string()); } else { marked.remove(did); } } Some(t) if crate::blobstore::ENTRIES.contains(&t) => { doomed.replay(e).map_err(at)? } Some(crate::events::SEQS) => { let upto = e["upto"] .as_u64() .ok_or_else(|| at(format!("seqs with no upto: {e}")))?; ceiling = Some(ceiling.unwrap_or(0).max(upto)); } t => return Err(at(format!("a log entry of no known type: {t:?}"))), } Ok(None) }; // The table's entries that are not one account's, a row at a time. if let Some(t) = accounts.table() { t.rest(|entries| { entries .iter() .try_for_each(|e| apply(e, &accounts).map(drop)) }) .map_err(|why| format!("{}: {why}", layout.checkpoint().display()))?; } // The logs after it, replayed a frame at a time as they are read: // an entry of one account is noted, not applied. let mut frames = 0; let mut failed = None; let log = Wal::open_log_with( &layout, generation, size, None, crate::store::service_policy(&config), durable, &mut |pos, entries| { frames += 1; let replayed: Result<(), String> = entries.iter().try_for_each(|e| { let of = crate::compact::of_account(e) .map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?; // A mark is the account's, for its handle, and not. if (of.is_none() || e["t"] == crate::cache::DIR) && let Some(did) = apply(e, &accounts)? { accounts.note_deleted(&did, pos.generation); } if let Some(did) = of { accounts .note(pos, did, e) .map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?; } Ok(()) }); if let Err(why) = &replayed { failed = Some(why.clone()); } replayed }, ); let log = match (log, failed) { (_, Some(why)) => return Err(why), (Ok(log), None) => log, (Err(e), None) => return Err(format!("{ACCOUNTS_LOG}: {e}")), }; // In no pool, and given the journal all the same. if let Some(j) = &journal { log.use_journal(j.clone()); } sessions.keyed(dir, new && frames == 0)?; oauth.recovered(dir, sessions.now())?; // New events are numbered after every seq reserved before (`events::SEQS`). let logged = lifecycle .last_seq() .max(service_seq) .max(lifecycle.gaps.iter().copied().max().unwrap_or(0)); let last = logged.max(ceiling.unwrap_or(0)); // A data directory from before seqs were reserved: every account's // log is read for the greatest seq it holds. let upgrade = ceiling.is_none() && accounts.count() > 0; let log = held(dir, log); let logged_events = lifecycle .events .drain(..) .map(|e| events::Recovered { seq: e.seq, did: Arc::from(e.did.as_str()), frame: Bytes::from(e.frame), }) .collect(); let files = match journal { Some(j) => Files::with_journal(config.share, j), None => Files::new(config.share), }; let firehose = Firehose::new( logged_events, lifecycle.gaps.iter().copied().collect(), last, logged, config.backfill, log.clone(), events::index::At { dir: dir.to_path_buf(), pool: files.clone(), policy: crate::store::service_policy(&config), durable, }, ); let mut marks: Vec = marked.iter().cloned().collect(); marks.sort_unstable(); let eager = Arc::new(crate::cache::Eager::new(log.clone(), marked)); let bucket = match config.blobstore.clone() { Some(s) => { let b = crate::blobstore::Bucket::new(s)?; b.doomed(doomed); Some(Arc::new(b)) } None => None, }; let ctx = Arc::new(Ctx { dir: dir.to_path_buf(), files: files.clone(), policy: crate::store::policy(&config), window, events: firehose.clone(), eager: eager.clone(), cache: crate::cache::Cache::new(config.cache_bytes / 2), blocks: blocks.clone(), starting: std::sync::atomic::AtomicBool::new(true), durable, found: std::sync::atomic::AtomicBool::new(found), failed: std::sync::atomic::AtomicBool::new(false), bucket, }); let limits = crate::limits::Limits::new(dir, &config); let outbound = crate::outbound::Outbound::new(config.ssrf_protection)?; let load_every = config.load_all || upgrade; let directory = Arc::new(Accounts::new(layout, blocks, accounts)); // Compacted from now on, its table put in place with the directory held. log.set_fold(Arc::new(crate::store::ServiceFold { window, accounts: Arc::downgrade(&directory), })); let svc = Service { space_notify: crate::space_notify::Notify::new(), config, dir: dir.to_path_buf(), log, ctx: ctx.clone(), directory, sessions, oauth, replay, invites, events: firehose, http: crate::plc::client().map_err(|e| format!("an HTTP client: {e}"))?, changes: crate::identity::Changes::default(), space_hooks: crate::space_repo::Hooks::default(), single_use, space_credentials, space_authority: crate::space_authority::Host::default(), limits, resets: email::Resets::default(), outbound, did_docs: Default::default(), handle_resolver: Default::default(), prefs: Default::default(), }; // The accounts a start loads: those marked, or every one. let dids = if load_every { crate::read(&svc.directory).dids() } else { marks.clone() }; let mut loading: Vec> = Vec::with_capacity(dids.len()); for did in &dids { loading.extend(svc.build(did)?); } { let made: HashSet<&str> = loading.iter().map(|a| a.did.as_str()).collect(); if let Some(did) = marks.iter().find(|did| !made.contains(did.as_str())) { return Err(format!( "{ACCOUNTS_LOG}: a mark of {did:?}, which no account has" )); } } load_all(&loading)?; // With every account's log read, the greatest seq any holds is known: // the next follows it, whatever was reserved. let mut last = if load_every { logged } else { last }; // Each one's handle change in flight, if it has one. let mut pending = Vec::with_capacity(loading.len()); for a in &loading { let st = crate::read(&a.store); last = last.max(st.last_seq); // Its handle is the one its last landed change gave it. if let Some(h) = st.logged_handle.as_ref().filter(|h| **h != a.handle()) { *crate::write(&a.handle) = h.clone(); } pending.push(st.pending.clone()); } // Each handle, an account's or one a change in flight holds, and // each email, is one account's, of those loaded. { let mut d = crate::write(&svc.directory); for (a, pending) in loading.iter().zip(pending) { d.insert_loaded(a.clone()) .map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?; let Some(p) = pending else { continue }; // The handle stays taken until the change is settled. if p.handle != a.handle() { if let Some(other) = d.handles.get(&p.handle) { return Err(format!( "{ACCOUNTS_LOG}: {} is the handle of both {other} and {}", p.handle, a.did )); } if !d.claimed_handles.insert(p.handle.clone()) { return Err(format!( "{ACCOUNTS_LOG}: {} is the handle two accounts are being given", p.handle )); } } svc.changes.unsettled(&a.did, p); } } if load_every { svc.events.exact(last); } else { svc.events .raise(last) .map_err(|e| format!("{ACCOUNTS_LOG}: {e}"))?; } // A data directory with accounts and no index of their events is // given one, of every account's files, before it serves. svc.events .adopt(&svc) .map_err(|e| format!("the firehose's index: {e}"))?; // An account whose repository has no commit yet gets its first, // with no events if it was made deactivated, moving here. for a in &loading { let mut st = crate::write(&a.store); if st.repo.rev.is_empty() { let quiet = crate::read(&a.state).deactivated; let landing = first_commit(&svc.events, &a.did, &a.handle(), &mut st, quiet) .map_err(|e| format!("{}: {e}", a.did))?; landing .as_ref() .map_or(Ok(()), Landing::wait_entry) .and_then(|()| st.wal.wait(st.wal.written())) .map_err(|e| format!("{}: {e}", a.did))?; if let Some(l) = landing { svc.events.land(l); } } // Loaded and needing no start: unmarked, with the handle it has. if eager.is_marked(&a.did) && !st.needs_start() { eager.relax(&a.did, Some(&a.handle())); } } ctx.starting.store(false, Ordering::SeqCst); crate::space_notify::resume(&svc); Ok(svc) } } #[cfg(test)] mod tests { use super::*; use crate::sessions; use crate::testing::*; use pds_store as store; use rustix::process::{Resource, Rlimit, Signal, getrlimit, setrlimit}; use std::collections::BTreeMap; use std::path::Path; /// Everything a restart must carry over, as one comparable value: each /// account's repository, rev and KEM key, the sessions, the events, and /// the invite codes, each account's email and the tokens mailed for it, /// and each account's state, its records and blobs taken down. type State = ( BTreeMap, String, Option)>, BTreeMap, Vec>, Vec, Mails, Lives, ); /// Each account's mail, the directory's emails, and which account each reset token is of. type Mails = ( BTreeMap, BTreeMap, BTreeMap, ); /// Each account's state, its records taken down, and its blobs; and the directory's handles. type Lives = ( BTreeMap, String)>, BTreeMap, ); fn state(s: &Service) -> State { let d = crate::read(&s.directory); let repos = d .by_did .iter() .map(|(did, a)| { let st = crate::read(&a.store); ( did.clone(), ( crate::sync::CarStream::of(&st, None) .unwrap() .bytes() .unwrap(), st.repo.rev.clone(), a.kem.as_ref().map(|k| hex::encode(k.to_bytes())), ), ) }) .collect(); let mails = d .by_did .iter() .map(|(did, a)| (did.clone(), email::lock(&a.mail).clone())) .collect(); let emails = d .emails .iter() .map(|(e, did)| (e.clone(), did.clone())) .collect(); let lives = d .by_did .iter() .map(|(did, a)| { let st = crate::read(&a.store); ( did.clone(), ( crate::read(&a.state).clone(), st.takedowns.clone(), st.blobs.snapshot(), ), ) }) .collect(); let handles = d .handles .iter() .map(|(h, did)| (h.clone(), did.clone())) .collect(); drop(d); let sessions = s.sessions.snapshot(); let events = s .events .get(s, 0, u64::MAX - 1, s.events.drops()) .iter() .map(|e| e.to_vec()) .collect(); ( repos, sessions, events, s.invites.snapshot(), (mails, emails, s.resets.snapshot()), (lives, handles), ) } /// The files with the service log as a kill cut short in an account's /// log could have left it: without the notes of handles and the marks /// let go, which are written only once a frame of that log is on disk /// (`cache::Eager::relax`). The marks made before stay. fn unnoted(mut files: BTreeMap>) -> BTreeMap> { let p = PathBuf::from(ACCOUNTS_LOG_FILE); if let Some(bytes) = files.get(&p) { let note = |e: &Json| { e["t"] == crate::cache::DIR && (e.get("handle").is_some() || e["eager"] == json!(false)) }; let frames = store::scan(bytes).unwrap().0; let kept: Vec = frames .iter() .filter(|f| !f.iter().all(note)) .flat_map(|f| store::frame(f)) .collect(); files.insert(p, kept); } files } /// The files a step changed, but the service log, where marks go /// (`cache`), and the firehose's index, which a step that makes events /// appends their entries to (`events::index`). fn changed_logs<'a>( before: &BTreeMap>, after: &'a BTreeMap>, ) -> Vec<&'a PathBuf> { after .keys() .filter(|p| { before.get(*p) != after.get(*p) && p.as_path() != Path::new(ACCOUNTS_LOG_FILE) && !of_index(p) }) .collect() } /// Whether a file of the data directory, by its path there, is the /// firehose's index's (`events::index`). fn of_index(p: &Path) -> bool { p.to_str().is_some_and(|p| p.starts_with("events.")) } /// `files` with the index's files as `from` holds them. fn with_index_of( mut files: BTreeMap>, from: &BTreeMap>, ) -> BTreeMap> { files.retain(|p, _| !of_index(p)); files.extend( from.iter() .filter(|(p, _)| of_index(p)) .map(|(p, b)| (p.clone(), b.clone())), ); files } /// Of a step that made events, the index cut short at every byte of /// the entries it appended, the step's other writes lost, as a power /// cut may leave it, with `room` zeros after the cut where it is given: /// recovery must give the state before the step. Answers how many cuts. fn index_cut_short(work: &Path, plc: &str, w: &[Step], room: Option) -> usize { let mut cuts = 0; for (p, whole) in w[1] .files .iter() .filter(|(p, b)| of_index(p) && w[0].files.get(*p) != Some(*b)) { let from = w[0].files.get(p).map_or(0, Vec::len); for cut in from + 1..whole.len() { let mut files = w[0].files.clone(); let bytes = match room { Some(room) => with_room(&whole[..cut], whole.len() - cut + room), None => whole[..cut].to_vec(), }; files.insert(p.clone(), bytes); put_files(work, &files); assert_eq!( &state(&open(work, plc)), &w[0].state, "{} cut at {cut} of {from}..{}, the step's other writes lost", p.display(), whole.len() ); cuts += 1; } } cuts } /// An event's body, from its frame: its `seq`, `did` and the rest. fn body_of(frame: &[u8]) -> Json { let (_, at) = pds_repo::data::decode_prefix(frame).unwrap(); pds_repo::data::to_json(&pds_repo::data::decode(&frame[at..]).unwrap()) } /// The events a subscription from cursor 0 begun now is sent, each its /// body. fn sent(s: &Service) -> Vec { s.events .get(s, 0, u64::MAX - 1, s.events.drops()) .iter() .map(|f| body_of(f)) .collect() } /// The seqs of the events a subscription from cursor 0 is sent, in order. fn sent_seqs(s: &Service) -> Vec { sent(s).iter().map(|e| e["seq"].as_u64().unwrap()).collect() } /// Every file in the data directory, and its bytes: of a log, those it /// holds, without the room after its frames (`pds_store`), as a power /// cut the room did not outlive leaves it. The journal's segments are /// left out: what they hold is in the logs, as the system has them, and /// a step's frames are cut or kept in their logs alone, as after a /// checkpoint (`journal_cut_at_every_byte` cuts the journal). fn files(dir: &Path) -> BTreeMap> { let mut files = files_whole(dir); files.retain(|p, _| !p.starts_with(pds_store::journal::DIR)); for (_, bytes) in files .iter_mut() .filter(|(p, _)| p.extension().is_some_and(|e| e == "log")) { *bytes = super::without_room(std::mem::take(bytes)); } files } /// Every file in the data directory, and its bytes, room and all. fn files_whole(dir: &Path) -> BTreeMap> { let mut out = BTreeMap::new(); let mut todo = vec![dir.to_path_buf()]; while let Some(d) = todo.pop() { for e in std::fs::read_dir(&d).unwrap() { let p = e.unwrap().path(); if p.is_dir() { todo.push(p); } else { out.insert( p.strip_prefix(dir).unwrap().to_path_buf(), std::fs::read(&p).unwrap(), ); } } } out } /// Make `dir` hold exactly these files, writing only those that differ, /// so that the logs recovery makes durable are mostly durable already. fn put_files(dir: &Path, files: &BTreeMap>) { let have = if dir.exists() { files_whole(dir) } else { BTreeMap::new() }; for p in have.keys().filter(|p| !files.contains_key(*p)) { std::fs::remove_file(dir.join(p)).unwrap(); } for (p, bytes) in files.iter().filter(|(p, b)| have.get(*p) != Some(*b)) { let p = dir.join(p); std::fs::create_dir_all(p.parent().unwrap()).unwrap(); std::fs::write(p, bytes).unwrap(); } } /// A log, or the first of one, with `room` zeros after it: as a kill /// leaves a log that was open (`pds_store`). fn with_room(log: &[u8], room: usize) -> Vec { [log, &vec![0u8; room]].concat() } /// The room each log in the data directory holds: the zeros after its /// frames. fn rooms(dir: &Path) -> BTreeMap { files_whole(dir) .into_iter() .filter(|(p, _)| p.extension().is_some_and(|e| e == "log")) .map(|(p, whole)| { let room = whole.len() - super::without_room(whole).len(); (p, room) }) .collect() } /// A step of the script: the files after it, the journal's segments /// apart, each without its room, and the state. struct Step { files: BTreeMap>, journal: BTreeMap>, state: State, } /// The journal's segments in the data directory, each without its room. fn segments(dir: &Path) -> BTreeMap> { let mut j = files_whole(dir); j.retain(|p, _| p.starts_with(pds_store::journal::DIR)); for bytes in j.values_mut() { *bytes = super::without_room(std::mem::take(bytes)); } j } /// Run a script of requests over two accounts; after each, the files and the state. async fn script(dir: &Path, plc: &str) -> (Vec, Vec<(String, String)>) { let s = open(dir, plc); let mut steps = Vec::new(); let snap = |s: &Service, steps: &mut Vec| { steps.push(Step { files: files(dir), journal: segments(dir), state: state(s), }) }; snap(&s, &mut steps); let (a, ta) = create(&s, "alice").await; snap(&s, &mut steps); let (b, tb) = create(&s, "bob").await; snap(&s, &mut steps); let ta2 = session(&s, "alice").await; snap(&s, &mut steps); let w = |did: &str, k: &str, n: i64| json!({ "repo": did, "collection": "a.b.c", "rkey": k, "record": record(n) }); call(&s, "com.atproto.repo.putRecord", Some(&ta), w(&a, "one", 1)) .await .unwrap(); snap(&s, &mut steps); call(&s, "com.atproto.repo.putRecord", Some(&tb), w(&b, "one", 1)) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.repo.putRecord", Some(&ta2), w(&a, "two", 2), ) .await .unwrap(); snap(&s, &mut steps); call(&s, "com.atproto.repo.applyWrites", Some(&tb), json!({ "repo": b, "writes": [ { "$type": "com.atproto.repo.applyWrites#create", "collection": "a.b.c", "rkey": "three", "value": record(3) }, { "$type": "com.atproto.repo.applyWrites#update", "collection": "a.b.c", "rkey": "one", "value": record(4) } ] })).await.unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.repo.deleteRecord", Some(&ta), json!({ "repo": a, "collection": "a.b.c", "rkey": "one" }), ) .await .unwrap(); snap(&s, &mut steps); // A refused write changes nothing and logs nothing. assert!( call( &s, "com.atproto.repo.applyWrites", Some(&ta), json!({ "repo": b, "writes": [] }) ) .await .is_err() ); snap(&s, &mut steps); let n = steps.len(); assert_eq!(steps[n - 1].files, steps[n - 2].files); (steps, vec![(a, ta), (b, tb)]) } #[tokio::test] async fn restart_resumes_every_account() { let plc = plc().await; let dir = scratch("resume"); let (steps, accounts) = script(&dir, &plc).await; let s = open(&dir, &plc); assert_eq!(state(&s), steps.last().unwrap().state); // Four events open each account's stream; each of the five commits adds one. assert_eq!(s.events.len(), 4 + 4 + 5); assert_eq!(*s.events.subscribe().borrow(), s.events.len() as u64); drop(s); let s = open(&dir, &plc); for (i, (did, token)) in accounts.iter().enumerate() { let before = repo_of(&s, did, |r| r.rev.clone()); let body = json!({ "repo": did, "collection": "a.b.c", "rkey": "x", "record": { "n": 1 } }); call(&s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); assert!(repo_of(&s, did, |r| r.rev.clone()) > before); // The firehose carries on from the last event. let n = s.events.len(); assert_eq!(n, 13 + i + 1); let last = s.events.get(&s, n as u64 - 1, n as u64, 0).pop().unwrap(); let (_, at) = pds_repo::data::decode_prefix(&last).unwrap(); let body = pds_repo::data::to_json(&pds_repo::data::decode(&last[at..]).unwrap()); assert_eq!(body["seq"], json!(n)); assert_eq!(body["repo"], json!(did)); } // Accounts and sessions too: each logs in again, and each old token works. session(&s, "alice").await; for (did, token) in &accounts { assert_eq!( call(&s, "com.atproto.server.getSession", Some(token), json!({})) .await .unwrap()["did"], json!(did) ); } let _ = std::fs::remove_dir_all(&dir); } /// A data directory written before logs had room, each log its frames /// alone, is started on and carried on from. Its logs are given room as /// they are written to, which no count of their bytes includes; and a /// server killed with that room in them starts as one killed without. /// Under `each`: a log given a journal (`together`) makes no room. #[tokio::test] async fn logs_with_no_room_are_carried_on() { syncing(crate::Syncing::Each); let plc = plc().await; let dir = scratch("room-src"); let (steps, accounts) = script(&dir, &plc).await; let last = steps.last().unwrap(); let work = scratch("room-work"); put_files(&work, &last.files); assert_eq!(files_whole(&work), last.files); let s = open(&work, &plc); assert_eq!(state(&s), last.state); for n in 0..3 { for (did, token) in &accounts { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": format!("more{n}"), "record": record(n) }); call(&s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); } } let (killed, bare) = (files_whole(&work), files(&work)); for (did, _) in &accounts { let log = repo_log(Path::new(""), did); let (file, frames) = (killed[&log].len(), bare[&log].len()); assert!(file > frames, "{file} bytes of file, {frames} of frames"); assert_eq!(store::scan(&killed[&log]).unwrap().1, frames); // The log's bytes are its frames', by its own count and by its file. let counted = s.account(did).unwrap().store.log_bytes(); assert_eq!(counted, frames as u64); assert_eq!(crate::directory::files_bytes(&work, did), counted); } let after = state(&s); drop(s); for (name, disk) in [("room-killed", &killed), ("room-bare", &bare)] { let again = scratch(name); put_files(&again, disk); let s = open(&again, &plc); assert_eq!(state(&s), after, "{name}"); let (bob, token) = &accounts[1]; let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": record(9) }); call(&s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); let now = state(&s); drop(s); assert_eq!(state(&open(&again, &plc)), now, "{name}"); let _ = std::fs::remove_dir_all(&again); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Room is held by the logs open to be written to, and no others: a /// store let go leaves its account's log its frames alone, and a /// shutdown every log. A kill leaves each log the room it had: the /// service log until the start after it, an account's until its store /// is next loaded. Under `each`: a log given a journal (`together`) /// makes no room. #[tokio::test] async fn room_is_held_by_open_logs_alone() { syncing(crate::Syncing::Each); let plc = plc().await; let dir = scratch("room-held"); let s = open(&dir, &plc); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; for (did, token) in [(&alice, &ta), (&bob, &tb)] { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": "one", "record": record(1) }); call(&s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); } let service = PathBuf::from(ACCOUNTS_LOG_FILE); let [a, b] = [&alice, &bob].map(|did| repo_log(Path::new(""), did)); let held = rooms(&dir); assert!(held[&a] > 0 && held[&b] > 0, "{held:?}"); // Let go, a store leaves its log its frames alone. assert!(s.account(&alice).unwrap().store.evict()); let held = rooms(&dir); assert!(held[&a] == 0 && held[&b] > 0, "{held:?}"); let mut killed = files_whole(&dir); let log = killed.get_mut(&service).unwrap(); *log = with_room(&super::without_room(std::mem::take(log)), 1500); // A shutdown leaves every log so. crate::ops::settle(&s).unwrap(); assert!(rooms(&dir).values().all(|room| *room == 0)); drop(s); // Killed, the logs keep their room: the service log until the start // after, an account's until its store is next loaded. let work = scratch("room-held-killed"); put_files(&work, &killed); let s = open_with(&work, &plc, |c| c.load_all = false); assert!(!s.account(&bob).unwrap().store.is_loaded()); let held = rooms(&work); assert!(held[&service] == 0 && held[&b] > 0, "{held:?}"); assert_eq!( crate::directory::files_bytes(&work, &bob), super::without_room(killed[&b].clone()).len() as u64 ); s.account(&bob).unwrap().store.load().unwrap(); assert_eq!(rooms(&work)[&b], 0); drop(s); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Logs whose frames are intact but whose entries are not what the /// server wrote are refused with a reason, not a panic. #[tokio::test] async fn bad_entries_are_refused() { let plc = plc().await; let dir = scratch("bad-src"); let (steps, accounts) = script(&dir, &plc).await; let last = &steps.last().unwrap().files; let work = scratch("bad-work"); let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let alice = repo_log(Path::new(""), &accounts[0].0); let frames = |p: &PathBuf| store::scan(&last[p]).unwrap().0; let reopen = |p: &PathBuf, frames: &[Vec]| { let mut files = last.clone(); files.insert( p.clone(), frames.iter().flat_map(|f| store::frame(f)).collect(), ); put_files(&work, &files); try_open(&work, &plc).err() }; assert_eq!(reopen(&alice, &frames(&alice)), None); // A record the latest commit did not sign. let mut tampered = frames(&alice); let commit = tampered .iter_mut() .rev() .find_map(|f| { f.iter_mut() .find(|e| e["t"] == "commit" && !e["puts"].as_array().unwrap().is_empty()) }) .unwrap(); commit["puts"][0][1]["n"] = json!(99); assert!(reopen(&alice, &tampered).unwrap().contains("does not sign")); for (at, bad) in [ (1, json!({ "t": "nonsense" })), ( 1, json!({ "t": "event", "ord": 1, "seq": 1, "frame": "00" }), ), (2, json!({ "t": "commit", "rev": "x" })), ] { let mut broken = frames(&alice); broken[at].push(bad.clone()); assert!(reopen(&alice, &broken).is_some(), "{bad}"); } let a = accounts[0].0.as_str(); let session = |did: &str, app: Json, scope: &str| json!({ "t": "session", "did": did, "id": "s", "app": app, "scope": scope, "exp": 1 }); for bad in [ json!({ "t": "nonsense" }), json!({ "t": "session", "did": a }), session("did:plc:nobody", Json::Null, "com.atproto.access"), session(a, Json::Null, "com.atproto.appPass"), // A session of an app password not had is kept only with an app password's scope (`dropAppPasswords`). session(a, json!("no such app password"), "com.atproto.access"), json!({ "t": "refresh", "did": a, "id": "not kept", "next": "n", "exp": 1, "nextExp": 2 }), json!({ "t": "endSession", "did": a, "id": "not kept" }), json!({ "t": "appPassword", "did": a, "name": "x", "hash": "h", "createdAt": "c" }), json!({ "t": "revokeAppPassword", "did": a, "name": "not had" }), json!({ "t": "useToken", "did": a, "purpose": "confirm_email" }), json!({ "t": "mailToken", "did": a, "purpose": "nonsense", "token": "AAAAA-AAAAA", "made": 1 }), json!({ "t": "mailToken", "did": a, "purpose": "confirm_email", "token": "AAAAAAAAAA", "made": 1 }), json!({ "t": "mailToken", "did": a, "purpose": "confirm_email", "token": "AAAAA-AAAAA" }), json!({ "t": "mailToken", "did": "did:plc:nobody", "purpose": "confirm_email", "token": "AAAAA-AAAAA", "made": 1 }), json!({ "t": "emailAuthFactor", "did": a, "on": true }), json!({ "t": "updateEmail", "did": a, "email": "bob@mail.com" }), json!({ "t": "updateEmail", "did": a }), json!({ "t": "password", "did": a }), ] { let mut broken = frames(&accounts_log); broken[1].push(bad.clone()); assert!(reopen(&accounts_log, &broken).is_some(), "{bad}"); } // The same session twice. let mut twice = frames(&accounts_log); let first = twice .iter() .flatten() .find(|e| e["t"] == "session") .unwrap() .clone(); twice.push(vec![first]); assert!( reopen(&accounts_log, &twice) .unwrap() .contains("a second session") ); // A second account with the first's handle. // A second account with the first's email, or its handle. let mut twice = frames(&accounts_log); let mut copy = twice .iter() .flatten() .find(|e| e["t"] == "account") .unwrap() .clone(); copy["did"] = json!("did:plc:another"); twice.push(vec![copy.clone()]); assert!( reopen(&accounts_log, &twice) .unwrap() .contains("a second account") ); copy["email"] = json!("another@mail.com"); *twice.last_mut().unwrap() = vec![copy]; assert!( reopen(&accounts_log, &twice) .unwrap() .contains("is the handle of both") ); // A KEM key that is not one. for bad in [ json!("zz"), json!("00"), json!(hex::encode([0u8; 32])), json!(1), ] { let mut broken = frames(&accounts_log); broken .iter_mut() .flatten() .find(|e| e["t"] == "account") .unwrap()["kemKey"] = bad.clone(); assert!( reopen(&accounts_log, &broken) .unwrap() .contains("bad kemKey"), "{bad}" ); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// An account from before accounts had a KEM key is given none: it /// starts without one, its credentials name only its signing key, and /// its entry stays as it was. #[tokio::test] async fn an_account_without_a_kem_key() { let plc = plc().await; let dir = scratch("no-kem"); let s = open(&dir, &plc); let (alice, token) = create(&s, "alice").await; drop(s); redone(&dir); let path = accounts_log(&dir); let mut frames = store::scan(&std::fs::read(&path).unwrap()).unwrap().0; let entry = frames .iter_mut() .flatten() .find(|e| e["t"] == "account") .unwrap(); assert!(entry["kemKey"].is_string()); entry.as_object_mut().unwrap().remove("kemKey"); let bytes: Vec = frames.iter().flat_map(|f| store::frame(f)).collect(); std::fs::write(&path, &bytes).unwrap(); let s = open(&dir, &plc); assert!(s.account(&alice).unwrap().kem.is_none()); let got = call( &s, "com.atproto.identity.getRecommendedDidCredentials", Some(&token), json!({}), ) .await .unwrap(); assert_eq!( got["verificationMethods"] .as_object() .unwrap() .keys() .collect::>(), ["atproto"] ); drop(s); assert_eq!(std::fs::read(&path).unwrap(), bytes); let _ = std::fs::remove_dir_all(&dir); } /// Cut each log at every byte of the last frame the script wrote to it /// (and flip a byte in it): recovery must give the state before that /// write or after it, and keep working. #[tokio::test] async fn torn_writes_recover_to_before_or_after() { for sync in SYNCS { syncing(sync); torn_writes_once().await; } } async fn torn_writes_once() { let plc = plc().await; let dir = scratch("torn-src"); let (steps, accounts) = script(&dir, &plc).await; let (bob, token) = &accounts[1]; let work = scratch("torn-work"); let mut cuts = 0; // The steps after both accounts exist, each of which changes one log // besides the index, which takes the entries of the events a write // makes before the frame that holds them. let changes = |w: &[Step]| -> Vec { w[1].files .keys() .filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p)) .cloned() .collect() }; let mut index_cuts = 0; let windows: Vec<&[Step]> = steps.windows(2).skip(2).collect(); for (i, w) in windows.iter().enumerate() { let changed = changes(w); let [path] = &changed[..] else { assert_eq!(changed.len(), 0); continue; }; if windows[i + 1..] .iter() .any(|later| changes(later).contains(path)) { continue; } let (from, to) = (w[0].files[path].len(), w[1].files[path].len()); let whole = &w[1].files[path]; for cut in from..=to { let mut files = w[1].files.clone(); files.insert(path.clone(), whole[..cut].to_vec()); put_files(&work, &files); let s = open(&work, &plc); let want = if cut == to { &w[1].state } else { &w[0].state }; assert_eq!( &state(&s), want, "{} cut at {cut} of {from}..{to}", path.display() ); cuts += 1; // After recovery the logs take new writes and replay them. if cut % 61 == 0 || cut == from + 1 || cut == to { let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": { "n": 9 } }); call(&s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); let now = state(&s); drop(s); assert_eq!( state(&open(&work, &plc)), now, "{} cut at {cut}", path.display() ); } } // A damaged byte anywhere in the frame drops it. for at in (from..to).step_by(5) { let mut files = w[1].files.clone(); files.get_mut(path).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!( &state(&open(&work, &plc)), &w[0].state, "{} flip at {at} of {from}..{to}", path.display() ); } // The write's entries cut short, its frame lost. index_cuts += index_cut_short(&work, &plc, w, None); } assert!(cuts > 500, "{cuts}"); assert!(index_cuts > 100, "{index_cuts}"); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// A power cut as `PDS_SYNC` `together` leaves it, the journal cut at /// every byte of a write (`spec/DURABILITY.md`, "Writes waiting at /// once"): of each step that writes one log, the log as it was before /// the step, its frames never synced in it, a log the step made there /// and empty, as its making was synced; and the journal's segment cut /// at every byte of what the step wrote to it, every record before it /// whole. A start puts back what the journal holds whole: the state /// before the step, or after it once the step's records are whole. #[tokio::test] async fn journal_cut_at_every_byte() { let plc = plc().await; let dir = scratch("journal-cut-src"); let (steps, _) = script(&dir, &plc).await; let work = scratch("journal-cut-work"); let (mut cuts, mut stepped) = (0, 0); for w in steps.windows(2) { let changed: Vec<&PathBuf> = w[1] .files .keys() .filter(|p| p.extension().is_some_and(|e| e == "log")) .filter(|p| w[0].files.get(*p) != w[1].files.get(*p)) .collect(); if changed.is_empty() { continue; } // The segment the step wrote to, grown from what it was: none // where a checkpoint came between, which is no cut of this. let Some((segment, from, to)) = w[1].journal.iter().find_map(|(p, after)| { let before = w[0].journal.get(p)?; (after.len() > before.len() && after.starts_with(before)) .then(|| (p.clone(), before.len(), after.len())) }) else { continue; }; stepped += 1; let mut files = w[1].files.clone(); for p in &changed { files.insert( (*p).clone(), w[0].files.get(*p).cloned().unwrap_or_default(), ); } for cut in from..=to { let mut disk = files.clone(); disk.extend(w[1].journal.clone()); disk.insert(segment.clone(), w[1].journal[&segment][..cut].to_vec()); put_files(&work, &disk); let s = open(&work, &plc); let got = state(&s); if cut == to { assert_eq!( got, w[1].state, "{} cut at {cut} of {from}..{to}", segment.display() ); } else if cut == from { assert_eq!(got, w[0].state, "{} cut at {cut}", segment.display()); } else { assert!( got == w[0].state || got == w[1].state, "{} cut at {cut} of {from}..{to}", segment.display() ); } drop(s); cuts += 1; } } assert!(stepped >= 5 && cuts > 500, "{stepped} steps, {cuts} cuts"); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// A frame written over room and cut at every byte, the room after the /// cut as it was: the shortest write the script made to an account's /// log, and the one it made to the service log alone. Recovery must /// give the state before that write or after it, and keep working, as /// where the file ends at the cut /// (`torn_writes_recover_to_before_or_after`). #[tokio::test] async fn writes_torn_in_room_recover_to_before_or_after() { for sync in SYNCS { syncing(sync); writes_torn_in_room_once().await; } } async fn writes_torn_in_room_once() { let plc = plc().await; let dir = scratch("torn-room-src"); let (steps, accounts) = script(&dir, &plc).await; let (bob, token) = &accounts[1]; let work = scratch("torn-room-work"); let service = PathBuf::from(ACCOUNTS_LOG_FILE); // Of the steps that change one log, the service log or an account's, // the one of each kind whose frame is shortest. let mut shortest: BTreeMap = BTreeMap::new(); for w in steps.windows(2).skip(2) { let changed: Vec<&PathBuf> = w[1] .files .keys() .filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p)) .collect(); let [path] = changed[..] else { continue }; let len = w[1].files[path].len() - w[0].files[path].len(); let of = shortest.entry(*path == service).or_insert((len, w, path)); if len < of.0 { *of = (len, w, path); } } assert_eq!(shortest.len(), 2, "a write to each kind of log"); let mut cuts = 0; for (_, w, path) in shortest.into_values() { let (from, to) = (w[0].files[path].len(), w[1].files[path].len()); let whole = &w[1].files[path]; assert!(whole.starts_with(&w[0].files[path])); for cut in from..=to { // What of the frame reached the disk, then zeros to past // where it ends. let mut files = w[1].files.clone(); files.insert(path.clone(), with_room(&whole[..cut], to - cut + 1500)); put_files(&work, &files); let s = open(&work, &plc); let want = if cut == to { &w[1].state } else { &w[0].state }; assert_eq!( &state(&s), want, "{} cut at {cut} of {from}..{to}", path.display() ); cuts += 1; if cut % 61 == 0 || cut == from + 1 || cut == to { let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": { "n": 9 } }); call(&s, "com.atproto.repo.putRecord", Some(token), body) .await .unwrap(); let now = state(&s); drop(s); assert_eq!( state(&open(&work, &plc)), now, "{} cut at {cut}", path.display() ); } } // The write's entries cut short in room, its frame lost. cuts += index_cut_short(&work, &plc, w, Some(1500)); } assert!(cuts > 500, "{cuts}"); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Cut `accounts.log` at every byte of each frame a change to sessions or /// app passwords wrote (and flip a byte in it): recovery must give the /// state before that change or after it. Each kind of entry is cut: the /// account starts with as many sessions as it keeps, so that the next /// one drops one. #[tokio::test] async fn torn_session_writes() { for sync in SYNCS { syncing(sync); torn_session_writes_once().await; } } async fn torn_session_writes_once() { let plc = plc().await; let dir = scratch("torn-sessions"); let s = open(&dir, &plc); let (alice, _) = create(&s, "alice").await; let tokens = |got: Json| { ( got["accessJwt"].as_str().unwrap().to_string(), got["refreshJwt"].as_str().unwrap().to_string(), ) }; let login = |pw: &str| json!({ "identifier": "alice.test", "password": pw }); let (_, mut chain) = tokens( call( &s, "com.atproto.server.createSession", None, login("alice-pw"), ) .await .unwrap(), ); for _ in 2..sessions::MAX_SESSIONS { chain = tokens( call( &s, "com.atproto.server.refreshSession", Some(&chain), Json::Null, ) .await .unwrap(), ) .1; } assert_eq!(s.sessions.count(&alice), sessions::MAX_SESSIONS); let mut steps = vec![Step { files: files(&dir), journal: segments(&dir), state: state(&s), }]; let snap = |s: &Service, steps: &mut Vec| { steps.push(Step { files: files(&dir), journal: segments(&dir), state: state(s), }) }; let (access, r0) = tokens( call( &s, "com.atproto.server.createSession", None, login("alice-pw"), ) .await .unwrap(), ); assert_eq!(s.sessions.count(&alice), sessions::MAX_SESSIONS); snap(&s, &mut steps); let app = call( &s, "com.atproto.server.createAppPassword", Some(&access), json!({ "name": "phone", "privileged": true }), ) .await .unwrap(); snap(&s, &mut steps); let (_, app_r0) = tokens( call( &s, "com.atproto.server.createSession", None, login(app["password"].as_str().unwrap()), ) .await .unwrap(), ); snap(&s, &mut steps); let (_, r1) = tokens( call( &s, "com.atproto.server.refreshSession", Some(&r0), Json::Null, ) .await .unwrap(), ); snap(&s, &mut steps); call( &s, "com.atproto.server.refreshSession", Some(&app_r0), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.deleteSession", Some(&r1), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.revokeAppPassword", Some(&access), json!({ "name": "phone" }), ) .await .unwrap(); snap(&s, &mut steps); drop(s); let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let work = scratch("torn-sessions-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let changed: Vec<&PathBuf> = w[1] .files .keys() .filter(|p| w[0].files.get(*p) != w[1].files.get(*p)) .collect(); assert_eq!(changed, [&accounts_log]); assert_ne!(w[0].state, w[1].state); let whole = &w[1].files[&accounts_log]; let (from, to) = (w[0].files[&accounts_log].len(), whole.len()); let (frames, _) = store::scan(&whole[from..]).unwrap(); kinds.extend( frames .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = w[1].files.clone(); files.insert(accounts_log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { &w[1].state } else { &w[0].state }; assert_eq!( &state(&open(&work, &plc)), want, "cut at {cut} of {from}..{to}" ); } for at in (from..to).step_by(3) { let mut files = w[1].files.clone(); files.get_mut(&accounts_log).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!( &state(&open(&work, &plc)), &w[0].state, "flip at {at} of {from}..{to}" ); } } kinds.sort(); kinds.dedup(); // A deactivation with an OAuth token writes `dropAppPasswords`, in its // lifecycle frame: compact.rs's account_states_end_oauth_sessions. let mut want: Vec<&str> = crate::sessions::ENTRIES .iter() .copied() .filter(|k| *k != "dropAppPasswords") .collect(); want.sort(); assert_eq!(kinds, want); // After recovery from the last cut, the log takes changes, and replays them. put_files(&work, &steps.last().unwrap().files); let s = open(&work, &plc); call( &s, "com.atproto.server.createSession", None, login("alice-pw"), ) .await .unwrap(); let now = state(&s); drop(s); assert_eq!(state(&open(&work, &plc)), now); assert_ne!(now.1[&alice], steps.last().unwrap().state.1[&alice]); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Cut the log each lifecycle change wrote at every byte of its frame /// (and flip a byte in it): recovery must give the state before that /// change or after it. A deactivation, an activation, a takedown with /// the sessions it ends and its lifting, an event of a DID no account /// has, a record's and a blob's takedown, a deletion's token, and a /// deletion, whose account's files may be there still or not: each /// kind of entry is cut. #[tokio::test] async fn torn_lifecycle_writes() { for sync in SYNCS { syncing(sync); torn_lifecycle_writes_once().await; } } async fn torn_lifecycle_writes_once() { use base64::Engine; let plc = plc().await; let dir = scratch("torn-life"); let sink = scratch("torn-life-mail"); let s = crate::email::testing::open_with_sink(&dir, &plc, &sink); let (alice, ta) = create(&s, "alice").await; let (bob, tb) = create(&s, "bob").await; session(&s, "alice").await; let blob = crate::blobs::test_upload( &s, Some(&ta), Some("text/plain"), b"taken down".to_vec(), false, ) .await .unwrap()["blob"] .clone(); let w = |k: &str, v: Json| json!({ "repo": alice, "collection": "a.b.c", "rkey": k, "record": v }); call( &s, "com.atproto.repo.putRecord", Some(&ta), w("one", json!({ "b": blob })), ) .await .unwrap(); let admin = format!( "Basic {}", base64::engine::general_purpose::STANDARD.encode(format!("admin:{ADMIN_PASSWORD}")) ); let update = |body: Json| { let (s, admin) = (s.clone(), admin.clone()); async move { call_raw( &s, "com.atproto.admin.updateSubjectStatus", sessions::Bearer::of(Some(&admin)), &[], &body.to_string(), ) .await .unwrap() } }; let repo_ref = |did: &str| json!({ "$type": "com.atproto.admin.defs#repoRef", "did": did }); let mut steps = vec![Step { files: files(&dir), journal: segments(&dir), state: state(&s), }]; let snap = |s: &Service, steps: &mut Vec| { steps.push(Step { files: files(&dir), journal: segments(&dir), state: state(s), }) }; call( &s, "com.atproto.server.deactivateAccount", Some(&ta), json!({}), ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.activateAccount", Some(&ta), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); update(json!({ "subject": repo_ref(&alice), "takedown": { "applied": true, "ref": "r" } })) .await; snap(&s, &mut steps); update(json!({ "subject": repo_ref(&alice), "takedown": { "applied": false } })).await; snap(&s, &mut steps); update(json!({ "subject": repo_ref("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa") })).await; snap(&s, &mut steps); let uri = format!("at://{alice}/a.b.c/one"); update(json!({ "subject": { "$type": "com.atproto.repo.strongRef", "uri": uri, "cid": blob["ref"]["$link"] }, "takedown": { "applied": true } })).await; snap(&s, &mut steps); let blob_ref = json!({ "$type": "com.atproto.admin.defs#repoBlobRef", "did": alice, "cid": blob["ref"]["$link"] }); update(json!({ "subject": blob_ref, "takedown": { "applied": true, "ref": "b" } })).await; snap(&s, &mut steps); call( &s, "com.atproto.server.requestAccountDelete", Some(&tb), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); let token = crate::email::testing::last_token(&sink); call( &s, "com.atproto.server.deleteAccount", None, json!({ "did": bob, "password": "bob-pw", "token": token }), ) .await .unwrap(); snap(&s, &mut steps); drop(s); let work = scratch("torn-life-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { // The log a change wrote to, but the index, which takes the // entries of the events it makes before the frame that holds them. let written: Vec<&PathBuf> = w[1] .files .keys() .filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p)) .collect(); let [path] = written[..] else { panic!("{written:?}") }; let indexed = w[1] .files .iter() .any(|(p, b)| of_index(p) && w[0].files.get(p) != Some(b)); // A deletion's account's files go once it is logged: a cut keeps them. let gone: Vec<&PathBuf> = w[0] .files .keys() .filter(|p| !w[1].files.contains_key(*p)) .collect(); assert_ne!(w[0].state, w[1].state); let whole = &w[1].files[path]; let (from, to) = (w[0].files[path].len(), whole.len()); let (frames, _) = store::scan(&whole[from..]).unwrap(); kinds.extend( frames .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = if gone.is_empty() { w[1].files.clone() } else { with_index_of(w[0].files.clone(), &w[1].files) }; files.insert(path.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { &w[1].state } else { &w[0].state }; let s = open(&work, &plc); assert_eq!( &state(&s), want, "{} cut at {cut} of {from}..{to}", path.display() ); if cut == to { // What a deletion left of its account is gone once the // directories of no account are swept. s.sweep_repos(); assert!(gone.iter().all(|p| !work.join(p).exists())); drop(s); // The change's frame whole and its entries lost: the // service log's own events need none to be sent. if indexed { put_files(&work, &with_index_of(files, &w[0].files)); assert_eq!( &state(&open(&work, &plc)), want, "{} whole, its entries lost", path.display() ); } } } for at in (from..to).step_by(7) { let mut files = if gone.is_empty() { w[1].files.clone() } else { w[0].files.clone() }; let mut flipped = whole.clone(); flipped[at] ^= 0x55; files.insert(path.clone(), flipped); put_files(&work, &files); assert_eq!( &state(&open(&work, &plc)), &w[0].state, "{} flip at {at} of {from}..{to}", path.display() ); } // The change's entries cut short, its frame lost. if indexed { assert!(index_cut_short(&work, &plc, w, None) > 0); } } kinds.sort(); kinds.dedup(); for k in crate::lifecycle::ENTRIES.iter().chain(&[ crate::lifecycle::RECORD_TAKEDOWN, "takedownBlob", "endSession", "mailToken", ]) { assert!(kinds.iter().any(|x| x == k), "{k} not cut: {kinds:?}"); } // After recovery from the last cut, the logs take changes, and replay them. put_files(&work, &steps.last().unwrap().files); let s = open(&work, &plc); call( &s, "com.atproto.server.deactivateAccount", Some(&ta), json!({}), ) .await .unwrap(); let now = state(&s); drop(s); assert_eq!(state(&open(&work, &plc)), now); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&sink); let _ = std::fs::remove_dir_all(&work); } /// Cut `accounts.log` at every byte of each frame a change to an /// account's email, password or tokens wrote (and flip a byte in it): /// recovery must give the state before that change or after it, the /// sessions a reset password ends with it. Each kind of entry is cut. #[tokio::test] async fn torn_email_writes() { for sync in SYNCS { syncing(sync); torn_email_writes_once().await; } } async fn torn_email_writes_once() { use crate::email::testing::{last_token, open_with_sink}; let plc = plc().await; let dir = scratch("torn-email"); let sink = scratch("torn-email-mail"); let s = open_with_sink(&dir, &plc, &sink); let (_, access) = create(&s, "alice").await; create(&s, "bob").await; let mut steps = vec![Step { files: files(&dir), journal: segments(&dir), state: state(&s), }]; let snap = |s: &Service, steps: &mut Vec| { steps.push(Step { files: files(&dir), journal: segments(&dir), state: state(s), }) }; let token = || last_token(&sink); let login = |code: Option| { let mut b = json!({ "identifier": "alice.test", "password": "alice-pw" }); if let Some(c) = code { b["authFactorToken"] = json!(c); } b }; let factor = |on: bool, token: Option| { let mut b = json!({ "email": "alice@mail.com", "emailAuthFactor": on }); if let Some(t) = token { b["token"] = json!(t); } b }; call( &s, "com.atproto.server.requestEmailConfirmation", Some(&access), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.confirmEmail", Some(&access), json!({ "email": "alice@mail.com", "token": token() }), ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.updateEmail", Some(&access), factor(true, None), ) .await .unwrap(); snap(&s, &mut steps); assert_eq!( call(&s, "com.atproto.server.createSession", None, login(None)).await, Err("AuthFactorTokenRequired".into()) ); snap(&s, &mut steps); call( &s, "com.atproto.server.createSession", None, login(Some(token())), ) .await .unwrap(); snap(&s, &mut steps); assert_eq!( call( &s, "com.atproto.server.updateEmail", Some(&access), factor(false, None) ) .await, Err("TokenRequired".into()) ); snap(&s, &mut steps); call( &s, "com.atproto.server.updateEmail", Some(&access), factor(false, Some(token())), ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.requestEmailUpdate", Some(&access), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.updateEmail", Some(&access), json!({ "email": "alice2@mail.com", "token": token() }), ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.requestPasswordReset", None, json!({ "email": "alice2@mail.com" }), ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.resetPassword", None, json!({ "token": token(), "password": "new" }), ) .await .unwrap(); snap(&s, &mut steps); drop(s); let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let work = scratch("torn-email-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let changed: Vec<&PathBuf> = w[1] .files .keys() .filter(|p| w[0].files.get(*p) != w[1].files.get(*p)) .collect(); assert_eq!(changed, [&accounts_log]); assert_ne!(w[0].state, w[1].state); let whole = &w[1].files[&accounts_log]; let (from, to) = (w[0].files[&accounts_log].len(), whole.len()); let (frames, _) = store::scan(&whole[from..]).unwrap(); assert_eq!(frames.len(), 1); kinds.extend( frames .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = w[1].files.clone(); files.insert(accounts_log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { &w[1].state } else { &w[0].state }; assert_eq!( &state(&open(&work, &plc)), want, "cut at {cut} of {from}..{to}" ); } for at in (from..to).step_by(3) { let mut files = w[1].files.clone(); files.get_mut(&accounts_log).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!( &state(&open(&work, &plc)), &w[0].state, "flip at {at} of {from}..{to}" ); } } for k in crate::email::ENTRIES { assert!(kinds.iter().any(|got| got == k), "no {k} entry was cut"); } assert!(kinds.iter().any(|k| k == "endSession")); // After recovery from the last cut, the log takes changes, and replays them. put_files(&work, &steps.last().unwrap().files); let s = open_with_sink(&work, &plc, &sink); call( &s, "com.atproto.server.requestPasswordReset", None, json!({ "email": "alice2@mail.com" }), ) .await .unwrap(); let now = state(&s); drop(s); assert_eq!(state(&open(&work, &plc)), now); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&sink); let _ = std::fs::remove_dir_all(&work); } /// Cut `accounts.log` at every byte of each frame `putPreferences` wrote /// (and flip a byte in it): recovery must give every account's /// preferences as before that put or after it. #[tokio::test] async fn torn_preference_writes() { for sync in SYNCS { syncing(sync); torn_preference_writes_once().await; } } async fn torn_preference_writes_once() { let plc = plc().await; let dir = scratch("torn-prefs"); let cfg = || { let mut c = config(&plc); c.proxy.appview = Some(crate::proxy::AppView { url: "http://127.0.0.1:1".into(), did: "did:web:appview.test".into(), }); c }; let s = Arc::new(Service::open(&dir, cfg()).unwrap()); let (_, alice) = create(&s, "alice").await; let (_, bob) = create(&s, "bob").await; let prefs = |s: &Service| s.prefs.snapshot(); let mut steps = vec![(files(&dir), prefs(&s))]; let puts = [ ( &alice, json!([{ "$type": "app.bsky.actor.defs#adultContentPref", "enabled": true }]), ), (&bob, json!([{ "$type": "app.bsky.x" }])), (&alice, json!([])), ( &alice, json!([{ "$type": "app.bsky.actor.defs#personalDetailsPref", "birthDate": "2000-01-01T00:00:00Z" }, { "$type": "app.bsky.y", "n": "é" }]), ), ]; for (token, list) in puts { call( &s, "app.bsky.actor.putPreferences", Some(token), json!({ "preferences": list }), ) .await .unwrap(); steps.push((files(&dir), prefs(&s))); } drop(s); let log = PathBuf::from(ACCOUNTS_LOG_FILE); let work = scratch("torn-prefs-work"); let recovered = || prefs(&Service::open(&work, cfg()).unwrap()); let mut cuts = 0; for w in steps.windows(2) { let ((before_files, before), (after_files, after)) = (&w[0], &w[1]); assert_ne!(before, after); // Preferences are the service log's own. let changed: Vec<&PathBuf> = after_files .keys() .filter(|p| before_files.get(*p) != after_files.get(*p)) .collect(); assert_eq!(changed, [&log]); let whole = &after_files[&log]; let (from, to) = (before_files[&log].len(), whole.len()); let (frames, _) = store::scan(&whole[from..]).unwrap(); assert!( frames .iter() .flatten() .all(|e| e["t"] == crate::preferences::ENTRY) ); for cut in from..=to { let mut files = unnoted(after_files.clone()); files.insert(log.clone(), whole[..cut].to_vec()); put_files(&work, &files); assert_eq!( &recovered(), if cut == to { after } else { before }, "cut at {cut} of {from}..{to}" ); cuts += 1; } for at in (from..to).step_by(7) { let mut files = after_files.clone(); files.get_mut(&log).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!(&recovered(), before, "flip at {at} of {from}..{to}"); } } assert!(cuts > 100, "{cuts}"); // After recovery the log takes more, and replays them. put_files(&work, &steps.last().unwrap().0); let s = Arc::new(Service::open(&work, cfg()).unwrap()); call( &s, "app.bsky.actor.putPreferences", Some(&bob), json!({ "preferences": [] }), ) .await .unwrap(); assert_eq!(recovered(), prefs(&s)); assert_eq!(prefs(&s).len(), 1); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Cut `accounts.log` at every byte of each frame an OAuth change wrote /// (and flip a byte in it): recovery must give what OAuth keeps, and the /// single-use values, as before that change or after it. Each kind of /// entry is cut. #[tokio::test] async fn torn_oauth_writes() { for sync in SYNCS { syncing(sync); torn_oauth_writes_once().await; } } async fn torn_oauth_writes_once() { use crate::oauth::tests::{App, Browser, held_at, kept, serve, text}; let plc = plc().await; let dir = scratch("torn-oauth"); let srv = serve(open(&dir, &plc)).await; let (alice, alice_token) = create(&srv.svc, "alice").await; let mut steps = vec![(files(&dir), kept(&srv.svc))]; let snap = |steps: &mut Vec<_>| steps.push((files(&dir), kept(&srv.svc))); let mut app = App::new(&srv, "atproto"); let mut browser = Browser::new(&srv); let cid = app.client_id.clone(); let (_, pushed) = app.par(&[]).await; snap(&mut steps); browser.page(&cid, &pushed.request_uri).await; snap(&mut steps); // The device's cookies from before the sign-in, which it renews. let old_cookies = browser.cookies.clone(); let sign_in = json!({ "locale": "en", "identifier": "alice.test", "password": "alice-pw", "remember": true }); browser .api(&cid, &pushed.request_uri, "sign-in", sign_in, None) .await; snap(&mut steps); let consent = browser .api( &cid, &pushed.request_uri, "consent", json!({ "did": alice }), None, ) .await .json(); let url = url::Url::parse(consent["url"].as_str().unwrap()).unwrap(); let code = url .query_pairs() .find(|(k, _)| k == "code") .unwrap() .1 .into_owned(); snap(&mut steps); let t = app.exchange(&code, &pushed).await.json(); snap(&mut steps); let r = app.refresh(&text(&t, "refresh_token")).await.json(); snap(&mut steps); app.get_session(&text(&r, "access_token")).await; snap(&mut steps); app.revoke(&text(&r, "access_token")).await; snap(&mut steps); // A code refused, and spent. let (other, code) = crate::oauth::tests::approved(&mut app, &mut browser, "alice").await; steps.push((files(&dir), kept(&srv.svc))); let wrong = crate::oauth::tests::Pushed { verifier: "x".repeat(43), ..other }; app.exchange(&code, &wrong).await; snap(&mut steps); let (_, rejected) = app.par(&[]).await; browser.page(&cid, &rejected.request_uri).await; steps.push((files(&dir), kept(&srv.svc))); browser .api(&cid, &rejected.request_uri, "reject", json!({}), None) .await; snap(&mut steps); browser .api( &cid, &rejected.request_uri, "sign-out", json!({ "did": alice }), None, ) .await; snap(&mut steps); // Signed in again; past 30 seconds, the old cookies end the device. browser.api(&cid, &rejected.request_uri, "sign-in", json!({ "locale": "en", "identifier": "alice.test", "password": "alice-pw", "remember": true }), None).await; snap(&mut steps); srv.svc .sessions .skew .store(31, std::sync::atomic::Ordering::SeqCst); let mut old = Browser::new(&srv); old.cookies = old_cookies; old.api( &cid, &rejected.request_uri, "sign-out", json!({ "did": alice }), None, ) .await; snap(&mut steps); // The clients an account approved forgotten, as a deactivation with // an OAuth token does: one approved first, as a discoverable // client's is (two frames, not cut). let now = srv.svc.sessions.now(); let request = json!({ "t": "oauthRequest", "id": "req-forget", "client": "https://app.example/client.json", "redirectUri": "https://app.example/cb", "scope": ["atproto"], "state": "s", "codeChallenge": "c".repeat(43), "responseMode": "query", "jkt": "j", "expires": now + 300, }); let approve = json!({ "t": "oauthApprove", "id": "req-forget", "code": "cod-forget", "did": alice, "scope": ["atproto"], "at": now }); for e in [request, approve] { let n = srv.svc.log.append(std::slice::from_ref(&e)).unwrap(); assert!(crate::durable(srv.svc.log.clone(), n).await.is_ok()); srv.svc.oauth.replay(&e, &|_| true).unwrap(); } steps.push((files(&dir), kept(&srv.svc))); let ends = crate::oauth::Ends { clients: true, ..Default::default() }; let n = srv .svc .oauth .end_for(&alice, ends, |e| srv.svc.log.append(&e)) .unwrap(); assert!(crate::durable(srv.svc.log.clone(), n).await.is_ok()); snap(&mut steps); // A passkey added, signing in, and removed (`spec/PASSKEYS.md`): // its link and challenge are held in memory alone. let auth = format!("Bearer {alice_token}"); let (mut key, handle) = crate::oauth::passkeys::tests::add(&srv, &auth, "Laptop", 3).await; snap(&mut steps); let signed = crate::oauth::passkeys::tests::sign_in( &mut browser, &cid, &rejected.request_uri, &mut key, &handle, true, ) .await; assert_eq!(signed.status, 200, "{}", signed.body); snap(&mut steps); let id = base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, &key.id); crate::oauth::passkeys::tests::own( &srv, "POST", "_passkeyRemove", &auth, Some(json!({ "id": id })), ) .await; snap(&mut steps); drop(srv); let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let work = scratch("torn-oauth-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let ((before_files, before), (after_files, after)) = (&w[0], &w[1]); let changed: Vec<&PathBuf> = after_files .keys() .filter(|p| before_files.get(*p) != after_files.get(*p)) .collect(); assert_eq!(changed, [&accounts_log]); assert_ne!(before, after); let whole = &after_files[&accounts_log]; let from = before_files[&accounts_log].len(); let (frames, _) = store::scan(&whole[from..]).unwrap(); if frames.len() != 1 { // A step of several requests, as approving one: only the last is cut. continue; } kinds.extend( frames .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); let to = whole.len(); for cut in from..=to { let mut files = after_files.clone(); files.insert(accounts_log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { after } else { before }; let s = open(&work, &plc); let now = s.sessions.now(); assert_eq!( held_at(&(s.oauth.snapshot(), s.replay.snapshot(0)), now), held_at(want, now), "cut at {cut} of {from}..{to}" ); } for at in (from..to).step_by(7) { let mut files = after_files.clone(); files.get_mut(&accounts_log).unwrap()[at] ^= 0x55; put_files(&work, &files); let s = open(&work, &plc); let now = s.sessions.now(); assert_eq!( held_at(&(s.oauth.snapshot(), s.replay.snapshot(0)), now), held_at(before, now), "flip at {at} of {from}..{to}" ); } } kinds.sort(); kinds.dedup(); let mut want: Vec<&str> = crate::oauth::ENTRIES.to_vec(); want.push(crate::replay::ENTRY); want.sort(); assert_eq!(kinds, want); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Cut `accounts.log` at every byte of each frame a single-use value or /// a revocation wrote (and flip a byte in it): recovery must give the /// values and revocations held as before that write or after. Each kind /// of entry is cut. #[tokio::test] async fn torn_space_credential_writes() { for sync in SYNCS { syncing(sync); torn_space_credential_writes_once().await; } } async fn torn_space_credential_writes_once() { use crate::space_credentials::tests as creds; let plc = plc().await; let dir = scratch("torn-space-credentials"); let s = open(&dir, &plc); let (alice, token) = create(&s, "alice").await; let (bob, _) = create(&s, "bob").await; let personal = json!({ "$type": "com.atproto.personalspace.policy" }); call( &s, "com.atproto.space.createSpace", Some(&token), json!({ "spaceType": "com.example.forum", "skey": "main", "policy": personal }), ) .await .unwrap(); type Held = (BTreeMap<[u8; 32], u64>, BTreeMap<[u8; 32], u64>); let held = |s: &Service| -> Held { (s.single_use.snapshot(0), s.space_credentials.snapshot(0)) }; let mut steps = vec![(files(&dir), held(&s))]; let space = format!("at://{alice}/space/com.example.forum/main"); let t = creds::delegation_token(&s, &token, &space).await.unwrap(); creds::credential( &s, json!({ "space": space, "credentialKey": creds::client(1).kid, "delegationToken": t }), ) .await .unwrap(); steps.push((files(&dir), held(&s))); let auth = format!("Bearer {}", creds::service_auth_token(&s, &alice, &bob)); creds::revoke( &s, Some(&auth), json!({ "space": space, "credentials": ["a", "b"] }), ) .await .unwrap(); steps.push((files(&dir), held(&s))); drop(s); let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let work = scratch("torn-space-credentials-work"); // What a recovered server holds against what it should, at its own // time: a delegation token's value is held only a minute past its // issue, and a slow run may recover after that, rightly without it. let recovers_to = |want: &Held, why: String| { let s = open(&work, &plc); let now = s.sessions.now(); let live = |m: &BTreeMap<[u8; 32], u64>| { m.iter() .filter(|(_, t)| **t > now) .map(|(k, t)| (*k, *t)) .collect::>() }; assert_eq!( ( s.single_use.snapshot(now), s.space_credentials.snapshot(now) ), (live(&want.0), live(&want.1)), "{why}" ); }; let mut kinds = Vec::new(); for w in steps.windows(2) { let changed: Vec<&PathBuf> = w[1] .0 .keys() .filter(|p| w[0].0.get(*p) != w[1].0.get(*p)) .collect(); assert_eq!(changed, [&accounts_log]); assert_ne!(w[0].1, w[1].1); let whole = &w[1].0[&accounts_log]; let (from, to) = (w[0].0[&accounts_log].len(), whole.len()); kinds.extend( store::scan(&whole[from..]) .unwrap() .0 .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = w[1].0.clone(); files.insert(accounts_log.clone(), whole[..cut].to_vec()); put_files(&work, &files); recovers_to( if cut == to { &w[1].1 } else { &w[0].1 }, format!("cut at {cut} of {from}..{to}"), ); } for at in (from..to).step_by(3) { let mut files = w[1].0.clone(); files.get_mut(&accounts_log).unwrap()[at] ^= 0x55; put_files(&work, &files); recovers_to(&w[0].1, format!("flip at {at} of {from}..{to}")); } } assert_eq!( kinds, [crate::single_use::ENTRY, crate::space_credentials::ENTRY] ); // A malformed entry of either kind is refused with a reason. let last = &steps.last().unwrap().0; for bad in [ json!({ "t": crate::single_use::ENTRY, "v": "x", "until": 1 }), json!({ "t": crate::space_credentials::ENTRY, "space": "s", "credentials": [1], "until": 1 }), ] { let mut frames = store::scan(&last[&accounts_log]).unwrap().0; frames.push(vec![bad.clone()]); let mut files = last.clone(); files.insert( accounts_log.clone(), frames.iter().flat_map(|f| store::frame(f)).collect(), ); put_files(&work, &files); assert!(try_open(&work, &plc).is_err(), "{bad}"); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Cut an authority's `repo.log` at every byte of each frame a change to /// its spaces wrote (and flip a byte in it): recovery gives its spaces as /// before the change or after it, and the log takes changes after. Each /// kind of entry is cut. #[tokio::test] async fn torn_space_authority_writes() { for sync in SYNCS { syncing(sync); torn_space_authority_writes_once().await; } } async fn torn_space_authority_writes_once() { use crate::space_authority::{ENTRIES, Writer, track_writer}; use pds_repo::data::{self, V}; let plc = plc().await; let dir = scratch("torn-spaces"); let s = open(&dir, &plc); let (alice, ta) = create(&s, "alice").await; let (bob, _) = create(&s, "bob").await; let log = repo_log(Path::new(""), &alice); let spaces = |s: &Service| { crate::read(&s.account(&alice).unwrap().store) .authority .clone() }; let mut steps = vec![(files(&dir), spaces(&s))]; let snap = |s: &Service, steps: &mut Vec<_>| steps.push((files(&dir), spaces(s))); let m = format!("at://{alice}/space/com.example.forum/m"); let space = pds_space::SpaceRef::parse(&m).unwrap(); let track = |s: &Service| { let w = Writer { rev: "3jzfcijpj2z2a".into(), data: vec![7; 32], space_rev: "3jzfcijpj2z2b".into(), removed: false, }; assert!(track_writer( &mut crate::write(&s.account(&alice).unwrap().store), &space, &bob, &w )); }; let change = |nsid: &'static str, body: Json| { let (s, ta) = (s.clone(), ta.clone()); async move { assert_eq!( call(&s, nsid, Some(&ta), body).await, Ok(Json::Null), "{nsid}" ) } }; let policy = json!({ "$type": "com.atproto.memberspace.policy", "publicWrite": true }); call( &s, "com.atproto.space.createSpace", Some(&ta), json!({ "spaceType": "com.example.forum", "skey": "m", "policy": policy }), ) .await .unwrap(); snap(&s, &mut steps); change( "com.atproto.memberspace.putMembers", json!({ "space": m, "members": [{ "did": bob, "read": true, "write": true }] }), ) .await; snap(&s, &mut steps); track(&s); snap(&s, &mut steps); change( "com.atproto.space.removeWriter", json!({ "space": m, "repo": bob }), ) .await; snap(&s, &mut steps); track(&s); snap(&s, &mut steps); change( "com.atproto.memberspace.removeMembers", json!({ "space": m, "dids": [bob] }), ) .await; snap(&s, &mut steps); change( "com.atproto.space.updateSpace", json!({ "space": m, "policy": { "$type": "com.atproto.personalspace.policy" } }), ) .await; snap(&s, &mut steps); change("com.atproto.space.deleteSpace", json!({ "space": m })).await; snap(&s, &mut steps); let v = |j: Json| data::encode(&data::from_json(&j).unwrap()); let meta = data::from_json(&json!({ "$type": "atproto-space-meta", "ver": 1, "space": m })) .unwrap(); let writer = data::encode(&V::Map(vec![ ("repoRev".into(), V::Str("3jzfcijpj2z2a".into())), ("data".into(), V::Bytes(vec![1; 32])), ("spaceRev".into(), V::Str("3jzfcijpj2z2c".into())), ])); let (policy, member, writer_key) = ( v(json!({ "$type": "com.atproto.memberspace.policy" })), v(json!({ "$type": "com.atproto.memberspace.member", "read": true, "write": false })), format!("writer/{bob}"), ); let member_key = format!("com.atproto.memberspace.member/{bob}"); let archive = pds_space::star::write( &meta, [ (b"policy".as_slice(), policy.as_slice()), (member_key.as_bytes(), member.as_slice()), (writer_key.as_bytes(), writer.as_slice()), ], ); let mut headers = axum::http::HeaderMap::new(); headers.insert( axum::http::header::CONTENT_TYPE, axum::http::HeaderValue::from_static(crate::STAR_TYPE), ); let import = crate::Call { nsid: "com.atproto.space.importSpaceMeta", method: &axum::http::Method::POST, params: &[], body: &archive, bearer: sessions::Bearer::Token(&ta), headers: &headers, client: "127.0.0.1".into(), charged: Default::default(), http: None, }; assert!(matches!( crate::answer(&s, &import).await, Ok(crate::Resp::Nothing) )); snap(&s, &mut steps); drop(s); let work = scratch("torn-spaces-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let ((before_files, before), (after_files, after)) = (&w[0], &w[1]); let changed = changed_logs(before_files, after_files); assert_eq!(changed, [&log]); assert_ne!(before, after); let whole = &after_files[&log]; let (from, to) = (before_files[&log].len(), whole.len()); kinds.extend( store::scan(&whole[from..]) .unwrap() .0 .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = unnoted(after_files.clone()); files.insert(log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { after } else { before }; assert_eq!( &spaces(&open(&work, &plc)), want, "cut at {cut} of {from}..{to}" ); } for at in (from..to).step_by(5) { let mut files = after_files.clone(); files.get_mut(&log).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!( &spaces(&open(&work, &plc)), before, "flip at {at} of {from}..{to}" ); } } kinds.sort(); kinds.dedup(); let mut want: Vec<&str> = ENTRIES.to_vec(); want.sort(); assert_eq!(kinds.iter().map(String::as_str).collect::>(), want); // After recovery from the last cut, the log takes changes, and replays them. put_files(&work, &steps.last().unwrap().0); let s = open(&work, &plc); assert_eq!(&spaces(&s), &steps.last().unwrap().1); call( &s, "com.atproto.memberspace.putMembers", Some(&ta), json!({ "space": m, "members": [{ "did": "did:plc:carol", "read": true, "write": false }] }), ) .await .unwrap(); let now = spaces(&s); drop(s); assert_eq!(spaces(&open(&work, &plc)), now); assert_ne!(now, steps.last().unwrap().1); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Cut an account's `repo.log` at every byte of each frame the write /// notifications' part wrote (and flip a byte in it): recovery gives /// what it keeps (its retry queue, registrations and deletion notices) /// as before the frame or after it, and the log takes changes after. /// Each kind of entry is cut, and a malformed one is refused. #[tokio::test] async fn torn_space_notify_writes() { for sync in SYNCS { syncing(sync); torn_space_notify_writes_once().await; } } async fn torn_space_notify_writes_once() { use crate::space_notify::{ENTRIES, tests as notify}; let plc = plc().await; let dir = scratch("torn-notify"); let s = open(&dir, &plc); let (alice, _) = create(&s, "alice").await; let log = repo_log(Path::new(""), &alice); let kept = |s: &Service| { crate::read(&s.account(&alice).unwrap().store) .notify .clone() }; let (q, m) = ( "at://did:plc:elsewhere/space/com.example.forum/q".to_string(), format!("at://{alice}/space/com.example.forum/m"), ); let mut steps = vec![(files(&dir), kept(&s))]; for frame in notify::frames_of_each_kind(&q, &m) { notify::keep(&s, &alice, &frame); steps.push((files(&dir), kept(&s))); } drop(s); let work = scratch("torn-notify-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let ((before_files, before), (after_files, after)) = (&w[0], &w[1]); let changed = changed_logs(before_files, after_files); assert_eq!(changed, [&log]); assert_ne!(before, after); let whole = &after_files[&log]; let (from, to) = (before_files[&log].len(), whole.len()); kinds.extend( store::scan(&whole[from..]) .unwrap() .0 .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = unnoted(after_files.clone()); files.insert(log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { after } else { before }; assert_eq!( &kept(&open(&work, &plc)), want, "cut at {cut} of {from}..{to}" ); } for at in (from..to).step_by(3) { let mut files = unnoted(after_files.clone()); files.get_mut(&log).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!( &kept(&open(&work, &plc)), before, "flip at {at} of {from}..{to}" ); } } kinds.sort(); kinds.dedup(); let mut want: Vec<&str> = ENTRIES.to_vec(); want.sort(); assert_eq!(kinds.iter().map(String::as_str).collect::>(), want); // After recovery from the last cut, the log takes changes, and replays them. let last = &steps.last().unwrap(); put_files(&work, &last.0); let s = open(&work, &plc); assert_eq!(&kept(&s), &last.1); notify::keep(&s, &alice, ¬ify::frames_of_each_kind(&q, &m)[0]); let now = kept(&s); drop(s); assert_eq!(kept(&open(&work, &plc)), now); assert_ne!(now, last.1); // A malformed entry of each kind is refused with a reason, and so is // one of a kind this part does not keep, in a log marked as of this // version (`checkpoint::unmarked`). let space = json!(q); let mine = json!(m); for bad in [ json!({ "t": "notifyQueued", "space": space, "rev": "x", "data": "00", "n": 1, "due": 1, "until": 2 }), json!({ "t": "notifyQueued", "space": space, "rev": "3jzfcijpj2z2a", "data": "zz", "n": 1, "due": 1, "until": 2 }), json!({ "t": "notifyQueued", "space": space, "rev": "3jzfcijpj2z2a", "data": "00", "n": 1, "due": 1 }), json!({ "t": "notifyUnqueued", "space": "nope", "rev": "3jzfcijpj2z2a" }), json!({ "t": "syncRegistered", "space": mine, "endpoint": "https://s.example", "token": "t" }), json!({ "t": "syncRegistered", "space": mine, "endpoint": "https://s.example", "expires": 1 }), json!({ "t": "syncDropped", "space": mine, "token": "t" }), json!({ "t": "syncNoticed", "space": mine, "endpoint": "https://never.example", "token": "t" }), json!({ "t": "syncEmptied", "space": "nope" }), json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "other", "n": 0, "due": 1, "until": 2 }), json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "expired", "n": 0, "due": 1, "until": 2 }), json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "deleted", "n": -1, "due": 1, "until": 2 }), json!({ "t": "syncNoticeDone", "space": mine, "endpoint": "https://s.example" }), json!({ "t": "notifyRegistered", "space": mine, "service": "did:plc:x", "expires": 1 }), ] { let mut frames = store::scan(&last.0[&log]).unwrap().0; frames.push(vec![bad.clone()]); let mut files = last.0.clone(); files.insert( log.clone(), frames.iter().flat_map(|f| store::frame(f)).collect(), ); put_files(&work, &files); assert!(try_open(&work, &plc).is_err(), "{bad}"); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// An account's blobs, and the files of its blob directory. fn blob_state(s: &Service, dir: &Path, did: &str) -> (String, Vec) { let held = crate::read(&s.account(did).unwrap().store).blobs.snapshot(); let mut names: Vec = std::fs::read_dir(blob_dir(dir, did)) .map(|d| { d.map(|e| e.unwrap().file_name().into_string().unwrap()) .collect() }) .unwrap_or_default(); names.sort(); (held, names) } /// Cut the account's log at every byte of each blob write: an upload, a /// commit referencing it, and one dropping it, with the files as they /// are while that frame is on its way. An upload's file is in place /// before its entry is written, and a dropped blob's file goes only once /// the commit is on disk, so through each write the blob's file is /// there. Recovery gives the blobs as before the write or after, and /// leaves no file no entry holds: a kill between a commit dropping a blob /// and the removal of its file loses no blob still held, and leaks none. #[tokio::test] async fn torn_blob_writes_recover_to_before_or_after() { for sync in SYNCS { syncing(sync); torn_blob_writes_once().await; } } async fn torn_blob_writes_once() { let plc = plc().await; let dir = scratch("torn-blob-src"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let log = repo_log(Path::new(""), &did); let blob = b"spec-pds blob torn".to_vec(); let cid = pds_repo::data::cid_string(&pds_repo::data::cid_of(0x55, &blob)); let snap = |s: &Service| (files(&dir), blob_state(s, &dir, &did)); let mut steps = vec![snap(&s)]; crate::blobs::test_upload(&s, Some(&token), Some("text/plain"), blob.clone(), false) .await .unwrap(); steps.push(snap(&s)); let r = json!({ "f": { "$type": "blob", "ref": { "$link": cid }, "mimeType": "text/plain", "size": blob.len() } }); call( &s, "com.atproto.repo.putRecord", Some(&token), json!({ "repo": did, "collection": "a.b.c", "rkey": "a", "record": r }), ) .await .unwrap(); steps.push(snap(&s)); call( &s, "com.atproto.repo.deleteRecord", Some(&token), json!({ "repo": did, "collection": "a.b.c", "rkey": "a" }), ) .await .unwrap(); steps.push(snap(&s)); drop(s); assert_eq!(steps[1].1.1, vec![cid.clone()]); assert_eq!(steps[3].1.1, Vec::::new()); let work = scratch("torn-blob-work"); let mut cuts = 0; for w in steps.windows(2) { let ((before_files, before), (after_files, after)) = (&w[0], &w[1]); let (from, to) = (before_files[&log].len(), after_files[&log].len()); assert!(to > from); let mut files = before_files.clone(); files.extend(after_files.clone()); for cut in from..=to { files.insert(log.clone(), after_files[&log][..cut].to_vec()); put_files(&work, &files); let s = open(&work, &plc); let want = if cut == to { after } else { before }; assert_eq!( &blob_state(&s, &work, &did), want, "cut at {cut} of {from}..{to}" ); cuts += 1; } } assert!(cuts > 100, "{cuts}"); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Everything a restart must carry over, and each account's space repos. fn with_spaces(s: &Service) -> (State, BTreeMap) { let d = crate::read(&s.directory); let spaces = d .by_did .iter() .map(|(did, a)| { ( did.clone(), format!( "{:?}", crate::space_repo::tests::snapshot(&crate::read(&a.store)) ), ) }) .collect(); drop(d); (state(s), spaces) } /// Cut the account's log at every byte of each space commit's frame (and /// flip a byte in it): recovery gives the space repos as before that /// commit or after it, the repository and the events as they were, and /// takes new space writes, which replay. Each commit's notification goes /// to a space host that cannot be reached: the frames after the commit's /// keep it in the retry queue (`space_notify`), which this does not cut. #[tokio::test] async fn torn_space_writes_recover_to_before_or_after() { for sync in SYNCS { syncing(sync); torn_space_writes("log", false).await; } } /// The same of an account's log after a table that holds space repos: /// each commit replayed reads the records it replaces from the table, /// and a `clearRecords` and an import void what the table holds. #[tokio::test] async fn torn_space_writes_after_a_table() { for sync in SYNCS { syncing(sync); torn_space_writes("table", true).await; } } async fn torn_space_writes(name: &str, table: bool) { use crate::space_notify::tests::idle; use crate::space_repo::tests::{SPACE, w}; let plc = plc().await; let dir = scratch(&format!("torn-space-src-{name}")); let s = open(&dir, &plc); let (a, ta) = create(&s, "alice").await; let (b, tb) = create(&s, "bob").await; let snap = |s: &Service| (files(&dir), with_spaces(s)); let w = |s: &crate::Shared, nsid: &'static str, token: &str, body: Json| { let (s, token) = (s.clone(), token.to_string()); async move { let got = w(&s, nsid, &token, body).await; idle(&s).await; got } }; let rec = |did: &str, rkey: &str, n: i64| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) }); if table { for (did, token, rkey) in [(&a, &ta, "zero"), (&b, &tb, "zero"), (&b, &tb, "one")] { w(&s, "putRecord", token, rec(did, rkey, 0)).await.unwrap(); } for did in [&a, &b] { let wal = crate::read(&s.account(did).unwrap().store).wal.clone(); wal.compact().unwrap(); let layout = crate::store::account_layout(&dir, did); assert!(layout.checkpoint().exists()); } } let mut steps = vec![snap(&s)]; w(&s, "createRecord", &ta, rec(&a, "one", 1)).await.unwrap(); steps.push(snap(&s)); w(&s, "putRecord", &tb, rec(&b, "one", 2)).await.unwrap(); steps.push(snap(&s)); let writes = json!([ { "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "rkey": "two", "value": record(3) }, { "$type": "com.atproto.space.applyWrites#update", "collection": "a.b.c", "rkey": "one", "value": record(4) }, { "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "value": record(5) }, ]); w( &s, "applyWrites", &ta, json!({ "space": SPACE, "repo": a, "writes": writes }), ) .await .unwrap(); steps.push(snap(&s)); w( &s, "deleteRecord", &ta, json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "one" }), ) .await .unwrap(); steps.push(snap(&s)); w( &s, "clearRecords", &tb, json!({ "space": SPACE, "repo": b }), ) .await .unwrap(); steps.push(snap(&s)); // An import of alice's own archive back: one frame, of one entry. let params = [ ("space".to_string(), SPACE.to_string()), ("repo".to_string(), a.clone()), ]; let archive = { let none = axum::http::HeaderMap::new(); let get = crate::Call { nsid: "com.atproto.space.getRepo", method: &axum::http::Method::GET, params: ¶ms, body: b"", bearer: sessions::Bearer::Token(&ta), headers: &none, client: "127.0.0.1".into(), charged: Default::default(), http: None, }; let Ok(crate::Resp::Archive(archive, _)) = crate::answer(&s, &get).await else { panic!("getRepo answered no archive") }; archive.bytes().unwrap() }; let mut headers = axum::http::HeaderMap::new(); headers.insert( axum::http::header::CONTENT_TYPE, axum::http::HeaderValue::from_static("application/vnd.dasl.star"), ); let import = crate::Call { nsid: "com.atproto.space.importRepo", method: &axum::http::Method::POST, params: &[], body: &archive, bearer: sessions::Bearer::Token(&ta), headers: &headers, client: "127.0.0.1".into(), charged: Default::default(), http: None, }; assert!(matches!( crate::answer(&s, &import).await, Ok(crate::Resp::Nothing) )); steps.push(snap(&s)); // A refused write, or one making no commit, logs nothing. assert!(w(&s, "createRecord", &ta, rec(&a, "two", 6)).await.is_err()); w( &s, "deleteRecord", &ta, json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "none" }), ) .await .unwrap(); assert_eq!(files(&dir), steps.last().unwrap().0); drop(s); let work = scratch(&format!("torn-space-work-{name}")); let mut cuts = 0; for (i, win) in steps.windows(2).enumerate() { let ((before_files, before), (after_files, after)) = (&win[0], &win[1]); let changed = changed_logs(before_files, after_files); let [path] = changed[..] else { panic!("step {i} changed {changed:?}") }; assert_ne!(before, after); // The repository and the events are as they were. assert_eq!(before.0, after.0); let whole = &after_files[path]; let from = before_files[path].len(); let frames = store::scan(&whole[from..]).unwrap().0; let kinds: Vec<&str> = frames[0].iter().map(|e| e["t"].as_str().unwrap()).collect(); assert!( kinds == [crate::space_repo::ENTRY] || kinds == [crate::space_repo::IMPORT], "{kinds:?}" ); assert!( frames[1..] .iter() .flatten() .all(|e| crate::space_notify::ENTRIES.contains(&e["t"].as_str().unwrap())) ); let to = from + store::frame(&frames[0]).len(); for cut in from..=to { let mut files = unnoted(after_files.clone()); files.insert(path.clone(), whole[..cut].to_vec()); put_files(&work, &files); let s = open(&work, &plc); let want = if cut == to { after } else { before }; assert_eq!( &with_spaces(&s), want, "step {i}: cut at {cut} of {from}..{to}" ); cuts += 1; if cut % 97 == 0 || cut == from + 1 || cut == to { let got = w(&s, "putRecord", &tb, rec(&b, "after", 9)).await.unwrap(); assert!(got["cid"].is_string()); idle(&s).await; assert_eq!( with_spaces(&open(&work, &plc)), with_spaces(&s), "step {i}: cut at {cut}" ); } } for at in (from..to).step_by(7) { let mut files = unnoted(after_files.clone()); files.get_mut(path).unwrap()[at] ^= 0x55; put_files(&work, &files); assert_eq!( &with_spaces(&open(&work, &plc)), before, "step {i}: flip at {at} of {from}..{to}" ); } } assert!(cuts > 500, "{cuts}"); // A space commit whose records do not make its data is refused. let alice = changed_logs(&steps[0].0, &steps[1].0)[0].clone(); let last = &steps.last().unwrap().0; let mut frames = store::scan(&last[&alice]).unwrap().0; let entry = frames .iter_mut() .flatten() .find(|e| { e["t"] == crate::space_repo::ENTRY && !e["puts"].as_array().unwrap().is_empty() }) .unwrap(); entry["puts"][0][1]["n"] = json!(99); let mut files = last.clone(); files.insert(alice, frames.iter().flat_map(|f| store::frame(f)).collect()); put_files(&work, &files); assert!( try_open(&work, &plc) .err() .unwrap() .contains("do not make the data") ); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// An account whose entry is cut short is not there; one whose entry is /// whole but whose repository's first frame is not starts with a fresh /// first commit, once. #[tokio::test] async fn torn_account_creation() { for sync in SYNCS { syncing(sync); torn_account_creation_once().await; } } async fn torn_account_creation_once() { let plc = plc().await; let dir = scratch("torn-create"); let (steps, accounts) = script(&dir, &plc).await; let (before, after) = (&steps[1], &steps[2]); let bob = &accounts[1].0; let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let bob_log = repo_log(Path::new(""), bob); let work = scratch("torn-create-work"); let (from, to) = ( before.files[&accounts_log].len(), after.files[&accounts_log].len(), ); for cut in from..to { let mut files = before.files.clone(); files.insert( accounts_log.clone(), after.files[&accounts_log][..cut].to_vec(), ); put_files(&work, &files); assert_eq!( state(&open(&work, &plc)), before.state, "accounts.log cut at {cut}" ); } for cut in 0..after.files[&bob_log].len() { let mut files = after.files.clone(); files.insert(bob_log.clone(), after.files[&bob_log][..cut].to_vec()); put_files(&work, &files); let s = open(&work, &plc); assert!(repo_of(&s, bob, |r| r.record_count() == 0 && !r.rev.is_empty())); // Sent: alice's four events as before, then the four of bob's // fresh first commit, numbered after every seq given out: the // index keeps the entries of his first ones, whose frames were // cut, and a replay passes over them. let got = sent(&s); let seq = |e: &Json| e["seq"].as_u64().unwrap(); let first: Vec = after.state.2.iter().map(|f| seq(&body_of(f))).collect(); assert_eq!(got.len(), 8, "cut at {cut}: {got:?}"); assert_eq!(got[..4].iter().map(seq).collect::>(), first[..4]); // A `#commit` names its account as `repo`, the others as `did`. let of = |e: &Json| e.get("did").or_else(|| e.get("repo")).cloned(); assert!( got[..4] .iter() .all(|e| of(e).is_some_and(|d| d != json!(bob))) ); assert!( got[4..].iter().all(|e| of(e) == Some(json!(bob))), "cut at {cut}: {got:?}" ); assert!( got[4..].iter().map(seq).all(|q| q > first[7]), "cut at {cut}: {got:?}" ); assert!(got.windows(2).all(|w| seq(&w[0]) < seq(&w[1]))); // Its KEM key is the one its whole entry has. let kem = state(&s).0[bob.as_str()].2.clone(); assert!(kem.is_some()); assert_eq!(kem, after.state.0[bob.as_str()].2); if cut % 61 == 0 { let now = state(&s); drop(s); assert_eq!(state(&open(&work, &plc)), now); } } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// A handle change writes two frames to its account's log: the change /// recorded, then settled with its `#identity` event. Cut at every byte /// of them (or with a byte of the second flipped), recovery gives the /// account before the change, with it recorded and not settled, or with /// it landed; and one recorded is settled, landing whole, once the /// server serves. #[tokio::test] async fn torn_handle_changes() { for sync in SYNCS { syncing(sync); torn_handle_changes_once().await; } } async fn torn_handle_changes_once() { let plc = crate::identity::stub::Plc::start().await; let dir = scratch("torn-handle"); let s = open(&dir, &plc.url); let (alice, ta) = create(&s, "alice").await; create(&s, "bob").await; let before = files(&dir); let body = json!({ "handle": "carol.test" }); call(&s, "com.atproto.identity.updateHandle", Some(&ta), body) .await .unwrap(); let after = files(&dir); let after_state = state(&s); drop(s); let path = repo_log(Path::new(""), &alice); let changed = changed_logs(&before, &after); assert_eq!(changed, [&path]); // The settled change's entry is in the index. assert!( after .iter() .any(|(p, b)| of_index(p) && before.get(p) != Some(b)) ); let (from, to) = (before[&path].len(), after[&path].len()); let frames = store::scan(&after[&path]).unwrap().0; let recorded = from + store::frame(&frames[frames.len() - 2]).len(); // What recovery gives: alice's handle, the handles held, and how // many events a subscription from cursor 0 is sent. let identity = |s: &Service| { let d = crate::read(&s.directory); let mut held: Vec = d.claimed_handles.iter().cloned().collect(); held.sort(); let handle = d.by_did[&alice].handle(); drop(d); (handle, held, sent(s).len()) }; let n = after_state.2.len(); let work = scratch("torn-handle-work"); let mut unsettled = 0; let check = |files: BTreeMap>, at: String, cut: usize| { put_files(&work, &files); let s = open(&work, &plc.url); // Repositories and sessions are as they were throughout. let got = state(&s); assert_eq!((&got.0, &got.1), (&after_state.0, &after_state.1), "{at}"); let want = if cut < recorded { ("alice.test".to_string(), vec![], n - 1) } else if cut < to { ( "alice.test".to_string(), vec!["carol.test".to_string()], n - 1, ) } else { ("carol.test".to_string(), vec![], n) }; assert_eq!(identity(&s), want, "{at}"); if want.1.is_empty() { assert_eq!(state(&open(&work, &plc.url)), got, "{at}"); } s }; for cut in from..=to { let mut files = unnoted(after.clone()); files.insert(path.clone(), after[&path][..cut].to_vec()); let s = check(files, format!("cut at {cut} of {from}..{to}"), cut); // One recorded and not settled lands whole once the server serves. if (recorded..to).contains(&cut) && (cut == recorded || cut % 23 == 0) { crate::identity::settle_unfinished(&s); for _ in 0..100 { if identity(&s).0 == "carol.test" { break; } tokio::time::sleep(std::time::Duration::from_millis(20)).await; } assert_eq!( identity(&s), ("carol.test".to_string(), vec![], n), "settling after a cut at {cut}" ); drop(s); let s = open(&work, &plc.url); assert_eq!( identity(&s), ("carol.test".to_string(), vec![], n), "after settling a cut at {cut}" ); unsettled += 1; } } for at in (recorded..to).step_by(3) { let mut files = unnoted(after.clone()); files.get_mut(&path).unwrap()[at] ^= 0x55; check(files, format!("flip at {at} of {recorded}..{to}"), recorded); } assert!(unsettled > 3, "{unsettled}"); assert_eq!(plc.log(&alice).len(), 2); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// An event that never reached the disk, while a later one of another /// account's did: a restart leaves its number unused, the same way every /// time, and the next event is numbered after every one given out. #[tokio::test] async fn lost_events_leave_their_numbers_unused() { for sync in SYNCS { syncing(sync); lost_events_once().await; } } async fn lost_events_once() { let plc = plc().await; let dir = scratch("gap"); let (steps, accounts) = script(&dir, &plc).await; let (a, ta) = &accounts[0]; // Drop alice's last frame, her deleteRecord, and put bob's next commit after it. let s = open(&dir, &plc); let n = sent(&s).len() as u64; let body = json!({ "repo": accounts[1].0, "collection": "a.b.c", "rkey": "late", "record": { "n": 1 } }); call(&s, "com.atproto.repo.putRecord", Some(&accounts[1].1), body) .await .unwrap(); drop(s); let mut files = files(&dir); let alice = repo_log(Path::new(""), a); let cut = steps[steps.len() - 3].files[&alice].len(); files.get_mut(&alice).unwrap().truncate(cut); let work = scratch("gap-work"); put_files(&work, &files); let s = open(&work, &plc); assert_eq!(sent(&s).len() as u64, n); let seqs = sent_seqs; let got = seqs(&s); let lost: Vec = (1..=n + 1).filter(|i| !got.contains(i)).collect(); assert_eq!(lost.len(), 1, "{got:?}"); assert_eq!(*s.events.subscribe().borrow(), n + 1); let first = state(&s); drop(s); let s = open(&work, &plc); assert_eq!(state(&s), first); // A new event follows on, and every restart after numbers them alike. let body = json!({ "repo": a, "collection": "a.b.c", "rkey": "new", "record": { "n": 1 } }); call(&s, "com.atproto.repo.putRecord", Some(ta), body) .await .unwrap(); assert_eq!(seqs(&s).last(), Some(&(n + 2))); let want = state(&s); drop(s); assert_eq!(state(&open(&work, &plc)), want); assert_eq!(state(&open(&work, &plc)), want); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// The seal that closes a log. fn seal() -> Vec { store::frame(&[json!({ "t": "seal" })]) } /// A compaction killed at each step of `spec/STORE.md`'s crash table, /// with the writes of every file it had not made durable lost: an /// account's log, then the service log, each with a write after its seal. /// The account's table and its log hold space repos, which the fold /// merges with what the log voids of them. Every start recovers the /// state before the compaction, or with that write too where all of it /// is in the new log. #[tokio::test] async fn compaction_killed_at_every_step() { for sync in SYNCS { syncing(sync); compaction_killed_at_every_step_once().await; } } async fn compaction_killed_at_every_step_once() { let plc = plc().await; let dir = scratch("compact-killed"); let (_, accounts) = script(&dir, &plc).await; let (a, ta) = &accounts[0]; // The account's space repos: in its table, and in its log after it, // which writes over the table's records, deletes one and clears a repo. { use crate::space_notify::tests::idle; use crate::space_repo::tests::{SPACE, w}; let other = "at://did:web:example.com/space/com.example.chat/room"; let s = open(&dir, &plc); let rec = |space: &str, rkey: &str, n: i64| json!({ "space": space, "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record(n) }); for (space, rkey, n) in [(SPACE, "one", 1), (SPACE, "two", 2), (other, "one", 3)] { w(&s, "putRecord", ta, rec(space, rkey, n)).await.unwrap(); } idle(&s).await; let wal = crate::read(&s.account(a).unwrap().store).wal.clone(); wal.compact().unwrap(); drop(wal); w(&s, "putRecord", ta, rec(SPACE, "two", 22)).await.unwrap(); w(&s, "putRecord", ta, rec(SPACE, "three", 3)) .await .unwrap(); let gone = json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "one" }); w(&s, "deleteRecord", ta, gone).await.unwrap(); w(&s, "clearRecords", ta, json!({ "space": other, "repo": a })) .await .unwrap(); w(&s, "putRecord", ta, rec(other, "after", 4)) .await .unwrap(); idle(&s).await; } let work = scratch("compact-killed-work"); let state = with_spaces; for which in ["account", "service"] { let s = open(&dir, &plc); let before = (files(&dir), state(&s)); let (log, layout) = match which { "account" => ( crate::read(&s.account(a).unwrap().store).wal.clone(), crate::store::account_layout(Path::new(""), a), ), _ => (s.log.clone(), crate::store::service_layout(Path::new(""))), }; let g = log.generation(); log.compact().unwrap(); assert_eq!(log.generation(), g + 1); let compacted = files(&dir); match which { "account" => { let body = json!({ "repo": a, "collection": "a.b.c", "rkey": format!("after-{g}"), "record": record(9) }); call(&s, "com.atproto.repo.putRecord", Some(ta), body) .await .unwrap(); } _ => drop(session(&s, "alice").await), } let after = (files(&dir), state(&s)); drop(s); let (old, new, ckpt) = (layout.log(g), layout.log(g + 1), layout.checkpoint()); assert!(!compacted.contains_key(&old) && compacted.contains_key(&ckpt)); let mut sealed = before.0[&old].clone(); sealed.extend(seal()); let written = after.0[&new].clone(); let check = |files: BTreeMap>, why: String, whole: bool| { put_files(&work, &files); let s = open(&work, &plc); let got = state(&s); let want = if whole { &after.1 } else { &before.1 }; assert!(got == *want, "{which}: {why}"); // A start makes it whole again: the same state once more. drop(s); assert!( state(&open(&work, &plc)) == *want, "{which}: {why}, started twice" ); }; // The files but the log's, as after the write. let base: BTreeMap> = after .0 .iter() .filter(|(p, _)| ![&old, &new, &ckpt].contains(p)) .map(|(p, b)| (p.clone(), b.clone())) .collect(); let with = |logs: &[(&PathBuf, &[u8])]| { let mut f = base.clone(); // The table before, if it had one. if let Some(table) = before.0.get(&ckpt) { f.insert(ckpt.clone(), table.clone()); } for (p, b) in logs { f.insert((*p).clone(), b.to_vec()); } f }; // In step 1: the seal cut at every byte, log g + 1 not made. for cut in 0..seal().len() { check( with(&[(&old, &sealed[..before.0[&old].len() + cut])]), format!("the seal cut at {cut}"), false, ); } // Sealed, log g + 1 not made, or made: in step 2, as in step 1. check( with(&[(&old, &sealed)]), "sealed, no next log".into(), false, ); for cut in 0..=written.len() { let whole = cut == written.len(); check( with(&[(&old, &sealed), (&new, &written[..cut])]), format!("the next log cut at {cut}"), whole, ); // In step 3: the checkpoint's `.tmp` cut short, at some lengths. if cut == 0 || whole { let tmp = layout.tmp(); let bytes = &compacted[&ckpt]; for at in (0..bytes.len()) .step_by((bytes.len() / 7).max(1)) .chain([bytes.len()]) { let mut f = with(&[(&old, &sealed), (&new, &written[..cut])]); f.insert(tmp.clone(), bytes[..at].to_vec()); check( f, format!("the .tmp cut at {at}, the next log at {cut}"), whole, ); } // In step 4 after the rename, and in step 5 with the old log gone or not. let mut f = with(&[(&old, &sealed), (&new, &written[..cut])]); f.insert(ckpt.clone(), bytes.clone()); check(f.clone(), format!("renamed, the next log at {cut}"), whole); f.remove(&old); check( f, format!("the old log deleted, the next log at {cut}"), whole, ); } } // A log missing between the checkpoint's and the last is data not recovered. let mut f = with(&[(&new, &written)]); f.insert(ckpt.clone(), compacted[&ckpt].clone()); f.insert(layout.log(g + 3), Vec::new()); put_files(&work, &f); assert!( Service::open(&work, config(&plc)).is_err(), "{which}: a gap" ); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Everything a restart must carry over of the accounts, with each /// one's space repos, the spaces it is the authority of, and its write /// notifications. fn with_all(s: &Service) -> (State, BTreeMap, BTreeMap) { let kept = crate::read(&s.directory) .by_did .iter() .map(|(did, a)| { let st = crate::read(&a.store); (did.clone(), format!("{:?}", (&st.authority, &st.notify))) }) .collect(); let (state, spaces) = with_spaces(s); (state, spaces, kept) } /// Account tables of the version before, and logs after them as that /// version wrote them: the first load of each folds it into a table of /// this version, once, and serves all of it, the same wherever the fold /// was cut short, when the server is started again, and when each /// account is loaded as it is first asked for; and takes writes after. /// Of spaces, the state in their `S` and in the logs is let go, with /// the blobs only space records held (`checkpoint::unmarked`), and /// spaces are written after as ever. #[tokio::test] async fn a_table_of_the_version_before_is_folded_once() { use crate::space_notify::tests::idle; use crate::space_repo::tests::{SPACE, w}; let plc = plc().await; let dir = scratch("before-src"); let s = open(&dir, &plc); let (a, ta) = create(&s, "alice").await; let (b, tb) = create(&s, "bob").await; // An account no start loads: nothing of its is waiting to be sent. let (c, tc) = create(&s, "carol").await; let (one, two) = ( b"spec-pds before one".to_vec(), b"spec-pds before two".to_vec(), ); let mut refs = Vec::new(); for body in [&one, &two] { let up = crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), body.clone(), false) .await .unwrap(); refs.push(up["blob"].clone()); } let made = call(&s, "com.atproto.space.createSpace", Some(&ta), json!({ "spaceType": "com.example.forum", "skey": "mine", "policy": { "$type": crate::space_authority::MEMBER, "publishPolicy": true, "publishMembers": true } })) .await .unwrap(); let mine = made["space"].as_str().unwrap().to_string(); let member = |did: &str, write: bool| json!({ "space": mine, "members": [{ "did": did, "read": true, "write": write }] }); call( &s, "com.atproto.memberspace.putMembers", Some(&ta), member(&b, true), ) .await .unwrap(); let rec = |space: &str, did: &str, rkey: &str, record: Json| json!({ "space": space, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record }); let public = |rkey: &str, record: Json| json!({ "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record }); call( &s, "com.atproto.repo.putRecord", Some(&ta), public("pic", json!({ "img": refs[0] })), ) .await .unwrap(); for (space, rkey, record) in [ (SPACE, "one", record(1)), (SPACE, "two", json!({ "img": refs[1] })), (mine.as_str(), "one", record(3)), (mine.as_str(), "two", record(4)), ] { w(&s, "putRecord", &ta, rec(space, &a, rkey, record)) .await .unwrap(); } w( &s, "deleteRecord", &ta, json!({ "space": mine, "repo": a, "collection": "a.b.c", "rkey": "two" }), ) .await .unwrap(); for rkey in ["one", "zero"] { w(&s, "putRecord", &tb, rec(SPACE, &b, rkey, record(5))) .await .unwrap(); } let carols = json!({ "repo": c, "collection": "a.b.c", "rkey": "one", "record": record(1) }); call(&s, "com.atproto.repo.putRecord", Some(&tc), carols) .await .unwrap(); idle(&s).await; // The tables, then the logs after them. for did in [&a, &b, &c] { let wal = crate::read(&s.account(did).unwrap().store).wal.clone(); wal.compact().unwrap(); } w(&s, "putRecord", &ta, rec(SPACE, &a, "one", record(11))) .await .unwrap(); w( &s, "putRecord", &ta, rec(SPACE, &a, "three", json!({ "img": refs[1] })), ) .await .unwrap(); w( &s, "deleteRecord", &ta, json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "two" }), ) .await .unwrap(); w(&s, "putRecord", &ta, rec(&mine, &a, "five", record(5))) .await .unwrap(); call( &s, "com.atproto.memberspace.putMembers", Some(&ta), member("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", false), ) .await .unwrap(); call( &s, "com.atproto.repo.putRecord", Some(&ta), public("after", record(2)), ) .await .unwrap(); w(&s, "putRecord", &tb, rec(SPACE, &b, "one", record(6))) .await .unwrap(); w( &s, "clearRecords", &tb, json!({ "space": SPACE, "repo": b }), ) .await .unwrap(); w(&s, "putRecord", &tb, rec(SPACE, &b, "after", record(7))) .await .unwrap(); idle(&s).await; let want = with_all(&s); let cid = |r: &Json| r["ref"]["$link"].as_str().unwrap().to_string(); let (one, two) = (cid(&refs[0]), cid(&refs[1])); assert!(want.0.5.0[&a].2.contains(&two)); let g = crate::read(&s.account(&a).unwrap().store).wal.generation(); drop(s); // The tables as the version before wrote them; and in the logs, a // `clearRecords` that names each record it deletes, as spaces once // logged it. for did in [&a, &b, &c] { let layout = crate::store::account_layout(&dir, did); crate::checkpoint::downgrade(&layout.checkpoint()); let log = layout.log(g); let mut frames = store::scan(&std::fs::read(&log).unwrap()).unwrap().0; for e in frames.iter_mut().flatten() { if e.get("kept") == Some(&json!([])) { e.as_object_mut().unwrap().remove("kept"); e["dels"] = json!(["a.b.c/one", "a.b.c/zero"]); } } let bytes: Vec = frames.iter().flat_map(|f| store::frame(f)).collect(); std::fs::write(&log, bytes).unwrap(); } let before = files(&dir); let layout = crate::store::account_layout(Path::new(""), &a); let (old, new, ckpt) = (layout.log(g), layout.log(g + 1), layout.checkpoint()); assert!(!store::scan(&before[&old]).unwrap().0.is_empty()); // Folded on its first load, and served. let work = scratch("before-work"); put_files(&work, &before); let s = open(&work, &plc); // Each account's state as it was but for its blobs; no spaces state; // and of alice's blobs, the one a public record references alone. let public = |mut state: State| { for (_, _, blobs) in state.5.0.values_mut() { blobs.clear(); } state }; let served = with_all(&s); assert!(public(served.0.clone()) == public(want.0.clone()), "folded"); for (did, spaces) in &served.1 { assert_eq!(spaces, "[]", "{did}"); } let none = format!( "{:?}", ( &crate::space_authority::Authority::default(), &crate::space_notify::Kept::default() ) ); assert!(served.2.values().all(|k| *k == none)); { let account = s.account(&a).unwrap(); let st = crate::read(&account.store); let held: Vec<&str> = st.blobs.held_list().map(|(cid, _, _)| cid).collect(); assert_eq!(held, [one.as_str()]); assert!(st.blobs.space_ref_list().is_empty()); } assert!(!blob_dir(&work, &a).join(&two).exists()); assert!(blob_dir(&work, &a).join(&one).exists()); drop(s); let folded = files(&work); assert!(!folded.contains_key(&old) && folded.contains_key(&new)); assert!(folded[&ckpt] != before[&ckpt]); // Once: a start after it reads the table it made, and writes go on. let s = open(&work, &plc); assert!(with_all(&s) == served, "started again"); assert!(files(&work)[&ckpt] == folded[&ckpt]); w(&s, "putRecord", &ta, rec(SPACE, &a, "later", record(8))) .await .unwrap(); w(&s, "putRecord", &tb, rec(SPACE, &b, "zero", record(9))) .await .unwrap(); idle(&s).await; let later = with_all(&s); assert!(later != served); drop(s); assert!(with_all(&open(&work, &plc)) == later, "written after"); // An account not loaded at the start: its table has no header to // find its events by, which the firehose's index has; it is loaded // as first asked for, and folded then. put_files(&work, &before); let s = open_with(&work, &plc, |c| c.load_all = false); assert!(!s.account(&c).is_some_and(|a| a.store.is_loaded())); assert!(crate::checkpoint::events_on_disk(&s.ctx, &c).is_err()); s.events.find_old(); assert!(!s.account(&c).is_some_and(|a| a.store.is_loaded())); for did in [&a, &b, &c] { assert!(s.by_did(did).is_some()); } assert!(with_all(&s) == served, "loaded as first asked for"); drop(s); // The fold cut at every step: the seal at every byte, the next log // made or not, the table written in part, put in place, and the log // it covers deleted or not. let check = |files: BTreeMap>, why: String| { put_files(&work, &files); let s = open(&work, &plc); assert!(with_all(&s) == served, "{why}"); drop(s); assert!( with_all(&open(&work, &plc)) == served, "{why}, started twice" ); }; let with = |logs: &[(&PathBuf, &[u8])]| { let mut f = before.clone(); for (p, b) in logs { f.insert((*p).clone(), b.to_vec()); } f }; let mut sealed = before[&old].clone(); sealed.extend(seal()); for cut in 0..seal().len() { check( with(&[(&old, &sealed[..before[&old].len() + cut])]), format!("the seal cut at {cut}"), ); } check(with(&[(&old, &sealed)]), "sealed, no next log".into()); check( with(&[(&old, &sealed), (&new, &[])]), "sealed, the next log made".into(), ); let bytes = &folded[&ckpt]; for at in (0..bytes.len()) .step_by((bytes.len() / 9).max(1)) .chain([bytes.len()]) { let mut f = with(&[(&old, &sealed), (&new, &[])]); f.insert(layout.tmp(), bytes[..at].to_vec()); check(f, format!("the .tmp cut at {at}")); } let mut f = with(&[(&old, &sealed), (&new, &[])]); f.insert(ckpt.clone(), bytes.clone()); check(f.clone(), "put in place".into()); f.remove(&old); check(f, "the log it covers deleted".into()); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// An account's log with no mark, as an earlier version left it, sealed /// with a log after it: its spaces entries, of the kinds this server /// logs and of others, are let go as its store loads, and its other /// entries replayed as they were; a blob only its space records /// referenced is no longer held. The mark is appended once, and the /// account takes spaces writes after, which replay, and fold. #[tokio::test] async fn spaces_before_the_mark_are_let_go() { use crate::space_notify::tests::idle; use crate::space_repo::tests::{SPACE, r, w}; let plc = plc().await; let dir = scratch("unmarked-src"); let s = open(&dir, &plc); let (a, ta) = create(&s, "alice").await; let mut refs = Vec::new(); for body in [b"unmarked one".to_vec(), b"unmarked two".to_vec()] { let up = crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), body, false) .await .unwrap(); refs.push(up["blob"].clone()); } let cid = |r: &Json| r["ref"]["$link"].as_str().unwrap().to_string(); let (one, two) = (cid(&refs[0]), cid(&refs[1])); call( &s, "com.atproto.repo.putRecord", Some(&ta), json!({ "repo": a, "collection": "a.b.c", "rkey": "pic", "record": { "img": refs[0] } }), ) .await .unwrap(); let space = |s: &crate::Shared| { let (s, ta) = (s.clone(), ta.clone()); let body = json!({ "spaceType": "com.example.forum", "skey": "mine", "policy": { "$type": crate::space_authority::MEMBER, "publishMembers": true } }); async move { call(&s, "com.atproto.space.createSpace", Some(&ta), body) .await .unwrap()["space"] .as_str() .unwrap() .to_string() } }; let mine = space(&s).await; call( &s, "com.atproto.memberspace.putMembers", Some(&ta), json!({ "space": mine, "members": [{ "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", "read": true, "write": false }] }), ) .await .unwrap(); let rec = |space: &str, rkey: &str, record: Json| json!({ "space": space, "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record }); for (space, rkey, record) in [ (SPACE, "one", json!({ "img": refs[1] })), (SPACE, "two", json!({ "img": refs[0] })), (mine.as_str(), "x", record(1)), ] { w(&s, "putRecord", &ta, rec(space, rkey, record)) .await .unwrap(); } idle(&s).await; let want = with_all(&s); assert!(want.0.5.0[&a].2.contains(&two) && want.1[&a] != "[]"); let g = crate::read(&s.account(&a).unwrap().store).wal.generation(); drop(s); // As an earlier version left it: no mark, and entries of kinds this // server does not log; sealed, with a log after it. let layout = crate::store::account_layout(Path::new(""), &a); let (log, next) = (layout.log(g), layout.log(g + 1)); let mut before = files(&dir); let mark = |f: &Vec| f.iter().any(|e| e["t"] == "spaces"); let mut frames = store::scan(&before[&log]).unwrap().0; assert_eq!(frames.iter().filter(|f| mark(f)).count(), 1); frames.retain(|f| !mark(f)); let mut bytes: Vec = frames.iter().flat_map(|f| store::frame(f)).collect(); bytes.extend(seal()); before.insert(log.clone(), bytes); let old = [ json!({ "t": "notifyRegistered", "space": mine, "service": "did:plc:x#s", "expires": 1 }), json!({ "t": "spaceUnwrite", "space": mine, "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" }), ]; before.insert( next.clone(), old.iter() .flat_map(|e| store::frame(std::slice::from_ref(e))) .collect(), ); let work = scratch("unmarked-work"); put_files(&work, &before); let s = open(&work, &plc); let public = |mut state: State| { for (_, _, blobs) in state.5.0.values_mut() { blobs.clear(); } state }; let served = with_all(&s); assert!(public(served.0.clone()) == public(want.0.clone())); assert_eq!(served.1[&a], "[]"); let none = format!( "{:?}", ( &crate::space_authority::Authority::default(), &crate::space_notify::Kept::default() ) ); assert_eq!(served.2[&a], none); { let account = s.account(&a).unwrap(); let st = crate::read(&account.store); let held: Vec<&str> = st.blobs.held_list().map(|(cid, _, _)| cid).collect(); assert_eq!(held, [one.as_str()]); } assert!(!blob_dir(&work, &a).join(&two).exists()); assert_eq!( r(&s, "listSpaces", &ta, &[]).await, Ok(json!({ "spaces": [] })) ); assert_eq!( r(&s, "getSpace", &ta, &[("space", mine.as_str())]).await, Err("SpaceNotFound".into()) ); drop(s); // Marked once: started again, it is as it was. let marks = |work: &Path| { let files = files(work); files .iter() .filter(|(p, _)| p.starts_with(&layout.dir)) .flat_map(|(_, b)| store::scan(b).map(|f| f.0).unwrap_or_default()) .filter(|f| mark(f)) .count() }; assert_eq!(marks(&work), 1); let s = open(&work, &plc); assert!(with_all(&s) == served); // Spaces writes go on, and replay, and fold. assert_eq!(space(&s).await, mine); w( &s, "putRecord", &ta, rec(SPACE, "one", json!({ "img": refs[0] })), ) .await .unwrap(); idle(&s).await; let later = with_all(&s); assert!(later.1[&a] != "[]" && later != served); drop(s); assert_eq!(marks(&work), 1); let s = open(&work, &plc); assert!(with_all(&s) == later, "written after"); let wal = crate::read(&s.account(&a).unwrap().store).wal.clone(); wal.compact().unwrap(); drop(wal); drop(s); let s = open(&work, &plc); assert!(with_all(&s) == later, "folded"); drop(s); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Make the table at `path` one of the table format's version before /// (`pds_store::table::Version::Sha256`), holding the same entries. fn as_the_version_before(path: &Path) { use store::table::{BLOCK, BlockCache, Table, TableWriter, Version}; let cache = BlockCache::new(1 << 20); let t = Table::open(path, &cache).unwrap(); assert_eq!(t.version(), Version::Xxh3); let mut w = TableWriter::of_version(Vec::new(), t.len(), BLOCK, Version::Sha256); for e in t.iter(b"", None, &cache) { let (k, v) = e.unwrap(); w.add(&k, &v).unwrap(); } std::fs::write(path, w.finish().unwrap().0).unwrap(); } /// The version of the table format the table at `path` is of. fn table_version(path: &Path) -> store::table::Version { let cache = store::table::BlockCache::new(0); store::table::Table::open(path, &cache).unwrap().version() } /// Tables of the table format's version before, whose checks and bloom /// filters are SHA-256's, among tables of this one: two accounts' and /// the service's, a third account's as it is written now, each with a /// log after it. A start serves all of it as it was, loading every /// account or each as it is first asked for, and finds an account's /// events in its files, its store not loaded, as in a table of this /// version. The accounts take writes, and a compaction folds a table /// of the version before, with its log, into one of this version /// holding the same: an account's, with its space repo, and the /// service's, beside an account's still of the version before. With a /// byte changed, a table of either version is refused when it is /// opened, one of the version before that says it has more entries /// than it could hold too, or the block changed is not served. #[tokio::test] async fn tables_of_the_version_before_are_served_and_folded() { use crate::space_notify::tests::idle; use crate::space_repo::tests::{SPACE, w}; use store::table::Version; const PUT: &str = "com.atproto.repo.putRecord"; const DELETE: &str = "com.atproto.repo.deleteRecord"; let plc = plc().await; let dir = scratch("checks-src"); let s = open(&dir, &plc); let mut accounts = Vec::new(); for name in ["alice", "bob", "carol"] { accounts.push(create(&s, name).await); } let put = |did: &str, rkey: &str, n: i64| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) }); let del = |did: &str, rkey: &str| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey }); let space = |did: &str, rkey: &str, n: i64| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) }); let gone = |did: &str, rkey: &str| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey }); // In each account's table: records and their events; in Alice's, // a space repo too. for (did, token) in &accounts { for n in 0..24 { let body = put(did, &format!("r{n:02}"), n); call(&s, PUT, Some(token), body).await.unwrap(); } } let [a, b, c] = [0, 1, 2].map(|i| accounts[i].0.clone()); let ta = &accounts[0].1; for (rkey, n) in [("one", 1), ("two", 2)] { w(&s, "putRecord", ta, space(&a, rkey, n)).await.unwrap(); } idle(&s).await; for (did, _) in &accounts { let wal = crate::read(&s.account(did).unwrap().store).wal.clone(); wal.compact().unwrap(); } s.log.compact().unwrap(); // In the log after each: records written over and deleted. for (did, token) in &accounts { call(&s, PUT, Some(token), put(did, "r00", 100)) .await .unwrap(); call(&s, DELETE, Some(token), del(did, "r01")) .await .unwrap(); } w(&s, "putRecord", ta, space(&a, "one", 11)).await.unwrap(); w(&s, "deleteRecord", ta, gone(&a, "two")).await.unwrap(); session(&s, "alice").await; idle(&s).await; let want = with_all(&s); drop(s); let ckpt = |did: &str| crate::store::account_layout(Path::new(""), did).checkpoint(); let service = crate::store::service_layout(Path::new("")).checkpoint(); // Alice's, Carol's and the service's of the version before; Bob's as it is. let written = files(&dir); let old = [ckpt(&a), ckpt(&c), service.clone()]; for p in &old { as_the_version_before(&dir.join(p)); } let before = files(&dir); for p in &old { assert_eq!(table_version(&dir.join(p)), Version::Sha256); assert!(before[p] != written[p]); } assert_eq!(table_version(&dir.join(ckpt(&b))), Version::Xxh3); // Bob's events and Carol's, found in their files with no store // loaded: neither has anything waiting to be sent, so a start that // loads each account as it is first asked for loads neither. let work = scratch("checks-work"); let unloaded = || { let s = open_with(&work, &plc, |c| c.load_all = false); for did in [&b, &c] { assert!(!s.account(did).is_some_and(|a| a.store.is_loaded())); } assert!(s.account(&a).is_some()); s }; let on_disk = |files: &BTreeMap>| { put_files(&work, files); let s = unloaded(); [&b, &c].map(|did| crate::checkpoint::events_on_disk(&s.ctx, did).unwrap()) }; let events = on_disk(&written); assert!(events.iter().all(|(seqs, _, _)| seqs.len() > 24)); assert_eq!(on_disk(&before), events); // Served as it was, the tables left as they are. put_files(&work, &before); assert!(with_all(&open(&work, &plc)) == want, "served"); assert!(with_all(&unloaded()) == want, "loaded on first use"); let served = files(&work); assert!(old.iter().all(|p| served[p] == before[p])); // Written to, then folded: Alice's table and the service's are of // this version and hold the same; Carol's is as it was. let s = open(&work, &plc); for (did, token) in &accounts { call(&s, PUT, Some(token), put(did, "after", 7)) .await .unwrap(); call(&s, DELETE, Some(token), del(did, "r02")) .await .unwrap(); } w(&s, "putRecord", ta, space(&a, "three", 3)).await.unwrap(); idle(&s).await; let later = with_all(&s); assert!(later != want); let wal = crate::read(&s.account(&a).unwrap().store).wal.clone(); wal.compact().unwrap(); drop(wal); s.log.compact().unwrap(); assert!(with_all(&s) == later, "folded"); drop(s); let folded = files(&work); for p in [ckpt(&a), service] { assert_eq!(table_version(&work.join(p)), Version::Xxh3); } assert!(folded[&ckpt(&c)] == before[&ckpt(&c)]); assert!(with_all(&open(&work, &plc)) == later, "started again"); assert!( with_all(&unloaded()) == later, "started again, loaded on first use" ); // A byte changed, of Carol's table, of the version before, and of // Bob's, of this one: the last of the file, which is of its number // of entries in the version before and of its footer's check in // this one; the last of its bloom filter's check; and the first // of its first block. for (did, version, last) in [ ( &c, Version::Sha256, "more entries than its blocks could hold", ), (&b, Version::Xxh3, "its footer fails its check"), ] { assert_eq!(table_version(&work.join(ckpt(did))), version); let len = folded[&ckpt(did)].len(); for (at, why) in [ (len - 1, last), ( len - version.footer() - 1, "its bloom filter fails its check", ), (0, "fails its check"), ] { let mut f = folded.clone(); f.get_mut(&ckpt(did)).unwrap()[at] ^= 0x55; put_files(&work, &f); let refused = match try_open(&work, &plc) { Err(e) => e, Ok(s) => { let account = s.account(did).unwrap(); let st = crate::read(&account.store); crate::sync::CarStream::of(&st, None) .and_then(|car| car.bytes()) .unwrap_err() .to_string() } }; assert!(refused.contains(why), "{refused}"); } } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// The state of a server that loads each account when it is first /// asked for, once every account `want` holds is: none is loaded /// before. fn lazily(dir: &Path, plc: &str, want: &State) -> State { let s = open_with(dir, plc, |c| c.load_all = false); assert!(crate::read(&s.directory).by_did.is_empty()); for did in want.0.keys() { s.account(did); } state(&s) } /// What the service log keeps of the accounts, as one comparable value /// read with no account's store: each account's handle, mail and state, /// the sessions and app passwords, which account each reset token is /// of, and the invite codes. type Kept = ( BTreeMap, BTreeMap, BTreeMap, Vec, ); fn kept(s: &Service) -> Kept { let accounts = crate::read(&s.directory) .by_did .iter() .map(|(did, a)| { let (mail, state) = (email::lock(&a.mail).clone(), crate::read(&a.state).clone()); (did.clone(), (a.handle(), mail, state)) }) .collect(); ( accounts, s.sessions.snapshot(), s.resets.snapshot(), s.invites.snapshot(), ) } /// With the service checkpoint a table: each kind of write to the /// service log after it, of an account in the table and of one made /// after it, cut at every byte. A start that loads each account when /// it is first asked for finds what the service log keeps as before /// the write, or as after it once it is whole; and so does one that /// loads every account, of everything, at every few bytes. #[tokio::test] async fn torn_writes_after_a_table() { for sync in SYNCS { syncing(sync); torn_writes_after_a_table_once().await; } } async fn torn_writes_after_a_table_once() { let plc = plc().await; let dir = scratch("torn-table"); let s = open(&dir, &plc); let (_, ta) = create(&s, "alice").await; let (_, tb) = create(&s, "bob").await; s.log.compact().unwrap(); let layout = crate::store::service_layout(Path::new("")); let (log, ckpt) = (layout.log(s.log.generation()), layout.checkpoint()); let mut steps = Vec::new(); let snap = |s: &Service, steps: &mut Vec<(Step, Kept)>| { steps.push(( Step { files: files(&dir), journal: segments(&dir), state: state(s), }, kept(s), )) }; snap(&s, &mut steps); assert!(store::table::is_table(&steps[0].0.files[&ckpt])); let app = |token: &str| { let (s, token) = (s.clone(), token.to_string()); async move { call( &s, "com.atproto.server.createAppPassword", Some(&token), json!({ "name": "phone" }), ) .await .unwrap() } }; // Of accounts in the table. session(&s, "alice").await; snap(&s, &mut steps); app(&ta).await; snap(&s, &mut steps); call( &s, "com.atproto.server.updateEmail", Some(&tb), json!({ "email": "bob2@mail.com" }), ) .await .unwrap(); snap(&s, &mut steps); call( &s, "com.atproto.server.deactivateAccount", Some(&ta), json!({}), ) .await .unwrap(); snap(&s, &mut steps); // An account made after it, and a write of its. let (_, tc) = create(&s, "carol").await; snap(&s, &mut steps); app(&tc).await; snap(&s, &mut steps); call( &s, "com.atproto.server.activateAccount", Some(&ta), Json::Null, ) .await .unwrap(); snap(&s, &mut steps); drop(s); let dids: Vec = steps.last().unwrap().1.0.keys().cloned().collect(); let work = scratch("torn-table-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let whole = &w[1].0.files[&log]; let (from, to) = (w[0].0.files[&log].len(), whole.len()); assert!(to > from); kinds.extend( store::scan(&whole[from..]) .unwrap() .0 .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = w[1].0.files.clone(); files.insert(log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { &w[1] } else { &w[0] }; // Each account loaded when it is asked for, none before. let s = open_with(&work, &plc, |c| c.load_all = false); assert!(crate::read(&s.directory).by_did.is_empty()); for did in &dids { s.account(did); } assert!(kept(&s) == want.1, "cut at {cut} of {from}..{to}"); drop(s); if cut % 61 == 0 || cut + 1 >= to { assert!( state(&open(&work, &plc)) == want.0.state, "cut at {cut} of {from}..{to}, every account loaded" ); assert!( lazily(&work, &plc, &want.0.state) == want.0.state, "cut at {cut} of {from}..{to}, loaded on first use" ); } } } for kind in [ "account", "session", "appPassword", "updateEmail", "deactivated", "event", ] { assert!( kinds.iter().any(|k| k == kind), "{kind} never cut: {kinds:?}" ); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// The service log's table, cut at every byte where a compaction /// writes it beside the table before, the log sealed and the next one /// made: a start finds everything as before the compaction, and the /// part written is gone. #[tokio::test] async fn a_table_cut_short_where_it_is_written() { let plc = plc().await; let dir = scratch("torn-tmp"); let s = open(&dir, &plc); let (alice, _) = create(&s, "alice").await; s.log.compact().unwrap(); session(&s, "alice").await; let layout = crate::store::service_layout(Path::new("")); let g = s.log.generation(); let before = (files(&dir), state(&s)); s.log.compact().unwrap(); assert!(state(&s) == before.1); let table = files(&dir)[&layout.checkpoint()].clone(); assert!(table != before.0[&layout.checkpoint()]); drop(s); let mut killed = before.0.clone(); let mut sealed = killed[&layout.log(g)].clone(); sealed.extend(seal()); killed.insert(layout.log(g), sealed); killed.insert(layout.log(g + 1), Vec::new()); let work = scratch("torn-tmp-work"); for cut in 0..=table.len() { let mut files = killed.clone(); files.insert(layout.tmp(), table[..cut].to_vec()); put_files(&work, &files); if cut % 499 == 0 || cut == table.len() { assert!(state(&open(&work, &plc)) == before.1, "cut at {cut}"); assert!( lazily(&work, &plc, &before.1) == before.1, "cut at {cut}, loaded on first use" ); } else { let s = open_with(&work, &plc, |c| c.load_all = false); assert_eq!(crate::read(&s.directory).count(), 1, "cut at {cut}"); assert_eq!(s.sessions.count(&alice), 0, "cut at {cut}"); assert_eq!(s.account(&alice).unwrap().handle(), "alice.test"); assert_eq!(s.sessions.count(&alice), 2, "cut at {cut}"); } assert!(!work.join(layout.tmp()).exists(), "cut at {cut}"); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// A service checkpoint of the format before a table, a log of frames /// after its header, with the log after it: a start makes it a table /// of the same generation and serves all of it, the same wherever the /// entries were divided between the checkpoint and the log, wherever /// the conversion was cut short, and when it is started again. #[tokio::test] async fn a_checkpoint_of_the_format_before_is_made_a_table() { let plc = plc().await; let dir = scratch("convert"); let (_, accounts) = script(&dir, &plc).await; let s = open(&dir, &plc); let (ta, tb) = (&accounts[0].1, &accounts[1].1); call( &s, "com.atproto.server.createAppPassword", Some(ta), json!({ "name": "phone" }), ) .await .unwrap(); call( &s, "com.atproto.server.updateEmail", Some(tb), json!({ "email": "bob2@mail.com" }), ) .await .unwrap(); call( &s, "com.atproto.server.deactivateAccount", Some(tb), json!({}), ) .await .unwrap(); create(&s, "carol").await; let want = state(&s); drop(s); let last = files(&dir); let log = PathBuf::from(ACCOUNTS_LOG_FILE); let layout = crate::store::service_layout(Path::new("")); let (ckpt, tmp) = (layout.checkpoint(), layout.tmp()); assert!(!last.contains_key(&ckpt)); let frames = store::scan(&last[&log]).unwrap().0; // The files as the server before left them, having compacted the // first `at` frames: its checkpoint, a header naming its generation // and the last seq of its events, then what those frames hold. let before = |at: usize| { let seq = frames[..at] .iter() .flatten() .filter(|e| e["t"] == "event" || e["t"] == "deleted") .filter_map(|e| e["seq"].as_u64()) .max() .unwrap_or(0); let mut checkpoint = store::frame(&[json!({ "t": "checkpoint", "generation": 1, "lastSeq": seq })]); checkpoint.extend(frames[..at].iter().flat_map(|f| store::frame(f))); let mut files = last.clone(); files.remove(&log); files.insert(ckpt.clone(), checkpoint); files.insert( layout.log(1), frames[at..].iter().flat_map(|f| store::frame(f)).collect(), ); files }; let work = scratch("convert-work"); for at in 0..=frames.len() { put_files(&work, &before(at)); assert!(state(&open(&work, &plc)) == want, "divided at {at}"); let made = files(&work); assert!(store::table::is_table(&made[&ckpt]), "divided at {at}"); assert!(!made.contains_key(&tmp), "divided at {at}"); assert!( state(&open(&work, &plc)) == want, "divided at {at}, started again" ); assert!(files(&work)[&ckpt] == made[&ckpt], "made a table twice"); assert!( lazily(&work, &plc, &want) == want, "divided at {at}, loaded on first use" ); } // Cut short while the table is written beside it, at every byte; // and whole, not yet renamed over it. Of a checkpoint that holds // the first account, the log the rest. let at = 1 + frames .iter() .position(|f| f.iter().any(|e| e["t"] == "account")) .unwrap(); put_files(&work, &before(at)); drop(open_with(&work, &plc, |c| c.load_all = false)); let table = std::fs::read(work.join(&ckpt)).unwrap(); let accounts = want.0.len() as u64; for cut in 0..=table.len() { let mut f = before(at); f.insert(tmp.clone(), table[..cut].to_vec()); put_files(&work, &f); if cut % 499 == 0 || cut == table.len() { assert!(state(&open(&work, &plc)) == want, "cut at {cut}"); } else { let s = open_with(&work, &plc, |c| c.load_all = false); assert_eq!(crate::read(&s.directory).count(), accounts, "cut at {cut}"); } assert!( std::fs::read(work.join(&ckpt)).unwrap() == table && !work.join(&tmp).exists(), "cut at {cut}" ); } // One that is not whole is refused, not taken for less than it held. let whole = before(at); for cut in [0, 5, whole[&ckpt].len() - 1] { let mut f = whole.clone(); f.insert(ckpt.clone(), whole[&ckpt][..cut].to_vec()); put_files(&work, &f); assert!(try_open(&work, &plc).is_err(), "a checkpoint cut at {cut}"); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// Compacted, an account's log and the service log keep only the events /// in the window; a restart serves the same, and numbers new events after /// every one given out, those dropped too. #[tokio::test] async fn compaction_drops_events_out_of_the_window() { let plc = plc().await; let dir = scratch("window"); let cfg = || Config { backfill: std::time::Duration::from_millis(1500), ..config(&plc) }; let s: Arc = Arc::new(Service::open(&dir, cfg()).unwrap()); let (a, ta) = create(&s, "alice").await; let w = |k: &str| json!({ "repo": a, "collection": "a.b.c", "rkey": k, "record": record(1) }); call(&s, "com.atproto.repo.putRecord", Some(&ta), w("one")) .await .unwrap(); let old = *s.events.subscribe().borrow(); tokio::time::sleep(std::time::Duration::from_millis(3000)).await; call(&s, "com.atproto.repo.putRecord", Some(&ta), w("two")) .await .unwrap(); let last = *s.events.subscribe().borrow(); assert_eq!(last, old + 1); let wal = crate::read(&s.account(&a).unwrap().store).wal.clone(); // The write may have made one due by age already. wal.wait_compacted(); wal.compact().unwrap(); s.log.wait_compacted(); s.log.compact().unwrap(); // The window holds the last commit's event alone. let kept = s.events.get(&s, 0, last, 0); assert_eq!(kept.len(), 1); drop(s); let s = Service::open(&dir, cfg()).unwrap(); assert_eq!(*s.events.subscribe().borrow(), last); assert_eq!(s.events.get(&s, 0, last, 0), kept); drop(s); tokio::time::sleep(std::time::Duration::from_millis(3000)).await; // Out of the window, every event dropped: numbers go on all the same. let s: Arc = Arc::new(Service::open(&dir, cfg()).unwrap()); assert_eq!(s.events.window_start(last), None); let wal = crate::read(&s.account(&a).unwrap().store).wal.clone(); wal.wait_compacted(); wal.compact().unwrap(); drop(s); let s: Arc = Arc::new(Service::open(&dir, cfg()).unwrap()); assert_eq!(s.events.len(), 0); call(&s, "com.atproto.repo.putRecord", Some(&ta), w("three")) .await .unwrap(); assert_eq!(*s.events.subscribe().borrow(), last + 1); let _ = std::fs::remove_dir_all(&dir); } /// The layout before generations, `accounts.log` and each account's /// `repo.log`, with events written with seqs other than those served (as /// the server before numbered them again at each start): moved to /// generation 0 on the first start, its events with the seqs it served, /// the same if the move is cut short and made again. #[tokio::test] async fn the_layout_before_generations_is_moved() { let plc = plc().await; let dir = scratch("migrate"); let (_, accounts) = script(&dir, &plc).await; let s = open(&dir, &plc); let want = state(&s); drop(s); // Make it the layout before: ords, and seqs off by some. let legacy = |frames: Vec>| -> Vec { let mut out = Vec::new(); for mut f in frames { for e in f.iter_mut().filter(|e| e["t"] == "event") { let seq = e["seq"].as_u64().unwrap(); e["ord"] = json!(seq * 3); e["seq"] = json!(seq + 100); } out.extend(store::frame(&f)); } out }; let mut old = BTreeMap::new(); for (p, b) in files(&dir) { let name = p.file_name().unwrap().to_string_lossy().to_string(); let renamed = match name.as_str() { "accounts.0.log" => p.with_file_name("accounts.log"), "repo.0.log" => p.with_file_name("repo.log"), _ => p.clone(), }; let b = if renamed != p { legacy(store::scan(&b).unwrap().0) } else { b }; old.insert(renamed, b); } let work = scratch("migrate-work"); put_files(&work, &old); assert!(state(&open(&work, &plc)) == want, "moved"); assert!(!work.join("accounts.log").exists() && work.join("accounts.0.log").exists()); assert!(state(&open(&work, &plc)) == want, "started again"); // Cut short: one account's log moved, the service log not. let _ = std::fs::remove_dir_all(&work); put_files(&work, &old); let alice = Path::new("repos").join(&accounts[0].0); std::fs::rename( work.join(alice.join("repo.log")), work.join(alice.join("repo.0.log")), ) .unwrap(); assert!( state(&open(&work, &plc)) == want, "moved after a move cut short" ); let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } #[test] fn wal_group_commit_covers_all_waiters() { let dir = scratch("wal"); let path = dir.join("x.log"); let (wal, _) = Wal::open(&path, None).unwrap(); let a = wal .append(&[json!({ "t": "session", "token": "a" })]) .unwrap(); let b = wal .append(&[json!({ "t": "session", "token": "b" })]) .unwrap(); assert!(!wal.is_durable(a)); wal.wait(b).unwrap(); assert!(wal.is_durable(a) && wal.is_durable(b)); wal.wait(a).unwrap(); let (frames, _) = store::scan(&std::fs::read(&path).unwrap()).unwrap(); assert_eq!(frames.len(), 2); let _ = std::fs::remove_dir_all(&dir); } /// Cut `accounts.log` at every byte of each frame invite codes wrote: one /// code, several for several accounts, and an account made with one, /// whose use is in its frame. Recovery gives the state before or after. #[tokio::test] async fn torn_invite_writes() { for sync in SYNCS { syncing(sync); torn_invite_writes_once().await; } } async fn torn_invite_writes_once() { let plc = plc().await; let dir = scratch("torn-invites"); let s = open(&dir, &plc); let (alice, _) = create(&s, "alice").await; let admin = format!( "Basic {}", base64::Engine::encode( &base64::engine::general_purpose::STANDARD, format!("admin:{ADMIN_PASSWORD}") ) ); let as_admin = |nsid: &'static str, body: Json| { let (s, admin) = (s.clone(), admin.clone()); async move { call_raw( &s, nsid, sessions::Bearer::of(Some(&admin)), &[], &body.to_string(), ) .await .unwrap() } }; let mut steps = vec![Step { files: files(&dir), journal: segments(&dir), state: state(&s), }]; let snap = |s: &Service, steps: &mut Vec| { steps.push(Step { files: files(&dir), journal: segments(&dir), state: state(s), }) }; let code = as_admin( "com.atproto.server.createInviteCode", json!({ "useCount": 2, "forAccount": alice }), ) .await; snap(&s, &mut steps); as_admin( "com.atproto.server.createInviteCodes", json!({ "codeCount": 2, "useCount": 1, "forAccounts": [alice, "did:plc:other"] }), ) .await; snap(&s, &mut steps); let body = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw", "inviteCode": code["code"] }); call(&s, "com.atproto.server.createAccount", None, body) .await .unwrap(); snap(&s, &mut steps); drop(s); let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE); let work = scratch("torn-invites-work"); let mut kinds = Vec::new(); for w in steps.windows(2) { let whole = &w[1].files[&accounts_log]; let (from, to) = (w[0].files[&accounts_log].len(), whole.len()); kinds.extend( store::scan(&whole[from..]) .unwrap() .0 .iter() .flatten() .map(|e| e["t"].as_str().unwrap().to_string()), ); for cut in from..=to { let mut files = w[1].files.clone(); files.insert(accounts_log.clone(), whole[..cut].to_vec()); put_files(&work, &files); let want = if cut == to { &w[1].state } else { &w[0].state }; assert_eq!( &state(&open(&work, &plc)), want, "cut at {cut} of {from}..{to}" ); } } for kind in crate::invites::ENTRIES { assert!( kinds.iter().any(|k| k == kind), "{kind} never cut: {kinds:?}" ); } let _ = std::fs::remove_dir_all(&dir); let _ = std::fs::remove_dir_all(&work); } /// A `createAccount` whose account's log cannot be made answers `500` /// and writes nothing to `accounts.log`: the account does not exist, then /// or after a restart, and the next one is made. #[tokio::test] async fn an_account_whose_log_cannot_be_made() { let plc = plc().await; let dir = scratch("unmade"); let s = open(&dir, &plc); create(&s, "alice").await; let before = (std::fs::read(accounts_log(&dir)).unwrap(), state(&s)); // A file where the accounts' directories must go: no directory can be // made in it, for any user, root too. Alice's log stays open meanwhile. let (repos, aside) = (dir.join("repos"), dir.join("repos.aside")); std::fs::rename(&repos, &aside).unwrap(); std::fs::write(&repos, b"not a directory").unwrap(); let bob = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw" }); assert_eq!( call(&s, "com.atproto.server.createAccount", None, bob.clone()).await, Err("InternalServerError".into()) ); std::fs::remove_file(&repos).unwrap(); std::fs::rename(&aside, &repos).unwrap(); assert_eq!( (std::fs::read(accounts_log(&dir)).unwrap(), state(&s)), before ); drop(s); let s = open(&dir, &plc); assert_eq!(state(&s), before.1); call(&s, "com.atproto.server.createAccount", None, bob) .await .unwrap(); let _ = std::fs::remove_dir_all(&dir); } /// Every file the server makes is `0600` and every directory `0700`, /// whatever the umask; the data directory is made `0700` whatever it was. #[tokio::test] async fn files_are_the_owners_alone() { use std::os::unix::fs::PermissionsExt; let plc = plc().await; let dir = scratch("modes"); std::fs::create_dir_all(&dir).unwrap(); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; crate::blobs::test_upload( &s, Some(&token), Some("text/plain"), b"a blob".to_vec(), false, ) .await .unwrap(); call( &s, "com.atproto.server.createAppPassword", Some(&token), json!({ "name": "x" }), ) .await .unwrap(); assert!(dir.join(format!("repos/{did}/blobs")).is_dir()); let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; assert_eq!(mode(&dir), 0o700); let mut todo = vec![dir.clone()]; let mut seen = 0; while let Some(d) = todo.pop() { for e in std::fs::read_dir(&d).unwrap() { let p = e.unwrap().path(); seen += 1; if p.is_dir() { assert_eq!(mode(&p), 0o700, "{}", p.display()); todo.push(p); } else { assert_eq!(mode(&p), 0o600, "{}", p.display()); } } } assert!(seen >= 6, "{seen}"); let _ = std::fs::remove_dir_all(&dir); } /// Accounts are loaded on no more threads than the machine has cores, /// with no more of their logs open than a share. #[tokio::test] async fn recovery_is_bounded() { let plc = plc().await; let dir = scratch("recover-threads"); let s = open(&dir, &plc); let cores = std::thread::available_parallelism().unwrap().get(); let share = s.config.share; let n = (cores * 3).max(share + 8); for i in 0..n { let body = json!({ "handle": format!("user{i}.test"), "email": format!("u{i}@mail.com") }); call(&s, "com.atproto.server.createAccount", None, body) .await .unwrap(); } assert!(s.ctx.files.open_count() <= share); drop(s); let s = open(&dir, &plc); let d = crate::read(&s.directory); assert_eq!(d.by_did.len(), n); assert!(d.by_did.values().all(|a| a.store.is_loaded())); let threads: std::collections::HashSet<_> = d .by_did .values() .map(|a| crate::read(&a.store).thread) .collect(); assert!( threads.len() <= cores, "{} threads, {cores} cores", threads.len() ); assert!(s.ctx.files.open_count() <= share); drop(d); let _ = std::fs::remove_dir_all(&dir); } /// Run the test `name` again in a process of its own, with `PDS_TEST_CHILD` /// naming it, so that it may change the limits of its process: whether /// this is that process. No log is opened while the child is spawned, /// which holds a copy of every log's file open in this process, and its /// lock, until it execs (`pds_store::spawning`): the tests beside this /// one close their logs and open them again meanwhile. fn in_child(name: &str) -> bool { use std::process::{Command, Stdio}; if std::env::var("PDS_TEST_CHILD").as_deref() == Ok(name) { return true; } let child = { let _alone = pds_store::spawning(); Command::new(std::env::current_exe().unwrap()) .args([ &format!("log::tests::{name}"), "--exact", "--nocapture", "--test-threads=1", ]) .env("PDS_TEST_CHILD", name) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .unwrap() }; let out = child.wait_with_output().unwrap(); let text = format!( "{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr) ); assert!(out.status.success() && text.contains("1 passed"), "{text}"); false } /// Set the soft limit of `resource` to `soft`, or with none, to the hard limit. fn set_limit(resource: Resource, soft: Option) { let l = getrlimit(resource); let current = match soft { Some(s) => Some(l.maximum.map_or(s, |m| s.min(m))), None => l.maximum, }; setrlimit(resource, Rlimit { current, ..l }).unwrap(); } /// The file descriptors the process holds open. fn open_fds() -> usize { let dir = if Path::new("/proc/self/fd").exists() { "/proc/self/fd" } else { "/dev/fd" }; std::fs::read_dir(dir).unwrap().count() } /// With 256 file descriptors, and the share of them the server takes /// under that limit, 300 accounts are made, and the server started /// again serves every one of them, and takes writes from the first and /// the last: what it holds open does not grow with them, and is no more /// than a share of logs, a share of tables and 72 files besides. #[test] fn many_accounts_few_files() { if !in_child("many_accounts_few_files") { return; } set_limit(Resource::Nofile, Some(256)); let share = store::files::share(store::files::limit()); assert_eq!(share, 64); let most = 2 * share + 72; let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .enable_all() .build() .unwrap(); rt.block_on(async { let plc = plc().await; let dir = scratch("many-accounts"); let s = open_with(&dir, &plc, |c| c.share = share); let mut made = Vec::new(); for i in 0..300 { let body = json!({ "handle": format!("user{i}.test"), "email": format!("u{i}@mail.com"), "password": "" }); made.push(call(&s, "com.atproto.server.createAccount", None, body).await.unwrap()); } assert!(s.ctx.files.open_count() <= share); assert!(open_fds() < most, "{} open", open_fds()); drop(s); let s = open_with(&dir, &plc, |c| c.share = share); assert!(open_fds() < most, "{} open", open_fds()); for m in &made { let p = [("did".to_string(), m["did"].as_str().unwrap().to_string())]; call_with(&s, "com.atproto.sync.getLatestCommit", None, &p, Json::Null).await.unwrap(); } for m in [&made[0], &made[299]] { let body = json!({ "repo": m["did"], "collection": "a.b.c", "record": { "n": 1 } }); call(&s, "com.atproto.repo.createRecord", m["accessJwt"].as_str(), body).await.unwrap(); } assert!(s.ctx.files.open_count() <= share); let _ = std::fs::remove_dir_all(&dir); }); } /// A write whose append fails part way, as on a full disk, is `500` and /// cut back from its log, leaving nothing of itself in the repository or /// the firehose; the next write goes on from there, and a restart finds /// the same. #[test] fn a_failed_append_is_cut_back() { if !in_child("a_failed_append_is_cut_back") { return; } let rt = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .enable_all() .build() .unwrap(); rt.block_on(async { // A handler for the signal, so that a write past the size limit fails instead. let _xfsz = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::from_raw(Signal::XFSZ.as_raw())).unwrap(); let plc = plc().await; let dir = scratch("cut-back"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let (bob, bob_token) = create(&s, "bob").await; let put = |did: &str, rkey: &str, n: usize| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": { "t": "x".repeat(n) } }); call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "a", 10)).await.unwrap(); let log = dir.join(repo_log(Path::new(""), &did)); let len = log_bytes(&log).len() as u64; let before = state(&s); let largest = files_whole(&dir).values().map(|b| b.len() as u64).max().unwrap(); set_limit(Resource::Fsize, Some(largest.max(len) + 8000)); assert_eq!(call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "b", 40_000)).await, Err("InternalServerError".into())); assert_eq!(std::fs::metadata(&log).unwrap().len(), len); assert_eq!(state(&s), before); // Another account's write goes on, and so does this one's. call(&s, "com.atproto.repo.putRecord", Some(&bob_token), put(&bob, "c", 10)).await.unwrap(); call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "d", 10)).await.unwrap(); set_limit(Resource::Fsize, None); let after = state(&s); assert_eq!(after.2.len(), before.2.len() + 2); drop(s); assert_eq!(state(&open(&dir, &plc)), after); let _ = std::fs::remove_dir_all(&dir); }); } /// A commit's frame of the first kind: JSON, its bytes as hex, as /// `v0.1.0` wrote every commit's, and as a log may hold them. mod json_frames { use super::super::Hex; use pds_repo::Change; use serde_json::Value as Json; struct CommitEntry<'a>(&'a Change); impl serde::Serialize for CommitEntry<'_> { fn serialize(&self, s: S) -> Result { use serde::ser::SerializeMap; let c = self.0; let mut m = s.serialize_map(Some(10))?; m.serialize_entry("t", "commit")?; m.serialize_entry("rev", &c.rev)?; m.serialize_entry("head", &Hex(&c.head_bytes))?; m.serialize_entry( "puts", &c.puts .iter() .zip(&c.put_cids) .map(|((k, v), cid)| (k, v, Hex(cid))) .collect::>(), )?; m.serialize_entry("dels", &c.dels)?; m.serialize_entry( "tree", &c.tree .iter() .map(|(cid, b)| (Hex(cid), Hex(b))) .collect::>(), )?; m.serialize_entry("gone", &c.gone.iter().map(|g| Hex(g)).collect::>())?; m.serialize_entry( "refs", &c.refs .iter() .map(|(cid, n, rev)| (Hex(cid), n, rev)) .collect::>(), )?; m.serialize_entry("counts", &[c.counts.0, c.counts.1])?; m.end() } } struct EventEntry<'a>(u64, u64, &'a [u8]); impl serde::Serialize for EventEntry<'_> { fn serialize(&self, s: S) -> Result { use serde::ser::SerializeMap; let mut m = s.serialize_map(Some(4))?; m.serialize_entry("t", "event")?; m.serialize_entry("seq", &self.0)?; m.serialize_entry("time", &self.1)?; m.serialize_entry("frame", &Hex(self.2))?; m.end() } } pub(super) fn commit_frame( commit: Option<&Change>, entries: &[Json], seq: u64, frames: &[Vec], times: &[u64], ) -> Vec { let capacity = frames.iter().map(|f| f.len() * 3).sum::() + 4096; pds_store::frame_of(capacity, |out| { use serde::ser::{SerializeSeq, Serializer}; let mut s = serde_json::Serializer::new(out); let mut seq_out = s.serialize_seq(None).unwrap(); if let Some(c) = commit { seq_out.serialize_element(&CommitEntry(c)).unwrap(); } for e in entries { seq_out.serialize_element(e).unwrap(); } for (i, f) in frames.iter().enumerate() { seq_out .serialize_element(&EventEntry(seq + i as u64, times[i], f)) .unwrap(); } seq_out.end().unwrap(); }) } } /// A commit's frame with holes, filled, is byte for byte its frame of /// the first kind: with and without its commit, events and other /// entries, bytes that end with zero and none. #[test] fn a_commit_frame_filled_is_its_json_frame() { let bytes = |n: usize, seed: u8, last: u8| -> Vec { let mut b: Vec = (0..n) .map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed)) .collect(); if let Some(x) = b.last_mut() { *x = last; } b }; let cid = |b: &[u8]| pds_repo::data::cid_of(0x71, b); let record = json!({ "$type": "app.bsky.feed.post", "text": "a \"post\"\n", "langs": ["en"] }); let (n1, n2) = (bytes(1000, 3, 7), bytes(1100, 5, 0)); let c = Change { rev: "3lzzzzzzzzz2a".into(), head_bytes: bytes(190, 1, 9), puts: vec![("app.bsky.feed.post/3lzz".into(), record.clone())], put_cids: vec![cid(&record.to_string().into_bytes())], dels: vec!["app.bsky.feed.post/3lyy".into()], tree: vec![(cid(&n1), n1), (cid(&n2), n2)], gone: vec![cid(b"gone")], refs: vec![(cid(b"ref"), 1, "3lzzzzzzzzz2a".into())], counts: (12, 30), ..Default::default() }; let empty = Change::default(); let handled = [json!({ "t": "handled", "cid": "bafy", "landed": true })]; let events = [bytes(3000, 11, 4), bytes(2900, 13, 0)]; for (commit, entries, frames) in [ (Some(&c), &[][..], &events[..1]), (Some(&c), &handled[..], &events[..]), (Some(&c), &[][..], &[][..]), (Some(&empty), &[][..], &events[1..]), (None, &handled[..], &events[1..]), (None, &handled[..], &[][..]), ] { let times: Vec = (0..frames.len() as u64) .map(|i| 1_700_000_000_000 + i) .collect(); let with = commit_frame(commit, entries, 9, frames, ×); let json = json_frames::commit_frame(commit, entries, 9, frames, ×); assert_eq!(with[store::HEADER], store::HOLES); assert_eq!( store::fill(&with[store::HEADER..]).unwrap().as_ref(), &json[store::HEADER..] ); assert_eq!(store::parse_frame(&with), store::parse_frame(&json)); assert_ne!(*with.last().unwrap(), 0); } } /// The frame at the end of the log of `did`'s current generation: its /// bytes. fn last_frame(s: &Service, dir: &Path, did: &str) -> Vec { let g = store_of(s, did, |st| st.wal.generation()); let path = crate::store::account_layout(dir, did).log(g); let mut last = None; store::scan_file(&path, |at, len, _| { last = Some((at, len)); Ok(()) }) .unwrap() .unwrap(); let (at, len) = last.unwrap(); std::fs::read(&path).unwrap()[at as usize..][..len as usize].to_vec() } /// The bytes of a commit's frame of a post, in a tree of fifty: with /// holes, and as JSON with its bytes in hex. #[tokio::test] async fn the_bytes_of_a_commit_frame() { let plc = plc().await; let dir = scratch("commit-bytes"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; let words = "the weather here is lovely today and I went for a long walk by the river "; for i in 0..50 { let post = json!({ "$type": "app.bsky.feed.post", "text": format!("{i}: {}", words.repeat(3)), "langs": ["en"], "createdAt": "2026-11-12T10:00:00.000Z", }); call( &s, "com.atproto.repo.createRecord", Some(&token), json!({ "repo": did, "collection": "app.bsky.feed.post", "record": post }), ) .await .unwrap(); } let with = last_frame(&s, &dir, &did); let json = store::HEADER + store::fill(&with[store::HEADER..]).unwrap().len(); println!( "a commit's frame: {json} bytes as JSON, bytes as hex; {} with holes", with.len() ); assert!(with.len() * 10 < json * 6, "{} of {json}", with.len()); let _ = std::fs::remove_dir_all(&dir); } /// A log with its first `upto` frames with holes made JSON, their /// bytes in hex, as `v0.1.0` wrote every commit's: and how many it /// made so. fn as_json(log: &[u8], upto: usize) -> (Vec, usize) { let (mut out, mut made, mut at) = (Vec::new(), 0, 0); while at < log.len() { let len = u32::from_le_bytes(log[at..at + 4].try_into().unwrap()) as usize; let frame = &log[at..at + store::HEADER + len]; if frame[store::HEADER] == store::HOLES && made < upto { let text = store::fill(&frame[store::HEADER..]).unwrap(); out.extend(store::frame_of(text.len(), |o| o.extend_from_slice(&text))); made += 1; } else { out.extend_from_slice(frame); } at += store::HEADER + len; } (out, made) } /// An account's log of commits `v0.1.0` wrote, JSON, and then commits /// this version wrote, with holes, in any proportion: it loads, serves /// every block, record and event as the server that wrote it did, from /// frames of both kinds, and compacts into the same table as the log /// written all one way or all the other. #[tokio::test] async fn a_log_of_both_kinds() { let plc = plc().await; let dir = scratch("kinds"); let s = open(&dir, &plc); let (did, token) = create(&s, "alice").await; for i in 0..30 { let (nsid, body) = if i % 7 == 6 { let rkey = format!("k{}", (i - 3) % 12); ( "com.atproto.repo.deleteRecord", json!({ "repo": did, "collection": "a.b.c", "rkey": rkey }), ) } else { let rkey = format!("k{}", i % 12); ( "com.atproto.repo.putRecord", json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(i) }), ) }; call(&s, nsid, Some(&token), body).await.unwrap(); } let want = state(&s); drop(s); let written = files(&dir); let layout = crate::store::account_layout(Path::new(""), &did); let logs: Vec = written .keys() .filter(|p| p.starts_with(&layout.dir) && p.extension().is_some_and(|e| e == "log")) .cloned() .collect(); let holed: usize = logs .iter() .map(|l| as_json(&written[l], usize::MAX).1) .sum(); assert!(holed > 25, "{holed} frames with holes"); let mut tables = Vec::new(); // Of each log's frames with holes, none, all, or the first half // made JSON. for (name, of, by) in [("holes", 0, 1), ("json", 1, 1), ("both", 1, 2)] { let mut files = written.clone(); let mut made = 0; for l in &logs { let frames = as_json(&written[l], usize::MAX).1; let (bytes, n) = as_json(&written[l], frames * of / by); files.insert(l.clone(), bytes); made += n; } match (of, by) { (0, _) => assert_eq!(made, 0), (1, 1) => assert_eq!(made, holed), _ => assert!(made > 0 && made < holed, "{made} of {holed}"), } let work = scratch(&format!("kinds-{name}")); put_files(&work, &files); let s = open(&work, &plc); assert_eq!(state(&s), want, "{name}: served"); let wal = store_of(&s, &did, |st| st.wal.clone()); wal.wait_compacted(); wal.compact().unwrap(); drop(wal); assert_eq!(state(&s), want, "{name}: compacted"); tables.push(std::fs::read(work.join(layout.checkpoint())).unwrap()); drop(s); let _ = std::fs::remove_dir_all(&work); } assert!(tables.iter().all(|t| *t == tables[0]), "one table"); let _ = std::fs::remove_dir_all(&dir); } }