~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
6141 lines · 241.8 KB Raw History
//! What the server keeps, and how it starts again from it, as
//! `spec/DURABILITY.md` and `spec/STORE.md` say. In `PDS_DATA_DIR`, which is
//! its owner's alone:
//!
//! - the service log, `accounts.<g>.log` from its checkpoint
//!   `accounts.checkpoint` on (`compact`): each account (DID, handle, email,
//!   password hash, signing key, KEM key, DID document), each change to the
//!   sessions it gave out and its app passwords (`sessions::ENTRIES`), each
//!   change to its email and password and the tokens mailed for them
//!   (`email::ENTRIES`), each change to what OAuth keeps (`oauth::ENTRIES`),
//!   each single-use value OAuth spent (`replay::ENTRY`), the invite codes
//!   and their uses (`invites::ENTRIES`), the single-use values the spaces
//!   took (`single_use`), the spaces' credentials revoked here
//!   (`space_credentials`), each account's state and its events
//!   (`lifecycle::ENTRIES`), each account's preferences as last put
//!   (`preferences::ENTRY`), and with blobs in a bucket, the objects of
//!   deleted accounts not yet deleted (`blobstore::ENTRIES`);
//! - `jwt.key`, the secret its tokens are signed with, and `oauth.key`, the
//!   one OAuth seals its tokens, nonces and cookies with;
//! - each account's own log, `repos/<did>/repo.<g>.log` from its checkpoint
//!   `repo.checkpoint` on (`checkpoint`): the mark that its spaces state is
//!   this version's from there (`checkpoint::unmarked`), first; its
//!   commits, each in one frame
//!   with its firehose events, the blobs it stored and stopped holding, its
//!   handle changes: each recorded before its operation goes to the PLC
//!   directory, then settled, in one frame with its `#identity` event if it
//!   landed; its space repos' commits and imports, each one frame
//!   (`space_repo`); each change to the spaces it is the authority of, one
//!   frame each (`space_authority::ENTRIES`), with the commits of the
//!   records it publishes and the repo it drops (`space_repo::DROP`); and its
//!   write notifications waiting to be sent again, and its spaces' syncers'
//!   registrations and the notices kept for them (`space_notify::ENTRIES`);
//! - `repos/<did>/blobs/<cid>`, the bytes of each blob it holds (`blobs`),
//!   or with blobs in a bucket, none, and the mark `blobstore` (`blobstore`);
//! - the firehose's index, `events.<g>.log` from its checkpoint
//!   `events.checkpoint` on: of each event, as it is numbered, its seq,
//!   account and time, so that a start learns the events made before it
//!   without reading the accounts' logs (`events::index`). Its entry of an
//!   event is appended before the frame that holds the event, and both are
//!   on disk before anything is made of it;
//! - `clean-shutdown`, the mark a shutdown leaves and the next start takes
//!   away (`ops`): it holds the word that every log was closed and synced,
//!   for which the next start syncs no log as it opens it (`store`, "What
//!   a start syncs").
//!
//! Each log is compacted into its checkpoint, generation by generation
//! (`store`, `pds_store`). A start reads the service checkpoint's summary
//! and its entries that are not one account's, and replays the service log
//! after it, noting where each account's entries are without applying them
//! (`directory`); each account, and then its own checkpoint and log, is
//! read on its first use. A data directory with the layout before
//! generations is moved to generation 0 first (`store::migrate`), and a
//! service checkpoint of the format before a table is made one
//! (`compact::convert`).
//!
//! An account's log, with its repository's first commit, is on disk before
//! the account is in the service log: if the server stops between the two,
//! the log is of no account, and goes once the server serves again
//! (`directory::Accounts::sweep`). An account in the service log whose log
//! has no commit, as one older servers left, gets its first commit when its
//! store is loaded. Events are numbered in one sequence across all the
//! accounts' logs (`events::Firehose`), each keeping its seq, the next
//! after the greatest any log or checkpoint names. An account's handle is
//! the one its last landed change gave it, or else the one the service log
//! has.
//!
//! At most a share of accounts' logs are open at once (`Config::share`: a
//! quarter of the file descriptors the process may have, as
//! `pds_store::files::share` counts it), and at most a share of their
//! tables' files, and they are recovered on at most as many threads as the
//! machine has cores.

use crate::blobs::blob_dir;
use crate::cache::{Ctx, Slot};
use crate::compact::ServiceTable;
use crate::directory::{Accounts, Directory};
use crate::email::{self, Mail};
use crate::events::{self, Firehose, Landing, Reserved};
use crate::sessions::Sessions;
use crate::{Account, Config, Service, Store};
use axum::body::Bytes;
use k256::ecdsa::SigningKey;
use pds_repo::*;
use pds_store::{Files, Holed, Wal};
use rustix::io::Errno;
use serde_json::{Value as Json, json};
use std::collections::HashSet;
use std::io;
use std::path::Path as FsPath;
#[cfg(test)]
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, PoisonError, RwLock};

/// What error messages call the service log.
pub(crate) const ACCOUNTS_LOG: &str = "the service log";

/// The service log of generation 0, by its name.
#[cfg(test)]
pub(crate) const ACCOUNTS_LOG_FILE: &str = "accounts.0.log";

/// The service log of generation 0, which a server that has not compacted it holds.
#[cfg(test)]
pub(crate) fn accounts_log(dir: &FsPath) -> PathBuf {
    crate::store::service_layout(dir).log(0)
}

/// An account's log of generation 0, which an account whose log has not
/// been compacted holds.
#[cfg(test)]
pub(crate) fn repo_log(dir: &FsPath, did: &str) -> PathBuf {
    crate::store::account_layout(dir, did).log(0)
}

/// A copy of the data directory `dir` as it is now, beside it: what a
/// server started again would find, for one to start on while the server
/// on `dir`, which holds its logs, goes on.
#[cfg(test)]
pub(crate) fn copied(dir: &FsPath) -> PathBuf {
    let to = dir.with_extension("copy");
    let _ = std::fs::remove_dir_all(&to);
    let mut todo = vec![dir.to_path_buf()];
    while let Some(d) = todo.pop() {
        std::fs::create_dir_all(to.join(d.strip_prefix(dir).unwrap())).unwrap();
        for e in std::fs::read_dir(&d).unwrap() {
            let p = e.unwrap().path();
            if p.is_dir() {
                todo.push(p);
            } else {
                std::fs::copy(&p, to.join(p.strip_prefix(dir).unwrap())).unwrap();
            }
        }
    }
    to
}

/// What the log at `path` holds: its file's bytes, without the room after
/// its frames, which a server running or killed leaves there
/// (`pds_store`).
#[cfg(test)]
pub(crate) fn log_bytes(path: &FsPath) -> Vec<u8> {
    without_room(std::fs::read(path).unwrap())
}

/// A log's bytes without the room after its frames: the zeros they end with.
#[cfg(test)]
fn without_room(mut log: Vec<u8>) -> Vec<u8> {
    log.truncate(log.iter().rposition(|&b| b != 0).map_or(0, |last| last + 1));
    log
}

/// The logs opened on each data directory, in tests alone, where one
/// process starts a server on a directory again and again while tasks of
/// the server before still hold it: a start closes that server's logs
/// first, as the restart of a process ends its writers (`Wal::close`).
#[cfg(test)]
static OPENED: Mutex<Vec<(PathBuf, std::sync::Weak<Wal>)>> = Mutex::new(Vec::new());

/// A log as the server holds it, opened on the data directory `dir`.
pub(crate) fn held(dir: &FsPath, wal: Wal) -> Arc<Wal> {
    let wal = Arc::new(wal);
    #[cfg(test)]
    OPENED
        .lock()
        .unwrap_or_else(PoisonError::into_inner)
        .push((dir.to_path_buf(), Arc::downgrade(&wal)));
    let _ = dir;
    wal
}

/// The journals opened on each data directory, in tests alone, for the
/// same reason: a start lets the one before go, as a kill does.
#[cfg(test)]
static JOURNALS: Mutex<Vec<(PathBuf, std::sync::Weak<pds_store::Journal>)>> =
    Mutex::new(Vec::new());

/// Close every log a server before opened on `dir` and still holds: its
/// files are let go whether or not what it had written could be synced.
/// Then let its journal go, its segments left for this start to redo.
#[cfg(test)]
fn stop_writers(dir: &FsPath) {
    OPENED
        .lock()
        .unwrap_or_else(PoisonError::into_inner)
        .retain(|(of, wal)| match wal.upgrade() {
            Some(wal) if of == dir => {
                let _ = wal.close();
                false
            }
            held => held.is_some(),
        });
    JOURNALS
        .lock()
        .unwrap_or_else(PoisonError::into_inner)
        .retain(|(of, j)| match j.upgrade() {
            Some(j) if of == dir => {
                j.let_go();
                false
            }
            held => held.is_some(),
        });
}

/// An account as `accounts.log` has it.
pub(crate) struct AccountEntry {
    pub did: String,
    pub handle: String,
    /// None for an account without a password.
    pub email: Option<String>,
    pub password: Option<String>,
    /// Made on first use: an account is loaded for much that signs nothing.
    pub key: crate::cache::Key,
    /// Its KEM key (`spec/SPACES.md`); None for an account made before
    /// accounts had one, which is given none.
    pub kem: Option<pds_space::KemKey>,
    /// Its DID document, as JSON text: held so from the start, a few
    /// allocations rather than one for each of its fields.
    pub did_doc: Box<str>,
}

impl AccountEntry {
    pub fn entry(&self) -> Json {
        json!({
            "t": "account", "did": self.did, "handle": self.handle, "email": self.email,
            "password": self.password, "key": hex::encode(self.key.to_bytes()),
            "kemKey": self.kem.as_ref().map(|k| hex::encode(k.to_bytes())), "didDoc": serde_json::from_str::<Json>(&self.did_doc).unwrap_or(Json::Null),
        })
    }

    pub(crate) fn parse(e: &Json) -> Result<AccountEntry, String> {
        let text = |k: &str| {
            e[k].as_str()
                .map(str::to_string)
                .ok_or_else(|| format!("an account with no {k}: {e}"))
        };
        let key = hex::decode(text("key")?)
            .ok()
            .and_then(|b| crate::cache::Key::from_slice(&b))
            .ok_or_else(|| format!("an account with a bad key: {e}"))?;
        let optional = |k: &str| match &e[k] {
            Json::Null => Ok(None),
            Json::String(p) => Ok(Some(p.clone())),
            _ => Err(format!("an account with a bad {k}: {e}")),
        };
        let (password, email) = (optional("password")?, optional("email")?);
        let kem = match optional("kemKey")? {
            None => None,
            Some(k) => Some(
                hex::decode(k)
                    .ok()
                    .and_then(|b| pds_space::KemKey::from_bytes(&b).ok())
                    .ok_or_else(|| format!("an account with a bad kemKey: {e}"))?,
            ),
        };
        if password.is_some() != email.is_some() {
            return Err(format!(
                "an account with a password and no email, or an email and no password: {e}"
            ));
        }
        if !e["didDoc"].is_object() {
            return Err(format!("an account with no didDoc: {e}"));
        }
        Ok(AccountEntry {
            did: text("did")?,
            handle: text("handle")?,
            email,
            password,
            key,
            kem,
            did_doc: e["didDoc"].to_string().into(),
        })
    }

    /// The account, its store held if given, else loaded on first use; its
    /// DID document as fetched at `fetched`, or none if it may be out of
    /// date; its mail, if not as made, and its state.
    pub fn account(
        self,
        store: Option<Store>,
        fetched: Option<u64>,
        ctx: &Arc<Ctx>,
        mail: Option<Mail>,
        state: crate::lifecycle::State,
    ) -> Arc<Account> {
        let mail = mail.unwrap_or_else(|| Mail::new(self.email, self.password));
        let a = Arc::new_cyclic(|me| Account {
            did: self.did,
            handle: RwLock::new(self.handle),
            mail: Mutex::new(mail),
            kem: self.kem,
            did_doc: RwLock::new((self.did_doc, fetched)),
            fetching: tokio::sync::Mutex::new(()),
            store: Slot::new(me.clone(), self.key, ctx.clone(), store),
            state: RwLock::new(state),
        });
        a.store.counted();
        a
    }
}

/// A commit's entry: its rev and block, each record it put, with its CID,
/// and each path it deleted; and what it changed of the tree: the nodes it
/// made, with their bytes, those it took out, each record whose count of
/// paths changed, and the counts after (`checkpoint`). Written as it is
/// serialized, borrowing the commit: no copy of its records is made. Its
/// block and its nodes' bytes are holes of its frame, written as they are
/// (`pds_store::frame_with_holes`); CIDs are hex.
struct CommitEntry<'h, 'a>(&'a Change, &'h Holed<'a>);

impl serde::Serialize for CommitEntry<'_, '_> {
    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        use serde::ser::SerializeMap;
        let (c, w) = (self.0, self.1);
        let mut m = s.serialize_map(Some(10))?;
        m.serialize_entry("t", "commit")?;
        m.serialize_entry("rev", &c.rev)?;
        m.serialize_entry("head", &Hole(w, &c.head_bytes))?;
        m.serialize_entry(
            "puts",
            &c.puts
                .iter()
                .zip(&c.put_cids)
                .map(|((k, v), cid)| (k, v, Hex(cid)))
                .collect::<Vec<_>>(),
        )?;
        m.serialize_entry("dels", &c.dels)?;
        m.serialize_entry(
            "tree",
            &c.tree
                .iter()
                .map(|(cid, b)| (Hex(cid), Hole(w, b)))
                .collect::<Vec<_>>(),
        )?;
        m.serialize_entry("gone", &c.gone.iter().map(|g| Hex(g)).collect::<Vec<_>>())?;
        m.serialize_entry(
            "refs",
            &c.refs
                .iter()
                .map(|(cid, n, rev)| (Hex(cid), n, rev))
                .collect::<Vec<_>>(),
        )?;
        m.serialize_entry("counts", &[c.counts.0, c.counts.1])?;
        m.end()
    }
}

/// An event's entry: its seq, when it was made, and its frame, a hole of
/// the log's frame.
struct EventEntry<'h, 'a>(u64, u64, &'a [u8], &'h Holed<'a>);

impl serde::Serialize for EventEntry<'_, '_> {
    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        use serde::ser::SerializeMap;
        let mut m = s.serialize_map(Some(4))?;
        m.serialize_entry("t", "event")?;
        m.serialize_entry("seq", &self.0)?;
        m.serialize_entry("time", &self.1)?;
        m.serialize_entry("frame", &Hole(self.3, self.2))?;
        m.end()
    }
}

/// Bytes in a hole of the frame `Holed` writes: an empty string in its
/// text, and the bytes, as they are, after it.
struct Hole<'h, 'a>(&'h Holed<'a>, &'a [u8]);

impl serde::Serialize for Hole<'_, '_> {
    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        let written = s.serialize_str("")?;
        self.0.hole(self.1);
        Ok(written)
    }
}

/// Bytes as hex, written as they are serialized, never held as a string.
struct Hex<'a>(&'a [u8]);

impl serde::Serialize for Hex<'_> {
    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        s.collect_str(self)
    }
}

impl std::fmt::Display for Hex<'_> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let mut buf = [0u8; 2048];
        for chunk in self.0.chunks(buf.len() / 2) {
            let out = &mut buf[..chunk.len() * 2];
            hex::encode_to_slice(chunk, out).expect("twice the length");
            f.write_str(std::str::from_utf8(out).expect("hex"))?;
        }
        Ok(())
    }
}

/// A commit's frame, JSON with holes: its entry, if any, then `entries`,
/// then the events numbered from `seq`, each with when it was made.
fn commit_frame(
    commit: Option<&Change>,
    entries: &[Json],
    seq: u64,
    frames: &[Vec<u8>],
    times: &[u64],
) -> Vec<u8> {
    // The records, as JSON, are about the size of the events' blocks, and
    // so are the nodes and the block, besides the events themselves.
    let capacity = frames.iter().map(|f| f.len() * 3).sum::<usize>() + 4096;
    pds_store::frame_with_holes(capacity, |w| {
        use serde::ser::{SerializeSeq, Serializer};
        let mut s = serde_json::Serializer::new(w);
        let mut seq_out = s.serialize_seq(None).expect("a frame serializes");
        if let Some(c) = commit {
            seq_out
                .serialize_element(&CommitEntry(c, w))
                .expect("a commit serializes");
        }
        for e in entries {
            seq_out.serialize_element(e).expect("an entry serializes");
        }
        for (i, f) in frames.iter().enumerate() {
            seq_out
                .serialize_element(&EventEntry(seq + i as u64, times[i], f, w))
                .expect("an event serializes");
        }
        seq_out.end().expect("a frame serializes");
    })
}

/// A handle change, recorded before its operation, by its CID, goes to the directory.
pub(crate) fn handle_entry(handle: &str, cid: &str) -> Json {
    json!({ "t": "handle", "handle": handle, "cid": cid })
}

/// Append `commit`'s entry, if given, and `entries`, with the events
/// `numbered`, if any, numbered and made as their reservation says, the
/// commit's change kept in the store's memtable: the events to land once
/// the log, and their index entry, are on disk, or, if the append failed,
/// none of them, their numbers left unused. With blobs in a bucket, a
/// commit that stops the account holding a blob has it marked for a start
/// to load first, so that the blob's object is deleted (`blobstore`).
fn append_events(
    events: &Firehose,
    st: &mut Store,
    entries: &[Json],
    numbered: Option<(&Reserved, Vec<Vec<u8>>)>,
    commit: Option<&Change>,
) -> io::Result<Option<Landing>> {
    let (r, frames) = match numbered {
        Some((r, frames)) => (Some(r), frames),
        None => (None, Vec::new()),
    };
    let (seq, at) = r.map_or((0, 0), |r| (r.seq, r.at));
    let fail = |e: io::Error| {
        if !frames.is_empty() {
            events.fail(seq, frames.len());
        }
        e
    };
    if st.ctx.bucket.is_some()
        && st.blobs.dooms()
        && let Err(e) = st.ctx.eager.mark(&st.repo.did)
    {
        return Err(fail(e));
    }
    let times = vec![at; frames.len()];
    let bytes = commit_frame(commit, entries, seq, &frames, &times);
    let pos = match st.wal.append_frame_at(&bytes) {
        Ok((_, pos)) => pos,
        Err(e) => return Err(fail(e)),
    };
    st.disk
        .indexed(&st.ctx.blocks, pos, &bytes[pds_store::HEADER..]);
    drop(bytes);
    if let Some(c) = commit {
        st.disk.mem.commit(pos, c);
    }
    for (i, t) in times.iter().enumerate() {
        st.disk.mem.event(seq + i as u64, *t, pos);
    }
    let Some(r) = r else {
        return Ok(None);
    };
    st.last_seq = st.last_seq.max(seq + frames.len() as u64 - 1);
    Ok(Some(
        r.landing(frames.into_iter().map(Bytes::from).collect()),
    ))
}

/// Settle the account's handle change in flight, under its account's lock:
/// it landed, with its `#identity` event in the same frame, or it was
/// dropped. Answers the event to land once the log is on disk.
pub(crate) fn log_settled(
    events: &Firehose,
    did: &str,
    st: &mut Store,
    cid: &str,
    landed: Option<&str>,
) -> io::Result<Option<Landing>> {
    let settled = json!({ "t": "handled", "cid": cid, "landed": landed.is_some() });
    let Some(handle) = landed else {
        st.wal.append(&[settled])?;
        st.pending = None;
        return Ok(None);
    };
    let r = events.reserve(did, 1)?;
    let frame = events::identity_event(did, handle, r.seq, &r.time());
    let landing = append_events(events, st, &[settled], Some((&r, vec![frame])), None)?;
    st.pending = None;
    st.logged_handle = Some(handle.to_string());
    Ok(landing)
}

/// Log a commit made under its account's lock, with its event; land the
/// event once the log is on disk.
pub(crate) fn log_commit(
    events: &Firehose,
    did: &str,
    st: &mut Store,
    c: &Change,
) -> io::Result<Landing> {
    let r = events.reserve(did, 1)?;
    let frame = events::commit_event(did, r.seq, &r.time(), c);
    let landing = append_events(events, st, &[], Some((&r, vec![frame])), Some(c))?;
    Ok(landing.expect("a commit's event"))
}

/// Log a repository imported whole (`spec/MIGRATION.md`), its commit made
/// under its account's lock: with its `#sync` event if the account is
/// `active`, to land once the log is on disk, or with none.
pub(crate) fn log_import(
    events: &Firehose,
    did: &str,
    st: &mut Store,
    c: &Change,
    active: bool,
) -> io::Result<Option<Landing>> {
    if !active {
        return append_events(events, st, &[], None, Some(c));
    }
    let r = events.reserve(did, 1)?;
    let frame = events::sync_event(did, r.seq, &r.time(), &c.head_bytes, &c.rev);
    append_events(events, st, &[], Some((&r, vec![frame])), Some(c))
}

/// Make a new repository's first commit, the empty repository, and log it
/// with the events a new account's stream starts with; with `quiet`, an
/// account moving here (`spec/MIGRATION.md`), with none.
pub(crate) fn first_commit(
    events: &Firehose,
    did: &str,
    handle: &str,
    st: &mut Store,
    quiet: bool,
) -> io::Result<Option<Landing>> {
    let mut c = st.repo.init();
    let landed = if quiet {
        append_events(events, st, &[], None, Some(&c))
    } else {
        events.reserve(did, 4).and_then(|r| {
            let frames = events::initial_events(did, handle, r.seq, &r.time(), &c);
            append_events(events, st, &[], Some((&r, frames)), Some(&c))
        })
    };
    if landed.is_err()
        && let Some(u) = c.undo.take()
    {
        st.repo.undo(u);
    }
    landed
}

/// An account's store as its checkpoint and log have it.
pub(crate) struct Recovered {
    pub store: Store,
    /// Whether it has no commit yet.
    pub fresh: bool,
}

/// Try `f` again while it fails for want of a file descriptor, waiting for
/// one to be closed, up to a point.
pub(crate) fn retrying<T>(mut f: impl FnMut() -> io::Result<T>) -> io::Result<T> {
    for _ in 0..200 {
        match f() {
            Err(e) if matches!(Errno::from_io_error(&e), Some(Errno::MFILE | Errno::NFILE)) => {
                std::thread::sleep(std::time::Duration::from_millis(50))
            }
            done => return done,
        }
    }
    f()
}

/// Load an account's store: its checkpoint's table opened, the logs after
/// it replayed, or begun; compacted from now on as `store::policy` says,
/// once the account's slot sets its fold (`cache::Slot`). `durable` says
/// that every frame its log's files hold is on disk, and nothing of them
/// is then synced but what is mended (`checkpoint::load`).
pub(crate) fn open_repo(
    ctx: &Arc<Ctx>,
    did: &str,
    key: &SigningKey,
    durable: bool,
) -> Result<Recovered, String> {
    let dir = &ctx.dir;
    let layout = crate::store::account_layout(dir, did);
    let at = |why: String| format!("{}: {why}", layout.dir.display());
    crate::store::remove_legacy(dir, Some(did)).map_err(|e| at(e.to_string()))?;
    // Uploads on their way when the server stopped: not this process's own.
    if let Ok(entries) = std::fs::read_dir(blob_dir(dir, did)) {
        for e in entries.flatten() {
            let name = e.file_name();
            let name = name.to_string_lossy();
            if name.starts_with(".tmp-") && !name.starts_with(crate::blobs::temp_prefix()) {
                std::fs::remove_file(e.path()).map_err(|why| at(why.to_string()))?;
            }
        }
    }
    let crate::checkpoint::Loaded {
        repo,
        wal,
        disk,
        mut small,
    } = crate::checkpoint::load(ctx, did, key, durable)?;
    // Spaces state of no version but this one is let go, once: from its
    // mark on, the log is of this version's (`checkpoint::unmarked`).
    if small.unmarked.is_some() {
        wal.append(&[crate::checkpoint::mark()])
            .map_err(|e| at(e.to_string()))?;
        small.mark();
    }
    match &ctx.bucket {
        None => small.blobs.recover(&blob_dir(dir, did))?,
        Some(bucket) => {
            small.blobs.recover_kept();
            // An upload the server stopped during, or an object not yet deleted.
            if small.blobs.strays().next().is_some() {
                bucket.wake(did);
            }
        }
    }
    let fresh = repo.rev.is_empty();
    Ok(Recovered {
        store: Store {
            repo,
            wal: held(dir, wal),
            blobs: small.blobs,
            takedowns: small.takedowns,
            spaces: small.spaces,
            authority: small.authority,
            notify: small.notify,
            disk,
            last_seq: small.last_seq,
            logged_handle: small.handle,
            pending: small.pending,
            ctx: ctx.clone(),
            #[cfg(test)]
            thread: std::thread::current().id(),
        },
        fresh,
    })
}

/// Begin a new account's log, with its mark of spaces state of this
/// version (`checkpoint::unmarked`): its store, with no commit yet.
pub(crate) fn new_repo(ctx: &Arc<Ctx>, did: &str, key: &SigningKey) -> io::Result<Store> {
    let layout = crate::store::account_layout(&ctx.dir, did);
    let (wal, frames) = Wal::open_log(&layout, 0, 0, Some(ctx.files.clone()), ctx.policy)?;
    if !frames.is_empty() {
        return Err(io::Error::other("its log holds something already"));
    }
    wal.append(&[crate::checkpoint::mark()])?;
    Ok(Store {
        repo: Repo::new(did.to_string(), key.clone()),
        wal: held(&ctx.dir, wal),
        blobs: Default::default(),
        takedowns: Default::default(),
        spaces: Default::default(),
        authority: Default::default(),
        notify: Default::default(),
        disk: crate::checkpoint::Disk::empty(layout),
        last_seq: 0,
        logged_handle: None,
        pending: None,
        ctx: ctx.clone(),
        #[cfg(test)]
        thread: std::thread::current().id(),
    })
}

/// Load the stores of `accounts`, side by side on at most as many threads
/// as the machine has cores, this one among them: the first error, if one
/// cannot be.
fn load_all(accounts: &[Arc<Account>]) -> Result<(), String> {
    let next = AtomicUsize::new(0);
    let failed: Mutex<Option<String>> = Mutex::new(None);
    let work = || {
        while failed
            .lock()
            .unwrap_or_else(PoisonError::into_inner)
            .is_none()
        {
            let i = next.fetch_add(1, Ordering::Relaxed);
            let Some(a) = accounts.get(i) else { return };
            if let Err(why) = a.store.load() {
                failed
                    .lock()
                    .unwrap_or_else(PoisonError::into_inner)
                    .get_or_insert(why);
            }
        }
    };
    let threads = std::thread::available_parallelism()
        .map_or(1, |n| n.get())
        .min(accounts.len())
        .max(1);
    std::thread::scope(|s| {
        for i in 1..threads {
            if std::thread::Builder::new()
                .name(format!("pds-recover-{i}"))
                .spawn_scoped(s, work)
                .is_err()
            {
                break;
            }
        }
        work();
    });
    failed
        .into_inner()
        .unwrap_or_else(PoisonError::into_inner)
        .map_or(Ok(()), Err)
}

impl Service {
    /// Start from the logs in `dir`: read the service checkpoint's summary
    /// and its entries that are not one account's, and replay the service
    /// log after it, or begin it. Of an account's entries in the log, only
    /// where they are is noted (`directory`): no account is loaded, nor its
    /// own log read, but those the service log marks (`cache`), or every one
    /// with `Config::load_all`, or on a data directory from before seqs
    /// were reserved (`events::SEQS`).
    pub(crate) fn open(dir: &FsPath, config: Config) -> Result<Service, String> {
        use std::os::unix::fs::PermissionsExt;
        #[cfg(test)]
        stop_writers(dir);
        pds_store::make_dirs(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
        // Whether the last server left every log durable, in this very
        // directory, asked before anything of it is changed or written:
        // no log is then synced as it is opened, the service log now and
        // each account's on its first use. The directory's mode is set
        // after, which changes when it last changed: the word names that
        // too, and from here on names this directory no more.
        let durable = crate::ops::durable(dir)?;
        std::fs::set_permissions(dir, std::fs::Permissions::from_mode(pds_store::DIR_MODE))
            .map_err(|e| format!("{}: {e}", dir.display()))?;
        // What the journal holds of writes answered before a kill or a
        // power cut is put back where it is kept, before anything else of
        // the directory is read or changed, whichever way this run syncs.
        // To sync together, the journal begins again, adopting the segments
        // where that start left them, for its first checkpoint to make
        // durable and let go; to sync each by itself, there is no journal
        // to adopt them, and the start makes them durable and deletes them.
        let journal = match config.sync {
            crate::Syncing::Together => {
                let redone = pds_store::journal::redo(dir, pds_store::journal::Redo::Adopting)
                    .map_err(|e| e.to_string())?;
                Some(pds_store::Journal::open(dir, redone.adopted).map_err(|e| e.to_string())?)
            }
            crate::Syncing::Each => {
                pds_store::journal::redo(dir, pds_store::journal::Redo::Eager)
                    .map_err(|e| e.to_string())?;
                None
            }
        };
        #[cfg(test)]
        if let Some(j) = &journal {
            JOURNALS
                .lock()
                .unwrap_or_else(PoisonError::into_inner)
                .push((dir.to_path_buf(), Arc::downgrade(j)));
        }
        // What this run finds is durable already where that was so, or
        // where it finds no data; else once it has made it so
        // (`ops::sync_found`).
        let found = durable || !crate::store::holds_data(dir);
        crate::store::migrate(dir)?;
        crate::store::remove_legacy(dir, None).map_err(|e| format!("{}: {e}", dir.display()))?;
        let layout = crate::store::service_layout(dir);
        let window = config.backfill.as_millis() as u64;
        // A checkpoint of the format before a table is made one, once.
        let now_ms = events::now_ms();
        crate::compact::convert(&layout, now_ms / 1000, |_, f| {
            crate::store::in_window(f, now_ms, window)
        })?;
        // Of `PDS_CACHE_BYTES`: half the stores held, a quarter the blocks
        // read, and the firehose's tail (`events::TAIL`) the rest.
        let blocks = Arc::new(pds_store::table::BlockCache::with_open(
            config.cache_bytes / 4,
            config.share,
        ));
        let table = ServiceTable::open(&layout.checkpoint(), blocks.clone())?;
        let (generation, size, service_seq, mut ceiling) =
            table.as_ref().map_or((0, 0, 0, None), |t| {
                (
                    t.summary.generation,
                    t.size(),
                    t.summary.last_seq,
                    t.summary.ceiling,
                )
            });
        let new = table.is_none();
        // No account loaded: what the log holds of each is noted in it.
        let mut accounts = Directory::new(layout.clone(), table);
        // The accounts marked.
        let mut marked: HashSet<String> = HashSet::new();
        // Keyed once the replay has said whether the data directory is new.
        let mut sessions = Sessions::unkeyed(&config.hostname);
        let oauth =
            crate::oauth::OAuth::open(dir, &config.public_url, config.oauth_client_fetch.clone())?;
        let replay = crate::replay::Replay::default();
        let (single_use, space_credentials) = (
            crate::single_use::Spent::default(),
            crate::space_credentials::Credentials::default(),
        );
        let now = sessions.now();
        let invites = crate::invites::Invites::default();
        // The events this log holds, and the places of a deleted account's.
        let mut lifecycle = crate::lifecycle::Replayed::default();
        // Deleted accounts' objects in a bucket not yet deleted.
        let mut doomed = crate::blobstore::Doomed::default();
        // Replay an entry that is not one account's, given what is noted of
        // the accounts so far: answers the DID of an account it deletes.
        let mut apply = |e: &Json, accounts: &Directory| -> Result<Option<String>, String> {
            let at = |why: String| format!("{ACCOUNTS_LOG}: {why}");
            match e["t"].as_str() {
                Some(crate::replay::ENTRY) => replay.replay(e, now).map_err(at)?,
                Some(t) if crate::oauth::ENTRIES.contains(&t) => {
                    oauth.replay(e, &|did| accounts.may_be(did)).map_err(at)?
                }
                Some(crate::single_use::ENTRY) => single_use.replay(e, now).map_err(at)?,
                Some(crate::space_credentials::ENTRY) => {
                    space_credentials.replay(e, now).map_err(at)?
                }
                Some(t) if crate::invites::ENTRIES.contains(&t) => invites.replay(e).map_err(at)?,
                Some(t) if crate::lifecycle::ENTRIES.contains(&t) => {
                    if let Some(did) = lifecycle.replay(e).map_err(at)? {
                        // Deleted: it is no account; its files, if any are
                        // left, go once the server serves.
                        marked.remove(&did);
                        oauth.forget_account(&did);
                        return Ok(Some(did));
                    }
                }
                Some(crate::cache::DIR) => {
                    let (Some(did), Some(eager)) = (e["did"].as_str(), e["eager"].as_bool()) else {
                        return Err(at(format!("a mark with no did or eager: {e}")));
                    };
                    // Nothing is kept of an account deleted.
                    if accounts.is_noted_deleted(did) {
                        return Ok(None);
                    }
                    if !accounts.may_be(did) {
                        return Err(at(format!("a mark of {did:?}, which no account has")));
                    }
                    if eager {
                        marked.insert(did.to_string());
                    } else {
                        marked.remove(did);
                    }
                }
                Some(t) if crate::blobstore::ENTRIES.contains(&t) => {
                    doomed.replay(e).map_err(at)?
                }
                Some(crate::events::SEQS) => {
                    let upto = e["upto"]
                        .as_u64()
                        .ok_or_else(|| at(format!("seqs with no upto: {e}")))?;
                    ceiling = Some(ceiling.unwrap_or(0).max(upto));
                }
                t => return Err(at(format!("a log entry of no known type: {t:?}"))),
            }
            Ok(None)
        };
        // The table's entries that are not one account's, a row at a time.
        if let Some(t) = accounts.table() {
            t.rest(|entries| {
                entries
                    .iter()
                    .try_for_each(|e| apply(e, &accounts).map(drop))
            })
            .map_err(|why| format!("{}: {why}", layout.checkpoint().display()))?;
        }
        // The logs after it, replayed a frame at a time as they are read:
        // an entry of one account is noted, not applied.
        let mut frames = 0;
        let mut failed = None;
        let log = Wal::open_log_with(
            &layout,
            generation,
            size,
            None,
            crate::store::service_policy(&config),
            durable,
            &mut |pos, entries| {
                frames += 1;
                let replayed: Result<(), String> = entries.iter().try_for_each(|e| {
                    let of = crate::compact::of_account(e)
                        .map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?;
                    // A mark is the account's, for its handle, and not.
                    if (of.is_none() || e["t"] == crate::cache::DIR)
                        && let Some(did) = apply(e, &accounts)?
                    {
                        accounts.note_deleted(&did, pos.generation);
                    }
                    if let Some(did) = of {
                        accounts
                            .note(pos, did, e)
                            .map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?;
                    }
                    Ok(())
                });
                if let Err(why) = &replayed {
                    failed = Some(why.clone());
                }
                replayed
            },
        );
        let log = match (log, failed) {
            (_, Some(why)) => return Err(why),
            (Ok(log), None) => log,
            (Err(e), None) => return Err(format!("{ACCOUNTS_LOG}: {e}")),
        };
        // In no pool, and given the journal all the same.
        if let Some(j) = &journal {
            log.use_journal(j.clone());
        }
        sessions.keyed(dir, new && frames == 0)?;
        oauth.recovered(dir, sessions.now())?;
        // New events are numbered after every seq reserved before (`events::SEQS`).
        let logged = lifecycle
            .last_seq()
            .max(service_seq)
            .max(lifecycle.gaps.iter().copied().max().unwrap_or(0));
        let last = logged.max(ceiling.unwrap_or(0));
        // A data directory from before seqs were reserved: every account's
        // log is read for the greatest seq it holds.
        let upgrade = ceiling.is_none() && accounts.count() > 0;
        let log = held(dir, log);
        let logged_events = lifecycle
            .events
            .drain(..)
            .map(|e| events::Recovered {
                seq: e.seq,
                did: Arc::from(e.did.as_str()),
                frame: Bytes::from(e.frame),
            })
            .collect();
        let files = match journal {
            Some(j) => Files::with_journal(config.share, j),
            None => Files::new(config.share),
        };
        let firehose = Firehose::new(
            logged_events,
            lifecycle.gaps.iter().copied().collect(),
            last,
            logged,
            config.backfill,
            log.clone(),
            events::index::At {
                dir: dir.to_path_buf(),
                pool: files.clone(),
                policy: crate::store::service_policy(&config),
                durable,
            },
        );
        let mut marks: Vec<String> = marked.iter().cloned().collect();
        marks.sort_unstable();
        let eager = Arc::new(crate::cache::Eager::new(log.clone(), marked));
        let bucket = match config.blobstore.clone() {
            Some(s) => {
                let b = crate::blobstore::Bucket::new(s)?;
                b.doomed(doomed);
                Some(Arc::new(b))
            }
            None => None,
        };
        let ctx = Arc::new(Ctx {
            dir: dir.to_path_buf(),
            files: files.clone(),
            policy: crate::store::policy(&config),
            window,
            events: firehose.clone(),
            eager: eager.clone(),
            cache: crate::cache::Cache::new(config.cache_bytes / 2),
            blocks: blocks.clone(),
            starting: std::sync::atomic::AtomicBool::new(true),
            durable,
            found: std::sync::atomic::AtomicBool::new(found),
            failed: std::sync::atomic::AtomicBool::new(false),
            bucket,
        });
        let limits = crate::limits::Limits::new(dir, &config);
        let outbound = crate::outbound::Outbound::new(config.ssrf_protection)?;
        let load_every = config.load_all || upgrade;
        let directory = Arc::new(Accounts::new(layout, blocks, accounts));
        // Compacted from now on, its table put in place with the directory held.
        log.set_fold(Arc::new(crate::store::ServiceFold {
            window,
            accounts: Arc::downgrade(&directory),
        }));
        let svc = Service {
            space_notify: crate::space_notify::Notify::new(),
            config,
            dir: dir.to_path_buf(),
            log,
            ctx: ctx.clone(),
            directory,
            sessions,
            oauth,
            replay,
            invites,
            events: firehose,
            http: crate::plc::client().map_err(|e| format!("an HTTP client: {e}"))?,
            changes: crate::identity::Changes::default(),
            space_hooks: crate::space_repo::Hooks::default(),
            single_use,
            space_credentials,
            space_authority: crate::space_authority::Host::default(),
            limits,
            resets: email::Resets::default(),
            outbound,
            did_docs: Default::default(),
            handle_resolver: Default::default(),
            prefs: Default::default(),
        };
        // The accounts a start loads: those marked, or every one.
        let dids = if load_every {
            crate::read(&svc.directory).dids()
        } else {
            marks.clone()
        };
        let mut loading: Vec<Arc<Account>> = Vec::with_capacity(dids.len());
        for did in &dids {
            loading.extend(svc.build(did)?);
        }
        {
            let made: HashSet<&str> = loading.iter().map(|a| a.did.as_str()).collect();
            if let Some(did) = marks.iter().find(|did| !made.contains(did.as_str())) {
                return Err(format!(
                    "{ACCOUNTS_LOG}: a mark of {did:?}, which no account has"
                ));
            }
        }
        load_all(&loading)?;
        // With every account's log read, the greatest seq any holds is known:
        // the next follows it, whatever was reserved.
        let mut last = if load_every { logged } else { last };
        // Each one's handle change in flight, if it has one.
        let mut pending = Vec::with_capacity(loading.len());
        for a in &loading {
            let st = crate::read(&a.store);
            last = last.max(st.last_seq);
            // Its handle is the one its last landed change gave it.
            if let Some(h) = st.logged_handle.as_ref().filter(|h| **h != a.handle()) {
                *crate::write(&a.handle) = h.clone();
            }
            pending.push(st.pending.clone());
        }
        // Each handle, an account's or one a change in flight holds, and
        // each email, is one account's, of those loaded.
        {
            let mut d = crate::write(&svc.directory);
            for (a, pending) in loading.iter().zip(pending) {
                d.insert_loaded(a.clone())
                    .map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?;
                let Some(p) = pending else { continue };
                // The handle stays taken until the change is settled.
                if p.handle != a.handle() {
                    if let Some(other) = d.handles.get(&p.handle) {
                        return Err(format!(
                            "{ACCOUNTS_LOG}: {} is the handle of both {other} and {}",
                            p.handle, a.did
                        ));
                    }
                    if !d.claimed_handles.insert(p.handle.clone()) {
                        return Err(format!(
                            "{ACCOUNTS_LOG}: {} is the handle two accounts are being given",
                            p.handle
                        ));
                    }
                }
                svc.changes.unsettled(&a.did, p);
            }
        }
        if load_every {
            svc.events.exact(last);
        } else {
            svc.events
                .raise(last)
                .map_err(|e| format!("{ACCOUNTS_LOG}: {e}"))?;
        }
        // A data directory with accounts and no index of their events is
        // given one, of every account's files, before it serves.
        svc.events
            .adopt(&svc)
            .map_err(|e| format!("the firehose's index: {e}"))?;
        // An account whose repository has no commit yet gets its first,
        // with no events if it was made deactivated, moving here.
        for a in &loading {
            let mut st = crate::write(&a.store);
            if st.repo.rev.is_empty() {
                let quiet = crate::read(&a.state).deactivated;
                let landing = first_commit(&svc.events, &a.did, &a.handle(), &mut st, quiet)
                    .map_err(|e| format!("{}: {e}", a.did))?;
                landing
                    .as_ref()
                    .map_or(Ok(()), Landing::wait_entry)
                    .and_then(|()| st.wal.wait(st.wal.written()))
                    .map_err(|e| format!("{}: {e}", a.did))?;
                if let Some(l) = landing {
                    svc.events.land(l);
                }
            }
            // Loaded and needing no start: unmarked, with the handle it has.
            if eager.is_marked(&a.did) && !st.needs_start() {
                eager.relax(&a.did, Some(&a.handle()));
            }
        }
        ctx.starting.store(false, Ordering::SeqCst);
        crate::space_notify::resume(&svc);
        Ok(svc)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::sessions;
    use crate::testing::*;
    use pds_store as store;
    use rustix::process::{Resource, Rlimit, Signal, getrlimit, setrlimit};
    use std::collections::BTreeMap;
    use std::path::Path;

    /// Everything a restart must carry over, as one comparable value: each
    /// account's repository, rev and KEM key, the sessions, the events, and
    /// the invite codes, each account's email and the tokens mailed for it,
    /// and each account's state, its records and blobs taken down.
    type State = (
        BTreeMap<String, (Vec<u8>, String, Option<String>)>,
        BTreeMap<String, sessions::Keep>,
        Vec<Vec<u8>>,
        Vec<String>,
        Mails,
        Lives,
    );

    /// Each account's mail, the directory's emails, and which account each reset token is of.
    type Mails = (
        BTreeMap<String, Mail>,
        BTreeMap<String, String>,
        BTreeMap<String, String>,
    );

    /// Each account's state, its records taken down, and its blobs; and the directory's handles.
    type Lives = (
        BTreeMap<String, (crate::lifecycle::State, BTreeMap<String, String>, String)>,
        BTreeMap<String, String>,
    );

    fn state(s: &Service) -> State {
        let d = crate::read(&s.directory);
        let repos = d
            .by_did
            .iter()
            .map(|(did, a)| {
                let st = crate::read(&a.store);
                (
                    did.clone(),
                    (
                        crate::sync::CarStream::of(&st, None)
                            .unwrap()
                            .bytes()
                            .unwrap(),
                        st.repo.rev.clone(),
                        a.kem.as_ref().map(|k| hex::encode(k.to_bytes())),
                    ),
                )
            })
            .collect();
        let mails = d
            .by_did
            .iter()
            .map(|(did, a)| (did.clone(), email::lock(&a.mail).clone()))
            .collect();
        let emails = d
            .emails
            .iter()
            .map(|(e, did)| (e.clone(), did.clone()))
            .collect();
        let lives = d
            .by_did
            .iter()
            .map(|(did, a)| {
                let st = crate::read(&a.store);
                (
                    did.clone(),
                    (
                        crate::read(&a.state).clone(),
                        st.takedowns.clone(),
                        st.blobs.snapshot(),
                    ),
                )
            })
            .collect();
        let handles = d
            .handles
            .iter()
            .map(|(h, did)| (h.clone(), did.clone()))
            .collect();
        drop(d);
        let sessions = s.sessions.snapshot();
        let events = s
            .events
            .get(s, 0, u64::MAX - 1, s.events.drops())
            .iter()
            .map(|e| e.to_vec())
            .collect();
        (
            repos,
            sessions,
            events,
            s.invites.snapshot(),
            (mails, emails, s.resets.snapshot()),
            (lives, handles),
        )
    }

    /// The files with the service log as a kill cut short in an account's
    /// log could have left it: without the notes of handles and the marks
    /// let go, which are written only once a frame of that log is on disk
    /// (`cache::Eager::relax`). The marks made before stay.
    fn unnoted(mut files: BTreeMap<PathBuf, Vec<u8>>) -> BTreeMap<PathBuf, Vec<u8>> {
        let p = PathBuf::from(ACCOUNTS_LOG_FILE);
        if let Some(bytes) = files.get(&p) {
            let note = |e: &Json| {
                e["t"] == crate::cache::DIR
                    && (e.get("handle").is_some() || e["eager"] == json!(false))
            };
            let frames = store::scan(bytes).unwrap().0;
            let kept: Vec<u8> = frames
                .iter()
                .filter(|f| !f.iter().all(note))
                .flat_map(|f| store::frame(f))
                .collect();
            files.insert(p, kept);
        }
        files
    }

    /// The files a step changed, but the service log, where marks go
    /// (`cache`), and the firehose's index, which a step that makes events
    /// appends their entries to (`events::index`).
    fn changed_logs<'a>(
        before: &BTreeMap<PathBuf, Vec<u8>>,
        after: &'a BTreeMap<PathBuf, Vec<u8>>,
    ) -> Vec<&'a PathBuf> {
        after
            .keys()
            .filter(|p| {
                before.get(*p) != after.get(*p)
                    && p.as_path() != Path::new(ACCOUNTS_LOG_FILE)
                    && !of_index(p)
            })
            .collect()
    }

    /// Whether a file of the data directory, by its path there, is the
    /// firehose's index's (`events::index`).
    fn of_index(p: &Path) -> bool {
        p.to_str().is_some_and(|p| p.starts_with("events."))
    }

    /// `files` with the index's files as `from` holds them.
    fn with_index_of(
        mut files: BTreeMap<PathBuf, Vec<u8>>,
        from: &BTreeMap<PathBuf, Vec<u8>>,
    ) -> BTreeMap<PathBuf, Vec<u8>> {
        files.retain(|p, _| !of_index(p));
        files.extend(
            from.iter()
                .filter(|(p, _)| of_index(p))
                .map(|(p, b)| (p.clone(), b.clone())),
        );
        files
    }

    /// Of a step that made events, the index cut short at every byte of
    /// the entries it appended, the step's other writes lost, as a power
    /// cut may leave it, with `room` zeros after the cut where it is given:
    /// recovery must give the state before the step. Answers how many cuts.
    fn index_cut_short(work: &Path, plc: &str, w: &[Step], room: Option<usize>) -> usize {
        let mut cuts = 0;
        for (p, whole) in w[1]
            .files
            .iter()
            .filter(|(p, b)| of_index(p) && w[0].files.get(*p) != Some(*b))
        {
            let from = w[0].files.get(p).map_or(0, Vec::len);
            for cut in from + 1..whole.len() {
                let mut files = w[0].files.clone();
                let bytes = match room {
                    Some(room) => with_room(&whole[..cut], whole.len() - cut + room),
                    None => whole[..cut].to_vec(),
                };
                files.insert(p.clone(), bytes);
                put_files(work, &files);
                assert_eq!(
                    &state(&open(work, plc)),
                    &w[0].state,
                    "{} cut at {cut} of {from}..{}, the step's other writes lost",
                    p.display(),
                    whole.len()
                );
                cuts += 1;
            }
        }
        cuts
    }

    /// An event's body, from its frame: its `seq`, `did` and the rest.
    fn body_of(frame: &[u8]) -> Json {
        let (_, at) = pds_repo::data::decode_prefix(frame).unwrap();
        pds_repo::data::to_json(&pds_repo::data::decode(&frame[at..]).unwrap())
    }

    /// The events a subscription from cursor 0 begun now is sent, each its
    /// body.
    fn sent(s: &Service) -> Vec<Json> {
        s.events
            .get(s, 0, u64::MAX - 1, s.events.drops())
            .iter()
            .map(|f| body_of(f))
            .collect()
    }

    /// The seqs of the events a subscription from cursor 0 is sent, in order.
    fn sent_seqs(s: &Service) -> Vec<u64> {
        sent(s).iter().map(|e| e["seq"].as_u64().unwrap()).collect()
    }

    /// Every file in the data directory, and its bytes: of a log, those it
    /// holds, without the room after its frames (`pds_store`), as a power
    /// cut the room did not outlive leaves it. The journal's segments are
    /// left out: what they hold is in the logs, as the system has them, and
    /// a step's frames are cut or kept in their logs alone, as after a
    /// checkpoint (`journal_cut_at_every_byte` cuts the journal).
    fn files(dir: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
        let mut files = files_whole(dir);
        files.retain(|p, _| !p.starts_with(pds_store::journal::DIR));
        for (_, bytes) in files
            .iter_mut()
            .filter(|(p, _)| p.extension().is_some_and(|e| e == "log"))
        {
            *bytes = super::without_room(std::mem::take(bytes));
        }
        files
    }

    /// Every file in the data directory, and its bytes, room and all.
    fn files_whole(dir: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
        let mut out = BTreeMap::new();
        let mut todo = vec![dir.to_path_buf()];
        while let Some(d) = todo.pop() {
            for e in std::fs::read_dir(&d).unwrap() {
                let p = e.unwrap().path();
                if p.is_dir() {
                    todo.push(p);
                } else {
                    out.insert(
                        p.strip_prefix(dir).unwrap().to_path_buf(),
                        std::fs::read(&p).unwrap(),
                    );
                }
            }
        }
        out
    }

    /// Make `dir` hold exactly these files, writing only those that differ,
    /// so that the logs recovery makes durable are mostly durable already.
    fn put_files(dir: &Path, files: &BTreeMap<PathBuf, Vec<u8>>) {
        let have = if dir.exists() {
            files_whole(dir)
        } else {
            BTreeMap::new()
        };
        for p in have.keys().filter(|p| !files.contains_key(*p)) {
            std::fs::remove_file(dir.join(p)).unwrap();
        }
        for (p, bytes) in files.iter().filter(|(p, b)| have.get(*p) != Some(*b)) {
            let p = dir.join(p);
            std::fs::create_dir_all(p.parent().unwrap()).unwrap();
            std::fs::write(p, bytes).unwrap();
        }
    }

    /// A log, or the first of one, with `room` zeros after it: as a kill
    /// leaves a log that was open (`pds_store`).
    fn with_room(log: &[u8], room: usize) -> Vec<u8> {
        [log, &vec![0u8; room]].concat()
    }

    /// The room each log in the data directory holds: the zeros after its
    /// frames.
    fn rooms(dir: &Path) -> BTreeMap<PathBuf, usize> {
        files_whole(dir)
            .into_iter()
            .filter(|(p, _)| p.extension().is_some_and(|e| e == "log"))
            .map(|(p, whole)| {
                let room = whole.len() - super::without_room(whole).len();
                (p, room)
            })
            .collect()
    }

    /// A step of the script: the files after it, the journal's segments
    /// apart, each without its room, and the state.
    struct Step {
        files: BTreeMap<PathBuf, Vec<u8>>,
        journal: BTreeMap<PathBuf, Vec<u8>>,
        state: State,
    }

    /// The journal's segments in the data directory, each without its room.
    fn segments(dir: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
        let mut j = files_whole(dir);
        j.retain(|p, _| p.starts_with(pds_store::journal::DIR));
        for bytes in j.values_mut() {
            *bytes = super::without_room(std::mem::take(bytes));
        }
        j
    }

    /// Run a script of requests over two accounts; after each, the files and the state.
    async fn script(dir: &Path, plc: &str) -> (Vec<Step>, Vec<(String, String)>) {
        let s = open(dir, plc);
        let mut steps = Vec::new();
        let snap = |s: &Service, steps: &mut Vec<Step>| {
            steps.push(Step {
                files: files(dir),
                journal: segments(dir),
                state: state(s),
            })
        };
        snap(&s, &mut steps);
        let (a, ta) = create(&s, "alice").await;
        snap(&s, &mut steps);
        let (b, tb) = create(&s, "bob").await;
        snap(&s, &mut steps);
        let ta2 = session(&s, "alice").await;
        snap(&s, &mut steps);
        let w = |did: &str, k: &str, n: i64| json!({ "repo": did, "collection": "a.b.c", "rkey": k, "record": record(n) });
        call(&s, "com.atproto.repo.putRecord", Some(&ta), w(&a, "one", 1))
            .await
            .unwrap();
        snap(&s, &mut steps);
        call(&s, "com.atproto.repo.putRecord", Some(&tb), w(&b, "one", 1))
            .await
            .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.repo.putRecord",
            Some(&ta2),
            w(&a, "two", 2),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(&s, "com.atproto.repo.applyWrites", Some(&tb), json!({ "repo": b, "writes": [
            { "$type": "com.atproto.repo.applyWrites#create", "collection": "a.b.c", "rkey": "three", "value": record(3) },
            { "$type": "com.atproto.repo.applyWrites#update", "collection": "a.b.c", "rkey": "one", "value": record(4) } ] })).await.unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.repo.deleteRecord",
            Some(&ta),
            json!({ "repo": a, "collection": "a.b.c", "rkey": "one" }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        // A refused write changes nothing and logs nothing.
        assert!(
            call(
                &s,
                "com.atproto.repo.applyWrites",
                Some(&ta),
                json!({ "repo": b, "writes": [] })
            )
            .await
            .is_err()
        );
        snap(&s, &mut steps);
        let n = steps.len();
        assert_eq!(steps[n - 1].files, steps[n - 2].files);
        (steps, vec![(a, ta), (b, tb)])
    }

    #[tokio::test]
    async fn restart_resumes_every_account() {
        let plc = plc().await;
        let dir = scratch("resume");
        let (steps, accounts) = script(&dir, &plc).await;
        let s = open(&dir, &plc);
        assert_eq!(state(&s), steps.last().unwrap().state);
        // Four events open each account's stream; each of the five commits adds one.
        assert_eq!(s.events.len(), 4 + 4 + 5);
        assert_eq!(*s.events.subscribe().borrow(), s.events.len() as u64);
        drop(s);
        let s = open(&dir, &plc);
        for (i, (did, token)) in accounts.iter().enumerate() {
            let before = repo_of(&s, did, |r| r.rev.clone());
            let body =
                json!({ "repo": did, "collection": "a.b.c", "rkey": "x", "record": { "n": 1 } });
            call(&s, "com.atproto.repo.putRecord", Some(token), body)
                .await
                .unwrap();
            assert!(repo_of(&s, did, |r| r.rev.clone()) > before);
            // The firehose carries on from the last event.
            let n = s.events.len();
            assert_eq!(n, 13 + i + 1);
            let last = s.events.get(&s, n as u64 - 1, n as u64, 0).pop().unwrap();
            let (_, at) = pds_repo::data::decode_prefix(&last).unwrap();
            let body = pds_repo::data::to_json(&pds_repo::data::decode(&last[at..]).unwrap());
            assert_eq!(body["seq"], json!(n));
            assert_eq!(body["repo"], json!(did));
        }
        // Accounts and sessions too: each logs in again, and each old token works.
        session(&s, "alice").await;
        for (did, token) in &accounts {
            assert_eq!(
                call(&s, "com.atproto.server.getSession", Some(token), json!({}))
                    .await
                    .unwrap()["did"],
                json!(did)
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A data directory written before logs had room, each log its frames
    /// alone, is started on and carried on from. Its logs are given room as
    /// they are written to, which no count of their bytes includes; and a
    /// server killed with that room in them starts as one killed without.
    /// Under `each`: a log given a journal (`together`) makes no room.
    #[tokio::test]
    async fn logs_with_no_room_are_carried_on() {
        syncing(crate::Syncing::Each);
        let plc = plc().await;
        let dir = scratch("room-src");
        let (steps, accounts) = script(&dir, &plc).await;
        let last = steps.last().unwrap();
        let work = scratch("room-work");
        put_files(&work, &last.files);
        assert_eq!(files_whole(&work), last.files);
        let s = open(&work, &plc);
        assert_eq!(state(&s), last.state);
        for n in 0..3 {
            for (did, token) in &accounts {
                let body = json!({ "repo": did, "collection": "a.b.c", "rkey": format!("more{n}"), "record": record(n) });
                call(&s, "com.atproto.repo.putRecord", Some(token), body)
                    .await
                    .unwrap();
            }
        }
        let (killed, bare) = (files_whole(&work), files(&work));
        for (did, _) in &accounts {
            let log = repo_log(Path::new(""), did);
            let (file, frames) = (killed[&log].len(), bare[&log].len());
            assert!(file > frames, "{file} bytes of file, {frames} of frames");
            assert_eq!(store::scan(&killed[&log]).unwrap().1, frames);
            // The log's bytes are its frames', by its own count and by its file.
            let counted = s.account(did).unwrap().store.log_bytes();
            assert_eq!(counted, frames as u64);
            assert_eq!(crate::directory::files_bytes(&work, did), counted);
        }
        let after = state(&s);
        drop(s);
        for (name, disk) in [("room-killed", &killed), ("room-bare", &bare)] {
            let again = scratch(name);
            put_files(&again, disk);
            let s = open(&again, &plc);
            assert_eq!(state(&s), after, "{name}");
            let (bob, token) = &accounts[1];
            let body =
                json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": record(9) });
            call(&s, "com.atproto.repo.putRecord", Some(token), body)
                .await
                .unwrap();
            let now = state(&s);
            drop(s);
            assert_eq!(state(&open(&again, &plc)), now, "{name}");
            let _ = std::fs::remove_dir_all(&again);
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Room is held by the logs open to be written to, and no others: a
    /// store let go leaves its account's log its frames alone, and a
    /// shutdown every log. A kill leaves each log the room it had: the
    /// service log until the start after it, an account's until its store
    /// is next loaded. Under `each`: a log given a journal (`together`)
    /// makes no room.
    #[tokio::test]
    async fn room_is_held_by_open_logs_alone() {
        syncing(crate::Syncing::Each);
        let plc = plc().await;
        let dir = scratch("room-held");
        let s = open(&dir, &plc);
        let (alice, ta) = create(&s, "alice").await;
        let (bob, tb) = create(&s, "bob").await;
        for (did, token) in [(&alice, &ta), (&bob, &tb)] {
            let body =
                json!({ "repo": did, "collection": "a.b.c", "rkey": "one", "record": record(1) });
            call(&s, "com.atproto.repo.putRecord", Some(token), body)
                .await
                .unwrap();
        }
        let service = PathBuf::from(ACCOUNTS_LOG_FILE);
        let [a, b] = [&alice, &bob].map(|did| repo_log(Path::new(""), did));
        let held = rooms(&dir);
        assert!(held[&a] > 0 && held[&b] > 0, "{held:?}");
        // Let go, a store leaves its log its frames alone.
        assert!(s.account(&alice).unwrap().store.evict());
        let held = rooms(&dir);
        assert!(held[&a] == 0 && held[&b] > 0, "{held:?}");
        let mut killed = files_whole(&dir);
        let log = killed.get_mut(&service).unwrap();
        *log = with_room(&super::without_room(std::mem::take(log)), 1500);
        // A shutdown leaves every log so.
        crate::ops::settle(&s).unwrap();
        assert!(rooms(&dir).values().all(|room| *room == 0));
        drop(s);
        // Killed, the logs keep their room: the service log until the start
        // after, an account's until its store is next loaded.
        let work = scratch("room-held-killed");
        put_files(&work, &killed);
        let s = open_with(&work, &plc, |c| c.load_all = false);
        assert!(!s.account(&bob).unwrap().store.is_loaded());
        let held = rooms(&work);
        assert!(held[&service] == 0 && held[&b] > 0, "{held:?}");
        assert_eq!(
            crate::directory::files_bytes(&work, &bob),
            super::without_room(killed[&b].clone()).len() as u64
        );
        s.account(&bob).unwrap().store.load().unwrap();
        assert_eq!(rooms(&work)[&b], 0);
        drop(s);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Logs whose frames are intact but whose entries are not what the
    /// server wrote are refused with a reason, not a panic.
    #[tokio::test]
    async fn bad_entries_are_refused() {
        let plc = plc().await;
        let dir = scratch("bad-src");
        let (steps, accounts) = script(&dir, &plc).await;
        let last = &steps.last().unwrap().files;
        let work = scratch("bad-work");
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let alice = repo_log(Path::new(""), &accounts[0].0);
        let frames = |p: &PathBuf| store::scan(&last[p]).unwrap().0;
        let reopen = |p: &PathBuf, frames: &[Vec<Json>]| {
            let mut files = last.clone();
            files.insert(
                p.clone(),
                frames.iter().flat_map(|f| store::frame(f)).collect(),
            );
            put_files(&work, &files);
            try_open(&work, &plc).err()
        };
        assert_eq!(reopen(&alice, &frames(&alice)), None);
        // A record the latest commit did not sign.
        let mut tampered = frames(&alice);
        let commit = tampered
            .iter_mut()
            .rev()
            .find_map(|f| {
                f.iter_mut()
                    .find(|e| e["t"] == "commit" && !e["puts"].as_array().unwrap().is_empty())
            })
            .unwrap();
        commit["puts"][0][1]["n"] = json!(99);
        assert!(reopen(&alice, &tampered).unwrap().contains("does not sign"));
        for (at, bad) in [
            (1, json!({ "t": "nonsense" })),
            (
                1,
                json!({ "t": "event", "ord": 1, "seq": 1, "frame": "00" }),
            ),
            (2, json!({ "t": "commit", "rev": "x" })),
        ] {
            let mut broken = frames(&alice);
            broken[at].push(bad.clone());
            assert!(reopen(&alice, &broken).is_some(), "{bad}");
        }
        let a = accounts[0].0.as_str();
        let session = |did: &str, app: Json, scope: &str| json!({ "t": "session", "did": did, "id": "s", "app": app, "scope": scope, "exp": 1 });
        for bad in [
            json!({ "t": "nonsense" }),
            json!({ "t": "session", "did": a }),
            session("did:plc:nobody", Json::Null, "com.atproto.access"),
            session(a, Json::Null, "com.atproto.appPass"),
            // A session of an app password not had is kept only with an app password's scope (`dropAppPasswords`).
            session(a, json!("no such app password"), "com.atproto.access"),
            json!({ "t": "refresh", "did": a, "id": "not kept", "next": "n", "exp": 1, "nextExp": 2 }),
            json!({ "t": "endSession", "did": a, "id": "not kept" }),
            json!({ "t": "appPassword", "did": a, "name": "x", "hash": "h", "createdAt": "c" }),
            json!({ "t": "revokeAppPassword", "did": a, "name": "not had" }),
            json!({ "t": "useToken", "did": a, "purpose": "confirm_email" }),
            json!({ "t": "mailToken", "did": a, "purpose": "nonsense", "token": "AAAAA-AAAAA", "made": 1 }),
            json!({ "t": "mailToken", "did": a, "purpose": "confirm_email", "token": "AAAAAAAAAA", "made": 1 }),
            json!({ "t": "mailToken", "did": a, "purpose": "confirm_email", "token": "AAAAA-AAAAA" }),
            json!({ "t": "mailToken", "did": "did:plc:nobody", "purpose": "confirm_email", "token": "AAAAA-AAAAA", "made": 1 }),
            json!({ "t": "emailAuthFactor", "did": a, "on": true }),
            json!({ "t": "updateEmail", "did": a, "email": "bob@mail.com" }),
            json!({ "t": "updateEmail", "did": a }),
            json!({ "t": "password", "did": a }),
        ] {
            let mut broken = frames(&accounts_log);
            broken[1].push(bad.clone());
            assert!(reopen(&accounts_log, &broken).is_some(), "{bad}");
        }
        // The same session twice.
        let mut twice = frames(&accounts_log);
        let first = twice
            .iter()
            .flatten()
            .find(|e| e["t"] == "session")
            .unwrap()
            .clone();
        twice.push(vec![first]);
        assert!(
            reopen(&accounts_log, &twice)
                .unwrap()
                .contains("a second session")
        );
        // A second account with the first's handle.
        // A second account with the first's email, or its handle.
        let mut twice = frames(&accounts_log);
        let mut copy = twice
            .iter()
            .flatten()
            .find(|e| e["t"] == "account")
            .unwrap()
            .clone();
        copy["did"] = json!("did:plc:another");
        twice.push(vec![copy.clone()]);
        assert!(
            reopen(&accounts_log, &twice)
                .unwrap()
                .contains("a second account")
        );
        copy["email"] = json!("another@mail.com");
        *twice.last_mut().unwrap() = vec![copy];
        assert!(
            reopen(&accounts_log, &twice)
                .unwrap()
                .contains("is the handle of both")
        );
        // A KEM key that is not one.
        for bad in [
            json!("zz"),
            json!("00"),
            json!(hex::encode([0u8; 32])),
            json!(1),
        ] {
            let mut broken = frames(&accounts_log);
            broken
                .iter_mut()
                .flatten()
                .find(|e| e["t"] == "account")
                .unwrap()["kemKey"] = bad.clone();
            assert!(
                reopen(&accounts_log, &broken)
                    .unwrap()
                    .contains("bad kemKey"),
                "{bad}"
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// An account from before accounts had a KEM key is given none: it
    /// starts without one, its credentials name only its signing key, and
    /// its entry stays as it was.
    #[tokio::test]
    async fn an_account_without_a_kem_key() {
        let plc = plc().await;
        let dir = scratch("no-kem");
        let s = open(&dir, &plc);
        let (alice, token) = create(&s, "alice").await;
        drop(s);
        redone(&dir);
        let path = accounts_log(&dir);
        let mut frames = store::scan(&std::fs::read(&path).unwrap()).unwrap().0;
        let entry = frames
            .iter_mut()
            .flatten()
            .find(|e| e["t"] == "account")
            .unwrap();
        assert!(entry["kemKey"].is_string());
        entry.as_object_mut().unwrap().remove("kemKey");
        let bytes: Vec<u8> = frames.iter().flat_map(|f| store::frame(f)).collect();
        std::fs::write(&path, &bytes).unwrap();
        let s = open(&dir, &plc);
        assert!(s.account(&alice).unwrap().kem.is_none());
        let got = call(
            &s,
            "com.atproto.identity.getRecommendedDidCredentials",
            Some(&token),
            json!({}),
        )
        .await
        .unwrap();
        assert_eq!(
            got["verificationMethods"]
                .as_object()
                .unwrap()
                .keys()
                .collect::<Vec<_>>(),
            ["atproto"]
        );
        drop(s);
        assert_eq!(std::fs::read(&path).unwrap(), bytes);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Cut each log at every byte of the last frame the script wrote to it
    /// (and flip a byte in it): recovery must give the state before that
    /// write or after it, and keep working.
    #[tokio::test]
    async fn torn_writes_recover_to_before_or_after() {
        for sync in SYNCS {
            syncing(sync);
            torn_writes_once().await;
        }
    }

    async fn torn_writes_once() {
        let plc = plc().await;
        let dir = scratch("torn-src");
        let (steps, accounts) = script(&dir, &plc).await;
        let (bob, token) = &accounts[1];
        let work = scratch("torn-work");
        let mut cuts = 0;
        // The steps after both accounts exist, each of which changes one log
        // besides the index, which takes the entries of the events a write
        // makes before the frame that holds them.
        let changes = |w: &[Step]| -> Vec<PathBuf> {
            w[1].files
                .keys()
                .filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p))
                .cloned()
                .collect()
        };
        let mut index_cuts = 0;
        let windows: Vec<&[Step]> = steps.windows(2).skip(2).collect();
        for (i, w) in windows.iter().enumerate() {
            let changed = changes(w);
            let [path] = &changed[..] else {
                assert_eq!(changed.len(), 0);
                continue;
            };
            if windows[i + 1..]
                .iter()
                .any(|later| changes(later).contains(path))
            {
                continue;
            }
            let (from, to) = (w[0].files[path].len(), w[1].files[path].len());
            let whole = &w[1].files[path];
            for cut in from..=to {
                let mut files = w[1].files.clone();
                files.insert(path.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let s = open(&work, &plc);
                let want = if cut == to { &w[1].state } else { &w[0].state };
                assert_eq!(
                    &state(&s),
                    want,
                    "{} cut at {cut} of {from}..{to}",
                    path.display()
                );
                cuts += 1;
                // After recovery the logs take new writes and replay them.
                if cut % 61 == 0 || cut == from + 1 || cut == to {
                    let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": { "n": 9 } });
                    call(&s, "com.atproto.repo.putRecord", Some(token), body)
                        .await
                        .unwrap();
                    let now = state(&s);
                    drop(s);
                    assert_eq!(
                        state(&open(&work, &plc)),
                        now,
                        "{} cut at {cut}",
                        path.display()
                    );
                }
            }
            // A damaged byte anywhere in the frame drops it.
            for at in (from..to).step_by(5) {
                let mut files = w[1].files.clone();
                files.get_mut(path).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(
                    &state(&open(&work, &plc)),
                    &w[0].state,
                    "{} flip at {at} of {from}..{to}",
                    path.display()
                );
            }
            // The write's entries cut short, its frame lost.
            index_cuts += index_cut_short(&work, &plc, w, None);
        }
        assert!(cuts > 500, "{cuts}");
        assert!(index_cuts > 100, "{index_cuts}");
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// A power cut as `PDS_SYNC` `together` leaves it, the journal cut at
    /// every byte of a write (`spec/DURABILITY.md`, "Writes waiting at
    /// once"): of each step that writes one log, the log as it was before
    /// the step, its frames never synced in it, a log the step made there
    /// and empty, as its making was synced; and the journal's segment cut
    /// at every byte of what the step wrote to it, every record before it
    /// whole. A start puts back what the journal holds whole: the state
    /// before the step, or after it once the step's records are whole.
    #[tokio::test]
    async fn journal_cut_at_every_byte() {
        let plc = plc().await;
        let dir = scratch("journal-cut-src");
        let (steps, _) = script(&dir, &plc).await;
        let work = scratch("journal-cut-work");
        let (mut cuts, mut stepped) = (0, 0);
        for w in steps.windows(2) {
            let changed: Vec<&PathBuf> = w[1]
                .files
                .keys()
                .filter(|p| p.extension().is_some_and(|e| e == "log"))
                .filter(|p| w[0].files.get(*p) != w[1].files.get(*p))
                .collect();
            if changed.is_empty() {
                continue;
            }
            // The segment the step wrote to, grown from what it was: none
            // where a checkpoint came between, which is no cut of this.
            let Some((segment, from, to)) = w[1].journal.iter().find_map(|(p, after)| {
                let before = w[0].journal.get(p)?;
                (after.len() > before.len() && after.starts_with(before))
                    .then(|| (p.clone(), before.len(), after.len()))
            }) else {
                continue;
            };
            stepped += 1;
            let mut files = w[1].files.clone();
            for p in &changed {
                files.insert(
                    (*p).clone(),
                    w[0].files.get(*p).cloned().unwrap_or_default(),
                );
            }
            for cut in from..=to {
                let mut disk = files.clone();
                disk.extend(w[1].journal.clone());
                disk.insert(segment.clone(), w[1].journal[&segment][..cut].to_vec());
                put_files(&work, &disk);
                let s = open(&work, &plc);
                let got = state(&s);
                if cut == to {
                    assert_eq!(
                        got,
                        w[1].state,
                        "{} cut at {cut} of {from}..{to}",
                        segment.display()
                    );
                } else if cut == from {
                    assert_eq!(got, w[0].state, "{} cut at {cut}", segment.display());
                } else {
                    assert!(
                        got == w[0].state || got == w[1].state,
                        "{} cut at {cut} of {from}..{to}",
                        segment.display()
                    );
                }
                drop(s);
                cuts += 1;
            }
        }
        assert!(stepped >= 5 && cuts > 500, "{stepped} steps, {cuts} cuts");
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// A frame written over room and cut at every byte, the room after the
    /// cut as it was: the shortest write the script made to an account's
    /// log, and the one it made to the service log alone. Recovery must
    /// give the state before that write or after it, and keep working, as
    /// where the file ends at the cut
    /// (`torn_writes_recover_to_before_or_after`).
    #[tokio::test]
    async fn writes_torn_in_room_recover_to_before_or_after() {
        for sync in SYNCS {
            syncing(sync);
            writes_torn_in_room_once().await;
        }
    }

    async fn writes_torn_in_room_once() {
        let plc = plc().await;
        let dir = scratch("torn-room-src");
        let (steps, accounts) = script(&dir, &plc).await;
        let (bob, token) = &accounts[1];
        let work = scratch("torn-room-work");
        let service = PathBuf::from(ACCOUNTS_LOG_FILE);
        // Of the steps that change one log, the service log or an account's,
        // the one of each kind whose frame is shortest.
        let mut shortest: BTreeMap<bool, (usize, &[Step], &PathBuf)> = BTreeMap::new();
        for w in steps.windows(2).skip(2) {
            let changed: Vec<&PathBuf> = w[1]
                .files
                .keys()
                .filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p))
                .collect();
            let [path] = changed[..] else { continue };
            let len = w[1].files[path].len() - w[0].files[path].len();
            let of = shortest.entry(*path == service).or_insert((len, w, path));
            if len < of.0 {
                *of = (len, w, path);
            }
        }
        assert_eq!(shortest.len(), 2, "a write to each kind of log");
        let mut cuts = 0;
        for (_, w, path) in shortest.into_values() {
            let (from, to) = (w[0].files[path].len(), w[1].files[path].len());
            let whole = &w[1].files[path];
            assert!(whole.starts_with(&w[0].files[path]));
            for cut in from..=to {
                // What of the frame reached the disk, then zeros to past
                // where it ends.
                let mut files = w[1].files.clone();
                files.insert(path.clone(), with_room(&whole[..cut], to - cut + 1500));
                put_files(&work, &files);
                let s = open(&work, &plc);
                let want = if cut == to { &w[1].state } else { &w[0].state };
                assert_eq!(
                    &state(&s),
                    want,
                    "{} cut at {cut} of {from}..{to}",
                    path.display()
                );
                cuts += 1;
                if cut % 61 == 0 || cut == from + 1 || cut == to {
                    let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": { "n": 9 } });
                    call(&s, "com.atproto.repo.putRecord", Some(token), body)
                        .await
                        .unwrap();
                    let now = state(&s);
                    drop(s);
                    assert_eq!(
                        state(&open(&work, &plc)),
                        now,
                        "{} cut at {cut}",
                        path.display()
                    );
                }
            }
            // The write's entries cut short in room, its frame lost.
            cuts += index_cut_short(&work, &plc, w, Some(1500));
        }
        assert!(cuts > 500, "{cuts}");
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut `accounts.log` at every byte of each frame a change to sessions or
    /// app passwords wrote (and flip a byte in it): recovery must give the
    /// state before that change or after it. Each kind of entry is cut: the
    /// account starts with as many sessions as it keeps, so that the next
    /// one drops one.
    #[tokio::test]
    async fn torn_session_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_session_writes_once().await;
        }
    }

    async fn torn_session_writes_once() {
        let plc = plc().await;
        let dir = scratch("torn-sessions");
        let s = open(&dir, &plc);
        let (alice, _) = create(&s, "alice").await;
        let tokens = |got: Json| {
            (
                got["accessJwt"].as_str().unwrap().to_string(),
                got["refreshJwt"].as_str().unwrap().to_string(),
            )
        };
        let login = |pw: &str| json!({ "identifier": "alice.test", "password": pw });
        let (_, mut chain) = tokens(
            call(
                &s,
                "com.atproto.server.createSession",
                None,
                login("alice-pw"),
            )
            .await
            .unwrap(),
        );
        for _ in 2..sessions::MAX_SESSIONS {
            chain = tokens(
                call(
                    &s,
                    "com.atproto.server.refreshSession",
                    Some(&chain),
                    Json::Null,
                )
                .await
                .unwrap(),
            )
            .1;
        }
        assert_eq!(s.sessions.count(&alice), sessions::MAX_SESSIONS);
        let mut steps = vec![Step {
            files: files(&dir),
            journal: segments(&dir),
            state: state(&s),
        }];
        let snap = |s: &Service, steps: &mut Vec<Step>| {
            steps.push(Step {
                files: files(&dir),
                journal: segments(&dir),
                state: state(s),
            })
        };
        let (access, r0) = tokens(
            call(
                &s,
                "com.atproto.server.createSession",
                None,
                login("alice-pw"),
            )
            .await
            .unwrap(),
        );
        assert_eq!(s.sessions.count(&alice), sessions::MAX_SESSIONS);
        snap(&s, &mut steps);
        let app = call(
            &s,
            "com.atproto.server.createAppPassword",
            Some(&access),
            json!({ "name": "phone", "privileged": true }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        let (_, app_r0) = tokens(
            call(
                &s,
                "com.atproto.server.createSession",
                None,
                login(app["password"].as_str().unwrap()),
            )
            .await
            .unwrap(),
        );
        snap(&s, &mut steps);
        let (_, r1) = tokens(
            call(
                &s,
                "com.atproto.server.refreshSession",
                Some(&r0),
                Json::Null,
            )
            .await
            .unwrap(),
        );
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.refreshSession",
            Some(&app_r0),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.deleteSession",
            Some(&r1),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.revokeAppPassword",
            Some(&access),
            json!({ "name": "phone" }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        drop(s);
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let work = scratch("torn-sessions-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let changed: Vec<&PathBuf> = w[1]
                .files
                .keys()
                .filter(|p| w[0].files.get(*p) != w[1].files.get(*p))
                .collect();
            assert_eq!(changed, [&accounts_log]);
            assert_ne!(w[0].state, w[1].state);
            let whole = &w[1].files[&accounts_log];
            let (from, to) = (w[0].files[&accounts_log].len(), whole.len());
            let (frames, _) = store::scan(&whole[from..]).unwrap();
            kinds.extend(
                frames
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = w[1].files.clone();
                files.insert(accounts_log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { &w[1].state } else { &w[0].state };
                assert_eq!(
                    &state(&open(&work, &plc)),
                    want,
                    "cut at {cut} of {from}..{to}"
                );
            }
            for at in (from..to).step_by(3) {
                let mut files = w[1].files.clone();
                files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(
                    &state(&open(&work, &plc)),
                    &w[0].state,
                    "flip at {at} of {from}..{to}"
                );
            }
        }
        kinds.sort();
        kinds.dedup();
        // A deactivation with an OAuth token writes `dropAppPasswords`, in its
        // lifecycle frame: compact.rs's account_states_end_oauth_sessions.
        let mut want: Vec<&str> = crate::sessions::ENTRIES
            .iter()
            .copied()
            .filter(|k| *k != "dropAppPasswords")
            .collect();
        want.sort();
        assert_eq!(kinds, want);
        // After recovery from the last cut, the log takes changes, and replays them.
        put_files(&work, &steps.last().unwrap().files);
        let s = open(&work, &plc);
        call(
            &s,
            "com.atproto.server.createSession",
            None,
            login("alice-pw"),
        )
        .await
        .unwrap();
        let now = state(&s);
        drop(s);
        assert_eq!(state(&open(&work, &plc)), now);
        assert_ne!(now.1[&alice], steps.last().unwrap().state.1[&alice]);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut the log each lifecycle change wrote at every byte of its frame
    /// (and flip a byte in it): recovery must give the state before that
    /// change or after it. A deactivation, an activation, a takedown with
    /// the sessions it ends and its lifting, an event of a DID no account
    /// has, a record's and a blob's takedown, a deletion's token, and a
    /// deletion, whose account's files may be there still or not: each
    /// kind of entry is cut.
    #[tokio::test]
    async fn torn_lifecycle_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_lifecycle_writes_once().await;
        }
    }

    async fn torn_lifecycle_writes_once() {
        use base64::Engine;
        let plc = plc().await;
        let dir = scratch("torn-life");
        let sink = scratch("torn-life-mail");
        let s = crate::email::testing::open_with_sink(&dir, &plc, &sink);
        let (alice, ta) = create(&s, "alice").await;
        let (bob, tb) = create(&s, "bob").await;
        session(&s, "alice").await;
        let blob = crate::blobs::test_upload(
            &s,
            Some(&ta),
            Some("text/plain"),
            b"taken down".to_vec(),
            false,
        )
        .await
        .unwrap()["blob"]
            .clone();
        let w = |k: &str, v: Json| json!({ "repo": alice, "collection": "a.b.c", "rkey": k, "record": v });
        call(
            &s,
            "com.atproto.repo.putRecord",
            Some(&ta),
            w("one", json!({ "b": blob })),
        )
        .await
        .unwrap();
        let admin = format!(
            "Basic {}",
            base64::engine::general_purpose::STANDARD.encode(format!("admin:{ADMIN_PASSWORD}"))
        );
        let update = |body: Json| {
            let (s, admin) = (s.clone(), admin.clone());
            async move {
                call_raw(
                    &s,
                    "com.atproto.admin.updateSubjectStatus",
                    sessions::Bearer::of(Some(&admin)),
                    &[],
                    &body.to_string(),
                )
                .await
                .unwrap()
            }
        };
        let repo_ref = |did: &str| json!({ "$type": "com.atproto.admin.defs#repoRef", "did": did });
        let mut steps = vec![Step {
            files: files(&dir),
            journal: segments(&dir),
            state: state(&s),
        }];
        let snap = |s: &Service, steps: &mut Vec<Step>| {
            steps.push(Step {
                files: files(&dir),
                journal: segments(&dir),
                state: state(s),
            })
        };
        call(
            &s,
            "com.atproto.server.deactivateAccount",
            Some(&ta),
            json!({}),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.activateAccount",
            Some(&ta),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        update(json!({ "subject": repo_ref(&alice), "takedown": { "applied": true, "ref": "r" } }))
            .await;
        snap(&s, &mut steps);
        update(json!({ "subject": repo_ref(&alice), "takedown": { "applied": false } })).await;
        snap(&s, &mut steps);
        update(json!({ "subject": repo_ref("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa") })).await;
        snap(&s, &mut steps);
        let uri = format!("at://{alice}/a.b.c/one");
        update(json!({ "subject": { "$type": "com.atproto.repo.strongRef", "uri": uri, "cid": blob["ref"]["$link"] }, "takedown": { "applied": true } })).await;
        snap(&s, &mut steps);
        let blob_ref = json!({ "$type": "com.atproto.admin.defs#repoBlobRef", "did": alice, "cid": blob["ref"]["$link"] });
        update(json!({ "subject": blob_ref, "takedown": { "applied": true, "ref": "b" } })).await;
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.requestAccountDelete",
            Some(&tb),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        let token = crate::email::testing::last_token(&sink);
        call(
            &s,
            "com.atproto.server.deleteAccount",
            None,
            json!({ "did": bob, "password": "bob-pw", "token": token }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        drop(s);
        let work = scratch("torn-life-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            // The log a change wrote to, but the index, which takes the
            // entries of the events it makes before the frame that holds them.
            let written: Vec<&PathBuf> = w[1]
                .files
                .keys()
                .filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p))
                .collect();
            let [path] = written[..] else {
                panic!("{written:?}")
            };
            let indexed = w[1]
                .files
                .iter()
                .any(|(p, b)| of_index(p) && w[0].files.get(p) != Some(b));
            // A deletion's account's files go once it is logged: a cut keeps them.
            let gone: Vec<&PathBuf> = w[0]
                .files
                .keys()
                .filter(|p| !w[1].files.contains_key(*p))
                .collect();
            assert_ne!(w[0].state, w[1].state);
            let whole = &w[1].files[path];
            let (from, to) = (w[0].files[path].len(), whole.len());
            let (frames, _) = store::scan(&whole[from..]).unwrap();
            kinds.extend(
                frames
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = if gone.is_empty() {
                    w[1].files.clone()
                } else {
                    with_index_of(w[0].files.clone(), &w[1].files)
                };
                files.insert(path.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { &w[1].state } else { &w[0].state };
                let s = open(&work, &plc);
                assert_eq!(
                    &state(&s),
                    want,
                    "{} cut at {cut} of {from}..{to}",
                    path.display()
                );
                if cut == to {
                    // What a deletion left of its account is gone once the
                    // directories of no account are swept.
                    s.sweep_repos();
                    assert!(gone.iter().all(|p| !work.join(p).exists()));
                    drop(s);
                    // The change's frame whole and its entries lost: the
                    // service log's own events need none to be sent.
                    if indexed {
                        put_files(&work, &with_index_of(files, &w[0].files));
                        assert_eq!(
                            &state(&open(&work, &plc)),
                            want,
                            "{} whole, its entries lost",
                            path.display()
                        );
                    }
                }
            }
            for at in (from..to).step_by(7) {
                let mut files = if gone.is_empty() {
                    w[1].files.clone()
                } else {
                    w[0].files.clone()
                };
                let mut flipped = whole.clone();
                flipped[at] ^= 0x55;
                files.insert(path.clone(), flipped);
                put_files(&work, &files);
                assert_eq!(
                    &state(&open(&work, &plc)),
                    &w[0].state,
                    "{} flip at {at} of {from}..{to}",
                    path.display()
                );
            }
            // The change's entries cut short, its frame lost.
            if indexed {
                assert!(index_cut_short(&work, &plc, w, None) > 0);
            }
        }
        kinds.sort();
        kinds.dedup();
        for k in crate::lifecycle::ENTRIES.iter().chain(&[
            crate::lifecycle::RECORD_TAKEDOWN,
            "takedownBlob",
            "endSession",
            "mailToken",
        ]) {
            assert!(kinds.iter().any(|x| x == k), "{k} not cut: {kinds:?}");
        }
        // After recovery from the last cut, the logs take changes, and replay them.
        put_files(&work, &steps.last().unwrap().files);
        let s = open(&work, &plc);
        call(
            &s,
            "com.atproto.server.deactivateAccount",
            Some(&ta),
            json!({}),
        )
        .await
        .unwrap();
        let now = state(&s);
        drop(s);
        assert_eq!(state(&open(&work, &plc)), now);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut `accounts.log` at every byte of each frame a change to an
    /// account's email, password or tokens wrote (and flip a byte in it):
    /// recovery must give the state before that change or after it, the
    /// sessions a reset password ends with it. Each kind of entry is cut.
    #[tokio::test]
    async fn torn_email_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_email_writes_once().await;
        }
    }

    async fn torn_email_writes_once() {
        use crate::email::testing::{last_token, open_with_sink};
        let plc = plc().await;
        let dir = scratch("torn-email");
        let sink = scratch("torn-email-mail");
        let s = open_with_sink(&dir, &plc, &sink);
        let (_, access) = create(&s, "alice").await;
        create(&s, "bob").await;
        let mut steps = vec![Step {
            files: files(&dir),
            journal: segments(&dir),
            state: state(&s),
        }];
        let snap = |s: &Service, steps: &mut Vec<Step>| {
            steps.push(Step {
                files: files(&dir),
                journal: segments(&dir),
                state: state(s),
            })
        };
        let token = || last_token(&sink);
        let login = |code: Option<String>| {
            let mut b = json!({ "identifier": "alice.test", "password": "alice-pw" });
            if let Some(c) = code {
                b["authFactorToken"] = json!(c);
            }
            b
        };
        let factor = |on: bool, token: Option<String>| {
            let mut b = json!({ "email": "alice@mail.com", "emailAuthFactor": on });
            if let Some(t) = token {
                b["token"] = json!(t);
            }
            b
        };
        call(
            &s,
            "com.atproto.server.requestEmailConfirmation",
            Some(&access),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.confirmEmail",
            Some(&access),
            json!({ "email": "alice@mail.com", "token": token() }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.updateEmail",
            Some(&access),
            factor(true, None),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        assert_eq!(
            call(&s, "com.atproto.server.createSession", None, login(None)).await,
            Err("AuthFactorTokenRequired".into())
        );
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.createSession",
            None,
            login(Some(token())),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        assert_eq!(
            call(
                &s,
                "com.atproto.server.updateEmail",
                Some(&access),
                factor(false, None)
            )
            .await,
            Err("TokenRequired".into())
        );
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.updateEmail",
            Some(&access),
            factor(false, Some(token())),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.requestEmailUpdate",
            Some(&access),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.updateEmail",
            Some(&access),
            json!({ "email": "alice2@mail.com", "token": token() }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.requestPasswordReset",
            None,
            json!({ "email": "alice2@mail.com" }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.resetPassword",
            None,
            json!({ "token": token(), "password": "new" }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        drop(s);
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let work = scratch("torn-email-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let changed: Vec<&PathBuf> = w[1]
                .files
                .keys()
                .filter(|p| w[0].files.get(*p) != w[1].files.get(*p))
                .collect();
            assert_eq!(changed, [&accounts_log]);
            assert_ne!(w[0].state, w[1].state);
            let whole = &w[1].files[&accounts_log];
            let (from, to) = (w[0].files[&accounts_log].len(), whole.len());
            let (frames, _) = store::scan(&whole[from..]).unwrap();
            assert_eq!(frames.len(), 1);
            kinds.extend(
                frames
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = w[1].files.clone();
                files.insert(accounts_log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { &w[1].state } else { &w[0].state };
                assert_eq!(
                    &state(&open(&work, &plc)),
                    want,
                    "cut at {cut} of {from}..{to}"
                );
            }
            for at in (from..to).step_by(3) {
                let mut files = w[1].files.clone();
                files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(
                    &state(&open(&work, &plc)),
                    &w[0].state,
                    "flip at {at} of {from}..{to}"
                );
            }
        }
        for k in crate::email::ENTRIES {
            assert!(kinds.iter().any(|got| got == k), "no {k} entry was cut");
        }
        assert!(kinds.iter().any(|k| k == "endSession"));
        // After recovery from the last cut, the log takes changes, and replays them.
        put_files(&work, &steps.last().unwrap().files);
        let s = open_with_sink(&work, &plc, &sink);
        call(
            &s,
            "com.atproto.server.requestPasswordReset",
            None,
            json!({ "email": "alice2@mail.com" }),
        )
        .await
        .unwrap();
        let now = state(&s);
        drop(s);
        assert_eq!(state(&open(&work, &plc)), now);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut `accounts.log` at every byte of each frame `putPreferences` wrote
    /// (and flip a byte in it): recovery must give every account's
    /// preferences as before that put or after it.
    #[tokio::test]
    async fn torn_preference_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_preference_writes_once().await;
        }
    }

    async fn torn_preference_writes_once() {
        let plc = plc().await;
        let dir = scratch("torn-prefs");
        let cfg = || {
            let mut c = config(&plc);
            c.proxy.appview = Some(crate::proxy::AppView {
                url: "http://127.0.0.1:1".into(),
                did: "did:web:appview.test".into(),
            });
            c
        };
        let s = Arc::new(Service::open(&dir, cfg()).unwrap());
        let (_, alice) = create(&s, "alice").await;
        let (_, bob) = create(&s, "bob").await;
        let prefs = |s: &Service| s.prefs.snapshot();
        let mut steps = vec![(files(&dir), prefs(&s))];
        let puts = [
            (
                &alice,
                json!([{ "$type": "app.bsky.actor.defs#adultContentPref", "enabled": true }]),
            ),
            (&bob, json!([{ "$type": "app.bsky.x" }])),
            (&alice, json!([])),
            (
                &alice,
                json!([{ "$type": "app.bsky.actor.defs#personalDetailsPref", "birthDate": "2000-01-01T00:00:00Z" }, { "$type": "app.bsky.y", "n": "é" }]),
            ),
        ];
        for (token, list) in puts {
            call(
                &s,
                "app.bsky.actor.putPreferences",
                Some(token),
                json!({ "preferences": list }),
            )
            .await
            .unwrap();
            steps.push((files(&dir), prefs(&s)));
        }
        drop(s);
        let log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let work = scratch("torn-prefs-work");
        let recovered = || prefs(&Service::open(&work, cfg()).unwrap());
        let mut cuts = 0;
        for w in steps.windows(2) {
            let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
            assert_ne!(before, after);
            // Preferences are the service log's own.
            let changed: Vec<&PathBuf> = after_files
                .keys()
                .filter(|p| before_files.get(*p) != after_files.get(*p))
                .collect();
            assert_eq!(changed, [&log]);
            let whole = &after_files[&log];
            let (from, to) = (before_files[&log].len(), whole.len());
            let (frames, _) = store::scan(&whole[from..]).unwrap();
            assert!(
                frames
                    .iter()
                    .flatten()
                    .all(|e| e["t"] == crate::preferences::ENTRY)
            );
            for cut in from..=to {
                let mut files = unnoted(after_files.clone());
                files.insert(log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                assert_eq!(
                    &recovered(),
                    if cut == to { after } else { before },
                    "cut at {cut} of {from}..{to}"
                );
                cuts += 1;
            }
            for at in (from..to).step_by(7) {
                let mut files = after_files.clone();
                files.get_mut(&log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(&recovered(), before, "flip at {at} of {from}..{to}");
            }
        }
        assert!(cuts > 100, "{cuts}");
        // After recovery the log takes more, and replays them.
        put_files(&work, &steps.last().unwrap().0);
        let s = Arc::new(Service::open(&work, cfg()).unwrap());
        call(
            &s,
            "app.bsky.actor.putPreferences",
            Some(&bob),
            json!({ "preferences": [] }),
        )
        .await
        .unwrap();
        assert_eq!(recovered(), prefs(&s));
        assert_eq!(prefs(&s).len(), 1);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut `accounts.log` at every byte of each frame an OAuth change wrote
    /// (and flip a byte in it): recovery must give what OAuth keeps, and the
    /// single-use values, as before that change or after it. Each kind of
    /// entry is cut.
    #[tokio::test]
    async fn torn_oauth_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_oauth_writes_once().await;
        }
    }

    async fn torn_oauth_writes_once() {
        use crate::oauth::tests::{App, Browser, held_at, kept, serve, text};
        let plc = plc().await;
        let dir = scratch("torn-oauth");
        let srv = serve(open(&dir, &plc)).await;
        let (alice, alice_token) = create(&srv.svc, "alice").await;
        let mut steps = vec![(files(&dir), kept(&srv.svc))];
        let snap = |steps: &mut Vec<_>| steps.push((files(&dir), kept(&srv.svc)));
        let mut app = App::new(&srv, "atproto");
        let mut browser = Browser::new(&srv);
        let cid = app.client_id.clone();
        let (_, pushed) = app.par(&[]).await;
        snap(&mut steps);
        browser.page(&cid, &pushed.request_uri).await;
        snap(&mut steps);
        // The device's cookies from before the sign-in, which it renews.
        let old_cookies = browser.cookies.clone();
        let sign_in = json!({ "locale": "en", "identifier": "alice.test", "password": "alice-pw", "remember": true });
        browser
            .api(&cid, &pushed.request_uri, "sign-in", sign_in, None)
            .await;
        snap(&mut steps);
        let consent = browser
            .api(
                &cid,
                &pushed.request_uri,
                "consent",
                json!({ "did": alice }),
                None,
            )
            .await
            .json();
        let url = url::Url::parse(consent["url"].as_str().unwrap()).unwrap();
        let code = url
            .query_pairs()
            .find(|(k, _)| k == "code")
            .unwrap()
            .1
            .into_owned();
        snap(&mut steps);
        let t = app.exchange(&code, &pushed).await.json();
        snap(&mut steps);
        let r = app.refresh(&text(&t, "refresh_token")).await.json();
        snap(&mut steps);
        app.get_session(&text(&r, "access_token")).await;
        snap(&mut steps);
        app.revoke(&text(&r, "access_token")).await;
        snap(&mut steps);
        // A code refused, and spent.
        let (other, code) = crate::oauth::tests::approved(&mut app, &mut browser, "alice").await;
        steps.push((files(&dir), kept(&srv.svc)));
        let wrong = crate::oauth::tests::Pushed {
            verifier: "x".repeat(43),
            ..other
        };
        app.exchange(&code, &wrong).await;
        snap(&mut steps);
        let (_, rejected) = app.par(&[]).await;
        browser.page(&cid, &rejected.request_uri).await;
        steps.push((files(&dir), kept(&srv.svc)));
        browser
            .api(&cid, &rejected.request_uri, "reject", json!({}), None)
            .await;
        snap(&mut steps);
        browser
            .api(
                &cid,
                &rejected.request_uri,
                "sign-out",
                json!({ "did": alice }),
                None,
            )
            .await;
        snap(&mut steps);
        // Signed in again; past 30 seconds, the old cookies end the device.
        browser.api(&cid, &rejected.request_uri, "sign-in", json!({ "locale": "en", "identifier": "alice.test", "password": "alice-pw", "remember": true }), None).await;
        snap(&mut steps);
        srv.svc
            .sessions
            .skew
            .store(31, std::sync::atomic::Ordering::SeqCst);
        let mut old = Browser::new(&srv);
        old.cookies = old_cookies;
        old.api(
            &cid,
            &rejected.request_uri,
            "sign-out",
            json!({ "did": alice }),
            None,
        )
        .await;
        snap(&mut steps);
        // The clients an account approved forgotten, as a deactivation with
        // an OAuth token does: one approved first, as a discoverable
        // client's is (two frames, not cut).
        let now = srv.svc.sessions.now();
        let request = json!({
            "t": "oauthRequest", "id": "req-forget", "client": "https://app.example/client.json", "redirectUri": "https://app.example/cb",
            "scope": ["atproto"], "state": "s", "codeChallenge": "c".repeat(43), "responseMode": "query", "jkt": "j", "expires": now + 300,
        });
        let approve = json!({ "t": "oauthApprove", "id": "req-forget", "code": "cod-forget", "did": alice, "scope": ["atproto"], "at": now });
        for e in [request, approve] {
            let n = srv.svc.log.append(std::slice::from_ref(&e)).unwrap();
            assert!(crate::durable(srv.svc.log.clone(), n).await.is_ok());
            srv.svc.oauth.replay(&e, &|_| true).unwrap();
        }
        steps.push((files(&dir), kept(&srv.svc)));
        let ends = crate::oauth::Ends {
            clients: true,
            ..Default::default()
        };
        let n = srv
            .svc
            .oauth
            .end_for(&alice, ends, |e| srv.svc.log.append(&e))
            .unwrap();
        assert!(crate::durable(srv.svc.log.clone(), n).await.is_ok());
        snap(&mut steps);
        // A passkey added, signing in, and removed (`spec/PASSKEYS.md`):
        // its link and challenge are held in memory alone.
        let auth = format!("Bearer {alice_token}");
        let (mut key, handle) = crate::oauth::passkeys::tests::add(&srv, &auth, "Laptop", 3).await;
        snap(&mut steps);
        let signed = crate::oauth::passkeys::tests::sign_in(
            &mut browser,
            &cid,
            &rejected.request_uri,
            &mut key,
            &handle,
            true,
        )
        .await;
        assert_eq!(signed.status, 200, "{}", signed.body);
        snap(&mut steps);
        let id = base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, &key.id);
        crate::oauth::passkeys::tests::own(
            &srv,
            "POST",
            "_passkeyRemove",
            &auth,
            Some(json!({ "id": id })),
        )
        .await;
        snap(&mut steps);
        drop(srv);
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let work = scratch("torn-oauth-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
            let changed: Vec<&PathBuf> = after_files
                .keys()
                .filter(|p| before_files.get(*p) != after_files.get(*p))
                .collect();
            assert_eq!(changed, [&accounts_log]);
            assert_ne!(before, after);
            let whole = &after_files[&accounts_log];
            let from = before_files[&accounts_log].len();
            let (frames, _) = store::scan(&whole[from..]).unwrap();
            if frames.len() != 1 {
                // A step of several requests, as approving one: only the last is cut.
                continue;
            }
            kinds.extend(
                frames
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            let to = whole.len();
            for cut in from..=to {
                let mut files = after_files.clone();
                files.insert(accounts_log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { after } else { before };
                let s = open(&work, &plc);
                let now = s.sessions.now();
                assert_eq!(
                    held_at(&(s.oauth.snapshot(), s.replay.snapshot(0)), now),
                    held_at(want, now),
                    "cut at {cut} of {from}..{to}"
                );
            }
            for at in (from..to).step_by(7) {
                let mut files = after_files.clone();
                files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                let s = open(&work, &plc);
                let now = s.sessions.now();
                assert_eq!(
                    held_at(&(s.oauth.snapshot(), s.replay.snapshot(0)), now),
                    held_at(before, now),
                    "flip at {at} of {from}..{to}"
                );
            }
        }
        kinds.sort();
        kinds.dedup();
        let mut want: Vec<&str> = crate::oauth::ENTRIES.to_vec();
        want.push(crate::replay::ENTRY);
        want.sort();
        assert_eq!(kinds, want);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut `accounts.log` at every byte of each frame a single-use value or
    /// a revocation wrote (and flip a byte in it): recovery must give the
    /// values and revocations held as before that write or after. Each kind
    /// of entry is cut.
    #[tokio::test]
    async fn torn_space_credential_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_space_credential_writes_once().await;
        }
    }

    async fn torn_space_credential_writes_once() {
        use crate::space_credentials::tests as creds;
        let plc = plc().await;
        let dir = scratch("torn-space-credentials");
        let s = open(&dir, &plc);
        let (alice, token) = create(&s, "alice").await;
        let (bob, _) = create(&s, "bob").await;
        let personal = json!({ "$type": "com.atproto.personalspace.policy" });
        call(
            &s,
            "com.atproto.space.createSpace",
            Some(&token),
            json!({ "spaceType": "com.example.forum", "skey": "main", "policy": personal }),
        )
        .await
        .unwrap();
        type Held = (BTreeMap<[u8; 32], u64>, BTreeMap<[u8; 32], u64>);
        let held =
            |s: &Service| -> Held { (s.single_use.snapshot(0), s.space_credentials.snapshot(0)) };
        let mut steps = vec![(files(&dir), held(&s))];
        let space = format!("at://{alice}/space/com.example.forum/main");
        let t = creds::delegation_token(&s, &token, &space).await.unwrap();
        creds::credential(
            &s,
            json!({ "space": space, "credentialKey": creds::client(1).kid, "delegationToken": t }),
        )
        .await
        .unwrap();
        steps.push((files(&dir), held(&s)));
        let auth = format!("Bearer {}", creds::service_auth_token(&s, &alice, &bob));
        creds::revoke(
            &s,
            Some(&auth),
            json!({ "space": space, "credentials": ["a", "b"] }),
        )
        .await
        .unwrap();
        steps.push((files(&dir), held(&s)));
        drop(s);
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let work = scratch("torn-space-credentials-work");
        // What a recovered server holds against what it should, at its own
        // time: a delegation token's value is held only a minute past its
        // issue, and a slow run may recover after that, rightly without it.
        let recovers_to = |want: &Held, why: String| {
            let s = open(&work, &plc);
            let now = s.sessions.now();
            let live = |m: &BTreeMap<[u8; 32], u64>| {
                m.iter()
                    .filter(|(_, t)| **t > now)
                    .map(|(k, t)| (*k, *t))
                    .collect::<BTreeMap<_, _>>()
            };
            assert_eq!(
                (
                    s.single_use.snapshot(now),
                    s.space_credentials.snapshot(now)
                ),
                (live(&want.0), live(&want.1)),
                "{why}"
            );
        };
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let changed: Vec<&PathBuf> = w[1]
                .0
                .keys()
                .filter(|p| w[0].0.get(*p) != w[1].0.get(*p))
                .collect();
            assert_eq!(changed, [&accounts_log]);
            assert_ne!(w[0].1, w[1].1);
            let whole = &w[1].0[&accounts_log];
            let (from, to) = (w[0].0[&accounts_log].len(), whole.len());
            kinds.extend(
                store::scan(&whole[from..])
                    .unwrap()
                    .0
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = w[1].0.clone();
                files.insert(accounts_log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                recovers_to(
                    if cut == to { &w[1].1 } else { &w[0].1 },
                    format!("cut at {cut} of {from}..{to}"),
                );
            }
            for at in (from..to).step_by(3) {
                let mut files = w[1].0.clone();
                files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                recovers_to(&w[0].1, format!("flip at {at} of {from}..{to}"));
            }
        }
        assert_eq!(
            kinds,
            [crate::single_use::ENTRY, crate::space_credentials::ENTRY]
        );
        // A malformed entry of either kind is refused with a reason.
        let last = &steps.last().unwrap().0;
        for bad in [
            json!({ "t": crate::single_use::ENTRY, "v": "x", "until": 1 }),
            json!({ "t": crate::space_credentials::ENTRY, "space": "s", "credentials": [1], "until": 1 }),
        ] {
            let mut frames = store::scan(&last[&accounts_log]).unwrap().0;
            frames.push(vec![bad.clone()]);
            let mut files = last.clone();
            files.insert(
                accounts_log.clone(),
                frames.iter().flat_map(|f| store::frame(f)).collect(),
            );
            put_files(&work, &files);
            assert!(try_open(&work, &plc).is_err(), "{bad}");
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut an authority's `repo.log` at every byte of each frame a change to
    /// its spaces wrote (and flip a byte in it): recovery gives its spaces as
    /// before the change or after it, and the log takes changes after. Each
    /// kind of entry is cut.
    #[tokio::test]
    async fn torn_space_authority_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_space_authority_writes_once().await;
        }
    }

    async fn torn_space_authority_writes_once() {
        use crate::space_authority::{ENTRIES, Writer, track_writer};
        use pds_repo::data::{self, V};
        let plc = plc().await;
        let dir = scratch("torn-spaces");
        let s = open(&dir, &plc);
        let (alice, ta) = create(&s, "alice").await;
        let (bob, _) = create(&s, "bob").await;
        let log = repo_log(Path::new(""), &alice);
        let spaces = |s: &Service| {
            crate::read(&s.account(&alice).unwrap().store)
                .authority
                .clone()
        };
        let mut steps = vec![(files(&dir), spaces(&s))];
        let snap = |s: &Service, steps: &mut Vec<_>| steps.push((files(&dir), spaces(s)));
        let m = format!("at://{alice}/space/com.example.forum/m");
        let space = pds_space::SpaceRef::parse(&m).unwrap();
        let track = |s: &Service| {
            let w = Writer {
                rev: "3jzfcijpj2z2a".into(),
                data: vec![7; 32],
                space_rev: "3jzfcijpj2z2b".into(),
                removed: false,
            };
            assert!(track_writer(
                &mut crate::write(&s.account(&alice).unwrap().store),
                &space,
                &bob,
                &w
            ));
        };
        let change = |nsid: &'static str, body: Json| {
            let (s, ta) = (s.clone(), ta.clone());
            async move {
                assert_eq!(
                    call(&s, nsid, Some(&ta), body).await,
                    Ok(Json::Null),
                    "{nsid}"
                )
            }
        };
        let policy = json!({ "$type": "com.atproto.memberspace.policy", "publicWrite": true });
        call(
            &s,
            "com.atproto.space.createSpace",
            Some(&ta),
            json!({ "spaceType": "com.example.forum", "skey": "m", "policy": policy }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        change(
            "com.atproto.memberspace.putMembers",
            json!({ "space": m, "members": [{ "did": bob, "read": true, "write": true }] }),
        )
        .await;
        snap(&s, &mut steps);
        track(&s);
        snap(&s, &mut steps);
        change(
            "com.atproto.space.removeWriter",
            json!({ "space": m, "repo": bob }),
        )
        .await;
        snap(&s, &mut steps);
        track(&s);
        snap(&s, &mut steps);
        change(
            "com.atproto.memberspace.removeMembers",
            json!({ "space": m, "dids": [bob] }),
        )
        .await;
        snap(&s, &mut steps);
        change(
            "com.atproto.space.updateSpace",
            json!({ "space": m, "policy": { "$type": "com.atproto.personalspace.policy" } }),
        )
        .await;
        snap(&s, &mut steps);
        change("com.atproto.space.deleteSpace", json!({ "space": m })).await;
        snap(&s, &mut steps);
        let v = |j: Json| data::encode(&data::from_json(&j).unwrap());
        let meta = data::from_json(&json!({ "$type": "atproto-space-meta", "ver": 1, "space": m }))
            .unwrap();
        let writer = data::encode(&V::Map(vec![
            ("repoRev".into(), V::Str("3jzfcijpj2z2a".into())),
            ("data".into(), V::Bytes(vec![1; 32])),
            ("spaceRev".into(), V::Str("3jzfcijpj2z2c".into())),
        ]));
        let (policy, member, writer_key) = (
            v(json!({ "$type": "com.atproto.memberspace.policy" })),
            v(json!({ "$type": "com.atproto.memberspace.member", "read": true, "write": false })),
            format!("writer/{bob}"),
        );
        let member_key = format!("com.atproto.memberspace.member/{bob}");
        let archive = pds_space::star::write(
            &meta,
            [
                (b"policy".as_slice(), policy.as_slice()),
                (member_key.as_bytes(), member.as_slice()),
                (writer_key.as_bytes(), writer.as_slice()),
            ],
        );
        let mut headers = axum::http::HeaderMap::new();
        headers.insert(
            axum::http::header::CONTENT_TYPE,
            axum::http::HeaderValue::from_static(crate::STAR_TYPE),
        );
        let import = crate::Call {
            nsid: "com.atproto.space.importSpaceMeta",
            method: &axum::http::Method::POST,
            params: &[],
            body: &archive,
            bearer: sessions::Bearer::Token(&ta),
            headers: &headers,
            client: "127.0.0.1".into(),
            charged: Default::default(),
            http: None,
        };
        assert!(matches!(
            crate::answer(&s, &import).await,
            Ok(crate::Resp::Nothing)
        ));
        snap(&s, &mut steps);
        drop(s);
        let work = scratch("torn-spaces-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
            let changed = changed_logs(before_files, after_files);
            assert_eq!(changed, [&log]);
            assert_ne!(before, after);
            let whole = &after_files[&log];
            let (from, to) = (before_files[&log].len(), whole.len());
            kinds.extend(
                store::scan(&whole[from..])
                    .unwrap()
                    .0
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = unnoted(after_files.clone());
                files.insert(log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { after } else { before };
                assert_eq!(
                    &spaces(&open(&work, &plc)),
                    want,
                    "cut at {cut} of {from}..{to}"
                );
            }
            for at in (from..to).step_by(5) {
                let mut files = after_files.clone();
                files.get_mut(&log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(
                    &spaces(&open(&work, &plc)),
                    before,
                    "flip at {at} of {from}..{to}"
                );
            }
        }
        kinds.sort();
        kinds.dedup();
        let mut want: Vec<&str> = ENTRIES.to_vec();
        want.sort();
        assert_eq!(kinds.iter().map(String::as_str).collect::<Vec<_>>(), want);
        // After recovery from the last cut, the log takes changes, and replays them.
        put_files(&work, &steps.last().unwrap().0);
        let s = open(&work, &plc);
        assert_eq!(&spaces(&s), &steps.last().unwrap().1);
        call(
            &s,
            "com.atproto.memberspace.putMembers",
            Some(&ta),
            json!({ "space": m, "members": [{ "did": "did:plc:carol", "read": true, "write": false }] }),
        )
        .await
        .unwrap();
        let now = spaces(&s);
        drop(s);
        assert_eq!(spaces(&open(&work, &plc)), now);
        assert_ne!(now, steps.last().unwrap().1);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Cut an account's `repo.log` at every byte of each frame the write
    /// notifications' part wrote (and flip a byte in it): recovery gives
    /// what it keeps (its retry queue, registrations and deletion notices)
    /// as before the frame or after it, and the log takes changes after.
    /// Each kind of entry is cut, and a malformed one is refused.
    #[tokio::test]
    async fn torn_space_notify_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_space_notify_writes_once().await;
        }
    }

    async fn torn_space_notify_writes_once() {
        use crate::space_notify::{ENTRIES, tests as notify};
        let plc = plc().await;
        let dir = scratch("torn-notify");
        let s = open(&dir, &plc);
        let (alice, _) = create(&s, "alice").await;
        let log = repo_log(Path::new(""), &alice);
        let kept = |s: &Service| {
            crate::read(&s.account(&alice).unwrap().store)
                .notify
                .clone()
        };
        let (q, m) = (
            "at://did:plc:elsewhere/space/com.example.forum/q".to_string(),
            format!("at://{alice}/space/com.example.forum/m"),
        );
        let mut steps = vec![(files(&dir), kept(&s))];
        for frame in notify::frames_of_each_kind(&q, &m) {
            notify::keep(&s, &alice, &frame);
            steps.push((files(&dir), kept(&s)));
        }
        drop(s);
        let work = scratch("torn-notify-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
            let changed = changed_logs(before_files, after_files);
            assert_eq!(changed, [&log]);
            assert_ne!(before, after);
            let whole = &after_files[&log];
            let (from, to) = (before_files[&log].len(), whole.len());
            kinds.extend(
                store::scan(&whole[from..])
                    .unwrap()
                    .0
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = unnoted(after_files.clone());
                files.insert(log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { after } else { before };
                assert_eq!(
                    &kept(&open(&work, &plc)),
                    want,
                    "cut at {cut} of {from}..{to}"
                );
            }
            for at in (from..to).step_by(3) {
                let mut files = unnoted(after_files.clone());
                files.get_mut(&log).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(
                    &kept(&open(&work, &plc)),
                    before,
                    "flip at {at} of {from}..{to}"
                );
            }
        }
        kinds.sort();
        kinds.dedup();
        let mut want: Vec<&str> = ENTRIES.to_vec();
        want.sort();
        assert_eq!(kinds.iter().map(String::as_str).collect::<Vec<_>>(), want);
        // After recovery from the last cut, the log takes changes, and replays them.
        let last = &steps.last().unwrap();
        put_files(&work, &last.0);
        let s = open(&work, &plc);
        assert_eq!(&kept(&s), &last.1);
        notify::keep(&s, &alice, &notify::frames_of_each_kind(&q, &m)[0]);
        let now = kept(&s);
        drop(s);
        assert_eq!(kept(&open(&work, &plc)), now);
        assert_ne!(now, last.1);
        // A malformed entry of each kind is refused with a reason, and so is
        // one of a kind this part does not keep, in a log marked as of this
        // version (`checkpoint::unmarked`).
        let space = json!(q);
        let mine = json!(m);
        for bad in [
            json!({ "t": "notifyQueued", "space": space, "rev": "x", "data": "00", "n": 1, "due": 1, "until": 2 }),
            json!({ "t": "notifyQueued", "space": space, "rev": "3jzfcijpj2z2a", "data": "zz", "n": 1, "due": 1, "until": 2 }),
            json!({ "t": "notifyQueued", "space": space, "rev": "3jzfcijpj2z2a", "data": "00", "n": 1, "due": 1 }),
            json!({ "t": "notifyUnqueued", "space": "nope", "rev": "3jzfcijpj2z2a" }),
            json!({ "t": "syncRegistered", "space": mine, "endpoint": "https://s.example", "token": "t" }),
            json!({ "t": "syncRegistered", "space": mine, "endpoint": "https://s.example", "expires": 1 }),
            json!({ "t": "syncDropped", "space": mine, "token": "t" }),
            json!({ "t": "syncNoticed", "space": mine, "endpoint": "https://never.example", "token": "t" }),
            json!({ "t": "syncEmptied", "space": "nope" }),
            json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "other", "n": 0, "due": 1, "until": 2 }),
            json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "expired", "n": 0, "due": 1, "until": 2 }),
            json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "deleted", "n": -1, "due": 1, "until": 2 }),
            json!({ "t": "syncNoticeDone", "space": mine, "endpoint": "https://s.example" }),
            json!({ "t": "notifyRegistered", "space": mine, "service": "did:plc:x", "expires": 1 }),
        ] {
            let mut frames = store::scan(&last.0[&log]).unwrap().0;
            frames.push(vec![bad.clone()]);
            let mut files = last.0.clone();
            files.insert(
                log.clone(),
                frames.iter().flat_map(|f| store::frame(f)).collect(),
            );
            put_files(&work, &files);
            assert!(try_open(&work, &plc).is_err(), "{bad}");
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// An account's blobs, and the files of its blob directory.
    fn blob_state(s: &Service, dir: &Path, did: &str) -> (String, Vec<String>) {
        let held = crate::read(&s.account(did).unwrap().store).blobs.snapshot();
        let mut names: Vec<String> = std::fs::read_dir(blob_dir(dir, did))
            .map(|d| {
                d.map(|e| e.unwrap().file_name().into_string().unwrap())
                    .collect()
            })
            .unwrap_or_default();
        names.sort();
        (held, names)
    }

    /// Cut the account's log at every byte of each blob write: an upload, a
    /// commit referencing it, and one dropping it, with the files as they
    /// are while that frame is on its way. An upload's file is in place
    /// before its entry is written, and a dropped blob's file goes only once
    /// the commit is on disk, so through each write the blob's file is
    /// there. Recovery gives the blobs as before the write or after, and
    /// leaves no file no entry holds: a kill between a commit dropping a blob
    /// and the removal of its file loses no blob still held, and leaks none.
    #[tokio::test]
    async fn torn_blob_writes_recover_to_before_or_after() {
        for sync in SYNCS {
            syncing(sync);
            torn_blob_writes_once().await;
        }
    }

    async fn torn_blob_writes_once() {
        let plc = plc().await;
        let dir = scratch("torn-blob-src");
        let s = open(&dir, &plc);
        let (did, token) = create(&s, "alice").await;
        let log = repo_log(Path::new(""), &did);
        let blob = b"spec-pds blob torn".to_vec();
        let cid = pds_repo::data::cid_string(&pds_repo::data::cid_of(0x55, &blob));
        let snap = |s: &Service| (files(&dir), blob_state(s, &dir, &did));
        let mut steps = vec![snap(&s)];
        crate::blobs::test_upload(&s, Some(&token), Some("text/plain"), blob.clone(), false)
            .await
            .unwrap();
        steps.push(snap(&s));
        let r = json!({ "f": { "$type": "blob", "ref": { "$link": cid }, "mimeType": "text/plain", "size": blob.len() } });
        call(
            &s,
            "com.atproto.repo.putRecord",
            Some(&token),
            json!({ "repo": did, "collection": "a.b.c", "rkey": "a", "record": r }),
        )
        .await
        .unwrap();
        steps.push(snap(&s));
        call(
            &s,
            "com.atproto.repo.deleteRecord",
            Some(&token),
            json!({ "repo": did, "collection": "a.b.c", "rkey": "a" }),
        )
        .await
        .unwrap();
        steps.push(snap(&s));
        drop(s);
        assert_eq!(steps[1].1.1, vec![cid.clone()]);
        assert_eq!(steps[3].1.1, Vec::<String>::new());
        let work = scratch("torn-blob-work");
        let mut cuts = 0;
        for w in steps.windows(2) {
            let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
            let (from, to) = (before_files[&log].len(), after_files[&log].len());
            assert!(to > from);
            let mut files = before_files.clone();
            files.extend(after_files.clone());
            for cut in from..=to {
                files.insert(log.clone(), after_files[&log][..cut].to_vec());
                put_files(&work, &files);
                let s = open(&work, &plc);
                let want = if cut == to { after } else { before };
                assert_eq!(
                    &blob_state(&s, &work, &did),
                    want,
                    "cut at {cut} of {from}..{to}"
                );
                cuts += 1;
            }
        }
        assert!(cuts > 100, "{cuts}");
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Everything a restart must carry over, and each account's space repos.
    fn with_spaces(s: &Service) -> (State, BTreeMap<String, String>) {
        let d = crate::read(&s.directory);
        let spaces = d
            .by_did
            .iter()
            .map(|(did, a)| {
                (
                    did.clone(),
                    format!(
                        "{:?}",
                        crate::space_repo::tests::snapshot(&crate::read(&a.store))
                    ),
                )
            })
            .collect();
        drop(d);
        (state(s), spaces)
    }

    /// Cut the account's log at every byte of each space commit's frame (and
    /// flip a byte in it): recovery gives the space repos as before that
    /// commit or after it, the repository and the events as they were, and
    /// takes new space writes, which replay. Each commit's notification goes
    /// to a space host that cannot be reached: the frames after the commit's
    /// keep it in the retry queue (`space_notify`), which this does not cut.
    #[tokio::test]
    async fn torn_space_writes_recover_to_before_or_after() {
        for sync in SYNCS {
            syncing(sync);
            torn_space_writes("log", false).await;
        }
    }

    /// The same of an account's log after a table that holds space repos:
    /// each commit replayed reads the records it replaces from the table,
    /// and a `clearRecords` and an import void what the table holds.
    #[tokio::test]
    async fn torn_space_writes_after_a_table() {
        for sync in SYNCS {
            syncing(sync);
            torn_space_writes("table", true).await;
        }
    }

    async fn torn_space_writes(name: &str, table: bool) {
        use crate::space_notify::tests::idle;
        use crate::space_repo::tests::{SPACE, w};
        let plc = plc().await;
        let dir = scratch(&format!("torn-space-src-{name}"));
        let s = open(&dir, &plc);
        let (a, ta) = create(&s, "alice").await;
        let (b, tb) = create(&s, "bob").await;
        let snap = |s: &Service| (files(&dir), with_spaces(s));
        let w = |s: &crate::Shared, nsid: &'static str, token: &str, body: Json| {
            let (s, token) = (s.clone(), token.to_string());
            async move {
                let got = w(&s, nsid, &token, body).await;
                idle(&s).await;
                got
            }
        };
        let rec = |did: &str, rkey: &str, n: i64| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
        if table {
            for (did, token, rkey) in [(&a, &ta, "zero"), (&b, &tb, "zero"), (&b, &tb, "one")] {
                w(&s, "putRecord", token, rec(did, rkey, 0)).await.unwrap();
            }
            for did in [&a, &b] {
                let wal = crate::read(&s.account(did).unwrap().store).wal.clone();
                wal.compact().unwrap();
                let layout = crate::store::account_layout(&dir, did);
                assert!(layout.checkpoint().exists());
            }
        }
        let mut steps = vec![snap(&s)];
        w(&s, "createRecord", &ta, rec(&a, "one", 1)).await.unwrap();
        steps.push(snap(&s));
        w(&s, "putRecord", &tb, rec(&b, "one", 2)).await.unwrap();
        steps.push(snap(&s));
        let writes = json!([
            { "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "rkey": "two", "value": record(3) },
            { "$type": "com.atproto.space.applyWrites#update", "collection": "a.b.c", "rkey": "one", "value": record(4) },
            { "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "value": record(5) },
        ]);
        w(
            &s,
            "applyWrites",
            &ta,
            json!({ "space": SPACE, "repo": a, "writes": writes }),
        )
        .await
        .unwrap();
        steps.push(snap(&s));
        w(
            &s,
            "deleteRecord",
            &ta,
            json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "one" }),
        )
        .await
        .unwrap();
        steps.push(snap(&s));
        w(
            &s,
            "clearRecords",
            &tb,
            json!({ "space": SPACE, "repo": b }),
        )
        .await
        .unwrap();
        steps.push(snap(&s));
        // An import of alice's own archive back: one frame, of one entry.
        let params = [
            ("space".to_string(), SPACE.to_string()),
            ("repo".to_string(), a.clone()),
        ];
        let archive = {
            let none = axum::http::HeaderMap::new();
            let get = crate::Call {
                nsid: "com.atproto.space.getRepo",
                method: &axum::http::Method::GET,
                params: &params,
                body: b"",
                bearer: sessions::Bearer::Token(&ta),
                headers: &none,
                client: "127.0.0.1".into(),
                charged: Default::default(),
                http: None,
            };
            let Ok(crate::Resp::Archive(archive, _)) = crate::answer(&s, &get).await else {
                panic!("getRepo answered no archive")
            };
            archive.bytes().unwrap()
        };
        let mut headers = axum::http::HeaderMap::new();
        headers.insert(
            axum::http::header::CONTENT_TYPE,
            axum::http::HeaderValue::from_static("application/vnd.dasl.star"),
        );
        let import = crate::Call {
            nsid: "com.atproto.space.importRepo",
            method: &axum::http::Method::POST,
            params: &[],
            body: &archive,
            bearer: sessions::Bearer::Token(&ta),
            headers: &headers,
            client: "127.0.0.1".into(),
            charged: Default::default(),
            http: None,
        };
        assert!(matches!(
            crate::answer(&s, &import).await,
            Ok(crate::Resp::Nothing)
        ));
        steps.push(snap(&s));
        // A refused write, or one making no commit, logs nothing.
        assert!(w(&s, "createRecord", &ta, rec(&a, "two", 6)).await.is_err());
        w(
            &s,
            "deleteRecord",
            &ta,
            json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "none" }),
        )
        .await
        .unwrap();
        assert_eq!(files(&dir), steps.last().unwrap().0);
        drop(s);
        let work = scratch(&format!("torn-space-work-{name}"));
        let mut cuts = 0;
        for (i, win) in steps.windows(2).enumerate() {
            let ((before_files, before), (after_files, after)) = (&win[0], &win[1]);
            let changed = changed_logs(before_files, after_files);
            let [path] = changed[..] else {
                panic!("step {i} changed {changed:?}")
            };
            assert_ne!(before, after);
            // The repository and the events are as they were.
            assert_eq!(before.0, after.0);
            let whole = &after_files[path];
            let from = before_files[path].len();
            let frames = store::scan(&whole[from..]).unwrap().0;
            let kinds: Vec<&str> = frames[0].iter().map(|e| e["t"].as_str().unwrap()).collect();
            assert!(
                kinds == [crate::space_repo::ENTRY] || kinds == [crate::space_repo::IMPORT],
                "{kinds:?}"
            );
            assert!(
                frames[1..]
                    .iter()
                    .flatten()
                    .all(|e| crate::space_notify::ENTRIES.contains(&e["t"].as_str().unwrap()))
            );
            let to = from + store::frame(&frames[0]).len();
            for cut in from..=to {
                let mut files = unnoted(after_files.clone());
                files.insert(path.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let s = open(&work, &plc);
                let want = if cut == to { after } else { before };
                assert_eq!(
                    &with_spaces(&s),
                    want,
                    "step {i}: cut at {cut} of {from}..{to}"
                );
                cuts += 1;
                if cut % 97 == 0 || cut == from + 1 || cut == to {
                    let got = w(&s, "putRecord", &tb, rec(&b, "after", 9)).await.unwrap();
                    assert!(got["cid"].is_string());
                    idle(&s).await;
                    assert_eq!(
                        with_spaces(&open(&work, &plc)),
                        with_spaces(&s),
                        "step {i}: cut at {cut}"
                    );
                }
            }
            for at in (from..to).step_by(7) {
                let mut files = unnoted(after_files.clone());
                files.get_mut(path).unwrap()[at] ^= 0x55;
                put_files(&work, &files);
                assert_eq!(
                    &with_spaces(&open(&work, &plc)),
                    before,
                    "step {i}: flip at {at} of {from}..{to}"
                );
            }
        }
        assert!(cuts > 500, "{cuts}");
        // A space commit whose records do not make its data is refused.
        let alice = changed_logs(&steps[0].0, &steps[1].0)[0].clone();
        let last = &steps.last().unwrap().0;
        let mut frames = store::scan(&last[&alice]).unwrap().0;
        let entry = frames
            .iter_mut()
            .flatten()
            .find(|e| {
                e["t"] == crate::space_repo::ENTRY && !e["puts"].as_array().unwrap().is_empty()
            })
            .unwrap();
        entry["puts"][0][1]["n"] = json!(99);
        let mut files = last.clone();
        files.insert(alice, frames.iter().flat_map(|f| store::frame(f)).collect());
        put_files(&work, &files);
        assert!(
            try_open(&work, &plc)
                .err()
                .unwrap()
                .contains("do not make the data")
        );
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// An account whose entry is cut short is not there; one whose entry is
    /// whole but whose repository's first frame is not starts with a fresh
    /// first commit, once.
    #[tokio::test]
    async fn torn_account_creation() {
        for sync in SYNCS {
            syncing(sync);
            torn_account_creation_once().await;
        }
    }

    async fn torn_account_creation_once() {
        let plc = plc().await;
        let dir = scratch("torn-create");
        let (steps, accounts) = script(&dir, &plc).await;
        let (before, after) = (&steps[1], &steps[2]);
        let bob = &accounts[1].0;
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let bob_log = repo_log(Path::new(""), bob);
        let work = scratch("torn-create-work");
        let (from, to) = (
            before.files[&accounts_log].len(),
            after.files[&accounts_log].len(),
        );
        for cut in from..to {
            let mut files = before.files.clone();
            files.insert(
                accounts_log.clone(),
                after.files[&accounts_log][..cut].to_vec(),
            );
            put_files(&work, &files);
            assert_eq!(
                state(&open(&work, &plc)),
                before.state,
                "accounts.log cut at {cut}"
            );
        }
        for cut in 0..after.files[&bob_log].len() {
            let mut files = after.files.clone();
            files.insert(bob_log.clone(), after.files[&bob_log][..cut].to_vec());
            put_files(&work, &files);
            let s = open(&work, &plc);
            assert!(repo_of(&s, bob, |r| r.record_count() == 0 && !r.rev.is_empty()));
            // Sent: alice's four events as before, then the four of bob's
            // fresh first commit, numbered after every seq given out: the
            // index keeps the entries of his first ones, whose frames were
            // cut, and a replay passes over them.
            let got = sent(&s);
            let seq = |e: &Json| e["seq"].as_u64().unwrap();
            let first: Vec<u64> = after.state.2.iter().map(|f| seq(&body_of(f))).collect();
            assert_eq!(got.len(), 8, "cut at {cut}: {got:?}");
            assert_eq!(got[..4].iter().map(seq).collect::<Vec<_>>(), first[..4]);
            // A `#commit` names its account as `repo`, the others as `did`.
            let of = |e: &Json| e.get("did").or_else(|| e.get("repo")).cloned();
            assert!(
                got[..4]
                    .iter()
                    .all(|e| of(e).is_some_and(|d| d != json!(bob)))
            );
            assert!(
                got[4..].iter().all(|e| of(e) == Some(json!(bob))),
                "cut at {cut}: {got:?}"
            );
            assert!(
                got[4..].iter().map(seq).all(|q| q > first[7]),
                "cut at {cut}: {got:?}"
            );
            assert!(got.windows(2).all(|w| seq(&w[0]) < seq(&w[1])));
            // Its KEM key is the one its whole entry has.
            let kem = state(&s).0[bob.as_str()].2.clone();
            assert!(kem.is_some());
            assert_eq!(kem, after.state.0[bob.as_str()].2);
            if cut % 61 == 0 {
                let now = state(&s);
                drop(s);
                assert_eq!(state(&open(&work, &plc)), now);
            }
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// A handle change writes two frames to its account's log: the change
    /// recorded, then settled with its `#identity` event. Cut at every byte
    /// of them (or with a byte of the second flipped), recovery gives the
    /// account before the change, with it recorded and not settled, or with
    /// it landed; and one recorded is settled, landing whole, once the
    /// server serves.
    #[tokio::test]
    async fn torn_handle_changes() {
        for sync in SYNCS {
            syncing(sync);
            torn_handle_changes_once().await;
        }
    }

    async fn torn_handle_changes_once() {
        let plc = crate::identity::stub::Plc::start().await;
        let dir = scratch("torn-handle");
        let s = open(&dir, &plc.url);
        let (alice, ta) = create(&s, "alice").await;
        create(&s, "bob").await;
        let before = files(&dir);
        let body = json!({ "handle": "carol.test" });
        call(&s, "com.atproto.identity.updateHandle", Some(&ta), body)
            .await
            .unwrap();
        let after = files(&dir);
        let after_state = state(&s);
        drop(s);
        let path = repo_log(Path::new(""), &alice);
        let changed = changed_logs(&before, &after);
        assert_eq!(changed, [&path]);
        // The settled change's entry is in the index.
        assert!(
            after
                .iter()
                .any(|(p, b)| of_index(p) && before.get(p) != Some(b))
        );
        let (from, to) = (before[&path].len(), after[&path].len());
        let frames = store::scan(&after[&path]).unwrap().0;
        let recorded = from + store::frame(&frames[frames.len() - 2]).len();
        // What recovery gives: alice's handle, the handles held, and how
        // many events a subscription from cursor 0 is sent.
        let identity = |s: &Service| {
            let d = crate::read(&s.directory);
            let mut held: Vec<String> = d.claimed_handles.iter().cloned().collect();
            held.sort();
            let handle = d.by_did[&alice].handle();
            drop(d);
            (handle, held, sent(s).len())
        };
        let n = after_state.2.len();
        let work = scratch("torn-handle-work");
        let mut unsettled = 0;
        let check = |files: BTreeMap<PathBuf, Vec<u8>>, at: String, cut: usize| {
            put_files(&work, &files);
            let s = open(&work, &plc.url);
            // Repositories and sessions are as they were throughout.
            let got = state(&s);
            assert_eq!((&got.0, &got.1), (&after_state.0, &after_state.1), "{at}");
            let want = if cut < recorded {
                ("alice.test".to_string(), vec![], n - 1)
            } else if cut < to {
                (
                    "alice.test".to_string(),
                    vec!["carol.test".to_string()],
                    n - 1,
                )
            } else {
                ("carol.test".to_string(), vec![], n)
            };
            assert_eq!(identity(&s), want, "{at}");
            if want.1.is_empty() {
                assert_eq!(state(&open(&work, &plc.url)), got, "{at}");
            }
            s
        };
        for cut in from..=to {
            let mut files = unnoted(after.clone());
            files.insert(path.clone(), after[&path][..cut].to_vec());
            let s = check(files, format!("cut at {cut} of {from}..{to}"), cut);
            // One recorded and not settled lands whole once the server serves.
            if (recorded..to).contains(&cut) && (cut == recorded || cut % 23 == 0) {
                crate::identity::settle_unfinished(&s);
                for _ in 0..100 {
                    if identity(&s).0 == "carol.test" {
                        break;
                    }
                    tokio::time::sleep(std::time::Duration::from_millis(20)).await;
                }
                assert_eq!(
                    identity(&s),
                    ("carol.test".to_string(), vec![], n),
                    "settling after a cut at {cut}"
                );
                drop(s);
                let s = open(&work, &plc.url);
                assert_eq!(
                    identity(&s),
                    ("carol.test".to_string(), vec![], n),
                    "after settling a cut at {cut}"
                );
                unsettled += 1;
            }
        }
        for at in (recorded..to).step_by(3) {
            let mut files = unnoted(after.clone());
            files.get_mut(&path).unwrap()[at] ^= 0x55;
            check(files, format!("flip at {at} of {recorded}..{to}"), recorded);
        }
        assert!(unsettled > 3, "{unsettled}");
        assert_eq!(plc.log(&alice).len(), 2);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// An event that never reached the disk, while a later one of another
    /// account's did: a restart leaves its number unused, the same way every
    /// time, and the next event is numbered after every one given out.
    #[tokio::test]
    async fn lost_events_leave_their_numbers_unused() {
        for sync in SYNCS {
            syncing(sync);
            lost_events_once().await;
        }
    }

    async fn lost_events_once() {
        let plc = plc().await;
        let dir = scratch("gap");
        let (steps, accounts) = script(&dir, &plc).await;
        let (a, ta) = &accounts[0];
        // Drop alice's last frame, her deleteRecord, and put bob's next commit after it.
        let s = open(&dir, &plc);
        let n = sent(&s).len() as u64;
        let body = json!({ "repo": accounts[1].0, "collection": "a.b.c", "rkey": "late", "record": { "n": 1 } });
        call(&s, "com.atproto.repo.putRecord", Some(&accounts[1].1), body)
            .await
            .unwrap();
        drop(s);
        let mut files = files(&dir);
        let alice = repo_log(Path::new(""), a);
        let cut = steps[steps.len() - 3].files[&alice].len();
        files.get_mut(&alice).unwrap().truncate(cut);
        let work = scratch("gap-work");
        put_files(&work, &files);
        let s = open(&work, &plc);
        assert_eq!(sent(&s).len() as u64, n);
        let seqs = sent_seqs;
        let got = seqs(&s);
        let lost: Vec<u64> = (1..=n + 1).filter(|i| !got.contains(i)).collect();
        assert_eq!(lost.len(), 1, "{got:?}");
        assert_eq!(*s.events.subscribe().borrow(), n + 1);
        let first = state(&s);
        drop(s);
        let s = open(&work, &plc);
        assert_eq!(state(&s), first);
        // A new event follows on, and every restart after numbers them alike.
        let body = json!({ "repo": a, "collection": "a.b.c", "rkey": "new", "record": { "n": 1 } });
        call(&s, "com.atproto.repo.putRecord", Some(ta), body)
            .await
            .unwrap();
        assert_eq!(seqs(&s).last(), Some(&(n + 2)));
        let want = state(&s);
        drop(s);
        assert_eq!(state(&open(&work, &plc)), want);
        assert_eq!(state(&open(&work, &plc)), want);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// The seal that closes a log.
    fn seal() -> Vec<u8> {
        store::frame(&[json!({ "t": "seal" })])
    }

    /// A compaction killed at each step of `spec/STORE.md`'s crash table,
    /// with the writes of every file it had not made durable lost: an
    /// account's log, then the service log, each with a write after its seal.
    /// The account's table and its log hold space repos, which the fold
    /// merges with what the log voids of them. Every start recovers the
    /// state before the compaction, or with that write too where all of it
    /// is in the new log.
    #[tokio::test]
    async fn compaction_killed_at_every_step() {
        for sync in SYNCS {
            syncing(sync);
            compaction_killed_at_every_step_once().await;
        }
    }

    async fn compaction_killed_at_every_step_once() {
        let plc = plc().await;
        let dir = scratch("compact-killed");
        let (_, accounts) = script(&dir, &plc).await;
        let (a, ta) = &accounts[0];
        // The account's space repos: in its table, and in its log after it,
        // which writes over the table's records, deletes one and clears a repo.
        {
            use crate::space_notify::tests::idle;
            use crate::space_repo::tests::{SPACE, w};
            let other = "at://did:web:example.com/space/com.example.chat/room";
            let s = open(&dir, &plc);
            let rec = |space: &str, rkey: &str, n: i64| json!({ "space": space, "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
            for (space, rkey, n) in [(SPACE, "one", 1), (SPACE, "two", 2), (other, "one", 3)] {
                w(&s, "putRecord", ta, rec(space, rkey, n)).await.unwrap();
            }
            idle(&s).await;
            let wal = crate::read(&s.account(a).unwrap().store).wal.clone();
            wal.compact().unwrap();
            drop(wal);
            w(&s, "putRecord", ta, rec(SPACE, "two", 22)).await.unwrap();
            w(&s, "putRecord", ta, rec(SPACE, "three", 3))
                .await
                .unwrap();
            let gone = json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "one" });
            w(&s, "deleteRecord", ta, gone).await.unwrap();
            w(&s, "clearRecords", ta, json!({ "space": other, "repo": a }))
                .await
                .unwrap();
            w(&s, "putRecord", ta, rec(other, "after", 4))
                .await
                .unwrap();
            idle(&s).await;
        }
        let work = scratch("compact-killed-work");
        let state = with_spaces;
        for which in ["account", "service"] {
            let s = open(&dir, &plc);
            let before = (files(&dir), state(&s));
            let (log, layout) = match which {
                "account" => (
                    crate::read(&s.account(a).unwrap().store).wal.clone(),
                    crate::store::account_layout(Path::new(""), a),
                ),
                _ => (s.log.clone(), crate::store::service_layout(Path::new(""))),
            };
            let g = log.generation();
            log.compact().unwrap();
            assert_eq!(log.generation(), g + 1);
            let compacted = files(&dir);
            match which {
                "account" => {
                    let body = json!({ "repo": a, "collection": "a.b.c", "rkey": format!("after-{g}"), "record": record(9) });
                    call(&s, "com.atproto.repo.putRecord", Some(ta), body)
                        .await
                        .unwrap();
                }
                _ => drop(session(&s, "alice").await),
            }
            let after = (files(&dir), state(&s));
            drop(s);
            let (old, new, ckpt) = (layout.log(g), layout.log(g + 1), layout.checkpoint());
            assert!(!compacted.contains_key(&old) && compacted.contains_key(&ckpt));
            let mut sealed = before.0[&old].clone();
            sealed.extend(seal());
            let written = after.0[&new].clone();
            let check = |files: BTreeMap<PathBuf, Vec<u8>>, why: String, whole: bool| {
                put_files(&work, &files);
                let s = open(&work, &plc);
                let got = state(&s);
                let want = if whole { &after.1 } else { &before.1 };
                assert!(got == *want, "{which}: {why}");
                // A start makes it whole again: the same state once more.
                drop(s);
                assert!(
                    state(&open(&work, &plc)) == *want,
                    "{which}: {why}, started twice"
                );
            };
            // The files but the log's, as after the write.
            let base: BTreeMap<PathBuf, Vec<u8>> = after
                .0
                .iter()
                .filter(|(p, _)| ![&old, &new, &ckpt].contains(p))
                .map(|(p, b)| (p.clone(), b.clone()))
                .collect();
            let with = |logs: &[(&PathBuf, &[u8])]| {
                let mut f = base.clone();
                // The table before, if it had one.
                if let Some(table) = before.0.get(&ckpt) {
                    f.insert(ckpt.clone(), table.clone());
                }
                for (p, b) in logs {
                    f.insert((*p).clone(), b.to_vec());
                }
                f
            };
            // In step 1: the seal cut at every byte, log g + 1 not made.
            for cut in 0..seal().len() {
                check(
                    with(&[(&old, &sealed[..before.0[&old].len() + cut])]),
                    format!("the seal cut at {cut}"),
                    false,
                );
            }
            // Sealed, log g + 1 not made, or made: in step 2, as in step 1.
            check(
                with(&[(&old, &sealed)]),
                "sealed, no next log".into(),
                false,
            );
            for cut in 0..=written.len() {
                let whole = cut == written.len();
                check(
                    with(&[(&old, &sealed), (&new, &written[..cut])]),
                    format!("the next log cut at {cut}"),
                    whole,
                );
                // In step 3: the checkpoint's `.tmp` cut short, at some lengths.
                if cut == 0 || whole {
                    let tmp = layout.tmp();
                    let bytes = &compacted[&ckpt];
                    for at in (0..bytes.len())
                        .step_by((bytes.len() / 7).max(1))
                        .chain([bytes.len()])
                    {
                        let mut f = with(&[(&old, &sealed), (&new, &written[..cut])]);
                        f.insert(tmp.clone(), bytes[..at].to_vec());
                        check(
                            f,
                            format!("the .tmp cut at {at}, the next log at {cut}"),
                            whole,
                        );
                    }
                    // In step 4 after the rename, and in step 5 with the old log gone or not.
                    let mut f = with(&[(&old, &sealed), (&new, &written[..cut])]);
                    f.insert(ckpt.clone(), bytes.clone());
                    check(f.clone(), format!("renamed, the next log at {cut}"), whole);
                    f.remove(&old);
                    check(
                        f,
                        format!("the old log deleted, the next log at {cut}"),
                        whole,
                    );
                }
            }
            // A log missing between the checkpoint's and the last is data not recovered.
            let mut f = with(&[(&new, &written)]);
            f.insert(ckpt.clone(), compacted[&ckpt].clone());
            f.insert(layout.log(g + 3), Vec::new());
            put_files(&work, &f);
            assert!(
                Service::open(&work, config(&plc)).is_err(),
                "{which}: a gap"
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Everything a restart must carry over of the accounts, with each
    /// one's space repos, the spaces it is the authority of, and its write
    /// notifications.
    fn with_all(s: &Service) -> (State, BTreeMap<String, String>, BTreeMap<String, String>) {
        let kept = crate::read(&s.directory)
            .by_did
            .iter()
            .map(|(did, a)| {
                let st = crate::read(&a.store);
                (did.clone(), format!("{:?}", (&st.authority, &st.notify)))
            })
            .collect();
        let (state, spaces) = with_spaces(s);
        (state, spaces, kept)
    }

    /// Account tables of the version before, and logs after them as that
    /// version wrote them: the first load of each folds it into a table of
    /// this version, once, and serves all of it, the same wherever the fold
    /// was cut short, when the server is started again, and when each
    /// account is loaded as it is first asked for; and takes writes after.
    /// Of spaces, the state in their `S` and in the logs is let go, with
    /// the blobs only space records held (`checkpoint::unmarked`), and
    /// spaces are written after as ever.
    #[tokio::test]
    async fn a_table_of_the_version_before_is_folded_once() {
        use crate::space_notify::tests::idle;
        use crate::space_repo::tests::{SPACE, w};
        let plc = plc().await;
        let dir = scratch("before-src");
        let s = open(&dir, &plc);
        let (a, ta) = create(&s, "alice").await;
        let (b, tb) = create(&s, "bob").await;
        // An account no start loads: nothing of its is waiting to be sent.
        let (c, tc) = create(&s, "carol").await;
        let (one, two) = (
            b"spec-pds before one".to_vec(),
            b"spec-pds before two".to_vec(),
        );
        let mut refs = Vec::new();
        for body in [&one, &two] {
            let up =
                crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), body.clone(), false)
                    .await
                    .unwrap();
            refs.push(up["blob"].clone());
        }
        let made = call(&s, "com.atproto.space.createSpace", Some(&ta), json!({ "spaceType": "com.example.forum", "skey": "mine", "policy": { "$type": crate::space_authority::MEMBER, "publishPolicy": true, "publishMembers": true } }))
            .await
            .unwrap();
        let mine = made["space"].as_str().unwrap().to_string();
        let member = |did: &str, write: bool| json!({ "space": mine, "members": [{ "did": did, "read": true, "write": write }] });
        call(
            &s,
            "com.atproto.memberspace.putMembers",
            Some(&ta),
            member(&b, true),
        )
        .await
        .unwrap();
        let rec = |space: &str, did: &str, rkey: &str, record: Json| json!({ "space": space, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record });
        let public = |rkey: &str, record: Json| json!({ "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record });
        call(
            &s,
            "com.atproto.repo.putRecord",
            Some(&ta),
            public("pic", json!({ "img": refs[0] })),
        )
        .await
        .unwrap();
        for (space, rkey, record) in [
            (SPACE, "one", record(1)),
            (SPACE, "two", json!({ "img": refs[1] })),
            (mine.as_str(), "one", record(3)),
            (mine.as_str(), "two", record(4)),
        ] {
            w(&s, "putRecord", &ta, rec(space, &a, rkey, record))
                .await
                .unwrap();
        }
        w(
            &s,
            "deleteRecord",
            &ta,
            json!({ "space": mine, "repo": a, "collection": "a.b.c", "rkey": "two" }),
        )
        .await
        .unwrap();
        for rkey in ["one", "zero"] {
            w(&s, "putRecord", &tb, rec(SPACE, &b, rkey, record(5)))
                .await
                .unwrap();
        }
        let carols =
            json!({ "repo": c, "collection": "a.b.c", "rkey": "one", "record": record(1) });
        call(&s, "com.atproto.repo.putRecord", Some(&tc), carols)
            .await
            .unwrap();
        idle(&s).await;
        // The tables, then the logs after them.
        for did in [&a, &b, &c] {
            let wal = crate::read(&s.account(did).unwrap().store).wal.clone();
            wal.compact().unwrap();
        }
        w(&s, "putRecord", &ta, rec(SPACE, &a, "one", record(11)))
            .await
            .unwrap();
        w(
            &s,
            "putRecord",
            &ta,
            rec(SPACE, &a, "three", json!({ "img": refs[1] })),
        )
        .await
        .unwrap();
        w(
            &s,
            "deleteRecord",
            &ta,
            json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "two" }),
        )
        .await
        .unwrap();
        w(&s, "putRecord", &ta, rec(&mine, &a, "five", record(5)))
            .await
            .unwrap();
        call(
            &s,
            "com.atproto.memberspace.putMembers",
            Some(&ta),
            member("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", false),
        )
        .await
        .unwrap();
        call(
            &s,
            "com.atproto.repo.putRecord",
            Some(&ta),
            public("after", record(2)),
        )
        .await
        .unwrap();
        w(&s, "putRecord", &tb, rec(SPACE, &b, "one", record(6)))
            .await
            .unwrap();
        w(
            &s,
            "clearRecords",
            &tb,
            json!({ "space": SPACE, "repo": b }),
        )
        .await
        .unwrap();
        w(&s, "putRecord", &tb, rec(SPACE, &b, "after", record(7)))
            .await
            .unwrap();
        idle(&s).await;
        let want = with_all(&s);
        let cid = |r: &Json| r["ref"]["$link"].as_str().unwrap().to_string();
        let (one, two) = (cid(&refs[0]), cid(&refs[1]));
        assert!(want.0.5.0[&a].2.contains(&two));
        let g = crate::read(&s.account(&a).unwrap().store).wal.generation();
        drop(s);
        // The tables as the version before wrote them; and in the logs, a
        // `clearRecords` that names each record it deletes, as spaces once
        // logged it.
        for did in [&a, &b, &c] {
            let layout = crate::store::account_layout(&dir, did);
            crate::checkpoint::downgrade(&layout.checkpoint());
            let log = layout.log(g);
            let mut frames = store::scan(&std::fs::read(&log).unwrap()).unwrap().0;
            for e in frames.iter_mut().flatten() {
                if e.get("kept") == Some(&json!([])) {
                    e.as_object_mut().unwrap().remove("kept");
                    e["dels"] = json!(["a.b.c/one", "a.b.c/zero"]);
                }
            }
            let bytes: Vec<u8> = frames.iter().flat_map(|f| store::frame(f)).collect();
            std::fs::write(&log, bytes).unwrap();
        }
        let before = files(&dir);
        let layout = crate::store::account_layout(Path::new(""), &a);
        let (old, new, ckpt) = (layout.log(g), layout.log(g + 1), layout.checkpoint());
        assert!(!store::scan(&before[&old]).unwrap().0.is_empty());
        // Folded on its first load, and served.
        let work = scratch("before-work");
        put_files(&work, &before);
        let s = open(&work, &plc);
        // Each account's state as it was but for its blobs; no spaces state;
        // and of alice's blobs, the one a public record references alone.
        let public = |mut state: State| {
            for (_, _, blobs) in state.5.0.values_mut() {
                blobs.clear();
            }
            state
        };
        let served = with_all(&s);
        assert!(public(served.0.clone()) == public(want.0.clone()), "folded");
        for (did, spaces) in &served.1 {
            assert_eq!(spaces, "[]", "{did}");
        }
        let none = format!(
            "{:?}",
            (
                &crate::space_authority::Authority::default(),
                &crate::space_notify::Kept::default()
            )
        );
        assert!(served.2.values().all(|k| *k == none));
        {
            let account = s.account(&a).unwrap();
            let st = crate::read(&account.store);
            let held: Vec<&str> = st.blobs.held_list().map(|(cid, _, _)| cid).collect();
            assert_eq!(held, [one.as_str()]);
            assert!(st.blobs.space_ref_list().is_empty());
        }
        assert!(!blob_dir(&work, &a).join(&two).exists());
        assert!(blob_dir(&work, &a).join(&one).exists());
        drop(s);
        let folded = files(&work);
        assert!(!folded.contains_key(&old) && folded.contains_key(&new));
        assert!(folded[&ckpt] != before[&ckpt]);
        // Once: a start after it reads the table it made, and writes go on.
        let s = open(&work, &plc);
        assert!(with_all(&s) == served, "started again");
        assert!(files(&work)[&ckpt] == folded[&ckpt]);
        w(&s, "putRecord", &ta, rec(SPACE, &a, "later", record(8)))
            .await
            .unwrap();
        w(&s, "putRecord", &tb, rec(SPACE, &b, "zero", record(9)))
            .await
            .unwrap();
        idle(&s).await;
        let later = with_all(&s);
        assert!(later != served);
        drop(s);
        assert!(with_all(&open(&work, &plc)) == later, "written after");
        // An account not loaded at the start: its table has no header to
        // find its events by, which the firehose's index has; it is loaded
        // as first asked for, and folded then.
        put_files(&work, &before);
        let s = open_with(&work, &plc, |c| c.load_all = false);
        assert!(!s.account(&c).is_some_and(|a| a.store.is_loaded()));
        assert!(crate::checkpoint::events_on_disk(&s.ctx, &c).is_err());
        s.events.find_old();
        assert!(!s.account(&c).is_some_and(|a| a.store.is_loaded()));
        for did in [&a, &b, &c] {
            assert!(s.by_did(did).is_some());
        }
        assert!(with_all(&s) == served, "loaded as first asked for");
        drop(s);
        // The fold cut at every step: the seal at every byte, the next log
        // made or not, the table written in part, put in place, and the log
        // it covers deleted or not.
        let check = |files: BTreeMap<PathBuf, Vec<u8>>, why: String| {
            put_files(&work, &files);
            let s = open(&work, &plc);
            assert!(with_all(&s) == served, "{why}");
            drop(s);
            assert!(
                with_all(&open(&work, &plc)) == served,
                "{why}, started twice"
            );
        };
        let with = |logs: &[(&PathBuf, &[u8])]| {
            let mut f = before.clone();
            for (p, b) in logs {
                f.insert((*p).clone(), b.to_vec());
            }
            f
        };
        let mut sealed = before[&old].clone();
        sealed.extend(seal());
        for cut in 0..seal().len() {
            check(
                with(&[(&old, &sealed[..before[&old].len() + cut])]),
                format!("the seal cut at {cut}"),
            );
        }
        check(with(&[(&old, &sealed)]), "sealed, no next log".into());
        check(
            with(&[(&old, &sealed), (&new, &[])]),
            "sealed, the next log made".into(),
        );
        let bytes = &folded[&ckpt];
        for at in (0..bytes.len())
            .step_by((bytes.len() / 9).max(1))
            .chain([bytes.len()])
        {
            let mut f = with(&[(&old, &sealed), (&new, &[])]);
            f.insert(layout.tmp(), bytes[..at].to_vec());
            check(f, format!("the .tmp cut at {at}"));
        }
        let mut f = with(&[(&old, &sealed), (&new, &[])]);
        f.insert(ckpt.clone(), bytes.clone());
        check(f.clone(), "put in place".into());
        f.remove(&old);
        check(f, "the log it covers deleted".into());
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// An account's log with no mark, as an earlier version left it, sealed
    /// with a log after it: its spaces entries, of the kinds this server
    /// logs and of others, are let go as its store loads, and its other
    /// entries replayed as they were; a blob only its space records
    /// referenced is no longer held. The mark is appended once, and the
    /// account takes spaces writes after, which replay, and fold.
    #[tokio::test]
    async fn spaces_before_the_mark_are_let_go() {
        use crate::space_notify::tests::idle;
        use crate::space_repo::tests::{SPACE, r, w};
        let plc = plc().await;
        let dir = scratch("unmarked-src");
        let s = open(&dir, &plc);
        let (a, ta) = create(&s, "alice").await;
        let mut refs = Vec::new();
        for body in [b"unmarked one".to_vec(), b"unmarked two".to_vec()] {
            let up = crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), body, false)
                .await
                .unwrap();
            refs.push(up["blob"].clone());
        }
        let cid = |r: &Json| r["ref"]["$link"].as_str().unwrap().to_string();
        let (one, two) = (cid(&refs[0]), cid(&refs[1]));
        call(
            &s,
            "com.atproto.repo.putRecord",
            Some(&ta),
            json!({ "repo": a, "collection": "a.b.c", "rkey": "pic", "record": { "img": refs[0] } }),
        )
        .await
        .unwrap();
        let space = |s: &crate::Shared| {
            let (s, ta) = (s.clone(), ta.clone());
            let body = json!({ "spaceType": "com.example.forum", "skey": "mine", "policy": { "$type": crate::space_authority::MEMBER, "publishMembers": true } });
            async move {
                call(&s, "com.atproto.space.createSpace", Some(&ta), body)
                    .await
                    .unwrap()["space"]
                    .as_str()
                    .unwrap()
                    .to_string()
            }
        };
        let mine = space(&s).await;
        call(
            &s,
            "com.atproto.memberspace.putMembers",
            Some(&ta),
            json!({ "space": mine, "members": [{ "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", "read": true, "write": false }] }),
        )
        .await
        .unwrap();
        let rec = |space: &str, rkey: &str, record: Json| json!({ "space": space, "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record });
        for (space, rkey, record) in [
            (SPACE, "one", json!({ "img": refs[1] })),
            (SPACE, "two", json!({ "img": refs[0] })),
            (mine.as_str(), "x", record(1)),
        ] {
            w(&s, "putRecord", &ta, rec(space, rkey, record))
                .await
                .unwrap();
        }
        idle(&s).await;
        let want = with_all(&s);
        assert!(want.0.5.0[&a].2.contains(&two) && want.1[&a] != "[]");
        let g = crate::read(&s.account(&a).unwrap().store).wal.generation();
        drop(s);
        // As an earlier version left it: no mark, and entries of kinds this
        // server does not log; sealed, with a log after it.
        let layout = crate::store::account_layout(Path::new(""), &a);
        let (log, next) = (layout.log(g), layout.log(g + 1));
        let mut before = files(&dir);
        let mark = |f: &Vec<Json>| f.iter().any(|e| e["t"] == "spaces");
        let mut frames = store::scan(&before[&log]).unwrap().0;
        assert_eq!(frames.iter().filter(|f| mark(f)).count(), 1);
        frames.retain(|f| !mark(f));
        let mut bytes: Vec<u8> = frames.iter().flat_map(|f| store::frame(f)).collect();
        bytes.extend(seal());
        before.insert(log.clone(), bytes);
        let old = [
            json!({ "t": "notifyRegistered", "space": mine, "service": "did:plc:x#s", "expires": 1 }),
            json!({ "t": "spaceUnwrite", "space": mine, "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" }),
        ];
        before.insert(
            next.clone(),
            old.iter()
                .flat_map(|e| store::frame(std::slice::from_ref(e)))
                .collect(),
        );
        let work = scratch("unmarked-work");
        put_files(&work, &before);
        let s = open(&work, &plc);
        let public = |mut state: State| {
            for (_, _, blobs) in state.5.0.values_mut() {
                blobs.clear();
            }
            state
        };
        let served = with_all(&s);
        assert!(public(served.0.clone()) == public(want.0.clone()));
        assert_eq!(served.1[&a], "[]");
        let none = format!(
            "{:?}",
            (
                &crate::space_authority::Authority::default(),
                &crate::space_notify::Kept::default()
            )
        );
        assert_eq!(served.2[&a], none);
        {
            let account = s.account(&a).unwrap();
            let st = crate::read(&account.store);
            let held: Vec<&str> = st.blobs.held_list().map(|(cid, _, _)| cid).collect();
            assert_eq!(held, [one.as_str()]);
        }
        assert!(!blob_dir(&work, &a).join(&two).exists());
        assert_eq!(
            r(&s, "listSpaces", &ta, &[]).await,
            Ok(json!({ "spaces": [] }))
        );
        assert_eq!(
            r(&s, "getSpace", &ta, &[("space", mine.as_str())]).await,
            Err("SpaceNotFound".into())
        );
        drop(s);
        // Marked once: started again, it is as it was.
        let marks = |work: &Path| {
            let files = files(work);
            files
                .iter()
                .filter(|(p, _)| p.starts_with(&layout.dir))
                .flat_map(|(_, b)| store::scan(b).map(|f| f.0).unwrap_or_default())
                .filter(|f| mark(f))
                .count()
        };
        assert_eq!(marks(&work), 1);
        let s = open(&work, &plc);
        assert!(with_all(&s) == served);
        // Spaces writes go on, and replay, and fold.
        assert_eq!(space(&s).await, mine);
        w(
            &s,
            "putRecord",
            &ta,
            rec(SPACE, "one", json!({ "img": refs[0] })),
        )
        .await
        .unwrap();
        idle(&s).await;
        let later = with_all(&s);
        assert!(later.1[&a] != "[]" && later != served);
        drop(s);
        assert_eq!(marks(&work), 1);
        let s = open(&work, &plc);
        assert!(with_all(&s) == later, "written after");
        let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
        wal.compact().unwrap();
        drop(wal);
        drop(s);
        let s = open(&work, &plc);
        assert!(with_all(&s) == later, "folded");
        drop(s);
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Make the table at `path` one of the table format's version before
    /// (`pds_store::table::Version::Sha256`), holding the same entries.
    fn as_the_version_before(path: &Path) {
        use store::table::{BLOCK, BlockCache, Table, TableWriter, Version};
        let cache = BlockCache::new(1 << 20);
        let t = Table::open(path, &cache).unwrap();
        assert_eq!(t.version(), Version::Xxh3);
        let mut w = TableWriter::of_version(Vec::new(), t.len(), BLOCK, Version::Sha256);
        for e in t.iter(b"", None, &cache) {
            let (k, v) = e.unwrap();
            w.add(&k, &v).unwrap();
        }
        std::fs::write(path, w.finish().unwrap().0).unwrap();
    }

    /// The version of the table format the table at `path` is of.
    fn table_version(path: &Path) -> store::table::Version {
        let cache = store::table::BlockCache::new(0);
        store::table::Table::open(path, &cache).unwrap().version()
    }

    /// Tables of the table format's version before, whose checks and bloom
    /// filters are SHA-256's, among tables of this one: two accounts' and
    /// the service's, a third account's as it is written now, each with a
    /// log after it. A start serves all of it as it was, loading every
    /// account or each as it is first asked for, and finds an account's
    /// events in its files, its store not loaded, as in a table of this
    /// version. The accounts take writes, and a compaction folds a table
    /// of the version before, with its log, into one of this version
    /// holding the same: an account's, with its space repo, and the
    /// service's, beside an account's still of the version before. With a
    /// byte changed, a table of either version is refused when it is
    /// opened, one of the version before that says it has more entries
    /// than it could hold too, or the block changed is not served.
    #[tokio::test]
    async fn tables_of_the_version_before_are_served_and_folded() {
        use crate::space_notify::tests::idle;
        use crate::space_repo::tests::{SPACE, w};
        use store::table::Version;
        const PUT: &str = "com.atproto.repo.putRecord";
        const DELETE: &str = "com.atproto.repo.deleteRecord";
        let plc = plc().await;
        let dir = scratch("checks-src");
        let s = open(&dir, &plc);
        let mut accounts = Vec::new();
        for name in ["alice", "bob", "carol"] {
            accounts.push(create(&s, name).await);
        }
        let put = |did: &str, rkey: &str, n: i64| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
        let del =
            |did: &str, rkey: &str| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey });
        let space = |did: &str, rkey: &str, n: i64| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
        let gone = |did: &str, rkey: &str| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey });
        // In each account's table: records and their events; in Alice's,
        // a space repo too.
        for (did, token) in &accounts {
            for n in 0..24 {
                let body = put(did, &format!("r{n:02}"), n);
                call(&s, PUT, Some(token), body).await.unwrap();
            }
        }
        let [a, b, c] = [0, 1, 2].map(|i| accounts[i].0.clone());
        let ta = &accounts[0].1;
        for (rkey, n) in [("one", 1), ("two", 2)] {
            w(&s, "putRecord", ta, space(&a, rkey, n)).await.unwrap();
        }
        idle(&s).await;
        for (did, _) in &accounts {
            let wal = crate::read(&s.account(did).unwrap().store).wal.clone();
            wal.compact().unwrap();
        }
        s.log.compact().unwrap();
        // In the log after each: records written over and deleted.
        for (did, token) in &accounts {
            call(&s, PUT, Some(token), put(did, "r00", 100))
                .await
                .unwrap();
            call(&s, DELETE, Some(token), del(did, "r01"))
                .await
                .unwrap();
        }
        w(&s, "putRecord", ta, space(&a, "one", 11)).await.unwrap();
        w(&s, "deleteRecord", ta, gone(&a, "two")).await.unwrap();
        session(&s, "alice").await;
        idle(&s).await;
        let want = with_all(&s);
        drop(s);
        let ckpt = |did: &str| crate::store::account_layout(Path::new(""), did).checkpoint();
        let service = crate::store::service_layout(Path::new("")).checkpoint();
        // Alice's, Carol's and the service's of the version before; Bob's as it is.
        let written = files(&dir);
        let old = [ckpt(&a), ckpt(&c), service.clone()];
        for p in &old {
            as_the_version_before(&dir.join(p));
        }
        let before = files(&dir);
        for p in &old {
            assert_eq!(table_version(&dir.join(p)), Version::Sha256);
            assert!(before[p] != written[p]);
        }
        assert_eq!(table_version(&dir.join(ckpt(&b))), Version::Xxh3);
        // Bob's events and Carol's, found in their files with no store
        // loaded: neither has anything waiting to be sent, so a start that
        // loads each account as it is first asked for loads neither.
        let work = scratch("checks-work");
        let unloaded = || {
            let s = open_with(&work, &plc, |c| c.load_all = false);
            for did in [&b, &c] {
                assert!(!s.account(did).is_some_and(|a| a.store.is_loaded()));
            }
            assert!(s.account(&a).is_some());
            s
        };
        let on_disk = |files: &BTreeMap<PathBuf, Vec<u8>>| {
            put_files(&work, files);
            let s = unloaded();
            [&b, &c].map(|did| crate::checkpoint::events_on_disk(&s.ctx, did).unwrap())
        };
        let events = on_disk(&written);
        assert!(events.iter().all(|(seqs, _, _)| seqs.len() > 24));
        assert_eq!(on_disk(&before), events);
        // Served as it was, the tables left as they are.
        put_files(&work, &before);
        assert!(with_all(&open(&work, &plc)) == want, "served");
        assert!(with_all(&unloaded()) == want, "loaded on first use");
        let served = files(&work);
        assert!(old.iter().all(|p| served[p] == before[p]));
        // Written to, then folded: Alice's table and the service's are of
        // this version and hold the same; Carol's is as it was.
        let s = open(&work, &plc);
        for (did, token) in &accounts {
            call(&s, PUT, Some(token), put(did, "after", 7))
                .await
                .unwrap();
            call(&s, DELETE, Some(token), del(did, "r02"))
                .await
                .unwrap();
        }
        w(&s, "putRecord", ta, space(&a, "three", 3)).await.unwrap();
        idle(&s).await;
        let later = with_all(&s);
        assert!(later != want);
        let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
        wal.compact().unwrap();
        drop(wal);
        s.log.compact().unwrap();
        assert!(with_all(&s) == later, "folded");
        drop(s);
        let folded = files(&work);
        for p in [ckpt(&a), service] {
            assert_eq!(table_version(&work.join(p)), Version::Xxh3);
        }
        assert!(folded[&ckpt(&c)] == before[&ckpt(&c)]);
        assert!(with_all(&open(&work, &plc)) == later, "started again");
        assert!(
            with_all(&unloaded()) == later,
            "started again, loaded on first use"
        );
        // A byte changed, of Carol's table, of the version before, and of
        // Bob's, of this one: the last of the file, which is of its number
        // of entries in the version before and of its footer's check in
        // this one; the last of its bloom filter's check; and the first
        // of its first block.
        for (did, version, last) in [
            (
                &c,
                Version::Sha256,
                "more entries than its blocks could hold",
            ),
            (&b, Version::Xxh3, "its footer fails its check"),
        ] {
            assert_eq!(table_version(&work.join(ckpt(did))), version);
            let len = folded[&ckpt(did)].len();
            for (at, why) in [
                (len - 1, last),
                (
                    len - version.footer() - 1,
                    "its bloom filter fails its check",
                ),
                (0, "fails its check"),
            ] {
                let mut f = folded.clone();
                f.get_mut(&ckpt(did)).unwrap()[at] ^= 0x55;
                put_files(&work, &f);
                let refused = match try_open(&work, &plc) {
                    Err(e) => e,
                    Ok(s) => {
                        let account = s.account(did).unwrap();
                        let st = crate::read(&account.store);
                        crate::sync::CarStream::of(&st, None)
                            .and_then(|car| car.bytes())
                            .unwrap_err()
                            .to_string()
                    }
                };
                assert!(refused.contains(why), "{refused}");
            }
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// The state of a server that loads each account when it is first
    /// asked for, once every account `want` holds is: none is loaded
    /// before.
    fn lazily(dir: &Path, plc: &str, want: &State) -> State {
        let s = open_with(dir, plc, |c| c.load_all = false);
        assert!(crate::read(&s.directory).by_did.is_empty());
        for did in want.0.keys() {
            s.account(did);
        }
        state(&s)
    }

    /// What the service log keeps of the accounts, as one comparable value
    /// read with no account's store: each account's handle, mail and state,
    /// the sessions and app passwords, which account each reset token is
    /// of, and the invite codes.
    type Kept = (
        BTreeMap<String, (String, Mail, crate::lifecycle::State)>,
        BTreeMap<String, sessions::Keep>,
        BTreeMap<String, String>,
        Vec<String>,
    );

    fn kept(s: &Service) -> Kept {
        let accounts = crate::read(&s.directory)
            .by_did
            .iter()
            .map(|(did, a)| {
                let (mail, state) = (email::lock(&a.mail).clone(), crate::read(&a.state).clone());
                (did.clone(), (a.handle(), mail, state))
            })
            .collect();
        (
            accounts,
            s.sessions.snapshot(),
            s.resets.snapshot(),
            s.invites.snapshot(),
        )
    }

    /// With the service checkpoint a table: each kind of write to the
    /// service log after it, of an account in the table and of one made
    /// after it, cut at every byte. A start that loads each account when
    /// it is first asked for finds what the service log keeps as before
    /// the write, or as after it once it is whole; and so does one that
    /// loads every account, of everything, at every few bytes.
    #[tokio::test]
    async fn torn_writes_after_a_table() {
        for sync in SYNCS {
            syncing(sync);
            torn_writes_after_a_table_once().await;
        }
    }

    async fn torn_writes_after_a_table_once() {
        let plc = plc().await;
        let dir = scratch("torn-table");
        let s = open(&dir, &plc);
        let (_, ta) = create(&s, "alice").await;
        let (_, tb) = create(&s, "bob").await;
        s.log.compact().unwrap();
        let layout = crate::store::service_layout(Path::new(""));
        let (log, ckpt) = (layout.log(s.log.generation()), layout.checkpoint());
        let mut steps = Vec::new();
        let snap = |s: &Service, steps: &mut Vec<(Step, Kept)>| {
            steps.push((
                Step {
                    files: files(&dir),
                    journal: segments(&dir),
                    state: state(s),
                },
                kept(s),
            ))
        };
        snap(&s, &mut steps);
        assert!(store::table::is_table(&steps[0].0.files[&ckpt]));
        let app = |token: &str| {
            let (s, token) = (s.clone(), token.to_string());
            async move {
                call(
                    &s,
                    "com.atproto.server.createAppPassword",
                    Some(&token),
                    json!({ "name": "phone" }),
                )
                .await
                .unwrap()
            }
        };
        // Of accounts in the table.
        session(&s, "alice").await;
        snap(&s, &mut steps);
        app(&ta).await;
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.updateEmail",
            Some(&tb),
            json!({ "email": "bob2@mail.com" }),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.deactivateAccount",
            Some(&ta),
            json!({}),
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        // An account made after it, and a write of its.
        let (_, tc) = create(&s, "carol").await;
        snap(&s, &mut steps);
        app(&tc).await;
        snap(&s, &mut steps);
        call(
            &s,
            "com.atproto.server.activateAccount",
            Some(&ta),
            Json::Null,
        )
        .await
        .unwrap();
        snap(&s, &mut steps);
        drop(s);
        let dids: Vec<String> = steps.last().unwrap().1.0.keys().cloned().collect();
        let work = scratch("torn-table-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let whole = &w[1].0.files[&log];
            let (from, to) = (w[0].0.files[&log].len(), whole.len());
            assert!(to > from);
            kinds.extend(
                store::scan(&whole[from..])
                    .unwrap()
                    .0
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = w[1].0.files.clone();
                files.insert(log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { &w[1] } else { &w[0] };
                // Each account loaded when it is asked for, none before.
                let s = open_with(&work, &plc, |c| c.load_all = false);
                assert!(crate::read(&s.directory).by_did.is_empty());
                for did in &dids {
                    s.account(did);
                }
                assert!(kept(&s) == want.1, "cut at {cut} of {from}..{to}");
                drop(s);
                if cut % 61 == 0 || cut + 1 >= to {
                    assert!(
                        state(&open(&work, &plc)) == want.0.state,
                        "cut at {cut} of {from}..{to}, every account loaded"
                    );
                    assert!(
                        lazily(&work, &plc, &want.0.state) == want.0.state,
                        "cut at {cut} of {from}..{to}, loaded on first use"
                    );
                }
            }
        }
        for kind in [
            "account",
            "session",
            "appPassword",
            "updateEmail",
            "deactivated",
            "event",
        ] {
            assert!(
                kinds.iter().any(|k| k == kind),
                "{kind} never cut: {kinds:?}"
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// The service log's table, cut at every byte where a compaction
    /// writes it beside the table before, the log sealed and the next one
    /// made: a start finds everything as before the compaction, and the
    /// part written is gone.
    #[tokio::test]
    async fn a_table_cut_short_where_it_is_written() {
        let plc = plc().await;
        let dir = scratch("torn-tmp");
        let s = open(&dir, &plc);
        let (alice, _) = create(&s, "alice").await;
        s.log.compact().unwrap();
        session(&s, "alice").await;
        let layout = crate::store::service_layout(Path::new(""));
        let g = s.log.generation();
        let before = (files(&dir), state(&s));
        s.log.compact().unwrap();
        assert!(state(&s) == before.1);
        let table = files(&dir)[&layout.checkpoint()].clone();
        assert!(table != before.0[&layout.checkpoint()]);
        drop(s);
        let mut killed = before.0.clone();
        let mut sealed = killed[&layout.log(g)].clone();
        sealed.extend(seal());
        killed.insert(layout.log(g), sealed);
        killed.insert(layout.log(g + 1), Vec::new());
        let work = scratch("torn-tmp-work");
        for cut in 0..=table.len() {
            let mut files = killed.clone();
            files.insert(layout.tmp(), table[..cut].to_vec());
            put_files(&work, &files);
            if cut % 499 == 0 || cut == table.len() {
                assert!(state(&open(&work, &plc)) == before.1, "cut at {cut}");
                assert!(
                    lazily(&work, &plc, &before.1) == before.1,
                    "cut at {cut}, loaded on first use"
                );
            } else {
                let s = open_with(&work, &plc, |c| c.load_all = false);
                assert_eq!(crate::read(&s.directory).count(), 1, "cut at {cut}");
                assert_eq!(s.sessions.count(&alice), 0, "cut at {cut}");
                assert_eq!(s.account(&alice).unwrap().handle(), "alice.test");
                assert_eq!(s.sessions.count(&alice), 2, "cut at {cut}");
            }
            assert!(!work.join(layout.tmp()).exists(), "cut at {cut}");
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// A service checkpoint of the format before a table, a log of frames
    /// after its header, with the log after it: a start makes it a table
    /// of the same generation and serves all of it, the same wherever the
    /// entries were divided between the checkpoint and the log, wherever
    /// the conversion was cut short, and when it is started again.
    #[tokio::test]
    async fn a_checkpoint_of_the_format_before_is_made_a_table() {
        let plc = plc().await;
        let dir = scratch("convert");
        let (_, accounts) = script(&dir, &plc).await;
        let s = open(&dir, &plc);
        let (ta, tb) = (&accounts[0].1, &accounts[1].1);
        call(
            &s,
            "com.atproto.server.createAppPassword",
            Some(ta),
            json!({ "name": "phone" }),
        )
        .await
        .unwrap();
        call(
            &s,
            "com.atproto.server.updateEmail",
            Some(tb),
            json!({ "email": "bob2@mail.com" }),
        )
        .await
        .unwrap();
        call(
            &s,
            "com.atproto.server.deactivateAccount",
            Some(tb),
            json!({}),
        )
        .await
        .unwrap();
        create(&s, "carol").await;
        let want = state(&s);
        drop(s);
        let last = files(&dir);
        let log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let layout = crate::store::service_layout(Path::new(""));
        let (ckpt, tmp) = (layout.checkpoint(), layout.tmp());
        assert!(!last.contains_key(&ckpt));
        let frames = store::scan(&last[&log]).unwrap().0;
        // The files as the server before left them, having compacted the
        // first `at` frames: its checkpoint, a header naming its generation
        // and the last seq of its events, then what those frames hold.
        let before = |at: usize| {
            let seq = frames[..at]
                .iter()
                .flatten()
                .filter(|e| e["t"] == "event" || e["t"] == "deleted")
                .filter_map(|e| e["seq"].as_u64())
                .max()
                .unwrap_or(0);
            let mut checkpoint =
                store::frame(&[json!({ "t": "checkpoint", "generation": 1, "lastSeq": seq })]);
            checkpoint.extend(frames[..at].iter().flat_map(|f| store::frame(f)));
            let mut files = last.clone();
            files.remove(&log);
            files.insert(ckpt.clone(), checkpoint);
            files.insert(
                layout.log(1),
                frames[at..].iter().flat_map(|f| store::frame(f)).collect(),
            );
            files
        };
        let work = scratch("convert-work");
        for at in 0..=frames.len() {
            put_files(&work, &before(at));
            assert!(state(&open(&work, &plc)) == want, "divided at {at}");
            let made = files(&work);
            assert!(store::table::is_table(&made[&ckpt]), "divided at {at}");
            assert!(!made.contains_key(&tmp), "divided at {at}");
            assert!(
                state(&open(&work, &plc)) == want,
                "divided at {at}, started again"
            );
            assert!(files(&work)[&ckpt] == made[&ckpt], "made a table twice");
            assert!(
                lazily(&work, &plc, &want) == want,
                "divided at {at}, loaded on first use"
            );
        }
        // Cut short while the table is written beside it, at every byte;
        // and whole, not yet renamed over it. Of a checkpoint that holds
        // the first account, the log the rest.
        let at = 1 + frames
            .iter()
            .position(|f| f.iter().any(|e| e["t"] == "account"))
            .unwrap();
        put_files(&work, &before(at));
        drop(open_with(&work, &plc, |c| c.load_all = false));
        let table = std::fs::read(work.join(&ckpt)).unwrap();
        let accounts = want.0.len() as u64;
        for cut in 0..=table.len() {
            let mut f = before(at);
            f.insert(tmp.clone(), table[..cut].to_vec());
            put_files(&work, &f);
            if cut % 499 == 0 || cut == table.len() {
                assert!(state(&open(&work, &plc)) == want, "cut at {cut}");
            } else {
                let s = open_with(&work, &plc, |c| c.load_all = false);
                assert_eq!(crate::read(&s.directory).count(), accounts, "cut at {cut}");
            }
            assert!(
                std::fs::read(work.join(&ckpt)).unwrap() == table && !work.join(&tmp).exists(),
                "cut at {cut}"
            );
        }
        // One that is not whole is refused, not taken for less than it held.
        let whole = before(at);
        for cut in [0, 5, whole[&ckpt].len() - 1] {
            let mut f = whole.clone();
            f.insert(ckpt.clone(), whole[&ckpt][..cut].to_vec());
            put_files(&work, &f);
            assert!(try_open(&work, &plc).is_err(), "a checkpoint cut at {cut}");
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// Compacted, an account's log and the service log keep only the events
    /// in the window; a restart serves the same, and numbers new events after
    /// every one given out, those dropped too.
    #[tokio::test]
    async fn compaction_drops_events_out_of_the_window() {
        let plc = plc().await;
        let dir = scratch("window");
        let cfg = || Config {
            backfill: std::time::Duration::from_millis(1500),
            ..config(&plc)
        };
        let s: Arc<Service> = Arc::new(Service::open(&dir, cfg()).unwrap());
        let (a, ta) = create(&s, "alice").await;
        let w =
            |k: &str| json!({ "repo": a, "collection": "a.b.c", "rkey": k, "record": record(1) });
        call(&s, "com.atproto.repo.putRecord", Some(&ta), w("one"))
            .await
            .unwrap();
        let old = *s.events.subscribe().borrow();
        tokio::time::sleep(std::time::Duration::from_millis(3000)).await;
        call(&s, "com.atproto.repo.putRecord", Some(&ta), w("two"))
            .await
            .unwrap();
        let last = *s.events.subscribe().borrow();
        assert_eq!(last, old + 1);
        let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
        // The write may have made one due by age already.
        wal.wait_compacted();
        wal.compact().unwrap();
        s.log.wait_compacted();
        s.log.compact().unwrap();
        // The window holds the last commit's event alone.
        let kept = s.events.get(&s, 0, last, 0);
        assert_eq!(kept.len(), 1);
        drop(s);
        let s = Service::open(&dir, cfg()).unwrap();
        assert_eq!(*s.events.subscribe().borrow(), last);
        assert_eq!(s.events.get(&s, 0, last, 0), kept);
        drop(s);
        tokio::time::sleep(std::time::Duration::from_millis(3000)).await;
        // Out of the window, every event dropped: numbers go on all the same.
        let s: Arc<Service> = Arc::new(Service::open(&dir, cfg()).unwrap());
        assert_eq!(s.events.window_start(last), None);
        let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
        wal.wait_compacted();
        wal.compact().unwrap();
        drop(s);
        let s: Arc<Service> = Arc::new(Service::open(&dir, cfg()).unwrap());
        assert_eq!(s.events.len(), 0);
        call(&s, "com.atproto.repo.putRecord", Some(&ta), w("three"))
            .await
            .unwrap();
        assert_eq!(*s.events.subscribe().borrow(), last + 1);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// The layout before generations, `accounts.log` and each account's
    /// `repo.log`, with events written with seqs other than those served (as
    /// the server before numbered them again at each start): moved to
    /// generation 0 on the first start, its events with the seqs it served,
    /// the same if the move is cut short and made again.
    #[tokio::test]
    async fn the_layout_before_generations_is_moved() {
        let plc = plc().await;
        let dir = scratch("migrate");
        let (_, accounts) = script(&dir, &plc).await;
        let s = open(&dir, &plc);
        let want = state(&s);
        drop(s);
        // Make it the layout before: ords, and seqs off by some.
        let legacy = |frames: Vec<Vec<Json>>| -> Vec<u8> {
            let mut out = Vec::new();
            for mut f in frames {
                for e in f.iter_mut().filter(|e| e["t"] == "event") {
                    let seq = e["seq"].as_u64().unwrap();
                    e["ord"] = json!(seq * 3);
                    e["seq"] = json!(seq + 100);
                }
                out.extend(store::frame(&f));
            }
            out
        };
        let mut old = BTreeMap::new();
        for (p, b) in files(&dir) {
            let name = p.file_name().unwrap().to_string_lossy().to_string();
            let renamed = match name.as_str() {
                "accounts.0.log" => p.with_file_name("accounts.log"),
                "repo.0.log" => p.with_file_name("repo.log"),
                _ => p.clone(),
            };
            let b = if renamed != p {
                legacy(store::scan(&b).unwrap().0)
            } else {
                b
            };
            old.insert(renamed, b);
        }
        let work = scratch("migrate-work");
        put_files(&work, &old);
        assert!(state(&open(&work, &plc)) == want, "moved");
        assert!(!work.join("accounts.log").exists() && work.join("accounts.0.log").exists());
        assert!(state(&open(&work, &plc)) == want, "started again");
        // Cut short: one account's log moved, the service log not.
        let _ = std::fs::remove_dir_all(&work);
        put_files(&work, &old);
        let alice = Path::new("repos").join(&accounts[0].0);
        std::fs::rename(
            work.join(alice.join("repo.log")),
            work.join(alice.join("repo.0.log")),
        )
        .unwrap();
        assert!(
            state(&open(&work, &plc)) == want,
            "moved after a move cut short"
        );
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    #[test]
    fn wal_group_commit_covers_all_waiters() {
        let dir = scratch("wal");
        let path = dir.join("x.log");
        let (wal, _) = Wal::open(&path, None).unwrap();
        let a = wal
            .append(&[json!({ "t": "session", "token": "a" })])
            .unwrap();
        let b = wal
            .append(&[json!({ "t": "session", "token": "b" })])
            .unwrap();
        assert!(!wal.is_durable(a));
        wal.wait(b).unwrap();
        assert!(wal.is_durable(a) && wal.is_durable(b));
        wal.wait(a).unwrap();
        let (frames, _) = store::scan(&std::fs::read(&path).unwrap()).unwrap();
        assert_eq!(frames.len(), 2);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Cut `accounts.log` at every byte of each frame invite codes wrote: one
    /// code, several for several accounts, and an account made with one,
    /// whose use is in its frame. Recovery gives the state before or after.
    #[tokio::test]
    async fn torn_invite_writes() {
        for sync in SYNCS {
            syncing(sync);
            torn_invite_writes_once().await;
        }
    }

    async fn torn_invite_writes_once() {
        let plc = plc().await;
        let dir = scratch("torn-invites");
        let s = open(&dir, &plc);
        let (alice, _) = create(&s, "alice").await;
        let admin = format!(
            "Basic {}",
            base64::Engine::encode(
                &base64::engine::general_purpose::STANDARD,
                format!("admin:{ADMIN_PASSWORD}")
            )
        );
        let as_admin = |nsid: &'static str, body: Json| {
            let (s, admin) = (s.clone(), admin.clone());
            async move {
                call_raw(
                    &s,
                    nsid,
                    sessions::Bearer::of(Some(&admin)),
                    &[],
                    &body.to_string(),
                )
                .await
                .unwrap()
            }
        };
        let mut steps = vec![Step {
            files: files(&dir),
            journal: segments(&dir),
            state: state(&s),
        }];
        let snap = |s: &Service, steps: &mut Vec<Step>| {
            steps.push(Step {
                files: files(&dir),
                journal: segments(&dir),
                state: state(s),
            })
        };
        let code = as_admin(
            "com.atproto.server.createInviteCode",
            json!({ "useCount": 2, "forAccount": alice }),
        )
        .await;
        snap(&s, &mut steps);
        as_admin(
            "com.atproto.server.createInviteCodes",
            json!({ "codeCount": 2, "useCount": 1, "forAccounts": [alice, "did:plc:other"] }),
        )
        .await;
        snap(&s, &mut steps);
        let body = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw", "inviteCode": code["code"] });
        call(&s, "com.atproto.server.createAccount", None, body)
            .await
            .unwrap();
        snap(&s, &mut steps);
        drop(s);
        let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
        let work = scratch("torn-invites-work");
        let mut kinds = Vec::new();
        for w in steps.windows(2) {
            let whole = &w[1].files[&accounts_log];
            let (from, to) = (w[0].files[&accounts_log].len(), whole.len());
            kinds.extend(
                store::scan(&whole[from..])
                    .unwrap()
                    .0
                    .iter()
                    .flatten()
                    .map(|e| e["t"].as_str().unwrap().to_string()),
            );
            for cut in from..=to {
                let mut files = w[1].files.clone();
                files.insert(accounts_log.clone(), whole[..cut].to_vec());
                put_files(&work, &files);
                let want = if cut == to { &w[1].state } else { &w[0].state };
                assert_eq!(
                    &state(&open(&work, &plc)),
                    want,
                    "cut at {cut} of {from}..{to}"
                );
            }
        }
        for kind in crate::invites::ENTRIES {
            assert!(
                kinds.iter().any(|k| k == kind),
                "{kind} never cut: {kinds:?}"
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&work);
    }

    /// A `createAccount` whose account's log cannot be made answers `500`
    /// and writes nothing to `accounts.log`: the account does not exist, then
    /// or after a restart, and the next one is made.
    #[tokio::test]
    async fn an_account_whose_log_cannot_be_made() {
        let plc = plc().await;
        let dir = scratch("unmade");
        let s = open(&dir, &plc);
        create(&s, "alice").await;
        let before = (std::fs::read(accounts_log(&dir)).unwrap(), state(&s));
        // A file where the accounts' directories must go: no directory can be
        // made in it, for any user, root too. Alice's log stays open meanwhile.
        let (repos, aside) = (dir.join("repos"), dir.join("repos.aside"));
        std::fs::rename(&repos, &aside).unwrap();
        std::fs::write(&repos, b"not a directory").unwrap();
        let bob = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw" });
        assert_eq!(
            call(&s, "com.atproto.server.createAccount", None, bob.clone()).await,
            Err("InternalServerError".into())
        );
        std::fs::remove_file(&repos).unwrap();
        std::fs::rename(&aside, &repos).unwrap();
        assert_eq!(
            (std::fs::read(accounts_log(&dir)).unwrap(), state(&s)),
            before
        );
        drop(s);
        let s = open(&dir, &plc);
        assert_eq!(state(&s), before.1);
        call(&s, "com.atproto.server.createAccount", None, bob)
            .await
            .unwrap();
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Every file the server makes is `0600` and every directory `0700`,
    /// whatever the umask; the data directory is made `0700` whatever it was.
    #[tokio::test]
    async fn files_are_the_owners_alone() {
        use std::os::unix::fs::PermissionsExt;
        let plc = plc().await;
        let dir = scratch("modes");
        std::fs::create_dir_all(&dir).unwrap();
        std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap();
        let s = open(&dir, &plc);
        let (did, token) = create(&s, "alice").await;
        crate::blobs::test_upload(
            &s,
            Some(&token),
            Some("text/plain"),
            b"a blob".to_vec(),
            false,
        )
        .await
        .unwrap();
        call(
            &s,
            "com.atproto.server.createAppPassword",
            Some(&token),
            json!({ "name": "x" }),
        )
        .await
        .unwrap();
        assert!(dir.join(format!("repos/{did}/blobs")).is_dir());
        let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777;
        assert_eq!(mode(&dir), 0o700);
        let mut todo = vec![dir.clone()];
        let mut seen = 0;
        while let Some(d) = todo.pop() {
            for e in std::fs::read_dir(&d).unwrap() {
                let p = e.unwrap().path();
                seen += 1;
                if p.is_dir() {
                    assert_eq!(mode(&p), 0o700, "{}", p.display());
                    todo.push(p);
                } else {
                    assert_eq!(mode(&p), 0o600, "{}", p.display());
                }
            }
        }
        assert!(seen >= 6, "{seen}");
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Accounts are loaded on no more threads than the machine has cores,
    /// with no more of their logs open than a share.
    #[tokio::test]
    async fn recovery_is_bounded() {
        let plc = plc().await;
        let dir = scratch("recover-threads");
        let s = open(&dir, &plc);
        let cores = std::thread::available_parallelism().unwrap().get();
        let share = s.config.share;
        let n = (cores * 3).max(share + 8);
        for i in 0..n {
            let body =
                json!({ "handle": format!("user{i}.test"), "email": format!("u{i}@mail.com") });
            call(&s, "com.atproto.server.createAccount", None, body)
                .await
                .unwrap();
        }
        assert!(s.ctx.files.open_count() <= share);
        drop(s);
        let s = open(&dir, &plc);
        let d = crate::read(&s.directory);
        assert_eq!(d.by_did.len(), n);
        assert!(d.by_did.values().all(|a| a.store.is_loaded()));
        let threads: std::collections::HashSet<_> = d
            .by_did
            .values()
            .map(|a| crate::read(&a.store).thread)
            .collect();
        assert!(
            threads.len() <= cores,
            "{} threads, {cores} cores",
            threads.len()
        );
        assert!(s.ctx.files.open_count() <= share);
        drop(d);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Run the test `name` again in a process of its own, with `PDS_TEST_CHILD`
    /// naming it, so that it may change the limits of its process: whether
    /// this is that process. No log is opened while the child is spawned,
    /// which holds a copy of every log's file open in this process, and its
    /// lock, until it execs (`pds_store::spawning`): the tests beside this
    /// one close their logs and open them again meanwhile.
    fn in_child(name: &str) -> bool {
        use std::process::{Command, Stdio};
        if std::env::var("PDS_TEST_CHILD").as_deref() == Ok(name) {
            return true;
        }
        let child = {
            let _alone = pds_store::spawning();
            Command::new(std::env::current_exe().unwrap())
                .args([
                    &format!("log::tests::{name}"),
                    "--exact",
                    "--nocapture",
                    "--test-threads=1",
                ])
                .env("PDS_TEST_CHILD", name)
                .stdin(Stdio::null())
                .stdout(Stdio::piped())
                .stderr(Stdio::piped())
                .spawn()
                .unwrap()
        };
        let out = child.wait_with_output().unwrap();
        let text = format!(
            "{}{}",
            String::from_utf8_lossy(&out.stdout),
            String::from_utf8_lossy(&out.stderr)
        );
        assert!(out.status.success() && text.contains("1 passed"), "{text}");
        false
    }

    /// Set the soft limit of `resource` to `soft`, or with none, to the hard limit.
    fn set_limit(resource: Resource, soft: Option<u64>) {
        let l = getrlimit(resource);
        let current = match soft {
            Some(s) => Some(l.maximum.map_or(s, |m| s.min(m))),
            None => l.maximum,
        };
        setrlimit(resource, Rlimit { current, ..l }).unwrap();
    }

    /// The file descriptors the process holds open.
    fn open_fds() -> usize {
        let dir = if Path::new("/proc/self/fd").exists() {
            "/proc/self/fd"
        } else {
            "/dev/fd"
        };
        std::fs::read_dir(dir).unwrap().count()
    }

    /// With 256 file descriptors, and the share of them the server takes
    /// under that limit, 300 accounts are made, and the server started
    /// again serves every one of them, and takes writes from the first and
    /// the last: what it holds open does not grow with them, and is no more
    /// than a share of logs, a share of tables and 72 files besides.
    #[test]
    fn many_accounts_few_files() {
        if !in_child("many_accounts_few_files") {
            return;
        }
        set_limit(Resource::Nofile, Some(256));
        let share = store::files::share(store::files::limit());
        assert_eq!(share, 64);
        let most = 2 * share + 72;
        let rt = tokio::runtime::Builder::new_multi_thread()
            .worker_threads(2)
            .enable_all()
            .build()
            .unwrap();
        rt.block_on(async {
            let plc = plc().await;
            let dir = scratch("many-accounts");
            let s = open_with(&dir, &plc, |c| c.share = share);
            let mut made = Vec::new();
            for i in 0..300 {
                let body = json!({ "handle": format!("user{i}.test"), "email": format!("u{i}@mail.com"), "password": "" });
                made.push(call(&s, "com.atproto.server.createAccount", None, body).await.unwrap());
            }
            assert!(s.ctx.files.open_count() <= share);
            assert!(open_fds() < most, "{} open", open_fds());
            drop(s);
            let s = open_with(&dir, &plc, |c| c.share = share);
            assert!(open_fds() < most, "{} open", open_fds());
            for m in &made {
                let p = [("did".to_string(), m["did"].as_str().unwrap().to_string())];
                call_with(&s, "com.atproto.sync.getLatestCommit", None, &p, Json::Null).await.unwrap();
            }
            for m in [&made[0], &made[299]] {
                let body = json!({ "repo": m["did"], "collection": "a.b.c", "record": { "n": 1 } });
                call(&s, "com.atproto.repo.createRecord", m["accessJwt"].as_str(), body).await.unwrap();
            }
            assert!(s.ctx.files.open_count() <= share);
            let _ = std::fs::remove_dir_all(&dir);
        });
    }

    /// A write whose append fails part way, as on a full disk, is `500` and
    /// cut back from its log, leaving nothing of itself in the repository or
    /// the firehose; the next write goes on from there, and a restart finds
    /// the same.
    #[test]
    fn a_failed_append_is_cut_back() {
        if !in_child("a_failed_append_is_cut_back") {
            return;
        }
        let rt = tokio::runtime::Builder::new_multi_thread()
            .worker_threads(2)
            .enable_all()
            .build()
            .unwrap();
        rt.block_on(async {
            // A handler for the signal, so that a write past the size limit fails instead.
            let _xfsz = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::from_raw(Signal::XFSZ.as_raw())).unwrap();
            let plc = plc().await;
            let dir = scratch("cut-back");
            let s = open(&dir, &plc);
            let (did, token) = create(&s, "alice").await;
            let (bob, bob_token) = create(&s, "bob").await;
            let put = |did: &str, rkey: &str, n: usize| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": { "t": "x".repeat(n) } });
            call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "a", 10)).await.unwrap();
            let log = dir.join(repo_log(Path::new(""), &did));
            let len = log_bytes(&log).len() as u64;
            let before = state(&s);
            let largest = files_whole(&dir).values().map(|b| b.len() as u64).max().unwrap();
            set_limit(Resource::Fsize, Some(largest.max(len) + 8000));
            assert_eq!(call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "b", 40_000)).await, Err("InternalServerError".into()));
            assert_eq!(std::fs::metadata(&log).unwrap().len(), len);
            assert_eq!(state(&s), before);
            // Another account's write goes on, and so does this one's.
            call(&s, "com.atproto.repo.putRecord", Some(&bob_token), put(&bob, "c", 10)).await.unwrap();
            call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "d", 10)).await.unwrap();
            set_limit(Resource::Fsize, None);
            let after = state(&s);
            assert_eq!(after.2.len(), before.2.len() + 2);
            drop(s);
            assert_eq!(state(&open(&dir, &plc)), after);
            let _ = std::fs::remove_dir_all(&dir);
        });
    }

    /// A commit's frame of the first kind: JSON, its bytes as hex, as
    /// `v0.1.0` wrote every commit's, and as a log may hold them.
    mod json_frames {
        use super::super::Hex;
        use pds_repo::Change;
        use serde_json::Value as Json;

        struct CommitEntry<'a>(&'a Change);

        impl serde::Serialize for CommitEntry<'_> {
            fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
                use serde::ser::SerializeMap;
                let c = self.0;
                let mut m = s.serialize_map(Some(10))?;
                m.serialize_entry("t", "commit")?;
                m.serialize_entry("rev", &c.rev)?;
                m.serialize_entry("head", &Hex(&c.head_bytes))?;
                m.serialize_entry(
                    "puts",
                    &c.puts
                        .iter()
                        .zip(&c.put_cids)
                        .map(|((k, v), cid)| (k, v, Hex(cid)))
                        .collect::<Vec<_>>(),
                )?;
                m.serialize_entry("dels", &c.dels)?;
                m.serialize_entry(
                    "tree",
                    &c.tree
                        .iter()
                        .map(|(cid, b)| (Hex(cid), Hex(b)))
                        .collect::<Vec<_>>(),
                )?;
                m.serialize_entry("gone", &c.gone.iter().map(|g| Hex(g)).collect::<Vec<_>>())?;
                m.serialize_entry(
                    "refs",
                    &c.refs
                        .iter()
                        .map(|(cid, n, rev)| (Hex(cid), n, rev))
                        .collect::<Vec<_>>(),
                )?;
                m.serialize_entry("counts", &[c.counts.0, c.counts.1])?;
                m.end()
            }
        }

        struct EventEntry<'a>(u64, u64, &'a [u8]);

        impl serde::Serialize for EventEntry<'_> {
            fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
                use serde::ser::SerializeMap;
                let mut m = s.serialize_map(Some(4))?;
                m.serialize_entry("t", "event")?;
                m.serialize_entry("seq", &self.0)?;
                m.serialize_entry("time", &self.1)?;
                m.serialize_entry("frame", &Hex(self.2))?;
                m.end()
            }
        }

        pub(super) fn commit_frame(
            commit: Option<&Change>,
            entries: &[Json],
            seq: u64,
            frames: &[Vec<u8>],
            times: &[u64],
        ) -> Vec<u8> {
            let capacity = frames.iter().map(|f| f.len() * 3).sum::<usize>() + 4096;
            pds_store::frame_of(capacity, |out| {
                use serde::ser::{SerializeSeq, Serializer};
                let mut s = serde_json::Serializer::new(out);
                let mut seq_out = s.serialize_seq(None).unwrap();
                if let Some(c) = commit {
                    seq_out.serialize_element(&CommitEntry(c)).unwrap();
                }
                for e in entries {
                    seq_out.serialize_element(e).unwrap();
                }
                for (i, f) in frames.iter().enumerate() {
                    seq_out
                        .serialize_element(&EventEntry(seq + i as u64, times[i], f))
                        .unwrap();
                }
                seq_out.end().unwrap();
            })
        }
    }

    /// A commit's frame with holes, filled, is byte for byte its frame of
    /// the first kind: with and without its commit, events and other
    /// entries, bytes that end with zero and none.
    #[test]
    fn a_commit_frame_filled_is_its_json_frame() {
        let bytes = |n: usize, seed: u8, last: u8| -> Vec<u8> {
            let mut b: Vec<u8> = (0..n)
                .map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed))
                .collect();
            if let Some(x) = b.last_mut() {
                *x = last;
            }
            b
        };
        let cid = |b: &[u8]| pds_repo::data::cid_of(0x71, b);
        let record =
            json!({ "$type": "app.bsky.feed.post", "text": "a \"post\"\n", "langs": ["en"] });
        let (n1, n2) = (bytes(1000, 3, 7), bytes(1100, 5, 0));
        let c = Change {
            rev: "3lzzzzzzzzz2a".into(),
            head_bytes: bytes(190, 1, 9),
            puts: vec![("app.bsky.feed.post/3lzz".into(), record.clone())],
            put_cids: vec![cid(&record.to_string().into_bytes())],
            dels: vec!["app.bsky.feed.post/3lyy".into()],
            tree: vec![(cid(&n1), n1), (cid(&n2), n2)],
            gone: vec![cid(b"gone")],
            refs: vec![(cid(b"ref"), 1, "3lzzzzzzzzz2a".into())],
            counts: (12, 30),
            ..Default::default()
        };
        let empty = Change::default();
        let handled = [json!({ "t": "handled", "cid": "bafy", "landed": true })];
        let events = [bytes(3000, 11, 4), bytes(2900, 13, 0)];
        for (commit, entries, frames) in [
            (Some(&c), &[][..], &events[..1]),
            (Some(&c), &handled[..], &events[..]),
            (Some(&c), &[][..], &[][..]),
            (Some(&empty), &[][..], &events[1..]),
            (None, &handled[..], &events[1..]),
            (None, &handled[..], &[][..]),
        ] {
            let times: Vec<u64> = (0..frames.len() as u64)
                .map(|i| 1_700_000_000_000 + i)
                .collect();
            let with = commit_frame(commit, entries, 9, frames, &times);
            let json = json_frames::commit_frame(commit, entries, 9, frames, &times);
            assert_eq!(with[store::HEADER], store::HOLES);
            assert_eq!(
                store::fill(&with[store::HEADER..]).unwrap().as_ref(),
                &json[store::HEADER..]
            );
            assert_eq!(store::parse_frame(&with), store::parse_frame(&json));
            assert_ne!(*with.last().unwrap(), 0);
        }
    }

    /// The frame at the end of the log of `did`'s current generation: its
    /// bytes.
    fn last_frame(s: &Service, dir: &Path, did: &str) -> Vec<u8> {
        let g = store_of(s, did, |st| st.wal.generation());
        let path = crate::store::account_layout(dir, did).log(g);
        let mut last = None;
        store::scan_file(&path, |at, len, _| {
            last = Some((at, len));
            Ok(())
        })
        .unwrap()
        .unwrap();
        let (at, len) = last.unwrap();
        std::fs::read(&path).unwrap()[at as usize..][..len as usize].to_vec()
    }

    /// The bytes of a commit's frame of a post, in a tree of fifty: with
    /// holes, and as JSON with its bytes in hex.
    #[tokio::test]
    async fn the_bytes_of_a_commit_frame() {
        let plc = plc().await;
        let dir = scratch("commit-bytes");
        let s = open(&dir, &plc);
        let (did, token) = create(&s, "alice").await;
        let words = "the weather here is lovely today and I went for a long walk by the river ";
        for i in 0..50 {
            let post = json!({
                "$type": "app.bsky.feed.post",
                "text": format!("{i}: {}", words.repeat(3)),
                "langs": ["en"],
                "createdAt": "2026-11-12T10:00:00.000Z",
            });
            call(
                &s,
                "com.atproto.repo.createRecord",
                Some(&token),
                json!({ "repo": did, "collection": "app.bsky.feed.post", "record": post }),
            )
            .await
            .unwrap();
        }
        let with = last_frame(&s, &dir, &did);
        let json = store::HEADER + store::fill(&with[store::HEADER..]).unwrap().len();
        println!(
            "a commit's frame: {json} bytes as JSON, bytes as hex; {} with holes",
            with.len()
        );
        assert!(with.len() * 10 < json * 6, "{} of {json}", with.len());
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A log with its first `upto` frames with holes made JSON, their
    /// bytes in hex, as `v0.1.0` wrote every commit's: and how many it
    /// made so.
    fn as_json(log: &[u8], upto: usize) -> (Vec<u8>, usize) {
        let (mut out, mut made, mut at) = (Vec::new(), 0, 0);
        while at < log.len() {
            let len = u32::from_le_bytes(log[at..at + 4].try_into().unwrap()) as usize;
            let frame = &log[at..at + store::HEADER + len];
            if frame[store::HEADER] == store::HOLES && made < upto {
                let text = store::fill(&frame[store::HEADER..]).unwrap();
                out.extend(store::frame_of(text.len(), |o| o.extend_from_slice(&text)));
                made += 1;
            } else {
                out.extend_from_slice(frame);
            }
            at += store::HEADER + len;
        }
        (out, made)
    }

    /// An account's log of commits `v0.1.0` wrote, JSON, and then commits
    /// this version wrote, with holes, in any proportion: it loads, serves
    /// every block, record and event as the server that wrote it did, from
    /// frames of both kinds, and compacts into the same table as the log
    /// written all one way or all the other.
    #[tokio::test]
    async fn a_log_of_both_kinds() {
        let plc = plc().await;
        let dir = scratch("kinds");
        let s = open(&dir, &plc);
        let (did, token) = create(&s, "alice").await;
        for i in 0..30 {
            let (nsid, body) = if i % 7 == 6 {
                let rkey = format!("k{}", (i - 3) % 12);
                (
                    "com.atproto.repo.deleteRecord",
                    json!({ "repo": did, "collection": "a.b.c", "rkey": rkey }),
                )
            } else {
                let rkey = format!("k{}", i % 12);
                (
                    "com.atproto.repo.putRecord",
                    json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(i) }),
                )
            };
            call(&s, nsid, Some(&token), body).await.unwrap();
        }
        let want = state(&s);
        drop(s);
        let written = files(&dir);
        let layout = crate::store::account_layout(Path::new(""), &did);
        let logs: Vec<PathBuf> = written
            .keys()
            .filter(|p| p.starts_with(&layout.dir) && p.extension().is_some_and(|e| e == "log"))
            .cloned()
            .collect();
        let holed: usize = logs
            .iter()
            .map(|l| as_json(&written[l], usize::MAX).1)
            .sum();
        assert!(holed > 25, "{holed} frames with holes");
        let mut tables = Vec::new();
        // Of each log's frames with holes, none, all, or the first half
        // made JSON.
        for (name, of, by) in [("holes", 0, 1), ("json", 1, 1), ("both", 1, 2)] {
            let mut files = written.clone();
            let mut made = 0;
            for l in &logs {
                let frames = as_json(&written[l], usize::MAX).1;
                let (bytes, n) = as_json(&written[l], frames * of / by);
                files.insert(l.clone(), bytes);
                made += n;
            }
            match (of, by) {
                (0, _) => assert_eq!(made, 0),
                (1, 1) => assert_eq!(made, holed),
                _ => assert!(made > 0 && made < holed, "{made} of {holed}"),
            }
            let work = scratch(&format!("kinds-{name}"));
            put_files(&work, &files);
            let s = open(&work, &plc);
            assert_eq!(state(&s), want, "{name}: served");
            let wal = store_of(&s, &did, |st| st.wal.clone());
            wal.wait_compacted();
            wal.compact().unwrap();
            drop(wal);
            assert_eq!(state(&s), want, "{name}: compacted");
            tables.push(std::fs::read(work.join(layout.checkpoint())).unwrap());
            drop(s);
            let _ = std::fs::remove_dir_all(&work);
        }
        assert!(tables.iter().all(|t| *t == tables[0]), "one table");
        let _ = std::fs::remove_dir_all(&dir);
    }
}