chadtmiller.com/monolith
An atproto PDS
0
0
//! What the server keeps, and how it starts again from it, as
//! `spec/DURABILITY.md` and `spec/STORE.md` say. In `PDS_DATA_DIR`, which is
//! its owner's alone:
//!
//! - the service log, `accounts.<g>.log` from its checkpoint
//! `accounts.checkpoint` on (`compact`): each account (DID, handle, email,
//! password hash, signing key, KEM key, DID document), each change to the
//! sessions it gave out and its app passwords (`sessions::ENTRIES`), each
//! change to its email and password and the tokens mailed for them
//! (`email::ENTRIES`), each change to what OAuth keeps (`oauth::ENTRIES`),
//! each single-use value OAuth spent (`replay::ENTRY`), the invite codes
//! and their uses (`invites::ENTRIES`), the single-use values the spaces
//! took (`single_use`), the spaces' credentials revoked here
//! (`space_credentials`), each account's state and its events
//! (`lifecycle::ENTRIES`), each account's preferences as last put
//! (`preferences::ENTRY`), and with blobs in a bucket, the objects of
//! deleted accounts not yet deleted (`blobstore::ENTRIES`);
//! - `jwt.key`, the secret its tokens are signed with, and `oauth.key`, the
//! one OAuth seals its tokens, nonces and cookies with;
//! - each account's own log, `repos/<did>/repo.<g>.log` from its checkpoint
//! `repo.checkpoint` on (`checkpoint`): the mark that its spaces state is
//! this version's from there (`checkpoint::unmarked`), first; its
//! commits, each in one frame
//! with its firehose events, the blobs it stored and stopped holding, its
//! handle changes: each recorded before its operation goes to the PLC
//! directory, then settled, in one frame with its `#identity` event if it
//! landed; its space repos' commits and imports, each one frame
//! (`space_repo`); each change to the spaces it is the authority of, one
//! frame each (`space_authority::ENTRIES`), with the commits of the
//! records it publishes and the repo it drops (`space_repo::DROP`); and its
//! write notifications waiting to be sent again, and its spaces' syncers'
//! registrations and the notices kept for them (`space_notify::ENTRIES`);
//! - `repos/<did>/blobs/<cid>`, the bytes of each blob it holds (`blobs`),
//! or with blobs in a bucket, none, and the mark `blobstore` (`blobstore`);
//! - the firehose's index, `events.<g>.log` from its checkpoint
//! `events.checkpoint` on: of each event, as it is numbered, its seq,
//! account and time, so that a start learns the events made before it
//! without reading the accounts' logs (`events::index`). Its entry of an
//! event is appended before the frame that holds the event, and both are
//! on disk before anything is made of it;
//! - `clean-shutdown`, the mark a shutdown leaves and the next start takes
//! away (`ops`): it holds the word that every log was closed and synced,
//! for which the next start syncs no log as it opens it (`store`, "What
//! a start syncs").
//!
//! Each log is compacted into its checkpoint, generation by generation
//! (`store`, `pds_store`). A start reads the service checkpoint's summary
//! and its entries that are not one account's, and replays the service log
//! after it, noting where each account's entries are without applying them
//! (`directory`); each account, and then its own checkpoint and log, is
//! read on its first use. A data directory with the layout before
//! generations is moved to generation 0 first (`store::migrate`), and a
//! service checkpoint of the format before a table is made one
//! (`compact::convert`).
//!
//! An account's log, with its repository's first commit, is on disk before
//! the account is in the service log: if the server stops between the two,
//! the log is of no account, and goes once the server serves again
//! (`directory::Accounts::sweep`). An account in the service log whose log
//! has no commit, as one older servers left, gets its first commit when its
//! store is loaded. Events are numbered in one sequence across all the
//! accounts' logs (`events::Firehose`), each keeping its seq, the next
//! after the greatest any log or checkpoint names. An account's handle is
//! the one its last landed change gave it, or else the one the service log
//! has.
//!
//! At most a share of accounts' logs are open at once (`Config::share`: a
//! quarter of the file descriptors the process may have, as
//! `pds_store::files::share` counts it), and at most a share of their
//! tables' files, and they are recovered on at most as many threads as the
//! machine has cores.
use crate::blobs::blob_dir;
use crate::cache::{Ctx, Slot};
use crate::compact::ServiceTable;
use crate::directory::{Accounts, Directory};
use crate::email::{self, Mail};
use crate::events::{self, Firehose, Landing, Reserved};
use crate::sessions::Sessions;
use crate::{Account, Config, Service, Store};
use axum::body::Bytes;
use k256::ecdsa::SigningKey;
use pds_repo::*;
use pds_store::{Files, Holed, Wal};
use rustix::io::Errno;
use serde_json::{Value as Json, json};
use std::collections::HashSet;
use std::io;
use std::path::Path as FsPath;
#[cfg(test)]
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, PoisonError, RwLock};
/// What error messages call the service log.
pub(crate) const ACCOUNTS_LOG: &str = "the service log";
/// The service log of generation 0, by its name.
#[cfg(test)]
pub(crate) const ACCOUNTS_LOG_FILE: &str = "accounts.0.log";
/// The service log of generation 0, which a server that has not compacted it holds.
#[cfg(test)]
pub(crate) fn accounts_log(dir: &FsPath) -> PathBuf {
crate::store::service_layout(dir).log(0)
}
/// An account's log of generation 0, which an account whose log has not
/// been compacted holds.
#[cfg(test)]
pub(crate) fn repo_log(dir: &FsPath, did: &str) -> PathBuf {
crate::store::account_layout(dir, did).log(0)
}
/// A copy of the data directory `dir` as it is now, beside it: what a
/// server started again would find, for one to start on while the server
/// on `dir`, which holds its logs, goes on.
#[cfg(test)]
pub(crate) fn copied(dir: &FsPath) -> PathBuf {
let to = dir.with_extension("copy");
let _ = std::fs::remove_dir_all(&to);
let mut todo = vec![dir.to_path_buf()];
while let Some(d) = todo.pop() {
std::fs::create_dir_all(to.join(d.strip_prefix(dir).unwrap())).unwrap();
for e in std::fs::read_dir(&d).unwrap() {
let p = e.unwrap().path();
if p.is_dir() {
todo.push(p);
} else {
std::fs::copy(&p, to.join(p.strip_prefix(dir).unwrap())).unwrap();
}
}
}
to
}
/// What the log at `path` holds: its file's bytes, without the room after
/// its frames, which a server running or killed leaves there
/// (`pds_store`).
#[cfg(test)]
pub(crate) fn log_bytes(path: &FsPath) -> Vec<u8> {
without_room(std::fs::read(path).unwrap())
}
/// A log's bytes without the room after its frames: the zeros they end with.
#[cfg(test)]
fn without_room(mut log: Vec<u8>) -> Vec<u8> {
log.truncate(log.iter().rposition(|&b| b != 0).map_or(0, |last| last + 1));
log
}
/// The logs opened on each data directory, in tests alone, where one
/// process starts a server on a directory again and again while tasks of
/// the server before still hold it: a start closes that server's logs
/// first, as the restart of a process ends its writers (`Wal::close`).
#[cfg(test)]
static OPENED: Mutex<Vec<(PathBuf, std::sync::Weak<Wal>)>> = Mutex::new(Vec::new());
/// A log as the server holds it, opened on the data directory `dir`.
pub(crate) fn held(dir: &FsPath, wal: Wal) -> Arc<Wal> {
let wal = Arc::new(wal);
#[cfg(test)]
OPENED
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push((dir.to_path_buf(), Arc::downgrade(&wal)));
let _ = dir;
wal
}
/// The journals opened on each data directory, in tests alone, for the
/// same reason: a start lets the one before go, as a kill does.
#[cfg(test)]
static JOURNALS: Mutex<Vec<(PathBuf, std::sync::Weak<pds_store::Journal>)>> =
Mutex::new(Vec::new());
/// Close every log a server before opened on `dir` and still holds: its
/// files are let go whether or not what it had written could be synced.
/// Then let its journal go, its segments left for this start to redo.
#[cfg(test)]
fn stop_writers(dir: &FsPath) {
OPENED
.lock()
.unwrap_or_else(PoisonError::into_inner)
.retain(|(of, wal)| match wal.upgrade() {
Some(wal) if of == dir => {
let _ = wal.close();
false
}
held => held.is_some(),
});
JOURNALS
.lock()
.unwrap_or_else(PoisonError::into_inner)
.retain(|(of, j)| match j.upgrade() {
Some(j) if of == dir => {
j.let_go();
false
}
held => held.is_some(),
});
}
/// An account as `accounts.log` has it.
pub(crate) struct AccountEntry {
pub did: String,
pub handle: String,
/// None for an account without a password.
pub email: Option<String>,
pub password: Option<String>,
/// Made on first use: an account is loaded for much that signs nothing.
pub key: crate::cache::Key,
/// Its KEM key (`spec/SPACES.md`); None for an account made before
/// accounts had one, which is given none.
pub kem: Option<pds_space::KemKey>,
/// Its DID document, as JSON text: held so from the start, a few
/// allocations rather than one for each of its fields.
pub did_doc: Box<str>,
}
impl AccountEntry {
pub fn entry(&self) -> Json {
json!({
"t": "account", "did": self.did, "handle": self.handle, "email": self.email,
"password": self.password, "key": hex::encode(self.key.to_bytes()),
"kemKey": self.kem.as_ref().map(|k| hex::encode(k.to_bytes())), "didDoc": serde_json::from_str::<Json>(&self.did_doc).unwrap_or(Json::Null),
})
}
pub(crate) fn parse(e: &Json) -> Result<AccountEntry, String> {
let text = |k: &str| {
e[k].as_str()
.map(str::to_string)
.ok_or_else(|| format!("an account with no {k}: {e}"))
};
let key = hex::decode(text("key")?)
.ok()
.and_then(|b| crate::cache::Key::from_slice(&b))
.ok_or_else(|| format!("an account with a bad key: {e}"))?;
let optional = |k: &str| match &e[k] {
Json::Null => Ok(None),
Json::String(p) => Ok(Some(p.clone())),
_ => Err(format!("an account with a bad {k}: {e}")),
};
let (password, email) = (optional("password")?, optional("email")?);
let kem = match optional("kemKey")? {
None => None,
Some(k) => Some(
hex::decode(k)
.ok()
.and_then(|b| pds_space::KemKey::from_bytes(&b).ok())
.ok_or_else(|| format!("an account with a bad kemKey: {e}"))?,
),
};
if password.is_some() != email.is_some() {
return Err(format!(
"an account with a password and no email, or an email and no password: {e}"
));
}
if !e["didDoc"].is_object() {
return Err(format!("an account with no didDoc: {e}"));
}
Ok(AccountEntry {
did: text("did")?,
handle: text("handle")?,
email,
password,
key,
kem,
did_doc: e["didDoc"].to_string().into(),
})
}
/// The account, its store held if given, else loaded on first use; its
/// DID document as fetched at `fetched`, or none if it may be out of
/// date; its mail, if not as made, and its state.
pub fn account(
self,
store: Option<Store>,
fetched: Option<u64>,
ctx: &Arc<Ctx>,
mail: Option<Mail>,
state: crate::lifecycle::State,
) -> Arc<Account> {
let mail = mail.unwrap_or_else(|| Mail::new(self.email, self.password));
let a = Arc::new_cyclic(|me| Account {
did: self.did,
handle: RwLock::new(self.handle),
mail: Mutex::new(mail),
kem: self.kem,
did_doc: RwLock::new((self.did_doc, fetched)),
fetching: tokio::sync::Mutex::new(()),
store: Slot::new(me.clone(), self.key, ctx.clone(), store),
state: RwLock::new(state),
});
a.store.counted();
a
}
}
/// A commit's entry: its rev and block, each record it put, with its CID,
/// and each path it deleted; and what it changed of the tree: the nodes it
/// made, with their bytes, those it took out, each record whose count of
/// paths changed, and the counts after (`checkpoint`). Written as it is
/// serialized, borrowing the commit: no copy of its records is made. Its
/// block and its nodes' bytes are holes of its frame, written as they are
/// (`pds_store::frame_with_holes`); CIDs are hex.
struct CommitEntry<'h, 'a>(&'a Change, &'h Holed<'a>);
impl serde::Serialize for CommitEntry<'_, '_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let (c, w) = (self.0, self.1);
let mut m = s.serialize_map(Some(10))?;
m.serialize_entry("t", "commit")?;
m.serialize_entry("rev", &c.rev)?;
m.serialize_entry("head", &Hole(w, &c.head_bytes))?;
m.serialize_entry(
"puts",
&c.puts
.iter()
.zip(&c.put_cids)
.map(|((k, v), cid)| (k, v, Hex(cid)))
.collect::<Vec<_>>(),
)?;
m.serialize_entry("dels", &c.dels)?;
m.serialize_entry(
"tree",
&c.tree
.iter()
.map(|(cid, b)| (Hex(cid), Hole(w, b)))
.collect::<Vec<_>>(),
)?;
m.serialize_entry("gone", &c.gone.iter().map(|g| Hex(g)).collect::<Vec<_>>())?;
m.serialize_entry(
"refs",
&c.refs
.iter()
.map(|(cid, n, rev)| (Hex(cid), n, rev))
.collect::<Vec<_>>(),
)?;
m.serialize_entry("counts", &[c.counts.0, c.counts.1])?;
m.end()
}
}
/// An event's entry: its seq, when it was made, and its frame, a hole of
/// the log's frame.
struct EventEntry<'h, 'a>(u64, u64, &'a [u8], &'h Holed<'a>);
impl serde::Serialize for EventEntry<'_, '_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let mut m = s.serialize_map(Some(4))?;
m.serialize_entry("t", "event")?;
m.serialize_entry("seq", &self.0)?;
m.serialize_entry("time", &self.1)?;
m.serialize_entry("frame", &Hole(self.3, self.2))?;
m.end()
}
}
/// Bytes in a hole of the frame `Holed` writes: an empty string in its
/// text, and the bytes, as they are, after it.
struct Hole<'h, 'a>(&'h Holed<'a>, &'a [u8]);
impl serde::Serialize for Hole<'_, '_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
let written = s.serialize_str("")?;
self.0.hole(self.1);
Ok(written)
}
}
/// Bytes as hex, written as they are serialized, never held as a string.
struct Hex<'a>(&'a [u8]);
impl serde::Serialize for Hex<'_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
s.collect_str(self)
}
}
impl std::fmt::Display for Hex<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut buf = [0u8; 2048];
for chunk in self.0.chunks(buf.len() / 2) {
let out = &mut buf[..chunk.len() * 2];
hex::encode_to_slice(chunk, out).expect("twice the length");
f.write_str(std::str::from_utf8(out).expect("hex"))?;
}
Ok(())
}
}
/// A commit's frame, JSON with holes: its entry, if any, then `entries`,
/// then the events numbered from `seq`, each with when it was made.
fn commit_frame(
commit: Option<&Change>,
entries: &[Json],
seq: u64,
frames: &[Vec<u8>],
times: &[u64],
) -> Vec<u8> {
// The records, as JSON, are about the size of the events' blocks, and
// so are the nodes and the block, besides the events themselves.
let capacity = frames.iter().map(|f| f.len() * 3).sum::<usize>() + 4096;
pds_store::frame_with_holes(capacity, |w| {
use serde::ser::{SerializeSeq, Serializer};
let mut s = serde_json::Serializer::new(w);
let mut seq_out = s.serialize_seq(None).expect("a frame serializes");
if let Some(c) = commit {
seq_out
.serialize_element(&CommitEntry(c, w))
.expect("a commit serializes");
}
for e in entries {
seq_out.serialize_element(e).expect("an entry serializes");
}
for (i, f) in frames.iter().enumerate() {
seq_out
.serialize_element(&EventEntry(seq + i as u64, times[i], f, w))
.expect("an event serializes");
}
seq_out.end().expect("a frame serializes");
})
}
/// A handle change, recorded before its operation, by its CID, goes to the directory.
pub(crate) fn handle_entry(handle: &str, cid: &str) -> Json {
json!({ "t": "handle", "handle": handle, "cid": cid })
}
/// Append `commit`'s entry, if given, and `entries`, with the events
/// `numbered`, if any, numbered and made as their reservation says, the
/// commit's change kept in the store's memtable: the events to land once
/// the log, and their index entry, are on disk, or, if the append failed,
/// none of them, their numbers left unused. With blobs in a bucket, a
/// commit that stops the account holding a blob has it marked for a start
/// to load first, so that the blob's object is deleted (`blobstore`).
fn append_events(
events: &Firehose,
st: &mut Store,
entries: &[Json],
numbered: Option<(&Reserved, Vec<Vec<u8>>)>,
commit: Option<&Change>,
) -> io::Result<Option<Landing>> {
let (r, frames) = match numbered {
Some((r, frames)) => (Some(r), frames),
None => (None, Vec::new()),
};
let (seq, at) = r.map_or((0, 0), |r| (r.seq, r.at));
let fail = |e: io::Error| {
if !frames.is_empty() {
events.fail(seq, frames.len());
}
e
};
if st.ctx.bucket.is_some()
&& st.blobs.dooms()
&& let Err(e) = st.ctx.eager.mark(&st.repo.did)
{
return Err(fail(e));
}
let times = vec![at; frames.len()];
let bytes = commit_frame(commit, entries, seq, &frames, ×);
let pos = match st.wal.append_frame_at(&bytes) {
Ok((_, pos)) => pos,
Err(e) => return Err(fail(e)),
};
st.disk
.indexed(&st.ctx.blocks, pos, &bytes[pds_store::HEADER..]);
drop(bytes);
if let Some(c) = commit {
st.disk.mem.commit(pos, c);
}
for (i, t) in times.iter().enumerate() {
st.disk.mem.event(seq + i as u64, *t, pos);
}
let Some(r) = r else {
return Ok(None);
};
st.last_seq = st.last_seq.max(seq + frames.len() as u64 - 1);
Ok(Some(
r.landing(frames.into_iter().map(Bytes::from).collect()),
))
}
/// Settle the account's handle change in flight, under its account's lock:
/// it landed, with its `#identity` event in the same frame, or it was
/// dropped. Answers the event to land once the log is on disk.
pub(crate) fn log_settled(
events: &Firehose,
did: &str,
st: &mut Store,
cid: &str,
landed: Option<&str>,
) -> io::Result<Option<Landing>> {
let settled = json!({ "t": "handled", "cid": cid, "landed": landed.is_some() });
let Some(handle) = landed else {
st.wal.append(&[settled])?;
st.pending = None;
return Ok(None);
};
let r = events.reserve(did, 1)?;
let frame = events::identity_event(did, handle, r.seq, &r.time());
let landing = append_events(events, st, &[settled], Some((&r, vec![frame])), None)?;
st.pending = None;
st.logged_handle = Some(handle.to_string());
Ok(landing)
}
/// Log a commit made under its account's lock, with its event; land the
/// event once the log is on disk.
pub(crate) fn log_commit(
events: &Firehose,
did: &str,
st: &mut Store,
c: &Change,
) -> io::Result<Landing> {
let r = events.reserve(did, 1)?;
let frame = events::commit_event(did, r.seq, &r.time(), c);
let landing = append_events(events, st, &[], Some((&r, vec![frame])), Some(c))?;
Ok(landing.expect("a commit's event"))
}
/// Log a repository imported whole (`spec/MIGRATION.md`), its commit made
/// under its account's lock: with its `#sync` event if the account is
/// `active`, to land once the log is on disk, or with none.
pub(crate) fn log_import(
events: &Firehose,
did: &str,
st: &mut Store,
c: &Change,
active: bool,
) -> io::Result<Option<Landing>> {
if !active {
return append_events(events, st, &[], None, Some(c));
}
let r = events.reserve(did, 1)?;
let frame = events::sync_event(did, r.seq, &r.time(), &c.head_bytes, &c.rev);
append_events(events, st, &[], Some((&r, vec![frame])), Some(c))
}
/// Make a new repository's first commit, the empty repository, and log it
/// with the events a new account's stream starts with; with `quiet`, an
/// account moving here (`spec/MIGRATION.md`), with none.
pub(crate) fn first_commit(
events: &Firehose,
did: &str,
handle: &str,
st: &mut Store,
quiet: bool,
) -> io::Result<Option<Landing>> {
let mut c = st.repo.init();
let landed = if quiet {
append_events(events, st, &[], None, Some(&c))
} else {
events.reserve(did, 4).and_then(|r| {
let frames = events::initial_events(did, handle, r.seq, &r.time(), &c);
append_events(events, st, &[], Some((&r, frames)), Some(&c))
})
};
if landed.is_err()
&& let Some(u) = c.undo.take()
{
st.repo.undo(u);
}
landed
}
/// An account's store as its checkpoint and log have it.
pub(crate) struct Recovered {
pub store: Store,
/// Whether it has no commit yet.
pub fresh: bool,
}
/// Try `f` again while it fails for want of a file descriptor, waiting for
/// one to be closed, up to a point.
pub(crate) fn retrying<T>(mut f: impl FnMut() -> io::Result<T>) -> io::Result<T> {
for _ in 0..200 {
match f() {
Err(e) if matches!(Errno::from_io_error(&e), Some(Errno::MFILE | Errno::NFILE)) => {
std::thread::sleep(std::time::Duration::from_millis(50))
}
done => return done,
}
}
f()
}
/// Load an account's store: its checkpoint's table opened, the logs after
/// it replayed, or begun; compacted from now on as `store::policy` says,
/// once the account's slot sets its fold (`cache::Slot`). `durable` says
/// that every frame its log's files hold is on disk, and nothing of them
/// is then synced but what is mended (`checkpoint::load`).
pub(crate) fn open_repo(
ctx: &Arc<Ctx>,
did: &str,
key: &SigningKey,
durable: bool,
) -> Result<Recovered, String> {
let dir = &ctx.dir;
let layout = crate::store::account_layout(dir, did);
let at = |why: String| format!("{}: {why}", layout.dir.display());
crate::store::remove_legacy(dir, Some(did)).map_err(|e| at(e.to_string()))?;
// Uploads on their way when the server stopped: not this process's own.
if let Ok(entries) = std::fs::read_dir(blob_dir(dir, did)) {
for e in entries.flatten() {
let name = e.file_name();
let name = name.to_string_lossy();
if name.starts_with(".tmp-") && !name.starts_with(crate::blobs::temp_prefix()) {
std::fs::remove_file(e.path()).map_err(|why| at(why.to_string()))?;
}
}
}
let crate::checkpoint::Loaded {
repo,
wal,
disk,
mut small,
} = crate::checkpoint::load(ctx, did, key, durable)?;
// Spaces state of no version but this one is let go, once: from its
// mark on, the log is of this version's (`checkpoint::unmarked`).
if small.unmarked.is_some() {
wal.append(&[crate::checkpoint::mark()])
.map_err(|e| at(e.to_string()))?;
small.mark();
}
match &ctx.bucket {
None => small.blobs.recover(&blob_dir(dir, did))?,
Some(bucket) => {
small.blobs.recover_kept();
// An upload the server stopped during, or an object not yet deleted.
if small.blobs.strays().next().is_some() {
bucket.wake(did);
}
}
}
let fresh = repo.rev.is_empty();
Ok(Recovered {
store: Store {
repo,
wal: held(dir, wal),
blobs: small.blobs,
takedowns: small.takedowns,
spaces: small.spaces,
authority: small.authority,
notify: small.notify,
disk,
last_seq: small.last_seq,
logged_handle: small.handle,
pending: small.pending,
ctx: ctx.clone(),
#[cfg(test)]
thread: std::thread::current().id(),
},
fresh,
})
}
/// Begin a new account's log, with its mark of spaces state of this
/// version (`checkpoint::unmarked`): its store, with no commit yet.
pub(crate) fn new_repo(ctx: &Arc<Ctx>, did: &str, key: &SigningKey) -> io::Result<Store> {
let layout = crate::store::account_layout(&ctx.dir, did);
let (wal, frames) = Wal::open_log(&layout, 0, 0, Some(ctx.files.clone()), ctx.policy)?;
if !frames.is_empty() {
return Err(io::Error::other("its log holds something already"));
}
wal.append(&[crate::checkpoint::mark()])?;
Ok(Store {
repo: Repo::new(did.to_string(), key.clone()),
wal: held(&ctx.dir, wal),
blobs: Default::default(),
takedowns: Default::default(),
spaces: Default::default(),
authority: Default::default(),
notify: Default::default(),
disk: crate::checkpoint::Disk::empty(layout),
last_seq: 0,
logged_handle: None,
pending: None,
ctx: ctx.clone(),
#[cfg(test)]
thread: std::thread::current().id(),
})
}
/// Load the stores of `accounts`, side by side on at most as many threads
/// as the machine has cores, this one among them: the first error, if one
/// cannot be.
fn load_all(accounts: &[Arc<Account>]) -> Result<(), String> {
let next = AtomicUsize::new(0);
let failed: Mutex<Option<String>> = Mutex::new(None);
let work = || {
while failed
.lock()
.unwrap_or_else(PoisonError::into_inner)
.is_none()
{
let i = next.fetch_add(1, Ordering::Relaxed);
let Some(a) = accounts.get(i) else { return };
if let Err(why) = a.store.load() {
failed
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get_or_insert(why);
}
}
};
let threads = std::thread::available_parallelism()
.map_or(1, |n| n.get())
.min(accounts.len())
.max(1);
std::thread::scope(|s| {
for i in 1..threads {
if std::thread::Builder::new()
.name(format!("pds-recover-{i}"))
.spawn_scoped(s, work)
.is_err()
{
break;
}
}
work();
});
failed
.into_inner()
.unwrap_or_else(PoisonError::into_inner)
.map_or(Ok(()), Err)
}
impl Service {
/// Start from the logs in `dir`: read the service checkpoint's summary
/// and its entries that are not one account's, and replay the service
/// log after it, or begin it. Of an account's entries in the log, only
/// where they are is noted (`directory`): no account is loaded, nor its
/// own log read, but those the service log marks (`cache`), or every one
/// with `Config::load_all`, or on a data directory from before seqs
/// were reserved (`events::SEQS`).
pub(crate) fn open(dir: &FsPath, config: Config) -> Result<Service, String> {
use std::os::unix::fs::PermissionsExt;
#[cfg(test)]
stop_writers(dir);
pds_store::make_dirs(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
// Whether the last server left every log durable, in this very
// directory, asked before anything of it is changed or written:
// no log is then synced as it is opened, the service log now and
// each account's on its first use. The directory's mode is set
// after, which changes when it last changed: the word names that
// too, and from here on names this directory no more.
let durable = crate::ops::durable(dir)?;
std::fs::set_permissions(dir, std::fs::Permissions::from_mode(pds_store::DIR_MODE))
.map_err(|e| format!("{}: {e}", dir.display()))?;
// What the journal holds of writes answered before a kill or a
// power cut is put back where it is kept, before anything else of
// the directory is read or changed, whichever way this run syncs.
// To sync together, the journal begins again, adopting the segments
// where that start left them, for its first checkpoint to make
// durable and let go; to sync each by itself, there is no journal
// to adopt them, and the start makes them durable and deletes them.
let journal = match config.sync {
crate::Syncing::Together => {
let redone = pds_store::journal::redo(dir, pds_store::journal::Redo::Adopting)
.map_err(|e| e.to_string())?;
Some(pds_store::Journal::open(dir, redone.adopted).map_err(|e| e.to_string())?)
}
crate::Syncing::Each => {
pds_store::journal::redo(dir, pds_store::journal::Redo::Eager)
.map_err(|e| e.to_string())?;
None
}
};
#[cfg(test)]
if let Some(j) = &journal {
JOURNALS
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push((dir.to_path_buf(), Arc::downgrade(j)));
}
// What this run finds is durable already where that was so, or
// where it finds no data; else once it has made it so
// (`ops::sync_found`).
let found = durable || !crate::store::holds_data(dir);
crate::store::migrate(dir)?;
crate::store::remove_legacy(dir, None).map_err(|e| format!("{}: {e}", dir.display()))?;
let layout = crate::store::service_layout(dir);
let window = config.backfill.as_millis() as u64;
// A checkpoint of the format before a table is made one, once.
let now_ms = events::now_ms();
crate::compact::convert(&layout, now_ms / 1000, |_, f| {
crate::store::in_window(f, now_ms, window)
})?;
// Of `PDS_CACHE_BYTES`: half the stores held, a quarter the blocks
// read, and the firehose's tail (`events::TAIL`) the rest.
let blocks = Arc::new(pds_store::table::BlockCache::with_open(
config.cache_bytes / 4,
config.share,
));
let table = ServiceTable::open(&layout.checkpoint(), blocks.clone())?;
let (generation, size, service_seq, mut ceiling) =
table.as_ref().map_or((0, 0, 0, None), |t| {
(
t.summary.generation,
t.size(),
t.summary.last_seq,
t.summary.ceiling,
)
});
let new = table.is_none();
// No account loaded: what the log holds of each is noted in it.
let mut accounts = Directory::new(layout.clone(), table);
// The accounts marked.
let mut marked: HashSet<String> = HashSet::new();
// Keyed once the replay has said whether the data directory is new.
let mut sessions = Sessions::unkeyed(&config.hostname);
let oauth =
crate::oauth::OAuth::open(dir, &config.public_url, config.oauth_client_fetch.clone())?;
let replay = crate::replay::Replay::default();
let (single_use, space_credentials) = (
crate::single_use::Spent::default(),
crate::space_credentials::Credentials::default(),
);
let now = sessions.now();
let invites = crate::invites::Invites::default();
// The events this log holds, and the places of a deleted account's.
let mut lifecycle = crate::lifecycle::Replayed::default();
// Deleted accounts' objects in a bucket not yet deleted.
let mut doomed = crate::blobstore::Doomed::default();
// Replay an entry that is not one account's, given what is noted of
// the accounts so far: answers the DID of an account it deletes.
let mut apply = |e: &Json, accounts: &Directory| -> Result<Option<String>, String> {
let at = |why: String| format!("{ACCOUNTS_LOG}: {why}");
match e["t"].as_str() {
Some(crate::replay::ENTRY) => replay.replay(e, now).map_err(at)?,
Some(t) if crate::oauth::ENTRIES.contains(&t) => {
oauth.replay(e, &|did| accounts.may_be(did)).map_err(at)?
}
Some(crate::single_use::ENTRY) => single_use.replay(e, now).map_err(at)?,
Some(crate::space_credentials::ENTRY) => {
space_credentials.replay(e, now).map_err(at)?
}
Some(t) if crate::invites::ENTRIES.contains(&t) => invites.replay(e).map_err(at)?,
Some(t) if crate::lifecycle::ENTRIES.contains(&t) => {
if let Some(did) = lifecycle.replay(e).map_err(at)? {
// Deleted: it is no account; its files, if any are
// left, go once the server serves.
marked.remove(&did);
oauth.forget_account(&did);
return Ok(Some(did));
}
}
Some(crate::cache::DIR) => {
let (Some(did), Some(eager)) = (e["did"].as_str(), e["eager"].as_bool()) else {
return Err(at(format!("a mark with no did or eager: {e}")));
};
// Nothing is kept of an account deleted.
if accounts.is_noted_deleted(did) {
return Ok(None);
}
if !accounts.may_be(did) {
return Err(at(format!("a mark of {did:?}, which no account has")));
}
if eager {
marked.insert(did.to_string());
} else {
marked.remove(did);
}
}
Some(t) if crate::blobstore::ENTRIES.contains(&t) => {
doomed.replay(e).map_err(at)?
}
Some(crate::events::SEQS) => {
let upto = e["upto"]
.as_u64()
.ok_or_else(|| at(format!("seqs with no upto: {e}")))?;
ceiling = Some(ceiling.unwrap_or(0).max(upto));
}
t => return Err(at(format!("a log entry of no known type: {t:?}"))),
}
Ok(None)
};
// The table's entries that are not one account's, a row at a time.
if let Some(t) = accounts.table() {
t.rest(|entries| {
entries
.iter()
.try_for_each(|e| apply(e, &accounts).map(drop))
})
.map_err(|why| format!("{}: {why}", layout.checkpoint().display()))?;
}
// The logs after it, replayed a frame at a time as they are read:
// an entry of one account is noted, not applied.
let mut frames = 0;
let mut failed = None;
let log = Wal::open_log_with(
&layout,
generation,
size,
None,
crate::store::service_policy(&config),
durable,
&mut |pos, entries| {
frames += 1;
let replayed: Result<(), String> = entries.iter().try_for_each(|e| {
let of = crate::compact::of_account(e)
.map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?;
// A mark is the account's, for its handle, and not.
if (of.is_none() || e["t"] == crate::cache::DIR)
&& let Some(did) = apply(e, &accounts)?
{
accounts.note_deleted(&did, pos.generation);
}
if let Some(did) = of {
accounts
.note(pos, did, e)
.map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?;
}
Ok(())
});
if let Err(why) = &replayed {
failed = Some(why.clone());
}
replayed
},
);
let log = match (log, failed) {
(_, Some(why)) => return Err(why),
(Ok(log), None) => log,
(Err(e), None) => return Err(format!("{ACCOUNTS_LOG}: {e}")),
};
// In no pool, and given the journal all the same.
if let Some(j) = &journal {
log.use_journal(j.clone());
}
sessions.keyed(dir, new && frames == 0)?;
oauth.recovered(dir, sessions.now())?;
// New events are numbered after every seq reserved before (`events::SEQS`).
let logged = lifecycle
.last_seq()
.max(service_seq)
.max(lifecycle.gaps.iter().copied().max().unwrap_or(0));
let last = logged.max(ceiling.unwrap_or(0));
// A data directory from before seqs were reserved: every account's
// log is read for the greatest seq it holds.
let upgrade = ceiling.is_none() && accounts.count() > 0;
let log = held(dir, log);
let logged_events = lifecycle
.events
.drain(..)
.map(|e| events::Recovered {
seq: e.seq,
did: Arc::from(e.did.as_str()),
frame: Bytes::from(e.frame),
})
.collect();
let files = match journal {
Some(j) => Files::with_journal(config.share, j),
None => Files::new(config.share),
};
let firehose = Firehose::new(
logged_events,
lifecycle.gaps.iter().copied().collect(),
last,
logged,
config.backfill,
log.clone(),
events::index::At {
dir: dir.to_path_buf(),
pool: files.clone(),
policy: crate::store::service_policy(&config),
durable,
},
);
let mut marks: Vec<String> = marked.iter().cloned().collect();
marks.sort_unstable();
let eager = Arc::new(crate::cache::Eager::new(log.clone(), marked));
let bucket = match config.blobstore.clone() {
Some(s) => {
let b = crate::blobstore::Bucket::new(s)?;
b.doomed(doomed);
Some(Arc::new(b))
}
None => None,
};
let ctx = Arc::new(Ctx {
dir: dir.to_path_buf(),
files: files.clone(),
policy: crate::store::policy(&config),
window,
events: firehose.clone(),
eager: eager.clone(),
cache: crate::cache::Cache::new(config.cache_bytes / 2),
blocks: blocks.clone(),
starting: std::sync::atomic::AtomicBool::new(true),
durable,
found: std::sync::atomic::AtomicBool::new(found),
failed: std::sync::atomic::AtomicBool::new(false),
bucket,
});
let limits = crate::limits::Limits::new(dir, &config);
let outbound = crate::outbound::Outbound::new(config.ssrf_protection)?;
let load_every = config.load_all || upgrade;
let directory = Arc::new(Accounts::new(layout, blocks, accounts));
// Compacted from now on, its table put in place with the directory held.
log.set_fold(Arc::new(crate::store::ServiceFold {
window,
accounts: Arc::downgrade(&directory),
}));
let svc = Service {
space_notify: crate::space_notify::Notify::new(),
config,
dir: dir.to_path_buf(),
log,
ctx: ctx.clone(),
directory,
sessions,
oauth,
replay,
invites,
events: firehose,
http: crate::plc::client().map_err(|e| format!("an HTTP client: {e}"))?,
changes: crate::identity::Changes::default(),
space_hooks: crate::space_repo::Hooks::default(),
single_use,
space_credentials,
space_authority: crate::space_authority::Host::default(),
limits,
resets: email::Resets::default(),
outbound,
did_docs: Default::default(),
handle_resolver: Default::default(),
prefs: Default::default(),
};
// The accounts a start loads: those marked, or every one.
let dids = if load_every {
crate::read(&svc.directory).dids()
} else {
marks.clone()
};
let mut loading: Vec<Arc<Account>> = Vec::with_capacity(dids.len());
for did in &dids {
loading.extend(svc.build(did)?);
}
{
let made: HashSet<&str> = loading.iter().map(|a| a.did.as_str()).collect();
if let Some(did) = marks.iter().find(|did| !made.contains(did.as_str())) {
return Err(format!(
"{ACCOUNTS_LOG}: a mark of {did:?}, which no account has"
));
}
}
load_all(&loading)?;
// With every account's log read, the greatest seq any holds is known:
// the next follows it, whatever was reserved.
let mut last = if load_every { logged } else { last };
// Each one's handle change in flight, if it has one.
let mut pending = Vec::with_capacity(loading.len());
for a in &loading {
let st = crate::read(&a.store);
last = last.max(st.last_seq);
// Its handle is the one its last landed change gave it.
if let Some(h) = st.logged_handle.as_ref().filter(|h| **h != a.handle()) {
*crate::write(&a.handle) = h.clone();
}
pending.push(st.pending.clone());
}
// Each handle, an account's or one a change in flight holds, and
// each email, is one account's, of those loaded.
{
let mut d = crate::write(&svc.directory);
for (a, pending) in loading.iter().zip(pending) {
d.insert_loaded(a.clone())
.map_err(|why| format!("{ACCOUNTS_LOG}: {why}"))?;
let Some(p) = pending else { continue };
// The handle stays taken until the change is settled.
if p.handle != a.handle() {
if let Some(other) = d.handles.get(&p.handle) {
return Err(format!(
"{ACCOUNTS_LOG}: {} is the handle of both {other} and {}",
p.handle, a.did
));
}
if !d.claimed_handles.insert(p.handle.clone()) {
return Err(format!(
"{ACCOUNTS_LOG}: {} is the handle two accounts are being given",
p.handle
));
}
}
svc.changes.unsettled(&a.did, p);
}
}
if load_every {
svc.events.exact(last);
} else {
svc.events
.raise(last)
.map_err(|e| format!("{ACCOUNTS_LOG}: {e}"))?;
}
// A data directory with accounts and no index of their events is
// given one, of every account's files, before it serves.
svc.events
.adopt(&svc)
.map_err(|e| format!("the firehose's index: {e}"))?;
// An account whose repository has no commit yet gets its first,
// with no events if it was made deactivated, moving here.
for a in &loading {
let mut st = crate::write(&a.store);
if st.repo.rev.is_empty() {
let quiet = crate::read(&a.state).deactivated;
let landing = first_commit(&svc.events, &a.did, &a.handle(), &mut st, quiet)
.map_err(|e| format!("{}: {e}", a.did))?;
landing
.as_ref()
.map_or(Ok(()), Landing::wait_entry)
.and_then(|()| st.wal.wait(st.wal.written()))
.map_err(|e| format!("{}: {e}", a.did))?;
if let Some(l) = landing {
svc.events.land(l);
}
}
// Loaded and needing no start: unmarked, with the handle it has.
if eager.is_marked(&a.did) && !st.needs_start() {
eager.relax(&a.did, Some(&a.handle()));
}
}
ctx.starting.store(false, Ordering::SeqCst);
crate::space_notify::resume(&svc);
Ok(svc)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sessions;
use crate::testing::*;
use pds_store as store;
use rustix::process::{Resource, Rlimit, Signal, getrlimit, setrlimit};
use std::collections::BTreeMap;
use std::path::Path;
/// Everything a restart must carry over, as one comparable value: each
/// account's repository, rev and KEM key, the sessions, the events, and
/// the invite codes, each account's email and the tokens mailed for it,
/// and each account's state, its records and blobs taken down.
type State = (
BTreeMap<String, (Vec<u8>, String, Option<String>)>,
BTreeMap<String, sessions::Keep>,
Vec<Vec<u8>>,
Vec<String>,
Mails,
Lives,
);
/// Each account's mail, the directory's emails, and which account each reset token is of.
type Mails = (
BTreeMap<String, Mail>,
BTreeMap<String, String>,
BTreeMap<String, String>,
);
/// Each account's state, its records taken down, and its blobs; and the directory's handles.
type Lives = (
BTreeMap<String, (crate::lifecycle::State, BTreeMap<String, String>, String)>,
BTreeMap<String, String>,
);
fn state(s: &Service) -> State {
let d = crate::read(&s.directory);
let repos = d
.by_did
.iter()
.map(|(did, a)| {
let st = crate::read(&a.store);
(
did.clone(),
(
crate::sync::CarStream::of(&st, None)
.unwrap()
.bytes()
.unwrap(),
st.repo.rev.clone(),
a.kem.as_ref().map(|k| hex::encode(k.to_bytes())),
),
)
})
.collect();
let mails = d
.by_did
.iter()
.map(|(did, a)| (did.clone(), email::lock(&a.mail).clone()))
.collect();
let emails = d
.emails
.iter()
.map(|(e, did)| (e.clone(), did.clone()))
.collect();
let lives = d
.by_did
.iter()
.map(|(did, a)| {
let st = crate::read(&a.store);
(
did.clone(),
(
crate::read(&a.state).clone(),
st.takedowns.clone(),
st.blobs.snapshot(),
),
)
})
.collect();
let handles = d
.handles
.iter()
.map(|(h, did)| (h.clone(), did.clone()))
.collect();
drop(d);
let sessions = s.sessions.snapshot();
let events = s
.events
.get(s, 0, u64::MAX - 1, s.events.drops())
.iter()
.map(|e| e.to_vec())
.collect();
(
repos,
sessions,
events,
s.invites.snapshot(),
(mails, emails, s.resets.snapshot()),
(lives, handles),
)
}
/// The files with the service log as a kill cut short in an account's
/// log could have left it: without the notes of handles and the marks
/// let go, which are written only once a frame of that log is on disk
/// (`cache::Eager::relax`). The marks made before stay.
fn unnoted(mut files: BTreeMap<PathBuf, Vec<u8>>) -> BTreeMap<PathBuf, Vec<u8>> {
let p = PathBuf::from(ACCOUNTS_LOG_FILE);
if let Some(bytes) = files.get(&p) {
let note = |e: &Json| {
e["t"] == crate::cache::DIR
&& (e.get("handle").is_some() || e["eager"] == json!(false))
};
let frames = store::scan(bytes).unwrap().0;
let kept: Vec<u8> = frames
.iter()
.filter(|f| !f.iter().all(note))
.flat_map(|f| store::frame(f))
.collect();
files.insert(p, kept);
}
files
}
/// The files a step changed, but the service log, where marks go
/// (`cache`), and the firehose's index, which a step that makes events
/// appends their entries to (`events::index`).
fn changed_logs<'a>(
before: &BTreeMap<PathBuf, Vec<u8>>,
after: &'a BTreeMap<PathBuf, Vec<u8>>,
) -> Vec<&'a PathBuf> {
after
.keys()
.filter(|p| {
before.get(*p) != after.get(*p)
&& p.as_path() != Path::new(ACCOUNTS_LOG_FILE)
&& !of_index(p)
})
.collect()
}
/// Whether a file of the data directory, by its path there, is the
/// firehose's index's (`events::index`).
fn of_index(p: &Path) -> bool {
p.to_str().is_some_and(|p| p.starts_with("events."))
}
/// `files` with the index's files as `from` holds them.
fn with_index_of(
mut files: BTreeMap<PathBuf, Vec<u8>>,
from: &BTreeMap<PathBuf, Vec<u8>>,
) -> BTreeMap<PathBuf, Vec<u8>> {
files.retain(|p, _| !of_index(p));
files.extend(
from.iter()
.filter(|(p, _)| of_index(p))
.map(|(p, b)| (p.clone(), b.clone())),
);
files
}
/// Of a step that made events, the index cut short at every byte of
/// the entries it appended, the step's other writes lost, as a power
/// cut may leave it, with `room` zeros after the cut where it is given:
/// recovery must give the state before the step. Answers how many cuts.
fn index_cut_short(work: &Path, plc: &str, w: &[Step], room: Option<usize>) -> usize {
let mut cuts = 0;
for (p, whole) in w[1]
.files
.iter()
.filter(|(p, b)| of_index(p) && w[0].files.get(*p) != Some(*b))
{
let from = w[0].files.get(p).map_or(0, Vec::len);
for cut in from + 1..whole.len() {
let mut files = w[0].files.clone();
let bytes = match room {
Some(room) => with_room(&whole[..cut], whole.len() - cut + room),
None => whole[..cut].to_vec(),
};
files.insert(p.clone(), bytes);
put_files(work, &files);
assert_eq!(
&state(&open(work, plc)),
&w[0].state,
"{} cut at {cut} of {from}..{}, the step's other writes lost",
p.display(),
whole.len()
);
cuts += 1;
}
}
cuts
}
/// An event's body, from its frame: its `seq`, `did` and the rest.
fn body_of(frame: &[u8]) -> Json {
let (_, at) = pds_repo::data::decode_prefix(frame).unwrap();
pds_repo::data::to_json(&pds_repo::data::decode(&frame[at..]).unwrap())
}
/// The events a subscription from cursor 0 begun now is sent, each its
/// body.
fn sent(s: &Service) -> Vec<Json> {
s.events
.get(s, 0, u64::MAX - 1, s.events.drops())
.iter()
.map(|f| body_of(f))
.collect()
}
/// The seqs of the events a subscription from cursor 0 is sent, in order.
fn sent_seqs(s: &Service) -> Vec<u64> {
sent(s).iter().map(|e| e["seq"].as_u64().unwrap()).collect()
}
/// Every file in the data directory, and its bytes: of a log, those it
/// holds, without the room after its frames (`pds_store`), as a power
/// cut the room did not outlive leaves it. The journal's segments are
/// left out: what they hold is in the logs, as the system has them, and
/// a step's frames are cut or kept in their logs alone, as after a
/// checkpoint (`journal_cut_at_every_byte` cuts the journal).
fn files(dir: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
let mut files = files_whole(dir);
files.retain(|p, _| !p.starts_with(pds_store::journal::DIR));
for (_, bytes) in files
.iter_mut()
.filter(|(p, _)| p.extension().is_some_and(|e| e == "log"))
{
*bytes = super::without_room(std::mem::take(bytes));
}
files
}
/// Every file in the data directory, and its bytes, room and all.
fn files_whole(dir: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
let mut out = BTreeMap::new();
let mut todo = vec![dir.to_path_buf()];
while let Some(d) = todo.pop() {
for e in std::fs::read_dir(&d).unwrap() {
let p = e.unwrap().path();
if p.is_dir() {
todo.push(p);
} else {
out.insert(
p.strip_prefix(dir).unwrap().to_path_buf(),
std::fs::read(&p).unwrap(),
);
}
}
}
out
}
/// Make `dir` hold exactly these files, writing only those that differ,
/// so that the logs recovery makes durable are mostly durable already.
fn put_files(dir: &Path, files: &BTreeMap<PathBuf, Vec<u8>>) {
let have = if dir.exists() {
files_whole(dir)
} else {
BTreeMap::new()
};
for p in have.keys().filter(|p| !files.contains_key(*p)) {
std::fs::remove_file(dir.join(p)).unwrap();
}
for (p, bytes) in files.iter().filter(|(p, b)| have.get(*p) != Some(*b)) {
let p = dir.join(p);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, bytes).unwrap();
}
}
/// A log, or the first of one, with `room` zeros after it: as a kill
/// leaves a log that was open (`pds_store`).
fn with_room(log: &[u8], room: usize) -> Vec<u8> {
[log, &vec![0u8; room]].concat()
}
/// The room each log in the data directory holds: the zeros after its
/// frames.
fn rooms(dir: &Path) -> BTreeMap<PathBuf, usize> {
files_whole(dir)
.into_iter()
.filter(|(p, _)| p.extension().is_some_and(|e| e == "log"))
.map(|(p, whole)| {
let room = whole.len() - super::without_room(whole).len();
(p, room)
})
.collect()
}
/// A step of the script: the files after it, the journal's segments
/// apart, each without its room, and the state.
struct Step {
files: BTreeMap<PathBuf, Vec<u8>>,
journal: BTreeMap<PathBuf, Vec<u8>>,
state: State,
}
/// The journal's segments in the data directory, each without its room.
fn segments(dir: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
let mut j = files_whole(dir);
j.retain(|p, _| p.starts_with(pds_store::journal::DIR));
for bytes in j.values_mut() {
*bytes = super::without_room(std::mem::take(bytes));
}
j
}
/// Run a script of requests over two accounts; after each, the files and the state.
async fn script(dir: &Path, plc: &str) -> (Vec<Step>, Vec<(String, String)>) {
let s = open(dir, plc);
let mut steps = Vec::new();
let snap = |s: &Service, steps: &mut Vec<Step>| {
steps.push(Step {
files: files(dir),
journal: segments(dir),
state: state(s),
})
};
snap(&s, &mut steps);
let (a, ta) = create(&s, "alice").await;
snap(&s, &mut steps);
let (b, tb) = create(&s, "bob").await;
snap(&s, &mut steps);
let ta2 = session(&s, "alice").await;
snap(&s, &mut steps);
let w = |did: &str, k: &str, n: i64| json!({ "repo": did, "collection": "a.b.c", "rkey": k, "record": record(n) });
call(&s, "com.atproto.repo.putRecord", Some(&ta), w(&a, "one", 1))
.await
.unwrap();
snap(&s, &mut steps);
call(&s, "com.atproto.repo.putRecord", Some(&tb), w(&b, "one", 1))
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.repo.putRecord",
Some(&ta2),
w(&a, "two", 2),
)
.await
.unwrap();
snap(&s, &mut steps);
call(&s, "com.atproto.repo.applyWrites", Some(&tb), json!({ "repo": b, "writes": [
{ "$type": "com.atproto.repo.applyWrites#create", "collection": "a.b.c", "rkey": "three", "value": record(3) },
{ "$type": "com.atproto.repo.applyWrites#update", "collection": "a.b.c", "rkey": "one", "value": record(4) } ] })).await.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.repo.deleteRecord",
Some(&ta),
json!({ "repo": a, "collection": "a.b.c", "rkey": "one" }),
)
.await
.unwrap();
snap(&s, &mut steps);
// A refused write changes nothing and logs nothing.
assert!(
call(
&s,
"com.atproto.repo.applyWrites",
Some(&ta),
json!({ "repo": b, "writes": [] })
)
.await
.is_err()
);
snap(&s, &mut steps);
let n = steps.len();
assert_eq!(steps[n - 1].files, steps[n - 2].files);
(steps, vec![(a, ta), (b, tb)])
}
#[tokio::test]
async fn restart_resumes_every_account() {
let plc = plc().await;
let dir = scratch("resume");
let (steps, accounts) = script(&dir, &plc).await;
let s = open(&dir, &plc);
assert_eq!(state(&s), steps.last().unwrap().state);
// Four events open each account's stream; each of the five commits adds one.
assert_eq!(s.events.len(), 4 + 4 + 5);
assert_eq!(*s.events.subscribe().borrow(), s.events.len() as u64);
drop(s);
let s = open(&dir, &plc);
for (i, (did, token)) in accounts.iter().enumerate() {
let before = repo_of(&s, did, |r| r.rev.clone());
let body =
json!({ "repo": did, "collection": "a.b.c", "rkey": "x", "record": { "n": 1 } });
call(&s, "com.atproto.repo.putRecord", Some(token), body)
.await
.unwrap();
assert!(repo_of(&s, did, |r| r.rev.clone()) > before);
// The firehose carries on from the last event.
let n = s.events.len();
assert_eq!(n, 13 + i + 1);
let last = s.events.get(&s, n as u64 - 1, n as u64, 0).pop().unwrap();
let (_, at) = pds_repo::data::decode_prefix(&last).unwrap();
let body = pds_repo::data::to_json(&pds_repo::data::decode(&last[at..]).unwrap());
assert_eq!(body["seq"], json!(n));
assert_eq!(body["repo"], json!(did));
}
// Accounts and sessions too: each logs in again, and each old token works.
session(&s, "alice").await;
for (did, token) in &accounts {
assert_eq!(
call(&s, "com.atproto.server.getSession", Some(token), json!({}))
.await
.unwrap()["did"],
json!(did)
);
}
let _ = std::fs::remove_dir_all(&dir);
}
/// A data directory written before logs had room, each log its frames
/// alone, is started on and carried on from. Its logs are given room as
/// they are written to, which no count of their bytes includes; and a
/// server killed with that room in them starts as one killed without.
/// Under `each`: a log given a journal (`together`) makes no room.
#[tokio::test]
async fn logs_with_no_room_are_carried_on() {
syncing(crate::Syncing::Each);
let plc = plc().await;
let dir = scratch("room-src");
let (steps, accounts) = script(&dir, &plc).await;
let last = steps.last().unwrap();
let work = scratch("room-work");
put_files(&work, &last.files);
assert_eq!(files_whole(&work), last.files);
let s = open(&work, &plc);
assert_eq!(state(&s), last.state);
for n in 0..3 {
for (did, token) in &accounts {
let body = json!({ "repo": did, "collection": "a.b.c", "rkey": format!("more{n}"), "record": record(n) });
call(&s, "com.atproto.repo.putRecord", Some(token), body)
.await
.unwrap();
}
}
let (killed, bare) = (files_whole(&work), files(&work));
for (did, _) in &accounts {
let log = repo_log(Path::new(""), did);
let (file, frames) = (killed[&log].len(), bare[&log].len());
assert!(file > frames, "{file} bytes of file, {frames} of frames");
assert_eq!(store::scan(&killed[&log]).unwrap().1, frames);
// The log's bytes are its frames', by its own count and by its file.
let counted = s.account(did).unwrap().store.log_bytes();
assert_eq!(counted, frames as u64);
assert_eq!(crate::directory::files_bytes(&work, did), counted);
}
let after = state(&s);
drop(s);
for (name, disk) in [("room-killed", &killed), ("room-bare", &bare)] {
let again = scratch(name);
put_files(&again, disk);
let s = open(&again, &plc);
assert_eq!(state(&s), after, "{name}");
let (bob, token) = &accounts[1];
let body =
json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": record(9) });
call(&s, "com.atproto.repo.putRecord", Some(token), body)
.await
.unwrap();
let now = state(&s);
drop(s);
assert_eq!(state(&open(&again, &plc)), now, "{name}");
let _ = std::fs::remove_dir_all(&again);
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Room is held by the logs open to be written to, and no others: a
/// store let go leaves its account's log its frames alone, and a
/// shutdown every log. A kill leaves each log the room it had: the
/// service log until the start after it, an account's until its store
/// is next loaded. Under `each`: a log given a journal (`together`)
/// makes no room.
#[tokio::test]
async fn room_is_held_by_open_logs_alone() {
syncing(crate::Syncing::Each);
let plc = plc().await;
let dir = scratch("room-held");
let s = open(&dir, &plc);
let (alice, ta) = create(&s, "alice").await;
let (bob, tb) = create(&s, "bob").await;
for (did, token) in [(&alice, &ta), (&bob, &tb)] {
let body =
json!({ "repo": did, "collection": "a.b.c", "rkey": "one", "record": record(1) });
call(&s, "com.atproto.repo.putRecord", Some(token), body)
.await
.unwrap();
}
let service = PathBuf::from(ACCOUNTS_LOG_FILE);
let [a, b] = [&alice, &bob].map(|did| repo_log(Path::new(""), did));
let held = rooms(&dir);
assert!(held[&a] > 0 && held[&b] > 0, "{held:?}");
// Let go, a store leaves its log its frames alone.
assert!(s.account(&alice).unwrap().store.evict());
let held = rooms(&dir);
assert!(held[&a] == 0 && held[&b] > 0, "{held:?}");
let mut killed = files_whole(&dir);
let log = killed.get_mut(&service).unwrap();
*log = with_room(&super::without_room(std::mem::take(log)), 1500);
// A shutdown leaves every log so.
crate::ops::settle(&s).unwrap();
assert!(rooms(&dir).values().all(|room| *room == 0));
drop(s);
// Killed, the logs keep their room: the service log until the start
// after, an account's until its store is next loaded.
let work = scratch("room-held-killed");
put_files(&work, &killed);
let s = open_with(&work, &plc, |c| c.load_all = false);
assert!(!s.account(&bob).unwrap().store.is_loaded());
let held = rooms(&work);
assert!(held[&service] == 0 && held[&b] > 0, "{held:?}");
assert_eq!(
crate::directory::files_bytes(&work, &bob),
super::without_room(killed[&b].clone()).len() as u64
);
s.account(&bob).unwrap().store.load().unwrap();
assert_eq!(rooms(&work)[&b], 0);
drop(s);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Logs whose frames are intact but whose entries are not what the
/// server wrote are refused with a reason, not a panic.
#[tokio::test]
async fn bad_entries_are_refused() {
let plc = plc().await;
let dir = scratch("bad-src");
let (steps, accounts) = script(&dir, &plc).await;
let last = &steps.last().unwrap().files;
let work = scratch("bad-work");
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let alice = repo_log(Path::new(""), &accounts[0].0);
let frames = |p: &PathBuf| store::scan(&last[p]).unwrap().0;
let reopen = |p: &PathBuf, frames: &[Vec<Json>]| {
let mut files = last.clone();
files.insert(
p.clone(),
frames.iter().flat_map(|f| store::frame(f)).collect(),
);
put_files(&work, &files);
try_open(&work, &plc).err()
};
assert_eq!(reopen(&alice, &frames(&alice)), None);
// A record the latest commit did not sign.
let mut tampered = frames(&alice);
let commit = tampered
.iter_mut()
.rev()
.find_map(|f| {
f.iter_mut()
.find(|e| e["t"] == "commit" && !e["puts"].as_array().unwrap().is_empty())
})
.unwrap();
commit["puts"][0][1]["n"] = json!(99);
assert!(reopen(&alice, &tampered).unwrap().contains("does not sign"));
for (at, bad) in [
(1, json!({ "t": "nonsense" })),
(
1,
json!({ "t": "event", "ord": 1, "seq": 1, "frame": "00" }),
),
(2, json!({ "t": "commit", "rev": "x" })),
] {
let mut broken = frames(&alice);
broken[at].push(bad.clone());
assert!(reopen(&alice, &broken).is_some(), "{bad}");
}
let a = accounts[0].0.as_str();
let session = |did: &str, app: Json, scope: &str| json!({ "t": "session", "did": did, "id": "s", "app": app, "scope": scope, "exp": 1 });
for bad in [
json!({ "t": "nonsense" }),
json!({ "t": "session", "did": a }),
session("did:plc:nobody", Json::Null, "com.atproto.access"),
session(a, Json::Null, "com.atproto.appPass"),
// A session of an app password not had is kept only with an app password's scope (`dropAppPasswords`).
session(a, json!("no such app password"), "com.atproto.access"),
json!({ "t": "refresh", "did": a, "id": "not kept", "next": "n", "exp": 1, "nextExp": 2 }),
json!({ "t": "endSession", "did": a, "id": "not kept" }),
json!({ "t": "appPassword", "did": a, "name": "x", "hash": "h", "createdAt": "c" }),
json!({ "t": "revokeAppPassword", "did": a, "name": "not had" }),
json!({ "t": "useToken", "did": a, "purpose": "confirm_email" }),
json!({ "t": "mailToken", "did": a, "purpose": "nonsense", "token": "AAAAA-AAAAA", "made": 1 }),
json!({ "t": "mailToken", "did": a, "purpose": "confirm_email", "token": "AAAAAAAAAA", "made": 1 }),
json!({ "t": "mailToken", "did": a, "purpose": "confirm_email", "token": "AAAAA-AAAAA" }),
json!({ "t": "mailToken", "did": "did:plc:nobody", "purpose": "confirm_email", "token": "AAAAA-AAAAA", "made": 1 }),
json!({ "t": "emailAuthFactor", "did": a, "on": true }),
json!({ "t": "updateEmail", "did": a, "email": "bob@mail.com" }),
json!({ "t": "updateEmail", "did": a }),
json!({ "t": "password", "did": a }),
] {
let mut broken = frames(&accounts_log);
broken[1].push(bad.clone());
assert!(reopen(&accounts_log, &broken).is_some(), "{bad}");
}
// The same session twice.
let mut twice = frames(&accounts_log);
let first = twice
.iter()
.flatten()
.find(|e| e["t"] == "session")
.unwrap()
.clone();
twice.push(vec![first]);
assert!(
reopen(&accounts_log, &twice)
.unwrap()
.contains("a second session")
);
// A second account with the first's handle.
// A second account with the first's email, or its handle.
let mut twice = frames(&accounts_log);
let mut copy = twice
.iter()
.flatten()
.find(|e| e["t"] == "account")
.unwrap()
.clone();
copy["did"] = json!("did:plc:another");
twice.push(vec![copy.clone()]);
assert!(
reopen(&accounts_log, &twice)
.unwrap()
.contains("a second account")
);
copy["email"] = json!("another@mail.com");
*twice.last_mut().unwrap() = vec![copy];
assert!(
reopen(&accounts_log, &twice)
.unwrap()
.contains("is the handle of both")
);
// A KEM key that is not one.
for bad in [
json!("zz"),
json!("00"),
json!(hex::encode([0u8; 32])),
json!(1),
] {
let mut broken = frames(&accounts_log);
broken
.iter_mut()
.flatten()
.find(|e| e["t"] == "account")
.unwrap()["kemKey"] = bad.clone();
assert!(
reopen(&accounts_log, &broken)
.unwrap()
.contains("bad kemKey"),
"{bad}"
);
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// An account from before accounts had a KEM key is given none: it
/// starts without one, its credentials name only its signing key, and
/// its entry stays as it was.
#[tokio::test]
async fn an_account_without_a_kem_key() {
let plc = plc().await;
let dir = scratch("no-kem");
let s = open(&dir, &plc);
let (alice, token) = create(&s, "alice").await;
drop(s);
redone(&dir);
let path = accounts_log(&dir);
let mut frames = store::scan(&std::fs::read(&path).unwrap()).unwrap().0;
let entry = frames
.iter_mut()
.flatten()
.find(|e| e["t"] == "account")
.unwrap();
assert!(entry["kemKey"].is_string());
entry.as_object_mut().unwrap().remove("kemKey");
let bytes: Vec<u8> = frames.iter().flat_map(|f| store::frame(f)).collect();
std::fs::write(&path, &bytes).unwrap();
let s = open(&dir, &plc);
assert!(s.account(&alice).unwrap().kem.is_none());
let got = call(
&s,
"com.atproto.identity.getRecommendedDidCredentials",
Some(&token),
json!({}),
)
.await
.unwrap();
assert_eq!(
got["verificationMethods"]
.as_object()
.unwrap()
.keys()
.collect::<Vec<_>>(),
["atproto"]
);
drop(s);
assert_eq!(std::fs::read(&path).unwrap(), bytes);
let _ = std::fs::remove_dir_all(&dir);
}
/// Cut each log at every byte of the last frame the script wrote to it
/// (and flip a byte in it): recovery must give the state before that
/// write or after it, and keep working.
#[tokio::test]
async fn torn_writes_recover_to_before_or_after() {
for sync in SYNCS {
syncing(sync);
torn_writes_once().await;
}
}
async fn torn_writes_once() {
let plc = plc().await;
let dir = scratch("torn-src");
let (steps, accounts) = script(&dir, &plc).await;
let (bob, token) = &accounts[1];
let work = scratch("torn-work");
let mut cuts = 0;
// The steps after both accounts exist, each of which changes one log
// besides the index, which takes the entries of the events a write
// makes before the frame that holds them.
let changes = |w: &[Step]| -> Vec<PathBuf> {
w[1].files
.keys()
.filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p))
.cloned()
.collect()
};
let mut index_cuts = 0;
let windows: Vec<&[Step]> = steps.windows(2).skip(2).collect();
for (i, w) in windows.iter().enumerate() {
let changed = changes(w);
let [path] = &changed[..] else {
assert_eq!(changed.len(), 0);
continue;
};
if windows[i + 1..]
.iter()
.any(|later| changes(later).contains(path))
{
continue;
}
let (from, to) = (w[0].files[path].len(), w[1].files[path].len());
let whole = &w[1].files[path];
for cut in from..=to {
let mut files = w[1].files.clone();
files.insert(path.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let s = open(&work, &plc);
let want = if cut == to { &w[1].state } else { &w[0].state };
assert_eq!(
&state(&s),
want,
"{} cut at {cut} of {from}..{to}",
path.display()
);
cuts += 1;
// After recovery the logs take new writes and replay them.
if cut % 61 == 0 || cut == from + 1 || cut == to {
let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": { "n": 9 } });
call(&s, "com.atproto.repo.putRecord", Some(token), body)
.await
.unwrap();
let now = state(&s);
drop(s);
assert_eq!(
state(&open(&work, &plc)),
now,
"{} cut at {cut}",
path.display()
);
}
}
// A damaged byte anywhere in the frame drops it.
for at in (from..to).step_by(5) {
let mut files = w[1].files.clone();
files.get_mut(path).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(
&state(&open(&work, &plc)),
&w[0].state,
"{} flip at {at} of {from}..{to}",
path.display()
);
}
// The write's entries cut short, its frame lost.
index_cuts += index_cut_short(&work, &plc, w, None);
}
assert!(cuts > 500, "{cuts}");
assert!(index_cuts > 100, "{index_cuts}");
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// A power cut as `PDS_SYNC` `together` leaves it, the journal cut at
/// every byte of a write (`spec/DURABILITY.md`, "Writes waiting at
/// once"): of each step that writes one log, the log as it was before
/// the step, its frames never synced in it, a log the step made there
/// and empty, as its making was synced; and the journal's segment cut
/// at every byte of what the step wrote to it, every record before it
/// whole. A start puts back what the journal holds whole: the state
/// before the step, or after it once the step's records are whole.
#[tokio::test]
async fn journal_cut_at_every_byte() {
let plc = plc().await;
let dir = scratch("journal-cut-src");
let (steps, _) = script(&dir, &plc).await;
let work = scratch("journal-cut-work");
let (mut cuts, mut stepped) = (0, 0);
for w in steps.windows(2) {
let changed: Vec<&PathBuf> = w[1]
.files
.keys()
.filter(|p| p.extension().is_some_and(|e| e == "log"))
.filter(|p| w[0].files.get(*p) != w[1].files.get(*p))
.collect();
if changed.is_empty() {
continue;
}
// The segment the step wrote to, grown from what it was: none
// where a checkpoint came between, which is no cut of this.
let Some((segment, from, to)) = w[1].journal.iter().find_map(|(p, after)| {
let before = w[0].journal.get(p)?;
(after.len() > before.len() && after.starts_with(before))
.then(|| (p.clone(), before.len(), after.len()))
}) else {
continue;
};
stepped += 1;
let mut files = w[1].files.clone();
for p in &changed {
files.insert(
(*p).clone(),
w[0].files.get(*p).cloned().unwrap_or_default(),
);
}
for cut in from..=to {
let mut disk = files.clone();
disk.extend(w[1].journal.clone());
disk.insert(segment.clone(), w[1].journal[&segment][..cut].to_vec());
put_files(&work, &disk);
let s = open(&work, &plc);
let got = state(&s);
if cut == to {
assert_eq!(
got,
w[1].state,
"{} cut at {cut} of {from}..{to}",
segment.display()
);
} else if cut == from {
assert_eq!(got, w[0].state, "{} cut at {cut}", segment.display());
} else {
assert!(
got == w[0].state || got == w[1].state,
"{} cut at {cut} of {from}..{to}",
segment.display()
);
}
drop(s);
cuts += 1;
}
}
assert!(stepped >= 5 && cuts > 500, "{stepped} steps, {cuts} cuts");
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// A frame written over room and cut at every byte, the room after the
/// cut as it was: the shortest write the script made to an account's
/// log, and the one it made to the service log alone. Recovery must
/// give the state before that write or after it, and keep working, as
/// where the file ends at the cut
/// (`torn_writes_recover_to_before_or_after`).
#[tokio::test]
async fn writes_torn_in_room_recover_to_before_or_after() {
for sync in SYNCS {
syncing(sync);
writes_torn_in_room_once().await;
}
}
async fn writes_torn_in_room_once() {
let plc = plc().await;
let dir = scratch("torn-room-src");
let (steps, accounts) = script(&dir, &plc).await;
let (bob, token) = &accounts[1];
let work = scratch("torn-room-work");
let service = PathBuf::from(ACCOUNTS_LOG_FILE);
// Of the steps that change one log, the service log or an account's,
// the one of each kind whose frame is shortest.
let mut shortest: BTreeMap<bool, (usize, &[Step], &PathBuf)> = BTreeMap::new();
for w in steps.windows(2).skip(2) {
let changed: Vec<&PathBuf> = w[1]
.files
.keys()
.filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p))
.collect();
let [path] = changed[..] else { continue };
let len = w[1].files[path].len() - w[0].files[path].len();
let of = shortest.entry(*path == service).or_insert((len, w, path));
if len < of.0 {
*of = (len, w, path);
}
}
assert_eq!(shortest.len(), 2, "a write to each kind of log");
let mut cuts = 0;
for (_, w, path) in shortest.into_values() {
let (from, to) = (w[0].files[path].len(), w[1].files[path].len());
let whole = &w[1].files[path];
assert!(whole.starts_with(&w[0].files[path]));
for cut in from..=to {
// What of the frame reached the disk, then zeros to past
// where it ends.
let mut files = w[1].files.clone();
files.insert(path.clone(), with_room(&whole[..cut], to - cut + 1500));
put_files(&work, &files);
let s = open(&work, &plc);
let want = if cut == to { &w[1].state } else { &w[0].state };
assert_eq!(
&state(&s),
want,
"{} cut at {cut} of {from}..{to}",
path.display()
);
cuts += 1;
if cut % 61 == 0 || cut == from + 1 || cut == to {
let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": "after", "record": { "n": 9 } });
call(&s, "com.atproto.repo.putRecord", Some(token), body)
.await
.unwrap();
let now = state(&s);
drop(s);
assert_eq!(
state(&open(&work, &plc)),
now,
"{} cut at {cut}",
path.display()
);
}
}
// The write's entries cut short in room, its frame lost.
cuts += index_cut_short(&work, &plc, w, Some(1500));
}
assert!(cuts > 500, "{cuts}");
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut `accounts.log` at every byte of each frame a change to sessions or
/// app passwords wrote (and flip a byte in it): recovery must give the
/// state before that change or after it. Each kind of entry is cut: the
/// account starts with as many sessions as it keeps, so that the next
/// one drops one.
#[tokio::test]
async fn torn_session_writes() {
for sync in SYNCS {
syncing(sync);
torn_session_writes_once().await;
}
}
async fn torn_session_writes_once() {
let plc = plc().await;
let dir = scratch("torn-sessions");
let s = open(&dir, &plc);
let (alice, _) = create(&s, "alice").await;
let tokens = |got: Json| {
(
got["accessJwt"].as_str().unwrap().to_string(),
got["refreshJwt"].as_str().unwrap().to_string(),
)
};
let login = |pw: &str| json!({ "identifier": "alice.test", "password": pw });
let (_, mut chain) = tokens(
call(
&s,
"com.atproto.server.createSession",
None,
login("alice-pw"),
)
.await
.unwrap(),
);
for _ in 2..sessions::MAX_SESSIONS {
chain = tokens(
call(
&s,
"com.atproto.server.refreshSession",
Some(&chain),
Json::Null,
)
.await
.unwrap(),
)
.1;
}
assert_eq!(s.sessions.count(&alice), sessions::MAX_SESSIONS);
let mut steps = vec![Step {
files: files(&dir),
journal: segments(&dir),
state: state(&s),
}];
let snap = |s: &Service, steps: &mut Vec<Step>| {
steps.push(Step {
files: files(&dir),
journal: segments(&dir),
state: state(s),
})
};
let (access, r0) = tokens(
call(
&s,
"com.atproto.server.createSession",
None,
login("alice-pw"),
)
.await
.unwrap(),
);
assert_eq!(s.sessions.count(&alice), sessions::MAX_SESSIONS);
snap(&s, &mut steps);
let app = call(
&s,
"com.atproto.server.createAppPassword",
Some(&access),
json!({ "name": "phone", "privileged": true }),
)
.await
.unwrap();
snap(&s, &mut steps);
let (_, app_r0) = tokens(
call(
&s,
"com.atproto.server.createSession",
None,
login(app["password"].as_str().unwrap()),
)
.await
.unwrap(),
);
snap(&s, &mut steps);
let (_, r1) = tokens(
call(
&s,
"com.atproto.server.refreshSession",
Some(&r0),
Json::Null,
)
.await
.unwrap(),
);
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.refreshSession",
Some(&app_r0),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.deleteSession",
Some(&r1),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.revokeAppPassword",
Some(&access),
json!({ "name": "phone" }),
)
.await
.unwrap();
snap(&s, &mut steps);
drop(s);
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let work = scratch("torn-sessions-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let changed: Vec<&PathBuf> = w[1]
.files
.keys()
.filter(|p| w[0].files.get(*p) != w[1].files.get(*p))
.collect();
assert_eq!(changed, [&accounts_log]);
assert_ne!(w[0].state, w[1].state);
let whole = &w[1].files[&accounts_log];
let (from, to) = (w[0].files[&accounts_log].len(), whole.len());
let (frames, _) = store::scan(&whole[from..]).unwrap();
kinds.extend(
frames
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = w[1].files.clone();
files.insert(accounts_log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { &w[1].state } else { &w[0].state };
assert_eq!(
&state(&open(&work, &plc)),
want,
"cut at {cut} of {from}..{to}"
);
}
for at in (from..to).step_by(3) {
let mut files = w[1].files.clone();
files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(
&state(&open(&work, &plc)),
&w[0].state,
"flip at {at} of {from}..{to}"
);
}
}
kinds.sort();
kinds.dedup();
// A deactivation with an OAuth token writes `dropAppPasswords`, in its
// lifecycle frame: compact.rs's account_states_end_oauth_sessions.
let mut want: Vec<&str> = crate::sessions::ENTRIES
.iter()
.copied()
.filter(|k| *k != "dropAppPasswords")
.collect();
want.sort();
assert_eq!(kinds, want);
// After recovery from the last cut, the log takes changes, and replays them.
put_files(&work, &steps.last().unwrap().files);
let s = open(&work, &plc);
call(
&s,
"com.atproto.server.createSession",
None,
login("alice-pw"),
)
.await
.unwrap();
let now = state(&s);
drop(s);
assert_eq!(state(&open(&work, &plc)), now);
assert_ne!(now.1[&alice], steps.last().unwrap().state.1[&alice]);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut the log each lifecycle change wrote at every byte of its frame
/// (and flip a byte in it): recovery must give the state before that
/// change or after it. A deactivation, an activation, a takedown with
/// the sessions it ends and its lifting, an event of a DID no account
/// has, a record's and a blob's takedown, a deletion's token, and a
/// deletion, whose account's files may be there still or not: each
/// kind of entry is cut.
#[tokio::test]
async fn torn_lifecycle_writes() {
for sync in SYNCS {
syncing(sync);
torn_lifecycle_writes_once().await;
}
}
async fn torn_lifecycle_writes_once() {
use base64::Engine;
let plc = plc().await;
let dir = scratch("torn-life");
let sink = scratch("torn-life-mail");
let s = crate::email::testing::open_with_sink(&dir, &plc, &sink);
let (alice, ta) = create(&s, "alice").await;
let (bob, tb) = create(&s, "bob").await;
session(&s, "alice").await;
let blob = crate::blobs::test_upload(
&s,
Some(&ta),
Some("text/plain"),
b"taken down".to_vec(),
false,
)
.await
.unwrap()["blob"]
.clone();
let w = |k: &str, v: Json| json!({ "repo": alice, "collection": "a.b.c", "rkey": k, "record": v });
call(
&s,
"com.atproto.repo.putRecord",
Some(&ta),
w("one", json!({ "b": blob })),
)
.await
.unwrap();
let admin = format!(
"Basic {}",
base64::engine::general_purpose::STANDARD.encode(format!("admin:{ADMIN_PASSWORD}"))
);
let update = |body: Json| {
let (s, admin) = (s.clone(), admin.clone());
async move {
call_raw(
&s,
"com.atproto.admin.updateSubjectStatus",
sessions::Bearer::of(Some(&admin)),
&[],
&body.to_string(),
)
.await
.unwrap()
}
};
let repo_ref = |did: &str| json!({ "$type": "com.atproto.admin.defs#repoRef", "did": did });
let mut steps = vec![Step {
files: files(&dir),
journal: segments(&dir),
state: state(&s),
}];
let snap = |s: &Service, steps: &mut Vec<Step>| {
steps.push(Step {
files: files(&dir),
journal: segments(&dir),
state: state(s),
})
};
call(
&s,
"com.atproto.server.deactivateAccount",
Some(&ta),
json!({}),
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.activateAccount",
Some(&ta),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
update(json!({ "subject": repo_ref(&alice), "takedown": { "applied": true, "ref": "r" } }))
.await;
snap(&s, &mut steps);
update(json!({ "subject": repo_ref(&alice), "takedown": { "applied": false } })).await;
snap(&s, &mut steps);
update(json!({ "subject": repo_ref("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa") })).await;
snap(&s, &mut steps);
let uri = format!("at://{alice}/a.b.c/one");
update(json!({ "subject": { "$type": "com.atproto.repo.strongRef", "uri": uri, "cid": blob["ref"]["$link"] }, "takedown": { "applied": true } })).await;
snap(&s, &mut steps);
let blob_ref = json!({ "$type": "com.atproto.admin.defs#repoBlobRef", "did": alice, "cid": blob["ref"]["$link"] });
update(json!({ "subject": blob_ref, "takedown": { "applied": true, "ref": "b" } })).await;
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.requestAccountDelete",
Some(&tb),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
let token = crate::email::testing::last_token(&sink);
call(
&s,
"com.atproto.server.deleteAccount",
None,
json!({ "did": bob, "password": "bob-pw", "token": token }),
)
.await
.unwrap();
snap(&s, &mut steps);
drop(s);
let work = scratch("torn-life-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
// The log a change wrote to, but the index, which takes the
// entries of the events it makes before the frame that holds them.
let written: Vec<&PathBuf> = w[1]
.files
.keys()
.filter(|p| !of_index(p) && w[0].files.get(*p) != w[1].files.get(*p))
.collect();
let [path] = written[..] else {
panic!("{written:?}")
};
let indexed = w[1]
.files
.iter()
.any(|(p, b)| of_index(p) && w[0].files.get(p) != Some(b));
// A deletion's account's files go once it is logged: a cut keeps them.
let gone: Vec<&PathBuf> = w[0]
.files
.keys()
.filter(|p| !w[1].files.contains_key(*p))
.collect();
assert_ne!(w[0].state, w[1].state);
let whole = &w[1].files[path];
let (from, to) = (w[0].files[path].len(), whole.len());
let (frames, _) = store::scan(&whole[from..]).unwrap();
kinds.extend(
frames
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = if gone.is_empty() {
w[1].files.clone()
} else {
with_index_of(w[0].files.clone(), &w[1].files)
};
files.insert(path.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { &w[1].state } else { &w[0].state };
let s = open(&work, &plc);
assert_eq!(
&state(&s),
want,
"{} cut at {cut} of {from}..{to}",
path.display()
);
if cut == to {
// What a deletion left of its account is gone once the
// directories of no account are swept.
s.sweep_repos();
assert!(gone.iter().all(|p| !work.join(p).exists()));
drop(s);
// The change's frame whole and its entries lost: the
// service log's own events need none to be sent.
if indexed {
put_files(&work, &with_index_of(files, &w[0].files));
assert_eq!(
&state(&open(&work, &plc)),
want,
"{} whole, its entries lost",
path.display()
);
}
}
}
for at in (from..to).step_by(7) {
let mut files = if gone.is_empty() {
w[1].files.clone()
} else {
w[0].files.clone()
};
let mut flipped = whole.clone();
flipped[at] ^= 0x55;
files.insert(path.clone(), flipped);
put_files(&work, &files);
assert_eq!(
&state(&open(&work, &plc)),
&w[0].state,
"{} flip at {at} of {from}..{to}",
path.display()
);
}
// The change's entries cut short, its frame lost.
if indexed {
assert!(index_cut_short(&work, &plc, w, None) > 0);
}
}
kinds.sort();
kinds.dedup();
for k in crate::lifecycle::ENTRIES.iter().chain(&[
crate::lifecycle::RECORD_TAKEDOWN,
"takedownBlob",
"endSession",
"mailToken",
]) {
assert!(kinds.iter().any(|x| x == k), "{k} not cut: {kinds:?}");
}
// After recovery from the last cut, the logs take changes, and replay them.
put_files(&work, &steps.last().unwrap().files);
let s = open(&work, &plc);
call(
&s,
"com.atproto.server.deactivateAccount",
Some(&ta),
json!({}),
)
.await
.unwrap();
let now = state(&s);
drop(s);
assert_eq!(state(&open(&work, &plc)), now);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut `accounts.log` at every byte of each frame a change to an
/// account's email, password or tokens wrote (and flip a byte in it):
/// recovery must give the state before that change or after it, the
/// sessions a reset password ends with it. Each kind of entry is cut.
#[tokio::test]
async fn torn_email_writes() {
for sync in SYNCS {
syncing(sync);
torn_email_writes_once().await;
}
}
async fn torn_email_writes_once() {
use crate::email::testing::{last_token, open_with_sink};
let plc = plc().await;
let dir = scratch("torn-email");
let sink = scratch("torn-email-mail");
let s = open_with_sink(&dir, &plc, &sink);
let (_, access) = create(&s, "alice").await;
create(&s, "bob").await;
let mut steps = vec![Step {
files: files(&dir),
journal: segments(&dir),
state: state(&s),
}];
let snap = |s: &Service, steps: &mut Vec<Step>| {
steps.push(Step {
files: files(&dir),
journal: segments(&dir),
state: state(s),
})
};
let token = || last_token(&sink);
let login = |code: Option<String>| {
let mut b = json!({ "identifier": "alice.test", "password": "alice-pw" });
if let Some(c) = code {
b["authFactorToken"] = json!(c);
}
b
};
let factor = |on: bool, token: Option<String>| {
let mut b = json!({ "email": "alice@mail.com", "emailAuthFactor": on });
if let Some(t) = token {
b["token"] = json!(t);
}
b
};
call(
&s,
"com.atproto.server.requestEmailConfirmation",
Some(&access),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.confirmEmail",
Some(&access),
json!({ "email": "alice@mail.com", "token": token() }),
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.updateEmail",
Some(&access),
factor(true, None),
)
.await
.unwrap();
snap(&s, &mut steps);
assert_eq!(
call(&s, "com.atproto.server.createSession", None, login(None)).await,
Err("AuthFactorTokenRequired".into())
);
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.createSession",
None,
login(Some(token())),
)
.await
.unwrap();
snap(&s, &mut steps);
assert_eq!(
call(
&s,
"com.atproto.server.updateEmail",
Some(&access),
factor(false, None)
)
.await,
Err("TokenRequired".into())
);
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.updateEmail",
Some(&access),
factor(false, Some(token())),
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.requestEmailUpdate",
Some(&access),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.updateEmail",
Some(&access),
json!({ "email": "alice2@mail.com", "token": token() }),
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.requestPasswordReset",
None,
json!({ "email": "alice2@mail.com" }),
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.resetPassword",
None,
json!({ "token": token(), "password": "new" }),
)
.await
.unwrap();
snap(&s, &mut steps);
drop(s);
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let work = scratch("torn-email-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let changed: Vec<&PathBuf> = w[1]
.files
.keys()
.filter(|p| w[0].files.get(*p) != w[1].files.get(*p))
.collect();
assert_eq!(changed, [&accounts_log]);
assert_ne!(w[0].state, w[1].state);
let whole = &w[1].files[&accounts_log];
let (from, to) = (w[0].files[&accounts_log].len(), whole.len());
let (frames, _) = store::scan(&whole[from..]).unwrap();
assert_eq!(frames.len(), 1);
kinds.extend(
frames
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = w[1].files.clone();
files.insert(accounts_log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { &w[1].state } else { &w[0].state };
assert_eq!(
&state(&open(&work, &plc)),
want,
"cut at {cut} of {from}..{to}"
);
}
for at in (from..to).step_by(3) {
let mut files = w[1].files.clone();
files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(
&state(&open(&work, &plc)),
&w[0].state,
"flip at {at} of {from}..{to}"
);
}
}
for k in crate::email::ENTRIES {
assert!(kinds.iter().any(|got| got == k), "no {k} entry was cut");
}
assert!(kinds.iter().any(|k| k == "endSession"));
// After recovery from the last cut, the log takes changes, and replays them.
put_files(&work, &steps.last().unwrap().files);
let s = open_with_sink(&work, &plc, &sink);
call(
&s,
"com.atproto.server.requestPasswordReset",
None,
json!({ "email": "alice2@mail.com" }),
)
.await
.unwrap();
let now = state(&s);
drop(s);
assert_eq!(state(&open(&work, &plc)), now);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&sink);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut `accounts.log` at every byte of each frame `putPreferences` wrote
/// (and flip a byte in it): recovery must give every account's
/// preferences as before that put or after it.
#[tokio::test]
async fn torn_preference_writes() {
for sync in SYNCS {
syncing(sync);
torn_preference_writes_once().await;
}
}
async fn torn_preference_writes_once() {
let plc = plc().await;
let dir = scratch("torn-prefs");
let cfg = || {
let mut c = config(&plc);
c.proxy.appview = Some(crate::proxy::AppView {
url: "http://127.0.0.1:1".into(),
did: "did:web:appview.test".into(),
});
c
};
let s = Arc::new(Service::open(&dir, cfg()).unwrap());
let (_, alice) = create(&s, "alice").await;
let (_, bob) = create(&s, "bob").await;
let prefs = |s: &Service| s.prefs.snapshot();
let mut steps = vec![(files(&dir), prefs(&s))];
let puts = [
(
&alice,
json!([{ "$type": "app.bsky.actor.defs#adultContentPref", "enabled": true }]),
),
(&bob, json!([{ "$type": "app.bsky.x" }])),
(&alice, json!([])),
(
&alice,
json!([{ "$type": "app.bsky.actor.defs#personalDetailsPref", "birthDate": "2000-01-01T00:00:00Z" }, { "$type": "app.bsky.y", "n": "é" }]),
),
];
for (token, list) in puts {
call(
&s,
"app.bsky.actor.putPreferences",
Some(token),
json!({ "preferences": list }),
)
.await
.unwrap();
steps.push((files(&dir), prefs(&s)));
}
drop(s);
let log = PathBuf::from(ACCOUNTS_LOG_FILE);
let work = scratch("torn-prefs-work");
let recovered = || prefs(&Service::open(&work, cfg()).unwrap());
let mut cuts = 0;
for w in steps.windows(2) {
let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
assert_ne!(before, after);
// Preferences are the service log's own.
let changed: Vec<&PathBuf> = after_files
.keys()
.filter(|p| before_files.get(*p) != after_files.get(*p))
.collect();
assert_eq!(changed, [&log]);
let whole = &after_files[&log];
let (from, to) = (before_files[&log].len(), whole.len());
let (frames, _) = store::scan(&whole[from..]).unwrap();
assert!(
frames
.iter()
.flatten()
.all(|e| e["t"] == crate::preferences::ENTRY)
);
for cut in from..=to {
let mut files = unnoted(after_files.clone());
files.insert(log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
assert_eq!(
&recovered(),
if cut == to { after } else { before },
"cut at {cut} of {from}..{to}"
);
cuts += 1;
}
for at in (from..to).step_by(7) {
let mut files = after_files.clone();
files.get_mut(&log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(&recovered(), before, "flip at {at} of {from}..{to}");
}
}
assert!(cuts > 100, "{cuts}");
// After recovery the log takes more, and replays them.
put_files(&work, &steps.last().unwrap().0);
let s = Arc::new(Service::open(&work, cfg()).unwrap());
call(
&s,
"app.bsky.actor.putPreferences",
Some(&bob),
json!({ "preferences": [] }),
)
.await
.unwrap();
assert_eq!(recovered(), prefs(&s));
assert_eq!(prefs(&s).len(), 1);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut `accounts.log` at every byte of each frame an OAuth change wrote
/// (and flip a byte in it): recovery must give what OAuth keeps, and the
/// single-use values, as before that change or after it. Each kind of
/// entry is cut.
#[tokio::test]
async fn torn_oauth_writes() {
for sync in SYNCS {
syncing(sync);
torn_oauth_writes_once().await;
}
}
async fn torn_oauth_writes_once() {
use crate::oauth::tests::{App, Browser, held_at, kept, serve, text};
let plc = plc().await;
let dir = scratch("torn-oauth");
let srv = serve(open(&dir, &plc)).await;
let (alice, alice_token) = create(&srv.svc, "alice").await;
let mut steps = vec![(files(&dir), kept(&srv.svc))];
let snap = |steps: &mut Vec<_>| steps.push((files(&dir), kept(&srv.svc)));
let mut app = App::new(&srv, "atproto");
let mut browser = Browser::new(&srv);
let cid = app.client_id.clone();
let (_, pushed) = app.par(&[]).await;
snap(&mut steps);
browser.page(&cid, &pushed.request_uri).await;
snap(&mut steps);
// The device's cookies from before the sign-in, which it renews.
let old_cookies = browser.cookies.clone();
let sign_in = json!({ "locale": "en", "identifier": "alice.test", "password": "alice-pw", "remember": true });
browser
.api(&cid, &pushed.request_uri, "sign-in", sign_in, None)
.await;
snap(&mut steps);
let consent = browser
.api(
&cid,
&pushed.request_uri,
"consent",
json!({ "did": alice }),
None,
)
.await
.json();
let url = url::Url::parse(consent["url"].as_str().unwrap()).unwrap();
let code = url
.query_pairs()
.find(|(k, _)| k == "code")
.unwrap()
.1
.into_owned();
snap(&mut steps);
let t = app.exchange(&code, &pushed).await.json();
snap(&mut steps);
let r = app.refresh(&text(&t, "refresh_token")).await.json();
snap(&mut steps);
app.get_session(&text(&r, "access_token")).await;
snap(&mut steps);
app.revoke(&text(&r, "access_token")).await;
snap(&mut steps);
// A code refused, and spent.
let (other, code) = crate::oauth::tests::approved(&mut app, &mut browser, "alice").await;
steps.push((files(&dir), kept(&srv.svc)));
let wrong = crate::oauth::tests::Pushed {
verifier: "x".repeat(43),
..other
};
app.exchange(&code, &wrong).await;
snap(&mut steps);
let (_, rejected) = app.par(&[]).await;
browser.page(&cid, &rejected.request_uri).await;
steps.push((files(&dir), kept(&srv.svc)));
browser
.api(&cid, &rejected.request_uri, "reject", json!({}), None)
.await;
snap(&mut steps);
browser
.api(
&cid,
&rejected.request_uri,
"sign-out",
json!({ "did": alice }),
None,
)
.await;
snap(&mut steps);
// Signed in again; past 30 seconds, the old cookies end the device.
browser.api(&cid, &rejected.request_uri, "sign-in", json!({ "locale": "en", "identifier": "alice.test", "password": "alice-pw", "remember": true }), None).await;
snap(&mut steps);
srv.svc
.sessions
.skew
.store(31, std::sync::atomic::Ordering::SeqCst);
let mut old = Browser::new(&srv);
old.cookies = old_cookies;
old.api(
&cid,
&rejected.request_uri,
"sign-out",
json!({ "did": alice }),
None,
)
.await;
snap(&mut steps);
// The clients an account approved forgotten, as a deactivation with
// an OAuth token does: one approved first, as a discoverable
// client's is (two frames, not cut).
let now = srv.svc.sessions.now();
let request = json!({
"t": "oauthRequest", "id": "req-forget", "client": "https://app.example/client.json", "redirectUri": "https://app.example/cb",
"scope": ["atproto"], "state": "s", "codeChallenge": "c".repeat(43), "responseMode": "query", "jkt": "j", "expires": now + 300,
});
let approve = json!({ "t": "oauthApprove", "id": "req-forget", "code": "cod-forget", "did": alice, "scope": ["atproto"], "at": now });
for e in [request, approve] {
let n = srv.svc.log.append(std::slice::from_ref(&e)).unwrap();
assert!(crate::durable(srv.svc.log.clone(), n).await.is_ok());
srv.svc.oauth.replay(&e, &|_| true).unwrap();
}
steps.push((files(&dir), kept(&srv.svc)));
let ends = crate::oauth::Ends {
clients: true,
..Default::default()
};
let n = srv
.svc
.oauth
.end_for(&alice, ends, |e| srv.svc.log.append(&e))
.unwrap();
assert!(crate::durable(srv.svc.log.clone(), n).await.is_ok());
snap(&mut steps);
// A passkey added, signing in, and removed (`spec/PASSKEYS.md`):
// its link and challenge are held in memory alone.
let auth = format!("Bearer {alice_token}");
let (mut key, handle) = crate::oauth::passkeys::tests::add(&srv, &auth, "Laptop", 3).await;
snap(&mut steps);
let signed = crate::oauth::passkeys::tests::sign_in(
&mut browser,
&cid,
&rejected.request_uri,
&mut key,
&handle,
true,
)
.await;
assert_eq!(signed.status, 200, "{}", signed.body);
snap(&mut steps);
let id = base64::Engine::encode(&base64::engine::general_purpose::URL_SAFE_NO_PAD, &key.id);
crate::oauth::passkeys::tests::own(
&srv,
"POST",
"_passkeyRemove",
&auth,
Some(json!({ "id": id })),
)
.await;
snap(&mut steps);
drop(srv);
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let work = scratch("torn-oauth-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
let changed: Vec<&PathBuf> = after_files
.keys()
.filter(|p| before_files.get(*p) != after_files.get(*p))
.collect();
assert_eq!(changed, [&accounts_log]);
assert_ne!(before, after);
let whole = &after_files[&accounts_log];
let from = before_files[&accounts_log].len();
let (frames, _) = store::scan(&whole[from..]).unwrap();
if frames.len() != 1 {
// A step of several requests, as approving one: only the last is cut.
continue;
}
kinds.extend(
frames
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
let to = whole.len();
for cut in from..=to {
let mut files = after_files.clone();
files.insert(accounts_log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { after } else { before };
let s = open(&work, &plc);
let now = s.sessions.now();
assert_eq!(
held_at(&(s.oauth.snapshot(), s.replay.snapshot(0)), now),
held_at(want, now),
"cut at {cut} of {from}..{to}"
);
}
for at in (from..to).step_by(7) {
let mut files = after_files.clone();
files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
let s = open(&work, &plc);
let now = s.sessions.now();
assert_eq!(
held_at(&(s.oauth.snapshot(), s.replay.snapshot(0)), now),
held_at(before, now),
"flip at {at} of {from}..{to}"
);
}
}
kinds.sort();
kinds.dedup();
let mut want: Vec<&str> = crate::oauth::ENTRIES.to_vec();
want.push(crate::replay::ENTRY);
want.sort();
assert_eq!(kinds, want);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut `accounts.log` at every byte of each frame a single-use value or
/// a revocation wrote (and flip a byte in it): recovery must give the
/// values and revocations held as before that write or after. Each kind
/// of entry is cut.
#[tokio::test]
async fn torn_space_credential_writes() {
for sync in SYNCS {
syncing(sync);
torn_space_credential_writes_once().await;
}
}
async fn torn_space_credential_writes_once() {
use crate::space_credentials::tests as creds;
let plc = plc().await;
let dir = scratch("torn-space-credentials");
let s = open(&dir, &plc);
let (alice, token) = create(&s, "alice").await;
let (bob, _) = create(&s, "bob").await;
let personal = json!({ "$type": "com.atproto.personalspace.policy" });
call(
&s,
"com.atproto.space.createSpace",
Some(&token),
json!({ "spaceType": "com.example.forum", "skey": "main", "policy": personal }),
)
.await
.unwrap();
type Held = (BTreeMap<[u8; 32], u64>, BTreeMap<[u8; 32], u64>);
let held =
|s: &Service| -> Held { (s.single_use.snapshot(0), s.space_credentials.snapshot(0)) };
let mut steps = vec![(files(&dir), held(&s))];
let space = format!("at://{alice}/space/com.example.forum/main");
let t = creds::delegation_token(&s, &token, &space).await.unwrap();
creds::credential(
&s,
json!({ "space": space, "credentialKey": creds::client(1).kid, "delegationToken": t }),
)
.await
.unwrap();
steps.push((files(&dir), held(&s)));
let auth = format!("Bearer {}", creds::service_auth_token(&s, &alice, &bob));
creds::revoke(
&s,
Some(&auth),
json!({ "space": space, "credentials": ["a", "b"] }),
)
.await
.unwrap();
steps.push((files(&dir), held(&s)));
drop(s);
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let work = scratch("torn-space-credentials-work");
// What a recovered server holds against what it should, at its own
// time: a delegation token's value is held only a minute past its
// issue, and a slow run may recover after that, rightly without it.
let recovers_to = |want: &Held, why: String| {
let s = open(&work, &plc);
let now = s.sessions.now();
let live = |m: &BTreeMap<[u8; 32], u64>| {
m.iter()
.filter(|(_, t)| **t > now)
.map(|(k, t)| (*k, *t))
.collect::<BTreeMap<_, _>>()
};
assert_eq!(
(
s.single_use.snapshot(now),
s.space_credentials.snapshot(now)
),
(live(&want.0), live(&want.1)),
"{why}"
);
};
let mut kinds = Vec::new();
for w in steps.windows(2) {
let changed: Vec<&PathBuf> = w[1]
.0
.keys()
.filter(|p| w[0].0.get(*p) != w[1].0.get(*p))
.collect();
assert_eq!(changed, [&accounts_log]);
assert_ne!(w[0].1, w[1].1);
let whole = &w[1].0[&accounts_log];
let (from, to) = (w[0].0[&accounts_log].len(), whole.len());
kinds.extend(
store::scan(&whole[from..])
.unwrap()
.0
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = w[1].0.clone();
files.insert(accounts_log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
recovers_to(
if cut == to { &w[1].1 } else { &w[0].1 },
format!("cut at {cut} of {from}..{to}"),
);
}
for at in (from..to).step_by(3) {
let mut files = w[1].0.clone();
files.get_mut(&accounts_log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
recovers_to(&w[0].1, format!("flip at {at} of {from}..{to}"));
}
}
assert_eq!(
kinds,
[crate::single_use::ENTRY, crate::space_credentials::ENTRY]
);
// A malformed entry of either kind is refused with a reason.
let last = &steps.last().unwrap().0;
for bad in [
json!({ "t": crate::single_use::ENTRY, "v": "x", "until": 1 }),
json!({ "t": crate::space_credentials::ENTRY, "space": "s", "credentials": [1], "until": 1 }),
] {
let mut frames = store::scan(&last[&accounts_log]).unwrap().0;
frames.push(vec![bad.clone()]);
let mut files = last.clone();
files.insert(
accounts_log.clone(),
frames.iter().flat_map(|f| store::frame(f)).collect(),
);
put_files(&work, &files);
assert!(try_open(&work, &plc).is_err(), "{bad}");
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut an authority's `repo.log` at every byte of each frame a change to
/// its spaces wrote (and flip a byte in it): recovery gives its spaces as
/// before the change or after it, and the log takes changes after. Each
/// kind of entry is cut.
#[tokio::test]
async fn torn_space_authority_writes() {
for sync in SYNCS {
syncing(sync);
torn_space_authority_writes_once().await;
}
}
async fn torn_space_authority_writes_once() {
use crate::space_authority::{ENTRIES, Writer, track_writer};
use pds_repo::data::{self, V};
let plc = plc().await;
let dir = scratch("torn-spaces");
let s = open(&dir, &plc);
let (alice, ta) = create(&s, "alice").await;
let (bob, _) = create(&s, "bob").await;
let log = repo_log(Path::new(""), &alice);
let spaces = |s: &Service| {
crate::read(&s.account(&alice).unwrap().store)
.authority
.clone()
};
let mut steps = vec![(files(&dir), spaces(&s))];
let snap = |s: &Service, steps: &mut Vec<_>| steps.push((files(&dir), spaces(s)));
let m = format!("at://{alice}/space/com.example.forum/m");
let space = pds_space::SpaceRef::parse(&m).unwrap();
let track = |s: &Service| {
let w = Writer {
rev: "3jzfcijpj2z2a".into(),
data: vec![7; 32],
space_rev: "3jzfcijpj2z2b".into(),
removed: false,
};
assert!(track_writer(
&mut crate::write(&s.account(&alice).unwrap().store),
&space,
&bob,
&w
));
};
let change = |nsid: &'static str, body: Json| {
let (s, ta) = (s.clone(), ta.clone());
async move {
assert_eq!(
call(&s, nsid, Some(&ta), body).await,
Ok(Json::Null),
"{nsid}"
)
}
};
let policy = json!({ "$type": "com.atproto.memberspace.policy", "publicWrite": true });
call(
&s,
"com.atproto.space.createSpace",
Some(&ta),
json!({ "spaceType": "com.example.forum", "skey": "m", "policy": policy }),
)
.await
.unwrap();
snap(&s, &mut steps);
change(
"com.atproto.memberspace.putMembers",
json!({ "space": m, "members": [{ "did": bob, "read": true, "write": true }] }),
)
.await;
snap(&s, &mut steps);
track(&s);
snap(&s, &mut steps);
change(
"com.atproto.space.removeWriter",
json!({ "space": m, "repo": bob }),
)
.await;
snap(&s, &mut steps);
track(&s);
snap(&s, &mut steps);
change(
"com.atproto.memberspace.removeMembers",
json!({ "space": m, "dids": [bob] }),
)
.await;
snap(&s, &mut steps);
change(
"com.atproto.space.updateSpace",
json!({ "space": m, "policy": { "$type": "com.atproto.personalspace.policy" } }),
)
.await;
snap(&s, &mut steps);
change("com.atproto.space.deleteSpace", json!({ "space": m })).await;
snap(&s, &mut steps);
let v = |j: Json| data::encode(&data::from_json(&j).unwrap());
let meta = data::from_json(&json!({ "$type": "atproto-space-meta", "ver": 1, "space": m }))
.unwrap();
let writer = data::encode(&V::Map(vec![
("repoRev".into(), V::Str("3jzfcijpj2z2a".into())),
("data".into(), V::Bytes(vec![1; 32])),
("spaceRev".into(), V::Str("3jzfcijpj2z2c".into())),
]));
let (policy, member, writer_key) = (
v(json!({ "$type": "com.atproto.memberspace.policy" })),
v(json!({ "$type": "com.atproto.memberspace.member", "read": true, "write": false })),
format!("writer/{bob}"),
);
let member_key = format!("com.atproto.memberspace.member/{bob}");
let archive = pds_space::star::write(
&meta,
[
(b"policy".as_slice(), policy.as_slice()),
(member_key.as_bytes(), member.as_slice()),
(writer_key.as_bytes(), writer.as_slice()),
],
);
let mut headers = axum::http::HeaderMap::new();
headers.insert(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static(crate::STAR_TYPE),
);
let import = crate::Call {
nsid: "com.atproto.space.importSpaceMeta",
method: &axum::http::Method::POST,
params: &[],
body: &archive,
bearer: sessions::Bearer::Token(&ta),
headers: &headers,
client: "127.0.0.1".into(),
charged: Default::default(),
http: None,
};
assert!(matches!(
crate::answer(&s, &import).await,
Ok(crate::Resp::Nothing)
));
snap(&s, &mut steps);
drop(s);
let work = scratch("torn-spaces-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
let changed = changed_logs(before_files, after_files);
assert_eq!(changed, [&log]);
assert_ne!(before, after);
let whole = &after_files[&log];
let (from, to) = (before_files[&log].len(), whole.len());
kinds.extend(
store::scan(&whole[from..])
.unwrap()
.0
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = unnoted(after_files.clone());
files.insert(log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { after } else { before };
assert_eq!(
&spaces(&open(&work, &plc)),
want,
"cut at {cut} of {from}..{to}"
);
}
for at in (from..to).step_by(5) {
let mut files = after_files.clone();
files.get_mut(&log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(
&spaces(&open(&work, &plc)),
before,
"flip at {at} of {from}..{to}"
);
}
}
kinds.sort();
kinds.dedup();
let mut want: Vec<&str> = ENTRIES.to_vec();
want.sort();
assert_eq!(kinds.iter().map(String::as_str).collect::<Vec<_>>(), want);
// After recovery from the last cut, the log takes changes, and replays them.
put_files(&work, &steps.last().unwrap().0);
let s = open(&work, &plc);
assert_eq!(&spaces(&s), &steps.last().unwrap().1);
call(
&s,
"com.atproto.memberspace.putMembers",
Some(&ta),
json!({ "space": m, "members": [{ "did": "did:plc:carol", "read": true, "write": false }] }),
)
.await
.unwrap();
let now = spaces(&s);
drop(s);
assert_eq!(spaces(&open(&work, &plc)), now);
assert_ne!(now, steps.last().unwrap().1);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Cut an account's `repo.log` at every byte of each frame the write
/// notifications' part wrote (and flip a byte in it): recovery gives
/// what it keeps (its retry queue, registrations and deletion notices)
/// as before the frame or after it, and the log takes changes after.
/// Each kind of entry is cut, and a malformed one is refused.
#[tokio::test]
async fn torn_space_notify_writes() {
for sync in SYNCS {
syncing(sync);
torn_space_notify_writes_once().await;
}
}
async fn torn_space_notify_writes_once() {
use crate::space_notify::{ENTRIES, tests as notify};
let plc = plc().await;
let dir = scratch("torn-notify");
let s = open(&dir, &plc);
let (alice, _) = create(&s, "alice").await;
let log = repo_log(Path::new(""), &alice);
let kept = |s: &Service| {
crate::read(&s.account(&alice).unwrap().store)
.notify
.clone()
};
let (q, m) = (
"at://did:plc:elsewhere/space/com.example.forum/q".to_string(),
format!("at://{alice}/space/com.example.forum/m"),
);
let mut steps = vec![(files(&dir), kept(&s))];
for frame in notify::frames_of_each_kind(&q, &m) {
notify::keep(&s, &alice, &frame);
steps.push((files(&dir), kept(&s)));
}
drop(s);
let work = scratch("torn-notify-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
let changed = changed_logs(before_files, after_files);
assert_eq!(changed, [&log]);
assert_ne!(before, after);
let whole = &after_files[&log];
let (from, to) = (before_files[&log].len(), whole.len());
kinds.extend(
store::scan(&whole[from..])
.unwrap()
.0
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = unnoted(after_files.clone());
files.insert(log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { after } else { before };
assert_eq!(
&kept(&open(&work, &plc)),
want,
"cut at {cut} of {from}..{to}"
);
}
for at in (from..to).step_by(3) {
let mut files = unnoted(after_files.clone());
files.get_mut(&log).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(
&kept(&open(&work, &plc)),
before,
"flip at {at} of {from}..{to}"
);
}
}
kinds.sort();
kinds.dedup();
let mut want: Vec<&str> = ENTRIES.to_vec();
want.sort();
assert_eq!(kinds.iter().map(String::as_str).collect::<Vec<_>>(), want);
// After recovery from the last cut, the log takes changes, and replays them.
let last = &steps.last().unwrap();
put_files(&work, &last.0);
let s = open(&work, &plc);
assert_eq!(&kept(&s), &last.1);
notify::keep(&s, &alice, ¬ify::frames_of_each_kind(&q, &m)[0]);
let now = kept(&s);
drop(s);
assert_eq!(kept(&open(&work, &plc)), now);
assert_ne!(now, last.1);
// A malformed entry of each kind is refused with a reason, and so is
// one of a kind this part does not keep, in a log marked as of this
// version (`checkpoint::unmarked`).
let space = json!(q);
let mine = json!(m);
for bad in [
json!({ "t": "notifyQueued", "space": space, "rev": "x", "data": "00", "n": 1, "due": 1, "until": 2 }),
json!({ "t": "notifyQueued", "space": space, "rev": "3jzfcijpj2z2a", "data": "zz", "n": 1, "due": 1, "until": 2 }),
json!({ "t": "notifyQueued", "space": space, "rev": "3jzfcijpj2z2a", "data": "00", "n": 1, "due": 1 }),
json!({ "t": "notifyUnqueued", "space": "nope", "rev": "3jzfcijpj2z2a" }),
json!({ "t": "syncRegistered", "space": mine, "endpoint": "https://s.example", "token": "t" }),
json!({ "t": "syncRegistered", "space": mine, "endpoint": "https://s.example", "expires": 1 }),
json!({ "t": "syncDropped", "space": mine, "token": "t" }),
json!({ "t": "syncNoticed", "space": mine, "endpoint": "https://never.example", "token": "t" }),
json!({ "t": "syncEmptied", "space": "nope" }),
json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "other", "n": 0, "due": 1, "until": 2 }),
json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "expired", "n": 0, "due": 1, "until": 2 }),
json!({ "t": "syncNotice", "space": mine, "endpoint": "https://s.example", "token": "t", "update": "deleted", "n": -1, "due": 1, "until": 2 }),
json!({ "t": "syncNoticeDone", "space": mine, "endpoint": "https://s.example" }),
json!({ "t": "notifyRegistered", "space": mine, "service": "did:plc:x", "expires": 1 }),
] {
let mut frames = store::scan(&last.0[&log]).unwrap().0;
frames.push(vec![bad.clone()]);
let mut files = last.0.clone();
files.insert(
log.clone(),
frames.iter().flat_map(|f| store::frame(f)).collect(),
);
put_files(&work, &files);
assert!(try_open(&work, &plc).is_err(), "{bad}");
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// An account's blobs, and the files of its blob directory.
fn blob_state(s: &Service, dir: &Path, did: &str) -> (String, Vec<String>) {
let held = crate::read(&s.account(did).unwrap().store).blobs.snapshot();
let mut names: Vec<String> = std::fs::read_dir(blob_dir(dir, did))
.map(|d| {
d.map(|e| e.unwrap().file_name().into_string().unwrap())
.collect()
})
.unwrap_or_default();
names.sort();
(held, names)
}
/// Cut the account's log at every byte of each blob write: an upload, a
/// commit referencing it, and one dropping it, with the files as they
/// are while that frame is on its way. An upload's file is in place
/// before its entry is written, and a dropped blob's file goes only once
/// the commit is on disk, so through each write the blob's file is
/// there. Recovery gives the blobs as before the write or after, and
/// leaves no file no entry holds: a kill between a commit dropping a blob
/// and the removal of its file loses no blob still held, and leaks none.
#[tokio::test]
async fn torn_blob_writes_recover_to_before_or_after() {
for sync in SYNCS {
syncing(sync);
torn_blob_writes_once().await;
}
}
async fn torn_blob_writes_once() {
let plc = plc().await;
let dir = scratch("torn-blob-src");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let log = repo_log(Path::new(""), &did);
let blob = b"spec-pds blob torn".to_vec();
let cid = pds_repo::data::cid_string(&pds_repo::data::cid_of(0x55, &blob));
let snap = |s: &Service| (files(&dir), blob_state(s, &dir, &did));
let mut steps = vec![snap(&s)];
crate::blobs::test_upload(&s, Some(&token), Some("text/plain"), blob.clone(), false)
.await
.unwrap();
steps.push(snap(&s));
let r = json!({ "f": { "$type": "blob", "ref": { "$link": cid }, "mimeType": "text/plain", "size": blob.len() } });
call(
&s,
"com.atproto.repo.putRecord",
Some(&token),
json!({ "repo": did, "collection": "a.b.c", "rkey": "a", "record": r }),
)
.await
.unwrap();
steps.push(snap(&s));
call(
&s,
"com.atproto.repo.deleteRecord",
Some(&token),
json!({ "repo": did, "collection": "a.b.c", "rkey": "a" }),
)
.await
.unwrap();
steps.push(snap(&s));
drop(s);
assert_eq!(steps[1].1.1, vec![cid.clone()]);
assert_eq!(steps[3].1.1, Vec::<String>::new());
let work = scratch("torn-blob-work");
let mut cuts = 0;
for w in steps.windows(2) {
let ((before_files, before), (after_files, after)) = (&w[0], &w[1]);
let (from, to) = (before_files[&log].len(), after_files[&log].len());
assert!(to > from);
let mut files = before_files.clone();
files.extend(after_files.clone());
for cut in from..=to {
files.insert(log.clone(), after_files[&log][..cut].to_vec());
put_files(&work, &files);
let s = open(&work, &plc);
let want = if cut == to { after } else { before };
assert_eq!(
&blob_state(&s, &work, &did),
want,
"cut at {cut} of {from}..{to}"
);
cuts += 1;
}
}
assert!(cuts > 100, "{cuts}");
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Everything a restart must carry over, and each account's space repos.
fn with_spaces(s: &Service) -> (State, BTreeMap<String, String>) {
let d = crate::read(&s.directory);
let spaces = d
.by_did
.iter()
.map(|(did, a)| {
(
did.clone(),
format!(
"{:?}",
crate::space_repo::tests::snapshot(&crate::read(&a.store))
),
)
})
.collect();
drop(d);
(state(s), spaces)
}
/// Cut the account's log at every byte of each space commit's frame (and
/// flip a byte in it): recovery gives the space repos as before that
/// commit or after it, the repository and the events as they were, and
/// takes new space writes, which replay. Each commit's notification goes
/// to a space host that cannot be reached: the frames after the commit's
/// keep it in the retry queue (`space_notify`), which this does not cut.
#[tokio::test]
async fn torn_space_writes_recover_to_before_or_after() {
for sync in SYNCS {
syncing(sync);
torn_space_writes("log", false).await;
}
}
/// The same of an account's log after a table that holds space repos:
/// each commit replayed reads the records it replaces from the table,
/// and a `clearRecords` and an import void what the table holds.
#[tokio::test]
async fn torn_space_writes_after_a_table() {
for sync in SYNCS {
syncing(sync);
torn_space_writes("table", true).await;
}
}
async fn torn_space_writes(name: &str, table: bool) {
use crate::space_notify::tests::idle;
use crate::space_repo::tests::{SPACE, w};
let plc = plc().await;
let dir = scratch(&format!("torn-space-src-{name}"));
let s = open(&dir, &plc);
let (a, ta) = create(&s, "alice").await;
let (b, tb) = create(&s, "bob").await;
let snap = |s: &Service| (files(&dir), with_spaces(s));
let w = |s: &crate::Shared, nsid: &'static str, token: &str, body: Json| {
let (s, token) = (s.clone(), token.to_string());
async move {
let got = w(&s, nsid, &token, body).await;
idle(&s).await;
got
}
};
let rec = |did: &str, rkey: &str, n: i64| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
if table {
for (did, token, rkey) in [(&a, &ta, "zero"), (&b, &tb, "zero"), (&b, &tb, "one")] {
w(&s, "putRecord", token, rec(did, rkey, 0)).await.unwrap();
}
for did in [&a, &b] {
let wal = crate::read(&s.account(did).unwrap().store).wal.clone();
wal.compact().unwrap();
let layout = crate::store::account_layout(&dir, did);
assert!(layout.checkpoint().exists());
}
}
let mut steps = vec![snap(&s)];
w(&s, "createRecord", &ta, rec(&a, "one", 1)).await.unwrap();
steps.push(snap(&s));
w(&s, "putRecord", &tb, rec(&b, "one", 2)).await.unwrap();
steps.push(snap(&s));
let writes = json!([
{ "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "rkey": "two", "value": record(3) },
{ "$type": "com.atproto.space.applyWrites#update", "collection": "a.b.c", "rkey": "one", "value": record(4) },
{ "$type": "com.atproto.space.applyWrites#create", "collection": "a.b.c", "value": record(5) },
]);
w(
&s,
"applyWrites",
&ta,
json!({ "space": SPACE, "repo": a, "writes": writes }),
)
.await
.unwrap();
steps.push(snap(&s));
w(
&s,
"deleteRecord",
&ta,
json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "one" }),
)
.await
.unwrap();
steps.push(snap(&s));
w(
&s,
"clearRecords",
&tb,
json!({ "space": SPACE, "repo": b }),
)
.await
.unwrap();
steps.push(snap(&s));
// An import of alice's own archive back: one frame, of one entry.
let params = [
("space".to_string(), SPACE.to_string()),
("repo".to_string(), a.clone()),
];
let archive = {
let none = axum::http::HeaderMap::new();
let get = crate::Call {
nsid: "com.atproto.space.getRepo",
method: &axum::http::Method::GET,
params: ¶ms,
body: b"",
bearer: sessions::Bearer::Token(&ta),
headers: &none,
client: "127.0.0.1".into(),
charged: Default::default(),
http: None,
};
let Ok(crate::Resp::Archive(archive, _)) = crate::answer(&s, &get).await else {
panic!("getRepo answered no archive")
};
archive.bytes().unwrap()
};
let mut headers = axum::http::HeaderMap::new();
headers.insert(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static("application/vnd.dasl.star"),
);
let import = crate::Call {
nsid: "com.atproto.space.importRepo",
method: &axum::http::Method::POST,
params: &[],
body: &archive,
bearer: sessions::Bearer::Token(&ta),
headers: &headers,
client: "127.0.0.1".into(),
charged: Default::default(),
http: None,
};
assert!(matches!(
crate::answer(&s, &import).await,
Ok(crate::Resp::Nothing)
));
steps.push(snap(&s));
// A refused write, or one making no commit, logs nothing.
assert!(w(&s, "createRecord", &ta, rec(&a, "two", 6)).await.is_err());
w(
&s,
"deleteRecord",
&ta,
json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "none" }),
)
.await
.unwrap();
assert_eq!(files(&dir), steps.last().unwrap().0);
drop(s);
let work = scratch(&format!("torn-space-work-{name}"));
let mut cuts = 0;
for (i, win) in steps.windows(2).enumerate() {
let ((before_files, before), (after_files, after)) = (&win[0], &win[1]);
let changed = changed_logs(before_files, after_files);
let [path] = changed[..] else {
panic!("step {i} changed {changed:?}")
};
assert_ne!(before, after);
// The repository and the events are as they were.
assert_eq!(before.0, after.0);
let whole = &after_files[path];
let from = before_files[path].len();
let frames = store::scan(&whole[from..]).unwrap().0;
let kinds: Vec<&str> = frames[0].iter().map(|e| e["t"].as_str().unwrap()).collect();
assert!(
kinds == [crate::space_repo::ENTRY] || kinds == [crate::space_repo::IMPORT],
"{kinds:?}"
);
assert!(
frames[1..]
.iter()
.flatten()
.all(|e| crate::space_notify::ENTRIES.contains(&e["t"].as_str().unwrap()))
);
let to = from + store::frame(&frames[0]).len();
for cut in from..=to {
let mut files = unnoted(after_files.clone());
files.insert(path.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let s = open(&work, &plc);
let want = if cut == to { after } else { before };
assert_eq!(
&with_spaces(&s),
want,
"step {i}: cut at {cut} of {from}..{to}"
);
cuts += 1;
if cut % 97 == 0 || cut == from + 1 || cut == to {
let got = w(&s, "putRecord", &tb, rec(&b, "after", 9)).await.unwrap();
assert!(got["cid"].is_string());
idle(&s).await;
assert_eq!(
with_spaces(&open(&work, &plc)),
with_spaces(&s),
"step {i}: cut at {cut}"
);
}
}
for at in (from..to).step_by(7) {
let mut files = unnoted(after_files.clone());
files.get_mut(path).unwrap()[at] ^= 0x55;
put_files(&work, &files);
assert_eq!(
&with_spaces(&open(&work, &plc)),
before,
"step {i}: flip at {at} of {from}..{to}"
);
}
}
assert!(cuts > 500, "{cuts}");
// A space commit whose records do not make its data is refused.
let alice = changed_logs(&steps[0].0, &steps[1].0)[0].clone();
let last = &steps.last().unwrap().0;
let mut frames = store::scan(&last[&alice]).unwrap().0;
let entry = frames
.iter_mut()
.flatten()
.find(|e| {
e["t"] == crate::space_repo::ENTRY && !e["puts"].as_array().unwrap().is_empty()
})
.unwrap();
entry["puts"][0][1]["n"] = json!(99);
let mut files = last.clone();
files.insert(alice, frames.iter().flat_map(|f| store::frame(f)).collect());
put_files(&work, &files);
assert!(
try_open(&work, &plc)
.err()
.unwrap()
.contains("do not make the data")
);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// An account whose entry is cut short is not there; one whose entry is
/// whole but whose repository's first frame is not starts with a fresh
/// first commit, once.
#[tokio::test]
async fn torn_account_creation() {
for sync in SYNCS {
syncing(sync);
torn_account_creation_once().await;
}
}
async fn torn_account_creation_once() {
let plc = plc().await;
let dir = scratch("torn-create");
let (steps, accounts) = script(&dir, &plc).await;
let (before, after) = (&steps[1], &steps[2]);
let bob = &accounts[1].0;
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let bob_log = repo_log(Path::new(""), bob);
let work = scratch("torn-create-work");
let (from, to) = (
before.files[&accounts_log].len(),
after.files[&accounts_log].len(),
);
for cut in from..to {
let mut files = before.files.clone();
files.insert(
accounts_log.clone(),
after.files[&accounts_log][..cut].to_vec(),
);
put_files(&work, &files);
assert_eq!(
state(&open(&work, &plc)),
before.state,
"accounts.log cut at {cut}"
);
}
for cut in 0..after.files[&bob_log].len() {
let mut files = after.files.clone();
files.insert(bob_log.clone(), after.files[&bob_log][..cut].to_vec());
put_files(&work, &files);
let s = open(&work, &plc);
assert!(repo_of(&s, bob, |r| r.record_count() == 0 && !r.rev.is_empty()));
// Sent: alice's four events as before, then the four of bob's
// fresh first commit, numbered after every seq given out: the
// index keeps the entries of his first ones, whose frames were
// cut, and a replay passes over them.
let got = sent(&s);
let seq = |e: &Json| e["seq"].as_u64().unwrap();
let first: Vec<u64> = after.state.2.iter().map(|f| seq(&body_of(f))).collect();
assert_eq!(got.len(), 8, "cut at {cut}: {got:?}");
assert_eq!(got[..4].iter().map(seq).collect::<Vec<_>>(), first[..4]);
// A `#commit` names its account as `repo`, the others as `did`.
let of = |e: &Json| e.get("did").or_else(|| e.get("repo")).cloned();
assert!(
got[..4]
.iter()
.all(|e| of(e).is_some_and(|d| d != json!(bob)))
);
assert!(
got[4..].iter().all(|e| of(e) == Some(json!(bob))),
"cut at {cut}: {got:?}"
);
assert!(
got[4..].iter().map(seq).all(|q| q > first[7]),
"cut at {cut}: {got:?}"
);
assert!(got.windows(2).all(|w| seq(&w[0]) < seq(&w[1])));
// Its KEM key is the one its whole entry has.
let kem = state(&s).0[bob.as_str()].2.clone();
assert!(kem.is_some());
assert_eq!(kem, after.state.0[bob.as_str()].2);
if cut % 61 == 0 {
let now = state(&s);
drop(s);
assert_eq!(state(&open(&work, &plc)), now);
}
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// A handle change writes two frames to its account's log: the change
/// recorded, then settled with its `#identity` event. Cut at every byte
/// of them (or with a byte of the second flipped), recovery gives the
/// account before the change, with it recorded and not settled, or with
/// it landed; and one recorded is settled, landing whole, once the
/// server serves.
#[tokio::test]
async fn torn_handle_changes() {
for sync in SYNCS {
syncing(sync);
torn_handle_changes_once().await;
}
}
async fn torn_handle_changes_once() {
let plc = crate::identity::stub::Plc::start().await;
let dir = scratch("torn-handle");
let s = open(&dir, &plc.url);
let (alice, ta) = create(&s, "alice").await;
create(&s, "bob").await;
let before = files(&dir);
let body = json!({ "handle": "carol.test" });
call(&s, "com.atproto.identity.updateHandle", Some(&ta), body)
.await
.unwrap();
let after = files(&dir);
let after_state = state(&s);
drop(s);
let path = repo_log(Path::new(""), &alice);
let changed = changed_logs(&before, &after);
assert_eq!(changed, [&path]);
// The settled change's entry is in the index.
assert!(
after
.iter()
.any(|(p, b)| of_index(p) && before.get(p) != Some(b))
);
let (from, to) = (before[&path].len(), after[&path].len());
let frames = store::scan(&after[&path]).unwrap().0;
let recorded = from + store::frame(&frames[frames.len() - 2]).len();
// What recovery gives: alice's handle, the handles held, and how
// many events a subscription from cursor 0 is sent.
let identity = |s: &Service| {
let d = crate::read(&s.directory);
let mut held: Vec<String> = d.claimed_handles.iter().cloned().collect();
held.sort();
let handle = d.by_did[&alice].handle();
drop(d);
(handle, held, sent(s).len())
};
let n = after_state.2.len();
let work = scratch("torn-handle-work");
let mut unsettled = 0;
let check = |files: BTreeMap<PathBuf, Vec<u8>>, at: String, cut: usize| {
put_files(&work, &files);
let s = open(&work, &plc.url);
// Repositories and sessions are as they were throughout.
let got = state(&s);
assert_eq!((&got.0, &got.1), (&after_state.0, &after_state.1), "{at}");
let want = if cut < recorded {
("alice.test".to_string(), vec![], n - 1)
} else if cut < to {
(
"alice.test".to_string(),
vec!["carol.test".to_string()],
n - 1,
)
} else {
("carol.test".to_string(), vec![], n)
};
assert_eq!(identity(&s), want, "{at}");
if want.1.is_empty() {
assert_eq!(state(&open(&work, &plc.url)), got, "{at}");
}
s
};
for cut in from..=to {
let mut files = unnoted(after.clone());
files.insert(path.clone(), after[&path][..cut].to_vec());
let s = check(files, format!("cut at {cut} of {from}..{to}"), cut);
// One recorded and not settled lands whole once the server serves.
if (recorded..to).contains(&cut) && (cut == recorded || cut % 23 == 0) {
crate::identity::settle_unfinished(&s);
for _ in 0..100 {
if identity(&s).0 == "carol.test" {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
assert_eq!(
identity(&s),
("carol.test".to_string(), vec![], n),
"settling after a cut at {cut}"
);
drop(s);
let s = open(&work, &plc.url);
assert_eq!(
identity(&s),
("carol.test".to_string(), vec![], n),
"after settling a cut at {cut}"
);
unsettled += 1;
}
}
for at in (recorded..to).step_by(3) {
let mut files = unnoted(after.clone());
files.get_mut(&path).unwrap()[at] ^= 0x55;
check(files, format!("flip at {at} of {recorded}..{to}"), recorded);
}
assert!(unsettled > 3, "{unsettled}");
assert_eq!(plc.log(&alice).len(), 2);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// An event that never reached the disk, while a later one of another
/// account's did: a restart leaves its number unused, the same way every
/// time, and the next event is numbered after every one given out.
#[tokio::test]
async fn lost_events_leave_their_numbers_unused() {
for sync in SYNCS {
syncing(sync);
lost_events_once().await;
}
}
async fn lost_events_once() {
let plc = plc().await;
let dir = scratch("gap");
let (steps, accounts) = script(&dir, &plc).await;
let (a, ta) = &accounts[0];
// Drop alice's last frame, her deleteRecord, and put bob's next commit after it.
let s = open(&dir, &plc);
let n = sent(&s).len() as u64;
let body = json!({ "repo": accounts[1].0, "collection": "a.b.c", "rkey": "late", "record": { "n": 1 } });
call(&s, "com.atproto.repo.putRecord", Some(&accounts[1].1), body)
.await
.unwrap();
drop(s);
let mut files = files(&dir);
let alice = repo_log(Path::new(""), a);
let cut = steps[steps.len() - 3].files[&alice].len();
files.get_mut(&alice).unwrap().truncate(cut);
let work = scratch("gap-work");
put_files(&work, &files);
let s = open(&work, &plc);
assert_eq!(sent(&s).len() as u64, n);
let seqs = sent_seqs;
let got = seqs(&s);
let lost: Vec<u64> = (1..=n + 1).filter(|i| !got.contains(i)).collect();
assert_eq!(lost.len(), 1, "{got:?}");
assert_eq!(*s.events.subscribe().borrow(), n + 1);
let first = state(&s);
drop(s);
let s = open(&work, &plc);
assert_eq!(state(&s), first);
// A new event follows on, and every restart after numbers them alike.
let body = json!({ "repo": a, "collection": "a.b.c", "rkey": "new", "record": { "n": 1 } });
call(&s, "com.atproto.repo.putRecord", Some(ta), body)
.await
.unwrap();
assert_eq!(seqs(&s).last(), Some(&(n + 2)));
let want = state(&s);
drop(s);
assert_eq!(state(&open(&work, &plc)), want);
assert_eq!(state(&open(&work, &plc)), want);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// The seal that closes a log.
fn seal() -> Vec<u8> {
store::frame(&[json!({ "t": "seal" })])
}
/// A compaction killed at each step of `spec/STORE.md`'s crash table,
/// with the writes of every file it had not made durable lost: an
/// account's log, then the service log, each with a write after its seal.
/// The account's table and its log hold space repos, which the fold
/// merges with what the log voids of them. Every start recovers the
/// state before the compaction, or with that write too where all of it
/// is in the new log.
#[tokio::test]
async fn compaction_killed_at_every_step() {
for sync in SYNCS {
syncing(sync);
compaction_killed_at_every_step_once().await;
}
}
async fn compaction_killed_at_every_step_once() {
let plc = plc().await;
let dir = scratch("compact-killed");
let (_, accounts) = script(&dir, &plc).await;
let (a, ta) = &accounts[0];
// The account's space repos: in its table, and in its log after it,
// which writes over the table's records, deletes one and clears a repo.
{
use crate::space_notify::tests::idle;
use crate::space_repo::tests::{SPACE, w};
let other = "at://did:web:example.com/space/com.example.chat/room";
let s = open(&dir, &plc);
let rec = |space: &str, rkey: &str, n: i64| json!({ "space": space, "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
for (space, rkey, n) in [(SPACE, "one", 1), (SPACE, "two", 2), (other, "one", 3)] {
w(&s, "putRecord", ta, rec(space, rkey, n)).await.unwrap();
}
idle(&s).await;
let wal = crate::read(&s.account(a).unwrap().store).wal.clone();
wal.compact().unwrap();
drop(wal);
w(&s, "putRecord", ta, rec(SPACE, "two", 22)).await.unwrap();
w(&s, "putRecord", ta, rec(SPACE, "three", 3))
.await
.unwrap();
let gone = json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "one" });
w(&s, "deleteRecord", ta, gone).await.unwrap();
w(&s, "clearRecords", ta, json!({ "space": other, "repo": a }))
.await
.unwrap();
w(&s, "putRecord", ta, rec(other, "after", 4))
.await
.unwrap();
idle(&s).await;
}
let work = scratch("compact-killed-work");
let state = with_spaces;
for which in ["account", "service"] {
let s = open(&dir, &plc);
let before = (files(&dir), state(&s));
let (log, layout) = match which {
"account" => (
crate::read(&s.account(a).unwrap().store).wal.clone(),
crate::store::account_layout(Path::new(""), a),
),
_ => (s.log.clone(), crate::store::service_layout(Path::new(""))),
};
let g = log.generation();
log.compact().unwrap();
assert_eq!(log.generation(), g + 1);
let compacted = files(&dir);
match which {
"account" => {
let body = json!({ "repo": a, "collection": "a.b.c", "rkey": format!("after-{g}"), "record": record(9) });
call(&s, "com.atproto.repo.putRecord", Some(ta), body)
.await
.unwrap();
}
_ => drop(session(&s, "alice").await),
}
let after = (files(&dir), state(&s));
drop(s);
let (old, new, ckpt) = (layout.log(g), layout.log(g + 1), layout.checkpoint());
assert!(!compacted.contains_key(&old) && compacted.contains_key(&ckpt));
let mut sealed = before.0[&old].clone();
sealed.extend(seal());
let written = after.0[&new].clone();
let check = |files: BTreeMap<PathBuf, Vec<u8>>, why: String, whole: bool| {
put_files(&work, &files);
let s = open(&work, &plc);
let got = state(&s);
let want = if whole { &after.1 } else { &before.1 };
assert!(got == *want, "{which}: {why}");
// A start makes it whole again: the same state once more.
drop(s);
assert!(
state(&open(&work, &plc)) == *want,
"{which}: {why}, started twice"
);
};
// The files but the log's, as after the write.
let base: BTreeMap<PathBuf, Vec<u8>> = after
.0
.iter()
.filter(|(p, _)| ![&old, &new, &ckpt].contains(p))
.map(|(p, b)| (p.clone(), b.clone()))
.collect();
let with = |logs: &[(&PathBuf, &[u8])]| {
let mut f = base.clone();
// The table before, if it had one.
if let Some(table) = before.0.get(&ckpt) {
f.insert(ckpt.clone(), table.clone());
}
for (p, b) in logs {
f.insert((*p).clone(), b.to_vec());
}
f
};
// In step 1: the seal cut at every byte, log g + 1 not made.
for cut in 0..seal().len() {
check(
with(&[(&old, &sealed[..before.0[&old].len() + cut])]),
format!("the seal cut at {cut}"),
false,
);
}
// Sealed, log g + 1 not made, or made: in step 2, as in step 1.
check(
with(&[(&old, &sealed)]),
"sealed, no next log".into(),
false,
);
for cut in 0..=written.len() {
let whole = cut == written.len();
check(
with(&[(&old, &sealed), (&new, &written[..cut])]),
format!("the next log cut at {cut}"),
whole,
);
// In step 3: the checkpoint's `.tmp` cut short, at some lengths.
if cut == 0 || whole {
let tmp = layout.tmp();
let bytes = &compacted[&ckpt];
for at in (0..bytes.len())
.step_by((bytes.len() / 7).max(1))
.chain([bytes.len()])
{
let mut f = with(&[(&old, &sealed), (&new, &written[..cut])]);
f.insert(tmp.clone(), bytes[..at].to_vec());
check(
f,
format!("the .tmp cut at {at}, the next log at {cut}"),
whole,
);
}
// In step 4 after the rename, and in step 5 with the old log gone or not.
let mut f = with(&[(&old, &sealed), (&new, &written[..cut])]);
f.insert(ckpt.clone(), bytes.clone());
check(f.clone(), format!("renamed, the next log at {cut}"), whole);
f.remove(&old);
check(
f,
format!("the old log deleted, the next log at {cut}"),
whole,
);
}
}
// A log missing between the checkpoint's and the last is data not recovered.
let mut f = with(&[(&new, &written)]);
f.insert(ckpt.clone(), compacted[&ckpt].clone());
f.insert(layout.log(g + 3), Vec::new());
put_files(&work, &f);
assert!(
Service::open(&work, config(&plc)).is_err(),
"{which}: a gap"
);
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Everything a restart must carry over of the accounts, with each
/// one's space repos, the spaces it is the authority of, and its write
/// notifications.
fn with_all(s: &Service) -> (State, BTreeMap<String, String>, BTreeMap<String, String>) {
let kept = crate::read(&s.directory)
.by_did
.iter()
.map(|(did, a)| {
let st = crate::read(&a.store);
(did.clone(), format!("{:?}", (&st.authority, &st.notify)))
})
.collect();
let (state, spaces) = with_spaces(s);
(state, spaces, kept)
}
/// Account tables of the version before, and logs after them as that
/// version wrote them: the first load of each folds it into a table of
/// this version, once, and serves all of it, the same wherever the fold
/// was cut short, when the server is started again, and when each
/// account is loaded as it is first asked for; and takes writes after.
/// Of spaces, the state in their `S` and in the logs is let go, with
/// the blobs only space records held (`checkpoint::unmarked`), and
/// spaces are written after as ever.
#[tokio::test]
async fn a_table_of_the_version_before_is_folded_once() {
use crate::space_notify::tests::idle;
use crate::space_repo::tests::{SPACE, w};
let plc = plc().await;
let dir = scratch("before-src");
let s = open(&dir, &plc);
let (a, ta) = create(&s, "alice").await;
let (b, tb) = create(&s, "bob").await;
// An account no start loads: nothing of its is waiting to be sent.
let (c, tc) = create(&s, "carol").await;
let (one, two) = (
b"spec-pds before one".to_vec(),
b"spec-pds before two".to_vec(),
);
let mut refs = Vec::new();
for body in [&one, &two] {
let up =
crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), body.clone(), false)
.await
.unwrap();
refs.push(up["blob"].clone());
}
let made = call(&s, "com.atproto.space.createSpace", Some(&ta), json!({ "spaceType": "com.example.forum", "skey": "mine", "policy": { "$type": crate::space_authority::MEMBER, "publishPolicy": true, "publishMembers": true } }))
.await
.unwrap();
let mine = made["space"].as_str().unwrap().to_string();
let member = |did: &str, write: bool| json!({ "space": mine, "members": [{ "did": did, "read": true, "write": write }] });
call(
&s,
"com.atproto.memberspace.putMembers",
Some(&ta),
member(&b, true),
)
.await
.unwrap();
let rec = |space: &str, did: &str, rkey: &str, record: Json| json!({ "space": space, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record });
let public = |rkey: &str, record: Json| json!({ "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record });
call(
&s,
"com.atproto.repo.putRecord",
Some(&ta),
public("pic", json!({ "img": refs[0] })),
)
.await
.unwrap();
for (space, rkey, record) in [
(SPACE, "one", record(1)),
(SPACE, "two", json!({ "img": refs[1] })),
(mine.as_str(), "one", record(3)),
(mine.as_str(), "two", record(4)),
] {
w(&s, "putRecord", &ta, rec(space, &a, rkey, record))
.await
.unwrap();
}
w(
&s,
"deleteRecord",
&ta,
json!({ "space": mine, "repo": a, "collection": "a.b.c", "rkey": "two" }),
)
.await
.unwrap();
for rkey in ["one", "zero"] {
w(&s, "putRecord", &tb, rec(SPACE, &b, rkey, record(5)))
.await
.unwrap();
}
let carols =
json!({ "repo": c, "collection": "a.b.c", "rkey": "one", "record": record(1) });
call(&s, "com.atproto.repo.putRecord", Some(&tc), carols)
.await
.unwrap();
idle(&s).await;
// The tables, then the logs after them.
for did in [&a, &b, &c] {
let wal = crate::read(&s.account(did).unwrap().store).wal.clone();
wal.compact().unwrap();
}
w(&s, "putRecord", &ta, rec(SPACE, &a, "one", record(11)))
.await
.unwrap();
w(
&s,
"putRecord",
&ta,
rec(SPACE, &a, "three", json!({ "img": refs[1] })),
)
.await
.unwrap();
w(
&s,
"deleteRecord",
&ta,
json!({ "space": SPACE, "repo": a, "collection": "a.b.c", "rkey": "two" }),
)
.await
.unwrap();
w(&s, "putRecord", &ta, rec(&mine, &a, "five", record(5)))
.await
.unwrap();
call(
&s,
"com.atproto.memberspace.putMembers",
Some(&ta),
member("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", false),
)
.await
.unwrap();
call(
&s,
"com.atproto.repo.putRecord",
Some(&ta),
public("after", record(2)),
)
.await
.unwrap();
w(&s, "putRecord", &tb, rec(SPACE, &b, "one", record(6)))
.await
.unwrap();
w(
&s,
"clearRecords",
&tb,
json!({ "space": SPACE, "repo": b }),
)
.await
.unwrap();
w(&s, "putRecord", &tb, rec(SPACE, &b, "after", record(7)))
.await
.unwrap();
idle(&s).await;
let want = with_all(&s);
let cid = |r: &Json| r["ref"]["$link"].as_str().unwrap().to_string();
let (one, two) = (cid(&refs[0]), cid(&refs[1]));
assert!(want.0.5.0[&a].2.contains(&two));
let g = crate::read(&s.account(&a).unwrap().store).wal.generation();
drop(s);
// The tables as the version before wrote them; and in the logs, a
// `clearRecords` that names each record it deletes, as spaces once
// logged it.
for did in [&a, &b, &c] {
let layout = crate::store::account_layout(&dir, did);
crate::checkpoint::downgrade(&layout.checkpoint());
let log = layout.log(g);
let mut frames = store::scan(&std::fs::read(&log).unwrap()).unwrap().0;
for e in frames.iter_mut().flatten() {
if e.get("kept") == Some(&json!([])) {
e.as_object_mut().unwrap().remove("kept");
e["dels"] = json!(["a.b.c/one", "a.b.c/zero"]);
}
}
let bytes: Vec<u8> = frames.iter().flat_map(|f| store::frame(f)).collect();
std::fs::write(&log, bytes).unwrap();
}
let before = files(&dir);
let layout = crate::store::account_layout(Path::new(""), &a);
let (old, new, ckpt) = (layout.log(g), layout.log(g + 1), layout.checkpoint());
assert!(!store::scan(&before[&old]).unwrap().0.is_empty());
// Folded on its first load, and served.
let work = scratch("before-work");
put_files(&work, &before);
let s = open(&work, &plc);
// Each account's state as it was but for its blobs; no spaces state;
// and of alice's blobs, the one a public record references alone.
let public = |mut state: State| {
for (_, _, blobs) in state.5.0.values_mut() {
blobs.clear();
}
state
};
let served = with_all(&s);
assert!(public(served.0.clone()) == public(want.0.clone()), "folded");
for (did, spaces) in &served.1 {
assert_eq!(spaces, "[]", "{did}");
}
let none = format!(
"{:?}",
(
&crate::space_authority::Authority::default(),
&crate::space_notify::Kept::default()
)
);
assert!(served.2.values().all(|k| *k == none));
{
let account = s.account(&a).unwrap();
let st = crate::read(&account.store);
let held: Vec<&str> = st.blobs.held_list().map(|(cid, _, _)| cid).collect();
assert_eq!(held, [one.as_str()]);
assert!(st.blobs.space_ref_list().is_empty());
}
assert!(!blob_dir(&work, &a).join(&two).exists());
assert!(blob_dir(&work, &a).join(&one).exists());
drop(s);
let folded = files(&work);
assert!(!folded.contains_key(&old) && folded.contains_key(&new));
assert!(folded[&ckpt] != before[&ckpt]);
// Once: a start after it reads the table it made, and writes go on.
let s = open(&work, &plc);
assert!(with_all(&s) == served, "started again");
assert!(files(&work)[&ckpt] == folded[&ckpt]);
w(&s, "putRecord", &ta, rec(SPACE, &a, "later", record(8)))
.await
.unwrap();
w(&s, "putRecord", &tb, rec(SPACE, &b, "zero", record(9)))
.await
.unwrap();
idle(&s).await;
let later = with_all(&s);
assert!(later != served);
drop(s);
assert!(with_all(&open(&work, &plc)) == later, "written after");
// An account not loaded at the start: its table has no header to
// find its events by, which the firehose's index has; it is loaded
// as first asked for, and folded then.
put_files(&work, &before);
let s = open_with(&work, &plc, |c| c.load_all = false);
assert!(!s.account(&c).is_some_and(|a| a.store.is_loaded()));
assert!(crate::checkpoint::events_on_disk(&s.ctx, &c).is_err());
s.events.find_old();
assert!(!s.account(&c).is_some_and(|a| a.store.is_loaded()));
for did in [&a, &b, &c] {
assert!(s.by_did(did).is_some());
}
assert!(with_all(&s) == served, "loaded as first asked for");
drop(s);
// The fold cut at every step: the seal at every byte, the next log
// made or not, the table written in part, put in place, and the log
// it covers deleted or not.
let check = |files: BTreeMap<PathBuf, Vec<u8>>, why: String| {
put_files(&work, &files);
let s = open(&work, &plc);
assert!(with_all(&s) == served, "{why}");
drop(s);
assert!(
with_all(&open(&work, &plc)) == served,
"{why}, started twice"
);
};
let with = |logs: &[(&PathBuf, &[u8])]| {
let mut f = before.clone();
for (p, b) in logs {
f.insert((*p).clone(), b.to_vec());
}
f
};
let mut sealed = before[&old].clone();
sealed.extend(seal());
for cut in 0..seal().len() {
check(
with(&[(&old, &sealed[..before[&old].len() + cut])]),
format!("the seal cut at {cut}"),
);
}
check(with(&[(&old, &sealed)]), "sealed, no next log".into());
check(
with(&[(&old, &sealed), (&new, &[])]),
"sealed, the next log made".into(),
);
let bytes = &folded[&ckpt];
for at in (0..bytes.len())
.step_by((bytes.len() / 9).max(1))
.chain([bytes.len()])
{
let mut f = with(&[(&old, &sealed), (&new, &[])]);
f.insert(layout.tmp(), bytes[..at].to_vec());
check(f, format!("the .tmp cut at {at}"));
}
let mut f = with(&[(&old, &sealed), (&new, &[])]);
f.insert(ckpt.clone(), bytes.clone());
check(f.clone(), "put in place".into());
f.remove(&old);
check(f, "the log it covers deleted".into());
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// An account's log with no mark, as an earlier version left it, sealed
/// with a log after it: its spaces entries, of the kinds this server
/// logs and of others, are let go as its store loads, and its other
/// entries replayed as they were; a blob only its space records
/// referenced is no longer held. The mark is appended once, and the
/// account takes spaces writes after, which replay, and fold.
#[tokio::test]
async fn spaces_before_the_mark_are_let_go() {
use crate::space_notify::tests::idle;
use crate::space_repo::tests::{SPACE, r, w};
let plc = plc().await;
let dir = scratch("unmarked-src");
let s = open(&dir, &plc);
let (a, ta) = create(&s, "alice").await;
let mut refs = Vec::new();
for body in [b"unmarked one".to_vec(), b"unmarked two".to_vec()] {
let up = crate::blobs::test_upload(&s, Some(&ta), Some("text/plain"), body, false)
.await
.unwrap();
refs.push(up["blob"].clone());
}
let cid = |r: &Json| r["ref"]["$link"].as_str().unwrap().to_string();
let (one, two) = (cid(&refs[0]), cid(&refs[1]));
call(
&s,
"com.atproto.repo.putRecord",
Some(&ta),
json!({ "repo": a, "collection": "a.b.c", "rkey": "pic", "record": { "img": refs[0] } }),
)
.await
.unwrap();
let space = |s: &crate::Shared| {
let (s, ta) = (s.clone(), ta.clone());
let body = json!({ "spaceType": "com.example.forum", "skey": "mine", "policy": { "$type": crate::space_authority::MEMBER, "publishMembers": true } });
async move {
call(&s, "com.atproto.space.createSpace", Some(&ta), body)
.await
.unwrap()["space"]
.as_str()
.unwrap()
.to_string()
}
};
let mine = space(&s).await;
call(
&s,
"com.atproto.memberspace.putMembers",
Some(&ta),
json!({ "space": mine, "members": [{ "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", "read": true, "write": false }] }),
)
.await
.unwrap();
let rec = |space: &str, rkey: &str, record: Json| json!({ "space": space, "repo": a, "collection": "a.b.c", "rkey": rkey, "record": record });
for (space, rkey, record) in [
(SPACE, "one", json!({ "img": refs[1] })),
(SPACE, "two", json!({ "img": refs[0] })),
(mine.as_str(), "x", record(1)),
] {
w(&s, "putRecord", &ta, rec(space, rkey, record))
.await
.unwrap();
}
idle(&s).await;
let want = with_all(&s);
assert!(want.0.5.0[&a].2.contains(&two) && want.1[&a] != "[]");
let g = crate::read(&s.account(&a).unwrap().store).wal.generation();
drop(s);
// As an earlier version left it: no mark, and entries of kinds this
// server does not log; sealed, with a log after it.
let layout = crate::store::account_layout(Path::new(""), &a);
let (log, next) = (layout.log(g), layout.log(g + 1));
let mut before = files(&dir);
let mark = |f: &Vec<Json>| f.iter().any(|e| e["t"] == "spaces");
let mut frames = store::scan(&before[&log]).unwrap().0;
assert_eq!(frames.iter().filter(|f| mark(f)).count(), 1);
frames.retain(|f| !mark(f));
let mut bytes: Vec<u8> = frames.iter().flat_map(|f| store::frame(f)).collect();
bytes.extend(seal());
before.insert(log.clone(), bytes);
let old = [
json!({ "t": "notifyRegistered", "space": mine, "service": "did:plc:x#s", "expires": 1 }),
json!({ "t": "spaceUnwrite", "space": mine, "did": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" }),
];
before.insert(
next.clone(),
old.iter()
.flat_map(|e| store::frame(std::slice::from_ref(e)))
.collect(),
);
let work = scratch("unmarked-work");
put_files(&work, &before);
let s = open(&work, &plc);
let public = |mut state: State| {
for (_, _, blobs) in state.5.0.values_mut() {
blobs.clear();
}
state
};
let served = with_all(&s);
assert!(public(served.0.clone()) == public(want.0.clone()));
assert_eq!(served.1[&a], "[]");
let none = format!(
"{:?}",
(
&crate::space_authority::Authority::default(),
&crate::space_notify::Kept::default()
)
);
assert_eq!(served.2[&a], none);
{
let account = s.account(&a).unwrap();
let st = crate::read(&account.store);
let held: Vec<&str> = st.blobs.held_list().map(|(cid, _, _)| cid).collect();
assert_eq!(held, [one.as_str()]);
}
assert!(!blob_dir(&work, &a).join(&two).exists());
assert_eq!(
r(&s, "listSpaces", &ta, &[]).await,
Ok(json!({ "spaces": [] }))
);
assert_eq!(
r(&s, "getSpace", &ta, &[("space", mine.as_str())]).await,
Err("SpaceNotFound".into())
);
drop(s);
// Marked once: started again, it is as it was.
let marks = |work: &Path| {
let files = files(work);
files
.iter()
.filter(|(p, _)| p.starts_with(&layout.dir))
.flat_map(|(_, b)| store::scan(b).map(|f| f.0).unwrap_or_default())
.filter(|f| mark(f))
.count()
};
assert_eq!(marks(&work), 1);
let s = open(&work, &plc);
assert!(with_all(&s) == served);
// Spaces writes go on, and replay, and fold.
assert_eq!(space(&s).await, mine);
w(
&s,
"putRecord",
&ta,
rec(SPACE, "one", json!({ "img": refs[0] })),
)
.await
.unwrap();
idle(&s).await;
let later = with_all(&s);
assert!(later.1[&a] != "[]" && later != served);
drop(s);
assert_eq!(marks(&work), 1);
let s = open(&work, &plc);
assert!(with_all(&s) == later, "written after");
let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
wal.compact().unwrap();
drop(wal);
drop(s);
let s = open(&work, &plc);
assert!(with_all(&s) == later, "folded");
drop(s);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Make the table at `path` one of the table format's version before
/// (`pds_store::table::Version::Sha256`), holding the same entries.
fn as_the_version_before(path: &Path) {
use store::table::{BLOCK, BlockCache, Table, TableWriter, Version};
let cache = BlockCache::new(1 << 20);
let t = Table::open(path, &cache).unwrap();
assert_eq!(t.version(), Version::Xxh3);
let mut w = TableWriter::of_version(Vec::new(), t.len(), BLOCK, Version::Sha256);
for e in t.iter(b"", None, &cache) {
let (k, v) = e.unwrap();
w.add(&k, &v).unwrap();
}
std::fs::write(path, w.finish().unwrap().0).unwrap();
}
/// The version of the table format the table at `path` is of.
fn table_version(path: &Path) -> store::table::Version {
let cache = store::table::BlockCache::new(0);
store::table::Table::open(path, &cache).unwrap().version()
}
/// Tables of the table format's version before, whose checks and bloom
/// filters are SHA-256's, among tables of this one: two accounts' and
/// the service's, a third account's as it is written now, each with a
/// log after it. A start serves all of it as it was, loading every
/// account or each as it is first asked for, and finds an account's
/// events in its files, its store not loaded, as in a table of this
/// version. The accounts take writes, and a compaction folds a table
/// of the version before, with its log, into one of this version
/// holding the same: an account's, with its space repo, and the
/// service's, beside an account's still of the version before. With a
/// byte changed, a table of either version is refused when it is
/// opened, one of the version before that says it has more entries
/// than it could hold too, or the block changed is not served.
#[tokio::test]
async fn tables_of_the_version_before_are_served_and_folded() {
use crate::space_notify::tests::idle;
use crate::space_repo::tests::{SPACE, w};
use store::table::Version;
const PUT: &str = "com.atproto.repo.putRecord";
const DELETE: &str = "com.atproto.repo.deleteRecord";
let plc = plc().await;
let dir = scratch("checks-src");
let s = open(&dir, &plc);
let mut accounts = Vec::new();
for name in ["alice", "bob", "carol"] {
accounts.push(create(&s, name).await);
}
let put = |did: &str, rkey: &str, n: i64| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
let del =
|did: &str, rkey: &str| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey });
let space = |did: &str, rkey: &str, n: i64| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(n) });
let gone = |did: &str, rkey: &str| json!({ "space": SPACE, "repo": did, "collection": "a.b.c", "rkey": rkey });
// In each account's table: records and their events; in Alice's,
// a space repo too.
for (did, token) in &accounts {
for n in 0..24 {
let body = put(did, &format!("r{n:02}"), n);
call(&s, PUT, Some(token), body).await.unwrap();
}
}
let [a, b, c] = [0, 1, 2].map(|i| accounts[i].0.clone());
let ta = &accounts[0].1;
for (rkey, n) in [("one", 1), ("two", 2)] {
w(&s, "putRecord", ta, space(&a, rkey, n)).await.unwrap();
}
idle(&s).await;
for (did, _) in &accounts {
let wal = crate::read(&s.account(did).unwrap().store).wal.clone();
wal.compact().unwrap();
}
s.log.compact().unwrap();
// In the log after each: records written over and deleted.
for (did, token) in &accounts {
call(&s, PUT, Some(token), put(did, "r00", 100))
.await
.unwrap();
call(&s, DELETE, Some(token), del(did, "r01"))
.await
.unwrap();
}
w(&s, "putRecord", ta, space(&a, "one", 11)).await.unwrap();
w(&s, "deleteRecord", ta, gone(&a, "two")).await.unwrap();
session(&s, "alice").await;
idle(&s).await;
let want = with_all(&s);
drop(s);
let ckpt = |did: &str| crate::store::account_layout(Path::new(""), did).checkpoint();
let service = crate::store::service_layout(Path::new("")).checkpoint();
// Alice's, Carol's and the service's of the version before; Bob's as it is.
let written = files(&dir);
let old = [ckpt(&a), ckpt(&c), service.clone()];
for p in &old {
as_the_version_before(&dir.join(p));
}
let before = files(&dir);
for p in &old {
assert_eq!(table_version(&dir.join(p)), Version::Sha256);
assert!(before[p] != written[p]);
}
assert_eq!(table_version(&dir.join(ckpt(&b))), Version::Xxh3);
// Bob's events and Carol's, found in their files with no store
// loaded: neither has anything waiting to be sent, so a start that
// loads each account as it is first asked for loads neither.
let work = scratch("checks-work");
let unloaded = || {
let s = open_with(&work, &plc, |c| c.load_all = false);
for did in [&b, &c] {
assert!(!s.account(did).is_some_and(|a| a.store.is_loaded()));
}
assert!(s.account(&a).is_some());
s
};
let on_disk = |files: &BTreeMap<PathBuf, Vec<u8>>| {
put_files(&work, files);
let s = unloaded();
[&b, &c].map(|did| crate::checkpoint::events_on_disk(&s.ctx, did).unwrap())
};
let events = on_disk(&written);
assert!(events.iter().all(|(seqs, _, _)| seqs.len() > 24));
assert_eq!(on_disk(&before), events);
// Served as it was, the tables left as they are.
put_files(&work, &before);
assert!(with_all(&open(&work, &plc)) == want, "served");
assert!(with_all(&unloaded()) == want, "loaded on first use");
let served = files(&work);
assert!(old.iter().all(|p| served[p] == before[p]));
// Written to, then folded: Alice's table and the service's are of
// this version and hold the same; Carol's is as it was.
let s = open(&work, &plc);
for (did, token) in &accounts {
call(&s, PUT, Some(token), put(did, "after", 7))
.await
.unwrap();
call(&s, DELETE, Some(token), del(did, "r02"))
.await
.unwrap();
}
w(&s, "putRecord", ta, space(&a, "three", 3)).await.unwrap();
idle(&s).await;
let later = with_all(&s);
assert!(later != want);
let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
wal.compact().unwrap();
drop(wal);
s.log.compact().unwrap();
assert!(with_all(&s) == later, "folded");
drop(s);
let folded = files(&work);
for p in [ckpt(&a), service] {
assert_eq!(table_version(&work.join(p)), Version::Xxh3);
}
assert!(folded[&ckpt(&c)] == before[&ckpt(&c)]);
assert!(with_all(&open(&work, &plc)) == later, "started again");
assert!(
with_all(&unloaded()) == later,
"started again, loaded on first use"
);
// A byte changed, of Carol's table, of the version before, and of
// Bob's, of this one: the last of the file, which is of its number
// of entries in the version before and of its footer's check in
// this one; the last of its bloom filter's check; and the first
// of its first block.
for (did, version, last) in [
(
&c,
Version::Sha256,
"more entries than its blocks could hold",
),
(&b, Version::Xxh3, "its footer fails its check"),
] {
assert_eq!(table_version(&work.join(ckpt(did))), version);
let len = folded[&ckpt(did)].len();
for (at, why) in [
(len - 1, last),
(
len - version.footer() - 1,
"its bloom filter fails its check",
),
(0, "fails its check"),
] {
let mut f = folded.clone();
f.get_mut(&ckpt(did)).unwrap()[at] ^= 0x55;
put_files(&work, &f);
let refused = match try_open(&work, &plc) {
Err(e) => e,
Ok(s) => {
let account = s.account(did).unwrap();
let st = crate::read(&account.store);
crate::sync::CarStream::of(&st, None)
.and_then(|car| car.bytes())
.unwrap_err()
.to_string()
}
};
assert!(refused.contains(why), "{refused}");
}
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// The state of a server that loads each account when it is first
/// asked for, once every account `want` holds is: none is loaded
/// before.
fn lazily(dir: &Path, plc: &str, want: &State) -> State {
let s = open_with(dir, plc, |c| c.load_all = false);
assert!(crate::read(&s.directory).by_did.is_empty());
for did in want.0.keys() {
s.account(did);
}
state(&s)
}
/// What the service log keeps of the accounts, as one comparable value
/// read with no account's store: each account's handle, mail and state,
/// the sessions and app passwords, which account each reset token is
/// of, and the invite codes.
type Kept = (
BTreeMap<String, (String, Mail, crate::lifecycle::State)>,
BTreeMap<String, sessions::Keep>,
BTreeMap<String, String>,
Vec<String>,
);
fn kept(s: &Service) -> Kept {
let accounts = crate::read(&s.directory)
.by_did
.iter()
.map(|(did, a)| {
let (mail, state) = (email::lock(&a.mail).clone(), crate::read(&a.state).clone());
(did.clone(), (a.handle(), mail, state))
})
.collect();
(
accounts,
s.sessions.snapshot(),
s.resets.snapshot(),
s.invites.snapshot(),
)
}
/// With the service checkpoint a table: each kind of write to the
/// service log after it, of an account in the table and of one made
/// after it, cut at every byte. A start that loads each account when
/// it is first asked for finds what the service log keeps as before
/// the write, or as after it once it is whole; and so does one that
/// loads every account, of everything, at every few bytes.
#[tokio::test]
async fn torn_writes_after_a_table() {
for sync in SYNCS {
syncing(sync);
torn_writes_after_a_table_once().await;
}
}
async fn torn_writes_after_a_table_once() {
let plc = plc().await;
let dir = scratch("torn-table");
let s = open(&dir, &plc);
let (_, ta) = create(&s, "alice").await;
let (_, tb) = create(&s, "bob").await;
s.log.compact().unwrap();
let layout = crate::store::service_layout(Path::new(""));
let (log, ckpt) = (layout.log(s.log.generation()), layout.checkpoint());
let mut steps = Vec::new();
let snap = |s: &Service, steps: &mut Vec<(Step, Kept)>| {
steps.push((
Step {
files: files(&dir),
journal: segments(&dir),
state: state(s),
},
kept(s),
))
};
snap(&s, &mut steps);
assert!(store::table::is_table(&steps[0].0.files[&ckpt]));
let app = |token: &str| {
let (s, token) = (s.clone(), token.to_string());
async move {
call(
&s,
"com.atproto.server.createAppPassword",
Some(&token),
json!({ "name": "phone" }),
)
.await
.unwrap()
}
};
// Of accounts in the table.
session(&s, "alice").await;
snap(&s, &mut steps);
app(&ta).await;
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.updateEmail",
Some(&tb),
json!({ "email": "bob2@mail.com" }),
)
.await
.unwrap();
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.deactivateAccount",
Some(&ta),
json!({}),
)
.await
.unwrap();
snap(&s, &mut steps);
// An account made after it, and a write of its.
let (_, tc) = create(&s, "carol").await;
snap(&s, &mut steps);
app(&tc).await;
snap(&s, &mut steps);
call(
&s,
"com.atproto.server.activateAccount",
Some(&ta),
Json::Null,
)
.await
.unwrap();
snap(&s, &mut steps);
drop(s);
let dids: Vec<String> = steps.last().unwrap().1.0.keys().cloned().collect();
let work = scratch("torn-table-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let whole = &w[1].0.files[&log];
let (from, to) = (w[0].0.files[&log].len(), whole.len());
assert!(to > from);
kinds.extend(
store::scan(&whole[from..])
.unwrap()
.0
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = w[1].0.files.clone();
files.insert(log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { &w[1] } else { &w[0] };
// Each account loaded when it is asked for, none before.
let s = open_with(&work, &plc, |c| c.load_all = false);
assert!(crate::read(&s.directory).by_did.is_empty());
for did in &dids {
s.account(did);
}
assert!(kept(&s) == want.1, "cut at {cut} of {from}..{to}");
drop(s);
if cut % 61 == 0 || cut + 1 >= to {
assert!(
state(&open(&work, &plc)) == want.0.state,
"cut at {cut} of {from}..{to}, every account loaded"
);
assert!(
lazily(&work, &plc, &want.0.state) == want.0.state,
"cut at {cut} of {from}..{to}, loaded on first use"
);
}
}
}
for kind in [
"account",
"session",
"appPassword",
"updateEmail",
"deactivated",
"event",
] {
assert!(
kinds.iter().any(|k| k == kind),
"{kind} never cut: {kinds:?}"
);
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// The service log's table, cut at every byte where a compaction
/// writes it beside the table before, the log sealed and the next one
/// made: a start finds everything as before the compaction, and the
/// part written is gone.
#[tokio::test]
async fn a_table_cut_short_where_it_is_written() {
let plc = plc().await;
let dir = scratch("torn-tmp");
let s = open(&dir, &plc);
let (alice, _) = create(&s, "alice").await;
s.log.compact().unwrap();
session(&s, "alice").await;
let layout = crate::store::service_layout(Path::new(""));
let g = s.log.generation();
let before = (files(&dir), state(&s));
s.log.compact().unwrap();
assert!(state(&s) == before.1);
let table = files(&dir)[&layout.checkpoint()].clone();
assert!(table != before.0[&layout.checkpoint()]);
drop(s);
let mut killed = before.0.clone();
let mut sealed = killed[&layout.log(g)].clone();
sealed.extend(seal());
killed.insert(layout.log(g), sealed);
killed.insert(layout.log(g + 1), Vec::new());
let work = scratch("torn-tmp-work");
for cut in 0..=table.len() {
let mut files = killed.clone();
files.insert(layout.tmp(), table[..cut].to_vec());
put_files(&work, &files);
if cut % 499 == 0 || cut == table.len() {
assert!(state(&open(&work, &plc)) == before.1, "cut at {cut}");
assert!(
lazily(&work, &plc, &before.1) == before.1,
"cut at {cut}, loaded on first use"
);
} else {
let s = open_with(&work, &plc, |c| c.load_all = false);
assert_eq!(crate::read(&s.directory).count(), 1, "cut at {cut}");
assert_eq!(s.sessions.count(&alice), 0, "cut at {cut}");
assert_eq!(s.account(&alice).unwrap().handle(), "alice.test");
assert_eq!(s.sessions.count(&alice), 2, "cut at {cut}");
}
assert!(!work.join(layout.tmp()).exists(), "cut at {cut}");
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// A service checkpoint of the format before a table, a log of frames
/// after its header, with the log after it: a start makes it a table
/// of the same generation and serves all of it, the same wherever the
/// entries were divided between the checkpoint and the log, wherever
/// the conversion was cut short, and when it is started again.
#[tokio::test]
async fn a_checkpoint_of_the_format_before_is_made_a_table() {
let plc = plc().await;
let dir = scratch("convert");
let (_, accounts) = script(&dir, &plc).await;
let s = open(&dir, &plc);
let (ta, tb) = (&accounts[0].1, &accounts[1].1);
call(
&s,
"com.atproto.server.createAppPassword",
Some(ta),
json!({ "name": "phone" }),
)
.await
.unwrap();
call(
&s,
"com.atproto.server.updateEmail",
Some(tb),
json!({ "email": "bob2@mail.com" }),
)
.await
.unwrap();
call(
&s,
"com.atproto.server.deactivateAccount",
Some(tb),
json!({}),
)
.await
.unwrap();
create(&s, "carol").await;
let want = state(&s);
drop(s);
let last = files(&dir);
let log = PathBuf::from(ACCOUNTS_LOG_FILE);
let layout = crate::store::service_layout(Path::new(""));
let (ckpt, tmp) = (layout.checkpoint(), layout.tmp());
assert!(!last.contains_key(&ckpt));
let frames = store::scan(&last[&log]).unwrap().0;
// The files as the server before left them, having compacted the
// first `at` frames: its checkpoint, a header naming its generation
// and the last seq of its events, then what those frames hold.
let before = |at: usize| {
let seq = frames[..at]
.iter()
.flatten()
.filter(|e| e["t"] == "event" || e["t"] == "deleted")
.filter_map(|e| e["seq"].as_u64())
.max()
.unwrap_or(0);
let mut checkpoint =
store::frame(&[json!({ "t": "checkpoint", "generation": 1, "lastSeq": seq })]);
checkpoint.extend(frames[..at].iter().flat_map(|f| store::frame(f)));
let mut files = last.clone();
files.remove(&log);
files.insert(ckpt.clone(), checkpoint);
files.insert(
layout.log(1),
frames[at..].iter().flat_map(|f| store::frame(f)).collect(),
);
files
};
let work = scratch("convert-work");
for at in 0..=frames.len() {
put_files(&work, &before(at));
assert!(state(&open(&work, &plc)) == want, "divided at {at}");
let made = files(&work);
assert!(store::table::is_table(&made[&ckpt]), "divided at {at}");
assert!(!made.contains_key(&tmp), "divided at {at}");
assert!(
state(&open(&work, &plc)) == want,
"divided at {at}, started again"
);
assert!(files(&work)[&ckpt] == made[&ckpt], "made a table twice");
assert!(
lazily(&work, &plc, &want) == want,
"divided at {at}, loaded on first use"
);
}
// Cut short while the table is written beside it, at every byte;
// and whole, not yet renamed over it. Of a checkpoint that holds
// the first account, the log the rest.
let at = 1 + frames
.iter()
.position(|f| f.iter().any(|e| e["t"] == "account"))
.unwrap();
put_files(&work, &before(at));
drop(open_with(&work, &plc, |c| c.load_all = false));
let table = std::fs::read(work.join(&ckpt)).unwrap();
let accounts = want.0.len() as u64;
for cut in 0..=table.len() {
let mut f = before(at);
f.insert(tmp.clone(), table[..cut].to_vec());
put_files(&work, &f);
if cut % 499 == 0 || cut == table.len() {
assert!(state(&open(&work, &plc)) == want, "cut at {cut}");
} else {
let s = open_with(&work, &plc, |c| c.load_all = false);
assert_eq!(crate::read(&s.directory).count(), accounts, "cut at {cut}");
}
assert!(
std::fs::read(work.join(&ckpt)).unwrap() == table && !work.join(&tmp).exists(),
"cut at {cut}"
);
}
// One that is not whole is refused, not taken for less than it held.
let whole = before(at);
for cut in [0, 5, whole[&ckpt].len() - 1] {
let mut f = whole.clone();
f.insert(ckpt.clone(), whole[&ckpt][..cut].to_vec());
put_files(&work, &f);
assert!(try_open(&work, &plc).is_err(), "a checkpoint cut at {cut}");
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// Compacted, an account's log and the service log keep only the events
/// in the window; a restart serves the same, and numbers new events after
/// every one given out, those dropped too.
#[tokio::test]
async fn compaction_drops_events_out_of_the_window() {
let plc = plc().await;
let dir = scratch("window");
let cfg = || Config {
backfill: std::time::Duration::from_millis(1500),
..config(&plc)
};
let s: Arc<Service> = Arc::new(Service::open(&dir, cfg()).unwrap());
let (a, ta) = create(&s, "alice").await;
let w =
|k: &str| json!({ "repo": a, "collection": "a.b.c", "rkey": k, "record": record(1) });
call(&s, "com.atproto.repo.putRecord", Some(&ta), w("one"))
.await
.unwrap();
let old = *s.events.subscribe().borrow();
tokio::time::sleep(std::time::Duration::from_millis(3000)).await;
call(&s, "com.atproto.repo.putRecord", Some(&ta), w("two"))
.await
.unwrap();
let last = *s.events.subscribe().borrow();
assert_eq!(last, old + 1);
let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
// The write may have made one due by age already.
wal.wait_compacted();
wal.compact().unwrap();
s.log.wait_compacted();
s.log.compact().unwrap();
// The window holds the last commit's event alone.
let kept = s.events.get(&s, 0, last, 0);
assert_eq!(kept.len(), 1);
drop(s);
let s = Service::open(&dir, cfg()).unwrap();
assert_eq!(*s.events.subscribe().borrow(), last);
assert_eq!(s.events.get(&s, 0, last, 0), kept);
drop(s);
tokio::time::sleep(std::time::Duration::from_millis(3000)).await;
// Out of the window, every event dropped: numbers go on all the same.
let s: Arc<Service> = Arc::new(Service::open(&dir, cfg()).unwrap());
assert_eq!(s.events.window_start(last), None);
let wal = crate::read(&s.account(&a).unwrap().store).wal.clone();
wal.wait_compacted();
wal.compact().unwrap();
drop(s);
let s: Arc<Service> = Arc::new(Service::open(&dir, cfg()).unwrap());
assert_eq!(s.events.len(), 0);
call(&s, "com.atproto.repo.putRecord", Some(&ta), w("three"))
.await
.unwrap();
assert_eq!(*s.events.subscribe().borrow(), last + 1);
let _ = std::fs::remove_dir_all(&dir);
}
/// The layout before generations, `accounts.log` and each account's
/// `repo.log`, with events written with seqs other than those served (as
/// the server before numbered them again at each start): moved to
/// generation 0 on the first start, its events with the seqs it served,
/// the same if the move is cut short and made again.
#[tokio::test]
async fn the_layout_before_generations_is_moved() {
let plc = plc().await;
let dir = scratch("migrate");
let (_, accounts) = script(&dir, &plc).await;
let s = open(&dir, &plc);
let want = state(&s);
drop(s);
// Make it the layout before: ords, and seqs off by some.
let legacy = |frames: Vec<Vec<Json>>| -> Vec<u8> {
let mut out = Vec::new();
for mut f in frames {
for e in f.iter_mut().filter(|e| e["t"] == "event") {
let seq = e["seq"].as_u64().unwrap();
e["ord"] = json!(seq * 3);
e["seq"] = json!(seq + 100);
}
out.extend(store::frame(&f));
}
out
};
let mut old = BTreeMap::new();
for (p, b) in files(&dir) {
let name = p.file_name().unwrap().to_string_lossy().to_string();
let renamed = match name.as_str() {
"accounts.0.log" => p.with_file_name("accounts.log"),
"repo.0.log" => p.with_file_name("repo.log"),
_ => p.clone(),
};
let b = if renamed != p {
legacy(store::scan(&b).unwrap().0)
} else {
b
};
old.insert(renamed, b);
}
let work = scratch("migrate-work");
put_files(&work, &old);
assert!(state(&open(&work, &plc)) == want, "moved");
assert!(!work.join("accounts.log").exists() && work.join("accounts.0.log").exists());
assert!(state(&open(&work, &plc)) == want, "started again");
// Cut short: one account's log moved, the service log not.
let _ = std::fs::remove_dir_all(&work);
put_files(&work, &old);
let alice = Path::new("repos").join(&accounts[0].0);
std::fs::rename(
work.join(alice.join("repo.log")),
work.join(alice.join("repo.0.log")),
)
.unwrap();
assert!(
state(&open(&work, &plc)) == want,
"moved after a move cut short"
);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
#[test]
fn wal_group_commit_covers_all_waiters() {
let dir = scratch("wal");
let path = dir.join("x.log");
let (wal, _) = Wal::open(&path, None).unwrap();
let a = wal
.append(&[json!({ "t": "session", "token": "a" })])
.unwrap();
let b = wal
.append(&[json!({ "t": "session", "token": "b" })])
.unwrap();
assert!(!wal.is_durable(a));
wal.wait(b).unwrap();
assert!(wal.is_durable(a) && wal.is_durable(b));
wal.wait(a).unwrap();
let (frames, _) = store::scan(&std::fs::read(&path).unwrap()).unwrap();
assert_eq!(frames.len(), 2);
let _ = std::fs::remove_dir_all(&dir);
}
/// Cut `accounts.log` at every byte of each frame invite codes wrote: one
/// code, several for several accounts, and an account made with one,
/// whose use is in its frame. Recovery gives the state before or after.
#[tokio::test]
async fn torn_invite_writes() {
for sync in SYNCS {
syncing(sync);
torn_invite_writes_once().await;
}
}
async fn torn_invite_writes_once() {
let plc = plc().await;
let dir = scratch("torn-invites");
let s = open(&dir, &plc);
let (alice, _) = create(&s, "alice").await;
let admin = format!(
"Basic {}",
base64::Engine::encode(
&base64::engine::general_purpose::STANDARD,
format!("admin:{ADMIN_PASSWORD}")
)
);
let as_admin = |nsid: &'static str, body: Json| {
let (s, admin) = (s.clone(), admin.clone());
async move {
call_raw(
&s,
nsid,
sessions::Bearer::of(Some(&admin)),
&[],
&body.to_string(),
)
.await
.unwrap()
}
};
let mut steps = vec![Step {
files: files(&dir),
journal: segments(&dir),
state: state(&s),
}];
let snap = |s: &Service, steps: &mut Vec<Step>| {
steps.push(Step {
files: files(&dir),
journal: segments(&dir),
state: state(s),
})
};
let code = as_admin(
"com.atproto.server.createInviteCode",
json!({ "useCount": 2, "forAccount": alice }),
)
.await;
snap(&s, &mut steps);
as_admin(
"com.atproto.server.createInviteCodes",
json!({ "codeCount": 2, "useCount": 1, "forAccounts": [alice, "did:plc:other"] }),
)
.await;
snap(&s, &mut steps);
let body = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw", "inviteCode": code["code"] });
call(&s, "com.atproto.server.createAccount", None, body)
.await
.unwrap();
snap(&s, &mut steps);
drop(s);
let accounts_log = PathBuf::from(ACCOUNTS_LOG_FILE);
let work = scratch("torn-invites-work");
let mut kinds = Vec::new();
for w in steps.windows(2) {
let whole = &w[1].files[&accounts_log];
let (from, to) = (w[0].files[&accounts_log].len(), whole.len());
kinds.extend(
store::scan(&whole[from..])
.unwrap()
.0
.iter()
.flatten()
.map(|e| e["t"].as_str().unwrap().to_string()),
);
for cut in from..=to {
let mut files = w[1].files.clone();
files.insert(accounts_log.clone(), whole[..cut].to_vec());
put_files(&work, &files);
let want = if cut == to { &w[1].state } else { &w[0].state };
assert_eq!(
&state(&open(&work, &plc)),
want,
"cut at {cut} of {from}..{to}"
);
}
}
for kind in crate::invites::ENTRIES {
assert!(
kinds.iter().any(|k| k == kind),
"{kind} never cut: {kinds:?}"
);
}
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&work);
}
/// A `createAccount` whose account's log cannot be made answers `500`
/// and writes nothing to `accounts.log`: the account does not exist, then
/// or after a restart, and the next one is made.
#[tokio::test]
async fn an_account_whose_log_cannot_be_made() {
let plc = plc().await;
let dir = scratch("unmade");
let s = open(&dir, &plc);
create(&s, "alice").await;
let before = (std::fs::read(accounts_log(&dir)).unwrap(), state(&s));
// A file where the accounts' directories must go: no directory can be
// made in it, for any user, root too. Alice's log stays open meanwhile.
let (repos, aside) = (dir.join("repos"), dir.join("repos.aside"));
std::fs::rename(&repos, &aside).unwrap();
std::fs::write(&repos, b"not a directory").unwrap();
let bob = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw" });
assert_eq!(
call(&s, "com.atproto.server.createAccount", None, bob.clone()).await,
Err("InternalServerError".into())
);
std::fs::remove_file(&repos).unwrap();
std::fs::rename(&aside, &repos).unwrap();
assert_eq!(
(std::fs::read(accounts_log(&dir)).unwrap(), state(&s)),
before
);
drop(s);
let s = open(&dir, &plc);
assert_eq!(state(&s), before.1);
call(&s, "com.atproto.server.createAccount", None, bob)
.await
.unwrap();
let _ = std::fs::remove_dir_all(&dir);
}
/// Every file the server makes is `0600` and every directory `0700`,
/// whatever the umask; the data directory is made `0700` whatever it was.
#[tokio::test]
async fn files_are_the_owners_alone() {
use std::os::unix::fs::PermissionsExt;
let plc = plc().await;
let dir = scratch("modes");
std::fs::create_dir_all(&dir).unwrap();
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap();
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
crate::blobs::test_upload(
&s,
Some(&token),
Some("text/plain"),
b"a blob".to_vec(),
false,
)
.await
.unwrap();
call(
&s,
"com.atproto.server.createAppPassword",
Some(&token),
json!({ "name": "x" }),
)
.await
.unwrap();
assert!(dir.join(format!("repos/{did}/blobs")).is_dir());
let mode = |p: &Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777;
assert_eq!(mode(&dir), 0o700);
let mut todo = vec![dir.clone()];
let mut seen = 0;
while let Some(d) = todo.pop() {
for e in std::fs::read_dir(&d).unwrap() {
let p = e.unwrap().path();
seen += 1;
if p.is_dir() {
assert_eq!(mode(&p), 0o700, "{}", p.display());
todo.push(p);
} else {
assert_eq!(mode(&p), 0o600, "{}", p.display());
}
}
}
assert!(seen >= 6, "{seen}");
let _ = std::fs::remove_dir_all(&dir);
}
/// Accounts are loaded on no more threads than the machine has cores,
/// with no more of their logs open than a share.
#[tokio::test]
async fn recovery_is_bounded() {
let plc = plc().await;
let dir = scratch("recover-threads");
let s = open(&dir, &plc);
let cores = std::thread::available_parallelism().unwrap().get();
let share = s.config.share;
let n = (cores * 3).max(share + 8);
for i in 0..n {
let body =
json!({ "handle": format!("user{i}.test"), "email": format!("u{i}@mail.com") });
call(&s, "com.atproto.server.createAccount", None, body)
.await
.unwrap();
}
assert!(s.ctx.files.open_count() <= share);
drop(s);
let s = open(&dir, &plc);
let d = crate::read(&s.directory);
assert_eq!(d.by_did.len(), n);
assert!(d.by_did.values().all(|a| a.store.is_loaded()));
let threads: std::collections::HashSet<_> = d
.by_did
.values()
.map(|a| crate::read(&a.store).thread)
.collect();
assert!(
threads.len() <= cores,
"{} threads, {cores} cores",
threads.len()
);
assert!(s.ctx.files.open_count() <= share);
drop(d);
let _ = std::fs::remove_dir_all(&dir);
}
/// Run the test `name` again in a process of its own, with `PDS_TEST_CHILD`
/// naming it, so that it may change the limits of its process: whether
/// this is that process. No log is opened while the child is spawned,
/// which holds a copy of every log's file open in this process, and its
/// lock, until it execs (`pds_store::spawning`): the tests beside this
/// one close their logs and open them again meanwhile.
fn in_child(name: &str) -> bool {
use std::process::{Command, Stdio};
if std::env::var("PDS_TEST_CHILD").as_deref() == Ok(name) {
return true;
}
let child = {
let _alone = pds_store::spawning();
Command::new(std::env::current_exe().unwrap())
.args([
&format!("log::tests::{name}"),
"--exact",
"--nocapture",
"--test-threads=1",
])
.env("PDS_TEST_CHILD", name)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap()
};
let out = child.wait_with_output().unwrap();
let text = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
assert!(out.status.success() && text.contains("1 passed"), "{text}");
false
}
/// Set the soft limit of `resource` to `soft`, or with none, to the hard limit.
fn set_limit(resource: Resource, soft: Option<u64>) {
let l = getrlimit(resource);
let current = match soft {
Some(s) => Some(l.maximum.map_or(s, |m| s.min(m))),
None => l.maximum,
};
setrlimit(resource, Rlimit { current, ..l }).unwrap();
}
/// The file descriptors the process holds open.
fn open_fds() -> usize {
let dir = if Path::new("/proc/self/fd").exists() {
"/proc/self/fd"
} else {
"/dev/fd"
};
std::fs::read_dir(dir).unwrap().count()
}
/// With 256 file descriptors, and the share of them the server takes
/// under that limit, 300 accounts are made, and the server started
/// again serves every one of them, and takes writes from the first and
/// the last: what it holds open does not grow with them, and is no more
/// than a share of logs, a share of tables and 72 files besides.
#[test]
fn many_accounts_few_files() {
if !in_child("many_accounts_few_files") {
return;
}
set_limit(Resource::Nofile, Some(256));
let share = store::files::share(store::files::limit());
assert_eq!(share, 64);
let most = 2 * share + 72;
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.unwrap();
rt.block_on(async {
let plc = plc().await;
let dir = scratch("many-accounts");
let s = open_with(&dir, &plc, |c| c.share = share);
let mut made = Vec::new();
for i in 0..300 {
let body = json!({ "handle": format!("user{i}.test"), "email": format!("u{i}@mail.com"), "password": "" });
made.push(call(&s, "com.atproto.server.createAccount", None, body).await.unwrap());
}
assert!(s.ctx.files.open_count() <= share);
assert!(open_fds() < most, "{} open", open_fds());
drop(s);
let s = open_with(&dir, &plc, |c| c.share = share);
assert!(open_fds() < most, "{} open", open_fds());
for m in &made {
let p = [("did".to_string(), m["did"].as_str().unwrap().to_string())];
call_with(&s, "com.atproto.sync.getLatestCommit", None, &p, Json::Null).await.unwrap();
}
for m in [&made[0], &made[299]] {
let body = json!({ "repo": m["did"], "collection": "a.b.c", "record": { "n": 1 } });
call(&s, "com.atproto.repo.createRecord", m["accessJwt"].as_str(), body).await.unwrap();
}
assert!(s.ctx.files.open_count() <= share);
let _ = std::fs::remove_dir_all(&dir);
});
}
/// A write whose append fails part way, as on a full disk, is `500` and
/// cut back from its log, leaving nothing of itself in the repository or
/// the firehose; the next write goes on from there, and a restart finds
/// the same.
#[test]
fn a_failed_append_is_cut_back() {
if !in_child("a_failed_append_is_cut_back") {
return;
}
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.unwrap();
rt.block_on(async {
// A handler for the signal, so that a write past the size limit fails instead.
let _xfsz = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::from_raw(Signal::XFSZ.as_raw())).unwrap();
let plc = plc().await;
let dir = scratch("cut-back");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let (bob, bob_token) = create(&s, "bob").await;
let put = |did: &str, rkey: &str, n: usize| json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": { "t": "x".repeat(n) } });
call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "a", 10)).await.unwrap();
let log = dir.join(repo_log(Path::new(""), &did));
let len = log_bytes(&log).len() as u64;
let before = state(&s);
let largest = files_whole(&dir).values().map(|b| b.len() as u64).max().unwrap();
set_limit(Resource::Fsize, Some(largest.max(len) + 8000));
assert_eq!(call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "b", 40_000)).await, Err("InternalServerError".into()));
assert_eq!(std::fs::metadata(&log).unwrap().len(), len);
assert_eq!(state(&s), before);
// Another account's write goes on, and so does this one's.
call(&s, "com.atproto.repo.putRecord", Some(&bob_token), put(&bob, "c", 10)).await.unwrap();
call(&s, "com.atproto.repo.putRecord", Some(&token), put(&did, "d", 10)).await.unwrap();
set_limit(Resource::Fsize, None);
let after = state(&s);
assert_eq!(after.2.len(), before.2.len() + 2);
drop(s);
assert_eq!(state(&open(&dir, &plc)), after);
let _ = std::fs::remove_dir_all(&dir);
});
}
/// A commit's frame of the first kind: JSON, its bytes as hex, as
/// `v0.1.0` wrote every commit's, and as a log may hold them.
mod json_frames {
use super::super::Hex;
use pds_repo::Change;
use serde_json::Value as Json;
struct CommitEntry<'a>(&'a Change);
impl serde::Serialize for CommitEntry<'_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let c = self.0;
let mut m = s.serialize_map(Some(10))?;
m.serialize_entry("t", "commit")?;
m.serialize_entry("rev", &c.rev)?;
m.serialize_entry("head", &Hex(&c.head_bytes))?;
m.serialize_entry(
"puts",
&c.puts
.iter()
.zip(&c.put_cids)
.map(|((k, v), cid)| (k, v, Hex(cid)))
.collect::<Vec<_>>(),
)?;
m.serialize_entry("dels", &c.dels)?;
m.serialize_entry(
"tree",
&c.tree
.iter()
.map(|(cid, b)| (Hex(cid), Hex(b)))
.collect::<Vec<_>>(),
)?;
m.serialize_entry("gone", &c.gone.iter().map(|g| Hex(g)).collect::<Vec<_>>())?;
m.serialize_entry(
"refs",
&c.refs
.iter()
.map(|(cid, n, rev)| (Hex(cid), n, rev))
.collect::<Vec<_>>(),
)?;
m.serialize_entry("counts", &[c.counts.0, c.counts.1])?;
m.end()
}
}
struct EventEntry<'a>(u64, u64, &'a [u8]);
impl serde::Serialize for EventEntry<'_> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let mut m = s.serialize_map(Some(4))?;
m.serialize_entry("t", "event")?;
m.serialize_entry("seq", &self.0)?;
m.serialize_entry("time", &self.1)?;
m.serialize_entry("frame", &Hex(self.2))?;
m.end()
}
}
pub(super) fn commit_frame(
commit: Option<&Change>,
entries: &[Json],
seq: u64,
frames: &[Vec<u8>],
times: &[u64],
) -> Vec<u8> {
let capacity = frames.iter().map(|f| f.len() * 3).sum::<usize>() + 4096;
pds_store::frame_of(capacity, |out| {
use serde::ser::{SerializeSeq, Serializer};
let mut s = serde_json::Serializer::new(out);
let mut seq_out = s.serialize_seq(None).unwrap();
if let Some(c) = commit {
seq_out.serialize_element(&CommitEntry(c)).unwrap();
}
for e in entries {
seq_out.serialize_element(e).unwrap();
}
for (i, f) in frames.iter().enumerate() {
seq_out
.serialize_element(&EventEntry(seq + i as u64, times[i], f))
.unwrap();
}
seq_out.end().unwrap();
})
}
}
/// A commit's frame with holes, filled, is byte for byte its frame of
/// the first kind: with and without its commit, events and other
/// entries, bytes that end with zero and none.
#[test]
fn a_commit_frame_filled_is_its_json_frame() {
let bytes = |n: usize, seed: u8, last: u8| -> Vec<u8> {
let mut b: Vec<u8> = (0..n)
.map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed))
.collect();
if let Some(x) = b.last_mut() {
*x = last;
}
b
};
let cid = |b: &[u8]| pds_repo::data::cid_of(0x71, b);
let record =
json!({ "$type": "app.bsky.feed.post", "text": "a \"post\"\n", "langs": ["en"] });
let (n1, n2) = (bytes(1000, 3, 7), bytes(1100, 5, 0));
let c = Change {
rev: "3lzzzzzzzzz2a".into(),
head_bytes: bytes(190, 1, 9),
puts: vec![("app.bsky.feed.post/3lzz".into(), record.clone())],
put_cids: vec![cid(&record.to_string().into_bytes())],
dels: vec!["app.bsky.feed.post/3lyy".into()],
tree: vec![(cid(&n1), n1), (cid(&n2), n2)],
gone: vec![cid(b"gone")],
refs: vec![(cid(b"ref"), 1, "3lzzzzzzzzz2a".into())],
counts: (12, 30),
..Default::default()
};
let empty = Change::default();
let handled = [json!({ "t": "handled", "cid": "bafy", "landed": true })];
let events = [bytes(3000, 11, 4), bytes(2900, 13, 0)];
for (commit, entries, frames) in [
(Some(&c), &[][..], &events[..1]),
(Some(&c), &handled[..], &events[..]),
(Some(&c), &[][..], &[][..]),
(Some(&empty), &[][..], &events[1..]),
(None, &handled[..], &events[1..]),
(None, &handled[..], &[][..]),
] {
let times: Vec<u64> = (0..frames.len() as u64)
.map(|i| 1_700_000_000_000 + i)
.collect();
let with = commit_frame(commit, entries, 9, frames, ×);
let json = json_frames::commit_frame(commit, entries, 9, frames, ×);
assert_eq!(with[store::HEADER], store::HOLES);
assert_eq!(
store::fill(&with[store::HEADER..]).unwrap().as_ref(),
&json[store::HEADER..]
);
assert_eq!(store::parse_frame(&with), store::parse_frame(&json));
assert_ne!(*with.last().unwrap(), 0);
}
}
/// The frame at the end of the log of `did`'s current generation: its
/// bytes.
fn last_frame(s: &Service, dir: &Path, did: &str) -> Vec<u8> {
let g = store_of(s, did, |st| st.wal.generation());
let path = crate::store::account_layout(dir, did).log(g);
let mut last = None;
store::scan_file(&path, |at, len, _| {
last = Some((at, len));
Ok(())
})
.unwrap()
.unwrap();
let (at, len) = last.unwrap();
std::fs::read(&path).unwrap()[at as usize..][..len as usize].to_vec()
}
/// The bytes of a commit's frame of a post, in a tree of fifty: with
/// holes, and as JSON with its bytes in hex.
#[tokio::test]
async fn the_bytes_of_a_commit_frame() {
let plc = plc().await;
let dir = scratch("commit-bytes");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
let words = "the weather here is lovely today and I went for a long walk by the river ";
for i in 0..50 {
let post = json!({
"$type": "app.bsky.feed.post",
"text": format!("{i}: {}", words.repeat(3)),
"langs": ["en"],
"createdAt": "2026-11-12T10:00:00.000Z",
});
call(
&s,
"com.atproto.repo.createRecord",
Some(&token),
json!({ "repo": did, "collection": "app.bsky.feed.post", "record": post }),
)
.await
.unwrap();
}
let with = last_frame(&s, &dir, &did);
let json = store::HEADER + store::fill(&with[store::HEADER..]).unwrap().len();
println!(
"a commit's frame: {json} bytes as JSON, bytes as hex; {} with holes",
with.len()
);
assert!(with.len() * 10 < json * 6, "{} of {json}", with.len());
let _ = std::fs::remove_dir_all(&dir);
}
/// A log with its first `upto` frames with holes made JSON, their
/// bytes in hex, as `v0.1.0` wrote every commit's: and how many it
/// made so.
fn as_json(log: &[u8], upto: usize) -> (Vec<u8>, usize) {
let (mut out, mut made, mut at) = (Vec::new(), 0, 0);
while at < log.len() {
let len = u32::from_le_bytes(log[at..at + 4].try_into().unwrap()) as usize;
let frame = &log[at..at + store::HEADER + len];
if frame[store::HEADER] == store::HOLES && made < upto {
let text = store::fill(&frame[store::HEADER..]).unwrap();
out.extend(store::frame_of(text.len(), |o| o.extend_from_slice(&text)));
made += 1;
} else {
out.extend_from_slice(frame);
}
at += store::HEADER + len;
}
(out, made)
}
/// An account's log of commits `v0.1.0` wrote, JSON, and then commits
/// this version wrote, with holes, in any proportion: it loads, serves
/// every block, record and event as the server that wrote it did, from
/// frames of both kinds, and compacts into the same table as the log
/// written all one way or all the other.
#[tokio::test]
async fn a_log_of_both_kinds() {
let plc = plc().await;
let dir = scratch("kinds");
let s = open(&dir, &plc);
let (did, token) = create(&s, "alice").await;
for i in 0..30 {
let (nsid, body) = if i % 7 == 6 {
let rkey = format!("k{}", (i - 3) % 12);
(
"com.atproto.repo.deleteRecord",
json!({ "repo": did, "collection": "a.b.c", "rkey": rkey }),
)
} else {
let rkey = format!("k{}", i % 12);
(
"com.atproto.repo.putRecord",
json!({ "repo": did, "collection": "a.b.c", "rkey": rkey, "record": record(i) }),
)
};
call(&s, nsid, Some(&token), body).await.unwrap();
}
let want = state(&s);
drop(s);
let written = files(&dir);
let layout = crate::store::account_layout(Path::new(""), &did);
let logs: Vec<PathBuf> = written
.keys()
.filter(|p| p.starts_with(&layout.dir) && p.extension().is_some_and(|e| e == "log"))
.cloned()
.collect();
let holed: usize = logs
.iter()
.map(|l| as_json(&written[l], usize::MAX).1)
.sum();
assert!(holed > 25, "{holed} frames with holes");
let mut tables = Vec::new();
// Of each log's frames with holes, none, all, or the first half
// made JSON.
for (name, of, by) in [("holes", 0, 1), ("json", 1, 1), ("both", 1, 2)] {
let mut files = written.clone();
let mut made = 0;
for l in &logs {
let frames = as_json(&written[l], usize::MAX).1;
let (bytes, n) = as_json(&written[l], frames * of / by);
files.insert(l.clone(), bytes);
made += n;
}
match (of, by) {
(0, _) => assert_eq!(made, 0),
(1, 1) => assert_eq!(made, holed),
_ => assert!(made > 0 && made < holed, "{made} of {holed}"),
}
let work = scratch(&format!("kinds-{name}"));
put_files(&work, &files);
let s = open(&work, &plc);
assert_eq!(state(&s), want, "{name}: served");
let wal = store_of(&s, &did, |st| st.wal.clone());
wal.wait_compacted();
wal.compact().unwrap();
drop(wal);
assert_eq!(state(&s), want, "{name}: compacted");
tables.push(std::fs::read(work.join(layout.checkpoint())).unwrap());
drop(s);
let _ = std::fs::remove_dir_all(&work);
}
assert!(tables.iter().all(|t| *t == tables[0]), "one table");
let _ = std::fs::remove_dir_all(&dir);
}
}