~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
1722 lines · 61.6 KB Raw History
//! The write path of each layout under the same commits: logs of
//! `pds_store`, which give themselves room as they are written to, each
//! synced by itself or all of them waiting on the server's journal, and
//! this benchmark's own, growing or allocated ahead, with JSON frames or
//! binary ones, one for each account or one for all, or one for each
//! account that no commit syncs alone, beside a journal for all that is
//! synced; SQLite likewise, a transaction a commit or one for all that
//! wait; and RocksDB likewise, a batch a commit. A commit is answered once
//! it is on disk; or, to see what the processor alone allows, never synced
//! at all.

use crate::check::Check;
use crate::data::{self, Commit, SEED};
use crate::fj;
use crate::frame::{self, Encoding, Frames};
use crate::rocks;
use crate::util::{Rng, Rss, at, cores, dir_bytes, files, mib, ms, par, scratch, us};
use pds_store::{Files, HEADER, Wal};
use rusqlite::{Connection, params};
use std::collections::HashMap;
use std::fs::File;
use std::os::unix::fs::FileExt;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Barrier, Condvar, Mutex, PoisonError};
use std::time::{Duration, Instant};

/// The accounts the commits are spread over, as the PDS benchmark's
/// createRecord part spreads its own (`bench/README.md`).
const ACCOUNTS: usize = 40;

/// Whether SQLite's `fullfsync` changes anything: only where a plain fsync
/// leaves the drive's cache unflushed, which the log's syncs never do
/// (`File::sync_data` there is `F_FULLFSYNC`).
const FULLFSYNC: bool = cfg!(target_os = "macos");

/// The threads that open a layout's files: each is synced as it is made,
/// and a few thousand, one after another, take longer than the commits.
const OPENERS: usize = 64;

/// The number of the first commit that warms a layout.
const WARM: u64 = 1_000_000;

static SYNCS: AtomicU64 = AtomicU64::new(0);
static SYNC_NANOS: AtomicU64 = AtomicU64::new(0);
/// The bytes of zeros written to give logs room, since a row began.
static ZEROS: AtomicU64 = AtomicU64::new(0);

/// Count a sync of a log, and how long it took.
pub fn synced(d: Duration) {
    SYNCS.fetch_add(1, Ordering::Relaxed);
    SYNC_NANOS.fetch_add(d.as_nanos() as u64, Ordering::Relaxed);
}

fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
    m.lock().unwrap_or_else(PoisonError::into_inner)
}

/// A way of keeping the accounts' commits: `write` returns once the commit
/// is on disk, or, of a layout that never syncs, once it is written.
trait Layout: Sync {
    fn write(&self, account: usize, c: &Arc<Commit>);

    /// The syncs it has made and the nanoseconds they took, if it counts
    /// its own: a log's are counted as they are made (`synced`).
    fn syncs(&self) -> Option<(u64, u64)> {
        None
    }

    /// The timed commits begin: what it counts of its own, it counts
    /// from here.
    fn begin(&self) {}

    /// The timed commits are over: a layout that checks what its files
    /// hold does so. Of a layout that makes checkpoints: those made
    /// during the timed commits, and how long one more took, made now.
    fn end(&self) -> Option<(Checkpoints, Duration)> {
        None
    }
}

/// A log: `append` hands it a frame and answers the frame's number, and
/// `wait` returns once the frame of that number is on disk. A commit says
/// when it has `begun` and when its frame is `written`, to a log whose
/// syncs wait for those between the two.
trait Log: Send + Sync {
    fn append(&self, frame: &[u8]) -> u64;
    fn wait(&self, n: u64);
    fn begun(&self) {}
    fn written(&self) {}
}

impl Log for Wal {
    fn append(&self, frame: &[u8]) -> u64 {
        self.append_frame_at(frame).expect("an append").0
    }

    fn wait(&self, n: u64) {
        Wal::wait(self, n).expect("a sync");
    }
}

/// How a log allocated ahead gets its room: from `fallocate`, where there
/// is one, which leaves it unwritten to the file system; or as zeros
/// written once.
#[derive(Clone, Copy)]
enum Room {
    #[cfg(target_os = "linux")]
    Fallocate,
    Zeros,
}

impl Room {
    /// Give `file` `len` more bytes at `at`.
    fn give(self, file: &File, at: u64, len: u64) {
        match self {
            #[cfg(target_os = "linux")]
            Room::Fallocate => {
                rustix::fs::fallocate(file, rustix::fs::FallocateFlags::empty(), at, len)
                    .expect("room allocated")
            }
            Room::Zeros => {
                ZEROS.fetch_add(len, Ordering::Relaxed);
                let zeros = vec![0u8; len.min(1 << 20) as usize];
                let mut done = 0;
                while done < len {
                    let n = (len - done).min(zeros.len() as u64) as usize;
                    file.write_all_at(&zeros[..n], at + done).expect("zeros");
                    done += n as u64;
                }
            }
        }
    }
}

/// How the commits that wait on a log at once come to be answered.
#[derive(Clone, Copy, PartialEq)]
enum Wait {
    /// As a `Wal`'s are: the lock is let go through each sync, one at a
    /// time, and every commit waiting is woken when it ends: those it
    /// covered are answered then, whatever sync begins next.
    Together,
    /// As a `Wal`'s were before: the lock on what is durable is held
    /// through each sync, and a commit is answered once it has had that
    /// lock, in its turn, behind any commit that began another sync
    /// before it.
    InTurn,
    /// As `Together`, but a sync first lets the commits that have begun
    /// be written, for at most `BEGUN`, and so covers them: with writers
    /// that each begin their next commit as the last is answered, every
    /// one of them is in every sync, where without it those that begin
    /// during a sync wait for the one after.
    Gathered,
}

/// The longest a sync waits for commits that have begun to be written.
const BEGUN: Duration = Duration::from_millis(1);

/// Wait, for at most `BEGUN`, until none of `writing` commits is left to
/// be written.
fn gather(writing: &AtomicU64) {
    let began = Instant::now();
    while writing.load(Ordering::SeqCst) > 0 && began.elapsed() < BEGUN {
        std::thread::yield_now();
    }
}

/// A log of this benchmark's own, each frame written at its offset: in a
/// file that grows as they are written, or, with a chunk, in one allocated
/// ahead a chunk at a time, so that a sync has no new size to make
/// durable. The first is what the second is compared with: all else of
/// them is the same. Commits that wait at once share a sync. Where the
/// frames of a file allocated ahead end is found, at a replay, by the
/// first that fails its check.
struct Placed {
    file: File,
    ahead: Option<(u64, Room)>,
    wait: Wait,
    /// The bytes of frames written, the bytes allocated, and the frames written.
    inner: Mutex<(u64, u64, u64)>,
    /// The frames known to be on disk, and whether a sync is going on.
    synced: Mutex<(u64, bool)>,
    /// Signalled, with `synced`, when a sync ends.
    ended: Condvar,
    /// The commits begun and not yet written.
    writing: AtomicU64,
}

impl Placed {
    /// A log at `path`, made durable empty, as a `Wal` is, if `durable`.
    fn open(path: &Path, ahead: Option<(u64, Room)>, wait: Wait, durable: bool) -> Placed {
        let dir = path.parent().expect("a directory");
        std::fs::create_dir_all(dir).expect("a directory");
        let file = File::options()
            .read(true)
            .write(true)
            .create_new(true)
            .open(path)
            .expect("a log");
        if let Some((chunk, room)) = ahead {
            room.give(&file, 0, chunk);
        }
        if durable {
            file.sync_all().expect("a sync");
            pds_store::sync_dir(dir).expect("a sync");
        }
        Placed {
            file,
            ahead,
            wait,
            inner: Mutex::new((0, ahead.map_or(0, |(chunk, _)| chunk), 0)),
            synced: Mutex::new((0, false)),
            ended: Condvar::new(),
            writing: AtomicU64::new(0),
        }
    }

    /// Sync the frames written so far: answers how many.
    fn sync(&self) -> u64 {
        let target = lock(&self.inner).2;
        let started = Instant::now();
        self.file.sync_data().expect("a sync");
        synced(started.elapsed());
        target
    }
}

impl Log for Placed {
    fn append(&self, frame: &[u8]) -> u64 {
        let mut inner = lock(&self.inner);
        let (len, allocated, written) = &mut *inner;
        if let Some((chunk, room)) = self.ahead {
            while *len + frame.len() as u64 > *allocated {
                room.give(&self.file, *allocated, chunk);
                *allocated += chunk;
            }
        }
        self.file.write_all_at(frame, *len).expect("an append");
        *len += frame.len() as u64;
        *written += 1;
        *written
    }

    fn begun(&self) {
        self.writing.fetch_add(1, Ordering::SeqCst);
    }

    fn written(&self) {
        self.writing.fetch_sub(1, Ordering::SeqCst);
    }

    fn wait(&self, n: u64) {
        let mut state = lock(&self.synced);
        while state.0 < n {
            if self.wait == Wait::InTurn {
                state.0 = self.sync();
            } else if state.1 {
                state = self
                    .ended
                    .wait(state)
                    .unwrap_or_else(PoisonError::into_inner);
            } else {
                state.1 = true;
                drop(state);
                if self.wait == Wait::Gathered {
                    gather(&self.writing);
                }
                let target = self.sync();
                state = lock(&self.synced);
                *state = (target, false);
                self.ended.notify_all();
            }
        }
    }
}

/// Which log a layout's are: the server's (`Wal`), with the room it gives
/// itself as it is written to, each synced by itself (`PDS_SYNC` `each`)
/// or all of a pool written to its journal too and waiting on it
/// (`together`); this benchmark's, growing as it is written; that with
/// each waiting commit answered in its turn, or with each sync letting the
/// commits begun be written first; or this benchmark's allocated ahead in
/// chunks of so many bytes, got so.
#[derive(Clone, Copy)]
enum Grow {
    Server,
    ServerJournal,
    Placed,
    InTurn,
    Gathered,
    Ahead(u64, Room),
}

/// Logs: one for each account, as the server keeps them, or one for all.
struct Logs {
    logs: Vec<Box<dyn Log>>,
    encoding: Encoding,
    /// Whether a commit waits for its frame to be on disk.
    durable: bool,
    last: Vec<Mutex<Vec<Vec<u8>>>>,
}

impl Logs {
    fn new(dir: &Path, n: usize, accounts: usize, kind: (Encoding, Grow), durable: bool) -> Logs {
        let (encoding, grow) = kind;
        // The server's pool, of the share of file descriptors it takes, and
        // its journal, with its checkpoint thread, as `pds serve` has them.
        let pool = match grow {
            Grow::ServerJournal => Some(Files::with_journal(
                pds_store::files::share(pds_store::files::limit()),
                pds_store::Journal::open(dir, None).expect("a journal"),
            )),
            _ => None,
        };
        Logs {
            logs: par(n, OPENERS, |i| {
                let path = dir.join(format!("{i}/repo.log"));
                let placed = |ahead, wait| Box::new(Placed::open(&path, ahead, wait, durable));
                let log: Box<dyn Log> = match grow {
                    Grow::Server => Box::new(Wal::open(&path, None).expect("a log").0),
                    Grow::ServerJournal => {
                        Box::new(Wal::open(&path, pool.clone()).expect("a log").0)
                    }
                    Grow::Placed => placed(None, Wait::Together),
                    Grow::InTurn => placed(None, Wait::InTurn),
                    Grow::Gathered => placed(None, Wait::Gathered),
                    Grow::Ahead(chunk, room) => placed(Some((chunk, room)), Wait::Together),
                };
                log
            }),
            encoding,
            durable,
            last: (0..accounts).map(|_| Mutex::new(Vec::new())).collect(),
        }
    }
}

impl Layout for Logs {
    fn write(&self, account: usize, c: &Arc<Commit>) {
        let log = &self.logs[account % self.logs.len()];
        // Appended under the account's lock, as the server appends; synced
        // outside it, so that commits waiting at once share a sync.
        log.begun();
        let n = {
            let mut last = self.last[account].lock().expect("the account");
            let n = log.append(&self.encoding.frame(c, &last));
            *last = data::nodes(c);
            n
        };
        log.written();
        if self.durable {
            log.wait(n);
        }
    }
}

/// The checkpoints a layout made: how many, and the nanoseconds they took
/// together and the longest alone.
#[derive(Clone, Copy, Default)]
struct Checkpoints {
    count: u64,
    nanos: u64,
    longest: u64,
}

/// The bytes a journal grows by from one checkpoint to the next.
const JOURNAL_CHECKPOINT: u64 = 16 << 20;

/// `JOURNAL_CHECKPOINT`, or the bytes `BENCH_JOURNAL_CHECKPOINT` names: a
/// small run, which writes less than the one, sets the other.
fn journal_checkpoint() -> u64 {
    std::env::var("BENCH_JOURNAL_CHECKPOINT").map_or(JOURNAL_CHECKPOINT, |v| {
        v.parse().expect("BENCH_JOURNAL_CHECKPOINT: bytes")
    })
}

/// What a journal's record has before its frame: its account's number,
/// and the frame's offset in that account's log.
const PLACE: usize = 12;

/// A journal's record of `frame`, written at `offset` of `account`'s log:
/// a frame itself, its payload the account, the offset and `frame`, its
/// check XXH3-64 (`Check::Xxh3`).
fn record(account: usize, offset: u64, frame: &[u8]) -> Vec<u8> {
    let len = PLACE + frame.len();
    let mut out = Vec::with_capacity(HEADER + len);
    out.extend_from_slice(&(len as u32).to_le_bytes());
    out.extend_from_slice(&[0; HEADER - 4]);
    out.extend_from_slice(&(account as u32).to_le_bytes());
    out.extend_from_slice(&offset.to_le_bytes());
    out.extend_from_slice(frame);
    let check = Check::Xxh3.of_frame(&out[HEADER..]);
    out[4..HEADER].copy_from_slice(&check);
    out
}

/// An account's log beside a journal: written and never synced by a
/// commit.
struct Account {
    file: File,
    /// The bytes of frames written, and the nodes its last commit made.
    inner: Mutex<(u64, Vec<Vec<u8>>)>,
    /// Whether it was written to since a checkpoint last took it.
    #[cfg(not(target_os = "linux"))]
    written: AtomicBool,
}

/// A journal for all the accounts, and their logs. A commit's frame is
/// written to its account's log, which no commit syncs, and a record of
/// it to the journal, a log for all allocated ahead (`Placed`), whose
/// sync the commit waits for. A checkpoint makes the accounts' logs
/// durable all at once: the journal before it is then not needed to
/// recover them.
struct Journal {
    log: Placed,
    accounts: Vec<Account>,
    /// The directory all of it is under.
    #[cfg(target_os = "linux")]
    top: File,
    /// The bytes the journal grows by from one checkpoint to the next.
    every: u64,
    /// The bytes of records written to the journal, and those of them a
    /// checkpoint has covered.
    journaled: AtomicU64,
    checkpointed: AtomicU64,
    closed: AtomicBool,
    made: Mutex<Checkpoints>,
}

impl Journal {
    /// Make durable every account's log as it is now, which covers every
    /// record the journal has as this begins: a record is written after
    /// its frame, and after its log is marked as written to. On Linux one
    /// `syncfs` of the directory's file system, as the server's
    /// `sync_found`; elsewhere an `fsync` of each log written to since
    /// the last checkpoint, which hands it to the drive, and one full
    /// sync of the journal, which flushes the drive. It takes no lock:
    /// commits go on through it.
    fn checkpoint(&self) -> Duration {
        let started = Instant::now();
        let upto = self.journaled.load(Ordering::SeqCst);
        #[cfg(target_os = "linux")]
        rustix::fs::syncfs(&self.top).expect("a sync");
        #[cfg(not(target_os = "linux"))]
        {
            for account in &self.accounts {
                if account.written.swap(false, Ordering::SeqCst) {
                    rustix::fs::fsync(&account.file).expect("a sync");
                }
            }
            self.log.file.sync_data().expect("a sync");
        }
        self.checkpointed.store(upto, Ordering::SeqCst);
        started.elapsed()
    }

    /// Whether the journal has grown by a checkpoint's bytes since the
    /// last.
    fn due(&self) -> bool {
        let checkpointed = self.checkpointed.load(Ordering::SeqCst);
        self.journaled.load(Ordering::SeqCst) - checkpointed >= self.every
    }

    /// The checkpoint thread: a checkpoint, counted, whenever one is due,
    /// until the journal is closed.
    fn checkpoints(&self) {
        while !self.closed.load(Ordering::SeqCst) {
            if self.due() {
                let nanos = self.checkpoint().as_nanos() as u64;
                let mut made = lock(&self.made);
                made.count += 1;
                made.nanos += nanos;
                made.longest = made.longest.max(nanos);
            } else {
                std::thread::park();
            }
        }
    }
}

/// A log for each account, written and not synced, and one journal for
/// all, with the thread that makes its checkpoints.
struct Journaled {
    journal: Arc<Journal>,
    encoding: Encoding,
    checkpointer: Mutex<Option<std::thread::JoinHandle<()>>>,
    /// The checkpoint thread, to wake.
    waker: std::thread::Thread,
}

impl Journaled {
    fn new(dir: &Path, accounts: usize, encoding: Encoding) -> Journaled {
        let journal = Arc::new(Journal {
            log: Placed::open(
                &dir.join("journal.log"),
                Some((CHUNK, Room::Zeros)),
                Wait::Together,
                true,
            ),
            accounts: par(accounts, OPENERS, |i| {
                let path = dir.join(format!("{i}/repo.log"));
                let parent = path.parent().expect("a directory");
                std::fs::create_dir_all(parent).expect("a directory");
                let file = File::options()
                    .read(true)
                    .write(true)
                    .create_new(true)
                    .open(&path)
                    .expect("a log");
                file.sync_all().expect("a sync");
                pds_store::sync_dir(parent).expect("a sync");
                Account {
                    file,
                    inner: Mutex::default(),
                    #[cfg(not(target_os = "linux"))]
                    written: AtomicBool::new(false),
                }
            }),
            #[cfg(target_os = "linux")]
            top: File::open(dir).expect("the directory"),
            every: journal_checkpoint(),
            journaled: AtomicU64::new(0),
            checkpointed: AtomicU64::new(0),
            closed: AtomicBool::new(false),
            made: Mutex::default(),
        });
        let checkpointer = std::thread::spawn({
            let journal = journal.clone();
            move || journal.checkpoints()
        });
        Journaled {
            journal,
            encoding,
            waker: checkpointer.thread().clone(),
            checkpointer: Mutex::new(Some(checkpointer)),
        }
    }

    /// Stop the checkpoint thread, the checkpoint it is making done.
    fn stop(&self) {
        self.journal.closed.store(true, Ordering::SeqCst);
        self.waker.unpark();
        if let Some(checkpointer) = lock(&self.checkpointer).take() {
            checkpointer.join().expect("the checkpoint thread");
        }
    }
}

impl Drop for Journaled {
    fn drop(&mut self) {
        self.stop();
    }
}

impl Layout for Journaled {
    fn write(&self, account: usize, c: &Arc<Commit>) {
        let journal = &*self.journal;
        let log = &journal.accounts[account];
        // The frame and then its record, under the account's lock: the
        // journal never has a record of a frame further on in a log
        // before one of a frame before it.
        let n = {
            let mut inner = lock(&log.inner);
            let (len, last) = &mut *inner;
            let frame = self.encoding.frame(c, last);
            log.file.write_all_at(&frame, *len).expect("an append");
            #[cfg(not(target_os = "linux"))]
            log.written.store(true, Ordering::SeqCst);
            let record = record(account, *len, &frame);
            let n = journal.log.append(&record);
            journal
                .journaled
                .fetch_add(record.len() as u64, Ordering::SeqCst);
            *len += frame.len() as u64;
            *last = data::nodes(c);
            n
        };
        if journal.due() {
            self.waker.unpark();
        }
        journal.log.wait(n);
    }

    fn begin(&self) {
        *lock(&self.journal.made) = Checkpoints::default();
    }

    fn end(&self) -> Option<(Checkpoints, Duration)> {
        self.stop();
        let made = *lock(&self.journal.made);
        Some((made, self.journal.checkpoint()))
    }
}

/// See that each account's log under `dir` holds the frames the journal
/// there has records of for it, at the offsets they say, and nothing
/// else, and that the journal has `wrote` records: every commit's.
fn journal_kept(dir: &Path, accounts: usize, wrote: u64, name: &str) {
    let file = File::open(dir.join("journal.log")).expect("the journal");
    let size = file.metadata().expect("the journal's size").len();
    let mut records = Frames::new(file, size, Some(Check::Xxh3));
    let mut logs = vec![Vec::new(); accounts];
    let mut found = 0;
    while let Some(payload) = records.next_payload() {
        let (place, frame) = payload.split_at(PLACE);
        let account = u32::from_le_bytes(place[..4].try_into().expect("4 bytes")) as usize;
        let offset = u64::from_le_bytes(place[4..].try_into().expect("8 bytes"));
        let log = &mut logs[account];
        assert_eq!(
            offset,
            log.len() as u64,
            "a record of {name} for account {account} is where the one before it ends"
        );
        log.extend_from_slice(frame);
        found += 1;
    }
    assert_eq!(found, wrote, "every commit journaled by {name}");
    par(accounts, cores(), |i| {
        let held = std::fs::read(dir.join(format!("{i}/repo.log"))).expect("the log read");
        assert!(
            held == logs[i],
            "account {i}'s log holds what {name} journaled for it"
        );
    });
}

/// The tables a SQLite layout keeps: blocks by CID, records by path, and
/// events by seq.
pub const SCHEMA: &str = "CREATE TABLE block(account INTEGER, cid BLOB, rev TEXT, bytes BLOB, PRIMARY KEY(account, cid)) WITHOUT ROWID;
     CREATE TABLE record(account INTEGER, path TEXT, cid BLOB, rev TEXT, PRIMARY KEY(account, path)) WITHOUT ROWID;
     CREATE TABLE event(seq INTEGER PRIMARY KEY, account INTEGER, time INTEGER, frame BLOB);";

/// What SQLite does to put a transaction on disk: a sync that flushes the
/// drive's cache (`F_FULLFSYNC` where `fsync` does not), `fsync` alone, or
/// nothing.
#[derive(Clone, Copy, PartialEq)]
enum Flush {
    Disk,
    Fsync,
    Never,
}

/// A database in WAL mode at `dir/<i>/store.sqlite`, its tables made.
fn database(dir: &Path, i: usize, flush: Flush) -> Connection {
    std::fs::create_dir_all(dir.join(i.to_string())).expect("a directory");
    let c = Connection::open(dir.join(format!("{i}/store.sqlite"))).expect("a database");
    let synchronous = if flush == Flush::Never { "OFF" } else { "FULL" };
    let on = if flush == Flush::Disk && FULLFSYNC {
        "ON"
    } else {
        "OFF"
    };
    // The tables in one transaction: one sync of them, not three.
    c.execute_batch(&format!(
        "PRAGMA journal_mode=WAL; PRAGMA synchronous={synchronous}; PRAGMA fullfsync={on}; PRAGMA checkpoint_fullfsync={on};
             BEGIN; {SCHEMA} COMMIT;"
    ))
    .expect("a schema");
    c
}

/// What a commit puts in a database, inside a transaction: its blocks, the
/// nodes the account's commit before made taken out, its record's path and
/// its event.
fn put(db: &Connection, account: usize, c: &Commit, last: &[Vec<u8>]) {
    let a = account as i64;
    let mut block = db
        .prepare_cached("INSERT INTO block(account, cid, rev, bytes) VALUES (?1, ?2, ?3, ?4)")
        .expect("a statement");
    for (cid, b) in &c.blocks {
        block.execute(params![a, cid, c.rev, b]).expect("a block");
    }
    let mut gone = db
        .prepare_cached("DELETE FROM block WHERE account = ?1 AND cid = ?2")
        .expect("a statement");
    for cid in last {
        gone.execute(params![a, cid]).expect("a block taken out");
    }
    db.prepare_cached(
        "INSERT OR REPLACE INTO record(account, path, cid, rev) VALUES (?1, ?2, ?3, ?4)",
    )
    .expect("a statement")
    .execute(params![a, c.path, c.blocks[1].0, c.rev])
    .expect("a record");
    db.prepare_cached("INSERT INTO event(seq, account, time, frame) VALUES (?1, ?2, ?3, ?4)")
        .expect("a statement")
        .execute(params![
            c.op as i64,
            a,
            1_700_000_000_000i64 + c.op as i64,
            c.event
        ])
        .expect("an event");
}

/// SQLite databases in WAL mode with `synchronous=FULL`, a commit a
/// transaction: one database for each account, or one for all.
struct Sqlite {
    dbs: Vec<Mutex<Connection>>,
    last: Vec<Mutex<Vec<Vec<u8>>>>,
}

impl Sqlite {
    fn new(dir: &Path, n: usize, accounts: usize, flush: Flush) -> Sqlite {
        Sqlite {
            dbs: par(n, OPENERS, |i| Mutex::new(database(dir, i, flush))),
            last: (0..accounts).map(|_| Mutex::new(Vec::new())).collect(),
        }
    }
}

/// Closed unsynced: closing a database folds its WAL into it, and the
/// files are removed next.
impl Drop for Sqlite {
    fn drop(&mut self) {
        for db in &self.dbs {
            let _ = lock(db).execute_batch("PRAGMA synchronous=OFF;");
        }
    }
}

impl Layout for Sqlite {
    fn write(&self, account: usize, c: &Arc<Commit>) {
        let mut last = self.last[account].lock().expect("the account");
        let mut db = self.dbs[account % self.dbs.len()]
            .lock()
            .expect("the database");
        let tx = db.transaction().expect("a transaction");
        put(&tx, account, c, &last);
        tx.commit().expect("a commit");
        *last = data::nodes(c);
    }
}

/// The commits waiting for the one writer of a database, and how far it
/// has got.
#[derive(Default)]
struct Waiting {
    commits: Vec<(usize, Arc<Commit>)>,
    /// The commits given, and those in transactions that are on disk.
    given: u64,
    done: u64,
    closed: bool,
}

struct Queue {
    waiting: Mutex<Waiting>,
    arrived: Condvar,
    committed: Condvar,
}

/// One SQLite database for all, as a server built on it would share a
/// sync: one thread writes, taking every commit that waits and putting
/// them all in one transaction, and each is answered when that is on
/// disk.
struct Batched {
    queue: Arc<Queue>,
    writer: Option<std::thread::JoinHandle<()>>,
}

impl Batched {
    fn new(dir: &Path, accounts: usize, flush: Flush) -> Batched {
        let queue = Arc::new(Queue {
            waiting: Mutex::default(),
            arrived: Condvar::new(),
            committed: Condvar::new(),
        });
        let mut db = database(dir, 0, flush);
        let theirs = queue.clone();
        let writer = std::thread::spawn(move || {
            let queue = theirs;
            let mut last: Vec<Vec<Vec<u8>>> = vec![Vec::new(); accounts];
            loop {
                let (batch, upto) = {
                    let mut w = lock(&queue.waiting);
                    while w.commits.is_empty() && !w.closed {
                        w = queue
                            .arrived
                            .wait(w)
                            .unwrap_or_else(PoisonError::into_inner);
                    }
                    if w.commits.is_empty() {
                        break;
                    }
                    (std::mem::take(&mut w.commits), w.given)
                };
                let tx = db.transaction().expect("a transaction");
                for (account, c) in &batch {
                    put(&tx, *account, c, &last[*account]);
                    last[*account] = data::nodes(c);
                }
                tx.commit().expect("a commit");
                lock(&queue.waiting).done = upto;
                queue.committed.notify_all();
            }
            // Closed unsynced, as the others: its files are removed next.
            let _ = db.execute_batch("PRAGMA synchronous=OFF;");
        });
        Batched {
            queue,
            writer: Some(writer),
        }
    }
}

impl Drop for Batched {
    fn drop(&mut self) {
        lock(&self.queue.waiting).closed = true;
        self.queue.arrived.notify_all();
        if let Some(writer) = self.writer.take() {
            let _ = writer.join();
        }
    }
}

impl Layout for Batched {
    fn write(&self, account: usize, c: &Arc<Commit>) {
        let mut w = lock(&self.queue.waiting);
        w.commits.push((account, c.clone()));
        w.given += 1;
        let mine = w.given;
        self.queue.arrived.notify_one();
        while w.done < mine {
            w = self
                .queue
                .committed
                .wait(w)
                .unwrap_or_else(PoisonError::into_inner);
        }
    }
}

/// A RocksDB for one account: a family's memtable holds 1 MiB before it
/// is written out, where the one for all has RocksDB's 64 MiB.
const ACCOUNT: &str = "write_buffer_size=1048576;";

/// A RocksDB whose write groups may grow to 8 MiB. The commits one sync
/// covers are a write group, and one that a small batch leads takes
/// batches up to an eighth of `max_write_batch_group_size_bytes` more
/// than its own: of RocksDB's 1 MiB, 128 KiB, twenty of these commits.
const GROUP: &str = "max_write_batch_group_size_bytes=67108864;";

/// A RocksDB that writes its log over files written before: a log file
/// no memtable needs any longer is kept, up to four of them, and the next
/// log is written over one from its start, so that a sync has no new
/// block and no new size to make durable, as a log allocated ahead has
/// none. Its log is cut at 4 MiB, each family's memtable written out
/// then, so that the files come round within a row's commits; and
/// `RECYCLE_WARM` commits, five such files' worth, are written before the
/// row is warmed, so that by then every log is written over one.
const RECYCLE: &str = "recycle_log_file_num=4;max_total_wal_size=4194304;";
const RECYCLE_WARM: usize = 3000;

/// The log files of the RocksDB at `dir`: their bytes, by inode, which a
/// file keeps when it is taken for the next log.
fn log_files(dir: &Path) -> HashMap<u64, u64> {
    use std::os::unix::fs::MetadataExt;
    let Ok(entries) = std::fs::read_dir(dir) else {
        return HashMap::new();
    };
    entries
        .flatten()
        .filter(|e| e.path().extension().is_some_and(|x| x == "log"))
        .filter_map(|e| e.metadata().ok())
        .map(|m| (m.ino(), m.len()))
        .collect()
}

/// RocksDB, one database for all or one for each account: a commit is one
/// `WriteBatch`, written under its account's lock, and answered once
/// RocksDB has synced its log (`WriteOptions::set_sync`). Commits to one
/// database that wait at once are written to its log together by one of
/// their threads, and share its sync: RocksDB's own group commit. The
/// databases for each account share a block cache and the threads that
/// write their files. Or, with a group, a batch is written unsynced and
/// the commits that wait share one `FlushWAL(true)`, each sync first
/// letting the commits that have begun be written, as `Wait::Gathered`.
struct Rocks {
    setup: rocks::Setup,
    dbs: Vec<rocksdb::DB>,
    options: rocksdb::WriteOptions,
    group: Option<Group>,
    last: Vec<Mutex<Vec<Vec<u8>>>>,
}

impl Rocks {
    fn new(dir: &Path, n: usize, accounts: usize, sync: Persist, more: &str) -> Rocks {
        let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE);
        let setup = rocks::Setup::new(&cache, rocks::Pack::Snappy, more, true);
        let mut options = rocksdb::WriteOptions::default();
        options.set_sync(sync == Persist::Each);
        Rocks {
            dbs: par(n, OPENERS, |i| setup.open(&dir.join(i.to_string()))),
            setup,
            options,
            group: (sync == Persist::Gathered).then(Group::default),
            last: (0..accounts).map(|_| Mutex::new(Vec::new())).collect(),
        }
    }
}

impl Layout for Rocks {
    fn write(&self, account: usize, c: &Arc<Commit>) {
        let db = &self.dbs[account % self.dbs.len()];
        let commit = || {
            let mut last = lock(&self.last[account]);
            db.write_opt(rocks::commit(db, account, c, &last), &self.options)
                .expect("a commit");
            *last = data::nodes(c);
        };
        let Some(group) = &self.group else {
            return commit();
        };
        group.writing.fetch_add(1, Ordering::SeqCst);
        commit();
        let n = group.written.fetch_add(1, Ordering::SeqCst) + 1;
        group.writing.fetch_sub(1, Ordering::SeqCst);
        group.wait(n, true, || db.flush_wal(true).expect("a sync"));
    }

    fn syncs(&self) -> Option<(u64, u64)> {
        self.group.is_none().then(|| self.setup.syncs())
    }
}

/// How a RocksDB's or a fjall database's commits come to be on disk.
#[derive(Clone, Copy, PartialEq)]
enum Persist {
    /// As the store does it. RocksDB writes the batches that wait at once
    /// to its log together and syncs it once. fjall writes each batch to
    /// its journal and syncs it (`PersistMode::SyncAll`), under the
    /// journal's lock, so each commit makes a sync of its own, one after
    /// another.
    Each,
    /// A batch is written to the journal and left unsynced, and the
    /// commits that wait at once share one `Database::persist`, as a
    /// `Wal`'s share a sync: group commit made over fjall, which has none.
    /// fjall holds its journal's lock through a sync, so a commit that
    /// begins during one is written only when it ends, where a `Wal`
    /// takes it meanwhile.
    Shared,
    /// As `Shared`, each sync first letting the commits that have begun
    /// be written, as `Wait::Gathered`.
    Gathered,
    /// Written to the system's cache (`PersistMode::Buffer`), never synced.
    Never,
}

/// The commits written to a fjall database and those known to be on
/// disk, for the commits that share its syncs.
#[derive(Default)]
struct Group {
    written: AtomicU64,
    /// The commits begun and not yet written.
    writing: AtomicU64,
    /// The commits on disk, and whether a sync is going on.
    synced: Mutex<(u64, bool)>,
    ended: Condvar,
}

impl Group {
    /// Return once commit `n` is on disk: after a `sync` that began when
    /// it was written, this thread's or another's; with `gathered`, a
    /// sync that first lets the commits that have begun be written.
    fn wait(&self, n: u64, gathered: bool, sync: impl Fn()) {
        let mut state = lock(&self.synced);
        while state.0 < n {
            if state.1 {
                state = self
                    .ended
                    .wait(state)
                    .unwrap_or_else(PoisonError::into_inner);
            } else {
                state.1 = true;
                drop(state);
                if gathered {
                    gather(&self.writing);
                }
                let target = self.written.load(Ordering::SeqCst);
                let started = Instant::now();
                sync();
                synced(started.elapsed());
                state = lock(&self.synced);
                *state = (target.max(state.0), false);
                self.ended.notify_all();
            }
        }
    }
}

/// fjall, one database for all or one for each account: a commit is one
/// batch across its keyspaces, written under its account's lock.
struct Fjall {
    stores: Vec<(fj::Store, Group)>,
    persist: Persist,
    last: Vec<Mutex<Vec<Vec<u8>>>>,
}

impl Fjall {
    fn new(dir: &Path, n: usize, accounts: usize, persist: Persist) -> Fjall {
        Fjall {
            stores: par(n, OPENERS, |i| {
                let path = dir.join(i.to_string());
                let store = fj::Store::open(&path, fj::CACHE, fj::Pack::Lz4, n > 1);
                (store, Group::default())
            }),
            persist,
            last: (0..accounts).map(|_| Mutex::new(Vec::new())).collect(),
        }
    }
}

impl Layout for Fjall {
    fn write(&self, account: usize, c: &Arc<Commit>) {
        use fjall::PersistMode::{Buffer, SyncAll};
        let (store, group) = &self.stores[account % self.stores.len()];
        let mode = match self.persist {
            Persist::Each => Some(SyncAll),
            Persist::Shared | Persist::Gathered => None,
            Persist::Never => Some(Buffer),
        };
        group.writing.fetch_add(1, Ordering::SeqCst);
        let n = {
            let mut last = lock(&self.last[account]);
            store.commit(account, c, &last, mode);
            *last = data::nodes(c);
            group.written.fetch_add(1, Ordering::SeqCst) + 1
        };
        group.writing.fetch_sub(1, Ordering::SeqCst);
        if matches!(self.persist, Persist::Shared | Persist::Gathered) {
            let gathered = self.persist == Persist::Gathered;
            group.wait(n, gathered, || store.db.persist(SyncAll).expect("a sync"));
        }
    }
}

/// A commit to write, and the account it is of.
type Given = (usize, Arc<Commit>);

/// The commits a run writes.
enum Commits<'a> {
    /// Made as they are written, each by its number and the account its
    /// writer draws for it: making one is no part of its time, which is
    /// the disk's.
    Made(&'a (dyn Fn(u64, usize) -> Commit + Sync)),
    /// Made before any is written, those that warm a layout and those that
    /// are timed, so that the writers do nothing but write: gone round
    /// again if more are asked for than there are.
    Ready {
        warm: &'a [Given],
        timed: &'a [Given],
    },
}

/// What a run writes, and how many accounts it is spread over.
struct Load<'a> {
    commits: Commits<'a>,
    accounts: usize,
}

impl Load<'_> {
    /// Commit `i` of those numbered from `first`, for the writer whose
    /// generator `x` is.
    fn commit(&self, first: u64, i: usize, x: &mut u64) -> Given {
        match self.commits {
            Commits::Made(make) => {
                *x ^= *x << 13;
                *x ^= *x >> 7;
                *x ^= *x << 17;
                let account = *x as usize % self.accounts;
                (account, Arc::new(make(first + i as u64, account)))
            }
            Commits::Ready { warm, timed } => {
                let list = if first == WARM { warm } else { timed };
                list[i % list.len()].clone()
            }
        }
    }
}

/// `commits` commits to `layout`, numbered from `first`, from `writers`
/// threads that begin together, each to an account drawn the same way
/// every run: how long they took together, and each.
fn drive(
    layout: &dyn Layout,
    load: &Load,
    first: u64,
    commits: usize,
    writers: usize,
) -> (Duration, Vec<Duration>) {
    let next = AtomicUsize::new(0);
    let times = Mutex::new(Vec::with_capacity(commits));
    let begin = Barrier::new(writers + 1);
    let mut started = Instant::now();
    std::thread::scope(|s| {
        for t in 0..writers {
            let (next, times, begin) = (&next, &times, &begin);
            s.spawn(move || {
                let mut mine = Vec::new();
                let mut x = (t as u64 + 1).wrapping_mul(0x2545_F491_4F6C_DD1D);
                begin.wait();
                loop {
                    let i = next.fetch_add(1, Ordering::Relaxed);
                    if i >= commits {
                        break;
                    }
                    let (account, c) = load.commit(first, i, &mut x);
                    let at = Instant::now();
                    layout.write(account, &c);
                    mine.push(at.elapsed());
                }
                times.lock().expect("the times").extend(mine);
            });
        }
        begin.wait();
        started = Instant::now();
    });
    let took = started.elapsed();
    (took, times.into_inner().expect("the times"))
}

/// How a layout is warmed, with a tenth as many commits as are measured:
/// one after another, or from the writers.
#[derive(Clone, Copy)]
enum Warm {
    InTurn,
    Writers,
}

/// A table of rows: what each is run with.
struct Rows<'a> {
    load: Load<'a>,
    warm: Warm,
    commits: usize,
    writers: usize,
    /// Whether a commit is on disk before it is answered.
    durable: bool,
    /// Whether its rows say what zeros were written to give logs room, and
    /// what checkpoints were made.
    zeros: bool,
    /// What is said under the table.
    notes: Mutex<Vec<String>>,
}

/// The times a layout that never syncs writes the commits made ready for
/// it, if it can write one again: a log can, and writes too fast to time
/// in one round; a database, which holds each by its key, cannot.
const ROUNDS: usize = 5;

impl Rows<'_> {
    fn head(&self, what: &str) {
        let sync = if FULLFSYNC { "F_FULLFSYNC" } else { "fsync" };
        if self.durable {
            println!(
                "{} {what} from {} writers over {} accounts, each on disk ({sync}) before it is answered.\n",
                self.commits, self.writers, self.load.accounts
            );
            if self.zeros {
                println!(
                    "| layout | per s | p50 ms | p99 ms | log syncs | ms per sync | zeros written a commit, bytes | checkpoints | ms per checkpoint | longest checkpoint, ms |"
                );
                println!("|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|");
            } else {
                println!("| layout | per s | p50 ms | p99 ms | log syncs | ms per sync |");
                println!("|---|--:|--:|--:|--:|--:|");
            }
        } else {
            println!(
                "{what} from {} writers over {} accounts, **none synced: not durable**. A commit is answered once it is written, to the system's cache, and making it is no part of its time.\n",
                self.writers, self.load.accounts
            );
            println!("| layout | commits | per s | p50 µs | p99 µs |");
            println!("|---|--:|--:|--:|--:|");
        }
    }

    /// A row: `commits` commits to `layout` from the writers, after it is
    /// warmed.
    fn run(&self, name: &str, layout: &dyn Layout, commits: usize) {
        match (self.warm, &self.load.commits) {
            (Warm::InTurn, Commits::Made(make)) => {
                for i in 0..commits / 10 {
                    let account = i % self.load.accounts;
                    layout.write(account, &Arc::new(make(WARM + i as u64, account)));
                }
            }
            _ => {
                drive(layout, &self.load, WARM, commits / 10, self.writers);
            }
        }
        SYNCS.store(0, Ordering::Relaxed);
        SYNC_NANOS.store(0, Ordering::Relaxed);
        let before = layout.syncs();
        layout.begin();
        let (took, mut times) = drive(layout, &self.load, 1, commits, self.writers);
        let (syncs, nanos) = match (before, layout.syncs()) {
            (Some(before), Some(after)) => (after.0 - before.0, after.1 - before.1),
            _ => (
                SYNCS.load(Ordering::Relaxed),
                SYNC_NANOS.load(Ordering::Relaxed),
            ),
        };
        let checkpoints = layout.end();
        times.sort();
        let per_s = commits as f64 / took.as_secs_f64();
        let (p50, p99) = (times[times.len() / 2], at(&times, 99));
        if !self.durable {
            println!(
                "| {name} | {commits} | {per_s:.0} | {:.0} | {:.0} |",
                us(p50),
                us(p99)
            );
            return;
        }
        let (count, each) = if syncs > 0 {
            (
                syncs.to_string(),
                format!("{:.2}", nanos as f64 / syncs as f64 / 1e6),
            )
        } else {
            ("".into(), "".into())
        };
        let zeros = match ZEROS.load(Ordering::Relaxed) {
            0 => String::new(),
            z => (z as usize / (commits + commits / 10)).to_string(),
        };
        let zeros = if self.zeros {
            let checkpoints = match checkpoints {
                Some((made, last)) => {
                    lock(&self.notes).push(format!(
                        "{name}: one more checkpoint, after the commits, took {:.1} ms.",
                        ms(last)
                    ));
                    let mean = match made.count {
                        0 => String::new(),
                        n => format!("{:.1}", made.nanos as f64 / n as f64 / 1e6),
                    };
                    format!(
                        " {} | {mean} | {:.1} |",
                        made.count,
                        made.longest as f64 / 1e6
                    )
                }
                None => " | | |".into(),
            };
            format!(" {zeros} |{checkpoints}")
        } else {
            String::new()
        };
        println!(
            "| {name} | {per_s:.0} | {:.1} | {:.1} | {count} | {each} |{zeros}",
            ms(p50),
            ms(p99),
        );
    }

    /// A layout made in a scratch directory, run, and its files removed
    /// once `after` has seen them.
    fn row<L: Layout>(
        &self,
        name: &str,
        commits: usize,
        make: impl FnOnce(&Path) -> L,
        after: impl FnOnce(&Path),
    ) {
        let dir = scratch("write");
        ZEROS.store(0, Ordering::Relaxed);
        let layout = make(&dir);
        self.run(name, &layout, commits);
        drop(layout);
        after(&dir);
        let _ = std::fs::remove_dir_all(&dir);
    }

    fn flush(&self, flush: Flush) -> Flush {
        if self.durable { flush } else { Flush::Never }
    }

    fn sqlite(&self, name: &str, n: usize, flush: Flush) {
        let (accounts, flush) = (self.load.accounts, self.flush(flush));
        self.row(
            name,
            self.commits,
            |d| Sqlite::new(d, n, accounts, flush),
            |_| {},
        );
    }

    /// A row of RocksDB, in `n` databases, with `more` options; of more
    /// than one database, a note of what that many cost to make and to
    /// hold open.
    fn rocks(&self, name: &str, n: usize, sync: Persist, more: &str) {
        let dir = scratch("write");
        let mut rss = Rss::start();
        let (before, started) = (rss.kib(), Instant::now());
        let sync = if self.durable { sync } else { Persist::Never };
        let layout = Rocks::new(&dir, n, self.load.accounts, sync, more);
        let (made, open) = (started.elapsed(), rss.kib());
        let recycled = self.durable && more.contains(RECYCLE);
        if recycled {
            drive(&layout, &self.load, 2 * WARM, RECYCLE_WARM, self.writers);
        }
        let (logs, wrote) = (log_files(&dir.join("0")), layout.setup.written()[1]);
        self.run(name, &layout, self.commits);
        if recycled {
            // What the commits wrote past the bytes its log files held
            // before them, in files there then or made since.
            let (mut new, mut past) = (0, 0);
            for (inode, len) in log_files(&dir.join("0")) {
                match logs.get(&inode) {
                    Some(before) => past += len.saturating_sub(*before),
                    None => (new, past) = (new + 1, past + len),
                }
            }
            lock(&self.notes).push(format!(
                "{name}: of the {:.1} MiB the commits wrote to its log, {:.2} MiB went past what its log files held before them, {new} of those files new; the rest went over bytes written before.",
                mib(layout.setup.written()[1] - wrote),
                mib(past),
            ));
        }
        if n > 1 {
            let files = std::fs::read_dir(dir.join("0")).map_or(0, Iterator::count);
            lock(&self.notes).push(format!(
                "{name}: {n} databases, made by {OPENERS} threads in {:.1} s; the process holds {:.0} MiB more with them open and {:.0} MiB more after the commits; each is {files} files, and all are {:.0} MiB on disk.",
                made.as_secs_f64(),
                (open.saturating_sub(before)) as f64 / 1024.0,
                (rss.kib().saturating_sub(before)) as f64 / 1024.0,
                mib(dir_bytes(&dir)),
            ));
        }
        drop(layout);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A row of fjall, in `n` databases; of more than one, a note of what
    /// that many cost to make and to hold open.
    fn fjall(&self, name: &str, n: usize, persist: Persist) {
        let persist = if self.durable {
            persist
        } else {
            Persist::Never
        };
        let dir = scratch("write");
        let mut rss = Rss::start();
        let (before, started) = (rss.kib(), Instant::now());
        let layout = Fjall::new(&dir, n, self.load.accounts, persist);
        let (made, open) = (started.elapsed(), rss.kib());
        self.run(name, &layout, self.commits);
        if n > 1 {
            lock(&self.notes).push(format!(
                "{name}: {n} databases, made by {OPENERS} threads in {:.1} s; the process holds {:.0} MiB more with them open and {:.0} MiB more after the commits.",
                made.as_secs_f64(),
                (open.saturating_sub(before)) as f64 / 1024.0,
                (rss.kib().saturating_sub(before)) as f64 / 1024.0,
            ));
        }
        drop(layout);
        let _ = std::fs::remove_dir_all(&dir);
    }

    fn batched(&self, name: &str) {
        let (accounts, flush) = (self.load.accounts, self.flush(Flush::Disk));
        self.row(
            name,
            self.commits,
            |d| Batched::new(d, accounts, flush),
            |_| {},
        );
    }

    fn logs(&self, name: &str, n: usize, encoding: Encoding, grow: Grow) {
        let accounts = self.load.accounts;
        let commits = self.commits * if self.durable { 1 } else { ROUNDS };
        self.row(
            name,
            commits,
            |d| Logs::new(d, n, accounts, (encoding, grow), self.durable),
            |d| {
                // This benchmark's own logs replay as the server's do, and
                // one allocated ahead ends where its frames do, not its
                // file: every commit is there.
                if !matches!(grow, Grow::Server | Grow::ServerJournal) {
                    let found: u64 = par(n, cores(), |i| {
                        frame::count(&d.join(format!("{i}/repo.log"))).expect("the log read")
                    })
                    .iter()
                    .sum();
                    let wrote = (commits + commits / 10) as u64;
                    assert_eq!(found, wrote, "every commit replayed from {name}");
                }
            },
        );
    }

    /// A row of a log for each account beside one journal for all: after
    /// it, each log is seen to hold what the journal has of it.
    fn journaled(&self, name: &str, encoding: Encoding) {
        let accounts = self.load.accounts;
        self.row(
            name,
            self.commits,
            |d| Journaled::new(d, accounts, encoding),
            |d| {
                let wrote = (self.commits + self.commits / 10) as u64;
                journal_kept(d, accounts, wrote, name);
            },
        );
    }

    /// The rows of realistic commits: every layout; the server's logs,
    /// which make their own room, with its frames waiting on its journal
    /// and each synced by itself, with JSON frames, bytes as hex, and with
    /// binary ones; and this benchmark's own of binary frames, in a file
    /// that grows and in one allocated ahead.
    fn layouts(&self) {
        use Encoding::{Binary, Holes, Json};
        let n = self.load.accounts;
        // What makes the server's logs durable, as `PDS_SYNC` has it; of a
        // row that never syncs, what is written all the same.
        let (together, each) = if self.durable {
            (
                "its pool's journal synced (PDS_SYNC together)",
                ", each log synced by itself",
            )
        } else {
            ("written to its pool's journal too (PDS_SYNC together)", "")
        };
        self.logs(
            &format!("a log for each account, JSON frames with holes (the server's), {together}"),
            n,
            Holes,
            Grow::ServerJournal,
        );
        self.logs(
            &format!(
                "a log for each account, the same, no pool and no journal{each} (PDS_SYNC each)"
            ),
            n,
            Holes,
            Grow::Server,
        );
        self.logs(
            &format!("a log for each account, JSON frames, bytes as hex{each}"),
            n,
            Json,
            Grow::Server,
        );
        self.logs(
            &format!("a log for each account, binary frames{each}"),
            n,
            Binary,
            Grow::Server,
        );
        self.logs(
            "a log for each account, binary frames, each written at its offset",
            n,
            Binary,
            Grow::Placed,
        );
        #[cfg(target_os = "linux")]
        self.logs(
            "a log for each account, the same, allocated ahead by 64 KiB with fallocate",
            n,
            Binary,
            Grow::Ahead(ACCOUNT_CHUNKS[0], Room::Fallocate),
        );
        for chunk in ACCOUNT_CHUNKS {
            self.logs(
                &format!(
                    "a log for each account, the same, allocated ahead by {} KiB of zeros",
                    chunk >> 10
                ),
                n,
                Binary,
                Grow::Ahead(chunk, Room::Zeros),
            );
        }
        self.logs(
            "one log for all, JSON frames with holes",
            1,
            Holes,
            Grow::Server,
        );
        self.logs("one log for all, binary frames", 1, Binary, Grow::Server);
        self.logs(
            "one log for all, binary frames, each written at its offset",
            1,
            Binary,
            Grow::Placed,
        );
        #[cfg(target_os = "linux")]
        self.logs(
            "one log for all, the same, allocated ahead by 16 MiB with fallocate",
            1,
            Binary,
            Grow::Ahead(CHUNK, Room::Fallocate),
        );
        self.logs(
            "one log for all, the same, allocated ahead by 16 MiB of zeros",
            1,
            Binary,
            Grow::Ahead(CHUNK, Room::Zeros),
        );
        if self.durable {
            const JOURNALED: &str = "a log for each account written and not synced, each commit made durable by one journal for all";
            for (encoding, frames) in [
                (Holes, "JSON frames with holes (the server's)"),
                (Binary, "binary frames"),
            ] {
                self.journaled(&format!("{JOURNALED}, {frames}"), encoding);
            }
            self.logs(
                "one log for all, binary frames, each written at its offset, its waiting commits answered in turn (as a `Wal`'s were)",
                1,
                Binary,
                Grow::InTurn,
            );
            self.logs(
                "one log for all, binary frames, each written at its offset, a sync first letting the commits begun be written",
                1,
                Binary,
                Grow::Gathered,
            );
        }
        self.sqlite("SQLite for each account", n, Flush::Disk);
        self.sqlite("one SQLite for all", 1, Flush::Disk);
        self.batched("one SQLite for all, one writer, every commit waiting in one transaction");
        let synced = if self.durable { ", its log synced" } else { "" };
        self.rocks(
            &format!("one RocksDB for all, a commit a batch{synced}"),
            1,
            Persist::Each,
            "",
        );
        self.rocks(
            "one RocksDB for all, the same, its write groups let grow to 8 MiB",
            1,
            Persist::Each,
            GROUP,
        );
        if self.durable {
            self.rocks(
                "one RocksDB for all, a commit a batch left unsynced, those that wait sharing one sync of its log, a sync first letting the commits begun be written",
                1,
                Persist::Gathered,
                "",
            );
        }
        if self.durable {
            self.rocks(
                "one RocksDB for all, a commit a batch, its log synced, its log files written over",
                1,
                Persist::Each,
                RECYCLE,
            );
            self.rocks(
                "one RocksDB for all, the same, its write groups let grow to 8 MiB, its log files written over",
                1,
                Persist::Each,
                &[GROUP, RECYCLE].concat(),
            );
        }
        // Two thousand databases take over a minute to make, so one table
        // of them has this row; and each holds nine files open.
        let name = format!("RocksDB for each account, a commit a batch{synced}");
        let here = n <= ACCOUNTS || (self.durable && self.writers <= 64);
        if here && n as u64 * 10 > files() {
            lock(&self.notes).push(format!(
                "{name}: left out, since {n} databases hold more files open than the {} this process may.",
                files()
            ));
        } else if here {
            self.rocks(&name, n, Persist::Each, ACCOUNT);
        }
        let synced = if self.durable {
            ", its journal synced"
        } else {
            ""
        };
        self.fjall(
            &format!("one fjall for all, a commit a batch{synced}"),
            1,
            Persist::Each,
        );
        if self.durable {
            self.fjall(
                "one fjall for all, a commit a batch left unsynced, those that wait sharing one sync of its journal",
                1,
                Persist::Shared,
            );
            self.fjall(
                "one fjall for all, the same, a sync first letting the commits begun be written",
                1,
                Persist::Gathered,
            );
        }
        // Two thousand of these take over three minutes to make.
        if n <= ACCOUNTS {
            self.fjall(
                &format!("fjall for each account, a commit a batch{synced}"),
                n,
                Persist::Each,
            );
        }
        for note in lock(&self.notes).drain(..) {
            println!("\n{note}");
        }
    }
}

/// The write part as it was first: commits of random bytes over 40
/// accounts, the server's frames, each layout warmed in turn.
pub fn write(commits: usize, writers: usize) {
    let t = Rows {
        load: Load {
            commits: Commits::Made(&|op, _| data::random(op)),
            accounts: ACCOUNTS,
        },
        warm: Warm::InTurn,
        commits,
        writers,
        durable: true,
        zeros: false,
        notes: Mutex::default(),
    };
    t.head("commits");
    t.logs(
        "a log for each account (the server's)",
        ACCOUNTS,
        Encoding::Holes,
        Grow::Server,
    );
    t.logs("one log for all", 1, Encoding::Holes, Grow::Server);
    t.sqlite("SQLite for each account", ACCOUNTS, Flush::Disk);
    t.sqlite("one SQLite for all", 1, Flush::Disk);
    t.rocks(
        "one RocksDB for all, a commit a batch, its log synced",
        1,
        Persist::Each,
        "",
    );
    t.fjall(
        "one fjall for all, a commit a batch, its journal synced",
        1,
        Persist::Each,
    );
    if FULLFSYNC {
        // What SQLite does there by default: synced, the drive's cache not flushed.
        t.sqlite(
            "SQLite for each account, fsync without F_FULLFSYNC",
            ACCOUNTS,
            Flush::Fsync,
        );
        t.sqlite(
            "one SQLite for all, fsync without F_FULLFSYNC",
            1,
            Flush::Fsync,
        );
    }
}

/// A chunk of a log for all allocated ahead, and the chunks of an
/// account's that are tried: small, as a server of many accounts, most
/// with little to write, could afford of zeros.
const CHUNK: u64 = 16 << 20;
const ACCOUNT_CHUNKS: [u64; 2] = [64 << 10, 16 << 10];

/// Realistic commits over `accounts` accounts from `writers` writers, each
/// on disk before it is answered: every layout.
pub fn spread(commits: usize, writers: usize, accounts: usize) {
    let t = Rows {
        load: Load {
            commits: Commits::Made(&|op, account| data::real(SEED, op, account as u64)),
            accounts,
        },
        warm: Warm::Writers,
        commits,
        writers,
        durable: true,
        zeros: true,
        notes: Mutex::default(),
    };
    t.head("realistic commits");
    t.layouts();
}

/// `n` realistic commits numbered from `first`, each with the account
/// drawn for it.
fn ready(first: u64, n: usize, accounts: usize) -> Vec<Given> {
    par(n, cores(), |i| {
        let op = first + i as u64;
        let account = Rng::of(&[SEED, 12, op]).below(accounts as u64);
        (account as usize, Arc::new(data::real(SEED, op, account)))
    })
}

/// The same with no sync at all, the commits made before any is written:
/// what the processor allows each layout, which a drive whose syncs cost
/// next to nothing would approach. Nothing here is durable.
pub fn unsynced(commits: usize, writers: usize, accounts: usize) {
    let (warm, timed) = (
        ready(WARM, commits / 10, accounts),
        ready(1, commits, accounts),
    );
    let t = Rows {
        load: Load {
            commits: Commits::Ready {
                warm: &warm,
                timed: &timed,
            },
            accounts,
        },
        warm: Warm::Writers,
        commits,
        writers,
        durable: false,
        zeros: false,
        notes: Mutex::default(),
    };
    t.head("Realistic commits");
    t.layouts();
}