chadtmiller.com/monolith
An atproto PDS
0
0
//! The firehose replayed, four ways, of the same realistic commits: from
//! one log for all, whose order is the firehose's; from a SQLite table of
//! events by seq; from RocksDB's family of events by seq; and from a log
//! for each account, as the server keeps them, where the order is made
//! again from every log.
use crate::check::Check;
use crate::data::{self, SEED};
use crate::fj;
use crate::frame::{self, Encoding, Frames};
use crate::rocks::{self, Pack, Setup};
use crate::stores::{fjall_path, rocks_path, settle};
use crate::util::{Rng, at, dir_bytes, mib, ms, scratch};
use rocksdb::{DB, ReadOptions, WriteBatch};
use rusqlite::{Connection, params};
use std::fs::File;
use std::hint::black_box;
use std::io::{BufWriter, Seek, SeekFrom, Write};
use std::path::Path;
use std::time::{Duration, Instant};
/// The cursors a first event is timed from, spread over the middle half of
/// the events.
const CURSORS: u64 = 200;
/// What a replay gives each event to: here, a count of them and of their bytes.
#[derive(Default)]
struct Sink {
events: u64,
bytes: u64,
last: u64,
}
impl Sink {
fn give(&mut self, seq: u64, frame: &[u8]) {
assert!(seq > self.last, "events in order");
self.last = seq;
self.events += 1;
self.bytes += black_box(frame).len() as u64;
}
}
/// Write the four: `all.log` with `all.index`, an entry of seq and offset
/// for each `every` frames, written as the log is, and `all.index.each`,
/// an entry for each frame; `logs/<account>.log`; `sequencer.sqlite`; and
/// a RocksDB for each of `PACKS`, left to settle. Unsynced: they are only
/// read.
fn build(dir: &Path, commits: u64, accounts: u64, every: u64) {
std::fs::create_dir_all(dir.join("logs")).expect("a directory");
let create = |name: String| File::create(dir.join(name)).expect("a file");
let mut log = BufWriter::with_capacity(1 << 20, create("all.log".into()));
let mut index = BufWriter::new(create("all.index".into()));
let mut each = BufWriter::new(create("all.index.each".into()));
let mut logs: Vec<File> = (0..accounts)
.map(|a| create(format!("logs/{a}.log")))
.collect();
let db = Connection::open(dir.join("sequencer.sqlite")).expect("a database");
db.execute_batch(
"PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF;
CREATE TABLE event(seq INTEGER PRIMARY KEY, account INTEGER, time INTEGER, frame BLOB);
BEGIN;",
)
.expect("a schema");
let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE);
let mut rocks: Vec<(DB, WriteBatch)> = PACKS
.iter()
.map(|p| {
let db = Setup::new(&cache, *p, "", false).open(&rocks_path(dir, *p));
(db, WriteBatch::default())
})
.collect();
let fjalls: Vec<fj::Store> = fj::PACKS
.iter()
.map(|p| fj::Store::open(&fjall_path(dir, *p), fj::CACHE, *p, false))
.collect();
let mut batches: Vec<_> = fjalls.iter().map(|s| s.db.batch()).collect();
let mut last: Vec<Vec<Vec<u8>>> = vec![Vec::new(); accounts as usize];
let mut pick = Rng::of(&[SEED, 9]);
let mut offset = 0u64;
for seq in 1..=commits {
let a = pick.below(accounts);
let c = data::real(SEED, seq, a);
let gone = &last[a as usize];
let frame = Encoding::Binary.frame(&c, gone);
let entry = [seq.to_le_bytes(), offset.to_le_bytes()].concat();
if (seq - 1) % every == 0 {
index.write_all(&entry).expect("the index");
}
each.write_all(&entry).expect("the index");
log.write_all(&frame).expect("an append");
offset += frame.len() as u64;
logs[a as usize]
.write_all(&Encoding::Holes.frame(&c, gone))
.expect("an append");
db.prepare_cached("INSERT INTO event(seq, account, time, frame) VALUES (?1, ?2, ?3, ?4)")
.expect("a statement")
.execute(params![
seq as i64,
a as i64,
1_700_000_000_000i64 + seq as i64,
c.event
])
.expect("an event");
for (db, batch) in &mut rocks {
let event = rocks::event(a as usize, seq, &c.event);
batch.put_cf(rocks::family(db, rocks::EVENT), seq.to_be_bytes(), event);
}
for (store, batch) in fjalls.iter().zip(&mut batches) {
let event = rocks::event(a as usize, seq, &c.event);
batch.insert(&store.event, seq.to_be_bytes(), event);
}
if seq % 1000 == 0 || seq == commits {
db.execute_batch("COMMIT; BEGIN;").expect("a commit");
for (db, batch) in &mut rocks {
db.write_without_wal(std::mem::take(batch))
.expect("a batch");
}
for (store, batch) in fjalls.iter().zip(&mut batches) {
let batch = std::mem::replace(batch, store.db.batch());
batch.commit().expect("a batch");
}
}
last[a as usize] = data::nodes(&c);
}
db.execute_batch("COMMIT;").expect("a commit");
crate::stores::close_db(db);
rocks.iter().for_each(|(db, _)| settle(db));
fjalls.iter().for_each(fj::Store::settle);
log.flush().expect("the log");
index.flush().expect("the index");
each.flush().expect("the index");
}
/// The one log's sparse index, read whole: seqs and the offsets of their frames.
fn read_index(path: &Path) -> Vec<(u64, u64)> {
std::fs::read(path)
.expect("the index")
.chunks_exact(16)
.map(|e| {
let n = |i: usize| u64::from_le_bytes(e[i..i + 8].try_into().expect("8 bytes"));
(n(0), n(8))
})
.collect()
}
/// The events after `cursor` from the one log, at most `limit`: from the
/// frame the index names at or before the first of them, each frame's
/// check computed, if it has one to compute, and its fields walked.
fn from_log(
mut file: &File,
size: u64,
index: &[(u64, u64)],
cursor: u64,
limit: u64,
check: Option<Check>,
sink: &mut Sink,
) {
let i = index.partition_point(|&(seq, _)| seq <= cursor + 1);
let offset = index[i.saturating_sub(1)].1;
file.seek(SeekFrom::Start(offset)).expect("a seek");
let mut frames = Frames::new(file, size - offset, check);
while sink.events < limit {
let Some(p) = frames.next_payload() else {
break;
};
let e = frame::walk(p).expect("a commit's frame");
if e.seq > cursor {
sink.give(e.seq, e.frame);
}
}
}
/// The one log again, at `path`, each frame with `check` in the place of
/// SHA-256's.
fn rechecked(mut log: &File, size: u64, path: &Path, check: Check) -> File {
let mut out = BufWriter::with_capacity(1 << 20, File::create(path).expect("a log"));
log.seek(SeekFrom::Start(0)).expect("a seek");
let mut frames = Frames::new(log, size, None);
while let Some(p) = frames.next_payload() {
out.write_all(&(p.len() as u32).to_le_bytes())
.expect("an append");
out.write_all(&check.of_frame(p)).expect("an append");
out.write_all(p).expect("an append");
}
drop(out.into_inner().expect("the log written"));
File::open(path).expect("the log")
}
/// The same from the table of events.
fn from_sqlite(db: &Connection, cursor: u64, limit: u64, sink: &mut Sink) {
let mut st = db
.prepare_cached("SELECT seq, frame FROM event WHERE seq > ?1 ORDER BY seq")
.expect("a statement");
let mut rows = st.query(params![cursor as i64]).expect("a query");
while sink.events < limit {
let Some(row) = rows.next().expect("a row") else {
break;
};
let seq: i64 = row.get(0).expect("a seq");
let frame = row.get_ref(1).expect("a frame");
sink.give(seq as u64, frame.as_blob().expect("a blob"));
}
}
/// The ways RocksDB's blocks are compressed that the events are kept in.
const PACKS: [Pack; 2] = [Pack::Snappy, Pack::None];
/// The same from RocksDB's family of events, with an iterator: one that
/// is to give more than the first event reads its files 2 MiB at a time
/// and leaves the blocks it reads out of the block cache, as a long scan
/// is told to.
fn from_rocks(db: &DB, cursor: u64, limit: u64, sink: &mut Sink) {
let mut options = ReadOptions::default();
options.fill_cache(limit == 1);
if limit > 1 {
options.set_readahead_size(2 << 20);
}
let mut events = db.raw_iterator_cf_opt(rocks::family(db, rocks::EVENT), options);
events.seek((cursor + 1).to_be_bytes());
while sink.events < limit {
let (Some(seq), Some(value)) = (events.key(), events.value()) else {
break;
};
let seq = u64::from_be_bytes(seq.try_into().expect("a seq"));
sink.give(seq, rocks::frame(value));
events.next();
}
events.status().expect("the events read");
}
/// The same from fjall's keyspace of events, with an iterator over the
/// seqs from the one after the cursor.
fn from_fjall(store: &fj::Store, cursor: u64, limit: u64, sink: &mut Sink) {
for event in store.event.range((cursor + 1).to_be_bytes()..) {
if sink.events >= limit {
break;
}
let (seq, value) = event.into_inner().expect("the events read");
let seq = u64::from_be_bytes(seq[..].try_into().expect("a seq"));
sink.give(seq, rocks::frame(&value));
}
}
/// Every account's log read as the server reads one (each frame's check
/// computed, its holes filled, its JSON parsed), its events' frames taken
/// out of their hex, and all of them put in order of seq: the events, and
/// how long until the first could be given.
fn from_logs(dir: &Path, accounts: u64, sink: &mut Sink) -> Duration {
let started = Instant::now();
let mut events: Vec<(u64, Vec<u8>)> = Vec::new();
for a in 0..accounts {
pds_store::scan_file(&dir.join(format!("logs/{a}.log")), |_, _, entries| {
for e in &entries {
if e["t"] == "event" {
let frame = e["frame"].as_str().ok_or("an event with no frame")?;
events.push((
e["seq"].as_u64().ok_or("an event with no seq")?,
hex::decode(frame).map_err(|e| e.to_string())?,
));
}
}
Ok(())
})
.expect("the log read")
.expect("every frame");
}
events.sort_unstable_by_key(|(seq, _)| *seq);
let first = started.elapsed();
for (seq, frame) in &events {
sink.give(*seq, frame);
}
first
}
/// The less of two times of a whole replay: every event given.
fn whole(commits: u64, mut f: impl FnMut(&mut Sink)) -> (Duration, u64) {
let mut best = (Duration::MAX, 0);
for _ in 0..2 {
let mut sink = Sink::default();
let at = Instant::now();
f(&mut sink);
let took = at.elapsed();
assert_eq!(sink.events, commits, "every event replayed");
best = best.min((took, sink.bytes));
}
best
}
/// How long until the first event after each cursor, in order.
fn firsts(commits: u64, mut f: impl FnMut(u64, &mut Sink)) -> Vec<Duration> {
let mut times: Vec<Duration> = (0..CURSORS)
.map(|i| {
let cursor = commits / 4 + i * (commits / 2) / CURSORS;
let mut sink = Sink::default();
let at = Instant::now();
f(cursor, &mut sink);
let took = at.elapsed();
assert_eq!(sink.last, cursor + 1, "the event after the cursor");
took
})
.collect();
times.sort();
times
}
pub fn firehose(commits: u64, accounts: u64, every: u64) {
let dir = scratch("firehose");
build(&dir, commits, accounts, every);
println!(
"The events of {commits} realistic commits over {accounts} accounts. A whole replay is the faster of two, so of files the system's cache holds, if it has the memory to; a first event is timed after those, from each of {CURSORS} cursors in the middle half, the files open.\n"
);
println!(
"| from | MiB on disk | whole replay ms | events per s | MiB of events per s | first event from a cursor, p50 ms | p99 ms | with an index entry each frame, p50 ms | p99 ms |"
);
println!("|---|--:|--:|--:|--:|--:|--:|--:|--:|");
let row = |name: &str,
bytes: u64,
(took, events): (Duration, u64),
first: &[Duration],
dense: Option<&[Duration]>| {
let dense = dense.map_or("|".to_string(), |d| {
format!("{:.3} | {:.3}", ms(d[d.len() / 2]), ms(at(d, 99)))
});
println!(
"| {name} | {:.0} | {:.0} | {:.0} | {:.0} | {:.3} | {:.3} | {dense} |",
mib(bytes),
ms(took),
commits as f64 / took.as_secs_f64(),
mib(events) / took.as_secs_f64(),
ms(first[first.len() / 2]),
ms(at(first, 99)),
);
};
let log = File::open(dir.join("all.log")).expect("the log");
let size = log.metadata().expect("the log").len();
let index = read_index(&dir.join("all.index"));
let dense = read_index(&dir.join("all.index.each"));
let index_bytes = index.len() as u64 * 16;
let log_row = |name: &str, log: &File, check: Option<Check>| {
row(
name,
size + index_bytes,
whole(commits, |sink| {
from_log(log, size, &index, 0, u64::MAX, check, sink)
}),
&firsts(commits, |cursor, sink| {
from_log(log, size, &index, cursor, 1, check, sink)
}),
Some(&firsts(commits, |cursor, sink| {
from_log(log, size, &dense, cursor, 1, check, sink)
})),
);
};
log_row(
&format!(
"one log for all, binary frames, an index entry each {every} frames, each frame's check SHA-256"
),
&log,
Some(Check::Sha256),
);
// The same frames with each other check, in a log of their own, one at a time.
for check in &Check::ALL[1..] {
let path = dir.join("all.other.log");
let other = rechecked(&log, size, &path, *check);
log_row(
&format!("the same, each frame's check {}", check.name()),
&other,
Some(*check),
);
drop(other);
std::fs::remove_file(&path).expect("the log removed");
}
log_row("the same, no frame's check computed", &log, None);
let db = Connection::open(dir.join("sequencer.sqlite")).expect("the database");
row(
"SQLite, a table of events by seq",
dir_bytes(&dir.join("sequencer.sqlite")),
whole(commits, |sink| from_sqlite(&db, 0, u64::MAX, sink)),
&firsts(commits, |cursor, sink| from_sqlite(&db, cursor, 1, sink)),
None,
);
drop(db);
let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE);
for pack in PACKS {
let path = rocks_path(&dir, pack);
let db = Setup::new(&cache, pack, "", false).open(&path);
row(
&format!(
"RocksDB, a family of events by seq, an iterator, its blocks: {}",
pack.name()
),
dir_bytes(&path),
whole(commits, |sink| from_rocks(&db, 0, u64::MAX, sink)),
&firsts(commits, |cursor, sink| from_rocks(&db, cursor, 1, sink)),
None,
);
}
for pack in fj::PACKS {
let path = fjall_path(&dir, pack);
let store = fj::Store::open(&path, fj::CACHE, pack, false);
row(
&format!(
"fjall, a keyspace of events by seq, an iterator, its blocks: {}",
pack.name()
),
fj::disk(&path),
whole(commits, |sink| from_fjall(&store, 0, u64::MAX, sink)),
&firsts(commits, |cursor, sink| from_fjall(&store, cursor, 1, sink)),
None,
);
}
let mut first = Duration::MAX;
let took = whole(commits, |sink| {
first = first.min(from_logs(&dir, accounts, sink))
});
row(
"a log for each account, JSON frames with holes (the server's), put in order",
dir_bytes(&dir.join("logs")),
took,
&[first],
None,
);
let _ = std::fs::remove_dir_all(&dir);
}