//! The firehose replayed, four ways, of the same realistic commits: from //! one log for all, whose order is the firehose's; from a SQLite table of //! events by seq; from RocksDB's family of events by seq; and from a log //! for each account, as the server keeps them, where the order is made //! again from every log. use crate::check::Check; use crate::data::{self, SEED}; use crate::fj; use crate::frame::{self, Encoding, Frames}; use crate::rocks::{self, Pack, Setup}; use crate::stores::{fjall_path, rocks_path, settle}; use crate::util::{Rng, at, dir_bytes, mib, ms, scratch}; use rocksdb::{DB, ReadOptions, WriteBatch}; use rusqlite::{Connection, params}; use std::fs::File; use std::hint::black_box; use std::io::{BufWriter, Seek, SeekFrom, Write}; use std::path::Path; use std::time::{Duration, Instant}; /// The cursors a first event is timed from, spread over the middle half of /// the events. const CURSORS: u64 = 200; /// What a replay gives each event to: here, a count of them and of their bytes. #[derive(Default)] struct Sink { events: u64, bytes: u64, last: u64, } impl Sink { fn give(&mut self, seq: u64, frame: &[u8]) { assert!(seq > self.last, "events in order"); self.last = seq; self.events += 1; self.bytes += black_box(frame).len() as u64; } } /// Write the four: `all.log` with `all.index`, an entry of seq and offset /// for each `every` frames, written as the log is, and `all.index.each`, /// an entry for each frame; `logs/.log`; `sequencer.sqlite`; and /// a RocksDB for each of `PACKS`, left to settle. Unsynced: they are only /// read. fn build(dir: &Path, commits: u64, accounts: u64, every: u64) { std::fs::create_dir_all(dir.join("logs")).expect("a directory"); let create = |name: String| File::create(dir.join(name)).expect("a file"); let mut log = BufWriter::with_capacity(1 << 20, create("all.log".into())); let mut index = BufWriter::new(create("all.index".into())); let mut each = BufWriter::new(create("all.index.each".into())); let mut logs: Vec = (0..accounts) .map(|a| create(format!("logs/{a}.log"))) .collect(); let db = Connection::open(dir.join("sequencer.sqlite")).expect("a database"); db.execute_batch( "PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF; CREATE TABLE event(seq INTEGER PRIMARY KEY, account INTEGER, time INTEGER, frame BLOB); BEGIN;", ) .expect("a schema"); let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE); let mut rocks: Vec<(DB, WriteBatch)> = PACKS .iter() .map(|p| { let db = Setup::new(&cache, *p, "", false).open(&rocks_path(dir, *p)); (db, WriteBatch::default()) }) .collect(); let fjalls: Vec = fj::PACKS .iter() .map(|p| fj::Store::open(&fjall_path(dir, *p), fj::CACHE, *p, false)) .collect(); let mut batches: Vec<_> = fjalls.iter().map(|s| s.db.batch()).collect(); let mut last: Vec>> = vec![Vec::new(); accounts as usize]; let mut pick = Rng::of(&[SEED, 9]); let mut offset = 0u64; for seq in 1..=commits { let a = pick.below(accounts); let c = data::real(SEED, seq, a); let gone = &last[a as usize]; let frame = Encoding::Binary.frame(&c, gone); let entry = [seq.to_le_bytes(), offset.to_le_bytes()].concat(); if (seq - 1) % every == 0 { index.write_all(&entry).expect("the index"); } each.write_all(&entry).expect("the index"); log.write_all(&frame).expect("an append"); offset += frame.len() as u64; logs[a as usize] .write_all(&Encoding::Holes.frame(&c, gone)) .expect("an append"); db.prepare_cached("INSERT INTO event(seq, account, time, frame) VALUES (?1, ?2, ?3, ?4)") .expect("a statement") .execute(params![ seq as i64, a as i64, 1_700_000_000_000i64 + seq as i64, c.event ]) .expect("an event"); for (db, batch) in &mut rocks { let event = rocks::event(a as usize, seq, &c.event); batch.put_cf(rocks::family(db, rocks::EVENT), seq.to_be_bytes(), event); } for (store, batch) in fjalls.iter().zip(&mut batches) { let event = rocks::event(a as usize, seq, &c.event); batch.insert(&store.event, seq.to_be_bytes(), event); } if seq % 1000 == 0 || seq == commits { db.execute_batch("COMMIT; BEGIN;").expect("a commit"); for (db, batch) in &mut rocks { db.write_without_wal(std::mem::take(batch)) .expect("a batch"); } for (store, batch) in fjalls.iter().zip(&mut batches) { let batch = std::mem::replace(batch, store.db.batch()); batch.commit().expect("a batch"); } } last[a as usize] = data::nodes(&c); } db.execute_batch("COMMIT;").expect("a commit"); crate::stores::close_db(db); rocks.iter().for_each(|(db, _)| settle(db)); fjalls.iter().for_each(fj::Store::settle); log.flush().expect("the log"); index.flush().expect("the index"); each.flush().expect("the index"); } /// The one log's sparse index, read whole: seqs and the offsets of their frames. fn read_index(path: &Path) -> Vec<(u64, u64)> { std::fs::read(path) .expect("the index") .chunks_exact(16) .map(|e| { let n = |i: usize| u64::from_le_bytes(e[i..i + 8].try_into().expect("8 bytes")); (n(0), n(8)) }) .collect() } /// The events after `cursor` from the one log, at most `limit`: from the /// frame the index names at or before the first of them, each frame's /// check computed, if it has one to compute, and its fields walked. fn from_log( mut file: &File, size: u64, index: &[(u64, u64)], cursor: u64, limit: u64, check: Option, sink: &mut Sink, ) { let i = index.partition_point(|&(seq, _)| seq <= cursor + 1); let offset = index[i.saturating_sub(1)].1; file.seek(SeekFrom::Start(offset)).expect("a seek"); let mut frames = Frames::new(file, size - offset, check); while sink.events < limit { let Some(p) = frames.next_payload() else { break; }; let e = frame::walk(p).expect("a commit's frame"); if e.seq > cursor { sink.give(e.seq, e.frame); } } } /// The one log again, at `path`, each frame with `check` in the place of /// SHA-256's. fn rechecked(mut log: &File, size: u64, path: &Path, check: Check) -> File { let mut out = BufWriter::with_capacity(1 << 20, File::create(path).expect("a log")); log.seek(SeekFrom::Start(0)).expect("a seek"); let mut frames = Frames::new(log, size, None); while let Some(p) = frames.next_payload() { out.write_all(&(p.len() as u32).to_le_bytes()) .expect("an append"); out.write_all(&check.of_frame(p)).expect("an append"); out.write_all(p).expect("an append"); } drop(out.into_inner().expect("the log written")); File::open(path).expect("the log") } /// The same from the table of events. fn from_sqlite(db: &Connection, cursor: u64, limit: u64, sink: &mut Sink) { let mut st = db .prepare_cached("SELECT seq, frame FROM event WHERE seq > ?1 ORDER BY seq") .expect("a statement"); let mut rows = st.query(params![cursor as i64]).expect("a query"); while sink.events < limit { let Some(row) = rows.next().expect("a row") else { break; }; let seq: i64 = row.get(0).expect("a seq"); let frame = row.get_ref(1).expect("a frame"); sink.give(seq as u64, frame.as_blob().expect("a blob")); } } /// The ways RocksDB's blocks are compressed that the events are kept in. const PACKS: [Pack; 2] = [Pack::Snappy, Pack::None]; /// The same from RocksDB's family of events, with an iterator: one that /// is to give more than the first event reads its files 2 MiB at a time /// and leaves the blocks it reads out of the block cache, as a long scan /// is told to. fn from_rocks(db: &DB, cursor: u64, limit: u64, sink: &mut Sink) { let mut options = ReadOptions::default(); options.fill_cache(limit == 1); if limit > 1 { options.set_readahead_size(2 << 20); } let mut events = db.raw_iterator_cf_opt(rocks::family(db, rocks::EVENT), options); events.seek((cursor + 1).to_be_bytes()); while sink.events < limit { let (Some(seq), Some(value)) = (events.key(), events.value()) else { break; }; let seq = u64::from_be_bytes(seq.try_into().expect("a seq")); sink.give(seq, rocks::frame(value)); events.next(); } events.status().expect("the events read"); } /// The same from fjall's keyspace of events, with an iterator over the /// seqs from the one after the cursor. fn from_fjall(store: &fj::Store, cursor: u64, limit: u64, sink: &mut Sink) { for event in store.event.range((cursor + 1).to_be_bytes()..) { if sink.events >= limit { break; } let (seq, value) = event.into_inner().expect("the events read"); let seq = u64::from_be_bytes(seq[..].try_into().expect("a seq")); sink.give(seq, rocks::frame(&value)); } } /// Every account's log read as the server reads one (each frame's check /// computed, its holes filled, its JSON parsed), its events' frames taken /// out of their hex, and all of them put in order of seq: the events, and /// how long until the first could be given. fn from_logs(dir: &Path, accounts: u64, sink: &mut Sink) -> Duration { let started = Instant::now(); let mut events: Vec<(u64, Vec)> = Vec::new(); for a in 0..accounts { pds_store::scan_file(&dir.join(format!("logs/{a}.log")), |_, _, entries| { for e in &entries { if e["t"] == "event" { let frame = e["frame"].as_str().ok_or("an event with no frame")?; events.push(( e["seq"].as_u64().ok_or("an event with no seq")?, hex::decode(frame).map_err(|e| e.to_string())?, )); } } Ok(()) }) .expect("the log read") .expect("every frame"); } events.sort_unstable_by_key(|(seq, _)| *seq); let first = started.elapsed(); for (seq, frame) in &events { sink.give(*seq, frame); } first } /// The less of two times of a whole replay: every event given. fn whole(commits: u64, mut f: impl FnMut(&mut Sink)) -> (Duration, u64) { let mut best = (Duration::MAX, 0); for _ in 0..2 { let mut sink = Sink::default(); let at = Instant::now(); f(&mut sink); let took = at.elapsed(); assert_eq!(sink.events, commits, "every event replayed"); best = best.min((took, sink.bytes)); } best } /// How long until the first event after each cursor, in order. fn firsts(commits: u64, mut f: impl FnMut(u64, &mut Sink)) -> Vec { let mut times: Vec = (0..CURSORS) .map(|i| { let cursor = commits / 4 + i * (commits / 2) / CURSORS; let mut sink = Sink::default(); let at = Instant::now(); f(cursor, &mut sink); let took = at.elapsed(); assert_eq!(sink.last, cursor + 1, "the event after the cursor"); took }) .collect(); times.sort(); times } pub fn firehose(commits: u64, accounts: u64, every: u64) { let dir = scratch("firehose"); build(&dir, commits, accounts, every); println!( "The events of {commits} realistic commits over {accounts} accounts. A whole replay is the faster of two, so of files the system's cache holds, if it has the memory to; a first event is timed after those, from each of {CURSORS} cursors in the middle half, the files open.\n" ); println!( "| from | MiB on disk | whole replay ms | events per s | MiB of events per s | first event from a cursor, p50 ms | p99 ms | with an index entry each frame, p50 ms | p99 ms |" ); println!("|---|--:|--:|--:|--:|--:|--:|--:|--:|"); let row = |name: &str, bytes: u64, (took, events): (Duration, u64), first: &[Duration], dense: Option<&[Duration]>| { let dense = dense.map_or("|".to_string(), |d| { format!("{:.3} | {:.3}", ms(d[d.len() / 2]), ms(at(d, 99))) }); println!( "| {name} | {:.0} | {:.0} | {:.0} | {:.0} | {:.3} | {:.3} | {dense} |", mib(bytes), ms(took), commits as f64 / took.as_secs_f64(), mib(events) / took.as_secs_f64(), ms(first[first.len() / 2]), ms(at(first, 99)), ); }; let log = File::open(dir.join("all.log")).expect("the log"); let size = log.metadata().expect("the log").len(); let index = read_index(&dir.join("all.index")); let dense = read_index(&dir.join("all.index.each")); let index_bytes = index.len() as u64 * 16; let log_row = |name: &str, log: &File, check: Option| { row( name, size + index_bytes, whole(commits, |sink| { from_log(log, size, &index, 0, u64::MAX, check, sink) }), &firsts(commits, |cursor, sink| { from_log(log, size, &index, cursor, 1, check, sink) }), Some(&firsts(commits, |cursor, sink| { from_log(log, size, &dense, cursor, 1, check, sink) })), ); }; log_row( &format!( "one log for all, binary frames, an index entry each {every} frames, each frame's check SHA-256" ), &log, Some(Check::Sha256), ); // The same frames with each other check, in a log of their own, one at a time. for check in &Check::ALL[1..] { let path = dir.join("all.other.log"); let other = rechecked(&log, size, &path, *check); log_row( &format!("the same, each frame's check {}", check.name()), &other, Some(*check), ); drop(other); std::fs::remove_file(&path).expect("the log removed"); } log_row("the same, no frame's check computed", &log, None); let db = Connection::open(dir.join("sequencer.sqlite")).expect("the database"); row( "SQLite, a table of events by seq", dir_bytes(&dir.join("sequencer.sqlite")), whole(commits, |sink| from_sqlite(&db, 0, u64::MAX, sink)), &firsts(commits, |cursor, sink| from_sqlite(&db, cursor, 1, sink)), None, ); drop(db); let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE); for pack in PACKS { let path = rocks_path(&dir, pack); let db = Setup::new(&cache, pack, "", false).open(&path); row( &format!( "RocksDB, a family of events by seq, an iterator, its blocks: {}", pack.name() ), dir_bytes(&path), whole(commits, |sink| from_rocks(&db, 0, u64::MAX, sink)), &firsts(commits, |cursor, sink| from_rocks(&db, cursor, 1, sink)), None, ); } for pack in fj::PACKS { let path = fjall_path(&dir, pack); let store = fj::Store::open(&path, fj::CACHE, pack, false); row( &format!( "fjall, a keyspace of events by seq, an iterator, its blocks: {}", pack.name() ), fj::disk(&path), whole(commits, |sink| from_fjall(&store, 0, u64::MAX, sink)), &firsts(commits, |cursor, sink| from_fjall(&store, cursor, 1, sink)), None, ); } let mut first = Duration::MAX; let took = whole(commits, |sink| { first = first.min(from_logs(&dir, accounts, sink)) }); row( "a log for each account, JSON frames with holes (the server's), put in order", dir_bytes(&dir.join("logs")), took, &[first], None, ); let _ = std::fs::remove_dir_all(&dir); }