~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
429 lines · 16.1 KB Raw History
//! The firehose replayed, four ways, of the same realistic commits: from
//! one log for all, whose order is the firehose's; from a SQLite table of
//! events by seq; from RocksDB's family of events by seq; and from a log
//! for each account, as the server keeps them, where the order is made
//! again from every log.

use crate::check::Check;
use crate::data::{self, SEED};
use crate::fj;
use crate::frame::{self, Encoding, Frames};
use crate::rocks::{self, Pack, Setup};
use crate::stores::{fjall_path, rocks_path, settle};
use crate::util::{Rng, at, dir_bytes, mib, ms, scratch};
use rocksdb::{DB, ReadOptions, WriteBatch};
use rusqlite::{Connection, params};
use std::fs::File;
use std::hint::black_box;
use std::io::{BufWriter, Seek, SeekFrom, Write};
use std::path::Path;
use std::time::{Duration, Instant};

/// The cursors a first event is timed from, spread over the middle half of
/// the events.
const CURSORS: u64 = 200;

/// What a replay gives each event to: here, a count of them and of their bytes.
#[derive(Default)]
struct Sink {
    events: u64,
    bytes: u64,
    last: u64,
}

impl Sink {
    fn give(&mut self, seq: u64, frame: &[u8]) {
        assert!(seq > self.last, "events in order");
        self.last = seq;
        self.events += 1;
        self.bytes += black_box(frame).len() as u64;
    }
}

/// Write the four: `all.log` with `all.index`, an entry of seq and offset
/// for each `every` frames, written as the log is, and `all.index.each`,
/// an entry for each frame; `logs/<account>.log`; `sequencer.sqlite`; and
/// a RocksDB for each of `PACKS`, left to settle. Unsynced: they are only
/// read.
fn build(dir: &Path, commits: u64, accounts: u64, every: u64) {
    std::fs::create_dir_all(dir.join("logs")).expect("a directory");
    let create = |name: String| File::create(dir.join(name)).expect("a file");
    let mut log = BufWriter::with_capacity(1 << 20, create("all.log".into()));
    let mut index = BufWriter::new(create("all.index".into()));
    let mut each = BufWriter::new(create("all.index.each".into()));
    let mut logs: Vec<File> = (0..accounts)
        .map(|a| create(format!("logs/{a}.log")))
        .collect();
    let db = Connection::open(dir.join("sequencer.sqlite")).expect("a database");
    db.execute_batch(
        "PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF;
         CREATE TABLE event(seq INTEGER PRIMARY KEY, account INTEGER, time INTEGER, frame BLOB);
         BEGIN;",
    )
    .expect("a schema");
    let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE);
    let mut rocks: Vec<(DB, WriteBatch)> = PACKS
        .iter()
        .map(|p| {
            let db = Setup::new(&cache, *p, "", false).open(&rocks_path(dir, *p));
            (db, WriteBatch::default())
        })
        .collect();
    let fjalls: Vec<fj::Store> = fj::PACKS
        .iter()
        .map(|p| fj::Store::open(&fjall_path(dir, *p), fj::CACHE, *p, false))
        .collect();
    let mut batches: Vec<_> = fjalls.iter().map(|s| s.db.batch()).collect();
    let mut last: Vec<Vec<Vec<u8>>> = vec![Vec::new(); accounts as usize];
    let mut pick = Rng::of(&[SEED, 9]);
    let mut offset = 0u64;
    for seq in 1..=commits {
        let a = pick.below(accounts);
        let c = data::real(SEED, seq, a);
        let gone = &last[a as usize];
        let frame = Encoding::Binary.frame(&c, gone);
        let entry = [seq.to_le_bytes(), offset.to_le_bytes()].concat();
        if (seq - 1) % every == 0 {
            index.write_all(&entry).expect("the index");
        }
        each.write_all(&entry).expect("the index");
        log.write_all(&frame).expect("an append");
        offset += frame.len() as u64;
        logs[a as usize]
            .write_all(&Encoding::Holes.frame(&c, gone))
            .expect("an append");
        db.prepare_cached("INSERT INTO event(seq, account, time, frame) VALUES (?1, ?2, ?3, ?4)")
            .expect("a statement")
            .execute(params![
                seq as i64,
                a as i64,
                1_700_000_000_000i64 + seq as i64,
                c.event
            ])
            .expect("an event");
        for (db, batch) in &mut rocks {
            let event = rocks::event(a as usize, seq, &c.event);
            batch.put_cf(rocks::family(db, rocks::EVENT), seq.to_be_bytes(), event);
        }
        for (store, batch) in fjalls.iter().zip(&mut batches) {
            let event = rocks::event(a as usize, seq, &c.event);
            batch.insert(&store.event, seq.to_be_bytes(), event);
        }
        if seq % 1000 == 0 || seq == commits {
            db.execute_batch("COMMIT; BEGIN;").expect("a commit");
            for (db, batch) in &mut rocks {
                db.write_without_wal(std::mem::take(batch))
                    .expect("a batch");
            }
            for (store, batch) in fjalls.iter().zip(&mut batches) {
                let batch = std::mem::replace(batch, store.db.batch());
                batch.commit().expect("a batch");
            }
        }
        last[a as usize] = data::nodes(&c);
    }
    db.execute_batch("COMMIT;").expect("a commit");
    crate::stores::close_db(db);
    rocks.iter().for_each(|(db, _)| settle(db));
    fjalls.iter().for_each(fj::Store::settle);
    log.flush().expect("the log");
    index.flush().expect("the index");
    each.flush().expect("the index");
}

/// The one log's sparse index, read whole: seqs and the offsets of their frames.
fn read_index(path: &Path) -> Vec<(u64, u64)> {
    std::fs::read(path)
        .expect("the index")
        .chunks_exact(16)
        .map(|e| {
            let n = |i: usize| u64::from_le_bytes(e[i..i + 8].try_into().expect("8 bytes"));
            (n(0), n(8))
        })
        .collect()
}

/// The events after `cursor` from the one log, at most `limit`: from the
/// frame the index names at or before the first of them, each frame's
/// check computed, if it has one to compute, and its fields walked.
fn from_log(
    mut file: &File,
    size: u64,
    index: &[(u64, u64)],
    cursor: u64,
    limit: u64,
    check: Option<Check>,
    sink: &mut Sink,
) {
    let i = index.partition_point(|&(seq, _)| seq <= cursor + 1);
    let offset = index[i.saturating_sub(1)].1;
    file.seek(SeekFrom::Start(offset)).expect("a seek");
    let mut frames = Frames::new(file, size - offset, check);
    while sink.events < limit {
        let Some(p) = frames.next_payload() else {
            break;
        };
        let e = frame::walk(p).expect("a commit's frame");
        if e.seq > cursor {
            sink.give(e.seq, e.frame);
        }
    }
}

/// The one log again, at `path`, each frame with `check` in the place of
/// SHA-256's.
fn rechecked(mut log: &File, size: u64, path: &Path, check: Check) -> File {
    let mut out = BufWriter::with_capacity(1 << 20, File::create(path).expect("a log"));
    log.seek(SeekFrom::Start(0)).expect("a seek");
    let mut frames = Frames::new(log, size, None);
    while let Some(p) = frames.next_payload() {
        out.write_all(&(p.len() as u32).to_le_bytes())
            .expect("an append");
        out.write_all(&check.of_frame(p)).expect("an append");
        out.write_all(p).expect("an append");
    }
    drop(out.into_inner().expect("the log written"));
    File::open(path).expect("the log")
}

/// The same from the table of events.
fn from_sqlite(db: &Connection, cursor: u64, limit: u64, sink: &mut Sink) {
    let mut st = db
        .prepare_cached("SELECT seq, frame FROM event WHERE seq > ?1 ORDER BY seq")
        .expect("a statement");
    let mut rows = st.query(params![cursor as i64]).expect("a query");
    while sink.events < limit {
        let Some(row) = rows.next().expect("a row") else {
            break;
        };
        let seq: i64 = row.get(0).expect("a seq");
        let frame = row.get_ref(1).expect("a frame");
        sink.give(seq as u64, frame.as_blob().expect("a blob"));
    }
}

/// The ways RocksDB's blocks are compressed that the events are kept in.
const PACKS: [Pack; 2] = [Pack::Snappy, Pack::None];

/// The same from RocksDB's family of events, with an iterator: one that
/// is to give more than the first event reads its files 2 MiB at a time
/// and leaves the blocks it reads out of the block cache, as a long scan
/// is told to.
fn from_rocks(db: &DB, cursor: u64, limit: u64, sink: &mut Sink) {
    let mut options = ReadOptions::default();
    options.fill_cache(limit == 1);
    if limit > 1 {
        options.set_readahead_size(2 << 20);
    }
    let mut events = db.raw_iterator_cf_opt(rocks::family(db, rocks::EVENT), options);
    events.seek((cursor + 1).to_be_bytes());
    while sink.events < limit {
        let (Some(seq), Some(value)) = (events.key(), events.value()) else {
            break;
        };
        let seq = u64::from_be_bytes(seq.try_into().expect("a seq"));
        sink.give(seq, rocks::frame(value));
        events.next();
    }
    events.status().expect("the events read");
}

/// The same from fjall's keyspace of events, with an iterator over the
/// seqs from the one after the cursor.
fn from_fjall(store: &fj::Store, cursor: u64, limit: u64, sink: &mut Sink) {
    for event in store.event.range((cursor + 1).to_be_bytes()..) {
        if sink.events >= limit {
            break;
        }
        let (seq, value) = event.into_inner().expect("the events read");
        let seq = u64::from_be_bytes(seq[..].try_into().expect("a seq"));
        sink.give(seq, rocks::frame(&value));
    }
}

/// Every account's log read as the server reads one (each frame's check
/// computed, its holes filled, its JSON parsed), its events' frames taken
/// out of their hex, and all of them put in order of seq: the events, and
/// how long until the first could be given.
fn from_logs(dir: &Path, accounts: u64, sink: &mut Sink) -> Duration {
    let started = Instant::now();
    let mut events: Vec<(u64, Vec<u8>)> = Vec::new();
    for a in 0..accounts {
        pds_store::scan_file(&dir.join(format!("logs/{a}.log")), |_, _, entries| {
            for e in &entries {
                if e["t"] == "event" {
                    let frame = e["frame"].as_str().ok_or("an event with no frame")?;
                    events.push((
                        e["seq"].as_u64().ok_or("an event with no seq")?,
                        hex::decode(frame).map_err(|e| e.to_string())?,
                    ));
                }
            }
            Ok(())
        })
        .expect("the log read")
        .expect("every frame");
    }
    events.sort_unstable_by_key(|(seq, _)| *seq);
    let first = started.elapsed();
    for (seq, frame) in &events {
        sink.give(*seq, frame);
    }
    first
}

/// The less of two times of a whole replay: every event given.
fn whole(commits: u64, mut f: impl FnMut(&mut Sink)) -> (Duration, u64) {
    let mut best = (Duration::MAX, 0);
    for _ in 0..2 {
        let mut sink = Sink::default();
        let at = Instant::now();
        f(&mut sink);
        let took = at.elapsed();
        assert_eq!(sink.events, commits, "every event replayed");
        best = best.min((took, sink.bytes));
    }
    best
}

/// How long until the first event after each cursor, in order.
fn firsts(commits: u64, mut f: impl FnMut(u64, &mut Sink)) -> Vec<Duration> {
    let mut times: Vec<Duration> = (0..CURSORS)
        .map(|i| {
            let cursor = commits / 4 + i * (commits / 2) / CURSORS;
            let mut sink = Sink::default();
            let at = Instant::now();
            f(cursor, &mut sink);
            let took = at.elapsed();
            assert_eq!(sink.last, cursor + 1, "the event after the cursor");
            took
        })
        .collect();
    times.sort();
    times
}

pub fn firehose(commits: u64, accounts: u64, every: u64) {
    let dir = scratch("firehose");
    build(&dir, commits, accounts, every);
    println!(
        "The events of {commits} realistic commits over {accounts} accounts. A whole replay is the faster of two, so of files the system's cache holds, if it has the memory to; a first event is timed after those, from each of {CURSORS} cursors in the middle half, the files open.\n"
    );
    println!(
        "| from | MiB on disk | whole replay ms | events per s | MiB of events per s | first event from a cursor, p50 ms | p99 ms | with an index entry each frame, p50 ms | p99 ms |"
    );
    println!("|---|--:|--:|--:|--:|--:|--:|--:|--:|");
    let row = |name: &str,
               bytes: u64,
               (took, events): (Duration, u64),
               first: &[Duration],
               dense: Option<&[Duration]>| {
        let dense = dense.map_or("|".to_string(), |d| {
            format!("{:.3} | {:.3}", ms(d[d.len() / 2]), ms(at(d, 99)))
        });
        println!(
            "| {name} | {:.0} | {:.0} | {:.0} | {:.0} | {:.3} | {:.3} | {dense} |",
            mib(bytes),
            ms(took),
            commits as f64 / took.as_secs_f64(),
            mib(events) / took.as_secs_f64(),
            ms(first[first.len() / 2]),
            ms(at(first, 99)),
        );
    };

    let log = File::open(dir.join("all.log")).expect("the log");
    let size = log.metadata().expect("the log").len();
    let index = read_index(&dir.join("all.index"));
    let dense = read_index(&dir.join("all.index.each"));
    let index_bytes = index.len() as u64 * 16;
    let log_row = |name: &str, log: &File, check: Option<Check>| {
        row(
            name,
            size + index_bytes,
            whole(commits, |sink| {
                from_log(log, size, &index, 0, u64::MAX, check, sink)
            }),
            &firsts(commits, |cursor, sink| {
                from_log(log, size, &index, cursor, 1, check, sink)
            }),
            Some(&firsts(commits, |cursor, sink| {
                from_log(log, size, &dense, cursor, 1, check, sink)
            })),
        );
    };
    log_row(
        &format!(
            "one log for all, binary frames, an index entry each {every} frames, each frame's check SHA-256"
        ),
        &log,
        Some(Check::Sha256),
    );
    // The same frames with each other check, in a log of their own, one at a time.
    for check in &Check::ALL[1..] {
        let path = dir.join("all.other.log");
        let other = rechecked(&log, size, &path, *check);
        log_row(
            &format!("the same, each frame's check {}", check.name()),
            &other,
            Some(*check),
        );
        drop(other);
        std::fs::remove_file(&path).expect("the log removed");
    }
    log_row("the same, no frame's check computed", &log, None);

    let db = Connection::open(dir.join("sequencer.sqlite")).expect("the database");
    row(
        "SQLite, a table of events by seq",
        dir_bytes(&dir.join("sequencer.sqlite")),
        whole(commits, |sink| from_sqlite(&db, 0, u64::MAX, sink)),
        &firsts(commits, |cursor, sink| from_sqlite(&db, cursor, 1, sink)),
        None,
    );
    drop(db);

    let cache = rocksdb::Cache::new_lru_cache(rocks::CACHE);
    for pack in PACKS {
        let path = rocks_path(&dir, pack);
        let db = Setup::new(&cache, pack, "", false).open(&path);
        row(
            &format!(
                "RocksDB, a family of events by seq, an iterator, its blocks: {}",
                pack.name()
            ),
            dir_bytes(&path),
            whole(commits, |sink| from_rocks(&db, 0, u64::MAX, sink)),
            &firsts(commits, |cursor, sink| from_rocks(&db, cursor, 1, sink)),
            None,
        );
    }

    for pack in fj::PACKS {
        let path = fjall_path(&dir, pack);
        let store = fj::Store::open(&path, fj::CACHE, pack, false);
        row(
            &format!(
                "fjall, a keyspace of events by seq, an iterator, its blocks: {}",
                pack.name()
            ),
            fj::disk(&path),
            whole(commits, |sink| from_fjall(&store, 0, u64::MAX, sink)),
            &firsts(commits, |cursor, sink| from_fjall(&store, cursor, 1, sink)),
            None,
        );
    }

    let mut first = Duration::MAX;
    let took = whole(commits, |sink| {
        first = first.min(from_logs(&dir, accounts, sink))
    });
    row(
        "a log for each account, JSON frames with holes (the server's), put in order",
        dir_bytes(&dir.join("logs")),
        took,
        &[first],
        None,
    );
    let _ = std::fs::remove_dir_all(&dir);
}