chadtmiller.com/monolith
An atproto PDS
0
0
//! How fast a log is replayed: the server's frames of random bytes, then
//! the same realistic commits as the server's frames and as binary ones.
use crate::check::Check;
use crate::data::{self, SEED};
use crate::frame::{self, Encoding, Frames};
use crate::util::{mib, ms, scratch};
use pds_store::Wal;
use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::time::{Duration, Instant};
/// The least of three times of `f`: the file is in the system's cache by
/// the second.
fn best(mut f: impl FnMut()) -> Duration {
(0..3)
.map(|_| {
let at = Instant::now();
f();
at.elapsed()
})
.min()
.expect("three times")
}
/// Replay the log at `path` as the server does (`pds_store::scan_file`):
/// each frame's check computed, its holes filled and its JSON parsed.
fn json(path: &Path, frames: u64) {
let mut found = 0u64;
pds_store::scan_file(path, |_, _, _| {
found += 1;
Ok(())
})
.expect("the log read")
.expect("every frame");
assert_eq!(found, frames, "every frame replayed");
}
/// A log of the server's frames of random bytes, `size` MiB of them,
/// replayed: a row.
fn random(size: u64) {
let dir = scratch("scan");
let path = dir.join("repo.log");
let (wal, _) = Wal::open(&path, None).expect("a log");
let mut op = 0u64;
let mut gone: Vec<Vec<u8>> = Vec::new();
while wal.len() < size << 20 {
op += 1;
let c = data::random(op);
wal.append_frame_at(&Encoding::Holes.frame(&c, &gone))
.expect("an append");
gone = data::nodes(&c);
}
wal.wait(wal.written()).expect("a sync");
drop(wal);
let size = mib(std::fs::metadata(&path).expect("the log").len());
let took = best(|| json(&path, op));
let _ = std::fs::remove_dir_all(&dir);
println!(
"| {size:.0} | {op} | {:.0} | {:.0} |",
ms(took),
size / took.as_secs_f64()
);
}
/// One account's realistic commits, as many as make `size` MiB of the
/// server's frames, in a log of those and in one of binary frames, each
/// replayed: two rows.
fn real(size: u64) {
let dir = scratch("scan");
let paths = [dir.join("json.log"), dir.join("binary.log")];
let mut logs = paths
.each_ref()
.map(|p| std::io::BufWriter::new(File::create(p).expect("a log")));
let mut commits = 0u64;
let mut written = 0u64;
let mut gone: Vec<Vec<u8>> = Vec::new();
while written < size << 20 {
commits += 1;
let c = data::real(SEED, commits, 0);
let frame = Encoding::Holes.frame(&c, &gone);
written += frame.len() as u64;
logs[0].write_all(&frame).expect("an append");
logs[1]
.write_all(&Encoding::Binary.frame(&c, &gone))
.expect("an append");
gone = data::nodes(&c);
}
for log in logs {
log.into_inner()
.expect("a log written")
.sync_all()
.expect("a sync");
}
let row = |name: &str, path: &Path, took: Duration| {
let bytes = std::fs::metadata(path).expect("the log").len();
println!(
"| {name} | {commits} | {:.0} | {} | {:.0} | {:.0} | {:.0} |",
mib(bytes),
bytes / commits,
ms(took),
mib(bytes) / took.as_secs_f64(),
commits as f64 / took.as_secs_f64(),
);
};
row(
"JSON frames with holes (the server's)",
&paths[0],
best(|| json(&paths[0], commits)),
);
row(
"binary frames",
&paths[1],
best(|| {
let file = File::open(&paths[1]).expect("the log");
let size = file.metadata().expect("the log").len();
let mut frames = Frames::new(file, size, Some(Check::Sha256));
let (mut found, mut bytes) = (0u64, 0usize);
while let Some(p) = frames.next_payload() {
bytes += frame::walk(p).expect("a commit's frame").frame.len();
found += 1;
}
std::hint::black_box(bytes);
assert_eq!(found, commits, "every frame replayed");
}),
);
let _ = std::fs::remove_dir_all(&dir);
}
/// A table of each: logs of the sizes given, in MiB, replayed.
pub fn scan(sizes: &[u64]) {
println!("| log MiB | frames | replay ms | MiB per s |");
println!("|--:|--:|--:|--:|");
for &size in sizes {
random(size);
}
println!(
"\nRealistic commits, the same in both logs, as many as make that many MiB of the server's frames.\n"
);
println!(
"| frames | commits | log MiB | bytes a commit | replay ms | MiB per s | commits per s |"
);
println!("|---|--:|--:|--:|--:|--:|--:|");
for &size in sizes {
real(size);
}
}