chadtmiller.com/monolith
An atproto PDS
0
0
//! What the parts share: a generator of numbers that is the same for the
//! same seed, scratch directories, work spread over threads, and the
//! figures of a table.
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
/// Numbers that are the same for the same seed (splitmix64).
pub struct Rng(u64);
fn mix(mut z: u64) -> u64 {
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
impl Rng {
/// A generator of what `parts` name: a seed, and which thing of it.
pub fn of(parts: &[u64]) -> Rng {
let mut x = 0x9E37_79B9_7F4A_7C15u64;
for &p in parts {
x = mix(x ^ p).wrapping_add(0x9E37_79B9_7F4A_7C15);
}
Rng(x)
}
pub fn next_u64(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
mix(self.0)
}
/// A number below `n`.
pub fn below(&mut self, n: u64) -> u64 {
((u128::from(self.next_u64()) * u128::from(n)) >> 64) as u64
}
/// `n` more bytes on `out`, none of which compress.
pub fn fill(&mut self, out: &mut Vec<u8>, n: usize) {
let end = out.len() + n;
while out.len() < end {
let x = self.next_u64().to_le_bytes();
let take = (end - out.len()).min(8);
out.extend_from_slice(&x[..take]);
}
}
/// `items` in an order drawn from it.
pub fn shuffle<T>(&mut self, items: &mut [T]) {
for i in (1..items.len()).rev() {
items.swap(i, self.below(i as u64 + 1) as usize);
}
}
}
/// Where the benchmark's files go: `BENCH_DIR`, or `target/data` beside
/// this crate.
pub fn root() -> PathBuf {
std::env::var_os("BENCH_DIR").map_or_else(
|| Path::new(env!("CARGO_MANIFEST_DIR")).join("target/data"),
PathBuf::from,
)
}
/// An empty directory of this name under `BENCH_DIR`.
pub fn scratch(name: &str) -> PathBuf {
let dir = root().join(name);
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("a scratch directory");
dir
}
/// The bytes of every file under `dir`, as their lengths say.
pub fn dir_bytes(dir: &Path) -> u64 {
let Ok(entries) = std::fs::read_dir(dir) else {
return std::fs::metadata(dir).map_or(0, |m| m.len());
};
entries
.flatten()
.map(|e| match e.metadata() {
Ok(m) if m.is_dir() => dir_bytes(&e.path()),
Ok(m) => m.len(),
Err(_) => 0,
})
.sum()
}
/// Read every file under `dir` through, so that the system's cache holds
/// them, as far as it keeps what it reads: a file only written may have
/// been let go of once it was on disk.
pub fn read_through(dir: &Path) {
let Ok(entries) = std::fs::read_dir(dir) else {
if let Ok(mut f) = std::fs::File::open(dir) {
let _ = std::io::copy(&mut f, &mut std::io::sink());
}
return;
};
entries.flatten().for_each(|e| read_through(&e.path()));
}
/// `f(0)` to `f(n - 1)`, on `threads` threads, in order of their number.
pub fn par<T: Send>(n: usize, threads: usize, f: impl Fn(usize) -> T + Sync) -> Vec<T> {
let next = AtomicUsize::new(0);
let out = Mutex::new(Vec::with_capacity(n));
std::thread::scope(|s| {
for _ in 0..threads.clamp(1, n.max(1)) {
s.spawn(|| {
loop {
let i = next.fetch_add(1, Ordering::Relaxed);
if i >= n {
break;
}
let x = f(i);
out.lock().expect("the results").push((i, x));
}
});
}
});
let mut out = out.into_inner().expect("the results");
out.sort_by_key(|(i, _)| *i);
out.into_iter().map(|(_, x)| x).collect()
}
/// The threads work that waits on no disk is spread over.
pub fn cores() -> usize {
std::thread::available_parallelism().map_or(4, |n| n.get())
}
/// This process's resident memory, in KiB, as `ps` reports it.
pub fn rss_kib() -> u64 {
let out = std::process::Command::new("ps")
.args(["-o", "rss=", "-p", &std::process::id().to_string()])
.output()
.expect("ps");
String::from_utf8_lossy(&out.stdout)
.trim()
.parse()
.expect("ps's rss")
}
/// A child that reports this process's resident memory whenever it is
/// asked, for a process that will hold more files than `ps` can be started
/// under: macOS starts no child whose pipes are numbered past 10240, which
/// two thousand open RocksDB databases put them. Started before they are.
pub struct Rss {
child: std::process::Child,
ask: Option<std::process::ChildStdin>,
answers: std::io::BufReader<std::process::ChildStdout>,
}
impl Rss {
pub fn start() -> Rss {
use std::process::{Command, Stdio};
let script = format!(
"while read _; do ps -o rss= -p {}; done",
std::process::id()
);
let mut child = Command::new("sh")
.args(["-c", &script])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.spawn()
.expect("sh");
Rss {
ask: child.stdin.take(),
answers: std::io::BufReader::new(child.stdout.take().expect("its output")),
child,
}
}
/// This process's resident memory now, in KiB.
pub fn kib(&mut self) -> u64 {
use std::io::{BufRead, Write};
let ask = self.ask.as_mut().expect("a child to ask");
ask.write_all(b"\n").expect("asked");
let mut line = String::new();
self.answers.read_line(&mut line).expect("an answer");
line.trim().parse().expect("ps's rss")
}
}
impl Drop for Rss {
fn drop(&mut self) {
self.ask = None;
let _ = self.child.wait();
}
}
/// Let the process hold as many files as it may, as the server does when
/// it starts (`pds_store::files::raise`): one for each of a few thousand
/// accounts, three for a SQLite database in WAL mode and nine for a
/// RocksDB.
pub fn raise_files() {
pds_store::files::raise();
}
/// The files the process may hold open at once.
pub fn files() -> u64 {
pds_store::files::limit().unwrap_or(u64::MAX)
}
/// The tables' files the server would hold open at once in this process:
/// its share of the files the process may hold.
pub fn share() -> usize {
pds_store::files::share(pds_store::files::limit())
}
/// The time `per_cent` of `sorted` are within.
pub fn at(sorted: &[Duration], per_cent: usize) -> Duration {
sorted[(sorted.len() * per_cent / 100).min(sorted.len() - 1)]
}
pub fn ms(d: Duration) -> f64 {
d.as_secs_f64() * 1e3
}
pub fn us(d: Duration) -> f64 {
d.as_secs_f64() * 1e6
}
pub fn mib(bytes: u64) -> f64 {
bytes as f64 / 1_048_576.0
}