//! Reads of what accounts hold (`stores`): a record by its path, a listing //! of 50 records, the first read of an account not yet open, and what the //! open accounts keep resident. Each store is measured in processes of its //! own, started after the files are written: nothing of them is in the //! process, and all of them are in the system's cache. use crate::check::Check; use crate::data::{self, SEED}; use crate::fj; use crate::rocks::{self, Pack, Setup}; use crate::stores::{self, block_bytes, key, path_cid}; use crate::tables::{self, Reader}; use crate::util::{Rng, at, cores, dir_bytes, mib, par, rss_kib, scratch, share, us}; use pds_store::table::{BLOCK, BlockCache, Table, Version}; use rocksdb::DB; use rusqlite::{Connection, params}; use std::collections::HashMap; use std::hint::black_box; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; /// The bytes the tables' block cache may hold, each connection's page /// cache of the one database for all, and RocksDB's block cache: more /// than the data, so that nothing read is let go. const CACHE: u64 = 1 << 30; /// The records looked up one after another, on each of the threads, and /// the listings. const LOOKUPS: usize = 20_000; const THREADS: usize = 16; const EACH: usize = 5_000; const LISTINGS: usize = 2_000; const LISTED: usize = 50; const GET: &str = "SELECT b.bytes FROM record r JOIN block b ON b.account = r.account AND b.cid = r.cid WHERE r.account = ?1 AND r.path = ?2"; const LIST: &str = "SELECT r.path, b.bytes FROM record r JOIN block b ON b.account = r.account AND b.cid = r.cid WHERE r.account = ?1 AND r.path >= ?2 ORDER BY r.path LIMIT ?3"; /// A way of reading one of the stores, each account's file opened when it /// is first read. enum Store { /// A table for each account read as the server reads one /// (`pds_store::table::Table`), through one block cache, which keeps /// the tables' files open: at most the server's share of them /// (`util::share`), or every one. Tables { dir: PathBuf, tables: Vec>, cache: BlockCache, }, /// Tables read by this benchmark's reader (`tables::Reader`), which /// keeps every block it reads, has no bloom filter, and computes the /// check it is given: each file held open, or opened for each block /// read. Own { dir: PathBuf, readers: Vec>, check: Check, each: bool, }, /// A database for each account, one connection each, its page cache /// SQLite's own 2 MiB: more than the database. Dbs { dir: PathBuf, dbs: Vec>>, }, /// One database for all, a connection for each thread that reads. Db { dir: PathBuf, conns: Vec>>, }, /// One RocksDB for all, opened when it is first read, its files held /// open, read from every thread through one block cache. Rocks { dir: PathBuf, db: OnceLock, pack: Pack, }, /// One fjall database for all, opened when it is first read, read /// from every thread through one block cache. Fjall { dir: PathBuf, store: OnceLock, pack: fj::Pack, }, } /// The directory a way of reading reads: `tables`, `tables16` (the same /// entries in the layout before), `dbs` or `db`. A way is named for it; /// then, for this benchmark's reader, `:` and a check; then `+open` for /// tables whose files the block cache holds open every one, or `+each` /// for a reader that opens its file for each block. fn files(way: &str) -> &str { way.split(['+', ':']).next().unwrap_or(way) } impl Store { fn open(way: &str, dir: &Path, accounts: usize, threads: usize) -> Store { let dir = dir.join(files(way)); fn cells(n: usize) -> Vec> { (0..n).map(|_| OnceLock::new()).collect() } let (named, flag) = way.split_once('+').unwrap_or((way, "")); match (files(way), named.split_once(':')) { ("tables" | "tables16", Some((_, check))) => Store::Own { dir, readers: cells(accounts), check: Check::named(check).expect("a check"), each: flag == "each", }, ("tables", None) => Store::Tables { dir, tables: cells(accounts), cache: match flag { "open" => BlockCache::with_open(CACHE, accounts), _ => BlockCache::with_open(CACHE, share()), }, }, ("dbs", _) => Store::Dbs { dir, dbs: cells(accounts), }, ("db", _) => Store::Db { dir, conns: cells(threads), }, ("rocks", _) | ("rocks-plain", _) => Store::Rocks { pack: if files(way) == "rocks" { Pack::Snappy } else { Pack::None }, dir, db: OnceLock::new(), }, ("fjall", _) | ("fjall-plain", _) => Store::Fjall { pack: if files(way) == "fjall" { fj::Pack::Lz4 } else { fj::Pack::None }, dir, store: OnceLock::new(), }, _ => panic!("no way of reading named {way}"), } } /// The account's table, opened if it is not: its index and bloom /// filter read, and its file one of those the cache holds open. fn table(&self, account: usize) -> (&Table, &BlockCache) { let Store::Tables { dir, tables, cache } = self else { unreachable!("no table") }; let table = tables[account].get_or_init(|| { Table::open(&dir.join(format!("{account}.table")), cache).expect("the account's table") }); (table, cache) } /// The account's table in this benchmark's reader, opened if it is not. fn reader(&self, account: usize) -> &Reader { let Store::Own { dir, readers, check, each, } = self else { unreachable!("no reader") }; readers[account].get_or_init(|| { let path = dir.join(format!("{account}.table")); let reader = Reader::open(&path, Some(*check), true); if *each { reader.opening_each() } else { reader } }) } /// The one RocksDB, opened if it is not. fn rocks(&self) -> &DB { let Store::Rocks { dir, db, pack } = self else { unreachable!("no RocksDB") }; db.get_or_init(|| { let cache = rocksdb::Cache::new_lru_cache(CACHE as usize); Setup::new(&cache, *pack, "", false).open(dir) }) } /// The one fjall database, opened if it is not. fn fjall(&self) -> &fj::Store { let Store::Fjall { dir, store, pack } = self else { unreachable!("no fjall") }; store.get_or_init(|| fj::Store::open(dir, CACHE, *pack, false)) } /// The connection the read goes through, opened if it is not. fn conn(&self, thread: usize, account: usize) -> &Mutex { match self { Store::Dbs { dir, dbs } => dbs[account].get_or_init(|| { let path = dir.join(format!("{account}.sqlite")); Mutex::new(Connection::open(path).expect("a database")) }), Store::Db { dir, conns } => conns[thread].get_or_init(|| { let db = Connection::open(dir.join("store.sqlite")).expect("the database"); db.execute_batch(&format!("PRAGMA cache_size=-{};", CACHE >> 10)) .expect("a cache"); Mutex::new(db) }), _ => unreachable!("no connection"), } } /// Open what a read of `account` on `thread` goes through. fn touch(&self, thread: usize, account: usize) { match self { Store::Tables { .. } => { self.table(account); } Store::Own { .. } => { self.reader(account); } Store::Rocks { .. } => { self.rocks(); } Store::Fjall { .. } => { self.fjall(); } _ => { self.conn(thread, account); } } } /// The record at `path` of `account`: its bytes. fn get(&self, thread: usize, account: usize, path: &str) -> Vec { let path_key = key(b'P', path.as_bytes()); match self { Store::Tables { .. } => { let (t, cache) = self.table(account); let at = t .get(&path_key, cache) .expect("the table read") .expect("the path"); let block = t .get(&key(b'B', path_cid(&at)), cache) .expect("the table read") .expect("the record's block"); block_bytes(&block).to_vec() } Store::Own { .. } => { let r = self.reader(account); let at = r.get(&path_key, &mut None); block_bytes(&r.get(&key(b'B', path_cid(&at)), &mut None)).to_vec() } Store::Rocks { .. } => { let db = self.rocks(); let at = db .get_pinned_cf( rocks::family(db, rocks::RECORD), rocks::key(account, path.as_bytes()), ) .expect("the database read") .expect("the path"); let block = db .get_pinned_cf( rocks::family(db, rocks::BLOCK), rocks::key(account, path_cid(&at)), ) .expect("the database read") .expect("the record's block"); block_bytes(&block).to_vec() } Store::Fjall { .. } => { let s = self.fjall(); let at = s .record .get(rocks::key(account, path.as_bytes())) .expect("the database read") .expect("the path"); let block = s .block .get(rocks::key(account, path_cid(&at))) .expect("the database read") .expect("the record's block"); block_bytes(&block).to_vec() } _ => { let db = self.conn(thread, account).lock().expect("the database"); db.prepare_cached(GET) .expect("a statement") .query_row(params![account as i64, path], |row| row.get(0)) .expect("the record") } } } /// The `LISTED` records of `account` from `path` on, in order: how /// many, and their bytes. fn list(&self, thread: usize, account: usize, path: &str) -> (usize, usize) { let (mut n, mut bytes) = (0, 0); let from = key(b'P', path.as_bytes()); match self { Store::Tables { .. } => { let (t, cache) = self.table(account); for e in t.iter(&from, None, cache).take(LISTED) { let (k, at) = e.expect("the table read"); if k[0] != b'P' { break; } let block = t .get(&key(b'B', path_cid(&at)), cache) .expect("the table read") .expect("the record's block"); n += 1; bytes += black_box(block_bytes(&block).to_vec()).len() + k.len(); } } Store::Own { .. } => { let r = self.reader(account); for (k, at) in r.list(&from, LISTED) { let block = r.get(&key(b'B', path_cid(&at)), &mut None); n += 1; bytes += black_box(block_bytes(&block).to_vec()).len() + k.len(); } } // The paths with an iterator, then their blocks asked for // together (`MultiGet`), as RocksDB is meant to be asked for // many keys. Store::Rocks { .. } => { let db = self.rocks(); let mut paths = db.raw_iterator_cf(rocks::family(db, rocks::RECORD)); paths.seek(rocks::key(account, path.as_bytes())); let mut blocks = Vec::with_capacity(LISTED); while blocks.len() < LISTED { let (Some(k), Some(at)) = (paths.key(), paths.value()) else { break; }; if k[..4] != (account as u32).to_be_bytes() { break; } bytes += k.len() - 3; blocks.push(rocks::key(account, path_cid(at))); paths.next(); } paths.status().expect("the paths read"); for b in db.batched_multi_get_cf(rocks::family(db, rocks::BLOCK), &blocks, false) { let b = b.expect("the database read").expect("the record's block"); n += 1; bytes += black_box(block_bytes(&b).to_vec()).len(); } } // The paths with an iterator, and each one's block with a // read of its own: fjall has no read of many keys at once. Store::Fjall { .. } => { let s = self.fjall(); let paths = s.record.range(rocks::key(account, path.as_bytes())..); for path in paths.take(LISTED) { let (k, at) = path.into_inner().expect("the paths read"); if k[..4] != (account as u32).to_be_bytes() { break; } let b = s .block .get(rocks::key(account, path_cid(&at))) .expect("the database read") .expect("the record's block"); n += 1; bytes += black_box(block_bytes(&b).to_vec()).len() + k.len() - 3; } } _ => { let db = self.conn(thread, account).lock().expect("the database"); let mut st = db.prepare_cached(LIST).expect("a statement"); let mut rows = st .query(params![account as i64, path, LISTED as i64]) .expect("a query"); while let Some(row) = rows.next().expect("a row") { let path: String = row.get(0).expect("a path"); let block: Vec = row.get(1).expect("a block"); n += 1; bytes += black_box(block).len() + path.len(); } } } (n, bytes) } } /// A record to read: its account and its path. type Want = (usize, String); /// `n` records drawn from all of them, the same for the same `which`; the /// last `LISTED` of an account left out when `listing`, so that every /// listing is whole. fn wants(which: u64, n: usize, accounts: usize, records: usize, listing: bool) -> Vec { let mut r = Rng::of(&[SEED, 7, which]); let among = if listing { records - LISTED } else { records } as u64; (0..n) .map(|_| { let (a, i) = (r.below(accounts as u64), r.below(among)); (a as usize, data::path(SEED, i * accounts as u64 + a + 1, a)) }) .collect() } /// Time `f` of each of `wants`, in an order of its own. fn each(wants: &[Want], order: u64, mut f: impl FnMut(&Want)) -> Vec { let mut order_of: Vec = (0..wants.len()).collect(); Rng::of(&[SEED, 8, order]).shuffle(&mut order_of); let mut times: Vec = order_of .into_iter() .map(|i| { let at = Instant::now(); f(&wants[i]); at.elapsed() }) .collect(); times.sort(); times } fn say(name: &str, value: f64) { println!("{name} {value}"); } fn say_times(name: &str, times: &[Duration]) { say(&format!("{name}_p50"), us(times[times.len() / 2])); say(&format!("{name}_p99"), us(at(times, 99))); } /// One store measured one way, in a process of its own: figures by name, /// a line each. pub fn child(dir: &Path, store: &str, test: &str, accounts: usize, records: usize) { let before = rss_kib(); match test { // The first read of each account, then records by path one after another. "get" => { let s = Store::open(store, dir, accounts, 1); let firsts: Vec = (0..accounts) .map(|a| { let i = Rng::of(&[SEED, 6, a as u64]).below(records as u64); ( a, data::path(SEED, i * accounts as u64 + a as u64 + 1, a as u64), ) }) .collect(); say_times( "first", &each(&firsts, 0, |(a, path)| { black_box(s.get(0, *a, path)); }), ); let open = rss_kib(); say("rss_before", before as f64); say("rss_open", open as f64); let wants = wants(1, LOOKUPS, accounts, records, false); for (name, order) in [("cold", 1), ("warm", 2)] { say_times( name, &each(&wants, order, |(a, path)| { black_box(s.get(0, *a, path)); }), ); } say("rss_after", rss_kib() as f64); } // Records by path from 16 threads, each its own. "threads" => { let s = Store::open(store, dir, accounts, THREADS); let wants: Vec> = (0..THREADS) .map(|t| wants(100 + t as u64, EACH, accounts, records, false)) .collect(); // The accounts opened first, so that both passes time reads alone. match &s { Store::Db { .. } => (0..THREADS).for_each(|t| s.touch(t, 0)), _ => (0..accounts).for_each(|a| s.touch(0, a)), } for name in ["cold", "warm"] { let started = Instant::now(); std::thread::scope(|scope| { for (t, mine) in wants.iter().enumerate() { let s = &s; scope.spawn(move || { for (a, path) in mine { black_box(s.get(t, *a, path)); } }); } }); say( name, (THREADS * EACH) as f64 / started.elapsed().as_secs_f64(), ); } } // Listings of 50 records from a path. "list" => { let s = Store::open(store, dir, accounts, 1); let wants = wants(2, LISTINGS, accounts, records, true); for (name, order) in [("cold", 1), ("warm", 2)] { say_times( name, &each(&wants, order, |(a, path)| { assert_eq!(s.list(0, *a, path).0, LISTED, "a whole listing"); }), ); } } _ => panic!("no test named {test}"), } } /// Run a child and read its figures, the store's files read through first. fn measure( dir: &Path, store: &str, test: &str, accounts: usize, records: usize, ) -> HashMap { crate::util::read_through(&dir.join(files(store))); let out = std::process::Command::new(std::env::current_exe().expect("this program")) .arg("reads-child") .arg(dir) .args([store, test, &accounts.to_string(), &records.to_string()]) .stderr(std::process::Stdio::inherit()) .output() .expect("a child"); assert!(out.status.success(), "reading {store} ({test}) failed"); String::from_utf8_lossy(&out.stdout) .lines() .filter_map(|l| { let (name, value) = l.split_once(' ')?; Some((name.to_string(), value.parse().ok()?)) }) .collect() } /// See that the table at `before`, which this benchmark wrote in the /// layout before, is one of the version before that `Table` reads, and /// holds the entries of the table at `now`. fn same_entries(now: &Path, before: &Path) { let cache = BlockCache::new(CACHE); let entries = |path: &Path, version: Version| { let t = Table::open(path, &cache).expect("a table"); assert_eq!(t.version(), version, "{}", path.display()); t.iter(b"", None, &cache) .map(|e| e.expect("the table read")) .collect::>() }; assert!( entries(now, Version::WRITTEN) == entries(before, Version::Sha256), "the layout before holds other entries" ); } pub fn reads(accounts: usize, records: usize) { assert!(records > LISTED, "more records than a listing"); let dir = scratch("reads"); std::thread::scope(|s| { s.spawn(|| { stores::build_trees( &dir, accounts, records, &[Pack::Snappy, Pack::None], &[fj::Pack::Lz4, fj::Pack::None], ) }); stores::build(&dir, accounts, records); }); let share = share(); let as_the_server = format!( "a table for each account, as the server reads one, at most {share} of their files open" ); let stores = [ ("tables", as_the_server.as_str(), ""), ( "dbs", "SQLite for each account", "4 KiB pages | none | held open", ), ("db", "one SQLite for all", "4 KiB pages | none | held open"), ( "rocks", "one RocksDB for all, its blocks compressed (Snappy)", "4 KiB before Snappy | XXH3 | held open", ), ( "rocks-plain", "one RocksDB for all, its blocks as they are", "4 KiB | XXH3 | held open", ), ( "fjall", "one fjall for all, its blocks compressed below the second level (LZ4)", "4 KiB before LZ4 | XXH3 | held open", ), ( "fjall-plain", "one fjall for all, its blocks as they are", "4 KiB | XXH3 | held open", ), ]; let figures: Vec<[HashMap; 3]> = stores .iter() .map(|(store, _, _)| { ["get", "threads", "list"].map(|test| measure(&dir, store, test, accounts, records)) }) .collect(); println!( "{accounts} accounts of {records} realistic records each. Cold is a process that has read nothing, its accounts open, the store's files read through just before, so in the system's cache if it has the memory to keep them; warm is the same reads again, everything they touch held by the process.\n" ); println!( "| store | a record by path, cold: p50 µs | p99 µs | warm: p50 µs | p99 µs | on {THREADS} threads, cold: per s | warm: per s |" ); println!("|---|--:|--:|--:|--:|--:|--:|"); for ((_, name, _), [get, threads, _]) in stores.iter().zip(&figures) { println!( "| {name} | {:.1} | {:.1} | {:.1} | {:.1} | {:.0} | {:.0} |", get["cold_p50"], get["cold_p99"], get["warm_p50"], get["warm_p99"], threads["cold"], threads["warm"] ); } println!( "\n| store | a listing of {LISTED} records, cold: p50 µs | p99 µs | warm: p50 µs | p99 µs | first read of an account not yet open: p50 µs | p99 µs |" ); println!("|---|--:|--:|--:|--:|--:|--:|"); for ((_, name, _), [get, _, list]) in stores.iter().zip(&figures) { println!( "| {name} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} |", list["cold_p50"], list["cold_p99"], list["warm_p50"], list["warm_p99"], get["first_p50"], get["first_p99"] ); } println!( "\n| store | resident before any is open, MiB | with {accounts} open, a record read from each, MiB | KiB an account | after {LOOKUPS} more records, twice, MiB | MiB on disk | KiB an account |" ); println!("|---|--:|--:|--:|--:|--:|--:|"); for ((store, name, _), [get, _, _]) in stores.iter().zip(&figures) { let disk = if store.starts_with("fjall") { fj::disk(&dir.join(store)) } else { dir_bytes(&dir.join(store)) }; println!( "| {name} | {:.1} | {:.1} | {:.1} | {:.1} | {:.0} | {:.0} |", get["rss_before"] / 1024.0, get["rss_open"] / 1024.0, (get["rss_open"] - get["rss_before"]) / accounts as f64, get["rss_after"] / 1024.0, mib(disk), disk as f64 / 1024.0 / accounts as f64, ); } // The tables' entries again in the layout before, and each other way // of reading a table. std::fs::create_dir_all(dir.join("tables16")).expect("a directory"); par(accounts, cores(), |a| { let table = std::fs::read(stores::table_path(&dir, a)).expect("a table"); std::fs::write( dir.join(format!("tables16/{a}.table")), tables::reblocked(&table, 16 << 10), ) .expect("a table"); }); same_entries(&stores::table_path(&dir, 0), &dir.join("tables16/0.table")); let blocks = format!("{} KiB", BLOCK >> 10); // The check of the tables as they are written, and of the layout before. let own = tables::held(Version::WRITTEN).name(); let before = tables::held(Version::Sha256).name(); println!( "\nThe tables read other ways. By `pds_store::table::Table`: as the server reads one (as above), its block cache holding at most {share} of the tables' files open, the server's share of the file descriptors this process may have, the least recently read closed to make room and opened again when next read; and with every file held open, whatever the share. By this benchmark's reader, which holds every file open, keeps the blocks it reads in a map for each table where `Table` has one cache for all, has no bloom filter, and computes the check named over each block it reads: {own} is the one these files hold, and is compared with it; each other is compared with nothing. And by that reader from the layout before, a table of the version before: the same entries in blocks of 16 KiB, each block's last key whole in the index, its check {before}, which that file holds and is compared with, the file opened for each block read. The first read of an account is its table opened and a record read.\n" ); println!( "| read by | blocks | check | file | a record by path, cold: p50 µs | p99 µs | a listing of {LISTED}, cold: p50 µs | p99 µs | first read of an account: p50 µs | p99 µs | on {THREADS} threads, cold: per s | warm: per s |" ); println!("|---|---|---|---|--:|--:|--:|--:|--:|--:|--:|--:|"); let way = |cells: &str, [get, threads, list]: &[HashMap; 3]| { println!( "| {cells} | {:.1} | {:.1} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} | {:.0} |", get["cold_p50"], get["cold_p99"], list["cold_p50"], list["cold_p99"], get["first_p50"], get["first_p99"], threads["cold"], threads["warm"] ); }; way( &format!("`Table`, as the server's | {blocks} | {own} | at most {share} held open"), &figures[0], ); let mut ways = vec![( "tables+open".to_string(), format!("`Table`, every file open | {blocks} | {own} | all held open"), )]; ways.extend(Check::ALL.map(|c| { ( format!("tables:{}", c.name()), format!("this benchmark's | {blocks} | {} | all held open", c.name()), ) })); ways.push(( format!("tables16:{before}+each"), format!("this benchmark's, the layout before | 16 KiB, last keys whole | {before} | opened for each block"), )); for (name, cells) in ways { way( &cells, &["get", "threads", "list"].map(|test| measure(&dir, &name, test, accounts, records)), ); } for ((_, name, cells), figures) in stores.iter().zip(&figures).skip(1) { way(&format!("{name} | {cells}"), figures); } let _ = std::fs::remove_dir_all(&dir); }