chadtmiller.com/monolith
An atproto PDS
0
0
//! What accounts hold after their commits (`data::State`), kept four ways
//! for the read and size parts: a table for each account, as the server's
//! checkpoint of one has its blocks and paths; a SQLite database for
//! each; one SQLite database for all; and one RocksDB for all. Each holds
//! the same: every record's block and every tree node's by CID, and every
//! record's CID by its path. None holds an event.
use crate::data::{self, Block, Record, SEED, State};
use crate::fj;
use crate::rocks::{self, Pack, Setup};
use crate::util::{cores, par};
use crate::write::SCHEMA;
use pds_store::table::TableWriter;
use rocksdb::{Cache, DB, WaitForCompactOptions, WriteBatch};
use rusqlite::{Connection, params};
use std::fs::File;
use std::io::BufWriter;
use std::path::{Path, PathBuf};
/// A table's key: its section, `B` for a block or `P` for a path, and the
/// CID or path.
pub fn key(section: u8, k: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(1 + k.len());
out.push(section);
out.extend_from_slice(k);
out
}
/// A `B` value, as the server writes one: `[refs u32 LE][rev length
/// u8][rev][bytes]`.
pub fn block_value(rev: &str, bytes: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(5 + rev.len() + bytes.len());
out.extend_from_slice(&1u32.to_le_bytes());
out.push(rev.len() as u8);
out.extend_from_slice(rev.as_bytes());
out.extend_from_slice(bytes);
out
}
/// The bytes of the block a `B` value holds.
pub fn block_bytes(value: &[u8]) -> &[u8] {
&value[5 + usize::from(value[4])..]
}
/// The CID a `P` value holds: `[CID length u8][CID][rev]`.
pub fn path_cid(value: &[u8]) -> &[u8] {
&value[1..1 + usize::from(value[0])]
}
pub fn table_path(dir: &Path, account: usize) -> PathBuf {
dir.join(format!("tables/{account}.table"))
}
pub fn dbs_path(dir: &Path, account: usize) -> PathBuf {
dir.join(format!("dbs/{account}.sqlite"))
}
pub fn db_path(dir: &Path) -> PathBuf {
dir.join("db/store.sqlite")
}
/// Write an account's table: its blocks in order of CID, then its paths.
pub fn write_table(path: &Path, s: &State) {
let mut blocks: Vec<(&[u8], Vec<u8>)> = s
.records
.iter()
.map(|r| (r.cid.as_slice(), block_value(&r.rev, &r.bytes)))
.chain(
s.nodes()
.map(|n| (n.cid.as_slice(), block_value(&n.rev, &n.bytes))),
)
.collect();
blocks.sort();
let file = File::create(path).expect("a table");
let mut w = TableWriter::new(
BufWriter::new(file),
(blocks.len() + s.records.len()) as u64,
);
for (cid, value) in &blocks {
w.add(&key(b'B', cid), value).expect("a block");
}
for r in &s.records {
let mut value = Vec::with_capacity(1 + r.cid.len() + r.rev.len());
value.push(r.cid.len() as u8);
value.extend_from_slice(&r.cid);
value.extend_from_slice(r.rev.as_bytes());
w.add(&key(b'P', r.path.as_bytes()), &value)
.expect("a path");
}
w.finish().expect("a table written");
}
/// A database to load or rewrite: in WAL mode as the write part's, but
/// unsynced, which changes how long the writes take and not what they
/// leave; with `cache` KiB of pages, or SQLite's own 2 MiB.
pub fn open_db(path: &Path, cache: Option<u64>) -> Connection {
let db = Connection::open(path).expect("a database");
db.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=OFF;")
.expect("pragmas");
if let Some(kib) = cache {
db.execute_batch(&format!("PRAGMA cache_size=-{kib};"))
.expect("a cache");
}
db
}
/// Fold a database's WAL into it and close it, leaving one file.
pub fn close_db(db: Connection) {
db.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
.expect("a checkpoint");
db.close().map_err(|(_, e)| e).expect("a database closed");
}
fn put_record(db: &Connection, account: usize, r: &Record) {
let a = account as i64;
db.prepare_cached("INSERT INTO block(account, cid, rev, bytes) VALUES (?1, ?2, ?3, ?4)")
.expect("a statement")
.execute(params![a, r.cid, r.rev, r.bytes])
.expect("a block");
db.prepare_cached(
"INSERT OR REPLACE INTO record(account, path, cid, rev) VALUES (?1, ?2, ?3, ?4)",
)
.expect("a statement")
.execute(params![a, r.path, r.cid, r.rev])
.expect("a record");
}
fn put_node(db: &Connection, account: usize, n: &Block) {
db.prepare_cached("INSERT INTO block(account, cid, rev, bytes) VALUES (?1, ?2, ?3, ?4)")
.expect("a statement")
.execute(params![account as i64, n.cid, n.rev, n.bytes])
.expect("a block");
}
/// Put another record in record `i`'s place, in `s` and in `db`, as a
/// commit does: one transaction that takes out the record's block and each
/// node's from it to the root, and puts in the new ones and the path's row.
pub fn rewrite(db: &Connection, account: usize, s: &mut State, i: usize, version: u64) {
let (gone, made) = s.rewrite(i, version);
db.execute_batch("BEGIN").expect("a transaction");
{
let mut out = db
.prepare_cached("DELETE FROM block WHERE account = ?1 AND cid = ?2")
.expect("a statement");
for cid in &gone {
out.execute(params![account as i64, cid])
.expect("a block taken out");
}
}
put_record(db, account, &s.records[i]);
for n in &made {
put_node(db, account, n);
}
db.execute_batch("COMMIT").expect("a commit");
}
/// The three stores of `accounts` accounts of `records` records each,
/// under `dir`: `tables/`, `dbs/` and `db/`. The databases get their rows
/// in the order commits would give them, the records as they are made (in
/// the one for all, the accounts' taking turns) and the tree after, and
/// are not vacuumed.
pub fn build(dir: &Path, accounts: usize, records: usize) {
for d in ["tables", "dbs", "db"] {
std::fs::create_dir_all(dir.join(d)).expect("a directory");
}
let state = |a: usize| State::new(SEED, a as u64, accounts as u64, records);
std::thread::scope(|s| {
s.spawn(|| {
let db = open_db(&db_path(dir), Some(1 << 20));
db.execute_batch(SCHEMA).expect("a schema");
db.execute_batch("BEGIN").expect("a transaction");
for i in 0..records {
for a in 0..accounts {
let op = (i * accounts + a + 1) as u64;
put_record(&db, a, &data::record(SEED, op, a as u64, 0));
}
}
for a in 0..accounts {
for n in state(a).nodes() {
put_node(&db, a, n);
}
}
db.execute_batch("COMMIT").expect("a commit");
close_db(db);
});
par(accounts, cores().saturating_sub(1), |a| {
let s = state(a);
write_table(&table_path(dir, a), &s);
let db = open_db(&dbs_path(dir, a), None);
db.execute_batch(SCHEMA).expect("a schema");
db.execute_batch("BEGIN").expect("a transaction");
for r in &s.records {
put_record(&db, a, r);
}
for n in s.nodes() {
put_node(&db, a, n);
}
db.execute_batch("COMMIT").expect("a commit");
close_db(db);
});
});
}
pub fn rocks_path(dir: &Path, pack: Pack) -> PathBuf {
dir.join(match pack {
Pack::Snappy => "rocks",
Pack::None => "rocks-plain",
Pack::Zstd => "rocks-zstd",
})
}
pub fn fjall_path(dir: &Path, pack: fj::Pack) -> PathBuf {
dir.join(match pack {
fj::Pack::Lz4 => "fjall",
fj::Pack::None => "fjall-plain",
})
}
/// The rows a RocksDB or a fjall database is given at once as it is loaded.
const BATCH: usize = 1000;
/// Give each of `dbs` and of `stores` what `accounts` accounts of
/// `records` records hold, in the order the one SQLite database for all
/// gets it: the records as they are made, the accounts' taking turns, and
/// the tree after. Written through each database's log, unsynced.
pub fn load(dbs: &[&DB], stores: &[&fj::Store], accounts: usize, records: usize) {
let mut batches: Vec<WriteBatch> = dbs.iter().map(|_| WriteBatch::default()).collect();
let mut others: Vec<_> = stores.iter().map(|s| s.db.batch()).collect();
let mut rows = 0;
let mut put = |family: &str, key: Vec<u8>, value: Vec<u8>, end: bool| {
let full = rows % BATCH == BATCH - 1 || end;
for (db, batch) in dbs.iter().zip(&mut batches) {
batch.put_cf(rocks::family(db, family), &key, &value);
if full {
db.write(std::mem::take(batch)).expect("a batch");
}
}
for (store, batch) in stores.iter().zip(&mut others) {
let keyspace = if family == rocks::BLOCK {
&store.block
} else {
&store.record
};
batch.insert(keyspace, key.as_slice(), value.as_slice());
if full {
let batch = std::mem::replace(batch, store.db.batch());
batch.commit().expect("a batch");
}
}
rows += 1;
};
for i in 0..records {
for a in 0..accounts {
let r = data::record(SEED, (i * accounts + a + 1) as u64, a as u64, 0);
let block = block_value(&r.rev, &r.bytes);
put(rocks::BLOCK, rocks::key(a, &r.cid), block, false);
let at = rocks::record_value(&r.cid, &r.rev);
put(rocks::RECORD, rocks::key(a, r.path.as_bytes()), at, false);
}
}
for a in 0..accounts {
let s = State::new(SEED, a as u64, accounts as u64, records);
let nodes: Vec<&Block> = s.nodes().collect();
for (i, n) in nodes.iter().enumerate() {
let end = a + 1 == accounts && i + 1 == nodes.len();
let block = block_value(&n.rev, &n.bytes);
put(rocks::BLOCK, rocks::key(a, &n.cid), block, end);
}
}
}
/// Write what a RocksDB holds in memory to its files, and wait until it
/// has no compaction left to do of its own accord: as a database that has
/// been let alone a while is.
pub fn settle(db: &DB) {
for family in [rocks::BLOCK, rocks::RECORD, rocks::EVENT] {
db.flush_cf(rocks::family(db, family)).expect("a flush");
}
db.wait_for_compact(&WaitForCompactOptions::default())
.expect("the compactions done");
}
/// One RocksDB for all at `rocks_path` for each of `packs`, and one fjall
/// database for all at `fjall_path` for each of `others`, loaded and
/// settled: the setups the RocksDB were opened from.
pub fn build_trees(
dir: &Path,
accounts: usize,
records: usize,
packs: &[Pack],
others: &[fj::Pack],
) -> Vec<Setup> {
let cache = Cache::new_lru_cache(rocks::CACHE);
let setups: Vec<Setup> = packs
.iter()
.map(|p| Setup::new(&cache, *p, "", true))
.collect();
let dbs: Vec<DB> = setups
.iter()
.zip(packs)
.map(|(s, p)| s.open(&rocks_path(dir, *p)))
.collect();
let stores: Vec<fj::Store> = others
.iter()
.map(|p| fj::Store::open(&fjall_path(dir, *p), fj::CACHE, *p, false))
.collect();
load(
&dbs.iter().collect::<Vec<_>>(),
&stores.iter().collect::<Vec<_>>(),
accounts,
records,
);
dbs.iter().for_each(settle);
stores.iter().for_each(fj::Store::settle);
setups
}