//! fjall beside RocksDB, set up the same way: one database with three //! keyspaces, as fjall calls its column families, `block`, `record` and //! `event`, under one journal, so that a commit is one batch across the //! three. Its keys and values are RocksDB's (`rocks`), byte for byte. //! //! Blocks are 4 KiB before compression, each with a check, XXH3, computed //! when it is read from its file, and every keyspace has bloom filters, 10 //! bits a key but in its first level: fjall's own of all three. As it //! comes it compresses with LZ4 from the third level down and nothing //! above; it has no other compression. use crate::data::Commit; use crate::rocks::{event, key, record_value}; use crate::stores::block_value; use fjall::config::CompressionPolicy; use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, PersistMode}; use std::path::Path; use std::time::Duration; /// The bytes a block cache may hold where a part gives it no other size: /// what fjall gives one it makes for itself. pub const CACHE: u64 = 32 << 20; /// How the blocks of a database's files are compressed: with LZ4 below /// its second level, as fjall does when it is not told, or not at all. #[derive(Clone, Copy, PartialEq)] pub enum Pack { Lz4, None, } /// Both ways. pub const PACKS: [Pack; 2] = [Pack::Lz4, Pack::None]; impl Pack { pub fn name(self) -> &'static str { match self { Pack::Lz4 => "LZ4 below the second level", Pack::None => "no compression", } } } /// A database and its three keyspaces. pub struct Store { pub db: Database, pub block: Keyspace, pub record: Keyspace, pub event: Keyspace, } impl Store { /// The database at `path`, made if it is not there: its block cache /// may hold `cache` bytes. `small` is a database for one account: a /// keyspace's memtable holds 1 MiB before it is written out, where it /// has fjall's own 64 MiB otherwise, and one thread writes its files /// and compacts them, where it has up to four. pub fn open(path: &Path, cache: u64, pack: Pack, small: bool) -> Store { let buffer = small.then_some(1 << 20); let mut builder = Database::builder(path).cache_size(cache); if small { builder = builder.worker_threads(1); } let db = builder.open().expect("a fjall database"); let keyspace = |name: &str| { db.keyspace(name, || { let mut options = KeyspaceCreateOptions::default(); if pack == Pack::None { options = options.data_block_compression_policy(CompressionPolicy::all( CompressionType::None, )); } match buffer { Some(bytes) => options.max_memtable_size(bytes), None => options, } }) .expect("a keyspace") }; Store { block: keyspace("block"), record: keyspace("record"), event: keyspace("event"), db, } } /// What a commit puts in the database, as one batch, as /// `rocks::commit`: made durable as `mode` says before it is answered, /// or left in the journal's buffer if it says nothing. pub fn commit(&self, account: usize, c: &Commit, last: &[Vec], mode: Option) { let mut batch = self.db.batch().durability(mode); for (cid, b) in &c.blocks { batch.insert(&self.block, key(account, cid), block_value(&c.rev, b)); } for cid in last { batch.remove(&self.block, key(account, cid)); } batch.insert( &self.record, key(account, c.path.as_bytes()), record_value(&c.blocks[1].0, &c.rev), ); batch.insert( &self.event, c.op.to_be_bytes(), event(account, c.op, &c.event), ); batch.commit().expect("a commit"); } /// Write what the database holds in memory to its files, and wait /// until it has no compaction left to do of its own accord. pub fn settle(&self) { for keyspace in [&self.block, &self.record, &self.event] { keyspace.rotate_memtable_and_wait().expect("a flush"); } // It says how many compactions are going on and how many are done, // not whether another is due: idle is none of either for a while. let (mut done, mut idle) = (self.db.compactions_completed(), 0); while idle < 50 { std::thread::sleep(Duration::from_millis(10)); let now = self.db.compactions_completed(); if self.db.active_compactions() == 0 && now == done { idle += 1; } else { (done, idle) = (now, 0); } } } } /// The bytes of the files of the database at `path`, its journal left /// out: that is 64 MiB allocated ahead, whatever it holds. pub fn disk(path: &Path) -> u64 { crate::util::dir_bytes(&path.join("keyspaces")) }