~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
137 lines · 5.0 KB Raw History
//! fjall beside RocksDB, set up the same way: one database with three
//! keyspaces, as fjall calls its column families, `block`, `record` and
//! `event`, under one journal, so that a commit is one batch across the
//! three. Its keys and values are RocksDB's (`rocks`), byte for byte.
//!
//! Blocks are 4 KiB before compression, each with a check, XXH3, computed
//! when it is read from its file, and every keyspace has bloom filters, 10
//! bits a key but in its first level: fjall's own of all three. As it
//! comes it compresses with LZ4 from the third level down and nothing
//! above; it has no other compression.

use crate::data::Commit;
use crate::rocks::{event, key, record_value};
use crate::stores::block_value;
use fjall::config::CompressionPolicy;
use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, PersistMode};
use std::path::Path;
use std::time::Duration;

/// The bytes a block cache may hold where a part gives it no other size:
/// what fjall gives one it makes for itself.
pub const CACHE: u64 = 32 << 20;

/// How the blocks of a database's files are compressed: with LZ4 below
/// its second level, as fjall does when it is not told, or not at all.
#[derive(Clone, Copy, PartialEq)]
pub enum Pack {
    Lz4,
    None,
}

/// Both ways.
pub const PACKS: [Pack; 2] = [Pack::Lz4, Pack::None];

impl Pack {
    pub fn name(self) -> &'static str {
        match self {
            Pack::Lz4 => "LZ4 below the second level",
            Pack::None => "no compression",
        }
    }
}

/// A database and its three keyspaces.
pub struct Store {
    pub db: Database,
    pub block: Keyspace,
    pub record: Keyspace,
    pub event: Keyspace,
}

impl Store {
    /// The database at `path`, made if it is not there: its block cache
    /// may hold `cache` bytes. `small` is a database for one account: a
    /// keyspace's memtable holds 1 MiB before it is written out, where it
    /// has fjall's own 64 MiB otherwise, and one thread writes its files
    /// and compacts them, where it has up to four.
    pub fn open(path: &Path, cache: u64, pack: Pack, small: bool) -> Store {
        let buffer = small.then_some(1 << 20);
        let mut builder = Database::builder(path).cache_size(cache);
        if small {
            builder = builder.worker_threads(1);
        }
        let db = builder.open().expect("a fjall database");
        let keyspace = |name: &str| {
            db.keyspace(name, || {
                let mut options = KeyspaceCreateOptions::default();
                if pack == Pack::None {
                    options = options.data_block_compression_policy(CompressionPolicy::all(
                        CompressionType::None,
                    ));
                }
                match buffer {
                    Some(bytes) => options.max_memtable_size(bytes),
                    None => options,
                }
            })
            .expect("a keyspace")
        };
        Store {
            block: keyspace("block"),
            record: keyspace("record"),
            event: keyspace("event"),
            db,
        }
    }

    /// What a commit puts in the database, as one batch, as
    /// `rocks::commit`: made durable as `mode` says before it is answered,
    /// or left in the journal's buffer if it says nothing.
    pub fn commit(&self, account: usize, c: &Commit, last: &[Vec<u8>], mode: Option<PersistMode>) {
        let mut batch = self.db.batch().durability(mode);
        for (cid, b) in &c.blocks {
            batch.insert(&self.block, key(account, cid), block_value(&c.rev, b));
        }
        for cid in last {
            batch.remove(&self.block, key(account, cid));
        }
        batch.insert(
            &self.record,
            key(account, c.path.as_bytes()),
            record_value(&c.blocks[1].0, &c.rev),
        );
        batch.insert(
            &self.event,
            c.op.to_be_bytes(),
            event(account, c.op, &c.event),
        );
        batch.commit().expect("a commit");
    }

    /// Write what the database holds in memory to its files, and wait
    /// until it has no compaction left to do of its own accord.
    pub fn settle(&self) {
        for keyspace in [&self.block, &self.record, &self.event] {
            keyspace.rotate_memtable_and_wait().expect("a flush");
        }
        // It says how many compactions are going on and how many are done,
        // not whether another is due: idle is none of either for a while.
        let (mut done, mut idle) = (self.db.compactions_completed(), 0);
        while idle < 50 {
            std::thread::sleep(Duration::from_millis(10));
            let now = self.db.compactions_completed();
            if self.db.active_compactions() == 0 && now == done {
                idle += 1;
            } else {
                (done, idle) = (now, 0);
            }
        }
    }
}

/// The bytes of the files of the database at `path`, its journal left
/// out: that is 64 MiB allocated ahead, whatever it holds.
pub fn disk(path: &Path) -> u64 {
    crate::util::dir_bytes(&path.join("keyspaces"))
}