~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
315 lines · 9.6 KB Raw History
//! A commit's frame, three ways: the server's, JSON with holes, its bytes
//! as they are after the JSON text; a JSON array with every byte string in
//! hex; and a binary one of the same content; and a reader of a log of the
//! binary ones.
//!
//! A binary frame's payload is its fields in the order the JSON has them,
//! every length and number a varint, every string and byte string its
//! length then its bytes:
//!
//! - `1`, one byte: a commit and its event;
//! - the rev, and the commit's block;
//! - the records put, a count, then each one's path, CID and bytes;
//! - the paths deleted, a count, then each;
//! - the tree's nodes made, a count, then each one's CID and bytes;
//! - the nodes taken out, a count, then each one's CID;
//! - the records whose count of paths changed, a count, then each one's
//!   CID and count;
//! - the counts of records and of blocks;
//! - the event's seq, its time, and its frame.

use crate::check::Check;
use crate::data::Commit;
use pds_store::HEADER;
use serde_json::{Value as Json, json};
use std::io::{self, Read, Write};

/// How a commit's frame is written.
#[derive(Clone, Copy, PartialEq)]
pub enum Encoding {
    /// The server's: JSON with holes (`holes`).
    Holes,
    /// JSON, bytes as hex (`entries`).
    Json,
    Binary,
}

impl Encoding {
    /// The frame of commit `c`, header and all, of an account whose commit
    /// before made the nodes `gone`.
    pub fn frame(self, c: &Commit, gone: &[Vec<u8>]) -> Vec<u8> {
        match self {
            Encoding::Holes => holes(c, gone),
            Encoding::Json => pds_store::frame(&entries(c, gone)),
            Encoding::Binary => binary(c, gone),
        }
    }
}

/// When a commit's event was made, as its entry says.
pub fn time(c: &Commit) -> u64 {
    1_700_000_000_000 + c.op
}

/// A commit's entry and its event's, as JSON, bytes as hex: a frame of
/// `pds_store`'s first kind, its payload a JSON array.
pub fn entries(c: &Commit, gone: &[Vec<u8>]) -> Vec<Json> {
    vec![
        json!({
            "t": "commit", "rev": c.rev, "head": hex::encode(&c.blocks[0].1),
            "puts": [[c.path, hex::encode(&c.blocks[1].0), hex::encode(&c.blocks[1].1)]],
            "dels": [],
            "tree": c.blocks[2..].iter().map(|(cid, b)| json!([hex::encode(cid), hex::encode(b)])).collect::<Vec<_>>(),
            "gone": gone.iter().map(hex::encode).collect::<Vec<_>>(),
            "refs": [], "counts": [c.op, c.op * 2],
        }),
        json!({ "t": "event", "seq": c.op, "time": time(c), "frame": hex::encode(&c.event) }),
    ]
}

/// The frame the server writes of a commit (`pds-server`'s `log`,
/// `commit_frame`): the same entries, its fields in the order the server
/// writes them, as JSON with holes (`pds_store::frame_with_holes`): the
/// commit's block, the tree's nodes and the event's frame are holes, their
/// bytes as they are after the text; CIDs are hex, and so is the record,
/// which the server writes as its JSON.
pub fn holes(c: &Commit, gone: &[Vec<u8>]) -> Vec<u8> {
    let text = 2 * (c.blocks[1].0.len() + c.blocks[1].1.len())
        + (c.blocks.len() + gone.len()) * 80
        + c.path.len()
        + 256;
    pds_store::frame_with_holes(text, |w| {
        let put = |text: &[u8]| {
            let mut out = w;
            out.write_all(text).expect("a frame")
        };
        let json = |v: Json| put(&serde_json::to_vec(&v).expect("a frame"));
        put(br#"[{"t":"commit","rev":"#);
        json(json!(c.rev));
        put(br#","head":"""#);
        w.hole(&c.blocks[0].1);
        put(br#","puts":"#);
        json(json!([[
            c.path,
            hex::encode(&c.blocks[1].1),
            hex::encode(&c.blocks[1].0)
        ]]));
        put(br#","dels":[],"tree":["#);
        for (i, (cid, b)) in c.blocks[2..].iter().enumerate() {
            put(if i == 0 { b"[" } else { b",[" });
            json(json!(hex::encode(cid)));
            put(br#","""#);
            w.hole(b);
            put(b"]");
        }
        put(br#"],"gone":"#);
        json(json!(gone.iter().map(hex::encode).collect::<Vec<_>>()));
        put(br#","refs":[],"counts":"#);
        json(json!([c.op, c.op * 2]));
        put(br#"},{"t":"event","seq":"#);
        json(json!(c.op));
        put(br#","time":"#);
        json(json!(time(c)));
        put(br#","frame":"""#);
        w.hole(&c.event);
        put(b"}]");
    })
}

const COMMIT: u8 = 1;

fn varint(out: &mut Vec<u8>, mut n: u64) {
    while n >= 0x80 {
        out.push(n as u8 | 0x80);
        n >>= 7;
    }
    out.push(n as u8);
}

fn bytes(out: &mut Vec<u8>, b: &[u8]) {
    varint(out, b.len() as u64);
    out.extend_from_slice(b);
}

/// The same content as `entries`, as a binary frame.
pub fn binary(c: &Commit, gone: &[Vec<u8>]) -> Vec<u8> {
    let size = c
        .blocks
        .iter()
        .map(|(k, b)| k.len() + b.len() + 6)
        .sum::<usize>()
        + gone.iter().map(|g| g.len() + 1).sum::<usize>()
        + c.event.len()
        + c.rev.len()
        + c.path.len()
        + 48;
    pds_store::frame_of(size, |out| {
        out.push(COMMIT);
        bytes(out, c.rev.as_bytes());
        bytes(out, &c.blocks[0].1);
        varint(out, 1);
        bytes(out, c.path.as_bytes());
        bytes(out, &c.blocks[1].0);
        bytes(out, &c.blocks[1].1);
        varint(out, 0);
        varint(out, c.blocks.len() as u64 - 2);
        for (cid, b) in &c.blocks[2..] {
            bytes(out, cid);
            bytes(out, b);
        }
        varint(out, gone.len() as u64);
        for cid in gone {
            bytes(out, cid);
        }
        varint(out, 0);
        varint(out, c.op);
        varint(out, c.op * 2);
        varint(out, c.op);
        varint(out, time(c));
        bytes(out, &c.event);
    })
}

/// A binary payload, read field by field.
struct Walk<'a> {
    b: &'a [u8],
    at: usize,
}

impl<'a> Walk<'a> {
    fn varint(&mut self) -> Option<u64> {
        let mut n = 0u64;
        for shift in (0..64).step_by(7) {
            let b = *self.b.get(self.at)?;
            self.at += 1;
            n |= u64::from(b & 0x7f) << shift;
            if b & 0x80 == 0 {
                return Some(n);
            }
        }
        None
    }

    fn bytes(&mut self) -> Option<&'a [u8]> {
        let n = self.varint()? as usize;
        let b = self.b.get(self.at..self.at.checked_add(n)?)?;
        self.at += n;
        Some(b)
    }

    fn text(&mut self) -> Option<&'a str> {
        std::str::from_utf8(self.bytes()?).ok()
    }
}

/// A commit's event, as its frame holds it.
pub struct Event<'a> {
    pub seq: u64,
    pub frame: &'a [u8],
}

/// Walk every field of a binary payload, as a replay does, each string
/// checked to be one: its event, or `None` if the payload is not a
/// commit's, whole.
pub fn walk(payload: &[u8]) -> Option<Event<'_>> {
    let mut w = Walk { b: payload, at: 0 };
    if *payload.first()? != COMMIT {
        return None;
    }
    w.at = 1;
    w.text()?;
    w.bytes()?;
    for _ in 0..w.varint()? {
        w.text()?;
        w.bytes()?;
        w.bytes()?;
    }
    for _ in 0..w.varint()? {
        w.text()?;
    }
    for _ in 0..w.varint()? {
        w.bytes()?;
        w.bytes()?;
    }
    for _ in 0..w.varint()? {
        w.bytes()?;
    }
    for _ in 0..w.varint()? {
        w.bytes()?;
        w.varint()?;
    }
    w.varint()?;
    w.varint()?;
    let seq = w.varint()?;
    w.varint()?;
    let frame = w.bytes()?;
    (w.at == payload.len()).then_some(Event { seq, frame })
}

/// The frames of a log, one at a time, over the header `pds-store` gives
/// every frame (`[len u32 LE][8-byte check][payload]`), through a buffer
/// of the size `pds_store::scan_file` reads with.
pub struct Frames<R> {
    r: io::BufReader<R>,
    /// The bytes left to read, and those of the whole frames read.
    left: u64,
    pub pos: u64,
    payload: Vec<u8>,
    check: Option<Check>,
}

impl<R: Read> Frames<R> {
    /// The frames `r` holds in its next `left` bytes, each one's check
    /// computed, as `check`, if there is one.
    pub fn new(r: R, left: u64, check: Option<Check>) -> Frames<R> {
        Frames {
            r: io::BufReader::with_capacity(1 << 16, r),
            left,
            pos: 0,
            payload: Vec::new(),
            check,
        }
    }

    /// The next frame's payload: `None` at the first that is cut short or
    /// fails its check, where a log ends.
    pub fn next_payload(&mut self) -> Option<&[u8]> {
        let mut head = [0u8; HEADER];
        if self.left < HEADER as u64 || self.r.read_exact(&mut head).is_err() {
            return None;
        }
        let len = u64::from(u32::from_le_bytes(head[..4].try_into().expect("4 bytes")));
        if self.left - (HEADER as u64) < len {
            return None;
        }
        self.payload.resize(len as usize, 0);
        if self.r.read_exact(&mut self.payload).is_err()
            || self
                .check
                .is_some_and(|c| head[4..] != c.of_frame(&self.payload))
        {
            return None;
        }
        self.left -= HEADER as u64 + len;
        self.pos += HEADER as u64 + len;
        Some(&self.payload)
    }
}

/// The commits in the binary log at `path`, each frame checked and walked.
pub fn count(path: &std::path::Path) -> io::Result<u64> {
    let file = std::fs::File::open(path)?;
    let size = file.metadata()?.len();
    let mut frames = Frames::new(file, size, Some(Check::Sha256));
    let mut n = 0;
    while let Some(p) = frames.next_payload() {
        walk(p).ok_or_else(|| {
            io::Error::new(
                io::ErrorKind::InvalidData,
                format!("{}: a frame that is no commit's", path.display()),
            )
        })?;
        n += 1;
    }
    Ok(n)
}