//! A commit's frame, three ways: the server's, JSON with holes, its bytes //! as they are after the JSON text; a JSON array with every byte string in //! hex; and a binary one of the same content; and a reader of a log of the //! binary ones. //! //! A binary frame's payload is its fields in the order the JSON has them, //! every length and number a varint, every string and byte string its //! length then its bytes: //! //! - `1`, one byte: a commit and its event; //! - the rev, and the commit's block; //! - the records put, a count, then each one's path, CID and bytes; //! - the paths deleted, a count, then each; //! - the tree's nodes made, a count, then each one's CID and bytes; //! - the nodes taken out, a count, then each one's CID; //! - the records whose count of paths changed, a count, then each one's //! CID and count; //! - the counts of records and of blocks; //! - the event's seq, its time, and its frame. use crate::check::Check; use crate::data::Commit; use pds_store::HEADER; use serde_json::{Value as Json, json}; use std::io::{self, Read, Write}; /// How a commit's frame is written. #[derive(Clone, Copy, PartialEq)] pub enum Encoding { /// The server's: JSON with holes (`holes`). Holes, /// JSON, bytes as hex (`entries`). Json, Binary, } impl Encoding { /// The frame of commit `c`, header and all, of an account whose commit /// before made the nodes `gone`. pub fn frame(self, c: &Commit, gone: &[Vec]) -> Vec { match self { Encoding::Holes => holes(c, gone), Encoding::Json => pds_store::frame(&entries(c, gone)), Encoding::Binary => binary(c, gone), } } } /// When a commit's event was made, as its entry says. pub fn time(c: &Commit) -> u64 { 1_700_000_000_000 + c.op } /// A commit's entry and its event's, as JSON, bytes as hex: a frame of /// `pds_store`'s first kind, its payload a JSON array. pub fn entries(c: &Commit, gone: &[Vec]) -> Vec { vec![ json!({ "t": "commit", "rev": c.rev, "head": hex::encode(&c.blocks[0].1), "puts": [[c.path, hex::encode(&c.blocks[1].0), hex::encode(&c.blocks[1].1)]], "dels": [], "tree": c.blocks[2..].iter().map(|(cid, b)| json!([hex::encode(cid), hex::encode(b)])).collect::>(), "gone": gone.iter().map(hex::encode).collect::>(), "refs": [], "counts": [c.op, c.op * 2], }), json!({ "t": "event", "seq": c.op, "time": time(c), "frame": hex::encode(&c.event) }), ] } /// The frame the server writes of a commit (`pds-server`'s `log`, /// `commit_frame`): the same entries, its fields in the order the server /// writes them, as JSON with holes (`pds_store::frame_with_holes`): the /// commit's block, the tree's nodes and the event's frame are holes, their /// bytes as they are after the text; CIDs are hex, and so is the record, /// which the server writes as its JSON. pub fn holes(c: &Commit, gone: &[Vec]) -> Vec { let text = 2 * (c.blocks[1].0.len() + c.blocks[1].1.len()) + (c.blocks.len() + gone.len()) * 80 + c.path.len() + 256; pds_store::frame_with_holes(text, |w| { let put = |text: &[u8]| { let mut out = w; out.write_all(text).expect("a frame") }; let json = |v: Json| put(&serde_json::to_vec(&v).expect("a frame")); put(br#"[{"t":"commit","rev":"#); json(json!(c.rev)); put(br#","head":"""#); w.hole(&c.blocks[0].1); put(br#","puts":"#); json(json!([[ c.path, hex::encode(&c.blocks[1].1), hex::encode(&c.blocks[1].0) ]])); put(br#","dels":[],"tree":["#); for (i, (cid, b)) in c.blocks[2..].iter().enumerate() { put(if i == 0 { b"[" } else { b",[" }); json(json!(hex::encode(cid))); put(br#","""#); w.hole(b); put(b"]"); } put(br#"],"gone":"#); json(json!(gone.iter().map(hex::encode).collect::>())); put(br#","refs":[],"counts":"#); json(json!([c.op, c.op * 2])); put(br#"},{"t":"event","seq":"#); json(json!(c.op)); put(br#","time":"#); json(json!(time(c))); put(br#","frame":"""#); w.hole(&c.event); put(b"}]"); }) } const COMMIT: u8 = 1; fn varint(out: &mut Vec, mut n: u64) { while n >= 0x80 { out.push(n as u8 | 0x80); n >>= 7; } out.push(n as u8); } fn bytes(out: &mut Vec, b: &[u8]) { varint(out, b.len() as u64); out.extend_from_slice(b); } /// The same content as `entries`, as a binary frame. pub fn binary(c: &Commit, gone: &[Vec]) -> Vec { let size = c .blocks .iter() .map(|(k, b)| k.len() + b.len() + 6) .sum::() + gone.iter().map(|g| g.len() + 1).sum::() + c.event.len() + c.rev.len() + c.path.len() + 48; pds_store::frame_of(size, |out| { out.push(COMMIT); bytes(out, c.rev.as_bytes()); bytes(out, &c.blocks[0].1); varint(out, 1); bytes(out, c.path.as_bytes()); bytes(out, &c.blocks[1].0); bytes(out, &c.blocks[1].1); varint(out, 0); varint(out, c.blocks.len() as u64 - 2); for (cid, b) in &c.blocks[2..] { bytes(out, cid); bytes(out, b); } varint(out, gone.len() as u64); for cid in gone { bytes(out, cid); } varint(out, 0); varint(out, c.op); varint(out, c.op * 2); varint(out, c.op); varint(out, time(c)); bytes(out, &c.event); }) } /// A binary payload, read field by field. struct Walk<'a> { b: &'a [u8], at: usize, } impl<'a> Walk<'a> { fn varint(&mut self) -> Option { let mut n = 0u64; for shift in (0..64).step_by(7) { let b = *self.b.get(self.at)?; self.at += 1; n |= u64::from(b & 0x7f) << shift; if b & 0x80 == 0 { return Some(n); } } None } fn bytes(&mut self) -> Option<&'a [u8]> { let n = self.varint()? as usize; let b = self.b.get(self.at..self.at.checked_add(n)?)?; self.at += n; Some(b) } fn text(&mut self) -> Option<&'a str> { std::str::from_utf8(self.bytes()?).ok() } } /// A commit's event, as its frame holds it. pub struct Event<'a> { pub seq: u64, pub frame: &'a [u8], } /// Walk every field of a binary payload, as a replay does, each string /// checked to be one: its event, or `None` if the payload is not a /// commit's, whole. pub fn walk(payload: &[u8]) -> Option> { let mut w = Walk { b: payload, at: 0 }; if *payload.first()? != COMMIT { return None; } w.at = 1; w.text()?; w.bytes()?; for _ in 0..w.varint()? { w.text()?; w.bytes()?; w.bytes()?; } for _ in 0..w.varint()? { w.text()?; } for _ in 0..w.varint()? { w.bytes()?; w.bytes()?; } for _ in 0..w.varint()? { w.bytes()?; } for _ in 0..w.varint()? { w.bytes()?; w.varint()?; } w.varint()?; w.varint()?; let seq = w.varint()?; w.varint()?; let frame = w.bytes()?; (w.at == payload.len()).then_some(Event { seq, frame }) } /// The frames of a log, one at a time, over the header `pds-store` gives /// every frame (`[len u32 LE][8-byte check][payload]`), through a buffer /// of the size `pds_store::scan_file` reads with. pub struct Frames { r: io::BufReader, /// The bytes left to read, and those of the whole frames read. left: u64, pub pos: u64, payload: Vec, check: Option, } impl Frames { /// The frames `r` holds in its next `left` bytes, each one's check /// computed, as `check`, if there is one. pub fn new(r: R, left: u64, check: Option) -> Frames { Frames { r: io::BufReader::with_capacity(1 << 16, r), left, pos: 0, payload: Vec::new(), check, } } /// The next frame's payload: `None` at the first that is cut short or /// fails its check, where a log ends. pub fn next_payload(&mut self) -> Option<&[u8]> { let mut head = [0u8; HEADER]; if self.left < HEADER as u64 || self.r.read_exact(&mut head).is_err() { return None; } let len = u64::from(u32::from_le_bytes(head[..4].try_into().expect("4 bytes"))); if self.left - (HEADER as u64) < len { return None; } self.payload.resize(len as usize, 0); if self.r.read_exact(&mut self.payload).is_err() || self .check .is_some_and(|c| head[4..] != c.of_frame(&self.payload)) { return None; } self.left -= HEADER as u64 + len; self.pos += HEADER as u64 + len; Some(&self.payload) } } /// The commits in the binary log at `path`, each frame checked and walked. pub fn count(path: &std::path::Path) -> io::Result { let file = std::fs::File::open(path)?; let size = file.metadata()?.len(); let mut frames = Frames::new(file, size, Some(Check::Sha256)); let mut n = 0; while let Some(p) = frames.next_payload() { walk(p).ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, format!("{}: a frame that is no commit's", path.display()), ) })?; n += 1; } Ok(n) }