chadtmiller.com/monolith
An atproto PDS
0
0
//! A commit's frame, three ways: the server's, JSON with holes, its bytes
//! as they are after the JSON text; a JSON array with every byte string in
//! hex; and a binary one of the same content; and a reader of a log of the
//! binary ones.
//!
//! A binary frame's payload is its fields in the order the JSON has them,
//! every length and number a varint, every string and byte string its
//! length then its bytes:
//!
//! - `1`, one byte: a commit and its event;
//! - the rev, and the commit's block;
//! - the records put, a count, then each one's path, CID and bytes;
//! - the paths deleted, a count, then each;
//! - the tree's nodes made, a count, then each one's CID and bytes;
//! - the nodes taken out, a count, then each one's CID;
//! - the records whose count of paths changed, a count, then each one's
//! CID and count;
//! - the counts of records and of blocks;
//! - the event's seq, its time, and its frame.
use crate::check::Check;
use crate::data::Commit;
use pds_store::HEADER;
use serde_json::{Value as Json, json};
use std::io::{self, Read, Write};
/// How a commit's frame is written.
#[derive(Clone, Copy, PartialEq)]
pub enum Encoding {
/// The server's: JSON with holes (`holes`).
Holes,
/// JSON, bytes as hex (`entries`).
Json,
Binary,
}
impl Encoding {
/// The frame of commit `c`, header and all, of an account whose commit
/// before made the nodes `gone`.
pub fn frame(self, c: &Commit, gone: &[Vec<u8>]) -> Vec<u8> {
match self {
Encoding::Holes => holes(c, gone),
Encoding::Json => pds_store::frame(&entries(c, gone)),
Encoding::Binary => binary(c, gone),
}
}
}
/// When a commit's event was made, as its entry says.
pub fn time(c: &Commit) -> u64 {
1_700_000_000_000 + c.op
}
/// A commit's entry and its event's, as JSON, bytes as hex: a frame of
/// `pds_store`'s first kind, its payload a JSON array.
pub fn entries(c: &Commit, gone: &[Vec<u8>]) -> Vec<Json> {
vec![
json!({
"t": "commit", "rev": c.rev, "head": hex::encode(&c.blocks[0].1),
"puts": [[c.path, hex::encode(&c.blocks[1].0), hex::encode(&c.blocks[1].1)]],
"dels": [],
"tree": c.blocks[2..].iter().map(|(cid, b)| json!([hex::encode(cid), hex::encode(b)])).collect::<Vec<_>>(),
"gone": gone.iter().map(hex::encode).collect::<Vec<_>>(),
"refs": [], "counts": [c.op, c.op * 2],
}),
json!({ "t": "event", "seq": c.op, "time": time(c), "frame": hex::encode(&c.event) }),
]
}
/// The frame the server writes of a commit (`pds-server`'s `log`,
/// `commit_frame`): the same entries, its fields in the order the server
/// writes them, as JSON with holes (`pds_store::frame_with_holes`): the
/// commit's block, the tree's nodes and the event's frame are holes, their
/// bytes as they are after the text; CIDs are hex, and so is the record,
/// which the server writes as its JSON.
pub fn holes(c: &Commit, gone: &[Vec<u8>]) -> Vec<u8> {
let text = 2 * (c.blocks[1].0.len() + c.blocks[1].1.len())
+ (c.blocks.len() + gone.len()) * 80
+ c.path.len()
+ 256;
pds_store::frame_with_holes(text, |w| {
let put = |text: &[u8]| {
let mut out = w;
out.write_all(text).expect("a frame")
};
let json = |v: Json| put(&serde_json::to_vec(&v).expect("a frame"));
put(br#"[{"t":"commit","rev":"#);
json(json!(c.rev));
put(br#","head":"""#);
w.hole(&c.blocks[0].1);
put(br#","puts":"#);
json(json!([[
c.path,
hex::encode(&c.blocks[1].1),
hex::encode(&c.blocks[1].0)
]]));
put(br#","dels":[],"tree":["#);
for (i, (cid, b)) in c.blocks[2..].iter().enumerate() {
put(if i == 0 { b"[" } else { b",[" });
json(json!(hex::encode(cid)));
put(br#","""#);
w.hole(b);
put(b"]");
}
put(br#"],"gone":"#);
json(json!(gone.iter().map(hex::encode).collect::<Vec<_>>()));
put(br#","refs":[],"counts":"#);
json(json!([c.op, c.op * 2]));
put(br#"},{"t":"event","seq":"#);
json(json!(c.op));
put(br#","time":"#);
json(json!(time(c)));
put(br#","frame":"""#);
w.hole(&c.event);
put(b"}]");
})
}
const COMMIT: u8 = 1;
fn varint(out: &mut Vec<u8>, mut n: u64) {
while n >= 0x80 {
out.push(n as u8 | 0x80);
n >>= 7;
}
out.push(n as u8);
}
fn bytes(out: &mut Vec<u8>, b: &[u8]) {
varint(out, b.len() as u64);
out.extend_from_slice(b);
}
/// The same content as `entries`, as a binary frame.
pub fn binary(c: &Commit, gone: &[Vec<u8>]) -> Vec<u8> {
let size = c
.blocks
.iter()
.map(|(k, b)| k.len() + b.len() + 6)
.sum::<usize>()
+ gone.iter().map(|g| g.len() + 1).sum::<usize>()
+ c.event.len()
+ c.rev.len()
+ c.path.len()
+ 48;
pds_store::frame_of(size, |out| {
out.push(COMMIT);
bytes(out, c.rev.as_bytes());
bytes(out, &c.blocks[0].1);
varint(out, 1);
bytes(out, c.path.as_bytes());
bytes(out, &c.blocks[1].0);
bytes(out, &c.blocks[1].1);
varint(out, 0);
varint(out, c.blocks.len() as u64 - 2);
for (cid, b) in &c.blocks[2..] {
bytes(out, cid);
bytes(out, b);
}
varint(out, gone.len() as u64);
for cid in gone {
bytes(out, cid);
}
varint(out, 0);
varint(out, c.op);
varint(out, c.op * 2);
varint(out, c.op);
varint(out, time(c));
bytes(out, &c.event);
})
}
/// A binary payload, read field by field.
struct Walk<'a> {
b: &'a [u8],
at: usize,
}
impl<'a> Walk<'a> {
fn varint(&mut self) -> Option<u64> {
let mut n = 0u64;
for shift in (0..64).step_by(7) {
let b = *self.b.get(self.at)?;
self.at += 1;
n |= u64::from(b & 0x7f) << shift;
if b & 0x80 == 0 {
return Some(n);
}
}
None
}
fn bytes(&mut self) -> Option<&'a [u8]> {
let n = self.varint()? as usize;
let b = self.b.get(self.at..self.at.checked_add(n)?)?;
self.at += n;
Some(b)
}
fn text(&mut self) -> Option<&'a str> {
std::str::from_utf8(self.bytes()?).ok()
}
}
/// A commit's event, as its frame holds it.
pub struct Event<'a> {
pub seq: u64,
pub frame: &'a [u8],
}
/// Walk every field of a binary payload, as a replay does, each string
/// checked to be one: its event, or `None` if the payload is not a
/// commit's, whole.
pub fn walk(payload: &[u8]) -> Option<Event<'_>> {
let mut w = Walk { b: payload, at: 0 };
if *payload.first()? != COMMIT {
return None;
}
w.at = 1;
w.text()?;
w.bytes()?;
for _ in 0..w.varint()? {
w.text()?;
w.bytes()?;
w.bytes()?;
}
for _ in 0..w.varint()? {
w.text()?;
}
for _ in 0..w.varint()? {
w.bytes()?;
w.bytes()?;
}
for _ in 0..w.varint()? {
w.bytes()?;
}
for _ in 0..w.varint()? {
w.bytes()?;
w.varint()?;
}
w.varint()?;
w.varint()?;
let seq = w.varint()?;
w.varint()?;
let frame = w.bytes()?;
(w.at == payload.len()).then_some(Event { seq, frame })
}
/// The frames of a log, one at a time, over the header `pds-store` gives
/// every frame (`[len u32 LE][8-byte check][payload]`), through a buffer
/// of the size `pds_store::scan_file` reads with.
pub struct Frames<R> {
r: io::BufReader<R>,
/// The bytes left to read, and those of the whole frames read.
left: u64,
pub pos: u64,
payload: Vec<u8>,
check: Option<Check>,
}
impl<R: Read> Frames<R> {
/// The frames `r` holds in its next `left` bytes, each one's check
/// computed, as `check`, if there is one.
pub fn new(r: R, left: u64, check: Option<Check>) -> Frames<R> {
Frames {
r: io::BufReader::with_capacity(1 << 16, r),
left,
pos: 0,
payload: Vec::new(),
check,
}
}
/// The next frame's payload: `None` at the first that is cut short or
/// fails its check, where a log ends.
pub fn next_payload(&mut self) -> Option<&[u8]> {
let mut head = [0u8; HEADER];
if self.left < HEADER as u64 || self.r.read_exact(&mut head).is_err() {
return None;
}
let len = u64::from(u32::from_le_bytes(head[..4].try_into().expect("4 bytes")));
if self.left - (HEADER as u64) < len {
return None;
}
self.payload.resize(len as usize, 0);
if self.r.read_exact(&mut self.payload).is_err()
|| self
.check
.is_some_and(|c| head[4..] != c.of_frame(&self.payload))
{
return None;
}
self.left -= HEADER as u64 + len;
self.pos += HEADER as u64 + len;
Some(&self.payload)
}
}
/// The commits in the binary log at `path`, each frame checked and walked.
pub fn count(path: &std::path::Path) -> io::Result<u64> {
let file = std::fs::File::open(path)?;
let size = file.metadata()?.len();
let mut frames = Frames::new(file, size, Some(Check::Sha256));
let mut n = 0;
while let Some(p) = frames.next_payload() {
walk(p).ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("{}: a frame that is no commit's", path.display()),
)
})?;
n += 1;
}
Ok(n)
}