chadtmiller.com/monolith
An atproto PDS
0
0
//! A table of the version before, which the last release wrote: it has no
//! `H`, its `S` says version 2 and holds the table's generation, the last
//! `seq` and the greatest seq let go. The rest of its `S`, and its other
//! sections, are as this version's, but for what it holds of spaces.
//!
//! It is read once, by the fold that makes a table of this version of it
//! (`load`). Of spaces it keeps nothing: its space repos, held whole in
//! its `S`, the spaces it is the authority of and its notifications are
//! let go, unread but for what its space repos' records reference of
//! blobs, which is counted until the log's mark (`unmarked`).
use super::unmarked::Unmarked;
use super::{Small, VERSION, as_uint, map};
use crate::blobs::cids_in;
use pds_repo::data::{V, decode};
use pds_repo::valid_tid;
/// The version a table of the version before says it is.
const BEFORE: i64 = 2;
/// Take the value of `k` out of a map.
fn take(m: &mut Vec<(String, V)>, k: &str) -> Result<V, String> {
m.iter()
.position(|(n, _)| n == k)
.map(|i| m.swap_remove(i).1)
.ok_or_else(|| format!("a checkpoint of the version before with no {k}"))
}
/// An `S` value of the version before, decoded, less its version.
fn state(bytes: &[u8]) -> Result<Vec<(String, V)>, String> {
let V::Map(mut m) = decode(bytes).map_err(|e| format!("the state: {e}"))? else {
return Err("the state is not a map".into());
};
let version = take(&mut m, "version")?;
if version != V::Int(BEFORE) {
return Err(format!(
"a checkpoint of version {version:?}, which this server does not know"
));
}
Ok(m)
}
/// The generation of the table whose `S` value this is.
pub(super) fn generation(bytes: &[u8]) -> Result<u64, String> {
as_uint(take(&mut state(bytes)?, "generation")?, "generation")
}
/// Read an `S` value of the version before, of the account `did`: its
/// state, unmarked, with no spaces state.
pub(super) fn read(bytes: &[u8], did: &str) -> Result<Small, String> {
let mut m = state(bytes)?;
let mut number = |k: &str| as_uint(take(&mut m, k)?, k);
let header = (number("generation")?, number("lastSeq")?, number("letGo")?);
let mut spaces = None;
for k in ["spaces", "authority", "notify", "spaceRefs"] {
if let Ok(v) = take(&mut m, k)
&& k == "spaces"
{
spaces = Some(v);
}
}
let empty = || V::Arr(Vec::new());
m.push(("version".into(), V::Int(VERSION)));
m.push(("spaces".into(), empty()));
m.push(("authority".into(), empty()));
m.push((
"notify".into(),
map([
("queue", empty()),
("registrations", empty()),
("notices", empty()),
]),
));
m.push(("spaceRefs".into(), empty()));
let mut small = Small::of(V::Map(m), header, did)?;
let mut unmarked = Unmarked::default();
for (space, rev, records) in spaces.map(space_repos).into_iter().flatten() {
if let Some(rev) = rev.filter(|r| valid_tid(r)) {
small.tids.after(&rev);
}
for (path, bytes) in records {
let cids = decode(&bytes).map(|v| cids_in(&v)).unwrap_or_default();
unmarked.kept(&space, &path, cids, &mut small.blobs);
}
}
small.unmarked = Some(unmarked);
Ok(small)
}
/// A space repo its `spaces` holds whole, as far as it reads: its space's
/// URI, its rev, and each record's path and DRISL bytes.
type Whole = (String, Option<String>, Vec<(String, Vec<u8>)>);
/// Each space repo its `spaces` holds whole (`Whole`).
fn space_repos(spaces: V) -> Vec<Whole> {
let V::Arr(repos) = spaces else {
return Vec::new();
};
let get = |m: &[(String, V)], k: &str| m.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());
repos
.into_iter()
.filter_map(|r| {
let V::Map(m) = r else { return None };
let Some(V::Str(space)) = get(&m, "space") else {
return None;
};
let rev = match get(&m, "rev") {
Some(V::Str(rev)) => Some(rev),
_ => None,
};
let records = match get(&m, "records") {
Some(V::Arr(records)) => records
.into_iter()
.filter_map(|x| match x {
V::Arr(p) => match <[V; 2]>::try_from(p) {
Ok([V::Str(path), V::Bytes(bytes)]) => Some((path, bytes)),
_ => None,
},
_ => None,
})
.collect(),
_ => Vec::new(),
};
Some((space, rev, records))
})
.collect()
}
#[cfg(test)]
pub(crate) mod tests {
use super::super::spaces::{self, Meta, space_key};
use super::super::{as_text, fields, header_value, int, key, read_header, text};
use super::*;
use pds_repo::data::encode;
use pds_store::table::{BLOCK, BlockCache, Table, TableWriter, Version};
use std::path::Path;
/// Make the table at `path`, of this version, one of the version
/// before holding the same: its space repos whole in its `S`, with its
/// generation, last `seq` and greatest seq let go, and no `H` nor any
/// of the space repos' sections; and of the table format's version
/// before, as every such table is.
pub(crate) fn downgrade(path: &Path) {
let cache = BlockCache::new(1 << 20);
let all: Vec<(Vec<u8>, Vec<u8>)> = Table::open(path, &cache)
.unwrap()
.iter(b"", None, &cache)
.map(Result::unwrap)
.collect();
let get = |k: &[u8]| all.iter().find(|(x, _)| x == k).unwrap().1.clone();
let (generation, last_seq, let_go) = read_header(&get(b"H")).unwrap();
let V::Map(mut m) = decode(&get(b"S")).unwrap() else {
panic!("a state that is no map")
};
assert_eq!(take(&mut m, "version"), Ok(V::Int(VERSION)));
take(&mut m, "spaceRefs").unwrap();
let whole = |s: V| {
let [space, rev, data, _, _, _] = fields(
s,
["space", "rev", "data", "hash", "records", "ops"],
"a space",
)
.unwrap();
let uri = as_text(space.clone(), "a space").unwrap();
let of = |section: u8| {
let prefix = key(section, &space_key(&uri, b""));
all.iter()
.filter(move |(k, _)| k.starts_with(&prefix))
.map(|(k, v)| (&k[uri.len() + 2..], v))
};
let path = |k: &[u8]| text(std::str::from_utf8(k).unwrap());
let records =
of(spaces::VALUES).map(|(k, v)| V::Arr(vec![path(k), V::Bytes(v.clone())]));
let written = of(spaces::RECORDS)
.map(|(k, v)| V::Arr(vec![path(k), text(&Meta::decode(v).unwrap().rev)]));
let ops = of(spaces::OPS).map(|(k, v)| {
let o = spaces::read_op(k, v).unwrap();
let (collection, rkey) = o.path.split_once('/').unwrap();
let link = |c: Option<Vec<u8>>| c.map_or(V::Null, V::Link);
map([
("rev", text(&o.rev)),
("collection", text(collection)),
("rkey", text(rkey)),
("cid", link(o.cid)),
("prev", link(o.prev)),
])
});
map([
("space", space),
("rev", rev),
("data", data),
("records", V::Arr(records.collect())),
("written", V::Arr(written.collect())),
("ops", V::Arr(ops.collect())),
])
};
let V::Arr(kept) = take(&mut m, "spaces").unwrap() else {
panic!("spaces that are no list")
};
m.push((
"spaces".into(),
V::Arr(kept.into_iter().map(whole).collect()),
));
m.push(("version".into(), V::Int(BEFORE)));
m.push(("generation".into(), int(generation)));
m.push(("lastSeq".into(), int(last_seq)));
m.push(("letGo".into(), int(let_go)));
assert_eq!(header_value(generation, last_seq, let_go), get(b"H"));
let out = std::io::BufWriter::new(std::fs::File::create(path).unwrap());
let mut w = TableWriter::of_version(out, all.len() as u64, BLOCK, Version::Sha256);
for (k, v) in &all {
match k[0] {
b'B' | b'D' | b'E' | b'P' => w.add(k, v).unwrap(),
b'S' => w.add(k, &encode(&V::Map(m.clone()))).unwrap(),
_ => {}
}
}
w.finish().unwrap();
}
}