chadtmiller.com/monolith
An atproto PDS
0
0
//! The log kept in `PDS_LOG_DIR`, for the admin to read back
//! (`spec/OPERATIONS.md`, "Kept"). Each line written to standard output is
//! appended to `pds.log` too, by one `write` under the log's lock. Before a
//! line would take `pds.log` past half of `PDS_LOG_KEEP_BYTES`, it is renamed
//! over `pds.log.1` and a new one started: so neither file is ever past half,
//! the two together never past the whole, and the newest lines of half of it
//! are always in them. A line longer than half is not kept.
//!
//! `_logs` reads them (`Snapshot`) from handles and lengths taken under the
//! lock, with no lock held as it reads, newest line first: a rotation since
//! leaves the handles on the files they were, so the read sees the lines kept
//! when it began.
use std::fs::{File, OpenOptions};
use std::io::{self, Write};
use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};
/// The file lines are appended to.
const CURRENT: &str = "pds.log";
/// The one before, renamed over by `CURRENT` once it is full.
const PREVIOUS: &str = "pds.log.1";
/// `PDS_LOG_KEEP_BYTES`: its least, and its default.
const LEAST: u64 = 65_536;
const DEFAULT: u64 = 64 << 20;
/// How much of a file a read takes at once, from its end.
const CHUNK: u64 = 64 << 10;
/// What the environment says of where the log is kept: the directory made,
/// and `CURRENT` opened, so that one that cannot be is found before the start.
pub(super) struct Settings {
dir: PathBuf,
keep: u64,
current: File,
}
impl Settings {
/// `PDS_LOG_DIR` and `PDS_LOG_KEEP_BYTES`; None with no directory.
pub fn from_env() -> Result<Option<Settings>, String> {
let keep = match super::var("PDS_LOG_KEEP_BYTES") {
None => DEFAULT,
Some(v) => v
.trim()
.parse::<u64>()
.ok()
.filter(|n| *n >= LEAST)
.ok_or_else(|| {
format!("PDS_LOG_KEEP_BYTES is {v:?}, not a number of bytes of {LEAST} or more")
})?,
};
let Some(dir) = super::var("PDS_LOG_DIR") else {
return Ok(None);
};
let dir = PathBuf::from(dir);
let cannot = |e: &dyn std::fmt::Display| {
format!(
"PDS_LOG_DIR {} cannot be made or written to: {e}",
dir.display()
)
};
pds_store::make_dirs(&dir).map_err(|e| cannot(&e))?;
rustix::fs::access(&dir, rustix::fs::Access::WRITE_OK).map_err(|e| cannot(&e))?;
let current = open_current(&dir.join(CURRENT)).map_err(|e| cannot(&e))?;
Ok(Some(Settings { dir, keep, current }))
}
}
/// `CURRENT`, made if missing, open to append to and read.
fn open_current(path: &Path) -> io::Result<File> {
pds_store::open_private(
path,
OpenOptions::new().create(true).append(true).read(true),
)
}
/// The kept files, as the writer has them.
pub(super) struct Kept {
dir: PathBuf,
/// Half of `PDS_LOG_KEEP_BYTES`: the most each file holds.
half: u64,
/// None once starting a new one failed, until one is made.
current: Option<File>,
current_len: u64,
previous: Option<File>,
previous_len: u64,
/// `current` ends in no `\n`, as a write cut short, or a power lost
/// mid-line, leaves it: the next line is written after one.
torn: bool,
}
impl Kept {
/// Carry on after the lines the directory holds, as the last start left
/// them; with more than this start's half in either file, as when
/// `PDS_LOG_KEEP_BYTES` was lowered, only the newest lines of half.
pub fn open(s: Settings) -> Kept {
let previous = File::open(s.dir.join(PREVIOUS)).ok();
let len = |f: &File| f.metadata().map_or(0, |m| m.len());
let mut k = Kept {
half: s.keep / 2,
current_len: len(&s.current),
previous_len: previous.as_ref().map_or(0, len),
current: Some(s.current),
previous,
dir: s.dir,
torn: false,
};
if k.current_len > k.half || k.previous_len > k.half {
k.trim();
} else if k.current_len > 0 {
let mut last = [0u8];
let read = k
.current
.as_ref()
.map(|f| f.read_exact_at(&mut last, k.current_len - 1));
k.torn = matches!(read, Some(Ok(()))) && last[0] != b'\n';
}
k
}
/// Keep only the newest whole lines of at most half, in `PREVIOUS`,
/// and start `CURRENT` again: what is let go is let go first.
fn trim(&mut self) {
let mut tail = self.snapshot().newest_bytes(self.half);
// A line cut short at the end is no line.
let whole = tail.iter().rposition(|b| *b == b'\n').map_or(0, |i| i + 1);
tail.truncate(whole);
let previous = self.dir.join(PREVIOUS);
self.previous = None;
self.previous_len = 0;
let _ = std::fs::remove_file(&previous);
if let Some(f) = &self.current {
let _ = f.set_len(0);
}
self.current_len = 0;
let written = pds_store::open_private(
&previous,
OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.read(true),
)
.and_then(|mut f| f.write_all(&tail).map(|()| f));
if let Ok(f) = written {
self.previous = Some(f);
self.previous_len = tail.len() as u64;
}
}
/// Keep a line, `\n` and all, failures ignored.
pub fn write(&mut self, line: &[u8]) {
let len = line.len() as u64;
if len > self.half {
return;
}
if (self.current.is_none() || self.current_len + len + u64::from(self.torn) > self.half)
&& self.rotate().is_err()
{
return;
}
let Some(f) = &self.current else { return };
let mut f = f;
let wrote = if self.torn {
let mut b = Vec::with_capacity(line.len() + 1);
b.push(b'\n');
b.extend_from_slice(line);
f.write(&b).map(|n| (n, b.len()))
} else {
f.write(line).map(|n| (n, line.len()))
};
if let Ok((n, of)) = wrote {
self.current_len += n as u64;
self.torn = n < of;
}
}
/// `CURRENT` becomes `PREVIOUS`, and a new one is started; with none
/// (starting one failed), only the new one.
fn rotate(&mut self) -> io::Result<()> {
let current = self.dir.join(CURRENT);
if let Some(f) = self.current.take() {
if let Err(e) = std::fs::rename(¤t, self.dir.join(PREVIOUS)) {
self.current = Some(f);
return Err(e);
}
self.previous = Some(f);
self.previous_len = self.current_len;
}
self.current_len = 0;
self.torn = false;
let f = open_current(¤t)?;
f.set_len(0)?;
self.current = Some(f);
Ok(())
}
/// The files as they are now, to read with no lock held.
pub fn snapshot(&self) -> Snapshot {
let mut files = Vec::with_capacity(2);
for (f, len) in [
(&self.current, self.current_len),
(&self.previous, self.previous_len),
] {
if let Some(f) = f.as_ref().and_then(|f| f.try_clone().ok()) {
files.push((f, len));
}
}
Snapshot { files }
}
}
/// The kept files as they were when it was taken, newest first, each to
/// the length it had then.
pub(crate) struct Snapshot {
files: Vec<(File, u64)>,
}
impl Snapshot {
/// Each line, newest first, until `each` answers false. Lines do not run
/// from one file into the other; empty ones are passed over.
pub fn lines_back(&self, mut each: impl FnMut(&[u8]) -> bool) -> io::Result<()> {
let mut buf: Vec<u8> = Vec::new();
for (f, len) in &self.files {
// `buf` holds, past what is read next, a line's start not yet found.
buf.clear();
let mut pos = *len;
loop {
let start = pos.saturating_sub(CHUNK);
let n = (pos - start) as usize;
buf.splice(0..0, std::iter::repeat_n(0, n));
f.read_exact_at(&mut buf[..n], start)?;
let mut end = buf.len();
while let Some(i) = buf[..end].iter().rposition(|b| *b == b'\n') {
if end > i + 1 && !each(&buf[i + 1..end]) {
return Ok(());
}
end = i;
}
buf.truncate(end);
pos = start;
if pos == 0 {
break;
}
}
if !buf.is_empty() && !each(&buf) {
return Ok(());
}
}
Ok(())
}
/// The last `n` bytes of the files, oldest first, starting at a line's
/// start: the whole lines that fit.
fn newest_bytes(&self, n: u64) -> Vec<u8> {
let mut lines: Vec<Vec<u8>> = Vec::new();
let mut total = 0u64;
let _ = self.lines_back(|line| {
total += line.len() as u64 + 1;
if total > n {
return false;
}
lines.push(line.to_vec());
true
});
let mut out = Vec::with_capacity(total.min(n) as usize);
for line in lines.iter().rev() {
out.extend_from_slice(line);
out.push(b'\n');
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
fn scratch(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("pds-kept-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn open(dir: &Path, keep: u64) -> Kept {
Kept::open(Settings {
dir: dir.to_path_buf(),
keep,
current: open_current(&dir.join(CURRENT)).unwrap(),
})
}
fn back(k: &Kept) -> Vec<String> {
let mut out = Vec::new();
k.snapshot()
.lines_back(|l| {
out.push(String::from_utf8(l.to_vec()).unwrap());
true
})
.unwrap();
out
}
fn size(dir: &Path) -> u64 {
std::fs::read_dir(dir)
.unwrap()
.map(|e| e.unwrap().metadata().unwrap().len())
.sum()
}
/// The files never hold more than the whole, the newest lines of half
/// are always there, and a line longer than half is not kept.
#[test]
fn rotates_within_bounds() {
let dir = scratch("rotate");
let mut k = open(&dir, 1000);
let mut all = Vec::new();
for i in 0..400 {
let line = format!("{{\"n\":{i},\"pad\":\"{}\"}}", "x".repeat(i % 37));
k.write(format!("{line}\n").as_bytes());
all.push(line);
assert!(size(&dir) <= 1000);
let kept = back(&k);
let mut want = 0;
let mut sum = 0;
for l in all.iter().rev() {
sum += l.len() + 1;
if sum > 500 {
break;
}
want += 1;
}
assert!(kept.len() >= want, "{i}: {} < {want}", kept.len());
assert_eq!(
kept,
all.iter()
.rev()
.take(kept.len())
.cloned()
.collect::<Vec<_>>()
);
}
k.write(format!("{}\n", "y".repeat(500)).as_bytes());
assert_eq!(back(&k)[0], all[399]);
let _ = std::fs::remove_dir_all(&dir);
}
/// A start carries on after the lines there; a torn line is not glued to
/// the next; a lower `PDS_LOG_KEEP_BYTES` keeps only the newest of half.
#[test]
fn starts_again() {
let dir = scratch("again");
let mut k = open(&dir, 1000);
k.write(b"{\"a\":1}\n");
drop(k);
std::fs::OpenOptions::new()
.append(true)
.open(dir.join(CURRENT))
.unwrap()
.write_all(b"{\"cut")
.unwrap();
let mut k = open(&dir, 1000);
k.write(b"{\"b\":2}\n");
assert_eq!(back(&k), ["{\"b\":2}", "{\"cut", "{\"a\":1}"]);
for i in 0..100 {
k.write(format!("{{\"c\":{i:03}}}\n").as_bytes());
}
drop(k);
let k = open(&dir, 200);
assert!(size(&dir) <= 200);
let kept = back(&k);
assert_eq!(kept.len(), 10);
assert_eq!(kept[0], "{\"c\":099}");
assert_eq!(kept[9], "{\"c\":090}");
let _ = std::fs::remove_dir_all(&dir);
}
}