//! The log kept in `PDS_LOG_DIR`, for the admin to read back //! (`spec/OPERATIONS.md`, "Kept"). Each line written to standard output is //! appended to `pds.log` too, by one `write` under the log's lock. Before a //! line would take `pds.log` past half of `PDS_LOG_KEEP_BYTES`, it is renamed //! over `pds.log.1` and a new one started: so neither file is ever past half, //! the two together never past the whole, and the newest lines of half of it //! are always in them. A line longer than half is not kept. //! //! `_logs` reads them (`Snapshot`) from handles and lengths taken under the //! lock, with no lock held as it reads, newest line first: a rotation since //! leaves the handles on the files they were, so the read sees the lines kept //! when it began. use std::fs::{File, OpenOptions}; use std::io::{self, Write}; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; /// The file lines are appended to. const CURRENT: &str = "pds.log"; /// The one before, renamed over by `CURRENT` once it is full. const PREVIOUS: &str = "pds.log.1"; /// `PDS_LOG_KEEP_BYTES`: its least, and its default. const LEAST: u64 = 65_536; const DEFAULT: u64 = 64 << 20; /// How much of a file a read takes at once, from its end. const CHUNK: u64 = 64 << 10; /// What the environment says of where the log is kept: the directory made, /// and `CURRENT` opened, so that one that cannot be is found before the start. pub(super) struct Settings { dir: PathBuf, keep: u64, current: File, } impl Settings { /// `PDS_LOG_DIR` and `PDS_LOG_KEEP_BYTES`; None with no directory. pub fn from_env() -> Result, String> { let keep = match super::var("PDS_LOG_KEEP_BYTES") { None => DEFAULT, Some(v) => v .trim() .parse::() .ok() .filter(|n| *n >= LEAST) .ok_or_else(|| { format!("PDS_LOG_KEEP_BYTES is {v:?}, not a number of bytes of {LEAST} or more") })?, }; let Some(dir) = super::var("PDS_LOG_DIR") else { return Ok(None); }; let dir = PathBuf::from(dir); let cannot = |e: &dyn std::fmt::Display| { format!( "PDS_LOG_DIR {} cannot be made or written to: {e}", dir.display() ) }; pds_store::make_dirs(&dir).map_err(|e| cannot(&e))?; rustix::fs::access(&dir, rustix::fs::Access::WRITE_OK).map_err(|e| cannot(&e))?; let current = open_current(&dir.join(CURRENT)).map_err(|e| cannot(&e))?; Ok(Some(Settings { dir, keep, current })) } } /// `CURRENT`, made if missing, open to append to and read. fn open_current(path: &Path) -> io::Result { pds_store::open_private( path, OpenOptions::new().create(true).append(true).read(true), ) } /// The kept files, as the writer has them. pub(super) struct Kept { dir: PathBuf, /// Half of `PDS_LOG_KEEP_BYTES`: the most each file holds. half: u64, /// None once starting a new one failed, until one is made. current: Option, current_len: u64, previous: Option, previous_len: u64, /// `current` ends in no `\n`, as a write cut short, or a power lost /// mid-line, leaves it: the next line is written after one. torn: bool, } impl Kept { /// Carry on after the lines the directory holds, as the last start left /// them; with more than this start's half in either file, as when /// `PDS_LOG_KEEP_BYTES` was lowered, only the newest lines of half. pub fn open(s: Settings) -> Kept { let previous = File::open(s.dir.join(PREVIOUS)).ok(); let len = |f: &File| f.metadata().map_or(0, |m| m.len()); let mut k = Kept { half: s.keep / 2, current_len: len(&s.current), previous_len: previous.as_ref().map_or(0, len), current: Some(s.current), previous, dir: s.dir, torn: false, }; if k.current_len > k.half || k.previous_len > k.half { k.trim(); } else if k.current_len > 0 { let mut last = [0u8]; let read = k .current .as_ref() .map(|f| f.read_exact_at(&mut last, k.current_len - 1)); k.torn = matches!(read, Some(Ok(()))) && last[0] != b'\n'; } k } /// Keep only the newest whole lines of at most half, in `PREVIOUS`, /// and start `CURRENT` again: what is let go is let go first. fn trim(&mut self) { let mut tail = self.snapshot().newest_bytes(self.half); // A line cut short at the end is no line. let whole = tail.iter().rposition(|b| *b == b'\n').map_or(0, |i| i + 1); tail.truncate(whole); let previous = self.dir.join(PREVIOUS); self.previous = None; self.previous_len = 0; let _ = std::fs::remove_file(&previous); if let Some(f) = &self.current { let _ = f.set_len(0); } self.current_len = 0; let written = pds_store::open_private( &previous, OpenOptions::new() .create(true) .truncate(true) .write(true) .read(true), ) .and_then(|mut f| f.write_all(&tail).map(|()| f)); if let Ok(f) = written { self.previous = Some(f); self.previous_len = tail.len() as u64; } } /// Keep a line, `\n` and all, failures ignored. pub fn write(&mut self, line: &[u8]) { let len = line.len() as u64; if len > self.half { return; } if (self.current.is_none() || self.current_len + len + u64::from(self.torn) > self.half) && self.rotate().is_err() { return; } let Some(f) = &self.current else { return }; let mut f = f; let wrote = if self.torn { let mut b = Vec::with_capacity(line.len() + 1); b.push(b'\n'); b.extend_from_slice(line); f.write(&b).map(|n| (n, b.len())) } else { f.write(line).map(|n| (n, line.len())) }; if let Ok((n, of)) = wrote { self.current_len += n as u64; self.torn = n < of; } } /// `CURRENT` becomes `PREVIOUS`, and a new one is started; with none /// (starting one failed), only the new one. fn rotate(&mut self) -> io::Result<()> { let current = self.dir.join(CURRENT); if let Some(f) = self.current.take() { if let Err(e) = std::fs::rename(¤t, self.dir.join(PREVIOUS)) { self.current = Some(f); return Err(e); } self.previous = Some(f); self.previous_len = self.current_len; } self.current_len = 0; self.torn = false; let f = open_current(¤t)?; f.set_len(0)?; self.current = Some(f); Ok(()) } /// The files as they are now, to read with no lock held. pub fn snapshot(&self) -> Snapshot { let mut files = Vec::with_capacity(2); for (f, len) in [ (&self.current, self.current_len), (&self.previous, self.previous_len), ] { if let Some(f) = f.as_ref().and_then(|f| f.try_clone().ok()) { files.push((f, len)); } } Snapshot { files } } } /// The kept files as they were when it was taken, newest first, each to /// the length it had then. pub(crate) struct Snapshot { files: Vec<(File, u64)>, } impl Snapshot { /// Each line, newest first, until `each` answers false. Lines do not run /// from one file into the other; empty ones are passed over. pub fn lines_back(&self, mut each: impl FnMut(&[u8]) -> bool) -> io::Result<()> { let mut buf: Vec = Vec::new(); for (f, len) in &self.files { // `buf` holds, past what is read next, a line's start not yet found. buf.clear(); let mut pos = *len; loop { let start = pos.saturating_sub(CHUNK); let n = (pos - start) as usize; buf.splice(0..0, std::iter::repeat_n(0, n)); f.read_exact_at(&mut buf[..n], start)?; let mut end = buf.len(); while let Some(i) = buf[..end].iter().rposition(|b| *b == b'\n') { if end > i + 1 && !each(&buf[i + 1..end]) { return Ok(()); } end = i; } buf.truncate(end); pos = start; if pos == 0 { break; } } if !buf.is_empty() && !each(&buf) { return Ok(()); } } Ok(()) } /// The last `n` bytes of the files, oldest first, starting at a line's /// start: the whole lines that fit. fn newest_bytes(&self, n: u64) -> Vec { let mut lines: Vec> = Vec::new(); let mut total = 0u64; let _ = self.lines_back(|line| { total += line.len() as u64 + 1; if total > n { return false; } lines.push(line.to_vec()); true }); let mut out = Vec::with_capacity(total.min(n) as usize); for line in lines.iter().rev() { out.extend_from_slice(line); out.push(b'\n'); } out } } #[cfg(test)] mod tests { use super::*; fn scratch(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("pds-kept-{name}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&d); std::fs::create_dir_all(&d).unwrap(); d } fn open(dir: &Path, keep: u64) -> Kept { Kept::open(Settings { dir: dir.to_path_buf(), keep, current: open_current(&dir.join(CURRENT)).unwrap(), }) } fn back(k: &Kept) -> Vec { let mut out = Vec::new(); k.snapshot() .lines_back(|l| { out.push(String::from_utf8(l.to_vec()).unwrap()); true }) .unwrap(); out } fn size(dir: &Path) -> u64 { std::fs::read_dir(dir) .unwrap() .map(|e| e.unwrap().metadata().unwrap().len()) .sum() } /// The files never hold more than the whole, the newest lines of half /// are always there, and a line longer than half is not kept. #[test] fn rotates_within_bounds() { let dir = scratch("rotate"); let mut k = open(&dir, 1000); let mut all = Vec::new(); for i in 0..400 { let line = format!("{{\"n\":{i},\"pad\":\"{}\"}}", "x".repeat(i % 37)); k.write(format!("{line}\n").as_bytes()); all.push(line); assert!(size(&dir) <= 1000); let kept = back(&k); let mut want = 0; let mut sum = 0; for l in all.iter().rev() { sum += l.len() + 1; if sum > 500 { break; } want += 1; } assert!(kept.len() >= want, "{i}: {} < {want}", kept.len()); assert_eq!( kept, all.iter() .rev() .take(kept.len()) .cloned() .collect::>() ); } k.write(format!("{}\n", "y".repeat(500)).as_bytes()); assert_eq!(back(&k)[0], all[399]); let _ = std::fs::remove_dir_all(&dir); } /// A start carries on after the lines there; a torn line is not glued to /// the next; a lower `PDS_LOG_KEEP_BYTES` keeps only the newest of half. #[test] fn starts_again() { let dir = scratch("again"); let mut k = open(&dir, 1000); k.write(b"{\"a\":1}\n"); drop(k); std::fs::OpenOptions::new() .append(true) .open(dir.join(CURRENT)) .unwrap() .write_all(b"{\"cut") .unwrap(); let mut k = open(&dir, 1000); k.write(b"{\"b\":2}\n"); assert_eq!(back(&k), ["{\"b\":2}", "{\"cut", "{\"a\":1}"]); for i in 0..100 { k.write(format!("{{\"c\":{i:03}}}\n").as_bytes()); } drop(k); let k = open(&dir, 200); assert!(size(&dir) <= 200); let kept = back(&k); assert_eq!(kept.len(), 10); assert_eq!(kept[0], "{\"c\":099}"); assert_eq!(kept[9], "{\"c\":090}"); let _ = std::fs::remove_dir_all(&dir); } }