~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
377 lines · 12.6 KB Raw History
//! The log kept in `PDS_LOG_DIR`, for the admin to read back
//! (`spec/OPERATIONS.md`, "Kept"). Each line written to standard output is
//! appended to `pds.log` too, by one `write` under the log's lock. Before a
//! line would take `pds.log` past half of `PDS_LOG_KEEP_BYTES`, it is renamed
//! over `pds.log.1` and a new one started: so neither file is ever past half,
//! the two together never past the whole, and the newest lines of half of it
//! are always in them. A line longer than half is not kept.
//!
//! `_logs` reads them (`Snapshot`) from handles and lengths taken under the
//! lock, with no lock held as it reads, newest line first: a rotation since
//! leaves the handles on the files they were, so the read sees the lines kept
//! when it began.

use std::fs::{File, OpenOptions};
use std::io::{self, Write};
use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};

/// The file lines are appended to.
const CURRENT: &str = "pds.log";
/// The one before, renamed over by `CURRENT` once it is full.
const PREVIOUS: &str = "pds.log.1";
/// `PDS_LOG_KEEP_BYTES`: its least, and its default.
const LEAST: u64 = 65_536;
const DEFAULT: u64 = 64 << 20;
/// How much of a file a read takes at once, from its end.
const CHUNK: u64 = 64 << 10;

/// What the environment says of where the log is kept: the directory made,
/// and `CURRENT` opened, so that one that cannot be is found before the start.
pub(super) struct Settings {
    dir: PathBuf,
    keep: u64,
    current: File,
}

impl Settings {
    /// `PDS_LOG_DIR` and `PDS_LOG_KEEP_BYTES`; None with no directory.
    pub fn from_env() -> Result<Option<Settings>, String> {
        let keep = match super::var("PDS_LOG_KEEP_BYTES") {
            None => DEFAULT,
            Some(v) => v
                .trim()
                .parse::<u64>()
                .ok()
                .filter(|n| *n >= LEAST)
                .ok_or_else(|| {
                    format!("PDS_LOG_KEEP_BYTES is {v:?}, not a number of bytes of {LEAST} or more")
                })?,
        };
        let Some(dir) = super::var("PDS_LOG_DIR") else {
            return Ok(None);
        };
        let dir = PathBuf::from(dir);
        let cannot = |e: &dyn std::fmt::Display| {
            format!(
                "PDS_LOG_DIR {} cannot be made or written to: {e}",
                dir.display()
            )
        };
        pds_store::make_dirs(&dir).map_err(|e| cannot(&e))?;
        rustix::fs::access(&dir, rustix::fs::Access::WRITE_OK).map_err(|e| cannot(&e))?;
        let current = open_current(&dir.join(CURRENT)).map_err(|e| cannot(&e))?;
        Ok(Some(Settings { dir, keep, current }))
    }
}

/// `CURRENT`, made if missing, open to append to and read.
fn open_current(path: &Path) -> io::Result<File> {
    pds_store::open_private(
        path,
        OpenOptions::new().create(true).append(true).read(true),
    )
}

/// The kept files, as the writer has them.
pub(super) struct Kept {
    dir: PathBuf,
    /// Half of `PDS_LOG_KEEP_BYTES`: the most each file holds.
    half: u64,
    /// None once starting a new one failed, until one is made.
    current: Option<File>,
    current_len: u64,
    previous: Option<File>,
    previous_len: u64,
    /// `current` ends in no `\n`, as a write cut short, or a power lost
    /// mid-line, leaves it: the next line is written after one.
    torn: bool,
}

impl Kept {
    /// Carry on after the lines the directory holds, as the last start left
    /// them; with more than this start's half in either file, as when
    /// `PDS_LOG_KEEP_BYTES` was lowered, only the newest lines of half.
    pub fn open(s: Settings) -> Kept {
        let previous = File::open(s.dir.join(PREVIOUS)).ok();
        let len = |f: &File| f.metadata().map_or(0, |m| m.len());
        let mut k = Kept {
            half: s.keep / 2,
            current_len: len(&s.current),
            previous_len: previous.as_ref().map_or(0, len),
            current: Some(s.current),
            previous,
            dir: s.dir,
            torn: false,
        };
        if k.current_len > k.half || k.previous_len > k.half {
            k.trim();
        } else if k.current_len > 0 {
            let mut last = [0u8];
            let read = k
                .current
                .as_ref()
                .map(|f| f.read_exact_at(&mut last, k.current_len - 1));
            k.torn = matches!(read, Some(Ok(()))) && last[0] != b'\n';
        }
        k
    }

    /// Keep only the newest whole lines of at most half, in `PREVIOUS`,
    /// and start `CURRENT` again: what is let go is let go first.
    fn trim(&mut self) {
        let mut tail = self.snapshot().newest_bytes(self.half);
        // A line cut short at the end is no line.
        let whole = tail.iter().rposition(|b| *b == b'\n').map_or(0, |i| i + 1);
        tail.truncate(whole);
        let previous = self.dir.join(PREVIOUS);
        self.previous = None;
        self.previous_len = 0;
        let _ = std::fs::remove_file(&previous);
        if let Some(f) = &self.current {
            let _ = f.set_len(0);
        }
        self.current_len = 0;
        let written = pds_store::open_private(
            &previous,
            OpenOptions::new()
                .create(true)
                .truncate(true)
                .write(true)
                .read(true),
        )
        .and_then(|mut f| f.write_all(&tail).map(|()| f));
        if let Ok(f) = written {
            self.previous = Some(f);
            self.previous_len = tail.len() as u64;
        }
    }

    /// Keep a line, `\n` and all, failures ignored.
    pub fn write(&mut self, line: &[u8]) {
        let len = line.len() as u64;
        if len > self.half {
            return;
        }
        if (self.current.is_none() || self.current_len + len + u64::from(self.torn) > self.half)
            && self.rotate().is_err()
        {
            return;
        }
        let Some(f) = &self.current else { return };
        let mut f = f;
        let wrote = if self.torn {
            let mut b = Vec::with_capacity(line.len() + 1);
            b.push(b'\n');
            b.extend_from_slice(line);
            f.write(&b).map(|n| (n, b.len()))
        } else {
            f.write(line).map(|n| (n, line.len()))
        };
        if let Ok((n, of)) = wrote {
            self.current_len += n as u64;
            self.torn = n < of;
        }
    }

    /// `CURRENT` becomes `PREVIOUS`, and a new one is started; with none
    /// (starting one failed), only the new one.
    fn rotate(&mut self) -> io::Result<()> {
        let current = self.dir.join(CURRENT);
        if let Some(f) = self.current.take() {
            if let Err(e) = std::fs::rename(&current, self.dir.join(PREVIOUS)) {
                self.current = Some(f);
                return Err(e);
            }
            self.previous = Some(f);
            self.previous_len = self.current_len;
        }
        self.current_len = 0;
        self.torn = false;
        let f = open_current(&current)?;
        f.set_len(0)?;
        self.current = Some(f);
        Ok(())
    }

    /// The files as they are now, to read with no lock held.
    pub fn snapshot(&self) -> Snapshot {
        let mut files = Vec::with_capacity(2);
        for (f, len) in [
            (&self.current, self.current_len),
            (&self.previous, self.previous_len),
        ] {
            if let Some(f) = f.as_ref().and_then(|f| f.try_clone().ok()) {
                files.push((f, len));
            }
        }
        Snapshot { files }
    }
}

/// The kept files as they were when it was taken, newest first, each to
/// the length it had then.
pub(crate) struct Snapshot {
    files: Vec<(File, u64)>,
}

impl Snapshot {
    /// Each line, newest first, until `each` answers false. Lines do not run
    /// from one file into the other; empty ones are passed over.
    pub fn lines_back(&self, mut each: impl FnMut(&[u8]) -> bool) -> io::Result<()> {
        let mut buf: Vec<u8> = Vec::new();
        for (f, len) in &self.files {
            // `buf` holds, past what is read next, a line's start not yet found.
            buf.clear();
            let mut pos = *len;
            loop {
                let start = pos.saturating_sub(CHUNK);
                let n = (pos - start) as usize;
                buf.splice(0..0, std::iter::repeat_n(0, n));
                f.read_exact_at(&mut buf[..n], start)?;
                let mut end = buf.len();
                while let Some(i) = buf[..end].iter().rposition(|b| *b == b'\n') {
                    if end > i + 1 && !each(&buf[i + 1..end]) {
                        return Ok(());
                    }
                    end = i;
                }
                buf.truncate(end);
                pos = start;
                if pos == 0 {
                    break;
                }
            }
            if !buf.is_empty() && !each(&buf) {
                return Ok(());
            }
        }
        Ok(())
    }

    /// The last `n` bytes of the files, oldest first, starting at a line's
    /// start: the whole lines that fit.
    fn newest_bytes(&self, n: u64) -> Vec<u8> {
        let mut lines: Vec<Vec<u8>> = Vec::new();
        let mut total = 0u64;
        let _ = self.lines_back(|line| {
            total += line.len() as u64 + 1;
            if total > n {
                return false;
            }
            lines.push(line.to_vec());
            true
        });
        let mut out = Vec::with_capacity(total.min(n) as usize);
        for line in lines.iter().rev() {
            out.extend_from_slice(line);
            out.push(b'\n');
        }
        out
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn scratch(name: &str) -> PathBuf {
        let d = std::env::temp_dir().join(format!("pds-kept-{name}-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&d);
        std::fs::create_dir_all(&d).unwrap();
        d
    }

    fn open(dir: &Path, keep: u64) -> Kept {
        Kept::open(Settings {
            dir: dir.to_path_buf(),
            keep,
            current: open_current(&dir.join(CURRENT)).unwrap(),
        })
    }

    fn back(k: &Kept) -> Vec<String> {
        let mut out = Vec::new();
        k.snapshot()
            .lines_back(|l| {
                out.push(String::from_utf8(l.to_vec()).unwrap());
                true
            })
            .unwrap();
        out
    }

    fn size(dir: &Path) -> u64 {
        std::fs::read_dir(dir)
            .unwrap()
            .map(|e| e.unwrap().metadata().unwrap().len())
            .sum()
    }

    /// The files never hold more than the whole, the newest lines of half
    /// are always there, and a line longer than half is not kept.
    #[test]
    fn rotates_within_bounds() {
        let dir = scratch("rotate");
        let mut k = open(&dir, 1000);
        let mut all = Vec::new();
        for i in 0..400 {
            let line = format!("{{\"n\":{i},\"pad\":\"{}\"}}", "x".repeat(i % 37));
            k.write(format!("{line}\n").as_bytes());
            all.push(line);
            assert!(size(&dir) <= 1000);
            let kept = back(&k);
            let mut want = 0;
            let mut sum = 0;
            for l in all.iter().rev() {
                sum += l.len() + 1;
                if sum > 500 {
                    break;
                }
                want += 1;
            }
            assert!(kept.len() >= want, "{i}: {} < {want}", kept.len());
            assert_eq!(
                kept,
                all.iter()
                    .rev()
                    .take(kept.len())
                    .cloned()
                    .collect::<Vec<_>>()
            );
        }
        k.write(format!("{}\n", "y".repeat(500)).as_bytes());
        assert_eq!(back(&k)[0], all[399]);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A start carries on after the lines there; a torn line is not glued to
    /// the next; a lower `PDS_LOG_KEEP_BYTES` keeps only the newest of half.
    #[test]
    fn starts_again() {
        let dir = scratch("again");
        let mut k = open(&dir, 1000);
        k.write(b"{\"a\":1}\n");
        drop(k);
        std::fs::OpenOptions::new()
            .append(true)
            .open(dir.join(CURRENT))
            .unwrap()
            .write_all(b"{\"cut")
            .unwrap();
        let mut k = open(&dir, 1000);
        k.write(b"{\"b\":2}\n");
        assert_eq!(back(&k), ["{\"b\":2}", "{\"cut", "{\"a\":1}"]);
        for i in 0..100 {
            k.write(format!("{{\"c\":{i:03}}}\n").as_bytes());
        }
        drop(k);
        let k = open(&dir, 200);
        assert!(size(&dir) <= 200);
        let kept = back(&k);
        assert_eq!(kept.len(), 10);
        assert_eq!(kept[0], "{\"c\":099}");
        assert_eq!(kept[9], "{\"c\":090}");
        let _ = std::fs::remove_dir_all(&dir);
    }
}