~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
275 lines · 8.4 KB Raw History
//! Rate governor behavior tests.

use std::sync::{Arc, Mutex};
use std::time::Duration;

use async_trait::async_trait;
use chrono::{DateTime, Utc};
use vuln_notif::db::DbHandle;
use vuln_notif::log::{CaptureSink, Level, Logger};
use vuln_notif::pds::RateGovernor;
use vuln_notif::time::Time;

const DID: &str = "did:plc:governor-parity";

/// Records every sleep, advances the clock, and returns zero jitter. With a
/// gate installed, sleeps block until [`FakeTime::release`] grants a permit,
/// which lets a test observe a governor task parked inside a pacing wait.
struct FakeTime {
    now: Mutex<f64>,
    sleeps: Mutex<Vec<f64>>,
    gate: Option<Arc<tokio::sync::Semaphore>>,
}

impl FakeTime {
    fn new(start: f64) -> Self {
        Self {
            now: Mutex::new(start),
            sleeps: Mutex::new(Vec::new()),
            gate: None,
        }
    }

    fn gated(start: f64) -> Self {
        Self {
            gate: Some(Arc::new(tokio::sync::Semaphore::new(0))),
            ..Self::new(start)
        }
    }

    fn now(&self) -> f64 {
        *self.now.lock().expect("fake time poisoned")
    }

    fn sleeps(&self) -> Vec<f64> {
        self.sleeps.lock().expect("fake sleeps poisoned").clone()
    }

    fn release(&self, permits: usize) {
        self.gate
            .as_ref()
            .expect("fake time is not gated")
            .add_permits(permits);
    }
}

#[async_trait]
impl Time for FakeTime {
    fn now_unix(&self) -> f64 {
        self.now()
    }

    fn now_utc(&self) -> DateTime<Utc> {
        DateTime::from_timestamp_millis((self.now() * 1000.0).round() as i64)
            .expect("fake timestamp")
    }

    async fn sleep(&self, duration: Duration) {
        let seconds = duration.as_secs_f64();
        self.sleeps
            .lock()
            .expect("fake sleeps poisoned")
            .push(seconds);
        if let Some(gate) = &self.gate {
            gate.clone()
                .acquire_owned()
                .await
                .expect("fake gate closed")
                .forget();
        }
        *self.now.lock().expect("fake time poisoned") += seconds;
    }

    fn jitter(&self, _scale: f64) -> f64 {
        0.0
    }
}

#[tokio::test]
async fn hour_budget_exhaustion_waits_to_boundary_and_persists() -> anyhow::Result<()> {
    let base = 1_700_000_000.0;
    let dir = tempfile::tempdir()?;
    let path = dir.path().join("cache.db");
    let time = Arc::new(FakeTime::new(base));
    let db = DbHandle::open(&path).await?;
    let sink = Arc::new(CaptureSink::default());
    let governor = RateGovernor::new(db.clone(), 4.0, 40.0, 3000.0, time.clone())
        .with_logger(Logger::new(sink.clone(), Level::Info));
    governor.bind(DID).await?;

    // hour cap = 3.6, day cap = 36: the second 3-point write spills the hour.
    governor
        .reserve_write(3.0, "createRecord", None, None)
        .await?;
    governor
        .reserve_write(3.0, "createRecord", None, None)
        .await?;

    let expected = base - base.rem_euclid(3600.0) + 3600.0 - base;
    assert_eq!(time.sleeps(), vec![expected]);
    assert_eq!(governor.waits(), 1);
    assert_eq!(governor.wait_seconds(), expected);
    assert_eq!(governor.used().await, (3.0, 6.0));

    let line = sink
        .lines()
        .last()
        .cloned()
        .expect("rate_limit_wait log line");
    let event: serde_json::Value = serde_json::from_str(&line)?;
    assert_eq!(event["event"], "rate_limit_wait");
    assert_eq!(event["op"], "createRecord");
    assert_eq!(event["wait_seconds"].as_f64(), Some(expected));

    drop(governor);
    db.close().await?;

    // A fresh DB handle + governor must resume the persisted window budget.
    let db = DbHandle::open(&path).await?;
    let reloaded = RateGovernor::new(db.clone(), 4.0, 40.0, 3000.0, time.clone());
    reloaded.bind(DID).await?;
    assert_eq!(reloaded.used().await, (3.0, 6.0));
    let (hour_left, day_left) = reloaded.remaining().await;
    assert!((hour_left - 0.6).abs() < 1e-9, "hour_left={hour_left}");
    assert_eq!(day_left, 30.0);
    drop(reloaded);
    db.close().await?;
    Ok(())
}

#[tokio::test]
async fn day_budget_exhaustion_waits_to_boundary() -> anyhow::Result<()> {
    let base = 1_700_000_000.0;
    let time = Arc::new(FakeTime::new(base));
    let db = DbHandle::open_in_memory().await?;
    let governor = RateGovernor::new(db.clone(), 1000.0, 4.0, 3000.0, time.clone());
    governor.bind(DID).await?;

    // day cap = 3.6, hour cap = 900: the hour never binds, the day does.
    governor
        .reserve_write(3.0, "createRecord", None, None)
        .await?;
    governor
        .reserve_write(3.0, "createRecord", None, None)
        .await?;

    let expected = base - base.rem_euclid(86400.0) + 86400.0 - base;
    assert_eq!(time.sleeps(), vec![expected]);
    assert_eq!(governor.waits(), 1);
    assert_eq!(governor.wait_seconds(), expected);
    assert_eq!(governor.used().await, (3.0, 3.0));
    db.close().await?;
    Ok(())
}

#[tokio::test]
async fn oversized_reservation_bypasses_instead_of_hanging() -> anyhow::Result<()> {
    let time = Arc::new(FakeTime::new(1_700_000_000.0));
    let db = DbHandle::open_in_memory().await?;
    let governor = RateGovernor::new(db.clone(), 4.0, 40.0, 3000.0, time.clone());
    governor.bind(DID).await?;

    // 10 points can never fit the 3.6-point hour cap; do not wait forever.
    governor
        .reserve_write(10.0, "applyWrites", Some("CVE-2024-0001"), None)
        .await?;

    assert!(time.sleeps().is_empty());
    assert_eq!(governor.waits(), 0);
    assert_eq!(governor.used().await, (10.0, 10.0));
    db.close().await?;
    Ok(())
}

#[tokio::test]
async fn request_pacing_stays_under_5min_cap() -> anyhow::Result<()> {
    let time = Arc::new(FakeTime::new(0.0));
    let db = DbHandle::open_in_memory().await?;
    let governor = RateGovernor::new(db.clone(), 5000.0, 35000.0, 2.0, time.clone());
    governor.bind(DID).await?;

    for _ in 0..3 {
        governor.request().await?;
    }

    let sleeps = time.sleeps();
    assert_eq!(sleeps.len(), 2);
    assert!(
        sleeps.iter().all(|seconds| (seconds - 150.0).abs() < 1e-6),
        "sleeps={sleeps:?}"
    );
    assert!(sleeps.iter().sum::<f64>() >= 300.0 - 1e-6);
    assert_eq!(governor.requests_5min(), 2);
    // Pacing sleeps are not budget waits.
    assert_eq!(governor.waits(), 0);
    assert_eq!(governor.wait_seconds(), 0.0);
    db.close().await?;
    Ok(())
}

#[tokio::test]
async fn request_token_bucket_spaces_beyond_burst() -> anyhow::Result<()> {
    let time = Arc::new(FakeTime::new(0.0));
    let db = DbHandle::open_in_memory().await?;
    // 10 req/s, burst 100: requests 101+ are spaced 0.1s apart.
    let governor = RateGovernor::new(db.clone(), 5000.0, 35000.0, 3000.0, time.clone());
    governor.bind(DID).await?;

    for _ in 0..101 {
        governor.request().await?;
    }

    let sleeps = time.sleeps();
    assert_eq!(sleeps.len(), 1);
    assert!((sleeps[0] - 0.1).abs() < 1e-9, "sleeps={sleeps:?}");
    assert_eq!(governor.requests_5min(), 101);
    assert_eq!(governor.waits(), 0);
    db.close().await?;
    Ok(())
}

#[tokio::test]
async fn request_never_sleeps_while_holding_the_mutex() -> anyhow::Result<()> {
    let time = Arc::new(FakeTime::gated(0.0));
    let db = DbHandle::open_in_memory().await?;
    let governor = Arc::new(RateGovernor::new(
        db.clone(),
        5000.0,
        35000.0,
        1.0,
        time.clone(),
    ));
    governor.bind(DID).await?;

    // Consume the burst token, then park a second request in a 300s pacing wait.
    governor.request().await?;
    let sleeper = {
        let governor = governor.clone();
        tokio::spawn(async move { governor.request().await })
    };
    tokio::time::timeout(Duration::from_secs(2), async {
        while time.sleeps().is_empty() {
            tokio::task::yield_now().await;
        }
    })
    .await
    .expect("request did not enter its pacing sleep");

    // Both need governor state; they hang if the sleeper kept the lock.
    tokio::time::timeout(Duration::from_secs(1), governor.remaining())
        .await
        .expect("remaining() blocked behind the pacing sleep");
    tokio::time::timeout(
        Duration::from_secs(1),
        governor.reserve_write(1.0, "createRecord", None, None),
    )
    .await
    .expect("reserve_write() blocked behind the pacing sleep")?;

    time.release(1);
    sleeper.await??;
    assert_eq!(governor.requests_5min(), 1);
    assert_eq!(governor.waits(), 0);
    db.close().await?;
    Ok(())
}