~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
570 lines · 17.0 KB Raw History
//! Per-host release rate governance tests.
//!
//! Every request is scripted through a local [`RegistryHttp`] fake and every
//! clock read comes from a fake [`Time`], so nothing sleeps or touches the
//! network.

use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use async_trait::async_trait;
use chrono::{DateTime, Utc};
use serde_json::{Value, json};

pub use vuln_notif::{db, release, time};

use vuln_notif::db::DbHandle;
use vuln_notif::release::http::{HttpStatusError, RegistryHttp, redact_url};
use vuln_notif::time::Time;

#[path = "../src/release/host.rs"]
mod host;

use host::{GovernedHttp, HostLimiter, HostLimiterConfig, host_of};

// ---------------------------------------------------------------------------
// Fake clock and registry transport
// ---------------------------------------------------------------------------

struct FakeTime {
    now: Mutex<f64>,
}

fn fake_time(now: f64) -> Arc<FakeTime> {
    Arc::new(FakeTime {
        now: Mutex::new(now),
    })
}

impl FakeTime {
    fn advance(&self, seconds: f64) {
        *self.now.lock().expect("fake clock") += seconds;
    }
}

#[async_trait]
impl Time for FakeTime {
    fn now_unix(&self) -> f64 {
        *self.now.lock().expect("fake clock")
    }

    fn now_utc(&self) -> DateTime<Utc> {
        DateTime::from_timestamp_millis((self.now_unix() * 1000.0) as i64).expect("timestamp")
    }

    async fn sleep(&self, duration: Duration) {
        self.advance(duration.as_secs_f64());
    }

    fn jitter(&self, _scale: f64) -> f64 {
        0.0
    }
}

#[derive(Clone, Debug)]
enum Reply {
    Json(Value),
    Text(String),
    Size(u64),
    Bytes(Vec<u8>),
    Status(HttpStatusError),
}

#[derive(Clone, Debug)]
struct Call {
    kind: &'static str,
    at: f64,
}

struct FakeHttp {
    time: Arc<FakeTime>,
    replies: Mutex<HashMap<String, VecDeque<Reply>>>,
    calls: Mutex<Vec<Call>>,
    headers: Mutex<Vec<(String, String)>>,
    body: Mutex<Option<Value>>,
    limit: Mutex<Option<u64>>,
}

impl FakeHttp {
    fn new(time: Arc<FakeTime>) -> Self {
        Self {
            time,
            replies: Mutex::new(HashMap::new()),
            calls: Mutex::new(Vec::new()),
            headers: Mutex::new(Vec::new()),
            body: Mutex::new(None),
            limit: Mutex::new(None),
        }
    }

    fn script(&self, url: &str, replies: Vec<Reply>) {
        self.replies
            .lock()
            .expect("replies")
            .insert(url.to_owned(), replies.into());
    }

    fn times(&self) -> Vec<f64> {
        self.calls
            .lock()
            .expect("calls")
            .iter()
            .map(|call| call.at)
            .collect()
    }

    fn kinds(&self) -> Vec<&'static str> {
        self.calls
            .lock()
            .expect("calls")
            .iter()
            .map(|call| call.kind)
            .collect()
    }

    fn take(&self, kind: &'static str, url: &str) -> anyhow::Result<Reply> {
        self.calls.lock().expect("calls").push(Call {
            kind,
            at: self.time.now_unix(),
        });
        self.replies
            .lock()
            .expect("replies")
            .get_mut(url)
            .and_then(VecDeque::pop_front)
            .ok_or_else(|| anyhow::anyhow!("unexpected {kind} {url}"))
    }
}

fn as_json(reply: Reply) -> anyhow::Result<Value> {
    match reply {
        Reply::Json(value) => Ok(value),
        Reply::Status(error) => Err(error.into()),
        other => Err(anyhow::anyhow!("expected JSON, got {other:?}")),
    }
}

fn as_bytes(reply: Reply) -> anyhow::Result<Vec<u8>> {
    match reply {
        Reply::Bytes(bytes) => Ok(bytes),
        Reply::Status(error) => Err(error.into()),
        other => Err(anyhow::anyhow!("expected bytes, got {other:?}")),
    }
}

#[async_trait]
impl RegistryHttp for FakeHttp {
    async fn get_json(&self, url: &str) -> anyhow::Result<Value> {
        as_json(self.take("json", url)?)
    }

    async fn get_json_with_headers(
        &self,
        url: &str,
        headers: &[(String, String)],
    ) -> anyhow::Result<Value> {
        *self.headers.lock().expect("headers") = headers.to_vec();
        as_json(self.take("json_headers", url)?)
    }

    async fn get_text(&self, url: &str) -> anyhow::Result<String> {
        match self.take("text", url)? {
            Reply::Text(text) => Ok(text),
            Reply::Status(error) => Err(error.into()),
            other => Err(anyhow::anyhow!("expected text, got {other:?}")),
        }
    }

    async fn head_size(&self, url: &str) -> anyhow::Result<u64> {
        match self.take("head", url)? {
            Reply::Size(size) => Ok(size),
            Reply::Status(error) => Err(error.into()),
            other => Err(anyhow::anyhow!("expected size, got {other:?}")),
        }
    }

    async fn download(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        as_bytes(self.take("download", url)?)
    }

    async fn download_limited(&self, url: &str, max_bytes: u64) -> anyhow::Result<Vec<u8>> {
        *self.limit.lock().expect("limit") = Some(max_bytes);
        as_bytes(self.take("download_limited", url)?)
    }

    async fn post_json(&self, url: &str, body: &Value) -> anyhow::Result<Value> {
        *self.body.lock().expect("body") = Some(body.clone());
        as_json(self.take("post", url)?)
    }
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

fn make_limiter(time: &Arc<FakeTime>, rate: f64, burst: u32) -> Arc<HostLimiter> {
    let clock: Arc<dyn Time> = time.clone();
    Arc::new(HostLimiter::with_clock(
        HostLimiterConfig {
            requests_per_second: rate,
            burst,
            jitter_seconds: 0.0,
        },
        clock,
    ))
}

fn governed(http: &Arc<FakeHttp>, limiter: Arc<HostLimiter>) -> GovernedHttp {
    let inner: Arc<dyn RegistryHttp> = http.clone();
    GovernedHttp::in_memory(inner, limiter, "did:plc:test")
}

fn retry_after_error(status: u16, url: &str, retry_after: &str) -> HttpStatusError {
    let mut error = HttpStatusError::new(status, url);
    error.retry_after = Some(retry_after.to_owned());
    error
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[tokio::test]
async fn default_limiter_uses_the_production_clock() {
    let limiter = HostLimiter::new(HostLimiterConfig::default());
    assert!(limiter.is_blocked("idle.example").is_none());
}

#[tokio::test]
async fn token_bucket_limits_sustained_rate() {
    let time = fake_time(1_000.0);
    let http = Arc::new(FakeHttp::new(time.clone()));
    let url = "https://bucket.example/pkg";
    http.script(url, vec![Reply::Json(json!({"ok": true})); 4]);
    let governor = governed(&http, make_limiter(&time, 2.0, 2));

    for _ in 0..4 {
        governor.get_json(url).await.expect("get");
    }

    let times = http.times();
    assert!((times[0] - 1_000.0).abs() < 1e-9, "{times:?}");
    assert!((times[1] - 1_000.0).abs() < 1e-9, "{times:?}");
    assert!((times[2] - 1_000.5).abs() < 1e-9, "{times:?}");
    assert!((times[3] - 1_001.0).abs() < 1e-9, "{times:?}");
}

#[tokio::test]
async fn retry_after_seconds_blocks_the_host_until_release() {
    let time = fake_time(1_000.0);
    let http = Arc::new(FakeHttp::new(time.clone()));
    let url = "https://rate.example/pkg";
    http.script(
        url,
        vec![
            Reply::Status(retry_after_error(429, url, "2")),
            Reply::Json(json!({"ok": true})),
        ],
    );
    let limiter = make_limiter(&time, 100.0, 100);
    let governor = governed(&http, limiter.clone());

    let error = governor.get_json(url).await.expect_err("429 propagates");
    assert!(error.downcast_ref::<HttpStatusError>().is_some());
    let blocked = limiter.is_blocked("rate.example").expect("host is blocked");
    assert!((blocked.as_secs_f64() - 2.0).abs() < 1e-6, "{blocked:?}");

    let value = governor.get_json(url).await.expect("after release");
    assert_eq!(value, json!({"ok": true}));
    let times = http.times();
    assert!((times[1] - times[0] - 2.0).abs() < 1e-6, "{times:?}");
    assert!(limiter.is_blocked("rate.example").is_none());
}

#[tokio::test]
async fn retry_after_http_date_is_parsed() {
    let now = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .expect("clock")
        .as_secs_f64();
    let time = fake_time(now);
    let http = Arc::new(FakeHttp::new(time.clone()));
    let url = "https://date.example/pkg";
    let retry_at = SystemTime::now() + Duration::from_secs(30);
    http.script(
        url,
        vec![Reply::Status(retry_after_error(
            429,
            url,
            &httpdate::fmt_http_date(retry_at),
        ))],
    );
    let limiter = make_limiter(&time, 100.0, 100);
    let governor = governed(&http, limiter.clone());

    assert!(governor.get_json(url).await.is_err());
    let blocked = limiter
        .is_blocked("date.example")
        .expect("host is blocked")
        .as_secs_f64();
    assert!(blocked > 25.0 && blocked <= 31.0, "{blocked}");
}

#[tokio::test]
async fn hosts_have_independent_buckets() {
    let time = fake_time(5_000.0);
    let http = Arc::new(FakeHttp::new(time.clone()));
    http.script("https://one.example/a", vec![Reply::Json(json!(1)); 2]);
    http.script("https://two.example/b", vec![Reply::Json(json!(2))]);
    let governor = governed(&http, make_limiter(&time, 1.0, 1));

    governor
        .get_json("https://one.example/a")
        .await
        .expect("one");
    governor
        .get_json("https://two.example/b")
        .await
        .expect("two");
    governor
        .get_json("https://one.example/a")
        .await
        .expect("one again");

    let times = http.times();
    assert!((times[1] - times[0]).abs() < 1e-9, "{times:?}");
    assert!((times[2] - times[0] - 1.0).abs() < 1e-9, "{times:?}");
}

#[tokio::test]
async fn state_round_trips_through_source_host_state() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = DbHandle::open(temp.path().join("cache.db"))
        .await
        .expect("open db");
    let now = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .expect("clock")
        .as_secs_f64();
    let time = fake_time(now);
    let http = Arc::new(FakeHttp::new(time.clone()));
    let url = "https://persist.example/pkg";
    http.script(
        url,
        vec![
            Reply::Status(retry_after_error(429, url, "2")),
            Reply::Json(json!({"ok": true})),
        ],
    );

    let limiter = make_limiter(&time, 100.0, 100);
    let governor = GovernedHttp::new(http.clone(), limiter, Some(db.clone()), "did:plc:test")
        .await
        .expect("governor");
    assert_eq!(governor.repo_did(), "did:plc:test");
    assert!(governor.get_json(url).await.is_err());

    let row = db
        .get_source_host_state("persist.example")
        .await
        .expect("get state")
        .expect("persisted row");
    assert_eq!(row.requests, 1);
    assert_eq!(row.window_start, now);
    assert!(row.blocked_until.is_some());

    // A fresh governor at the same instant restores the live block.
    let limiter = make_limiter(&time, 100.0, 100);
    let governor = GovernedHttp::new(
        http.clone(),
        limiter.clone(),
        Some(db.clone()),
        "did:plc:test",
    )
    .await
    .expect("reload");
    let blocked = limiter
        .is_blocked("persist.example")
        .expect("block restored");
    assert!((blocked.as_secs_f64() - 2.0).abs() < 0.01, "{blocked:?}");

    governor.get_json(url).await.expect("after release");
    let row = db
        .get_source_host_state("persist.example")
        .await
        .expect("get state")
        .expect("persisted row");
    assert_eq!(row.requests, 1);
    assert!(row.window_start > now + 1.9, "{}", row.window_start);
    assert!(row.blocked_until.is_none());
    db.close().await.expect("close db");
}

#[tokio::test]
async fn every_transport_method_is_governed() {
    let time = fake_time(9_000.0);
    let http = Arc::new(FakeHttp::new(time.clone()));
    http.script(
        "https://all.example/json",
        vec![Reply::Json(json!({"a": 1}))],
    );
    http.script(
        "https://all.example/headers",
        vec![Reply::Json(json!({"b": 2}))],
    );
    http.script(
        "https://all.example/text",
        vec![Reply::Text("hello".to_owned())],
    );
    http.script("https://all.example/head", vec![Reply::Size(42)]);
    http.script(
        "https://all.example/download",
        vec![Reply::Bytes(vec![1, 2, 3])],
    );
    http.script(
        "https://all.example/limited",
        vec![Reply::Bytes(vec![4, 5])],
    );
    http.script(
        "https://all.example/post",
        vec![Reply::Json(json!({"posted": true}))],
    );
    let governor = governed(&http, make_limiter(&time, 1.0, 1));

    assert_eq!(
        governor
            .get_json("https://all.example/json")
            .await
            .expect("json"),
        json!({"a": 1})
    );
    assert_eq!(
        governor
            .get_json_with_headers(
                "https://all.example/headers",
                &[("authorization".to_owned(), "Bearer secret".to_owned())],
            )
            .await
            .expect("headers"),
        json!({"b": 2})
    );
    assert_eq!(
        governor
            .get_text("https://all.example/text")
            .await
            .expect("text"),
        "hello"
    );
    assert_eq!(
        governor
            .head_size("https://all.example/head")
            .await
            .expect("head"),
        42
    );
    assert_eq!(
        governor
            .download("https://all.example/download")
            .await
            .expect("download"),
        vec![1, 2, 3]
    );
    assert_eq!(
        governor
            .download_limited("https://all.example/limited", 2)
            .await
            .expect("limited"),
        vec![4, 5]
    );
    assert_eq!(
        governor
            .post_json("https://all.example/post", &json!({"x": 1}))
            .await
            .expect("post"),
        json!({"posted": true})
    );

    assert_eq!(
        http.kinds(),
        vec![
            "json",
            "json_headers",
            "text",
            "head",
            "download",
            "download_limited",
            "post",
        ]
    );
    assert_eq!(
        *http.headers.lock().expect("headers"),
        vec![("authorization".to_owned(), "Bearer secret".to_owned())]
    );
    assert_eq!(*http.body.lock().expect("body"), Some(json!({"x": 1})));
    assert_eq!(*http.limit.lock().expect("limit"), Some(2));

    let times = http.times();
    for window in times.windows(2) {
        assert!((window[1] - window[0] - 1.0).abs() < 1e-9, "{times:?}");
    }
    assert!((times[6] - 9_006.0).abs() < 1e-9, "{times:?}");
}

#[tokio::test]
async fn forbidden_with_rate_limit_headers_backs_off() {
    let now = 20_000.0;
    let time = fake_time(now);
    let http = Arc::new(FakeHttp::new(time.clone()));
    let url = "https://gh.example/repos";
    let mut forbidden = HttpStatusError::new(403, url);
    forbidden.rate_limit_remaining = Some(0);
    forbidden.rate_limit_reset = Some((now + 10.0) as u64);
    http.script(
        url,
        vec![Reply::Status(forbidden), Reply::Json(json!({"ok": true}))],
    );
    let auth_url = "https://auth.example/private";
    http.script(
        auth_url,
        vec![Reply::Status(HttpStatusError::new(403, auth_url))],
    );
    let limiter = make_limiter(&time, 100.0, 100);
    let governor = governed(&http, limiter.clone());

    assert!(governor.get_json(url).await.is_err());
    let blocked = limiter
        .is_blocked("gh.example")
        .expect("rate-limited 403 blocks")
        .as_secs_f64();
    assert!((blocked - 10.0).abs() < 1e-6, "{blocked}");
    governor.get_json(url).await.expect("after reset");
    assert!((http.times()[1] - now - 10.0).abs() < 1e-6);

    // A plain authorization 403 is not a rate limit.
    assert!(governor.get_json(auth_url).await.is_err());
    assert!(limiter.is_blocked("auth.example").is_none());
}

#[tokio::test]
async fn errors_and_logs_do_not_leak_credentials() {
    let url = "https://user:sekret@api.example/v1/pkg?token=supersecret&x=1";
    let time = fake_time(100.0);
    let http = Arc::new(FakeHttp::new(time.clone()));
    http.script(url, vec![Reply::Status(retry_after_error(429, url, "0"))]);
    let governor = governed(&http, make_limiter(&time, 100.0, 100));

    let error = governor.get_json(url).await.expect_err("429 propagates");
    let text = format!("{error:#}");
    assert!(!text.contains("sekret"), "{text}");
    assert!(!text.contains("supersecret"), "{text}");
    assert!(!text.contains("user:"), "{text}");

    let status = error
        .downcast_ref::<HttpStatusError>()
        .expect("status error");
    assert_eq!(status.url, "https://api.example/v1/pkg");
    assert_eq!(host_of(url), "api.example");
    assert_eq!(redact_url(url), "https://api.example/v1/pkg");
    assert_eq!(redact_url("/relative?token=x"), "/relative");
}