~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
635 lines · 18.5 KB Raw History
//! NVD client: probe, pagination, request pacing, and outer retry.
//!
//! The client owns an injectable [`NvdTransport`] so integration tests never
//! touch the network.

use std::collections::HashSet;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::Duration;

use async_trait::async_trait;
use chrono::{DateTime, Utc};
use serde_json::{Value, json};
use tokio::sync::{Mutex as AsyncMutex, Semaphore};

use crate::constants::{NVD_RETRY_DELAYS, NVD_SOCK_TIMEOUT, NVD_TIMEOUT};
use crate::log::{Level, Logger};
use crate::time::{Time, nvd_time, parse_nvd_time};

pub const NVD_USER_AGENT: &str = "cve-bin-tool/3.4.1 (https://github.com/ossf/cve-bin-tool/)";

const RATE: f64 = 10.0;
const MAX_TOKENS: f64 = 10.0;
const INTERVAL_SECONDS: f64 = 6.0;
const KEY_INTERVAL_SECONDS: f64 = 1.0;
const JITTER_SCALE: f64 = 5.0;
const NO_KEY_CONCURRENCY: usize = 1;
const KEY_CONCURRENCY: usize = 19;
/// Page requests are bounded: one initial attempt plus one per configured
/// backoff delay (30/60/120/300 seconds), then a typed failure.
const PAGE_ATTEMPTS: usize = NVD_RETRY_DELAYS.len() + 1;

/// One NVD HTTP response. `body` is [`Value::Null`] when the payload is not JSON.
#[derive(Debug, Clone)]
pub struct NvdResponse {
    pub status: u16,
    pub body: Value,
}

/// HTTP seam for the NVD client. Production uses [`ReqwestTransport`]; tests
/// script responses.
#[async_trait]
pub trait NvdTransport: Send + Sync {
    async fn get(
        &self,
        url: &str,
        params: &[(&str, String)],
        api_key: Option<&str>,
    ) -> anyhow::Result<NvdResponse>;
}

/// reqwest-backed transport with the cve-bin-tool timeouts and user agent.
pub struct ReqwestTransport {
    client: reqwest::Client,
}

impl ReqwestTransport {
    pub fn new() -> anyhow::Result<Self> {
        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs_f64(NVD_TIMEOUT))
            .connect_timeout(Duration::from_secs_f64(NVD_SOCK_TIMEOUT))
            .read_timeout(Duration::from_secs_f64(NVD_SOCK_TIMEOUT))
            .user_agent(NVD_USER_AGENT)
            .build()?;
        Ok(Self { client })
    }
}

#[async_trait]
impl NvdTransport for ReqwestTransport {
    async fn get(
        &self,
        url: &str,
        params: &[(&str, String)],
        api_key: Option<&str>,
    ) -> anyhow::Result<NvdResponse> {
        let mut request = self.client.get(url).query(params);
        if let Some(key) = api_key.filter(|key| !key.is_empty()) {
            request = request.header("apiKey", key);
        }
        let response = request.send().await?;
        let status = response.status().as_u16();
        let text = response.text().await?;
        let body = serde_json::from_str(&text).unwrap_or(Value::Null);
        if status == 200 && body.is_null() {
            anyhow::bail!("NVD returned HTTP 200 with a non-JSON body");
        }
        Ok(NvdResponse { status, body })
    }
}

pub struct NvdClient {
    api_key: Option<String>,
    page_size: usize,
    base_url: String,
    time: Arc<dyn Time>,
    logger: Logger,
    transport: Arc<dyn NvdTransport>,
    key_logged: Arc<AtomicBool>,
}

impl Clone for NvdClient {
    fn clone(&self) -> Self {
        Self {
            api_key: self.api_key.clone(),
            page_size: self.page_size,
            base_url: self.base_url.clone(),
            time: Arc::clone(&self.time),
            logger: self.logger.clone(),
            transport: Arc::clone(&self.transport),
            key_logged: Arc::clone(&self.key_logged),
        }
    }
}

impl std::fmt::Debug for NvdClient {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("NvdClient")
            .field("api_key_configured", &self.effective_key().is_some())
            .field("page_size", &self.page_size)
            .field("base_url", &self.base_url)
            .finish_non_exhaustive()
    }
}

impl NvdClient {
    pub fn new(
        api_key: Option<String>,
        page_size: usize,
        base_url: String,
        time: Arc<dyn Time>,
    ) -> Self {
        let transport = ReqwestTransport::new().expect("build NVD HTTP client");
        Self {
            api_key,
            page_size,
            base_url,
            time,
            logger: Logger::stderr(Level::Info).with_target("nvd"),
            transport: Arc::new(transport),
            key_logged: Arc::new(AtomicBool::new(false)),
        }
    }

    /// Replace the HTTP transport (tests).
    pub fn with_transport(mut self, transport: Arc<dyn NvdTransport>) -> Self {
        self.transport = transport;
        self
    }

    /// Replace the logger (tests / runtime wiring).
    pub fn with_logger(mut self, logger: Logger) -> Self {
        self.logger = logger;
        self
    }

    pub fn api_key(&self) -> Option<&str> {
        self.api_key.as_deref()
    }

    pub fn page_size(&self) -> usize {
        self.page_size
    }

    pub fn base_url(&self) -> &str {
        &self.base_url
    }

    fn effective_key(&self) -> Option<&str> {
        self.api_key.as_deref().filter(|key| !key.is_empty())
    }
}

/// Startup signal mirroring `vuln_notif.nvd.log_nvd_key_configured`; never
/// includes the key itself.
pub fn log_nvd_key_configured(logger: &Logger, api_key: Option<&str>) {
    if api_key.is_some_and(|key| !key.is_empty()) {
        logger.info(
            "nvd_api_key_configured",
            None,
            json!({ "configured": true }),
        );
    } else {
        logger.warn(
            "nvd_api_key_configured",
            None,
            json!({
                "configured": false,
                "hint": "NVD_API_KEY unset: 5 requests/30s, ~6s between calls",
            }),
        );
    }
}

#[derive(Debug, Clone, Default)]
pub struct FetchResult {
    pub cves: Vec<Value>,
    pub cursor: String,
    pub requests: u64,
    pub seconds: f64,
    pub retries: u64,
}

/// Errors classified for the outer retry loop.
#[derive(Debug)]
struct NvdFailure {
    retryable: bool,
    name: &'static str,
    message: String,
}

impl std::fmt::Display for NvdFailure {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str(&self.message)
    }
}

impl std::error::Error for NvdFailure {}

impl NvdFailure {
    fn rate_limit(message: impl Into<String>) -> anyhow::Error {
        anyhow::Error::new(Self {
            retryable: true,
            name: "NVDRateLimit",
            message: message.into(),
        })
    }

    fn fatal(message: impl Into<String>) -> anyhow::Error {
        anyhow::Error::new(Self {
            retryable: false,
            name: "NVDServiceError",
            message: message.into(),
        })
    }
}

fn is_retryable(error: &anyhow::Error) -> bool {
    error
        .downcast_ref::<NvdFailure>()
        .is_none_or(|failure| failure.retryable)
}

fn error_name(error: &anyhow::Error) -> String {
    if let Some(failure) = error.downcast_ref::<NvdFailure>() {
        return failure.name.to_owned();
    }
    if error
        .downcast_ref::<reqwest::Error>()
        .is_some_and(reqwest::Error::is_timeout)
    {
        return "TimeoutError".to_owned();
    }
    "ClientError".to_owned()
}

struct TokenBucket {
    tokens: f64,
    updated_at: f64,
}

impl TokenBucket {
    fn new(now: f64) -> Self {
        Self {
            tokens: MAX_TOKENS,
            updated_at: now,
        }
    }

    async fn take(&mut self, time: &dyn Time) {
        loop {
            self.refill(time.now_unix());
            if self.tokens >= 1.0 {
                self.tokens -= 1.0;
                return;
            }
            time.sleep(Duration::from_millis(100)).await;
        }
    }

    fn refill(&mut self, now: f64) {
        let new_tokens = (now - self.updated_at) * RATE;
        if self.tokens + new_tokens >= 1.0 {
            self.tokens = (self.tokens + new_tokens).min(MAX_TOKENS);
            self.updated_at = now;
        }
    }
}

struct FetchState {
    effective_key: Option<String>,
    requests: AtomicU64,
    tokens: AsyncMutex<TokenBucket>,
    semaphore: Option<Arc<Semaphore>>,
}

impl FetchState {
    fn pacing_interval(&self) -> f64 {
        if self.effective_key.is_some() {
            KEY_INTERVAL_SECONDS
        } else {
            INTERVAL_SECONDS
        }
    }

    fn request_count(&self) -> u64 {
        self.requests.load(Ordering::Relaxed)
    }
}

struct Page {
    entries: Vec<Value>,
    total_results: Option<u64>,
}

/// Fetch NVD CVEs since `cursor` (None = full backfill).
pub async fn fetch_nvd(client: &NvdClient, cursor: Option<&str>) -> anyhow::Result<FetchResult> {
    let since =
        match cursor {
            Some(raw) => Some(parse_nvd_time(raw).map_err(|error| {
                NvdFailure::fatal(format!("invalid NVD cursor {raw:?}: {error}"))
            })?),
            None => None,
        };

    if !client.key_logged.swap(true, Ordering::Relaxed) {
        log_nvd_key_configured(&client.logger, client.api_key.as_deref());
    }

    let mut retries = 0;
    let attempts = NVD_RETRY_DELAYS.len() + 1;
    for attempt in 1..=attempts {
        match fetch_once(client, since).await {
            Ok(mut result) => {
                result.retries = retries;
                return Ok(result);
            }
            Err(error) => {
                if !is_retryable(&error) || attempt == attempts {
                    return Err(error);
                }
                let delay = NVD_RETRY_DELAYS[attempt - 1] + client.time.jitter(JITTER_SCALE);
                retries += 1;
                client.logger.warn(
                    "nvd_retry",
                    None,
                    json!({
                        "attempt": attempt,
                        "error": error_name(&error),
                        "sleep_seconds": round3(delay),
                    }),
                );
                client.time.sleep(Duration::from_secs_f64(delay)).await;
            }
        }
    }
    unreachable!("outer retry loop always returns")
}

async fn fetch_once(
    client: &NvdClient,
    since: Option<DateTime<Utc>>,
) -> anyhow::Result<FetchResult> {
    let started_at = client.time.now_utc();
    let started_unix = client.time.now_unix();
    let mut state = FetchState {
        effective_key: client.effective_key().map(str::to_owned),
        requests: AtomicU64::new(0),
        tokens: AsyncMutex::new(TokenBucket::new(started_unix)),
        semaphore: None,
    };

    client.logger.debug(
        "nvd_fetch_start",
        None,
        json!({
            "since": since.map(nvd_time),
            "page_size": client.page_size,
        }),
    );

    let Some(probe_total) = probe(client, &mut state).await? else {
        // Invalid API key: mark the API invalid and fetch nothing.
        return Ok(fetch_result(
            started_at,
            started_unix,
            client,
            &state,
            Vec::new(),
        ));
    };

    state.semaphore = Some(Arc::new(Semaphore::new(if state.effective_key.is_some() {
        KEY_CONCURRENCY
    } else {
        NO_KEY_CONCURRENCY
    })));

    let date_params = since.map(|since| {
        (
            nvd_time(since - chrono::Duration::minutes(2)),
            nvd_time(started_at),
        )
    });

    let mut entries = Vec::new();
    let mut total = probe_total;
    let page_size = client.page_size.max(1) as u64;
    let mut page_count = total.div_ceil(page_size) as usize;

    if date_params.is_some() {
        let page = fetch_page(client, &state, 0, date_params.as_ref()).await?;
        total = page.total_results.unwrap_or(total);
        page_count = total.div_ceil(page_size) as usize;
        entries.extend(page.entries);
    }

    let first_page = usize::from(date_params.is_some());
    let pages = (first_page..page_count)
        .map(|index| fetch_page(client, &state, index, date_params.as_ref()));
    for page in futures::future::join_all(pages).await {
        entries.extend(page?.entries);
    }

    let cves = unwrap_entries(&client.logger, &entries);
    Ok(fetch_result(started_at, started_unix, client, &state, cves))
}

fn fetch_result(
    started_at: DateTime<Utc>,
    started_unix: f64,
    client: &NvdClient,
    state: &FetchState,
    cves: Vec<Value>,
) -> FetchResult {
    FetchResult {
        cves,
        cursor: nvd_time(started_at),
        requests: state.request_count(),
        seconds: (client.time.now_unix() - started_unix).max(0.0),
        retries: 0,
    }
}

/// Probe `startIndex=0&resultsPerPage=1` without date filters.
///
/// `Ok(None)` means the API key was rejected and no CVE fetch should happen.
async fn probe(client: &NvdClient, state: &mut FetchState) -> anyhow::Result<Option<u64>> {
    let response = probe_request(client, state).await?;
    check_probe_status(client, &response)?;

    if !truthy(response.body.get("error")) {
        return Ok(total_results(&response.body));
    }

    // Invalid key/error body: retry once without the key, then treat as invalid.
    if state.effective_key.is_none() {
        warn_invalid_key(client);
        return Ok(None);
    }
    state.effective_key = None;
    let response = probe_request(client, state).await?;
    check_probe_status(client, &response)?;
    if truthy(response.body.get("error")) {
        warn_invalid_key(client);
        return Ok(None);
    }
    Ok(total_results(&response.body))
}

async fn probe_request(client: &NvdClient, state: &mut FetchState) -> anyhow::Result<NvdResponse> {
    let params = [
        ("startIndex", "0".to_owned()),
        ("resultsPerPage", "1".to_owned()),
    ];
    request(client, state, &params).await
}

fn check_probe_status(client: &NvdClient, response: &NvdResponse) -> anyhow::Result<()> {
    match response.status {
        200 => Ok(()),
        403 => Err(NvdFailure::rate_limit(format!(
            "NVD {}: got 403 Forbidden; rate-limited or egress IP blocked",
            client.base_url
        ))),
        status => Err(NvdFailure::fatal(format!(
            "NVD {}: probe returned HTTP {status}",
            client.base_url
        ))),
    }
}

fn warn_invalid_key(client: &NvdClient) {
    client
        .logger
        .warn("nvd_api_key_invalid", None, json!({ "configured": false }));
}

fn total_results(body: &Value) -> Option<u64> {
    body.get("totalResults").and_then(Value::as_u64)
}

/// JSON truthiness for the probe body's `error` field.
fn truthy(value: Option<&Value>) -> bool {
    match value {
        None | Some(Value::Null) => false,
        Some(Value::Bool(value)) => *value,
        Some(Value::Number(number)) => number.as_f64().is_some_and(|value| value != 0.0),
        Some(Value::String(value)) => !value.is_empty(),
        Some(Value::Array(value)) => !value.is_empty(),
        Some(Value::Object(value)) => !value.is_empty(),
    }
}

async fn fetch_page(
    client: &NvdClient,
    state: &FetchState,
    index: usize,
    date_params: Option<&(String, String)>,
) -> anyhow::Result<Page> {
    let page_size = client.page_size.max(1);
    let mut params = vec![
        ("startIndex", (index * page_size).to_string()),
        ("resultsPerPage", page_size.to_string()),
    ];
    if let Some((start, end)) = date_params {
        params.push(("lastModStartDate", start.clone()));
        params.push(("lastModEndDate", end.clone()));
    }

    for attempt in 1..=PAGE_ATTEMPTS {
        let failure = match request(client, state, &params).await {
            Ok(response) if response.status == 200 => {
                if let Some(entries) = response
                    .body
                    .get("vulnerabilities")
                    .and_then(Value::as_array)
                {
                    let interval = state.pacing_interval();
                    client.time.sleep(Duration::from_secs_f64(interval)).await;
                    return Ok(Page {
                        entries: entries.clone(),
                        total_results: total_results(&response.body),
                    });
                }
                "HTTP 200 without vulnerabilities".to_owned()
            }
            Ok(response) => format!("HTTP {}", response.status),
            Err(error) => error_name(&error),
        };
        if attempt == PAGE_ATTEMPTS {
            return Err(NvdFailure::fatal(format!(
                "NVD {}: page {index} failed after {PAGE_ATTEMPTS} attempts ({failure})",
                client.base_url
            )));
        }
        let delay = NVD_RETRY_DELAYS[attempt - 1] + client.time.jitter(JITTER_SCALE);
        client.logger.warn(
            "nvd_page_retry",
            None,
            json!({
                "page": index,
                "attempt": attempt,
                "failure": failure,
                "sleep_seconds": round3(delay),
            }),
        );
        client.time.sleep(Duration::from_secs_f64(delay)).await;
    }
    unreachable!("page retry loop always returns")
}

/// One paced HTTP request. The concurrency permit is released before the
/// caller sleeps, so a sleeping page never blocks another request.
async fn request(
    client: &NvdClient,
    state: &FetchState,
    params: &[(&str, String)],
) -> anyhow::Result<NvdResponse> {
    {
        let mut tokens = state.tokens.lock().await;
        tokens.take(client.time.as_ref()).await;
    }
    let permit = match &state.semaphore {
        Some(semaphore) => Some(
            Arc::clone(semaphore)
                .acquire_owned()
                .await
                .expect("NVD semaphore closed"),
        ),
        None => None,
    };
    let response = client
        .transport
        .get(&client.base_url, params, state.effective_key.as_deref())
        .await;
    drop(permit);
    state.requests.fetch_add(1, Ordering::Relaxed);
    response
}

fn unwrap_entries(logger: &Logger, entries: &[Value]) -> Vec<Value> {
    let mut seen = HashSet::new();
    let mut cves = Vec::new();
    for entry in entries {
        let cve = entry
            .as_object()
            .and_then(|entry| entry.get("cve"))
            .and_then(Value::as_object);
        let id = cve
            .and_then(|cve| cve.get("id"))
            .and_then(Value::as_str)
            .filter(|id| !id.is_empty());
        let (Some(cve), Some(id)) = (cve, id) else {
            logger.warn(
                "nvd_entry_skipped",
                None,
                json!({ "entry": truncate_chars(&entry.to_string(), 200) }),
            );
            continue;
        };
        if seen.insert(id.to_owned()) {
            cves.push(Value::Object(cve.clone()));
        }
    }
    cves
}

fn truncate_chars(value: &str, limit: usize) -> String {
    value.chars().take(limit).collect()
}

fn round3(value: f64) -> f64 {
    (value * 1000.0).round() / 1000.0
}