~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
575 lines · 18.4 KB Raw History
//! clap-native CLI: every flag/environment variable from the reference service,
//! resolved into a validated [`RunConfig`]. No config file.

use std::ffi::OsString;
use std::path::PathBuf;
use std::process::ExitCode;

use clap::{ArgAction, Parser};
use serde_json::json;

use crate::config::{
    DEFAULT_CACHE, DEFAULT_PLC_URL, DEFAULT_PURLDB, EXIT_FATAL, EXIT_OK, EXIT_USAGE, Order,
    ReleaseMode, RunConfig, S3Config,
};
use crate::constants::{
    COLLECTION, DEFAULT_S3_PATH, NVD_API_URL, PDS_REQUESTS_PER_5MIN, PDS_WRITE_DAILY,
    PDS_WRITE_HOURLY, RELEASE_RECHECK_SECONDS,
};
use crate::log::{Level, Logger};

#[derive(Parser)]
#[command(
    name = "vuln-notif",
    version,
    about = "Mirror NVD CVEs into atproto as com.nahdig.cve records."
)]
pub struct Cli {
    /// Run one cycle, then exit.
    #[arg(long)]
    pub once: bool,

    /// Fetch but publish nothing.
    #[arg(long)]
    pub dry_run: bool,

    /// Max cache rows to publish per cycle (cursor still advances).
    #[arg(long, default_value_t = 0)]
    pub limit: i64,

    /// No cursor: backfill from 1999.
    #[arg(long)]
    pub backfill: bool,

    /// Initial cursor lookback when the cache has no cursor.
    #[arg(long, default_value_t = 7)]
    pub initial_lookback_days: i64,

    /// Pending publish order.
    #[arg(long, value_enum, env = "ORDER", default_value_t = Order::Cache)]
    pub order: Order,

    /// After NVD v2 publish, create software.release + atprotoV1 records.
    #[arg(long)]
    pub publish_atproto: bool,

    /// Max CVEs to enrich with software/atprotoV1 this run (0 = all pending).
    #[arg(long, default_value_t = 0)]
    pub software_limit: i64,

    /// Publish latest-version software.release.* records for known software.
    #[arg(long)]
    pub publish_releases: bool,

    /// Release ingestion mode: latest, affected, history, extras, or all.
    #[arg(
        long,
        env = "RELEASE_MODE",
        value_enum,
        default_value_t = ReleaseMode::Latest
    )]
    pub release_mode: ReleaseMode,

    /// Max software projects to inspect for release records (0 = all).
    #[arg(long, default_value_t = 0)]
    pub release_limit: i64,

    /// Restrict release work to these purl types (comma-separated).
    #[arg(long, value_delimiter = ',')]
    pub release_ecosystems: Vec<String>,

    /// Restrict release work to these sources (comma-separated; also matches
    /// adapter names such as github or docker).
    #[arg(long, value_delimiter = ',')]
    pub release_sources: Vec<String>,

    /// Max versions per project for history mode (0 = all).
    #[arg(long, default_value_t = 0)]
    pub release_history_limit: usize,

    /// Concurrent release source requests (history projects; affected runs in
    /// order).
    #[arg(long, default_value_t = 4)]
    pub release_concurrency: usize,

    /// Source base-URL override (local e2e; sources with configurable URLs).
    #[arg(long, env = "RELEASE_SOURCE_URL")]
    pub release_source_url: Option<String>,

    /// Artifact download byte cap (0 = no cap).
    #[arg(long, default_value_t = 0)]
    pub artifact_max_bytes: u64,

    /// S3 bucket for the artifact mirror (disabled when unset).
    #[arg(long, env = "ARTIFACT_S3_BUCKET")]
    pub artifact_s3_bucket: Option<String>,

    /// S3 key prefix for mirrored artifacts.
    #[arg(long, default_value = "artifacts")]
    pub artifact_s3_path: String,

    /// GitHub token for release sources (never logged).
    #[arg(long, env = "GITHUB_TOKEN")]
    pub github_token: Option<String>,

    /// GitLab token for release sources (never logged).
    #[arg(long, env = "GITLAB_TOKEN")]
    pub gitlab_token: Option<String>,

    /// Gitee token for release sources (never logged).
    #[arg(long, env = "GITEE_TOKEN")]
    pub gitee_token: Option<String>,

    /// Query OSV online (default). Disable with --osv-offline.
    #[arg(
        long,
        action = ArgAction::Set,
        num_args = 0..=1,
        default_missing_value = "true",
        default_value_t = true,
        value_parser = clap::builder::BoolishValueParser::new()
    )]
    pub osv_online: bool,

    /// Load OSV advisories from disk instead of the network.
    #[arg(long)]
    pub osv_offline: bool,

    /// Directory of OSV bulk advisory JSON files (implies offline).
    #[arg(long)]
    pub osv_bulk_dir: Option<PathBuf>,

    /// Skip release projects checked more recently than this.
    #[arg(
        long,
        env = "RELEASE_RECHECK_SECONDS",
        default_value_t = RELEASE_RECHECK_SECONDS
    )]
    pub release_recheck_seconds: f64,

    /// purl2cpe SQLite path (downloaded if missing).
    #[arg(long, default_value = DEFAULT_PURLDB)]
    pub purldb: PathBuf,

    /// SQLite cache path.
    #[arg(long, default_value = DEFAULT_CACHE)]
    pub cache_db: PathBuf,

    /// Seconds between cycles.
    #[arg(long, env = "POLL_INTERVAL", default_value_t = 300.0)]
    pub poll_interval: f64,

    /// NVD results per page.
    #[arg(long, default_value_t = 2000)]
    pub page_size: usize,

    /// Concurrent PDS row operations.
    #[arg(long, env = "CONCURRENCY", default_value_t = 8)]
    pub concurrency: i64,

    /// PDS request timeout in seconds.
    #[arg(long, env = "PDS_TIMEOUT", default_value_t = 60.0)]
    pub pds_timeout: f64,

    /// NVD API key (never logged).
    #[arg(long, env = "NVD_API_KEY")]
    pub nvd_api_key: Option<String>,

    /// Fail --backfill when no NVD API key is configured.
    #[arg(
        long,
        env = "REQUIRE_NVD_KEY",
        action = ArgAction::SetTrue,
        value_parser = clap::builder::BoolishValueParser::new()
    )]
    pub require_nvd_key: bool,

    /// PDS write-point budget per hour.
    #[arg(
        long,
        env = "PDS_WRITE_HOURLY",
        default_value_t = PDS_WRITE_HOURLY
    )]
    pub pds_hourly_points: f64,

    /// PDS write-point budget per day.
    #[arg(long, env = "PDS_WRITE_DAILY", default_value_t = PDS_WRITE_DAILY)]
    pub pds_daily_points: f64,

    /// PDS request cap per 5 minutes.
    #[arg(
        long,
        env = "PDS_REQUESTS_PER_5MIN",
        default_value_t = PDS_REQUESTS_PER_5MIN
    )]
    pub pds_requests_per_5min: f64,

    /// PDS base URL; resolved from the handle's DID document if omitted.
    #[arg(long, env = "ATP_PDS_URL")]
    pub pds_url: Option<String>,

    /// atproto handle or DID.
    #[arg(long, env = "ATP_HANDLE")]
    pub handle: Option<String>,

    /// atproto app password (never logged).
    #[arg(long, env = "ATP_APP_PASSWORD")]
    pub app_password: Option<String>,

    /// NVD record collection.
    #[arg(long, env = "ATP_COLLECTION", default_value = COLLECTION)]
    pub collection: String,

    /// S3 bucket for the cache snapshot (disabled when unset).
    #[arg(long, env = "S3_BUCKET")]
    pub s3_bucket: Option<String>,

    /// S3 object path.
    #[arg(long, env = "S3_PATH", default_value = DEFAULT_S3_PATH)]
    pub s3_path: String,

    /// S3 endpoint URL (path-style addressing when set).
    #[arg(long, env = "S3_ENDPOINT_URL")]
    pub s3_endpoint_url: Option<String>,

    /// S3 access key (falls back to AWS_ACCESS_KEY_ID).
    #[arg(long, env = "S3_ACCESS_KEY_ID")]
    pub s3_access_key_id: Option<String>,

    /// S3 secret key (falls back to AWS_SECRET_ACCESS_KEY; never logged).
    #[arg(long, env = "S3_SECRET_ACCESS_KEY")]
    pub s3_secret_access_key: Option<String>,

    /// S3 region.
    #[arg(long, env = "S3_REGION", default_value = "us-east-1")]
    pub s3_region: String,

    /// Emit debug logs.
    #[arg(
        long,
        env = "DEBUG",
        action = ArgAction::SetTrue,
        value_parser = clap::builder::BoolishValueParser::new()
    )]
    pub debug: bool,

    /// NVD API base URL (test seam).
    #[arg(long, env = "NVD_API_URL", default_value = NVD_API_URL)]
    pub nvd_url: String,

    /// PLC directory base URL (test seam).
    #[arg(long, env = "PLC_URL", default_value = DEFAULT_PLC_URL)]
    pub plc_url: String,
}

/// Trim, lowercase, and drop empty CSV filter entries.
fn clean_csv(values: Vec<String>) -> Vec<String> {
    let mut values: Vec<String> = values
        .into_iter()
        .map(|value| value.trim().to_ascii_lowercase())
        .filter(|value| !value.is_empty())
        .collect();
    values.sort();
    values.dedup();
    values
}

impl Cli {
    pub fn resolve(self) -> Result<RunConfig, String> {
        let handle = self
            .handle
            .map(|value| value.trim_start_matches('@').to_owned())
            .filter(|value| !value.is_empty());
        let app_password = self.app_password.filter(|value| !value.is_empty());
        let nvd_api_key = self.nvd_api_key.filter(|value| !value.is_empty());
        let dry_run = self.dry_run;

        let osv_bulk_dir = self
            .osv_bulk_dir
            .filter(|path| !path.as_os_str().is_empty());
        let osv_online = self.osv_online && !self.osv_offline && osv_bulk_dir.is_none();
        if !osv_online && osv_bulk_dir.is_none() {
            return Err(
                "OSV is offline but no advisory source is configured; pass --osv-bulk-dir <dir>"
                    .to_owned(),
            );
        }
        if self.require_nvd_key && self.backfill && nvd_api_key.is_none() {
            return Err("--require-nvd-key: NVD_API_KEY is required for --backfill".to_owned());
        }
        if !dry_run {
            let mut missing = Vec::new();
            if handle.is_none() {
                missing.push("ATP_HANDLE");
            }
            if app_password.is_none() {
                missing.push("ATP_APP_PASSWORD");
            }
            if !missing.is_empty() {
                return Err(format!("missing credentials: {}", missing.join(", ")));
            }
        }

        let endpoint_url = self.s3_endpoint_url.clone();
        let access_key_id = self
            .s3_access_key_id
            .clone()
            .filter(|v| !v.is_empty())
            .or_else(|| {
                std::env::var("AWS_ACCESS_KEY_ID")
                    .ok()
                    .filter(|v| !v.is_empty())
            });
        let secret_access_key = self
            .s3_secret_access_key
            .clone()
            .filter(|v| !v.is_empty())
            .or_else(|| {
                std::env::var("AWS_SECRET_ACCESS_KEY")
                    .ok()
                    .filter(|v| !v.is_empty())
            });
        let region = self.s3_region.clone();
        let s3 = self
            .s3_bucket
            .clone()
            .filter(|bucket| !bucket.is_empty())
            .map(|bucket| S3Config {
                bucket,
                path: self.s3_path.clone(),
                endpoint_url: endpoint_url.clone(),
                access_key_id: access_key_id.clone(),
                secret_access_key: secret_access_key.clone(),
                region: region.clone(),
            });
        let artifact_s3 = self
            .artifact_s3_bucket
            .clone()
            .filter(|bucket| !bucket.is_empty())
            .map(|bucket| S3Config {
                bucket,
                path: self.artifact_s3_path.clone(),
                endpoint_url,
                access_key_id,
                secret_access_key,
                region,
            });

        Ok(RunConfig {
            once: self.once,
            dry_run,
            limit: self.limit.max(0) as usize,
            backfill: self.backfill,
            initial_lookback_days: self.initial_lookback_days,
            order: self.order,
            publish_atproto: self.publish_atproto,
            software_limit: self.software_limit.max(0) as usize,
            publish_releases: self.publish_releases,
            release_mode: self.release_mode,
            release_limit: self.release_limit.max(0) as usize,
            release_recheck_seconds: self.release_recheck_seconds.max(0.0),
            release_ecosystems: clean_csv(self.release_ecosystems),
            release_sources: clean_csv(self.release_sources),
            release_history_limit: self.release_history_limit,
            release_concurrency: self.release_concurrency.max(1),
            release_source_url: self.release_source_url.filter(|url| !url.is_empty()),
            artifact_max_bytes: self.artifact_max_bytes,
            artifact_s3,
            github_token: self.github_token.filter(|token| !token.is_empty()),
            gitlab_token: self.gitlab_token.filter(|token| !token.is_empty()),
            gitee_token: self.gitee_token.filter(|token| !token.is_empty()),
            osv_online,
            osv_bulk_dir,
            purldb: self.purldb,
            cache_db: self.cache_db,
            poll_interval: self.poll_interval,
            page_size: self.page_size,
            concurrency: self.concurrency.max(1) as usize,
            pds_timeout: self.pds_timeout,
            nvd_api_key,
            require_nvd_key: self.require_nvd_key,
            pds_hourly_points: self.pds_hourly_points,
            pds_daily_points: self.pds_daily_points,
            pds_requests_per_5min: self.pds_requests_per_5min,
            pds_url: self.pds_url,
            handle,
            app_password,
            collection: self.collection,
            s3,
            debug: self.debug,
            nvd_url: self.nvd_url,
            plc_url: self.plc_url,
        })
    }
}

/// Parse, validate, run, and map failures to the process exit codes.
pub async fn main<I, T>(args: I) -> ExitCode
where
    I: IntoIterator<Item = T>,
    T: Into<OsString> + Clone,
{
    let cli = match Cli::try_parse_from(args) {
        Ok(cli) => cli,
        Err(error) => {
            let code = if error.use_stderr() {
                EXIT_USAGE
            } else {
                EXIT_OK
            };
            let _ = error.print();
            return ExitCode::from(code);
        }
    };
    let debug = cli.debug;
    let logger = Logger::stderr(if debug { Level::Debug } else { Level::Info });

    let config = match cli.resolve() {
        Ok(config) => config,
        Err(message) => {
            logger.error("cli_error", Some("run"), json!({ "error": message }));
            return ExitCode::from(EXIT_USAGE);
        }
    };

    match crate::runtime::run(config).await {
        Ok(code) => code,
        Err(error) => {
            logger.error("fatal", Some("run"), json!({ "error": error.to_string() }));
            ExitCode::from(EXIT_FATAL)
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn parse(args: &[&str]) -> Cli {
        Cli::try_parse_from(std::iter::once("vuln-notif").chain(args.iter().copied()))
            .expect("parse")
    }

    #[test]
    fn dry_run_needs_no_credentials_and_strips_handle_at() {
        let config = parse(&["--dry-run", "--handle", "@alice.test", "--concurrency", "0"])
            .resolve()
            .expect("resolve");
        assert!(config.dry_run);
        assert_eq!(config.handle.as_deref(), Some("alice.test"));
        assert_eq!(config.concurrency, 1);
        assert_eq!(config.limit, 0);
    }

    #[test]
    fn live_run_requires_credentials() {
        // Explicit empty values override any ambient ATP_* environment.
        let error = parse(&["--handle", "", "--app-password", ""])
            .resolve()
            .expect_err("credentials required");
        assert!(error.contains("ATP_HANDLE"));
        assert!(error.contains("ATP_APP_PASSWORD"));
    }

    #[test]
    fn release_mode_defaults_to_latest_and_accepts_affected() {
        let config = parse(&["--dry-run"]).resolve().expect("default mode");
        assert_eq!(config.release_mode, ReleaseMode::Latest);
        assert!(config.osv_online);
        assert!(config.osv_bulk_dir.is_none());

        let config = parse(&["--dry-run", "--release-mode", "affected"])
            .resolve()
            .expect("affected mode");
        assert_eq!(config.release_mode, ReleaseMode::Affected);
    }

    #[test]
    fn extras_and_all_modes_resolve() {
        for (mode, expected) in [("extras", ReleaseMode::Extras), ("all", ReleaseMode::All)] {
            let config = parse(&["--dry-run", "--release-mode", mode])
                .resolve()
                .expect("mode is implemented");
            assert_eq!(config.release_mode, expected);
        }
    }

    #[test]
    fn history_mode_is_implemented_and_uses_its_limits() {
        let config = parse(&[
            "--dry-run",
            "--release-mode",
            "history",
            "--release-history-limit",
            "7",
            "--release-concurrency",
            "2",
            "--artifact-max-bytes",
            "1024",
        ])
        .resolve()
        .expect("history mode");
        assert_eq!(config.release_mode, ReleaseMode::History);
        assert_eq!(config.release_history_limit, 7);
        assert_eq!(config.release_concurrency, 2);
        assert_eq!(config.artifact_max_bytes, 1024);
        assert!(config.artifact_s3.is_none());
    }

    #[test]
    fn osv_offline_requires_a_bulk_dir_and_bulk_dir_implies_offline() {
        let error = parse(&["--dry-run", "--osv-offline"])
            .resolve()
            .expect_err("offline without bulk dir");
        assert!(error.contains("--osv-bulk-dir"));

        let config = parse(&["--dry-run", "--osv-bulk-dir", "/tmp/osv"])
            .resolve()
            .expect("bulk dir");
        assert!(!config.osv_online);
        assert_eq!(
            config.osv_bulk_dir.as_deref(),
            Some(std::path::Path::new("/tmp/osv"))
        );

        let config = parse(&["--dry-run", "--osv-online"])
            .resolve()
            .expect("bare --osv-online is accepted");
        assert!(config.osv_online);

        let error = parse(&["--dry-run", "--osv-online=false"])
            .resolve()
            .expect_err("explicit offline without bulk dir");
        assert!(error.contains("--osv-bulk-dir"));
    }

    #[test]
    fn release_filters_are_trimmed_sorted_and_deduplicated() {
        let config = parse(&[
            "--dry-run",
            "--release-ecosystems",
            "PyPI, npm ,,pypi",
            "--release-sources",
            "OSV",
        ])
        .resolve()
        .expect("filters");
        assert_eq!(config.release_ecosystems, vec!["npm", "pypi"]);
        assert_eq!(config.release_sources, vec!["osv"]);
        assert_eq!(config.release_concurrency, 4);
    }

    #[test]
    fn require_key_only_applies_to_backfill() {
        let error = parse(&[
            "--dry-run",
            "--backfill",
            "--require-nvd-key",
            "--nvd-api-key",
            "",
        ])
        .resolve()
        .expect_err("key required");
        assert!(error.contains("NVD_API_KEY"));
    }
}