~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
590 lines · 19.1 KB Raw History
//! `--release-mode affected`: OSV advisories -> planned versions -> immutable
//! version records -> `atprotoV1` channel/version refs.
//!
//! Every source call goes through the scripted [`FakeHttp`] seam and every PDS
//! write through [`common::FakePds`]; no live network is used.

use std::collections::HashMap;
use std::path::Path;
use std::sync::{Arc, Mutex};

use async_trait::async_trait;
use serde_json::{Value, json};
use tempfile::TempDir;
use tokio::sync::mpsc;

use vuln_notif::constants::{
    ATPROTO_COLLECTION, CHANNEL_COLLECTION, MAX_ATTEMPTS, PROJECT_COLLECTION, VERSION_COLLECTION,
};
use vuln_notif::db::DbHandle;
use vuln_notif::db::model::{
    ReleaseChannelMark, ReleaseVersionMark, SoftwareIdentityMark, SoftwareMark,
};
use vuln_notif::links;
use vuln_notif::pds::Publisher;
use vuln_notif::release::http::RegistryHttp;
use vuln_notif::release::{AffectedConfig, sync_affected};
use vuln_notif::runtime::events::ProgressEvent;

mod common;

const DID: &str = "did:plc:test";
const CVE: &str = "CVE-2024-3094";
const RHEL: &str = "red-hat-enterprise-linux";
const RHEL_PURL: &str = "pkg:pypi/enterprise-linux";

// ---------------------------------------------------------------------------
// Scripted source HTTP
// ---------------------------------------------------------------------------

#[derive(Default)]
struct FakeHttp {
    gets: Mutex<HashMap<String, Value>>,
    posts: Mutex<HashMap<String, Value>>,
    calls: Mutex<Vec<String>>,
}

impl FakeHttp {
    fn get(&self, url: &str, value: Value) {
        self.gets.lock().unwrap().insert(url.to_owned(), value);
    }

    fn post(&self, purl: &str, value: Value) {
        self.posts.lock().unwrap().insert(purl.to_owned(), value);
    }

    fn calls(&self) -> Vec<String> {
        self.calls.lock().unwrap().clone()
    }

    fn reset_calls(&self) {
        self.calls.lock().unwrap().clear();
    }
}

#[async_trait]
impl RegistryHttp for FakeHttp {
    async fn get_json(&self, url: &str) -> anyhow::Result<Value> {
        self.calls.lock().unwrap().push(format!("GET {url}"));
        self.gets
            .lock()
            .unwrap()
            .get(url)
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("unexpected GET {url}"))
    }

    async fn get_text(&self, url: &str) -> anyhow::Result<String> {
        Err(anyhow::anyhow!("unexpected text GET {url}"))
    }

    async fn head_size(&self, url: &str) -> anyhow::Result<u64> {
        Err(anyhow::anyhow!("unexpected HEAD {url}"))
    }

    async fn download(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        Err(anyhow::anyhow!("unexpected download {url}"))
    }

    async fn post_json(&self, url: &str, body: &Value) -> anyhow::Result<Value> {
        let purl = body
            .get("package")
            .and_then(|package| package.get("purl"))
            .and_then(Value::as_str)
            .unwrap_or_default()
            .to_owned();
        self.calls
            .lock()
            .unwrap()
            .push(format!("POST {url} {purl}"));
        self.posts
            .lock()
            .unwrap()
            .get(&purl)
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("unexpected POST {purl}"))
    }
}

fn fixture(name: &str) -> Value {
    let path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("tests/fixtures/release_affected")
        .join(name);
    let text = std::fs::read_to_string(&path)
        .unwrap_or_else(|error| panic!("read {}: {error}", path.display()));
    serde_json::from_str(&text).unwrap_or_else(|error| panic!("parse {}: {error}", path.display()))
}

fn advisory_response(advisory: Value) -> Value {
    json!({ "vulns": [advisory] })
}

fn no_vulns() -> Value {
    json!({ "vulns": [] })
}

fn depsdev_versions(versions: &[&str]) -> Value {
    json!({
        "versions": versions
            .iter()
            .map(|version| json!({
                "versionKey": {"version": version},
                "isDefault": false,
                "isDeprecated": false,
            }))
            .collect::<Vec<_>>(),
    })
}

fn pypi_metadata(name: &str, versions: &[&str]) -> Value {
    let mut releases = serde_json::Map::new();
    for version in versions {
        releases.insert(
            (*version).to_owned(),
            json!([{
                "digests": {"sha256": format!("{:0<64}", version.replace('.', ""))},
                "filename": format!("{name}-{version}.tar.gz"),
                "packagetype": "sdist",
                "size": 1000,
                "url": format!("https://files.invalid/{name}-{version}.tar.gz"),
                "yanked": false,
            }]),
        );
    }
    json!({ "releases": releases })
}

// ---------------------------------------------------------------------------
// Cache + PDS seeding
// ---------------------------------------------------------------------------

async fn test_db() -> (DbHandle, TempDir) {
    let dir = tempfile::tempdir().expect("tempdir");
    let db = DbHandle::open(dir.path().join("cache.db"))
        .await
        .expect("open db");
    (db, dir)
}

async fn seed_project(db: &DbHandle, slug: &str, product: &str, purl: &str) -> (String, String) {
    let project_cid = format!("bafyproject{slug}");
    let project_uri = format!("at://{DID}/{PROJECT_COLLECTION}/{slug}");
    let raw_slug = format!("{slug}-raw");
    db.mark_software(SoftwareMark {
        repo_did: DID.to_owned(),
        slug: raw_slug.clone(),
        part: "a".to_owned(),
        vendor: "redhat".to_owned(),
        product: product.to_owned(),
        project_uri: project_uri.clone(),
        project_cid: project_cid.clone(),
        software_uri: format!("at://{DID}/com.nahdig.software/{raw_slug}"),
        software_cid: "bafysoftware".to_owned(),
    })
    .await
    .expect("mark software");
    db.mark_software_identity(SoftwareIdentityMark {
        repo_did: DID.to_owned(),
        raw_slug,
        canonical_slug: slug.to_owned(),
        identity_key: Some(slug.to_owned()),
        source: Some("test".to_owned()),
        confidence: Some(1.0),
    })
    .await
    .expect("mark identity");
    db.mark_purl(
        DID,
        purl,
        slug,
        &format!(
            "at://{DID}/com.nahdig.purl/{}",
            purl.replace([':', '/'], "-")
        ),
        "bafypurl",
    )
    .await
    .expect("mark purl");
    (project_uri, project_cid)
}

async fn seed_versions(db: &DbHandle, slug: &str, versions: &[&str]) {
    for version in versions {
        db.mark_release_version(ReleaseVersionMark {
            repo_did: DID.to_owned(),
            project_slug: slug.to_owned(),
            version: (*version).to_owned(),
            version_raw: (*version).to_owned(),
            rkey: format!("{slug}:{version}"),
            uri: version_uri(slug, version),
            cid: version_cid(slug, version),
            manifest_uri: format!("at://{DID}/software.release.manifest/m{version}"),
            manifest_cid: format!("bafymanifest{version}"),
        })
        .await
        .expect("mark version");
    }
}

async fn seed_channel(db: &DbHandle, slug: &str, project_uri: &str, target: &str) {
    db.mark_release_channel(ReleaseChannelMark {
        repo_did: DID.to_owned(),
        project_slug: slug.to_owned(),
        name: "9".to_owned(),
        rkey: format!("{slug}:9"),
        uri: format!("at://{DID}/{CHANNEL_COLLECTION}/{slug}:9"),
        cid: "bafychannel".to_owned(),
        version_uri: version_uri(slug, target),
        version_cid: version_cid(slug, target),
    })
    .await
    .expect("mark channel");
    let _ = project_uri;
}

fn version_uri(slug: &str, version: &str) -> String {
    format!("at://{DID}/{VERSION_COLLECTION}/{slug}:{version}")
}

fn version_cid(slug: &str, version: &str) -> String {
    format!(
        "bafyver{}{}",
        slug.replace('-', ""),
        version.replace('.', "")
    )
}

fn atproto_record(cve: &str, project_uri: &str, project_cid: &str, purl: &str) -> Value {
    json!({
        "$type": ATPROTO_COLLECTION,
        "createdAt": "2025-03-04T05:06:07Z",
        "cve": cve,
        "nvd": {
            "uri": format!("at://{DID}/com.nahdig.cve.nvdV2/{cve}"),
            "cid": "bafynvd",
        },
        "software": [{
            "project": {"uri": project_uri, "cid": project_cid},
            "part": "a",
            "vendor": "redhat",
            "product": "enterprise_linux",
            "cpes": [],
            "purls": [purl],
        }],
    })
}

/// Store an atprotoV1 record on the fake PDS and index its backlinks locally.
async fn seed_atproto(api: &common::FakePds, db: &DbHandle, cve: &str, record: &Value) {
    api.seed_record(ATPROTO_COLLECTION, cve, record.clone());
    let uri = format!("at://{DID}/{ATPROTO_COLLECTION}/{cve}");
    links::index_record(db, DID, &uri, Some("bafycve"), ATPROTO_COLLECTION, record)
        .await
        .expect("index cve links");
}

async fn test_publisher(api: &Arc<common::FakePds>, db: &DbHandle) -> Publisher {
    let publisher = common::make_publisher(api.clone(), db);
    publisher.login().await.expect("login");
    publisher
}

fn config() -> AffectedConfig {
    AffectedConfig {
        repo_did: DID.to_owned(),
        limit: 0,
        recheck_seconds: 0.0,
        ecosystems: Vec::new(),
        sources: Vec::new(),
        concurrency: 1,
        osv_online: true,
        osv_bulk_dir: None,
    }
}

async fn run_affected(
    publisher: &Publisher,
    db: &DbHandle,
    config: &AffectedConfig,
    http: &Arc<FakeHttp>,
) -> vuln_notif::release::AffectedCounters {
    let (events, _receiver) = mpsc::channel::<ProgressEvent>(32);
    let http: Arc<dyn RegistryHttp> = http.clone();
    sync_affected(publisher, db, config, events, http)
        .await
        .expect("affected sync")
}

/// RHEL project: rows for 9.0/9.1/9.5, OSV says 9.0 <= v < 9.4.
async fn seed_rhel_case(db: &DbHandle) -> (String, String, Arc<common::FakePds>, Arc<FakeHttp>) {
    let (project_uri, project_cid) = seed_project(db, RHEL, "enterprise_linux", RHEL_PURL).await;
    seed_versions(db, RHEL, &["9.0", "9.1", "9.5"]).await;
    db.commit().await.expect("commit seed");

    let api = common::FakePds::new();
    let record = atproto_record(CVE, &project_uri, &project_cid, RHEL_PURL);
    seed_atproto(&api, db, CVE, &record).await;

    let http = Arc::new(FakeHttp::default());
    http.post(RHEL_PURL, advisory_response(fixture("osv-pypi.json")));
    http.get(
        "https://api.deps.dev/v3/systems/PYPI/packages/enterprise-linux",
        depsdev_versions(&["9.0", "9.1", "9.2", "9.3", "9.4", "9.5"]),
    );
    http.get(
        "https://pypi.org/pypi/enterprise-linux/json",
        pypi_metadata("enterprise-linux", &["9.2", "9.3"]),
    );
    (project_uri, project_cid, api, http)
}

fn software_of(record: &Value) -> &Value {
    &record["software"][0]
}

fn ref_uris(reference: &Value, field: &str) -> Vec<String> {
    reference
        .get(field)
        .and_then(Value::as_array)
        .into_iter()
        .flatten()
        .filter_map(|item| item.get("uri").and_then(Value::as_str))
        .map(str::to_owned)
        .collect()
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[tokio::test]
async fn rhel_advisory_creates_missing_versions_and_channel_version_refs() {
    let (db, _dir) = test_db().await;
    let (project_uri, _cid, api, http) = seed_rhel_case(&db).await;
    let publisher = test_publisher(&api, &db).await;

    let counters = run_affected(&publisher, &db, &config(), &http).await;

    assert_eq!(counters.projects_checked, 1);
    assert_eq!(counters.projects_linked, 1);
    assert_eq!(counters.versions_planned, 4);
    assert_eq!(counters.versions_created, 2);
    assert_eq!(counters.versions_label_only, 0);
    assert_eq!(counters.cves_linked, 1);
    assert_eq!(counters.unchanged, 0);
    assert_eq!(counters.failures, 0);

    // Versions 9.2 and 9.3 had no row and got real manifests+records.
    for version in ["9.2", "9.3"] {
        let row = db
            .get_release_version(DID, RHEL, version)
            .await
            .expect("version row query")
            .unwrap_or_else(|| panic!("missing release_versions row for {version}"));
        assert_eq!(
            row.uri.as_deref(),
            Some(version_uri(RHEL, version).as_str())
        );
    }
    assert_eq!(
        api.count("putRecord"),
        6,
        "2 manifests + 2 versions + 1 channel + 1 CVE"
    );

    let record = api.record(ATPROTO_COLLECTION, CVE).expect("cve record");
    let software = software_of(&record);
    assert_eq!(
        software["project"]["uri"],
        Value::String(project_uri),
        "one canonical project"
    );
    assert_eq!(
        ref_uris(software, "versions"),
        ["9.0", "9.1", "9.2", "9.3"]
            .iter()
            .map(|version| version_uri(RHEL, version))
            .collect::<Vec<_>>()
    );
    assert_eq!(
        ref_uris(software, "channels"),
        vec![format!("at://{DID}/{CHANNEL_COLLECTION}/{RHEL}:9")]
    );
}

#[tokio::test]
async fn no_osv_match_is_a_typed_skip_without_version_writes() {
    let (db, _dir) = test_db().await;
    let slug = "acme-nomatch";
    let purl = "pkg:pypi/no-match-pkg";
    let (project_uri, project_cid) = seed_project(&db, slug, "no-match-pkg", purl).await;
    seed_versions(&db, slug, &["9.0", "9.1"]).await;
    db.commit().await.expect("commit seed");

    let api = common::FakePds::new();
    let record = atproto_record("CVE-2024-1111", &project_uri, &project_cid, purl);
    seed_atproto(&api, &db, "CVE-2024-1111", &record).await;
    let publisher = test_publisher(&api, &db).await;

    let http = Arc::new(FakeHttp::default());
    http.post(purl, no_vulns());
    http.get(
        "https://api.deps.dev/v3/systems/PYPI/packages/no-match-pkg",
        depsdev_versions(&[]),
    );

    let counters = run_affected(&publisher, &db, &config(), &http).await;
    assert_eq!(counters.projects_checked, 1);
    assert_eq!(counters.skipped, 1);
    assert_eq!(counters.versions_planned, 0);
    assert_eq!(counters.versions_created, 0);
    assert_eq!(counters.versions_label_only, 0);
    assert_eq!(counters.cves_linked, 0);
    assert!(api.puts().is_empty(), "no version fabrication");
    assert_eq!(
        api.record(ATPROTO_COLLECTION, "CVE-2024-1111")
            .expect("cve record"),
        record
    );
}

#[tokio::test]
async fn offline_bulk_dir_plans_from_disk_without_osv_network() {
    let (db, _dir) = test_db().await;
    let (_project_uri, _cid, api, http) = seed_rhel_case(&db).await;
    let publisher = test_publisher(&api, &db).await;
    let mut config = config();
    config.osv_online = false;
    config.osv_bulk_dir =
        Some(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/release_affected"));

    let counters = run_affected(&publisher, &db, &config, &http).await;

    assert_eq!(counters.versions_planned, 4);
    assert_eq!(counters.versions_created, 2);
    assert_eq!(counters.cves_linked, 1);
    assert!(
        !http.calls().iter().any(|call| call.contains("api.osv.dev")),
        "offline mode must not call the OSV API: {:?}",
        http.calls()
    );
}

#[tokio::test]
async fn second_run_is_idempotent_and_writes_nothing() {
    let (db, _dir) = test_db().await;
    let (_project_uri, _cid, api, http) = seed_rhel_case(&db).await;
    let publisher = test_publisher(&api, &db).await;

    let first = run_affected(&publisher, &db, &config(), &http).await;
    assert_eq!(first.cves_linked, 1);
    let writes = api.puts().len();
    assert!(writes > 0);

    http.reset_calls();
    let second = run_affected(&publisher, &db, &config(), &http).await;
    assert_eq!(second.projects_checked, 1);
    assert_eq!(second.versions_created, 0);
    assert_eq!(second.versions_label_only, 0);
    assert_eq!(second.cves_linked, 0);
    assert_eq!(second.unchanged, 1);
    assert_eq!(
        api.puts().len(),
        writes,
        "second run must not write to the PDS"
    );
}

#[tokio::test]
async fn cve_without_a_project_backlink_is_ignored() {
    let (db, _dir) = test_db().await;
    let slug = "acme-unlinked";
    let purl = "pkg:pypi/unlinked-pkg";
    let (project_uri, project_cid) = seed_project(&db, slug, "unlinked-pkg", purl).await;
    seed_versions(&db, slug, &["9.0"]).await;
    db.commit().await.expect("commit seed");

    let api = common::FakePds::new();
    // The record exists but nothing links it to the project.
    let record = atproto_record("CVE-2024-2222", &project_uri, &project_cid, purl);
    api.seed_record(ATPROTO_COLLECTION, "CVE-2024-2222", record.clone());
    let publisher = test_publisher(&api, &db).await;

    let http = Arc::new(FakeHttp::default());
    let counters = run_affected(&publisher, &db, &config(), &http).await;

    assert_eq!(counters.skipped, 1);
    assert_eq!(counters.cves_linked, 0);
    assert!(api.puts().is_empty());
    assert!(
        http.calls().is_empty(),
        "no advisory query without backlinks"
    );
    assert_eq!(
        api.record(ATPROTO_COLLECTION, "CVE-2024-2222")
            .expect("record"),
        record
    );
}

#[tokio::test]
async fn pds_failure_is_counted_and_does_not_block_the_next_project() {
    let (db, _dir) = test_db().await;
    let api = common::FakePds::new();
    let http = FakeHttp::default();

    // "acme-a" sorts first, so the queued 500s hit its CVE write.
    for (suffix, cve) in [("a", "CVE-2024-0001"), ("b", "CVE-2024-0002")] {
        let slug = format!("acme-{suffix}");
        let purl = format!("pkg:pypi/pkg-{suffix}");
        let (project_uri, project_cid) =
            seed_project(&db, &slug, &format!("pkg-{suffix}"), &purl).await;
        seed_versions(&db, &slug, &["9.0", "9.1", "9.2", "9.3", "9.5"]).await;
        seed_channel(&db, &slug, &project_uri, "9.5").await;

        let mut advisory = fixture("osv-pypi.json");
        advisory["id"] = json!(format!("OSV-{cve}"));
        advisory["aliases"] = json!([cve]);
        advisory["affected"][0]["package"]["name"] = json!(format!("pkg-{suffix}"));
        advisory["affected"][0]["package"]["purl"] = json!(purl.clone());
        http.post(&purl, advisory_response(advisory));
        http.get(
            &format!("https://api.deps.dev/v3/systems/PYPI/packages/pkg-{suffix}"),
            depsdev_versions(&[]),
        );

        let mut record = atproto_record(cve, &project_uri, &project_cid, &purl);
        record["cve"] = json!(cve);
        seed_atproto(&api, &db, cve, &record).await;
    }
    db.commit().await.expect("commit seed");
    for _ in 0..MAX_ATTEMPTS {
        api.push_put(common::response(
            500,
            json!({"error": "InternalServerError"}),
        ));
    }

    let publisher = test_publisher(&api, &db).await;
    let http = Arc::new(http);
    let counters = run_affected(&publisher, &db, &config(), &http).await;

    assert_eq!(counters.projects_checked, 2);
    assert_eq!(
        counters.failures,
        1,
        "first project failure is isolated: {counters:?} puts={:?}",
        api.puts()
    );
    assert_eq!(counters.cves_linked, 1, "second project still links");
    assert_eq!(
        api.puts().len(),
        MAX_ATTEMPTS as usize + 1,
        "puts={:?}",
        api.puts()
    );
    assert!(
        api.record(ATPROTO_COLLECTION, "CVE-2024-0002")
            .is_some_and(|record| record["software"][0].get("versions").is_some())
    );
}