~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
539 lines · 17.7 KB Raw History
//! Software graph: canonical project identity, raw software aliases, purl
//! rows, and deduped atprotoV1 refs against a fake PDS.

mod common;

use std::collections::BTreeSet;
use std::path::PathBuf;

use serde_json::{Value, json};
use tokio::sync::mpsc;

use common::*;
use vuln_notif::constants::{
    ATPROTO_COLLECTION, ATPROTO_TYPE, COLLECTION, PROJECT_COLLECTION, PROJECT_TYPE,
    PURL_COLLECTION, SOFTWARE_COLLECTION, STATUS_CREATED,
};
use vuln_notif::db::DbHandle;
use vuln_notif::db::model::PublicationMark;
use vuln_notif::records::purl_rkey;
use vuln_notif::runtime::events::ProgressEvent;
use vuln_notif::software::{self, SoftwareConfig, load_purl_map, versionless_purl};

const RHEL_CPE: &str = "cpe:2.3:o:redhat:enterprise_linux:8.0:*:*:*:*:*:*:*";
const RHEL_PROJECT: &str = "red-hat-enterprise-linux";

fn fixture_purl_db() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/purl2cpe_sample.db")
}

fn rhel_cve() -> Value {
    serde_json::from_str(include_str!("fixtures/software/rhel_cve.json")).expect("rhel fixture")
}

fn two_cves() -> Vec<Value> {
    serde_json::from_str(include_str!("fixtures/software/two_cves.json")).expect("two cve fixture")
}

#[test]
fn purl_helpers() {
    assert_eq!(
        software::cpe_product_key("cpe:2.3:a:acme:widget:1.0:*:*:*:*:*:*:*").as_deref(),
        Some("a:acme:widget")
    );
    assert_eq!(
        software::cpe_product_key("cpe:/o:linux:kernel:5.0").as_deref(),
        Some("o:linux:kernel")
    );
    assert_eq!(software::cpe_product_key("cpe:2.3:x:a:b:1"), None);
    assert_eq!(software::cpe_product_key("nope"), None);

    assert_eq!(
        versionless_purl("pkg:npm/acme-widget"),
        "pkg:npm/acme-widget"
    );
    assert_eq!(
        versionless_purl("pkg:npm/%40scope/widget@1.2.3"),
        "pkg:npm/%40scope/widget"
    );
    assert_eq!(
        versionless_purl("pkg:pypi/django@4.2?extra=1#frag"),
        "pkg:pypi/django"
    );
}

#[tokio::test]
async fn purl2cpe_load_map() {
    let wanted: BTreeSet<String> = [
        "a:acme:widget",
        "a:djangoproject:django",
        "a:serde-rs:serde",
        "a:apache:log4j",
        "a:not:there",
    ]
    .into_iter()
    .map(str::to_owned)
    .collect();
    let map = load_purl_map(&fixture_purl_db(), &wanted).await.unwrap();

    assert_eq!(
        map["a:acme:widget"],
        BTreeSet::from([
            "pkg:npm/acme-widget".to_owned(),
            "pkg:pypi/acme-widget".to_owned(),
        ])
    );
    assert_eq!(
        map["a:djangoproject:django"],
        BTreeSet::from(["pkg:pypi/django".to_owned()])
    );
    assert_eq!(
        map["a:serde-rs:serde"],
        BTreeSet::from(["pkg:cargo/serde".to_owned()])
    );
    assert_eq!(
        map["a:apache:log4j"],
        BTreeSet::from(["pkg:maven/org.apache.logging.log4j/log4j-core".to_owned()])
    );
    assert!(!map.contains_key("a:not:there"));
    assert_eq!(map.len(), 4);
}

fn cve(id: &str, last_modified: &str) -> Value {
    json!({
        "id": id,
        "lastModified": last_modified,
        "configurations": [{"nodes": [{"cpeMatch": [{
            "criteria": "cpe:2.3:a:acme:widget:1.0:*:*:*:*:*:*:*"
        }]}]}],
    })
}

async fn seed(db: &DbHandle, cve: &Value) {
    let id = cve["id"].as_str().unwrap();
    db.upsert_cves(REPO, std::slice::from_ref(cve))
        .await
        .unwrap();
    let mut mark = PublicationMark::new(REPO, id, STATUS_CREATED);
    mark.uri = Some(format!("at://{REPO}/{COLLECTION}/{id}"));
    mark.cid = Some("bafynvd".to_owned());
    db.mark_publication(mark).await.unwrap();
}

fn config(limit: usize) -> SoftwareConfig {
    SoftwareConfig {
        repo_did: REPO.to_owned(),
        limit,
        purl_db: fixture_purl_db(),
    }
}

fn counters_json(counters: &vuln_notif::software::SoftwareCounters) -> Value {
    serde_json::to_value(counters).unwrap()
}

fn put_keys(api: &FakePds) -> Vec<(String, String)> {
    api.puts()
        .iter()
        .map(|call| (call.collection.clone(), call.rkey.clone()))
        .collect()
}

fn puts_for(api: &FakePds, collection: &str) -> Vec<ApiCall> {
    api.puts()
        .into_iter()
        .filter(|call| call.collection == collection)
        .collect()
}

#[tokio::test]
async fn affected_text_and_cpe_yield_one_canonical_project() {
    let temp = tempfile::tempdir().unwrap();
    let db = DbHandle::open(temp.path().join("cache.db")).await.unwrap();
    seed(&db, &rhel_cve()).await;

    let api = FakePds::new();
    let publisher = make_publisher(api.clone(), &db);
    publisher.login().await.unwrap();

    let (events, _received) = mpsc::channel::<ProgressEvent>(16);
    let counters = software::sync_software(&publisher, &db, &config(10), events)
        .await
        .unwrap();
    assert_eq!(
        counters_json(&counters),
        json!({"cves": 1, "software_created": 2, "purl_created": 1, "atproto_created": 1, "failures": 0})
    );

    let project_uri = format!("at://{REPO}/{PROJECT_COLLECTION}/{RHEL_PROJECT}");
    let text_key = "o:red_hat:red_hat_enterprise_linux_8";
    let cpe_key = "o:redhat:enterprise_linux";
    let purl = "pkg:rpm/redhat/enterprise-linux";
    assert_eq!(
        put_keys(&api),
        vec![
            (PROJECT_COLLECTION.to_owned(), RHEL_PROJECT.to_owned()),
            (ATPROTO_COLLECTION.to_owned(), "CVE-2024-6409".to_owned()),
        ]
    );

    let calls = api.puts();
    assert_eq!(
        calls[0].record.clone().unwrap(),
        json!({"$type": PROJECT_TYPE, "name": RHEL_PROJECT})
    );
    // One deduped ref for the canonical project, with CPEs/purls merged
    // across the raw aliases observed for the CVE.
    let atproto = calls[1].record.as_ref().unwrap();
    assert_eq!(atproto["$type"], ATPROTO_TYPE);
    assert_eq!(atproto["cve"], "CVE-2024-6409");
    assert_eq!(
        atproto["software"],
        json!([{
            "project": {"uri": project_uri, "cid": PUT_CID},
            "part": "o",
            "vendor": "Red Hat",
            "product": "Red Hat Enterprise Linux 8",
            "cpes": [RHEL_CPE],
            "purls": [purl],
        }])
    );

    let identity = db
        .get_software_identity(REPO, text_key)
        .await
        .unwrap()
        .expect("text alias identity");
    assert_eq!(identity.canonical_slug, RHEL_PROJECT);
    assert_eq!(
        identity.identity_key.as_deref(),
        Some("os:red-hat:red-hat-enterprise-linux")
    );
    assert_eq!(identity.source.as_deref(), Some("nvd-affected"));
    assert_eq!(identity.confidence, Some(1.0));
    assert_eq!(
        db.get_software_identity(REPO, cpe_key)
            .await
            .unwrap()
            .expect("cpe alias identity")
            .canonical_slug,
        RHEL_PROJECT
    );
    assert_eq!(
        db.list_software_identities(REPO).await.unwrap().len(),
        2,
        "one identity row per raw alias"
    );

    // purl rows store the canonical project slug and join release candidates.
    let purl_row = db.get_purl(REPO, purl).await.unwrap().expect("purl row");
    assert_eq!(purl_row.software_slug, RHEL_PROJECT);
    let candidates = db.release_candidates(REPO, 10, 0.0).await.unwrap();
    assert_eq!(candidates.len(), 1);
    assert_eq!(candidates[0].slug, RHEL_PROJECT);
    assert_eq!(candidates[0].purls, vec![purl.to_owned()]);
}

#[tokio::test]
async fn rhel_els_and_eus_variants_converge() {
    let temp = tempfile::tempdir().unwrap();
    let db = DbHandle::open(temp.path().join("cache.db")).await.unwrap();
    for cve in &two_cves() {
        seed(&db, cve).await;
    }

    let api = FakePds::new();
    let publisher = make_publisher(api.clone(), &db);
    publisher.login().await.unwrap();

    let (events, _received) = mpsc::channel::<ProgressEvent>(16);
    let counters = software::sync_software(&publisher, &db, &config(10), events)
        .await
        .unwrap();
    assert_eq!(
        counters_json(&counters),
        json!({"cves": 2, "software_created": 4, "purl_created": 0, "atproto_created": 2, "failures": 0})
    );

    let project_uri = format!("at://{REPO}/{PROJECT_COLLECTION}/{RHEL_PROJECT}");
    let projects = puts_for(&api, PROJECT_COLLECTION);
    assert_eq!(projects.len(), 1, "one canonical project across variants");
    assert_eq!(projects[0].rkey, RHEL_PROJECT);

    assert!(
        puts_for(&api, SOFTWARE_COLLECTION).is_empty(),
        "raw aliases are local-only; no software records are published"
    );
    for call in puts_for(&api, ATPROTO_COLLECTION) {
        let record = call.record.as_ref().unwrap();
        assert_eq!(record["software"].as_array().unwrap().len(), 1);
        assert_eq!(record["software"][0]["project"]["uri"], project_uri);
    }
    let identities = db.list_software_identities(REPO).await.unwrap();
    assert_eq!(identities.len(), 4);
    assert!(
        identities
            .iter()
            .all(|row| row.canonical_slug == RHEL_PROJECT)
    );
}

#[tokio::test]
async fn placeholder_identity_produces_no_records() {
    let temp = tempfile::tempdir().unwrap();
    let db = DbHandle::open(temp.path().join("cache.db")).await.unwrap();
    let unknown = json!({
        "id": "CVE-2024-9999",
        "lastModified": "2024-01-01T00:00:00.000",
        "affected": [{"vendor": "n/a", "product": "n/a"}],
    });
    seed(&db, &unknown).await;

    let api = FakePds::new();
    let publisher = make_publisher(api.clone(), &db);
    publisher.login().await.unwrap();

    let (events, _received) = mpsc::channel::<ProgressEvent>(16);
    let counters = software::sync_software(&publisher, &db, &config(10), events)
        .await
        .unwrap();
    assert_eq!(
        counters_json(&counters),
        json!({"cves": 0, "software_created": 0, "purl_created": 0, "atproto_created": 0, "failures": 0})
    );
    assert!(api.puts().is_empty());
    assert!(db.list_software_identities(REPO).await.unwrap().is_empty());
}

#[tokio::test]
async fn second_sync_reuses_project_software_and_purls() {
    let temp = tempfile::tempdir().unwrap();
    let db = DbHandle::open(temp.path().join("cache.db")).await.unwrap();
    let mut older = rhel_cve();
    older["id"] = json!("CVE-2024-6408");
    older["lastModified"] = json!("2024-11-20T00:00:00.000");
    let newer = rhel_cve();
    seed(&db, &older).await;
    seed(&db, &newer).await;

    let api = FakePds::new();
    let publisher = make_publisher(api.clone(), &db);
    publisher.login().await.unwrap();

    let (events, _received) = mpsc::channel::<ProgressEvent>(16);
    let first = software::sync_software(&publisher, &db, &config(1), events.clone())
        .await
        .unwrap();
    assert_eq!(
        counters_json(&first),
        json!({"cves": 1, "software_created": 2, "purl_created": 1, "atproto_created": 1, "failures": 0})
    );
    assert_eq!(api.puts().len(), 2);
    assert_eq!(api.puts()[1].rkey, "CVE-2024-6409");

    // The second CVE shares every raw alias with the first: no project,
    // software, or purl record is rewritten.
    let second = software::sync_software(&publisher, &db, &config(1), events.clone())
        .await
        .unwrap();
    assert_eq!(
        counters_json(&second),
        json!({"cves": 1, "software_created": 0, "purl_created": 0, "atproto_created": 1, "failures": 0})
    );
    let calls = api.puts();
    assert_eq!(calls.len(), 3);
    assert_eq!(calls[2].collection, ATPROTO_COLLECTION);
    assert_eq!(calls[2].rkey, "CVE-2024-6408");
    assert_eq!(puts_for(&api, PROJECT_COLLECTION).len(), 1);
    assert!(puts_for(&api, SOFTWARE_COLLECTION).is_empty());
    assert!(puts_for(&api, PURL_COLLECTION).is_empty());

    // Nothing is pending after both CVEs have atprotoV1 records.
    let third = software::sync_software(&publisher, &db, &config(0), events)
        .await
        .unwrap();
    assert_eq!(
        counters_json(&third),
        json!({"cves": 0, "software_created": 0, "purl_created": 0, "atproto_created": 0, "failures": 0})
    );
    assert_eq!(api.puts().len(), 3);
}

#[tokio::test]
async fn sync_software_links_idempotent() {
    let temp = tempfile::tempdir().unwrap();
    let db = DbHandle::open(temp.path().join("cache.db")).await.unwrap();
    let older = cve("CVE-2024-0001", "2024-01-01T00:00:00.000");
    let newer = cve("CVE-2024-0002", "2024-02-01T00:00:00.000");
    seed(&db, &older).await;
    seed(&db, &newer).await;
    db.commit().await.unwrap();

    let api = FakePds::new();
    let publisher = make_publisher(api.clone(), &db);
    publisher.login().await.unwrap();
    assert_eq!(publisher.repo(), REPO);

    let (events, mut received) = mpsc::channel::<ProgressEvent>(16);
    let first = software::sync_software(&publisher, &db, &config(1), events.clone())
        .await
        .unwrap();
    assert_eq!(
        counters_json(&first),
        json!({"cves": 1, "software_created": 1, "purl_created": 2, "atproto_created": 1, "failures": 0})
    );

    // Newest lastModified is processed first (limit 1). The canonical project
    // slug stays `acme-widget`; the raw software rkey is `a:acme:widget`.
    let project_uri = format!("at://{REPO}/{PROJECT_COLLECTION}/acme-widget");
    let first_atproto_uri = format!("at://{REPO}/{ATPROTO_COLLECTION}/CVE-2024-0002");
    let npm_uri = format!(
        "at://{REPO}/{PURL_COLLECTION}/{}",
        purl_rkey("pkg:npm/acme-widget")
    );
    let pypi_uri = format!(
        "at://{REPO}/{PURL_COLLECTION}/{}",
        purl_rkey("pkg:pypi/acme-widget")
    );
    let cpe = "cpe:2.3:a:acme:widget:1.0:*:*:*:*:*:*:*";
    let purls = json!(["pkg:npm/acme-widget", "pkg:pypi/acme-widget"]);

    let calls = api.puts();
    let expected_calls: Vec<(String, String)> = vec![
        (PROJECT_COLLECTION.to_owned(), "acme-widget".to_owned()),
        (ATPROTO_COLLECTION.to_owned(), "CVE-2024-0002".to_owned()),
    ];
    assert_eq!(
        calls
            .iter()
            .map(|call| (call.collection.clone(), call.rkey.clone()))
            .collect::<Vec<_>>(),
        expected_calls
    );
    assert!(calls.iter().all(|call| call.repo == REPO));

    assert_eq!(
        calls[0].record.clone().unwrap(),
        json!({"$type": PROJECT_TYPE, "name": "acme-widget"})
    );
    let atproto = calls[1].record.as_ref().unwrap();
    assert_eq!(atproto["$type"], ATPROTO_TYPE);
    assert_eq!(atproto["cve"], "CVE-2024-0002");
    assert_eq!(
        atproto["nvd"],
        json!({
            "uri": format!("at://{REPO}/{COLLECTION}/CVE-2024-0002"),
            "cid": "bafynvd",
        })
    );
    let created_at = atproto["createdAt"].as_str().unwrap();
    assert_eq!(created_at.len(), 24, "{created_at}");
    assert!(created_at.ends_with('Z'));
    assert_eq!(
        atproto["software"],
        json!([{
            "project": {"uri": project_uri, "cid": PUT_CID},
            "part": "a",
            "vendor": "acme",
            "product": "widget",
            "cpes": [cpe],
            "purls": purls,
        }])
    );

    // Links are indexed only after successful writes.
    for (purl_uri, purl) in [
        (&npm_uri, "pkg:npm/acme-widget"),
        (&pypi_uri, "pkg:pypi/acme-widget"),
    ] {
        let outgoing = db.outgoing(REPO, purl_uri).await.unwrap();
        assert!(outgoing.is_empty(), "{purl}: purls are local-only");
        // purl rows store the canonical project slug, not an AT-URI.
        assert_eq!(
            db.get_purl(REPO, purl)
                .await
                .unwrap()
                .expect("purl row")
                .software_slug,
            "acme-widget"
        );
    }
    let outgoing = db.outgoing(REPO, &first_atproto_uri).await.unwrap();
    let expected: BTreeSet<(String, String)> = BTreeSet::from([
        (
            "nvd.uri".to_owned(),
            format!("at://{REPO}/{COLLECTION}/CVE-2024-0002"),
        ),
        ("software[].project.uri".to_owned(), project_uri.clone()),
    ]);
    assert_eq!(
        outgoing
            .iter()
            .map(|row| (row.path.clone(), row.subject_uri.clone()))
            .collect::<BTreeSet<_>>(),
        expected
    );
    let backlinks = db
        .backlinks(REPO, &project_uri, None, None, 10)
        .await
        .unwrap();
    assert_eq!(
        backlinks
            .iter()
            .map(|row| row.src_uri.as_str())
            .collect::<BTreeSet<_>>(),
        BTreeSet::from([first_atproto_uri.as_str()])
    );

    // Second CVE reuses the existing software/purl records.
    let second = software::sync_software(&publisher, &db, &config(1), events.clone())
        .await
        .unwrap();
    assert_eq!(
        counters_json(&second),
        json!({"cves": 1, "software_created": 0, "purl_created": 0, "atproto_created": 1, "failures": 0})
    );
    let calls = api.puts();
    assert_eq!(calls.len(), 3);
    assert_eq!(calls[2].collection, ATPROTO_COLLECTION);
    assert_eq!(calls[2].rkey, "CVE-2024-0001");
    let second_atproto_uri = format!("at://{REPO}/{ATPROTO_COLLECTION}/CVE-2024-0001");
    assert!(
        db.outgoing(REPO, &second_atproto_uri)
            .await
            .unwrap()
            .iter()
            .any(|row| row.path == "software[].project.uri" && row.subject_uri == project_uri)
    );

    // Third run: nothing pending, no new writes.
    let third = software::sync_software(&publisher, &db, &config(0), events)
        .await
        .unwrap();
    assert_eq!(
        counters_json(&third),
        json!({"cves": 0, "software_created": 0, "purl_created": 0, "atproto_created": 0, "failures": 0})
    );
    assert_eq!(api.puts().len(), 3);

    // Phase events: started, completed for every run.
    let mut names = Vec::new();
    while let Ok(event) = received.try_recv() {
        names.push(match event {
            ProgressEvent::SoftwarePhaseStarted { .. } => "started",
            ProgressEvent::SoftwarePhaseCompleted { .. } => "completed",
            _ => "other",
        });
    }
    assert_eq!(
        names,
        vec![
            "started",
            "completed",
            "started",
            "completed",
            "started",
            "completed"
        ]
    );
}