~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
733 lines · 23.3 KB Raw History
//! Link extraction parity: paths, strongRefs, arrays, dedupe, indexing.

use serde_json::{Value, json};
use vuln_notif::constants::{
    ATPROTO_COLLECTION, CHANNEL_COLLECTION, COLLECTION, MANIFEST_COLLECTION, PROJECT_COLLECTION,
    PURL_COLLECTION, SOFTWARE_COLLECTION, STATUS_COLLECTION, VERSION_COLLECTION,
};
use vuln_notif::db::DbHandle;
use vuln_notif::db::model::LinkRepairMark;
use vuln_notif::links::{
    LINK_SPECS, Link, LinkRepairCounters, drain_link_repairs, extract_links, index_or_queue,
    index_record, repair_after_commit,
};

const DID: &str = "did:plc:test";

fn pairs(links: &[Link]) -> Vec<(String, String, Option<String>)> {
    links
        .iter()
        .map(|link| {
            (
                link.path.clone(),
                link.subject_uri.clone(),
                link.subject_cid.clone(),
            )
        })
        .collect()
}

fn expected(path: &str, uri: &str, cid: Option<&str>) -> (String, String, Option<String>) {
    (path.to_owned(), uri.to_owned(), cid.map(str::to_owned))
}

#[test]
fn extract_links_all_shapes() {
    let project_uri = format!("at://{DID}/software.release.project/acme-widget");
    let nvd_uri = format!("at://{DID}/com.nahdig.cve.nvdV2/CVE-2024-1");
    let other_project = format!("at://{DID}/software.release.project/other");
    let channel_uri = format!("at://{DID}/software.release.channel/acme-widget:9");
    let version_uri = format!("at://{DID}/software.release.version/acme-widget:9.4");
    let manifest_uri = format!("at://{DID}/software.release.manifest/3jzfcijpj2z2a");
    let atproto = json!({
        "nvd": {"uri": nvd_uri, "cid": "bafynvd"},
        "software": [
            {
                "project": {"uri": project_uri, "cid": "bafyproject"},
                "channels": [{"uri": channel_uri, "cid": "bafychannel"}],
                "versions": [{"uri": version_uri, "cid": "bafyversion"}],
                "manifests": [{"uri": manifest_uri, "cid": "bafymanifest"}],
            },
            {"project": {"uri": other_project}},
        ],
    });
    assert_eq!(
        pairs(&extract_links(ATPROTO_COLLECTION, &atproto)),
        vec![
            expected("nvd.uri", &nvd_uri, Some("bafynvd")),
            expected("software[].project.uri", &project_uri, Some("bafyproject")),
            expected("software[].project.uri", &other_project, None),
            expected(
                "software[].channels[].uri",
                &channel_uri,
                Some("bafychannel")
            ),
            expected(
                "software[].versions[].uri",
                &version_uri,
                Some("bafyversion")
            ),
            expected(
                "software[].manifests[].uri",
                &manifest_uri,
                Some("bafymanifest")
            ),
        ]
    );

    // Unknown collection and non-at:// subjects produce no links.
    assert!(extract_links("com.nahdig.cve.nvdV2", &json!({"id": "CVE-2024-1"})).is_empty());
    assert!(
        extract_links(
            SOFTWARE_COLLECTION,
            &json!({"project": {"uri": "https://example.com"}})
        )
        .is_empty()
    );
}

#[test]
fn extract_links_release_shapes() {
    let project = format!("at://{DID}/software.release.project/acme");
    let parent = format!("at://{DID}/software.release.project/acme-legacy");
    let renamed_to = format!("at://{DID}/software.release.project/acme-next");
    let manifest = format!("at://{DID}/software.release.manifest/acme");
    let version = format!("at://{DID}/software.release.version/acme");

    assert_eq!(
        pairs(&extract_links(
            PROJECT_COLLECTION,
            &json!({"name": "acme", "parent": parent, "renamedTo": renamed_to})
        )),
        vec![
            expected("parent", &parent, None),
            expected("renamedTo", &renamed_to, None),
        ]
    );
    assert_eq!(
        pairs(&extract_links(
            MANIFEST_COLLECTION,
            &json!({"project": {"uri": project, "cid": "bafyp"}})
        )),
        vec![expected("project", &project, Some("bafyp"))]
    );
    assert_eq!(
        pairs(&extract_links(
            VERSION_COLLECTION,
            &json!({"project": {"uri": project}, "manifest": {"uri": manifest, "cid": "bafym"}})
        )),
        vec![
            expected("project", &project, None),
            expected("manifest.uri", &manifest, Some("bafym")),
        ]
    );
    assert_eq!(
        pairs(&extract_links(
            CHANNEL_COLLECTION,
            &json!({"project": project, "version": {"uri": version, "cid": "bafyv"}})
        )),
        vec![
            expected("project", &project, None),
            expected("version.uri", &version, Some("bafyv")),
        ]
    );
    let subject = format!("at://{DID}/software.release.version/1");
    assert_eq!(
        pairs(&extract_links(
            STATUS_COLLECTION,
            &json!({"subject": subject})
        )),
        vec![expected("subject", &subject, None)]
    );

    // Malformed / wrong-shape fields are ignored, not panicked on.
    assert!(extract_links(VERSION_COLLECTION, &json!({"project": []})).is_empty());
    assert!(extract_links(PURL_COLLECTION, &json!({"projects": "nope"})).is_empty());
    assert!(extract_links(PURL_COLLECTION, &json!({"projects": [1, null, {}]})).is_empty());
    assert!(extract_links(STATUS_COLLECTION, &json!({"subject": "https://x"})).is_empty());
}

#[test]
fn link_specs_are_exact() {
    assert_eq!(
        LINK_SPECS,
        &[
            (
                ATPROTO_COLLECTION,
                &[
                    "nvd.uri",
                    "software[].project.uri",
                    "software[].channels[].uri",
                    "software[].versions[].uri",
                    "software[].manifests[].uri",
                ][..]
            ),
            (SOFTWARE_COLLECTION, &["project.uri"][..]),
            (PURL_COLLECTION, &["projects[].uri"][..]),
            (PROJECT_COLLECTION, &["parent", "renamedTo"][..]),
            (MANIFEST_COLLECTION, &["project"][..]),
            (VERSION_COLLECTION, &["project", "manifest.uri"][..]),
            (CHANNEL_COLLECTION, &["project", "version.uri"][..]),
            (STATUS_COLLECTION, &["subject"][..]),
            ("com.nahdig.release.projectMeta", &["project"][..]),
        ]
    );
}

#[tokio::test]
async fn index_record_replaces_links() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = format!("at://{DID}/com.nahdig.software/acme-widget");
    let first = format!("at://{DID}/software.release.project/first");
    let second = format!("at://{DID}/software.release.project/second");

    let record = json!({"project": {"uri": first, "cid": "bafyfirst"}});
    index_record(
        &db,
        DID,
        &src,
        Some("bafyold"),
        SOFTWARE_COLLECTION,
        &record,
    )
    .await
    .unwrap();
    let record = json!({"project": {"uri": second, "cid": "bafysecond"}});
    index_record(
        &db,
        DID,
        &src,
        Some("bafynew"),
        SOFTWARE_COLLECTION,
        &record,
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    assert!(
        db.backlinks(DID, &first, None, None, 10)
            .await
            .unwrap()
            .is_empty()
    );
    let rows = db.backlinks(DID, &second, None, None, 10).await.unwrap();
    assert_eq!(rows.len(), 1);
    assert_eq!(rows[0].src_uri, src);
    assert_eq!(rows[0].src_cid.as_deref(), Some("bafynew"));
    assert_eq!(rows[0].src_rkey, "acme-widget");
    assert_eq!(rows[0].path, "project.uri");
    assert!(
        db.backlinks(DID, &second, Some("other.collection"), None, 10)
            .await
            .unwrap()
            .is_empty()
    );
    assert!(
        db.backlinks(DID, &second, None, Some("other.path"), 10)
            .await
            .unwrap()
            .is_empty()
    );
    let outgoing = db.outgoing(DID, &src).await.unwrap();
    assert_eq!(
        outgoing
            .iter()
            .map(|row| (row.path.as_str(), row.subject_uri.as_str()))
            .collect::<Vec<_>>(),
        vec![("project.uri", second.as_str())]
    );
}

fn strong(uri: &str, cid: &str) -> Value {
    json!({"uri": uri, "cid": cid})
}

fn cve_record(nvd_uri: &str, project_uri: &str, channel_uri: &str, version_uri: &str) -> Value {
    json!({
        "$type": ATPROTO_COLLECTION,
        "createdAt": "2026-01-01T00:00:00Z",
        "cve": "CVE-2024-3094",
        "nvd": strong(nvd_uri, "bafynvd"),
        "software": [{
            "project": strong(project_uri, "bafyproject"),
            "part": "o",
            "vendor": "Red Hat",
            "product": "Red Hat Enterprise Linux 9",
            "channels": [strong(channel_uri, "bafychannel")],
            "versions": [strong(version_uri, "bafyversion")],
            "manifests": [strong(
                &format!("at://{DID}/software.release.manifest/3jzfcijpj2z2a"),
                "bafymanifest"
            )],
        }],
    })
}

#[tokio::test]
async fn cve_links_have_backlinks_and_outgoing_on_canonical_refs() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let cve_uri = format!("at://{DID}/com.nahdig.cve.atprotoV1/CVE-2024-3094");
    let nvd_uri = format!("at://{DID}/com.nahdig.cve.nvdV2/CVE-2024-3094");
    let project_uri = format!("at://{DID}/software.release.project/red-hat-enterprise-linux");
    let channel_uri = format!("at://{DID}/software.release.channel/red-hat-enterprise-linux:9");
    let version_uri = format!("at://{DID}/software.release.version/red-hat-enterprise-linux:9.4");
    let manifest_uri = format!("at://{DID}/software.release.manifest/3jzfcijpj2z2a");

    let record = cve_record(&nvd_uri, &project_uri, &channel_uri, &version_uri);
    index_record(
        &db,
        DID,
        &cve_uri,
        Some("bafycve"),
        ATPROTO_COLLECTION,
        &record,
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    for (path, subject, subject_cid) in [
        ("nvd.uri", &nvd_uri, "bafynvd"),
        ("software[].project.uri", &project_uri, "bafyproject"),
        ("software[].channels[].uri", &channel_uri, "bafychannel"),
        ("software[].versions[].uri", &version_uri, "bafyversion"),
        ("software[].manifests[].uri", &manifest_uri, "bafymanifest"),
    ] {
        let rows = db
            .backlinks(DID, subject, Some(ATPROTO_COLLECTION), Some(path), 10)
            .await
            .unwrap();
        assert_eq!(rows.len(), 1, "backlink {path}");
        assert_eq!(rows[0].src_uri, cve_uri);
        assert_eq!(rows[0].src_collection, ATPROTO_COLLECTION);
        assert_eq!(rows[0].src_cid.as_deref(), Some("bafycve"));
        assert_eq!(rows[0].subject_cid.as_deref(), Some(subject_cid));
    }

    let outgoing = db.outgoing(DID, &cve_uri).await.unwrap();
    // `outgoing` orders by path then subject URI.
    assert_eq!(
        outgoing
            .iter()
            .map(|row| (row.path.as_str(), row.subject_uri.as_str()))
            .collect::<Vec<_>>(),
        vec![
            ("nvd.uri", nvd_uri.as_str()),
            ("software[].channels[].uri", channel_uri.as_str()),
            ("software[].manifests[].uri", manifest_uri.as_str()),
            ("software[].project.uri", project_uri.as_str()),
            ("software[].versions[].uri", version_uri.as_str()),
        ]
    );
}

#[tokio::test]
async fn project_parent_and_renamed_to_are_indexed() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = format!("at://{DID}/software.release.project/acme");
    let parent = format!("at://{DID}/software.release.project/acme-legacy");
    let renamed_to = format!("at://{DID}/software.release.project/acme-next");
    let record = json!({
        "$type": PROJECT_COLLECTION,
        "name": "acme",
        "parent": parent,
        "renamedTo": renamed_to,
    });
    index_record(
        &db,
        DID,
        &src,
        Some("bafyproject"),
        PROJECT_COLLECTION,
        &record,
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    for (path, subject) in [("parent", &parent), ("renamedTo", &renamed_to)] {
        let rows = db
            .backlinks(DID, subject, Some(PROJECT_COLLECTION), Some(path), 10)
            .await
            .unwrap();
        assert_eq!(rows.len(), 1, "backlink {path}");
        assert_eq!(rows[0].src_uri, src);
        assert_eq!(rows[0].subject_cid, None);
    }
    let outgoing = db.outgoing(DID, &src).await.unwrap();
    assert_eq!(
        outgoing
            .iter()
            .map(|row| (row.path.as_str(), row.subject_uri.as_str()))
            .collect::<Vec<_>>(),
        vec![
            ("parent", parent.as_str()),
            ("renamedTo", renamed_to.as_str()),
        ]
    );
}

#[tokio::test]
async fn reindexing_changed_cve_removes_stale_links() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let cve_uri = format!("at://{DID}/com.nahdig.cve.atprotoV1/CVE-2024-3094");
    let nvd_uri = format!("at://{DID}/com.nahdig.cve.nvdV2/CVE-2024-3094");
    let project_uri = format!("at://{DID}/software.release.project/acme");
    let old_channel = format!("at://{DID}/software.release.channel/acme:9");
    let old_version = format!("at://{DID}/software.release.version/acme:9.4");
    let new_channel = format!("at://{DID}/software.release.channel/acme:8");

    let first = cve_record(&nvd_uri, &project_uri, &old_channel, &old_version);
    index_record(
        &db,
        DID,
        &cve_uri,
        Some("bafyold"),
        ATPROTO_COLLECTION,
        &first,
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    // The changed record drops the version and manifest refs and moves the
    // channel; every stale row for this source must disappear.
    let second = json!({
        "$type": ATPROTO_COLLECTION,
        "createdAt": "2026-01-02T00:00:00Z",
        "cve": "CVE-2024-3094",
        "nvd": strong(&nvd_uri, "bafynvd"),
        "software": [{
            "project": strong(&project_uri, "bafyproject2"),
            "part": "o",
            "vendor": "Red Hat",
            "product": "Red Hat Enterprise Linux 9",
            "channels": [strong(&new_channel, "bafychannel2")],
        }],
    });
    index_record(
        &db,
        DID,
        &cve_uri,
        Some("bafynew"),
        ATPROTO_COLLECTION,
        &second,
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    for (path, stale) in [
        ("software[].channels[].uri", &old_channel),
        ("software[].versions[].uri", &old_version),
    ] {
        assert!(
            db.backlinks(DID, stale, Some(ATPROTO_COLLECTION), Some(path), 10)
                .await
                .unwrap()
                .is_empty(),
            "{path} row for {stale} was not deleted"
        );
    }
    let rows = db
        .backlinks(
            DID,
            &new_channel,
            Some(ATPROTO_COLLECTION),
            Some("software[].channels[].uri"),
            10,
        )
        .await
        .unwrap();
    assert_eq!(rows.len(), 1);
    assert_eq!(rows[0].src_cid.as_deref(), Some("bafynew"));
    assert_eq!(rows[0].subject_cid.as_deref(), Some("bafychannel2"));
    let outgoing = db.outgoing(DID, &cve_uri).await.unwrap();
    assert_eq!(
        outgoing
            .iter()
            .map(|row| (row.path.as_str(), row.subject_uri.as_str()))
            .collect::<Vec<_>>(),
        vec![
            ("nvd.uri", nvd_uri.as_str()),
            ("software[].channels[].uri", new_channel.as_str()),
            ("software[].project.uri", project_uri.as_str()),
        ]
    );
}

#[tokio::test]
async fn raw_nvd_records_create_no_links() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = format!("at://{DID}/com.nahdig.cve.nvdV2/CVE-2024-3094");
    let record = json!({
        "$type": COLLECTION,
        "id": "CVE-2024-3094",
        "software": [{"project": {"uri": format!("at://{DID}/software.release.project/acme")}}],
        "configurations": [{
            "nodes": [{
                "cpeMatch": [{
                    "criteria": "cpe:2.3:a:acme:widget:1.0:*:*:*:*:*:*:*",
                }],
            }],
        }],
    });
    assert!(extract_links(COLLECTION, &record).is_empty());
    index_record(&db, DID, &src, Some("bafynvd"), COLLECTION, &record)
        .await
        .unwrap();
    db.commit().await.unwrap();
    assert!(db.outgoing(DID, &src).await.unwrap().is_empty());
}

const PAST: &str = "2000-01-01T00:00:00.000Z";

fn software_src() -> String {
    format!("at://{DID}/com.nahdig.software/acme-widget")
}

fn software_record(project: &str) -> Value {
    json!({
        "$type": SOFTWARE_COLLECTION,
        "project": {"uri": project, "cid": "bafyproject"},
    })
}

/// Upsert a due repair row, as a previous failed index would have left it.
async fn seed_repair(db: &DbHandle, src: &str, record_json: &str, attempts: i64) {
    db.mark_link_repair(LinkRepairMark {
        repo_did: DID.to_owned(),
        src_uri: src.to_owned(),
        src_cid: Some("bafyrepair".to_owned()),
        collection: Some(SOFTWARE_COLLECTION.to_owned()),
        record_json: record_json.to_owned(),
        attempts,
        last_error: Some("seeded failure".to_owned()),
        next_retry_at: Some(PAST.to_owned()),
    })
    .await
    .unwrap();
}

#[tokio::test]
async fn failed_index_queues_serialized_record() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = software_src();
    let record = json!(["not", "a", "record"]);

    index_or_queue(
        &db,
        DID,
        &src,
        Some("bafybad"),
        SOFTWARE_COLLECTION,
        &record,
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    let row = db
        .get_link_repair(DID, &src)
        .await
        .unwrap()
        .expect("failed index is queued");
    assert_eq!(row.record_json, serde_json::to_string(&record).unwrap());
    assert_eq!(row.collection.as_deref(), Some(SOFTWARE_COLLECTION));
    assert_eq!(row.attempts, 1);
    assert!(
        row.last_error
            .as_deref()
            .unwrap_or_default()
            .contains("JSON object"),
        "{:?}",
        row.last_error
    );
    assert!(row.next_retry_at.is_some());
    assert!(db.outgoing(DID, &src).await.unwrap().is_empty());
}

#[tokio::test]
async fn successful_index_does_not_queue() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = software_src();
    let project = format!("at://{DID}/software.release.project/acme");

    index_or_queue(
        &db,
        DID,
        &src,
        Some("bafysrc"),
        SOFTWARE_COLLECTION,
        &software_record(&project),
    )
    .await
    .unwrap();
    db.commit().await.unwrap();

    assert!(db.get_link_repair(DID, &src).await.unwrap().is_none());
    let rows = db
        .backlinks(
            DID,
            &project,
            Some(SOFTWARE_COLLECTION),
            Some("project.uri"),
            10,
        )
        .await
        .unwrap();
    assert_eq!(rows.len(), 1);
    assert_eq!(rows[0].src_uri, src);
    assert_eq!(rows[0].src_cid.as_deref(), Some("bafysrc"));
}

#[tokio::test]
async fn drain_reindexes_and_deletes_repair_across_restart() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("cache.db");
    let src = software_src();
    let project = format!("at://{DID}/software.release.project/acme");
    let record = software_record(&project);

    let db = DbHandle::open(&path).await.unwrap();
    // Fail the index, then repair the stored body and make it due.
    index_or_queue(
        &db,
        DID,
        &src,
        Some("bafybad"),
        SOFTWARE_COLLECTION,
        &json!("invalid"),
    )
    .await
    .unwrap();
    seed_repair(&db, &src, &record.to_string(), 1).await;
    db.commit().await.unwrap();
    db.close().await.unwrap();

    // Simulate a restart that drains from the persisted queue row.
    let db = DbHandle::open(&path).await.unwrap();
    let counters = drain_link_repairs(&db, 0).await.unwrap();
    assert_eq!(
        counters,
        LinkRepairCounters {
            attempted: 1,
            repaired: 1,
            failed: 0,
            deferred: 0,
        }
    );
    db.close().await.unwrap();

    // The drain committed both the link rows and the queue deletion.
    let db = DbHandle::open(&path).await.unwrap();
    assert!(db.get_link_repair(DID, &src).await.unwrap().is_none());
    let rows = db
        .backlinks(
            DID,
            &project,
            Some(SOFTWARE_COLLECTION),
            Some("project.uri"),
            10,
        )
        .await
        .unwrap();
    assert_eq!(rows.len(), 1);
    assert_eq!(rows[0].src_uri, src);
    // The queued source CID is restored on the repaired backlinks.
    assert_eq!(rows[0].src_cid.as_deref(), Some("bafyrepair"));
}

#[tokio::test]
async fn drain_retries_permanent_failure_with_backoff() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = software_src();
    seed_repair(&db, &src, "\"not an object\"", 1).await;

    let counters = drain_link_repairs(&db, 0).await.unwrap();
    assert_eq!(
        counters,
        LinkRepairCounters {
            attempted: 1,
            repaired: 0,
            failed: 1,
            deferred: 0,
        }
    );

    let row = db
        .get_link_repair(DID, &src)
        .await
        .unwrap()
        .expect("permanent failure stays queued");
    assert_eq!(row.attempts, 2);
    let retry_at =
        chrono::DateTime::parse_from_rfc3339(row.next_retry_at.as_deref().unwrap()).unwrap();
    assert!(retry_at.with_timezone(&chrono::Utc) > chrono::Utc::now());

    // Not due yet: the next pass defers without bumping attempts.
    let counters = drain_link_repairs(&db, 0).await.unwrap();
    assert_eq!(
        counters,
        LinkRepairCounters {
            attempted: 0,
            repaired: 0,
            failed: 0,
            deferred: 1,
        }
    );
    assert_eq!(
        db.get_link_repair(DID, &src)
            .await
            .unwrap()
            .unwrap()
            .attempts,
        2
    );

    // Backoff is capped even after many failures (plus at most 20% jitter).
    seed_repair(&db, &src, "\"not an object\"", 200).await;
    drain_link_repairs(&db, 0).await.unwrap();
    let row = db.get_link_repair(DID, &src).await.unwrap().unwrap();
    assert_eq!(row.attempts, 201);
    let retry_at =
        chrono::DateTime::parse_from_rfc3339(row.next_retry_at.as_deref().unwrap()).unwrap();
    let delay = retry_at.with_timezone(&chrono::Utc) - chrono::Utc::now();
    assert!(
        (3_590..=4_330).contains(&delay.num_seconds()),
        "capped backoff with jitter, got {delay}"
    );
}

#[tokio::test]
async fn drain_is_idempotent_and_makes_no_duplicate_links() {
    let db = DbHandle::open_in_memory().await.unwrap();
    let src = software_src();
    let project = format!("at://{DID}/software.release.project/acme");
    let record = software_record(&project);

    for _ in 0..2 {
        seed_repair(&db, &src, &record.to_string(), 1).await;
        let counters = repair_after_commit(&db, 0).await.unwrap();
        assert_eq!(counters.attempted, 1);
        assert_eq!(counters.repaired, 1);
        assert!(db.get_link_repair(DID, &src).await.unwrap().is_none());
    }

    let outgoing = db.outgoing(DID, &src).await.unwrap();
    assert_eq!(outgoing.len(), 1);
    assert_eq!(outgoing[0].subject_uri, project);
    assert_eq!(outgoing[0].path, "project.uri");
    let rows = db
        .backlinks(
            DID,
            &project,
            Some(SOFTWARE_COLLECTION),
            Some("project.uri"),
            10,
        )
        .await
        .unwrap();
    assert_eq!(rows.len(), 1);
}