~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
648 lines · 21.3 KB Raw History
//! Metadata and append-only status wiring across release modes.
//!
//! Latest publishes `projectMeta` and `release_version_meta` and attaches
//! source git facts to manifests; affected attaches git/target metadata; a
//! reason change appends a status event in latest and history; CVE presence
//! never creates a status; unchanged re-runs make zero PDS writes.
//!
//! Every HTTP and PDS seam is local (`WiringHttp`, `common::FakePds`).

mod common;

use std::collections::HashMap;
use std::sync::{Arc, Mutex};

use async_trait::async_trait;
use serde_json::{Value, json};
use tokio::sync::mpsc;

use common::FakePds;
use vuln_notif::constants::{
    ATPROTO_COLLECTION, MANIFEST_COLLECTION, STATUS_COLLECTION, VERSION_COLLECTION,
};
use vuln_notif::db::DbHandle;
use vuln_notif::db::model::{ReleaseVersionMark, SoftwareMark};
use vuln_notif::links;
use vuln_notif::pds::Publisher;
use vuln_notif::records::sha256_hex;
use vuln_notif::release::artifacts::ArtifactResolver;
use vuln_notif::release::http::RegistryHttp;
use vuln_notif::release::metadata::PROJECT_META_COLLECTION;
use vuln_notif::release::sources::composer::ComposerSource;
use vuln_notif::release::{
    AffectedConfig, HistoryProjectConfig, ReleaseConfig, sync_affected, sync_history,
    sync_releases_with_http,
};

const REPO: &str = common::REPO;
const SLUG: &str = "acme-widget";
const CVE: &str = "CVE-2024-9999";
const DEPSDEV: &str = "https://api.deps.dev/v3/systems/PYPI/packages/acme-widget";
const PYPI: &str = "https://pypi.org/pypi/acme-widget/json";
const GEM_VERSIONS: &str = "https://rubygems.org/api/v1/versions/acme-widget.json";
const GEM_META: &str = "https://rubygems.org/api/v1/gems/acme-widget.json";
const GEM_DEPSDEV: &str = "https://api.deps.dev/v3/systems/RUBYGEMS/packages/acme-widget";
const COMPOSER: &str = "https://packagist.example/p2/acme/widget.json";
const ARTIFACT_URL: &str = "https://files.example/acme_widget-1.2.3-py3-none-any.whl";
const GEM_ARTIFACT_URL: &str = "https://rubygems.org/gems/acme-widget-1.5.0.gem";
const GEM_DIGEST: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";

// ---------------------------------------------------------------------------
// Local HTTP seam
// ---------------------------------------------------------------------------

#[derive(Default)]
struct WiringHttp {
    json: Mutex<HashMap<String, Value>>,
    posts: Mutex<HashMap<String, Value>>,
    bytes: Mutex<HashMap<String, Vec<u8>>>,
}

impl WiringHttp {
    fn json(&self, url: &str, value: Value) {
        self.json.lock().unwrap().insert(url.to_owned(), value);
    }

    fn post(&self, purl: &str, value: Value) {
        self.posts.lock().unwrap().insert(purl.to_owned(), value);
    }

    fn bytes(&self, url: &str, value: &[u8]) {
        self.bytes
            .lock()
            .unwrap()
            .insert(url.to_owned(), value.to_vec());
    }
}

#[async_trait]
impl RegistryHttp for WiringHttp {
    async fn get_json(&self, url: &str) -> anyhow::Result<Value> {
        self.json
            .lock()
            .unwrap()
            .get(url)
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("unexpected GET {url}"))
    }

    async fn get_text(&self, url: &str) -> anyhow::Result<String> {
        anyhow::bail!("unexpected text GET {url}")
    }

    async fn head_size(&self, url: &str) -> anyhow::Result<u64> {
        anyhow::bail!("unexpected HEAD {url}")
    }

    async fn download(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        self.bytes
            .lock()
            .unwrap()
            .get(url)
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("unexpected download {url}"))
    }

    async fn post_json(&self, url: &str, body: &Value) -> anyhow::Result<Value> {
        let purl = body
            .get("package")
            .and_then(|package| package.get("purl"))
            .and_then(Value::as_str)
            .unwrap_or_default()
            .to_owned();
        self.posts
            .lock()
            .unwrap()
            .get(&purl)
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("unexpected POST {url} for {purl}"))
    }
}

// ---------------------------------------------------------------------------
// Fixtures and seeding
// ---------------------------------------------------------------------------

fn depsdev(version: &str, deprecated: bool, reason: &str) -> Value {
    json!({
        "versions": [{
            "versionKey": {"version": version},
            "isDefault": true,
            "isDeprecated": deprecated,
            "deprecatedReason": reason,
            "publishedAt": "2025-11-02T10:00:00Z",
        }]
    })
}

fn pypi_payload() -> Value {
    json!({
        "info": {
            "name": "acme-widget",
            "summary": "Widget toolkit",
            "home_page": "https://acme.example/widget",
            "license": "MIT",
            "project_urls": {"Source": "https://github.com/acme/widget.git"},
        },
        "releases": {
            "1.2.3": [{
                "filename": "acme_widget-1.2.3-py3-none-any.whl",
                "url": ARTIFACT_URL,
                "packagetype": "bdist_wheel",
                "size": 12345,
                "digests": {"sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},
                "yanked": false,
            }]
        }
    })
}

fn gem_versions() -> Value {
    json!([
        {
            "number": "1.0.0",
            "platform": "ruby",
            "gem_uri": "https://rubygems.org/gems/acme-widget-1.0.0.gem",
            "sha": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
            "size": 1000,
            "yanked": false,
        },
        {
            "number": "1.5.0",
            "platform": "ruby",
            "gem_uri": GEM_ARTIFACT_URL,
            "sha": GEM_DIGEST,
            "size": 2000,
            "yanked": false,
        }
    ])
}

fn gem_meta() -> Value {
    json!({
        "name": "acme-widget",
        "info": "Widget toolkit",
        "homepage_uri": "https://acme.example/widget",
        "source_code_uri": "https://github.com/acme/widget.git",
        "platform": "x86_64-linux",
        "licenses": ["MIT"],
    })
}

fn osv_advisory(purl: &str, name: &str, ecosystem: &str) -> Value {
    json!({
        "vulns": [{
            "id": "OSV-2024-1",
            "aliases": [CVE],
            "modified": "2024-01-01T00:00:00Z",
            "affected": [{
                "package": {"ecosystem": ecosystem, "name": name, "purl": purl},
                "ranges": [{
                    "type": "ECOSYSTEM",
                    "events": [{"introduced": "0"}, {"fixed": "2.0.0"}],
                }],
            }],
        }]
    })
}

fn composer_payload(replacement: &str) -> Value {
    json!({
        "packages": {
            "acme/widget": [{
                "name": "acme/widget",
                "version": "1.0.0",
                "abandoned": replacement,
                "dist": {
                    "type": "zip",
                    "url": "https://packagist.example/dist/acme-widget-1.0.0.zip",
                    "shasum": sha256_hex(COMPOSER_BYTES),
                },
            }]
        }
    })
}

const COMPOSER_BYTES: &[u8] = b"composer-history-artifact";

async fn open_db(temp: &tempfile::TempDir) -> DbHandle {
    let db = DbHandle::open(temp.path().join("metadata-wiring.db"))
        .await
        .expect("db");
    db.commit().await.expect("schema commit");
    db
}

async fn seed_software(db: &DbHandle, slug: &str, purl: &str, project_uri: &str) {
    db.mark_software(SoftwareMark {
        repo_did: REPO.to_owned(),
        slug: slug.to_owned(),
        part: "a".to_owned(),
        vendor: "acme".to_owned(),
        product: "widget".to_owned(),
        project_uri: project_uri.to_owned(),
        project_cid: if project_uri.is_empty() {
            String::new()
        } else {
            "bafyproject".to_owned()
        },
        software_uri: format!("at://{REPO}/com.nahdig.software/{slug}"),
        software_cid: "bafysoftware".to_owned(),
    })
    .await
    .expect("software");
    db.mark_purl(
        REPO,
        purl,
        slug,
        &format!(
            "at://{REPO}/com.nahdig.purl/{}",
            purl.replace([':', '/'], "-")
        ),
        "bafypurl",
    )
    .await
    .expect("purl");
    db.commit().await.expect("seed commit");
}

async fn seed_version(db: &DbHandle, slug: &str, version: &str) {
    db.mark_release_version(ReleaseVersionMark {
        repo_did: REPO.to_owned(),
        project_slug: slug.to_owned(),
        version: version.to_owned(),
        version_raw: version.to_owned(),
        rkey: format!("{slug}:{version}"),
        uri: format!("at://{REPO}/{VERSION_COLLECTION}/{slug}:{version}"),
        cid: format!("bafyver{version}"),
        manifest_uri: format!("at://{REPO}/software.release.manifest/m{version}"),
        manifest_cid: format!("bafymanifest{version}"),
    })
    .await
    .expect("version row");
    db.commit().await.expect("version commit");
}

async fn seed_atproto(api: &FakePds, db: &DbHandle, project_uri: &str, purl: &str) {
    let record = json!({
        "$type": ATPROTO_COLLECTION,
        "createdAt": "2025-03-04T05:06:07Z",
        "cve": CVE,
        "nvd": {
            "uri": format!("at://{REPO}/com.nahdig.cve.nvdV2/{CVE}"),
            "cid": "bafynvd",
        },
        "software": [{
            "project": {"uri": project_uri, "cid": "bafyproject"},
            "part": "a",
            "vendor": "acme",
            "product": "widget",
            "cpes": [],
            "purls": [purl],
        }],
    });
    api.seed_record(ATPROTO_COLLECTION, CVE, record.clone());
    links::index_record(
        db,
        REPO,
        &format!("at://{REPO}/{ATPROTO_COLLECTION}/{CVE}"),
        Some("bafycve"),
        ATPROTO_COLLECTION,
        &record,
    )
    .await
    .expect("index cve links");
}

async fn publisher(pds: &Arc<FakePds>, db: &DbHandle) -> Publisher {
    let publisher = common::make_publisher(pds.clone(), db);
    publisher.login().await.expect("publisher login");
    publisher
}

fn records(pds: &FakePds, collection: &str) -> Vec<Value> {
    pds.store()
        .into_iter()
        .filter(|((entry, _), _)| entry == collection)
        .map(|(_, value)| value)
        .collect()
}

fn puts(pds: &FakePds, collection: &str) -> Vec<Value> {
    pds.puts()
        .into_iter()
        .filter(|put| put.collection == collection)
        .filter_map(|put| put.record)
        .collect()
}

fn manifest_artifact(pds: &FakePds) -> Value {
    let manifest = puts(pds, MANIFEST_COLLECTION)
        .into_iter()
        .next()
        .expect("manifest record");
    manifest["artifacts"][0].clone()
}

// ---------------------------------------------------------------------------
// Latest: projectMeta + version metadata + idempotency
// ---------------------------------------------------------------------------

#[tokio::test]
async fn latest_publishes_project_meta_and_version_meta() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    seed_software(&db, SLUG, "pkg:pypi/acme-widget", "").await;
    let pds = FakePds::new();
    let publisher = publisher(&pds, &db).await;

    let http = Arc::new(WiringHttp::default());
    http.json(DEPSDEV, depsdev("1.2.3", false, ""));
    http.json(PYPI, pypi_payload());

    let config = ReleaseConfig {
        repo_did: REPO.to_owned(),
        limit: 0,
        recheck_seconds: 0.0,
    };
    let (events, mut receiver) = mpsc::channel(64);
    let counters = sync_releases_with_http(&publisher, &db, &config, events, http.clone())
        .await
        .expect("latest sync");
    while receiver.try_recv().is_ok() {}

    assert_eq!(counters.versions_created, 1);
    assert_eq!(counters.failures, 0);

    let meta = pds
        .record(PROJECT_META_COLLECTION, SLUG)
        .expect("projectMeta record");
    assert_eq!(meta["$type"], json!(PROJECT_META_COLLECTION));
    assert_eq!(
        meta["project"],
        format!("at://{REPO}/software.release.project/{SLUG}")
    );
    assert_eq!(meta["name"], SLUG);
    assert_eq!(meta["description"], "Widget toolkit");
    assert_eq!(meta["homepage"], "https://acme.example/widget");
    assert_eq!(meta["license"], "MIT");
    assert_eq!(meta["repository"], "https://github.com/acme/widget.git");
    assert_eq!(meta["versioningScheme"], "pep440");

    let version_meta = db
        .get_release_version_meta(REPO, SLUG, "1.2.3")
        .await
        .expect("version meta query")
        .expect("version meta row");
    assert_eq!(version_meta.source.as_deref(), Some("pypi"));
    assert_eq!(
        version_meta.published_at.as_deref(),
        Some("2025-11-02T10:00:00Z")
    );
    assert_eq!(version_meta.raw_version.as_deref(), Some("1.2.3"));
    assert!(version_meta.artifact_cid.is_some());
    assert_eq!(version_meta.artifact_size, Some(12345));

    let artifact = manifest_artifact(&pds);
    assert_eq!(
        artifact["metadata"],
        json!([{"repository": "https://github.com/acme/widget.git"}])
    );
    assert!(artifact.get("target").is_none());

    // Identical second pass: no new PDS writes.
    let writes = pds.count("putRecord");
    let (events, mut receiver) = mpsc::channel(64);
    let second = sync_releases_with_http(&publisher, &db, &config, events, http)
        .await
        .expect("second latest sync");
    while receiver.try_recv().is_ok() {}
    assert_eq!(second.unchanged, 1);
    assert_eq!(second.versions_created, 0);
    assert_eq!(pds.count("putRecord"), writes, "second run must not write");
}

// ---------------------------------------------------------------------------
// Affected: git/target metadata on manifests, no CVE-driven status
// ---------------------------------------------------------------------------

#[tokio::test]
async fn affected_attaches_git_and_target_metadata_to_manifests() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    let slug = "acme-widget-gem";
    let purl = "pkg:gem/acme-widget";
    let project_uri = format!("at://{REPO}/software.release.project/{slug}");
    seed_software(&db, slug, purl, &project_uri).await;
    seed_version(&db, slug, "1.0.0").await;
    let pds = FakePds::new();
    seed_atproto(&pds, &db, &project_uri, purl).await;
    let publisher = publisher(&pds, &db).await;

    let http = Arc::new(WiringHttp::default());
    http.json(GEM_VERSIONS, gem_versions());
    http.json(GEM_META, gem_meta());
    http.json(
        GEM_DEPSDEV,
        json!({
            "versions": [{
                "versionKey": {"version": "1.5.0"},
                "isDefault": false,
                "isDeprecated": false,
                "publishedAt": "2025-12-01T10:00:00Z",
            }]
        }),
    );
    http.post(purl, osv_advisory(purl, "acme-widget", "RubyGems"));

    let config = AffectedConfig {
        repo_did: REPO.to_owned(),
        limit: 0,
        recheck_seconds: 0.0,
        ecosystems: Vec::new(),
        sources: Vec::new(),
        concurrency: 1,
        osv_online: true,
        osv_bulk_dir: None,
    };
    let (events, mut receiver) = mpsc::channel(64);
    let counters = sync_affected(&publisher, &db, &config, events, http)
        .await
        .expect("affected sync");
    while receiver.try_recv().is_ok() {}

    assert_eq!(counters.versions_created, 1);
    assert_eq!(counters.failures, 0);

    let artifact = manifest_artifact(&pds);
    assert_eq!(
        artifact["metadata"],
        json!([{"repository": "https://github.com/acme/widget.git"}])
    );
    assert_eq!(
        artifact["target"],
        json!({"os": "linux", "architectures": ["x86_64"]})
    );

    let version_meta = db
        .get_release_version_meta(REPO, slug, "1.5.0")
        .await
        .expect("version meta query")
        .expect("version meta row");
    assert_eq!(version_meta.source.as_deref(), Some("gem"));
    assert_eq!(
        version_meta.published_at.as_deref(),
        Some("2025-12-01T10:00:00Z")
    );
    assert_eq!(version_meta.raw_version.as_deref(), Some("1.5.0"));

    // A CVE advisory alone never creates a status event.
    assert!(
        puts(&pds, STATUS_COLLECTION).is_empty(),
        "CVE presence must not create a status"
    );
    let version_uri = format!("at://{REPO}/{VERSION_COLLECTION}/{slug}:1.5.0");
    assert!(
        db.list_release_status_events(REPO, &version_uri, 0)
            .await
            .expect("status events")
            .is_empty()
    );
}

// ---------------------------------------------------------------------------
// Latest: reason change appends
// ---------------------------------------------------------------------------

#[tokio::test]
async fn status_reason_change_appends_in_latest() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    seed_software(&db, SLUG, "pkg:pypi/acme-widget", "").await;
    let pds = FakePds::new();
    let publisher = publisher(&pds, &db).await;

    let http = Arc::new(WiringHttp::default());
    http.json(DEPSDEV, depsdev("1.2.3", true, "superseded by 2.0"));
    http.json(PYPI, pypi_payload());
    let config = ReleaseConfig {
        repo_did: REPO.to_owned(),
        limit: 0,
        recheck_seconds: 0.0,
    };

    let (events, mut receiver) = mpsc::channel(64);
    sync_releases_with_http(&publisher, &db, &config, events, http.clone())
        .await
        .expect("first sync");
    while receiver.try_recv().is_ok() {}
    let first = puts(&pds, STATUS_COLLECTION);
    assert_eq!(first.len(), 1);
    assert_eq!(first[0]["status"], "deprecated");
    assert_eq!(first[0]["kind"], "superseded");
    assert_eq!(first[0]["reason"], "superseded by 2.0");

    http.json(DEPSDEV, depsdev("1.2.3", true, "superseded by 3.0"));
    let (events, mut receiver) = mpsc::channel(64);
    sync_releases_with_http(&publisher, &db, &config, events, http.clone())
        .await
        .expect("second sync");
    while receiver.try_recv().is_ok() {}

    let statuses = puts(&pds, STATUS_COLLECTION);
    assert_eq!(statuses.len(), 2, "reason change appends a new event");
    assert!(
        statuses
            .iter()
            .any(|record| record["reason"] == "superseded by 2.0")
    );
    assert!(
        statuses
            .iter()
            .any(|record| record["reason"] == "superseded by 3.0")
    );
    let version_uri = format!("at://{REPO}/{VERSION_COLLECTION}/{SLUG}:1.2.3");
    let events_rows = db
        .list_release_status_events(REPO, &version_uri, 0)
        .await
        .expect("status events");
    assert_eq!(events_rows.len(), 2);
    assert_eq!(records(&pds, STATUS_COLLECTION).len(), 2);

    // Re-observing the same reason is a no-op and writes nothing.
    let writes = pds.count("putRecord");
    let (events, mut receiver) = mpsc::channel(64);
    let third = sync_releases_with_http(&publisher, &db, &config, events, http)
        .await
        .expect("third sync");
    while receiver.try_recv().is_ok() {}
    assert_eq!(third.unchanged, 1);
    assert_eq!(pds.count("putRecord"), writes);
}

// ---------------------------------------------------------------------------
// History: reason change appends
// ---------------------------------------------------------------------------

#[tokio::test]
async fn status_reason_change_appends_in_history() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    let slug = "acme-widget-php";
    let pds = FakePds::new();
    let publisher = publisher(&pds, &db).await;

    let http = Arc::new(WiringHttp::default());
    http.json(COMPOSER, composer_payload("acme/new-widget"));
    http.bytes(
        "https://packagist.example/dist/acme-widget-1.0.0.zip",
        COMPOSER_BYTES,
    );
    let source = ComposerSource::with_base_url(http.clone(), "https://packagist.example");
    let resolver = ArtifactResolver::new(http.clone()).with_cache(db.clone(), REPO);
    let config = HistoryProjectConfig {
        repo_did: REPO.to_owned(),
        project_slug: slug.to_owned(),
        purl: "pkg:composer/acme/widget".to_owned(),
        source: "composer".to_owned(),
        limit: 0,
        max_bytes: 0,
    };

    let (events, mut receiver) = mpsc::channel(64);
    let first = sync_history(&publisher, &db, &config, events, &source, &resolver)
        .await
        .expect("first history sync");
    while receiver.try_recv().is_ok() {}
    assert_eq!(first.versions_created, 1);
    assert_eq!(first.failed, 0);
    let statuses = puts(&pds, STATUS_COLLECTION);
    assert_eq!(statuses.len(), 1);
    assert_eq!(statuses[0]["status"], "deprecated");
    assert_eq!(
        statuses[0]["reason"],
        "abandoned in favor of acme/new-widget"
    );

    http.json(COMPOSER, composer_payload("acme/other-widget"));
    let (events, mut receiver) = mpsc::channel(64);
    let second = sync_history(&publisher, &db, &config, events, &source, &resolver)
        .await
        .expect("second history sync");
    while receiver.try_recv().is_ok() {}
    assert_eq!(second.versions_created, 0);

    let statuses = puts(&pds, STATUS_COLLECTION);
    assert_eq!(statuses.len(), 2, "reason change appends a new event");
    assert!(
        statuses
            .iter()
            .any(|record| record["reason"] == "abandoned in favor of acme/other-widget")
    );
    let version_uri = format!("at://{REPO}/{VERSION_COLLECTION}/{slug}:1.0.0");
    let events_rows = db
        .list_release_status_events(REPO, &version_uri, 0)
        .await
        .expect("status events");
    assert_eq!(events_rows.len(), 2);
}