~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
415 lines · 12.2 KB Raw History
//! History-mode end-to-end: source catalog -> immutable versions/manifests,
//! per-cycle limits, resume after artifact failure, typed unsupported skips,
//! artifact cache hits, and the default no-op mirror.
//!
//! Every network seam is local: a fake `ReleaseSource` supplies versions, and
//! a fake `RegistryHttp` serves the PyPI metadata and artifact bytes.

mod common;

use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};

use async_trait::async_trait;
use serde_json::{Value, json};
use tokio::sync::mpsc;

use common::FakePds;
use vuln_notif::db::DbHandle;
use vuln_notif::db::model::SoftwareMark;
use vuln_notif::pds::Publisher;
use vuln_notif::records::sha256_hex;
use vuln_notif::release::artifact::Purl;
use vuln_notif::release::artifacts::ArtifactResolver;
use vuln_notif::release::http::RegistryHttp;
use vuln_notif::release::source::{
    ArtifactInfo, ProjectRef, ReleaseSource, SourceCapabilities, StatusInfo, VersionInfo,
};
use vuln_notif::release::{
    HistoryModeConfig, HistoryProjectConfig, sync_history, sync_history_mode,
};

const REPO: &str = common::REPO;
const SLUG: &str = "acme-widget";
const PURL: &str = "pkg:pypi/acme-widget";
const PYPI_URL: &str = "https://pypi.org/pypi/acme-widget/json";
const ARTIFACT_BYTES: &[u8] = b"history-e2e-artifact-bytes";
const CATALOG: &str = include_str!("fixtures/history_e2e/catalog.json");

fn catalog_versions() -> Vec<String> {
    let catalog: Value = serde_json::from_str(CATALOG).expect("catalog fixture");
    catalog["versions"]
        .as_array()
        .expect("versions")
        .iter()
        .map(|value| value.as_str().expect("version").to_owned())
        .collect()
}

fn version(raw: &str) -> VersionInfo {
    VersionInfo {
        raw: raw.to_owned(),
        version: raw.to_owned(),
        deprecated: false,
        reason: String::new(),
    }
}

struct FakeSource {
    versions: Vec<VersionInfo>,
}

#[async_trait]
impl ReleaseSource for FakeSource {
    fn name(&self) -> &'static str {
        "pypi"
    }

    fn capabilities(&self) -> SourceCapabilities {
        SourceCapabilities {
            versions: true,
            artifacts: true,
            statuses: true,
            project_meta: true,
        }
    }

    async fn versions(&self, _project: &ProjectRef) -> anyhow::Result<Vec<VersionInfo>> {
        Ok(self.versions.clone())
    }

    async fn artifacts(
        &self,
        _project: &ProjectRef,
        _version: &VersionInfo,
    ) -> anyhow::Result<Vec<ArtifactInfo>> {
        Ok(Vec::new())
    }

    async fn statuses(
        &self,
        _project: &ProjectRef,
        _version: &VersionInfo,
    ) -> anyhow::Result<Vec<StatusInfo>> {
        Ok(Vec::new())
    }

    async fn project_meta(&self, _project: &ProjectRef) -> anyhow::Result<Value> {
        Ok(Value::Null)
    }
}

/// Serves the PyPI JSON plus artifact bytes; can fail the next download once.
struct FakeHttp {
    pypi: Value,
    downloads: AtomicUsize,
    fail_next: AtomicBool,
    calls: Mutex<Vec<String>>,
}

impl FakeHttp {
    fn new(versions: &[String]) -> Self {
        let digest = sha256_hex(ARTIFACT_BYTES);
        let releases = versions
            .iter()
            .map(|raw| {
                (
                    raw.clone(),
                    json!([{
                        "filename": format!("acme_widget-{raw}.tar.gz"),
                        "url": format!("https://files.example/{raw}.tar.gz"),
                        "packagetype": "sdist",
                        "digests": { "sha256": digest },
                        "size": ARTIFACT_BYTES.len(),
                    }]),
                )
            })
            .collect::<serde_json::Map<String, Value>>();
        Self {
            pypi: json!({ "releases": releases }),
            downloads: AtomicUsize::new(0),
            fail_next: AtomicBool::new(false),
            calls: Mutex::new(Vec::new()),
        }
    }

    fn downloads(&self) -> usize {
        self.downloads.load(Ordering::SeqCst)
    }
}

#[async_trait]
impl RegistryHttp for FakeHttp {
    async fn get_json(&self, url: &str) -> anyhow::Result<Value> {
        self.calls.lock().unwrap().push(format!("json {url}"));
        if url == PYPI_URL {
            Ok(self.pypi.clone())
        } else {
            anyhow::bail!("unexpected JSON request {url}")
        }
    }

    async fn get_text(&self, url: &str) -> anyhow::Result<String> {
        anyhow::bail!("unexpected text request {url}")
    }

    async fn head_size(&self, _url: &str) -> anyhow::Result<u64> {
        Ok(ARTIFACT_BYTES.len() as u64)
    }

    async fn download(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        self.calls.lock().unwrap().push(format!("download {url}"));
        self.downloads.fetch_add(1, Ordering::SeqCst);
        if self.fail_next.swap(false, Ordering::SeqCst) {
            anyhow::bail!("injected download failure for {url}")
        }
        Ok(ARTIFACT_BYTES.to_vec())
    }
}

async fn open_db(temp: &tempfile::TempDir) -> DbHandle {
    let db = DbHandle::open(temp.path().join("history-e2e.db"))
        .await
        .expect("db");
    db.commit().await.expect("schema commit");
    db
}

async fn publisher(pds: &Arc<FakePds>, db: &DbHandle) -> Publisher {
    let publisher = common::make_publisher(pds.clone(), db);
    publisher.login().await.expect("publisher login");
    publisher
}

fn project_config(limit: usize) -> HistoryProjectConfig {
    HistoryProjectConfig {
        repo_did: REPO.to_owned(),
        project_slug: SLUG.to_owned(),
        purl: PURL.to_owned(),
        source: "pypi".to_owned(),
        limit,
        max_bytes: 0,
    }
}

async fn run_project(
    db: &DbHandle,
    publisher: &Publisher,
    http: &Arc<FakeHttp>,
    limit: usize,
) -> vuln_notif::release::HistoryJobCounters {
    let resolver = ArtifactResolver::new(http.clone()).with_cache(db.clone(), REPO);
    let source = FakeSource {
        versions: catalog_versions().iter().map(|raw| version(raw)).collect(),
    };
    let (events, mut rx) = mpsc::channel(64);
    let counters = sync_history(
        publisher,
        db,
        &project_config(limit),
        events,
        &source,
        &resolver,
    )
    .await
    .expect("history sync");
    while rx.try_recv().is_ok() {}
    counters
}

#[tokio::test]
async fn history_creates_versions_manifests_and_respects_the_limit() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    let pds = FakePds::new();
    let publisher = publisher(&pds, &db).await;
    let http = Arc::new(FakeHttp::new(&catalog_versions()));

    let first = run_project(&db, &publisher, &http, 2).await;
    assert_eq!(first.enqueued, 3);
    assert_eq!(first.processed, 2);
    assert_eq!(first.versions_created, 2);
    assert_eq!(first.manifests_created, 2);
    assert_eq!(first.failed, 0);

    let pending = db
        .list_release_history_jobs(REPO, Some("pending"), 0)
        .await
        .expect("pending jobs");
    assert_eq!(pending.len(), 1);
    assert_eq!(pending[0].version_raw, "2.0.0");

    let second = run_project(&db, &publisher, &http, 2).await;
    assert_eq!(second.processed, 1);
    assert_eq!(second.versions_created, 1);

    for raw in ["1.0.0", "1.1.0", "2.0.0"] {
        assert!(
            db.get_release_version(REPO, SLUG, raw)
                .await
                .expect("version query")
                .is_some(),
            "missing version row for {raw}"
        );
        assert!(
            pds.record(
                vuln_notif::constants::VERSION_COLLECTION,
                &format!("{SLUG}:{raw}")
            )
            .is_some(),
            "missing version record for {raw}"
        );
    }
    assert_eq!(pds.count("putRecord"), 6, "two records per version");
}

#[tokio::test]
async fn history_resumes_after_an_artifact_failure() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    let pds = FakePds::new();
    let publisher = publisher(&pds, &db).await;
    let http = Arc::new(FakeHttp::new(&catalog_versions()));
    http.fail_next.store(true, Ordering::SeqCst);

    let first = run_project(&db, &publisher, &http, 0).await;
    assert_eq!(first.processed, 3);
    assert_eq!(first.failed, 1);
    assert_eq!(first.versions_created, 2);

    let retry = db
        .list_release_history_jobs(REPO, Some("retry"), 0)
        .await
        .expect("retry jobs");
    assert_eq!(retry.len(), 1);
    assert_eq!(retry[0].attempts, 1);

    let second = run_project(&db, &publisher, &http, 0).await;
    assert_eq!(second.processed, 1);
    assert_eq!(second.versions_created, 1);
    assert_eq!(second.failed, 0);
    assert!(
        db.list_release_history_jobs(REPO, Some("done"), 0)
            .await
            .expect("done jobs")
            .len()
            == 3
    );
}

#[tokio::test]
async fn unsupported_source_type_is_a_typed_skip() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    let pds = FakePds::new();
    let publisher = publisher(&pds, &db).await;
    db.mark_software(SoftwareMark {
        repo_did: REPO.to_owned(),
        slug: "acme-oci".to_owned(),
        part: "a".to_owned(),
        vendor: "acme".to_owned(),
        product: "widget".to_owned(),
        project_uri: String::new(),
        project_cid: String::new(),
        software_uri: format!("at://{REPO}/com.nahdig.software/acme-oci"),
        software_cid: "bafysoftware".to_owned(),
    })
    .await
    .expect("software");
    db.mark_purl(
        REPO,
        "pkg:oci/acme/widget",
        "acme-oci",
        &format!("at://{REPO}/com.nahdig.purl/oci"),
        "bafypurl",
    )
    .await
    .expect("purl");
    db.commit().await.expect("commit");

    let resolver = ArtifactResolver::new(Arc::new(FakeHttp::new(&["1.0.0".to_owned()])));
    let config = HistoryModeConfig {
        repo_did: REPO.to_owned(),
        limit: 10,
        recheck_seconds: 0.0,
        history_limit: 0,
        concurrency: 1,
        ..HistoryModeConfig::default()
    };
    let (events, mut rx) = mpsc::channel(64);
    let counters = sync_history_mode(
        &publisher,
        &db,
        &config,
        &mut Default::default(),
        events,
        &resolver,
    )
    .await
    .expect("history mode");
    while rx.try_recv().is_ok() {}

    assert_eq!(counters.projects_checked, 1);
    assert_eq!(counters.unsupported, 1);
    assert!(counters.enqueued == 0 && counters.processed == 0);
    assert_eq!(pds.count("putRecord"), 0);
    let state = db
        .get_release_project_state(REPO, "acme-oci")
        .await
        .expect("state query")
        .expect("state row");
    let reason = state.unsupported_reason.unwrap_or_default();
    assert!(
        reason.contains("UnknownPurlType"),
        "expected typed reason, got {reason:?}"
    );
}

#[tokio::test]
async fn resolver_cache_hit_avoids_a_second_download() {
    let temp = tempfile::tempdir().expect("temp dir");
    let db = open_db(&temp).await;
    let http = Arc::new(FakeHttp::new(&["1.0.0".to_owned()]));
    let resolver = ArtifactResolver::new(http.clone()).with_cache(db.clone(), REPO);
    let purl = Purl {
        r#type: "pypi".to_owned(),
        name: "acme-widget".to_owned(),
        raw: PURL.to_owned(),
    };

    let first = resolver
        .resolve(&purl, "1.0.0", 0)
        .await
        .expect("first resolve");
    let second = resolver
        .resolve(&purl, "1.0.0", 0)
        .await
        .expect("cached resolve");
    assert_eq!(first, second);
    assert_eq!(
        http.downloads(),
        1,
        "second resolve must use the digest cache"
    );
    let counters = resolver.counters();
    assert_eq!(counters.downloads, 1);
    assert_eq!(counters.bytes, ARTIFACT_BYTES.len() as u64);
    assert_eq!(counters.cache_hits, 1);
    assert_eq!(counters.errors, 0);
}

#[tokio::test]
async fn unconfigured_mirror_is_a_noop() {
    let http = Arc::new(FakeHttp::new(&["1.0.0".to_owned()]));
    let resolver = ArtifactResolver::new(http.clone());
    let purl = Purl {
        r#type: "pypi".to_owned(),
        name: "acme-widget".to_owned(),
        raw: PURL.to_owned(),
    };
    resolver.resolve(&purl, "1.0.0", 0).await.expect("resolve");
    let counters = resolver.counters();
    assert_eq!(counters.downloads, 1);
    assert_eq!(counters.mirror_puts, 0, "NoopMirror reports no writes");
}