~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
551 lines · 18.7 KB Raw History
//! Artifact resolver tests: digest cache, mismatch rejection, computed
//! digests, byte caps, retries, mirror round-trip, and blob uploads.

mod common;

#[path = "../src/release/artifacts.rs"]
mod artifacts;

// `artifacts.rs` is not yet declared by `src/release/mod.rs`; these shims let
// the same file compile inside this test crate with its `crate::` imports.
mod db {
    pub use vuln_notif::db::*;
}
mod pds {
    pub use vuln_notif::pds::*;
}
mod records {
    pub use vuln_notif::records::*;
}
mod release {
    pub mod artifact {
        pub use vuln_notif::release::artifact::*;
    }
    pub mod http {
        pub use vuln_notif::release::http::*;
    }
}

use std::collections::HashMap;
use std::io::Read;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};

use async_trait::async_trait;
use serde_json::{Value, json};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

use vuln_notif::db::DbHandle;
use vuln_notif::release::artifact::{Purl, parse_purl};
use vuln_notif::release::http::{DownloadTooLarge, RegistryHttp, ReqwestRegistryHttp};

const REPO: &str = "did:plc:test";
const MAX_BYTES: u64 = 10 * 1024 * 1024;
const PYPI_JSON_URL: &str = "https://pypi.org/pypi/acme-widget/json";
const WHEEL_URL: &str =
    "https://files.pythonhosted.org/packages/ee/ff/acme_widget-1.2.3-py3-none-any.whl";
const SDIST_URL: &str = "https://files.pythonhosted.org/packages/cc/dd/acme_widget-1.2.3.tar.gz";
const WHEEL_SHA: &str = "1052c88adb8be0a046765e7a91d3a561d3f12b4c43c19ca0aec800167749b343";
const SDIST_SHA: &str = "3020aaf1136f4cee21cd01100d9bf88c0ad5a783dfd29c76ad3aa7c5db8d481e";
const WHEEL_CID: &str = "bafkreiaqkleivw4l4cqem5s6pki5hjlb2pyswtcdygokblwiaalhosntim";

const PYPI_FIXTURE: &str = include_str!("fixtures/artifacts/pypi.json");
const EXPECTED_HTTP_ARTIFACT: &str = include_str!("fixtures/artifacts/expected_http_artifact.json");
const WHEEL_BYTES: &[u8] = include_bytes!("fixtures/artifacts/acme_widget-1.2.3-py3-none-any.whl");
const SDIST_BYTES: &[u8] = include_bytes!("fixtures/artifacts/acme_widget-1.2.3.tar.gz");

// ---------------------------------------------------------------------------
// Registry fake
// ---------------------------------------------------------------------------

#[derive(Clone)]
enum Route {
    Json(Value),
    Text(String),
    Bytes(Vec<u8>),
}

#[derive(Default)]
struct FakeRegistry {
    routes: Mutex<HashMap<String, Route>>,
    calls: Mutex<Vec<String>>,
    downloads: AtomicUsize,
}

impl FakeRegistry {
    fn new(routes: Vec<(&str, Route)>) -> Arc<Self> {
        let registry = Arc::new(Self::default());
        for (url, route) in routes {
            registry
                .routes
                .lock()
                .unwrap()
                .insert(url.to_owned(), route);
        }
        registry
    }

    fn calls(&self) -> Vec<String> {
        self.calls.lock().unwrap().clone()
    }

    fn downloads(&self) -> usize {
        self.downloads.load(Ordering::SeqCst)
    }

    fn route(&self, url: &str) -> Option<Route> {
        self.routes.lock().unwrap().get(url).cloned()
    }
}

#[async_trait]
impl RegistryHttp for FakeRegistry {
    async fn get_json(&self, url: &str) -> anyhow::Result<Value> {
        self.calls.lock().unwrap().push(format!("json {url}"));
        match self.route(url) {
            Some(Route::Json(value)) => Ok(value),
            _ => Err(anyhow::anyhow!("unexpected JSON request {url}")),
        }
    }

    async fn get_text(&self, url: &str) -> anyhow::Result<String> {
        self.calls.lock().unwrap().push(format!("text {url}"));
        match self.route(url) {
            Some(Route::Text(value)) => Ok(value),
            _ => Err(anyhow::anyhow!("unexpected text request {url}")),
        }
    }

    async fn head_size(&self, url: &str) -> anyhow::Result<u64> {
        self.calls.lock().unwrap().push(format!("head {url}"));
        match self.route(url) {
            Some(Route::Bytes(bytes)) => Ok(bytes.len() as u64),
            _ => Err(anyhow::anyhow!("unexpected HEAD request {url}")),
        }
    }

    async fn download(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        self.downloads.fetch_add(1, Ordering::SeqCst);
        self.calls.lock().unwrap().push(format!("download {url}"));
        match self.route(url) {
            Some(Route::Bytes(bytes)) => Ok(bytes),
            _ => Err(anyhow::anyhow!("unexpected download request {url}")),
        }
    }
}

fn pypi_purl() -> Purl {
    parse_purl("pkg:pypi/acme-widget").expect("pypi purl")
}

fn pypi_registry() -> Arc<FakeRegistry> {
    FakeRegistry::new(vec![
        (
            PYPI_JSON_URL,
            Route::Json(serde_json::from_str(PYPI_FIXTURE).expect("pypi fixture")),
        ),
        (WHEEL_URL, Route::Bytes(WHEEL_BYTES.to_vec())),
        (SDIST_URL, Route::Bytes(SDIST_BYTES.to_vec())),
    ])
}

// ---------------------------------------------------------------------------
// Digest cache and verification
// ---------------------------------------------------------------------------

#[tokio::test]
async fn digest_cache_hit_skips_download() {
    let http = pypi_registry();
    let db = DbHandle::open_in_memory().await.expect("db");
    let resolver = artifacts::ArtifactResolver::new(http.clone()).with_cache(db.clone(), REPO);

    let first = resolver
        .resolve(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect("first resolve");
    assert_eq!(first.url, WHEEL_URL);
    assert_eq!(first.file_name, "acme_widget-1.2.3-py3-none-any.whl");
    assert_eq!(first.sha256, WHEEL_SHA);
    assert_eq!(first.cid, WHEEL_CID);
    assert_eq!(first.size, WHEEL_BYTES.len() as u64);
    assert_eq!(first.mime_type, "application/zip");
    assert_eq!(http.downloads(), 1);

    let second = resolver
        .resolve(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect("cached resolve");
    assert_eq!(first, second);
    assert_eq!(http.downloads(), 1, "cache hit must not download");

    let row = db
        .get_release_artifact(REPO, "pypi", "acme-widget", "1.2.3", WHEEL_URL)
        .await
        .expect("lookup")
        .expect("cached row");
    assert_eq!(row.sha256.as_deref(), Some(WHEEL_SHA));
    assert_eq!(row.cid.as_deref(), Some(WHEEL_CID));
    assert_eq!(row.size, Some(WHEEL_BYTES.len() as i64));
}

#[tokio::test]
async fn declared_digest_mismatch_is_rejected_and_not_cached() {
    let http = FakeRegistry::new(vec![
        (
            PYPI_JSON_URL,
            Route::Json(serde_json::from_str(PYPI_FIXTURE).expect("pypi fixture")),
        ),
        (WHEEL_URL, Route::Bytes(b"tampered artifact".to_vec())),
    ]);
    let db = DbHandle::open_in_memory().await.expect("db");
    let resolver = artifacts::ArtifactResolver::new(http.clone()).with_cache(db.clone(), REPO);

    let error = resolver
        .resolve(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect_err("digest mismatch must fail");
    let mismatch = error
        .downcast_ref::<artifacts::ArtifactError>()
        .expect("typed digest error");
    assert!(matches!(
        mismatch,
        artifacts::ArtifactError::DigestMismatch { .. }
    ));
    assert!(
        db.get_release_artifact(REPO, "pypi", "acme-widget", "1.2.3", WHEEL_URL)
            .await
            .expect("lookup")
            .is_none(),
        "unverified bytes must not be cached"
    );
    assert!(
        http.calls().contains(&format!("download {WHEEL_URL}")),
        "the declared file was fetched before verification"
    );
    assert_eq!(http.downloads(), 1);
}

#[tokio::test]
async fn computed_digest_matches_bytes_and_cid_is_stable() {
    let jar_url = "https://repo1.maven.org/maven2/com/acme/widget/2.0.0/widget-2.0.0.jar";
    let metadata_url = "https://repo1.maven.org/maven2/com/acme/widget/maven-metadata.xml";
    let http = FakeRegistry::new(vec![
        (
            metadata_url,
            Route::Text(
                "<metadata><versioning><versions><version>2.0.0</version>\
                 </versions></versioning></metadata>"
                    .to_owned(),
            ),
        ),
        (jar_url, Route::Bytes(WHEEL_BYTES.to_vec())),
    ]);
    let db = DbHandle::open_in_memory().await.expect("db");
    let resolver = artifacts::ArtifactResolver::new(http.clone()).with_cache(db.clone(), REPO);
    let purl = parse_purl("pkg:maven/com.acme/widget").expect("maven purl");

    let artifact = resolver
        .resolve(&purl, "2.0.0", MAX_BYTES)
        .await
        .expect("maven resolve");
    assert_eq!(artifact.file_name, "widget-2.0.0.jar");
    assert_eq!(artifact.sha256, WHEEL_SHA);
    assert_eq!(artifact.cid, vuln_notif::records::cidv1_raw(WHEEL_SHA));
    assert_eq!(artifact.size, WHEEL_BYTES.len() as u64);
    assert_eq!(artifact.mime_type, "application/java-archive");

    let again = resolver
        .resolve(&purl, "2.0.0", MAX_BYTES)
        .await
        .expect("cached maven resolve");
    assert_eq!(artifact.cid, again.cid, "CID must be stable");
}

#[tokio::test]
async fn http_artifact_record_matches_expected_fixture() {
    let resolver = artifacts::ArtifactResolver::new(pypi_registry());
    let resolved = resolver
        .resolve(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect("resolve");
    let expected: Value =
        serde_json::from_str(EXPECTED_HTTP_ARTIFACT).expect("expected artifact fixture");
    assert_eq!(artifacts::http_artifact_record(&resolved), expected);
}

// ---------------------------------------------------------------------------
// Multi-artifact selectors
// ---------------------------------------------------------------------------

struct SdistSelector;

#[async_trait]
impl artifacts::ArtifactSelector for SdistSelector {
    async fn candidates(
        &self,
        _purl: &Purl,
        _raw_version: &str,
    ) -> anyhow::Result<Vec<artifacts::ArtifactCandidate>> {
        Ok(vec![artifacts::ArtifactCandidate {
            file_name: "acme_widget-1.2.3.tar.gz".to_owned(),
            url: SDIST_URL.to_owned(),
            sha256: None,
            size: Some(SDIST_BYTES.len() as u64),
            mime_type: String::new(),
            metadata: Value::Null,
        }])
    }
}

#[tokio::test]
async fn resolve_many_adds_selector_artifacts() {
    let http = pypi_registry();
    let resolver =
        artifacts::ArtifactResolver::new(http.clone()).with_selector(Arc::new(SdistSelector));

    let resolved = resolver
        .resolve_many(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect("resolve many");
    assert_eq!(resolved.len(), 2);
    assert_eq!(resolved[0].url, WHEEL_URL);
    assert_eq!(resolved[1].file_name, "acme_widget-1.2.3.tar.gz");
    assert_eq!(resolved[1].sha256, SDIST_SHA);
    assert_eq!(resolved[1].mime_type, "application/gzip");
    assert_eq!(http.downloads(), 2);
}

// ---------------------------------------------------------------------------
// Byte caps and retries
// ---------------------------------------------------------------------------

#[tokio::test]
async fn content_length_over_cap_is_rejected_without_download_or_retry() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/big"))
        .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![b'x'; 64]))
        .mount(&server)
        .await;

    let http = ReqwestRegistryHttp::new().expect("http client");
    let error = http
        .download_limited(&format!("{}/big", server.uri()), 32)
        .await
        .expect_err("Content-Length over cap");
    let too_large = error
        .downcast_ref::<DownloadTooLarge>()
        .expect("typed size error");
    assert_eq!(too_large.limit, 32);
    assert_eq!(too_large.size, Some(64));
    assert_eq!(server.received_requests().await.unwrap().len(), 1);
}

/// Chunked HTTP/1.1 response with no `Content-Length`, so the cap can only be
/// enforced while streaming.
async fn chunked_server(chunks: Vec<Vec<u8>>) -> String {
    use tokio::io::{AsyncReadExt, AsyncWriteExt};
    use tokio::net::TcpListener;

    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
    let address = listener.local_addr().expect("local address");
    tokio::spawn(async move {
        let (mut socket, _) = listener.accept().await.expect("accept");
        let mut request = [0u8; 1024];
        let _ = socket.read(&mut request).await;
        let _ = socket
            .write_all(
                b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\
                  Transfer-Encoding: chunked\r\n\r\n",
            )
            .await;
        for chunk in chunks {
            let header = format!("{:x}\r\n", chunk.len());
            let _ = socket.write_all(header.as_bytes()).await;
            let _ = socket.write_all(&chunk).await;
            let _ = socket.write_all(b"\r\n").await;
        }
        let _ = socket.write_all(b"0\r\n\r\n").await;
    });
    format!("http://{address}/artifact")
}

#[tokio::test]
async fn streamed_body_over_cap_is_rejected() {
    let url = chunked_server(vec![b"123456".to_vec(), b"789012".to_vec()]).await;
    let http = ReqwestRegistryHttp::new().expect("http client");
    let error = http
        .download_limited(&url, 8)
        .await
        .expect_err("streamed body over cap");
    let too_large = error
        .downcast_ref::<DownloadTooLarge>()
        .expect("typed size error");
    assert_eq!(too_large.limit, 8);
}

#[tokio::test]
async fn download_limited_retries_503_then_succeeds() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/artifact"))
        .respond_with(ResponseTemplate::new(503))
        .up_to_n_times(1)
        .mount(&server)
        .await;
    Mock::given(method("GET"))
        .and(path("/artifact"))
        .respond_with(ResponseTemplate::new(200).set_body_bytes(b"payload".to_vec()))
        .mount(&server)
        .await;

    let http = ReqwestRegistryHttp::new().expect("http client");
    let bytes = http
        .download_limited(&format!("{}/artifact", server.uri()), 64)
        .await
        .expect("retry succeeds");
    assert_eq!(bytes, b"payload");
    assert_eq!(server.received_requests().await.unwrap().len(), 2);
}

#[tokio::test]
async fn download_limited_does_not_retry_413() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .respond_with(ResponseTemplate::new(413))
        .mount(&server)
        .await;

    let http = ReqwestRegistryHttp::new().expect("http client");
    let error = http
        .download_limited(&format!("{}/artifact", server.uri()), 64)
        .await
        .expect_err("413 is not retried");
    assert!(error.to_string().contains("413"));
    assert_eq!(server.received_requests().await.unwrap().len(), 1);
}

// ---------------------------------------------------------------------------
// Mirror and blob paths
// ---------------------------------------------------------------------------

#[derive(Default)]
struct MemoryMirror {
    entries: Mutex<HashMap<String, Vec<u8>>>,
}

#[async_trait]
impl artifacts::ArtifactMirror for MemoryMirror {
    async fn put(&self, key: &str, bytes: &[u8]) -> anyhow::Result<()> {
        self.entries
            .lock()
            .unwrap()
            .insert(key.to_owned(), bytes.to_vec());
        Ok(())
    }

    async fn get(&self, key: &str) -> anyhow::Result<Option<Vec<u8>>> {
        Ok(self.entries.lock().unwrap().get(key).cloned())
    }
}

#[tokio::test]
async fn mirror_round_trip_avoids_second_download() {
    let http = pypi_registry();
    let mirror = Arc::new(MemoryMirror::default());
    let db = DbHandle::open_in_memory().await.expect("db");
    let resolver = artifacts::ArtifactResolver::new(http.clone())
        .with_cache(db, REPO)
        .with_mirror(mirror.clone());

    let first = resolver
        .resolve(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect("first resolve");
    assert_eq!(http.downloads(), 1);

    let (key, mirrored) = {
        let entries = mirror.entries.lock().unwrap();
        assert_eq!(entries.len(), 1);
        let (key, bytes) = entries.iter().next().expect("mirrored entry");
        (key.clone(), bytes.clone())
    };
    assert_eq!(mirrored, WHEEL_BYTES);
    assert_eq!(
        artifacts::ArtifactMirror::get(mirror.as_ref(), &key)
            .await
            .expect("mirror get")
            .as_deref(),
        Some(WHEEL_BYTES)
    );

    // Fresh DB: only the mirror can satisfy this resolve.
    let db = DbHandle::open_in_memory().await.expect("db");
    let resolver = artifacts::ArtifactResolver::new(http.clone())
        .with_cache(db, REPO)
        .with_mirror(mirror);
    let second = resolver
        .resolve(&pypi_purl(), "1.2.3", MAX_BYTES)
        .await
        .expect("mirror resolve");
    assert_eq!(first, second);
    assert_eq!(http.downloads(), 1, "mirror supplied the bytes");
}

#[tokio::test]
async fn blob_path_gzips_and_uploads() {
    let fake = common::FakePds::new();
    fake.push_upload(common::response(
        200,
        json!({
            "blob": {
                "$type": "blob",
                "ref": {"$link": "bafyblob"},
                "mimeType": "application/gzip",
                "size": 42,
            }
        }),
    ));
    let time = common::FakeTime::new(0.0);
    let (logger, _sink) = common::logger();
    let (publisher, _governor, _db) =
        common::publisher_with(fake.clone(), time, "com.nahdig.cve.nvdV2", logger).await;

    let payload = b"artifact body\n";
    let sha256 = vuln_notif::records::sha256_hex(payload);
    let artifact = artifacts::ResolvedArtifact {
        file_name: "notes.txt".to_owned(),
        url: "https://example.test/notes.txt".to_owned(),
        cid: vuln_notif::records::cidv1_raw(&sha256),
        sha256,
        size: payload.len() as u64,
        mime_type: "text/plain".to_owned(),
        metadata: Value::Null,
    };

    let record = artifacts::blob_artifact_record(&publisher, &artifact, payload, 1024)
        .await
        .expect("blob record");
    assert_eq!(record["$type"], "software.release.defs#blobArtifact");
    assert_eq!(record["fileName"], "notes.txt");
    assert_eq!(record["blob"]["ref"]["$link"], "bafyblob");

    let calls = fake.calls();
    let upload = calls
        .iter()
        .find(|call| call.method == "uploadBlob")
        .expect("uploadBlob call");
    assert_eq!(upload.mime.as_deref(), Some("application/gzip"));
    let mut decoder = flate2::read::GzDecoder::new(upload.bytes.as_deref().expect("blob bytes"));
    let mut decoded = Vec::new();
    decoder.read_to_end(&mut decoded).expect("gunzip");
    assert_eq!(decoded, payload);

    let fallback = artifacts::blob_artifact_record(&publisher, &artifact, payload, 1)
        .await
        .expect("http fallback");
    assert_eq!(fallback["$type"], "software.release.defs#httpArtifact");
    assert_eq!(fallback["url"], artifact.url);
    assert_eq!(fake.count("uploadBlob"), 1, "oversized blob stays http");
}