~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
589 lines · 18.9 KB Raw History
//! Direct-hash registry sources: PyPI, crates.io, and RubyGems.
//!
//! Every request is served by the local [`FakeHttp`] seam; no test touches the
//! live registries. 429 retries belong to [`RegistryHttp`], so the source only
//! has to propagate the transport error unchanged.

use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::Duration;

use async_trait::async_trait;
use serde_json::{Value, json};

use vuln_notif::release::http::RegistryHttp;
use vuln_notif::release::source::{
    ArtifactInfo, ProjectRef, ReleaseSource, SourceCapabilities, StatusInfo, VersionInfo,
};
use vuln_notif::release::sources::cargo::CargoSource;
use vuln_notif::release::sources::gem::GemSource;
use vuln_notif::release::sources::pypi::{PyPiSource, pep503_name};

const PYPI_FIXTURE: &str = include_str!("fixtures/sources/direct/pypi.json");
const CRATES_FIXTURE: &str = include_str!("fixtures/sources/direct/crates.json");
const GEM_FIXTURE: &str = include_str!("fixtures/sources/direct/rubygems.json");

const PYPI_URL: &str = "https://pypi.org/pypi/foo-bar-baz/json";
const CRATES_URL: &str = "https://crates.io/api/v1/crates/acme-widget";
const CRATES_VERSIONS_URL: &str =
    "https://crates.io/api/v1/crates/acme-widget/versions?per_page=100";
const GEM_VERSIONS_URL: &str = "https://rubygems.org/api/v1/versions/acme-widget.json";
const GEM_META_URL: &str = "https://rubygems.org/api/v1/gems/acme-widget.json";

const AAAA: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
const BBBB: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
const CCCC: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
const DDDD: &str = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd";

// ---------------------------------------------------------------------------
// Local registry HTTP fake
// ---------------------------------------------------------------------------

#[derive(Debug, Clone)]
enum Reply {
    Json(Value),
    Size(u64),
    Error(String),
}

#[derive(Default)]
struct FakeHttp {
    replies: Mutex<HashMap<String, Reply>>,
    calls: Mutex<Vec<String>>,
}

impl FakeHttp {
    fn new(routes: Vec<(&str, Reply)>) -> Self {
        Self {
            replies: Mutex::new(
                routes
                    .into_iter()
                    .map(|(url, reply)| (url.to_owned(), reply))
                    .collect(),
            ),
            calls: Mutex::new(Vec::new()),
        }
    }

    fn calls(&self) -> Vec<String> {
        self.calls.lock().unwrap().clone()
    }

    fn take(&self, kind: &str, url: &str) -> anyhow::Result<Reply> {
        self.calls.lock().unwrap().push(format!("{kind} {url}"));
        self.replies
            .lock()
            .unwrap()
            .get(url)
            .cloned()
            .ok_or_else(|| anyhow::anyhow!("unexpected {kind} {url}"))
    }
}

#[async_trait]
impl RegistryHttp for FakeHttp {
    async fn get_json(&self, url: &str) -> anyhow::Result<Value> {
        match self.take("json", url)? {
            Reply::Json(value) => Ok(value),
            Reply::Error(message) => Err(anyhow::anyhow!(message)),
            other => Err(anyhow::anyhow!("expected JSON for {url}, got {other:?}")),
        }
    }

    async fn get_text(&self, url: &str) -> anyhow::Result<String> {
        Err(anyhow::anyhow!("unexpected text fetch for {url}"))
    }

    async fn head_size(&self, url: &str) -> anyhow::Result<u64> {
        match self.take("head", url)? {
            Reply::Size(value) => Ok(value),
            Reply::Error(message) => Err(anyhow::anyhow!(message)),
            other => Err(anyhow::anyhow!("expected size for {url}, got {other:?}")),
        }
    }

    async fn download(&self, url: &str) -> anyhow::Result<Vec<u8>> {
        Err(anyhow::anyhow!("unexpected download for {url}"))
    }
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

fn fixture(text: &str) -> Value {
    serde_json::from_str(text).expect("fixture JSON")
}

fn project(name: &str, purl_type: &str) -> ProjectRef {
    ProjectRef {
        purl_type: purl_type.to_owned(),
        name: name.to_owned(),
        purl: format!("pkg:{purl_type}/{name}"),
    }
}

fn version(raw: &str) -> VersionInfo {
    VersionInfo {
        raw: raw.to_owned(),
        version: raw.to_owned(),
        deprecated: false,
        reason: String::new(),
    }
}

fn all_capabilities() -> SourceCapabilities {
    SourceCapabilities {
        versions: true,
        artifacts: true,
        statuses: true,
        project_meta: true,
    }
}

// ---------------------------------------------------------------------------
// PyPI
// ---------------------------------------------------------------------------

#[tokio::test]
async fn pypi_versions_are_full_and_map_yanks() {
    let http = Arc::new(FakeHttp::new(vec![(
        PYPI_URL,
        Reply::Json(fixture(PYPI_FIXTURE)),
    )]));
    let source = PyPiSource::new(http.clone());
    let versions = source
        .versions(&project("Foo.Bar_baz", "pypi"))
        .await
        .expect("pypi versions");

    let summary = versions
        .iter()
        .map(|entry| {
            (
                entry.raw.as_str(),
                entry.version.as_str(),
                entry.deprecated,
                entry.reason.as_str(),
            )
        })
        .collect::<Vec<_>>();
    assert_eq!(
        summary,
        vec![
            ("1.0.0", "1.0.0", false, ""),
            ("1.0.1", "1.0.1", false, ""),
            ("1.1.0", "1.1.0", true, "Malware in the sdist"),
            ("1.2.2", "1.2.2", false, ""),
            ("1.2.3", "1.2.3", false, ""),
            ("2.0.0-Beta_1", "2.0.0-beta-1", false, ""),
        ]
    );
    assert_eq!(http.calls(), vec![format!("json {PYPI_URL}")]);
}

#[tokio::test]
async fn pypi_artifacts_carry_digests_sizes_and_mime() {
    let http = Arc::new(FakeHttp::new(vec![(
        PYPI_URL,
        Reply::Json(fixture(PYPI_FIXTURE)),
    )]));
    let source = PyPiSource::new(http);
    let artifacts = source
        .artifacts(&project("Foo.Bar_baz", "pypi"), &version("1.2.3"))
        .await
        .expect("pypi artifacts");

    let summary = artifacts
        .iter()
        .map(|artifact| {
            (
                artifact.url.as_str(),
                artifact.file_name.as_str(),
                artifact.sha256.as_deref(),
                artifact.size,
                artifact.mime_type.as_deref(),
            )
        })
        .collect::<Vec<_>>();
    assert_eq!(
        summary,
        vec![
            (
                "https://files.pythonhosted.org/packages/aa/11/foo_bar_baz-1.2.3-py2.py3-none-any.whl",
                "foo_bar_baz-1.2.3-py2.py3-none-any.whl",
                Some(BBBB),
                Some(12346),
                Some("application/zip"),
            ),
            (
                "https://files.pythonhosted.org/packages/bb/22/foo_bar_baz-1.2.3.tar.bz2",
                "foo_bar_baz-1.2.3.tar.bz2",
                None,
                None,
                Some("application/x-bzip2"),
            ),
            (
                "https://files.pythonhosted.org/packages/cc/33/foo_bar_baz-1.2.3-py3-none-any.whl",
                "foo_bar_baz-1.2.3-py3-none-any.whl",
                Some(AAAA),
                Some(12345),
                Some("application/zip"),
            ),
            (
                "https://files.pythonhosted.org/packages/dd/44/foo_bar_baz-1.2.3.tar.gz",
                "foo_bar_baz-1.2.3.tar.gz",
                Some(CCCC),
                Some(13000),
                Some("application/gzip"),
            ),
        ]
    );
    assert_eq!(artifacts.len(), 4, "duplicate URLs are deduped");
}

#[tokio::test]
async fn pypi_pep503_lookup_preserves_display_name() {
    assert_eq!(pep503_name("Foo.Bar_baz"), "foo-bar-baz");
    assert_eq!(pep503_name("foo---bar__baz"), "foo-bar-baz");
    assert_eq!(pep503_name("Already-Normal"), "already-normal");

    let http = Arc::new(FakeHttp::new(vec![(
        PYPI_URL,
        Reply::Json(fixture(PYPI_FIXTURE)),
    )]));
    let source = PyPiSource::new(http.clone());
    let meta = source
        .project_meta(&project("Foo.Bar_baz", "pypi"))
        .await
        .expect("pypi meta");

    assert_eq!(meta["name"], "Foo.Bar_baz");
    assert_eq!(meta["description"], "Synthetic fixture package");
    assert_eq!(meta["homepage"], "https://example.invalid/foo-bar-baz");
    assert_eq!(
        meta["repository"],
        "https://example.invalid/foo-bar-baz.git"
    );
    assert_eq!(meta["license"], "MIT");
    assert_eq!(meta["simple_index"], "https://pypi.org/simple/foo-bar-baz/");
    assert_eq!(http.calls(), vec![format!("json {PYPI_URL}")]);
}

#[tokio::test]
async fn pypi_statuses_distinguish_yanked_and_mixed_releases() {
    let http = Arc::new(FakeHttp::new(vec![(
        PYPI_URL,
        Reply::Json(fixture(PYPI_FIXTURE)),
    )]));
    let source = PyPiSource::new(http);
    let project = project("Foo.Bar_baz", "pypi");

    assert_eq!(
        source.statuses(&project, &version("1.1.0")).await.unwrap(),
        vec![StatusInfo {
            status: "yanked".to_owned(),
            kind: Some("security".to_owned()),
            reason: "Malware in the sdist".to_owned(),
        }]
    );
    assert!(
        source
            .statuses(&project, &version("1.2.2"))
            .await
            .unwrap()
            .is_empty(),
        "a partially yanked release is not yanked"
    );
    assert!(
        source
            .statuses(&project, &version("9.9.9"))
            .await
            .unwrap()
            .is_empty()
    );
}

// ---------------------------------------------------------------------------
// crates.io
// ---------------------------------------------------------------------------

#[tokio::test]
async fn cargo_versions_artifacts_statuses_and_meta() {
    let http = Arc::new(FakeHttp::new(vec![
        (CRATES_VERSIONS_URL, Reply::Json(fixture(CRATES_FIXTURE))),
        (CRATES_URL, Reply::Json(fixture(CRATES_FIXTURE))),
    ]));
    let source = CargoSource::with_interval(http.clone(), Duration::ZERO);
    let project = project("acme-widget", "cargo");

    let versions = source.versions(&project).await.expect("cargo versions");
    assert_eq!(
        versions
            .iter()
            .map(|entry| (entry.raw.as_str(), entry.deprecated, entry.reason.as_str()))
            .collect::<Vec<_>>(),
        vec![
            ("1.1.0", false, ""),
            ("1.2.2", true, "yanked due to a security issue"),
            ("1.2.3", false, ""),
        ]
    );

    assert_eq!(
        source.artifacts(&project, &version("1.2.3")).await.unwrap(),
        vec![ArtifactInfo {
            file_name: "acme-widget-1.2.3.crate".to_owned(),
            url: "https://static.crates.io/crates/acme-widget/acme-widget-1.2.3.crate".to_owned(),
            sha256: Some(AAAA.to_owned()),
            size: Some(14567),
            mime_type: Some("application/gzip".to_owned()),
        }]
    );
    assert_eq!(
        source.statuses(&project, &version("1.2.2")).await.unwrap(),
        vec![StatusInfo {
            status: "yanked".to_owned(),
            kind: Some("security".to_owned()),
            reason: "yanked due to a security issue".to_owned(),
        }]
    );
    assert!(
        source
            .statuses(&project, &version("1.2.3"))
            .await
            .unwrap()
            .is_empty()
    );

    let meta = source.project_meta(&project).await.expect("cargo meta");
    assert_eq!(meta["name"], "acme-widget");
    assert_eq!(meta["description"], "Synthetic crate for source tests");
    assert_eq!(meta["homepage"], "https://example.invalid/acme-widget");
    assert_eq!(
        meta["repository"],
        "https://example.invalid/acme-widget.git"
    );
    assert_eq!(
        meta["documentation"],
        "https://example.invalid/acme-widget/docs"
    );
    assert_eq!(meta["license"], "MIT OR Apache-2.0");
    assert!(http.calls().contains(&format!("json {CRATES_URL}")));
}

#[tokio::test]
async fn cargo_follows_keyset_pagination() {
    let second_page =
        "https://crates.io/api/v1/crates/acme-widget/versions?per_page=100&seek=cursor";
    let http = Arc::new(FakeHttp::new(vec![
        (
            CRATES_VERSIONS_URL,
            Reply::Json(json!({
                "versions": [{"crate": "acme-widget", "num": "1.0.0", "yanked": false}],
                "meta": {"total": 2, "next_page": "?per_page=100&seek=cursor"},
            })),
        ),
        (
            second_page,
            Reply::Json(json!({
                "versions": [{"crate": "acme-widget", "num": "1.1.0", "yanked": false}],
                "meta": {"total": 2, "next_page": null},
            })),
        ),
    ]));
    let source = CargoSource::with_interval(http, Duration::ZERO);
    let versions = source
        .versions(&project("acme-widget", "cargo"))
        .await
        .expect("cargo paginated versions");
    assert_eq!(
        versions
            .iter()
            .map(|entry| entry.raw.as_str())
            .collect::<Vec<_>>(),
        vec!["1.0.0", "1.1.0"]
    );
}

#[tokio::test]
async fn cargo_propagates_rate_limit_errors() {
    let http = Arc::new(FakeHttp::new(vec![(
        CRATES_VERSIONS_URL,
        Reply::Error("HTTP 429 Too Many Requests for https://crates.io".to_owned()),
    )]));
    let source = CargoSource::with_interval(http.clone(), Duration::ZERO);
    let error = source
        .versions(&project("acme-widget", "cargo"))
        .await
        .expect_err("429 propagates");
    assert!(error.to_string().contains("429"));
    assert_eq!(http.calls(), vec![format!("json {CRATES_VERSIONS_URL}")]);
}

// ---------------------------------------------------------------------------
// RubyGems
// ---------------------------------------------------------------------------

#[tokio::test]
async fn gem_versions_artifacts_statuses_and_meta() {
    let http = Arc::new(FakeHttp::new(vec![
        (GEM_VERSIONS_URL, Reply::Json(fixture(GEM_FIXTURE))),
        (
            GEM_META_URL,
            Reply::Json(json!({
                "name": "acme-widget",
                "info": "Synthetic gem for source tests",
                "homepage_uri": "https://example.invalid/acme-widget",
                "source_code_uri": "https://example.invalid/acme-widget.git",
                "licenses": ["MIT"],
            })),
        ),
    ]));
    let source = GemSource::new(http.clone());
    let project = project("acme-widget", "gem");

    let versions = source.versions(&project).await.expect("gem versions");
    assert_eq!(
        versions
            .iter()
            .map(|entry| (entry.raw.as_str(), entry.deprecated, entry.reason.as_str()))
            .collect::<Vec<_>>(),
        vec![
            ("1.1.0", true, "broken build"),
            ("1.2.2", false, ""),
            ("1.2.3", false, ""),
        ]
    );

    let artifacts = source
        .artifacts(&project, &version("1.2.3"))
        .await
        .expect("gem artifacts");
    assert_eq!(
        artifacts
            .iter()
            .map(|artifact| {
                (
                    artifact.url.as_str(),
                    artifact.sha256.as_deref(),
                    artifact.size,
                    artifact.mime_type.as_deref(),
                )
            })
            .collect::<Vec<_>>(),
        vec![
            (
                "https://rubygems.org/gems/acme-widget-1.2.3-java.gem",
                Some(BBBB),
                Some(16000),
                Some("application/gzip"),
            ),
            (
                "https://rubygems.org/gems/acme-widget-1.2.3.gem",
                Some(AAAA),
                Some(15678),
                Some("application/gzip"),
            ),
        ]
    );
    assert!(
        http.calls().iter().all(|call| !call.starts_with("head ")),
        "declared sizes need no HEAD"
    );
    assert!(
        http.calls().iter().all(|call| !call.contains("/api/v2/")),
        "declared digests need no detail request"
    );

    assert_eq!(
        source.statuses(&project, &version("1.1.0")).await.unwrap(),
        vec![StatusInfo {
            status: "yanked".to_owned(),
            kind: Some("registry".to_owned()),
            reason: "broken build".to_owned(),
        }]
    );
    assert!(
        source
            .statuses(&project, &version("1.2.2"))
            .await
            .unwrap()
            .is_empty()
    );

    let meta = source.project_meta(&project).await.expect("gem meta");
    assert_eq!(meta["name"], "acme-widget");
    assert_eq!(meta["description"], "Synthetic gem for source tests");
    assert_eq!(meta["homepage"], "https://example.invalid/acme-widget");
    assert_eq!(
        meta["repository"],
        "https://example.invalid/acme-widget.git"
    );
    assert_eq!(meta["license"], "MIT");
}

#[tokio::test]
async fn gem_missing_metadata_uses_detail_then_head() {
    let versions_url = "https://rubygems.org/api/v1/versions/acme-widget.json";
    let detail_url = "https://rubygems.org/api/v2/rubygems/acme-widget/versions/0.9.0.json";
    let gem_url = "https://rubygems.org/gems/acme-widget-0.9.0.gem";
    let http = Arc::new(FakeHttp::new(vec![
        (
            versions_url,
            Reply::Json(json!([{
                "number": "0.9.0",
                "platform": "ruby",
                "sha": "",
                "size": 0,
                "yanked": false,
            }])),
        ),
        (
            detail_url,
            Reply::Json(json!({
                "platform": "ruby",
                "sha": DDDD,
                "gem_uri": gem_url,
                "yanked": false,
            })),
        ),
        (gem_url, Reply::Size(4242)),
    ]));
    let source = GemSource::new(http.clone());
    let artifacts = source
        .artifacts(&project("acme-widget", "gem"), &version("0.9.0"))
        .await
        .expect("gem fallback artifact");
    assert_eq!(artifacts.len(), 1);
    assert_eq!(artifacts[0].sha256.as_deref(), Some(DDDD));
    assert_eq!(artifacts[0].size, Some(4242));
    assert_eq!(artifacts[0].file_name, "acme-widget-0.9.0.gem");
    assert_eq!(
        http.calls(),
        vec![
            format!("json {versions_url}"),
            format!("json {detail_url}"),
            format!("head {gem_url}"),
        ]
    );
}

// ---------------------------------------------------------------------------
// Capability flags
// ---------------------------------------------------------------------------

#[test]
fn sources_advertise_real_capabilities() {
    let http: Arc<dyn RegistryHttp> = Arc::new(FakeHttp::default());
    let pypi = PyPiSource::new(http.clone());
    let cargo = CargoSource::with_interval(http.clone(), Duration::ZERO);
    let gem = GemSource::new(http);

    assert_eq!(pypi.name(), "pypi");
    assert_eq!(cargo.name(), "cargo");
    assert_eq!(gem.name(), "gem");
    assert_eq!(pypi.capabilities(), all_capabilities());
    assert_eq!(cargo.capabilities(), all_capabilities());
    assert_eq!(gem.capabilities(), all_capabilities());
}