~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
398 lines · 13.7 KB Raw History
//! Software / purl / atprotoV1 graph sync.

use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;

use anyhow::Result;
use serde::Serialize;
use serde_json::{Value, json};

use crate::constants::{ATPROTO_COLLECTION, ATPROTO_TYPE, PROJECT_COLLECTION, PROJECT_TYPE};
use crate::db::DbHandle;
use crate::db::model::{SoftwareIdentityMark, SoftwareMark, SoftwareRow};
use crate::identity::{self, CanonicalIdentity, ObservedProduct};
use crate::links::index_record;
use crate::pds::Publisher;
use crate::publish::PublishOutcome;
use crate::runtime::events::{EventTx, ProgressEvent};
use crate::time::created_at;

pub use crate::db::purl2cpe::{cpe_product_key, ensure_purl_db, load_purl_map, versionless_purl};

#[derive(Debug, Clone)]
pub struct SoftwareConfig {
    pub repo_did: String,
    pub limit: usize,
    pub purl_db: PathBuf,
}

#[derive(Debug, Clone, Default, Serialize)]
pub struct SoftwareCounters {
    pub cves: u64,
    pub software_created: u64,
    pub purl_created: u64,
    pub atproto_created: u64,
    pub failures: u64,
}

/// One `com.nahdig.software` record: a raw CPE observation merged per
/// `part:vendor:product` alias, mapped to its canonical project identity.
#[derive(Debug, Clone, Default)]
struct RawAlias {
    part: String,
    vendor: String,
    product: String,
    cpes: BTreeSet<String>,
    purls: BTreeSet<String>,
    identity_key: String,
    canonical_slug: String,
    source: String,
}

/// Raw alias key used as the `com.nahdig.software` rkey: lowercased with
/// spaces replaced by underscores, preserving the CPE part.
fn raw_alias_key(part: &str, vendor: &str, product: &str) -> String {
    format!(
        "{}:{}:{}",
        part,
        vendor.trim().replace(' ', "_").to_lowercase(),
        product.trim().replace(' ', "_").to_lowercase()
    )
}

/// Part from the observation, else from its first parseable CPE, else `a` for
/// affected entries that carried no CPE at all.
fn observation_part(observed: &ObservedProduct) -> String {
    if let Some(part) = &observed.part {
        return part.clone();
    }
    observed
        .cpes
        .iter()
        .find_map(|cpe| identity::parse_cpe(cpe)?.part)
        .unwrap_or_else(|| "a".to_owned())
}

/// Project strongRef already persisted in `software`, if it points at the
/// canonical slug.
fn canonical_project_ref(row: &SoftwareRow, canonical_slug: &str) -> Option<(String, String)> {
    let project_uri = row
        .project_uri
        .as_deref()
        .filter(|uri| uri.rsplit('/').next() == Some(canonical_slug))?;
    let project_cid = row.project_cid.as_deref().filter(|cid| !cid.is_empty())?;
    Some((project_uri.to_owned(), project_cid.to_owned()))
}

fn strong_ref(uri: &str, cid: &str) -> Value {
    json!({ "uri": uri, "cid": cid })
}

fn outcome_created(outcome: &PublishOutcome) -> Option<(&str, &str)> {
    if !outcome.created {
        return None;
    }
    Some((outcome.uri.as_deref()?, outcome.cid.as_deref()?))
}

/// Canonical project -> raw software alias -> purl -> atprotoV1, then index
/// links for new records.
pub async fn sync_software(
    publisher: &Publisher,
    db: &DbHandle,
    config: &SoftwareConfig,
    events: EventTx,
) -> Result<SoftwareCounters> {
    let _ = events
        .send(ProgressEvent::SoftwarePhaseStarted {
            limit: config.limit,
        })
        .await;
    let mut counters = SoftwareCounters::default();
    let cve_ids = db.pending_atproto(&config.repo_did, config.limit).await?;
    if cve_ids.is_empty() {
        let _ = events
            .send(ProgressEvent::SoftwarePhaseCompleted {
                stats: counters.clone(),
            })
            .await;
        return Ok(counters);
    }

    // One observation per affected entry and per CPE criteria, canonicalized as
    // one batch so collision suffixes do not depend on CVE order.
    let mut all_observations: Vec<ObservedProduct> = Vec::new();
    let mut cve_observations: BTreeMap<String, Vec<ObservedProduct>> = BTreeMap::new();
    for cve_id in &cve_ids {
        let Some(cve) = db.get_cve_data(cve_id).await? else {
            continue;
        };
        let found = identity::observations_from_cve(&cve);
        if found.is_empty() {
            continue;
        }
        all_observations.extend(found.iter().cloned());
        cve_observations.insert(cve_id.clone(), found);
    }
    let identities = identity::canonicalize_all(&all_observations);
    let mut identity_by_key: BTreeMap<String, CanonicalIdentity> = BTreeMap::new();
    for identity in identities {
        identity_by_key
            .entry(identity.identity_key.clone())
            .or_insert(identity);
    }

    // Merge observations into raw `part:vendor:product` aliases; each alias
    // keeps its raw fields plus every CPE and purl seen for it.
    let mut aliases: BTreeMap<String, RawAlias> = BTreeMap::new();
    let mut cve_aliases: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
    for (cve_id, found) in &cve_observations {
        for observed in found {
            let Some(base) = identity::canonicalize(observed) else {
                continue;
            };
            let Some(canonical) = identity_by_key.get(&base.identity_key) else {
                continue;
            };
            let part = observation_part(observed);
            let key = raw_alias_key(&part, &observed.vendor, &observed.product);
            let entry = aliases.entry(key.clone()).or_insert_with(|| RawAlias {
                part,
                vendor: observed.vendor.trim().to_owned(),
                product: observed.product.trim().to_owned(),
                identity_key: canonical.identity_key.clone(),
                canonical_slug: canonical.slug.clone(),
                source: observed.source.clone(),
                ..RawAlias::default()
            });
            entry.cpes.extend(observed.cpes.iter().cloned());
            entry
                .purls
                .extend(observed.purls.iter().map(|purl| versionless_purl(purl)));
            cve_aliases.entry(cve_id.clone()).or_default().insert(key);
        }
    }
    if aliases.is_empty() {
        let _ = events
            .send(ProgressEvent::SoftwarePhaseCompleted {
                stats: counters.clone(),
            })
            .await;
        return Ok(counters);
    }

    let client = reqwest::Client::new();
    ensure_purl_db(&config.purl_db, &client).await?;
    let wanted: BTreeSet<String> = aliases.keys().cloned().collect();
    let purl_map = load_purl_map(&config.purl_db, &wanted).await?;
    for (key, purls) in &purl_map {
        if let Some(alias) = aliases.get_mut(key) {
            alias.purls.extend(purls.iter().cloned());
        }
    }

    // Reuse canonical project refs recorded by earlier syncs so a new raw
    // alias under a known project never re-writes the project record.
    let mut project_refs: BTreeMap<String, (String, String)> = BTreeMap::new();
    for row in db.list_software_identities(&config.repo_did).await? {
        if project_refs.contains_key(&row.canonical_slug) {
            continue;
        }
        if let Some(software) = db.get_software(&config.repo_did, &row.raw_slug).await?
            && let Some(reference) = canonical_project_ref(&software, &row.canonical_slug)
        {
            project_refs.insert(row.canonical_slug, reference);
        }
    }
    let canonical_slugs: BTreeSet<String> = aliases
        .values()
        .map(|alias| alias.canonical_slug.clone())
        .collect();
    for slug in &canonical_slugs {
        if project_refs.contains_key(slug) {
            continue;
        }
        let project = match publisher
            .put_record(
                PROJECT_COLLECTION,
                slug,
                json!({ "$type": PROJECT_TYPE, "name": slug }),
            )
            .await
        {
            Ok(outcome) => outcome,
            Err(_error) => {
                counters.failures += 1;
                continue;
            }
        };
        match outcome_created(&project) {
            Some((uri, cid)) => {
                project_refs.insert(slug.clone(), (uri.to_owned(), cid.to_owned()));
            }
            None => counters.failures += 1,
        }
    }

    // Raw aliases are local mapping rows only. Their atproto representation
    // lives inside the CVE record's `software[]` entries; no standalone
    // `com.nahdig.software` record is published.
    let mut alias_projects: BTreeMap<String, (String, String)> = BTreeMap::new();
    for (position, (key, alias)) in aliases.iter().enumerate() {
        let Some((project_uri, project_cid)) = project_refs.get(&alias.canonical_slug).cloned()
        else {
            counters.failures += 1;
            continue;
        };
        let is_new = db.get_software(&config.repo_did, key).await?.is_none();
        db.mark_software(SoftwareMark {
            repo_did: config.repo_did.clone(),
            slug: key.clone(),
            part: alias.part.clone(),
            vendor: alias.vendor.clone(),
            product: alias.product.clone(),
            project_uri: project_uri.clone(),
            project_cid: project_cid.clone(),
            software_uri: String::new(),
            software_cid: String::new(),
        })
        .await?;
        alias_projects.insert(key.clone(), (project_uri, project_cid));
        if is_new {
            counters.software_created += 1;
        }
        if (position + 1) % 50 == 0 {
            db.commit().await?;
        }
    }
    for (key, alias) in &aliases {
        db.mark_software_identity(SoftwareIdentityMark {
            repo_did: config.repo_did.clone(),
            raw_slug: key.clone(),
            canonical_slug: alias.canonical_slug.clone(),
            identity_key: Some(alias.identity_key.clone()),
            source: Some(alias.source.clone()),
            confidence: Some(1.0),
        })
        .await?;
    }

    // `software_purls.software_slug` stores the canonical project slug so
    // `release_candidates` can join purls to grouped projects.
    let mut purl_refs: BTreeMap<String, BTreeMap<String, (String, String)>> = BTreeMap::new();
    for (key, alias) in &aliases {
        let Some((uri, cid)) = alias_projects.get(key) else {
            continue;
        };
        for purl in &alias.purls {
            purl_refs
                .entry(purl.clone())
                .or_default()
                .insert(key.clone(), (uri.clone(), cid.clone()));
        }
    }

    for (purl, refs) in &purl_refs {
        if db.get_purl(&config.repo_did, purl).await?.is_some() {
            continue;
        }
        let canonical_slug = refs
            .keys()
            .find_map(|key| aliases.get(key))
            .map(|alias| alias.canonical_slug.as_str())
            .unwrap_or_default();
        db.mark_purl(&config.repo_did, purl, canonical_slug, "", "")
            .await?;
        counters.purl_created += 1;
    }
    db.commit().await?;

    for cve_id in &cve_ids {
        let Some(raw_keys) = cve_aliases.get(cve_id) else {
            continue;
        };
        let Some((nvd_uri, nvd_cid)) = db.get_cve_record(&config.repo_did, cve_id).await? else {
            continue;
        };
        // One ref per canonical project, with CPEs/purls merged across the raw
        // aliases observed for this CVE.
        let mut by_project: BTreeMap<String, (String, BTreeSet<String>, BTreeSet<String>)> =
            BTreeMap::new();
        for key in raw_keys {
            if !alias_projects.contains_key(key) {
                continue;
            }
            let Some(alias) = aliases.get(key) else {
                continue;
            };
            let entry = by_project
                .entry(alias.canonical_slug.clone())
                .or_insert_with(|| (key.clone(), BTreeSet::new(), BTreeSet::new()));
            entry.1.extend(alias.cpes.iter().cloned());
            entry.2.extend(alias.purls.iter().cloned());
        }
        let mut refs = Vec::new();
        for (canonical_slug, (key, cpes, purls)) in &by_project {
            let Some((project_uri, project_cid)) = project_refs.get(canonical_slug) else {
                continue;
            };
            let alias = &aliases[key];
            refs.push(json!({
                "project": strong_ref(project_uri, project_cid),
                "part": alias.part,
                "vendor": alias.vendor,
                "product": alias.product,
                "cpes": cpes.iter().collect::<Vec<_>>(),
                "purls": purls.iter().collect::<Vec<_>>(),
            }));
        }
        if refs.is_empty() {
            continue;
        }
        let atproto_record = json!({
            "$type": ATPROTO_TYPE,
            "createdAt": created_at(chrono::Utc::now()),
            "cve": cve_id,
            "nvd": strong_ref(&nvd_uri, &nvd_cid),
            "software": refs,
        });
        let result = match publisher
            .put_record(ATPROTO_COLLECTION, cve_id, atproto_record.clone())
            .await
        {
            Ok(outcome) => outcome,
            Err(_error) => {
                counters.failures += 1;
                continue;
            }
        };
        if let Some((uri, cid)) = outcome_created(&result) {
            index_record(
                db,
                &config.repo_did,
                uri,
                Some(cid),
                ATPROTO_COLLECTION,
                &atproto_record,
            )
            .await?;
            db.mark_cve_atproto(&config.repo_did, cve_id, uri, cid)
                .await?;
            counters.atproto_created += 1;
            counters.cves += 1;
        } else {
            counters.failures += 1;
        }
        if counters.cves % 50 == 0 {
            db.commit().await?;
        }
    }
    db.commit().await?;

    let _ = events
        .send(ProgressEvent::SoftwarePhaseCompleted {
            stats: counters.clone(),
        })
        .await;
    Ok(counters)
}