//! Software / purl / atprotoV1 graph sync. use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use anyhow::Result; use serde::Serialize; use serde_json::{Value, json}; use crate::constants::{ATPROTO_COLLECTION, ATPROTO_TYPE, PROJECT_COLLECTION, PROJECT_TYPE}; use crate::db::DbHandle; use crate::db::model::{SoftwareIdentityMark, SoftwareMark, SoftwareRow}; use crate::identity::{self, CanonicalIdentity, ObservedProduct}; use crate::links::index_record; use crate::pds::Publisher; use crate::publish::PublishOutcome; use crate::runtime::events::{EventTx, ProgressEvent}; use crate::time::created_at; pub use crate::db::purl2cpe::{cpe_product_key, ensure_purl_db, load_purl_map, versionless_purl}; #[derive(Debug, Clone)] pub struct SoftwareConfig { pub repo_did: String, pub limit: usize, pub purl_db: PathBuf, } #[derive(Debug, Clone, Default, Serialize)] pub struct SoftwareCounters { pub cves: u64, pub software_created: u64, pub purl_created: u64, pub atproto_created: u64, pub failures: u64, } /// One `com.nahdig.software` record: a raw CPE observation merged per /// `part:vendor:product` alias, mapped to its canonical project identity. #[derive(Debug, Clone, Default)] struct RawAlias { part: String, vendor: String, product: String, cpes: BTreeSet, purls: BTreeSet, identity_key: String, canonical_slug: String, source: String, } /// Raw alias key used as the `com.nahdig.software` rkey: lowercased with /// spaces replaced by underscores, preserving the CPE part. fn raw_alias_key(part: &str, vendor: &str, product: &str) -> String { format!( "{}:{}:{}", part, vendor.trim().replace(' ', "_").to_lowercase(), product.trim().replace(' ', "_").to_lowercase() ) } /// Part from the observation, else from its first parseable CPE, else `a` for /// affected entries that carried no CPE at all. fn observation_part(observed: &ObservedProduct) -> String { if let Some(part) = &observed.part { return part.clone(); } observed .cpes .iter() .find_map(|cpe| identity::parse_cpe(cpe)?.part) .unwrap_or_else(|| "a".to_owned()) } /// Project strongRef already persisted in `software`, if it points at the /// canonical slug. fn canonical_project_ref(row: &SoftwareRow, canonical_slug: &str) -> Option<(String, String)> { let project_uri = row .project_uri .as_deref() .filter(|uri| uri.rsplit('/').next() == Some(canonical_slug))?; let project_cid = row.project_cid.as_deref().filter(|cid| !cid.is_empty())?; Some((project_uri.to_owned(), project_cid.to_owned())) } fn strong_ref(uri: &str, cid: &str) -> Value { json!({ "uri": uri, "cid": cid }) } fn outcome_created(outcome: &PublishOutcome) -> Option<(&str, &str)> { if !outcome.created { return None; } Some((outcome.uri.as_deref()?, outcome.cid.as_deref()?)) } /// Canonical project -> raw software alias -> purl -> atprotoV1, then index /// links for new records. pub async fn sync_software( publisher: &Publisher, db: &DbHandle, config: &SoftwareConfig, events: EventTx, ) -> Result { let _ = events .send(ProgressEvent::SoftwarePhaseStarted { limit: config.limit, }) .await; let mut counters = SoftwareCounters::default(); let cve_ids = db.pending_atproto(&config.repo_did, config.limit).await?; if cve_ids.is_empty() { let _ = events .send(ProgressEvent::SoftwarePhaseCompleted { stats: counters.clone(), }) .await; return Ok(counters); } // One observation per affected entry and per CPE criteria, canonicalized as // one batch so collision suffixes do not depend on CVE order. let mut all_observations: Vec = Vec::new(); let mut cve_observations: BTreeMap> = BTreeMap::new(); for cve_id in &cve_ids { let Some(cve) = db.get_cve_data(cve_id).await? else { continue; }; let found = identity::observations_from_cve(&cve); if found.is_empty() { continue; } all_observations.extend(found.iter().cloned()); cve_observations.insert(cve_id.clone(), found); } let identities = identity::canonicalize_all(&all_observations); let mut identity_by_key: BTreeMap = BTreeMap::new(); for identity in identities { identity_by_key .entry(identity.identity_key.clone()) .or_insert(identity); } // Merge observations into raw `part:vendor:product` aliases; each alias // keeps its raw fields plus every CPE and purl seen for it. let mut aliases: BTreeMap = BTreeMap::new(); let mut cve_aliases: BTreeMap> = BTreeMap::new(); for (cve_id, found) in &cve_observations { for observed in found { let Some(base) = identity::canonicalize(observed) else { continue; }; let Some(canonical) = identity_by_key.get(&base.identity_key) else { continue; }; let part = observation_part(observed); let key = raw_alias_key(&part, &observed.vendor, &observed.product); let entry = aliases.entry(key.clone()).or_insert_with(|| RawAlias { part, vendor: observed.vendor.trim().to_owned(), product: observed.product.trim().to_owned(), identity_key: canonical.identity_key.clone(), canonical_slug: canonical.slug.clone(), source: observed.source.clone(), ..RawAlias::default() }); entry.cpes.extend(observed.cpes.iter().cloned()); entry .purls .extend(observed.purls.iter().map(|purl| versionless_purl(purl))); cve_aliases.entry(cve_id.clone()).or_default().insert(key); } } if aliases.is_empty() { let _ = events .send(ProgressEvent::SoftwarePhaseCompleted { stats: counters.clone(), }) .await; return Ok(counters); } let client = reqwest::Client::new(); ensure_purl_db(&config.purl_db, &client).await?; let wanted: BTreeSet = aliases.keys().cloned().collect(); let purl_map = load_purl_map(&config.purl_db, &wanted).await?; for (key, purls) in &purl_map { if let Some(alias) = aliases.get_mut(key) { alias.purls.extend(purls.iter().cloned()); } } // Reuse canonical project refs recorded by earlier syncs so a new raw // alias under a known project never re-writes the project record. let mut project_refs: BTreeMap = BTreeMap::new(); for row in db.list_software_identities(&config.repo_did).await? { if project_refs.contains_key(&row.canonical_slug) { continue; } if let Some(software) = db.get_software(&config.repo_did, &row.raw_slug).await? && let Some(reference) = canonical_project_ref(&software, &row.canonical_slug) { project_refs.insert(row.canonical_slug, reference); } } let canonical_slugs: BTreeSet = aliases .values() .map(|alias| alias.canonical_slug.clone()) .collect(); for slug in &canonical_slugs { if project_refs.contains_key(slug) { continue; } let project = match publisher .put_record( PROJECT_COLLECTION, slug, json!({ "$type": PROJECT_TYPE, "name": slug }), ) .await { Ok(outcome) => outcome, Err(_error) => { counters.failures += 1; continue; } }; match outcome_created(&project) { Some((uri, cid)) => { project_refs.insert(slug.clone(), (uri.to_owned(), cid.to_owned())); } None => counters.failures += 1, } } // Raw aliases are local mapping rows only. Their atproto representation // lives inside the CVE record's `software[]` entries; no standalone // `com.nahdig.software` record is published. let mut alias_projects: BTreeMap = BTreeMap::new(); for (position, (key, alias)) in aliases.iter().enumerate() { let Some((project_uri, project_cid)) = project_refs.get(&alias.canonical_slug).cloned() else { counters.failures += 1; continue; }; let is_new = db.get_software(&config.repo_did, key).await?.is_none(); db.mark_software(SoftwareMark { repo_did: config.repo_did.clone(), slug: key.clone(), part: alias.part.clone(), vendor: alias.vendor.clone(), product: alias.product.clone(), project_uri: project_uri.clone(), project_cid: project_cid.clone(), software_uri: String::new(), software_cid: String::new(), }) .await?; alias_projects.insert(key.clone(), (project_uri, project_cid)); if is_new { counters.software_created += 1; } if (position + 1) % 50 == 0 { db.commit().await?; } } for (key, alias) in &aliases { db.mark_software_identity(SoftwareIdentityMark { repo_did: config.repo_did.clone(), raw_slug: key.clone(), canonical_slug: alias.canonical_slug.clone(), identity_key: Some(alias.identity_key.clone()), source: Some(alias.source.clone()), confidence: Some(1.0), }) .await?; } // `software_purls.software_slug` stores the canonical project slug so // `release_candidates` can join purls to grouped projects. let mut purl_refs: BTreeMap> = BTreeMap::new(); for (key, alias) in &aliases { let Some((uri, cid)) = alias_projects.get(key) else { continue; }; for purl in &alias.purls { purl_refs .entry(purl.clone()) .or_default() .insert(key.clone(), (uri.clone(), cid.clone())); } } for (purl, refs) in &purl_refs { if db.get_purl(&config.repo_did, purl).await?.is_some() { continue; } let canonical_slug = refs .keys() .find_map(|key| aliases.get(key)) .map(|alias| alias.canonical_slug.as_str()) .unwrap_or_default(); db.mark_purl(&config.repo_did, purl, canonical_slug, "", "") .await?; counters.purl_created += 1; } db.commit().await?; for cve_id in &cve_ids { let Some(raw_keys) = cve_aliases.get(cve_id) else { continue; }; let Some((nvd_uri, nvd_cid)) = db.get_cve_record(&config.repo_did, cve_id).await? else { continue; }; // One ref per canonical project, with CPEs/purls merged across the raw // aliases observed for this CVE. let mut by_project: BTreeMap, BTreeSet)> = BTreeMap::new(); for key in raw_keys { if !alias_projects.contains_key(key) { continue; } let Some(alias) = aliases.get(key) else { continue; }; let entry = by_project .entry(alias.canonical_slug.clone()) .or_insert_with(|| (key.clone(), BTreeSet::new(), BTreeSet::new())); entry.1.extend(alias.cpes.iter().cloned()); entry.2.extend(alias.purls.iter().cloned()); } let mut refs = Vec::new(); for (canonical_slug, (key, cpes, purls)) in &by_project { let Some((project_uri, project_cid)) = project_refs.get(canonical_slug) else { continue; }; let alias = &aliases[key]; refs.push(json!({ "project": strong_ref(project_uri, project_cid), "part": alias.part, "vendor": alias.vendor, "product": alias.product, "cpes": cpes.iter().collect::>(), "purls": purls.iter().collect::>(), })); } if refs.is_empty() { continue; } let atproto_record = json!({ "$type": ATPROTO_TYPE, "createdAt": created_at(chrono::Utc::now()), "cve": cve_id, "nvd": strong_ref(&nvd_uri, &nvd_cid), "software": refs, }); let result = match publisher .put_record(ATPROTO_COLLECTION, cve_id, atproto_record.clone()) .await { Ok(outcome) => outcome, Err(_error) => { counters.failures += 1; continue; } }; if let Some((uri, cid)) = outcome_created(&result) { index_record( db, &config.repo_did, uri, Some(cid), ATPROTO_COLLECTION, &atproto_record, ) .await?; db.mark_cve_atproto(&config.repo_did, cve_id, uri, cid) .await?; counters.atproto_created += 1; counters.cves += 1; } else { counters.failures += 1; } if counters.cves % 50 == 0 { db.commit().await?; } } db.commit().await?; let _ = events .send(ProgressEvent::SoftwarePhaseCompleted { stats: counters.clone(), }) .await; Ok(counters) }