~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
399 lines · 12.2 KB Raw History
//! Cycle and run metrics.

use std::collections::BTreeMap;

use serde::Serialize;
use serde_json::{Map, Value, json};

/// Field names and rounding for cycle stats.
#[derive(Debug, Clone, Default, Serialize)]
pub struct CycleStats {
    pub cycle: u64,
    pub started_at: String,
    pub repo_did: String,
    pub duration_seconds: f64,
    pub dry_run: bool,
    pub limit: u64,
    pub concurrency: u64,
    pub nvd_requests: u64,
    pub nvd_cves: u64,
    pub nvd_fetch_seconds: f64,
    pub nvd_retries: u64,
    pub nvd_key_configured: bool,
    pub pds_budget_hour_remaining: f64,
    pub pds_budget_day_remaining: f64,
    pub pds_rate_limit_waits: u64,
    pub pds_rate_limit_seconds: f64,
    pub pds_requests_5min: usize,
    pub cache_hits: u64,
    pub cache_new: u64,
    pub cache_updated: u64,
    pub cache_pending_before: i64,
    pub cache_pending_after: i64,
    pub cache_total: i64,
    pub pds_login_calls: u64,
    pub pds_get_ops: u64,
    pub pds_create_ops: u64,
    pub pds_apply_ops: u64,
    pub pds_blob_ops: u64,
    pub pds_blob_bytes: u64,
    pub pds_http_calls: u64,
    pub pds_present: u64,
    pub pds_created: u64,
    pub pds_failures: u64,
    pub pds_retries: u64,
    pub pds_skipped: u64,
    pub pds_seconds_total: f64,
    pub pds_seconds_avg: f64,
    pub pds_seconds_max: f64,
    pub records_created: u64,
    pub records_present: u64,
    pub records_skipped: u64,
    pub records_failed: u64,
    pub records_oversized: u64,
    pub records_oversized_skipped: u64,
    pub record_bytes_max: u64,
    pub records_per_second: f64,
    pub publish_retried: u64,
    pub publish_retry_failed: u64,
    pub link_repair_attempted: u64,
    pub link_repair_repaired: u64,
    pub link_repair_failed: u64,
    pub link_repair_deferred: u64,
    pub software_cves: u64,
    pub software_created: u64,
    pub purl_created: u64,
    pub atproto_created: u64,
    pub software_failures: u64,
    pub s3_download_bytes: u64,
    pub s3_upload_bytes: u64,
    pub s3_download_seconds: f64,
    pub s3_upload_seconds: f64,
    pub cursor_before: Option<String>,
    pub cursor_after: Option<String>,
    pub release_mode: String,
    pub release_projects_checked: u64,
    pub release_projects_created: u64,
    pub release_versions_created: u64,
    pub release_channels_created: u64,
    pub release_statuses_created: u64,
    pub release_unchanged: u64,
    pub release_skipped: u64,
    pub release_failures: u64,
    pub release_affected_projects_checked: u64,
    pub release_affected_projects_linked: u64,
    pub release_affected_versions_planned: u64,
    pub release_affected_versions_created: u64,
    pub release_affected_versions_label_only: u64,
    pub release_affected_cves_linked: u64,
    pub release_affected_unchanged: u64,
    pub release_affected_skipped: u64,
    pub release_affected_failures: u64,
    pub release_affected_unsupported: u64,
    pub release_history_projects_checked: u64,
    pub release_history_projects_skipped: u64,
    pub release_history_unsupported: u64,
    pub release_history_enqueued: u64,
    pub release_history_processed: u64,
    pub release_history_created: u64,
    pub release_history_skipped: u64,
    pub release_history_failed: u64,
    pub release_history_reused: u64,
    pub release_extras_projects_checked: u64,
    pub release_extras_projects_created: u64,
    pub release_extras_versions_created: u64,
    pub release_extras_manifests_created: u64,
    pub release_extras_statuses_created: u64,
    pub release_extras_unchanged: u64,
    pub release_extras_skipped: u64,
    pub release_extras_unsupported: u64,
    pub release_extras_failures: u64,
    pub artifact_downloads: u64,
    pub artifact_bytes: u64,
    pub artifact_cache_hits: u64,
    pub artifact_mirror_puts: u64,
    pub artifact_errors: u64,
}

impl CycleStats {
    pub fn to_value(&self) -> Value {
        serde_json::to_value(self).unwrap_or(Value::Null)
    }
}

/// Keys summed across cycles.
pub const SUM_KEYS: &[&str] = &[
    "duration_seconds",
    "nvd_requests",
    "nvd_cves",
    "nvd_fetch_seconds",
    "nvd_retries",
    "cache_hits",
    "cache_new",
    "cache_updated",
    "pds_login_calls",
    "pds_get_ops",
    "pds_create_ops",
    "pds_apply_ops",
    "pds_blob_ops",
    "pds_blob_bytes",
    "pds_http_calls",
    "pds_present",
    "pds_created",
    "pds_failures",
    "pds_retries",
    "pds_skipped",
    "pds_rate_limit_waits",
    "pds_rate_limit_seconds",
    "pds_seconds_total",
    "records_created",
    "records_present",
    "records_skipped",
    "records_failed",
    "records_oversized",
    "records_oversized_skipped",
    "publish_retried",
    "publish_retry_failed",
    "link_repair_attempted",
    "link_repair_repaired",
    "link_repair_failed",
    "link_repair_deferred",
    "software_cves",
    "software_created",
    "purl_created",
    "atproto_created",
    "software_failures",
    "s3_download_bytes",
    "s3_upload_bytes",
    "s3_download_seconds",
    "s3_upload_seconds",
    "release_projects_checked",
    "release_projects_created",
    "release_versions_created",
    "release_channels_created",
    "release_statuses_created",
    "release_unchanged",
    "release_skipped",
    "release_failures",
    "release_affected_projects_checked",
    "release_affected_projects_linked",
    "release_affected_versions_planned",
    "release_affected_versions_created",
    "release_affected_versions_label_only",
    "release_affected_cves_linked",
    "release_affected_unchanged",
    "release_affected_skipped",
    "release_affected_failures",
    "release_affected_unsupported",
    "release_history_projects_checked",
    "release_history_projects_skipped",
    "release_history_unsupported",
    "release_history_enqueued",
    "release_history_processed",
    "release_history_created",
    "release_history_skipped",
    "release_history_failed",
    "release_history_reused",
    "release_extras_projects_checked",
    "release_extras_projects_created",
    "release_extras_versions_created",
    "release_extras_manifests_created",
    "release_extras_statuses_created",
    "release_extras_unchanged",
    "release_extras_skipped",
    "release_extras_unsupported",
    "release_extras_failures",
    "artifact_downloads",
    "artifact_bytes",
    "artifact_cache_hits",
    "artifact_mirror_puts",
    "artifact_errors",
    "errors",
];

/// Keys whose maximum is kept across cycles.
pub const MAX_KEYS: &[&str] = &[
    "pds_seconds_max",
    "cache_pending_after",
    "cache_total",
    "record_bytes_max",
];

/// Keys whose latest value wins.
pub const LAST_KEYS: &[&str] = &[
    "pds_budget_hour_remaining",
    "pds_budget_day_remaining",
    "pds_requests_5min",
    "nvd_key_configured",
    "pds_resolve_seconds",
    "release_mode",
];

/// Decimal rounding to `digits` places.
pub fn round_to(value: f64, digits: i32) -> f64 {
    if !value.is_finite() {
        return value;
    }
    let digits = digits.max(0) as usize;
    format!("{value:.digits$}").parse::<f64>().unwrap_or(value)
}

pub fn round6(value: f64) -> f64 {
    round_to(value, 6)
}

pub fn round3(value: f64) -> f64 {
    round_to(value, 3)
}

/// Accumulated run totals.
#[derive(Debug, Clone, Default)]
pub struct RunTotals {
    sums: BTreeMap<String, f64>,
    maxes: BTreeMap<String, f64>,
    lasts: Map<String, Value>,
}

impl RunTotals {
    pub fn new() -> Self {
        let mut totals = Self::default();
        totals.sums.insert("errors".to_owned(), 0.0);
        totals
            .lasts
            .insert("pds_resolve_seconds".to_owned(), json!(0.0));
        totals
    }

    pub fn set_last(&mut self, key: &str, value: Value) {
        self.lasts.insert(key.to_owned(), value);
    }

    /// Accumulate one cycle's stats into run totals.
    pub fn accumulate(&mut self, stats: &CycleStats) {
        let value = stats.to_value();
        for &key in SUM_KEYS {
            if let Some(number) = value.get(key).and_then(Value::as_f64) {
                *self.sums.entry(key.to_owned()).or_insert(0.0) += number;
            }
        }
        for &key in MAX_KEYS {
            if let Some(number) = value.get(key).and_then(Value::as_f64) {
                let entry = self.maxes.entry(key.to_owned()).or_insert(0.0);
                *entry = entry.max(number);
            }
        }
        for &key in LAST_KEYS {
            if let Some(last) = value.get(key)
                && !last.is_null()
            {
                self.lasts.insert(key.to_owned(), last.clone());
            }
        }
    }

    pub fn add_error(&mut self) {
        *self.sums.entry("errors".to_owned()).or_insert(0.0) += 1.0;
    }

    pub fn errors(&self) -> f64 {
        self.sums.get("errors").copied().unwrap_or(0.0)
    }

    /// Rounded totals plus derived summary fields.
    pub fn build_run_stats(&self, cycles: u64, started_at: &str) -> Value {
        let mut summary = Map::new();
        // Totals start with `errors` before accumulating, so it leads the
        // run_stats object.
        if let Some(value) = self.sums.get("errors") {
            summary.insert("errors".to_owned(), json!(round6(*value)));
        }
        for &key in SUM_KEYS {
            if key == "errors" {
                continue;
            }
            if let Some(value) = self.sums.get(key) {
                summary.insert(key.to_owned(), json!(round6(*value)));
            }
        }
        for &key in MAX_KEYS {
            if let Some(value) = self.maxes.get(key) {
                summary.insert(key.to_owned(), json!(round6(*value)));
            }
        }
        for &key in LAST_KEYS {
            if let Some(value) = self.lasts.get(key) {
                summary.insert(key.to_owned(), value.clone());
            }
        }
        let duration = self.sums.get("duration_seconds").copied().unwrap_or(0.0);
        let calls = self.sums.get("pds_http_calls").copied().unwrap_or(0.0);
        let records = self.sums.get("records_created").copied().unwrap_or(0.0)
            + self.sums.get("records_present").copied().unwrap_or(0.0)
            + self.sums.get("records_skipped").copied().unwrap_or(0.0);
        summary.insert("cycles".to_owned(), json!(cycles));
        summary.insert("started_at".to_owned(), json!(started_at));
        summary.insert(
            "pds_seconds_avg".to_owned(),
            json!(if calls != 0.0 {
                round6(self.sums.get("pds_seconds_total").copied().unwrap_or(0.0) / calls)
            } else {
                0.0
            }),
        );
        summary.insert(
            "records_per_second".to_owned(),
            json!(if duration != 0.0 {
                round3(records / duration)
            } else {
                0.0
            }),
        );
        Value::Object(summary)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn totals_sum_max_last_and_summary() {
        let mut totals = RunTotals::new();
        let first = CycleStats {
            duration_seconds: 2.0,
            cache_total: 3,
            pds_requests_5min: 7,
            nvd_key_configured: true,
            records_created: 4,
            software_cves: 2,
            publish_retried: 1,
            link_repair_repaired: 3,
            pds_seconds_total: 1.0,
            pds_http_calls: 2,
            ..CycleStats::default()
        };
        let second = CycleStats {
            duration_seconds: 3.0,
            cache_total: 9,
            pds_requests_5min: 5,
            records_present: 2,
            software_cves: 5,
            publish_retried: 4,
            link_repair_repaired: 1,
            ..CycleStats::default()
        };
        totals.accumulate(&first);
        totals.accumulate(&second);
        let summary = totals.build_run_stats(2, "2024-01-01T00:00:00.000Z");
        assert_eq!(summary["duration_seconds"], json!(5.0));
        assert_eq!(summary["cache_total"], json!(9.0));
        assert_eq!(summary["pds_requests_5min"], json!(5));
        assert_eq!(summary["nvd_key_configured"], json!(false));
        assert_eq!(summary["records_per_second"], json!(1.2));
        assert_eq!(summary["pds_seconds_avg"], json!(0.5));
        assert_eq!(summary["software_cves"], json!(7.0));
        assert_eq!(summary["publish_retried"], json!(5.0));
        assert_eq!(summary["link_repair_repaired"], json!(4.0));
        assert_eq!(summary["errors"], json!(0.0));
    }
}