~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
754 lines · 24.3 KB Raw History
//! Publish orchestration: check, prepare, create, and mark cache rows.

use std::collections::HashMap;

use chrono::{TimeDelta, Utc};
use futures::StreamExt;
use serde::Serialize;
use serde_json::json;

use crate::config::Order;
use crate::constants::{
    APPLY_WRITES_BATCH, CHECK_CONCURRENCY, PUBLISH_BATCH, STATUS_CREATED, STATUS_PRESENT,
    STATUS_SKIPPED,
};
use crate::db::DbHandle;
use crate::db::model::{Counts, CveRow, PublicationMark, PublishErrorMark};
use crate::log::{Level, Logger};
use crate::pds::Publisher;
use crate::records::{PreparedRow, prepare_row, sanitize_rkey};
use crate::runtime::events::{EventTx, ProgressEvent};
use crate::time::{created_at, jitter};

/// Due retries attempted per publish phase.
const PUBLISH_RETRY_BATCH: usize = 100;
const PUBLISH_RETRY_BASE_SECS: i64 = 30;
const PUBLISH_RETRY_MAX_SECS: i64 = 3_600;

#[derive(Debug, Clone, Default, Serialize)]
pub struct PublishOutcome {
    pub cve_id: Option<String>,
    pub found: bool,
    pub created: bool,
    pub skipped: bool,
    pub uri: Option<String>,
    pub cid: Option<String>,
    pub calls: u64,
    pub get_ops: u64,
    pub create_ops: u64,
    pub blob_ops: u64,
    pub blob_bytes: u64,
    pub blob_cid: Option<String>,
    pub record_bytes: u64,
    pub oversized: bool,
    pub retries: u64,
    pub durations: Vec<f64>,
    pub error: Option<String>,
}

#[derive(Debug, Clone, Default, Serialize)]
pub struct PublishStats {
    pub calls: u64,
    pub present: u64,
    pub created: u64,
    pub failures: u64,
    pub skipped: u64,
    pub retries: u64,
    pub get_ops: u64,
    pub create_ops: u64,
    pub apply_ops: u64,
    pub blob_ops: u64,
    pub blob_bytes: u64,
    pub oversized: u64,
    pub oversized_skipped: u64,
    pub record_bytes_max: u64,
    pub retried: u64,
    pub retry_failed: u64,
    pub durations: Vec<f64>,
}

#[derive(Debug, Clone, Serialize)]
pub struct RecordRef {
    pub uri: String,
    pub cid: String,
}

#[derive(Debug, Clone)]
pub struct PublishConfig {
    pub concurrency: usize,
    pub limit: usize,
    pub order: Order,
}

fn aggregate_outcome(stats: &mut PublishStats, outcome: &PublishOutcome) {
    stats.calls += outcome.calls;
    stats.get_ops += outcome.get_ops;
    stats.create_ops += outcome.create_ops;
    stats.blob_ops += outcome.blob_ops;
    stats.blob_bytes += outcome.blob_bytes;
    stats.retries += outcome.retries;
    stats.record_bytes_max = stats.record_bytes_max.max(outcome.record_bytes);
    stats.durations.extend(outcome.durations.iter().copied());
}

pub(crate) fn merge_stats(target: &mut PublishStats, part: &PublishStats) {
    target.calls += part.calls;
    target.present += part.present;
    target.created += part.created;
    target.failures += part.failures;
    target.skipped += part.skipped;
    target.retries += part.retries;
    target.get_ops += part.get_ops;
    target.create_ops += part.create_ops;
    target.apply_ops += part.apply_ops;
    target.blob_ops += part.blob_ops;
    target.blob_bytes += part.blob_bytes;
    target.oversized += part.oversized;
    target.oversized_skipped += part.oversized_skipped;
    target.record_bytes_max = target.record_bytes_max.max(part.record_bytes_max);
    target.retried += part.retried;
    target.retry_failed += part.retry_failed;
    target.durations.extend(part.durations.iter().copied());
}

fn error_outcome(cve_id: &str, error: impl Into<String>) -> PublishOutcome {
    PublishOutcome {
        cve_id: Some(cve_id.to_owned()),
        error: Some(error.into()),
        ..PublishOutcome::default()
    }
}

/// Bounded exponential retry stamp: `30 * 2^(attempts-1)` seconds, capped at
/// one hour, plus jitter.
fn retry_delay_seconds(attempts: i64) -> f64 {
    let exponent = attempts.clamp(1, 16) as u32 - 1;
    let delay = PUBLISH_RETRY_BASE_SECS
        .saturating_mul(1_i64 << exponent)
        .min(PUBLISH_RETRY_MAX_SECS);
    delay as f64 + jitter(delay as f64 * 0.2)
}

fn retry_due(next_retry_at: Option<&str>) -> bool {
    let Some(raw) = next_retry_at else {
        return true;
    };
    match chrono::DateTime::parse_from_rfc3339(raw) {
        Ok(when) => when.with_timezone(&Utc) <= Utc::now(),
        // An unparseable stamp must not wedge the queue.
        Err(_) => true,
    }
}

/// Record one failed publish attempt with its next backoff stamp.
async fn record_failure(
    db: &DbHandle,
    repo: &str,
    cve_id: &str,
    error: &str,
) -> anyhow::Result<()> {
    let attempts = db
        .get_publish_error(repo, cve_id)
        .await?
        .map_or(1, |row| row.attempts + 1);
    let delay = retry_delay_seconds(attempts);
    let next_retry_at = created_at(Utc::now() + TimeDelta::milliseconds((delay * 1000.0) as i64));
    db.mark_publish_error(PublishErrorMark {
        repo_did: repo.to_owned(),
        cve_id: cve_id.to_owned(),
        attempts,
        last_error: Some(error.to_owned()),
        next_retry_at: Some(next_retry_at),
    })
    .await
}

#[allow(clippy::too_many_arguments)]
async fn mark(
    db: &DbHandle,
    repo: &str,
    cve_id: &str,
    status: &str,
    uri: Option<String>,
    cid: Option<String>,
    blob_cid: Option<String>,
    blob_bytes: Option<u64>,
    record_bytes: Option<u64>,
    mode: Option<&str>,
) -> anyhow::Result<()> {
    let mut publication = PublicationMark::new(repo, cve_id, status);
    publication.uri = uri;
    publication.cid = cid;
    publication.blob_cid = blob_cid;
    publication.blob_bytes = blob_bytes.map(|value| value as i64);
    publication.record_bytes = record_bytes.map(|value| value as i64);
    publication.mode = mode.map(str::to_owned);
    db.mark_publication(publication).await?;
    // A terminal publication mark (present/created/skipped) clears any retry
    // record; failures never reach `mark`.
    db.delete_publish_error(repo, cve_id).await?;
    Ok(())
}

/// Apply a check outcome; return `true` when the record is missing.
async fn mark_checked(
    stats: &mut PublishStats,
    db: &DbHandle,
    repo: &str,
    outcome: &PublishOutcome,
    logger: &Logger,
) -> anyhow::Result<bool> {
    if outcome.found {
        stats.present += 1;
        mark(
            db,
            repo,
            outcome.cve_id.as_deref().unwrap_or_default(),
            STATUS_PRESENT,
            outcome.uri.clone(),
            outcome.cid.clone(),
            None,
            None,
            None,
            None,
        )
        .await?;
        logger.debug(
            "record_present",
            None,
            json!({"cve": outcome.cve_id, "uri": outcome.uri}),
        );
        return Ok(false);
    }
    if outcome.skipped {
        stats.skipped += 1;
        mark(
            db,
            repo,
            outcome.cve_id.as_deref().unwrap_or_default(),
            STATUS_SKIPPED,
            outcome.uri.clone(),
            outcome.cid.clone(),
            None,
            None,
            None,
            None,
        )
        .await?;
        logger.warn(
            "record_skipped",
            None,
            json!({"cve": outcome.cve_id, "error": outcome.error}),
        );
        return Ok(false);
    }
    if let Some(error) = &outcome.error {
        stats.failures += 1;
        record_failure(
            db,
            repo,
            outcome.cve_id.as_deref().unwrap_or_default(),
            error,
        )
        .await?;
        logger.error(
            "record_failed",
            None,
            json!({"cve": outcome.cve_id, "error": error}),
        );
        return Ok(false);
    }
    Ok(true)
}

async fn mark_created(
    stats: &mut PublishStats,
    db: &DbHandle,
    repo: &str,
    outcome: &PublishOutcome,
    cve_id: &str,
    mode: &str,
) -> anyhow::Result<()> {
    stats.created += 1;
    stats.record_bytes_max = stats.record_bytes_max.max(outcome.record_bytes);
    mark(
        db,
        repo,
        cve_id,
        STATUS_CREATED,
        outcome.uri.clone(),
        outcome.cid.clone(),
        outcome.blob_cid.clone(),
        (outcome.blob_bytes > 0).then_some(outcome.blob_bytes),
        (outcome.record_bytes > 0).then_some(outcome.record_bytes),
        Some(mode),
    )
    .await
}

async fn check_rows(publisher: &Publisher, rows: &[CveRow], limit: usize) -> Vec<PublishOutcome> {
    let rows = rows.to_vec();
    futures::stream::iter(rows.into_iter().map(|row| async move {
        match publisher.check_row(&row).await {
            Ok(outcome) => outcome,
            Err(error) => error_outcome(&row.id, format!("getRecord: {error}")),
        }
    }))
    .buffered(limit.max(1))
    .collect()
    .await
}

async fn create_rows(
    publisher: &Publisher,
    rows: &[PreparedRow],
    limit: usize,
) -> Vec<PublishOutcome> {
    let rows = rows.to_vec();
    futures::stream::iter(rows.into_iter().map(|prepared| async move {
        match publisher.create_row(&prepared).await {
            Ok(outcome) => outcome,
            Err(error) => error_outcome(&prepared.cve_id, format!("createRecord: {error}")),
        }
    }))
    .buffered(limit.max(1))
    .collect()
    .await
}

async fn create_large_rows(
    publisher: &Publisher,
    rows: &[PreparedRow],
    limit: usize,
) -> Vec<PublishOutcome> {
    let rows = rows.to_vec();
    futures::stream::iter(rows.into_iter().map(|prepared| async move {
        match publisher.create_large_row(&prepared).await {
            Ok(outcome) => outcome,
            Err(error) => error_outcome(&prepared.cve_id, format!("createLargeRow: {error}")),
        }
    }))
    .buffered(limit.max(1))
    .collect()
    .await
}

async fn publish_rows(
    publisher: &Publisher,
    db: &DbHandle,
    rows: &[CveRow],
    config: &PublishConfig,
    events: &EventTx,
    logger: &Logger,
) -> anyhow::Result<PublishStats> {
    let mut stats = PublishStats::default();
    if rows.is_empty() {
        return Ok(stats);
    }
    let repo = publisher.repo().to_owned();
    let check_limit = config.concurrency.max(CHECK_CONCURRENCY);

    for start in (0..rows.len()).step_by(APPLY_WRITES_BATCH) {
        let batch = &rows[start..rows.len().min(start + APPLY_WRITES_BATCH)];
        let _ = events
            .send(ProgressEvent::PublishBatchStarted {
                offset: start,
                size: batch.len(),
            })
            .await;

        let checks = check_rows(publisher, batch, check_limit).await;
        let mut missing: Vec<CveRow> = Vec::new();
        for (row, outcome) in batch.iter().zip(checks.iter()) {
            aggregate_outcome(&mut stats, outcome);
            let outcome = PublishOutcome {
                cve_id: outcome.cve_id.clone().or_else(|| Some(row.id.clone())),
                ..outcome.clone()
            };
            if mark_checked(&mut stats, db, &repo, &outcome, logger).await? {
                missing.push(row.clone());
            }
        }

        if missing.is_empty() {
            let _ = events
                .send(ProgressEvent::PublishBatchCompleted {
                    offset: start,
                    size: batch.len(),
                    missing: 0,
                    created: stats.created,
                    present: stats.present,
                    failed: stats.failures,
                })
                .await;
            continue;
        }

        let prepared_rows: Vec<PreparedRow> = missing
            .iter()
            .map(|row| prepare_row(row.clone(), &created_at(Utc::now())))
            .collect();
        let mut normal: Vec<PreparedRow> = Vec::new();
        let mut large: Vec<PreparedRow> = Vec::new();
        for prepared in prepared_rows {
            if prepared.mode == "skip" {
                stats.oversized += 1;
                stats.oversized_skipped += 1;
                stats.skipped += 1;
                mark(
                    db,
                    &repo,
                    &prepared.cve_id,
                    STATUS_SKIPPED,
                    None,
                    None,
                    None,
                    (prepared.blob_bytes > 0).then_some(prepared.blob_bytes),
                    (prepared.record_bytes > 0).then_some(prepared.record_bytes),
                    Some("skip"),
                )
                .await?;
                logger.warn(
                    "record_skipped",
                    None,
                    json!({"cve": prepared.cve_id, "reason": prepared.reason}),
                );
            } else if prepared.mode == "blob" {
                stats.oversized += 1;
                large.push(prepared);
            } else {
                normal.push(prepared);
            }
        }

        if !normal.is_empty() {
            let (batch_outcome, results) = match publisher.create_batch(&normal).await {
                Ok(pair) => pair,
                Err(error) => (
                    PublishOutcome {
                        error: Some(error.to_string()),
                        ..PublishOutcome::default()
                    },
                    None,
                ),
            };
            aggregate_outcome(&mut stats, &batch_outcome);
            stats.apply_ops += 1;
            if let Some(results) = results {
                stats.created += normal.len() as u64;
                for (index, prepared) in normal.iter().enumerate() {
                    let cid = results.get(index).map(|reference| reference.cid.clone());
                    let uri = format!(
                        "at://{}/{}/{}",
                        publisher.repo(),
                        publisher.collection(),
                        sanitize_rkey(&prepared.cve_id)
                    );
                    stats.record_bytes_max = stats.record_bytes_max.max(prepared.record_bytes);
                    mark(
                        db,
                        &repo,
                        &prepared.cve_id,
                        STATUS_CREATED,
                        Some(uri),
                        cid,
                        None,
                        None,
                        (prepared.record_bytes > 0).then_some(prepared.record_bytes),
                        Some("normal"),
                    )
                    .await?;
                }
                logger.debug(
                    "batch_created",
                    None,
                    json!({
                        "count": normal.len(),
                        "calls": batch_outcome.calls,
                        "retries": batch_outcome.retries,
                    }),
                );
            } else {
                logger.warn(
                    "apply_writes_failed",
                    None,
                    json!({"count": normal.len(), "error": batch_outcome.error}),
                );
                let fallbacks = create_rows(publisher, &normal, config.concurrency).await;
                for (prepared, outcome) in normal.iter().zip(fallbacks.iter()) {
                    let outcome = PublishOutcome {
                        cve_id: outcome
                            .cve_id
                            .clone()
                            .or_else(|| Some(prepared.cve_id.clone())),
                        ..outcome.clone()
                    };
                    aggregate_outcome(&mut stats, &outcome);
                    if outcome.created {
                        mark_created(&mut stats, db, &repo, &outcome, &prepared.cve_id, "normal")
                            .await?;
                    } else if outcome.found {
                        stats.present += 1;
                        mark(
                            db,
                            &repo,
                            &prepared.cve_id,
                            STATUS_PRESENT,
                            outcome.uri.clone(),
                            outcome.cid.clone(),
                            None,
                            None,
                            None,
                            None,
                        )
                        .await?;
                    } else if outcome.skipped {
                        stats.skipped += 1;
                        mark(
                            db,
                            &repo,
                            &prepared.cve_id,
                            STATUS_SKIPPED,
                            outcome.uri.clone(),
                            outcome.cid.clone(),
                            None,
                            None,
                            None,
                            None,
                        )
                        .await?;
                        logger.warn(
                            "record_skipped",
                            None,
                            json!({"cve": prepared.cve_id, "error": outcome.error}),
                        );
                    } else {
                        stats.failures += 1;
                        record_failure(
                            db,
                            &repo,
                            &prepared.cve_id,
                            outcome.error.as_deref().unwrap_or("create failed"),
                        )
                        .await?;
                        logger.error(
                            "record_failed",
                            None,
                            json!({"cve": prepared.cve_id, "error": outcome.error}),
                        );
                    }
                }
            }
        }

        if !large.is_empty() {
            let outcomes = create_large_rows(publisher, &large, config.concurrency).await;
            for (prepared, outcome) in large.iter().zip(outcomes.iter()) {
                let outcome = PublishOutcome {
                    cve_id: outcome
                        .cve_id
                        .clone()
                        .or_else(|| Some(prepared.cve_id.clone())),
                    ..outcome.clone()
                };
                aggregate_outcome(&mut stats, &outcome);
                if outcome.created {
                    mark_created(&mut stats, db, &repo, &outcome, &prepared.cve_id, "blob").await?;
                    logger.info(
                        "record_compacted",
                        None,
                        json!({
                            "cve": prepared.cve_id,
                            "blob_bytes": outcome.blob_bytes,
                            "record_bytes": outcome.record_bytes,
                            "blob_cid": outcome.blob_cid,
                            "sha256": prepared.sha256,
                        }),
                    );
                } else if outcome.found {
                    stats.present += 1;
                    mark(
                        db,
                        &repo,
                        &prepared.cve_id,
                        STATUS_PRESENT,
                        outcome.uri.clone(),
                        outcome.cid.clone(),
                        None,
                        None,
                        None,
                        None,
                    )
                    .await?;
                } else if outcome.skipped {
                    stats.oversized_skipped += 1;
                    stats.skipped += 1;
                    mark(
                        db,
                        &repo,
                        &prepared.cve_id,
                        STATUS_SKIPPED,
                        outcome.uri.clone(),
                        outcome.cid.clone(),
                        None,
                        None,
                        None,
                        None,
                    )
                    .await?;
                    logger.warn(
                        "record_skipped",
                        None,
                        json!({"cve": prepared.cve_id, "error": outcome.error}),
                    );
                } else {
                    stats.failures += 1;
                    logger.error(
                        "record_failed",
                        None,
                        json!({"cve": prepared.cve_id, "error": outcome.error}),
                    );
                }
            }
        }

        let _ = events
            .send(ProgressEvent::PublishBatchCompleted {
                offset: start,
                size: batch.len(),
                missing: missing.len(),
                created: stats.created,
                present: stats.present,
                failed: stats.failures,
            })
            .await;
    }

    Ok(stats)
}

pub async fn drain_pending(
    publisher: &Publisher,
    db: &DbHandle,
    config: &PublishConfig,
    events: EventTx,
) -> anyhow::Result<PublishStats> {
    let _ = events
        .send(ProgressEvent::PublishPhaseStarted {
            limit: config.limit,
        })
        .await;
    let result = drain_pending_inner(publisher, db, config, &events).await;
    let (stats, error) = match &result {
        Ok(stats) => (stats.clone(), None),
        Err(error) => (PublishStats::default(), Some(error.to_string())),
    };
    let _ = events
        .send(ProgressEvent::PublishPhaseCompleted { stats, error })
        .await;
    result
}

async fn drain_pending_inner(
    publisher: &Publisher,
    db: &DbHandle,
    config: &PublishConfig,
    events: &EventTx,
) -> anyhow::Result<PublishStats> {
    let logger = Logger::stderr(Level::Info);
    let mut total = PublishStats::default();
    let mut remaining = (config.limit > 0).then_some(config.limit);
    loop {
        let size = remaining.map_or(PUBLISH_BATCH, |left| PUBLISH_BATCH.min(left));
        if size == 0 {
            break;
        }
        let fetched = db.pending(publisher.repo(), size, config.order).await?;
        if fetched.is_empty() {
            break;
        }
        let fetched_len = fetched.len();
        let backoff: HashMap<String, Option<String>> = db
            .pending_publish_errors(0)
            .await?
            .into_iter()
            .filter(|row| row.repo_did == publisher.repo())
            .map(|row| (row.cve_id, row.next_retry_at))
            .collect();
        let rows: Vec<CveRow> = fetched
            .into_iter()
            .filter(|row| {
                backoff
                    .get(&row.id)
                    .is_none_or(|next| retry_due(next.as_deref()))
            })
            .collect();
        if let Some(left) = &mut remaining {
            *left = left.saturating_sub(fetched_len);
        }
        if rows.is_empty() {
            // Every pending row is backing off; retry on a later cycle.
            break;
        }
        let part = publish_rows(publisher, db, &rows, config, events, &logger).await?;
        merge_stats(&mut total, &part);
        db.commit().await?;
        let counts: Counts = db.counts(publisher.repo()).await?;
        logger.info(
            "publish_progress",
            None,
            json!({
                "created": total.created,
                "present": total.present,
                "skipped": total.skipped,
                "failed": total.failures,
                "pending": counts.pending,
            }),
        );
        if part.failures > 0 {
            // Failed rows stay pending and are retried when their backoff is due.
            break;
        }
        if remaining == Some(0) {
            break;
        }
    }
    Ok(total)
}

/// Publish CVEs whose recorded retry backoff is due.
///
/// Called before and after the publish phase. Successful rows are marked
/// present/created (and their retry rows cleared); failures bump `attempts`
/// and `next_retry_at` in `publish_errors`.
pub async fn drain_retries(
    publisher: &Publisher,
    db: &DbHandle,
    config: &PublishConfig,
    events: EventTx,
) -> anyhow::Result<PublishStats> {
    let errors = db.pending_publish_errors(PUBLISH_RETRY_BATCH).await?;
    let mut rows = Vec::new();
    for error in errors {
        if error.repo_did != publisher.repo() || !retry_due(error.next_retry_at.as_deref()) {
            continue;
        }
        match db.get_cve_data(&error.cve_id).await? {
            Some(data) => rows.push(CveRow::new(error.cve_id, data)),
            // The CVE left the cache; the retry can never succeed.
            None => {
                db.delete_publish_error(&error.repo_did, &error.cve_id)
                    .await?;
            }
        }
    }
    if rows.is_empty() {
        return Ok(PublishStats::default());
    }
    let logger = Logger::stderr(Level::Info);
    let mut stats = publish_rows(publisher, db, &rows, config, &events, &logger).await?;
    db.commit().await?;
    stats.retried = rows.len() as u64;
    stats.retry_failed = stats.failures;
    Ok(stats)
}