~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
819 lines · 23.8 KB Raw History
//! Publish parity against a fake `PdsApi` and an in-memory cache.

use std::collections::{HashMap, VecDeque};
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;

use async_trait::async_trait;
use chrono::{DateTime, Utc};
use serde_json::{Value, json};

use vuln_notif::config::Order;
use vuln_notif::db::DbHandle;
use vuln_notif::db::model::{CveRow, PublicationMark, PublishErrorMark};
use vuln_notif::log::{CaptureSink, Level, Logger};
use vuln_notif::pds::{PdsApi, Publisher, PublisherConfig, RateGovernor, XrpcResponse};
use vuln_notif::publish::{PublishConfig, PublishStats, drain_pending, drain_retries};
use vuln_notif::records::{prepare_row, sanitize_rkey};
use vuln_notif::runtime::events::ProgressEvent;
use vuln_notif::time::Time;

const REPO: &str = "did:plc:test";
const COLLECTION: &str = "com.nahdig.cve.nvdV2";

fn fixture(name: &str) -> Value {
    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .join("tests/fixtures/records/inputs")
        .join(format!("{name}.json"));
    serde_json::from_str(&std::fs::read_to_string(path).expect("read fixture")).expect("json")
}

fn ok(body: Value) -> XrpcResponse {
    XrpcResponse {
        status: 200,
        body,
        ..XrpcResponse::default()
    }
}

fn error(status: u16, code: &str, message: &str) -> XrpcResponse {
    XrpcResponse {
        status,
        body: json!({"error": code, "message": message}),
        ..XrpcResponse::default()
    }
}

fn not_found() -> XrpcResponse {
    error(400, "RecordNotFound", "Could not locate record")
}

#[derive(Default)]
struct ApiState {
    get: HashMap<String, XrpcResponse>,
    create: VecDeque<XrpcResponse>,
    apply: VecDeque<XrpcResponse>,
    upload: VecDeque<XrpcResponse>,
    get_calls: u64,
    create_calls: u64,
    apply_calls: u64,
    upload_calls: u64,
    uploads: Vec<(String, Vec<u8>)>,
    creates: Vec<Value>,
    writes: Vec<Vec<Value>>,
}

#[derive(Default)]
struct FakeApi {
    state: Mutex<ApiState>,
}

impl FakeApi {
    fn with_get(self, rkey: &str, response: XrpcResponse) -> Self {
        self.state
            .lock()
            .expect("fake api")
            .get
            .insert(rkey.to_owned(), response);
        self
    }

    fn with_create(self, response: XrpcResponse) -> Self {
        self.state
            .lock()
            .expect("fake api")
            .create
            .push_back(response);
        self
    }

    fn with_apply(self, response: XrpcResponse) -> Self {
        self.state
            .lock()
            .expect("fake api")
            .apply
            .push_back(response);
        self
    }

    fn counts(&self) -> (u64, u64, u64, u64) {
        let state = self.state.lock().expect("fake api");
        (
            state.get_calls,
            state.create_calls,
            state.apply_calls,
            state.upload_calls,
        )
    }

    fn creates(&self) -> Vec<Value> {
        self.state.lock().expect("fake api").creates.clone()
    }

    fn uploads(&self) -> Vec<(String, Vec<u8>)> {
        self.state.lock().expect("fake api").uploads.clone()
    }
}

#[async_trait]
impl PdsApi for FakeApi {
    async fn create_session(&self, _handle: &str, _password: &str) -> anyhow::Result<XrpcResponse> {
        Ok(ok(json!({
            "did": REPO,
            "handle": "alice.test",
            "accessJwt": "access-1",
            "refreshJwt": "refresh-1",
        })))
    }

    async fn refresh_session(&self, _refresh_jwt: &str) -> anyhow::Result<XrpcResponse> {
        Ok(ok(json!({
            "did": REPO,
            "handle": "alice.test",
            "accessJwt": "access-2",
            "refreshJwt": "refresh-2",
        })))
    }

    async fn get_record(
        &self,
        _repo: &str,
        _collection: &str,
        rkey: &str,
        _access_jwt: &str,
    ) -> anyhow::Result<XrpcResponse> {
        let mut state = self.state.lock().expect("fake api");
        state.get_calls += 1;
        Ok(state.get.get(rkey).cloned().unwrap_or_else(not_found))
    }

    async fn create_record(
        &self,
        repo: &str,
        collection: &str,
        rkey: &str,
        record: &Value,
        _access_jwt: &str,
    ) -> anyhow::Result<XrpcResponse> {
        let mut state = self.state.lock().expect("fake api");
        state.create_calls += 1;
        state.creates.push(record.clone());
        Ok(state.create.pop_front().unwrap_or_else(|| {
            ok(json!({
                "uri": format!("at://{repo}/{collection}/{rkey}"),
                "cid": "bafycreated",
            }))
        }))
    }

    async fn put_record(
        &self,
        _repo: &str,
        _collection: &str,
        _rkey: &str,
        _record: &Value,
        _access_jwt: &str,
    ) -> anyhow::Result<XrpcResponse> {
        Ok(ok(json!({})))
    }

    async fn apply_writes(
        &self,
        repo: &str,
        writes: &[Value],
        _access_jwt: &str,
    ) -> anyhow::Result<XrpcResponse> {
        let mut state = self.state.lock().expect("fake api");
        state.apply_calls += 1;
        state.writes.push(writes.to_vec());
        Ok(state.apply.pop_front().unwrap_or_else(|| {
            let results: Vec<Value> = writes
                .iter()
                .map(|write| {
                    json!({
                        "uri": format!(
                            "at://{repo}/{}/{}",
                            write["collection"].as_str().unwrap_or_default(),
                            write["rkey"].as_str().unwrap_or_default()
                        ),
                        "cid": "bafyapply",
                    })
                })
                .collect();
            ok(json!({"results": results}))
        }))
    }

    async fn upload_blob(
        &self,
        bytes: &[u8],
        mime_type: &str,
        _access_jwt: &str,
    ) -> anyhow::Result<XrpcResponse> {
        let mut state = self.state.lock().expect("fake api");
        state.upload_calls += 1;
        state.uploads.push((mime_type.to_owned(), bytes.to_vec()));
        Ok(state.upload.pop_front().unwrap_or_else(|| {
            ok(json!({
                "blob": {
                    "$type": "blob",
                    "ref": {"$link": "bafyreibvqn63urbv7lilphh4nsqh4d6yxj5xmbq7rc44ffi3euxsuxzpeu"},
                    "mimeType": mime_type,
                    "size": bytes.len(),
                }
            }))
        }))
    }
}

struct FakeTime {
    now: Mutex<f64>,
    sleeps: AtomicU64,
}

impl FakeTime {
    fn new() -> Self {
        Self {
            now: Mutex::new(1_700_000_000.0),
            sleeps: AtomicU64::new(0),
        }
    }
}

#[async_trait]
impl Time for FakeTime {
    fn now_unix(&self) -> f64 {
        *self.now.lock().expect("fake time")
    }

    fn now_utc(&self) -> DateTime<Utc> {
        DateTime::<Utc>::from_timestamp_millis((self.now_unix() * 1000.0) as i64)
            .expect("timestamp")
    }

    async fn sleep(&self, duration: Duration) {
        self.sleeps.fetch_add(1, Ordering::Relaxed);
        *self.now.lock().expect("fake time") += duration.as_secs_f64();
    }

    fn jitter(&self, _scale: f64) -> f64 {
        0.0
    }
}

struct Harness {
    db: DbHandle,
    publisher: Publisher,
    api: Arc<FakeApi>,
    events: tokio::sync::mpsc::Receiver<ProgressEvent>,
    event_tx: tokio::sync::mpsc::Sender<ProgressEvent>,
}

async fn harness(api: FakeApi) -> Harness {
    let db = DbHandle::open_in_memory().await.expect("open db");
    let api = Arc::new(api);
    let time: Arc<dyn Time> = Arc::new(FakeTime::new());
    let governor = Arc::new(RateGovernor::new(
        db.clone(),
        5_000.0,
        35_000.0,
        3_000.0,
        time.clone(),
    ));
    let config = PublisherConfig {
        pds_url: "http://127.0.0.1:1".to_owned(),
        handle: REPO.to_owned(),
        app_password: "password".to_owned(),
        collection: COLLECTION.to_owned(),
        timeout_secs: 1.0,
        governor,
    };
    let logger = Logger::new(Arc::new(CaptureSink::default()), Level::Error);
    let publisher =
        Publisher::with_api(config, api.clone(), logger, time).expect("build publisher");
    let (event_tx, events) = tokio::sync::mpsc::channel(128);
    Harness {
        db,
        publisher,
        api,
        events,
        event_tx,
    }
}

impl Harness {
    async fn seed(&self, cves: &[Value]) {
        self.db.upsert_cves(REPO, cves).await.expect("upsert");
        self.db.commit().await.expect("commit");
    }

    async fn mark_present(&self, cve_id: &str) {
        let mut mark = PublicationMark::new(REPO, cve_id, "present");
        mark.uri = Some(format!("at://{REPO}/{COLLECTION}/{cve_id}"));
        mark.cid = Some("bafypresent".to_owned());
        self.db.mark_publication(mark).await.expect("mark");
        self.db.commit().await.expect("commit");
    }

    async fn drain(&mut self, limit: usize) -> PublishStats {
        drain_pending(
            &self.publisher,
            &self.db,
            &PublishConfig {
                concurrency: 4,
                limit,
                order: Order::Cache,
            },
            self.event_tx.clone(),
        )
        .await
        .expect("drain")
    }

    fn drain_events(&mut self) -> Vec<ProgressEvent> {
        let mut events = Vec::new();
        while let Ok(event) = self.events.try_recv() {
            events.push(event);
        }
        events
    }
}

fn small_cve(identifier: &str) -> Value {
    let mut cve = fixture("small");
    cve["id"] = json!(identifier);
    cve
}

fn record_bytes(cve: &Value) -> u64 {
    let data = vuln_notif::db::model::cache_json(cve);
    prepare_row(
        CveRow {
            id: cve["id"].as_str().unwrap_or_default().to_owned(),
            data,
        },
        "2026-11-11T00:00:00.000Z",
    )
    .record_bytes
}

#[tokio::test]
async fn present_rows_are_marked_without_writes() {
    let cve = small_cve("CVE-2024-0001");
    let rkey = sanitize_rkey("CVE-2024-0001");
    let mut harness = harness(FakeApi::default().with_get(
        &rkey,
        ok(json!({
            "uri": format!("at://{REPO}/{COLLECTION}/{rkey}"),
            "cid": "bafypresent",
        })),
    ))
    .await;
    harness.seed(&[cve]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.present, stats.created, stats.failures), (1, 0, 0));
    assert_eq!(harness.api.counts(), (1, 0, 0, 0));
    assert_eq!(
        harness
            .db
            .get_cve_record(REPO, "CVE-2024-0001")
            .await
            .unwrap(),
        Some((
            format!("at://{REPO}/{COLLECTION}/{rkey}"),
            "bafypresent".to_owned()
        ))
    );
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!((counts.present, counts.pending), (1, 0));
}

#[tokio::test]
async fn missing_rows_use_apply_writes() {
    let cve = small_cve("CVE-2024-0002");
    let rkey = sanitize_rkey("CVE-2024-0002");
    let mut harness = harness(FakeApi::default()).await;
    harness.seed(std::slice::from_ref(&cve)).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.created, stats.failures, stats.apply_ops), (1, 0, 1));
    assert_eq!(stats.get_ops, 1);
    assert_eq!(stats.record_bytes_max, record_bytes(&cve));
    let (get_calls, create_calls, apply_calls, upload_calls) = harness.api.counts();
    assert_eq!(
        (get_calls, create_calls, apply_calls, upload_calls),
        (1, 0, 1, 0)
    );
    assert_eq!(
        harness
            .db
            .get_cve_record(REPO, "CVE-2024-0002")
            .await
            .unwrap(),
        Some((
            format!("at://{REPO}/{COLLECTION}/{rkey}"),
            "bafyapply".to_owned()
        ))
    );
}

#[tokio::test]
async fn cache_hits_make_no_calls() {
    let cve = small_cve("CVE-2024-0003");
    let mut harness = harness(FakeApi::default()).await;
    harness.seed(&[cve]).await;
    harness.mark_present("CVE-2024-0003").await;

    let stats = harness.drain(0).await;
    assert_eq!(stats.calls, 0);
    assert_eq!(harness.api.counts(), (0, 0, 0, 0));
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!(counts.pending, 0);
}

#[tokio::test]
async fn compact_too_large_is_skipped() {
    let mut harness = harness(FakeApi::default()).await;
    harness.seed(&[fixture("compact_too_large")]).await;

    let stats = harness.drain(0).await;
    assert_eq!(
        (stats.skipped, stats.oversized, stats.oversized_skipped),
        (1, 1, 1)
    );
    assert_eq!((stats.created, stats.failures), (0, 0));
    assert_eq!(harness.api.counts(), (1, 0, 0, 0));
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!((counts.skipped, counts.pending), (1, 0));
}

#[tokio::test]
async fn exists_race_becomes_present_after_fallback() {
    let mut harness = harness(
        FakeApi::default()
            .with_apply(error(409, "RecordAlreadyExists", "Record already exists"))
            .with_create(error(400, "RecordAlreadyExists", "Record already exists")),
    )
    .await;
    harness.seed(&[small_cve("CVE-2024-0004")]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.present, stats.created, stats.failures), (1, 0, 0));
    assert_eq!(stats.apply_ops, 1);
    let (_, create_calls, apply_calls, _) = harness.api.counts();
    assert_eq!((create_calls, apply_calls), (1, 1));
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!(counts.present, 1);
}

#[tokio::test]
async fn apply_writes_without_results_falls_back_per_row() {
    let mut harness = harness(FakeApi::default().with_apply(ok(json!({"results": []})))).await;
    harness.seed(&[small_cve("CVE-2024-0005")]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.created, stats.failures, stats.apply_ops), (1, 0, 1));
    let (_, create_calls, apply_calls, _) = harness.api.counts();
    assert_eq!((create_calls, apply_calls), (1, 1));
    assert_eq!(
        harness
            .db
            .get_cve_record(REPO, "CVE-2024-0005")
            .await
            .unwrap(),
        Some((
            format!("at://{REPO}/{COLLECTION}/CVE-2024-0005"),
            "bafycreated".to_owned()
        ))
    );
}

#[tokio::test]
async fn transient_create_retries_then_creates() {
    let mut harness = harness(
        FakeApi::default()
            .with_apply(error(400, "InvalidRequest", "fallback"))
            .with_create(error(500, "InternalServerError", "boom"))
            .with_create(ok(json!({
                "uri": format!("at://{REPO}/{COLLECTION}/CVE-2024-0006"),
                "cid": "bafyretried",
            }))),
    )
    .await;
    harness.seed(&[small_cve("CVE-2024-0006")]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.created, stats.failures, stats.retries), (1, 0, 1));
    let (_, create_calls, apply_calls, _) = harness.api.counts();
    assert_eq!((create_calls, apply_calls), (2, 1));
    assert_eq!(
        harness
            .db
            .get_cve_record(REPO, "CVE-2024-0006")
            .await
            .unwrap(),
        Some((
            format!("at://{REPO}/{COLLECTION}/CVE-2024-0006"),
            "bafyretried".to_owned()
        ))
    );
}

#[tokio::test]
async fn apply_writes_failure_falls_back_per_row() {
    let mut harness = harness(
        FakeApi::default()
            .with_apply(error(500, "InternalServerError", "boom"))
            .with_apply(error(500, "InternalServerError", "boom"))
            .with_apply(error(500, "InternalServerError", "boom")),
    )
    .await;
    harness.seed(&[small_cve("CVE-2024-0007")]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.created, stats.failures, stats.retries), (1, 0, 2));
    let (_, create_calls, apply_calls, _) = harness.api.counts();
    assert_eq!((create_calls, apply_calls), (1, 3));
    assert_eq!(stats.apply_ops, 1);
}

#[tokio::test]
async fn blob_rows_upload_then_create_pointer() {
    let cve = fixture("oversized");
    let prepared = prepare_row(
        CveRow {
            id: cve["id"].as_str().unwrap_or_default().to_owned(),
            data: vuln_notif::db::model::cache_json(&cve),
        },
        "2026-11-11T00:00:00.000Z",
    );
    assert_eq!(prepared.mode, "blob");
    let expected_blob = prepared.blob.clone().expect("blob");
    let mut harness = harness(FakeApi::default()).await;
    harness.seed(&[cve]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.created, stats.oversized, stats.blob_ops), (1, 1, 1));
    assert_eq!(stats.blob_bytes, expected_blob.len() as u64);
    let (get_calls, create_calls, apply_calls, upload_calls) = harness.api.counts();
    assert_eq!(
        (get_calls, create_calls, apply_calls, upload_calls),
        (1, 1, 0, 1)
    );

    let uploads = harness.api.uploads();
    assert_eq!(uploads.len(), 1);
    assert_eq!(uploads[0].0, "application/gzip");
    assert_eq!(uploads[0].1, expected_blob);

    let creates = harness.api.creates();
    assert_eq!(creates.len(), 1);
    assert_eq!(
        creates[0]["dataBlob"]["ref"]["$link"],
        "bafyreibvqn63urbv7lilphh4nsqh4d6yxj5xmbq7rc44ffi3euxsuxzpeu"
    );
    assert_eq!(creates[0]["dataBlob"]["mimeType"], "application/gzip");
    assert_eq!(creates[0]["id"], "CVE-2025-9999");
    assert!(creates[0].get("configurations").is_none());
}

#[tokio::test]
async fn stats_and_events_track_mixed_batches() {
    let present = small_cve("CVE-2024-0008");
    let missing = small_cve("CVE-2024-0009");
    let rkey = sanitize_rkey("CVE-2024-0008");
    let mut harness = harness(FakeApi::default().with_get(
        &rkey,
        ok(json!({
            "uri": format!("at://{REPO}/{COLLECTION}/{rkey}"),
            "cid": "bafypresent",
        })),
    ))
    .await;
    harness.seed(&[present, missing]).await;

    let stats = harness.drain(0).await;
    assert_eq!((stats.present, stats.created, stats.failures), (1, 1, 0));
    assert_eq!((stats.get_ops, stats.apply_ops), (2, 1));
    assert_eq!(stats.calls, 3);

    let events = harness.drain_events();
    assert!(matches!(
        events.first(),
        Some(ProgressEvent::PublishPhaseStarted { limit: 0 })
    ));
    assert!(matches!(
        events.get(1),
        Some(ProgressEvent::PublishBatchStarted { offset: 0, size: 2 })
    ));
    assert!(matches!(
        events.get(2),
        Some(ProgressEvent::PublishBatchCompleted {
            offset: 0,
            size: 2,
            missing: 1,
            created: 1,
            present: 1,
            failed: 0,
        })
    ));
    assert!(matches!(
        events.last(),
        Some(ProgressEvent::PublishPhaseCompleted { stats, error: None })
            if stats.created == 1 && stats.present == 1
    ));
}

#[tokio::test]
async fn limit_bounds_the_pending_batch() {
    let cves = [
        small_cve("CVE-2024-0010"),
        small_cve("CVE-2024-0011"),
        small_cve("CVE-2024-0012"),
    ];
    let mut harness = harness(FakeApi::default()).await;
    harness.seed(&cves).await;

    let stats = harness.drain(2).await;
    assert_eq!(stats.created, 2);
    assert_eq!(harness.api.counts().0, 2);
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!((counts.created, counts.pending), (2, 1));
}

#[tokio::test]
async fn failed_publish_records_attempt_and_backoff() {
    let id = "CVE-2024-0020";
    let mut api = FakeApi::default();
    for _ in 0..3 {
        api = api.with_apply(error(500, "InternalServerError", "apply boom"));
    }
    for _ in 0..3 {
        api = api.with_create(error(500, "InternalServerError", "create boom"));
    }
    let mut harness = harness(api).await;
    harness.seed(&[small_cve(id)]).await;

    let stats = harness.drain(0).await;
    assert_eq!(stats.failures, 1);
    let row = harness
        .db
        .get_publish_error(REPO, id)
        .await
        .unwrap()
        .expect("failed CVE has a retry row");
    assert_eq!(row.attempts, 1);
    assert!(
        row.last_error
            .as_deref()
            .is_some_and(|error| error.contains("InternalServerError")),
        "{:?}",
        row.last_error
    );
    assert!(row.next_retry_at.is_some());
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!((counts.present, counts.created, counts.pending), (0, 0, 1));
    assert!(harness.db.get_cve_record(REPO, id).await.unwrap().is_none());
}

#[tokio::test]
async fn due_retries_publish_and_clear_the_retry_row() {
    let id = "CVE-2024-0021";
    let harness = harness(FakeApi::default()).await;
    harness.seed(&[small_cve(id)]).await;
    harness
        .db
        .mark_publish_error(PublishErrorMark {
            repo_did: REPO.to_owned(),
            cve_id: id.to_owned(),
            attempts: 2,
            last_error: Some("previous failure".to_owned()),
            next_retry_at: Some("2000-01-01T00:00:00.000Z".to_owned()),
        })
        .await
        .unwrap();
    harness.db.commit().await.unwrap();

    let stats = drain_retries(
        &harness.publisher,
        &harness.db,
        &PublishConfig {
            concurrency: 4,
            limit: 0,
            order: Order::Cache,
        },
        harness.event_tx.clone(),
    )
    .await
    .expect("retry drains");

    assert_eq!((stats.retried, stats.retry_failed), (1, 0));
    assert_eq!(stats.created, 1);
    assert!(
        harness
            .db
            .get_publish_error(REPO, id)
            .await
            .unwrap()
            .is_none(),
        "successful retry clears the error row"
    );
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!((counts.created, counts.pending), (1, 0));
}

#[tokio::test]
async fn retries_before_backoff_are_not_attempted() {
    let id = "CVE-2024-0022";
    let harness = harness(FakeApi::default()).await;
    harness.seed(&[small_cve(id)]).await;
    harness
        .db
        .mark_publish_error(PublishErrorMark {
            repo_did: REPO.to_owned(),
            cve_id: id.to_owned(),
            attempts: 1,
            last_error: Some("previous failure".to_owned()),
            next_retry_at: Some("2999-01-01T00:00:00.000Z".to_owned()),
        })
        .await
        .unwrap();
    harness.db.commit().await.unwrap();

    let stats = drain_retries(
        &harness.publisher,
        &harness.db,
        &PublishConfig {
            concurrency: 4,
            limit: 0,
            order: Order::Cache,
        },
        harness.event_tx.clone(),
    )
    .await
    .expect("retry scan");

    assert_eq!((stats.retried, stats.retry_failed), (0, 0));
    assert_eq!(harness.api.counts(), (0, 0, 0, 0));
    assert!(
        harness
            .db
            .get_publish_error(REPO, id)
            .await
            .unwrap()
            .is_some()
    );
}

#[tokio::test]
async fn failed_retry_bumps_attempts_and_backoff() {
    let id = "CVE-2024-0023";
    let mut api = FakeApi::default();
    for _ in 0..3 {
        api = api.with_apply(error(500, "InternalServerError", "apply boom"));
    }
    for _ in 0..3 {
        api = api.with_create(error(500, "InternalServerError", "create boom"));
    }
    let harness = harness(api).await;
    harness.seed(&[small_cve(id)]).await;
    harness
        .db
        .mark_publish_error(PublishErrorMark {
            repo_did: REPO.to_owned(),
            cve_id: id.to_owned(),
            attempts: 1,
            last_error: Some("previous failure".to_owned()),
            next_retry_at: Some("2000-01-01T00:00:00.000Z".to_owned()),
        })
        .await
        .unwrap();
    harness.db.commit().await.unwrap();

    let stats = drain_retries(
        &harness.publisher,
        &harness.db,
        &PublishConfig {
            concurrency: 4,
            limit: 0,
            order: Order::Cache,
        },
        harness.event_tx.clone(),
    )
    .await
    .expect("retry drains");

    assert_eq!((stats.retried, stats.retry_failed), (1, 1));
    let row = harness
        .db
        .get_publish_error(REPO, id)
        .await
        .unwrap()
        .expect("failed retry stays queued");
    assert_eq!(row.attempts, 2);
    let counts = harness.db.counts(REPO).await.unwrap();
    assert_eq!(counts.pending, 1);
}