~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
221 lines · 7.6 KB Raw History
//! End-to-end blob publish against a live local PDS.
//!
//! Needs `ATP_PDS_URL`, `ATP_HANDLE`, `ATP_APP_PASSWORD` in the environment
//! (the variables `scripts/create_local_account.sh` prints). Without them the
//! test skips with a message instead of failing, so `cargo test` on a machine
//! with no local PDS stays green and `scripts/blob-e2e.sh` decides when to run
//! it for real.
//!
//! The blob flow exercised by `scripts/blob-e2e.sh`:
//! an oversized CVE goes into a temp cache DB, `records::prepare_row` turns it
//! into a compact pointer record plus a gzip blob, `Publisher::create_large_row`
//! uploads both, and `listRecords` + `com.atproto.sync.getBlob` read them back.

use std::io::Read;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use chrono::Utc;
use serde_json::{Value, json};
use vuln_notif::config::Order;
use vuln_notif::constants::{
    BLOB_MIME, COLLECTION, PDS_REQUESTS_PER_5MIN, PDS_WRITE_DAILY, PDS_WRITE_HOURLY, RECORD_TARGET,
};
use vuln_notif::db::DbHandle;
use vuln_notif::pds::{Publisher, PublisherConfig, RateGovernor};
use vuln_notif::records;
use vuln_notif::time::{RealTime, created_at};

const SKIP: &str = "SKIP e2e_blob: ATP_PDS_URL, ATP_HANDLE and ATP_APP_PASSWORD must be set \
(start scripts/local_plc.sh, scripts/local_atproto_pds.sh, \
scripts/create_local_account.sh, then export the printed ATP_* values)";

fn env(name: &str) -> Option<String> {
    std::env::var(name)
        .ok()
        .map(|value| value.trim().to_string())
        .filter(|value| !value.is_empty())
}

#[tokio::test(flavor = "multi_thread")]
async fn oversized_cve_publishes_compact_record_and_gzip_blob() {
    let (Some(pds_url), Some(handle), Some(app_password)) = (
        env("ATP_PDS_URL"),
        env("ATP_HANDLE"),
        env("ATP_APP_PASSWORD"),
    ) else {
        eprintln!("{SKIP}");
        return;
    };
    let base = pds_url.trim_end_matches('/').to_string();

    // The captured oversized fixture is 260 KB of `configurations`; a unique
    // id keeps repeated runs from colliding on the record key.
    let fixture = std::fs::read_to_string(concat!(
        env!("CARGO_MANIFEST_DIR"),
        "/tests/fixtures/records/inputs/oversized.json"
    ))
    .expect("oversized fixture");
    let mut cve: Value = serde_json::from_str(&fixture).expect("oversized fixture JSON");
    let stamp = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .expect("clock")
        .as_nanos();
    let cve_id = format!("CVE-2099-{stamp}");
    cve["id"] = json!(cve_id);

    let tmp = tempfile::TempDir::new().expect("tempdir");
    let db = DbHandle::open(tmp.path().join("cache.db"))
        .await
        .expect("temp cache DB");
    let governor = Arc::new(RateGovernor::new(
        db.clone(),
        PDS_WRITE_HOURLY,
        PDS_WRITE_DAILY,
        PDS_REQUESTS_PER_5MIN,
        Arc::new(RealTime),
    ));
    let publisher = Publisher::new(PublisherConfig {
        pds_url: base.clone(),
        handle,
        app_password,
        collection: COLLECTION.to_string(),
        timeout_secs: 60.0,
        governor,
    })
    .expect("publisher");
    publisher.login().await.expect("PDS login");
    let repo = publisher.repo().to_string();

    db.upsert_cves(&repo, std::slice::from_ref(&cve))
        .await
        .expect("cache the CVE");
    let rows = db
        .pending(&repo, 1, Order::Cache)
        .await
        .expect("pending rows");
    assert_eq!(rows.len(), 1, "the fixture is the only pending CVE");
    assert_eq!(rows[0].id, cve_id);

    let prepared = records::prepare_row(rows[0].clone(), &created_at(Utc::now()));
    assert_eq!(
        prepared.mode,
        "blob",
        "260 KB of configurations must publish as a blob: {}",
        prepared.reason.as_deref().unwrap_or("no reason")
    );
    let payload = prepared.blob.as_ref().expect("blob payload").clone();
    assert_eq!(prepared.blob_bytes, payload.len() as u64);

    let canonical =
        records::canonical_bytes(&records::normalize_cve(&cve)).expect("canonical payload bytes");
    let canonical_sha = records::sha256_hex(&canonical);
    assert_eq!(prepared.sha256.as_deref(), Some(canonical_sha.as_str()));

    let outcome = publisher
        .create_large_row(&prepared)
        .await
        .expect("create_large_row");
    assert!(outcome.created, "pointer record created: {outcome:?}");
    assert!(
        outcome.uri.is_some() && outcome.cid.is_some(),
        "publish returns the record address: {outcome:?}"
    );
    assert_eq!(outcome.blob_bytes, payload.len() as u64);
    assert!(
        outcome.record_bytes > 0 && outcome.record_bytes <= RECORD_TARGET as u64,
        "compact record stays under {RECORD_TARGET} bytes: {outcome:?}"
    );

    let http = reqwest::Client::builder()
        .timeout(Duration::from_secs(60))
        .build()
        .expect("http client");

    let listing: Value = http
        .get(format!("{base}/xrpc/com.atproto.repo.listRecords"))
        .query(&[
            ("repo", repo.as_str()),
            ("collection", COLLECTION),
            ("limit", "100"),
        ])
        .send()
        .await
        .expect("listRecords request")
        .error_for_status()
        .expect("listRecords status")
        .json()
        .await
        .expect("listRecords JSON");
    let record = listing["records"]
        .as_array()
        .expect("records array")
        .iter()
        .find(|entry| entry["value"]["id"].as_str() == Some(cve_id.as_str()))
        .unwrap_or_else(|| panic!("{cve_id} is not in listRecords: {listing}"));
    let value = &record["value"];

    assert!(
        value.get("configurations").is_none(),
        "the pointer record must not carry configurations: {value}"
    );
    assert_eq!(value["dataEncoding"], "gzip");
    let blob = &value["dataBlob"];
    assert_eq!(blob["mimeType"], BLOB_MIME);
    let blob_cid = blob["ref"]["$link"].as_str().expect("dataBlob.ref.$link");
    let blob_size = blob["size"].as_u64().expect("dataBlob.size");
    assert_eq!(
        blob_size,
        payload.len() as u64,
        "blob size is the gzip size"
    );
    assert_eq!(
        value["dataBytes"].as_u64(),
        Some(canonical.len() as u64),
        "dataBytes is the canonical payload length"
    );
    assert_eq!(
        value["dataSha256"].as_str(),
        Some(canonical_sha.as_str()),
        "dataSha256 covers the canonical payload"
    );

    let cbor = records::cbor_bytes(value).expect("compact record CBOR");
    assert!(
        cbor.len() <= RECORD_TARGET,
        "compact record CBOR is {} bytes, over the {RECORD_TARGET} target",
        cbor.len()
    );

    let response = http
        .get(format!("{base}/xrpc/com.atproto.sync.getBlob"))
        .query(&[("did", repo.as_str()), ("cid", blob_cid)])
        .send()
        .await
        .expect("getBlob request");
    assert_eq!(response.status(), 200, "getBlob must serve the blob");
    let gzipped = response.bytes().await.expect("getBlob bytes");
    assert_eq!(
        gzipped.len() as u64,
        blob_size,
        "served gzip matches blob.size"
    );

    let mut decoder = flate2::read::GzDecoder::new(&gzipped[..]);
    let mut served = Vec::new();
    decoder
        .read_to_end(&mut served)
        .expect("gzip-decode the served blob");
    assert_eq!(
        served, canonical,
        "served gzip decodes to the canonical bytes"
    );
    let served_json: Value = serde_json::from_slice(&served).expect("canonical payload is JSON");
    assert_eq!(
        served_json["configurations"], cve["configurations"],
        "the blob carries the record the pointer elides"
    );

    db.close().await.expect("close cache DB");
    publisher.close().await.expect("close publisher");
}