~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
261 lines · 8.3 KB Raw History
//! Record normalization and deterministic encodings.

use std::io::Write;
use std::sync::LazyLock;

use data_encoding::{BASE32_NOPAD, HEXLOWER};
use flate2::{Compression, GzBuilder};
use regex::Regex;
use serde_json::{Map, Value};
use sha2::{Digest, Sha256};

use crate::constants::{
    BLOB_MAX, COMPACT_INDEX_FIELDS, JSON_ENCODE_THRESHOLD, RECORD_MAX, RECORD_TARGET, RECORD_TYPE,
    RKEY_MAX_BYTES,
};
use crate::db::model::CveRow;

const TID_ALPHABET: &[u8; 32] = b"234567abcdefghijklmnopqrstuvwxyz";
const BLOB_REF_OVERHEAD: usize = 200;

/// reference `_ZONE_SUFFIX`: `(?:[Zz]|[+-]\d{2}:?\d{2})$`.
static ZONE_SUFFIX: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?:[Zz]|[+-]\d{2}:?\d{2})$").expect("zone regex"));

#[derive(Debug, Clone)]
pub struct PreparedRow {
    pub cve_id: String,
    pub record: Value,
    /// `normal` | `blob` | `skip`
    pub mode: String,
    pub blob: Option<Vec<u8>>,
    pub sha256: Option<String>,
    pub blob_bytes: u64,
    pub record_bytes: u64,
    pub reason: Option<String>,
}

pub fn sanitize_rkey(raw: &str) -> String {
    let filtered: String = raw
        .chars()
        .filter(|character| {
            character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | ':' | '~' | '-')
        })
        .take(RKEY_MAX_BYTES)
        .collect();
    if filtered.is_empty() {
        sha256_hex(raw.as_bytes())
    } else {
        filtered
    }
}

pub fn sha256_hex(input: &[u8]) -> String {
    HEXLOWER.encode(Sha256::digest(input).as_ref())
}

fn normalize_value(value: &Value) -> Value {
    match value {
        // ATProto has no float type; CVSS scores become decimal strings.
        Value::Number(number) if number.is_f64() => {
            Value::String(format!("{:?}", number.as_f64().unwrap_or_default()))
        }
        Value::Object(map) => Value::Object(
            map.iter()
                .map(|(key, item)| (key.clone(), normalize_value(item)))
                .collect(),
        ),
        Value::Array(items) => Value::Array(items.iter().map(normalize_value).collect()),
        other => other.clone(),
    }
}

fn normalize_timestamp(value: &str) -> String {
    if ZONE_SUFFIX.is_match(value) {
        value.to_owned()
    } else {
        format!("{value}Z")
    }
}

pub fn normalize_cve(cve: &Value) -> Value {
    let mut normalized = normalize_value(cve);
    if let Value::Object(map) = &mut normalized {
        for key in ["published", "lastModified"] {
            if let Some(Value::String(value)) = map.get_mut(key) {
                *value = normalize_timestamp(value);
            }
        }
    }
    normalized
}

/// `normal` | `blob` | `skip`, plus payload sizes mirroring `prepare_row`.
pub fn prepare_row(row: CveRow, created_at: &str) -> PreparedRow {
    let cve: Value = serde_json::from_str(&row.data).expect("cache row data is valid JSON");
    let normalized = normalize_cve(&cve);

    let mut record = normalized.clone();
    if let Value::Object(map) = &mut record {
        map.insert("$type".to_owned(), Value::String(RECORD_TYPE.to_owned()));
        map.insert("createdAt".to_owned(), Value::String(created_at.to_owned()));
    }

    let raw_size = row.data.chars().count() as u64;
    if raw_size < JSON_ENCODE_THRESHOLD as u64 {
        return PreparedRow {
            cve_id: row.id,
            record,
            mode: "normal".to_owned(),
            blob: None,
            sha256: None,
            blob_bytes: 0,
            record_bytes: raw_size,
            reason: None,
        };
    }

    if let Ok(normal_bytes) = cbor_bytes(&record)
        && normal_bytes.len() <= RECORD_TARGET
    {
        return PreparedRow {
            cve_id: row.id,
            record,
            mode: "normal".to_owned(),
            blob: None,
            sha256: None,
            blob_bytes: 0,
            record_bytes: normal_bytes.len() as u64,
            reason: None,
        };
    }

    let payload = canonical_bytes(&normalized).expect("canonical JSON encoding");
    let blob = gzip_bytes(&payload);
    let mut compact = Map::new();
    compact.insert("$type".to_owned(), Value::String(RECORD_TYPE.to_owned()));
    compact.insert("createdAt".to_owned(), Value::String(created_at.to_owned()));
    if let Value::Object(map) = &normalized {
        for key in COMPACT_INDEX_FIELDS {
            if let Some(value) = map.get(*key)
                && !value.is_null()
            {
                compact.insert((*key).to_owned(), value.clone());
            }
        }
    }
    let payload_sha = sha256_hex(&payload);
    compact.insert("dataSha256".to_owned(), Value::String(payload_sha.clone()));
    compact.insert(
        "dataBytes".to_owned(),
        Value::Number(serde_json::Number::from(payload.len() as u64)),
    );
    compact.insert("dataEncoding".to_owned(), Value::String("gzip".to_owned()));
    let compact = Value::Object(compact);

    // Reserve room for the blob ref, which is only known after upload.
    let blob_bytes = blob.len() as u64;
    if blob.len() > BLOB_MAX {
        return PreparedRow {
            cve_id: row.id,
            record,
            mode: "skip".to_owned(),
            blob: None,
            sha256: None,
            blob_bytes,
            record_bytes: 0,
            reason: Some("blob_too_large".to_owned()),
        };
    }
    let compact_bytes = cbor_bytes(&compact)
        .map(|bytes| bytes.len() + BLOB_REF_OVERHEAD)
        .unwrap_or(RECORD_MAX);
    if compact_bytes > RECORD_TARGET {
        return PreparedRow {
            cve_id: row.id,
            record,
            mode: "skip".to_owned(),
            blob: Some(blob),
            sha256: None,
            blob_bytes,
            record_bytes: compact_bytes as u64,
            reason: Some("compact_too_large".to_owned()),
        };
    }
    PreparedRow {
        cve_id: row.id,
        record: compact,
        mode: "blob".to_owned(),
        blob: Some(blob),
        sha256: Some(payload_sha),
        blob_bytes,
        record_bytes: compact_bytes as u64,
        reason: None,
    }
}

fn gzip_bytes(payload: &[u8]) -> Vec<u8> {
    let mut encoder = GzBuilder::new()
        .mtime(0)
        .operating_system(255)
        .write(Vec::new(), Compression::new(9));
    encoder.write_all(payload).expect("gzip write to memory");
    encoder.finish().expect("gzip finish to memory")
}

pub fn cbor_bytes(record: &Value) -> anyhow::Result<Vec<u8>> {
    Ok(atproto_dasl::to_vec_from_json(record)?)
}

fn sorted_value(value: &Value) -> Value {
    match value {
        Value::Object(map) => {
            let mut entries: Vec<_> = map.iter().collect();
            entries.sort_by(|left, right| left.0.cmp(right.0));
            let mut sorted = Map::new();
            for (key, item) in entries {
                sorted.insert(key.clone(), sorted_value(item));
            }
            Value::Object(sorted)
        }
        Value::Array(items) => Value::Array(items.iter().map(sorted_value).collect()),
        other => other.clone(),
    }
}

pub fn canonical_bytes(value: &Value) -> anyhow::Result<Vec<u8>> {
    Ok(serde_json::to_vec(&sorted_value(value))?)
}

/// DAG-CBOR CIDv1 (raw codec) as base32 lowercase without padding.
pub fn cidv1_raw(sha256_hex: &str) -> String {
    let digest = HEXLOWER.decode(sha256_hex.as_bytes()).unwrap_or_default();
    let mut bytes = Vec::with_capacity(4 + digest.len());
    bytes.extend_from_slice(&[0x01, 0x55, 0x12, 0x20]);
    bytes.extend_from_slice(&digest);
    format!("b{}", BASE32_NOPAD.encode(&bytes).to_ascii_lowercase())
}

/// 13-character TID-style rkey from sha256(parts joined by `|`).
pub fn digest_rkey(parts: &[&str]) -> String {
    let joined = parts.join("|");
    let digest = Sha256::digest(joined.as_bytes());
    let mut number = u64::from_be_bytes(digest[..8].try_into().expect("sha256 prefix"));
    let mut characters = [0u8; 13];
    for slot in &mut characters {
        *slot = TID_ALPHABET[(number & 0x1F) as usize];
        number >>= 5;
    }
    characters.reverse();
    // Draft-04 `tid` keys restrict the first character to `234567ab`; the
    // remaining twelve use the full base32-sortable alphabet. Content-digest
    // keys would otherwise be rejected roughly half the time.
    if !matches!(characters[0], b'2'..=b'7' | b'a' | b'b') {
        characters[0] = b'2';
    }
    String::from_utf8(characters.to_vec()).expect("tid alphabet is ascii")
}

pub fn purl_rkey(purl: &str) -> String {
    sha256_hex(purl.as_bytes())
}