~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
419 lines · 11.7 KB Raw History
pub const COLLECTION: &str = "com.nahdig.cve.nvdV2";
pub const RECORD_TYPE: &str = "com.nahdig.cve.nvdV2";
pub const ATPROTO_COLLECTION: &str = "com.nahdig.cve.atprotoV1";
pub const ATPROTO_TYPE: &str = "com.nahdig.cve.atprotoV1";
pub const SOFTWARE_COLLECTION: &str = "com.nahdig.software";
pub const SOFTWARE_TYPE: &str = "com.nahdig.software";
pub const PURL_COLLECTION: &str = "com.nahdig.purl";
pub const PURL_TYPE: &str = "com.nahdig.purl";
pub const PROJECT_COLLECTION: &str = "software.release.project";
pub const PROJECT_TYPE: &str = "software.release.project";
pub const MANIFEST_COLLECTION: &str = "software.release.manifest";
pub const MANIFEST_TYPE: &str = "software.release.manifest";
pub const VERSION_COLLECTION: &str = "software.release.version";
pub const VERSION_TYPE: &str = "software.release.version";
pub const CHANNEL_COLLECTION: &str = "software.release.channel";
pub const CHANNEL_TYPE: &str = "software.release.channel";
pub const STATUS_COLLECTION: &str = "software.release.status";
pub const STATUS_TYPE: &str = "software.release.status";

pub const DEPSDEV_URL: &str = "https://api.deps.dev";
pub const NVD_API_URL: &str = "https://services.nvd.nist.gov/rest/json/cves/2.0";
pub const DEFAULT_S3_PATH: &str = "vuln-notif/cve-cache.db";
pub const PURL2CPE_URL: &str =
    "https://raw.githubusercontent.com/scanoss/purl2cpe/main/purl2cpe.db.zip";

pub const RELEASE_RECHECK_SECONDS: f64 = 86_400.0;
pub const RELEASE_ECOSYSTEM_PRIORITY: &[(&str, u8)] = &[
    ("npm", 0),
    ("pypi", 1),
    ("maven", 2),
    ("nuget", 3),
    ("gem", 4),
    ("cargo", 5),
    ("golang", 6),
];
pub const RELEASE_SYSTEMS: &[(&str, &str)] = &[
    ("npm", "NPM"),
    ("pypi", "PYPI"),
    ("maven", "MAVEN"),
    ("golang", "GO"),
    ("gem", "RUBYGEMS"),
    ("nuget", "NUGET"),
    ("cargo", "CARGO"),
];

pub const PUBLISH_BATCH: usize = 500;
pub const APPLY_WRITES_BATCH: usize = 200;
pub const CHECK_CONCURRENCY: usize = 32;
pub const RECORD_MAX: usize = 65_536;
pub const RECORD_TARGET: usize = 61_440;
pub const JSON_ENCODE_THRESHOLD: usize = 49_152;
pub const BLOB_MAX: usize = 5 * 1024 * 1024;
pub const BLOB_MIME: &str = "application/gzip";
pub const COMPACT_INDEX_FIELDS: &[&str] = &[
    "id",
    "sourceIdentifier",
    "published",
    "lastModified",
    "vulnStatus",
];
pub const MAX_ATTEMPTS: u32 = 3;
pub const RETRY_DELAYS: [f64; 2] = [1.0, 2.0];
pub const RKEY_MAX_BYTES: usize = 512;
pub const NVD_TIMEOUT: f64 = 900.0;
pub const NVD_SOCK_TIMEOUT: f64 = 10.0;
pub const NVD_RETRY_DELAYS: [f64; 4] = [30.0, 60.0, 120.0, 300.0];
pub const PDS_WRITE_HOURLY: f64 = 5_000.0;
pub const PDS_WRITE_DAILY: f64 = 35_000.0;
pub const PDS_REQUESTS_PER_5MIN: f64 = 3_000.0;
pub const PDS_BUDGET_MARGIN: f64 = 0.9;
pub const PDS_REQUEST_WINDOW: f64 = 300.0;
pub const PDS_REQUEST_BURST: f64 = 100.0;
pub const PDS_WRITE_CREATE: i64 = 3;
pub const PDS_WRITE_PUT: i64 = 3;
pub const PDS_WRITE_UPDATE: i64 = 2;
pub const PDS_WRITE_DELETE: i64 = 1;
pub const MAX_RATE_LIMIT_ATTEMPTS: u32 = 5;
pub const RATE_LIMIT_BACKOFF_START: f64 = 60.0;
pub const RATE_LIMIT_BACKOFF_MAX: f64 = 1_800.0;
pub const CREATE_SESSION_ATTEMPTS: u32 = 3;
pub const STATUS_PENDING: &str = "pending";
pub const STATUS_PRESENT: &str = "present";
pub const STATUS_CREATED: &str = "created";
pub const STATUS_SKIPPED: &str = "skipped";

pub const RATE_BUDGET_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS rate_budget (
  repo_did TEXT PRIMARY KEY,
  hour_start REAL NOT NULL,
  hour_points REAL NOT NULL,
  day_start REAL NOT NULL,
  day_points REAL NOT NULL,
  updated_at TEXT NOT NULL
);
"#;

pub const SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS meta (
  key TEXT PRIMARY KEY,
  value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS cves (
  id TEXT PRIMARY KEY,
  last_modified TEXT,
  data TEXT NOT NULL,
  updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS publications (
  repo_did TEXT NOT NULL,
  cve_id TEXT NOT NULL,
  status TEXT NOT NULL DEFAULT 'pending',
  uri TEXT,
  cid TEXT,
  checked_at TEXT,
  published_at TEXT,
  blob_cid TEXT,
  blob_bytes INTEGER,
  record_bytes INTEGER,
  mode TEXT,
  PRIMARY KEY (repo_did, cve_id)
);
CREATE INDEX IF NOT EXISTS publications_status_idx
  ON publications(repo_did, status);
CREATE TABLE IF NOT EXISTS software (
  repo_did TEXT NOT NULL,
  slug TEXT NOT NULL,
  part TEXT,
  vendor TEXT,
  product TEXT,
  project_uri TEXT,
  project_cid TEXT,
  software_uri TEXT,
  software_cid TEXT,
  status TEXT NOT NULL DEFAULT 'pending',
  updated_at TEXT,
  PRIMARY KEY (repo_did, slug)
);
CREATE TABLE IF NOT EXISTS software_purls (
  repo_did TEXT NOT NULL,
  purl TEXT NOT NULL,
  software_slug TEXT NOT NULL,
  purl_uri TEXT,
  purl_cid TEXT,
  status TEXT NOT NULL DEFAULT 'pending',
  updated_at TEXT,
  PRIMARY KEY (repo_did, purl)
);
CREATE TABLE IF NOT EXISTS cve_atproto (
  repo_did TEXT NOT NULL,
  cve_id TEXT NOT NULL,
  status TEXT NOT NULL DEFAULT 'pending',
  uri TEXT,
  cid TEXT,
  updated_at TEXT,
  PRIMARY KEY (repo_did, cve_id)
);
CREATE TABLE IF NOT EXISTS links (
  repo_did TEXT NOT NULL,
  src_uri TEXT NOT NULL,
  src_collection TEXT NOT NULL,
  src_rkey TEXT NOT NULL,
  src_cid TEXT,
  path TEXT NOT NULL,
  subject_uri TEXT NOT NULL,
  subject_cid TEXT,
  indexed_at TEXT NOT NULL,
  PRIMARY KEY (src_uri, path, subject_uri)
);
CREATE INDEX IF NOT EXISTS links_subject_idx
  ON links(repo_did, subject_uri, src_collection, path);
CREATE INDEX IF NOT EXISTS links_source_idx
  ON links(repo_did, src_uri);
CREATE TABLE IF NOT EXISTS release_projects (
  repo_did TEXT NOT NULL,
  slug TEXT NOT NULL,
  purl TEXT NOT NULL,
  system TEXT NOT NULL,
  latest_version TEXT,
  latest_version_raw TEXT,
  project_uri TEXT,
  project_cid TEXT,
  manifest_uri TEXT,
  manifest_cid TEXT,
  version_uri TEXT,
  version_cid TEXT,
  channel_uri TEXT,
  channel_cid TEXT,
  status TEXT NOT NULL DEFAULT 'created',
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, slug)
);
CREATE TABLE IF NOT EXISTS release_versions (
  repo_did TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  version TEXT NOT NULL,
  version_raw TEXT NOT NULL,
  rkey TEXT NOT NULL,
  uri TEXT,
  cid TEXT,
  manifest_uri TEXT,
  manifest_cid TEXT,
  status TEXT NOT NULL DEFAULT 'created',
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, project_slug, version)
);
CREATE TABLE IF NOT EXISTS release_channels (
  repo_did TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  name TEXT NOT NULL,
  rkey TEXT NOT NULL,
  uri TEXT,
  cid TEXT,
  version_uri TEXT,
  version_cid TEXT,
  status TEXT NOT NULL DEFAULT 'created',
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, project_slug, name)
);
CREATE TABLE IF NOT EXISTS release_status (
  repo_did TEXT NOT NULL,
  subject_uri TEXT NOT NULL,
  status TEXT NOT NULL,
  rkey TEXT NOT NULL,
  uri TEXT,
  cid TEXT,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, subject_uri, status)
);
CREATE TABLE IF NOT EXISTS software_identity (
  repo_did TEXT NOT NULL,
  raw_slug TEXT NOT NULL,
  canonical_slug TEXT NOT NULL,
  identity_key TEXT,
  source TEXT,
  confidence REAL,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, raw_slug)
);
CREATE TABLE IF NOT EXISTS project_aliases (
  repo_did TEXT NOT NULL,
  alias_key TEXT NOT NULL,
  canonical_slug TEXT NOT NULL,
  source TEXT,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, alias_key)
);
CREATE TABLE IF NOT EXISTS project_channels (
  repo_did TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  channel TEXT NOT NULL,
  latest_version TEXT,
  eol_at TEXT,
  status TEXT NOT NULL DEFAULT 'active',
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, project_slug, channel)
);
CREATE TABLE IF NOT EXISTS release_artifacts (
  repo_did TEXT NOT NULL,
  source TEXT NOT NULL,
  package TEXT NOT NULL,
  version TEXT NOT NULL,
  file_name TEXT,
  url TEXT NOT NULL,
  sha256 TEXT,
  cid TEXT,
  size INTEGER,
  mime_type TEXT,
  metadata_json TEXT,
  fetched_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, source, package, version, url)
);
CREATE TABLE IF NOT EXISTS release_manifests (
  repo_did TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  manifest_key TEXT NOT NULL,
  uri TEXT,
  cid TEXT,
  artifacts_json TEXT,
  created_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, project_slug, manifest_key)
);
CREATE TABLE IF NOT EXISTS release_history_jobs (
  repo_did TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  version_raw TEXT NOT NULL,
  version TEXT,
  source TEXT,
  purl TEXT,
  status TEXT NOT NULL DEFAULT 'pending',
  attempts INTEGER NOT NULL DEFAULT 0,
  error TEXT,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, project_slug, version_raw)
);
CREATE INDEX IF NOT EXISTS release_history_jobs_status_idx
  ON release_history_jobs(status, updated_at);
CREATE TABLE IF NOT EXISTS release_osv (
  repo_did TEXT NOT NULL,
  purl TEXT NOT NULL,
  osv_id TEXT NOT NULL,
  aliases_json TEXT,
  affected_json TEXT,
  modified TEXT,
  fetched_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, purl, osv_id)
);
CREATE TABLE IF NOT EXISTS release_cve_links (
  repo_did TEXT NOT NULL,
  cve_id TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  version_uri TEXT,
  version_cid TEXT,
  channel_uri TEXT,
  channel_cid TEXT,
  manifest_uri TEXT,
  manifest_cid TEXT,
  version_label TEXT,
  affected_ranges TEXT,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, cve_id, project_slug)
);
CREATE INDEX IF NOT EXISTS release_cve_links_cve_idx
  ON release_cve_links(repo_did, cve_id);
CREATE TABLE IF NOT EXISTS release_source_state (
  repo_did TEXT NOT NULL,
  source TEXT NOT NULL,
  cursor TEXT,
  etag TEXT,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, source)
);
CREATE TABLE IF NOT EXISTS release_status_events (
  repo_did TEXT NOT NULL,
  subject_uri TEXT NOT NULL,
  rkey TEXT NOT NULL,
  status TEXT NOT NULL,
  kind TEXT,
  reason TEXT,
  cid TEXT,
  created_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, rkey)
);
CREATE TABLE IF NOT EXISTS release_project_state (
  repo_did TEXT NOT NULL,
  slug TEXT NOT NULL,
  canonical_purl TEXT,
  source TEXT,
  raw_versions_json TEXT,
  checked_at TEXT,
  unsupported_reason TEXT,
  PRIMARY KEY (repo_did, slug)
);
CREATE TABLE IF NOT EXISTS release_version_meta (
  repo_did TEXT NOT NULL,
  project_slug TEXT NOT NULL,
  version TEXT NOT NULL,
  source TEXT,
  published_at TEXT,
  raw_version TEXT,
  artifact_cid TEXT,
  artifact_size INTEGER,
  PRIMARY KEY (repo_did, project_slug, version)
);
CREATE TABLE IF NOT EXISTS identity_migration (
  repo_did TEXT NOT NULL,
  phase TEXT NOT NULL,
  raw_slug TEXT NOT NULL,
  canonical_slug TEXT,
  action TEXT,
  status TEXT NOT NULL DEFAULT 'pending',
  attempts INTEGER NOT NULL DEFAULT 0,
  error TEXT,
  updated_at TEXT NOT NULL,
  PRIMARY KEY (repo_did, phase, raw_slug)
);
CREATE INDEX IF NOT EXISTS identity_migration_status_idx
  ON identity_migration(status, updated_at);
CREATE TABLE IF NOT EXISTS link_repairs (
  repo_did TEXT NOT NULL,
  src_uri TEXT NOT NULL,
  src_cid TEXT,
  collection TEXT,
  record_json TEXT NOT NULL,
  attempts INTEGER NOT NULL DEFAULT 0,
  last_error TEXT,
  next_retry_at TEXT,
  PRIMARY KEY (repo_did, src_uri)
);
CREATE INDEX IF NOT EXISTS link_repairs_next_retry_idx
  ON link_repairs(next_retry_at);
CREATE TABLE IF NOT EXISTS publish_errors (
  repo_did TEXT NOT NULL,
  cve_id TEXT NOT NULL,
  attempts INTEGER NOT NULL DEFAULT 0,
  last_error TEXT,
  next_retry_at TEXT,
  PRIMARY KEY (repo_did, cve_id)
);
CREATE INDEX IF NOT EXISTS publish_errors_next_retry_idx
  ON publish_errors(next_retry_at);
CREATE TABLE IF NOT EXISTS source_host_state (
  host TEXT PRIMARY KEY,
  window_start REAL NOT NULL,
  requests INTEGER NOT NULL DEFAULT 0,
  blocked_until TEXT
);
"#;

pub fn release_system(purl_type: &str) -> Option<&'static str> {
    RELEASE_SYSTEMS
        .iter()
        .find_map(|(k, v)| (*k == purl_type).then_some(*v))
}

pub fn release_priority(purl_type: &str) -> Option<u8> {
    RELEASE_ECOSYSTEM_PRIORITY
        .iter()
        .find_map(|(k, v)| (*k == purl_type).then_some(*v))
}