1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
pub const COLLECTION: &str = "com.nahdig.cve.nvdV2";
pub const RECORD_TYPE: &str = "com.nahdig.cve.nvdV2";
pub const ATPROTO_COLLECTION: &str = "com.nahdig.cve.atprotoV1";
pub const ATPROTO_TYPE: &str = "com.nahdig.cve.atprotoV1";
pub const SOFTWARE_COLLECTION: &str = "com.nahdig.software";
pub const SOFTWARE_TYPE: &str = "com.nahdig.software";
pub const PURL_COLLECTION: &str = "com.nahdig.purl";
pub const PURL_TYPE: &str = "com.nahdig.purl";
pub const PROJECT_COLLECTION: &str = "software.release.project";
pub const PROJECT_TYPE: &str = "software.release.project";
pub const MANIFEST_COLLECTION: &str = "software.release.manifest";
pub const MANIFEST_TYPE: &str = "software.release.manifest";
pub const VERSION_COLLECTION: &str = "software.release.version";
pub const VERSION_TYPE: &str = "software.release.version";
pub const CHANNEL_COLLECTION: &str = "software.release.channel";
pub const CHANNEL_TYPE: &str = "software.release.channel";
pub const STATUS_COLLECTION: &str = "software.release.status";
pub const STATUS_TYPE: &str = "software.release.status";
pub const DEPSDEV_URL: &str = "https://api.deps.dev";
pub const NVD_API_URL: &str = "https://services.nvd.nist.gov/rest/json/cves/2.0";
pub const DEFAULT_S3_PATH: &str = "vuln-notif/cve-cache.db";
pub const PURL2CPE_URL: &str =
"https://raw.githubusercontent.com/scanoss/purl2cpe/main/purl2cpe.db.zip";
pub const RELEASE_RECHECK_SECONDS: f64 = 86_400.0;
pub const RELEASE_ECOSYSTEM_PRIORITY: &[(&str, u8)] = &[
("npm", 0),
("pypi", 1),
("maven", 2),
("nuget", 3),
("gem", 4),
("cargo", 5),
("golang", 6),
];
pub const RELEASE_SYSTEMS: &[(&str, &str)] = &[
("npm", "NPM"),
("pypi", "PYPI"),
("maven", "MAVEN"),
("golang", "GO"),
("gem", "RUBYGEMS"),
("nuget", "NUGET"),
("cargo", "CARGO"),
];
pub const PUBLISH_BATCH: usize = 500;
pub const APPLY_WRITES_BATCH: usize = 200;
pub const CHECK_CONCURRENCY: usize = 32;
pub const RECORD_MAX: usize = 65_536;
pub const RECORD_TARGET: usize = 61_440;
pub const JSON_ENCODE_THRESHOLD: usize = 49_152;
pub const BLOB_MAX: usize = 5 * 1024 * 1024;
pub const BLOB_MIME: &str = "application/gzip";
pub const COMPACT_INDEX_FIELDS: &[&str] = &[
"id",
"sourceIdentifier",
"published",
"lastModified",
"vulnStatus",
];
pub const MAX_ATTEMPTS: u32 = 3;
pub const RETRY_DELAYS: [f64; 2] = [1.0, 2.0];
pub const RKEY_MAX_BYTES: usize = 512;
pub const NVD_TIMEOUT: f64 = 900.0;
pub const NVD_SOCK_TIMEOUT: f64 = 10.0;
pub const NVD_RETRY_DELAYS: [f64; 4] = [30.0, 60.0, 120.0, 300.0];
pub const PDS_WRITE_HOURLY: f64 = 5_000.0;
pub const PDS_WRITE_DAILY: f64 = 35_000.0;
pub const PDS_REQUESTS_PER_5MIN: f64 = 3_000.0;
pub const PDS_BUDGET_MARGIN: f64 = 0.9;
pub const PDS_REQUEST_WINDOW: f64 = 300.0;
pub const PDS_REQUEST_BURST: f64 = 100.0;
pub const PDS_WRITE_CREATE: i64 = 3;
pub const PDS_WRITE_PUT: i64 = 3;
pub const PDS_WRITE_UPDATE: i64 = 2;
pub const PDS_WRITE_DELETE: i64 = 1;
pub const MAX_RATE_LIMIT_ATTEMPTS: u32 = 5;
pub const RATE_LIMIT_BACKOFF_START: f64 = 60.0;
pub const RATE_LIMIT_BACKOFF_MAX: f64 = 1_800.0;
pub const CREATE_SESSION_ATTEMPTS: u32 = 3;
pub const STATUS_PENDING: &str = "pending";
pub const STATUS_PRESENT: &str = "present";
pub const STATUS_CREATED: &str = "created";
pub const STATUS_SKIPPED: &str = "skipped";
pub const RATE_BUDGET_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS rate_budget (
repo_did TEXT PRIMARY KEY,
hour_start REAL NOT NULL,
hour_points REAL NOT NULL,
day_start REAL NOT NULL,
day_points REAL NOT NULL,
updated_at TEXT NOT NULL
);
"#;
pub const SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS cves (
id TEXT PRIMARY KEY,
last_modified TEXT,
data TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS publications (
repo_did TEXT NOT NULL,
cve_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
uri TEXT,
cid TEXT,
checked_at TEXT,
published_at TEXT,
blob_cid TEXT,
blob_bytes INTEGER,
record_bytes INTEGER,
mode TEXT,
PRIMARY KEY (repo_did, cve_id)
);
CREATE INDEX IF NOT EXISTS publications_status_idx
ON publications(repo_did, status);
CREATE TABLE IF NOT EXISTS software (
repo_did TEXT NOT NULL,
slug TEXT NOT NULL,
part TEXT,
vendor TEXT,
product TEXT,
project_uri TEXT,
project_cid TEXT,
software_uri TEXT,
software_cid TEXT,
status TEXT NOT NULL DEFAULT 'pending',
updated_at TEXT,
PRIMARY KEY (repo_did, slug)
);
CREATE TABLE IF NOT EXISTS software_purls (
repo_did TEXT NOT NULL,
purl TEXT NOT NULL,
software_slug TEXT NOT NULL,
purl_uri TEXT,
purl_cid TEXT,
status TEXT NOT NULL DEFAULT 'pending',
updated_at TEXT,
PRIMARY KEY (repo_did, purl)
);
CREATE TABLE IF NOT EXISTS cve_atproto (
repo_did TEXT NOT NULL,
cve_id TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
uri TEXT,
cid TEXT,
updated_at TEXT,
PRIMARY KEY (repo_did, cve_id)
);
CREATE TABLE IF NOT EXISTS links (
repo_did TEXT NOT NULL,
src_uri TEXT NOT NULL,
src_collection TEXT NOT NULL,
src_rkey TEXT NOT NULL,
src_cid TEXT,
path TEXT NOT NULL,
subject_uri TEXT NOT NULL,
subject_cid TEXT,
indexed_at TEXT NOT NULL,
PRIMARY KEY (src_uri, path, subject_uri)
);
CREATE INDEX IF NOT EXISTS links_subject_idx
ON links(repo_did, subject_uri, src_collection, path);
CREATE INDEX IF NOT EXISTS links_source_idx
ON links(repo_did, src_uri);
CREATE TABLE IF NOT EXISTS release_projects (
repo_did TEXT NOT NULL,
slug TEXT NOT NULL,
purl TEXT NOT NULL,
system TEXT NOT NULL,
latest_version TEXT,
latest_version_raw TEXT,
project_uri TEXT,
project_cid TEXT,
manifest_uri TEXT,
manifest_cid TEXT,
version_uri TEXT,
version_cid TEXT,
channel_uri TEXT,
channel_cid TEXT,
status TEXT NOT NULL DEFAULT 'created',
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, slug)
);
CREATE TABLE IF NOT EXISTS release_versions (
repo_did TEXT NOT NULL,
project_slug TEXT NOT NULL,
version TEXT NOT NULL,
version_raw TEXT NOT NULL,
rkey TEXT NOT NULL,
uri TEXT,
cid TEXT,
manifest_uri TEXT,
manifest_cid TEXT,
status TEXT NOT NULL DEFAULT 'created',
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, project_slug, version)
);
CREATE TABLE IF NOT EXISTS release_channels (
repo_did TEXT NOT NULL,
project_slug TEXT NOT NULL,
name TEXT NOT NULL,
rkey TEXT NOT NULL,
uri TEXT,
cid TEXT,
version_uri TEXT,
version_cid TEXT,
status TEXT NOT NULL DEFAULT 'created',
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, project_slug, name)
);
CREATE TABLE IF NOT EXISTS release_status (
repo_did TEXT NOT NULL,
subject_uri TEXT NOT NULL,
status TEXT NOT NULL,
rkey TEXT NOT NULL,
uri TEXT,
cid TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, subject_uri, status)
);
CREATE TABLE IF NOT EXISTS software_identity (
repo_did TEXT NOT NULL,
raw_slug TEXT NOT NULL,
canonical_slug TEXT NOT NULL,
identity_key TEXT,
source TEXT,
confidence REAL,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, raw_slug)
);
CREATE TABLE IF NOT EXISTS project_aliases (
repo_did TEXT NOT NULL,
alias_key TEXT NOT NULL,
canonical_slug TEXT NOT NULL,
source TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, alias_key)
);
CREATE TABLE IF NOT EXISTS project_channels (
repo_did TEXT NOT NULL,
project_slug TEXT NOT NULL,
channel TEXT NOT NULL,
latest_version TEXT,
eol_at TEXT,
status TEXT NOT NULL DEFAULT 'active',
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, project_slug, channel)
);
CREATE TABLE IF NOT EXISTS release_artifacts (
repo_did TEXT NOT NULL,
source TEXT NOT NULL,
package TEXT NOT NULL,
version TEXT NOT NULL,
file_name TEXT,
url TEXT NOT NULL,
sha256 TEXT,
cid TEXT,
size INTEGER,
mime_type TEXT,
metadata_json TEXT,
fetched_at TEXT NOT NULL,
PRIMARY KEY (repo_did, source, package, version, url)
);
CREATE TABLE IF NOT EXISTS release_manifests (
repo_did TEXT NOT NULL,
project_slug TEXT NOT NULL,
manifest_key TEXT NOT NULL,
uri TEXT,
cid TEXT,
artifacts_json TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (repo_did, project_slug, manifest_key)
);
CREATE TABLE IF NOT EXISTS release_history_jobs (
repo_did TEXT NOT NULL,
project_slug TEXT NOT NULL,
version_raw TEXT NOT NULL,
version TEXT,
source TEXT,
purl TEXT,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
error TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, project_slug, version_raw)
);
CREATE INDEX IF NOT EXISTS release_history_jobs_status_idx
ON release_history_jobs(status, updated_at);
CREATE TABLE IF NOT EXISTS release_osv (
repo_did TEXT NOT NULL,
purl TEXT NOT NULL,
osv_id TEXT NOT NULL,
aliases_json TEXT,
affected_json TEXT,
modified TEXT,
fetched_at TEXT NOT NULL,
PRIMARY KEY (repo_did, purl, osv_id)
);
CREATE TABLE IF NOT EXISTS release_cve_links (
repo_did TEXT NOT NULL,
cve_id TEXT NOT NULL,
project_slug TEXT NOT NULL,
version_uri TEXT,
version_cid TEXT,
channel_uri TEXT,
channel_cid TEXT,
manifest_uri TEXT,
manifest_cid TEXT,
version_label TEXT,
affected_ranges TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, cve_id, project_slug)
);
CREATE INDEX IF NOT EXISTS release_cve_links_cve_idx
ON release_cve_links(repo_did, cve_id);
CREATE TABLE IF NOT EXISTS release_source_state (
repo_did TEXT NOT NULL,
source TEXT NOT NULL,
cursor TEXT,
etag TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, source)
);
CREATE TABLE IF NOT EXISTS release_status_events (
repo_did TEXT NOT NULL,
subject_uri TEXT NOT NULL,
rkey TEXT NOT NULL,
status TEXT NOT NULL,
kind TEXT,
reason TEXT,
cid TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (repo_did, rkey)
);
CREATE TABLE IF NOT EXISTS release_project_state (
repo_did TEXT NOT NULL,
slug TEXT NOT NULL,
canonical_purl TEXT,
source TEXT,
raw_versions_json TEXT,
checked_at TEXT,
unsupported_reason TEXT,
PRIMARY KEY (repo_did, slug)
);
CREATE TABLE IF NOT EXISTS release_version_meta (
repo_did TEXT NOT NULL,
project_slug TEXT NOT NULL,
version TEXT NOT NULL,
source TEXT,
published_at TEXT,
raw_version TEXT,
artifact_cid TEXT,
artifact_size INTEGER,
PRIMARY KEY (repo_did, project_slug, version)
);
CREATE TABLE IF NOT EXISTS identity_migration (
repo_did TEXT NOT NULL,
phase TEXT NOT NULL,
raw_slug TEXT NOT NULL,
canonical_slug TEXT,
action TEXT,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
error TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (repo_did, phase, raw_slug)
);
CREATE INDEX IF NOT EXISTS identity_migration_status_idx
ON identity_migration(status, updated_at);
CREATE TABLE IF NOT EXISTS link_repairs (
repo_did TEXT NOT NULL,
src_uri TEXT NOT NULL,
src_cid TEXT,
collection TEXT,
record_json TEXT NOT NULL,
attempts INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
next_retry_at TEXT,
PRIMARY KEY (repo_did, src_uri)
);
CREATE INDEX IF NOT EXISTS link_repairs_next_retry_idx
ON link_repairs(next_retry_at);
CREATE TABLE IF NOT EXISTS publish_errors (
repo_did TEXT NOT NULL,
cve_id TEXT NOT NULL,
attempts INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
next_retry_at TEXT,
PRIMARY KEY (repo_did, cve_id)
);
CREATE INDEX IF NOT EXISTS publish_errors_next_retry_idx
ON publish_errors(next_retry_at);
CREATE TABLE IF NOT EXISTS source_host_state (
host TEXT PRIMARY KEY,
window_start REAL NOT NULL,
requests INTEGER NOT NULL DEFAULT 0,
blocked_until TEXT
);
"#;
pub fn release_system(purl_type: &str) -> Option<&'static str> {
RELEASE_SYSTEMS
.iter()
.find_map(|(k, v)| (*k == purl_type).then_some(*v))
}
pub fn release_priority(purl_type: &str) -> Option<u8> {
RELEASE_ECOSYSTEM_PRIORITY
.iter()
.find_map(|(k, v)| (*k == purl_type).then_some(*v))
}