1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
//! Cycle and run metrics.
use std::collections::BTreeMap;
use serde::Serialize;
use serde_json::{Map, Value, json};
/// Field names and rounding for cycle stats.
#[derive(Debug, Clone, Default, Serialize)]
pub struct CycleStats {
pub cycle: u64,
pub started_at: String,
pub repo_did: String,
pub duration_seconds: f64,
pub dry_run: bool,
pub limit: u64,
pub concurrency: u64,
pub nvd_requests: u64,
pub nvd_cves: u64,
pub nvd_fetch_seconds: f64,
pub nvd_retries: u64,
pub nvd_key_configured: bool,
pub pds_budget_hour_remaining: f64,
pub pds_budget_day_remaining: f64,
pub pds_rate_limit_waits: u64,
pub pds_rate_limit_seconds: f64,
pub pds_requests_5min: usize,
pub cache_hits: u64,
pub cache_new: u64,
pub cache_updated: u64,
pub cache_pending_before: i64,
pub cache_pending_after: i64,
pub cache_total: i64,
pub pds_login_calls: u64,
pub pds_get_ops: u64,
pub pds_create_ops: u64,
pub pds_apply_ops: u64,
pub pds_blob_ops: u64,
pub pds_blob_bytes: u64,
pub pds_http_calls: u64,
pub pds_present: u64,
pub pds_created: u64,
pub pds_failures: u64,
pub pds_retries: u64,
pub pds_skipped: u64,
pub pds_seconds_total: f64,
pub pds_seconds_avg: f64,
pub pds_seconds_max: f64,
pub records_created: u64,
pub records_present: u64,
pub records_skipped: u64,
pub records_failed: u64,
pub records_oversized: u64,
pub records_oversized_skipped: u64,
pub record_bytes_max: u64,
pub records_per_second: f64,
pub publish_retried: u64,
pub publish_retry_failed: u64,
pub link_repair_attempted: u64,
pub link_repair_repaired: u64,
pub link_repair_failed: u64,
pub link_repair_deferred: u64,
pub software_cves: u64,
pub software_created: u64,
pub purl_created: u64,
pub atproto_created: u64,
pub software_failures: u64,
pub s3_download_bytes: u64,
pub s3_upload_bytes: u64,
pub s3_download_seconds: f64,
pub s3_upload_seconds: f64,
pub cursor_before: Option<String>,
pub cursor_after: Option<String>,
pub release_mode: String,
pub release_projects_checked: u64,
pub release_projects_created: u64,
pub release_versions_created: u64,
pub release_channels_created: u64,
pub release_statuses_created: u64,
pub release_unchanged: u64,
pub release_skipped: u64,
pub release_failures: u64,
pub release_affected_projects_checked: u64,
pub release_affected_projects_linked: u64,
pub release_affected_versions_planned: u64,
pub release_affected_versions_created: u64,
pub release_affected_versions_label_only: u64,
pub release_affected_cves_linked: u64,
pub release_affected_unchanged: u64,
pub release_affected_skipped: u64,
pub release_affected_failures: u64,
pub release_affected_unsupported: u64,
pub release_history_projects_checked: u64,
pub release_history_projects_skipped: u64,
pub release_history_unsupported: u64,
pub release_history_enqueued: u64,
pub release_history_processed: u64,
pub release_history_created: u64,
pub release_history_skipped: u64,
pub release_history_failed: u64,
pub release_history_reused: u64,
pub release_extras_projects_checked: u64,
pub release_extras_projects_created: u64,
pub release_extras_versions_created: u64,
pub release_extras_manifests_created: u64,
pub release_extras_statuses_created: u64,
pub release_extras_unchanged: u64,
pub release_extras_skipped: u64,
pub release_extras_unsupported: u64,
pub release_extras_failures: u64,
pub artifact_downloads: u64,
pub artifact_bytes: u64,
pub artifact_cache_hits: u64,
pub artifact_mirror_puts: u64,
pub artifact_errors: u64,
}
impl CycleStats {
pub fn to_value(&self) -> Value {
serde_json::to_value(self).unwrap_or(Value::Null)
}
}
/// Keys summed across cycles.
pub const SUM_KEYS: &[&str] = &[
"duration_seconds",
"nvd_requests",
"nvd_cves",
"nvd_fetch_seconds",
"nvd_retries",
"cache_hits",
"cache_new",
"cache_updated",
"pds_login_calls",
"pds_get_ops",
"pds_create_ops",
"pds_apply_ops",
"pds_blob_ops",
"pds_blob_bytes",
"pds_http_calls",
"pds_present",
"pds_created",
"pds_failures",
"pds_retries",
"pds_skipped",
"pds_rate_limit_waits",
"pds_rate_limit_seconds",
"pds_seconds_total",
"records_created",
"records_present",
"records_skipped",
"records_failed",
"records_oversized",
"records_oversized_skipped",
"publish_retried",
"publish_retry_failed",
"link_repair_attempted",
"link_repair_repaired",
"link_repair_failed",
"link_repair_deferred",
"software_cves",
"software_created",
"purl_created",
"atproto_created",
"software_failures",
"s3_download_bytes",
"s3_upload_bytes",
"s3_download_seconds",
"s3_upload_seconds",
"release_projects_checked",
"release_projects_created",
"release_versions_created",
"release_channels_created",
"release_statuses_created",
"release_unchanged",
"release_skipped",
"release_failures",
"release_affected_projects_checked",
"release_affected_projects_linked",
"release_affected_versions_planned",
"release_affected_versions_created",
"release_affected_versions_label_only",
"release_affected_cves_linked",
"release_affected_unchanged",
"release_affected_skipped",
"release_affected_failures",
"release_affected_unsupported",
"release_history_projects_checked",
"release_history_projects_skipped",
"release_history_unsupported",
"release_history_enqueued",
"release_history_processed",
"release_history_created",
"release_history_skipped",
"release_history_failed",
"release_history_reused",
"release_extras_projects_checked",
"release_extras_projects_created",
"release_extras_versions_created",
"release_extras_manifests_created",
"release_extras_statuses_created",
"release_extras_unchanged",
"release_extras_skipped",
"release_extras_unsupported",
"release_extras_failures",
"artifact_downloads",
"artifact_bytes",
"artifact_cache_hits",
"artifact_mirror_puts",
"artifact_errors",
"errors",
];
/// Keys whose maximum is kept across cycles.
pub const MAX_KEYS: &[&str] = &[
"pds_seconds_max",
"cache_pending_after",
"cache_total",
"record_bytes_max",
];
/// Keys whose latest value wins.
pub const LAST_KEYS: &[&str] = &[
"pds_budget_hour_remaining",
"pds_budget_day_remaining",
"pds_requests_5min",
"nvd_key_configured",
"pds_resolve_seconds",
"release_mode",
];
/// Decimal rounding to `digits` places.
pub fn round_to(value: f64, digits: i32) -> f64 {
if !value.is_finite() {
return value;
}
let digits = digits.max(0) as usize;
format!("{value:.digits$}").parse::<f64>().unwrap_or(value)
}
pub fn round6(value: f64) -> f64 {
round_to(value, 6)
}
pub fn round3(value: f64) -> f64 {
round_to(value, 3)
}
/// Accumulated run totals.
#[derive(Debug, Clone, Default)]
pub struct RunTotals {
sums: BTreeMap<String, f64>,
maxes: BTreeMap<String, f64>,
lasts: Map<String, Value>,
}
impl RunTotals {
pub fn new() -> Self {
let mut totals = Self::default();
totals.sums.insert("errors".to_owned(), 0.0);
totals
.lasts
.insert("pds_resolve_seconds".to_owned(), json!(0.0));
totals
}
pub fn set_last(&mut self, key: &str, value: Value) {
self.lasts.insert(key.to_owned(), value);
}
/// Accumulate one cycle's stats into run totals.
pub fn accumulate(&mut self, stats: &CycleStats) {
let value = stats.to_value();
for &key in SUM_KEYS {
if let Some(number) = value.get(key).and_then(Value::as_f64) {
*self.sums.entry(key.to_owned()).or_insert(0.0) += number;
}
}
for &key in MAX_KEYS {
if let Some(number) = value.get(key).and_then(Value::as_f64) {
let entry = self.maxes.entry(key.to_owned()).or_insert(0.0);
*entry = entry.max(number);
}
}
for &key in LAST_KEYS {
if let Some(last) = value.get(key)
&& !last.is_null()
{
self.lasts.insert(key.to_owned(), last.clone());
}
}
}
pub fn add_error(&mut self) {
*self.sums.entry("errors".to_owned()).or_insert(0.0) += 1.0;
}
pub fn errors(&self) -> f64 {
self.sums.get("errors").copied().unwrap_or(0.0)
}
/// Rounded totals plus derived summary fields.
pub fn build_run_stats(&self, cycles: u64, started_at: &str) -> Value {
let mut summary = Map::new();
// Totals start with `errors` before accumulating, so it leads the
// run_stats object.
if let Some(value) = self.sums.get("errors") {
summary.insert("errors".to_owned(), json!(round6(*value)));
}
for &key in SUM_KEYS {
if key == "errors" {
continue;
}
if let Some(value) = self.sums.get(key) {
summary.insert(key.to_owned(), json!(round6(*value)));
}
}
for &key in MAX_KEYS {
if let Some(value) = self.maxes.get(key) {
summary.insert(key.to_owned(), json!(round6(*value)));
}
}
for &key in LAST_KEYS {
if let Some(value) = self.lasts.get(key) {
summary.insert(key.to_owned(), value.clone());
}
}
let duration = self.sums.get("duration_seconds").copied().unwrap_or(0.0);
let calls = self.sums.get("pds_http_calls").copied().unwrap_or(0.0);
let records = self.sums.get("records_created").copied().unwrap_or(0.0)
+ self.sums.get("records_present").copied().unwrap_or(0.0)
+ self.sums.get("records_skipped").copied().unwrap_or(0.0);
summary.insert("cycles".to_owned(), json!(cycles));
summary.insert("started_at".to_owned(), json!(started_at));
summary.insert(
"pds_seconds_avg".to_owned(),
json!(if calls != 0.0 {
round6(self.sums.get("pds_seconds_total").copied().unwrap_or(0.0) / calls)
} else {
0.0
}),
);
summary.insert(
"records_per_second".to_owned(),
json!(if duration != 0.0 {
round3(records / duration)
} else {
0.0
}),
);
Value::Object(summary)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn totals_sum_max_last_and_summary() {
let mut totals = RunTotals::new();
let first = CycleStats {
duration_seconds: 2.0,
cache_total: 3,
pds_requests_5min: 7,
nvd_key_configured: true,
records_created: 4,
software_cves: 2,
publish_retried: 1,
link_repair_repaired: 3,
pds_seconds_total: 1.0,
pds_http_calls: 2,
..CycleStats::default()
};
let second = CycleStats {
duration_seconds: 3.0,
cache_total: 9,
pds_requests_5min: 5,
records_present: 2,
software_cves: 5,
publish_retried: 4,
link_repair_repaired: 1,
..CycleStats::default()
};
totals.accumulate(&first);
totals.accumulate(&second);
let summary = totals.build_run_stats(2, "2024-01-01T00:00:00.000Z");
assert_eq!(summary["duration_seconds"], json!(5.0));
assert_eq!(summary["cache_total"], json!(9.0));
assert_eq!(summary["pds_requests_5min"], json!(5));
assert_eq!(summary["nvd_key_configured"], json!(false));
assert_eq!(summary["records_per_second"], json!(1.2));
assert_eq!(summary["pds_seconds_avg"], json!(0.5));
assert_eq!(summary["software_cves"], json!(7.0));
assert_eq!(summary["publish_retried"], json!(5.0));
assert_eq!(summary["link_repair_repaired"], json!(4.0));
assert_eq!(summary["errors"], json!(0.0));
}
}