chadtmiller.com/knotless
git that lives in your Atmosphere account
1
10
Assembled pulls and issues count kept, patched and rebuilt, with times
crates/knotless/src/telemetry.rs+36−0
| @@ -61,6 +61,7 @@ pub struct Fields<'a> { | ||
pub exit_code: Option<i64>, | ||
pub connected: Option<bool>, | ||
pub rebuilt: Option<bool>, | ||
pub how: Option<&'a str>, | ||
} | ||
| ||
macro_rules! event { | ||
| @@ -101,6 +102,7 @@ macro_rules! event { | ||
exit_code = $f.exit_code, | ||
connected = $f.connected, | ||
rebuilt = $f.rebuilt, | ||
how = $f.how, | ||
trace_id = $trace, | ||
"{}", | ||
$message | ||
| @@ -511,6 +513,8 @@ struct Stats { | ||
called: BTreeMap<(String, String), Timings>, | ||
/// Lookups in each cache: hits, misses, and fills that failed. | ||
caches: BTreeMap<&'static str, [u64; 3]>, | ||
/// Misses of each cache made good, by how: patched or rebuilt. | ||
built: BTreeMap<(&'static str, &'static str), Timings>, | ||
} | ||
| ||
impl Stats { | ||
| @@ -613,6 +617,15 @@ pub fn looked<T>(cache: &'static str, found: Option<T>) -> Option<T> { | ||
found | ||
} | ||
| ||
/// Counts a miss of `cache` made good in `took`, `how`: patched from what | ||
/// was kept, or rebuilt in full. | ||
pub fn built(cache: &'static str, how: &'static str, took: Duration) { | ||
record(|s| { | ||
let t = s.built.entry((cache, how)).or_default(); | ||
t.add(millis(took), false, slow(took)); | ||
}); | ||
} | ||
| ||
/// Counts and logs, at warn, a fill of `cache` under `key` that failed. | ||
pub fn fill<T, E: Display>(cache: &'static str, key: &str, done: &Result<T, E>) { | ||
if let Err(err) = done { | ||
| @@ -658,6 +671,14 @@ pub fn summarize() { | ||
}; | ||
note(Level::INFO, "summary: cache", &f); | ||
} | ||
for ((cache, how), t) in window.built { | ||
let f = Fields { | ||
cache: Some(cache), | ||
how: Some(how), | ||
..t.fields() | ||
}; | ||
note(Level::INFO, "summary: built", &f); | ||
} | ||
} | ||
| ||
/// What was counted since the start, in Prometheus's text format: rate, | ||
| @@ -705,6 +726,16 @@ pub fn metrics() -> String { | ||
); | ||
} | ||
} | ||
let name = "knotless_cache_build_duration_seconds"; | ||
let _ = writeln!(out, "# TYPE {name} histogram"); | ||
for ((cache, how), t) in &total.built { | ||
histogram( | ||
&mut out, | ||
name, | ||
&format!("cache=\"{cache}\",how=\"{how}\""), | ||
t, | ||
); | ||
} | ||
out | ||
} | ||
| ||
| @@ -1054,6 +1085,7 @@ mod tests { | ||
} | ||
assert_eq!(looked("avatars", Some(1)), Some(1)); | ||
assert_eq!(looked::<u8>("avatars", None), None); | ||
built("avatars", "patched", Duration::from_millis(3)); | ||
fill::<(), _>("avatars", "did:plc:x", &Err("no answer")); | ||
fill::<(), &str>("avatars", "did:plc:x", &Ok(())); | ||
summarize(); | ||
| @@ -1070,11 +1102,15 @@ mod tests { | ||
let requests = summary("summary: requests").unwrap(); | ||
assert!(requests["count"].as_u64().unwrap() >= 4); | ||
assert!(summary("summary: cache").is_some()); | ||
assert_eq!(summary("summary: built").unwrap()["how"], "patched"); | ||
let text = metrics(); | ||
assert!(text.contains("http_route=\"fallback\"")); | ||
assert!( | ||
text.contains("knotless_cache_lookups_total{cache=\"avatars\",result=\"fill_failed\"}") | ||
); | ||
assert!(text.contains( | ||
"knotless_cache_build_duration_seconds_count{cache=\"avatars\",how=\"patched\"}" | ||
)); | ||
assert!(text.contains("process_start_time_seconds")); | ||
assert_eq!(quoted("a\"b\\c\nd"), "a\\\"b\\\\c\\nd"); | ||
} | ||
crates/knotless-serve/src/index.rs+7−0
| @@ -2956,6 +2956,12 @@ impl Index { | ||
if let Some(found) = knotless::telemetry::looked("assembled", standing) { | ||
return Ok(found); | ||
} | ||
let began = Instant::now(); | ||
let how = if found.is_some() { | ||
"patched" | ||
} else { | ||
"rebuilt" | ||
}; | ||
let (entries, made, scopes, hides) = match found { | ||
Some((was, _, made, scopes, hides)) => { | ||
let entries = self.patched(did, key, &was.entries, &hides, &touched, &scopes)?; | ||
| @@ -2978,6 +2984,7 @@ impl Index { | ||
) | ||
} | ||
}; | ||
knotless::telemetry::built("assembled", how, began.elapsed()); | ||
let items = knotless::forge::assemble(record, did, &entries); | ||
let assembled = Arc::new(Assembled { entries, items }); | ||
let made = Made { | ||
deploy/README.md+3−1
| @@ -909,6 +909,8 @@ as the journal keeps them. They hold what the journal does, no more. | ||
`/metrics` gives what was counted since the start, in Prometheus's format: | ||
`http_server_request_duration_seconds` by route, method and status, | ||
`http_client_request_duration_seconds` by host and outcome, | ||
`knotless_cache_lookups_total` by cache and result, and each feed's | ||
`knotless_cache_lookups_total` by cache and result, | ||
`knotless_cache_build_duration_seconds` by cache and how a miss was made | ||
good (`patched` or `rebuilt`), and each feed's | ||
`knotless_feed_connected`, `knotless_feed_events_total` and | ||
`knotless_feed_lag_seconds`. | ||
docs/observability.md+5−2
| @@ -59,11 +59,14 @@ to read it is in [deploy/README.md](../deploy/README.md#logs). | ||
- **RED metrics** [4] at `/metrics`, in Prometheus's text format: | ||
`http_server_request_duration_seconds` (rate, errors by status, and | ||
duration, by route) and `http_client_request_duration_seconds` (by host | ||
and outcome), OpenTelemetry's metric names [5]; cache lookups by result; | ||
and outcome), OpenTelemetry's metric names [5]; cache lookups by result, and how long a | ||
miss took to make good (`knotless_cache_build_duration_seconds`, by cache and `how`: patched | ||
from what was kept, or rebuilt in full); | ||
each feed's connection, events and lag. Counted in a few maps under a | ||
mutex: no exporter, no allocation per request beyond the line. | ||
- **A summary each minute**: requests, calls by host, each cache's hits, | ||
misses and failed fills, and each feed's lag, as log lines, so a jq over | ||
misses and failed fills (and, for a miss made good, how many and how long | ||
by `how`), and each feed's lag, as log lines, so a jq over | ||
the log answers "what was it doing at 14:02" without a metrics store. | ||
- **The level changed at run time**: `PUT /log` with a `RUST_LOG` | ||
directive (`info,knotless::outgoing=debug`), through | ||