chadtmiller.com/hud
A remote MCP server that shows atproto records as interactive UI inside MCP Apps hosts
hud.hudkit 0.5.0 Latest 5 packages
0
0
//! What the server does, as JSON lines on stderr, one an event, and counts
//! of it: for `/metrics`, and for a summary each minute.
//!
//! Every line is written by one formatter (`Lines`): its time, level and
//! message, the event's fields at the top level, its target, and the
//! trace of the request it was logged while serving (`trace_id`), so a
//! line from anywhere in HUD is in its request's trace. Field names follow
//! OpenTelemetry's semantic conventions where they have one
//! (`http.route`, `server.address`, `http.response.status_code`), so a
//! collector can take the lines as they are. The calls a request makes
//! hand its trace on as a W3C `traceparent` (`net`).
//!
//! Nothing here logs a header or a body, nor a URL's query but a request's,
//! each value a secret could be in masked (`query`).
use rand_core::RngCore;
use serde_json::{Map, Value};
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
use std::time::{Duration, SystemTime};
pub use tracing::Level;
use tracing::field::{Field, Visit};
use tracing_subscriber::fmt::format::Writer;
use tracing_subscriber::fmt::time::FormatTime;
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Registry, reload};
/// What a line [`note`] logs says beyond its message, each only if set.
#[derive(Default)]
pub struct Fields<'a> {
pub version: Option<&'a str>,
pub commit: Option<&'a str>,
pub did: Option<&'a str>,
/// What was asked, as a read of the kept lines names it.
pub reason: Option<&'a str>,
pub error: Option<String>,
pub cache: Option<&'a str>,
/// A host called, as `server.address`.
pub host: Option<&'a str>,
pub count: Option<u64>,
pub failed: Option<u64>,
pub slow: Option<u64>,
pub hits: Option<u64>,
pub misses: Option<u64>,
pub mean_ms: Option<u64>,
pub max_ms: Option<u64>,
}
macro_rules! event {
($level:expr, $message:expr, $f:expr) => {
tracing::event!(
target: "hud",
$level,
version = $f.version,
commit = $f.commit,
did = $f.did,
reason = $f.reason,
error = $f.error.as_deref(),
cache = $f.cache,
server.address = $f.host,
count = $f.count,
failed = $f.failed,
slow = $f.slow,
hits = $f.hits,
misses = $f.misses,
mean_ms = $f.mean_ms,
max_ms = $f.max_ms,
"{}",
$message
)
};
}
/// Logs `message` at `level`, with `f`'s fields.
pub fn note(level: Level, message: &str, f: &Fields) {
match level {
Level::ERROR => event!(Level::ERROR, message, f),
Level::WARN => event!(Level::WARN, message, f),
Level::INFO => event!(Level::INFO, message, f),
_ => event!(Level::DEBUG, message, f),
}
}
/// Milliseconds, as a line gives a duration.
pub fn millis(d: Duration) -> u64 {
u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
}
/// The slowest a request, a call or a tool call is before it is logged
/// at warn: `HUD_SLOW_MS`, a second unless set.
static SLOW_MS: AtomicU64 = AtomicU64::new(1000);
/// Sets what [`slow`] counts as slow, in milliseconds (`HUD_SLOW_MS`).
pub fn set_slow(ms: u64) {
SLOW_MS.store(ms, Ordering::Relaxed);
}
/// Whether what took `took` was slow.
pub fn slow(took: Duration) -> bool {
millis(took) >= SLOW_MS.load(Ordering::Relaxed)
}
static RELOAD: OnceLock<reload::Handle<EnvFilter, Registry>> = OnceLock::new();
/// The version and commit [`init`] was given, for `/metrics`.
static BUILD: OnceLock<(String, String)> = OnceLock::new();
static STARTED: LazyLock<SystemTime> = LazyLock::new(SystemTime::now);
/// The commit a build was made from, as `stamped` gives it: `commit.txt`
/// beside the binary's crate, which `git archive` writes the commit into
/// (its `export-subst`, in `.gitattributes`); none in a checkout, where it
/// says only `$Format:%H$`.
pub fn commit(stamped: &str) -> Option<&str> {
let commit = stamped.trim();
(!commit.is_empty() && !commit.starts_with("$Format")).then_some(commit)
}
/// Logs from here on as JSON lines on stderr, at the level `RUST_LOG` sets
/// (info, unset), and says so: `hud starting`, its version and commit.
/// Panics are logged too, with where they happened.
pub fn init(version: &str, commit: Option<&str>) {
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let (filter, handle) = reload::Layer::new(filter);
let _ = tracing_subscriber::registry()
.with(filter)
.with(layer())
.try_init();
let _ = RELOAD.set(handle);
let _ = BUILD.set((version.to_string(), commit.unwrap_or("unknown").to_string()));
LazyLock::force(&STARTED);
std::panic::set_hook(Box::new(panicked));
let f = Fields {
version: Some(version),
commit,
..Fields::default()
};
note(Level::INFO, "hud starting", &f);
}
/// The layer writing the lines, to stderr and what [`keep_lines`] named.
fn layer<S>() -> impl tracing_subscriber::Layer<S>
where
S: tracing::Subscriber + for<'a> LookupSpan<'a>,
{
tracing_subscriber::fmt::layer()
.event_format(Lines)
.with_writer(|| Tee)
}
/// The JSON lines' format.
struct Lines;
impl<S, N> FormatEvent<S, N> for Lines
where
S: tracing::Subscriber + for<'a> LookupSpan<'a>,
N: for<'a> FormatFields<'a> + 'static,
{
fn format_event(
&self,
_: &FmtContext<'_, S, N>,
mut writer: Writer<'_>,
event: &tracing::Event<'_>,
) -> std::fmt::Result {
writeln!(writer, "{}", line(event))
}
}
/// `event` as its line: its time, level and message, its fields, its
/// target, and the trace it was logged in, unless it names one itself.
fn line(event: &tracing::Event<'_>) -> Value {
let mut at = String::new();
let _ = tracing_subscriber::fmt::time::SystemTime.format_time(&mut Writer::new(&mut at));
let mut line = Map::new();
line.insert("timestamp".into(), at.into());
line.insert("level".into(), event.metadata().level().as_str().into());
event.record(&mut Visited(&mut line));
line.insert("target".into(), event.metadata().target().into());
let trace = trace_id().filter(|_| !line.contains_key("trace_id"));
line.extend(trace.map(|id| ("trace_id".to_string(), id.into())));
Value::Object(line)
}
/// An event's fields, as they are recorded into its line.
struct Visited<'a>(&'a mut Map<String, Value>);
impl Visit for Visited<'_> {
fn record_str(&mut self, field: &Field, value: &str) {
self.0.insert(field.name().into(), value.into());
}
fn record_u64(&mut self, field: &Field, value: u64) {
self.0.insert(field.name().into(), value.into());
}
fn record_i64(&mut self, field: &Field, value: i64) {
self.0.insert(field.name().into(), value.into());
}
fn record_bool(&mut self, field: &Field, value: bool) {
self.0.insert(field.name().into(), value.into());
}
fn record_f64(&mut self, field: &Field, value: f64) {
self.0.insert(field.name().into(), value.into());
}
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
self.0
.insert(field.name().into(), format!("{value:?}").into());
}
}
/// What is handed each line as it is written, besides stderr.
type Keeper = Arc<dyn Fn(&[u8]) + Send + Sync>;
static KEEPER: RwLock<Option<Keeper>> = RwLock::new(None);
/// Hands each line logged from now on to `keep` as well, as it goes to
/// stderr: the operator's copy (`logs`). The last given wins.
pub fn keep_lines(keep: impl Fn(&[u8]) + Send + Sync + 'static) {
*KEEPER
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(keep));
}
/// Writes each line to stderr, and to what [`keep_lines`] named. The
/// layer writes a line whole, in one call.
struct Tee;
impl std::io::Write for Tee {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let keeper = KEEPER
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
keeper.inspect(|keep| keep(buf));
std::io::stderr().write_all(buf)?;
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
std::io::stderr().flush()
}
}
fn panicked(info: &std::panic::PanicHookInfo) {
let f = Fields {
error: Some(info.to_string()),
..Fields::default()
};
note(Level::ERROR, "panicked", &f);
}
/// Logs from now on at the levels `directives` sets, as `RUST_LOG` takes
/// them: `debug`, or `info,hud::outgoing=debug`.
pub fn set_filter(directives: &str) -> Result<(), String> {
let filter = EnvFilter::try_new(directives).map_err(|e| e.to_string())?;
let handle = RELOAD.get().ok_or("logging is not set up")?;
// A reload fails only once the subscriber is gone, which it never is.
let _ = handle.reload(filter);
Ok(())
}
tokio::task_local! {
static TRACE: String;
}
/// The trace the task is in: the request being served.
pub fn trace_id() -> Option<String> {
TRACE
.try_with(String::clone)
.ok()
.filter(|id| !id.is_empty())
}
/// Runs `f` in the trace `id`.
pub async fn traced<F: Future>(id: String, f: F) -> F::Output {
TRACE.scope(id, f).await
}
/// `tokio::spawn`, the task in the trace it was spawned from: what a
/// request starts of its own logs and calls in the request's trace.
pub fn spawn<F>(f: F) -> tokio::task::JoinHandle<F::Output>
where
F: Future + Send + 'static,
F::Output: Send + 'static,
{
tokio::spawn(TRACE.scope(trace_id().unwrap_or_default(), f))
}
/// The trace a `traceparent` header names, if it is a valid one; else a
/// new trace.
pub fn trace_from(traceparent: Option<&str>) -> String {
let given = traceparent.and_then(trace_of);
given.unwrap_or_else(|| hex(16))
}
fn trace_of(traceparent: &str) -> Option<String> {
let id = traceparent.split('-').nth(1)?;
let hex = id.len() == 32 && id.bytes().all(|b| b.is_ascii_hexdigit());
(hex && id.bytes().any(|b| b != b'0')).then(|| id.to_ascii_lowercase())
}
/// `bytes` random bytes, as hex.
pub(crate) fn hex(bytes: usize) -> String {
let mut buf = vec![0u8; bytes];
rand_core::OsRng.fill_bytes(&mut buf);
buf.iter().fold(String::new(), |mut s, b| {
let _ = write!(s, "{b:02x}");
s
})
}
/// The `traceparent` a call made in the trace `id` sends: a span of its own.
pub(crate) fn traceparent(id: &str) -> String {
format!("00-{id}-{}-01", hex(8))
}
/// The names of a query's parameters whose values a line leaves out:
/// sign-in's codes and state, tokens, passwords and keys.
const SECRETS: [&str; 9] = [
"code",
"state",
"iss",
"token",
"access_token",
"refresh_token",
"password",
"secret",
"key",
];
/// The query of a request to `path`, as its line gives it: each value a
/// secret could be in ([`SECRETS`]) replaced by `_`, its name read as a
/// query is (`%63ode` is `code`), cut to 200 characters; none for
/// sign-in's (`/oauth`), whose query is all codes and state.
pub fn query(path: &str, query: Option<&str>) -> Option<String> {
let query = query.filter(|_| !path.starts_with("/oauth"))?;
let pairs = query.split('&').map(|pair| {
let raw = pair.split('=').next().unwrap_or_default();
let name = url::form_urlencoded::parse(pair.as_bytes()).next();
match name.is_some_and(|(n, _)| SECRETS.iter().any(|s| n.eq_ignore_ascii_case(s))) {
true => format!("{raw}=_"),
false => pair.to_string(),
}
});
let kept = pairs.collect::<Vec<_>>().join("&");
Some(kept.chars().take(200).collect())
}
/// A path as a line gives it: no query, and not too long.
pub fn short(path: &str) -> String {
path.chars().take(120).collect()
}
/// A call made, as the line logged for it says: never its query, a
/// header or a body.
pub(crate) struct Call<'a> {
pub host: &'a str,
pub method: &'a str,
pub path: &'a str,
pub status: Option<u16>,
/// Why it failed, before an answer: `timeout`, `connect`, …
pub error: Option<&'a str>,
pub took: Duration,
}
/// What a call given up on by what asked for it, not by its wait, is
/// logged as: no failure, at debug.
pub(crate) const ABANDONED: &str = "abandoned";
/// Logs a call made and counts it: at debug, or warn if it failed,
/// answered 5xx or was slow; one [`ABANDONED`] is no failure.
pub(crate) fn called(c: &Call) {
let ms = millis(c.took);
let server_error = c.status.filter(|s| *s >= 500).map(|s| s.to_string());
let error = c.error.or(server_error.as_deref());
let failed = error.is_some_and(|e| e != ABANDONED);
let slow = slow(c.took);
let outcome = c
.status
.map_or(error.unwrap_or_default().to_string(), |s| s.to_string());
let host = record(|s| s.call(c.host, &outcome, ms, failed, slow));
let rpc = c.path.strip_prefix("/xrpc/");
macro_rules! line {
($level:expr) => {
tracing::event!(
target: "hud::outgoing",
$level,
server.address = host.as_str(),
http.request.method = c.method,
url.path = c.path,
rpc.method = rpc,
http.response.status_code = c.status,
error.type = error,
duration_ms = ms,
"outgoing"
)
};
}
match failed || slow {
true => line!(Level::WARN),
false => line!(Level::DEBUG),
}
}
/// What kind of failure `err` is, and why, without the URL it names.
pub(crate) fn error_type(err: &reqwest::Error) -> String {
let kind = match () {
_ if err.is_timeout() => "timeout",
_ if err.is_connect() => "connect",
_ if err.is_body() || err.is_decode() => "body",
_ => "request",
};
let mut said = kind.to_string();
let mut source = std::error::Error::source(err);
while let Some(cause) = source {
let _ = write!(said, ": {cause}");
source = cause.source();
}
said
}
/// A request served, as the line logged for it says.
pub struct Served<'a> {
/// The route it matched, as its template: `/blob/{did}/{cid}`.
pub route: Option<&'a str>,
pub method: &'a str,
pub path: &'a str,
/// Its query, as [`query`] gives it.
pub query: Option<&'a str>,
/// The client, as its `User-Agent` names itself.
pub agent: Option<&'a str>,
pub status: u16,
/// The body's length, when known before it is sent.
pub bytes: Option<u64>,
pub took: Duration,
pub trace: &'a str,
/// Why it failed, where its answer says.
pub cause: Option<&'a str>,
/// Whether it is long-lived on purpose, as an event stream is, so never
/// slow.
pub waits: bool,
}
/// Logs a request served and counts it: at error if it failed on this
/// side (5xx), warn if what HUD asked for it failed (502, 504: the call's
/// own line says which) or it was slow, info otherwise, and debug for a
/// health check.
pub fn served(r: &Served) {
let ms = millis(r.took);
let failed = r.status >= 500;
let upstream = matches!(r.status, 502 | 504);
let slow = slow(r.took) && !r.waits;
let route = r.route.unwrap_or("fallback");
let key = (route.to_string(), r.method.to_string(), r.status);
record(|s| {
s.served
.entry(key.clone())
.or_default()
.add(ms, failed, slow)
});
let quiet = route == "/_health" && !failed && !slow;
macro_rules! line {
($level:expr) => {
tracing::event!(
target: "hud::request",
$level,
http.route = r.route,
http.request.method = r.method,
url.path = r.path,
url.query = r.query,
user_agent.original = r.agent,
http.response.status_code = r.status,
http.response.body.size = r.bytes,
duration_ms = ms,
trace_id = r.trace,
error = r.cause,
"request"
)
};
}
match (failed && !upstream, failed || slow, quiet) {
(true, _, _) => line!(Level::ERROR),
(false, true, _) => line!(Level::WARN),
(false, false, true) => line!(Level::DEBUG),
(false, false, false) => line!(Level::INFO),
}
}
/// An MCP tool call, as the line logged for it says: never its arguments
/// or its result.
pub struct Tool<'a> {
pub name: &'a str,
/// The caller, signed in.
pub did: Option<&'a str>,
/// What the tool answered, where it failed.
pub error: Option<&'a str>,
/// Why it answered nothing, as when it asked the caller to sign in.
pub reason: Option<&'a str>,
pub took: Duration,
/// The kit drawing its tree, as it said itself (`kit::Kit`'s `said`).
pub kit: Option<&'a str>,
}
/// Logs an MCP tool call: `mcp tool failed` at warn, or `mcp tool`, at
/// warn when slow, else info.
pub fn tool(t: &Tool) {
let ms = millis(t.took);
let is_error = t.error.is_some();
macro_rules! line {
($level:expr, $message:expr) => {
tracing::event!(
target: "hud::mcp",
$level,
gen_ai.tool.name = t.name,
did = t.did,
kit = t.kit,
is_error,
error = t.error,
reason = t.reason,
duration_ms = ms,
$message
)
};
}
match (is_error, slow(t.took)) {
(true, _) => line!(Level::WARN, "mcp tool failed"),
(false, true) => line!(Level::WARN, "mcp tool"),
(false, false) => line!(Level::INFO, "mcp tool"),
}
}
/// Logs, at info, an MCP request other than a tool call that was answered
/// a JSON-RPC error: `mcp error`, its method and why.
pub fn rpc_failed(method: &str, error: &str) {
tracing::event!(target: "hud::mcp", Level::INFO, rpc.method = method, error, "mcp error");
}
/// The bounds of the duration histograms' buckets, in milliseconds.
const BOUNDS_MS: [u64; 11] = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000];
/// How many hosts calls are counted by; the rest are `other`.
const HOSTS: usize = 200;
#[derive(Default, Clone)]
struct Timings {
count: u64,
failed: u64,
slow: u64,
sum_ms: u64,
max_ms: u64,
buckets: [u64; BOUNDS_MS.len()],
}
impl Timings {
fn add(&mut self, ms: u64, failed: bool, slow: bool) {
self.count += 1;
self.failed += u64::from(failed);
self.slow += u64::from(slow);
self.sum_ms += ms;
self.max_ms = self.max_ms.max(ms);
for (bucket, bound) in self.buckets.iter_mut().zip(BOUNDS_MS) {
*bucket += u64::from(ms <= bound);
}
}
fn merge(&mut self, other: &Timings) {
self.count += other.count;
self.failed += other.failed;
self.slow += other.slow;
self.sum_ms += other.sum_ms;
self.max_ms = self.max_ms.max(other.max_ms);
for (bucket, more) in self.buckets.iter_mut().zip(other.buckets) {
*bucket += more;
}
}
fn fields(&self) -> Fields<'static> {
Fields {
count: Some(self.count),
failed: Some(self.failed),
slow: Some(self.slow),
mean_ms: Some(self.sum_ms / self.count.max(1)),
max_ms: Some(self.max_ms),
..Fields::default()
}
}
}
#[derive(Default)]
struct Stats {
/// Requests served, by route, method and status.
served: BTreeMap<(String, String, u16), Timings>,
/// Calls made, by host and status or failure.
called: BTreeMap<(String, String), Timings>,
}
impl Stats {
/// Counts a call to `host`, or to `other` past [`HOSTS`] of them: the
/// host it was counted under.
fn call(&mut self, host: &str, outcome: &str, ms: u64, failed: bool, slow: bool) -> String {
let hosts = self
.called
.keys()
.map(|(h, _)| h.as_str())
.collect::<std::collections::BTreeSet<_>>();
let host = match hosts.contains(host) || hosts.len() < HOSTS {
true => host.to_string(),
false => "other".to_string(),
};
let key = (host.clone(), outcome.to_string());
self.called.entry(key).or_default().add(ms, failed, slow);
host
}
}
/// Since the start, for `/metrics`, and since the last summary.
static TOTAL: LazyLock<Mutex<Stats>> = LazyLock::new(Mutex::default);
static WINDOW: LazyLock<Mutex<Stats>> = LazyLock::new(Mutex::default);
/// Counts into both, answering what the count since the start did.
fn record<T>(count: impl Fn(&mut Stats) -> T) -> T {
count(
&mut WINDOW
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
);
count(
&mut TOTAL
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
)
}
/// Each cache's hits and misses at the last summary, to say those since.
static LOOKED: LazyLock<Mutex<BTreeMap<&'static str, (u64, u64)>>> = LazyLock::new(Mutex::default);
/// Logs, at info, what was counted since the last summary: requests
/// served, calls made to each host, and the hits and misses of each cache
/// of `lookups` (each cache's since the start, as `Hud::lookups` gives
/// them). Nothing for what saw none.
pub fn summarize(lookups: &[(&'static str, u64, u64)]) {
let window = std::mem::take(
&mut *WINDOW
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
);
let mut served = Timings::default();
window.served.values().for_each(|t| served.merge(t));
// A minute with requests says so; one with none, nothing.
std::iter::once(served)
.filter(|t| t.count > 0)
.for_each(|t| note(Level::INFO, "summary: requests", &t.fields()));
let mut hosts: BTreeMap<&str, Timings> = BTreeMap::new();
for ((host, _), t) in &window.called {
hosts.entry(host).or_default().merge(t);
}
for (host, t) in hosts {
let f = Fields {
host: Some(host),
..t.fields()
};
note(Level::INFO, "summary: outgoing", &f);
}
let mut looked = LOOKED
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for &(cache, hits, misses) in lookups {
let (was_hits, was_misses) = looked.insert(cache, (hits, misses)).unwrap_or_default();
let (hits, misses) = (
hits.saturating_sub(was_hits),
misses.saturating_sub(was_misses),
);
if hits + misses > 0 {
let f = Fields {
cache: Some(cache),
hits: Some(hits),
misses: Some(misses),
..Fields::default()
};
note(Level::INFO, "summary: cache", &f);
}
}
}
/// What was counted since the start, in Prometheus's text format: rate,
/// errors and duration by route and by host called, by OpenTelemetry's
/// metric names, and the lookups of each cache of `lookups`.
pub fn metrics(lookups: &[(&'static str, u64, u64)]) -> String {
let total = TOTAL
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut out = String::new();
let (version, commit) = BUILD.get().cloned().unwrap_or_default();
let started = STARTED
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default();
let _ = writeln!(out, "# TYPE hud_build_info gauge");
let _ = writeln!(
out,
"hud_build_info{{version=\"{}\",commit=\"{}\"}} 1",
quoted(&version),
quoted(&commit)
);
let _ = writeln!(out, "# TYPE process_start_time_seconds gauge");
let _ = writeln!(out, "process_start_time_seconds {}", started.as_secs());
let name = "http_server_request_duration_seconds";
let _ = writeln!(out, "# TYPE {name} histogram");
for ((route, method, status), t) in &total.served {
let labels = format!(
"http_route=\"{}\",http_request_method=\"{}\",http_response_status_code=\"{status}\"",
quoted(route),
quoted(method)
);
histogram(&mut out, name, &labels, t);
}
let name = "http_client_request_duration_seconds";
let _ = writeln!(out, "# TYPE {name} histogram");
for ((host, outcome), t) in &total.called {
let labels = format!(
"server_address=\"{}\",outcome=\"{}\"",
quoted(host),
quoted(outcome)
);
histogram(&mut out, name, &labels, t);
}
let _ = writeln!(out, "# TYPE hud_cache_lookups_total counter");
for (cache, hits, misses) in lookups {
for (result, n) in [("hit", hits), ("miss", misses)] {
let _ = writeln!(
out,
"hud_cache_lookups_total{{cache=\"{cache}\",result=\"{result}\"}} {n}"
);
}
}
out
}
fn histogram(out: &mut String, name: &str, labels: &str, t: &Timings) {
for (bound, n) in BOUNDS_MS.iter().zip(t.buckets) {
let le = *bound as f64 / 1000.0;
let _ = writeln!(out, "{name}_bucket{{{labels},le=\"{le}\"}} {n}");
}
let _ = writeln!(out, "{name}_bucket{{{labels},le=\"+Inf\"}} {}", t.count);
let _ = writeln!(out, "{name}_sum{{{labels}}} {}", t.sum_ms as f64 / 1000.0);
let _ = writeln!(out, "{name}_count{{{labels}}} {}", t.count);
}
/// `value` as a Prometheus label value takes it.
fn quoted(value: &str) -> String {
value
.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
std::thread_local! {
/// The lines this thread logs while [`logged`] runs.
static CAUGHT: std::cell::RefCell<Option<Vec<u8>>> = const { std::cell::RefCell::new(None) };
}
/// What is logged on this thread while `f` runs, at every level, as
/// the lines say it. One subscriber, for every test and thread, logs
/// everything: a subscriber of each test's own would leave out what
/// another thread's test, logging at once, had tracing take as never
/// wanted.
pub(crate) fn logged(f: impl FnOnce()) -> Vec<Value> {
static SUBSCRIBED: std::sync::Once = std::sync::Once::new();
SUBSCRIBED.call_once(|| {
let layer = tracing_subscriber::fmt::layer()
.event_format(Lines)
.with_writer(|| Caught);
let subscriber = tracing_subscriber::registry()
.with(EnvFilter::new("trace"))
.with(layer);
tracing::subscriber::set_global_default(subscriber).unwrap();
});
CAUGHT.with(|c| *c.borrow_mut() = Some(Vec::new()));
f();
let caught = CAUGHT.with(|c| c.borrow_mut().take()).unwrap_or_default();
let text = String::from_utf8(caught).unwrap();
text.lines()
.map(|l| serde_json::from_str(l).unwrap())
.collect()
}
/// Writes a line to this thread's [`CAUGHT`], while a test catches them.
struct Caught;
impl std::io::Write for Caught {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let into = |c: &std::cell::RefCell<Option<Vec<u8>>>| {
c.borrow_mut()
.iter_mut()
.for_each(|v| v.extend_from_slice(buf))
};
CAUGHT.with(into);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
/// Each line goes to stderr and to what keeps them.
#[test]
fn lines_are_kept_as_written() {
use std::io::Write as _;
let kept = Arc::new(Mutex::new(Vec::new()));
let into = kept.clone();
keep_lines(move |line| into.lock().unwrap().push(line.to_vec()));
let line = b"{\"message\":\"tee\"}\n";
let mut tee = Tee;
assert_eq!(tee.write(line).unwrap(), line.len());
tee.flush().unwrap();
assert!(kept.lock().unwrap().iter().any(|l| l == line));
Caught.flush().unwrap();
}
#[test]
fn notes_at_each_level() {
let lines = logged(|| {
for level in [Level::ERROR, Level::WARN, Level::INFO, Level::TRACE] {
let f = Fields {
did: Some("did:plc:alice"),
error: Some("broke".into()),
count: Some(3),
..Fields::default()
};
note(level, "said", &f);
}
});
let levels: Vec<&str> = lines.iter().map(|l| l["level"].as_str().unwrap()).collect();
assert_eq!(levels, ["ERROR", "WARN", "INFO", "DEBUG"]);
assert_eq!(lines[0]["message"], "said");
assert_eq!(lines[0]["did"], "did:plc:alice");
assert_eq!(lines[0]["count"], 3);
assert_eq!(lines[0]["target"], "hud");
assert!(lines[0]["timestamp"].as_str().unwrap().ends_with('Z'));
assert!(lines[0].get("cache").is_none());
assert!(lines[0].get("trace_id").is_none());
}
/// Every kind of value a field holds, as JSON; a line in a trace
/// names it, unless it names one itself.
#[test]
fn fields_and_traces_in_lines() {
let rt = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let lines = logged(|| {
rt.block_on(traced("t1".into(), async {
tracing::info!(n = 1u64, i = -1i64, yes = true, x = 0.5, s = "a", e = %"shown", "kinds");
tracing::info!(trace_id = "t2", "named");
}));
});
assert_eq!(lines[0]["n"], 1);
assert_eq!(lines[0]["i"], -1);
assert_eq!(lines[0]["yes"], true);
assert_eq!(lines[0]["x"], 0.5);
assert_eq!(lines[0]["s"], "a");
assert_eq!(lines[0]["e"], "shown");
assert_eq!(lines[0]["message"], "kinds");
assert_eq!(lines[0]["trace_id"], "t1");
assert_eq!(lines[1]["trace_id"], "t2");
}
#[test]
fn layers_and_settings() {
let _ = layer::<Registry>();
assert!(slow(Duration::from_secs(1)));
assert!(!slow(Duration::from_millis(999)));
assert_eq!(millis(Duration::from_secs(2)), 2000);
assert_eq!(short(&"a".repeat(500)).len(), 120);
// Never set up here: refused either way.
assert!(set_filter("[[[").unwrap_err().contains("invalid"));
assert_eq!(set_filter("debug").unwrap_err(), "logging is not set up");
assert_eq!(commit("$Format:%H$\n"), None);
assert_eq!(commit(" \n"), None);
assert_eq!(commit("abc123\n"), Some("abc123"));
}
#[test]
fn panics_are_logged() {
let lines = logged(|| {
let before = std::panic::take_hook();
std::panic::set_hook(Box::new(panicked));
let _ = std::panic::catch_unwind(|| panic!("on fire"));
std::panic::set_hook(before);
});
assert_eq!(lines[0]["message"], "panicked");
let error = lines[0]["error"].as_str().unwrap();
assert!(error.contains("on fire"), "{error}");
assert!(error.contains("telemetry.rs"), "{error}");
}
#[test]
fn traces() {
let id = "4bf92f3577b34da6a3ce929d0e0e4736";
let parent = format!("00-{id}-00f067aa0ba902b7-01");
assert_eq!(trace_from(Some(&parent)), id);
assert_eq!(trace_from(Some(&parent.to_uppercase())), id);
for bad in ["junk", "00-xyz-00f067aa0ba902b7-01"] {
assert_ne!(trace_from(Some(bad)), id);
}
let zeros = format!("00-{}-00f067aa0ba902b7-01", "0".repeat(32));
assert_eq!(trace_from(Some(&zeros)).len(), 32);
assert_eq!(trace_from(None).len(), 32);
assert_eq!(trace_id(), None);
let sent = traceparent(id);
assert_eq!(trace_from(Some(&sent)), id);
assert_eq!(sent.len(), 55);
}
/// A task spawned in a trace is in it; one spawned in none, in none.
#[tokio::test]
async fn spawned_tasks_keep_their_trace() {
let within = traced("t".into(), async { spawn(async { trace_id() }).await })
.await
.unwrap();
assert_eq!(within.as_deref(), Some("t"));
assert_eq!(spawn(async { trace_id() }).await.unwrap(), None);
}
#[test]
fn hosts_past_the_most_are_other() {
let mut stats = Stats::default();
for n in 0..HOSTS {
stats.call(&format!("h{n}"), "200", 1, false, false);
}
assert_eq!(stats.call("h0", "200", 1, false, false), "h0");
assert_eq!(stats.call("new", "200", 1, false, false), "other");
}
#[test]
fn queries_leave_secrets_out() {
let logged = |q: &str| query("/xrpc/place.hud.getLogs", Some(q)).unwrap();
assert_eq!(logged("trace=abc&level=warn"), "trace=abc&level=warn");
assert_eq!(logged("code=abc&ISS=https://pds&x"), "code=_&ISS=_&x");
// Its name read as a query is, however it is written.
assert_eq!(
logged("%63ode=abc&acc%65ss_token=t&refresh+token=r&&x"),
"%63ode=_&acc%65ss_token=_&refresh+token=r&&x"
);
assert_eq!(logged(&"a".repeat(300)).len(), 200);
assert_eq!(query("/oauth/callback", Some("code=abc")), None);
assert_eq!(query("/xrpc/place.hud.getLogs", None), None);
}
/// The event stream is never slow, however long it is open.
#[test]
fn what_waits_is_not_slow() {
let lines = logged(|| {
served(&Served {
route: Some("/xrpc/place.hud.getWorkspaceEvents"),
method: "GET",
path: "/xrpc/place.hud.getWorkspaceEvents",
query: None,
agent: None,
status: 200,
bytes: None,
took: Duration::from_secs(25),
trace: "t",
cause: None,
waits: true,
});
});
assert_eq!(lines[0]["level"], "INFO");
}
#[test]
fn rpc_errors() {
let lines = logged(|| rpc_failed("tools/list", "Invalid cursor"));
assert_eq!(lines[0]["message"], "mcp error");
assert_eq!(lines[0]["rpc.method"], "tools/list");
assert_eq!(lines[0]["error"], "Invalid cursor");
}
#[test]
fn tool_calls() {
let lines = logged(|| {
for (error, reason, ms) in [
(Some("no such account"), None, 1),
(None, None, 5000),
(None, Some("sign-in asked"), 1),
] {
tool(&Tool {
name: "describe_repo",
did: Some("did:plc:alice"),
error,
reason,
took: Duration::from_millis(ms),
kit: Some("ios 0.3.0"),
});
}
});
let said: Vec<(&str, &str)> = lines
.iter()
.map(|l| (l["level"].as_str().unwrap(), l["message"].as_str().unwrap()))
.collect();
assert_eq!(
said,
[
("WARN", "mcp tool failed"),
("WARN", "mcp tool"),
("INFO", "mcp tool")
]
);
assert_eq!(lines[0]["gen_ai.tool.name"], "describe_repo");
assert_eq!(lines[0]["kit"], "ios 0.3.0");
assert_eq!(lines[0]["error"], "no such account");
assert_eq!(lines[0]["is_error"], true);
assert_eq!(lines[1]["duration_ms"], 5000);
assert_eq!(lines[2]["reason"], "sign-in asked");
assert_eq!(lines[2]["is_error"], false);
}
#[test]
fn calls_made() {
let lines = logged(|| {
for (status, error, ms) in [
(Some(200), None, 1),
(Some(503), None, 1),
(None, Some("connect: refused"), 1),
(Some(200), None, 5000),
] {
called(&Call {
host: "pds.calls.test",
method: "GET",
path: "/xrpc/com.atproto.repo.getRecord",
status,
error,
took: Duration::from_millis(ms),
});
}
});
let levels: Vec<&str> = lines.iter().map(|l| l["level"].as_str().unwrap()).collect();
assert_eq!(levels, ["DEBUG", "WARN", "WARN", "WARN"]);
assert_eq!(lines[0]["rpc.method"], "com.atproto.repo.getRecord");
assert_eq!(lines[0]["server.address"], "pds.calls.test");
assert_eq!(lines[0]["target"], "hud::outgoing");
assert_eq!(lines[1]["error.type"], "503");
assert_eq!(lines[2]["error.type"], "connect: refused");
assert!(lines[2].get("http.response.status_code").is_none());
let text = metrics(&[]);
assert!(text.contains(
"http_client_request_duration_seconds_count{server_address=\"pds.calls.test\",outcome=\"connect: refused\"} 1"
));
}
#[test]
fn served_and_summarized() {
let lines = logged(|| {
for (route, status, ms) in [
(Some("/_health"), 200, 1),
(Some("/blob/{did}/{cid}"), 200, 2),
(Some("/blob/{did}/{cid}"), 500, 2),
// What HUD asked for it failed: its call's line says so.
(Some("/icon/{domain}"), 502, 2),
(None, 200, 5000),
] {
served(&Served {
route,
method: "GET",
path: "/blob/did:plc:x/bafy",
query: Some("size=thumb"),
agent: Some("Mozilla/5.0"),
status,
bytes: Some(10),
took: Duration::from_millis(ms),
trace: "t",
cause: (status >= 500).then_some("PDS down"),
waits: false,
});
}
summarize(&[("docs", 3, 1), ("handles", 0, 0)]);
// Since then, nothing to summarize: nothing is said of it.
summarize(&[("docs", 3, 1), ("handles", 0, 0)]);
});
let levels: Vec<&str> = lines.iter().map(|l| l["level"].as_str().unwrap()).collect();
assert_eq!(&levels[..5], ["DEBUG", "INFO", "ERROR", "WARN", "WARN"]);
assert_eq!(lines[2]["error"], "PDS down");
assert_eq!(lines[2]["http.route"], "/blob/{did}/{cid}");
assert_eq!(lines[1]["http.response.status_code"], 200);
assert_eq!(lines[1]["user_agent.original"], "Mozilla/5.0");
assert_eq!(lines[1]["url.query"], "size=thumb");
assert_eq!(lines[1]["trace_id"], "t");
assert_eq!(lines[1]["target"], "hud::request");
let summary = |m: &str| lines.iter().filter(|l| l["message"] == m).count();
assert_eq!(summary("summary: cache"), 1);
assert!(summary("summary: requests") >= 1);
let cache = lines
.iter()
.find(|l| l["message"] == "summary: cache")
.unwrap();
assert_eq!(
(cache["cache"].clone(), cache["hits"].clone()),
("docs".into(), 3.into())
);
let text = metrics(&[("docs", 3, 1)]);
assert!(text.contains("http_route=\"fallback\""));
assert!(text.contains("hud_cache_lookups_total{cache=\"docs\",result=\"miss\"} 1"));
assert!(text.contains("process_start_time_seconds"));
assert!(text.contains("hud_build_info{version=\"\",commit=\"\"} 1"));
assert_eq!(quoted("a\"b\\c\nd"), "a\\\"b\\\\c\\nd");
}
#[tokio::test]
async fn error_types() {
let http = reqwest::Client::new();
// Asked of a listener that never answers, so the timeout comes first.
let silent = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let never = format!("http://{}/", silent.local_addr().unwrap());
let timeout = http.get(&never).timeout(Duration::ZERO).send().await;
assert!(error_type(&timeout.unwrap_err()).starts_with("timeout"));
let refused = http.get("http://127.0.0.1:9/").send().await.unwrap_err();
assert!(
error_type(&refused).starts_with("connect"),
"{}",
error_type(&refused)
);
let built = http
.get(&never)
.header("bad\nname", "x")
.build()
.unwrap_err();
assert!(error_type(&built).starts_with("request"));
// An answer cut short fails as its body is read.
let cut = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}/", cut.local_addr().unwrap());
tokio::spawn(async move {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let (mut s, _) = cut.accept().await.unwrap();
let _ = s.read(&mut [0; 1024]).await.unwrap();
let head = b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort";
s.write_all(head).await.unwrap();
});
let res = http.get(&url).send().await.unwrap();
let body = res.bytes().await.unwrap_err();
assert!(
error_type(&body).starts_with("body"),
"{}",
error_type(&body)
);
}
}