~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/hud

A remote MCP server that shows atproto records as interactive UI inside MCP Apps hosts

0
0
1158 lines · 40.9 KB Raw History
//! What the server does, as JSON lines on stderr, one an event, and counts
//! of it: for `/metrics`, and for a summary each minute.
//!
//! Every line is written by one formatter (`Lines`): its time, level and
//! message, the event's fields at the top level, its target, and the
//! trace of the request it was logged while serving (`trace_id`), so a
//! line from anywhere in HUD is in its request's trace. Field names follow
//! OpenTelemetry's semantic conventions where they have one
//! (`http.route`, `server.address`, `http.response.status_code`), so a
//! collector can take the lines as they are. The calls a request makes
//! hand its trace on as a W3C `traceparent` (`net`).
//!
//! Nothing here logs a header or a body, nor a URL's query but a request's,
//! each value a secret could be in masked (`query`).

use rand_core::RngCore;
use serde_json::{Map, Value};
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
use std::time::{Duration, SystemTime};
pub use tracing::Level;
use tracing::field::{Field, Visit};
use tracing_subscriber::fmt::format::Writer;
use tracing_subscriber::fmt::time::FormatTime;
use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Registry, reload};

/// What a line [`note`] logs says beyond its message, each only if set.
#[derive(Default)]
pub struct Fields<'a> {
    pub version: Option<&'a str>,
    pub commit: Option<&'a str>,
    pub did: Option<&'a str>,
    /// What was asked, as a read of the kept lines names it.
    pub reason: Option<&'a str>,
    pub error: Option<String>,
    pub cache: Option<&'a str>,
    /// A host called, as `server.address`.
    pub host: Option<&'a str>,
    pub count: Option<u64>,
    pub failed: Option<u64>,
    pub slow: Option<u64>,
    pub hits: Option<u64>,
    pub misses: Option<u64>,
    pub mean_ms: Option<u64>,
    pub max_ms: Option<u64>,
}

macro_rules! event {
    ($level:expr, $message:expr, $f:expr) => {
        tracing::event!(
            target: "hud",
            $level,
            version = $f.version,
            commit = $f.commit,
            did = $f.did,
            reason = $f.reason,
            error = $f.error.as_deref(),
            cache = $f.cache,
            server.address = $f.host,
            count = $f.count,
            failed = $f.failed,
            slow = $f.slow,
            hits = $f.hits,
            misses = $f.misses,
            mean_ms = $f.mean_ms,
            max_ms = $f.max_ms,
            "{}",
            $message
        )
    };
}

/// Logs `message` at `level`, with `f`'s fields.
pub fn note(level: Level, message: &str, f: &Fields) {
    match level {
        Level::ERROR => event!(Level::ERROR, message, f),
        Level::WARN => event!(Level::WARN, message, f),
        Level::INFO => event!(Level::INFO, message, f),
        _ => event!(Level::DEBUG, message, f),
    }
}

/// Milliseconds, as a line gives a duration.
pub fn millis(d: Duration) -> u64 {
    u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
}

/// The slowest a request, a call or a tool call is before it is logged
/// at warn: `HUD_SLOW_MS`, a second unless set.
static SLOW_MS: AtomicU64 = AtomicU64::new(1000);

/// Sets what [`slow`] counts as slow, in milliseconds (`HUD_SLOW_MS`).
pub fn set_slow(ms: u64) {
    SLOW_MS.store(ms, Ordering::Relaxed);
}

/// Whether what took `took` was slow.
pub fn slow(took: Duration) -> bool {
    millis(took) >= SLOW_MS.load(Ordering::Relaxed)
}

static RELOAD: OnceLock<reload::Handle<EnvFilter, Registry>> = OnceLock::new();

/// The version and commit [`init`] was given, for `/metrics`.
static BUILD: OnceLock<(String, String)> = OnceLock::new();

static STARTED: LazyLock<SystemTime> = LazyLock::new(SystemTime::now);

/// The commit a build was made from, as `stamped` gives it: `commit.txt`
/// beside the binary's crate, which `git archive` writes the commit into
/// (its `export-subst`, in `.gitattributes`); none in a checkout, where it
/// says only `$Format:%H$`.
pub fn commit(stamped: &str) -> Option<&str> {
    let commit = stamped.trim();
    (!commit.is_empty() && !commit.starts_with("$Format")).then_some(commit)
}

/// Logs from here on as JSON lines on stderr, at the level `RUST_LOG` sets
/// (info, unset), and says so: `hud starting`, its version and commit.
/// Panics are logged too, with where they happened.
pub fn init(version: &str, commit: Option<&str>) {
    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
    let (filter, handle) = reload::Layer::new(filter);
    let _ = tracing_subscriber::registry()
        .with(filter)
        .with(layer())
        .try_init();
    let _ = RELOAD.set(handle);
    let _ = BUILD.set((version.to_string(), commit.unwrap_or("unknown").to_string()));
    LazyLock::force(&STARTED);
    std::panic::set_hook(Box::new(panicked));
    let f = Fields {
        version: Some(version),
        commit,
        ..Fields::default()
    };
    note(Level::INFO, "hud starting", &f);
}

/// The layer writing the lines, to stderr and what [`keep_lines`] named.
fn layer<S>() -> impl tracing_subscriber::Layer<S>
where
    S: tracing::Subscriber + for<'a> LookupSpan<'a>,
{
    tracing_subscriber::fmt::layer()
        .event_format(Lines)
        .with_writer(|| Tee)
}

/// The JSON lines' format.
struct Lines;

impl<S, N> FormatEvent<S, N> for Lines
where
    S: tracing::Subscriber + for<'a> LookupSpan<'a>,
    N: for<'a> FormatFields<'a> + 'static,
{
    fn format_event(
        &self,
        _: &FmtContext<'_, S, N>,
        mut writer: Writer<'_>,
        event: &tracing::Event<'_>,
    ) -> std::fmt::Result {
        writeln!(writer, "{}", line(event))
    }
}

/// `event` as its line: its time, level and message, its fields, its
/// target, and the trace it was logged in, unless it names one itself.
fn line(event: &tracing::Event<'_>) -> Value {
    let mut at = String::new();
    let _ = tracing_subscriber::fmt::time::SystemTime.format_time(&mut Writer::new(&mut at));
    let mut line = Map::new();
    line.insert("timestamp".into(), at.into());
    line.insert("level".into(), event.metadata().level().as_str().into());
    event.record(&mut Visited(&mut line));
    line.insert("target".into(), event.metadata().target().into());
    let trace = trace_id().filter(|_| !line.contains_key("trace_id"));
    line.extend(trace.map(|id| ("trace_id".to_string(), id.into())));
    Value::Object(line)
}

/// An event's fields, as they are recorded into its line.
struct Visited<'a>(&'a mut Map<String, Value>);

impl Visit for Visited<'_> {
    fn record_str(&mut self, field: &Field, value: &str) {
        self.0.insert(field.name().into(), value.into());
    }

    fn record_u64(&mut self, field: &Field, value: u64) {
        self.0.insert(field.name().into(), value.into());
    }

    fn record_i64(&mut self, field: &Field, value: i64) {
        self.0.insert(field.name().into(), value.into());
    }

    fn record_bool(&mut self, field: &Field, value: bool) {
        self.0.insert(field.name().into(), value.into());
    }

    fn record_f64(&mut self, field: &Field, value: f64) {
        self.0.insert(field.name().into(), value.into());
    }

    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
        self.0
            .insert(field.name().into(), format!("{value:?}").into());
    }
}

/// What is handed each line as it is written, besides stderr.
type Keeper = Arc<dyn Fn(&[u8]) + Send + Sync>;

static KEEPER: RwLock<Option<Keeper>> = RwLock::new(None);

/// Hands each line logged from now on to `keep` as well, as it goes to
/// stderr: the operator's copy (`logs`). The last given wins.
pub fn keep_lines(keep: impl Fn(&[u8]) + Send + Sync + 'static) {
    *KEEPER
        .write()
        .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(keep));
}

/// Writes each line to stderr, and to what [`keep_lines`] named. The
/// layer writes a line whole, in one call.
struct Tee;

impl std::io::Write for Tee {
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
        let keeper = KEEPER
            .read()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
            .clone();
        keeper.inspect(|keep| keep(buf));
        std::io::stderr().write_all(buf)?;
        Ok(buf.len())
    }

    fn flush(&mut self) -> std::io::Result<()> {
        std::io::stderr().flush()
    }
}

fn panicked(info: &std::panic::PanicHookInfo) {
    let f = Fields {
        error: Some(info.to_string()),
        ..Fields::default()
    };
    note(Level::ERROR, "panicked", &f);
}

/// Logs from now on at the levels `directives` sets, as `RUST_LOG` takes
/// them: `debug`, or `info,hud::outgoing=debug`.
pub fn set_filter(directives: &str) -> Result<(), String> {
    let filter = EnvFilter::try_new(directives).map_err(|e| e.to_string())?;
    let handle = RELOAD.get().ok_or("logging is not set up")?;
    // A reload fails only once the subscriber is gone, which it never is.
    let _ = handle.reload(filter);
    Ok(())
}

tokio::task_local! {
    static TRACE: String;
}

/// The trace the task is in: the request being served.
pub fn trace_id() -> Option<String> {
    TRACE
        .try_with(String::clone)
        .ok()
        .filter(|id| !id.is_empty())
}

/// Runs `f` in the trace `id`.
pub async fn traced<F: Future>(id: String, f: F) -> F::Output {
    TRACE.scope(id, f).await
}

/// `tokio::spawn`, the task in the trace it was spawned from: what a
/// request starts of its own logs and calls in the request's trace.
pub fn spawn<F>(f: F) -> tokio::task::JoinHandle<F::Output>
where
    F: Future + Send + 'static,
    F::Output: Send + 'static,
{
    tokio::spawn(TRACE.scope(trace_id().unwrap_or_default(), f))
}

/// The trace a `traceparent` header names, if it is a valid one; else a
/// new trace.
pub fn trace_from(traceparent: Option<&str>) -> String {
    let given = traceparent.and_then(trace_of);
    given.unwrap_or_else(|| hex(16))
}

fn trace_of(traceparent: &str) -> Option<String> {
    let id = traceparent.split('-').nth(1)?;
    let hex = id.len() == 32 && id.bytes().all(|b| b.is_ascii_hexdigit());
    (hex && id.bytes().any(|b| b != b'0')).then(|| id.to_ascii_lowercase())
}

/// `bytes` random bytes, as hex.
pub(crate) fn hex(bytes: usize) -> String {
    let mut buf = vec![0u8; bytes];
    rand_core::OsRng.fill_bytes(&mut buf);
    buf.iter().fold(String::new(), |mut s, b| {
        let _ = write!(s, "{b:02x}");
        s
    })
}

/// The `traceparent` a call made in the trace `id` sends: a span of its own.
pub(crate) fn traceparent(id: &str) -> String {
    format!("00-{id}-{}-01", hex(8))
}

/// The names of a query's parameters whose values a line leaves out:
/// sign-in's codes and state, tokens, passwords and keys.
const SECRETS: [&str; 9] = [
    "code",
    "state",
    "iss",
    "token",
    "access_token",
    "refresh_token",
    "password",
    "secret",
    "key",
];

/// The query of a request to `path`, as its line gives it: each value a
/// secret could be in ([`SECRETS`]) replaced by `_`, its name read as a
/// query is (`%63ode` is `code`), cut to 200 characters; none for
/// sign-in's (`/oauth`), whose query is all codes and state.
pub fn query(path: &str, query: Option<&str>) -> Option<String> {
    let query = query.filter(|_| !path.starts_with("/oauth"))?;
    let pairs = query.split('&').map(|pair| {
        let raw = pair.split('=').next().unwrap_or_default();
        let name = url::form_urlencoded::parse(pair.as_bytes()).next();
        match name.is_some_and(|(n, _)| SECRETS.iter().any(|s| n.eq_ignore_ascii_case(s))) {
            true => format!("{raw}=_"),
            false => pair.to_string(),
        }
    });
    let kept = pairs.collect::<Vec<_>>().join("&");
    Some(kept.chars().take(200).collect())
}

/// A path as a line gives it: no query, and not too long.
pub fn short(path: &str) -> String {
    path.chars().take(120).collect()
}

/// A call made, as the line logged for it says: never its query, a
/// header or a body.
pub(crate) struct Call<'a> {
    pub host: &'a str,
    pub method: &'a str,
    pub path: &'a str,
    pub status: Option<u16>,
    /// Why it failed, before an answer: `timeout`, `connect`, …
    pub error: Option<&'a str>,
    pub took: Duration,
}

/// What a call given up on by what asked for it, not by its wait, is
/// logged as: no failure, at debug.
pub(crate) const ABANDONED: &str = "abandoned";

/// Logs a call made and counts it: at debug, or warn if it failed,
/// answered 5xx or was slow; one [`ABANDONED`] is no failure.
pub(crate) fn called(c: &Call) {
    let ms = millis(c.took);
    let server_error = c.status.filter(|s| *s >= 500).map(|s| s.to_string());
    let error = c.error.or(server_error.as_deref());
    let failed = error.is_some_and(|e| e != ABANDONED);
    let slow = slow(c.took);
    let outcome = c
        .status
        .map_or(error.unwrap_or_default().to_string(), |s| s.to_string());
    let host = record(|s| s.call(c.host, &outcome, ms, failed, slow));
    let rpc = c.path.strip_prefix("/xrpc/");
    macro_rules! line {
        ($level:expr) => {
            tracing::event!(
                target: "hud::outgoing",
                $level,
                server.address = host.as_str(),
                http.request.method = c.method,
                url.path = c.path,
                rpc.method = rpc,
                http.response.status_code = c.status,
                error.type = error,
                duration_ms = ms,
                "outgoing"
            )
        };
    }
    match failed || slow {
        true => line!(Level::WARN),
        false => line!(Level::DEBUG),
    }
}

/// What kind of failure `err` is, and why, without the URL it names.
pub(crate) fn error_type(err: &reqwest::Error) -> String {
    let kind = match () {
        _ if err.is_timeout() => "timeout",
        _ if err.is_connect() => "connect",
        _ if err.is_body() || err.is_decode() => "body",
        _ => "request",
    };
    let mut said = kind.to_string();
    let mut source = std::error::Error::source(err);
    while let Some(cause) = source {
        let _ = write!(said, ": {cause}");
        source = cause.source();
    }
    said
}

/// A request served, as the line logged for it says.
pub struct Served<'a> {
    /// The route it matched, as its template: `/blob/{did}/{cid}`.
    pub route: Option<&'a str>,
    pub method: &'a str,
    pub path: &'a str,
    /// Its query, as [`query`] gives it.
    pub query: Option<&'a str>,
    /// The client, as its `User-Agent` names itself.
    pub agent: Option<&'a str>,
    pub status: u16,
    /// The body's length, when known before it is sent.
    pub bytes: Option<u64>,
    pub took: Duration,
    pub trace: &'a str,
    /// Why it failed, where its answer says.
    pub cause: Option<&'a str>,
    /// Whether it is long-lived on purpose, as an event stream is, so never
    /// slow.
    pub waits: bool,
}

/// Logs a request served and counts it: at error if it failed on this
/// side (5xx), warn if what HUD asked for it failed (502, 504: the call's
/// own line says which) or it was slow, info otherwise, and debug for a
/// health check.
pub fn served(r: &Served) {
    let ms = millis(r.took);
    let failed = r.status >= 500;
    let upstream = matches!(r.status, 502 | 504);
    let slow = slow(r.took) && !r.waits;
    let route = r.route.unwrap_or("fallback");
    let key = (route.to_string(), r.method.to_string(), r.status);
    record(|s| {
        s.served
            .entry(key.clone())
            .or_default()
            .add(ms, failed, slow)
    });
    let quiet = route == "/_health" && !failed && !slow;
    macro_rules! line {
        ($level:expr) => {
            tracing::event!(
                target: "hud::request",
                $level,
                http.route = r.route,
                http.request.method = r.method,
                url.path = r.path,
                url.query = r.query,
                user_agent.original = r.agent,
                http.response.status_code = r.status,
                http.response.body.size = r.bytes,
                duration_ms = ms,
                trace_id = r.trace,
                error = r.cause,
                "request"
            )
        };
    }
    match (failed && !upstream, failed || slow, quiet) {
        (true, _, _) => line!(Level::ERROR),
        (false, true, _) => line!(Level::WARN),
        (false, false, true) => line!(Level::DEBUG),
        (false, false, false) => line!(Level::INFO),
    }
}

/// An MCP tool call, as the line logged for it says: never its arguments
/// or its result.
pub struct Tool<'a> {
    pub name: &'a str,
    /// The caller, signed in.
    pub did: Option<&'a str>,
    /// What the tool answered, where it failed.
    pub error: Option<&'a str>,
    /// Why it answered nothing, as when it asked the caller to sign in.
    pub reason: Option<&'a str>,
    pub took: Duration,
    /// The kit drawing its tree, as it said itself (`kit::Kit`'s `said`).
    pub kit: Option<&'a str>,
}

/// Logs an MCP tool call: `mcp tool failed` at warn, or `mcp tool`, at
/// warn when slow, else info.
pub fn tool(t: &Tool) {
    let ms = millis(t.took);
    let is_error = t.error.is_some();
    macro_rules! line {
        ($level:expr, $message:expr) => {
            tracing::event!(
                target: "hud::mcp",
                $level,
                gen_ai.tool.name = t.name,
                did = t.did,
                kit = t.kit,
                is_error,
                error = t.error,
                reason = t.reason,
                duration_ms = ms,
                $message
            )
        };
    }
    match (is_error, slow(t.took)) {
        (true, _) => line!(Level::WARN, "mcp tool failed"),
        (false, true) => line!(Level::WARN, "mcp tool"),
        (false, false) => line!(Level::INFO, "mcp tool"),
    }
}

/// Logs, at info, an MCP request other than a tool call that was answered
/// a JSON-RPC error: `mcp error`, its method and why.
pub fn rpc_failed(method: &str, error: &str) {
    tracing::event!(target: "hud::mcp", Level::INFO, rpc.method = method, error, "mcp error");
}

/// The bounds of the duration histograms' buckets, in milliseconds.
const BOUNDS_MS: [u64; 11] = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000];

/// How many hosts calls are counted by; the rest are `other`.
const HOSTS: usize = 200;

#[derive(Default, Clone)]
struct Timings {
    count: u64,
    failed: u64,
    slow: u64,
    sum_ms: u64,
    max_ms: u64,
    buckets: [u64; BOUNDS_MS.len()],
}

impl Timings {
    fn add(&mut self, ms: u64, failed: bool, slow: bool) {
        self.count += 1;
        self.failed += u64::from(failed);
        self.slow += u64::from(slow);
        self.sum_ms += ms;
        self.max_ms = self.max_ms.max(ms);
        for (bucket, bound) in self.buckets.iter_mut().zip(BOUNDS_MS) {
            *bucket += u64::from(ms <= bound);
        }
    }

    fn merge(&mut self, other: &Timings) {
        self.count += other.count;
        self.failed += other.failed;
        self.slow += other.slow;
        self.sum_ms += other.sum_ms;
        self.max_ms = self.max_ms.max(other.max_ms);
        for (bucket, more) in self.buckets.iter_mut().zip(other.buckets) {
            *bucket += more;
        }
    }

    fn fields(&self) -> Fields<'static> {
        Fields {
            count: Some(self.count),
            failed: Some(self.failed),
            slow: Some(self.slow),
            mean_ms: Some(self.sum_ms / self.count.max(1)),
            max_ms: Some(self.max_ms),
            ..Fields::default()
        }
    }
}

#[derive(Default)]
struct Stats {
    /// Requests served, by route, method and status.
    served: BTreeMap<(String, String, u16), Timings>,
    /// Calls made, by host and status or failure.
    called: BTreeMap<(String, String), Timings>,
}

impl Stats {
    /// Counts a call to `host`, or to `other` past [`HOSTS`] of them: the
    /// host it was counted under.
    fn call(&mut self, host: &str, outcome: &str, ms: u64, failed: bool, slow: bool) -> String {
        let hosts = self
            .called
            .keys()
            .map(|(h, _)| h.as_str())
            .collect::<std::collections::BTreeSet<_>>();
        let host = match hosts.contains(host) || hosts.len() < HOSTS {
            true => host.to_string(),
            false => "other".to_string(),
        };
        let key = (host.clone(), outcome.to_string());
        self.called.entry(key).or_default().add(ms, failed, slow);
        host
    }
}

/// Since the start, for `/metrics`, and since the last summary.
static TOTAL: LazyLock<Mutex<Stats>> = LazyLock::new(Mutex::default);
static WINDOW: LazyLock<Mutex<Stats>> = LazyLock::new(Mutex::default);

/// Counts into both, answering what the count since the start did.
fn record<T>(count: impl Fn(&mut Stats) -> T) -> T {
    count(
        &mut WINDOW
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner),
    );
    count(
        &mut TOTAL
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner),
    )
}

/// Each cache's hits and misses at the last summary, to say those since.
static LOOKED: LazyLock<Mutex<BTreeMap<&'static str, (u64, u64)>>> = LazyLock::new(Mutex::default);

/// Logs, at info, what was counted since the last summary: requests
/// served, calls made to each host, and the hits and misses of each cache
/// of `lookups` (each cache's since the start, as `Hud::lookups` gives
/// them). Nothing for what saw none.
pub fn summarize(lookups: &[(&'static str, u64, u64)]) {
    let window = std::mem::take(
        &mut *WINDOW
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner),
    );
    let mut served = Timings::default();
    window.served.values().for_each(|t| served.merge(t));
    // A minute with requests says so; one with none, nothing.
    std::iter::once(served)
        .filter(|t| t.count > 0)
        .for_each(|t| note(Level::INFO, "summary: requests", &t.fields()));
    let mut hosts: BTreeMap<&str, Timings> = BTreeMap::new();
    for ((host, _), t) in &window.called {
        hosts.entry(host).or_default().merge(t);
    }
    for (host, t) in hosts {
        let f = Fields {
            host: Some(host),
            ..t.fields()
        };
        note(Level::INFO, "summary: outgoing", &f);
    }
    let mut looked = LOOKED
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    for &(cache, hits, misses) in lookups {
        let (was_hits, was_misses) = looked.insert(cache, (hits, misses)).unwrap_or_default();
        let (hits, misses) = (
            hits.saturating_sub(was_hits),
            misses.saturating_sub(was_misses),
        );
        if hits + misses > 0 {
            let f = Fields {
                cache: Some(cache),
                hits: Some(hits),
                misses: Some(misses),
                ..Fields::default()
            };
            note(Level::INFO, "summary: cache", &f);
        }
    }
}

/// What was counted since the start, in Prometheus's text format: rate,
/// errors and duration by route and by host called, by OpenTelemetry's
/// metric names, and the lookups of each cache of `lookups`.
pub fn metrics(lookups: &[(&'static str, u64, u64)]) -> String {
    let total = TOTAL
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    let mut out = String::new();
    let (version, commit) = BUILD.get().cloned().unwrap_or_default();
    let started = STARTED
        .duration_since(SystemTime::UNIX_EPOCH)
        .unwrap_or_default();
    let _ = writeln!(out, "# TYPE hud_build_info gauge");
    let _ = writeln!(
        out,
        "hud_build_info{{version=\"{}\",commit=\"{}\"}} 1",
        quoted(&version),
        quoted(&commit)
    );
    let _ = writeln!(out, "# TYPE process_start_time_seconds gauge");
    let _ = writeln!(out, "process_start_time_seconds {}", started.as_secs());
    let name = "http_server_request_duration_seconds";
    let _ = writeln!(out, "# TYPE {name} histogram");
    for ((route, method, status), t) in &total.served {
        let labels = format!(
            "http_route=\"{}\",http_request_method=\"{}\",http_response_status_code=\"{status}\"",
            quoted(route),
            quoted(method)
        );
        histogram(&mut out, name, &labels, t);
    }
    let name = "http_client_request_duration_seconds";
    let _ = writeln!(out, "# TYPE {name} histogram");
    for ((host, outcome), t) in &total.called {
        let labels = format!(
            "server_address=\"{}\",outcome=\"{}\"",
            quoted(host),
            quoted(outcome)
        );
        histogram(&mut out, name, &labels, t);
    }
    let _ = writeln!(out, "# TYPE hud_cache_lookups_total counter");
    for (cache, hits, misses) in lookups {
        for (result, n) in [("hit", hits), ("miss", misses)] {
            let _ = writeln!(
                out,
                "hud_cache_lookups_total{{cache=\"{cache}\",result=\"{result}\"}} {n}"
            );
        }
    }
    out
}

fn histogram(out: &mut String, name: &str, labels: &str, t: &Timings) {
    for (bound, n) in BOUNDS_MS.iter().zip(t.buckets) {
        let le = *bound as f64 / 1000.0;
        let _ = writeln!(out, "{name}_bucket{{{labels},le=\"{le}\"}} {n}");
    }
    let _ = writeln!(out, "{name}_bucket{{{labels},le=\"+Inf\"}} {}", t.count);
    let _ = writeln!(out, "{name}_sum{{{labels}}} {}", t.sum_ms as f64 / 1000.0);
    let _ = writeln!(out, "{name}_count{{{labels}}} {}", t.count);
}

/// `value` as a Prometheus label value takes it.
fn quoted(value: &str) -> String {
    value
        .replace('\\', "\\\\")
        .replace('"', "\\\"")
        .replace('\n', "\\n")
}

#[cfg(test)]
pub(crate) mod tests {
    use super::*;

    std::thread_local! {
        /// The lines this thread logs while [`logged`] runs.
        static CAUGHT: std::cell::RefCell<Option<Vec<u8>>> = const { std::cell::RefCell::new(None) };
    }

    /// What is logged on this thread while `f` runs, at every level, as
    /// the lines say it. One subscriber, for every test and thread, logs
    /// everything: a subscriber of each test's own would leave out what
    /// another thread's test, logging at once, had tracing take as never
    /// wanted.
    pub(crate) fn logged(f: impl FnOnce()) -> Vec<Value> {
        static SUBSCRIBED: std::sync::Once = std::sync::Once::new();
        SUBSCRIBED.call_once(|| {
            let layer = tracing_subscriber::fmt::layer()
                .event_format(Lines)
                .with_writer(|| Caught);
            let subscriber = tracing_subscriber::registry()
                .with(EnvFilter::new("trace"))
                .with(layer);
            tracing::subscriber::set_global_default(subscriber).unwrap();
        });
        CAUGHT.with(|c| *c.borrow_mut() = Some(Vec::new()));
        f();
        let caught = CAUGHT.with(|c| c.borrow_mut().take()).unwrap_or_default();
        let text = String::from_utf8(caught).unwrap();
        text.lines()
            .map(|l| serde_json::from_str(l).unwrap())
            .collect()
    }

    /// Writes a line to this thread's [`CAUGHT`], while a test catches them.
    struct Caught;

    impl std::io::Write for Caught {
        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
            let into = |c: &std::cell::RefCell<Option<Vec<u8>>>| {
                c.borrow_mut()
                    .iter_mut()
                    .for_each(|v| v.extend_from_slice(buf))
            };
            CAUGHT.with(into);
            Ok(buf.len())
        }

        fn flush(&mut self) -> std::io::Result<()> {
            Ok(())
        }
    }

    /// Each line goes to stderr and to what keeps them.
    #[test]
    fn lines_are_kept_as_written() {
        use std::io::Write as _;
        let kept = Arc::new(Mutex::new(Vec::new()));
        let into = kept.clone();
        keep_lines(move |line| into.lock().unwrap().push(line.to_vec()));
        let line = b"{\"message\":\"tee\"}\n";
        let mut tee = Tee;
        assert_eq!(tee.write(line).unwrap(), line.len());
        tee.flush().unwrap();
        assert!(kept.lock().unwrap().iter().any(|l| l == line));
        Caught.flush().unwrap();
    }

    #[test]
    fn notes_at_each_level() {
        let lines = logged(|| {
            for level in [Level::ERROR, Level::WARN, Level::INFO, Level::TRACE] {
                let f = Fields {
                    did: Some("did:plc:alice"),
                    error: Some("broke".into()),
                    count: Some(3),
                    ..Fields::default()
                };
                note(level, "said", &f);
            }
        });
        let levels: Vec<&str> = lines.iter().map(|l| l["level"].as_str().unwrap()).collect();
        assert_eq!(levels, ["ERROR", "WARN", "INFO", "DEBUG"]);
        assert_eq!(lines[0]["message"], "said");
        assert_eq!(lines[0]["did"], "did:plc:alice");
        assert_eq!(lines[0]["count"], 3);
        assert_eq!(lines[0]["target"], "hud");
        assert!(lines[0]["timestamp"].as_str().unwrap().ends_with('Z'));
        assert!(lines[0].get("cache").is_none());
        assert!(lines[0].get("trace_id").is_none());
    }

    /// Every kind of value a field holds, as JSON; a line in a trace
    /// names it, unless it names one itself.
    #[test]
    fn fields_and_traces_in_lines() {
        let rt = tokio::runtime::Builder::new_current_thread()
            .build()
            .unwrap();
        let lines = logged(|| {
            rt.block_on(traced("t1".into(), async {
                tracing::info!(n = 1u64, i = -1i64, yes = true, x = 0.5, s = "a", e = %"shown", "kinds");
                tracing::info!(trace_id = "t2", "named");
            }));
        });
        assert_eq!(lines[0]["n"], 1);
        assert_eq!(lines[0]["i"], -1);
        assert_eq!(lines[0]["yes"], true);
        assert_eq!(lines[0]["x"], 0.5);
        assert_eq!(lines[0]["s"], "a");
        assert_eq!(lines[0]["e"], "shown");
        assert_eq!(lines[0]["message"], "kinds");
        assert_eq!(lines[0]["trace_id"], "t1");
        assert_eq!(lines[1]["trace_id"], "t2");
    }

    #[test]
    fn layers_and_settings() {
        let _ = layer::<Registry>();
        assert!(slow(Duration::from_secs(1)));
        assert!(!slow(Duration::from_millis(999)));
        assert_eq!(millis(Duration::from_secs(2)), 2000);
        assert_eq!(short(&"a".repeat(500)).len(), 120);
        // Never set up here: refused either way.
        assert!(set_filter("[[[").unwrap_err().contains("invalid"));
        assert_eq!(set_filter("debug").unwrap_err(), "logging is not set up");
        assert_eq!(commit("$Format:%H$\n"), None);
        assert_eq!(commit(" \n"), None);
        assert_eq!(commit("abc123\n"), Some("abc123"));
    }

    #[test]
    fn panics_are_logged() {
        let lines = logged(|| {
            let before = std::panic::take_hook();
            std::panic::set_hook(Box::new(panicked));
            let _ = std::panic::catch_unwind(|| panic!("on fire"));
            std::panic::set_hook(before);
        });
        assert_eq!(lines[0]["message"], "panicked");
        let error = lines[0]["error"].as_str().unwrap();
        assert!(error.contains("on fire"), "{error}");
        assert!(error.contains("telemetry.rs"), "{error}");
    }

    #[test]
    fn traces() {
        let id = "4bf92f3577b34da6a3ce929d0e0e4736";
        let parent = format!("00-{id}-00f067aa0ba902b7-01");
        assert_eq!(trace_from(Some(&parent)), id);
        assert_eq!(trace_from(Some(&parent.to_uppercase())), id);
        for bad in ["junk", "00-xyz-00f067aa0ba902b7-01"] {
            assert_ne!(trace_from(Some(bad)), id);
        }
        let zeros = format!("00-{}-00f067aa0ba902b7-01", "0".repeat(32));
        assert_eq!(trace_from(Some(&zeros)).len(), 32);
        assert_eq!(trace_from(None).len(), 32);
        assert_eq!(trace_id(), None);
        let sent = traceparent(id);
        assert_eq!(trace_from(Some(&sent)), id);
        assert_eq!(sent.len(), 55);
    }

    /// A task spawned in a trace is in it; one spawned in none, in none.
    #[tokio::test]
    async fn spawned_tasks_keep_their_trace() {
        let within = traced("t".into(), async { spawn(async { trace_id() }).await })
            .await
            .unwrap();
        assert_eq!(within.as_deref(), Some("t"));
        assert_eq!(spawn(async { trace_id() }).await.unwrap(), None);
    }

    #[test]
    fn hosts_past_the_most_are_other() {
        let mut stats = Stats::default();
        for n in 0..HOSTS {
            stats.call(&format!("h{n}"), "200", 1, false, false);
        }
        assert_eq!(stats.call("h0", "200", 1, false, false), "h0");
        assert_eq!(stats.call("new", "200", 1, false, false), "other");
    }

    #[test]
    fn queries_leave_secrets_out() {
        let logged = |q: &str| query("/xrpc/place.hud.getLogs", Some(q)).unwrap();
        assert_eq!(logged("trace=abc&level=warn"), "trace=abc&level=warn");
        assert_eq!(logged("code=abc&ISS=https://pds&x"), "code=_&ISS=_&x");
        // Its name read as a query is, however it is written.
        assert_eq!(
            logged("%63ode=abc&acc%65ss_token=t&refresh+token=r&&x"),
            "%63ode=_&acc%65ss_token=_&refresh+token=r&&x"
        );
        assert_eq!(logged(&"a".repeat(300)).len(), 200);
        assert_eq!(query("/oauth/callback", Some("code=abc")), None);
        assert_eq!(query("/xrpc/place.hud.getLogs", None), None);
    }

    /// The event stream is never slow, however long it is open.
    #[test]
    fn what_waits_is_not_slow() {
        let lines = logged(|| {
            served(&Served {
                route: Some("/xrpc/place.hud.getWorkspaceEvents"),
                method: "GET",
                path: "/xrpc/place.hud.getWorkspaceEvents",
                query: None,
                agent: None,
                status: 200,
                bytes: None,
                took: Duration::from_secs(25),
                trace: "t",
                cause: None,
                waits: true,
            });
        });
        assert_eq!(lines[0]["level"], "INFO");
    }

    #[test]
    fn rpc_errors() {
        let lines = logged(|| rpc_failed("tools/list", "Invalid cursor"));
        assert_eq!(lines[0]["message"], "mcp error");
        assert_eq!(lines[0]["rpc.method"], "tools/list");
        assert_eq!(lines[0]["error"], "Invalid cursor");
    }

    #[test]
    fn tool_calls() {
        let lines = logged(|| {
            for (error, reason, ms) in [
                (Some("no such account"), None, 1),
                (None, None, 5000),
                (None, Some("sign-in asked"), 1),
            ] {
                tool(&Tool {
                    name: "describe_repo",
                    did: Some("did:plc:alice"),
                    error,
                    reason,
                    took: Duration::from_millis(ms),
                    kit: Some("ios 0.3.0"),
                });
            }
        });
        let said: Vec<(&str, &str)> = lines
            .iter()
            .map(|l| (l["level"].as_str().unwrap(), l["message"].as_str().unwrap()))
            .collect();
        assert_eq!(
            said,
            [
                ("WARN", "mcp tool failed"),
                ("WARN", "mcp tool"),
                ("INFO", "mcp tool")
            ]
        );
        assert_eq!(lines[0]["gen_ai.tool.name"], "describe_repo");
        assert_eq!(lines[0]["kit"], "ios 0.3.0");
        assert_eq!(lines[0]["error"], "no such account");
        assert_eq!(lines[0]["is_error"], true);
        assert_eq!(lines[1]["duration_ms"], 5000);
        assert_eq!(lines[2]["reason"], "sign-in asked");
        assert_eq!(lines[2]["is_error"], false);
    }

    #[test]
    fn calls_made() {
        let lines = logged(|| {
            for (status, error, ms) in [
                (Some(200), None, 1),
                (Some(503), None, 1),
                (None, Some("connect: refused"), 1),
                (Some(200), None, 5000),
            ] {
                called(&Call {
                    host: "pds.calls.test",
                    method: "GET",
                    path: "/xrpc/com.atproto.repo.getRecord",
                    status,
                    error,
                    took: Duration::from_millis(ms),
                });
            }
        });
        let levels: Vec<&str> = lines.iter().map(|l| l["level"].as_str().unwrap()).collect();
        assert_eq!(levels, ["DEBUG", "WARN", "WARN", "WARN"]);
        assert_eq!(lines[0]["rpc.method"], "com.atproto.repo.getRecord");
        assert_eq!(lines[0]["server.address"], "pds.calls.test");
        assert_eq!(lines[0]["target"], "hud::outgoing");
        assert_eq!(lines[1]["error.type"], "503");
        assert_eq!(lines[2]["error.type"], "connect: refused");
        assert!(lines[2].get("http.response.status_code").is_none());
        let text = metrics(&[]);
        assert!(text.contains(
            "http_client_request_duration_seconds_count{server_address=\"pds.calls.test\",outcome=\"connect: refused\"} 1"
        ));
    }

    #[test]
    fn served_and_summarized() {
        let lines = logged(|| {
            for (route, status, ms) in [
                (Some("/_health"), 200, 1),
                (Some("/blob/{did}/{cid}"), 200, 2),
                (Some("/blob/{did}/{cid}"), 500, 2),
                // What HUD asked for it failed: its call's line says so.
                (Some("/icon/{domain}"), 502, 2),
                (None, 200, 5000),
            ] {
                served(&Served {
                    route,
                    method: "GET",
                    path: "/blob/did:plc:x/bafy",
                    query: Some("size=thumb"),
                    agent: Some("Mozilla/5.0"),
                    status,
                    bytes: Some(10),
                    took: Duration::from_millis(ms),
                    trace: "t",
                    cause: (status >= 500).then_some("PDS down"),
                    waits: false,
                });
            }
            summarize(&[("docs", 3, 1), ("handles", 0, 0)]);
            // Since then, nothing to summarize: nothing is said of it.
            summarize(&[("docs", 3, 1), ("handles", 0, 0)]);
        });
        let levels: Vec<&str> = lines.iter().map(|l| l["level"].as_str().unwrap()).collect();
        assert_eq!(&levels[..5], ["DEBUG", "INFO", "ERROR", "WARN", "WARN"]);
        assert_eq!(lines[2]["error"], "PDS down");
        assert_eq!(lines[2]["http.route"], "/blob/{did}/{cid}");
        assert_eq!(lines[1]["http.response.status_code"], 200);
        assert_eq!(lines[1]["user_agent.original"], "Mozilla/5.0");
        assert_eq!(lines[1]["url.query"], "size=thumb");
        assert_eq!(lines[1]["trace_id"], "t");
        assert_eq!(lines[1]["target"], "hud::request");
        let summary = |m: &str| lines.iter().filter(|l| l["message"] == m).count();
        assert_eq!(summary("summary: cache"), 1);
        assert!(summary("summary: requests") >= 1);
        let cache = lines
            .iter()
            .find(|l| l["message"] == "summary: cache")
            .unwrap();
        assert_eq!(
            (cache["cache"].clone(), cache["hits"].clone()),
            ("docs".into(), 3.into())
        );
        let text = metrics(&[("docs", 3, 1)]);
        assert!(text.contains("http_route=\"fallback\""));
        assert!(text.contains("hud_cache_lookups_total{cache=\"docs\",result=\"miss\"} 1"));
        assert!(text.contains("process_start_time_seconds"));
        assert!(text.contains("hud_build_info{version=\"\",commit=\"\"} 1"));
        assert_eq!(quoted("a\"b\\c\nd"), "a\\\"b\\\\c\\nd");
    }

    #[tokio::test]
    async fn error_types() {
        let http = reqwest::Client::new();
        // Asked of a listener that never answers, so the timeout comes first.
        let silent = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let never = format!("http://{}/", silent.local_addr().unwrap());
        let timeout = http.get(&never).timeout(Duration::ZERO).send().await;
        assert!(error_type(&timeout.unwrap_err()).starts_with("timeout"));
        let refused = http.get("http://127.0.0.1:9/").send().await.unwrap_err();
        assert!(
            error_type(&refused).starts_with("connect"),
            "{}",
            error_type(&refused)
        );
        let built = http
            .get(&never)
            .header("bad\nname", "x")
            .build()
            .unwrap_err();
        assert!(error_type(&built).starts_with("request"));
        // An answer cut short fails as its body is read.
        let cut = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let url = format!("http://{}/", cut.local_addr().unwrap());
        tokio::spawn(async move {
            use tokio::io::{AsyncReadExt, AsyncWriteExt};
            let (mut s, _) = cut.accept().await.unwrap();
            let _ = s.read(&mut [0; 1024]).await.unwrap();
            let head = b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort";
            s.write_all(head).await.unwrap();
        });
        let res = http.get(&url).send().await.unwrap();
        let body = res.bytes().await.unwrap_err();
        assert!(
            error_type(&body).starts_with("body"),
            "{}",
            error_type(&body)
        );
    }
}