//! What the server logs of what it serves (`hud_core::telemetry`): a line //! for each request, `request`, in a trace of its own or the one its W3C //! `traceparent` names, which every line logged while serving it carries //! and the answer gives back as `x-trace-id`; and `/metrics` and `PUT /log` //! on a listener of their own (`HUD_METRICS`), kept off the public port. use axum::Router; use axum::extract::{MatchedPath, Request, State}; use axum::http::{HeaderValue, StatusCode, header}; use axum::middleware::Next; use axum::response::{IntoResponse, Response}; use axum::routing::{get, put}; use hud_core::Hud; use hud_core::telemetry; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; /// Why a response failed, for the line logged for it. #[derive(Clone)] pub struct Cause(pub String); /// The route a request matched, as the routing finds it. #[derive(Clone, Default)] struct Route(Arc>); /// `routes`, each request through them logged and counted, in its trace. pub fn observed(routes: Router) -> Router { routes .route_layer(axum::middleware::from_fn(matched)) .layer(axum::middleware::from_fn(observe)) } /// The routes long-lived on purpose, never slow however long they are /// open: the workspace's event stream. const WAITS: [&str; 1] = ["/xrpc/place.hud.getWorkspaceEvents"]; async fn matched(req: Request, next: Next) -> Response { let route = req.extensions().get::().cloned().unwrap_or_default(); let template = req.extensions().get::(); let template = template.map_or("", MatchedPath::as_str); let _ = route.0.set(crate::xrpc::route(template, req.uri().path())); next.run(req).await } /// Names the route `req` is logged and counted by, where none of the /// routes took it: the web app's pages, `web`. pub(crate) fn name(req: &Request, route: &str) { let named = req.extensions().get::().cloned().unwrap_or_default(); let _ = named.0.set(route.to_string()); } async fn observe(mut req: Request, next: Next) -> Response { let started = Instant::now(); let route = Route::default(); req.extensions_mut().insert(route.clone()); let method = req.method().to_string(); let path = telemetry::short(req.uri().path()); let query = telemetry::query(&path, req.uri().query()); let agent = req.headers().get(header::USER_AGENT); let agent = agent.and_then(|v| v.to_str().ok()).map(str::to_string); let parent = req.headers().get("traceparent"); let trace = telemetry::trace_from(parent.and_then(|v| v.to_str().ok())); let mut res = telemetry::traced(trace.clone(), next.run(req)).await; let id = HeaderValue::try_from(trace.as_str()).expect("hex is a header value"); res.headers_mut().insert("x-trace-id", id); let cause = res.extensions().get::(); let bytes = axum::body::HttpBody::size_hint(res.body()).exact(); let matched = route.0.get().map(String::as_str); telemetry::served(&telemetry::Served { route: matched, method: &method, path: &path, query: query.as_deref(), agent: agent.as_deref(), status: res.status().as_u16(), bytes, took: started.elapsed(), trace: &trace, cause: cause.map(|c| c.0.as_str()), waits: matched.is_some_and(|r| WAITS.contains(&r)), }); res } /// What is answered on `HUD_METRICS`'s listener: the counts at /// `/metrics`, and `PUT /log` setting what is logged, as `RUST_LOG` does. pub fn admin(hud: Arc) -> Router { Router::new() .route("/metrics", get(metrics)) .route("/log", put(level)) .with_state(hud) } /// Answers [`admin`] on `listener`, until the server stops. pub fn serve_admin( listener: tokio::net::TcpListener, hud: Arc, ) -> impl Future> { std::future::IntoFuture::into_future(axum::serve(listener, admin(hud))) } async fn metrics(State(hud): State>) -> Response { let kind = "text/plain; version=0.0.4"; let text = telemetry::metrics(&hud.lookups()); ([(header::CONTENT_TYPE, kind)], text).into_response() } async fn level(body: String) -> Response { match telemetry::set_filter(body.trim()) { Ok(()) => format!("logging {}\n", body.trim()).into_response(), Err(err) => (StatusCode::BAD_REQUEST, format!("{err}\n")).into_response(), } } /// Logs a summary every `every`: the requests, calls and cache lookups /// since the last (`telemetry::summarize`). pub async fn summaries(hud: Arc, every: Duration) { let mut tick = tokio::time::interval(every); loop { tick.tick().await; telemetry::summarize(&hud.lookups()); } } #[cfg(test)] mod tests { use super::*; use axum::body::Body; use tower::ServiceExt; fn hud() -> Arc { let dir = std::env::temp_dir().join(format!("hud-telemetry-{}", atproto::jose::random(6))); Hud::new(hud_core::Config { port: 0, bind: "::".parse().unwrap(), public_origin: "https://hud.example".into(), cache_dir: dir.join("cache"), data_dir: dir, blob_cache_bytes: 0, cache_memory_bytes: 1 << 20, oauth: false, oauth_scope: hud_core::scope::asked(hud_core::view::views(), true, false), oauth_include: None, workspace: true, spaces: false, plc_url: "http://127.0.0.1:9".into(), handle_resolver: None, allow_private: true, backlinks: hud_core::backlinks::Backlinks::None, favicons: None, image_cdn: None, relay: None, sign_up_pds: hud_core::config::SIGN_UP_PDS.into(), client: Default::default(), operators: Vec::new(), logs: false, logs_keep: hud_core::logs::KEEP, }) .unwrap() } /// Each answer names its trace: the one its `traceparent` names, else /// a new one; a request no route takes is answered and logged too. #[tokio::test] async fn answers_name_their_trace() { let hud = hud(); let id = "4bf92f3577b34da6a3ce929d0e0e4736"; let parent = format!("00-{id}-00f067aa0ba902b7-01"); let asked = |path: &str, parent: Option<&str>| { let mut req = Request::get(path).header(header::USER_AGENT, "test"); if let Some(p) = parent { req = req.header("traceparent", p); } crate::router(hud.clone(), None).oneshot(req.body(Body::empty()).unwrap()) }; let r = asked("/_health", Some(&parent)).await.unwrap(); assert_eq!(r.headers()["x-trace-id"], id); let r = asked("/nowhere?code=x", None).await.unwrap(); assert_eq!(r.status(), StatusCode::NOT_FOUND); let own = r.headers()["x-trace-id"].to_str().unwrap(); assert!(own.len() == 32 && own != id, "{own}"); let text = telemetry::metrics(&hud.lookups()); assert!(text.contains("http_route=\"/_health\""), "{text}"); assert!(text.contains("http_route=\"fallback\""), "{text}"); } #[tokio::test] async fn metrics_and_levels_on_their_own_listener() { let hud = hud(); let get = |path: &str| Request::get(path).body(Body::empty()).unwrap(); let r = admin(hud.clone()).oneshot(get("/metrics")).await.unwrap(); assert_eq!(r.status(), StatusCode::OK); assert_eq!( r.headers()[header::CONTENT_TYPE], "text/plain; version=0.0.4" ); let body = axum::body::to_bytes(r.into_body(), usize::MAX) .await .unwrap(); let text = String::from_utf8(body.to_vec()).unwrap(); assert!(text.contains("hud_cache_lookups_total{cache=\"docs\",result=\"hit\"} 0")); // Logging is not set up in a test: refused, saying why. let put = Request::put("/log").body(Body::from("debug")).unwrap(); let r = admin(hud.clone()).oneshot(put).await.unwrap(); assert_eq!(r.status(), StatusCode::BAD_REQUEST); // Served on a listener of its own. let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let at = listener.local_addr().unwrap(); let serving = tokio::spawn(serve_admin(listener, hud)); let mut s = tokio::net::TcpStream::connect(at).await.unwrap(); use tokio::io::{AsyncReadExt, AsyncWriteExt}; s.write_all(b"GET /metrics HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n") .await .unwrap(); let mut answer = String::new(); s.read_to_string(&mut answer).await.unwrap(); assert!(answer.starts_with("HTTP/1.1 200"), "{answer}"); serving.abort(); } #[tokio::test(start_paused = true)] async fn summarized_each_minute() { let going = tokio::spawn(summaries(hud(), Duration::from_secs(60))); tokio::time::sleep(Duration::from_secs(61)).await; assert!(!going.is_finished()); going.abort(); } }