//! Traces, as `spec/OPERATIONS.md` says: each request's trace context, from //! its `traceparent` or new, sampled as `parentbased_always_on`; its span, //! the spans within it, and those of the requests the server makes; and //! their export in the background, in batches, by OTLP over HTTP in //! protobuf, which no request waits on. use super::{Level, line, ops, var}; use axum::body::Bytes; use axum::http::Method; use serde_json::json; use std::collections::VecDeque; use std::sync::{Mutex, OnceLock, PoisonError}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; pub(crate) const INTERNAL: u64 = 1; const SERVER: u64 = 2; const CLIENT: u64 = 3; /// Nanoseconds since the epoch. pub(crate) fn now_ns() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_nanos() as u64 } fn new_id() -> [u8; N] { loop { let id: [u8; N] = std::array::from_fn(|_| rand::random::()); if id.iter().any(|&b| b != 0) { return id; } } } // ------------------------------------------------------------ settings /// The `OTEL_*` variables. pub(crate) struct Settings { /// Where exports go; none if traces are not exported. url: Option, headers: reqwest::header::HeaderMap, gzip: bool, service: String, delay: Duration, max_queue: usize, max_batch: usize, timeout: Duration, } fn number(name: &str, default: u64) -> Result { match var(name) { None => Ok(default), Some(v) => v .trim() .parse() .map_err(|_| format!("{name} is {v:?}, not a whole number of 0 or more")), } } /// `%XX` in a header's value, as the byte it stands for. fn percent_decode(v: &str) -> Option { let b = v.as_bytes(); let mut out = Vec::with_capacity(b.len()); let mut i = 0; while i < b.len() { if b[i] == b'%' { let hex = std::str::from_utf8(b.get(i + 1..i + 3)?).ok()?; out.push(u8::from_str_radix(hex, 16).ok()?); i += 3; } else { out.push(b[i]); i += 1; } } String::from_utf8(out).ok() } /// `key=value` pairs joined by `,`, into `into`, each replacing any of its /// name. Its values are never said, as they may be secrets. fn headers( name: &str, value: Option, into: &mut reqwest::header::HeaderMap, ) -> Result<(), String> { let Some(v) = value else { return Ok(()) }; let bad = || { format!("{name} is not key=value pairs joined by commas, each a header's name and value") }; for pair in v.split(',').map(str::trim).filter(|p| !p.is_empty()) { let (k, v) = pair.split_once('=').ok_or_else(bad)?; let k = reqwest::header::HeaderName::from_bytes(k.trim().to_ascii_lowercase().as_bytes()) .map_err(|_| bad())?; let v = percent_decode(v.trim()) .and_then(|v| reqwest::header::HeaderValue::from_str(&v).ok()) .ok_or_else(bad)?; into.insert(k, v); } Ok(()) } impl Settings { pub fn from_env() -> Result { for name in [ "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", ] { if let Some(p) = var(name).filter(|p| p != "http/protobuf") { return Err(format!( "{name} is {p:?}: the server speaks only http/protobuf" )); } } let mut gzip = false; for name in [ "OTEL_EXPORTER_OTLP_COMPRESSION", "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", ] { match var(name).as_deref() { None => {} Some("gzip") => gzip = true, Some("none") => gzip = false, Some(c) => return Err(format!("{name} is {c:?}, not gzip or none")), } } let disabled = match var("OTEL_SDK_DISABLED") .map(|v| v.to_ascii_lowercase()) .as_deref() { None | Some("false") => false, Some("true") => true, Some(_) => return Err("OTEL_SDK_DISABLED is neither true nor false".into()), }; let parse = |name: &str, v: String| match reqwest::Url::parse(&v) { Ok(u) if matches!(u.scheme(), "http" | "https") && u.has_host() => Ok(u), _ => Err(format!("{name} is not an http or https URL")), }; let url = match ( var("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"), var("OTEL_EXPORTER_OTLP_ENDPOINT"), ) { (Some(u), _) => Some(parse("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", u)?), (None, Some(base)) => { let base = parse("OTEL_EXPORTER_OTLP_ENDPOINT", base)?; let sep = if base.as_str().ends_with('/') { "" } else { "/" }; Some(parse( "OTEL_EXPORTER_OTLP_ENDPOINT", format!("{base}{sep}v1/traces"), )?) } (None, None) => None, }; let mut hs = reqwest::header::HeaderMap::new(); headers( "OTEL_EXPORTER_OTLP_HEADERS", var("OTEL_EXPORTER_OTLP_HEADERS"), &mut hs, )?; headers( "OTEL_EXPORTER_OTLP_TRACES_HEADERS", var("OTEL_EXPORTER_OTLP_TRACES_HEADERS"), &mut hs, )?; let max_queue = number("OTEL_BSP_MAX_QUEUE_SIZE", 2048)? as usize; let max_batch = number("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", 512)? as usize; if max_batch == 0 { return Err("OTEL_BSP_MAX_EXPORT_BATCH_SIZE is 0".into()); } Ok(Settings { url: url.filter(|_| !disabled), headers: hs, gzip, service: var("OTEL_SERVICE_NAME").unwrap_or_else(|| "unknown_service:pds".into()), delay: Duration::from_millis(number("OTEL_BSP_SCHEDULE_DELAY", 5000)?), max_queue, max_batch: max_batch.min(max_queue.max(1)), timeout: Duration::from_millis(number("OTEL_BSP_EXPORT_TIMEOUT", 30_000)?), }) } /// What exports the spans, if they are exported. pub fn exporter(&self) -> Option { let url = self.url.clone()?; let http = reqwest::Client::builder().no_proxy().build().ok()?; let resource = encode_resource(&self.service); Some(Exporter { url, headers: self.headers.clone(), gzip: self.gzip, delay: self.delay, max_queue: self.max_queue, max_batch: self.max_batch, timeout: self.timeout, http, resource, queue: Mutex::new(VecDeque::new()), wake: tokio::sync::Notify::new(), logged: Mutex::new(None), }) } } // ------------------------------------------------------------ context /// A request's trace context: its trace, its own span, the span it is the /// child of, whether it is sampled, the account its token is of, and of a /// `getBlob` the blob it serves, by its account's DID and its CID. pub(crate) struct Ctx { pub trace_id: [u8; 16], pub span_id: [u8; 8], parent: Option<[u8; 8]>, pub sampled: bool, pub did: OnceLock, pub blob: OnceLock<(String, String)>, } tokio::task_local! { /// The trace context of the request being answered. pub(crate) static CTX: std::sync::Arc; } /// The trace context of the request being answered, if any. pub(crate) fn current() -> Option> { CTX.try_with(std::sync::Arc::clone).ok() } /// A `traceparent` (W3C Trace Context), if it is valid: its trace id, the /// parent's span id, and whether it is sampled. fn traceparent(h: &str) -> Option<([u8; 16], [u8; 8], bool)> { let lower_hex = |s: &str| { s.bytes() .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) }; let version = h.get(..2)?; if !lower_hex(version) || version == "ff" { return None; } // Version 00 is exactly its fields; a later one may have more after them. if h.len() < 55 || (version == "00" && h.len() != 55) || (h.len() > 55 && h.as_bytes()[55] != b'-') { return None; } let b = h.as_bytes(); if b[2] != b'-' || b[35] != b'-' || b[52] != b'-' { return None; } let (trace, parent, flags) = (&h[3..35], &h[36..52], &h[53..55]); if !lower_hex(trace) || !lower_hex(parent) || !lower_hex(flags) { return None; } let trace: [u8; 16] = hex::decode(trace).ok()?.try_into().ok()?; let parent: [u8; 8] = hex::decode(parent).ok()?.try_into().ok()?; if trace == [0; 16] || parent == [0; 8] { return None; } Some((trace, parent, u8::from_str_radix(flags, 16).ok()? & 1 == 1)) } impl Ctx { /// A request's, from its `traceparent` if it is valid; else the root of a new trace. pub fn of(header: Option<&str>) -> Ctx { let (trace_id, parent, sampled) = match header.and_then(traceparent) { Some((t, p, s)) => (t, Some(p), s), None => (new_id(), None, true), }; Ctx { trace_id, span_id: new_id(), parent, sampled, did: OnceLock::new(), blob: OnceLock::new(), } } /// A span within this one's, from `start` to now. pub fn child( &self, name: String, kind: u64, start: u64, attrs: Vec<(&'static str, Value)>, error: bool, ) -> Span { Span { trace_id: self.trace_id, span_id: new_id(), parent: Some(self.span_id), name, kind, start, end: now_ns(), attrs, error, } } } // ------------------------------------------------------------ spans pub(crate) enum Value { Str(String), Int(i64), } pub(crate) struct Span { trace_id: [u8; 16], span_id: [u8; 8], parent: Option<[u8; 8]>, name: String, kind: u64, start: u64, end: u64, attrs: Vec<(&'static str, Value)>, /// Whether its status is `ERROR`; else it is unset. error: bool, } /// The HTTP methods OpenTelemetry knows; any other is `_OTHER`. fn method_name(m: &Method) -> &'static str { match *m { Method::GET => "GET", Method::HEAD => "HEAD", Method::POST => "POST", Method::PUT => "PUT", Method::DELETE => "DELETE", Method::CONNECT => "CONNECT", Method::OPTIONS => "OPTIONS", Method::TRACE => "TRACE", Method::PATCH => "PATCH", _ => "_OTHER", } } /// A request's span, from `start` to now. pub(crate) fn server_span( ctx: &Ctx, method: &Method, path: &str, route: &str, status: u16, start: u64, ) -> Span { let method = method_name(method); let http_route = match route { "_unknown" => None, r if r.starts_with('/') => Some(r.to_string()), r => Some(format!("/xrpc/{r}")), }; let mut attrs = vec![ ("http.request.method", Value::Str(method.into())), ("url.path", Value::Str(path.into())), ("url.scheme", Value::Str("http".into())), ("http.response.status_code", Value::Int(status.into())), ]; if let Some(r) = &http_route { attrs.push(("http.route", Value::Str(r.clone()))); } if status >= 500 { attrs.push(("error.type", Value::Str(status.to_string()))); } let name = http_route.map_or_else(|| method.to_string(), |r| format!("{method} {r}")); Span { trace_id: ctx.trace_id, span_id: ctx.span_id, parent: ctx.parent, name, kind: SERVER, start, end: now_ns(), attrs, error: status >= 500, } } /// Where the server sends a request of its own: the PLC directory, a /// crawler, a service it forwards to or reads from (the AppView, those /// `atproto-proxy` names), a `did:web` document, a client's metadata or /// keys (OAuth's, or an attesting client's), a space's notification, a /// permission set's record, a handle's `/.well-known/atproto-did`, or the /// bucket blobs are kept in. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum Target { Plc, Crawler, Service, DidWeb, Client, Space, Lexicon, Handle, Blobstore, } impl Target { pub(super) fn label(self) -> &'static str { match self { Target::Plc => "plc", Target::Crawler => "crawler", Target::Service => "service", Target::DidWeb => "did-web", Target::Client => "client", Target::Space => "space", Target::Lexicon => "lexicon", Target::Handle => "handle", Target::Blobstore => "blobstore", } } /// The line a failure of a request to it is logged with, and at what level. fn failure(self) -> (&'static str, Level) { match self { Target::Plc => ("plc failure", Level::Warn), Target::Crawler => ("crawler failure", Level::Warn), Target::Service => ("service failure", Level::Debug), Target::DidWeb => ("did:web failure", Level::Debug), Target::Client => ("client failure", Level::Debug), Target::Space => ("space notification failure", Level::Debug), Target::Lexicon => ("lexicon failure", Level::Debug), Target::Handle => ("handle failure", Level::Debug), Target::Blobstore => ("blobstore failure", Level::Warn), } } } /// What a request to the bucket does, as its failure line names it: `put`, /// `get` or `delete`. fn blobstore_op(m: &Method) -> &'static str { match *m { Method::PUT => "put", Method::DELETE => "delete", _ => "get", } } /// Send a request of the server's own to `target`, as `execute` does. pub(crate) async fn send( target: Target, req: reqwest::RequestBuilder, ) -> reqwest::Result { let (client, request) = req.build_split(); execute(target, &client, request?).await } /// Send a request of the server's own to `target` on `client`, as the child /// of the request being answered or the root of a trace of its own, /// carrying a `traceparent` naming its span: counted, timed, its span /// exported, and a failure logged. Every request the server makes goes out /// here. One given up before its answer, as by a caller's timeout, is a /// `timeout`. pub(crate) async fn execute( target: Target, client: &reqwest::Client, mut request: reqwest::Request, ) -> reqwest::Result { let parent = current(); let (trace_id, parent_id, sampled) = match &parent { Some(c) => (c.trace_id, Some(c.span_id), c.sampled), None => (new_id(), None, true), }; let span_id: [u8; 8] = new_id(); let tp = format!( "00-{}-{}-{}", hex::encode(trace_id), hex::encode(span_id), if sampled { "01" } else { "00" } ); if let Ok(v) = reqwest::header::HeaderValue::from_str(&tp) { request.headers_mut().insert("traceparent", v); } let mut out = Outgoing { target, trace_id, span_id, parent: parent_id, sampled, method: request.method().clone(), url: request.url().clone(), start: now_ns(), started: Instant::now(), done: false, }; let res = client.execute(request).await; match &res { Ok(r) if r.status().is_success() => out.finish(Some(r.status().as_u16()), None), Ok(r) => out.finish( Some(r.status().as_u16()), Some(r.status().as_u16().to_string()), ), Err(e) if e.is_timeout() => out.finish(None, Some("timeout".into())), Err(_) => out.finish(None, Some("no answer".into())), } res } /// A request of the server's own on its way: once it is answered, fails, or /// is given up, its metrics, failure line and span. struct Outgoing { target: Target, trace_id: [u8; 16], span_id: [u8; 8], parent: Option<[u8; 8]>, sampled: bool, method: Method, url: reqwest::Url, start: u64, started: Instant, done: bool, } impl Outgoing { fn finish(&mut self, status: Option, error: Option) { self.done = true; let target = self.target; ops().metrics.outbound( target, error.is_none(), self.started.elapsed().as_secs_f64(), ); if let Some(why) = &error { let (msg, level) = target.failure(); // Only a directory's paths, and the bucket's keys, name a DID, // and nothing else of a URL is logged. let segments = || self.url.path_segments().into_iter().flatten(); let did = match target { Target::Plc => segments() .find(|p| p.starts_with("did:")) .map(str::to_string), Target::Blobstore => segments() .map(|p| percent_encoding::percent_decode_str(p).decode_utf8_lossy()) .find(|p| p.starts_with("did:")) .map(|p| p.into_owned()), _ => None, }; let op = (target == Target::Blobstore).then(|| blobstore_op(&self.method)); line( level, msg, &[ ("op", json!(op)), ("error", json!(why)), ("status", json!(status)), ("did", json!(did)), ], ); } if self.sampled && let Some(e) = &ops().exporter { // A query, a fragment or credentials of a URL are never in a span. let mut full = self.url.clone(); let _ = full.set_username(""); let _ = full.set_password(None); full.set_query(None); full.set_fragment(None); let mut attrs = vec![ ( "http.request.method", Value::Str(method_name(&self.method).into()), ), ( "server.address", Value::Str(self.url.host_str().unwrap_or_default().into()), ), ( "server.port", Value::Int(self.url.port_or_known_default().unwrap_or(0).into()), ), ("url.full", Value::Str(full.to_string())), ]; if let Some(s) = status { attrs.push(("http.response.status_code", Value::Int(s.into()))); } if let Some(why) = &error { attrs.push(( "error.type", Value::Str(why.replace("no answer", "connection")), )); } let name = method_name(&self.method).to_string(); e.push(Span { trace_id: self.trace_id, span_id: self.span_id, parent: self.parent, name, kind: CLIENT, start: self.start, end: now_ns(), attrs, error: error.is_some(), }); } } } impl Drop for Outgoing { fn drop(&mut self) { if !self.done { self.finish(None, Some("timeout".into())); } } } // ------------------------------------------------------------ export /// The spans held for export, and what exports them. pub(crate) struct Exporter { url: reqwest::Url, headers: reqwest::header::HeaderMap, gzip: bool, delay: Duration, max_queue: usize, max_batch: usize, timeout: Duration, http: reqwest::Client, /// The `Resource` every export names, encoded. resource: Vec, queue: Mutex>, /// Told once a batch is full. wake: tokio::sync::Notify, /// When an export last failed and was said to: none since one went. logged: Mutex>, } impl Exporter { /// Hold a span for export; dropped if the queue is full. pub fn push(&self, s: Span) { let mut q = self.queue.lock().unwrap_or_else(PoisonError::into_inner); if q.len() >= self.max_queue { return; } q.push_back(s); if q.len() == self.max_batch { self.wake.notify_one(); } } /// Export what is held every `delay`, or once a batch is full. pub async fn run(&'static self) { loop { tokio::select! { () = tokio::time::sleep(self.delay) => {} () = self.wake.notified() => {} } self.drain(None).await; } } /// Export what is held until it is all gone, or `until` passes. pub async fn flush(&self, until: Instant) { let _ = tokio::time::timeout_at(until.into(), self.drain(Some(until))).await; } async fn drain(&self, until: Option) { loop { let batch: Vec = { let mut q = self.queue.lock().unwrap_or_else(PoisonError::into_inner); let n = q.len().min(self.max_batch); q.drain(..n).collect() }; if batch.is_empty() { return; } self.export(&batch, until).await; } } /// Post one batch, tried again until the export timeout has passed since /// the first try, after which its spans are dropped. async fn export(&self, batch: &[Span], until: Option) { let body = Bytes::from(self.encode(batch)); let deadline = (Instant::now() + self.timeout) .min(until.unwrap_or_else(|| Instant::now() + self.timeout)); let mut pause = Duration::from_millis(100); let why = loop { let left = deadline.saturating_duration_since(Instant::now()); if left.is_zero() { break "timeout".to_string(); } let mut req = self .http .post(self.url.clone()) .headers(self.headers.clone()) .header(reqwest::header::CONTENT_TYPE, "application/x-protobuf") .timeout(left); if self.gzip { req = req.header(reqwest::header::CONTENT_ENCODING, "gzip"); } let why = match req.body(body.clone()).send().await { Ok(res) if res.status().is_success() => { *self.logged.lock().unwrap_or_else(PoisonError::into_inner) = None; return; } Ok(res) if ![429, 502, 503, 504].contains(&res.status().as_u16()) => { break format!("status {}", res.status().as_u16()); } Ok(res) => format!("status {}", res.status().as_u16()), Err(e) if e.is_timeout() => "timeout".to_string(), Err(_) => "no answer".to_string(), }; let left = deadline.saturating_duration_since(Instant::now()); if left <= pause { break why; } tokio::time::sleep(pause).await; pause = (pause * 2).min(Duration::from_secs(1)); }; let mut logged = self.logged.lock().unwrap_or_else(PoisonError::into_inner); if logged.is_none_or(|t| t.elapsed() >= Duration::from_secs(60)) { *logged = Some(Instant::now()); drop(logged); line( Level::Warn, "trace export failed", &[("error", json!(why)), ("spans", json!(batch.len()))], ); } } /// An `ExportTraceServiceRequest` of `spans`, gzipped if it is to be. fn encode(&self, spans: &[Span]) -> Vec { let mut scope_spans = Vec::new(); message(&mut scope_spans, 1, |m| { string(m, 1, "pds"); string(m, 2, super::VERSION); }); for s in spans { message(&mut scope_spans, 2, |m| encode_span(m, s)); } let mut resource_spans = Vec::new(); bytes(&mut resource_spans, 1, &self.resource); bytes(&mut resource_spans, 2, &scope_spans); let mut out = Vec::new(); bytes(&mut out, 1, &resource_spans); if !self.gzip { return out; } use std::io::Write; let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); let _ = gz.write_all(&out); gz.finish().unwrap_or_default() } } // ------------------------------------------------------------ protobuf fn varint(out: &mut Vec, mut v: u64) { while v >= 0x80 { out.push((v as u8) | 0x80); v >>= 7; } out.push(v as u8); } fn key(out: &mut Vec, field: u64, wire: u64) { varint(out, (field << 3) | wire); } fn bytes(out: &mut Vec, field: u64, b: &[u8]) { key(out, field, 2); varint(out, b.len() as u64); out.extend_from_slice(b); } fn string(out: &mut Vec, field: u64, s: &str) { bytes(out, field, s.as_bytes()); } fn fixed64(out: &mut Vec, field: u64, v: u64) { key(out, field, 1); out.extend_from_slice(&v.to_le_bytes()); } fn message(out: &mut Vec, field: u64, f: impl FnOnce(&mut Vec)) { let mut m = Vec::new(); f(&mut m); bytes(out, field, &m); } /// A `KeyValue`, as field `field`. fn attribute(out: &mut Vec, field: u64, k: &str, v: &Value) { message(out, field, |kv| { string(kv, 1, k); message(kv, 2, |any| match v { Value::Str(s) => string(any, 1, s), Value::Int(i) => { key(any, 3, 0); varint(any, *i as u64); } }); }); } /// The `Resource`: the service's name and version. fn encode_resource(service: &str) -> Vec { let mut out = Vec::new(); attribute(&mut out, 1, "service.name", &Value::Str(service.into())); attribute( &mut out, 1, "service.version", &Value::Str(super::VERSION.into()), ); out } fn encode_span(m: &mut Vec, s: &Span) { bytes(m, 1, &s.trace_id); bytes(m, 2, &s.span_id); if let Some(p) = &s.parent { bytes(m, 4, p); } string(m, 5, &s.name); key(m, 6, 0); varint(m, s.kind); fixed64(m, 7, s.start); fixed64(m, 8, s.end.max(s.start)); for (k, v) in &s.attrs { attribute(m, 9, k, v); } if s.error { message(m, 15, |st| { key(st, 3, 0); varint(st, 2); }); } } #[cfg(test)] mod tests { use super::*; #[test] fn traceparents() { let t = "4bf92f3577b34da6a3ce929d0e0e4736"; let p = "00f067aa0ba902b7"; assert_eq!( traceparent(&format!("00-{t}-{p}-01")).map(|x| x.2), Some(true) ); assert_eq!( traceparent(&format!("00-{t}-{p}-00")).map(|x| x.2), Some(false) ); assert!(traceparent(&format!("01-{t}-{p}-01-more")).is_some()); assert!(traceparent(&format!("00-{t}-{p}-01-more")).is_none()); assert!(traceparent(&format!("ff-{t}-{p}-01")).is_none()); assert!(traceparent(&format!("00-{}-{p}-01", t.to_uppercase())).is_none()); assert!(traceparent(&format!("00-{}-{p}-01", "0".repeat(32))).is_none()); assert!(traceparent(&format!("00-{t}-{}-01", "0".repeat(16))).is_none()); assert!(traceparent("garbage").is_none()); } #[test] fn header_values_are_percent_decoded() { let mut h = reqwest::header::HeaderMap::new(); headers("H", Some("a=b%20c, X-Key = k%3Dv ,".into()), &mut h).unwrap(); assert_eq!(h["a"], "b c"); assert_eq!(h["x-key"], "k=v"); } }