chadtmiller.com/monolith
An atproto PDS
0
0
//! Traces, as `spec/OPERATIONS.md` says: each request's trace context, from
//! its `traceparent` or new, sampled as `parentbased_always_on`; its span,
//! the spans within it, and those of the requests the server makes; and
//! their export in the background, in batches, by OTLP over HTTP in
//! protobuf, which no request waits on.
use super::{Level, line, ops, var};
use axum::body::Bytes;
use axum::http::Method;
use serde_json::json;
use std::collections::VecDeque;
use std::sync::{Mutex, OnceLock, PoisonError};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
pub(crate) const INTERNAL: u64 = 1;
const SERVER: u64 = 2;
const CLIENT: u64 = 3;
/// Nanoseconds since the epoch.
pub(crate) fn now_ns() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as u64
}
fn new_id<const N: usize>() -> [u8; N] {
loop {
let id: [u8; N] = std::array::from_fn(|_| rand::random::<u8>());
if id.iter().any(|&b| b != 0) {
return id;
}
}
}
// ------------------------------------------------------------ settings
/// The `OTEL_*` variables.
pub(crate) struct Settings {
/// Where exports go; none if traces are not exported.
url: Option<reqwest::Url>,
headers: reqwest::header::HeaderMap,
gzip: bool,
service: String,
delay: Duration,
max_queue: usize,
max_batch: usize,
timeout: Duration,
}
fn number(name: &str, default: u64) -> Result<u64, String> {
match var(name) {
None => Ok(default),
Some(v) => v
.trim()
.parse()
.map_err(|_| format!("{name} is {v:?}, not a whole number of 0 or more")),
}
}
/// `%XX` in a header's value, as the byte it stands for.
fn percent_decode(v: &str) -> Option<String> {
let b = v.as_bytes();
let mut out = Vec::with_capacity(b.len());
let mut i = 0;
while i < b.len() {
if b[i] == b'%' {
let hex = std::str::from_utf8(b.get(i + 1..i + 3)?).ok()?;
out.push(u8::from_str_radix(hex, 16).ok()?);
i += 3;
} else {
out.push(b[i]);
i += 1;
}
}
String::from_utf8(out).ok()
}
/// `key=value` pairs joined by `,`, into `into`, each replacing any of its
/// name. Its values are never said, as they may be secrets.
fn headers(
name: &str,
value: Option<String>,
into: &mut reqwest::header::HeaderMap,
) -> Result<(), String> {
let Some(v) = value else { return Ok(()) };
let bad = || {
format!("{name} is not key=value pairs joined by commas, each a header's name and value")
};
for pair in v.split(',').map(str::trim).filter(|p| !p.is_empty()) {
let (k, v) = pair.split_once('=').ok_or_else(bad)?;
let k = reqwest::header::HeaderName::from_bytes(k.trim().to_ascii_lowercase().as_bytes())
.map_err(|_| bad())?;
let v = percent_decode(v.trim())
.and_then(|v| reqwest::header::HeaderValue::from_str(&v).ok())
.ok_or_else(bad)?;
into.insert(k, v);
}
Ok(())
}
impl Settings {
pub fn from_env() -> Result<Settings, String> {
for name in [
"OTEL_EXPORTER_OTLP_PROTOCOL",
"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL",
] {
if let Some(p) = var(name).filter(|p| p != "http/protobuf") {
return Err(format!(
"{name} is {p:?}: the server speaks only http/protobuf"
));
}
}
let mut gzip = false;
for name in [
"OTEL_EXPORTER_OTLP_COMPRESSION",
"OTEL_EXPORTER_OTLP_TRACES_COMPRESSION",
] {
match var(name).as_deref() {
None => {}
Some("gzip") => gzip = true,
Some("none") => gzip = false,
Some(c) => return Err(format!("{name} is {c:?}, not gzip or none")),
}
}
let disabled = match var("OTEL_SDK_DISABLED")
.map(|v| v.to_ascii_lowercase())
.as_deref()
{
None | Some("false") => false,
Some("true") => true,
Some(_) => return Err("OTEL_SDK_DISABLED is neither true nor false".into()),
};
let parse = |name: &str, v: String| match reqwest::Url::parse(&v) {
Ok(u) if matches!(u.scheme(), "http" | "https") && u.has_host() => Ok(u),
_ => Err(format!("{name} is not an http or https URL")),
};
let url = match (
var("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"),
var("OTEL_EXPORTER_OTLP_ENDPOINT"),
) {
(Some(u), _) => Some(parse("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", u)?),
(None, Some(base)) => {
let base = parse("OTEL_EXPORTER_OTLP_ENDPOINT", base)?;
let sep = if base.as_str().ends_with('/') {
""
} else {
"/"
};
Some(parse(
"OTEL_EXPORTER_OTLP_ENDPOINT",
format!("{base}{sep}v1/traces"),
)?)
}
(None, None) => None,
};
let mut hs = reqwest::header::HeaderMap::new();
headers(
"OTEL_EXPORTER_OTLP_HEADERS",
var("OTEL_EXPORTER_OTLP_HEADERS"),
&mut hs,
)?;
headers(
"OTEL_EXPORTER_OTLP_TRACES_HEADERS",
var("OTEL_EXPORTER_OTLP_TRACES_HEADERS"),
&mut hs,
)?;
let max_queue = number("OTEL_BSP_MAX_QUEUE_SIZE", 2048)? as usize;
let max_batch = number("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", 512)? as usize;
if max_batch == 0 {
return Err("OTEL_BSP_MAX_EXPORT_BATCH_SIZE is 0".into());
}
Ok(Settings {
url: url.filter(|_| !disabled),
headers: hs,
gzip,
service: var("OTEL_SERVICE_NAME").unwrap_or_else(|| "unknown_service:pds".into()),
delay: Duration::from_millis(number("OTEL_BSP_SCHEDULE_DELAY", 5000)?),
max_queue,
max_batch: max_batch.min(max_queue.max(1)),
timeout: Duration::from_millis(number("OTEL_BSP_EXPORT_TIMEOUT", 30_000)?),
})
}
/// What exports the spans, if they are exported.
pub fn exporter(&self) -> Option<Exporter> {
let url = self.url.clone()?;
let http = reqwest::Client::builder().no_proxy().build().ok()?;
let resource = encode_resource(&self.service);
Some(Exporter {
url,
headers: self.headers.clone(),
gzip: self.gzip,
delay: self.delay,
max_queue: self.max_queue,
max_batch: self.max_batch,
timeout: self.timeout,
http,
resource,
queue: Mutex::new(VecDeque::new()),
wake: tokio::sync::Notify::new(),
logged: Mutex::new(None),
})
}
}
// ------------------------------------------------------------ context
/// A request's trace context: its trace, its own span, the span it is the
/// child of, whether it is sampled, the account its token is of, and of a
/// `getBlob` the blob it serves, by its account's DID and its CID.
pub(crate) struct Ctx {
pub trace_id: [u8; 16],
pub span_id: [u8; 8],
parent: Option<[u8; 8]>,
pub sampled: bool,
pub did: OnceLock<String>,
pub blob: OnceLock<(String, String)>,
}
tokio::task_local! {
/// The trace context of the request being answered.
pub(crate) static CTX: std::sync::Arc<Ctx>;
}
/// The trace context of the request being answered, if any.
pub(crate) fn current() -> Option<std::sync::Arc<Ctx>> {
CTX.try_with(std::sync::Arc::clone).ok()
}
/// A `traceparent` (W3C Trace Context), if it is valid: its trace id, the
/// parent's span id, and whether it is sampled.
fn traceparent(h: &str) -> Option<([u8; 16], [u8; 8], bool)> {
let lower_hex = |s: &str| {
s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
};
let version = h.get(..2)?;
if !lower_hex(version) || version == "ff" {
return None;
}
// Version 00 is exactly its fields; a later one may have more after them.
if h.len() < 55
|| (version == "00" && h.len() != 55)
|| (h.len() > 55 && h.as_bytes()[55] != b'-')
{
return None;
}
let b = h.as_bytes();
if b[2] != b'-' || b[35] != b'-' || b[52] != b'-' {
return None;
}
let (trace, parent, flags) = (&h[3..35], &h[36..52], &h[53..55]);
if !lower_hex(trace) || !lower_hex(parent) || !lower_hex(flags) {
return None;
}
let trace: [u8; 16] = hex::decode(trace).ok()?.try_into().ok()?;
let parent: [u8; 8] = hex::decode(parent).ok()?.try_into().ok()?;
if trace == [0; 16] || parent == [0; 8] {
return None;
}
Some((trace, parent, u8::from_str_radix(flags, 16).ok()? & 1 == 1))
}
impl Ctx {
/// A request's, from its `traceparent` if it is valid; else the root of a new trace.
pub fn of(header: Option<&str>) -> Ctx {
let (trace_id, parent, sampled) = match header.and_then(traceparent) {
Some((t, p, s)) => (t, Some(p), s),
None => (new_id(), None, true),
};
Ctx {
trace_id,
span_id: new_id(),
parent,
sampled,
did: OnceLock::new(),
blob: OnceLock::new(),
}
}
/// A span within this one's, from `start` to now.
pub fn child(
&self,
name: String,
kind: u64,
start: u64,
attrs: Vec<(&'static str, Value)>,
error: bool,
) -> Span {
Span {
trace_id: self.trace_id,
span_id: new_id(),
parent: Some(self.span_id),
name,
kind,
start,
end: now_ns(),
attrs,
error,
}
}
}
// ------------------------------------------------------------ spans
pub(crate) enum Value {
Str(String),
Int(i64),
}
pub(crate) struct Span {
trace_id: [u8; 16],
span_id: [u8; 8],
parent: Option<[u8; 8]>,
name: String,
kind: u64,
start: u64,
end: u64,
attrs: Vec<(&'static str, Value)>,
/// Whether its status is `ERROR`; else it is unset.
error: bool,
}
/// The HTTP methods OpenTelemetry knows; any other is `_OTHER`.
fn method_name(m: &Method) -> &'static str {
match *m {
Method::GET => "GET",
Method::HEAD => "HEAD",
Method::POST => "POST",
Method::PUT => "PUT",
Method::DELETE => "DELETE",
Method::CONNECT => "CONNECT",
Method::OPTIONS => "OPTIONS",
Method::TRACE => "TRACE",
Method::PATCH => "PATCH",
_ => "_OTHER",
}
}
/// A request's span, from `start` to now.
pub(crate) fn server_span(
ctx: &Ctx,
method: &Method,
path: &str,
route: &str,
status: u16,
start: u64,
) -> Span {
let method = method_name(method);
let http_route = match route {
"_unknown" => None,
r if r.starts_with('/') => Some(r.to_string()),
r => Some(format!("/xrpc/{r}")),
};
let mut attrs = vec![
("http.request.method", Value::Str(method.into())),
("url.path", Value::Str(path.into())),
("url.scheme", Value::Str("http".into())),
("http.response.status_code", Value::Int(status.into())),
];
if let Some(r) = &http_route {
attrs.push(("http.route", Value::Str(r.clone())));
}
if status >= 500 {
attrs.push(("error.type", Value::Str(status.to_string())));
}
let name = http_route.map_or_else(|| method.to_string(), |r| format!("{method} {r}"));
Span {
trace_id: ctx.trace_id,
span_id: ctx.span_id,
parent: ctx.parent,
name,
kind: SERVER,
start,
end: now_ns(),
attrs,
error: status >= 500,
}
}
/// Where the server sends a request of its own: the PLC directory, a
/// crawler, a service it forwards to or reads from (the AppView, those
/// `atproto-proxy` names), a `did:web` document, a client's metadata or
/// keys (OAuth's, or an attesting client's), a space's notification, a
/// permission set's record, a handle's `/.well-known/atproto-did`, or the
/// bucket blobs are kept in.
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Target {
Plc,
Crawler,
Service,
DidWeb,
Client,
Space,
Lexicon,
Handle,
Blobstore,
}
impl Target {
pub(super) fn label(self) -> &'static str {
match self {
Target::Plc => "plc",
Target::Crawler => "crawler",
Target::Service => "service",
Target::DidWeb => "did-web",
Target::Client => "client",
Target::Space => "space",
Target::Lexicon => "lexicon",
Target::Handle => "handle",
Target::Blobstore => "blobstore",
}
}
/// The line a failure of a request to it is logged with, and at what level.
fn failure(self) -> (&'static str, Level) {
match self {
Target::Plc => ("plc failure", Level::Warn),
Target::Crawler => ("crawler failure", Level::Warn),
Target::Service => ("service failure", Level::Debug),
Target::DidWeb => ("did:web failure", Level::Debug),
Target::Client => ("client failure", Level::Debug),
Target::Space => ("space notification failure", Level::Debug),
Target::Lexicon => ("lexicon failure", Level::Debug),
Target::Handle => ("handle failure", Level::Debug),
Target::Blobstore => ("blobstore failure", Level::Warn),
}
}
}
/// What a request to the bucket does, as its failure line names it: `put`,
/// `get` or `delete`.
fn blobstore_op(m: &Method) -> &'static str {
match *m {
Method::PUT => "put",
Method::DELETE => "delete",
_ => "get",
}
}
/// Send a request of the server's own to `target`, as `execute` does.
pub(crate) async fn send(
target: Target,
req: reqwest::RequestBuilder,
) -> reqwest::Result<reqwest::Response> {
let (client, request) = req.build_split();
execute(target, &client, request?).await
}
/// Send a request of the server's own to `target` on `client`, as the child
/// of the request being answered or the root of a trace of its own,
/// carrying a `traceparent` naming its span: counted, timed, its span
/// exported, and a failure logged. Every request the server makes goes out
/// here. One given up before its answer, as by a caller's timeout, is a
/// `timeout`.
pub(crate) async fn execute(
target: Target,
client: &reqwest::Client,
mut request: reqwest::Request,
) -> reqwest::Result<reqwest::Response> {
let parent = current();
let (trace_id, parent_id, sampled) = match &parent {
Some(c) => (c.trace_id, Some(c.span_id), c.sampled),
None => (new_id(), None, true),
};
let span_id: [u8; 8] = new_id();
let tp = format!(
"00-{}-{}-{}",
hex::encode(trace_id),
hex::encode(span_id),
if sampled { "01" } else { "00" }
);
if let Ok(v) = reqwest::header::HeaderValue::from_str(&tp) {
request.headers_mut().insert("traceparent", v);
}
let mut out = Outgoing {
target,
trace_id,
span_id,
parent: parent_id,
sampled,
method: request.method().clone(),
url: request.url().clone(),
start: now_ns(),
started: Instant::now(),
done: false,
};
let res = client.execute(request).await;
match &res {
Ok(r) if r.status().is_success() => out.finish(Some(r.status().as_u16()), None),
Ok(r) => out.finish(
Some(r.status().as_u16()),
Some(r.status().as_u16().to_string()),
),
Err(e) if e.is_timeout() => out.finish(None, Some("timeout".into())),
Err(_) => out.finish(None, Some("no answer".into())),
}
res
}
/// A request of the server's own on its way: once it is answered, fails, or
/// is given up, its metrics, failure line and span.
struct Outgoing {
target: Target,
trace_id: [u8; 16],
span_id: [u8; 8],
parent: Option<[u8; 8]>,
sampled: bool,
method: Method,
url: reqwest::Url,
start: u64,
started: Instant,
done: bool,
}
impl Outgoing {
fn finish(&mut self, status: Option<u16>, error: Option<String>) {
self.done = true;
let target = self.target;
ops().metrics.outbound(
target,
error.is_none(),
self.started.elapsed().as_secs_f64(),
);
if let Some(why) = &error {
let (msg, level) = target.failure();
// Only a directory's paths, and the bucket's keys, name a DID,
// and nothing else of a URL is logged.
let segments = || self.url.path_segments().into_iter().flatten();
let did = match target {
Target::Plc => segments()
.find(|p| p.starts_with("did:"))
.map(str::to_string),
Target::Blobstore => segments()
.map(|p| percent_encoding::percent_decode_str(p).decode_utf8_lossy())
.find(|p| p.starts_with("did:"))
.map(|p| p.into_owned()),
_ => None,
};
let op = (target == Target::Blobstore).then(|| blobstore_op(&self.method));
line(
level,
msg,
&[
("op", json!(op)),
("error", json!(why)),
("status", json!(status)),
("did", json!(did)),
],
);
}
if self.sampled
&& let Some(e) = &ops().exporter
{
// A query, a fragment or credentials of a URL are never in a span.
let mut full = self.url.clone();
let _ = full.set_username("");
let _ = full.set_password(None);
full.set_query(None);
full.set_fragment(None);
let mut attrs = vec![
(
"http.request.method",
Value::Str(method_name(&self.method).into()),
),
(
"server.address",
Value::Str(self.url.host_str().unwrap_or_default().into()),
),
(
"server.port",
Value::Int(self.url.port_or_known_default().unwrap_or(0).into()),
),
("url.full", Value::Str(full.to_string())),
];
if let Some(s) = status {
attrs.push(("http.response.status_code", Value::Int(s.into())));
}
if let Some(why) = &error {
attrs.push((
"error.type",
Value::Str(why.replace("no answer", "connection")),
));
}
let name = method_name(&self.method).to_string();
e.push(Span {
trace_id: self.trace_id,
span_id: self.span_id,
parent: self.parent,
name,
kind: CLIENT,
start: self.start,
end: now_ns(),
attrs,
error: error.is_some(),
});
}
}
}
impl Drop for Outgoing {
fn drop(&mut self) {
if !self.done {
self.finish(None, Some("timeout".into()));
}
}
}
// ------------------------------------------------------------ export
/// The spans held for export, and what exports them.
pub(crate) struct Exporter {
url: reqwest::Url,
headers: reqwest::header::HeaderMap,
gzip: bool,
delay: Duration,
max_queue: usize,
max_batch: usize,
timeout: Duration,
http: reqwest::Client,
/// The `Resource` every export names, encoded.
resource: Vec<u8>,
queue: Mutex<VecDeque<Span>>,
/// Told once a batch is full.
wake: tokio::sync::Notify,
/// When an export last failed and was said to: none since one went.
logged: Mutex<Option<Instant>>,
}
impl Exporter {
/// Hold a span for export; dropped if the queue is full.
pub fn push(&self, s: Span) {
let mut q = self.queue.lock().unwrap_or_else(PoisonError::into_inner);
if q.len() >= self.max_queue {
return;
}
q.push_back(s);
if q.len() == self.max_batch {
self.wake.notify_one();
}
}
/// Export what is held every `delay`, or once a batch is full.
pub async fn run(&'static self) {
loop {
tokio::select! {
() = tokio::time::sleep(self.delay) => {}
() = self.wake.notified() => {}
}
self.drain(None).await;
}
}
/// Export what is held until it is all gone, or `until` passes.
pub async fn flush(&self, until: Instant) {
let _ = tokio::time::timeout_at(until.into(), self.drain(Some(until))).await;
}
async fn drain(&self, until: Option<Instant>) {
loop {
let batch: Vec<Span> = {
let mut q = self.queue.lock().unwrap_or_else(PoisonError::into_inner);
let n = q.len().min(self.max_batch);
q.drain(..n).collect()
};
if batch.is_empty() {
return;
}
self.export(&batch, until).await;
}
}
/// Post one batch, tried again until the export timeout has passed since
/// the first try, after which its spans are dropped.
async fn export(&self, batch: &[Span], until: Option<Instant>) {
let body = Bytes::from(self.encode(batch));
let deadline = (Instant::now() + self.timeout)
.min(until.unwrap_or_else(|| Instant::now() + self.timeout));
let mut pause = Duration::from_millis(100);
let why = loop {
let left = deadline.saturating_duration_since(Instant::now());
if left.is_zero() {
break "timeout".to_string();
}
let mut req = self
.http
.post(self.url.clone())
.headers(self.headers.clone())
.header(reqwest::header::CONTENT_TYPE, "application/x-protobuf")
.timeout(left);
if self.gzip {
req = req.header(reqwest::header::CONTENT_ENCODING, "gzip");
}
let why = match req.body(body.clone()).send().await {
Ok(res) if res.status().is_success() => {
*self.logged.lock().unwrap_or_else(PoisonError::into_inner) = None;
return;
}
Ok(res) if ![429, 502, 503, 504].contains(&res.status().as_u16()) => {
break format!("status {}", res.status().as_u16());
}
Ok(res) => format!("status {}", res.status().as_u16()),
Err(e) if e.is_timeout() => "timeout".to_string(),
Err(_) => "no answer".to_string(),
};
let left = deadline.saturating_duration_since(Instant::now());
if left <= pause {
break why;
}
tokio::time::sleep(pause).await;
pause = (pause * 2).min(Duration::from_secs(1));
};
let mut logged = self.logged.lock().unwrap_or_else(PoisonError::into_inner);
if logged.is_none_or(|t| t.elapsed() >= Duration::from_secs(60)) {
*logged = Some(Instant::now());
drop(logged);
line(
Level::Warn,
"trace export failed",
&[("error", json!(why)), ("spans", json!(batch.len()))],
);
}
}
/// An `ExportTraceServiceRequest` of `spans`, gzipped if it is to be.
fn encode(&self, spans: &[Span]) -> Vec<u8> {
let mut scope_spans = Vec::new();
message(&mut scope_spans, 1, |m| {
string(m, 1, "pds");
string(m, 2, super::VERSION);
});
for s in spans {
message(&mut scope_spans, 2, |m| encode_span(m, s));
}
let mut resource_spans = Vec::new();
bytes(&mut resource_spans, 1, &self.resource);
bytes(&mut resource_spans, 2, &scope_spans);
let mut out = Vec::new();
bytes(&mut out, 1, &resource_spans);
if !self.gzip {
return out;
}
use std::io::Write;
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
let _ = gz.write_all(&out);
gz.finish().unwrap_or_default()
}
}
// ------------------------------------------------------------ protobuf
fn varint(out: &mut Vec<u8>, mut v: u64) {
while v >= 0x80 {
out.push((v as u8) | 0x80);
v >>= 7;
}
out.push(v as u8);
}
fn key(out: &mut Vec<u8>, field: u64, wire: u64) {
varint(out, (field << 3) | wire);
}
fn bytes(out: &mut Vec<u8>, field: u64, b: &[u8]) {
key(out, field, 2);
varint(out, b.len() as u64);
out.extend_from_slice(b);
}
fn string(out: &mut Vec<u8>, field: u64, s: &str) {
bytes(out, field, s.as_bytes());
}
fn fixed64(out: &mut Vec<u8>, field: u64, v: u64) {
key(out, field, 1);
out.extend_from_slice(&v.to_le_bytes());
}
fn message(out: &mut Vec<u8>, field: u64, f: impl FnOnce(&mut Vec<u8>)) {
let mut m = Vec::new();
f(&mut m);
bytes(out, field, &m);
}
/// A `KeyValue`, as field `field`.
fn attribute(out: &mut Vec<u8>, field: u64, k: &str, v: &Value) {
message(out, field, |kv| {
string(kv, 1, k);
message(kv, 2, |any| match v {
Value::Str(s) => string(any, 1, s),
Value::Int(i) => {
key(any, 3, 0);
varint(any, *i as u64);
}
});
});
}
/// The `Resource`: the service's name and version.
fn encode_resource(service: &str) -> Vec<u8> {
let mut out = Vec::new();
attribute(&mut out, 1, "service.name", &Value::Str(service.into()));
attribute(
&mut out,
1,
"service.version",
&Value::Str(super::VERSION.into()),
);
out
}
fn encode_span(m: &mut Vec<u8>, s: &Span) {
bytes(m, 1, &s.trace_id);
bytes(m, 2, &s.span_id);
if let Some(p) = &s.parent {
bytes(m, 4, p);
}
string(m, 5, &s.name);
key(m, 6, 0);
varint(m, s.kind);
fixed64(m, 7, s.start);
fixed64(m, 8, s.end.max(s.start));
for (k, v) in &s.attrs {
attribute(m, 9, k, v);
}
if s.error {
message(m, 15, |st| {
key(st, 3, 0);
varint(st, 2);
});
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn traceparents() {
let t = "4bf92f3577b34da6a3ce929d0e0e4736";
let p = "00f067aa0ba902b7";
assert_eq!(
traceparent(&format!("00-{t}-{p}-01")).map(|x| x.2),
Some(true)
);
assert_eq!(
traceparent(&format!("00-{t}-{p}-00")).map(|x| x.2),
Some(false)
);
assert!(traceparent(&format!("01-{t}-{p}-01-more")).is_some());
assert!(traceparent(&format!("00-{t}-{p}-01-more")).is_none());
assert!(traceparent(&format!("ff-{t}-{p}-01")).is_none());
assert!(traceparent(&format!("00-{}-{p}-01", t.to_uppercase())).is_none());
assert!(traceparent(&format!("00-{}-{p}-01", "0".repeat(32))).is_none());
assert!(traceparent(&format!("00-{t}-{}-01", "0".repeat(16))).is_none());
assert!(traceparent("garbage").is_none());
}
#[test]
fn header_values_are_percent_decoded() {
let mut h = reqwest::header::HeaderMap::new();
headers("H", Some("a=b%20c, X-Key = k%3Dv ,".into()), &mut h).unwrap();
assert_eq!(h["a"], "b c");
assert_eq!(h["x-key"], "k=v");
}
}