~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
978 lines · 35.4 KB Raw History
//! What the server tells whoever runs it, as `spec/OPERATIONS.md` says: its
//! log, JSON lines on standard output, and kept in `PDS_LOG_DIR` too
//! (`kept`); `/xrpc/_health`; its metrics (`metrics`); its traces
//! (`trace`); the admin's reads of the kept lines and of the metrics,
//! `/xrpc/_logs` and `/xrpc/_stats` (`admin`); and how it shuts down, and
//! says on its next start whether the last one did.
//!
//! Every request goes through `Seen`, in the one place connections are
//! served (`limits::serve`): its route, its trace, and once its answer is
//! whole, before the last byte of it is sent, its log line, its metrics and
//! its span.

mod admin;
mod kept;
mod metrics;
mod trace;

pub(crate) use admin::{Read, logs, read, stats};

pub(crate) use trace::{Target, execute, send};

use crate::Shared;
use axum::body::{Body, Bytes};
use axum::http::{Method, Request, Response, StatusCode, header};
use http_body::{Body as _, Frame, SizeHint};
use serde_json::{Value as Json, json};
use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::pin::Pin;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};

/// The server's version, as `/xrpc/_health` and the `start` line say it.
pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");

/// The mark a shutdown leaves in the data directory, taken away at the next start.
pub(crate) const CLEAN: &str = "clean-shutdown";
/// What the mark begins with when every frame of every log in the
/// directory is on disk: the shutdown that left it closed every log and
/// made durable all they held (`settle`), in a run that had found the rest
/// durable or made it so (`mark`). After it the mark names the directory
/// it was written in (`word`), and it is the word only there: the next to
/// open that directory then syncs no log as it opens it (`store`, "What a
/// start syncs"), and takes the word out of the mark before it writes
/// anything (`durable`). A mark without it, as a version before left, or a
/// run shut down before it had made durable what it found, says only that
/// the server shut down; and so does one that names another directory, as
/// a copy's does.
pub(crate) const DURABLE: &[u8] = b"durable\n";
/// How long a shutdown lets the requests in flight finish.
pub(crate) const DRAIN: Duration = Duration::from_secs(10);
/// How long a shutdown waits on the spans it holds being exported.
const FLUSH: Duration = Duration::from_secs(5);

#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
pub(crate) enum Level {
    Debug,
    Info,
    Warn,
    Error,
    /// No line at all.
    Off,
}

impl Level {
    fn name(self) -> &'static str {
        match self {
            Level::Debug => "debug",
            Level::Info => "info",
            Level::Warn => "warn",
            Level::Error => "error",
            Level::Off => "off",
        }
    }
}

/// A variable's value; an empty one is as if unset.
fn var(name: &str) -> Option<String> {
    std::env::var(name).ok().filter(|v| !v.is_empty())
}

/// What the environment says of the log, the metrics and the traces.
pub(crate) struct Settings {
    level: Level,
    metrics_port: Option<u16>,
    trace: trace::Settings,
    kept: Option<kept::Settings>,
}

impl Settings {
    /// Read from the environment, the server's own port being `port`: why
    /// not, if a variable does not parse.
    pub fn from_env(port: u16) -> Result<Settings, String> {
        let level = match var("PDS_LOG_LEVEL").as_deref() {
            None | Some("info") => Level::Info,
            Some("debug") => Level::Debug,
            Some("warn") => Level::Warn,
            Some("error") => Level::Error,
            Some("off") => Level::Off,
            Some(v) => {
                return Err(format!(
                    "PDS_LOG_LEVEL is {v:?}, not debug, info, warn, error or off"
                ));
            }
        };
        let metrics_port = match var("PDS_METRICS_PORT") {
            None => None,
            Some(v) => match v.trim().parse::<u16>() {
                Ok(p) if p == port => {
                    return Err(format!(
                        "PDS_METRICS_PORT is {p}, the port PDS_PORT serves on"
                    ));
                }
                Ok(p) if p > 0 => Some(p),
                _ => return Err(format!("PDS_METRICS_PORT is {v:?}, not a port")),
            },
        };
        Ok(Settings {
            level,
            metrics_port,
            trace: trace::Settings::from_env()?,
            kept: kept::Settings::from_env()?,
        })
    }
}

/// Where lines go: standard output, and the files they are kept in, if any.
struct Out {
    /// Written to with one `write` a line.
    stdout: Option<File>,
    kept: Option<kept::Kept>,
}

/// What every request and every line shares.
struct Ops {
    level: Level,
    /// Held while a line is written, to both.
    out: Mutex<Out>,
    metrics: metrics::Metrics,
    /// Where spans go, if they are exported.
    exporter: Option<trace::Exporter>,
    /// The `time` of this start's `start` line.
    started: String,
}

static OPS: OnceLock<Ops> = OnceLock::new();

/// As set at start; with none set, as in tests, nothing is written or exported.
fn ops() -> &'static Ops {
    OPS.get_or_init(|| Ops {
        level: Level::Off,
        out: Mutex::new(Out {
            stdout: None,
            kept: None,
        }),
        metrics: metrics::Metrics::default(),
        exporter: None,
        started: crate::events::timestamp(),
    })
}

/// Standard output, as a file of its own, written past Rust's buffering.
fn stdout() -> Option<File> {
    use std::os::fd::AsFd;
    std::io::stdout()
        .as_fd()
        .try_clone_to_owned()
        .ok()
        .map(File::from)
}

/// Take the settings, and write the `start` line: the environment is read,
/// and the data not yet. The lines kept in `PDS_LOG_DIR` go on after those
/// there.
pub(crate) fn start(s: Settings, port: u16, sync: crate::Syncing) {
    let exporter = s.trace.exporter();
    let started = crate::events::timestamp();
    let _ = OPS.set(Ops {
        level: s.level,
        out: Mutex::new(Out {
            stdout: stdout(),
            kept: s.kept.map(kept::Kept::open),
        }),
        metrics: metrics::Metrics::default(),
        exporter,
        started: started.clone(),
    });
    pds_store::on_sync(|d| ops().metrics.fsync(d));
    // What the store does in the background and could not: a compaction,
    // a checkpoint of the journal, a sync of it.
    pds_store::on_error(|why| failed("store failure", why));
    // A panic is a line of the log, as nothing but why the server would not
    // start goes to standard error.
    std::panic::set_hook(Box::new(|info| {
        let at = info
            .location()
            .map(|l| format!("{}:{}", l.file(), l.line()));
        error("panic", &[("at", json!(at))]);
    }));
    line_at(
        &started,
        Level::Info,
        "start",
        &[
            ("version", json!(VERSION)),
            ("port", json!(port)),
            ("sync", json!(sync.name())),
            ("metrics_port", json!(s.metrics_port)),
        ],
    );
    if let Some(p) = s.metrics_port {
        METRICS_PORT.set(p).ok();
    }
}

static METRICS_PORT: OnceLock<u16> = OnceLock::new();

/// Write a line at `level`, with `fields` after `time`, `level` and `msg`;
/// a field that is null is left out.
pub(crate) fn line(level: Level, msg: &str, fields: &[(&str, Json)]) {
    line_at(&crate::events::timestamp(), level, msg, fields);
}

/// Write a line whose `time` is `time`: to standard output, then, the same
/// bytes, to the files it is kept in, before the next line goes to either.
fn line_at(time: &str, level: Level, msg: &str, fields: &[(&str, Json)]) {
    let o = ops();
    if o.level == Level::Off || level < o.level {
        return;
    }
    let mut s = String::with_capacity(192);
    s.push_str("{\"time\":\"");
    s.push_str(time);
    s.push_str("\",\"level\":\"");
    s.push_str(level.name());
    s.push_str("\",\"msg\":");
    s.push_str(&Json::from(msg).to_string());
    for (k, v) in fields {
        if v.is_null() {
            continue;
        }
        s.push(',');
        s.push_str(&Json::from(*k).to_string());
        s.push(':');
        s.push_str(&v.to_string());
    }
    s.push_str("}\n");
    let mut out = o.out.lock().unwrap_or_else(PoisonError::into_inner);
    if let Some(mut f) = out.stdout.as_ref() {
        let _ = f.write_all(s.as_bytes());
    }
    if let Some(k) = &mut out.kept {
        k.write(s.as_bytes());
    }
}

/// The lines kept in `PDS_LOG_DIR`, as they are now, to read with no lock
/// held; None with no `PDS_LOG_DIR`.
fn kept_now() -> Option<kept::Snapshot> {
    let out = ops().out.lock().unwrap_or_else(PoisonError::into_inner);
    out.kept.as_ref().map(kept::Kept::snapshot)
}

pub(crate) fn info(msg: &str, fields: &[(&str, Json)]) {
    line(Level::Info, msg, fields);
}

pub(crate) fn warn(msg: &str, fields: &[(&str, Json)]) {
    line(Level::Warn, msg, fields);
}

pub(crate) fn error(msg: &str, fields: &[(&str, Json)]) {
    line(Level::Error, msg, fields);
}

/// A line of an error of the server's own: `msg` says what failed, and
/// `error` why.
pub(crate) fn failed(msg: &str, why: impl std::fmt::Display) {
    error(msg, &[("error", json!(why.to_string()))]);
}

/// A line at `level` of something that failed for an account, and why.
pub(crate) fn failed_for(level: Level, msg: &str, did: &str, why: impl std::fmt::Display) {
    line(
        level,
        msg,
        &[("did", json!(did)), ("error", json!(why.to_string()))],
    );
}

// ------------------------------------------------------------ starts

/// Whether the server starting on `dir` finds it as the last one left it
/// while running: it holds data, a service log of any generation, its
/// checkpoint, or the single log of the layout before generations, and no
/// mark of a shutdown.
pub(crate) fn unclean(dir: &Path) -> bool {
    crate::store::holds_data(dir) && !dir.join(CLEAN).exists()
}

/// The word as the mark `mark`, open, of the data directory `dir` holds
/// it, for the directory as it is now: `DURABLE`, and then what a copy of
/// the directory does not keep (`store`, "The word is its directory's"):
/// of the directory and of `repos`, the id of the file system each is on,
/// its inode and when it last changed (its `ctime`, to the nanosecond);
/// and the mark's own inode. With it, when the data directory last
/// changed, which the mark's own time of writing is set to (`mark`) and
/// must still be (`durable`). None on a file system that gives no id,
/// zeros, where the mark never holds the word.
///
/// The id is `f_fsid` as `fstatvfs` gives it. On Linux that is both
/// halves of the kernel's `f_fsid`, which a file system makes of its own
/// UUID (ext4, btrfs, tmpfs) or of its device's number (XFS), and which is
/// zeros where it makes none (NFS, FUSE). On macOS it is the first half
/// alone, the device's number.
fn word(dir: &Path, mark: &File) -> std::io::Result<Option<(Vec<u8>, std::time::SystemTime)>> {
    use std::os::unix::fs::MetadataExt;
    /// A directory's file system, inode and change time, as the mark
    /// names them, and the change time: none with no id, or a time before
    /// any clock's.
    fn place(dir: &File) -> std::io::Result<Option<(String, std::time::SystemTime)>> {
        let fs = rustix::fs::fstatvfs(dir)?.f_fsid;
        let m = dir.metadata()?;
        let (ino, at, nanos) = (m.ino(), m.ctime(), m.ctime_nsec());
        let (Ok(secs), Ok(sub), true) = (u64::try_from(at), u32::try_from(nanos), fs != 0) else {
            return Ok(None);
        };
        let changed = std::time::UNIX_EPOCH + Duration::new(secs, sub);
        Ok(Some((format!("{fs:x} {ino} {at}.{nanos:09}"), changed)))
    }
    let Some((top, changed)) = place(&File::open(dir)?)? else {
        return Ok(None);
    };
    let repos = match File::open(dir.join("repos")) {
        Ok(repos) => match place(&repos)? {
            Some((repos, _)) => repos,
            None => return Ok(None),
        },
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => "none".into(),
        Err(e) => return Err(e),
    };
    let mut word = DURABLE.to_vec();
    let mark = mark.metadata()?.ino();
    writeln!(word, "dir {top}\nrepos {repos}\nmark {mark}")?;
    Ok(Some((word, changed)))
}

/// Whether the mark `mark`, open, which holds `held`, is the word of the
/// data directory `dir` as it is now: it holds what `word` makes of the
/// directory, and was last written when the directory last changed. What
/// cannot be told of the directory is no word.
fn is_word(dir: &Path, mark: &File, held: &[u8]) -> bool {
    let Ok(Some((word, changed))) = word(dir, mark) else {
        return false;
    };
    let written = mark.metadata().and_then(|m| m.modified());
    word == held && written.is_ok_and(|at| at == changed)
}

/// Whether the last to use `dir` left every frame of every log in it on
/// disk: a server that shut down there, as its mark says (`mark`), with
/// the word of this directory as it is now (`is_word`). Any other mark is
/// no word: one that names another directory, as a copy of a stopped
/// server's does, whose bytes may be in the system's memory alone; one
/// written since its shutdown wrote it; one of a form this does not
/// write; an empty one. Whatever the mark held is taken out of it,
/// durably, before this answers, as whoever asks is about to write there:
/// a kill of this run, whatever it has written by then, is not taken for
/// that shutdown by the next. The mark itself stays, for `unclean` and
/// `recovered`.
///
/// Asked before anything of the directory is changed, its mode too
/// (`Service::open`), which is set after: setting it changes when the
/// directory last changed, so that what the mark held names the
/// directory no more, whatever puts it back.
pub(crate) fn durable(dir: &Path) -> Result<bool, String> {
    use std::io::Read;
    let path = dir.join(CLEAN);
    let at = |e: std::io::Error| format!("{}: {e}", path.display());
    let mut mark = match std::fs::OpenOptions::new()
        .read(true)
        .write(true)
        .open(&path)
    {
        Ok(mark) => mark,
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
        Err(e) => return Err(at(e)),
    };
    let mut held = Vec::new();
    mark.read_to_end(&mut held).map_err(at)?;
    if held.is_empty() {
        return Ok(false);
    }
    let here = is_word(dir, &mark, &held);
    mark.set_len(0)
        .and_then(|()| pds_store::sync_file(&mark))
        .map_err(at)?;
    Ok(here)
}

/// The data is read: say so if the last server did not shut down, and take
/// away the mark of its shutdown, so that this one leaves it only if it does.
pub(crate) fn recovered(svc: &Shared, unclean: bool) -> Result<(), String> {
    if unclean {
        // Of the logs the start read: the service's, and those of the
        // accounts it loaded. An account loaded later says what its own
        // log dropped then (`cache::Slot::load`): a start reads no more of
        // the accounts than it must (`spec/STORE.md`, "Scale").
        let mut dropped = svc.log.dropped();
        for a in crate::read(&svc.directory).by_did.values() {
            dropped += a.store.loaded_wal().map_or(0, |w| w.dropped());
        }
        warn("recovered", &[("dropped_bytes", json!(dropped))]);
    }
    let mark = svc.dir.join(CLEAN);
    if mark.exists() {
        std::fs::remove_file(&mark)
            .and_then(|()| pds_store::sync_dir(&svc.dir))
            .map_err(|e| format!("{}: {e}", mark.display()))?;
    }
    Ok(())
}

/// Leave the mark of a shutdown in the data directory, durably, `settle`
/// having answered. It holds the word that every frame of every log there
/// is on disk only where that is so of the logs this run never opened too
/// (`cache::Ctx::found`): the run started with the word, or has made
/// durable what it found (`sync_found`); and where no load of a store
/// failed in the run (`cache::Ctx::failed`), which may have left part of
/// a mend synced by nothing. Otherwise the mark says only that the server
/// shut down, and the next start syncs each log as it starts.
///
/// The mark is made anew, the one before deleted if a start left it: so
/// the directory last changed as this mark was made, which the word then
/// names (`word`), and nothing changes the directory after. The mark's
/// time of writing is set to that same time, once the word is written:
/// whatever writes the mark again, as a copy of the directory put back
/// over it file by file does, leaves it another.
pub(crate) fn mark(s: &crate::Service) -> std::io::Result<()> {
    let path = s.dir.join(CLEAN);
    match std::fs::remove_file(&path) {
        Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e),
        _ => {}
    }
    let mut f = pds_store::open_private(
        &path,
        std::fs::OpenOptions::new().create_new(true).write(true),
    )?;
    if s.ctx.found.load(Ordering::SeqCst)
        && !s.ctx.failed.load(Ordering::SeqCst)
        && !s.events.owes()
        && let Some((word, changed)) = word(&s.dir, &f)?
    {
        f.write_all(&word)?;
        f.set_modified(changed)?;
    }
    pds_store::sync_file(&f)?;
    pds_store::sync_dir(&s.dir)
}

/// Make durable what this run found in the data directory as it started,
/// unless that is so already: every account's logs and table, those of the
/// accounts nothing has loaded too (`store`, "What a start found, made
/// durable"), and the firehose's index, which is read first and so synced
/// as it is opened. Its shutdown leaves the word in its mark from then on.
pub(crate) fn sync_found(s: &crate::Service) -> std::io::Result<()> {
    if !s.ctx.found.load(Ordering::SeqCst) {
        s.events.find_old();
        crate::store::sync_found(&s.dir)?;
        s.ctx.found.store(true, Ordering::SeqCst);
    }
    Ok(())
}

/// The same, on a thread of its own while the server serves: a start
/// waits for none of it, nor does a shutdown, which leaves no word if it
/// comes first; one that fails says so, and leaves none either.
pub(crate) fn sync_found_soon(svc: &Shared) {
    if svc.ctx.found.load(Ordering::SeqCst) {
        return;
    }
    let svc = svc.clone();
    tokio::task::spawn_blocking(move || {
        if let Err(e) = sync_found(&svc) {
            failed("syncing what the start found", e);
        }
    });
}

/// What runs beside the server: the metrics' port, and the export of spans.
pub(crate) async fn serve_beside(svc: &Shared) -> Result<(), String> {
    if let Some(e) = &ops().exporter {
        tokio::spawn(e.run());
    }
    if let Some(&port) = METRICS_PORT.get() {
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", port))
            .await
            .map_err(|e| format!("listening on PDS_METRICS_PORT {port}: {e}"))?;
        tokio::spawn(metrics::serve(listener, svc.clone()));
    }
    Ok(())
}

/// The first SIGTERM or SIGINT, by name, from now on.
pub(crate) fn signals() -> impl Future<Output = &'static str> {
    use tokio::signal::unix::{SignalKind, signal};
    let term = signal(SignalKind::terminate());
    let int = signal(SignalKind::interrupt());
    async move {
        match (term, int) {
            (Ok(mut term), Ok(mut int)) => tokio::select! {
                _ = term.recv() => "SIGTERM",
                _ = int.recv() => "SIGINT",
            },
            _ => std::future::pending().await,
        }
    }
}

/// Shut down, the server having stopped taking connections and its requests
/// finished: close the firehose's subscriptions, write the issuers' keys
/// kept (`space_credentials::issuers`, a cache, not synced), close the logs
/// and sync what is written (`settle`), export the spans held, leave the
/// mark of a shutdown, say so, and exit 0. The mark is left only once
/// `settle` has answered that all this run wrote is on disk: where it
/// fails, or the process is killed before the mark, the next start is as
/// after a kill.
/// It is so too, but for its `recovered` line, where what this run found
/// was not yet made durable (`mark`), which nothing here waits for.
pub(crate) async fn shutdown(svc: &Shared, signal: &'static str) -> ! {
    svc.events.close();
    for _ in 0..40 {
        if svc.events.subscribers() == 0 {
            break;
        }
        tokio::time::sleep(Duration::from_millis(25)).await;
    }
    let s = svc.clone();
    let synced = tokio::task::spawn_blocking(move || {
        crate::space_credentials::issuers::write(&s);
        settle(&s)
    })
    .await;
    if let Some(e) = &ops().exporter {
        e.flush(Instant::now() + FLUSH).await;
    }
    match synced {
        Ok(Ok(())) => {
            if let Err(e) = mark(svc) {
                failed("marking the shutdown", e);
            }
        }
        Ok(Err(e)) => failed("syncing the logs", e),
        Err(e) => failed("syncing the logs", e),
    }
    info("shutdown", &[("signal", json!(signal))]);
    std::process::exit(0);
}

/// What a shutdown leaves on disk: no event is numbered from now on, and
/// every log is closed and what it holds made durable, the firehose's
/// index last (`events::Firehose::close_index`), which an account's store
/// loaded as the others close may still append void entries to.
///
/// Once this answers, every frame this run gave a log is on disk, with
/// every frame of each log it opened, and no log takes another; no log
/// this run opened holds room:
///
/// - the service log is closed (`pds_store::Wal::close`): an append from
///   now on is refused, its room is given back, and every frame it took is
///   synced;
/// - the accounts' logs are closed together, as the pool they are all of
///   (`pds_store::Files::close`), and from then on no account's store is
///   loaded (`cache::Slot::load`): a load mends what it finds, which is a
///   write, and the mark could be left before one begun after this had
///   synced what it mended. Then the log of each store held is closed in
///   turn, which gives back its room and syncs every frame it took, with
///   the logs a compaction sealed and the files its commits rest on. A
///   store is looked for under its slot's lock, which a load holds from
///   before it asks whether the logs are closed until its store is held:
///   so one being loaded as the logs were closed is waited for, what it
///   mended synced by its own open, and closed here as any held; and one
///   that failed is noted under that lock (`cache::Ctx::failed`), so that
///   the mark holds no word, before it stops the server;
/// - a store not held was let go with every frame of its log on disk and
///   no compaction of it running (`Store::idle`), its log's room given back
///   as it went, and nothing has been appended to its log since;
/// - an account being made, not yet among those listed here, has in its
///   log its first commit alone, synced before the service log is given
///   the account (`server`), and the room that commit made;
/// - with `PDS_SYNC` `together`, where "synced" above is the journal's
///   sync, the journal is closed last (`pds_store::Journal::close`): every
///   log written to since its last checkpoint, those let go and those of
///   accounts being made too, is made durable in its own file, and its
///   segments deleted, so that no journal is left. It takes no record from
///   then on, so that what writes to a log after is refused.
///
/// A log this run never opened, of an account nothing asked for, is not
/// among these: it has no store, and holds what the run found. That is on
/// disk where the run started with the mark's word, or has since made
/// durable what it found (`sync_found`), and only then does the mark say
/// that every frame of every log is (`mark`).
///
/// The cut that gives room back is not itself synced, and a log no run
/// since a kill has opened keeps the room it had then: zeros after a log's
/// frames are room to the open that finds them, cut off with no sync where
/// the frames are on disk (`pds_store`, "An open that syncs nothing"). A
/// log's compaction under way makes no `.tmp`, puts no checkpoint in place
/// and deletes no log once its log is closed, each of which it does only
/// under the log's lock with its files found not let go there
/// (`pds_store::generations`): it leaves its files as a kill at one of
/// its steps would. Whatever writes to a log from now on, a request that
/// outlived the shutdown's wait or the server's own housekeeping, is
/// refused as a write that fails is; and whatever asks for a store not
/// held is unwound, a request answered an internal error
/// (`cache::ShuttingDown`), with nothing of the account's files read,
/// made, mended or synced.
pub(crate) fn settle(s: &crate::Service) -> std::io::Result<()> {
    s.events.seal();
    s.log.close()?;
    s.ctx.files.close();
    let accounts: Vec<_> = crate::read(&s.directory).by_did.values().cloned().collect();
    for wal in accounts.iter().filter_map(|a| a.store.loaded_wal()) {
        wal.close()?;
    }
    s.events.close_index()?;
    if let Some(j) = s.ctx.files.journal() {
        j.close()?;
    }
    Ok(())
}

// ------------------------------------------------------------ requests

/// The trace context of the request being answered, if any, for what is
/// spawned on its behalf.
pub(crate) fn spawn<F>(f: F) -> tokio::task::JoinHandle<F::Output>
where
    F: Future + Send + 'static,
    F::Output: Send + 'static,
{
    match trace::current() {
        Some(ctx) => tokio::spawn(trace::CTX.scope(ctx, f)),
        None => tokio::spawn(f),
    }
}

/// The account whose bearer token the request being answered carried, the
/// server having verified it.
pub(crate) fn verified(did: &str) {
    if let Some(ctx) = trace::current() {
        let _ = ctx.did.set(did.to_string());
    }
}

/// The blob the `getBlob` being answered serves, of the account `did`: its
/// request line names it (`spec/OPERATIONS.md`, "Events").
pub(crate) fn serving_blob(did: &str, cid: &str) {
    if let Some(ctx) = trace::current() {
        let _ = ctx.blob.set((did.to_string(), cid.to_string()));
    }
}

/// Wait on `f`, a wait on writes reaching the disk, as a span `fsync` of
/// the request being answered.
pub(crate) async fn fsync<T>(f: impl Future<Output = T>) -> T {
    let ctx = trace::current().filter(|c| c.sampled && ops().exporter.is_some());
    let start = trace::now_ns();
    let out = f.await;
    if let (Some(ctx), Some(e)) = (ctx, &ops().exporter) {
        e.push(ctx.child("fsync".into(), trace::INTERNAL, start, vec![], false));
    }
    out
}

/// The routes outside `/xrpc/` the server serves, each its own label: the
/// root, the MCP endpoint, `atproto-did`, OAuth's metadata, endpoints and
/// pages, and the MCP issuer's (`spec/OAUTH-MCP.md`).
const ROUTES: &[&str] = &[
    "/",
    "/mcp",
    "/.well-known/atproto-did",
    "/.well-known/oauth-protected-resource",
    "/.well-known/oauth-authorization-server",
    "/oauth/jwks",
    "/oauth/par",
    "/oauth/token",
    "/oauth/revoke",
    "/oauth/authorize",
    "/oauth/authorize/redirect",
    "/@atproto/oauth-provider/~api/sign-in",
    "/@atproto/oauth-provider/~api/consent",
    "/@atproto/oauth-provider/~api/reject",
    "/@atproto/oauth-provider/~api/sign-out",
    "/@atproto/oauth-provider/~api/passkey-options",
    "/@atproto/oauth-provider/~api/passkey-sign-in",
    "/oauth/passkey",
    "/@atproto/oauth-provider/~api/passkey-register",
    "/.well-known/oauth-protected-resource/mcp",
    "/.well-known/oauth-authorization-server/oauth/mcp",
    "/oauth/mcp/authorize",
    "/oauth/mcp/token",
    "/oauth/mcp/revoke",
];

/// The route of every path under `/oauth/assets/`, whatever its name,
/// served or not: the pages' build (`spec/OAUTH-PAGES.md`, "Assets").
const ASSETS: &str = "/oauth/assets/{name}";

/// A request's route, as its log line and metrics label it: the NSID of a
/// method served, `_health`, `_logs` or `_stats`, the account's own routes
/// as `_oauthSessions` and `_passkeyList`, a path served outside `/xrpc/`,
/// or `_unknown`.
fn route_of(path: &str) -> &'static str {
    if let Some(own) = ["_health", "_logs", "_stats"]
        .into_iter()
        .chain(crate::oauth::account::Route::ALL.map(|r| r.name()))
        .find(|r| path.strip_prefix("/xrpc/") == Some(*r))
    {
        return own;
    }
    if let Some(nsid) = path.strip_prefix("/xrpc/") {
        return pds_lexicon::method(nsid).map_or("_unknown", |m| m.nsid.as_str());
    }
    if path.starts_with("/oauth/assets/") {
        return ASSETS;
    }
    ROUTES
        .iter()
        .find(|r| **r == path)
        .copied()
        .unwrap_or("_unknown")
}

/// A request, from when its head was read.
pub(crate) struct Seen {
    started: Instant,
    start_ns: u64,
    method: Method,
    path: String,
    route: &'static str,
    ctx: Arc<trace::Ctx>,
}

impl Seen {
    pub fn new<B>(req: &Request<B>) -> Seen {
        let ctx = trace::Ctx::of(
            req.headers()
                .get("traceparent")
                .and_then(|v| v.to_str().ok()),
        );
        Seen {
            started: Instant::now(),
            start_ns: trace::now_ns(),
            method: req.method().clone(),
            path: req.uri().path().to_string(),
            route: route_of(req.uri().path()),
            ctx: Arc::new(ctx),
        }
    }

    /// Answer the request with `f`, in its trace.
    pub async fn run<F: Future>(&self, f: F) -> F::Output {
        trace::CTX.scope(self.ctx.clone(), f).await
    }

    /// The answer, with its line written once it is whole: now, if it has no
    /// body to send, or is a WebSocket's upgrade; else as its last bytes go.
    pub fn answer(self, res: Response<Body>) -> Response<Body> {
        let status = res.status();
        let bodiless = self.method == Method::HEAD
            || status == StatusCode::SWITCHING_PROTOCOLS
            || status == StatusCode::NO_CONTENT
            || status == StatusCode::NOT_MODIFIED;
        if bodiless || res.body().is_end_stream() {
            self.done(status.as_u16(), 0);
            return res;
        }
        let exact = res.body().size_hint().exact();
        res.map(|inner| {
            Body::new(Logged {
                inner,
                seen: Some(self),
                status: status.as_u16(),
                sent: 0,
                exact,
            })
        })
    }

    /// The answer is whole, `sent` bytes of its body handed on: its line,
    /// metrics and span.
    fn done(self, status: u16, sent: u64) {
        let o = ops();
        let secs = self.started.elapsed().as_secs_f64();
        let blob = self
            .ctx
            .blob
            .get()
            .filter(|_| status == 200 || status == 500);
        let fields = [
            ("method", json!(self.route)),
            ("status", json!(status)),
            ("ms", json!(secs * 1000.0)),
            ("trace_id", json!(hex::encode(self.ctx.trace_id))),
            ("did", json!(self.ctx.did.get())),
            ("repo", json!(blob.map(|(did, _)| did))),
            ("cid", json!(blob.map(|(_, cid)| cid))),
            ("bytes", json!(blob.filter(|_| status == 200).map(|_| sent))),
        ];
        line(
            if status >= 500 {
                Level::Error
            } else {
                Level::Info
            },
            "request",
            &fields,
        );
        o.metrics.request(self.route, status, secs);
        if let Some(e) = &o.exporter
            && self.ctx.sampled
        {
            e.push(trace::server_span(
                &self.ctx,
                &self.method,
                &self.path,
                self.route,
                status,
                self.start_ns,
            ));
        }
    }
}

/// An answer's body, which writes its request's line before its last bytes
/// are handed on, or once it is dropped short of them, as when the client
/// leaves.
struct Logged {
    inner: Body,
    seen: Option<Seen>,
    status: u16,
    /// The bytes handed on, and those there are, if known.
    sent: u64,
    exact: Option<u64>,
}

impl Logged {
    fn done(&mut self) {
        if let Some(seen) = self.seen.take() {
            seen.done(self.status, self.sent);
        }
    }
}

impl http_body::Body for Logged {
    type Data = Bytes;
    type Error = axum::Error;

    fn poll_frame(
        mut self: Pin<&mut Self>,
        cx: &mut Context<'_>,
    ) -> Poll<Option<Result<Frame<Bytes>, axum::Error>>> {
        let this = &mut *self;
        match Pin::new(&mut this.inner).poll_frame(cx) {
            Poll::Ready(Some(Ok(frame))) => {
                if let Some(d) = frame.data_ref() {
                    this.sent += d.len() as u64;
                }
                if this.inner.is_end_stream() || this.exact.is_some_and(|n| this.sent >= n) {
                    this.done();
                }
                Poll::Ready(Some(Ok(frame)))
            }
            Poll::Ready(other) => {
                this.done();
                Poll::Ready(other)
            }
            Poll::Pending => Poll::Pending,
        }
    }

    fn is_end_stream(&self) -> bool {
        self.inner.is_end_stream()
    }

    fn size_hint(&self) -> SizeHint {
        self.inner.size_hint()
    }
}

impl Drop for Logged {
    fn drop(&mut self) {
        self.done();
    }
}

// ------------------------------------------------------------ health

/// `/xrpc/_health`: the version, to `GET` and `HEAD`, whatever the query.
pub(crate) async fn health(method: Method) -> Response<Body> {
    use axum::response::IntoResponse;
    if method != Method::GET && method != Method::HEAD {
        return crate::respond(
            crate::invalid(format!("Incorrect HTTP method ({method}) expected GET")),
            None,
        );
    }
    (
        StatusCode::OK,
        [(header::CONTENT_TYPE, "application/json; charset=utf-8")],
        json!({ "version": VERSION }).to_string(),
    )
        .into_response()
}

// ------------------------------------------------------------ the root

/// What `/` says, `<origin>` standing for the origin of `PDS_PUBLIC_URL`
/// (`spec/OPERATIONS.md`, "The root").
const ROOT: &str = r#"
         __                         __
        /\ \__                     /\ \__
    __  \ \ ,_\  _____   _ __   ___\ \ ,_\   ___
  /'__'\ \ \ \/ /\ '__'\/\''__\/ __'\ \ \/  / __'\
 /\ \L\.\_\ \ \_\ \ \L\ \ \ \//\ \L\ \ \ \_/\ \L\ \
 \ \__/.\_\\ \__\\ \ ,__/\ \_\\ \____/\ \__\ \____/
  \/__/\/_/ \/__/ \ \ \/  \/_/ \/___/  \/__/\/___/
                   \ \_\
                    \/_/


This is an AT Protocol Personal Data Server (aka, an atproto PDS)

Most API routes are under /xrpc/

Connect an AI app

  1. Add this address to your AI app as a connector:

       <origin>/mcp

     In Claude: Settings > Connectors > Add custom connector,
     and choose "Sign in when needed".

  2. When it asks, sign in here and approve it.

  3. Ask it things like:

       What have I posted this week?
       Post "hello from Claude" to Bluesky
       Who have I followed lately?
       Set my Bluesky bio to "Software engineer in Seattle"

      Code: https://knotless.dev/chadtmiller.com/monolith
  Protocol: https://atproto.com
"#;

/// `/`: what the server is, and how to connect an AI app to it, to `GET`
/// and `HEAD`, whatever the query; any other method is no route.
pub(crate) async fn root(
    axum::extract::State(svc): axum::extract::State<Shared>,
    method: Method,
) -> Response<Body> {
    use axum::response::IntoResponse;
    if method != Method::GET && method != Method::HEAD {
        return StatusCode::NOT_FOUND.into_response();
    }
    (
        StatusCode::OK,
        [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
        ROOT.replace("<origin>", &svc.oauth.issuer),
    )
        .into_response()
}