chadtmiller.com/monolith
An atproto PDS
0
0
//! What the server tells whoever runs it, as `spec/OPERATIONS.md` says: its
//! log, JSON lines on standard output, and kept in `PDS_LOG_DIR` too
//! (`kept`); `/xrpc/_health`; its metrics (`metrics`); its traces
//! (`trace`); the admin's reads of the kept lines and of the metrics,
//! `/xrpc/_logs` and `/xrpc/_stats` (`admin`); and how it shuts down, and
//! says on its next start whether the last one did.
//!
//! Every request goes through `Seen`, in the one place connections are
//! served (`limits::serve`): its route, its trace, and once its answer is
//! whole, before the last byte of it is sent, its log line, its metrics and
//! its span.
mod admin;
mod kept;
mod metrics;
mod trace;
pub(crate) use admin::{Read, logs, read, stats};
pub(crate) use trace::{Target, execute, send};
use crate::Shared;
use axum::body::{Body, Bytes};
use axum::http::{Method, Request, Response, StatusCode, header};
use http_body::{Body as _, Frame, SizeHint};
use serde_json::{Value as Json, json};
use std::fs::File;
use std::io::Write;
use std::path::Path;
use std::pin::Pin;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
/// The server's version, as `/xrpc/_health` and the `start` line say it.
pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
/// The mark a shutdown leaves in the data directory, taken away at the next start.
pub(crate) const CLEAN: &str = "clean-shutdown";
/// What the mark begins with when every frame of every log in the
/// directory is on disk: the shutdown that left it closed every log and
/// made durable all they held (`settle`), in a run that had found the rest
/// durable or made it so (`mark`). After it the mark names the directory
/// it was written in (`word`), and it is the word only there: the next to
/// open that directory then syncs no log as it opens it (`store`, "What a
/// start syncs"), and takes the word out of the mark before it writes
/// anything (`durable`). A mark without it, as a version before left, or a
/// run shut down before it had made durable what it found, says only that
/// the server shut down; and so does one that names another directory, as
/// a copy's does.
pub(crate) const DURABLE: &[u8] = b"durable\n";
/// How long a shutdown lets the requests in flight finish.
pub(crate) const DRAIN: Duration = Duration::from_secs(10);
/// How long a shutdown waits on the spans it holds being exported.
const FLUSH: Duration = Duration::from_secs(5);
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
pub(crate) enum Level {
Debug,
Info,
Warn,
Error,
/// No line at all.
Off,
}
impl Level {
fn name(self) -> &'static str {
match self {
Level::Debug => "debug",
Level::Info => "info",
Level::Warn => "warn",
Level::Error => "error",
Level::Off => "off",
}
}
}
/// A variable's value; an empty one is as if unset.
fn var(name: &str) -> Option<String> {
std::env::var(name).ok().filter(|v| !v.is_empty())
}
/// What the environment says of the log, the metrics and the traces.
pub(crate) struct Settings {
level: Level,
metrics_port: Option<u16>,
trace: trace::Settings,
kept: Option<kept::Settings>,
}
impl Settings {
/// Read from the environment, the server's own port being `port`: why
/// not, if a variable does not parse.
pub fn from_env(port: u16) -> Result<Settings, String> {
let level = match var("PDS_LOG_LEVEL").as_deref() {
None | Some("info") => Level::Info,
Some("debug") => Level::Debug,
Some("warn") => Level::Warn,
Some("error") => Level::Error,
Some("off") => Level::Off,
Some(v) => {
return Err(format!(
"PDS_LOG_LEVEL is {v:?}, not debug, info, warn, error or off"
));
}
};
let metrics_port = match var("PDS_METRICS_PORT") {
None => None,
Some(v) => match v.trim().parse::<u16>() {
Ok(p) if p == port => {
return Err(format!(
"PDS_METRICS_PORT is {p}, the port PDS_PORT serves on"
));
}
Ok(p) if p > 0 => Some(p),
_ => return Err(format!("PDS_METRICS_PORT is {v:?}, not a port")),
},
};
Ok(Settings {
level,
metrics_port,
trace: trace::Settings::from_env()?,
kept: kept::Settings::from_env()?,
})
}
}
/// Where lines go: standard output, and the files they are kept in, if any.
struct Out {
/// Written to with one `write` a line.
stdout: Option<File>,
kept: Option<kept::Kept>,
}
/// What every request and every line shares.
struct Ops {
level: Level,
/// Held while a line is written, to both.
out: Mutex<Out>,
metrics: metrics::Metrics,
/// Where spans go, if they are exported.
exporter: Option<trace::Exporter>,
/// The `time` of this start's `start` line.
started: String,
}
static OPS: OnceLock<Ops> = OnceLock::new();
/// As set at start; with none set, as in tests, nothing is written or exported.
fn ops() -> &'static Ops {
OPS.get_or_init(|| Ops {
level: Level::Off,
out: Mutex::new(Out {
stdout: None,
kept: None,
}),
metrics: metrics::Metrics::default(),
exporter: None,
started: crate::events::timestamp(),
})
}
/// Standard output, as a file of its own, written past Rust's buffering.
fn stdout() -> Option<File> {
use std::os::fd::AsFd;
std::io::stdout()
.as_fd()
.try_clone_to_owned()
.ok()
.map(File::from)
}
/// Take the settings, and write the `start` line: the environment is read,
/// and the data not yet. The lines kept in `PDS_LOG_DIR` go on after those
/// there.
pub(crate) fn start(s: Settings, port: u16, sync: crate::Syncing) {
let exporter = s.trace.exporter();
let started = crate::events::timestamp();
let _ = OPS.set(Ops {
level: s.level,
out: Mutex::new(Out {
stdout: stdout(),
kept: s.kept.map(kept::Kept::open),
}),
metrics: metrics::Metrics::default(),
exporter,
started: started.clone(),
});
pds_store::on_sync(|d| ops().metrics.fsync(d));
// What the store does in the background and could not: a compaction,
// a checkpoint of the journal, a sync of it.
pds_store::on_error(|why| failed("store failure", why));
// A panic is a line of the log, as nothing but why the server would not
// start goes to standard error.
std::panic::set_hook(Box::new(|info| {
let at = info
.location()
.map(|l| format!("{}:{}", l.file(), l.line()));
error("panic", &[("at", json!(at))]);
}));
line_at(
&started,
Level::Info,
"start",
&[
("version", json!(VERSION)),
("port", json!(port)),
("sync", json!(sync.name())),
("metrics_port", json!(s.metrics_port)),
],
);
if let Some(p) = s.metrics_port {
METRICS_PORT.set(p).ok();
}
}
static METRICS_PORT: OnceLock<u16> = OnceLock::new();
/// Write a line at `level`, with `fields` after `time`, `level` and `msg`;
/// a field that is null is left out.
pub(crate) fn line(level: Level, msg: &str, fields: &[(&str, Json)]) {
line_at(&crate::events::timestamp(), level, msg, fields);
}
/// Write a line whose `time` is `time`: to standard output, then, the same
/// bytes, to the files it is kept in, before the next line goes to either.
fn line_at(time: &str, level: Level, msg: &str, fields: &[(&str, Json)]) {
let o = ops();
if o.level == Level::Off || level < o.level {
return;
}
let mut s = String::with_capacity(192);
s.push_str("{\"time\":\"");
s.push_str(time);
s.push_str("\",\"level\":\"");
s.push_str(level.name());
s.push_str("\",\"msg\":");
s.push_str(&Json::from(msg).to_string());
for (k, v) in fields {
if v.is_null() {
continue;
}
s.push(',');
s.push_str(&Json::from(*k).to_string());
s.push(':');
s.push_str(&v.to_string());
}
s.push_str("}\n");
let mut out = o.out.lock().unwrap_or_else(PoisonError::into_inner);
if let Some(mut f) = out.stdout.as_ref() {
let _ = f.write_all(s.as_bytes());
}
if let Some(k) = &mut out.kept {
k.write(s.as_bytes());
}
}
/// The lines kept in `PDS_LOG_DIR`, as they are now, to read with no lock
/// held; None with no `PDS_LOG_DIR`.
fn kept_now() -> Option<kept::Snapshot> {
let out = ops().out.lock().unwrap_or_else(PoisonError::into_inner);
out.kept.as_ref().map(kept::Kept::snapshot)
}
pub(crate) fn info(msg: &str, fields: &[(&str, Json)]) {
line(Level::Info, msg, fields);
}
pub(crate) fn warn(msg: &str, fields: &[(&str, Json)]) {
line(Level::Warn, msg, fields);
}
pub(crate) fn error(msg: &str, fields: &[(&str, Json)]) {
line(Level::Error, msg, fields);
}
/// A line of an error of the server's own: `msg` says what failed, and
/// `error` why.
pub(crate) fn failed(msg: &str, why: impl std::fmt::Display) {
error(msg, &[("error", json!(why.to_string()))]);
}
/// A line at `level` of something that failed for an account, and why.
pub(crate) fn failed_for(level: Level, msg: &str, did: &str, why: impl std::fmt::Display) {
line(
level,
msg,
&[("did", json!(did)), ("error", json!(why.to_string()))],
);
}
// ------------------------------------------------------------ starts
/// Whether the server starting on `dir` finds it as the last one left it
/// while running: it holds data, a service log of any generation, its
/// checkpoint, or the single log of the layout before generations, and no
/// mark of a shutdown.
pub(crate) fn unclean(dir: &Path) -> bool {
crate::store::holds_data(dir) && !dir.join(CLEAN).exists()
}
/// The word as the mark `mark`, open, of the data directory `dir` holds
/// it, for the directory as it is now: `DURABLE`, and then what a copy of
/// the directory does not keep (`store`, "The word is its directory's"):
/// of the directory and of `repos`, the id of the file system each is on,
/// its inode and when it last changed (its `ctime`, to the nanosecond);
/// and the mark's own inode. With it, when the data directory last
/// changed, which the mark's own time of writing is set to (`mark`) and
/// must still be (`durable`). None on a file system that gives no id,
/// zeros, where the mark never holds the word.
///
/// The id is `f_fsid` as `fstatvfs` gives it. On Linux that is both
/// halves of the kernel's `f_fsid`, which a file system makes of its own
/// UUID (ext4, btrfs, tmpfs) or of its device's number (XFS), and which is
/// zeros where it makes none (NFS, FUSE). On macOS it is the first half
/// alone, the device's number.
fn word(dir: &Path, mark: &File) -> std::io::Result<Option<(Vec<u8>, std::time::SystemTime)>> {
use std::os::unix::fs::MetadataExt;
/// A directory's file system, inode and change time, as the mark
/// names them, and the change time: none with no id, or a time before
/// any clock's.
fn place(dir: &File) -> std::io::Result<Option<(String, std::time::SystemTime)>> {
let fs = rustix::fs::fstatvfs(dir)?.f_fsid;
let m = dir.metadata()?;
let (ino, at, nanos) = (m.ino(), m.ctime(), m.ctime_nsec());
let (Ok(secs), Ok(sub), true) = (u64::try_from(at), u32::try_from(nanos), fs != 0) else {
return Ok(None);
};
let changed = std::time::UNIX_EPOCH + Duration::new(secs, sub);
Ok(Some((format!("{fs:x} {ino} {at}.{nanos:09}"), changed)))
}
let Some((top, changed)) = place(&File::open(dir)?)? else {
return Ok(None);
};
let repos = match File::open(dir.join("repos")) {
Ok(repos) => match place(&repos)? {
Some((repos, _)) => repos,
None => return Ok(None),
},
Err(e) if e.kind() == std::io::ErrorKind::NotFound => "none".into(),
Err(e) => return Err(e),
};
let mut word = DURABLE.to_vec();
let mark = mark.metadata()?.ino();
writeln!(word, "dir {top}\nrepos {repos}\nmark {mark}")?;
Ok(Some((word, changed)))
}
/// Whether the mark `mark`, open, which holds `held`, is the word of the
/// data directory `dir` as it is now: it holds what `word` makes of the
/// directory, and was last written when the directory last changed. What
/// cannot be told of the directory is no word.
fn is_word(dir: &Path, mark: &File, held: &[u8]) -> bool {
let Ok(Some((word, changed))) = word(dir, mark) else {
return false;
};
let written = mark.metadata().and_then(|m| m.modified());
word == held && written.is_ok_and(|at| at == changed)
}
/// Whether the last to use `dir` left every frame of every log in it on
/// disk: a server that shut down there, as its mark says (`mark`), with
/// the word of this directory as it is now (`is_word`). Any other mark is
/// no word: one that names another directory, as a copy of a stopped
/// server's does, whose bytes may be in the system's memory alone; one
/// written since its shutdown wrote it; one of a form this does not
/// write; an empty one. Whatever the mark held is taken out of it,
/// durably, before this answers, as whoever asks is about to write there:
/// a kill of this run, whatever it has written by then, is not taken for
/// that shutdown by the next. The mark itself stays, for `unclean` and
/// `recovered`.
///
/// Asked before anything of the directory is changed, its mode too
/// (`Service::open`), which is set after: setting it changes when the
/// directory last changed, so that what the mark held names the
/// directory no more, whatever puts it back.
pub(crate) fn durable(dir: &Path) -> Result<bool, String> {
use std::io::Read;
let path = dir.join(CLEAN);
let at = |e: std::io::Error| format!("{}: {e}", path.display());
let mut mark = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&path)
{
Ok(mark) => mark,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
Err(e) => return Err(at(e)),
};
let mut held = Vec::new();
mark.read_to_end(&mut held).map_err(at)?;
if held.is_empty() {
return Ok(false);
}
let here = is_word(dir, &mark, &held);
mark.set_len(0)
.and_then(|()| pds_store::sync_file(&mark))
.map_err(at)?;
Ok(here)
}
/// The data is read: say so if the last server did not shut down, and take
/// away the mark of its shutdown, so that this one leaves it only if it does.
pub(crate) fn recovered(svc: &Shared, unclean: bool) -> Result<(), String> {
if unclean {
// Of the logs the start read: the service's, and those of the
// accounts it loaded. An account loaded later says what its own
// log dropped then (`cache::Slot::load`): a start reads no more of
// the accounts than it must (`spec/STORE.md`, "Scale").
let mut dropped = svc.log.dropped();
for a in crate::read(&svc.directory).by_did.values() {
dropped += a.store.loaded_wal().map_or(0, |w| w.dropped());
}
warn("recovered", &[("dropped_bytes", json!(dropped))]);
}
let mark = svc.dir.join(CLEAN);
if mark.exists() {
std::fs::remove_file(&mark)
.and_then(|()| pds_store::sync_dir(&svc.dir))
.map_err(|e| format!("{}: {e}", mark.display()))?;
}
Ok(())
}
/// Leave the mark of a shutdown in the data directory, durably, `settle`
/// having answered. It holds the word that every frame of every log there
/// is on disk only where that is so of the logs this run never opened too
/// (`cache::Ctx::found`): the run started with the word, or has made
/// durable what it found (`sync_found`); and where no load of a store
/// failed in the run (`cache::Ctx::failed`), which may have left part of
/// a mend synced by nothing. Otherwise the mark says only that the server
/// shut down, and the next start syncs each log as it starts.
///
/// The mark is made anew, the one before deleted if a start left it: so
/// the directory last changed as this mark was made, which the word then
/// names (`word`), and nothing changes the directory after. The mark's
/// time of writing is set to that same time, once the word is written:
/// whatever writes the mark again, as a copy of the directory put back
/// over it file by file does, leaves it another.
pub(crate) fn mark(s: &crate::Service) -> std::io::Result<()> {
let path = s.dir.join(CLEAN);
match std::fs::remove_file(&path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e),
_ => {}
}
let mut f = pds_store::open_private(
&path,
std::fs::OpenOptions::new().create_new(true).write(true),
)?;
if s.ctx.found.load(Ordering::SeqCst)
&& !s.ctx.failed.load(Ordering::SeqCst)
&& !s.events.owes()
&& let Some((word, changed)) = word(&s.dir, &f)?
{
f.write_all(&word)?;
f.set_modified(changed)?;
}
pds_store::sync_file(&f)?;
pds_store::sync_dir(&s.dir)
}
/// Make durable what this run found in the data directory as it started,
/// unless that is so already: every account's logs and table, those of the
/// accounts nothing has loaded too (`store`, "What a start found, made
/// durable"), and the firehose's index, which is read first and so synced
/// as it is opened. Its shutdown leaves the word in its mark from then on.
pub(crate) fn sync_found(s: &crate::Service) -> std::io::Result<()> {
if !s.ctx.found.load(Ordering::SeqCst) {
s.events.find_old();
crate::store::sync_found(&s.dir)?;
s.ctx.found.store(true, Ordering::SeqCst);
}
Ok(())
}
/// The same, on a thread of its own while the server serves: a start
/// waits for none of it, nor does a shutdown, which leaves no word if it
/// comes first; one that fails says so, and leaves none either.
pub(crate) fn sync_found_soon(svc: &Shared) {
if svc.ctx.found.load(Ordering::SeqCst) {
return;
}
let svc = svc.clone();
tokio::task::spawn_blocking(move || {
if let Err(e) = sync_found(&svc) {
failed("syncing what the start found", e);
}
});
}
/// What runs beside the server: the metrics' port, and the export of spans.
pub(crate) async fn serve_beside(svc: &Shared) -> Result<(), String> {
if let Some(e) = &ops().exporter {
tokio::spawn(e.run());
}
if let Some(&port) = METRICS_PORT.get() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", port))
.await
.map_err(|e| format!("listening on PDS_METRICS_PORT {port}: {e}"))?;
tokio::spawn(metrics::serve(listener, svc.clone()));
}
Ok(())
}
/// The first SIGTERM or SIGINT, by name, from now on.
pub(crate) fn signals() -> impl Future<Output = &'static str> {
use tokio::signal::unix::{SignalKind, signal};
let term = signal(SignalKind::terminate());
let int = signal(SignalKind::interrupt());
async move {
match (term, int) {
(Ok(mut term), Ok(mut int)) => tokio::select! {
_ = term.recv() => "SIGTERM",
_ = int.recv() => "SIGINT",
},
_ => std::future::pending().await,
}
}
}
/// Shut down, the server having stopped taking connections and its requests
/// finished: close the firehose's subscriptions, write the issuers' keys
/// kept (`space_credentials::issuers`, a cache, not synced), close the logs
/// and sync what is written (`settle`), export the spans held, leave the
/// mark of a shutdown, say so, and exit 0. The mark is left only once
/// `settle` has answered that all this run wrote is on disk: where it
/// fails, or the process is killed before the mark, the next start is as
/// after a kill.
/// It is so too, but for its `recovered` line, where what this run found
/// was not yet made durable (`mark`), which nothing here waits for.
pub(crate) async fn shutdown(svc: &Shared, signal: &'static str) -> ! {
svc.events.close();
for _ in 0..40 {
if svc.events.subscribers() == 0 {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
let s = svc.clone();
let synced = tokio::task::spawn_blocking(move || {
crate::space_credentials::issuers::write(&s);
settle(&s)
})
.await;
if let Some(e) = &ops().exporter {
e.flush(Instant::now() + FLUSH).await;
}
match synced {
Ok(Ok(())) => {
if let Err(e) = mark(svc) {
failed("marking the shutdown", e);
}
}
Ok(Err(e)) => failed("syncing the logs", e),
Err(e) => failed("syncing the logs", e),
}
info("shutdown", &[("signal", json!(signal))]);
std::process::exit(0);
}
/// What a shutdown leaves on disk: no event is numbered from now on, and
/// every log is closed and what it holds made durable, the firehose's
/// index last (`events::Firehose::close_index`), which an account's store
/// loaded as the others close may still append void entries to.
///
/// Once this answers, every frame this run gave a log is on disk, with
/// every frame of each log it opened, and no log takes another; no log
/// this run opened holds room:
///
/// - the service log is closed (`pds_store::Wal::close`): an append from
/// now on is refused, its room is given back, and every frame it took is
/// synced;
/// - the accounts' logs are closed together, as the pool they are all of
/// (`pds_store::Files::close`), and from then on no account's store is
/// loaded (`cache::Slot::load`): a load mends what it finds, which is a
/// write, and the mark could be left before one begun after this had
/// synced what it mended. Then the log of each store held is closed in
/// turn, which gives back its room and syncs every frame it took, with
/// the logs a compaction sealed and the files its commits rest on. A
/// store is looked for under its slot's lock, which a load holds from
/// before it asks whether the logs are closed until its store is held:
/// so one being loaded as the logs were closed is waited for, what it
/// mended synced by its own open, and closed here as any held; and one
/// that failed is noted under that lock (`cache::Ctx::failed`), so that
/// the mark holds no word, before it stops the server;
/// - a store not held was let go with every frame of its log on disk and
/// no compaction of it running (`Store::idle`), its log's room given back
/// as it went, and nothing has been appended to its log since;
/// - an account being made, not yet among those listed here, has in its
/// log its first commit alone, synced before the service log is given
/// the account (`server`), and the room that commit made;
/// - with `PDS_SYNC` `together`, where "synced" above is the journal's
/// sync, the journal is closed last (`pds_store::Journal::close`): every
/// log written to since its last checkpoint, those let go and those of
/// accounts being made too, is made durable in its own file, and its
/// segments deleted, so that no journal is left. It takes no record from
/// then on, so that what writes to a log after is refused.
///
/// A log this run never opened, of an account nothing asked for, is not
/// among these: it has no store, and holds what the run found. That is on
/// disk where the run started with the mark's word, or has since made
/// durable what it found (`sync_found`), and only then does the mark say
/// that every frame of every log is (`mark`).
///
/// The cut that gives room back is not itself synced, and a log no run
/// since a kill has opened keeps the room it had then: zeros after a log's
/// frames are room to the open that finds them, cut off with no sync where
/// the frames are on disk (`pds_store`, "An open that syncs nothing"). A
/// log's compaction under way makes no `.tmp`, puts no checkpoint in place
/// and deletes no log once its log is closed, each of which it does only
/// under the log's lock with its files found not let go there
/// (`pds_store::generations`): it leaves its files as a kill at one of
/// its steps would. Whatever writes to a log from now on, a request that
/// outlived the shutdown's wait or the server's own housekeeping, is
/// refused as a write that fails is; and whatever asks for a store not
/// held is unwound, a request answered an internal error
/// (`cache::ShuttingDown`), with nothing of the account's files read,
/// made, mended or synced.
pub(crate) fn settle(s: &crate::Service) -> std::io::Result<()> {
s.events.seal();
s.log.close()?;
s.ctx.files.close();
let accounts: Vec<_> = crate::read(&s.directory).by_did.values().cloned().collect();
for wal in accounts.iter().filter_map(|a| a.store.loaded_wal()) {
wal.close()?;
}
s.events.close_index()?;
if let Some(j) = s.ctx.files.journal() {
j.close()?;
}
Ok(())
}
// ------------------------------------------------------------ requests
/// The trace context of the request being answered, if any, for what is
/// spawned on its behalf.
pub(crate) fn spawn<F>(f: F) -> tokio::task::JoinHandle<F::Output>
where
F: Future + Send + 'static,
F::Output: Send + 'static,
{
match trace::current() {
Some(ctx) => tokio::spawn(trace::CTX.scope(ctx, f)),
None => tokio::spawn(f),
}
}
/// The account whose bearer token the request being answered carried, the
/// server having verified it.
pub(crate) fn verified(did: &str) {
if let Some(ctx) = trace::current() {
let _ = ctx.did.set(did.to_string());
}
}
/// The blob the `getBlob` being answered serves, of the account `did`: its
/// request line names it (`spec/OPERATIONS.md`, "Events").
pub(crate) fn serving_blob(did: &str, cid: &str) {
if let Some(ctx) = trace::current() {
let _ = ctx.blob.set((did.to_string(), cid.to_string()));
}
}
/// Wait on `f`, a wait on writes reaching the disk, as a span `fsync` of
/// the request being answered.
pub(crate) async fn fsync<T>(f: impl Future<Output = T>) -> T {
let ctx = trace::current().filter(|c| c.sampled && ops().exporter.is_some());
let start = trace::now_ns();
let out = f.await;
if let (Some(ctx), Some(e)) = (ctx, &ops().exporter) {
e.push(ctx.child("fsync".into(), trace::INTERNAL, start, vec![], false));
}
out
}
/// The routes outside `/xrpc/` the server serves, each its own label: the
/// root, the MCP endpoint, `atproto-did`, OAuth's metadata, endpoints and
/// pages, and the MCP issuer's (`spec/OAUTH-MCP.md`).
const ROUTES: &[&str] = &[
"/",
"/mcp",
"/.well-known/atproto-did",
"/.well-known/oauth-protected-resource",
"/.well-known/oauth-authorization-server",
"/oauth/jwks",
"/oauth/par",
"/oauth/token",
"/oauth/revoke",
"/oauth/authorize",
"/oauth/authorize/redirect",
"/@atproto/oauth-provider/~api/sign-in",
"/@atproto/oauth-provider/~api/consent",
"/@atproto/oauth-provider/~api/reject",
"/@atproto/oauth-provider/~api/sign-out",
"/@atproto/oauth-provider/~api/passkey-options",
"/@atproto/oauth-provider/~api/passkey-sign-in",
"/oauth/passkey",
"/@atproto/oauth-provider/~api/passkey-register",
"/.well-known/oauth-protected-resource/mcp",
"/.well-known/oauth-authorization-server/oauth/mcp",
"/oauth/mcp/authorize",
"/oauth/mcp/token",
"/oauth/mcp/revoke",
];
/// The route of every path under `/oauth/assets/`, whatever its name,
/// served or not: the pages' build (`spec/OAUTH-PAGES.md`, "Assets").
const ASSETS: &str = "/oauth/assets/{name}";
/// A request's route, as its log line and metrics label it: the NSID of a
/// method served, `_health`, `_logs` or `_stats`, the account's own routes
/// as `_oauthSessions` and `_passkeyList`, a path served outside `/xrpc/`,
/// or `_unknown`.
fn route_of(path: &str) -> &'static str {
if let Some(own) = ["_health", "_logs", "_stats"]
.into_iter()
.chain(crate::oauth::account::Route::ALL.map(|r| r.name()))
.find(|r| path.strip_prefix("/xrpc/") == Some(*r))
{
return own;
}
if let Some(nsid) = path.strip_prefix("/xrpc/") {
return pds_lexicon::method(nsid).map_or("_unknown", |m| m.nsid.as_str());
}
if path.starts_with("/oauth/assets/") {
return ASSETS;
}
ROUTES
.iter()
.find(|r| **r == path)
.copied()
.unwrap_or("_unknown")
}
/// A request, from when its head was read.
pub(crate) struct Seen {
started: Instant,
start_ns: u64,
method: Method,
path: String,
route: &'static str,
ctx: Arc<trace::Ctx>,
}
impl Seen {
pub fn new<B>(req: &Request<B>) -> Seen {
let ctx = trace::Ctx::of(
req.headers()
.get("traceparent")
.and_then(|v| v.to_str().ok()),
);
Seen {
started: Instant::now(),
start_ns: trace::now_ns(),
method: req.method().clone(),
path: req.uri().path().to_string(),
route: route_of(req.uri().path()),
ctx: Arc::new(ctx),
}
}
/// Answer the request with `f`, in its trace.
pub async fn run<F: Future>(&self, f: F) -> F::Output {
trace::CTX.scope(self.ctx.clone(), f).await
}
/// The answer, with its line written once it is whole: now, if it has no
/// body to send, or is a WebSocket's upgrade; else as its last bytes go.
pub fn answer(self, res: Response<Body>) -> Response<Body> {
let status = res.status();
let bodiless = self.method == Method::HEAD
|| status == StatusCode::SWITCHING_PROTOCOLS
|| status == StatusCode::NO_CONTENT
|| status == StatusCode::NOT_MODIFIED;
if bodiless || res.body().is_end_stream() {
self.done(status.as_u16(), 0);
return res;
}
let exact = res.body().size_hint().exact();
res.map(|inner| {
Body::new(Logged {
inner,
seen: Some(self),
status: status.as_u16(),
sent: 0,
exact,
})
})
}
/// The answer is whole, `sent` bytes of its body handed on: its line,
/// metrics and span.
fn done(self, status: u16, sent: u64) {
let o = ops();
let secs = self.started.elapsed().as_secs_f64();
let blob = self
.ctx
.blob
.get()
.filter(|_| status == 200 || status == 500);
let fields = [
("method", json!(self.route)),
("status", json!(status)),
("ms", json!(secs * 1000.0)),
("trace_id", json!(hex::encode(self.ctx.trace_id))),
("did", json!(self.ctx.did.get())),
("repo", json!(blob.map(|(did, _)| did))),
("cid", json!(blob.map(|(_, cid)| cid))),
("bytes", json!(blob.filter(|_| status == 200).map(|_| sent))),
];
line(
if status >= 500 {
Level::Error
} else {
Level::Info
},
"request",
&fields,
);
o.metrics.request(self.route, status, secs);
if let Some(e) = &o.exporter
&& self.ctx.sampled
{
e.push(trace::server_span(
&self.ctx,
&self.method,
&self.path,
self.route,
status,
self.start_ns,
));
}
}
}
/// An answer's body, which writes its request's line before its last bytes
/// are handed on, or once it is dropped short of them, as when the client
/// leaves.
struct Logged {
inner: Body,
seen: Option<Seen>,
status: u16,
/// The bytes handed on, and those there are, if known.
sent: u64,
exact: Option<u64>,
}
impl Logged {
fn done(&mut self) {
if let Some(seen) = self.seen.take() {
seen.done(self.status, self.sent);
}
}
}
impl http_body::Body for Logged {
type Data = Bytes;
type Error = axum::Error;
fn poll_frame(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, axum::Error>>> {
let this = &mut *self;
match Pin::new(&mut this.inner).poll_frame(cx) {
Poll::Ready(Some(Ok(frame))) => {
if let Some(d) = frame.data_ref() {
this.sent += d.len() as u64;
}
if this.inner.is_end_stream() || this.exact.is_some_and(|n| this.sent >= n) {
this.done();
}
Poll::Ready(Some(Ok(frame)))
}
Poll::Ready(other) => {
this.done();
Poll::Ready(other)
}
Poll::Pending => Poll::Pending,
}
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> SizeHint {
self.inner.size_hint()
}
}
impl Drop for Logged {
fn drop(&mut self) {
self.done();
}
}
// ------------------------------------------------------------ health
/// `/xrpc/_health`: the version, to `GET` and `HEAD`, whatever the query.
pub(crate) async fn health(method: Method) -> Response<Body> {
use axum::response::IntoResponse;
if method != Method::GET && method != Method::HEAD {
return crate::respond(
crate::invalid(format!("Incorrect HTTP method ({method}) expected GET")),
None,
);
}
(
StatusCode::OK,
[(header::CONTENT_TYPE, "application/json; charset=utf-8")],
json!({ "version": VERSION }).to_string(),
)
.into_response()
}
// ------------------------------------------------------------ the root
/// What `/` says, `<origin>` standing for the origin of `PDS_PUBLIC_URL`
/// (`spec/OPERATIONS.md`, "The root").
const ROOT: &str = r#"
__ __
/\ \__ /\ \__
__ \ \ ,_\ _____ _ __ ___\ \ ,_\ ___
/'__'\ \ \ \/ /\ '__'\/\''__\/ __'\ \ \/ / __'\
/\ \L\.\_\ \ \_\ \ \L\ \ \ \//\ \L\ \ \ \_/\ \L\ \
\ \__/.\_\\ \__\\ \ ,__/\ \_\\ \____/\ \__\ \____/
\/__/\/_/ \/__/ \ \ \/ \/_/ \/___/ \/__/\/___/
\ \_\
\/_/
This is an AT Protocol Personal Data Server (aka, an atproto PDS)
Most API routes are under /xrpc/
Connect an AI app
1. Add this address to your AI app as a connector:
<origin>/mcp
In Claude: Settings > Connectors > Add custom connector,
and choose "Sign in when needed".
2. When it asks, sign in here and approve it.
3. Ask it things like:
What have I posted this week?
Post "hello from Claude" to Bluesky
Who have I followed lately?
Set my Bluesky bio to "Software engineer in Seattle"
Code: https://knotless.dev/chadtmiller.com/monolith
Protocol: https://atproto.com
"#;
/// `/`: what the server is, and how to connect an AI app to it, to `GET`
/// and `HEAD`, whatever the query; any other method is no route.
pub(crate) async fn root(
axum::extract::State(svc): axum::extract::State<Shared>,
method: Method,
) -> Response<Body> {
use axum::response::IntoResponse;
if method != Method::GET && method != Method::HEAD {
return StatusCode::NOT_FOUND.into_response();
}
(
StatusCode::OK,
[(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
ROOT.replace("<origin>", &svc.oauth.issuer),
)
.into_response()
}