//! The PDS: XRPC, the firehose, and the MCP endpoint whose tools are its methods. //! //! This module is the router, the state every request shares, and the //! dispatch of a call to its method. Each area's methods are in a module of //! their own, named in `AREAS`: `server` (the server, its accounts and their //! sessions), `sessions` (tokens, refreshing and ending sessions, and app //! passwords), `email` (the account's email, the tokens mailed for it, and //! the mailer), `repo` (records), `sync` (the repository whole), //! `federation` (what a relay reads, and the crawlers the server tells), //! `identity` (handles), `blobs` (kept in the data directory, or in a //! bucket, `blobstore`, and moved from one to the other by `pds //! migrate-blobs`, `migrate_blobs`), `invites`, `lifecycle` (deactivating, //! taking down and deleting accounts, and what each state does to every //! method), `migration` (accounts moving between servers: `importRepo`, //! `listMissingBlobs` and the PLC operations that move a DID), `proxy` //! (requests for other services, service tokens) with //! `read_after_write` and `preferences`, `space_repo` (each account's space //! repos), `space_credentials` (spaces' delegation tokens, credentials and //! revocations), `space_authority` (the spaces each account is the //! authority of, with who their policies authorize in `space_authz`) and //! `space_notify` (write notifications, the writer set, syncers' //! registrations, and the updates and notices sent them); the firehose is `events`, the MCP //! endpoint `mcp`, the PLC directory `plc`, requests to URLs others name, //! with SSRF protection, `outbound`, OAuth's authorization server `oauth`, //! with the account's own routes for its OAuth sessions and devices, and //! for its passkeys, in `oauth::account`, and passkeys, which sign it in //! on the authorization page, in `oauth::passkeys` (they have no NSID, so //! no area of `AREAS` answers them), //! OAuth access tokens on every method and what their scopes allow //! `scopes`, the single-use values OAuth spends `replay`, single-use values //! the spaces spend `single_use`, what keeps the server up when anyone may //! call it `limits`, what the server keeps, how it starts again from it, //! and how it compacts it into checkpoints, `log`, `store`, `checkpoint` //! (an account's) and `compact` (the service log's), with every account, //! found by DID, handle, email or reset token and loaded on first use, //! `directory`, what it holds of each account's store in memory, loaded on //! first use and let go, `cache`, and what it tells whoever runs it, its //! log, kept lines, health, metrics, traces and shutdown, and the admin's //! reads of them, `ops`. #![forbid(unsafe_code)] mod blobs; mod blobstore; mod cache; mod checkpoint; mod compact; mod cors; mod directory; mod email; mod events; mod federation; mod identity; mod invites; mod lifecycle; mod limits; mod log; mod mcp; mod migrate_blobs; mod migration; mod oauth; mod ops; mod outbound; mod plc; mod preferences; mod proxy; mod read_after_write; mod replay; mod repo; mod scopes; mod server; mod sessions; mod single_use; mod space_authority; mod space_authz; mod space_credentials; mod space_notify; mod space_repo; mod store; mod sync; mod validation; pub use blobstore::Place; use axum::{ Router, body::{Body, Bytes}, extract::{Extension, State as AxState}, http::{HeaderMap, Method, StatusCode, Uri, header}, response::{IntoResponse, Response}, routing::{any, get}, }; use oauth::account::{self, Route as Own}; use pds_lexicon::Kind; use pds_repo::{Refused, Repo}; use pds_store::Wal; use serde_json::{Map, Value as Json, json}; use std::path::PathBuf; use std::pin::Pin; use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; /// The server's configuration, from its environment (`spec/ACCOUNTS.md`, /// `spec/FEDERATION.md`, `spec/LIMITS.md`). pub(crate) struct Config { /// Its DID is `did:web:`. pub hostname: String, /// The URL its accounts' DID documents name as their PDS. pub public_url: String, /// The domains handles may end in, each with its leading dot, in the order given. pub handle_domains: Vec, pub plc_url: String, /// The key that signs its PLC operations. pub rotation_key: k256::ecdsa::SigningKey, /// A `did:key` listed before the server's own as a rotation key. pub recovery_did_key: Option, /// The base URLs of the crawlers to tell of new events (`spec/FEDERATION.md`). pub crawlers: Vec, /// `PDS_OAUTH_CLIENT_FETCH`, for development only (`spec/OAUTH-CLIENTS.md`). pub oauth_client_fetch: Option, /// `PDS_LEXICON_AUTHORITY_DID`: every NSID's authority, in place of DNS /// (`spec/OAUTH-PERMISSION-SETS.md`). pub lexicon_authority: Option, /// The DNS server `_lexicon` records are looked up from, in place of the /// system's: the tests' stub; none from the environment. pub lexicon_dns: Option, /// `PDS_HANDLE_DNS_SERVER`: the DNS server handles' `_atproto` records /// are looked up from, in place of the system's (`spec/IDENTITY.md`). pub handle_dns: Option, /// `PDS_HANDLE_FETCH`, for development only: the base URL handles' /// `/.well-known/atproto-did` are fetched from instead (`spec/IDENTITY.md`). pub handle_fetch: Option, /// Whether `createAccount` needs an invite code. pub invite_required: bool, /// The password of the admin's `Basic` auth; with none (unset or empty), no request is the admin. pub admin_password: Option, /// The proxies whose `X-Forwarded-For` is read (`PDS_TRUSTED_PROXIES`, `spec/LIMITS.md`). pub trusted_proxies: limits::Trusted, /// Whether the rate limits apply. pub rate_limits: bool, /// The bytes of blobs an account may hold. pub blob_quota: u64, /// How long, in seconds, an upload no record references is held. pub blob_ttl: u64, /// The free space below which writes are refused. pub min_free: u64, /// The bytes of the CAR file `com.atproto.repo.importRepo` takes /// (`PDS_MAX_REPO_IMPORT_SIZE`, `spec/LIMITS.md`). pub max_import: u64, /// The bytes of a blob `uploadBlob` takes, and of a space repo's archive /// `com.atproto.space.importRepo` takes (`PDS_BLOB_UPLOAD_LIMIT`, /// `spec/LIMITS.md`); `describeServer` answers it. pub blob_upload_limit: u64, /// How long, in seconds, a DID document fetched from the directory is /// served, and a spaces token's issuer's key used with no read /// (`PDS_DID_CACHE_SECS`). pub did_cache: u64, /// How long, in seconds, a spaces token's issuer's key is used at all, /// read again in the background past `did_cache`: `did_cache` where /// that is more (`PDS_DID_CACHE_MAX_SECS`, `spec/SPACES-CREDENTIALS.md`, /// "Issuers' keys"). pub did_cache_max: u64, /// How long after its input is checked `createSession` answers, at the soonest. pub login_floor: std::time::Duration, /// How long after its body is checked OAuth's sign-in answers, at the soonest. pub sign_in_floor: std::time::Duration, /// Where emails go (`spec/EMAIL.md`). pub mailer: email::Mailer, /// The default AppView and its CDN (`spec/PROXY.md`). pub proxy: proxy::Settings, /// Whether requests to URLs others name are SSRF-protected (`PDS_DISABLE_SSRF_PROTECTION`, `spec/PROXY.md`). pub ssrf_protection: bool, /// The backfill window, `PDS_REPO_BACKFILL_LIMIT_MS` (`spec/STORE.md`). pub backfill: std::time::Duration, /// The bytes a log may grow to past its checkpoint's, whatever that is (`spec/STORE.md`). pub log_floor: u64, /// The most an account's log may grow to past its table, whatever the /// table's size, but `log_floor` if that is more (`store::ACCOUNT_CAP`, /// `spec/STORE.md`, "The first request to a cold account"). pub account_cap: u64, /// `PDS_CACHE_BYTES`: the most held in memory of accounts' state and data read from disk (`spec/STORE.md`). pub cache_bytes: u64, /// The accounts' logs open at once, and as many of their tables: a /// share of the file descriptors the process may have (`spec/LIMITS.md`, /// "Open files and threads"). pub share: usize, /// Whether a start loads every account's state, to find every fault in /// the data directory before serving: the tests' way; the server's own /// loads each on first use. pub load_all: bool, /// The bucket blobs are kept in, if they are not kept in the data /// directory (`spec/BLOBS.md`). pub blobstore: Option, /// `PDS_SYNC`: how waiting writes are made durable (`spec/DURABILITY.md`, /// "Writes waiting at once"). pub sync: Syncing, } /// How waiting writes are made durable (`PDS_SYNC`). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Syncing { /// Every account's logs written to one journal too, whose sync is what /// a write waits for (`pds_store::journal`). Together, /// Each log synced by itself. Each, } impl Syncing { /// Read from `PDS_SYNC`: `together` unset or empty. fn from_env() -> Result { match std::env::var("PDS_SYNC") .ok() .filter(|v| !v.is_empty()) .as_deref() { None | Some("together") => Ok(Syncing::Together), Some("each") => Ok(Syncing::Each), Some(v) => Err(format!("PDS_SYNC: {v:?} is neither together nor each")), } } /// As the `start` line says it. pub fn name(self) -> &'static str { match self { Syncing::Together => "together", Syncing::Each => "each", } } } impl Config { fn from_env() -> Result { let var = |n: &str| std::env::var(n).map_err(|_| format!("{n} is not set")); let key = var("PDS_PLC_ROTATION_KEY_HEX")?; let rotation_key = hex::decode(key.trim()) .ok() .and_then(|b| k256::ecdsa::SigningKey::from_slice(&b).ok()) .ok_or("PDS_PLC_ROTATION_KEY_HEX is not a secp256k1 key as 32 bytes of hex")?; Ok(Config { hostname: var("PDS_HOSTNAME")?, public_url: var("PDS_PUBLIC_URL")?, handle_domains: var("PDS_HANDLE_DOMAINS")? .split(',') .map(str::trim) .filter(|d| !d.is_empty()) .map(str::to_lowercase) .collect(), plc_url: var("PDS_PLC_URL")?, rotation_key, recovery_did_key: std::env::var("PDS_RECOVERY_DID_KEY") .ok() .filter(|k| !k.is_empty()), crawlers: std::env::var("PDS_CRAWLERS") .unwrap_or_default() .split(',') .map(str::trim) .filter(|u| !u.is_empty()) .map(str::to_string) .collect(), oauth_client_fetch: oauth::ClientFetch::from_env()?, lexicon_authority: match std::env::var("PDS_LEXICON_AUTHORITY_DID") { Ok(did) if pds_repo::data::valid_did(&did) => Some(did), Ok(did) => { return Err(format!("PDS_LEXICON_AUTHORITY_DID is not a DID: {did:?}")); } Err(_) => None, }, lexicon_dns: None, handle_dns: identity::resolve::dns_server_from_env()?, handle_fetch: identity::resolve::fetch_from_env()?, invite_required: limits::env_bool("PDS_INVITE_REQUIRED", false)?, admin_password: std::env::var("PDS_ADMIN_PASSWORD") .ok() .filter(|p| !p.is_empty()), trusted_proxies: limits::Trusted::from_env()?, rate_limits: limits::env_bool("PDS_RATE_LIMITS_ENABLED", true)?, blob_quota: limits::env_u64("PDS_BLOB_QUOTA_BYTES", 1 << 30)?, blob_ttl: limits::env_u64("PDS_BLOB_TTL_SECS", 3600)?, min_free: limits::env_u64("PDS_MIN_FREE_BYTES", 256 << 20)?, max_import: limits::env_u64("PDS_MAX_REPO_IMPORT_SIZE", 300 << 20)?, blob_upload_limit: limits::env_u64( "PDS_BLOB_UPLOAD_LIMIT", blobs::DEFAULT_UPLOAD_LIMIT, )?, did_cache: limits::env_u64("PDS_DID_CACHE_SECS", 3600)?, did_cache_max: limits::env_u64("PDS_DID_CACHE_MAX_SECS", 86_400)?, login_floor: std::time::Duration::from_millis(350), sign_in_floor: std::time::Duration::from_millis(400), mailer: email::Mailer::from_env()?, proxy: proxy::Settings::from_env()?, ssrf_protection: outbound::protection_from_env()?, backfill: std::time::Duration::from_millis(limits::env_u64( "PDS_REPO_BACKFILL_LIMIT_MS", 86_400_000, )?), log_floor: store::FLOOR, account_cap: store::ACCOUNT_CAP, cache_bytes: limits::env_u64("PDS_CACHE_BYTES", 256 << 20)?, share: pds_store::files::share(pds_store::files::limit()), load_all: false, blobstore: blobstore::Settings::from_env()?, sync: Syncing::from_env()?, }) } } /// An account, with its repository. pub(crate) struct Account { pub did: String, /// In lower case. Changed only with the directory's lock held, as its /// `handles` are. pub handle: RwLock, /// Its email and password, and what is known of the email /// (`spec/EMAIL.md`): changed only with this lock held. pub mail: Mutex, /// Its KEM key (`spec/SPACES.md`); None for an account made before /// accounts had one. pub kem: Option, /// Its DID document, as the PLC directory last served it, as JSON text /// (held so, a few allocations rather than one for each of its fields), /// and when, in seconds since the epoch: none if it may be out of date. pub did_doc: RwLock<(Box, Option)>, /// Held by the one fetch of its DID document at a time. pub fetching: tokio::sync::Mutex<()>, /// Queries read it side by side; a write has it alone. Loaded from disk /// on first use, and let go when the cache needs the room (`cache`). pub store: cache::Slot, /// Whether it is deactivated or taken down, or deleted /// (`spec/LIFECYCLE.md`): changed only with `store` held for writing too. pub state: RwLock, } impl Account { pub fn handle(&self) -> String { read(&self.handle).clone() } } /// An account's repository and the log it is kept in, apart from every other account's. pub(crate) struct Store { pub repo: Repo, pub wal: Arc, pub blobs: blobs::Blobs, /// Its records taken down, by path, each with its reference. pub takedowns: std::collections::BTreeMap, /// Its space repos (`spec/SPACES-REPO.md`), kept in the same log. pub spaces: space_repo::Spaces, /// The spaces it is the authority of (`spec/SPACES-POLICIES.md`), kept in the same log. pub authority: space_authority::Authority, /// Its write notifications waiting to be sent again, and the /// registrations of its spaces and the notices owed them (`space_notify`). pub notify: space_notify::Kept, /// Its checkpoint's table, and where in its log what the log changed is /// (`checkpoint`): its tree's blocks and paths, and its events. pub disk: checkpoint::Disk, /// The greatest seq of its events, those let go too. pub last_seq: u64, /// The handle its last landed change gave it, if any did. pub logged_handle: Option, /// A handle change of its recorded and not settled. pub pending: Option, /// What the accounts' stores share: the block cache, the marks of those a start loads. pub ctx: Arc, /// The thread that loaded it. #[cfg(test)] pub thread: std::thread::ThreadId, } /// The server, as every request shares it. pub(crate) struct Service { config: Config, dir: PathBuf, /// The service-wide log: accounts, sessions, app passwords and invite codes. log: Arc, /// What loading an account's state needs, and the cache it is held in. ctx: Arc, /// Every account: those loaded, and where the rest are. directory: Arc, /// The sessions given out, and app passwords. sessions: sessions::Sessions, /// OAuth's requests, codes, sessions and devices. oauth: oauth::OAuth, /// Single-use values, such as DPoP proofs' `jti`s. replay: replay::Replay, /// The invite codes, and their uses. invites: invites::Invites, /// The firehose: one sequence across all accounts. events: Arc, http: reqwest::Client, /// Each account's handle changes, one at a time, and any not yet settled. changes: identity::Changes, /// What other parts of spaces hear of each space repo commit. space_hooks: space_repo::Hooks, /// Single-use values: delegation tokens and client attestations spent. single_use: single_use::Spent, /// Spaces' credentials: revocations, and issuers' keys read. space_credentials: space_credentials::Credentials, /// What the spaces' authority shares across accounts: the managing apps /// asked, and the writes of published records. space_authority: space_authority::Host, /// Rate limits, the free space, the clock, and the password hashes' threads. limits: limits::Limits, /// Which account each password reset token is of. resets: email::Resets, /// Every request to another service goes out through it. outbound: outbound::Outbound, /// The DID documents of services read lately. did_docs: proxy::DidDocs, /// The resolver of handles outside the server's domains. handle_resolver: identity::resolve::Resolver, /// Each account's preferences. prefs: preferences::Preferences, /// Write notifications and notices to syncers being sent, and when next. space_notify: space_notify::Notify, } type Shared = Arc; /// Read what a lock guards: an `RwLock`'s value, or an account's store, /// loaded first if it is not (`cache::Slot`). fn read(l: &L) -> L::Read<'_> { l.read_lock() } fn write(l: &L) -> L::Write<'_> { l.write_lock() } /// What `read` and `write` take. trait Locked { type Read<'a> where Self: 'a; type Write<'a> where Self: 'a; fn read_lock(&self) -> Self::Read<'_>; fn write_lock(&self) -> Self::Write<'_>; } impl Locked for RwLock { type Read<'a> = RwLockReadGuard<'a, T> where T: 'a; type Write<'a> = RwLockWriteGuard<'a, T> where T: 'a; fn read_lock(&self) -> RwLockReadGuard<'_, T> { self.read().unwrap_or_else(PoisonError::into_inner) } fn write_lock(&self) -> RwLockWriteGuard<'_, T> { self.write().unwrap_or_else(PoisonError::into_inner) } } impl Locked for Arc { type Read<'a> = L::Read<'a> where L: 'a; type Write<'a> = L::Write<'a> where L: 'a; fn read_lock(&self) -> L::Read<'_> { (**self).read_lock() } fn write_lock(&self) -> L::Write<'_> { (**self).write_lock() } } /// Return once frame `n` of `wal` is on disk, sharing an fsync with whoever /// else waits: an error, logged, if it cannot be made so. async fn durable(wal: Arc, n: u64) -> Result<(), XrpcError> { ops::fsync(async move { if wal.is_durable(n) { return Ok(()); } match tokio::task::spawn_blocking(move || wal.wait(n)).await { Ok(Ok(())) => Ok(()), Ok(Err(e)) => { ops::failed("syncing a log", e); internal() } Err(_) => internal(), } }) .await } /// Who a request's access token says it is. enum Auth { None, /// An account's session, by its DID. Account(String), /// A token the server did not give out, or not an access token, or a /// header not of the `Bearer` scheme. Invalid, /// An access token past its `exp`. Expired, } impl Auth { /// The account's DID, or the error a method that needs one answers. fn did(self) -> Result { match self { Auth::Account(did) => Ok(did), Auth::None => err(401, "AuthMissing", "Authentication Required"), Auth::Invalid => invalid_token(), Auth::Expired => sessions::expired(), } } } impl Service { fn auth(&self, bearer: sessions::Bearer) -> Auth { match sessions::access_token(self, bearer) { Ok(t) => Auth::Account(t.did), Err(e) if e.name == "AuthMissing" => Auth::None, Err(e) if e.name == "ExpiredToken" => Auth::Expired, Err(_) => Auth::Invalid, } } /// Read an account's repository; `missing` is the error if no account has it. fn with_repo( &self, shown: &mut Shown, id: &str, missing: impl FnOnce() -> XrpcError, f: impl FnOnce(&Account, &Repo) -> Result, ) -> Result { self.with_store(shown, id, missing, |a, st| f(a, &st.repo)) } /// Read an account's store: its repository, blobs and takedowns. fn with_store( &self, shown: &mut Shown, id: &str, missing: impl FnOnce() -> XrpcError, f: impl FnOnce(&Account, &Store) -> Result, ) -> Result { let Some(a) = self.account(id) else { return Err(missing()); }; let st = read(&a.store); shown.0.push((st.wal.clone(), st.wal.written())); f(&a, &st) } } /// How far into each account's log an answer has read: it is sent only once /// that much is on disk, so that nothing it shows can be lost. #[derive(Default)] struct Shown(Vec<(Arc, u64)>); impl Shown { /// Return once each is on disk, in order: the first error, though each /// after it is waited for too. async fn durable(self) -> Result<(), XrpcError> { let mut out = Ok(()); for (wal, n) in self.0 { let synced = durable(wal, n).await; out = out.and(synced); } out } /// How far into each account's log an answer has read, after the /// entry in the index of the events `landing`, if any: waited for first. fn after(landing: Option<&events::Landing>, logs: Vec<(Arc, u64)>) -> Shown { Shown( landing .and_then(|l| l.entry()) .into_iter() .chain(logs) .collect(), ) } } /// The media type of a STAR archive (`spec/SPACES.md`), as a space repo's /// and a space meta archive are sent and taken. const STAR_TYPE: &str = "application/vnd.dasl.star"; enum Resp { Json(Json), Car(Vec), /// A repository's CAR file, read as it is sent (`sync::CarStream`). CarStream(sync::CarStream), /// No answer, as a method with no output gives: `200` with an empty body. Nothing, Blob(blobs::Served), /// A service's answer, as the server answers it (`spec/PROXY.md`). Upstream(proxy::Upstream), /// A JSON answer with headers of its own. Headed(Json, Vec<(&'static str, String)>), /// A STAR archive, as a space meta archive (`STAR_TYPE`). Star(Vec), /// A space repo's archive, read as it is sent (`space_repo::Archive`), /// with headers of its own, as its commit's MAC. Archive(space_repo::Archive, Vec<(&'static str, String)>), } /// An error as XRPC answers it: an HTTP status, a name and a message. struct XrpcError { status: u16, name: &'static str, message: String, } fn err(status: u16, name: &'static str, message: impl Into) -> Result { Err(XrpcError { status, name, message: message.into(), }) } fn invalid(message: impl Into) -> Result { err(400, "InvalidRequest", message) } fn internal() -> Result { err(500, "InternalServerError", "Internal Server Error") } fn invalid_token() -> Result { err(400, "InvalidToken", "Invalid token") } /// A refusal of the repository's, as the reference answers it. impl From for XrpcError { fn from(r: Refused) -> XrpcError { let (status, name, message) = match r { Refused::StaleCommit => (400, "InvalidSwap", "Commit was at a different CID"), Refused::Exists(_) | Refused::Missing(_) => { (500, "InternalServerError", "Internal Server Error") } Refused::TooBig => (400, "InvalidRequest", "Commit is too large"), }; XrpcError { status, name, message: message.into(), } } } /// One call of a method, over XRPC or as an MCP tool. struct Call<'a> { nsid: &'a str, method: &'a Method, params: &'a [(String, String)], body: &'a [u8], /// What the `Authorization` header holds. bearer: sessions::Bearer<'a>, /// The request's headers; none for a call not made over HTTP. headers: &'a HeaderMap, /// The client it is from, as the rate limits key on it. client: String, /// What it cost of the rate limits, for its answer's headers. charged: OnceLock, /// The HTTP request, for a call made over XRPC. http: Option<&'a Http>, } /// What a call made over XRPC was sent with, beyond its method, arguments and headers. struct Http { uri: Uri, /// The body, which `Call::body` is too, for a forwarded request to keep. body: Bytes, } /// A query's answer, given the call and its parameters; it notes in `Shown` /// the repositories it read. type QueryFn = fn(&Service, &Call, &mut Shown, &Map) -> Result; /// A repository write, once its input is checked, to apply to the repository its `repo` names. type Apply = Box Result + Send>; /// A repository write: its input checked, the write to apply. type WriteFn = fn(&Map) -> Result; type Answer = Pin> + Send>>; /// Any other procedure, given its input. It runs to its end even if its caller goes away. type ProcedureFn = fn(Shared, Map) -> Answer; /// The token a procedure takes, checked: the session it is of, or the error to answer. type CheckFn = fn(&Service, sessions::Bearer) -> Result; /// A procedure that takes a token, given the token checked and its input. type AuthedFn = fn(Shared, sessions::Token, Map) -> Answer; /// A query that asks outside the server, given its parameters. type LookupFn = fn(Shared, Map) -> Answer; /// A method that forwards its request to another service, given the /// request and, for a query, its parameters checked. type ProxyFn = fn(Shared, proxy::Request, Map) -> Answer; /// A method that takes its call as it came, and checks it in its own order. type CallFn = for<'a> fn( &'a Shared, &'a Call<'a>, &'static pds_lexicon::Method, ) -> Pin> + Send + 'a>>; /// What answers a method. enum Handler { /// A query, which reads and changes nothing. Query(QueryFn), /// A query that waits on something outside the server, such as the PLC directory. Lookup(LookupFn), /// A write to the repository its `repo` names: only that account may make /// it, and its token is checked before its input, its `repo` after. Write(WriteFn), Procedure(ProcedureFn), /// A procedure that takes a token, checked before its input; or a query, /// its token checked before its parameters. Authed(CheckFn, AuthedFn), /// A method that may forward its request (`spec/PROXY.md`); a query's /// parameters are checked first. Proxy(ProxyFn), /// A method that takes its call as it came, and checks it in its own order. Call(CallFn), } /// Each area's methods: what answers a method, by its NSID, if the area has it. const AREAS: &[fn(&str) -> Option] = &[ server::handler, sessions::handler, email::handler, repo::handler, sync::handler, federation::handler, identity::handler, blobs::handler, invites::handler, lifecycle::handler, migration::handler, proxy::handler, preferences::handler, space_repo::handler, space_credentials::handler, space_authority::handler, space_notify::handler, ]; fn handler(nsid: &str) -> Option { AREAS.iter().find_map(|area| area(nsid)) } /// What `pds serve` is given by its environment, as `spec/PDS.md` says. struct Environment { port: u16, dir: PathBuf, address: std::net::IpAddr, config: Config, ops: ops::Settings, } impl Environment { /// Read from the environment: why not, if a variable is missing or /// does not parse. fn read() -> Result { let var = |n: &str| std::env::var(n).map_err(|_| format!("{n} is not set")); let port = var("PDS_PORT")? .parse::() .map_err(|_| "PDS_PORT is not a port".to_string())?; let dir = PathBuf::from(var("PDS_DATA_DIR")?); let address = match std::env::var("PDS_BIND_ADDRESS") .ok() .filter(|a| !a.is_empty()) { None => std::net::IpAddr::from([127, 0, 0, 1]), Some(a) => a .parse() .map_err(|_| format!("PDS_BIND_ADDRESS is {a:?}, not an IPv4 or IPv6 address"))?, }; let config = Config::from_env()?; let ops = ops::Settings::from_env(port)?; Ok(Environment { port, dir, address, config, ops, }) } } /// Say why the process cannot go on, on standard error, and exit 1. fn fail(why: String) -> ! { eprintln!("pds: {why}"); std::process::exit(1); } /// Run the server, configured by the environment as `spec/PDS.md` says, /// until a SIGTERM or SIGINT shuts it down (`spec/OPERATIONS.md`). pub fn run() { // The file descriptors it may have, before the share of them the // configuration takes is read, and before anything is opened: left as // they are where they cannot be raised. pds_store::files::raise(); let Environment { port, dir, address, config, ops: settings, } = Environment::read().unwrap_or_else(|e| fail(e)); // The data directory is this process's alone while it runs, before // anything is written (`spec/COMMANDS.md`). let _held = store::hold(&dir).unwrap_or_else(|e| fail(e)); // Where the data directory keeps its blobs, before anything is written. blobstore::check_dir(&dir, config.blobstore.is_some()).unwrap_or_else(|e| fail(e)); ops::start(settings, port, config.sync); let unclean = ops::unclean(&dir); let svc = Arc::new( Service::open(&dir, config) .unwrap_or_else(|e| fail(format!("cannot recover {}: {e}", dir.display()))), ); ops::recovered(&svc, unclean).unwrap_or_else(|e| fail(e)); let app = router(svc.clone()); tokio::runtime::Runtime::new().unwrap().block_on(async { let stop = ops::signals(); ops::serve_beside(&svc).await.unwrap_or_else(|e| fail(e)); federation::tell_crawlers(&svc); let listener = tokio::net::TcpListener::bind((address, port)) .await .unwrap_or_else(|e| fail(format!("listening on {address} port {port}: {e}"))); // Handle changes in flight when the server stopped are settled while it serves. identity::settle_unfinished(&svc); // Directories of no account go while it serves. directory::sweep_once(&svc); // The firehose's index is read while it serves. events::find_soon(&svc); // What a start after a kill found is made durable while it serves, // the logs of accounts nothing asks for too. ops::sync_found_soon(&svc); blobs::sweep_every(&svc); blobstore::delete_every(&svc); cache::sweep_every(&svc); // Notifications kept when the server stopped are sent again when due. space_notify::start(&svc); // Issuers' keys the last run kept are read back while it serves. space_credentials::issuers::start(&svc); let signal = limits::serve(listener, app, stop).await; ops::shutdown(&svc, signal).await }) } /// `pds migrate-blobs --to `: move the data directory's blobs into the /// bucket or out of it, given the environment `pds serve` is, as /// `spec/BLOBS.md`'s "Moving them" says (`migrate_blobs`); print what it /// moved and exit 0, or say why not and exit 1. pub fn migrate_blobs(to: Place) -> ! { let env = Environment::read().unwrap_or_else(|e| fail(e)); if env.config.blobstore.is_none() { fail("PDS_BLOBSTORE_S3_BUCKET is not set: it names the bucket the blobs are moved into, or out of".into()); } let _held = store::hold(&env.dir).unwrap_or_else(|e| fail(e)); let moved = tokio::runtime::Runtime::new() .unwrap() .block_on(migrate_blobs::run(&env.dir, env.config, to)) .unwrap_or_else(|e| fail(e)); use std::io::Write; let mut out = std::io::stdout(); let _ = writeln!( out, "{{\"moved\":{},\"bytes\":{}}}", moved.blobs, moved.bytes ) .and_then(|()| out.flush()); std::process::exit(0); } /// The server's routes. Every path under `/xrpc/` is a method, served or /// forwarded. Each answers browsers as `cors` says. fn router(svc: Shared) -> Router { Router::new() .route( "/xrpc/com.atproto.sync.subscribeRepos", get(events::subscribe), ) .route("/", any(ops::root)) .route("/xrpc/_health", any(ops::health)) .route("/xrpc/_logs", any(ops::logs)) .route("/xrpc/_stats", any(ops::stats)) .route( "/xrpc/_oauthSessions", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::Sessions)), ) .route( "/xrpc/_oauthRevokeSession", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::RevokeSession)), ) .route( "/xrpc/_oauthDevices", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::Devices)), ) .route( "/xrpc/_oauthRevokeDevice", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::RevokeDevice)), ) .route( "/xrpc/_passkeyList", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::PasskeyList)), ) .route( "/xrpc/_passkeyAdd", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::PasskeyAdd)), ) .route( "/xrpc/_passkeyRemove", any(|s, m, h, p, b| account::xrpc(s, m, h, p, b, Own::PasskeyRemove)), ) .route("/xrpc/com.atproto.repo.uploadBlob", any(blobs::upload)) .route("/xrpc/{*method}", any(handle)) .route("/mcp", any(mcp::endpoint)) .route("/.well-known/atproto-did", get(identity::well_known)) .merge(oauth::routes()) .with_state(svc) .layer(axum::middleware::from_fn(cors::layer)) } /// A request's `Authorization` header, as text; one that is not is empty, which no scheme is. fn authorization(headers: &HeaderMap) -> Option { headers .get(header::AUTHORIZATION) .map(|v| v.to_str().unwrap_or_default().to_string()) } /// A call over XRPC: the method is what follows `/xrpc/` in the path, as sent. async fn handle( AxState(st): AxState, method: Method, uri: Uri, headers: HeaderMap, peer: Option>, body: Body, ) -> Response { let nsid = uri .path() .strip_prefix("/xrpc/") .unwrap_or_default() .to_string(); let params: Vec<(String, String)> = uri .query() .map(|q| { url::form_urlencoded::parse(q.as_bytes()) .into_owned() .collect() }) .unwrap_or_default(); // Read only so far past the method's limit: `input`, and `proxy::forward` // for a request it forwards, refuse a body longer than it. let Some(body) = limits::read_body(body, limits::read_limit(&st, &nsid)).await else { return respond(invalid("The body could not be read"), None); }; let authorization = authorization(&headers); let bearer = sessions::Bearer::of(authorization.as_deref()); let client = limits::client( &st.config.trusted_proxies, &headers, peer.map(|Extension(p)| p.0), ); let http = Http { uri, body }; let mut call = Call { nsid: &nsid, method: &method, params: ¶ms, body: &http.body, bearer, headers: &headers, client, charged: OnceLock::new(), http: Some(&http), }; let grant = match scopes::look(&st, &call, &headers).await { Ok(g) => g, Err(r) => return *r, }; if let Some(g) = &grant { call.bearer = sessions::Bearer::OAuth(g); } let answer = answer(&st, &call).await; scopes::answered(&st, grant.as_ref(), respond(answer, call.charged.get())) } /// An answer as XRPC sends it, with what it cost of the rate limits. fn respond(answer: Result, charged: Option<&limits::Charged>) -> Response { let mut res = match answer { Ok(Resp::Json(j)) => ( StatusCode::OK, [(header::CONTENT_TYPE, "application/json")], j.to_string(), ) .into_response(), Ok(Resp::Car(b)) => ( StatusCode::OK, [(header::CONTENT_TYPE, "application/vnd.ipld.car")], b, ) .into_response(), Ok(Resp::CarStream(c)) => c.response(), Ok(Resp::Nothing) => StatusCode::OK.into_response(), Ok(Resp::Blob(b)) => blobs::response(b), Ok(Resp::Upstream(u)) => { let mut r = Response::new(Body::from(u.body)); *r.status_mut() = StatusCode::from_u16(u.status).unwrap_or(StatusCode::BAD_GATEWAY); for (n, v) in u.headers { r.headers_mut().append(n, v); } r } Ok(Resp::Headed(j, headers)) => with_headers( ( StatusCode::OK, [(header::CONTENT_TYPE, "application/json")], j.to_string(), ) .into_response(), headers, ), Ok(Resp::Star(b)) => { (StatusCode::OK, [(header::CONTENT_TYPE, STAR_TYPE)], b).into_response() } Ok(Resp::Archive(a, headers)) => with_headers(a.response(), headers), Err(e) => ( StatusCode::from_u16(e.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), [(header::CONTENT_TYPE, "application/json")], json!({ "error": e.name, "message": e.message }).to_string(), ) .into_response(), }; if let Some(c) = charged { c.apply(res.headers_mut()); } res } /// An answer with headers of its own added. fn with_headers(mut r: Response, headers: Vec<(&'static str, String)>) -> Response { for (name, value) in headers { if let Ok(v) = header::HeaderValue::from_str(&value) { r.headers_mut() .append(header::HeaderName::from_static(name), v); } } r } /// Whether an error is a token's, which costs a call nothing of the rate limits. fn token_error(e: &XrpcError) -> bool { matches!(e.name, "AuthMissing" | "InvalidToken" | "ExpiredToken") } /// Answer a call. A query reads the repositories it needs side by side with /// other calls; a write has its account's repository alone, only to apply /// it and hand its log frame to the operating system. Either is answered /// only once what it read or wrote is on disk, waited for with no lock held, /// so that calls meanwhile share the fsync. Each pays the rate limits once /// its parameters, token and input are checked (`limits::guard`). Once they /// are, the state of the account the call is of may refuse it /// (`lifecycle::gate`). async fn answer(svc: &Shared, call: &Call<'_>) -> Result { // A method the server does not serve, in any case, is forwarded, and // costs its point first, unless it paid already as it arrived // (`scopes::look`). let nsid = match proxy::served(svc, call.nsid) { Ok(Some(nsid)) => nsid, served => { if call.charged.get().is_none() { limits::charge(svc, call, None, None)?; } served?; return proxy::forward(svc, call).await; } }; if nsid == call.nsid { return serve(svc, call).await; } // Named in another case: answered as the lexicon names it, its rate // limits' headers kept for the call's answer. let named = Call { nsid, method: call.method, params: call.params, body: call.body, bearer: call.bearer, headers: call.headers, client: call.client.clone(), charged: OnceLock::new(), http: call.http, }; let out = serve(svc, &named).await; if let Some(c) = named.charged.into_inner() { let _ = call.charged.set(c); } out } /// Answer a call of a method the server serves, as its lexicon names it. async fn serve(svc: &Shared, call: &Call<'_>) -> Result { let h = handler(call.nsid).expect("a method served has a handler"); let m = method_of(call)?; match h { Handler::Query(f) => { let p = params(call, m)?; scopes::admit(call)?; let mut shown = Shown::default(); // A query changes nothing, so it may pay once it has run: its // token, checked within it, costs nothing if it is refused, nor // a scope checked with it (`getServiceAuth`'s `rpc`), nor its // account's state checked with it (`lifecycle::with_token`). let out = lifecycle::gate_query(svc, call, &p).and_then(|()| f(svc, call, &mut shown, &p)); if !out.as_ref().is_err_and(|e| { token_error(e) || e.name == "ScopeMissingError" || lifecycle::refused_with_token(call.nsid, e) }) { limits::charge(svc, call, None, None)?; } shown.durable().await?; blobs::read_object(svc, out).await } Handler::Lookup(f) => { let p = params(call, m)?; scopes::admit(call)?; limits::guard(svc, call, None, None)?; lifecycle::gate_query(svc, call, &p)?; ops::spawn(f(svc.clone(), p)) .await .unwrap_or_else(|_| internal()) } Handler::Write(f) => write_repo(svc, call, m, f).await, Handler::Procedure(f) => { scopes::admit(call)?; let b = input(call, m)?; limits::guard(svc, call, None, Some(&b))?; ops::spawn(f(svc.clone(), b)) .await .unwrap_or_else(|_| internal()) } // An OAuth token was checked as the call arrived: what its scope // needs is before the token the method checks. Handler::Authed(check, f) => { scopes::admit(call)?; let token = check(svc, call.bearer)?; let early = lifecycle::with_token(call.nsid); if early { lifecycle::gate(svc, call.nsid, &token.did)?; } // A query that takes a token takes its parameters as a procedure its input. let b = if m.kind == Kind::Query { params(call, m)? } else { input(call, m)? }; limits::guard(svc, call, Some(&token.did), Some(&b))?; scopes::of_input(call.bearer, call.nsid, &b)?; if !early { lifecycle::gate(svc, call.nsid, &token.did)?; } ops::spawn(f(svc.clone(), token, b)) .await .unwrap_or_else(|_| internal()) } Handler::Proxy(f) => { let p = match m.kind { Kind::Query => m .parse_params(call.params.iter().map(|(k, v)| (k.as_str(), v.as_str()))) .or_else(|e| invalid(e.0))?, _ => Map::new(), }; // It pays once its token, what its scope needs of the service it // goes to, its account's state if it is checked with the token, // and its input are checked, as any other call does, before it // goes anywhere. let did = match proxy::needs_token(call.nsid) { true => Some(preferences::token(svc, call.nsid, call.bearer)?.did), false => None, }; proxy::scope(svc, call)?; if let Some(did) = did.as_deref().filter(|_| lifecycle::with_token(call.nsid)) { lifecycle::gate(svc, call.nsid, did)?; } let p = match m.kind { Kind::Procedure => filled_input(call, m)?, _ => p, }; limits::guard(svc, call, did.as_deref(), Some(&p))?; ops::spawn(f(svc.clone(), proxy::Request::of(call), p)) .await .unwrap_or_else(|_| internal()) } // A method that checks its call in its own order pays the limits itself (`limits::guard`). Handler::Call(f) => blobs::read_object(svc, f(svc, call, m).await).await, } } /// A query's parameters, checked against its lexicon. fn params(call: &Call, m: &pds_lexicon::Method) -> Result, XrpcError> { m.parse_params(call.params.iter().map(|(k, v)| (k.as_str(), v.as_str()))) .or_else(|e| invalid(e.0)) } /// A write checks, in this order, its token, its input, its `repo`, the /// number of its writes, what it needs of an OAuth scope, then its records' /// `$type`. async fn write_repo( svc: &Shared, call: &Call<'_>, m: &pds_lexicon::Method, f: WriteFn, ) -> Result { let did = svc.auth(call.bearer).did()?; let b = input(call, m)?; let apply = f(&b)?; limits::guard(svc, call, Some(&did), Some(&b))?; let Some(account) = svc.account(required(&b, "repo")?) else { return err(403, "AccountNotFound", "Could not find account"); }; // The repository's account's state decides first, whoever's token it is. lifecycle::gate(svc, call.nsid, &account.did)?; if account.did != did { return err(401, "AuthenticationRequired", "Bad token scope"); } if b.get("writes") .and_then(Json::as_array) .is_some_and(|w| w.len() > limits::MAX_WRITES) { return invalid(format!("Too many writes. Max: {}", limits::MAX_WRITES)); } scopes::records(call.bearer, call.nsid, &b)?; let (out, shown, landing, doomed) = { let mut st = write(&account.store); let st = &mut *st; // Deleted while it waited for the lock. lifecycle::not_deleted(&account)?; st.repo.change = None; st.blobs.time(svc.limits.clock.now(), svc.config.blob_ttl); let mut out = apply(st); let landing = match (&out, st.repo.change.take()) { (Ok(_), Some(mut c)) => match log::log_commit(&svc.events, &account.did, st, &c) { Ok(l) => { // A record deleted is no longer taken down. for path in &c.dels { st.takedowns.remove(path); } Some(l) } Err(e) => { // Nothing of it reached the log: nothing of it stays. ops::error( "a write failed", &[("did", json!(account.did)), ("error", json!(e.to_string()))], ); if let Some(u) = c.undo.take() { st.repo.undo(u); } st.blobs.undo(); out = internal(); None } }, _ => None, }; let shown = Shown::after(landing.as_ref(), vec![(st.wal.clone(), st.wal.written())]); (out, shown, landing, st.blobs.take_doomed()) }; // The write is in the log: the rest must happen even if the caller goes away, // or its event would never reach the firehose. let svc = svc.clone(); let finish = ops::spawn(async move { let synced = shown.durable().await; if let Some(l) = landing { svc.events.land(l); } blobs::forget(&svc, &account, doomed); synced }); finish.await.unwrap_or_else(|_| internal())?; out } /// The method a call names, called with the right HTTP method: a query by /// `POST` or a procedure by `GET` is refused before anything else is looked at. fn method_of(call: &Call) -> Result<&'static pds_lexicon::Method, XrpcError> { method_named(call.nsid, call.method) } /// The method `nsid` names, if it may be called with the HTTP method `method`. fn method_named(nsid: &str, method: &Method) -> Result<&'static pds_lexicon::Method, XrpcError> { let not_here = || err(501, "MethodNotImplemented", "Method not implemented"); let Some(m) = pds_lexicon::method(nsid) else { return not_here(); }; let want = match m.kind { Kind::Query => Method::GET, Kind::Procedure => Method::POST, Kind::Subscription => return not_here(), }; if *method != want && !(want == Method::GET && *method == Method::HEAD) { return invalid(format!("Incorrect HTTP method ({method}) expected {want}")); } Ok(m) } /// A procedure's input, checked against its lexicon, once its length is: a /// body longer than its method takes is `413 PayloadTooLargeError`. A /// procedure that takes none takes no body either. fn input(call: &Call, m: &pds_lexicon::Method) -> Result, XrpcError> { if m.input.is_none() { return if call.body.is_empty() { Ok(Map::new()) } else { invalid("Request body is not allowed") }; } if call.body.len() > limits::body_limit(call.nsid) { return limits::too_large(); } let Ok(input) = serde_json::from_slice::(call.body) else { return invalid("Invalid request body"); }; m.check_input(&input).or_else(|e| invalid(e.0))?; let Json::Object(b) = input else { return invalid("Invalid request body"); }; Ok(b) } /// A procedure's input, as `input` reads it, with the defaults its lexicon /// gives filled in before it is checked. fn filled_input(call: &Call, m: &pds_lexicon::Method) -> Result, XrpcError> { if call.body.len() > limits::body_limit(call.nsid) { return limits::too_large(); } let Ok(mut input) = serde_json::from_slice::(call.body) else { return invalid("Invalid request body"); }; m.fill_input_defaults(&mut input); m.check_input(&input).or_else(|e| invalid(e.0))?; let Json::Object(b) = input else { return invalid("Invalid request body"); }; Ok(b) } /// A string argument, which the lexicon checked is there. fn required<'a>(args: &'a Map, name: &str) -> Result<&'a str, XrpcError> { args.get(name) .and_then(Json::as_str) .map_or_else(|| invalid(format!("{name} must be a string")), Ok) } fn opt<'a>(args: &'a Map, name: &str) -> Option<&'a str> { args.get(name).and_then(Json::as_str) } /// The latest commit, as a write and `getLatestCommit` answer it. fn commit_meta(r: &Repo) -> Json { json!({ "cid": r.head_string(), "rev": r.rev }) } /// What the area modules' tests share: a PLC directory, a server on a /// scratch directory, and calls to it as a request would make them. #[cfg(test)] mod testing { use super::*; use axum::body::Bytes; use std::collections::HashMap; use std::path::Path as FsPath; /// A PLC directory, at the URL answered, which takes a genesis operation /// only if its signature and DID are as `atproto/did-plc.md` says, and /// refuses any for a handle starting `refused`. It serves each DID's /// document at `/`, and its current data at `//data`; a `PUT` /// there, which no real directory takes, names another PDS in it, as an /// operation moving the DID would. pub async fn plc() -> String { use axum::extract::{Path, State}; type Ops = Arc>>; async fn post(State(ops): State, Path(did): Path, body: Bytes) -> StatusCode { let Ok(op) = serde_json::from_slice::(&body) else { return StatusCode::BAD_REQUEST; }; if op["alsoKnownAs"][0] .as_str() .is_some_and(|h| h.starts_with("at://refused")) || crate::plc::verify(&did, &op).is_err() { return StatusCode::BAD_REQUEST; } ops.lock().unwrap().insert(did, op); StatusCode::OK } async fn get_doc(State(ops): State, Path(did): Path) -> Response { let Some(op) = ops.lock().unwrap().get(&did).cloned() else { return StatusCode::NOT_FOUND.into_response(); }; ( [(header::CONTENT_TYPE, "application/json")], did_document(&did, &op).to_string(), ) .into_response() } async fn get_data(State(ops): State, Path(did): Path) -> Response { let Some(op) = ops.lock().unwrap().get(&did).cloned() else { return StatusCode::NOT_FOUND.into_response(); }; let data = json!({ "did": did, "verificationMethods": op["verificationMethods"], "rotationKeys": op["rotationKeys"], "alsoKnownAs": op["alsoKnownAs"], "services": op["services"], }); ( [(header::CONTENT_TYPE, "application/json")], data.to_string(), ) .into_response() } async fn move_pds( State(ops): State, Path(did): Path, body: Bytes, ) -> StatusCode { let mut ops = ops.lock().unwrap(); let Some(op) = ops.get_mut(&did) else { return StatusCode::NOT_FOUND; }; op["services"]["atproto_pds"]["endpoint"] = json!(String::from_utf8_lossy(&body)); StatusCode::OK } let app = Router::new() .route("/{did}", get(get_doc).post(post)) .route("/{did}/data", get(get_data).put(move_pds)) .with_state(Ops::default()); let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await .unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); url } /// The DID document a directory renders from a DID's last operation: /// the signing key as a `Multikey`, and a KEM key in the legacy form /// `spec/SPACES.md` says the harness's directory renders it in. pub fn did_document(did: &str, op: &Json) -> Json { let key = op["verificationMethods"]["atproto"] .as_str() .unwrap() .trim_start_matches("did:key:") .to_string(); let mut context = vec![json!("https://www.w3.org/ns/did/v1")]; let mut methods = vec![ json!({ "id": format!("{did}#atproto"), "type": "Multikey", "controller": did, "publicKeyMultibase": key }), ]; if let Some(kem) = op["verificationMethods"]["atproto_kem"].as_str() { let point = pds_space::PublicKey::from_did_key(kem) .unwrap() .uncompressed(); context.push(json!("https://w3id.org/security/suites/ecdsa-2019/v1")); methods.push(json!({ "id": "#atproto_kem", "type": "EcdsaSecp256r1VerificationKey2019", "controller": did, "publicKeyMultibase": format!("z{}", bs58::encode(point).into_string()), })); } json!({ "@context": context, "id": did, "alsoKnownAs": op["alsoKnownAs"], "verificationMethod": methods, "service": [{ "id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": op["services"]["atproto_pds"]["endpoint"] }], }) } std::thread_local! { /// How the servers a test's thread opens sync (`syncing`). static SYNC: std::cell::Cell = const { std::cell::Cell::new(Syncing::Together) }; } /// Both ways a server syncs, for a test of what a power cut leaves to /// run in each. pub const SYNCS: [Syncing; 2] = [Syncing::Together, Syncing::Each]; /// Have the servers this thread opens from now on sync so: `together` /// until it is said. pub fn syncing(sync: Syncing) { SYNC.with(|s| s.set(sync)); } /// The configuration the tests run with: no rate limits, no floor of /// free space or of `createSession`'s time, and every DID document /// fetched again, so that each test sees what it changes at once. pub fn config(plc_url: &str) -> Config { Config { hostname: "localhost".into(), public_url: "http://127.0.0.1:2583".into(), handle_domains: vec![".test".into(), ".example".into()], plc_url: plc_url.into(), rotation_key: k256::ecdsa::SigningKey::from_slice(&[9u8; 32]).unwrap(), recovery_did_key: Some( "did:key:zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme".into(), ), crawlers: vec![], oauth_client_fetch: None, // A DID the directory does not have: no set resolves, and nothing asks DNS. lexicon_authority: Some("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa".into()), lexicon_dns: None, // The tests' own DNS server and web: a handle resolves only as a test serves it. handle_dns: Some(identity::resolve::stub::addrs().0), handle_fetch: Some(identity::resolve::stub::addrs().1), invite_required: false, admin_password: Some(ADMIN_PASSWORD.into()), trusted_proxies: Default::default(), rate_limits: false, blob_quota: 1 << 30, blob_ttl: 3600, min_free: 0, max_import: 5 << 20, blob_upload_limit: crate::blobs::DEFAULT_UPLOAD_LIMIT, did_cache: 0, did_cache_max: 86_400, login_floor: std::time::Duration::ZERO, sign_in_floor: std::time::Duration::ZERO, mailer: Default::default(), proxy: proxy::Settings::default(), ssrf_protection: false, backfill: std::time::Duration::from_millis(86_400_000), log_floor: crate::store::FLOOR, account_cap: crate::store::ACCOUNT_CAP, cache_bytes: 256 << 20, share: 64, load_all: true, blobstore: None, sync: SYNC.with(std::cell::Cell::get), } } /// The admin's password in `config`. pub const ADMIN_PASSWORD: &str = "admin-pw"; /// The server on `dir`, registering DIDs at `plc_url`. pub fn open(dir: &FsPath, plc_url: &str) -> Shared { Arc::new(try_open(dir, plc_url).unwrap()) } pub fn try_open(dir: &FsPath, plc_url: &str) -> Result { Service::open(dir, config(plc_url)) } /// The server on `dir`, configured as `config` and then `change` says. pub fn open_with(dir: &FsPath, plc_url: &str, change: impl FnOnce(&mut Config)) -> Shared { let mut c = config(plc_url); change(&mut c); Arc::new(Service::open(dir, c).unwrap()) } /// A call with a bearer token; a `null` body is none. pub async fn call_with( s: &Shared, nsid: &str, token: Option<&str>, params: &[(String, String)], body: Json, ) -> Result { let bearer = token.map_or(sessions::Bearer::None, sessions::Bearer::Token); call_raw( s, nsid, bearer, params, &if body.is_null() { String::new() } else { body.to_string() }, ) .await } /// A call as sent: its `Authorization` header and body. A method's empty answer is `null`. pub async fn call_raw( s: &Shared, nsid: &str, bearer: sessions::Bearer<'_>, params: &[(String, String)], body: &str, ) -> Result { let method = if pds_lexicon::method(nsid).is_some_and(|m| m.kind == Kind::Query) { Method::GET } else { Method::POST }; let headers = HeaderMap::new(); let call = Call { nsid, method: &method, params, body: body.as_bytes(), bearer, headers: &headers, client: "127.0.0.1".into(), charged: OnceLock::new(), http: None, }; match answer(s, &call).await { Ok(Resp::Json(j) | Resp::Headed(j, _)) => Ok(j), Ok(Resp::Car(_) | Resp::CarStream(_)) => Err("a CAR file".into()), Ok(Resp::Star(_) | Resp::Archive(..)) => Err("bytes".into()), Ok(Resp::Nothing) => Ok(Json::Null), Ok(Resp::Blob(_)) => Err("a blob".into()), // A service's answer: its JSON, or for an error, its name. Ok(Resp::Upstream(u)) if u.status < 400 => { Ok(serde_json::from_slice(&u.body).unwrap_or(Json::Null)) } Ok(Resp::Upstream(u)) => Err(serde_json::from_slice::(&u.body) .ok() .and_then(|j| j["error"].as_str().map(str::to_string)) .unwrap_or_else(|| u.status.to_string())), Err(e) => Err(e.name.to_string()), } } pub async fn call( s: &Shared, nsid: &str, token: Option<&str>, body: Json, ) -> Result { call_with(s, nsid, token, &[], body).await } pub fn scratch(name: &str) -> std::path::PathBuf { let d = std::env::temp_dir().join(format!("pds-test-{}-{name}", std::process::id())); let _ = std::fs::remove_dir_all(&d); d } /// How many syncs to the disk this thread has made (`pds_store::on_sync`) /// since this was first asked, on any thread: a test counts those of /// what it does on its own thread, as loading a store, whatever the /// tests beside it sync. pub fn syncs() -> u64 { use std::cell::Cell; thread_local!(static MADE: Cell = const { Cell::new(0) }); pds_store::on_sync(|_| MADE.with(|n| n.set(n.get() + 1))); MADE.with(Cell::get) } /// Shut the server down in order, as a SIGTERM does, but for exiting: /// its mark is left, with its word if the run may leave it. pub fn shut_down(s: Shared) { s.events.close(); ops::settle(&s).unwrap(); ops::mark(&s).unwrap(); } /// What the journal of the data directory `dir` holds put back where it /// is kept, and its segments let go, as a start does before anything /// else: the logs then hold every frame the server wrote, and a test /// changes or cuts them as a kill leaves them, a write a kill cut short /// in its log never having reached the journal. pub fn redone(dir: &FsPath) { pds_store::journal::redo(dir, pds_store::journal::Redo::Eager).unwrap(); } /// Create an account `.test`, with password `-pw`; answers its DID and access token. pub async fn create(s: &Shared, name: &str) -> (String, String) { let body = json!({ "handle": format!("{name}.test"), "email": format!("{name}@mail.com"), "password": format!("{name}-pw") }); let got = call(s, "com.atproto.server.createAccount", None, body) .await .unwrap(); ( got["did"].as_str().unwrap().to_string(), got["accessJwt"].as_str().unwrap().to_string(), ) } /// A new session of `.test`'s. pub async fn session(s: &Shared, name: &str) -> String { let body = json!({ "identifier": format!("{name}.test"), "password": format!("{name}-pw") }); let sess = call(s, "com.atproto.server.createSession", None, body) .await .unwrap(); sess["accessJwt"].as_str().unwrap().to_string() } /// The repository of the account `did`. pub fn repo_of(s: &Service, did: &str, f: impl FnOnce(&Repo) -> T) -> T { store_of(s, did, |st| f(&st.repo)) } /// The store of the account `did`. pub fn store_of(s: &Service, did: &str, f: impl FnOnce(&Store) -> T) -> T { let a = s.account(did).unwrap(); let st = read(&a.store); f(&st) } /// Every record of the account `did`, in order of path. pub fn records_of(s: &Service, did: &str) -> Vec<(String, pds_repo::Record)> { store_of(s, did, |st| { let mut paths = Vec::new(); st.each_path(|p, _, _| paths.push(p.to_string())); paths .into_iter() .map(|p| (p.clone(), st.record(&p).unwrap())) .collect() }) } /// A record as the tests write them. pub fn record(n: i64) -> Json { json!({ "$type": "a.b.c", "n": n, "b": { "$bytes": "AQID" } }) } }