~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
1881 lines · 67.8 KB Raw History
//! What keeps the server up when anyone may call it, as `spec/LIMITS.md`
//! says: how big a request may be, the rate limits and the client address
//! they key on, the free space a write needs, the threads password hashes
//! run on, and how long a connection may hold the server waiting.

use crate::{Call, Service, XrpcError, err};
use axum::body::{Body, Bytes, HttpBody};
use axum::http::{HeaderMap, HeaderValue, Request};
use http_body::{Frame, SizeHint};
use serde_json::{Map, Value as Json};
use std::collections::HashMap;
use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::task::{Context, Poll};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

/// The longest body `createRecord`, `putRecord` and `applyWrites` take.
pub(crate) const MAX_RECORD_BODY: usize = 1_000_000;
/// The longest body any other method with a JSON input takes.
pub(crate) const MAX_BODY: usize = 153_600;
/// The longest space meta archive `importSpaceMeta` reads.
pub(crate) const MAX_ARCHIVE_BODY: usize = 16 * 1024 * 1024;
/// The longest body `POST /mcp` takes.
pub(crate) const MAX_MCP_BODY: usize = 1_048_576;
/// The most writes one `applyWrites` may make.
pub(crate) const MAX_WRITES: usize = 200;
/// The longest message a firehose client may send without being closed.
pub(crate) const MAX_CLIENT_MESSAGE: usize = 4096;
/// The subscriptions one client address may hold, and the server in all.
pub(crate) const MAX_SUBSCRIPTIONS_PER_CLIENT: usize = 16;
pub(crate) const MAX_SUBSCRIPTIONS: usize = 1024;

/// How long a connection may take to send a whole request head.
const HEADER_TIMEOUT: Duration = Duration::from_secs(10);
/// How long a request's body may send nothing.
const BODY_IDLE: Duration = Duration::from_secs(30);
/// How much of a body past its limit is read and thrown away before the
/// refusal is answered, so that a client still sending reads it; and how
/// much of a body left unread is read and thrown away after its answer, so
/// that its connection is kept (`serve`).
const DRAIN: usize = 4 * 1024 * 1024;

/// The methods that keep something: each is refused below the free space
/// `PDS_MIN_FREE_BYTES` asks for.
const WRITES: &[&str] = &[
    "com.atproto.server.createAccount",
    "com.atproto.server.createSession",
    "com.atproto.server.refreshSession",
    "com.atproto.server.deleteSession",
    "com.atproto.server.createAppPassword",
    "com.atproto.server.revokeAppPassword",
    "com.atproto.identity.updateHandle",
    "com.atproto.repo.createRecord",
    "com.atproto.repo.putRecord",
    "com.atproto.repo.deleteRecord",
    "com.atproto.repo.applyWrites",
    "com.atproto.repo.uploadBlob",
    "com.atproto.repo.importRepo",
    "com.atproto.server.createInviteCode",
    "com.atproto.server.createInviteCodes",
    "com.atproto.server.deactivateAccount",
    "com.atproto.server.activateAccount",
    "com.atproto.server.requestAccountDelete",
    "com.atproto.server.deleteAccount",
    "com.atproto.admin.updateSubjectStatus",
    "app.bsky.actor.putPreferences",
    "com.atproto.space.createRecord",
    "com.atproto.space.putRecord",
    "com.atproto.space.deleteRecord",
    "com.atproto.space.applyWrites",
    "com.atproto.space.clearRecords",
    "com.atproto.space.createSpace",
    "com.atproto.space.updateSpace",
    "com.atproto.space.deleteSpace",
    "com.atproto.space.removeWriter",
    "com.atproto.space.importSpaceMeta",
    "com.atproto.space.importRepo",
    "com.atproto.space.notifyRepoUpdate",
    "com.atproto.memberspace.putMembers",
    "com.atproto.memberspace.removeMembers",
    "com.atproto.space.authorizeSpaceSync",
    "com.atproto.space.notifyCredentialsRevoked",
];

/// The longest body a method takes, named as the path after `/xrpc/` names
/// it, in any case: a JSON body, or any body forwarded (`spec/PROXY.md`).
pub(crate) fn body_limit(nsid: &str) -> usize {
    let nsid = nsid.strip_suffix('/').unwrap_or(nsid);
    let is = |names: &[&str]| names.iter().any(|n| n.eq_ignore_ascii_case(nsid));
    if is(&[
        "com.atproto.repo.createRecord",
        "com.atproto.repo.putRecord",
        "com.atproto.repo.applyWrites",
        "com.atproto.space.createRecord",
        "com.atproto.space.putRecord",
        "com.atproto.space.applyWrites",
    ]) {
        MAX_RECORD_BODY
    } else if is(&["com.atproto.space.importSpaceMeta"]) {
        MAX_ARCHIVE_BODY
    } else {
        MAX_BODY
    }
}

/// How much of a request's body is read before it is refused as too long:
/// `importRepo`'s CAR file up to `PDS_MAX_REPO_IMPORT_SIZE`, which it
/// checks itself (`migration`), a space repo's archive up to
/// `PDS_BLOB_UPLOAD_LIMIT` (`archive_limit`), and any other method's as
/// `body_limit` says.
pub(crate) fn read_limit(s: &Service, nsid: &str) -> usize {
    let nsid = nsid.strip_suffix('/').unwrap_or(nsid);
    if nsid.eq_ignore_ascii_case(crate::migration::IMPORT_REPO) {
        usize::try_from(s.config.max_import).map_or(usize::MAX - 1, |n| n.min(usize::MAX - 1))
    } else if nsid.eq_ignore_ascii_case(SPACE_IMPORT_REPO) {
        archive_limit(s)
    } else {
        body_limit(nsid)
    }
}

/// The space `importRepo`, whose archive is bounded as an upload is.
const SPACE_IMPORT_REPO: &str = "com.atproto.space.importRepo";

/// The longest space repo archive `importRepo` takes: `PDS_BLOB_UPLOAD_LIMIT`
/// bytes, as an upload (`spec/LIMITS.md`, "Request size").
pub(crate) fn archive_limit(s: &Service) -> usize {
    usize::try_from(s.config.blob_upload_limit).map_or(usize::MAX - 1, |n| n.min(usize::MAX - 1))
}

/// A body too big for its method.
pub(crate) fn too_large<T>() -> Result<T, XrpcError> {
    err(413, "PayloadTooLargeError", "request entity too large")
}

/// What the server keeps to limit what it is asked: the rate limits' windows,
/// the free space, the clock they read, and the threads hashes run on.
pub(crate) struct Limits {
    pub rates: Rates,
    pub disk: Disk,
    pub clock: Clock,
    pub hashes: Hashes,
}

impl Limits {
    pub fn new(dir: &std::path::Path, c: &crate::Config) -> Limits {
        Limits {
            rates: Rates::new(c.rate_limits),
            disk: Disk::new(dir.to_path_buf(), c.min_free),
            clock: Clock::default(),
            hashes: Hashes::start(),
        }
    }
}

// ---- Configuration ----------------------------------------------------------

/// A variable that is `true` or `false`, or unset for `default`.
pub(crate) fn env_bool(name: &str, default: bool) -> Result<bool, String> {
    match std::env::var(name) {
        Err(_) => Ok(default),
        Ok(v) if v == "true" => Ok(true),
        Ok(v) if v == "false" => Ok(false),
        Ok(v) => Err(format!("{name} is {v:?}, not true or false")),
    }
}

/// A variable that is a whole number of 0 or more, or unset for `default`.
pub(crate) fn env_u64(name: &str, default: u64) -> Result<u64, String> {
    match std::env::var(name) {
        Err(_) => Ok(default),
        Ok(v) => v
            .trim()
            .parse()
            .map_err(|_| format!("{name} is {v:?}, not a whole number of 0 or more")),
    }
}

// ---- The clock --------------------------------------------------------------

/// The time, which tests may move on, or stop.
#[derive(Default)]
pub(crate) struct Clock {
    #[cfg(test)]
    pub skew: AtomicU64,
    /// The milliseconds the clock stands at; 0 while it runs.
    #[cfg(test)]
    frozen: AtomicU64,
}

impl Clock {
    /// Milliseconds since the epoch.
    pub fn now_ms(&self) -> u64 {
        let now = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap_or_default()
            .as_millis() as u64;
        #[cfg(test)]
        let now = match self.frozen.load(Ordering::SeqCst) {
            0 => now,
            frozen => frozen,
        } + self.skew.load(Ordering::SeqCst) * 1000;
        now
    }

    /// Seconds since the epoch.
    pub fn now(&self) -> u64 {
        self.now_ms() / 1000
    }

    /// Stop the clock where it is, but for what `skew` adds.
    #[cfg(test)]
    pub fn freeze(&self) {
        let now = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap_or_default()
            .as_millis() as u64;
        self.frozen.store(now, Ordering::SeqCst);
    }
}

// ---- The client address -----------------------------------------------------

/// The address a connection came from, as the server's own accept loop sees it.
#[derive(Clone, Copy, Debug)]
pub(crate) struct Peer(pub SocketAddr);

/// The proxies before the server, whose `X-Forwarded-For` it reads
/// (`PDS_TRUSTED_PROXIES`): addresses and CIDR ranges.
#[derive(Clone, Debug, Default)]
pub(crate) struct Trusted(Vec<(IpAddr, u8)>);

impl Trusted {
    /// `PDS_TRUSTED_PROXIES`: unset or empty, none.
    pub fn from_env() -> Result<Trusted, String> {
        Trusted::parse(&std::env::var("PDS_TRUSTED_PROXIES").unwrap_or_default())
    }

    /// Addresses and CIDR ranges, IPv4 or IPv6, separated by commas.
    pub fn parse(list: &str) -> Result<Trusted, String> {
        let bad =
            |e: &str| format!("PDS_TRUSTED_PROXIES has {e:?}, not an address or a CIDR range");
        let mut out = Vec::new();
        for e in list.split(',').map(str::trim).filter(|e| !e.is_empty()) {
            let (ip, bits) = match e.split_once('/') {
                Some((ip, bits)) => (ip, Some(bits.parse::<u8>().map_err(|_| bad(e))?)),
                None => (e, None),
            };
            let ip = ip.parse::<IpAddr>().map_err(|_| bad(e))?.to_canonical();
            let max = if ip.is_ipv4() { 32 } else { 128 };
            let bits = bits.unwrap_or(max);
            if bits > max {
                return Err(bad(e));
            }
            out.push((ip, bits));
        }
        Ok(Trusted(out))
    }

    /// Whether `ip` is one of the proxies.
    pub fn contains(&self, ip: IpAddr) -> bool {
        let ip = ip.to_canonical();
        let prefix =
            |a: u128, b: u128, bits: u8, width: u8| bits == 0 || (a ^ b) >> (width - bits) == 0;
        self.0.iter().any(|&(net, bits)| match (net, ip) {
            (IpAddr::V4(n), IpAddr::V4(a)) => {
                prefix(u32::from(n).into(), u32::from(a).into(), bits, 32)
            }
            (IpAddr::V6(n), IpAddr::V6(a)) => prefix(u128::from(n), u128::from(a), bits, 128),
            _ => false,
        })
    }
}

/// The client a request is from: the connection's address, unless that is
/// one of the trusted proxies; then, from `X-Forwarded-For`, the last
/// address that is not one of them, reading from the end, or the first if
/// all are; and the connection's with no header, or one whose last entry is
/// not an address.
pub(crate) fn client(trusted: &Trusted, headers: &HeaderMap, peer: Option<SocketAddr>) -> String {
    let Some(peer) = peer.map(|p| p.ip().to_canonical()) else {
        return String::new();
    };
    if !trusted.contains(peer) {
        return peer.to_string();
    }
    let forwarded: Vec<&str> = headers
        .get_all("x-forwarded-for")
        .iter()
        .filter_map(|v| v.to_str().ok())
        .flat_map(|v| v.split(','))
        .map(str::trim)
        .filter(|a| !a.is_empty())
        .collect();
    let Some(last) = forwarded.last() else {
        return peer.to_string();
    };
    if last.parse::<IpAddr>().is_err() {
        return peer.to_string();
    }
    for a in forwarded.iter().rev() {
        match a.parse::<IpAddr>() {
            Ok(ip) if trusted.contains(ip) => continue,
            Ok(ip) => return ip.to_canonical().to_string(),
            Err(_) => return (*a).to_string(),
        }
    }
    forwarded[0].parse::<IpAddr>().map_or_else(
        |_| forwarded[0].to_string(),
        |ip| ip.to_canonical().to_string(),
    )
}

// ---- Rate limits ------------------------------------------------------------

/// A rate limit: so many points in a window of so many seconds.
struct Limit {
    points: u64,
    window: u64,
}

// The limits, in the order `spec/LIMITS.md`'s table has them, then
// `spec/LIFECYCLE.md`'s, then `spec/EMAIL.md`'s, and `global` last.
const SIGNUP: usize = 0;
const LOGIN_DAY: usize = 1;
const LOGIN: usize = 2;
const HANDLE: usize = 3;
const HANDLE_DAY: usize = 4;
const UPLOADS: usize = 5;
const WRITES_HOUR: usize = 6;
const WRITES_DAY: usize = 7;
const SYNC: usize = 8;
const DELETION_DAY: usize = 9;
const DELETION_HOUR: usize = 10;
const DELETE: usize = 11;
const CONFIRM_DAY: usize = 12;
const CONFIRM_HOUR: usize = 13;
const UPDATE_DAY: usize = 14;
const UPDATE_HOUR: usize = 15;
const RESET_REQUEST_DAY: usize = 16;
const RESET_REQUEST_HOUR: usize = 17;
const RESET: usize = 18;
const GLOBAL: usize = 19;

const DAY: u64 = 86_400;
const LIMITS: [Limit; 20] = [
    Limit {
        points: 100,
        window: 300,
    },
    Limit {
        points: 300,
        window: DAY,
    },
    Limit {
        points: 30,
        window: 300,
    },
    Limit {
        points: 10,
        window: 300,
    },
    Limit {
        points: 50,
        window: DAY,
    },
    Limit {
        points: 1000,
        window: DAY,
    },
    Limit {
        points: 5000,
        window: 3600,
    },
    Limit {
        points: 35000,
        window: DAY,
    },
    Limit {
        points: 6000,
        window: 300,
    },
    Limit {
        points: 15,
        window: DAY,
    },
    Limit {
        points: 5,
        window: 3600,
    },
    Limit {
        points: 50,
        window: 300,
    },
    Limit {
        points: 15,
        window: DAY,
    },
    Limit {
        points: 5,
        window: 3600,
    },
    Limit {
        points: 15,
        window: DAY,
    },
    Limit {
        points: 5,
        window: 3600,
    },
    Limit {
        points: 50,
        window: DAY,
    },
    Limit {
        points: 15,
        window: 3600,
    },
    Limit {
        points: 50,
        window: 300,
    },
    Limit {
        points: 3000,
        window: 300,
    },
];

/// A key's window of a limit: when it started, and the points spent in it.
struct Window {
    start_ms: u64,
    used: u64,
}

/// The rate limits' windows, kept in memory.
pub(crate) struct Rates {
    enabled: bool,
    windows: Mutex<HashMap<(usize, String), Window>>,
    /// Calls charged, to clear out windows that ended now and then.
    calls: AtomicU64,
}

/// What a call's answer says of the limits: the one with the fewest points
/// left, or the one it went past.
#[derive(Clone, Debug)]
pub(crate) struct Charged {
    points: u64,
    window: u64,
    remaining: u64,
    /// When its window ends, in seconds since the epoch.
    reset: u64,
    /// For a call refused: seconds until its window ends, rounded up.
    retry_after: Option<u64>,
}

impl Charged {
    /// Its headers, on an answer.
    pub(crate) fn apply(&self, h: &mut HeaderMap) {
        // Names the answer exposes already, as a space repo's MAC, stay first.
        let mut expose = match h
            .get("access-control-expose-headers")
            .and_then(|v| v.to_str().ok())
        {
            Some(own) if !own.is_empty() => format!("{own}, "),
            _ => String::new(),
        };
        let mut set = |name: &'static str, v: String| {
            if let Ok(v) = HeaderValue::from_str(&v) {
                h.insert(name, v);
            }
        };
        set("ratelimit-limit", self.points.to_string());
        set("ratelimit-remaining", self.remaining.to_string());
        set("ratelimit-reset", self.reset.to_string());
        set(
            "ratelimit-policy",
            format!("{};w={}", self.points, self.window),
        );
        expose.push_str("RateLimit-Limit, RateLimit-Reset, RateLimit-Remaining, RateLimit-Policy");
        if let Some(r) = self.retry_after {
            set("retry-after", r.to_string());
            expose.push_str(", Retry-After");
        }
        set("access-control-expose-headers", expose);
    }
}

/// What the writes of a write cost each writes limit.
fn write_points(nsid: &str, input: Option<&Map<String, Json>>) -> u64 {
    match nsid {
        "com.atproto.repo.createRecord" => 3,
        "com.atproto.repo.putRecord" => 2,
        "com.atproto.repo.deleteRecord" => 1,
        _ => input
            .and_then(|b| b.get("writes"))
            .and_then(Json::as_array)
            .map_or(0, |writes| {
                writes
                    .iter()
                    .map(|w| {
                        match w
                            .get("$type")
                            .and_then(Json::as_str)
                            .and_then(|t| t.rsplit_once('#'))
                            .map(|(_, kind)| kind)
                        {
                            Some("create") => 3,
                            Some("update") => 2,
                            Some("delete") => 1,
                            _ => 0,
                        }
                    })
                    .sum()
            }),
    }
}

/// What a login costs, `createSession` or OAuth's sign-in: a point of each
/// login limit, keyed by the identifier as sent and the client address.
fn login_costs(identifier: &str, client: &str) -> Vec<(usize, String, u64)> {
    let key = format!("{identifier}-{client}");
    vec![(LOGIN_DAY, key.clone(), 1), (LOGIN, key, 1)]
}

/// What a call of `nsid` costs: points of each limit, each under its key.
fn costs(
    nsid: &str,
    client: &str,
    did: Option<&str>,
    input: Option<&Map<String, Json>>,
) -> Vec<(usize, String, u64)> {
    let did = did.unwrap_or_default().to_string();
    let mut out = match nsid {
        "com.atproto.sync.getRepo" => return vec![(SYNC, client.to_string(), 1)],
        "com.atproto.server.createAccount" => vec![(SIGNUP, client.to_string(), 1)],
        "com.atproto.server.createSession" => login_costs(
            input
                .and_then(|b| b.get("identifier"))
                .and_then(Json::as_str)
                .unwrap_or_default(),
            client,
        ),
        "com.atproto.identity.updateHandle" => vec![(HANDLE, did.clone(), 1), (HANDLE_DAY, did, 1)],
        "com.atproto.repo.uploadBlob" => vec![(UPLOADS, did, 1)],
        "com.atproto.server.requestAccountDelete" => {
            vec![(DELETION_HOUR, did.clone(), 1), (DELETION_DAY, did, 1)]
        }
        "com.atproto.server.deleteAccount" => vec![(DELETE, client.to_string(), 1)],
        "com.atproto.server.requestEmailConfirmation" => {
            vec![(CONFIRM_HOUR, did.clone(), 1), (CONFIRM_DAY, did, 1)]
        }
        "com.atproto.server.requestEmailUpdate" => {
            vec![(UPDATE_HOUR, did.clone(), 1), (UPDATE_DAY, did, 1)]
        }
        "com.atproto.server.requestPasswordReset" => vec![
            (RESET_REQUEST_HOUR, client.to_string(), 1),
            (RESET_REQUEST_DAY, client.to_string(), 1),
        ],
        "com.atproto.server.resetPassword" => vec![(RESET, client.to_string(), 1)],
        "com.atproto.repo.createRecord"
        | "com.atproto.repo.putRecord"
        | "com.atproto.repo.deleteRecord"
        | "com.atproto.repo.applyWrites" => match write_points(nsid, input) {
            0 => vec![],
            n => vec![(WRITES_HOUR, did.clone(), n), (WRITES_DAY, did, n)],
        },
        _ => vec![],
    };
    out.push((GLOBAL, client.to_string(), 1));
    out
}

impl Rates {
    pub fn new(enabled: bool) -> Rates {
        Rates {
            enabled,
            windows: Mutex::default(),
            calls: AtomicU64::new(0),
        }
    }

    /// Spend each cost at `now_ms`: what the answer says of the limits, and
    /// whether a limit is past its points.
    fn spend(&self, now_ms: u64, costs: &[(usize, String, u64)]) -> (Charged, bool) {
        let mut windows = self.windows.lock().unwrap_or_else(PoisonError::into_inner);
        if self.calls.fetch_add(1, Ordering::Relaxed) % 4096 == 4095 {
            windows.retain(|(l, _), w| w.start_ms + LIMITS[*l].window * 1000 > now_ms);
        }
        // Each limit spent: (limit, used, window end), in the table's order.
        let mut spent: Vec<(usize, u64, u64)> = costs
            .iter()
            .map(|(l, key, points)| {
                let limit = &LIMITS[*l];
                let w = windows.entry((*l, key.clone())).or_insert(Window {
                    start_ms: now_ms,
                    used: 0,
                });
                if w.start_ms + limit.window * 1000 <= now_ms {
                    *w = Window {
                        start_ms: now_ms,
                        used: 0,
                    };
                }
                w.used += points;
                (*l, w.used, w.start_ms + limit.window * 1000)
            })
            .collect();
        spent.sort_by_key(|(l, _, _)| *l);
        let past = spent.iter().find(|(l, used, _)| *used > LIMITS[*l].points);
        let (l, used, end_ms) = *past.unwrap_or_else(|| {
            spent
                .iter()
                .min_by_key(|(l, used, _)| (LIMITS[*l].points.saturating_sub(*used), *l))
                .expect("every call costs a limit")
        });
        let limit = &LIMITS[l];
        let charged = Charged {
            points: limit.points,
            window: limit.window,
            remaining: limit.points.saturating_sub(used),
            reset: end_ms / 1000,
            retry_after: past
                .is_some()
                .then(|| end_ms.saturating_sub(now_ms).div_ceil(1000)),
        };
        (charged, past.is_some())
    }
}

/// Charge a call its points, once its parameters, token and input are
/// checked: `429 RateLimitExceeded` past a limit. Its answer's headers say
/// what it cost. A call pays once: one that paid already, as one whose
/// token pays before its issuer's document is fetched does, is charged
/// nothing more.
pub(crate) fn charge(
    s: &Service,
    call: &Call,
    did: Option<&str>,
    input: Option<&Map<String, Json>>,
) -> Result<(), XrpcError> {
    if !s.limits.rates.enabled || call.charged.get().is_some() {
        return Ok(());
    }
    let costs = costs(call.nsid, &call.client, did, input);
    let (charged, past) = s.limits.rates.spend(s.limits.clock.now_ms(), &costs);
    let _ = call.charged.set(charged);
    if past {
        return err(429, "RateLimitExceeded", "Rate Limit Exceeded");
    }
    Ok(())
}

/// Charge OAuth's sign-in its points (`spec/OAUTH.md`, "The API"): those of
/// the login limits alone, in `createSession`'s buckets. Past one, the
/// seconds until that window ends, rounded up.
pub(crate) fn charge_sign_in(s: &Service, identifier: &str, client: &str) -> Result<(), u64> {
    if !s.limits.rates.enabled {
        return Ok(());
    }
    match s
        .limits
        .rates
        .spend(s.limits.clock.now_ms(), &login_costs(identifier, client))
    {
        (
            Charged {
                retry_after: Some(secs),
                ..
            },
            true,
        ) => Err(secs),
        _ => Ok(()),
    }
}

/// What every call that keeps something passes once its token and input are
/// checked, before anything else: its rate limits, then the free space.
pub(crate) fn guard(
    s: &Service,
    call: &Call,
    did: Option<&str>,
    input: Option<&Map<String, Json>>,
) -> Result<(), XrpcError> {
    charge(s, call, did, input)?;
    if WRITES.contains(&call.nsid) {
        s.limits.disk.check()?;
    }
    Ok(())
}

// ---- Free space -------------------------------------------------------------

/// The free space of the filesystem the data is on, as at most a second ago.
pub(crate) struct Disk {
    dir: PathBuf,
    min: u64,
    seen: Mutex<Option<(Instant, u64)>>,
    /// Free space a test says there is.
    #[cfg(test)]
    pub free: Mutex<Option<u64>>,
}

impl Disk {
    pub fn new(dir: PathBuf, min: u64) -> Disk {
        Disk {
            dir,
            min,
            seen: Mutex::default(),
            #[cfg(test)]
            free: Mutex::default(),
        }
    }

    fn statvfs(&self) -> Option<u64> {
        #[cfg(test)]
        if let Some(f) = *self.free.lock().unwrap_or_else(PoisonError::into_inner) {
            return Some(f);
        }
        let st = rustix::fs::statvfs(&self.dir).ok()?;
        Some(st.f_bavail.saturating_mul(st.f_frsize))
    }

    /// The free space, in bytes; unknown, as much as anyone could want.
    fn free(&self) -> u64 {
        let mut seen = self.seen.lock().unwrap_or_else(PoisonError::into_inner);
        match *seen {
            Some((at, free)) if at.elapsed() < Duration::from_secs(1) => free,
            _ => {
                let free = self.statvfs().unwrap_or(u64::MAX);
                *seen = Some((Instant::now(), free));
                free
            }
        }
    }

    /// A write's check: `507 InsufficientStorage` below `PDS_MIN_FREE_BYTES`.
    pub fn check(&self) -> Result<(), XrpcError> {
        if self.min > 0 && self.free() < self.min {
            return err(
                507,
                "InsufficientStorage",
                "Not enough free space to store this",
            );
        }
        Ok(())
    }

    /// Forget the free space last seen, as a test that changes it must.
    #[cfg(test)]
    pub fn set_free(&self, free: Option<u64>) {
        *self.free.lock().unwrap() = free;
        *self.seen.lock().unwrap() = None;
    }
}

// ---- Password hashes --------------------------------------------------------

type Job = Box<dyn FnOnce() + Send>;

/// The threads password hashes run on, as many as the machine has cores, apart
/// from every other blocking task: a hash waits for a thread of these, and
/// nothing else waits behind a hash.
pub(crate) struct Hashes {
    jobs: Mutex<Option<std::sync::mpsc::Sender<Job>>>,
}

impl Hashes {
    /// Start the threads; they end once the pool is dropped.
    pub fn start() -> Hashes {
        let want = std::thread::available_parallelism().map_or(1, |n| n.get());
        let (tx, rx) = std::sync::mpsc::channel::<Job>();
        let rx = Arc::new(Mutex::new(rx));
        let mut threads = 0;
        for i in 0..want {
            let rx = rx.clone();
            let spawned = std::thread::Builder::new()
                .name(format!("pds-hash-{i}"))
                .spawn(move || {
                    loop {
                        let job = rx.lock().unwrap_or_else(PoisonError::into_inner).recv();
                        match job {
                            Ok(job) => job(),
                            Err(_) => return,
                        }
                    }
                });
            if let Err(e) = spawned {
                crate::ops::warn(
                    "starting a password hash thread failed",
                    &[
                        ("started", serde_json::json!(i)),
                        ("wanted", serde_json::json!(want)),
                        ("error", serde_json::json!(e.to_string())),
                    ],
                );
                break;
            }
            threads += 1;
        }
        Hashes {
            jobs: Mutex::new((threads > 0).then_some(tx)),
        }
    }

    /// Run a hash on one of the threads, waiting for one to be free.
    pub async fn run<T: Send + 'static>(
        &self,
        f: impl FnOnce() -> T + Send + 'static,
    ) -> Option<T> {
        let (tx, rx) = tokio::sync::oneshot::channel();
        let job: Job = Box::new(move || {
            let _ = tx.send(f());
        });
        let sent = match &*self.jobs.lock().unwrap_or_else(PoisonError::into_inner) {
            Some(jobs) => jobs.send(job).map_err(|e| e.0),
            None => Err(job),
        };
        if let Err(job) = sent {
            // No thread could be started: the hash runs on the blocking pool after all.
            tokio::task::spawn_blocking(job).await.ok()?;
        }
        rx.await.ok()
    }
}

// ---- Connections ------------------------------------------------------------

/// A request whose body sent nothing for `BODY_IDLE`: its connection is
/// closed with no answer.
#[derive(Debug)]
struct BodyTimedOut;

impl std::fmt::Display for BodyTimedOut {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str("the request's body sent nothing for too long")
    }
}

impl std::error::Error for BodyTimedOut {}

/// A request's body, which fails once it has sent nothing for `BODY_IDLE`
/// while it is read, and says so; says, in `ended`, once it was read to its
/// end; and, dropped before that, leaves the rest of it in `left`, for
/// `serve` to read and throw away after the answer.
struct Idle {
    inner: Option<hyper::body::Incoming>,
    sleep: Option<Pin<Box<tokio::time::Sleep>>>,
    timed_out: Arc<AtomicBool>,
    ended: Arc<AtomicBool>,
    left: Arc<Mutex<Option<hyper::body::Incoming>>>,
}

impl Drop for Idle {
    fn drop(&mut self) {
        if !self.ended.load(Ordering::SeqCst)
            && let Some(rest) = self.inner.take()
        {
            *self.left.lock().unwrap_or_else(PoisonError::into_inner) = Some(rest);
        }
    }
}

impl HttpBody for Idle {
    type Data = Bytes;
    type Error = axum::BoxError;

    fn poll_frame(
        self: Pin<&mut Self>,
        cx: &mut Context<'_>,
    ) -> Poll<Option<Result<Frame<Bytes>, axum::BoxError>>> {
        let this = self.get_mut();
        let Some(inner) = this.inner.as_mut() else {
            return Poll::Ready(None);
        };
        match Pin::new(&mut *inner).poll_frame(cx) {
            Poll::Ready(frame) => {
                this.sleep = None;
                if frame.is_none() || inner.is_end_stream() {
                    this.ended.store(true, Ordering::SeqCst);
                }
                Poll::Ready(frame.map(|f| f.map_err(Into::into)))
            }
            Poll::Pending => {
                let sleep = this
                    .sleep
                    .get_or_insert_with(|| Box::pin(tokio::time::sleep(BODY_IDLE)));
                if sleep.as_mut().poll(cx).is_ready() {
                    this.timed_out.store(true, Ordering::SeqCst);
                    return Poll::Ready(Some(Err(Box::new(BodyTimedOut))));
                }
                Poll::Pending
            }
        }
    }

    fn is_end_stream(&self) -> bool {
        self.inner.as_ref().is_none_or(HttpBody::is_end_stream)
    }

    fn size_hint(&self) -> SizeHint {
        self.inner
            .as_ref()
            .map_or_else(|| SizeHint::with_exact(0), HttpBody::size_hint)
    }
}

/// Read the rest of a body its request was answered without, and throw it
/// away, so that its connection takes the next request; given up, and the
/// connection closed, once it sends nothing for `BODY_IDLE`.
async fn drain(mut rest: hyper::body::Incoming) {
    loop {
        let frame = std::future::poll_fn(|cx| Pin::new(&mut rest).poll_frame(cx));
        match tokio::time::timeout(BODY_IDLE, frame).await {
            Ok(Some(Ok(_))) => {}
            _ => return,
        }
    }
}

/// Serve `app` on `listener` until `stop` is ready: HTTP/1.1, upgrades for
/// the firehose, a connection closed that sends no whole request head within
/// `HEADER_TIMEOUT` of opening or of its last answer, and a request closed,
/// with no answer, whose body sends nothing for `BODY_IDLE`. A request
/// answered before its body was read to its end has the rest read and thrown
/// away after the answer when its `Content-Length` leaves at most `DRAIN`
/// bytes of it, and is otherwise answered with `Connection: close`, its
/// connection closed after it (`spec/LIMITS.md`, "Request size"). Each
/// request is seen through `ops::Seen`. Once `stop` is ready, no connection
/// is taken, and those open finish the requests they have, for up to
/// `ops::DRAIN`; answers what `stop` did.
pub(crate) async fn serve<T>(
    listener: tokio::net::TcpListener,
    app: axum::Router,
    stop: impl Future<Output = T>,
) -> T {
    use hyper_util::rt::{TokioIo, TokioTimer};
    use tower_service::Service as _;
    // Each connection holds a sender: all are gone once every one is closed.
    let (open, mut closed) = tokio::sync::mpsc::channel::<()>(1);
    let (stopping, stopped) = tokio::sync::watch::channel(false);
    let mut stop = std::pin::pin!(stop);
    let why = loop {
        let (stream, peer) = tokio::select! {
            why = &mut stop => break why,
            accepted = listener.accept() => match accepted {
                Ok(x) => x,
                Err(e) => {
                    // Out of file descriptors, most likely: wait for one.
                    crate::ops::failed("accepting a connection", e);
                    tokio::time::sleep(Duration::from_millis(100)).await;
                    continue;
                }
            },
        };
        // Answers are written as they are made, a streamed `getRepo` in
        // chunks: without this its last small chunk waits on the client's
        // delayed acknowledgement, 40 ms on Linux.
        if let Err(e) = stream.set_nodelay(true) {
            crate::ops::failed("setting TCP_NODELAY", e);
        }
        let (app, open, mut stopped) = (app.clone(), open.clone(), stopped.clone());
        tokio::spawn(async move {
            let service = hyper::service::service_fn(move |req: Request<hyper::body::Incoming>| {
                let mut app = app.clone();
                async move {
                    let seen = crate::ops::Seen::new(&req);
                    let timed_out = Arc::new(AtomicBool::new(false));
                    let ended = Arc::new(AtomicBool::new(req.body().is_end_stream()));
                    let left = Arc::new(Mutex::new(None));
                    let (flag, read, rest) = (timed_out.clone(), ended.clone(), left.clone());
                    let mut req = req.map(move |inner| {
                        Body::new(Idle {
                            inner: Some(inner),
                            sleep: None,
                            timed_out: flag,
                            ended: read,
                            left: rest,
                        })
                    });
                    req.extensions_mut().insert(Peer(peer));
                    let mut res = match caught(seen.run(app.call(req))).await {
                        Ok(Ok(res)) => seen.answer(res),
                        Ok(Err(never)) => match never {},
                        // Its account's store was not held, and none is
                        // loaded once the shutdown has closed the logs.
                        Err(why) if why.is::<crate::cache::ShuttingDown>() => {
                            seen.answer(crate::respond(crate::internal(), None))
                        }
                        Err(panic) => std::panic::resume_unwind(panic),
                    };
                    // A body left unread: hyper gives up on it, and closes
                    // the connection after the answer, but only once it has
                    // written the answer's head, saying nothing of it, and a
                    // client sends its next request on a connection being
                    // closed. So the rest is read here, as hyper writes the
                    // answer, or, too long or of no known length, the answer
                    // says the connection closes.
                    if !ended.load(Ordering::SeqCst) {
                        let rest = left.lock().unwrap_or_else(PoisonError::into_inner).take();
                        match rest
                            .filter(|r| r.size_hint().exact().is_some_and(|n| n <= DRAIN as u64))
                        {
                            Some(rest) => {
                                tokio::spawn(drain(rest));
                            }
                            None => {
                                res.headers_mut().insert(
                                    axum::http::header::CONNECTION,
                                    HeaderValue::from_static("close"),
                                );
                            }
                        }
                    }
                    // An error closes the connection, and nothing is answered.
                    if timed_out.load(Ordering::SeqCst) {
                        Err(BodyTimedOut)
                    } else {
                        Ok(res)
                    }
                }
            });
            let mut http = hyper::server::conn::http1::Builder::new();
            http.timer(TokioTimer::new())
                .header_read_timeout(HEADER_TIMEOUT);
            let conn = http
                .serve_connection(TokioIo::new(stream), service)
                .with_upgrades();
            let mut conn = std::pin::pin!(conn);
            tokio::select! {
                _ = conn.as_mut() => {}
                () = async { let _ = stopped.wait_for(|s| *s).await; } => {
                    conn.as_mut().graceful_shutdown();
                    let _ = conn.await;
                }
            }
            drop(open);
        });
    };
    drop(listener);
    stopping.send_replace(true);
    drop(open);
    let _ = tokio::time::timeout(crate::ops::DRAIN, closed.recv()).await;
    why
}

/// Answer `f`, or what it unwound with if it did: a panic, or what a load
/// refused at shutdown unwinds its request with (`cache::ShuttingDown`).
async fn caught<F: Future>(f: F) -> Result<F::Output, Box<dyn std::any::Any + Send>> {
    let mut f = std::pin::pin!(f);
    std::future::poll_fn(|cx| {
        let polled = std::panic::AssertUnwindSafe(|| f.as_mut().poll(cx));
        match std::panic::catch_unwind(polled) {
            Ok(Poll::Ready(out)) => Poll::Ready(Ok(out)),
            Ok(Poll::Pending) => Poll::Pending,
            Err(unwound) => Poll::Ready(Err(unwound)),
        }
    })
    .await
}

/// Read a body of up to `limit` bytes: answers it, or `limit + 1` bytes of
/// it if it is longer, having read and thrown away some of the rest; None if
/// it could not be read.
pub(crate) async fn read_body(mut body: Body, limit: usize) -> Option<Bytes> {
    let mut out: Vec<u8> = Vec::new();
    let mut thrown = 0usize;
    while let Some(frame) = std::future::poll_fn(|cx| Pin::new(&mut body).poll_frame(cx)).await {
        let Ok(data) = frame.ok()?.into_data() else {
            continue;
        };
        if out.len() <= limit {
            let room = limit + 1 - out.len();
            out.extend_from_slice(&data[..data.len().min(room)]);
        } else {
            thrown += data.len();
            if thrown > DRAIN {
                break;
            }
        }
    }
    Some(Bytes::from(out))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::testing::*;
    use crate::{Shared, answer, sessions::Bearer};
    use axum::http::Method;
    use serde_json::json;
    use std::sync::OnceLock;
    use std::sync::atomic::AtomicUsize;

    #[test]
    fn client_addresses() {
        let reference = Trusted::parse("127.0.0.0/8,::1/128,169.254.0.0/16,fe80::/10,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fc00::/7").unwrap();
        let with = |trusted: &Trusted, peer: &str, xff: Option<&str>| {
            let mut h = HeaderMap::new();
            if let Some(x) = xff {
                h.insert("x-forwarded-for", HeaderValue::from_str(x).unwrap());
            }
            client(trusted, &h, Some(peer.parse().unwrap()))
        };
        let local = "127.0.0.1:1234";
        assert_eq!(with(&reference, local, None), "127.0.0.1");
        assert_eq!(
            with(&reference, local, Some("203.0.113.1, 10.0.0.5")),
            "203.0.113.1"
        );
        assert_eq!(
            with(&reference, local, Some("203.0.113.1, 198.51.100.7")),
            "198.51.100.7"
        );
        assert_eq!(
            with(&reference, local, Some("10.0.0.1, 127.0.0.1, 192.168.1.1")),
            "10.0.0.1"
        );
        assert_eq!(
            with(
                &reference,
                local,
                Some("203.0.113.1,172.16.0.1, ::1, fe80::1, fd00::1, 169.254.1.1")
            ),
            "203.0.113.1"
        );
        assert_eq!(with(&reference, local, Some("2001:db8::1")), "2001:db8::1");
        assert_eq!(
            with(&reference, local, Some("::ffff:203.0.113.1")),
            "203.0.113.1"
        );
        // A last entry that is not an address: the connection's.
        assert_eq!(
            with(&reference, local, Some("203.0.113.1, junk")),
            "127.0.0.1"
        );
        // From a peer not trusted, the header is ignored.
        assert_eq!(
            with(&reference, "192.0.2.9:1234", Some("203.0.113.1")),
            "192.0.2.9"
        );
        let none = Trusted::default();
        assert_eq!(with(&none, local, Some("203.0.113.1")), "127.0.0.1");
        // Trusting 127.0.0.1 alone: 10.0.0.5 is not a proxy.
        let one = Trusted::parse("127.0.0.1").unwrap();
        assert_eq!(with(&one, local, Some("203.0.113.1, 10.0.0.5")), "10.0.0.5");
        assert_eq!(
            client(
                &none,
                &HeaderMap::new(),
                Some("[::ffff:127.0.0.1]:9".parse().unwrap())
            ),
            "127.0.0.1"
        );
        assert!(
            Trusted::parse("10.0.0.0/33").is_err()
                && Trusted::parse("x").is_err()
                && Trusted::parse("").unwrap().0.is_empty()
        );
        assert!(
            Trusted::parse("::1")
                .unwrap()
                .contains("::1".parse().unwrap())
                && !one.contains("::1".parse().unwrap())
        );
    }

    /// A call as a client at `client` sends it: its answer, or its error's
    /// name, and what it cost of the rate limits.
    async fn charged(
        s: &Shared,
        nsid: &str,
        token: Option<&str>,
        client: &str,
        body: Json,
    ) -> (Result<Json, String>, Option<Charged>) {
        let method =
            if pds_lexicon::method(nsid).is_some_and(|m| m.kind == pds_lexicon::Kind::Query) {
                Method::GET
            } else {
                Method::POST
            };
        let body = if body.is_null() {
            String::new()
        } else {
            body.to_string()
        };
        let bearer = token.map_or(Bearer::None, Bearer::Token);
        let call = Call {
            nsid,
            method: &method,
            params: &[],
            body: body.as_bytes(),
            bearer,
            headers: &HeaderMap::new(),
            client: client.into(),
            charged: OnceLock::new(),
            http: None,
        };
        let got = match answer(s, &call).await {
            Ok(crate::Resp::Json(j)) => Ok(j),
            Ok(_) => Ok(Json::Null),
            Err(e) => Err(e.name.to_string()),
        };
        (got, call.charged.get().cloned())
    }

    /// `createSession`'s limits: by identifier and address, each window's
    /// points, the call past them refused with when to try again, input the
    /// lexicon refuses costing nothing, and the points back once the window ends.
    #[tokio::test]
    async fn login_limits() {
        let plc = plc().await;
        let dir = scratch("limits-login");
        let s = open_with(&dir, &plc, |c| c.rate_limits = true);
        create(&s, "alice").await;
        let login = |id: &str| json!({ "identifier": id, "password": "wrong" });
        const LOGIN: &str = "com.atproto.server.createSession";
        for i in 1..=30u64 {
            let (got, c) = charged(&s, LOGIN, None, "203.0.113.1", login("alice.test")).await;
            assert_eq!(got, Err("AuthenticationRequired".into()));
            let c = c.unwrap();
            assert_eq!(
                (c.points, c.window, c.remaining, c.retry_after),
                (30, 300, 30 - i, None),
                "{i}"
            );
        }
        let (got, c) = charged(&s, LOGIN, None, "203.0.113.1", login("alice.test")).await;
        assert_eq!(got, Err("RateLimitExceeded".into()));
        let c = c.unwrap();
        assert_eq!((c.points, c.remaining), (30, 0));
        // Until the window, begun at the first call, ends.
        assert!(
            (240..=300).contains(&c.retry_after.unwrap()),
            "{:?}",
            c.retry_after
        );
        let mut h = HeaderMap::new();
        c.apply(&mut h);
        assert_eq!(h["ratelimit-policy"], "30;w=300");
        assert_eq!(
            h["access-control-expose-headers"],
            "RateLimit-Limit, RateLimit-Reset, RateLimit-Remaining, RateLimit-Policy, Retry-After"
        );
        // An answer's own exposed headers stay, named first.
        let mut h = HeaderMap::new();
        h.insert(
            "access-control-expose-headers",
            HeaderValue::from_static("atproto-space-commit, signature-input, signature"),
        );
        c.apply(&mut h);
        assert_eq!(
            h["access-control-expose-headers"],
            "atproto-space-commit, signature-input, signature, RateLimit-Limit, RateLimit-Reset, RateLimit-Remaining, RateLimit-Policy, Retry-After"
        );
        // Another case of the identifier, or another address, is another key.
        assert_eq!(
            charged(&s, LOGIN, None, "203.0.113.1", login("ALICE.test"))
                .await
                .0,
            Err("AuthenticationRequired".into())
        );
        assert_eq!(
            charged(&s, LOGIN, None, "203.0.113.2", login("alice.test"))
                .await
                .0,
            Err("AuthenticationRequired".into())
        );
        // Input the lexicon refuses costs nothing.
        let (got, c) = charged(&s, LOGIN, None, "203.0.113.3", json!({ "identifier": 5 })).await;
        assert_eq!((got, c.is_none()), (Err("InvalidRequest".into()), true));
        // Once the window ends, its points are back.
        s.limits.clock.skew.store(301, Ordering::SeqCst);
        let (got, c) = charged(&s, LOGIN, None, "203.0.113.1", login("alice.test")).await;
        assert_eq!(
            (got, c.unwrap().remaining),
            (Err("AuthenticationRequired".into()), 29)
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// `global` by client address, a path not served costing its point, a bad
    /// token costing nothing, writes by their points, and `getRepo` only `sync`.
    #[tokio::test]
    async fn global_and_write_limits() {
        let plc = plc().await;
        let dir = scratch("limits-global");
        let s = open_with(&dir, &plc, |c| c.rate_limits = true);
        let (did, token) = create(&s, "alice").await;
        const DESCRIBE: &str = "com.atproto.server.describeServer";
        for i in 1..=3000u64 {
            let (got, c) = charged(&s, DESCRIBE, None, "198.51.100.1", Json::Null).await;
            assert!(got.is_ok());
            assert_eq!(c.unwrap().remaining, 3000 - i);
        }
        assert_eq!(
            charged(&s, DESCRIBE, None, "198.51.100.1", Json::Null)
                .await
                .0,
            Err("RateLimitExceeded".into())
        );
        assert_eq!(
            charged(
                &s,
                "com.atproto.server.refreshSession",
                Some(&token),
                "198.51.100.1",
                Json::Null
            )
            .await
            .0,
            Err("InvalidToken".into())
        );
        assert_eq!(
            charged(
                &s,
                "com.example.notServed",
                None,
                "198.51.100.1",
                Json::Null
            )
            .await
            .0,
            Err("RateLimitExceeded".into())
        );
        let (got, c) = charged(
            &s,
            "com.example.notServed",
            None,
            "198.51.100.2",
            Json::Null,
        )
        .await;
        // Forwarded, with no AppView to go to, once it has paid.
        assert_eq!(
            (got, c.unwrap().remaining),
            (Err("InvalidRequest".into()), 2999)
        );
        // getRepo is `sync` alone, past `global`.
        let p = [("did".to_string(), did.clone())];
        let method = Method::GET;
        let call = Call {
            nsid: "com.atproto.sync.getRepo",
            method: &method,
            params: &p,
            body: b"",
            bearer: Bearer::None,
            headers: &HeaderMap::new(),
            client: "198.51.100.1".into(),
            charged: OnceLock::new(),
            http: None,
        };
        assert!(answer(&s, &call).await.is_ok());
        assert_eq!(
            (
                call.charged.get().unwrap().points,
                call.charged.get().unwrap().remaining
            ),
            (6000, 5999)
        );
        // Writes: 3 points a create; 201 writes cost theirs, and are refused.
        let creates = |n: usize| {
            let writes: Vec<Json> = (0..n).map(|_| json!({ "$type": "com.atproto.repo.applyWrites#create", "collection": "a.b.c", "value": { "x": 1 } })).collect();
            json!({ "repo": did, "writes": writes })
        };
        const APPLY: &str = "com.atproto.repo.applyWrites";
        // The headers are of the limit with the fewest points left: `global`
        // at first, the writes' once they have fewer.
        let (got, c) = charged(&s, APPLY, Some(&token), "198.51.100.3", creates(201)).await;
        let c = c.unwrap();
        assert_eq!(
            (got, c.points, c.remaining),
            (Err("InvalidRequest".into()), 3000, 2999)
        );
        for i in 1..=7u64 {
            let (got, c) = charged(&s, APPLY, Some(&token), "198.51.100.3", creates(200)).await;
            assert!(got.is_ok(), "{got:?}");
            let (writes, global) = (5000 - 603 - 600 * i, 2999 - i);
            let c = c.unwrap();
            assert_eq!(
                (c.points, c.remaining),
                if writes < global {
                    (5000, writes)
                } else {
                    (3000, global)
                },
                "{i}"
            );
        }
        let before = repo_of(&s, &did, |r| r.record_count());
        let (got, c) = charged(&s, APPLY, Some(&token), "198.51.100.3", creates(200)).await;
        assert_eq!(got, Err("RateLimitExceeded".into()));
        assert_eq!(repo_of(&s, &did, |r| r.record_count()), before);
        assert_eq!(c.unwrap().points, 5000);
        // An empty batch costs only `global`.
        let (got, c) = charged(&s, APPLY, Some(&token), "198.51.100.4", creates(0)).await;
        assert!(got.is_ok());
        assert_eq!(c.unwrap().points, 3000);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Without `PDS_RATE_LIMITS_ENABLED`, nothing costs anything.
    #[tokio::test]
    async fn limits_off() {
        let plc = plc().await;
        let dir = scratch("limits-off");
        let s = open(&dir, &plc);
        for _ in 0..3100 {
            let (got, c) = charged(
                &s,
                "com.atproto.server.describeServer",
                None,
                "198.51.100.1",
                Json::Null,
            )
            .await;
            assert!(got.is_ok() && c.is_none());
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Below the free space asked for, every write is `507` and changes
    /// nothing, and reads answer; with space again, writes land.
    #[tokio::test]
    async fn the_free_space_floor() {
        let plc = plc().await;
        let dir = scratch("limits-disk");
        let s = open_with(&dir, &plc, |c| {
            c.min_free = 1 << 20;
            // So that `putPreferences` is served; nothing here goes to it.
            c.proxy.appview = Some(crate::proxy::AppView {
                url: "http://127.0.0.1:9".into(),
                did: "did:web:appview.test".into(),
            });
        });
        assert!(s.limits.disk.statvfs().is_some_and(|f| f > 0));
        let (did, token) = create(&s, "alice").await;
        s.limits.disk.set_free(Some(1000));
        let log_len = s.log.len();
        let record =
            json!({ "repo": did, "collection": "a.b.c", "rkey": "x", "record": { "n": 1 } });
        assert_eq!(
            call(
                &s,
                "com.atproto.repo.putRecord",
                Some(&token),
                record.clone()
            )
            .await,
            Err("InsufficientStorage".into())
        );
        let new = json!({ "handle": "bob.test", "email": "bob@mail.com", "password": "pw" });
        assert_eq!(
            call(&s, "com.atproto.server.createAccount", None, new.clone()).await,
            Err("InsufficientStorage".into())
        );
        assert_eq!(
            call(
                &s,
                "com.atproto.server.createSession",
                None,
                json!({ "identifier": "alice.test", "password": "alice-pw" })
            )
            .await,
            Err("InsufficientStorage".into())
        );
        assert_eq!(
            crate::blobs::test_upload(&s, Some(&token), Some("text/plain"), b"x".to_vec(), false)
                .await,
            Err((507, "InsufficientStorage".into()))
        );
        const PUT_PREFS: &str = "app.bsky.actor.putPreferences";
        let prefs = json!({ "preferences": [{ "$type": "app.bsky.actor.defs#adultContentPref", "enabled": true }] });
        assert_eq!(
            call(&s, PUT_PREFS, Some(&token), prefs.clone()).await,
            Err("InsufficientStorage".into())
        );
        // The token and input are checked first.
        assert_eq!(
            call(&s, PUT_PREFS, None, prefs.clone()).await,
            Err("AuthMissing".into())
        );
        assert_eq!(
            call(&s, PUT_PREFS, Some(&token), json!({})).await,
            Err("InvalidRequest".into())
        );
        assert_eq!(
            call(&s, "com.atproto.repo.putRecord", None, record.clone()).await,
            Err("AuthMissing".into())
        );
        assert_eq!(
            call(
                &s,
                "com.atproto.repo.putRecord",
                Some(&token),
                json!({ "repo": did })
            )
            .await,
            Err("InvalidRequest".into())
        );
        // Reads go on.
        call(
            &s,
            "com.atproto.server.getSession",
            Some(&token),
            Json::Null,
        )
        .await
        .unwrap();
        assert_eq!(s.log.len(), log_len);
        assert!(repo_of(&s, &did, |r| r.record_count() == 0));
        s.limits.disk.set_free(None);
        call(&s, "com.atproto.repo.putRecord", Some(&token), record)
            .await
            .unwrap();
        call(&s, "com.atproto.server.createAccount", None, new)
            .await
            .unwrap();
        call(&s, PUT_PREFS, Some(&token), prefs).await.unwrap();
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Hashes run on threads of their own, no more at once than the machine
    /// has cores, and an fsync waits behind none of them.
    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn the_hash_pool() {
        let hashes = Arc::new(Hashes::start());
        let cores = std::thread::available_parallelism().unwrap().get();
        let (now, most) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
        let jobs: Vec<_> = (0..cores * 4)
            .map(|_| {
                let (hashes, now, most) = (hashes.clone(), now.clone(), most.clone());
                tokio::spawn(async move {
                    hashes
                        .run(move || {
                            let n = now.fetch_add(1, Ordering::SeqCst) + 1;
                            most.fetch_max(n, Ordering::SeqCst);
                            std::thread::sleep(Duration::from_millis(100));
                            now.fetch_sub(1, Ordering::SeqCst);
                            std::thread::current()
                                .name()
                                .unwrap_or_default()
                                .to_string()
                        })
                        .await
                        .unwrap()
                })
            })
            .collect();
        // While every hash thread is busy, a log is synced at once.
        tokio::time::sleep(Duration::from_millis(20)).await;
        let dir = scratch("limits-hashes");
        let (wal, _) = pds_store::Wal::open(&dir.join("x.log"), None).unwrap();
        let wal = Arc::new(wal);
        let n = wal.append(&[json!(1)]).unwrap();
        let started = Instant::now();
        crate::durable(wal, n).await.ok().unwrap();
        assert!(
            started.elapsed() < Duration::from_millis(100),
            "{:?}",
            started.elapsed()
        );
        assert!(
            now.load(Ordering::SeqCst) > 0,
            "the hashes were done before the sync"
        );
        for j in jobs {
            assert!(j.await.unwrap().starts_with("pds-hash-"));
        }
        assert_eq!(most.load(Ordering::SeqCst), cores);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// `createSession` answers no sooner than its floor after its input is
    /// checked, for an identifier no account has as for one with a wrong
    /// password, or the right one.
    #[tokio::test]
    async fn logins_take_their_time() {
        let plc = plc().await;
        let dir = scratch("limits-login-time");
        let s = open_with(&dir, &plc, |c| c.login_floor = Duration::from_millis(350));
        create(&s, "alice").await;
        for (id, pw, ok) in [
            ("nobody.test", "x", false),
            ("alice.test", "x", false),
            ("alice.test", "alice-pw", true),
        ] {
            let started = Instant::now();
            let got = call(
                &s,
                "com.atproto.server.createSession",
                None,
                json!({ "identifier": id, "password": pw }),
            )
            .await;
            assert_eq!(got.is_ok(), ok);
            assert!(
                started.elapsed() >= Duration::from_millis(350),
                "{id}: {:?}",
                started.elapsed()
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// The server on a port of its own, as `pds serve` runs it.
    async fn serving(s: &Shared) -> SocketAddr {
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        let addr = listener.local_addr().unwrap();
        tokio::spawn(serve(
            listener,
            crate::router(s.clone()),
            std::future::pending::<()>(),
        ));
        addr
    }

    /// A path the server does not serve pays its point before anything is
    /// fetched for it: over its limit, a call with a `DPoP` token never
    /// given, whose `atproto-proxy` names a `did:web`, is `429` with nothing
    /// fetched of that host; under it, it pays once, and its target, which
    /// cannot be read, is `500`.
    #[tokio::test]
    async fn unserved_paths_pay_before_anything_is_fetched() {
        let plc = plc().await;
        let dir = scratch("limits-unserved");
        let s = open_with(&dir, &plc, |c| c.rate_limits = true);
        let addr = serving(&s).await;
        let host = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        let port = host.local_addr().unwrap().port();
        let connections = Arc::new(AtomicUsize::new(0));
        let seen = connections.clone();
        tokio::spawn(async move {
            while let Ok((c, _)) = host.accept().await {
                seen.fetch_add(1, Ordering::SeqCst);
                drop(c);
            }
        });
        let http = reqwest::Client::new();
        let ask = || {
            http.get(format!("http://{addr}/xrpc/app.example.never.served"))
                .header("authorization", "DPoP never-given")
                .header("dpop", "not.a.proof")
                .header("atproto-proxy", format!("did:web:localhost%3A{port}#x"))
                .send()
        };
        let res = ask().await.unwrap();
        assert_eq!(
            (
                res.status().as_u16(),
                res.headers()["ratelimit-remaining"].to_str().unwrap()
            ),
            (500, "2999")
        );
        let fetched = connections.load(Ordering::SeqCst);
        assert!(fetched >= 1);
        for _ in 0..2999 {
            let _ = charged(
                &s,
                "com.atproto.server.describeServer",
                None,
                "127.0.0.1",
                Json::Null,
            )
            .await;
        }
        let res = ask().await.unwrap();
        assert_eq!(res.status().as_u16(), 429);
        assert_eq!(
            connections.load(Ordering::SeqCst),
            fetched,
            "nothing fetched past the limit"
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Over HTTP: an answer that cost points says so in its headers, and a
    /// body over its method's limit, chunked or not, is `413`; one of exactly
    /// the limit is taken.
    #[tokio::test]
    async fn over_http() {
        let plc = plc().await;
        let dir = scratch("limits-http");
        // The harness's own address is a proxy here, so the header's client is a key of its own.
        let s = open_with(&dir, &plc, |c| {
            c.rate_limits = true;
            c.trusted_proxies = Trusted::parse("127.0.0.1").unwrap();
        });
        let (did, token) = create(&s, "alice").await;
        let addr = serving(&s).await;
        let http = reqwest::Client::new();
        let res = http
            .get(format!(
                "http://{addr}/xrpc/com.atproto.server.describeServer"
            ))
            .header("x-forwarded-for", "203.0.113.5")
            .send()
            .await
            .unwrap();
        assert_eq!(res.headers()["ratelimit-limit"], "3000");
        assert_eq!(res.headers()["ratelimit-remaining"], "2999");
        assert_eq!(res.headers()["ratelimit-policy"], "3000;w=300");
        let res = http
            .get(format!("http://{addr}/.well-known/atproto-did"))
            .header("host", "alice.test")
            .send()
            .await
            .unwrap();
        assert!(res.headers().get("ratelimit-limit").is_none());
        // A body of exactly the limit, and one byte over.
        let body = |len: usize| {
            let head = format!(r#"{{"repo":"{did}","collection":"a.b.c","record":{{"t":""#);
            let tail = r#""}}"#;
            format!("{head}{}{tail}", "x".repeat(len - head.len() - tail.len()))
        };
        let url = format!("http://{addr}/xrpc/com.atproto.repo.createRecord");
        for (len, want) in [(MAX_RECORD_BODY, 200), (MAX_RECORD_BODY + 1, 413)] {
            let text = body(len);
            assert_eq!(text.len(), len);
            let res = http
                .post(&url)
                .bearer_auth(&token)
                .header("content-type", "application/json")
                .body(text.clone())
                .send()
                .await
                .unwrap();
            let status = res.status().as_u16();
            let answer: Json = serde_json::from_slice(&res.bytes().await.unwrap()).unwrap();
            assert_eq!(status, want, "{len}: {answer}");
            if want == 413 {
                assert_eq!(answer["error"], "PayloadTooLargeError");
            }
            // The same, chunked.
            let mut req = format!(
                "POST /xrpc/com.atproto.repo.createRecord HTTP/1.1\r\nhost: x\r\nauthorization: Bearer {token}\r\ncontent-type: application/json\r\ntransfer-encoding: chunked\r\nconnection: close\r\n\r\n"
            )
            .into_bytes();
            for c in text.as_bytes().chunks(65536) {
                req.extend(format!("{:x}\r\n", c.len()).bytes());
                req.extend(c);
                req.extend(b"\r\n");
            }
            req.extend(b"0\r\n\r\n");
            let answer = raw(addr, &req).await;
            assert!(
                answer.starts_with(&format!("HTTP/1.1 {want} ")),
                "{len} chunked: {answer}"
            );
        }
        // MCP's own limit.
        let res = http
            .post(format!("http://{addr}/mcp"))
            .body(vec![b' '; MAX_MCP_BODY + 1])
            .send()
            .await
            .unwrap();
        assert_eq!(res.status().as_u16(), 413);
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Send `request` on a connection of its own, and read what comes back until it closes.
    async fn raw(addr: SocketAddr, request: &[u8]) -> String {
        use tokio::io::{AsyncReadExt, AsyncWriteExt};
        let mut c = tokio::net::TcpStream::connect(addr).await.unwrap();
        c.write_all(request).await.unwrap();
        let mut out = Vec::new();
        let _ = tokio::time::timeout(Duration::from_secs(20), c.read_to_end(&mut out)).await;
        String::from_utf8_lossy(&out).into_owned()
    }

    /// A connection that sends half a request head is closed once
    /// `HEADER_TIMEOUT` has passed, with no answer.
    #[tokio::test]
    async fn half_a_head_is_closed() {
        use tokio::io::{AsyncReadExt, AsyncWriteExt};
        let plc = plc().await;
        let dir = scratch("limits-head");
        let s = open(&dir, &plc);
        let addr = serving(&s).await;
        let mut c = tokio::net::TcpStream::connect(addr).await.unwrap();
        c.write_all(b"GET /xrpc/com.atproto.server.describeServer HTTP/1.1\r\nhost: x\r\n")
            .await
            .unwrap();
        let started = Instant::now();
        let mut out = Vec::new();
        let read = tokio::time::timeout(
            HEADER_TIMEOUT + Duration::from_secs(5),
            c.read_to_end(&mut out),
        )
        .await;
        assert!(read.is_ok(), "still open after {:?}", started.elapsed());
        assert!(started.elapsed() >= HEADER_TIMEOUT - Duration::from_millis(500));
        assert!(!String::from_utf8_lossy(&out).contains("200 OK"));
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// The firehose: a message from the client past the limit closes it with
    /// `1009`; one client address holds at most so many subscriptions.
    #[tokio::test]
    async fn firehose_clients() {
        use tokio::io::{AsyncReadExt, AsyncWriteExt};
        let plc = plc().await;
        let dir = scratch("limits-firehose");
        let s = open_with(&dir, &plc, |c| {
            c.trusted_proxies = Trusted::parse("127.0.0.1").unwrap()
        });
        let addr = serving(&s).await;
        let upgrade = |client: &str| {
            format!(
                "GET /xrpc/com.atproto.sync.subscribeRepos HTTP/1.1\r\nhost: x\r\nx-forwarded-for: {client}\r\nupgrade: websocket\r\nconnection: Upgrade\r\nsec-websocket-key: dGhlIHNhbXBsZSBub25jZQ==\r\nsec-websocket-version: 13\r\n\r\n"
            )
        };
        let open_one = |client: &'static str| {
            let req = upgrade(client);
            async move {
                let mut c = tokio::net::TcpStream::connect(addr).await.unwrap();
                c.write_all(req.as_bytes()).await.unwrap();
                let mut head = Vec::new();
                while !head.ends_with(b"\r\n\r\n") {
                    let mut b = [0u8; 1];
                    if c.read(&mut b).await.unwrap() == 0 {
                        break;
                    }
                    head.push(b[0]);
                }
                (c, String::from_utf8_lossy(&head).into_owned())
            }
        };
        let mut held = Vec::new();
        for _ in 0..MAX_SUBSCRIPTIONS_PER_CLIENT {
            let (c, head) = open_one("203.0.113.7").await;
            assert!(head.starts_with("HTTP/1.1 101"), "{head}");
            held.push(c);
        }
        let (_, head) = open_one("203.0.113.7").await;
        assert!(head.starts_with("HTTP/1.1 429"), "{head}");
        let (mut other, head) = open_one("203.0.113.8").await;
        assert!(head.starts_with("HTTP/1.1 101"), "{head}");
        // One closed is no longer held.
        drop(held.pop());
        tokio::time::sleep(Duration::from_millis(200)).await;
        let (_, head) = open_one("203.0.113.7").await;
        assert!(head.starts_with("HTTP/1.1 101"), "{head}");
        // A small message is ignored; a frame saying 2 MiB closes it with 1009.
        other
            .write_all(&[
                0x82,
                0x85,
                1,
                2,
                3,
                4,
                b'h' ^ 1,
                b'e' ^ 2,
                b'l' ^ 3,
                b'l' ^ 4,
                b'o' ^ 1,
            ])
            .await
            .unwrap();
        let mut big = vec![0x82, 0xff];
        big.extend((2u64 << 20).to_be_bytes());
        big.extend([1, 2, 3, 4]);
        other.write_all(&big).await.unwrap();
        let mut out = Vec::new();
        let _ = tokio::time::timeout(Duration::from_secs(5), other.read_to_end(&mut out)).await;
        assert!(
            out.windows(4).any(|w| w == [0x88, 0x11, 0x03, 0xf1]),
            "{out:?}"
        );
        let _ = std::fs::remove_dir_all(&dir);
    }
}