~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
1767 lines · 63.9 KB Raw History
//! Blobs kept in a bucket of an object store that speaks S3's API, rather
//! than in the data directory (`spec/BLOBS.md`, "Where blobs are kept"):
//! the variables that say so, the mark the data directory keeps of where
//! its blobs are, the requests to the store, signed with AWS Signature
//! Version 4, and the deletion of the objects of blobs no longer held.
//!
//! Each blob an account holds is the object `blocks/<did>/<cid>`. The
//! account's log says which objects the store may keep (`Blobs::kept`): an
//! upload logs `blobPut` before it puts an object no entry names yet, so
//! that one the server is killed during is found again, and `blobDeleted`
//! is logged once the store has answered an object's delete. The objects it
//! may keep of blobs the account does not hold, and no upload is putting,
//! are its strays: the deleter deletes them once what made them is on disk,
//! each in its object's turn (`Bucket::turn`), which an upload holds from
//! before it notes its object until it is logged, so that no object is
//! deleted of a blob held again by then. A delete that fails is tried again
//! on the next pass, a second after the first and at most `RETRY_MAX`
//! after the last.
//!
//! An account with strays, or an upload on its way, is marked for a start
//! to load (`cache::Eager`) before the entry that makes them reaches its
//! log: an upload's `blobPut`, a commit that stops it holding a blob
//! (`log`), a change to its space repos while they reference blobs
//! (`space_repo`, `space_authority`), and uploads past their time
//! (`blobs::sweep`). A start so finds every stray, and the deleter unmarks
//! an account once it has none. A deleted account's objects are named in
//! the service log with its deletion (`blobsDoomed`), and leave it as their
//! deletes are answered (`blobsDeleted`).
//!
//! The data directory keeps its blobs where its first start put them: a
//! start with a bucket on a directory that holds nothing yet leaves the mark
//! `blobstore` in it, and a start otherwise than the mark says is refused
//! (`check_dir`). `pds migrate-blobs` moves them from one place to the other
//! (`migrate_blobs`): the mark `blobstore-moving`, naming where they go,
//! is in the directory from its start until it finishes, and a start is
//! refused while it is; `blobstore` is made or taken away, each durably, as
//! the move switches where they are kept.

use crate::ops::Target;
use crate::{Account, Shared, Store, XrpcError, err, internal, read, write};
use axum::body::Bytes;
use hmac::{Hmac, Mac};
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
use reqwest::header::{AUTHORIZATION, CONTENT_LENGTH, HOST, HeaderName, HeaderValue};
use serde_json::{Value as Json, json};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;

const BUCKET: &str = "PDS_BLOBSTORE_S3_BUCKET";
const REGION: &str = "PDS_BLOBSTORE_S3_REGION";
const ENDPOINT: &str = "PDS_BLOBSTORE_S3_ENDPOINT";
const PATH_STYLE: &str = "PDS_BLOBSTORE_S3_FORCE_PATH_STYLE";
const KEY_ID: &str = "PDS_BLOBSTORE_S3_ACCESS_KEY_ID";
const SECRET: &str = "PDS_BLOBSTORE_S3_SECRET_ACCESS_KEY";
const UPLOAD_TIMEOUT: &str = "PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS";
const REQUEST_TIMEOUT: &str = "PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS";

/// The blobstore's variables but the bucket: none may be set without it.
const OTHERS: [&str; 7] = [
    REGION,
    ENDPOINT,
    PATH_STYLE,
    KEY_ID,
    SECRET,
    UPLOAD_TIMEOUT,
    REQUEST_TIMEOUT,
];

/// The mark in the data directory of a start that kept blobs in a bucket.
pub(crate) const MARK: &str = "blobstore";
/// What the mark holds.
const MARKED: &[u8] = b"s3\n";
/// The mark of a move of the blobs unfinished, holding where they go.
const MOVING: &str = "blobstore-moving";

/// The service log's entry naming a deleted account's objects, to delete.
pub(crate) const DOOMED: &str = "blobsDoomed";
/// The service log's entry of those of them deleted.
pub(crate) const DELETED: &str = "blobsDeleted";
/// The kinds of entry the service log has of the bucket.
pub(crate) const ENTRIES: &[&str] = &[DOOMED, DELETED];

/// How long the deleter waits, with nothing failing, before it looks
/// again unwoken.
const IDLE: Duration = Duration::from_secs(30);
/// The longest it waits after a pass with a delete that failed.
const RETRY_MAX: Duration = Duration::from_secs(30);
/// How many deletes it has in flight at once.
const AT_ONCE: usize = 16;

fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
    m.lock().unwrap_or_else(PoisonError::into_inner)
}

// ---- Settings ---------------------------------------------------------------

/// The credentials requests are signed with.
#[derive(Clone)]
pub(crate) struct Keys {
    pub id: String,
    pub secret: String,
    /// `AWS_SESSION_TOKEN`, with the keys of `AWS_*`.
    pub token: Option<String>,
}

/// Where blobs are kept, as the variables say: with no bucket, in the data
/// directory.
#[derive(Clone)]
pub(crate) struct Settings {
    pub bucket: String,
    pub region: String,
    /// The store's base URL.
    pub endpoint: url::Url,
    /// Whether the bucket is the path's first segment, rather than the
    /// host's first label.
    pub path_style: bool,
    pub keys: Keys,
    /// How long storing one upload may take.
    pub upload_timeout: Duration,
    /// How long any other request may take.
    pub request_timeout: Duration,
}

impl Settings {
    /// From the environment: none without a bucket.
    pub(crate) fn from_env() -> Result<Option<Settings>, String> {
        Settings::parse(|n| std::env::var(n).ok())
    }

    /// As `vars` gives the variables, an empty one as if unset.
    fn parse(vars: impl Fn(&str) -> Option<String>) -> Result<Option<Settings>, String> {
        let var = |n: &str| vars(n).filter(|v| !v.is_empty());
        let path_style = match var(PATH_STYLE).as_deref() {
            None | Some("false") => false,
            Some("true") => true,
            Some(v) => return Err(format!("{PATH_STYLE} is {v:?}, not true or false")),
        };
        let timeout = |n: &str, default: u64| match var(n) {
            None => Ok(Duration::from_millis(default)),
            Some(v) => v
                .parse::<u64>()
                .ok()
                .filter(|ms| *ms > 0)
                .map(Duration::from_millis)
                .ok_or_else(|| format!("{n} is {v:?}, not a whole number of milliseconds above 0")),
        };
        let upload_timeout = timeout(UPLOAD_TIMEOUT, 20_000)?;
        let request_timeout = timeout(REQUEST_TIMEOUT, 10_000)?;
        let given = var(ENDPOINT).map(|e| endpoint(&e)).transpose()?;
        let keys = match (var(KEY_ID), var(SECRET)) {
            (Some(id), Some(secret)) => Some(Keys {
                id,
                secret,
                token: None,
            }),
            (None, None) => None,
            _ => {
                return Err(format!(
                    "one of {KEY_ID} and {SECRET} is set without the other"
                ));
            }
        };
        let Some(bucket) = var(BUCKET) else {
            return match OTHERS.iter().find(|n| var(n).is_some()) {
                Some(n) => Err(format!("{n} is set, but {BUCKET} is not")),
                None => Ok(None),
            };
        };
        let region = var(REGION).unwrap_or_else(|| "us-east-1".into());
        let keys = keys
            .or_else(|| {
                Some(Keys {
                    id: var("AWS_ACCESS_KEY_ID")?,
                    secret: var("AWS_SECRET_ACCESS_KEY")?,
                    token: var("AWS_SESSION_TOKEN"),
                })
            })
            .ok_or_else(|| {
                format!(
                    "{BUCKET} is set, but no credentials are: neither {KEY_ID} and {SECRET}, nor AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY"
                )
            })?;
        let endpoint = match given {
            Some(e) => e,
            None => endpoint(&format!("https://s3.{region}.amazonaws.com"))
                .map_err(|_| format!("{REGION} is {region:?}, which makes no endpoint"))?,
        };
        Ok(Some(Settings {
            bucket,
            region,
            endpoint,
            path_style,
            keys,
            upload_timeout,
            request_timeout,
        }))
    }
}

/// An endpoint: an `http` or `https` URL with a host, and no query or fragment.
fn endpoint(v: &str) -> Result<url::Url, String> {
    let bad = || format!("{ENDPOINT} is {v:?}, not an http or https URL with no query or fragment");
    let u = url::Url::parse(v).map_err(|_| bad())?;
    if !matches!(u.scheme(), "http" | "https")
        || u.host_str().is_none_or(str::is_empty)
        || u.query().is_some()
        || u.fragment().is_some()
    {
        return Err(bad());
    }
    Ok(u)
}

/// Where a data directory keeps its blobs.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Place {
    /// In itself, each a file.
    Disk,
    /// In the bucket its variables name.
    Bucket,
}

impl Place {
    /// The place `pds migrate-blobs --to` names: `disk` or `bucket`.
    pub fn named(name: &str) -> Option<Place> {
        match name {
            "disk" => Some(Place::Disk),
            "bucket" => Some(Place::Bucket),
            _ => None,
        }
    }

    pub fn name(self) -> &'static str {
        match self {
            Place::Disk => "disk",
            Place::Bucket => "bucket",
        }
    }

    pub(crate) fn other(self) -> Place {
        match self {
            Place::Disk => Place::Bucket,
            Place::Bucket => Place::Disk,
        }
    }

    /// Where a move into it goes.
    pub(crate) fn towards(self) -> &'static str {
        match self {
            Place::Disk => "into the data directory",
            Place::Bucket => "into the bucket",
        }
    }
}

/// Where `dir` keeps its blobs, as its mark says.
pub(crate) fn kept_in(dir: &Path) -> Result<Place, String> {
    let mark = dir.join(MARK);
    match std::fs::read(&mark) {
        Ok(b) if b == MARKED => Ok(Place::Bucket),
        Ok(_) => Err(format!("{}: not a mark this server knows", mark.display())),
        Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(Place::Disk),
        Err(e) => Err(format!("{}: {e}", mark.display())),
    }
}

/// Whether a move of `dir`'s blobs is unfinished, as its mark says: none
/// if none is; if one is, where it takes them, or none if a power cut lost
/// the mark's bytes (`mark_moving`).
pub(crate) fn moving(dir: &Path) -> Result<Option<Option<Place>>, String> {
    let mark = dir.join(MOVING);
    match std::fs::read(&mark) {
        Ok(b) => Ok(Some(
            std::str::from_utf8(&b)
                .ok()
                .and_then(|s| s.strip_suffix('\n'))
                .and_then(Place::named),
        )),
        Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
        Err(e) => Err(format!("{}: {e}", mark.display())),
    }
}

/// Make `dir` keep its blobs in `place`, durably: its mark made, or taken
/// away. The mark is never there without its bytes.
pub(crate) fn keep_in(dir: &Path, place: Place) -> Result<(), String> {
    let mark = dir.join(MARK);
    let at = |e: io::Error| format!("{}: {e}", mark.display());
    match place {
        Place::Bucket => {
            let (tmp, f) = beside(dir, MARK, MARKED)?;
            pds_store::sync_file(&f)
                .and_then(|()| std::fs::rename(&tmp, &mark))
                .and_then(|()| pds_store::sync_dir(dir))
                .map_err(at)
        }
        Place::Disk => take_mark(dir, MARK),
    }
}

/// Mark `dir` as moving its blobs to `to`, or with none, as moving them no
/// more, durably, with what a mark's write killed before its rename left.
/// The mark is there as soon as it can be, its bytes synced after: what it
/// holds only says where the move goes.
pub(crate) fn mark_moving(dir: &Path, to: Option<Place>) -> Result<(), String> {
    let Some(to) = to else {
        for name in [MARK, MOVING] {
            let tmp = dir.join(format!("{name}.tmp"));
            match std::fs::remove_file(&tmp) {
                Err(e) if e.kind() != io::ErrorKind::NotFound => {
                    return Err(format!("{}: {e}", tmp.display()));
                }
                _ => {}
            }
        }
        return take_mark(dir, MOVING);
    };
    let mark = dir.join(MOVING);
    let (tmp, f) = beside(dir, MOVING, format!("{}\n", to.name()).as_bytes())?;
    std::fs::rename(&tmp, &mark)
        .and_then(|()| pds_store::sync_file(&f))
        .and_then(|()| pds_store::sync_dir(dir))
        .map_err(|e| format!("{}: {e}", mark.display()))
}

/// `bytes` written to a file beside the mark `name` in `dir`, to rename
/// over it: its path, and the file.
fn beside(dir: &Path, name: &str, bytes: &[u8]) -> Result<(PathBuf, std::fs::File), String> {
    pds_store::make_dirs(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
    let tmp = dir.join(format!("{name}.tmp"));
    let f = pds_store::open_private(
        &tmp,
        std::fs::OpenOptions::new()
            .create(true)
            .write(true)
            .truncate(true),
    )
    .and_then(|mut f| f.write_all(bytes).map(|()| f))
    .map_err(|e| format!("{}: {e}", tmp.display()))?;
    Ok((tmp, f))
}

/// Take the mark `name` out of `dir`, if it is there, synced.
fn take_mark(dir: &Path, name: &str) -> Result<(), String> {
    let mark = dir.join(name);
    let at = |e: io::Error| format!("{}: {e}", mark.display());
    match std::fs::remove_file(&mark) {
        Err(e) if e.kind() != io::ErrorKind::NotFound => Err(at(e)),
        _ => pds_store::sync_dir(dir).map_err(at),
    }
}

/// Whether a start on `dir`, with a bucket or not, keeps blobs where the
/// directory's first start put them, with no move of them unfinished: a
/// start with a bucket on a directory that holds nothing yet marks it so,
/// synced, before anything else is written there.
pub(crate) fn check_dir(dir: &Path, bucket: bool) -> Result<(), String> {
    match moving(dir)? {
        Some(Some(to)) => {
            return Err(format!(
                "{} is moving its blobs {}, unfinished: `pds migrate-blobs --to {}` finishes the move, or `pds migrate-blobs --to {}` moves them back, given the bucket's variables either way; then it starts",
                dir.display(),
                to.towards(),
                to.name(),
                to.other().name(),
            ));
        }
        Some(None) => {
            return Err(format!(
                "{} is moving its blobs, unfinished: `pds migrate-blobs --to bucket` or `pds migrate-blobs --to disk`, given the bucket's variables, finishes the move or moves them back; then it starts",
                dir.display(),
            ));
        }
        None => {}
    }
    match (bucket, kept_in(dir)?) {
        (true, Place::Bucket) | (false, Place::Disk) => Ok(()),
        (false, Place::Bucket) => Err(format!(
            "{} keeps its blobs in a bucket, and {BUCKET} is not set: `pds migrate-blobs --to disk`, given the bucket's variables, moves them into it",
            dir.display()
        )),
        (true, Place::Disk) if crate::store::holds_data(dir) => Err(format!(
            "{} keeps its blobs in itself, and {BUCKET} is set: `pds migrate-blobs --to bucket` moves them into the bucket",
            dir.display()
        )),
        (true, Place::Disk) => keep_in(dir, Place::Bucket),
    }
}

// ---- Signing ------------------------------------------------------------------

type HmacSha256 = Hmac<Sha256>;

fn hmac(key: &[u8], data: &[u8]) -> [u8; 32] {
    let mut m = HmacSha256::new_from_slice(key).expect("HMAC takes a key of any length");
    m.update(data);
    m.finalize().into_bytes().into()
}

/// What SigV4 leaves as it is in a path: letters, digits, `-`, `_`, `.` and `~`.
const UNRESERVED: &AsciiSet = &NON_ALPHANUMERIC
    .remove(b'-')
    .remove(b'_')
    .remove(b'.')
    .remove(b'~');

/// A path segment as SigV4 has it, and as it is sent.
fn uri_encode(s: &str) -> String {
    utf8_percent_encode(s, UNRESERVED).to_string()
}

/// The time `ms` after the epoch as SigV4 dates a request,
/// `YYYYMMDDTHHMMSSZ`, in UTC.
fn amz_date(ms: i64) -> String {
    let secs = ms.div_euclid(1000);
    let (y, m, d) = pds_repo::data::civil_from_days(secs.div_euclid(86_400));
    let t = secs.rem_euclid(86_400);
    format!(
        "{y:04}{m:02}{d:02}T{:02}{:02}{:02}Z",
        t / 3600,
        t / 60 % 60,
        t % 60
    )
}

/// The `Authorization` of a request signed with AWS Signature Version 4 for
/// the service `s3` in `region`, at `date` (`amz_date`): `headers`, each by
/// its lowercase name, are those it signs; `path` and `query` are as sent,
/// already canonical; `payload` is the hex SHA-256 of its body.
#[allow(clippy::too_many_arguments)]
fn authorization(
    keys: &Keys,
    region: &str,
    method: &str,
    path: &str,
    query: &str,
    headers: &[(&str, &str)],
    payload: &str,
    date: &str,
) -> String {
    let mut signed = headers.to_vec();
    signed.sort_unstable();
    let names = signed.iter().map(|(n, _)| *n).collect::<Vec<_>>().join(";");
    let canonical_headers: String = signed
        .iter()
        .map(|(n, v)| {
            format!(
                "{n}:{}\n",
                v.split_whitespace().collect::<Vec<_>>().join(" ")
            )
        })
        .collect();
    let request = format!("{method}\n{path}\n{query}\n{canonical_headers}\n{names}\n{payload}");
    let day = &date[..8];
    let scope = format!("{day}/{region}/s3/aws4_request");
    let to_sign = format!(
        "AWS4-HMAC-SHA256\n{date}\n{scope}\n{}",
        hex::encode(Sha256::digest(request.as_bytes()))
    );
    let mut key = hmac(format!("AWS4{}", keys.secret).as_bytes(), day.as_bytes());
    for part in [region, "s3", "aws4_request"] {
        key = hmac(&key, part.as_bytes());
    }
    format!(
        "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={}",
        keys.id,
        hex::encode(hmac(&key, to_sign.as_bytes()))
    )
}

// ---- The bucket -----------------------------------------------------------------

/// The bytes of an object as they come, for `getBlob` to send.
pub(crate) type Chunks = tokio::sync::mpsc::Receiver<io::Result<Bytes>>;

/// The turns of the objects some upload or delete waits on or holds, by
/// account and CID.
type Turns = HashMap<(String, String), Arc<tokio::sync::Mutex<()>>>;

/// An object's turn: held, no other upload or delete of it goes on.
pub(crate) struct Turn<'a> {
    bucket: &'a Bucket,
    key: (String, String),
    held: Option<tokio::sync::OwnedMutexGuard<()>>,
}

impl Drop for Turn<'_> {
    fn drop(&mut self) {
        drop(self.held.take());
        let mut turns = lock(&self.bucket.turns);
        if turns
            .get(&self.key)
            .is_some_and(|m| Arc::strong_count(m) == 1)
        {
            turns.remove(&self.key);
        }
    }
}

/// The bucket, and the deletes it is owed.
pub(crate) struct Bucket {
    s: Settings,
    /// Keeps a connection unused for up to 90 seconds.
    http: reqwest::Client,
    /// Keeps none: what a request that failed on its way is sent again with.
    fresh: reqwest::Client,
    /// Each object's turn while some upload or delete waits on it or holds it.
    turns: Mutex<Turns>,
    /// The accounts whose strays the deleter is to look at.
    due: Mutex<BTreeSet<String>>,
    /// Deleted accounts' objects not yet deleted.
    doomed: Mutex<Doomed>,
    wake: tokio::sync::Notify,
}

impl Bucket {
    pub(crate) fn new(s: Settings) -> Result<Bucket, String> {
        let builder = || {
            reqwest::Client::builder()
                .no_proxy()
                .redirect(reqwest::redirect::Policy::none())
                .connect_timeout(s.request_timeout.min(Duration::from_secs(5)))
        };
        let built =
            |b: reqwest::ClientBuilder| b.build().map_err(|e| format!("an HTTP client: {e}"));
        let http = built(builder().pool_idle_timeout(Duration::from_secs(90)))?;
        let fresh = built(builder().pool_max_idle_per_host(0))?;
        Ok(Bucket {
            s,
            http,
            fresh,
            turns: Mutex::default(),
            due: Mutex::default(),
            doomed: Mutex::default(),
            wake: tokio::sync::Notify::new(),
        })
    }

    /// Send `req` over a connection an earlier request left open, if one is;
    /// one that fails on its way, as when the store closes that connection
    /// just as it goes out, once more over a connection of its own. Every
    /// request to the store may be made twice to the same effect
    /// (`spec/BLOBS.md`, "Requests to the store").
    async fn send(&self, req: reqwest::Request) -> reqwest::Result<reqwest::Response> {
        let again = req.try_clone();
        match crate::ops::execute(Target::Blobstore, &self.http, req).await {
            Err(e) if !e.is_connect() && !e.is_timeout() => match again {
                Some(req) => crate::ops::execute(Target::Blobstore, &self.fresh, req).await,
                None => Err(e),
            },
            sent => sent,
        }
    }

    /// Wait for the turn of `did`'s object `cid`.
    pub(crate) async fn turn(&self, did: &str, cid: &str) -> Turn<'_> {
        let key = (did.to_string(), cid.to_string());
        let m = lock(&self.turns).entry(key.clone()).or_default().clone();
        Turn {
            bucket: self,
            key,
            held: Some(m.lock_owned().await),
        }
    }

    /// Have the deleter look at `did`'s strays, and at whether its mark may go.
    pub(crate) fn wake(&self, did: &str) {
        lock(&self.due).insert(did.to_string());
        self.wake.notify_one();
    }

    /// A deleted account's objects to delete, its deletion on disk.
    pub(crate) fn doom(&self, did: &str, cids: Vec<String>) {
        if cids.is_empty() {
            return;
        }
        lock(&self.doomed)
            .0
            .entry(did.to_string())
            .or_default()
            .extend(cids);
        self.wake.notify_one();
    }

    /// The deleted accounts' objects the service log names, as a start replayed it.
    pub(crate) fn doomed(&self, d: Doomed) {
        *lock(&self.doomed) = d;
    }

    /// The deleted accounts' objects not yet deleted, by DID.
    pub(crate) fn owed(&self) -> Vec<(String, Vec<String>)> {
        lock(&self.doomed)
            .0
            .iter()
            .map(|(did, cids)| (did.clone(), cids.iter().cloned().collect()))
            .collect()
    }

    /// A deleted account's objects deleted, their entry logged: owed no more.
    pub(crate) fn forget_doomed(&self, did: &str, cids: &[String]) {
        let mut d = lock(&self.doomed);
        if let Some(left) = d.0.get_mut(did) {
            for c in cids {
                left.remove(c);
            }
            if left.is_empty() {
                d.0.remove(did);
            }
        }
    }

    /// An object's URL, the `Host` it is sent with, and its path, as sent and
    /// as SigV4 has it.
    fn locate(&self, did: &str, cid: &str) -> Result<(url::Url, String, String), String> {
        let e = &self.s.endpoint;
        let mut path: String = e
            .path_segments()
            .into_iter()
            .flatten()
            .filter(|p| !p.is_empty())
            .map(|p| {
                format!(
                    "/{}",
                    uri_encode(&percent_decode_str(p).decode_utf8_lossy())
                )
            })
            .collect();
        let mut host = e.host_str().unwrap_or_default().to_string();
        if self.s.path_style {
            path.push('/');
            path.push_str(&uri_encode(&self.s.bucket));
        } else {
            host = format!("{}.{host}", self.s.bucket);
        }
        for p in ["blocks", did, cid] {
            path.push('/');
            path.push_str(&uri_encode(p));
        }
        if let Some(port) = e.port() {
            host = format!("{host}:{port}");
        }
        let url = url::Url::parse(&format!("{}://{host}{path}", e.scheme()))
            .map_err(|why| format!("the URL of {did}'s object {cid}: {why}"))?;
        Ok((url, host, path))
    }

    /// A request for `did`'s object `cid`, signed, with `body` if given.
    fn request(
        &self,
        method: reqwest::Method,
        did: &str,
        cid: &str,
        body: Option<Bytes>,
    ) -> Result<reqwest::Request, String> {
        let (url, host, path) = self.locate(did, cid)?;
        let payload = hex::encode(Sha256::digest(body.as_deref().unwrap_or_default()));
        let date = amz_date(crate::events::now_ms() as i64);
        let mut signed = vec![
            ("host", host.as_str()),
            ("x-amz-content-sha256", payload.as_str()),
            ("x-amz-date", date.as_str()),
        ];
        if let Some(t) = &self.s.keys.token {
            signed.push(("x-amz-security-token", t.as_str()));
        }
        let auth = authorization(
            &self.s.keys,
            &self.s.region,
            method.as_str(),
            &path,
            "",
            &signed,
            &payload,
            &date,
        );
        let mut req = reqwest::Request::new(method, url);
        let h = req.headers_mut();
        let value = |v: &str| HeaderValue::from_str(v).map_err(|e| format!("a header: {e}"));
        for (n, v) in &signed {
            let name = if *n == "host" {
                HOST
            } else {
                HeaderName::from_bytes(n.as_bytes()).map_err(|e| format!("a header: {e}"))?
            };
            h.insert(name, value(v)?);
        }
        h.insert(AUTHORIZATION, value(&auth)?);
        if let Some(b) = body {
            h.insert(CONTENT_LENGTH, HeaderValue::from(b.len()));
            *req.body_mut() = Some(b.into());
        }
        Ok(req)
    }

    /// Send `req`, and read its answer whole, within `wait`: the status the
    /// store answered, or why it gave none.
    async fn exchange(
        &self,
        req: reqwest::Request,
        wait: Duration,
    ) -> Result<reqwest::StatusCode, String> {
        let sent = tokio::time::timeout(wait, async {
            let res = self.send(req).await?;
            let status = res.status();
            let _ = res.bytes().await;
            Ok::<_, reqwest::Error>(status)
        })
        .await;
        match sent {
            Ok(Ok(s)) => Ok(s),
            Ok(Err(e)) => Err(e.to_string()),
            Err(_) => Err(format!("no answer within {} ms", wait.as_millis())),
        }
    }

    /// Put `did`'s object `cid`, within `PDS_BLOBSTORE_S3_UPLOAD_TIMEOUT_MS`:
    /// what `uploadBlob` answers if the store did not answer it with a 2xx.
    pub(crate) async fn put(&self, did: &str, cid: &str, bytes: Bytes) -> Result<(), XrpcError> {
        let req = match self.request(reqwest::Method::PUT, did, cid, Some(bytes)) {
            Ok(r) => r,
            Err(why) => {
                failure("put", did, &why);
                return internal();
            }
        };
        match self.exchange(req, self.s.upload_timeout).await {
            Ok(s) if s.is_success() => Ok(()),
            Ok(_) => internal(),
            Err(_) => err(504, "UpstreamTimeout", "The blob store did not answer"),
        }
    }

    /// Put `did`'s object `cid` as `put` does: why not, if the store did not
    /// answer it with a 2xx.
    pub(crate) async fn put_object(
        &self,
        did: &str,
        cid: &str,
        bytes: Bytes,
    ) -> Result<(), String> {
        let req = self
            .request(reqwest::Method::PUT, did, cid, Some(bytes))
            .inspect_err(|why| failure("put", did, why))?;
        match self.exchange(req, self.s.upload_timeout).await? {
            s if s.is_success() => Ok(()),
            s => Err(format!("the store answered {s}")),
        }
    }

    /// Read `did`'s object `cid`, of `size` bytes, as it comes: its first
    /// bytes in hand within `PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS`, the rest
    /// each within that of the last, or none if the store failed before.
    pub(crate) async fn get(&self, did: &str, cid: &str, size: u64) -> Option<Chunks> {
        let req = match self.request(reqwest::Method::GET, did, cid, None) {
            Ok(r) => r,
            Err(why) => {
                failure("get", did, &why);
                return None;
            }
        };
        let wait = self.s.request_timeout;
        let first = tokio::time::timeout(wait, async {
            let mut res = self.send(req).await.map_err(|_| None)?;
            if res.status() != reqwest::StatusCode::OK {
                return Err(None);
            }
            let first = match size {
                0 => None,
                _ => match res.chunk().await {
                    Ok(Some(b)) => Some(b),
                    Ok(None) => return Err(Some("the object ended before its first byte".into())),
                    Err(e) => return Err(Some(e.to_string())),
                },
            };
            Ok((res, first))
        })
        .await;
        let (mut res, first) = match first {
            Ok(Ok(got)) => got,
            // The request's own failure is logged as it is (`ops::execute`).
            Ok(Err(None)) => return None,
            Ok(Err(Some(why))) => {
                failure("get", did, &why);
                return None;
            }
            Err(_) => return None,
        };
        let (tx, chunks) = tokio::sync::mpsc::channel(4);
        let did = did.to_string();
        tokio::spawn(async move {
            if let Some(b) = first
                && tx.send(Ok(b)).await.is_err()
            {
                return;
            }
            loop {
                let why = match tokio::time::timeout(wait, res.chunk()).await {
                    Ok(Ok(Some(b))) => {
                        if tx.send(Ok(b)).await.is_err() {
                            return;
                        }
                        continue;
                    }
                    Ok(Ok(None)) => return,
                    Ok(Err(e)) => e.to_string(),
                    Err(_) => "timeout".to_string(),
                };
                failure("get", &did, &why);
                let _ = tx.send(Err(io::Error::other(why))).await;
                return;
            }
        });
        Some(chunks)
    }

    /// Read `did`'s object `cid`, of `size` bytes, whole, as `get` reads it:
    /// none if the store has no such object, a 404; why not if it failed
    /// otherwise. Reading stops past `size` bytes.
    pub(crate) async fn fetch(
        &self,
        did: &str,
        cid: &str,
        size: u64,
    ) -> Result<Option<Vec<u8>>, String> {
        let req = self
            .request(reqwest::Method::GET, did, cid, None)
            .inspect_err(|why| failure("get", did, why))?;
        let wait = self.s.request_timeout;
        let late = || format!("no answer within {} ms", wait.as_millis());
        let mut res = match tokio::time::timeout(wait, self.send(req)).await {
            Ok(Ok(res)) => res,
            Ok(Err(e)) => return Err(e.to_string()),
            Err(_) => return Err(late()),
        };
        match res.status() {
            reqwest::StatusCode::OK => {}
            reqwest::StatusCode::NOT_FOUND => return Ok(None),
            s => return Err(format!("the store answered {s}")),
        }
        let mut bytes = Vec::new();
        while bytes.len() as u64 <= size {
            let why = match tokio::time::timeout(wait, res.chunk()).await {
                Ok(Ok(Some(b))) => {
                    bytes.extend_from_slice(&b);
                    continue;
                }
                Ok(Ok(None)) => break,
                Ok(Err(e)) => e.to_string(),
                Err(_) => late(),
            };
            failure("get", did, &why);
            return Err(why);
        }
        Ok(Some(bytes))
    }

    /// Delete `did`'s object `cid`, within `PDS_BLOBSTORE_S3_REQUEST_TIMEOUT_MS`:
    /// once the store answered that it is gone, a 2xx, or a 404 of a store
    /// that answers so for a key it has not; why not otherwise.
    pub(crate) async fn delete(&self, did: &str, cid: &str) -> Result<(), String> {
        let req = self
            .request(reqwest::Method::DELETE, did, cid, None)
            .inspect_err(|why| failure("delete", did, why))?;
        match self.exchange(req, self.s.request_timeout).await? {
            s if s.is_success() || s == reqwest::StatusCode::NOT_FOUND => Ok(()),
            s => Err(format!("the store answered {s}")),
        }
    }
}

/// A request to the bucket that failed where `ops::execute` does not see it:
/// one that could not be made, or an answer that broke off.
fn failure(op: &str, did: &str, why: &str) {
    crate::ops::warn(
        "blobstore failure",
        &[
            ("op", json!(op)),
            ("error", json!(why)),
            ("did", json!(did)),
        ],
    );
}

// ---- Deleted accounts' objects ------------------------------------------------------

/// Deleted accounts' objects not yet deleted, by DID, as the service log
/// has them.
#[derive(Default)]
pub(crate) struct Doomed(BTreeMap<String, BTreeSet<String>>);

impl Doomed {
    /// Replay an entry of the kinds in `ENTRIES`.
    pub(crate) fn replay(&mut self, e: &Json) -> Result<(), String> {
        let (Some(did), Some(cids)) = (e["did"].as_str(), e["cids"].as_array()) else {
            return Err(format!("an entry of objects with no did or cids: {e}"));
        };
        let cids = cids
            .iter()
            .map(|c| c.as_str().map(str::to_string))
            .collect::<Option<Vec<String>>>()
            .ok_or_else(|| format!("an entry of objects whose cids are not strings: {e}"))?;
        if e["t"] == DOOMED {
            self.0.entry(did.to_string()).or_default().extend(cids);
        } else if let Some(left) = self.0.get_mut(did) {
            for c in &cids {
                left.remove(c);
            }
            if left.is_empty() {
                self.0.remove(did);
            }
        }
        Ok(())
    }

    /// One entry for each account with objects left: what a checkpoint keeps.
    pub(crate) fn entries(&self) -> impl Iterator<Item = Json> + '_ {
        self.0.iter().map(|(did, cids)| doomed_entry(did, cids))
    }
}

/// The service log's entry naming a deleted account's objects, to delete.
pub(crate) fn doomed_entry<'a>(did: &str, cids: impl IntoIterator<Item = &'a String>) -> Json {
    json!({ "t": DOOMED, "did": did, "cids": cids.into_iter().collect::<Vec<_>>() })
}

/// The service log's entry of a deleted account's objects deleted.
pub(crate) fn deleted_entry(did: &str, cids: &[String]) -> Json {
    json!({ "t": DELETED, "did": did, "cids": cids })
}

// ---- Marks ---------------------------------------------------------------------------

/// With blobs in a bucket, mark the account of `st` for a start to load
/// before a change to its space repos is logged while their records
/// reference blobs: the change may stop it holding one.
pub(crate) fn mark_for(st: &Store) -> io::Result<()> {
    if st.ctx.bucket.is_some() && st.blobs.space_held() {
        st.ctx.eager.mark(&st.repo.did)
    } else {
        Ok(())
    }
}

// ---- The deleter -------------------------------------------------------------------

/// Delete strays and deleted accounts' objects: at once, again whenever
/// woken or every `IDLE`, and after a pass with a delete that failed, a
/// second later, then twice as long each time up to `RETRY_MAX`.
pub(crate) fn delete_every(svc: &Shared) {
    let Some(bucket) = svc.ctx.bucket.clone() else {
        return;
    };
    let weak = Arc::downgrade(svc);
    tokio::spawn(async move {
        let mut retry = Duration::ZERO;
        loop {
            let Some(svc) = weak.upgrade() else { return };
            let clean = pass(&svc, &bucket).await;
            drop(svc);
            if clean {
                retry = Duration::ZERO;
                tokio::select! {
                    () = bucket.wake.notified() => {}
                    () = tokio::time::sleep(IDLE) => {}
                }
            } else {
                retry = (retry * 2).clamp(Duration::from_secs(1), RETRY_MAX);
                tokio::time::sleep(retry).await;
            }
        }
    });
}

/// What one delete of a pass came to.
enum Done {
    Stray { did: String, ok: bool },
    Doomed { did: String, cid: String, ok: bool },
}

/// One pass: every stray of the accounts due, and every deleted account's
/// object, deleted; the accounts left with none unmarked. Answers whether
/// every delete was answered.
pub(crate) async fn pass(svc: &Shared, bucket: &Arc<Bucket>) -> bool {
    let dids: Vec<String> = std::mem::take(&mut *lock(&bucket.due))
        .into_iter()
        .collect();
    // Read off the async threads: a store not held is loaded.
    let found = {
        let svc = svc.clone();
        tokio::task::spawn_blocking(move || {
            dids.iter()
                .filter_map(|did| strays(&svc, did))
                .collect::<Vec<_>>()
        })
        .await
        .unwrap_or_default()
    };
    let doomed: Vec<(String, String)> = lock(&bucket.doomed)
        .0
        .iter()
        .flat_map(|(did, cids)| cids.iter().map(|c| (did.clone(), c.clone())))
        .collect();
    let at_once = Arc::new(tokio::sync::Semaphore::new(AT_ONCE));
    let mut jobs = tokio::task::JoinSet::new();
    for (a, cids) in &found {
        for cid in cids {
            let (bucket, a, cid, at_once) =
                (bucket.clone(), a.clone(), cid.clone(), at_once.clone());
            jobs.spawn(async move {
                let _turn = at_once.acquire().await;
                let ok = delete_stray(&bucket, &a, &cid).await;
                Done::Stray {
                    did: a.did.clone(),
                    ok,
                }
            });
        }
    }
    for (did, cid) in doomed {
        let (svc, bucket, at_once) = (svc.clone(), bucket.clone(), at_once.clone());
        jobs.spawn(async move {
            let _turn = at_once.acquire().await;
            let ok = delete_doomed(&svc, &bucket, &did, &cid).await;
            Done::Doomed { did, cid, ok }
        });
    }
    let mut failed: BTreeSet<String> = BTreeSet::new();
    let mut deleted: BTreeMap<String, Vec<String>> = BTreeMap::new();
    let mut clean = true;
    while let Some(done) = jobs.join_next().await {
        match done {
            Ok(Done::Stray { did, ok: false }) => {
                failed.insert(did);
            }
            Ok(Done::Doomed { did, cid, ok: true }) => deleted.entry(did).or_default().push(cid),
            Ok(Done::Stray { ok: true, .. }) => {}
            Ok(Done::Doomed { ok: false, .. }) | Err(_) => clean = false,
        }
    }
    for (did, cids) in deleted {
        match svc.log.append(&[deleted_entry(&did, &cids)]) {
            Ok(_) => bucket.forget_doomed(&did, &cids),
            Err(e) => {
                crate::ops::failed_for(
                    crate::ops::Level::Error,
                    "logging objects deleted failed",
                    &did,
                    e,
                );
                clean = false;
            }
        }
    }
    clean &= failed.is_empty();
    for did in &failed {
        lock(&bucket.due).insert(did.clone());
    }
    let settled: Vec<Arc<Account>> = found
        .into_iter()
        .map(|(a, _)| a)
        .filter(|a| !failed.contains(&a.did))
        .collect();
    let _ = tokio::task::spawn_blocking(move || settled.iter().for_each(|a| unmark(a))).await;
    clean
}

/// An account's strays, unless it is deleted, whose objects are doomed.
fn strays(svc: &crate::Service, did: &str) -> Option<(Arc<Account>, Vec<String>)> {
    let a = svc.account(did)?;
    if read(&a.state).deleted {
        return None;
    }
    let cids = read(&a.store).blobs.strays().cloned().collect();
    Some((a, cids))
}

/// Unmark an account a start need not load any more.
fn unmark(a: &Account) {
    if read(&a.state).deleted {
        return;
    }
    let st = read(&a.store);
    if st.ctx.eager.is_marked(&a.did) && !st.needs_start() {
        st.ctx.eager.relax(&a.did, Some(&a.handle()));
    }
}

/// Delete a stray in its turn, once what made it is on disk, if it is a
/// stray still, and log that: whether nothing is left to do of it.
async fn delete_stray(bucket: &Bucket, a: &Account, cid: &str) -> bool {
    let _turn = bucket.turn(&a.did, cid).await;
    let on_disk = {
        if read(&a.state).deleted {
            return true;
        }
        let st = read(&a.store);
        if !st.blobs.is_stray(cid) {
            return true;
        }
        (st.wal.clone(), st.wal.written())
    };
    if crate::durable(on_disk.0, on_disk.1).await.is_err()
        || bucket.delete(&a.did, cid).await.is_err()
    {
        return false;
    }
    if read(&a.state).deleted {
        return true;
    }
    let mut st = write(&a.store);
    if !st.blobs.is_stray(cid) {
        return true;
    }
    match st.wal.append(&[crate::blobs::deleted_entry(cid)]) {
        Ok(_) => {
            st.blobs.forget_object(cid);
            true
        }
        Err(e) => {
            crate::ops::failed_for(
                crate::ops::Level::Error,
                "logging an object deleted failed",
                &a.did,
                e,
            );
            false
        }
    }
}

/// Delete a deleted account's object in its turn: whether it is gone, or is
/// an account's of that DID made since, which keeps it.
async fn delete_doomed(svc: &Shared, bucket: &Bucket, did: &str, cid: &str) -> bool {
    let _turn = bucket.turn(did, cid).await;
    if let Some(a) = svc.account(did)
        && !read(&a.state).deleted
        && read(&a.store).blobs.keeps(cid)
    {
        return true;
    }
    bucket.delete(did, cid).await.is_ok()
}

/// A store as S3 answers it, in memory, and a server with its blobs there.
#[cfg(test)]
pub(crate) mod testing {
    use super::*;
    use crate::testing::*;
    use axum::http::{Method, StatusCode};
    use axum::response::IntoResponse;

    /// A store as S3 answers, path-style, its bucket `bucket`, in memory:
    /// each request's method, key, and whether it was signed as `request`
    /// signs (and a `PUT` with its length), kept; failing a method as asked,
    /// with a status, or with none for 5 seconds (0).
    #[derive(Default)]
    pub(crate) struct Fake {
        pub objects: Mutex<BTreeMap<String, Vec<u8>>>,
        pub fail: Mutex<Option<(Method, u16)>>,
        pub seen: Mutex<Vec<(Method, String, bool)>>,
    }

    impl Fake {
        pub fn has(&self, key: &str) -> bool {
            lock(&self.objects).contains_key(key)
        }

        pub fn fail(&self, f: Option<(Method, u16)>) {
            *lock(&self.fail) = f;
        }

        /// The keys it holds, in order.
        pub fn keys(&self) -> Vec<String> {
            lock(&self.objects).keys().cloned().collect()
        }
    }

    pub(crate) async fn fake_store(fake: Arc<Fake>) -> String {
        async fn answer(
            axum::extract::State(f): axum::extract::State<Arc<Fake>>,
            method: Method,
            uri: axum::http::Uri,
            headers: axum::http::HeaderMap,
            body: Bytes,
        ) -> axum::response::Response {
            let h = |n: &str| {
                headers
                    .get(n)
                    .and_then(|v| v.to_str().ok())
                    .unwrap_or_default()
                    .to_string()
            };
            let payload = hex::encode(Sha256::digest(&body));
            let date = h("x-amz-date");
            let keys = Keys {
                id: "k".into(),
                secret: "s".into(),
                token: None,
            };
            let signed = authorization(
                &keys,
                "us-east-1",
                method.as_str(),
                uri.path(),
                "",
                &[
                    ("host", &h("host")),
                    ("x-amz-content-sha256", &h("x-amz-content-sha256")),
                    ("x-amz-date", &date),
                ],
                &payload,
                &date,
            );
            let ok = h("authorization") == signed
                && h("x-amz-content-sha256") == payload
                && (method != Method::PUT || headers.contains_key("content-length"));
            let key = uri
                .path()
                .strip_prefix("/bucket/")
                .unwrap_or_default()
                .split('/')
                .map(|p| percent_decode_str(p).decode_utf8_lossy().into_owned())
                .collect::<Vec<_>>()
                .join("/");
            lock(&f.seen).push((method.clone(), key.clone(), ok));
            if !ok {
                return StatusCode::FORBIDDEN.into_response();
            }
            let fail = lock(&f.fail).clone();
            if let Some((m, status)) = fail
                && m == method
            {
                if status == 0 {
                    tokio::time::sleep(Duration::from_secs(5)).await;
                }
                return StatusCode::from_u16(status.max(500))
                    .unwrap()
                    .into_response();
            }
            let mut objects = lock(&f.objects);
            match method {
                Method::PUT => {
                    objects.insert(key, body.to_vec());
                    StatusCode::OK.into_response()
                }
                Method::GET => match objects.get(&key) {
                    Some(b) => b.clone().into_response(),
                    None => StatusCode::NOT_FOUND.into_response(),
                },
                Method::DELETE => {
                    objects.remove(&key);
                    StatusCode::NO_CONTENT.into_response()
                }
                _ => StatusCode::NOT_IMPLEMENTED.into_response(),
            }
        }
        let app = axum::Router::new().fallback(answer).with_state(fake);
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        url
    }

    /// The bucket `bucket` of the store at `url`, path-style, with timeouts
    /// of half a second.
    pub(crate) fn settings(url: &str) -> Settings {
        Settings {
            bucket: "bucket".into(),
            region: "us-east-1".into(),
            endpoint: url::Url::parse(url).unwrap(),
            path_style: true,
            keys: Keys {
                id: "k".into(),
                secret: "s".into(),
                token: None,
            },
            upload_timeout: Duration::from_millis(500),
            request_timeout: Duration::from_millis(500),
        }
    }

    /// The configuration of a server registering DIDs at `plc`, its blobs
    /// in the bucket of the store at `url`, its mail in `sink`.
    pub(crate) fn bucket_config(
        plc: &str,
        sink: &Path,
        url: &str,
        load_all: bool,
    ) -> crate::Config {
        std::fs::create_dir_all(sink).unwrap();
        crate::Config {
            mailer: crate::email::Mailer::sink(sink),
            blobstore: Some(settings(url)),
            load_all,
            ..config(plc)
        }
    }

    /// The server on `dir`, configured as `bucket_config` says.
    pub(crate) fn open_bucket(
        dir: &Path,
        plc: &str,
        sink: &Path,
        url: &str,
        load_all: bool,
    ) -> Shared {
        check_dir(dir, true).unwrap();
        Arc::new(crate::Service::open(dir, bucket_config(plc, sink, url, load_all)).unwrap())
    }

    pub(crate) fn body(n: u8) -> Vec<u8> {
        format!("spec-pds blob {n} ").repeat(4).into_bytes()
    }

    pub(crate) fn raw_cid(b: &[u8]) -> String {
        pds_repo::data::cid_string(&pds_repo::data::cid_of(0x55, b))
    }

    /// A record `rkey` referencing the blob `b`.
    pub(crate) async fn put(
        s: &Shared,
        token: &str,
        did: &str,
        rkey: &str,
        b: &[u8],
    ) -> Result<Json, String> {
        let r = json!({ "$type": "blob", "ref": { "$link": raw_cid(b) }, "mimeType": "text/plain", "size": b.len() });
        call(s, "com.atproto.repo.putRecord", Some(token), json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey, "record": { "f": r } })).await
    }

    pub(crate) async fn unput(s: &Shared, token: &str, did: &str, rkey: &str) {
        call(
            s,
            "com.atproto.repo.deleteRecord",
            Some(token),
            json!({ "repo": did, "collection": "com.example.blob", "rkey": rkey }),
        )
        .await
        .unwrap();
    }

    /// getBlob through the router's handler: its status and bytes.
    pub(crate) async fn get(s: &Shared, did: &str, b: &[u8]) -> (u16, Vec<u8>) {
        let query = url::form_urlencoded::Serializer::new(String::new())
            .append_pair("did", did)
            .append_pair("cid", &raw_cid(b))
            .finish();
        let res = crate::handle(
            axum::extract::State(s.clone()),
            Method::GET,
            format!("/xrpc/com.atproto.sync.getBlob?{query}")
                .parse()
                .unwrap(),
            axum::http::HeaderMap::new(),
            None,
            axum::body::Body::empty(),
        )
        .await;
        let status = res.status().as_u16();
        let bytes = axum::body::to_bytes(res.into_body(), usize::MAX)
            .await
            .unwrap()
            .to_vec();
        (status, bytes)
    }
}

#[cfg(test)]
mod tests {
    use super::testing::*;
    use super::*;
    use crate::blobs::test_upload;
    use crate::testing::*;
    use axum::http::Method;

    fn vars(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
        let m: HashMap<String, String> = pairs
            .iter()
            .map(|(k, v)| (k.to_string(), v.to_string()))
            .collect();
        move |n| m.get(n).cloned()
    }

    #[test]
    fn settings_parse_as_the_spec_says() {
        assert!(Settings::parse(vars(&[])).unwrap().is_none());
        // Empty is as unset.
        assert!(
            Settings::parse(vars(&[(BUCKET, ""), (REGION, "")]))
                .unwrap()
                .is_none()
        );
        let s = Settings::parse(vars(&[(BUCKET, "b"), (KEY_ID, "k"), (SECRET, "s")]))
            .unwrap()
            .unwrap();
        assert_eq!(
            (s.region.as_str(), s.endpoint.as_str(), s.path_style),
            ("us-east-1", "https://s3.us-east-1.amazonaws.com/", false)
        );
        assert_eq!(
            (s.upload_timeout, s.request_timeout),
            (Duration::from_secs(20), Duration::from_secs(10))
        );
        // The environment's keys, with its session token, when neither of the bucket's is set.
        let s = Settings::parse(vars(&[
            (BUCKET, "b"),
            ("AWS_ACCESS_KEY_ID", "ak"),
            ("AWS_SECRET_ACCESS_KEY", "as"),
            ("AWS_SESSION_TOKEN", "t"),
        ]))
        .unwrap()
        .unwrap();
        assert_eq!(
            (s.keys.id.as_str(), s.keys.token.as_deref()),
            ("ak", Some("t"))
        );
        let bad: &[&[(&str, &str)]] = &[
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, "s"),
                (PATH_STYLE, "yes"),
            ],
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, "s"),
                (UPLOAD_TIMEOUT, "soon"),
            ],
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, "s"),
                (REQUEST_TIMEOUT, "0"),
            ],
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, "s"),
                (ENDPOINT, "ftp://store.example"),
            ],
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, "s"),
                (ENDPOINT, "http://store.example/?a=1"),
            ],
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, "s"),
                (ENDPOINT, "http://store.example/#f"),
            ],
            &[(BUCKET, "b"), (KEY_ID, "k")],
            &[
                (BUCKET, "b"),
                (KEY_ID, "k"),
                (SECRET, ""),
                ("AWS_ACCESS_KEY_ID", "ak"),
                ("AWS_SECRET_ACCESS_KEY", "as"),
            ],
            &[(BUCKET, "b")],
            &[(BUCKET, "b"), ("AWS_ACCESS_KEY_ID", "ak")],
            &[(PATH_STYLE, "true")],
            &[(PATH_STYLE, "yes")],
            &[(KEY_ID, "k"), (SECRET, "s")],
        ];
        for b in bad {
            assert!(Settings::parse(vars(b)).is_err(), "{b:?}");
        }
    }

    /// AWS's example of a signed `GET` of an object (Signature Version 4,
    /// "Examples: Signature Calculations", Amazon S3).
    #[test]
    fn signs_as_aws_does() {
        let keys = Keys {
            id: "AKIAIOSFODNN7EXAMPLE".into(),
            secret: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".into(),
            token: None,
        };
        let empty = hex::encode(Sha256::digest(b""));
        let got = authorization(
            &keys,
            "us-east-1",
            "GET",
            "/test.txt",
            "",
            &[
                ("host", "examplebucket.s3.amazonaws.com"),
                ("range", "bytes=0-9"),
                ("x-amz-content-sha256", &empty),
                ("x-amz-date", "20130524T000000Z"),
            ],
            &empty,
            "20130524T000000Z",
        );
        assert_eq!(
            got,
            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
        );
        // And its `PUT`, whose key needs encoding, with a body.
        let payload = hex::encode(Sha256::digest(b"Welcome to Amazon S3."));
        let got = authorization(
            &keys,
            "us-east-1",
            "PUT",
            &format!("/{}", uri_encode("test$file.text")),
            "",
            &[
                ("date", "Fri, 24 May 2013 00:00:00 GMT"),
                ("host", "examplebucket.s3.amazonaws.com"),
                ("x-amz-content-sha256", &payload),
                ("x-amz-date", "20130524T000000Z"),
                ("x-amz-storage-class", "REDUCED_REDUNDANCY"),
            ],
            &payload,
            "20130524T000000Z",
        );
        assert!(got.ends_with(
            "Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
        ));
        assert_eq!(amz_date(1_369_353_600_000), "20130524T000000Z");
        assert_eq!(amz_date(1_369_353_600_000 + 86_399_999), "20130524T235959Z");
    }

    /// Path-style and virtual-hosted URLs, each segment encoded once as
    /// SigV4 has it, the endpoint's path kept.
    #[test]
    fn objects_are_addressed_as_the_spec_says() {
        let mut s = Settings::parse(vars(&[
            (BUCKET, "blobs"),
            (KEY_ID, "k"),
            (SECRET, "s"),
            (ENDPOINT, "http://127.0.0.1:9000/base/"),
            (PATH_STYLE, "true"),
        ]))
        .unwrap()
        .unwrap();
        let b = Bucket::new(s.clone()).unwrap();
        let (url, host, path) = b.locate("did:web:localhost%3A2583", "bafkrei").unwrap();
        assert_eq!(host, "127.0.0.1:9000");
        assert_eq!(
            path,
            "/base/blobs/blocks/did%3Aweb%3Alocalhost%253A2583/bafkrei"
        );
        assert_eq!(url.as_str(), format!("http://127.0.0.1:9000{path}"));
        s.path_style = false;
        s.endpoint = url::Url::parse("https://store.example").unwrap();
        let b = Bucket::new(s).unwrap();
        let (url, host, path) = b.locate("did:plc:abc", "bafkrei").unwrap();
        assert_eq!(host, "blobs.store.example");
        assert_eq!(path, "/blocks/did%3Aplc%3Aabc/bafkrei");
        assert_eq!(
            url.as_str(),
            "https://blobs.store.example/blocks/did%3Aplc%3Aabc/bafkrei"
        );
    }

    /// A directory keeps its blobs where its first start put them.
    #[test]
    fn a_data_directory_keeps_its_blobs_where_they_are() {
        let dir = crate::testing::scratch("blobstore-mark");
        // A start in itself on a directory holding nothing leaves no mark;
        // a bucket's then may.
        check_dir(&dir, false).unwrap();
        assert!(!dir.join(MARK).exists());
        check_dir(&dir, true).unwrap();
        check_dir(&dir, true).unwrap();
        assert!(check_dir(&dir, false).is_err());
        // One whose first start kept them in itself.
        let other = crate::testing::scratch("blobstore-unmarked");
        std::fs::create_dir_all(&other).unwrap();
        std::fs::write(crate::store::service_layout(&other).log(0), b"").unwrap();
        check_dir(&other, false).unwrap();
        assert!(check_dir(&other, true).is_err());
        assert!(!other.join(MARK).exists());
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&other);
    }

    /// Each upload's object is put before it is answered, and no file of it
    /// kept; a read the store fails, or of an object gone, is `500`; an
    /// upload the store fails holds nothing, and its object, if any, is
    /// deleted; a blob no longer held has its object deleted once the store
    /// answers, the account marked for a start meanwhile, but not one held
    /// again by then. Every request is signed.
    #[tokio::test]
    async fn objects_follow_what_is_held() {
        let plc = plc().await;
        let fake = Arc::new(Fake::default());
        let url = fake_store(fake.clone()).await;
        let (dir, sink) = (scratch("bucket-held"), scratch("bucket-held-mail"));
        let s = open_bucket(&dir, &plc, &sink, &url, true);
        let bucket = s.ctx.bucket.clone().unwrap();
        let (did, token) = create(&s, "alice").await;
        let t = Some(token.as_str());
        let key = |b: &[u8]| format!("blocks/{did}/{}", raw_cid(b));
        let (a, b, c) = (body(1), body(2), body(3));
        test_upload(&s, t, Some("text/plain"), a.clone(), false)
            .await
            .unwrap();
        assert!(fake.has(&key(&a)));
        assert!(!crate::blobs::blob_dir(&dir, &did).exists());
        put(&s, &token, &did, "a", &a).await.unwrap();
        assert_eq!(get(&s, &did, &a).await, (200, a.clone()));
        fake.fail(Some((Method::GET, 503)));
        assert_eq!(get(&s, &did, &a).await.0, 500);
        fake.fail(Some((Method::GET, 0)));
        assert_eq!(get(&s, &did, &a).await.0, 500);
        fake.fail(None);
        let saved = lock(&fake.objects).remove(&key(&a)).unwrap();
        assert_eq!(get(&s, &did, &a).await.0, 500);
        lock(&fake.objects).insert(key(&a), saved);
        assert_eq!(get(&s, &did, &a).await, (200, a.clone()));
        // An upload the store fails: 500 for a status, 504 for none.
        fake.fail(Some((Method::PUT, 500)));
        assert_eq!(
            test_upload(&s, t, Some("text/plain"), b.clone(), false).await,
            Err((500, "InternalServerError".into()))
        );
        fake.fail(Some((Method::PUT, 0)));
        assert_eq!(
            test_upload(&s, t, Some("text/plain"), b.clone(), true).await,
            Err((504, "UpstreamTimeout".into()))
        );
        fake.fail(None);
        assert_eq!(
            put(&s, &token, &did, "b", &b).await,
            Err("BlobNotFound".into())
        );
        // As if the store kept it after all: the next pass deletes it, and
        // the mark taken for it goes.
        lock(&fake.objects).insert(key(&b), b.clone());
        assert!(s.ctx.eager.is_marked(&did));
        assert!(pass(&s, &bucket).await);
        assert!(!fake.has(&key(&b)));
        assert!(!s.ctx.eager.is_marked(&did));
        // A blob no longer held, the store failing deletes: kept, the
        // account marked, until the store answers.
        fake.fail(Some((Method::DELETE, 500)));
        unput(&s, &token, &did, "a").await;
        assert!(s.ctx.eager.is_marked(&did));
        assert!(!pass(&s, &bucket).await);
        assert!(fake.has(&key(&a)));
        fake.fail(None);
        assert!(pass(&s, &bucket).await);
        assert!(!fake.has(&key(&a)));
        assert!(!s.ctx.eager.is_marked(&did));
        // No longer held, and uploaded again at once: kept.
        test_upload(&s, t, Some("text/plain"), c.clone(), false)
            .await
            .unwrap();
        put(&s, &token, &did, "c", &c).await.unwrap();
        unput(&s, &token, &did, "c").await;
        test_upload(&s, t, Some("text/plain"), c.clone(), false)
            .await
            .unwrap();
        assert!(pass(&s, &bucket).await);
        put(&s, &token, &did, "c", &c).await.unwrap();
        assert_eq!(get(&s, &did, &c).await, (200, c.clone()));
        assert_eq!(
            lock(&fake.objects).keys().cloned().collect::<Vec<_>>(),
            vec![key(&c)]
        );
        assert!(lock(&fake.seen).iter().all(|(_, _, ok)| *ok));
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
    }

    /// Deletes not yet answered when the server stops are made after it
    /// starts again, from its log or its checkpoint: a stray's, its account
    /// loaded at the start for its mark alone; and a deleted account's, from
    /// the service log. Once answered, they are made no more.
    #[tokio::test]
    async fn deletes_owed_survive_a_restart() {
        let plc = plc().await;
        let fake = Arc::new(Fake::default());
        let url = fake_store(fake.clone()).await;
        let (dir, sink) = (scratch("bucket-restart"), scratch("bucket-restart-mail"));
        let open = |load_all| open_bucket(&dir, &plc, &sink, &url, load_all);
        let s = open(true);
        let (alice, token) = create(&s, "alice").await;
        let (bob, bob_token) = create(&s, "bob").await;
        let (x, y) = (body(10), body(11));
        let (kx, ky) = (
            format!("blocks/{alice}/{}", raw_cid(&x)),
            format!("blocks/{bob}/{}", raw_cid(&y)),
        );
        test_upload(&s, Some(&token), Some("text/plain"), x.clone(), false)
            .await
            .unwrap();
        put(&s, &token, &alice, "x", &x).await.unwrap();
        test_upload(&s, Some(&bob_token), Some("text/plain"), y.clone(), false)
            .await
            .unwrap();
        put(&s, &bob_token, &bob, "y", &y).await.unwrap();
        let bucket = s.ctx.bucket.clone().unwrap();
        assert!(pass(&s, &bucket).await);
        fake.fail(Some((Method::DELETE, 503)));
        unput(&s, &token, &alice, "x").await;
        call(
            &s,
            "com.atproto.server.requestAccountDelete",
            Some(&bob_token),
            Json::Null,
        )
        .await
        .unwrap();
        let mailed = crate::email::testing::last_token(&sink);
        call(
            &s,
            "com.atproto.server.deleteAccount",
            None,
            json!({ "did": bob, "password": "bob-pw", "token": mailed }),
        )
        .await
        .unwrap();
        assert!(!pass(&s, &bucket).await);
        assert!(fake.has(&kx) && fake.has(&ky));
        // Both logs compacted: what is owed is in their checkpoints.
        let wal = read(&s.account(&alice).unwrap().store).wal.clone();
        wal.wait_compacted();
        wal.compact().unwrap();
        s.log.wait_compacted();
        s.log.compact().unwrap();
        drop((s, bucket));
        fake.fail(None);
        // A start loads alice's store for her mark alone.
        let s = open(false);
        assert!(s.account(&alice).unwrap().store.is_loaded());
        let bucket = s.ctx.bucket.clone().unwrap();
        assert!(pass(&s, &bucket).await);
        assert!(!fake.has(&kx) && !fake.has(&ky));
        assert!(!s.ctx.eager.is_marked(&alice));
        drop((s, bucket));
        let deletes = |f: &Fake| {
            lock(&f.seen)
                .iter()
                .filter(|(m, _, _)| *m == Method::DELETE)
                .count()
        };
        let made = deletes(&fake);
        let s = open(false);
        assert!(!s.account(&alice).unwrap().store.is_loaded());
        let bucket = s.ctx.bucket.clone().unwrap();
        assert!(pass(&s, &bucket).await);
        assert_eq!(deletes(&fake), made);
        assert_eq!(
            read(&s.account(&alice).unwrap().store)
                .blobs
                .strays()
                .count(),
            0
        );
        assert!(lock(&fake.seen).iter().all(|(_, _, ok)| *ok));
        drop((s, bucket));
        let _ = std::fs::remove_dir_all(&dir);
        let _ = std::fs::remove_dir_all(&sink);
    }
}