~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/hud

A remote MCP server that shows atproto records as interactive UI inside MCP Apps hosts

0
0
916 lines · 36.3 KB Raw History
//! The blob proxy, `GET /blob/{did}/{cid}`: every track and video the view
//! plays is loaded through it, and every image but a public record's, which
//! the image CDN serves (`HUD_IMAGE_CDN`, hud-core's `blobs`) with the proxy
//! as its fallback. It resolves the DID's PDS and fetches
//! `com.atproto.sync.getBlob`, serving images (not SVG, 20 MB at most),
//! audio (100 MB at most) and MP4, QuickTime and WebM video (100 MB at
//! most) and nothing else, a blob the PDS sends untyped known by its first
//! bytes: the PDS comes from a DID document anyone can
//! write, so its URL goes through the outbound client's address checks,
//! and it has ten seconds to answer. A blob never changes, so it is cached
//! for good.
//!
//! Audio and video are kept on disk (hud-core's `blobcache`, which keeps
//! only bytes that hash to the CID), because a player seeks with `Range`
//! and PDSes answer only whole blobs. Each is downloaded once however many
//! ask for it, and served as it arrives, so a player starts on its first
//! bytes: a single range is answered `206`, an unsatisfiable one `416`, and
//! a plain request `200`, against the length its PDS said (one that said
//! none is served once it is all here). A blob already kept is served
//! without asking its PDS again: its bytes are what its CID names, whoever
//! asks.

use axum::body::Body;
use axum::extract::{Path, State};
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use futures::{StreamExt, TryStreamExt};
use hud_core::Hud;
use hud_core::blobcache::{Cached, Filling, PutError, READ_CHUNK};
use hud_core::blobs::{IMAGE_MAX, SNIFF_LEN, is_kept, max_size, sniff};
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio_util::io::ReaderStream;

/// How long the PDS has to answer.
const ANSWER_WAIT: Duration = Duration::from_secs(10);
/// How long the PDS has to send all of a blob.
const BODY_WAIT: Duration = Duration::from_secs(120);
// The caps, `IMAGE_MAX`, `AUDIO_MAX` and `VIDEO_MAX`, are hud-core's, which
// gives no proxy URL to a blob that declares itself past them.

/// A refusal: its status, and a line of plain text saying why, which the
/// request's line says too (`telemetry::Cause`).
fn refuse(status: StatusCode, why: &str) -> Response {
    let mut res = (
        status,
        [
            (header::CONTENT_TYPE, "text/plain; charset=utf-8"),
            (header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
        ],
        format!("{why}\n"),
    )
        .into_response();
    res.extensions_mut()
        .insert(crate::telemetry::Cause(why.to_string()));
    res
}

/// The headers every blob is served with: cached for good, never sniffed
/// or run, loadable from any origin.
fn blob_headers(h: &mut HeaderMap, content_type: &str) {
    if let Ok(v) = HeaderValue::from_str(content_type) {
        h.insert(header::CONTENT_TYPE, v);
    }
    for (name, value) in [
        (header::CACHE_CONTROL, "public, max-age=31536000, immutable"),
        (header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
        (header::ACCESS_CONTROL_ALLOW_ORIGIN, "*"),
        (
            header::HeaderName::from_static("cross-origin-resource-policy"),
            "cross-origin",
        ),
        (
            header::CONTENT_SECURITY_POLICY,
            "default-src 'none'; sandbox",
        ),
    ] {
        h.insert(name, HeaderValue::from_static(value));
    }
}

/// What a `Range` header asks of a blob of a given length.
#[derive(Debug, PartialEq, Eq)]
enum Range {
    /// All of it: no header, or one this server does not take (another
    /// unit, several ranges, a malformed one), which is ignored.
    Whole,
    /// The bytes from the first to the second, both included.
    Part(u64, u64),
    /// A range of bytes the blob does not have.
    Unsatisfiable,
}

/// A `Range` header's single byte range, against a blob of `len` bytes.
fn parse_range(value: &str, len: u64) -> Range {
    let Some(spec) = value.trim().strip_prefix("bytes=") else {
        return Range::Whole;
    };
    let Some((first, last)) = spec.split_once('-').filter(|_| !spec.contains(',')) else {
        return Range::Whole;
    };
    let number = |s: &str| {
        (!s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()))
            .then(|| s.parse::<u64>().ok())
            .flatten()
    };
    match (first.trim(), last.trim()) {
        ("", suffix) => match number(suffix) {
            None => Range::Whole,
            Some(0) => Range::Unsatisfiable,
            Some(_) if len == 0 => Range::Unsatisfiable,
            Some(n) => Range::Part(len.saturating_sub(n), len - 1),
        },
        (from, to) => {
            let Some(from) = number(from) else {
                return Range::Whole;
            };
            let to = match to {
                "" => None,
                to => match number(to) {
                    Some(to) if to >= from => Some(to),
                    _ => return Range::Whole,
                },
            };
            if from >= len {
                return Range::Unsatisfiable;
            }
            Range::Part(from, to.map_or(len - 1, |to| to.min(len - 1)))
        }
    }
}

/// A blob of `len` bytes, as a request's `Range` asks for it: `count` of
/// its bytes from `start`, as `read` gives them.
fn ranged<S>(
    len: u64,
    mime: &str,
    range: Option<&HeaderValue>,
    read: impl FnOnce(u64, u64) -> S,
) -> Response
where
    S: futures::Stream<Item = std::io::Result<bytes::Bytes>> + Send + 'static,
{
    let range = range
        .and_then(|r| r.to_str().ok())
        .map_or(Range::Whole, |r| parse_range(r, len));
    let (status, start, count) = match range {
        Range::Whole => (StatusCode::OK, 0, len),
        Range::Part(from, to) => (StatusCode::PARTIAL_CONTENT, from, to - from + 1),
        Range::Unsatisfiable => {
            let mut r = refuse(StatusCode::RANGE_NOT_SATISFIABLE, "No such range");
            let all = HeaderValue::from_str(&format!("bytes */{len}"))
                .expect("a length is a header's text");
            r.headers_mut().insert(header::CONTENT_RANGE, all);
            return r;
        }
    };
    let mut r = Response::new(Body::from_stream(read(start, count)));
    *r.status_mut() = status;
    let h = r.headers_mut();
    blob_headers(h, mime);
    h.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes"));
    h.insert(header::CONTENT_LENGTH, HeaderValue::from(count));
    if status == StatusCode::PARTIAL_CONTENT
        && let Ok(v) = HeaderValue::from_str(&format!("bytes {start}-{}/{len}", start + count - 1))
    {
        h.insert(header::CONTENT_RANGE, v);
    }
    r
}

/// A kept blob, as a request's `Range` asks for it.
fn serve_cached(c: Cached, range: Option<&HeaderValue>) -> Response {
    let Cached {
        mut file,
        len,
        mime,
    } = c;
    // Read from `start`: a kept blob is a file, so seeking it fails only as
    // reading it would, which ends the body.
    ranged(len, &mime, range, |start, count| {
        futures::stream::once(async move {
            file.seek(std::io::SeekFrom::Start(start)).await?;
            Ok::<_, std::io::Error>(ReaderStream::with_capacity(file.take(count), READ_CHUNK))
        })
        .try_flatten()
    })
}

/// A blob being downloaded, as a request's `Range` asks for it, each byte
/// as it arrives: against the length its PDS said, or where it said none,
/// once it is all here.
async fn serve_filling(f: Arc<Filling>, range: Option<&HeaderValue>) -> Response {
    let len = match f.said {
        Some(len) => len,
        None => match f.kept().await {
            Ok(len) => len,
            Err(e) => return unkept(&e),
        },
    };
    ranged(len, &f.mime, range, |start, count| f.read(start, count))
}

/// Why a blob was not kept, answered before any of it was sent.
fn unkept(e: &PutError) -> Response {
    match e {
        PutError::TooLarge => refuse(StatusCode::PAYLOAD_TOO_LARGE, "The blob is too large"),
        PutError::Disk(e) => {
            tracing::warn!(error = %e, "could not keep a blob");
            refuse(
                StatusCode::INTERNAL_SERVER_ERROR,
                "The blob could not be kept",
            )
        }
        e => refuse(StatusCode::BAD_GATEWAY, &e.to_string()),
    }
}

pub async fn blob(
    State(hud): State<Arc<Hud>>,
    Path((did, cid_text)): Path<(String, String)>,
    headers: HeaderMap,
) -> Response {
    if !atproto::syntax::is_did(&did) {
        return refuse(StatusCode::BAD_REQUEST, "Not a DID");
    }
    let Ok(cid) = atproto::Cid::parse(&cid_text) else {
        return refuse(StatusCode::BAD_REQUEST, "Not a CID");
    };
    let range = headers.get(header::RANGE);
    if let Some(kept) = hud.blob_cache.get(&cid).await {
        return serve_cached(kept, range);
    }
    if let Some(filling) = hud.blob_cache.filling(&did, &cid) {
        return serve_filling(filling, range).await;
    }
    let pds = match hud.pds(&did).await {
        Ok(pds) => pds,
        Err(why) => return refuse(StatusCode::BAD_GATEWAY, &why),
    };
    // The PDS's URL is one `hud.pds` checked, so building and sending the
    // request fail only as sending it does.
    let asked = async {
        let mut url = url::Url::parse(&format!("{pds}/xrpc/com.atproto.sync.getBlob"))?;
        url.query_pairs_mut()
            .append_pair("did", &did)
            .append_pair("cid", &cid_text);
        let req = hud.net.get(&url)?.timeout(BODY_WAIT);
        let res = hud.net.execute(req).await;
        let res = res.map_err(|e| hud_core::net::describe(&e))?;
        Ok::<_, Box<dyn std::error::Error + Send + Sync>>(res)
    };
    let res = match tokio::time::timeout(ANSWER_WAIT, asked).await {
        Ok(Ok(r)) => r,
        Ok(Err(why)) => return refuse(StatusCode::BAD_GATEWAY, &format!("{pds}: {why}")),
        Err(_) => {
            return refuse(
                StatusCode::BAD_GATEWAY,
                &format!("{pds}: it did not answer in time"),
            );
        }
    };
    match res.status().as_u16() {
        200..=299 => {}
        404 => return refuse(StatusCode::NOT_FOUND, "No such blob"),
        s => return refuse(StatusCode::BAD_GATEWAY, &format!("{pds} answered {s}")),
    }
    let declared = res
        .headers()
        .get(header::CONTENT_TYPE)
        .and_then(|v| v.to_str().ok())
        .unwrap_or("")
        .to_string();
    let length = res.content_length();
    let mut stream = res.bytes_stream().boxed();
    // A PDS that types every blob `application/octet-stream` (or not at
    // all): the blob's first bytes say what it is, if it is a kind served.
    let (content_type, head, max) = match max_size(&declared) {
        Some(max) => (declared, Vec::new(), max),
        None => {
            let mut head = Vec::new();
            while head.len() < SNIFF_LEN {
                match stream.next().await {
                    Some(Ok(chunk)) => head.extend_from_slice(&chunk),
                    Some(Err(e)) => return refuse(StatusCode::BAD_GATEWAY, &format!("{pds}: {e}")),
                    None => break,
                }
            }
            // What `sniff` knows is a kind served, so it has a cap.
            match sniff(&head).and_then(|t| Some((t.to_string(), max_size(t)?))) {
                Some((sniffed, max)) => (sniffed, head, max),
                None => {
                    return refuse(
                        StatusCode::UNSUPPORTED_MEDIA_TYPE,
                        "Not an image, audio or video",
                    );
                }
            }
        }
    };
    // What was read to sniff it goes first, then the rest as it comes.
    let stream = futures::stream::iter((!head.is_empty()).then(|| Ok(bytes::Bytes::from(head))))
        .chain(stream)
        .boxed();
    if length.is_some_and(|n| n > max) {
        return refuse(StatusCode::PAYLOAD_TOO_LARGE, "The blob is too large");
    }
    if is_kept(&content_type) {
        return match hud
            .blob_cache
            .fill(&did, &cid, &content_type, length, stream, max)
            .await
        {
            Ok(filling) => serve_filling(filling, range).await,
            Err(e) => unkept(&PutError::Disk(e)),
        };
    }
    // An image streams on, cut off past the cap whatever the PDS said its
    // length was.
    let mut sent = 0u64;
    let body = stream.map(move |chunk| {
        let chunk = chunk.map_err(std::io::Error::other)?;
        sent += chunk.len() as u64;
        if sent > IMAGE_MAX {
            return Err(std::io::Error::other("the blob is too large"));
        }
        Ok::<bytes::Bytes, std::io::Error>(chunk)
    });
    let mut r = Response::new(Body::from_stream(body));
    blob_headers(r.headers_mut(), &content_type);
    r
}

#[cfg(test)]
mod tests {
    use super::*;
    use axum::Router;
    use axum::extract::Query;
    use hud_core::Config;
    use hud_core::blobs::AUDIO_MAX;
    use std::collections::HashMap;
    use std::sync::Mutex;
    use tower::ServiceExt;

    /// What the fake PDS answers for a blob.
    #[derive(Clone)]
    enum Held {
        /// Its type (none sent if empty) and its bytes, its length said.
        Whole(&'static str, bytes::Bytes),
        /// Its type and its bytes as chunks, its length unsaid.
        Chunks(&'static str, Vec<bytes::Bytes>),
        /// An answer of this status alone.
        Status(u16),
        /// No answer, ever.
        Never,
        /// Its type and its bytes, its length said: the first half sent at
        /// once, the rest once the gate opens.
        Gated(&'static str, bytes::Bytes, Arc<tokio::sync::Notify>),
    }

    /// What the fake PDSes hold, by CID: `held` at did:plc:alice's, and how
    /// often it was asked for a blob; at did:plc:short's, each blob's type
    /// and the bytes it sends of it, a thousand fewer than it says, before
    /// it hangs up.
    #[derive(Default)]
    struct Shelves {
        held: HashMap<String, Held>,
        asked: usize,
        short: HashMap<String, (&'static str, bytes::Bytes)>,
    }

    type Fake = Arc<Mutex<Shelves>>;

    /// The PDSes holding `fake`, and PLC naming them, and a closed port
    /// for did:plc:closed: its origin.
    async fn pds(fake: Fake) -> String {
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let origin = format!("http://{}", listener.local_addr().unwrap());
        let short = short_pds(fake.clone()).await;
        let me = origin.clone();
        let plc = move |Path(did): Path<String>| {
            let (me, short) = (me.clone(), short.clone());
            async move {
                let endpoint = match did.as_str() {
                    "did:plc:alice" => me,
                    "did:plc:short" => short,
                    "did:plc:closed" => "http://127.0.0.1:1".to_string(),
                    _ => return StatusCode::NOT_FOUND.into_response(),
                };
                axum::Json(serde_json::json!({ "id": did, "alsoKnownAs": [],
                    "verificationMethod": [], "service": [{ "id": "#atproto_pds",
                    "type": "AtprotoPersonalDataServer", "serviceEndpoint": endpoint }] }))
                .into_response()
            }
        };
        let get_blob = move |Query(q): Query<HashMap<String, String>>| {
            let found = {
                let mut shelves = fake.lock().unwrap();
                shelves.asked += 1;
                shelves.held.get(&q["cid"]).cloned()
            };
            async move {
                let typed = |t: &'static str| {
                    let mut h = HeaderMap::new();
                    if !t.is_empty() {
                        h.insert(header::CONTENT_TYPE, HeaderValue::from_static(t));
                    }
                    h
                };
                match found {
                    Some(Held::Whole(t, b)) => (typed(t), b).into_response(),
                    Some(Held::Chunks(t, chunks)) => {
                        let body = futures::stream::iter(chunks.into_iter().map(Ok::<_, String>));
                        (typed(t), Body::from_stream(body)).into_response()
                    }
                    Some(Held::Status(s)) => StatusCode::from_u16(s).unwrap().into_response(),
                    Some(Held::Never) => std::future::pending::<Response>().await,
                    Some(Held::Gated(t, b, gate)) => {
                        let mut h = typed(t);
                        h.insert(header::CONTENT_LENGTH, HeaderValue::from(b.len()));
                        let (first, rest) = (b.slice(..b.len() / 2), b.slice(b.len() / 2..));
                        let body = futures::stream::iter([Ok::<_, String>(first)]).chain(
                            futures::stream::once(async move {
                                gate.notified().await;
                                Ok(rest)
                            }),
                        );
                        (h, Body::from_stream(body)).into_response()
                    }
                    None => StatusCode::NOT_FOUND.into_response(),
                }
            }
        };
        let app = Router::new()
            .route(
                "/xrpc/com.atproto.sync.getBlob",
                axum::routing::get(get_blob),
            )
            .route("/{did}", axum::routing::get(plc));
        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        origin
    }

    /// A PDS speaking HTTP by hand: a blob's head and bytes sent, then the
    /// connection closed short of the length it said. Its origin.
    async fn short_pds(fake: Fake) -> String {
        use tokio::io::{AsyncReadExt, AsyncWriteExt};
        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let origin = format!("http://{}", listener.local_addr().unwrap());
        tokio::spawn(async move {
            loop {
                let (mut conn, _) = listener.accept().await.unwrap();
                let fake = fake.clone();
                tokio::spawn(async move {
                    let (mut asked, mut buf) = (Vec::new(), [0; 1024]);
                    while !asked.windows(4).any(|w| w == b"\r\n\r\n") {
                        let n = conn.read(&mut buf).await.unwrap();
                        assert!(n > 0, "the proxy hung up");
                        asked.extend_from_slice(&buf[..n]);
                    }
                    let asked = String::from_utf8_lossy(&asked).to_string();
                    let cid = asked.split("cid=").nth(1).unwrap().split([' ', '&']).next();
                    let (kind, bytes) = fake.lock().unwrap().short[cid.unwrap()].clone();
                    let head = format!(
                        "HTTP/1.1 200 OK\r\ncontent-type: {kind}\r\ncontent-length: {}\r\n\r\n",
                        bytes.len() + 1000
                    );
                    conn.write_all(head.as_bytes()).await.unwrap();
                    conn.write_all(&bytes).await.unwrap();
                });
            }
        });
        origin
    }

    /// A server whose PLC is the fake one, its blobs cached in a directory
    /// of its own; and what the PDSes hold.
    async fn proxy() -> (Arc<Hud>, Fake) {
        let fake = Fake::default();
        let origin = pds(fake.clone()).await;
        let dir = std::env::temp_dir().join(format!("hud-blob-{}", atproto::jose::random(6)));
        let hud = Hud::new(Config {
            port: 0,
            bind: "::".parse().unwrap(),
            public_origin: "https://hud.example".into(),
            cache_dir: dir.join("cache"),
            data_dir: dir,
            oauth: false,
            oauth_scope: hud_core::scope::asked(hud_core::view::views(), true, false),
            oauth_include: None,
            workspace: true,
            sign_up_pds: hud_core::config::SIGN_UP_PDS.into(),
            client: Default::default(),
            spaces: false,
            plc_url: origin,
            handle_resolver: None,
            allow_private: true,
            blob_cache_bytes: 1 << 20,
            cache_memory_bytes: 1 << 24,
            backlinks: hud_core::backlinks::Backlinks::None,
            favicons: None,
            image_cdn: None,
            relay: None,
            operators: Vec::new(),
            logs: false,
            logs_keep: hud_core::logs::KEEP,
        })
        .unwrap();
        (hud, fake)
    }

    /// `bytes`' CID, as the proxy is asked for it.
    fn cid(bytes: &[u8]) -> String {
        atproto::Cid::of_raw(bytes).to_string()
    }

    /// A blob asked of the proxy: its status, headers and body, or the
    /// error its body ended with.
    async fn fetch(
        hud: &Arc<Hud>,
        path: &str,
        range: Option<&str>,
    ) -> (StatusCode, HeaderMap, Result<Vec<u8>, String>) {
        let mut req = axum::http::Request::get(path);
        if let Some(r) = range {
            req = req.header(header::RANGE, r);
        }
        let r = crate::router(hud.clone(), None)
            .oneshot(req.body(Body::empty()).unwrap())
            .await
            .unwrap();
        let (status, headers) = (r.status(), r.headers().clone());
        let body = axum::body::to_bytes(r.into_body(), usize::MAX)
            .await
            .map(|b| b.to_vec())
            .map_err(|e| e.to_string());
        (status, headers, body)
    }

    /// A refusal's status and line.
    async fn refused(hud: &Arc<Hud>, path: &str) -> (StatusCode, String) {
        let (status, h, body) = fetch(hud, path, None).await;
        assert_eq!(
            h[header::CONTENT_TYPE],
            "text/plain; charset=utf-8",
            "{path}"
        );
        (status, String::from_utf8(body.unwrap()).unwrap())
    }

    #[tokio::test]
    async fn refusals() {
        let (hud, held) = proxy().await;
        let html = cid(b"<html>");
        let gone = cid(b"gone");
        let broken = cid(b"broken");
        held.lock().unwrap().held.extend([
            (
                html.clone(),
                Held::Whole("application/octet-stream", "<html><body>".into()),
            ),
            (broken.clone(), Held::Status(500)),
        ]);
        let line = |s: &str| format!("{s}\n");
        assert_eq!(
            refused(&hud, &format!("/blob/alice/{html}")).await,
            (StatusCode::BAD_REQUEST, line("Not a DID"))
        );
        assert_eq!(
            refused(&hud, "/blob/did:plc:alice/nope").await,
            (StatusCode::BAD_REQUEST, line("Not a CID"))
        );
        // A DID with no document, and a PDS that is not there.
        let (status, why) = refused(&hud, &format!("/blob/did:plc:nobody/{gone}")).await;
        assert_eq!(status, StatusCode::BAD_GATEWAY);
        assert!(why.contains("did:plc:nobody"), "{why}");
        let (status, why) = refused(&hud, &format!("/blob/did:plc:closed/{gone}")).await;
        assert_eq!(status, StatusCode::BAD_GATEWAY);
        assert!(why.starts_with("http://127.0.0.1:1: "), "{why}");
        // What the PDS has not, what it fails, and what is not a kind served.
        assert_eq!(
            refused(&hud, &format!("/blob/did:plc:alice/{gone}")).await,
            (StatusCode::NOT_FOUND, line("No such blob"))
        );
        let (status, why) = refused(&hud, &format!("/blob/did:plc:alice/{broken}")).await;
        assert_eq!(status, StatusCode::BAD_GATEWAY);
        assert!(why.ends_with(" answered 500\n"), "{why}");
        assert_eq!(
            refused(&hud, &format!("/blob/did:plc:alice/{html}")).await,
            (
                StatusCode::UNSUPPORTED_MEDIA_TYPE,
                line("Not an image, audio or video")
            )
        );
    }

    #[tokio::test]
    async fn images_stream_through() {
        let (hud, held) = proxy().await;
        let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7; 100]].concat();
        let gif = b"GIF89a".to_vec();
        let (typed, sniffed, short, cut, big, endless, early) = (
            cid(b"typed"),
            cid(b"sniffed"),
            cid(b"short"),
            cid(b"cut"),
            cid(b"big"),
            cid(b"endless"),
            cid(b"early"),
        );
        let mib = bytes::Bytes::from(vec![0u8; 1 << 20]);
        held.lock().unwrap().held.extend([
            (typed.clone(), Held::Whole("image/png", png.clone().into())),
            // Untyped, in chunks shorter than what sniffing needs.
            (
                sniffed.clone(),
                Held::Chunks(
                    "",
                    png.chunks(5).map(bytes::Bytes::copy_from_slice).collect(),
                ),
            ),
            // Untyped, and shorter than that whole.
            (
                short.clone(),
                Held::Whole("application/octet-stream", gif.clone().into()),
            ),
            (
                big.clone(),
                Held::Whole(
                    "image/jpeg",
                    vec![0; usize::try_from(IMAGE_MAX).unwrap() + 1].into(),
                ),
            ),
            (endless.clone(), Held::Chunks("image/jpeg", vec![mib; 21])),
        ]);
        held.lock().unwrap().short.extend([
            (cut.clone(), ("image/png", png.clone().into())),
            (
                early.clone(),
                ("application/octet-stream", png[..4].to_vec().into()),
            ),
        ]);
        let at = |c: &str| format!("/blob/did:plc:alice/{c}");
        for (c, kind, bytes) in [
            (&typed, "image/png", &png),
            (&sniffed, "image/png", &png),
            (&short, "image/gif", &gif),
        ] {
            let (status, h, body) = fetch(&hud, &at(c), None).await;
            assert_eq!(status, StatusCode::OK, "{kind}");
            assert_eq!(h[header::CONTENT_TYPE], kind);
            assert_eq!(
                h[header::CACHE_CONTROL],
                "public, max-age=31536000, immutable"
            );
            assert_eq!(
                h[header::CONTENT_SECURITY_POLICY],
                "default-src 'none'; sandbox"
            );
            assert_eq!(&body.unwrap(), bytes);
        }
        // An image never kept: asked again, it is asked of the PDS again.
        held.lock().unwrap().held.remove(&typed);
        assert_eq!(
            fetch(&hud, &at(&typed), None).await.0,
            StatusCode::NOT_FOUND
        );
        // Said to be past the cap: refused before a byte is sent.
        assert_eq!(
            refused(&hud, &at(&big)).await,
            (
                StatusCode::PAYLOAD_TOO_LARGE,
                "The blob is too large\n".into()
            )
        );
        // Past it unsaid, or cut off by the PDS: the body ends in an error.
        let (status, _, body) = fetch(&hud, &at(&endless), None).await;
        assert_eq!(status, StatusCode::OK);
        assert!(body.is_err());
        let cut_off = |c: &str| format!("/blob/did:plc:short/{c}");
        let (status, h, body) = fetch(&hud, &cut_off(&cut), None).await;
        assert_eq!(
            (status, &h[header::CONTENT_TYPE]),
            (StatusCode::OK, &HeaderValue::from_static("image/png"))
        );
        assert!(body.is_err());
        // Cut off while it was being sniffed.
        let (status, why) = refused(&hud, &cut_off(&early)).await;
        assert_eq!(status, StatusCode::BAD_GATEWAY);
        assert!(why.starts_with("http://127.0.0.1:"), "{why}");
    }

    #[tokio::test]
    async fn audio_kept_then_served_from_the_cache() {
        let (hud, held) = proxy().await;
        let wav = [b"RIFF\x10\0\0\0WAVEfmt ".as_slice(), &[3; 200]].concat();
        let (song, wrong, unsaid, long) = (cid(&wav), cid(b"wrong"), cid(b"unsaid"), cid(b"long"));
        let mib = bytes::Bytes::from(vec![0u8; 1 << 20]);
        let mibs = usize::try_from(AUDIO_MAX >> 20).unwrap() + 1;
        held.lock().unwrap().held.extend([
            (song.clone(), Held::Chunks("", vec![wav.clone().into()])),
            // Bytes that are not what the CID names, their length said and unsaid.
            (wrong.clone(), Held::Whole("audio/wav", wav.clone().into())),
            (
                unsaid.clone(),
                Held::Chunks("audio/wav", vec![wav.clone().into()]),
            ),
            (long.clone(), Held::Chunks("audio/mpeg", vec![mib; mibs])),
        ]);
        let at = |c: &str| format!("/blob/did:plc:alice/{c}");
        // Its length unsaid: kept whole, then the range asked of it.
        let (status, h, body) = fetch(&hud, &at(&song), Some("bytes=4-11")).await;
        assert_eq!(status, StatusCode::PARTIAL_CONTENT);
        assert_eq!(h[header::CONTENT_TYPE], "audio/wav");
        assert_eq!(
            h[header::CONTENT_RANGE],
            format!("bytes 4-11/{}", wav.len())
        );
        assert_eq!(body.unwrap(), wav[4..12]);
        // Kept: served whole though the PDS has it no more.
        held.lock().unwrap().held.remove(&song);
        let (status, h, body) = fetch(&hud, &at(&song), None).await;
        assert_eq!(status, StatusCode::OK);
        assert_eq!(h[header::ACCEPT_RANGES], "bytes");
        assert_eq!(body.unwrap(), wav);
        // Never kept: bytes not its CID's. Their length said, they are
        // sent as they come, and the body ends in an error short of the
        // last; unsaid, nothing is sent.
        let (status, _, body) = fetch(&hud, &at(&wrong), None).await;
        assert_eq!(status, StatusCode::OK);
        assert!(body.is_err());
        let (status, why) = refused(&hud, &at(&unsaid)).await;
        assert_eq!(
            (status, why.as_str()),
            (
                StatusCode::BAD_GATEWAY,
                "the blob's bytes do not match its CID\n"
            )
        );
        // More than an audio blob's cap.
        assert_eq!(
            refused(&hud, &at(&long)).await,
            (
                StatusCode::PAYLOAD_TOO_LARGE,
                "The blob is too large\n".into()
            )
        );
    }

    #[tokio::test]
    async fn video_served_as_it_arrives_and_downloaded_once() {
        let (hud, held) = proxy().await;
        let mp4: Vec<u8> = (0..=255u8).cycle().take(400).collect();
        let gate = Arc::new(tokio::sync::Notify::new());
        let movie = cid(&mp4);
        held.lock().unwrap().held.insert(
            movie.clone(),
            Held::Gated("video/mp4", mp4.clone().into(), gate.clone()),
        );
        let at = format!("/blob/did:plc:alice/{movie}");
        // Its first bytes, and more of them asked again, before the rest has come.
        let (status, h, body) = fetch(&hud, &at, Some("bytes=0-1")).await;
        assert_eq!(status, StatusCode::PARTIAL_CONTENT);
        assert_eq!(h[header::CONTENT_RANGE], "bytes 0-1/400");
        assert_eq!(body.unwrap(), mp4[..2]);
        let (status, _, body) = fetch(&hud, &at, Some("bytes=100-149")).await;
        assert_eq!(status, StatusCode::PARTIAL_CONTENT);
        assert_eq!(body.unwrap(), mp4[100..150]);
        // The rest come: all of it, from the one download.
        gate.notify_one();
        let (status, _, body) = fetch(&hud, &at, None).await;
        assert_eq!(status, StatusCode::OK);
        assert_eq!(body.unwrap(), mp4);
        assert_eq!(held.lock().unwrap().asked, 1);
    }

    #[tokio::test]
    async fn audio_the_disk_cannot_keep() {
        let (hud, held) = proxy().await;
        let wav = [b"RIFF\x10\0\0\0WAVEfmt ".as_slice(), &[3; 20]].concat();
        held.lock()
            .unwrap()
            .held
            .insert(cid(&wav), Held::Whole("audio/wav", wav.clone().into()));
        // Where its blobs would go, a file.
        std::fs::create_dir_all(&hud.config.cache_dir).unwrap();
        std::fs::write(hud.config.cache_dir.join("blobs"), "").unwrap();
        let could_not = (
            StatusCode::INTERNAL_SERVER_ERROR,
            "The blob could not be kept\n".to_string(),
        );
        assert_eq!(
            refused(&hud, &format!("/blob/did:plc:alice/{}", cid(&wav))).await,
            could_not
        );
        // Downloaded, its length unsaid, but its type cannot be kept.
        let (hud, held) = proxy().await;
        held.lock().unwrap().held.insert(
            cid(&wav),
            Held::Chunks("audio/wav", vec![wav.clone().into()]),
        );
        let blobs = hud.config.cache_dir.join("blobs");
        std::fs::create_dir_all(blobs.join(format!("{}.type", cid(&wav))).join("in-the-way"))
            .unwrap();
        assert_eq!(
            refused(&hud, &format!("/blob/did:plc:alice/{}", cid(&wav))).await,
            could_not
        );
    }

    #[tokio::test]
    async fn a_pds_that_never_answers() {
        let (hud, held) = proxy().await;
        let silent = cid(b"silent");
        held.lock()
            .unwrap()
            .held
            .insert(silent.clone(), Held::Never);
        // Its DID's document read first, in real time.
        let (status, _) = refused(&hud, &format!("/blob/did:plc:alice/{}", cid(b"x"))).await;
        assert_eq!(status, StatusCode::NOT_FOUND);
        tokio::time::pause();
        let (status, why) = refused(&hud, &format!("/blob/did:plc:alice/{silent}")).await;
        assert_eq!(status, StatusCode::BAD_GATEWAY);
        assert!(why.ends_with(": it did not answer in time\n"), "{why}");
    }

    #[test]
    fn a_type_no_header_can_carry() {
        // As a kept blob's type, read from the disk, might be: it goes
        // untyped, and is still never sniffed.
        let mut h = HeaderMap::new();
        blob_headers(&mut h, "audio/\nwav");
        assert!(h.get(header::CONTENT_TYPE).is_none());
        assert_eq!(h[header::X_CONTENT_TYPE_OPTIONS], "nosniff");
    }

    #[test]
    fn ranges() {
        let r = |v: &str| parse_range(v, 1000);
        // A single range, its end past the blob's cut to it.
        assert_eq!(r("bytes=0-99"), Range::Part(0, 99));
        assert_eq!(r("bytes=500-500"), Range::Part(500, 500));
        assert_eq!(r("bytes=900-5000"), Range::Part(900, 999));
        // Open: to the end.
        assert_eq!(r("bytes=10-"), Range::Part(10, 999));
        // Suffix: the last N bytes, all of them if N is more.
        assert_eq!(r("bytes=-100"), Range::Part(900, 999));
        assert_eq!(r("bytes=-5000"), Range::Part(0, 999));
        // Unsatisfiable: starting past the end, or an empty suffix.
        assert_eq!(r("bytes=1000-"), Range::Unsatisfiable);
        assert_eq!(r("bytes=1000-2000"), Range::Unsatisfiable);
        assert_eq!(r("bytes=-0"), Range::Unsatisfiable);
        assert_eq!(parse_range("bytes=-10", 0), Range::Unsatisfiable);
        assert_eq!(parse_range("bytes=0-", 0), Range::Unsatisfiable);
        // Ignored, and the whole blob served: another unit, several ranges,
        // a malformed one.
        for ignored in [
            "items=0-1",
            "bytes=0-1,5-6",
            "bytes=5-1",
            "bytes=a-b",
            "bytes=-",
            "bytes=+1-2",
            "bytes",
        ] {
            assert_eq!(r(ignored), Range::Whole, "{ignored}");
        }
    }

    async fn served(range: Option<&str>) -> (StatusCode, HeaderMap, Vec<u8>) {
        let dir = std::env::temp_dir().join(format!(
            "hud-blob-serve-{}-{}",
            std::process::id(),
            range.unwrap_or("none")
        ));
        let cache = hud_core::blobcache::BlobCache::new(dir.clone(), 1 << 20);
        let bytes: Vec<u8> = (0..=255u8).collect();
        let cid = atproto::Cid::of_raw(&bytes);
        let body = futures::stream::iter([Ok::<_, String>(bytes)]);
        let filling = cache
            .fill("did:plc:alice", &cid, "audio/wav", None, body, 1 << 20)
            .await
            .unwrap();
        filling.kept().await.unwrap();
        let kept = cache.get(&cid).await.unwrap();
        let header = range.map(|r| HeaderValue::from_str(r).unwrap());
        let r = serve_cached(kept, header.as_ref());
        let (status, headers) = (r.status(), r.headers().clone());
        let body = axum::body::to_bytes(r.into_body(), usize::MAX)
            .await
            .unwrap();
        let _ = std::fs::remove_dir_all(&dir);
        (status, headers, body.to_vec())
    }

    #[tokio::test]
    async fn served_from_the_cache() {
        let (status, h, body) = served(Some("bytes=10-19")).await;
        assert_eq!(status, StatusCode::PARTIAL_CONTENT);
        assert_eq!(h[header::CONTENT_RANGE], "bytes 10-19/256");
        assert_eq!(h[header::CONTENT_LENGTH], "10");
        assert_eq!(h[header::ACCEPT_RANGES], "bytes");
        assert_eq!(body, (10..20u8).collect::<Vec<_>>());
        let (status, h, body) = served(None).await;
        assert_eq!(status, StatusCode::OK);
        assert_eq!(
            (h[header::CONTENT_TYPE].to_str().unwrap(), body.len()),
            ("audio/wav", 256)
        );
        let (status, h, _) = served(Some("bytes=300-")).await;
        assert_eq!(status, StatusCode::RANGE_NOT_SATISFIABLE);
        assert_eq!(h[header::CONTENT_RANGE], "bytes */256");
    }
}