chadtmiller.com/hud
A remote MCP server that shows atproto records as interactive UI inside MCP Apps hosts
@hud/shell 0.5.0 Latest 5 packages
0
0
//! The blob proxy, `GET /blob/{did}/{cid}`: every track and video the view
//! plays is loaded through it, and every image but a public record's, which
//! the image CDN serves (`HUD_IMAGE_CDN`, hud-core's `blobs`) with the proxy
//! as its fallback. It resolves the DID's PDS and fetches
//! `com.atproto.sync.getBlob`, serving images (not SVG, 20 MB at most),
//! audio (100 MB at most) and MP4, QuickTime and WebM video (100 MB at
//! most) and nothing else, a blob the PDS sends untyped known by its first
//! bytes: the PDS comes from a DID document anyone can
//! write, so its URL goes through the outbound client's address checks,
//! and it has ten seconds to answer. A blob never changes, so it is cached
//! for good.
//!
//! Audio and video are kept on disk (hud-core's `blobcache`, which keeps
//! only bytes that hash to the CID), because a player seeks with `Range`
//! and PDSes answer only whole blobs. Each is downloaded once however many
//! ask for it, and served as it arrives, so a player starts on its first
//! bytes: a single range is answered `206`, an unsatisfiable one `416`, and
//! a plain request `200`, against the length its PDS said (one that said
//! none is served once it is all here). A blob already kept is served
//! without asking its PDS again: its bytes are what its CID names, whoever
//! asks.
use axum::body::Body;
use axum::extract::{Path, State};
use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use futures::{StreamExt, TryStreamExt};
use hud_core::Hud;
use hud_core::blobcache::{Cached, Filling, PutError, READ_CHUNK};
use hud_core::blobs::{IMAGE_MAX, SNIFF_LEN, is_kept, max_size, sniff};
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio_util::io::ReaderStream;
/// How long the PDS has to answer.
const ANSWER_WAIT: Duration = Duration::from_secs(10);
/// How long the PDS has to send all of a blob.
const BODY_WAIT: Duration = Duration::from_secs(120);
// The caps, `IMAGE_MAX`, `AUDIO_MAX` and `VIDEO_MAX`, are hud-core's, which
// gives no proxy URL to a blob that declares itself past them.
/// A refusal: its status, and a line of plain text saying why, which the
/// request's line says too (`telemetry::Cause`).
fn refuse(status: StatusCode, why: &str) -> Response {
let mut res = (
status,
[
(header::CONTENT_TYPE, "text/plain; charset=utf-8"),
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
],
format!("{why}\n"),
)
.into_response();
res.extensions_mut()
.insert(crate::telemetry::Cause(why.to_string()));
res
}
/// The headers every blob is served with: cached for good, never sniffed
/// or run, loadable from any origin.
fn blob_headers(h: &mut HeaderMap, content_type: &str) {
if let Ok(v) = HeaderValue::from_str(content_type) {
h.insert(header::CONTENT_TYPE, v);
}
for (name, value) in [
(header::CACHE_CONTROL, "public, max-age=31536000, immutable"),
(header::X_CONTENT_TYPE_OPTIONS, "nosniff"),
(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*"),
(
header::HeaderName::from_static("cross-origin-resource-policy"),
"cross-origin",
),
(
header::CONTENT_SECURITY_POLICY,
"default-src 'none'; sandbox",
),
] {
h.insert(name, HeaderValue::from_static(value));
}
}
/// What a `Range` header asks of a blob of a given length.
#[derive(Debug, PartialEq, Eq)]
enum Range {
/// All of it: no header, or one this server does not take (another
/// unit, several ranges, a malformed one), which is ignored.
Whole,
/// The bytes from the first to the second, both included.
Part(u64, u64),
/// A range of bytes the blob does not have.
Unsatisfiable,
}
/// A `Range` header's single byte range, against a blob of `len` bytes.
fn parse_range(value: &str, len: u64) -> Range {
let Some(spec) = value.trim().strip_prefix("bytes=") else {
return Range::Whole;
};
let Some((first, last)) = spec.split_once('-').filter(|_| !spec.contains(',')) else {
return Range::Whole;
};
let number = |s: &str| {
(!s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()))
.then(|| s.parse::<u64>().ok())
.flatten()
};
match (first.trim(), last.trim()) {
("", suffix) => match number(suffix) {
None => Range::Whole,
Some(0) => Range::Unsatisfiable,
Some(_) if len == 0 => Range::Unsatisfiable,
Some(n) => Range::Part(len.saturating_sub(n), len - 1),
},
(from, to) => {
let Some(from) = number(from) else {
return Range::Whole;
};
let to = match to {
"" => None,
to => match number(to) {
Some(to) if to >= from => Some(to),
_ => return Range::Whole,
},
};
if from >= len {
return Range::Unsatisfiable;
}
Range::Part(from, to.map_or(len - 1, |to| to.min(len - 1)))
}
}
}
/// A blob of `len` bytes, as a request's `Range` asks for it: `count` of
/// its bytes from `start`, as `read` gives them.
fn ranged<S>(
len: u64,
mime: &str,
range: Option<&HeaderValue>,
read: impl FnOnce(u64, u64) -> S,
) -> Response
where
S: futures::Stream<Item = std::io::Result<bytes::Bytes>> + Send + 'static,
{
let range = range
.and_then(|r| r.to_str().ok())
.map_or(Range::Whole, |r| parse_range(r, len));
let (status, start, count) = match range {
Range::Whole => (StatusCode::OK, 0, len),
Range::Part(from, to) => (StatusCode::PARTIAL_CONTENT, from, to - from + 1),
Range::Unsatisfiable => {
let mut r = refuse(StatusCode::RANGE_NOT_SATISFIABLE, "No such range");
let all = HeaderValue::from_str(&format!("bytes */{len}"))
.expect("a length is a header's text");
r.headers_mut().insert(header::CONTENT_RANGE, all);
return r;
}
};
let mut r = Response::new(Body::from_stream(read(start, count)));
*r.status_mut() = status;
let h = r.headers_mut();
blob_headers(h, mime);
h.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes"));
h.insert(header::CONTENT_LENGTH, HeaderValue::from(count));
if status == StatusCode::PARTIAL_CONTENT
&& let Ok(v) = HeaderValue::from_str(&format!("bytes {start}-{}/{len}", start + count - 1))
{
h.insert(header::CONTENT_RANGE, v);
}
r
}
/// A kept blob, as a request's `Range` asks for it.
fn serve_cached(c: Cached, range: Option<&HeaderValue>) -> Response {
let Cached {
mut file,
len,
mime,
} = c;
// Read from `start`: a kept blob is a file, so seeking it fails only as
// reading it would, which ends the body.
ranged(len, &mime, range, |start, count| {
futures::stream::once(async move {
file.seek(std::io::SeekFrom::Start(start)).await?;
Ok::<_, std::io::Error>(ReaderStream::with_capacity(file.take(count), READ_CHUNK))
})
.try_flatten()
})
}
/// A blob being downloaded, as a request's `Range` asks for it, each byte
/// as it arrives: against the length its PDS said, or where it said none,
/// once it is all here.
async fn serve_filling(f: Arc<Filling>, range: Option<&HeaderValue>) -> Response {
let len = match f.said {
Some(len) => len,
None => match f.kept().await {
Ok(len) => len,
Err(e) => return unkept(&e),
},
};
ranged(len, &f.mime, range, |start, count| f.read(start, count))
}
/// Why a blob was not kept, answered before any of it was sent.
fn unkept(e: &PutError) -> Response {
match e {
PutError::TooLarge => refuse(StatusCode::PAYLOAD_TOO_LARGE, "The blob is too large"),
PutError::Disk(e) => {
tracing::warn!(error = %e, "could not keep a blob");
refuse(
StatusCode::INTERNAL_SERVER_ERROR,
"The blob could not be kept",
)
}
e => refuse(StatusCode::BAD_GATEWAY, &e.to_string()),
}
}
pub async fn blob(
State(hud): State<Arc<Hud>>,
Path((did, cid_text)): Path<(String, String)>,
headers: HeaderMap,
) -> Response {
if !atproto::syntax::is_did(&did) {
return refuse(StatusCode::BAD_REQUEST, "Not a DID");
}
let Ok(cid) = atproto::Cid::parse(&cid_text) else {
return refuse(StatusCode::BAD_REQUEST, "Not a CID");
};
let range = headers.get(header::RANGE);
if let Some(kept) = hud.blob_cache.get(&cid).await {
return serve_cached(kept, range);
}
if let Some(filling) = hud.blob_cache.filling(&did, &cid) {
return serve_filling(filling, range).await;
}
let pds = match hud.pds(&did).await {
Ok(pds) => pds,
Err(why) => return refuse(StatusCode::BAD_GATEWAY, &why),
};
// The PDS's URL is one `hud.pds` checked, so building and sending the
// request fail only as sending it does.
let asked = async {
let mut url = url::Url::parse(&format!("{pds}/xrpc/com.atproto.sync.getBlob"))?;
url.query_pairs_mut()
.append_pair("did", &did)
.append_pair("cid", &cid_text);
let req = hud.net.get(&url)?.timeout(BODY_WAIT);
let res = hud.net.execute(req).await;
let res = res.map_err(|e| hud_core::net::describe(&e))?;
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(res)
};
let res = match tokio::time::timeout(ANSWER_WAIT, asked).await {
Ok(Ok(r)) => r,
Ok(Err(why)) => return refuse(StatusCode::BAD_GATEWAY, &format!("{pds}: {why}")),
Err(_) => {
return refuse(
StatusCode::BAD_GATEWAY,
&format!("{pds}: it did not answer in time"),
);
}
};
match res.status().as_u16() {
200..=299 => {}
404 => return refuse(StatusCode::NOT_FOUND, "No such blob"),
s => return refuse(StatusCode::BAD_GATEWAY, &format!("{pds} answered {s}")),
}
let declared = res
.headers()
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let length = res.content_length();
let mut stream = res.bytes_stream().boxed();
// A PDS that types every blob `application/octet-stream` (or not at
// all): the blob's first bytes say what it is, if it is a kind served.
let (content_type, head, max) = match max_size(&declared) {
Some(max) => (declared, Vec::new(), max),
None => {
let mut head = Vec::new();
while head.len() < SNIFF_LEN {
match stream.next().await {
Some(Ok(chunk)) => head.extend_from_slice(&chunk),
Some(Err(e)) => return refuse(StatusCode::BAD_GATEWAY, &format!("{pds}: {e}")),
None => break,
}
}
// What `sniff` knows is a kind served, so it has a cap.
match sniff(&head).and_then(|t| Some((t.to_string(), max_size(t)?))) {
Some((sniffed, max)) => (sniffed, head, max),
None => {
return refuse(
StatusCode::UNSUPPORTED_MEDIA_TYPE,
"Not an image, audio or video",
);
}
}
}
};
// What was read to sniff it goes first, then the rest as it comes.
let stream = futures::stream::iter((!head.is_empty()).then(|| Ok(bytes::Bytes::from(head))))
.chain(stream)
.boxed();
if length.is_some_and(|n| n > max) {
return refuse(StatusCode::PAYLOAD_TOO_LARGE, "The blob is too large");
}
if is_kept(&content_type) {
return match hud
.blob_cache
.fill(&did, &cid, &content_type, length, stream, max)
.await
{
Ok(filling) => serve_filling(filling, range).await,
Err(e) => unkept(&PutError::Disk(e)),
};
}
// An image streams on, cut off past the cap whatever the PDS said its
// length was.
let mut sent = 0u64;
let body = stream.map(move |chunk| {
let chunk = chunk.map_err(std::io::Error::other)?;
sent += chunk.len() as u64;
if sent > IMAGE_MAX {
return Err(std::io::Error::other("the blob is too large"));
}
Ok::<bytes::Bytes, std::io::Error>(chunk)
});
let mut r = Response::new(Body::from_stream(body));
blob_headers(r.headers_mut(), &content_type);
r
}
#[cfg(test)]
mod tests {
use super::*;
use axum::Router;
use axum::extract::Query;
use hud_core::Config;
use hud_core::blobs::AUDIO_MAX;
use std::collections::HashMap;
use std::sync::Mutex;
use tower::ServiceExt;
/// What the fake PDS answers for a blob.
#[derive(Clone)]
enum Held {
/// Its type (none sent if empty) and its bytes, its length said.
Whole(&'static str, bytes::Bytes),
/// Its type and its bytes as chunks, its length unsaid.
Chunks(&'static str, Vec<bytes::Bytes>),
/// An answer of this status alone.
Status(u16),
/// No answer, ever.
Never,
/// Its type and its bytes, its length said: the first half sent at
/// once, the rest once the gate opens.
Gated(&'static str, bytes::Bytes, Arc<tokio::sync::Notify>),
}
/// What the fake PDSes hold, by CID: `held` at did:plc:alice's, and how
/// often it was asked for a blob; at did:plc:short's, each blob's type
/// and the bytes it sends of it, a thousand fewer than it says, before
/// it hangs up.
#[derive(Default)]
struct Shelves {
held: HashMap<String, Held>,
asked: usize,
short: HashMap<String, (&'static str, bytes::Bytes)>,
}
type Fake = Arc<Mutex<Shelves>>;
/// The PDSes holding `fake`, and PLC naming them, and a closed port
/// for did:plc:closed: its origin.
async fn pds(fake: Fake) -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://{}", listener.local_addr().unwrap());
let short = short_pds(fake.clone()).await;
let me = origin.clone();
let plc = move |Path(did): Path<String>| {
let (me, short) = (me.clone(), short.clone());
async move {
let endpoint = match did.as_str() {
"did:plc:alice" => me,
"did:plc:short" => short,
"did:plc:closed" => "http://127.0.0.1:1".to_string(),
_ => return StatusCode::NOT_FOUND.into_response(),
};
axum::Json(serde_json::json!({ "id": did, "alsoKnownAs": [],
"verificationMethod": [], "service": [{ "id": "#atproto_pds",
"type": "AtprotoPersonalDataServer", "serviceEndpoint": endpoint }] }))
.into_response()
}
};
let get_blob = move |Query(q): Query<HashMap<String, String>>| {
let found = {
let mut shelves = fake.lock().unwrap();
shelves.asked += 1;
shelves.held.get(&q["cid"]).cloned()
};
async move {
let typed = |t: &'static str| {
let mut h = HeaderMap::new();
if !t.is_empty() {
h.insert(header::CONTENT_TYPE, HeaderValue::from_static(t));
}
h
};
match found {
Some(Held::Whole(t, b)) => (typed(t), b).into_response(),
Some(Held::Chunks(t, chunks)) => {
let body = futures::stream::iter(chunks.into_iter().map(Ok::<_, String>));
(typed(t), Body::from_stream(body)).into_response()
}
Some(Held::Status(s)) => StatusCode::from_u16(s).unwrap().into_response(),
Some(Held::Never) => std::future::pending::<Response>().await,
Some(Held::Gated(t, b, gate)) => {
let mut h = typed(t);
h.insert(header::CONTENT_LENGTH, HeaderValue::from(b.len()));
let (first, rest) = (b.slice(..b.len() / 2), b.slice(b.len() / 2..));
let body = futures::stream::iter([Ok::<_, String>(first)]).chain(
futures::stream::once(async move {
gate.notified().await;
Ok(rest)
}),
);
(h, Body::from_stream(body)).into_response()
}
None => StatusCode::NOT_FOUND.into_response(),
}
}
};
let app = Router::new()
.route(
"/xrpc/com.atproto.sync.getBlob",
axum::routing::get(get_blob),
)
.route("/{did}", axum::routing::get(plc));
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
origin
}
/// A PDS speaking HTTP by hand: a blob's head and bytes sent, then the
/// connection closed short of the length it said. Its origin.
async fn short_pds(fake: Fake) -> String {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
loop {
let (mut conn, _) = listener.accept().await.unwrap();
let fake = fake.clone();
tokio::spawn(async move {
let (mut asked, mut buf) = (Vec::new(), [0; 1024]);
while !asked.windows(4).any(|w| w == b"\r\n\r\n") {
let n = conn.read(&mut buf).await.unwrap();
assert!(n > 0, "the proxy hung up");
asked.extend_from_slice(&buf[..n]);
}
let asked = String::from_utf8_lossy(&asked).to_string();
let cid = asked.split("cid=").nth(1).unwrap().split([' ', '&']).next();
let (kind, bytes) = fake.lock().unwrap().short[cid.unwrap()].clone();
let head = format!(
"HTTP/1.1 200 OK\r\ncontent-type: {kind}\r\ncontent-length: {}\r\n\r\n",
bytes.len() + 1000
);
conn.write_all(head.as_bytes()).await.unwrap();
conn.write_all(&bytes).await.unwrap();
});
}
});
origin
}
/// A server whose PLC is the fake one, its blobs cached in a directory
/// of its own; and what the PDSes hold.
async fn proxy() -> (Arc<Hud>, Fake) {
let fake = Fake::default();
let origin = pds(fake.clone()).await;
let dir = std::env::temp_dir().join(format!("hud-blob-{}", atproto::jose::random(6)));
let hud = Hud::new(Config {
port: 0,
bind: "::".parse().unwrap(),
public_origin: "https://hud.example".into(),
cache_dir: dir.join("cache"),
data_dir: dir,
oauth: false,
oauth_scope: hud_core::scope::asked(hud_core::view::views(), true, false),
oauth_include: None,
workspace: true,
sign_up_pds: hud_core::config::SIGN_UP_PDS.into(),
client: Default::default(),
spaces: false,
plc_url: origin,
handle_resolver: None,
allow_private: true,
blob_cache_bytes: 1 << 20,
cache_memory_bytes: 1 << 24,
backlinks: hud_core::backlinks::Backlinks::None,
favicons: None,
image_cdn: None,
relay: None,
operators: Vec::new(),
logs: false,
logs_keep: hud_core::logs::KEEP,
})
.unwrap();
(hud, fake)
}
/// `bytes`' CID, as the proxy is asked for it.
fn cid(bytes: &[u8]) -> String {
atproto::Cid::of_raw(bytes).to_string()
}
/// A blob asked of the proxy: its status, headers and body, or the
/// error its body ended with.
async fn fetch(
hud: &Arc<Hud>,
path: &str,
range: Option<&str>,
) -> (StatusCode, HeaderMap, Result<Vec<u8>, String>) {
let mut req = axum::http::Request::get(path);
if let Some(r) = range {
req = req.header(header::RANGE, r);
}
let r = crate::router(hud.clone(), None)
.oneshot(req.body(Body::empty()).unwrap())
.await
.unwrap();
let (status, headers) = (r.status(), r.headers().clone());
let body = axum::body::to_bytes(r.into_body(), usize::MAX)
.await
.map(|b| b.to_vec())
.map_err(|e| e.to_string());
(status, headers, body)
}
/// A refusal's status and line.
async fn refused(hud: &Arc<Hud>, path: &str) -> (StatusCode, String) {
let (status, h, body) = fetch(hud, path, None).await;
assert_eq!(
h[header::CONTENT_TYPE],
"text/plain; charset=utf-8",
"{path}"
);
(status, String::from_utf8(body.unwrap()).unwrap())
}
#[tokio::test]
async fn refusals() {
let (hud, held) = proxy().await;
let html = cid(b"<html>");
let gone = cid(b"gone");
let broken = cid(b"broken");
held.lock().unwrap().held.extend([
(
html.clone(),
Held::Whole("application/octet-stream", "<html><body>".into()),
),
(broken.clone(), Held::Status(500)),
]);
let line = |s: &str| format!("{s}\n");
assert_eq!(
refused(&hud, &format!("/blob/alice/{html}")).await,
(StatusCode::BAD_REQUEST, line("Not a DID"))
);
assert_eq!(
refused(&hud, "/blob/did:plc:alice/nope").await,
(StatusCode::BAD_REQUEST, line("Not a CID"))
);
// A DID with no document, and a PDS that is not there.
let (status, why) = refused(&hud, &format!("/blob/did:plc:nobody/{gone}")).await;
assert_eq!(status, StatusCode::BAD_GATEWAY);
assert!(why.contains("did:plc:nobody"), "{why}");
let (status, why) = refused(&hud, &format!("/blob/did:plc:closed/{gone}")).await;
assert_eq!(status, StatusCode::BAD_GATEWAY);
assert!(why.starts_with("http://127.0.0.1:1: "), "{why}");
// What the PDS has not, what it fails, and what is not a kind served.
assert_eq!(
refused(&hud, &format!("/blob/did:plc:alice/{gone}")).await,
(StatusCode::NOT_FOUND, line("No such blob"))
);
let (status, why) = refused(&hud, &format!("/blob/did:plc:alice/{broken}")).await;
assert_eq!(status, StatusCode::BAD_GATEWAY);
assert!(why.ends_with(" answered 500\n"), "{why}");
assert_eq!(
refused(&hud, &format!("/blob/did:plc:alice/{html}")).await,
(
StatusCode::UNSUPPORTED_MEDIA_TYPE,
line("Not an image, audio or video")
)
);
}
#[tokio::test]
async fn images_stream_through() {
let (hud, held) = proxy().await;
let png = [b"\x89PNG\r\n\x1a\n".as_slice(), &[7; 100]].concat();
let gif = b"GIF89a".to_vec();
let (typed, sniffed, short, cut, big, endless, early) = (
cid(b"typed"),
cid(b"sniffed"),
cid(b"short"),
cid(b"cut"),
cid(b"big"),
cid(b"endless"),
cid(b"early"),
);
let mib = bytes::Bytes::from(vec![0u8; 1 << 20]);
held.lock().unwrap().held.extend([
(typed.clone(), Held::Whole("image/png", png.clone().into())),
// Untyped, in chunks shorter than what sniffing needs.
(
sniffed.clone(),
Held::Chunks(
"",
png.chunks(5).map(bytes::Bytes::copy_from_slice).collect(),
),
),
// Untyped, and shorter than that whole.
(
short.clone(),
Held::Whole("application/octet-stream", gif.clone().into()),
),
(
big.clone(),
Held::Whole(
"image/jpeg",
vec![0; usize::try_from(IMAGE_MAX).unwrap() + 1].into(),
),
),
(endless.clone(), Held::Chunks("image/jpeg", vec![mib; 21])),
]);
held.lock().unwrap().short.extend([
(cut.clone(), ("image/png", png.clone().into())),
(
early.clone(),
("application/octet-stream", png[..4].to_vec().into()),
),
]);
let at = |c: &str| format!("/blob/did:plc:alice/{c}");
for (c, kind, bytes) in [
(&typed, "image/png", &png),
(&sniffed, "image/png", &png),
(&short, "image/gif", &gif),
] {
let (status, h, body) = fetch(&hud, &at(c), None).await;
assert_eq!(status, StatusCode::OK, "{kind}");
assert_eq!(h[header::CONTENT_TYPE], kind);
assert_eq!(
h[header::CACHE_CONTROL],
"public, max-age=31536000, immutable"
);
assert_eq!(
h[header::CONTENT_SECURITY_POLICY],
"default-src 'none'; sandbox"
);
assert_eq!(&body.unwrap(), bytes);
}
// An image never kept: asked again, it is asked of the PDS again.
held.lock().unwrap().held.remove(&typed);
assert_eq!(
fetch(&hud, &at(&typed), None).await.0,
StatusCode::NOT_FOUND
);
// Said to be past the cap: refused before a byte is sent.
assert_eq!(
refused(&hud, &at(&big)).await,
(
StatusCode::PAYLOAD_TOO_LARGE,
"The blob is too large\n".into()
)
);
// Past it unsaid, or cut off by the PDS: the body ends in an error.
let (status, _, body) = fetch(&hud, &at(&endless), None).await;
assert_eq!(status, StatusCode::OK);
assert!(body.is_err());
let cut_off = |c: &str| format!("/blob/did:plc:short/{c}");
let (status, h, body) = fetch(&hud, &cut_off(&cut), None).await;
assert_eq!(
(status, &h[header::CONTENT_TYPE]),
(StatusCode::OK, &HeaderValue::from_static("image/png"))
);
assert!(body.is_err());
// Cut off while it was being sniffed.
let (status, why) = refused(&hud, &cut_off(&early)).await;
assert_eq!(status, StatusCode::BAD_GATEWAY);
assert!(why.starts_with("http://127.0.0.1:"), "{why}");
}
#[tokio::test]
async fn audio_kept_then_served_from_the_cache() {
let (hud, held) = proxy().await;
let wav = [b"RIFF\x10\0\0\0WAVEfmt ".as_slice(), &[3; 200]].concat();
let (song, wrong, unsaid, long) = (cid(&wav), cid(b"wrong"), cid(b"unsaid"), cid(b"long"));
let mib = bytes::Bytes::from(vec![0u8; 1 << 20]);
let mibs = usize::try_from(AUDIO_MAX >> 20).unwrap() + 1;
held.lock().unwrap().held.extend([
(song.clone(), Held::Chunks("", vec![wav.clone().into()])),
// Bytes that are not what the CID names, their length said and unsaid.
(wrong.clone(), Held::Whole("audio/wav", wav.clone().into())),
(
unsaid.clone(),
Held::Chunks("audio/wav", vec![wav.clone().into()]),
),
(long.clone(), Held::Chunks("audio/mpeg", vec![mib; mibs])),
]);
let at = |c: &str| format!("/blob/did:plc:alice/{c}");
// Its length unsaid: kept whole, then the range asked of it.
let (status, h, body) = fetch(&hud, &at(&song), Some("bytes=4-11")).await;
assert_eq!(status, StatusCode::PARTIAL_CONTENT);
assert_eq!(h[header::CONTENT_TYPE], "audio/wav");
assert_eq!(
h[header::CONTENT_RANGE],
format!("bytes 4-11/{}", wav.len())
);
assert_eq!(body.unwrap(), wav[4..12]);
// Kept: served whole though the PDS has it no more.
held.lock().unwrap().held.remove(&song);
let (status, h, body) = fetch(&hud, &at(&song), None).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(h[header::ACCEPT_RANGES], "bytes");
assert_eq!(body.unwrap(), wav);
// Never kept: bytes not its CID's. Their length said, they are
// sent as they come, and the body ends in an error short of the
// last; unsaid, nothing is sent.
let (status, _, body) = fetch(&hud, &at(&wrong), None).await;
assert_eq!(status, StatusCode::OK);
assert!(body.is_err());
let (status, why) = refused(&hud, &at(&unsaid)).await;
assert_eq!(
(status, why.as_str()),
(
StatusCode::BAD_GATEWAY,
"the blob's bytes do not match its CID\n"
)
);
// More than an audio blob's cap.
assert_eq!(
refused(&hud, &at(&long)).await,
(
StatusCode::PAYLOAD_TOO_LARGE,
"The blob is too large\n".into()
)
);
}
#[tokio::test]
async fn video_served_as_it_arrives_and_downloaded_once() {
let (hud, held) = proxy().await;
let mp4: Vec<u8> = (0..=255u8).cycle().take(400).collect();
let gate = Arc::new(tokio::sync::Notify::new());
let movie = cid(&mp4);
held.lock().unwrap().held.insert(
movie.clone(),
Held::Gated("video/mp4", mp4.clone().into(), gate.clone()),
);
let at = format!("/blob/did:plc:alice/{movie}");
// Its first bytes, and more of them asked again, before the rest has come.
let (status, h, body) = fetch(&hud, &at, Some("bytes=0-1")).await;
assert_eq!(status, StatusCode::PARTIAL_CONTENT);
assert_eq!(h[header::CONTENT_RANGE], "bytes 0-1/400");
assert_eq!(body.unwrap(), mp4[..2]);
let (status, _, body) = fetch(&hud, &at, Some("bytes=100-149")).await;
assert_eq!(status, StatusCode::PARTIAL_CONTENT);
assert_eq!(body.unwrap(), mp4[100..150]);
// The rest come: all of it, from the one download.
gate.notify_one();
let (status, _, body) = fetch(&hud, &at, None).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(body.unwrap(), mp4);
assert_eq!(held.lock().unwrap().asked, 1);
}
#[tokio::test]
async fn audio_the_disk_cannot_keep() {
let (hud, held) = proxy().await;
let wav = [b"RIFF\x10\0\0\0WAVEfmt ".as_slice(), &[3; 20]].concat();
held.lock()
.unwrap()
.held
.insert(cid(&wav), Held::Whole("audio/wav", wav.clone().into()));
// Where its blobs would go, a file.
std::fs::create_dir_all(&hud.config.cache_dir).unwrap();
std::fs::write(hud.config.cache_dir.join("blobs"), "").unwrap();
let could_not = (
StatusCode::INTERNAL_SERVER_ERROR,
"The blob could not be kept\n".to_string(),
);
assert_eq!(
refused(&hud, &format!("/blob/did:plc:alice/{}", cid(&wav))).await,
could_not
);
// Downloaded, its length unsaid, but its type cannot be kept.
let (hud, held) = proxy().await;
held.lock().unwrap().held.insert(
cid(&wav),
Held::Chunks("audio/wav", vec![wav.clone().into()]),
);
let blobs = hud.config.cache_dir.join("blobs");
std::fs::create_dir_all(blobs.join(format!("{}.type", cid(&wav))).join("in-the-way"))
.unwrap();
assert_eq!(
refused(&hud, &format!("/blob/did:plc:alice/{}", cid(&wav))).await,
could_not
);
}
#[tokio::test]
async fn a_pds_that_never_answers() {
let (hud, held) = proxy().await;
let silent = cid(b"silent");
held.lock()
.unwrap()
.held
.insert(silent.clone(), Held::Never);
// Its DID's document read first, in real time.
let (status, _) = refused(&hud, &format!("/blob/did:plc:alice/{}", cid(b"x"))).await;
assert_eq!(status, StatusCode::NOT_FOUND);
tokio::time::pause();
let (status, why) = refused(&hud, &format!("/blob/did:plc:alice/{silent}")).await;
assert_eq!(status, StatusCode::BAD_GATEWAY);
assert!(why.ends_with(": it did not answer in time\n"), "{why}");
}
#[test]
fn a_type_no_header_can_carry() {
// As a kept blob's type, read from the disk, might be: it goes
// untyped, and is still never sniffed.
let mut h = HeaderMap::new();
blob_headers(&mut h, "audio/\nwav");
assert!(h.get(header::CONTENT_TYPE).is_none());
assert_eq!(h[header::X_CONTENT_TYPE_OPTIONS], "nosniff");
}
#[test]
fn ranges() {
let r = |v: &str| parse_range(v, 1000);
// A single range, its end past the blob's cut to it.
assert_eq!(r("bytes=0-99"), Range::Part(0, 99));
assert_eq!(r("bytes=500-500"), Range::Part(500, 500));
assert_eq!(r("bytes=900-5000"), Range::Part(900, 999));
// Open: to the end.
assert_eq!(r("bytes=10-"), Range::Part(10, 999));
// Suffix: the last N bytes, all of them if N is more.
assert_eq!(r("bytes=-100"), Range::Part(900, 999));
assert_eq!(r("bytes=-5000"), Range::Part(0, 999));
// Unsatisfiable: starting past the end, or an empty suffix.
assert_eq!(r("bytes=1000-"), Range::Unsatisfiable);
assert_eq!(r("bytes=1000-2000"), Range::Unsatisfiable);
assert_eq!(r("bytes=-0"), Range::Unsatisfiable);
assert_eq!(parse_range("bytes=-10", 0), Range::Unsatisfiable);
assert_eq!(parse_range("bytes=0-", 0), Range::Unsatisfiable);
// Ignored, and the whole blob served: another unit, several ranges,
// a malformed one.
for ignored in [
"items=0-1",
"bytes=0-1,5-6",
"bytes=5-1",
"bytes=a-b",
"bytes=-",
"bytes=+1-2",
"bytes",
] {
assert_eq!(r(ignored), Range::Whole, "{ignored}");
}
}
async fn served(range: Option<&str>) -> (StatusCode, HeaderMap, Vec<u8>) {
let dir = std::env::temp_dir().join(format!(
"hud-blob-serve-{}-{}",
std::process::id(),
range.unwrap_or("none")
));
let cache = hud_core::blobcache::BlobCache::new(dir.clone(), 1 << 20);
let bytes: Vec<u8> = (0..=255u8).collect();
let cid = atproto::Cid::of_raw(&bytes);
let body = futures::stream::iter([Ok::<_, String>(bytes)]);
let filling = cache
.fill("did:plc:alice", &cid, "audio/wav", None, body, 1 << 20)
.await
.unwrap();
filling.kept().await.unwrap();
let kept = cache.get(&cid).await.unwrap();
let header = range.map(|r| HeaderValue::from_str(r).unwrap());
let r = serve_cached(kept, header.as_ref());
let (status, headers) = (r.status(), r.headers().clone());
let body = axum::body::to_bytes(r.into_body(), usize::MAX)
.await
.unwrap();
let _ = std::fs::remove_dir_all(&dir);
(status, headers, body.to_vec())
}
#[tokio::test]
async fn served_from_the_cache() {
let (status, h, body) = served(Some("bytes=10-19")).await;
assert_eq!(status, StatusCode::PARTIAL_CONTENT);
assert_eq!(h[header::CONTENT_RANGE], "bytes 10-19/256");
assert_eq!(h[header::CONTENT_LENGTH], "10");
assert_eq!(h[header::ACCEPT_RANGES], "bytes");
assert_eq!(body, (10..20u8).collect::<Vec<_>>());
let (status, h, body) = served(None).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(
(h[header::CONTENT_TYPE].to_str().unwrap(), body.len()),
("audio/wav", 256)
);
let (status, h, _) = served(Some("bytes=300-")).await;
assert_eq!(status, StatusCode::RANGE_NOT_SATISFIABLE);
assert_eq!(h[header::CONTENT_RANGE], "bytes */256");
}
}