~ knotless Search/
↑↓ move↵ open⇧↵ all results

chadtmiller.com/hud

A remote MCP server that shows atproto records as interactive UI inside MCP Apps hosts

0
0
478 lines · 17.9 KB Raw History
//! Pointing a record's images, audio and video at where they are served.
//! Every blob ref in a returned record, at any depth, in its current form
//! (`{$type: "blob", ref: {$link}, mimeType, size}`) or the legacy one
//! (`{cid, mimeType}`), gets a sibling `url`, if it is an image, audio or
//! video the proxy serves and its declared size is within the proxy's cap
//! for its kind; the ref itself is left as it was. A blob past the cap gets
//! no `url`, as the proxy would refuse it: a view shows it as not playable,
//! or a trusted view's binding falls back to a URL.
//!
//! The `url` is the server's own proxy, `<public origin>/blob/<did>/<cid>`,
//! but for an image of a public record while there is an image CDN
//! (`HUD_IMAGE_CDN`, Bluesky's, which renders any blob on the network as a
//! JPEG at a few sizes): its `url` is the CDN's full size, beside it a
//! `thumb`, the CDN's thumbnail (an avatar's for an avatar-like field), and
//! `proxy`, the proxy's URL, for a renderer to fall back on. A GIF stays on
//! the proxy, which the CDN would turn into a still; and every blob of a
//! record not anyone can read (from a space, or a service answering the
//! signed-in account alone) stays on the proxy alone, never sent to a
//! public CDN.

use serde_json::{Map, Value};

/// A MIME type's essence, lowercased, without its parameters.
fn essence(mime: &str) -> String {
    mime.split(';')
        .next()
        .unwrap_or_default()
        .trim()
        .to_ascii_lowercase()
}

/// Whether the proxy serves an image of this MIME type: an image, but not
/// SVG, which can carry script.
pub fn is_served_image(mime: &str) -> bool {
    let mime = essence(mime);
    mime.starts_with("image/") && mime != "image/svg+xml"
}

/// Whether a MIME type is audio, which the proxy serves too.
pub fn is_audio(mime: &str) -> bool {
    essence(mime).starts_with("audio/")
}

/// Whether a MIME type is a video the proxy serves: one a browser plays
/// natively, MP4, QuickTime or WebM.
pub fn is_video(mime: &str) -> bool {
    matches!(
        essence(mime).as_str(),
        "video/mp4" | "video/quicktime" | "video/webm"
    )
}

/// Whether the proxy keeps a blob of this MIME type on disk before serving
/// it, so that a player can seek in it with `Range`: audio and video.
pub fn is_kept(mime: &str) -> bool {
    is_audio(mime) || is_video(mime)
}

/// Whether the proxy serves a blob of this MIME type at all.
pub fn is_served(mime: &str) -> bool {
    max_size(mime).is_some()
}

/// The most of an image the proxy sends on.
pub const IMAGE_MAX: u64 = 20 * 1024 * 1024;
/// The most of an audio blob the proxy keeps and serves.
pub const AUDIO_MAX: u64 = 100 * 1024 * 1024;
/// The most of a video blob the proxy keeps and serves.
pub const VIDEO_MAX: u64 = 100 * 1024 * 1024;

/// The proxy's cap for a blob of this MIME type; none if it serves none.
pub fn max_size(mime: &str) -> Option<u64> {
    if is_served_image(mime) {
        Some(IMAGE_MAX)
    } else if is_audio(mime) {
        Some(AUDIO_MAX)
    } else if is_video(mime) {
        Some(VIDEO_MAX)
    } else {
        None
    }
}

/// The bytes a blob starts with that `sniff` needs to know it.
pub const SNIFF_LEN: usize = 16;

/// The type of a blob a PDS sent untyped (`application/octet-stream`, as
/// some PDSes send every blob), from its first bytes: only a kind the proxy
/// serves (JPEG, PNG, GIF, WebP, AVIF, HEIC; MP3, MP4 audio, Ogg, WAV,
/// FLAC; MP4, QuickTime, WebM video), never one that runs (no HTML, no
/// SVG). None for anything else.
pub fn sniff(head: &[u8]) -> Option<&'static str> {
    let at = |i: usize, s: &[u8]| head.get(i..i + s.len()) == Some(s);
    if at(0, b"\xFF\xD8\xFF") {
        return Some("image/jpeg");
    }
    if at(0, b"\x89PNG\r\n\x1a\n") {
        return Some("image/png");
    }
    if at(0, b"GIF87a") || at(0, b"GIF89a") {
        return Some("image/gif");
    }
    if at(0, b"RIFF") && at(8, b"WEBP") {
        return Some("image/webp");
    }
    if at(0, b"RIFF") && at(8, b"WAVE") {
        return Some("audio/wav");
    }
    if at(0, b"ID3") || at(0, b"\xFF\xFB") || at(0, b"\xFF\xF3") || at(0, b"\xFF\xF2") {
        return Some("audio/mpeg");
    }
    if at(0, b"OggS") {
        return Some("audio/ogg");
    }
    if at(0, b"fLaC") {
        return Some("audio/flac");
    }
    if at(0, b"\x1A\x45\xDF\xA3") {
        return Some("video/webm");
    }
    // ISO base media: a box of type `ftyp`, its brand saying which.
    if at(4, b"ftyp") {
        let brand = head.get(8..12)?;
        return match brand {
            b"avif" | b"avis" => Some("image/avif"),
            b"heic" | b"heix" | b"mif1" | b"msf1" => Some("image/heic"),
            b"M4A " | b"M4B " => Some("audio/mp4"),
            b"qt  " => Some("video/quicktime"),
            _ => Some("video/mp4"),
        };
    }
    None
}

/// The proxy's URL for a blob of `did`'s repository.
pub fn url(origin: &str, did: &str, cid: &str) -> String {
    format!("{origin}/blob/{did}/{cid}")
}

/// A blob ref's CID and MIME type, in either form; none if `o` is not one.
pub fn blob_ref(o: &Map<String, Value>) -> Option<(&str, &str)> {
    let mime = o.get("mimeType")?.as_str()?;
    let cid = if o.get("$type").and_then(Value::as_str) == Some("blob") {
        o.get("ref")?.get("$link")?.as_str()?
    } else if !o.contains_key("$type") {
        o.get("cid")?.as_str()?
    } else {
        return None;
    };
    atproto::Cid::parse(cid).is_ok().then_some((cid, mime))
}

/// A blob ref the proxy would serve: its CID and MIME type, if its type is
/// one the proxy serves and the size it declares, when it declares one, is
/// within the cap for that type. A blob that understates its size is still
/// cut off by the proxy itself.
pub fn servable(o: &Map<String, Value>) -> Option<(&str, &str)> {
    let (cid, mime) = blob_ref(o)?;
    let max = max_size(mime)?;
    let size = o.get("size").and_then(Value::as_u64).unwrap_or(0);
    (size <= max).then_some((cid, mime))
}

/// The image CDN where `HUD_IMAGE_CDN` names none: Bluesky's.
pub const IMAGE_CDN: &str = "https://cdn.bsky.app/img";

/// Bluesky's video service, whose playlists a public post's video names,
/// and the CDN its segments are served from: where the view plays a video's
/// HLS from.
pub const VIDEO_ORIGINS: [&str; 2] = ["https://video.bsky.app", "https://video.cdn.bsky.app"];

/// The image CDN's URL for a blob of `did`'s repository at one of its sizes
/// (`feed_fullsize`, `feed_thumbnail`, `avatar_thumbnail`), as a JPEG.
pub fn cdn_url(cdn: &str, preset: &str, did: &str, cid: &str) -> String {
    format!("{cdn}/{preset}/plain/{did}/{cid}@jpeg")
}

/// Whether a record was read where anyone can: a repository's
/// `com.atproto.repo.*`, or the Bluesky AppView. Every caller of `rewrite`
/// says which, as only a public record's images may go through the CDN.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Visibility {
    /// Anyone can read it.
    Public,
    /// Not everyone can: a record of a space, or a service's answer to the
    /// signed-in account but the AppView's. Its blobs stay on the proxy.
    Private,
}

/// Where blob URLs point: the server's proxy at `origin`, and the image CDN
/// (`HUD_IMAGE_CDN`, no trailing `/`), where there is one.
#[derive(Clone, Copy, Debug)]
pub struct Urls<'a> {
    pub origin: &'a str,
    pub cdn: Option<&'a str>,
}

impl<'a> Urls<'a> {
    /// The proxy alone, no CDN.
    pub fn proxy(origin: &'a str) -> Urls<'a> {
        Urls { origin, cdn: None }
    }
}

/// Fields whose picture is drawn small and square, an avatar's: their
/// `thumb` is the CDN's avatar thumbnail.
const AVATARS: [&str; 2] = ["avatar", "icon"];

/// Add a `url` (with a CDN, `thumb` and `proxy` too) beside every served
/// blob ref in `value`, a record of `did`'s repository read where
/// `visibility` says.
pub fn rewrite(value: &mut Value, urls: Urls, did: &str, visibility: Visibility) {
    let cdn = match visibility {
        Visibility::Public => urls.cdn,
        Visibility::Private => None,
    };
    walk(value, None, urls.origin, cdn, did);
}

fn walk(value: &mut Value, field: Option<&str>, origin: &str, cdn: Option<&str>, did: &str) {
    match value {
        Value::Object(o) => {
            if blob_ref(o).is_some() {
                if let Some((cid, mime)) = servable(o) {
                    let proxy = url(origin, did, cid);
                    match cdn.filter(|_| is_served_image(mime) && essence(mime) != "image/gif") {
                        Some(cdn) => {
                            let small = match field {
                                Some(f) if AVATARS.contains(&f) => "avatar_thumbnail",
                                _ => "feed_thumbnail",
                            };
                            let full = cdn_url(cdn, "feed_fullsize", did, cid);
                            let thumb = cdn_url(cdn, small, did, cid);
                            o.insert("url".into(), Value::String(full));
                            o.insert("thumb".into(), Value::String(thumb));
                            o.insert("proxy".into(), Value::String(proxy));
                        }
                        None => {
                            o.insert("url".into(), Value::String(proxy));
                        }
                    }
                }
                return;
            }
            for (k, v) in o.iter_mut() {
                walk(v, Some(k), origin, cdn, did);
            }
        }
        Value::Array(items) => items
            .iter_mut()
            .for_each(|v| walk(v, field, origin, cdn, did)),
        _ => {}
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    const CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy";
    const O: &str = "https://hud.example";

    fn blob(mime: &str) -> Value {
        json!({ "$type": "blob", "ref": { "$link": CID }, "mimeType": mime, "size": 10 })
    }

    fn with_url(mut b: Value) -> Value {
        b["url"] = json!(format!("{O}/blob/did:plc:x/{CID}"));
        b
    }

    const CDN: &str = "https://cdn.example/img";

    /// Rewritten with the proxy alone, as with no CDN.
    fn proxied(v: &mut Value) {
        rewrite(v, Urls::proxy(O), "did:plc:x", Visibility::Public);
    }

    /// Rewritten with a CDN, as read where `visibility` says.
    fn with_cdn(v: &mut Value, visibility: Visibility) {
        let urls = Urls {
            origin: O,
            cdn: Some(CDN),
        };
        rewrite(v, urls, "did:plc:x", visibility);
    }

    /// A record with every kind of blob: images in an array, an avatar, a
    /// GIF, audio, video and an SVG.
    fn every_kind() -> Value {
        json!({
            "embed": { "images": [ { "alt": "a", "image": blob("image/jpeg") },
                                    { "alt": "b", "image": blob("image/png") } ] },
            "avatar": blob("image/webp"),
            "gif": blob("image/gif"),
            "audioBlob": blob("audio/mpeg"),
            "video": blob("video/mp4"),
            "svg": blob("image/svg+xml"),
        })
    }

    /// A public record's images: the CDN's full size as `url`, its
    /// thumbnail (an avatar's for an avatar) as `thumb`, the proxy as
    /// `proxy`. A GIF, audio and video stay on the proxy; an SVG gets none.
    #[test]
    fn public_images_through_the_cdn() {
        let mut v = every_kind();
        with_cdn(&mut v, Visibility::Public);
        let cdn = |preset: &str| json!(format!("{CDN}/{preset}/plain/did:plc:x/{CID}@jpeg"));
        let proxy = json!(format!("{O}/blob/did:plc:x/{CID}"));
        for image in [
            &v["embed"]["images"][0]["image"],
            &v["embed"]["images"][1]["image"],
        ] {
            assert_eq!(image["url"], cdn("feed_fullsize"));
            assert_eq!(image["thumb"], cdn("feed_thumbnail"));
            assert_eq!(image["proxy"], proxy);
        }
        assert_eq!(v["avatar"]["url"], cdn("feed_fullsize"));
        assert_eq!(v["avatar"]["thumb"], cdn("avatar_thumbnail"));
        assert_eq!(v["avatar"]["proxy"], proxy);
        for kept in ["gif", "audioBlob", "video"] {
            let mime = v[kept]["mimeType"].as_str().unwrap().to_string();
            assert_eq!(v[kept], with_url(blob(&mime)), "{kept}");
        }
        assert_eq!(v["svg"], blob("image/svg+xml"));
    }

    /// A record not anyone can read (a space's), or any record with no CDN:
    /// every blob on the proxy alone, nothing naming the CDN.
    #[test]
    fn private_records_and_no_cdn_stay_on_the_proxy() {
        let mut private = every_kind();
        with_cdn(&mut private, Visibility::Private);
        let mut off = every_kind();
        proxied(&mut off);
        for v in [&private, &off] {
            assert!(!v.to_string().contains(CDN), "{v}");
            assert_eq!(
                v["embed"]["images"][0]["image"],
                with_url(blob("image/jpeg"))
            );
            assert_eq!(v["avatar"], with_url(blob("image/webp")));
            assert_eq!(v["gif"], with_url(blob("image/gif")));
            assert_eq!(v["audioBlob"], with_url(blob("audio/mpeg")));
            assert_eq!(v["svg"], blob("image/svg+xml"));
        }
        assert_eq!(private, off);
    }

    #[test]
    fn untyped_blobs_known_by_their_first_bytes() {
        let iso = |brand: &[u8]| [b"\0\0\0\x20ftyp".as_slice(), brand, b"\0\0\0\0"].concat();
        for (head, want) in [
            (b"\xFF\xD8\xFF\xE0\0\x10JFIF".to_vec(), Some("image/jpeg")),
            (b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec(), Some("image/png")),
            (b"GIF89a\x01\0".to_vec(), Some("image/gif")),
            (b"RIFF\0\0\0\0WEBPVP8 ".to_vec(), Some("image/webp")),
            (b"RIFF\0\0\0\0WAVEfmt ".to_vec(), Some("audio/wav")),
            (b"ID3\x04\0\0\0\0".to_vec(), Some("audio/mpeg")),
            (b"OggS\0\x02".to_vec(), Some("audio/ogg")),
            (b"fLaC\0\0\0\x22".to_vec(), Some("audio/flac")),
            (b"\x1A\x45\xDF\xA3\x9F".to_vec(), Some("video/webm")),
            (iso(b"avif"), Some("image/avif")),
            (iso(b"heic"), Some("image/heic")),
            (iso(b"M4A "), Some("audio/mp4")),
            (iso(b"qt  "), Some("video/quicktime")),
            (iso(b"isom"), Some("video/mp4")),
            // What runs, or is unknown, or too short to say: none.
            (b"<svg xmlns=".to_vec(), None),
            (b"<!doctype html>".to_vec(), None),
            (b"\0\0\0\x20ftyp".to_vec(), None),
            (Vec::new(), None),
        ] {
            assert_eq!(sniff(&head), want, "{head:?}");
            if let Some(t) = want {
                assert!(is_served(t), "{t} is served");
            }
        }
    }

    #[test]
    fn nested_and_arrays() {
        let mut v = json!({
            "text": "hi",
            "embed": { "images": [ { "alt": "a", "image": blob("image/jpeg") },
                                    { "alt": "b", "image": blob("image/png") } ] },
            "avatar": blob("image/webp"),
        });
        proxied(&mut v);
        assert_eq!(
            v,
            json!({
                "text": "hi",
                "embed": { "images": [ { "alt": "a", "image": with_url(blob("image/jpeg")) },
                                        { "alt": "b", "image": with_url(blob("image/png")) } ] },
                "avatar": with_url(blob("image/webp")),
            })
        );
    }

    #[test]
    fn legacy_form() {
        let mut v = json!({ "photo": { "cid": CID, "mimeType": "image/jpeg" } });
        proxied(&mut v);
        assert_eq!(
            v,
            json!({ "photo": { "cid": CID, "mimeType": "image/jpeg",
                               "url": format!("{O}/blob/did:plc:x/{CID}") } })
        );
    }

    #[test]
    fn audio_too() {
        let mut v = json!({ "audioBlob": blob("audio/wav"), "art": blob("image/png") });
        proxied(&mut v);
        assert_eq!(
            v,
            json!({ "audioBlob": with_url(blob("audio/wav")), "art": with_url(blob("image/png")) })
        );
        assert!(is_audio("Audio/MPEG; codecs=mp3"));
        assert!(!is_served("image/svg+xml; charset=utf-8"));
    }

    #[test]
    fn video_too() {
        // A raw video embed's blob gets a url, kept on disk as audio is.
        let mut v =
            json!({ "embed": { "$type": "app.bsky.embed.video", "video": blob("video/mp4") } });
        proxied(&mut v);
        assert_eq!(v["embed"]["video"], with_url(blob("video/mp4")));
        for mime in ["video/mp4", "Video/QuickTime", "video/webm; codecs=vp9"] {
            assert!(is_video(mime) && is_kept(mime), "{mime}");
            assert_eq!(max_size(mime), Some(VIDEO_MAX), "{mime}");
        }
        for mime in [
            "video/x-msvideo",
            "video/mp2t",
            "application/vnd.apple.mpegurl",
        ] {
            assert!(!is_served(mime), "{mime}");
        }
        assert!(is_kept("audio/mpeg") && !is_kept("image/jpeg"));
    }

    #[test]
    fn others_left_alone() {
        let untouched = json!({
            "video": blob("video/x-msvideo"),
            "svg": blob("image/svg+xml"),
            "notcid": { "cid": "hello", "mimeType": "image/png" },
            "typed": { "$type": "app.x", "cid": CID, "mimeType": "image/png" },
            "strongRef": { "uri": "at://did:plc:x/a.b.c/1", "cid": CID },
        });
        let mut v = untouched.clone();
        with_cdn(&mut v, Visibility::Public);
        assert_eq!(v, untouched);
    }

    /// A blob declaring a size past the proxy's cap for its kind gets no
    /// `url`, as the proxy would answer it 413; one at the cap does.
    #[test]
    fn past_the_cap_left_alone() {
        let sized = |mime: &str, size: u64| json!({ "$type": "blob", "ref": { "$link": CID }, "mimeType": mime, "size": size });
        let mut v = json!({
            "mix": sized("audio/mpeg", 170_092_668),
            "photo": sized("image/jpeg", IMAGE_MAX + 1),
        });
        let untouched = v.clone();
        with_cdn(&mut v, Visibility::Public);
        assert_eq!(v, untouched);

        let mut v = json!({ "track": sized("audio/mpeg", AUDIO_MAX) });
        proxied(&mut v);
        assert!(v["track"]["url"].is_string());
    }
}