~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
594 lines · 21.0 KB Raw History
//! Read after write (`spec/PROXY.md`): `getProfile`, `getProfiles`,
//! `getTimeline`, `getAuthorFeed`, `getActorLikes` and `getPostThread` are
//! forwarded, and their answers show the account's own records the target
//! has not seen yet, those written by commits after the `atproto-repo-rev`
//! it answers with.
//!
//! When a record was written is when the commit that wrote it was made, as
//! its `rev`, a TID, says: so it survives a restart with the repository.

use crate::proxy::{self, Request, Upstream};
use crate::{Resp, Service, Shared, Shown, XrpcError, read, sessions};
use axum::body::Bytes;
use axum::http::{HeaderName, HeaderValue, header};
use pds_repo::data::{cid_string, datetime_millis, datetime_string, to_json};
use serde_json::{Map, Value as Json, json};

/// How many of the account's records an answer may show.
const MAX_RECORDS: usize = 10;

const PROFILE: &str = "app.bsky.actor.profile/self";
const POST: &str = "app.bsky.feed.post";

/// One of the account's records the target may not have seen.
struct Written {
    path: String,
    uri: String,
    cid: String,
    value: Json,
    /// When it was written, in milliseconds since the epoch, and as a datetime.
    ms: i64,
    indexed_at: String,
}

/// The account's records written after a rev, and what an answer showing them needs.
pub(crate) struct Local {
    did: String,
    handle: String,
    /// Oldest first.
    records: Vec<Written>,
    /// The account's profile as it is now.
    profile_now: Option<Json>,
    images: Images,
}

impl Local {
    fn profile(&self) -> Option<&Json> {
        self.records
            .iter()
            .find(|w| w.path == PROFILE)
            .map(|w| &w.value)
    }

    fn posts(&self) -> impl Iterator<Item = &Written> {
        self.records
            .iter()
            .filter(|w| w.path.split('/').next() == Some(POST))
    }

    /// The milliseconds since the oldest of the records that is a profile or a post was written.
    fn lag(&self, now_ms: i64) -> Option<i64> {
        self.records
            .iter()
            .filter(|w| w.path == PROFILE || w.path.split('/').next() == Some(POST))
            .map(|w| now_ms - w.ms)
            .max()
    }
}

/// When the commit of a rev was made, in milliseconds since the epoch: a
/// TID holds the microseconds it was made at, above its 10 bits of clock id.
fn tid_millis(rev: &str) -> i64 {
    const CHARS: &[u8; 32] = b"234567abcdefghijklmnopqrstuvwxyz";
    let t = rev.bytes().fold(0u64, |n, c| {
        (n << 5) | CHARS.iter().position(|&x| x == c).unwrap_or(0) as u64
    });
    ((t >> 10) / 1000) as i64
}

/// The images of an answer: on the AppView's CDN, or this server's `getBlob`.
struct Images {
    cdn: Option<String>,
    hostname: String,
    did: String,
}

impl Images {
    fn url(&self, kind: &str, cid: &str) -> String {
        match &self.cdn {
            Some(p) => p
                .replacen("%s", kind, 1)
                .replacen("%s", &self.did, 1)
                .replacen("%s", cid, 1),
            None => format!(
                "https://{}/xrpc/com.atproto.sync.getBlob?did={}&cid={cid}",
                self.hostname, self.did
            ),
        }
    }
}

/// The URL of an image of `did`'s, as answers show it: on the AppView's CDN
/// if `PDS_BSKY_APP_VIEW_CDN_URL_PATTERN` names one, else this server's `getBlob`.
pub(crate) fn image_url(s: &Service, kind: &str, did: &str, cid: &str) -> String {
    Images {
        cdn: s.config.proxy.cdn.clone(),
        hostname: s.config.hostname.clone(),
        did: did.to_string(),
    }
    .url(kind, cid)
}

/// The CID of a blob ref.
fn blob_cid(blob: &Json) -> Option<&str> {
    blob["ref"]["$link"].as_str()
}

/// The account's records written by commits after `rev`, the first
/// `MAX_RECORDS` by commit, of those the repository holds; None if there
/// are none, or no record of the account's was written at or before `rev`.
fn local_records(s: &Service, did: &str, rev: &str, shown: &mut Shown) -> Option<Local> {
    let account = s.account(did)?;
    let handle = account.handle();
    let st = read(&account.store);
    let repo = &st.repo;
    // Read path by path, keeping the first by commit.
    let mut before = false;
    let mut after: Vec<(String, String)> = Vec::new();
    st.each_path(|path, _, w| {
        if w <= rev {
            before = true;
            return;
        }
        after.push((w.to_string(), path.to_string()));
        if after.len() >= 4 * MAX_RECORDS {
            after.sort();
            after.truncate(MAX_RECORDS);
        }
    });
    if !before || after.is_empty() {
        return None;
    }
    after.sort();
    after.truncate(MAX_RECORDS);
    let records = after
        .into_iter()
        .filter_map(|(w, path)| {
            let rec = st.record(&path)?;
            let (collection, rkey) = path.split_once('/')?;
            let ms = tid_millis(&w);
            Some(Written {
                path: path.clone(),
                uri: repo.uri(collection, rkey),
                cid: cid_string(&rec.cid),
                value: to_json(&rec.value),
                ms,
                indexed_at: datetime_string(ms),
            })
        })
        .collect();
    let profile_now = st.record(PROFILE).map(|r| to_json(&r.value));
    shown.0.push((st.wal.clone(), st.wal.written()));
    let images = Images {
        cdn: s.config.proxy.cdn.clone(),
        hostname: s.config.hostname.clone(),
        did: did.to_string(),
    };
    Some(Local {
        did: did.to_string(),
        handle,
        records,
        profile_now,
        images,
    })
}

/// Answer a read-after-write method: forward it, and show in its answer
/// what the target has not seen yet.
pub(crate) async fn answer(s: Shared, r: Request) -> Result<Resp, XrpcError> {
    let t = sessions::access_token(&s, r.bearer())?;
    let target = proxy::target(&s, r.proxy.as_deref()).await?;
    let got = proxy::forward_query(&s, &r, &target, Some((&t.did, &target.did, &r.nsid))).await?;
    let rev = got
        .headers
        .get("atproto-repo-rev")
        .and_then(|v| v.to_str().ok())
        .map(str::to_string);
    let json_type = got
        .headers
        .get(header::CONTENT_TYPE)
        .is_none_or(proxy::is_json);
    let (Some(rev), true, false) = (
        rev,
        json_type,
        got.status >= 400 || r.nsid == "app.bsky.feed.getPostThread",
    ) else {
        return Ok(proxy::answered(got, true));
    };
    let mut shown = Shown::default();
    let Some(local) = local_records(&s, &t.did, &rev, &mut shown) else {
        return Ok(proxy::answered(got, true));
    };
    let valid = serde_json::from_slice::<Json>(&got.body)
        .ok()
        .filter(|b| pds_lexicon::method(&r.nsid).is_some_and(|m| m.check_output(b).is_ok()));
    let Some(body) = valid else {
        return Ok(proxy::answered(got, true));
    };
    let body = munge(&r.nsid, body, &local);
    shown.durable().await?;
    let mut headers = vec![(
        header::CONTENT_TYPE,
        HeaderValue::from_static("application/json"),
    )];
    let now = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap_or_default()
        .as_millis() as i64;
    if let Some(lag) = local.lag(now) {
        headers.push((
            HeaderName::from_static("atproto-upstream-lag"),
            HeaderValue::from(lag.max(0)),
        ));
    }
    Ok(Resp::Upstream(Upstream {
        status: 200,
        headers,
        body: Bytes::from(body.to_string()),
    }))
}

/// Set a view's fields from a profile record: `displayName` and
/// `description` as they are, `avatar` and `banner` as image URLs, each
/// left out where the profile has none.
fn set_profile(view: &mut Json, profile: &Json, images: &Images, fields: &[&str]) {
    let Some(view) = view.as_object_mut() else {
        return;
    };
    for &f in fields {
        view.remove(f);
        let value = match f {
            "avatar" | "banner" => blob_cid(&profile[f]).map(|cid| json!(images.url(f, cid))),
            _ => profile[f].as_str().map(|s| json!(s)),
        };
        if let Some(v) = value {
            view.insert(f.to_string(), v);
        }
    }
}

/// Whether datetime `a` is earlier than `b`.
fn earlier(a: &str, b: &str) -> bool {
    match (datetime_millis(a), datetime_millis(b)) {
        (Some(x), Some(y)) => x < y,
        _ => a < b,
    }
}

/// A post's view, as an AppView would show it before it saw any reply, like or repost.
fn post_view(w: &Written, local: &Local) -> Json {
    let mut author = json!({ "did": local.did, "handle": local.handle });
    if let Some(p) = &local.profile_now {
        set_profile(&mut author, p, &local.images, &["displayName", "avatar"]);
    }
    let mut view = json!({
        "uri": w.uri, "cid": w.cid, "likeCount": 0, "replyCount": 0, "repostCount": 0, "quoteCount": 0,
        "author": author, "record": w.value, "indexedAt": w.indexed_at,
    });
    if let Some(e) = embed_view(&w.value["embed"], &local.images) {
        view["embed"] = e;
    }
    view
}

/// The view of a post's embed of images or a link card; None for others.
fn embed_view(embed: &Json, images: &Images) -> Option<Json> {
    let copy = |from: &Json, keys: &[&str], to: &mut Map<String, Json>| {
        for &k in keys {
            if let Some(v) = from.get(k) {
                to.insert(k.to_string(), v.clone());
            }
        }
    };
    match embed["$type"].as_str()? {
        "app.bsky.embed.images" => {
            let views: Vec<Json> = embed["images"]
                .as_array()?
                .iter()
                .map(|img| {
                    let mut v = Map::new();
                    if let Some(cid) = blob_cid(&img["image"]) {
                        v.insert("thumb".into(), json!(images.url("feed_thumbnail", cid)));
                        v.insert("fullsize".into(), json!(images.url("feed_fullsize", cid)));
                    }
                    copy(img, &["alt", "aspectRatio"], &mut v);
                    Json::Object(v)
                })
                .collect();
            Some(json!({ "$type": "app.bsky.embed.images#view", "images": views }))
        }
        "app.bsky.embed.external" => {
            let ext = &embed["external"];
            let mut v = Map::new();
            copy(ext, &["uri", "title", "description"], &mut v);
            if let Some(cid) = blob_cid(&ext["thumb"]) {
                v.insert("thumb".into(), json!(images.url("feed_thumbnail", cid)));
            }
            Some(json!({ "$type": "app.bsky.embed.external#view", "external": v }))
        }
        _ => None,
    }
}

/// Put the account's posts into a feed: each written after the feed's last
/// item, newest first, before the first item indexed earlier than it, or at the end.
fn insert_posts(feed: &mut Vec<Json>, local: &Local) {
    let last = feed
        .last()
        .and_then(|i| i["post"]["indexedAt"].as_str())
        .map(str::to_string);
    let mut posts: Vec<&Written> = local
        .posts()
        .filter(|p| last.as_deref().is_none_or(|l| earlier(l, &p.indexed_at)))
        .collect();
    posts.sort_by_key(|p| std::cmp::Reverse(p.ms));
    for p in posts {
        let item = json!({ "post": post_view(p, local) });
        match feed.iter().position(|i| {
            i["post"]["indexedAt"]
                .as_str()
                .is_some_and(|t| earlier(t, &p.indexed_at))
        }) {
            Some(i) => feed.insert(i, item),
            None => feed.push(item),
        }
    }
}

/// The authors of a feed's posts that are the account, as its profile has them.
fn set_authors(feed: &mut [Json], local: &Local) {
    let Some(profile) = local.profile() else {
        return;
    };
    for item in feed {
        let author = &mut item["post"]["author"];
        if author["did"] == local.did.as_str() {
            set_profile(author, profile, &local.images, &["displayName", "avatar"]);
        }
    }
}

/// An answer, valid by its method's output lexicon, showing the account's records.
fn munge(nsid: &str, mut body: Json, local: &Local) -> Json {
    const ALL: &[&str] = &["displayName", "description", "avatar", "banner"];
    match nsid {
        "app.bsky.actor.getProfile" => {
            if let (Some(p), true) = (local.profile(), body["did"] == local.did.as_str()) {
                set_profile(&mut body, p, &local.images, ALL);
            }
        }
        "app.bsky.actor.getProfiles" => {
            if let (Some(p), Some(views)) = (local.profile(), body["profiles"].as_array_mut()) {
                for v in views.iter_mut().filter(|v| v["did"] == local.did.as_str()) {
                    set_profile(v, p, &local.images, ALL);
                }
            }
        }
        "app.bsky.feed.getActorLikes" => {
            if let Some(feed) = body["feed"].as_array_mut() {
                set_authors(feed, local);
            }
        }
        "app.bsky.feed.getAuthorFeed" => {
            if let Some(feed) = body["feed"].as_array_mut() {
                let own = feed.first().is_some_and(|first| match first.get("reason") {
                    None => first["post"]["author"]["did"] == local.did.as_str(),
                    Some(r) => {
                        r["$type"] == "app.bsky.feed.defs#reasonRepost"
                            && r["by"]["did"] == local.did.as_str()
                    }
                });
                if own {
                    set_authors(feed, local);
                    insert_posts(feed, local);
                }
            }
        }
        "app.bsky.feed.getTimeline" => {
            if let Some(feed) = body["feed"].as_array_mut() {
                insert_posts(feed, local);
            }
        }
        _ => {}
    }
    body
}

#[cfg(test)]
mod tests {
    use super::*;

    const DID: &str = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";

    fn images() -> Images {
        Images {
            cdn: Some("http://cdn.test/%s/%s/%s".into()),
            hostname: "pds.test".into(),
            did: DID.into(),
        }
    }

    fn blob(cid: &str) -> Json {
        json!({ "$type": "blob", "ref": { "$link": cid }, "mimeType": "image/png", "size": 3 })
    }

    fn written(path: &str, value: Json, ms: i64) -> Written {
        let (collection, rkey) = path.split_once('/').unwrap();
        Written {
            path: path.into(),
            uri: format!("at://{DID}/{collection}/{rkey}"),
            cid: format!("cid-{rkey}"),
            value,
            ms,
            indexed_at: datetime_string(ms),
        }
    }

    fn local(records: Vec<Written>) -> Local {
        let profile_now = records
            .iter()
            .find(|w| w.path == PROFILE)
            .map(|w| w.value.clone());
        Local {
            did: DID.into(),
            handle: "alice.test".into(),
            records,
            profile_now,
            images: images(),
        }
    }

    #[test]
    fn tids_tell_when() {
        let mut tids = pds_repo::Tids::new();
        let now = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_millis() as i64;
        let at = tid_millis(&tids.next_tid());
        assert!((now - at).abs() < 1000, "{now} {at}");
    }

    #[test]
    fn images_are_urls() {
        assert_eq!(
            images().url("avatar", "bafy"),
            format!("http://cdn.test/avatar/{DID}/bafy")
        );
        let plain = Images {
            cdn: None,
            ..images()
        };
        assert_eq!(
            plain.url("banner", "bafy"),
            format!("https://pds.test/xrpc/com.atproto.sync.getBlob?did={DID}&cid=bafy")
        );
    }

    #[test]
    fn profiles_show_the_new_profile() {
        let l = local(vec![written(
            PROFILE,
            json!({ "$type": "app.bsky.actor.profile", "displayName": "Al", "avatar": blob("bafya") }),
            1000,
        )]);
        let view = json!({ "did": DID, "handle": "alice.test", "displayName": "Old", "description": "old", "banner": "x", "followersCount": 3 });
        let got = munge("app.bsky.actor.getProfile", view.clone(), &l);
        assert_eq!(
            got,
            json!({ "did": DID, "handle": "alice.test", "displayName": "Al", "avatar": format!("http://cdn.test/avatar/{DID}/bafya"), "followersCount": 3 })
        );
        let other = json!({ "did": "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb", "handle": "bob.test", "displayName": "Bob" });
        assert_eq!(munge("app.bsky.actor.getProfile", other.clone(), &l), other);
        let many = munge(
            "app.bsky.actor.getProfiles",
            json!({ "profiles": [other, view] }),
            &l,
        );
        assert_eq!(many["profiles"][0]["displayName"], "Bob");
        assert_eq!(many["profiles"][1]["displayName"], "Al");
        // Without the profile among the records, nothing changes.
        let none = local(vec![written(
            "app.bsky.feed.post/a",
            json!({ "text": "hi" }),
            1000,
        )]);
        assert_eq!(
            munge(
                "app.bsky.actor.getProfile",
                json!({ "did": DID, "handle": "h", "displayName": "Old" }),
                &none
            )["displayName"],
            "Old"
        );
    }

    #[test]
    fn timelines_get_new_posts_in_place() {
        let at = |ms: i64| datetime_string(ms);
        let item = |ms: i64| json!({ "post": { "uri": "at://x", "indexedAt": at(ms), "author": { "did": "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" } } });
        let post = |rkey: &str, ms: i64, embed: Json| {
            let mut v = json!({ "$type": "app.bsky.feed.post", "text": rkey, "createdAt": at(ms) });
            if !embed.is_null() {
                v["embed"] = embed;
            }
            written(&format!("{POST}/{rkey}"), v, ms)
        };
        let images_embed = json!({ "$type": "app.bsky.embed.images", "images": [{ "image": blob("bafyi"), "alt": "a cat" }] });
        let link = json!({ "$type": "app.bsky.embed.external", "external": { "uri": "https://x.test", "title": "T", "description": "D", "thumb": blob("bafyt") } });
        let l = local(vec![
            post("p1", 1500, images_embed),
            post("p2", 2500, link),
            post("p0", 500, Json::Null),
        ]);
        let got = munge(
            "app.bsky.feed.getTimeline",
            json!({ "feed": [item(3000), item(2000), item(1000)], "cursor": "c" }),
            &l,
        );
        let times: Vec<i64> = got["feed"]
            .as_array()
            .unwrap()
            .iter()
            .map(|i| datetime_millis(i["post"]["indexedAt"].as_str().unwrap()).unwrap())
            .collect();
        // p0 is older than the feed's last item, and is left out.
        assert_eq!(times, [3000, 2500, 2000, 1500, 1000]);
        assert_eq!(got["cursor"], "c");
        let p2 = &got["feed"][1]["post"];
        assert_eq!(
            p2["embed"],
            json!({ "$type": "app.bsky.embed.external#view", "external": { "uri": "https://x.test", "title": "T", "description": "D", "thumb": format!("http://cdn.test/feed_thumbnail/{DID}/bafyt") } })
        );
        assert_eq!(p2["author"], json!({ "did": DID, "handle": "alice.test" }));
        assert_eq!(
            (p2["likeCount"].clone(), p2["record"]["text"].clone()),
            (json!(0), json!("p2"))
        );
        let p1 = &got["feed"][3]["post"];
        assert_eq!(
            p1["embed"],
            json!({ "$type": "app.bsky.embed.images#view", "images": [{ "thumb": format!("http://cdn.test/feed_thumbnail/{DID}/bafyi"), "fullsize": format!("http://cdn.test/feed_fullsize/{DID}/bafyi"), "alt": "a cat" }] })
        );
        // An empty feed takes every post, newest first.
        let empty = munge("app.bsky.feed.getTimeline", json!({ "feed": [] }), &l);
        let texts: Vec<&str> = empty["feed"]
            .as_array()
            .unwrap()
            .iter()
            .map(|i| i["post"]["record"]["text"].as_str().unwrap())
            .collect();
        assert_eq!(texts, ["p2", "p1", "p0"]);
        // An author feed changes only when it is the account's own.
        let others = json!({ "feed": [item(3000)] });
        assert_eq!(
            munge("app.bsky.feed.getAuthorFeed", others.clone(), &l),
            others
        );
        let own = json!({ "feed": [{ "post": { "uri": "at://x", "indexedAt": at(1000), "author": { "did": DID, "displayName": "Old" } } }] });
        assert_eq!(
            munge("app.bsky.feed.getAuthorFeed", own, &l)["feed"]
                .as_array()
                .unwrap()
                .len(),
            3
        );
        let lag = l.lag(10_000).unwrap();
        assert_eq!(lag, 9_500);
    }

    #[test]
    fn likes_show_the_new_profile() {
        let l = local(vec![written(PROFILE, json!({ "displayName": "Al" }), 1000)]);
        let feed = json!({ "feed": [{ "post": { "author": { "did": DID, "displayName": "Old", "avatar": "x" } } }, { "post": { "author": { "did": "did:plc:b" } } }] });
        let got = munge("app.bsky.feed.getActorLikes", feed, &l);
        assert_eq!(
            got["feed"][0]["post"]["author"],
            json!({ "did": DID, "displayName": "Al" })
        );
        assert_eq!(
            got["feed"][1]["post"]["author"],
            json!({ "did": "did:plc:b" })
        );
    }
}