//! Read after write (`spec/PROXY.md`): `getProfile`, `getProfiles`, //! `getTimeline`, `getAuthorFeed`, `getActorLikes` and `getPostThread` are //! forwarded, and their answers show the account's own records the target //! has not seen yet, those written by commits after the `atproto-repo-rev` //! it answers with. //! //! When a record was written is when the commit that wrote it was made, as //! its `rev`, a TID, says: so it survives a restart with the repository. use crate::proxy::{self, Request, Upstream}; use crate::{Resp, Service, Shared, Shown, XrpcError, read, sessions}; use axum::body::Bytes; use axum::http::{HeaderName, HeaderValue, header}; use pds_repo::data::{cid_string, datetime_millis, datetime_string, to_json}; use serde_json::{Map, Value as Json, json}; /// How many of the account's records an answer may show. const MAX_RECORDS: usize = 10; const PROFILE: &str = "app.bsky.actor.profile/self"; const POST: &str = "app.bsky.feed.post"; /// One of the account's records the target may not have seen. struct Written { path: String, uri: String, cid: String, value: Json, /// When it was written, in milliseconds since the epoch, and as a datetime. ms: i64, indexed_at: String, } /// The account's records written after a rev, and what an answer showing them needs. pub(crate) struct Local { did: String, handle: String, /// Oldest first. records: Vec, /// The account's profile as it is now. profile_now: Option, images: Images, } impl Local { fn profile(&self) -> Option<&Json> { self.records .iter() .find(|w| w.path == PROFILE) .map(|w| &w.value) } fn posts(&self) -> impl Iterator { self.records .iter() .filter(|w| w.path.split('/').next() == Some(POST)) } /// The milliseconds since the oldest of the records that is a profile or a post was written. fn lag(&self, now_ms: i64) -> Option { self.records .iter() .filter(|w| w.path == PROFILE || w.path.split('/').next() == Some(POST)) .map(|w| now_ms - w.ms) .max() } } /// When the commit of a rev was made, in milliseconds since the epoch: a /// TID holds the microseconds it was made at, above its 10 bits of clock id. fn tid_millis(rev: &str) -> i64 { const CHARS: &[u8; 32] = b"234567abcdefghijklmnopqrstuvwxyz"; let t = rev.bytes().fold(0u64, |n, c| { (n << 5) | CHARS.iter().position(|&x| x == c).unwrap_or(0) as u64 }); ((t >> 10) / 1000) as i64 } /// The images of an answer: on the AppView's CDN, or this server's `getBlob`. struct Images { cdn: Option, hostname: String, did: String, } impl Images { fn url(&self, kind: &str, cid: &str) -> String { match &self.cdn { Some(p) => p .replacen("%s", kind, 1) .replacen("%s", &self.did, 1) .replacen("%s", cid, 1), None => format!( "https://{}/xrpc/com.atproto.sync.getBlob?did={}&cid={cid}", self.hostname, self.did ), } } } /// The URL of an image of `did`'s, as answers show it: on the AppView's CDN /// if `PDS_BSKY_APP_VIEW_CDN_URL_PATTERN` names one, else this server's `getBlob`. pub(crate) fn image_url(s: &Service, kind: &str, did: &str, cid: &str) -> String { Images { cdn: s.config.proxy.cdn.clone(), hostname: s.config.hostname.clone(), did: did.to_string(), } .url(kind, cid) } /// The CID of a blob ref. fn blob_cid(blob: &Json) -> Option<&str> { blob["ref"]["$link"].as_str() } /// The account's records written by commits after `rev`, the first /// `MAX_RECORDS` by commit, of those the repository holds; None if there /// are none, or no record of the account's was written at or before `rev`. fn local_records(s: &Service, did: &str, rev: &str, shown: &mut Shown) -> Option { let account = s.account(did)?; let handle = account.handle(); let st = read(&account.store); let repo = &st.repo; // Read path by path, keeping the first by commit. let mut before = false; let mut after: Vec<(String, String)> = Vec::new(); st.each_path(|path, _, w| { if w <= rev { before = true; return; } after.push((w.to_string(), path.to_string())); if after.len() >= 4 * MAX_RECORDS { after.sort(); after.truncate(MAX_RECORDS); } }); if !before || after.is_empty() { return None; } after.sort(); after.truncate(MAX_RECORDS); let records = after .into_iter() .filter_map(|(w, path)| { let rec = st.record(&path)?; let (collection, rkey) = path.split_once('/')?; let ms = tid_millis(&w); Some(Written { path: path.clone(), uri: repo.uri(collection, rkey), cid: cid_string(&rec.cid), value: to_json(&rec.value), ms, indexed_at: datetime_string(ms), }) }) .collect(); let profile_now = st.record(PROFILE).map(|r| to_json(&r.value)); shown.0.push((st.wal.clone(), st.wal.written())); let images = Images { cdn: s.config.proxy.cdn.clone(), hostname: s.config.hostname.clone(), did: did.to_string(), }; Some(Local { did: did.to_string(), handle, records, profile_now, images, }) } /// Answer a read-after-write method: forward it, and show in its answer /// what the target has not seen yet. pub(crate) async fn answer(s: Shared, r: Request) -> Result { let t = sessions::access_token(&s, r.bearer())?; let target = proxy::target(&s, r.proxy.as_deref()).await?; let got = proxy::forward_query(&s, &r, &target, Some((&t.did, &target.did, &r.nsid))).await?; let rev = got .headers .get("atproto-repo-rev") .and_then(|v| v.to_str().ok()) .map(str::to_string); let json_type = got .headers .get(header::CONTENT_TYPE) .is_none_or(proxy::is_json); let (Some(rev), true, false) = ( rev, json_type, got.status >= 400 || r.nsid == "app.bsky.feed.getPostThread", ) else { return Ok(proxy::answered(got, true)); }; let mut shown = Shown::default(); let Some(local) = local_records(&s, &t.did, &rev, &mut shown) else { return Ok(proxy::answered(got, true)); }; let valid = serde_json::from_slice::(&got.body) .ok() .filter(|b| pds_lexicon::method(&r.nsid).is_some_and(|m| m.check_output(b).is_ok())); let Some(body) = valid else { return Ok(proxy::answered(got, true)); }; let body = munge(&r.nsid, body, &local); shown.durable().await?; let mut headers = vec![( header::CONTENT_TYPE, HeaderValue::from_static("application/json"), )]; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as i64; if let Some(lag) = local.lag(now) { headers.push(( HeaderName::from_static("atproto-upstream-lag"), HeaderValue::from(lag.max(0)), )); } Ok(Resp::Upstream(Upstream { status: 200, headers, body: Bytes::from(body.to_string()), })) } /// Set a view's fields from a profile record: `displayName` and /// `description` as they are, `avatar` and `banner` as image URLs, each /// left out where the profile has none. fn set_profile(view: &mut Json, profile: &Json, images: &Images, fields: &[&str]) { let Some(view) = view.as_object_mut() else { return; }; for &f in fields { view.remove(f); let value = match f { "avatar" | "banner" => blob_cid(&profile[f]).map(|cid| json!(images.url(f, cid))), _ => profile[f].as_str().map(|s| json!(s)), }; if let Some(v) = value { view.insert(f.to_string(), v); } } } /// Whether datetime `a` is earlier than `b`. fn earlier(a: &str, b: &str) -> bool { match (datetime_millis(a), datetime_millis(b)) { (Some(x), Some(y)) => x < y, _ => a < b, } } /// A post's view, as an AppView would show it before it saw any reply, like or repost. fn post_view(w: &Written, local: &Local) -> Json { let mut author = json!({ "did": local.did, "handle": local.handle }); if let Some(p) = &local.profile_now { set_profile(&mut author, p, &local.images, &["displayName", "avatar"]); } let mut view = json!({ "uri": w.uri, "cid": w.cid, "likeCount": 0, "replyCount": 0, "repostCount": 0, "quoteCount": 0, "author": author, "record": w.value, "indexedAt": w.indexed_at, }); if let Some(e) = embed_view(&w.value["embed"], &local.images) { view["embed"] = e; } view } /// The view of a post's embed of images or a link card; None for others. fn embed_view(embed: &Json, images: &Images) -> Option { let copy = |from: &Json, keys: &[&str], to: &mut Map| { for &k in keys { if let Some(v) = from.get(k) { to.insert(k.to_string(), v.clone()); } } }; match embed["$type"].as_str()? { "app.bsky.embed.images" => { let views: Vec = embed["images"] .as_array()? .iter() .map(|img| { let mut v = Map::new(); if let Some(cid) = blob_cid(&img["image"]) { v.insert("thumb".into(), json!(images.url("feed_thumbnail", cid))); v.insert("fullsize".into(), json!(images.url("feed_fullsize", cid))); } copy(img, &["alt", "aspectRatio"], &mut v); Json::Object(v) }) .collect(); Some(json!({ "$type": "app.bsky.embed.images#view", "images": views })) } "app.bsky.embed.external" => { let ext = &embed["external"]; let mut v = Map::new(); copy(ext, &["uri", "title", "description"], &mut v); if let Some(cid) = blob_cid(&ext["thumb"]) { v.insert("thumb".into(), json!(images.url("feed_thumbnail", cid))); } Some(json!({ "$type": "app.bsky.embed.external#view", "external": v })) } _ => None, } } /// Put the account's posts into a feed: each written after the feed's last /// item, newest first, before the first item indexed earlier than it, or at the end. fn insert_posts(feed: &mut Vec, local: &Local) { let last = feed .last() .and_then(|i| i["post"]["indexedAt"].as_str()) .map(str::to_string); let mut posts: Vec<&Written> = local .posts() .filter(|p| last.as_deref().is_none_or(|l| earlier(l, &p.indexed_at))) .collect(); posts.sort_by_key(|p| std::cmp::Reverse(p.ms)); for p in posts { let item = json!({ "post": post_view(p, local) }); match feed.iter().position(|i| { i["post"]["indexedAt"] .as_str() .is_some_and(|t| earlier(t, &p.indexed_at)) }) { Some(i) => feed.insert(i, item), None => feed.push(item), } } } /// The authors of a feed's posts that are the account, as its profile has them. fn set_authors(feed: &mut [Json], local: &Local) { let Some(profile) = local.profile() else { return; }; for item in feed { let author = &mut item["post"]["author"]; if author["did"] == local.did.as_str() { set_profile(author, profile, &local.images, &["displayName", "avatar"]); } } } /// An answer, valid by its method's output lexicon, showing the account's records. fn munge(nsid: &str, mut body: Json, local: &Local) -> Json { const ALL: &[&str] = &["displayName", "description", "avatar", "banner"]; match nsid { "app.bsky.actor.getProfile" => { if let (Some(p), true) = (local.profile(), body["did"] == local.did.as_str()) { set_profile(&mut body, p, &local.images, ALL); } } "app.bsky.actor.getProfiles" => { if let (Some(p), Some(views)) = (local.profile(), body["profiles"].as_array_mut()) { for v in views.iter_mut().filter(|v| v["did"] == local.did.as_str()) { set_profile(v, p, &local.images, ALL); } } } "app.bsky.feed.getActorLikes" => { if let Some(feed) = body["feed"].as_array_mut() { set_authors(feed, local); } } "app.bsky.feed.getAuthorFeed" => { if let Some(feed) = body["feed"].as_array_mut() { let own = feed.first().is_some_and(|first| match first.get("reason") { None => first["post"]["author"]["did"] == local.did.as_str(), Some(r) => { r["$type"] == "app.bsky.feed.defs#reasonRepost" && r["by"]["did"] == local.did.as_str() } }); if own { set_authors(feed, local); insert_posts(feed, local); } } } "app.bsky.feed.getTimeline" => { if let Some(feed) = body["feed"].as_array_mut() { insert_posts(feed, local); } } _ => {} } body } #[cfg(test)] mod tests { use super::*; const DID: &str = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; fn images() -> Images { Images { cdn: Some("http://cdn.test/%s/%s/%s".into()), hostname: "pds.test".into(), did: DID.into(), } } fn blob(cid: &str) -> Json { json!({ "$type": "blob", "ref": { "$link": cid }, "mimeType": "image/png", "size": 3 }) } fn written(path: &str, value: Json, ms: i64) -> Written { let (collection, rkey) = path.split_once('/').unwrap(); Written { path: path.into(), uri: format!("at://{DID}/{collection}/{rkey}"), cid: format!("cid-{rkey}"), value, ms, indexed_at: datetime_string(ms), } } fn local(records: Vec) -> Local { let profile_now = records .iter() .find(|w| w.path == PROFILE) .map(|w| w.value.clone()); Local { did: DID.into(), handle: "alice.test".into(), records, profile_now, images: images(), } } #[test] fn tids_tell_when() { let mut tids = pds_repo::Tids::new(); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_millis() as i64; let at = tid_millis(&tids.next_tid()); assert!((now - at).abs() < 1000, "{now} {at}"); } #[test] fn images_are_urls() { assert_eq!( images().url("avatar", "bafy"), format!("http://cdn.test/avatar/{DID}/bafy") ); let plain = Images { cdn: None, ..images() }; assert_eq!( plain.url("banner", "bafy"), format!("https://pds.test/xrpc/com.atproto.sync.getBlob?did={DID}&cid=bafy") ); } #[test] fn profiles_show_the_new_profile() { let l = local(vec![written( PROFILE, json!({ "$type": "app.bsky.actor.profile", "displayName": "Al", "avatar": blob("bafya") }), 1000, )]); let view = json!({ "did": DID, "handle": "alice.test", "displayName": "Old", "description": "old", "banner": "x", "followersCount": 3 }); let got = munge("app.bsky.actor.getProfile", view.clone(), &l); assert_eq!( got, json!({ "did": DID, "handle": "alice.test", "displayName": "Al", "avatar": format!("http://cdn.test/avatar/{DID}/bafya"), "followersCount": 3 }) ); let other = json!({ "did": "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb", "handle": "bob.test", "displayName": "Bob" }); assert_eq!(munge("app.bsky.actor.getProfile", other.clone(), &l), other); let many = munge( "app.bsky.actor.getProfiles", json!({ "profiles": [other, view] }), &l, ); assert_eq!(many["profiles"][0]["displayName"], "Bob"); assert_eq!(many["profiles"][1]["displayName"], "Al"); // Without the profile among the records, nothing changes. let none = local(vec![written( "app.bsky.feed.post/a", json!({ "text": "hi" }), 1000, )]); assert_eq!( munge( "app.bsky.actor.getProfile", json!({ "did": DID, "handle": "h", "displayName": "Old" }), &none )["displayName"], "Old" ); } #[test] fn timelines_get_new_posts_in_place() { let at = |ms: i64| datetime_string(ms); let item = |ms: i64| json!({ "post": { "uri": "at://x", "indexedAt": at(ms), "author": { "did": "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" } } }); let post = |rkey: &str, ms: i64, embed: Json| { let mut v = json!({ "$type": "app.bsky.feed.post", "text": rkey, "createdAt": at(ms) }); if !embed.is_null() { v["embed"] = embed; } written(&format!("{POST}/{rkey}"), v, ms) }; let images_embed = json!({ "$type": "app.bsky.embed.images", "images": [{ "image": blob("bafyi"), "alt": "a cat" }] }); let link = json!({ "$type": "app.bsky.embed.external", "external": { "uri": "https://x.test", "title": "T", "description": "D", "thumb": blob("bafyt") } }); let l = local(vec![ post("p1", 1500, images_embed), post("p2", 2500, link), post("p0", 500, Json::Null), ]); let got = munge( "app.bsky.feed.getTimeline", json!({ "feed": [item(3000), item(2000), item(1000)], "cursor": "c" }), &l, ); let times: Vec = got["feed"] .as_array() .unwrap() .iter() .map(|i| datetime_millis(i["post"]["indexedAt"].as_str().unwrap()).unwrap()) .collect(); // p0 is older than the feed's last item, and is left out. assert_eq!(times, [3000, 2500, 2000, 1500, 1000]); assert_eq!(got["cursor"], "c"); let p2 = &got["feed"][1]["post"]; assert_eq!( p2["embed"], json!({ "$type": "app.bsky.embed.external#view", "external": { "uri": "https://x.test", "title": "T", "description": "D", "thumb": format!("http://cdn.test/feed_thumbnail/{DID}/bafyt") } }) ); assert_eq!(p2["author"], json!({ "did": DID, "handle": "alice.test" })); assert_eq!( (p2["likeCount"].clone(), p2["record"]["text"].clone()), (json!(0), json!("p2")) ); let p1 = &got["feed"][3]["post"]; assert_eq!( p1["embed"], json!({ "$type": "app.bsky.embed.images#view", "images": [{ "thumb": format!("http://cdn.test/feed_thumbnail/{DID}/bafyi"), "fullsize": format!("http://cdn.test/feed_fullsize/{DID}/bafyi"), "alt": "a cat" }] }) ); // An empty feed takes every post, newest first. let empty = munge("app.bsky.feed.getTimeline", json!({ "feed": [] }), &l); let texts: Vec<&str> = empty["feed"] .as_array() .unwrap() .iter() .map(|i| i["post"]["record"]["text"].as_str().unwrap()) .collect(); assert_eq!(texts, ["p2", "p1", "p0"]); // An author feed changes only when it is the account's own. let others = json!({ "feed": [item(3000)] }); assert_eq!( munge("app.bsky.feed.getAuthorFeed", others.clone(), &l), others ); let own = json!({ "feed": [{ "post": { "uri": "at://x", "indexedAt": at(1000), "author": { "did": DID, "displayName": "Old" } } }] }); assert_eq!( munge("app.bsky.feed.getAuthorFeed", own, &l)["feed"] .as_array() .unwrap() .len(), 3 ); let lag = l.lag(10_000).unwrap(); assert_eq!(lag, 9_500); } #[test] fn likes_show_the_new_profile() { let l = local(vec![written(PROFILE, json!({ "displayName": "Al" }), 1000)]); let feed = json!({ "feed": [{ "post": { "author": { "did": DID, "displayName": "Old", "avatar": "x" } } }, { "post": { "author": { "did": "did:plc:b" } } }] }); let got = munge("app.bsky.feed.getActorLikes", feed, &l); assert_eq!( got["feed"][0]["post"]["author"], json!({ "did": DID, "displayName": "Al" }) ); assert_eq!( got["feed"][1]["post"]["author"], json!({ "did": "did:plc:b" }) ); } }