chadtmiller.com/monolith
An atproto PDS
0
0
//! Read after write (`spec/PROXY.md`): `getProfile`, `getProfiles`,
//! `getTimeline`, `getAuthorFeed`, `getActorLikes` and `getPostThread` are
//! forwarded, and their answers show the account's own records the target
//! has not seen yet, those written by commits after the `atproto-repo-rev`
//! it answers with.
//!
//! When a record was written is when the commit that wrote it was made, as
//! its `rev`, a TID, says: so it survives a restart with the repository.
use crate::proxy::{self, Request, Upstream};
use crate::{Resp, Service, Shared, Shown, XrpcError, read, sessions};
use axum::body::Bytes;
use axum::http::{HeaderName, HeaderValue, header};
use pds_repo::data::{cid_string, datetime_millis, datetime_string, to_json};
use serde_json::{Map, Value as Json, json};
/// How many of the account's records an answer may show.
const MAX_RECORDS: usize = 10;
const PROFILE: &str = "app.bsky.actor.profile/self";
const POST: &str = "app.bsky.feed.post";
/// One of the account's records the target may not have seen.
struct Written {
path: String,
uri: String,
cid: String,
value: Json,
/// When it was written, in milliseconds since the epoch, and as a datetime.
ms: i64,
indexed_at: String,
}
/// The account's records written after a rev, and what an answer showing them needs.
pub(crate) struct Local {
did: String,
handle: String,
/// Oldest first.
records: Vec<Written>,
/// The account's profile as it is now.
profile_now: Option<Json>,
images: Images,
}
impl Local {
fn profile(&self) -> Option<&Json> {
self.records
.iter()
.find(|w| w.path == PROFILE)
.map(|w| &w.value)
}
fn posts(&self) -> impl Iterator<Item = &Written> {
self.records
.iter()
.filter(|w| w.path.split('/').next() == Some(POST))
}
/// The milliseconds since the oldest of the records that is a profile or a post was written.
fn lag(&self, now_ms: i64) -> Option<i64> {
self.records
.iter()
.filter(|w| w.path == PROFILE || w.path.split('/').next() == Some(POST))
.map(|w| now_ms - w.ms)
.max()
}
}
/// When the commit of a rev was made, in milliseconds since the epoch: a
/// TID holds the microseconds it was made at, above its 10 bits of clock id.
fn tid_millis(rev: &str) -> i64 {
const CHARS: &[u8; 32] = b"234567abcdefghijklmnopqrstuvwxyz";
let t = rev.bytes().fold(0u64, |n, c| {
(n << 5) | CHARS.iter().position(|&x| x == c).unwrap_or(0) as u64
});
((t >> 10) / 1000) as i64
}
/// The images of an answer: on the AppView's CDN, or this server's `getBlob`.
struct Images {
cdn: Option<String>,
hostname: String,
did: String,
}
impl Images {
fn url(&self, kind: &str, cid: &str) -> String {
match &self.cdn {
Some(p) => p
.replacen("%s", kind, 1)
.replacen("%s", &self.did, 1)
.replacen("%s", cid, 1),
None => format!(
"https://{}/xrpc/com.atproto.sync.getBlob?did={}&cid={cid}",
self.hostname, self.did
),
}
}
}
/// The URL of an image of `did`'s, as answers show it: on the AppView's CDN
/// if `PDS_BSKY_APP_VIEW_CDN_URL_PATTERN` names one, else this server's `getBlob`.
pub(crate) fn image_url(s: &Service, kind: &str, did: &str, cid: &str) -> String {
Images {
cdn: s.config.proxy.cdn.clone(),
hostname: s.config.hostname.clone(),
did: did.to_string(),
}
.url(kind, cid)
}
/// The CID of a blob ref.
fn blob_cid(blob: &Json) -> Option<&str> {
blob["ref"]["$link"].as_str()
}
/// The account's records written by commits after `rev`, the first
/// `MAX_RECORDS` by commit, of those the repository holds; None if there
/// are none, or no record of the account's was written at or before `rev`.
fn local_records(s: &Service, did: &str, rev: &str, shown: &mut Shown) -> Option<Local> {
let account = s.account(did)?;
let handle = account.handle();
let st = read(&account.store);
let repo = &st.repo;
// Read path by path, keeping the first by commit.
let mut before = false;
let mut after: Vec<(String, String)> = Vec::new();
st.each_path(|path, _, w| {
if w <= rev {
before = true;
return;
}
after.push((w.to_string(), path.to_string()));
if after.len() >= 4 * MAX_RECORDS {
after.sort();
after.truncate(MAX_RECORDS);
}
});
if !before || after.is_empty() {
return None;
}
after.sort();
after.truncate(MAX_RECORDS);
let records = after
.into_iter()
.filter_map(|(w, path)| {
let rec = st.record(&path)?;
let (collection, rkey) = path.split_once('/')?;
let ms = tid_millis(&w);
Some(Written {
path: path.clone(),
uri: repo.uri(collection, rkey),
cid: cid_string(&rec.cid),
value: to_json(&rec.value),
ms,
indexed_at: datetime_string(ms),
})
})
.collect();
let profile_now = st.record(PROFILE).map(|r| to_json(&r.value));
shown.0.push((st.wal.clone(), st.wal.written()));
let images = Images {
cdn: s.config.proxy.cdn.clone(),
hostname: s.config.hostname.clone(),
did: did.to_string(),
};
Some(Local {
did: did.to_string(),
handle,
records,
profile_now,
images,
})
}
/// Answer a read-after-write method: forward it, and show in its answer
/// what the target has not seen yet.
pub(crate) async fn answer(s: Shared, r: Request) -> Result<Resp, XrpcError> {
let t = sessions::access_token(&s, r.bearer())?;
let target = proxy::target(&s, r.proxy.as_deref()).await?;
let got = proxy::forward_query(&s, &r, &target, Some((&t.did, &target.did, &r.nsid))).await?;
let rev = got
.headers
.get("atproto-repo-rev")
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let json_type = got
.headers
.get(header::CONTENT_TYPE)
.is_none_or(proxy::is_json);
let (Some(rev), true, false) = (
rev,
json_type,
got.status >= 400 || r.nsid == "app.bsky.feed.getPostThread",
) else {
return Ok(proxy::answered(got, true));
};
let mut shown = Shown::default();
let Some(local) = local_records(&s, &t.did, &rev, &mut shown) else {
return Ok(proxy::answered(got, true));
};
let valid = serde_json::from_slice::<Json>(&got.body)
.ok()
.filter(|b| pds_lexicon::method(&r.nsid).is_some_and(|m| m.check_output(b).is_ok()));
let Some(body) = valid else {
return Ok(proxy::answered(got, true));
};
let body = munge(&r.nsid, body, &local);
shown.durable().await?;
let mut headers = vec![(
header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
)];
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as i64;
if let Some(lag) = local.lag(now) {
headers.push((
HeaderName::from_static("atproto-upstream-lag"),
HeaderValue::from(lag.max(0)),
));
}
Ok(Resp::Upstream(Upstream {
status: 200,
headers,
body: Bytes::from(body.to_string()),
}))
}
/// Set a view's fields from a profile record: `displayName` and
/// `description` as they are, `avatar` and `banner` as image URLs, each
/// left out where the profile has none.
fn set_profile(view: &mut Json, profile: &Json, images: &Images, fields: &[&str]) {
let Some(view) = view.as_object_mut() else {
return;
};
for &f in fields {
view.remove(f);
let value = match f {
"avatar" | "banner" => blob_cid(&profile[f]).map(|cid| json!(images.url(f, cid))),
_ => profile[f].as_str().map(|s| json!(s)),
};
if let Some(v) = value {
view.insert(f.to_string(), v);
}
}
}
/// Whether datetime `a` is earlier than `b`.
fn earlier(a: &str, b: &str) -> bool {
match (datetime_millis(a), datetime_millis(b)) {
(Some(x), Some(y)) => x < y,
_ => a < b,
}
}
/// A post's view, as an AppView would show it before it saw any reply, like or repost.
fn post_view(w: &Written, local: &Local) -> Json {
let mut author = json!({ "did": local.did, "handle": local.handle });
if let Some(p) = &local.profile_now {
set_profile(&mut author, p, &local.images, &["displayName", "avatar"]);
}
let mut view = json!({
"uri": w.uri, "cid": w.cid, "likeCount": 0, "replyCount": 0, "repostCount": 0, "quoteCount": 0,
"author": author, "record": w.value, "indexedAt": w.indexed_at,
});
if let Some(e) = embed_view(&w.value["embed"], &local.images) {
view["embed"] = e;
}
view
}
/// The view of a post's embed of images or a link card; None for others.
fn embed_view(embed: &Json, images: &Images) -> Option<Json> {
let copy = |from: &Json, keys: &[&str], to: &mut Map<String, Json>| {
for &k in keys {
if let Some(v) = from.get(k) {
to.insert(k.to_string(), v.clone());
}
}
};
match embed["$type"].as_str()? {
"app.bsky.embed.images" => {
let views: Vec<Json> = embed["images"]
.as_array()?
.iter()
.map(|img| {
let mut v = Map::new();
if let Some(cid) = blob_cid(&img["image"]) {
v.insert("thumb".into(), json!(images.url("feed_thumbnail", cid)));
v.insert("fullsize".into(), json!(images.url("feed_fullsize", cid)));
}
copy(img, &["alt", "aspectRatio"], &mut v);
Json::Object(v)
})
.collect();
Some(json!({ "$type": "app.bsky.embed.images#view", "images": views }))
}
"app.bsky.embed.external" => {
let ext = &embed["external"];
let mut v = Map::new();
copy(ext, &["uri", "title", "description"], &mut v);
if let Some(cid) = blob_cid(&ext["thumb"]) {
v.insert("thumb".into(), json!(images.url("feed_thumbnail", cid)));
}
Some(json!({ "$type": "app.bsky.embed.external#view", "external": v }))
}
_ => None,
}
}
/// Put the account's posts into a feed: each written after the feed's last
/// item, newest first, before the first item indexed earlier than it, or at the end.
fn insert_posts(feed: &mut Vec<Json>, local: &Local) {
let last = feed
.last()
.and_then(|i| i["post"]["indexedAt"].as_str())
.map(str::to_string);
let mut posts: Vec<&Written> = local
.posts()
.filter(|p| last.as_deref().is_none_or(|l| earlier(l, &p.indexed_at)))
.collect();
posts.sort_by_key(|p| std::cmp::Reverse(p.ms));
for p in posts {
let item = json!({ "post": post_view(p, local) });
match feed.iter().position(|i| {
i["post"]["indexedAt"]
.as_str()
.is_some_and(|t| earlier(t, &p.indexed_at))
}) {
Some(i) => feed.insert(i, item),
None => feed.push(item),
}
}
}
/// The authors of a feed's posts that are the account, as its profile has them.
fn set_authors(feed: &mut [Json], local: &Local) {
let Some(profile) = local.profile() else {
return;
};
for item in feed {
let author = &mut item["post"]["author"];
if author["did"] == local.did.as_str() {
set_profile(author, profile, &local.images, &["displayName", "avatar"]);
}
}
}
/// An answer, valid by its method's output lexicon, showing the account's records.
fn munge(nsid: &str, mut body: Json, local: &Local) -> Json {
const ALL: &[&str] = &["displayName", "description", "avatar", "banner"];
match nsid {
"app.bsky.actor.getProfile" => {
if let (Some(p), true) = (local.profile(), body["did"] == local.did.as_str()) {
set_profile(&mut body, p, &local.images, ALL);
}
}
"app.bsky.actor.getProfiles" => {
if let (Some(p), Some(views)) = (local.profile(), body["profiles"].as_array_mut()) {
for v in views.iter_mut().filter(|v| v["did"] == local.did.as_str()) {
set_profile(v, p, &local.images, ALL);
}
}
}
"app.bsky.feed.getActorLikes" => {
if let Some(feed) = body["feed"].as_array_mut() {
set_authors(feed, local);
}
}
"app.bsky.feed.getAuthorFeed" => {
if let Some(feed) = body["feed"].as_array_mut() {
let own = feed.first().is_some_and(|first| match first.get("reason") {
None => first["post"]["author"]["did"] == local.did.as_str(),
Some(r) => {
r["$type"] == "app.bsky.feed.defs#reasonRepost"
&& r["by"]["did"] == local.did.as_str()
}
});
if own {
set_authors(feed, local);
insert_posts(feed, local);
}
}
}
"app.bsky.feed.getTimeline" => {
if let Some(feed) = body["feed"].as_array_mut() {
insert_posts(feed, local);
}
}
_ => {}
}
body
}
#[cfg(test)]
mod tests {
use super::*;
const DID: &str = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";
fn images() -> Images {
Images {
cdn: Some("http://cdn.test/%s/%s/%s".into()),
hostname: "pds.test".into(),
did: DID.into(),
}
}
fn blob(cid: &str) -> Json {
json!({ "$type": "blob", "ref": { "$link": cid }, "mimeType": "image/png", "size": 3 })
}
fn written(path: &str, value: Json, ms: i64) -> Written {
let (collection, rkey) = path.split_once('/').unwrap();
Written {
path: path.into(),
uri: format!("at://{DID}/{collection}/{rkey}"),
cid: format!("cid-{rkey}"),
value,
ms,
indexed_at: datetime_string(ms),
}
}
fn local(records: Vec<Written>) -> Local {
let profile_now = records
.iter()
.find(|w| w.path == PROFILE)
.map(|w| w.value.clone());
Local {
did: DID.into(),
handle: "alice.test".into(),
records,
profile_now,
images: images(),
}
}
#[test]
fn tids_tell_when() {
let mut tids = pds_repo::Tids::new();
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as i64;
let at = tid_millis(&tids.next_tid());
assert!((now - at).abs() < 1000, "{now} {at}");
}
#[test]
fn images_are_urls() {
assert_eq!(
images().url("avatar", "bafy"),
format!("http://cdn.test/avatar/{DID}/bafy")
);
let plain = Images {
cdn: None,
..images()
};
assert_eq!(
plain.url("banner", "bafy"),
format!("https://pds.test/xrpc/com.atproto.sync.getBlob?did={DID}&cid=bafy")
);
}
#[test]
fn profiles_show_the_new_profile() {
let l = local(vec![written(
PROFILE,
json!({ "$type": "app.bsky.actor.profile", "displayName": "Al", "avatar": blob("bafya") }),
1000,
)]);
let view = json!({ "did": DID, "handle": "alice.test", "displayName": "Old", "description": "old", "banner": "x", "followersCount": 3 });
let got = munge("app.bsky.actor.getProfile", view.clone(), &l);
assert_eq!(
got,
json!({ "did": DID, "handle": "alice.test", "displayName": "Al", "avatar": format!("http://cdn.test/avatar/{DID}/bafya"), "followersCount": 3 })
);
let other = json!({ "did": "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb", "handle": "bob.test", "displayName": "Bob" });
assert_eq!(munge("app.bsky.actor.getProfile", other.clone(), &l), other);
let many = munge(
"app.bsky.actor.getProfiles",
json!({ "profiles": [other, view] }),
&l,
);
assert_eq!(many["profiles"][0]["displayName"], "Bob");
assert_eq!(many["profiles"][1]["displayName"], "Al");
// Without the profile among the records, nothing changes.
let none = local(vec![written(
"app.bsky.feed.post/a",
json!({ "text": "hi" }),
1000,
)]);
assert_eq!(
munge(
"app.bsky.actor.getProfile",
json!({ "did": DID, "handle": "h", "displayName": "Old" }),
&none
)["displayName"],
"Old"
);
}
#[test]
fn timelines_get_new_posts_in_place() {
let at = |ms: i64| datetime_string(ms);
let item = |ms: i64| json!({ "post": { "uri": "at://x", "indexedAt": at(ms), "author": { "did": "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" } } });
let post = |rkey: &str, ms: i64, embed: Json| {
let mut v = json!({ "$type": "app.bsky.feed.post", "text": rkey, "createdAt": at(ms) });
if !embed.is_null() {
v["embed"] = embed;
}
written(&format!("{POST}/{rkey}"), v, ms)
};
let images_embed = json!({ "$type": "app.bsky.embed.images", "images": [{ "image": blob("bafyi"), "alt": "a cat" }] });
let link = json!({ "$type": "app.bsky.embed.external", "external": { "uri": "https://x.test", "title": "T", "description": "D", "thumb": blob("bafyt") } });
let l = local(vec![
post("p1", 1500, images_embed),
post("p2", 2500, link),
post("p0", 500, Json::Null),
]);
let got = munge(
"app.bsky.feed.getTimeline",
json!({ "feed": [item(3000), item(2000), item(1000)], "cursor": "c" }),
&l,
);
let times: Vec<i64> = got["feed"]
.as_array()
.unwrap()
.iter()
.map(|i| datetime_millis(i["post"]["indexedAt"].as_str().unwrap()).unwrap())
.collect();
// p0 is older than the feed's last item, and is left out.
assert_eq!(times, [3000, 2500, 2000, 1500, 1000]);
assert_eq!(got["cursor"], "c");
let p2 = &got["feed"][1]["post"];
assert_eq!(
p2["embed"],
json!({ "$type": "app.bsky.embed.external#view", "external": { "uri": "https://x.test", "title": "T", "description": "D", "thumb": format!("http://cdn.test/feed_thumbnail/{DID}/bafyt") } })
);
assert_eq!(p2["author"], json!({ "did": DID, "handle": "alice.test" }));
assert_eq!(
(p2["likeCount"].clone(), p2["record"]["text"].clone()),
(json!(0), json!("p2"))
);
let p1 = &got["feed"][3]["post"];
assert_eq!(
p1["embed"],
json!({ "$type": "app.bsky.embed.images#view", "images": [{ "thumb": format!("http://cdn.test/feed_thumbnail/{DID}/bafyi"), "fullsize": format!("http://cdn.test/feed_fullsize/{DID}/bafyi"), "alt": "a cat" }] })
);
// An empty feed takes every post, newest first.
let empty = munge("app.bsky.feed.getTimeline", json!({ "feed": [] }), &l);
let texts: Vec<&str> = empty["feed"]
.as_array()
.unwrap()
.iter()
.map(|i| i["post"]["record"]["text"].as_str().unwrap())
.collect();
assert_eq!(texts, ["p2", "p1", "p0"]);
// An author feed changes only when it is the account's own.
let others = json!({ "feed": [item(3000)] });
assert_eq!(
munge("app.bsky.feed.getAuthorFeed", others.clone(), &l),
others
);
let own = json!({ "feed": [{ "post": { "uri": "at://x", "indexedAt": at(1000), "author": { "did": DID, "displayName": "Old" } } }] });
assert_eq!(
munge("app.bsky.feed.getAuthorFeed", own, &l)["feed"]
.as_array()
.unwrap()
.len(),
3
);
let lag = l.lag(10_000).unwrap();
assert_eq!(lag, 9_500);
}
#[test]
fn likes_show_the_new_profile() {
let l = local(vec![written(PROFILE, json!({ "displayName": "Al" }), 1000)]);
let feed = json!({ "feed": [{ "post": { "author": { "did": DID, "displayName": "Old", "avatar": "x" } } }, { "post": { "author": { "did": "did:plc:b" } } }] });
let got = munge("app.bsky.feed.getActorLikes", feed, &l);
assert_eq!(
got["feed"][0]["post"]["author"],
json!({ "did": DID, "displayName": "Al" })
);
assert_eq!(
got["feed"][1]["post"]["author"],
json!({ "did": "did:plc:b" })
);
}
}