//! The join engine: runs a view's joins on records before its layout draws //! them. Each join runs on every record in scope and leaves its result at //! the record's `joins.`: the record an at-uri or strongRef names //! (`ref`), those an array's elements name (`each`), the records of the //! same repository that point back at it (`backlinks`), or how many records //! across the network point at it, from a backlink index (`counts`). A //! join's own joins run on its result records, in turn. //! //! Records are fetched from their own repositories' PDSes, and counts from //! the index, eight at a time for the whole request; a repository's //! records of a backlinks collection are all listed and kept a minute. //! Nothing here fails a tool: a record that cannot be fetched is a null or //! skipped result, a listing that fails is no backlinks, and an index that //! fails (or none) is null counts. //! //! A mode's query joins (`queries`) run once on the whole answer, not per //! record: each a related query, asked all at once after the main answer, //! as the person asking, as the query tool asks (`xrpc::ask`), each within //! three seconds. Its answer lands at `joins.`, or null when a binding //! of its params does not resolve (it is not asked), or it fails, runs out //! of time or answers what its lexicon does not allow (which is logged). use crate::Hud; use crate::backlinks::{self, Counts}; use crate::blobs; use crate::identity::Identity; use crate::pds::Record; use crate::tools::Refusal; use crate::uri::AtUri; use crate::view::{Join, Param, Path, QueryJoin, Source}; use crate::xrpc::{self, Asker}; use futures::future::{BoxFuture, join_all}; use serde_json::{Map, Value, json}; use std::cmp::Ordering; use std::sync::Arc; use std::time::Duration; use tokio::sync::Semaphore; /// How many fetches one request makes at once. const AT_ONCE: usize = 8; /// How long a repository's records of a collection are kept. const MEMBERS_KEEP: Duration = Duration::from_secs(60); /// The records listed per page, and the most pages listed. const MEMBERS_PAGE: u32 = 100; const MEMBERS_PAGES: usize = 50; /// Where the engine gets records: the network, or a test's own. pub trait Fetch: Sync { /// The record an at-uri names, its uri with its DID; none if it cannot /// be read. fn record<'a>(&'a self, uri: &'a AtUri) -> BoxFuture<'a, Option>; /// Every record of `collection` in a DID's repository, newest first; /// none if they cannot be listed. fn all<'a>( &'a self, did: &'a str, collection: &'a str, ) -> BoxFuture<'a, Option>>>; /// The backlink index's counts of records pointing at an at-uri; none /// with no index, or if it failed. fn counts<'a>(&'a self, target: &'a str) -> BoxFuture<'a, Option>>; } impl Fetch for Hud { fn record<'a>(&'a self, uri: &'a AtUri) -> BoxFuture<'a, Option> { Box::pin(async move { let did = if uri.authority.starts_with("did:") { uri.authority.clone() } else { self.handle_did(&uri.authority).await.ok()? }; let pds = self.pds(&did).await.ok()?; self.get_record(&pds, &did, &uri.collection, &uri.rkey) .await .ok() }) } fn all<'a>( &'a self, did: &'a str, collection: &'a str, ) -> BoxFuture<'a, Option>>> { Box::pin(async move { let key = format!("{did} {collection}"); if let Some(kept) = self.members.get(&key) { return Some(kept); } let pds = self.pds(did).await.ok()?; let mut all = Vec::new(); let mut cursor: Option = None; for _ in 0..MEMBERS_PAGES { let (page, next) = self .list_records(&pds, did, collection, MEMBERS_PAGE, cursor.as_deref()) .await .ok()?; let empty = page.is_empty(); all.extend(page); match next { Some(c) if !empty => cursor = Some(c), _ => break, } } let all = Arc::new(all); self.members.put(key, all.clone(), MEMBERS_KEEP); Some(all) }) } fn counts<'a>(&'a self, target: &'a str) -> BoxFuture<'a, Option>> { Box::pin(self.backlink_counts(target)) } } /// A record as a view's data has it: `{uri, cid, did, value, joins}`, its /// `did` the repository's, as its at-uri has it (a track's artist). pub fn rec(r: &Record) -> Value { json!({ "uri": r.uri, "cid": r.cid, "did": did_of(&r.uri), "value": r.value, "joins": {} }) } /// The DID of a record's at-uri. fn did_of(uri: &str) -> Option<&str> { uri.strip_prefix("at://")?.split('/').next() } /// The at-uri a value names, as a string or a strongRef's `uri`, if it is /// a record's (of `collection`, if given). fn target(v: Option<&Value>, collection: Option<&str>) -> Option { let v = v?; let s = v.as_str().or_else(|| v.get("uri")?.as_str())?; let u = AtUri::parse(s).ok()?; collection.is_none_or(|c| u.collection == c).then_some(u) } /// A field of a record as a join's paths read it: `uri`, `cid` or within /// `value`. fn field<'a>(r: &'a Record, path: &Path) -> Option> { let (first, rest) = path.split_first()?; let v = match (first, rest) { ("uri", []) => return Some(Key::Text(&r.uri)), ("cid", []) => return Some(Key::Text(&r.cid)), ("value", rest) => Path::get_rest(rest, &r.value)?, _ => return None, }; Some(match (v.as_i64(), v.as_str()) { _ if v.is_null() => Key::Int(0), (Some(i), _) => Key::Int(i), (_, Some(s)) => Key::Text(s), _ => Key::Other, }) } /// An order key: a missing value counts as the integer 0, and integers /// come before strings. #[derive(PartialEq, Eq, PartialOrd, Ord)] enum Key<'a> { Int(i64), Text(&'a str), Other, } fn rkey(r: &Record) -> &str { r.uri.rsplit('/').next().unwrap_or_default() } /// Records ordered by `keys` ascending, then by rkey; with no keys, as /// they were (newest first). pub fn order(keys: &[Path], records: &mut [&Record]) { if keys.is_empty() { return; } records.sort_by(|a, b| { keys.iter() .map(|k| { let ka = field(a, k).unwrap_or(Key::Int(0)); let kb = field(b, k).unwrap_or(Key::Int(0)); ka.cmp(&kb) }) .find(|o| *o != Ordering::Equal) .unwrap_or(Ordering::Equal) .then_with(|| rkey(a).cmp(rkey(b))) }); } /// Of a repository's records, those whose `path` names the record `of` /// (by its repository's DID, or `handle`), ordered. pub fn backlinks<'a>( all: &'a [Record], path: &Path, of: &AtUri, did: &str, handle: Option<&str>, order_by: &[Path], ) -> Vec<&'a Record> { let mut found: Vec<&Record> = all .iter() .filter(|r| { let Some(Key::Text(s)) = field(r, path) else { return false; }; AtUri::parse(s).is_ok_and(|u| { u.collection == of.collection && u.rkey == of.rkey && (u.authority == did || handle == Some(u.authority.as_str())) }) }) .collect(); order(order_by, &mut found); found } /// Runs joins for one request. pub struct Engine<'a, F: Fetch> { fetch: &'a F, permits: Semaphore, /// The repository the request reads, whose handle names its records too. who: &'a Identity, /// Where blob URLs point. urls: blobs::Urls<'a>, } impl<'a, F: Fetch> Engine<'a, F> { pub fn new(fetch: &'a F, who: &'a Identity, urls: blobs::Urls<'a>) -> Engine<'a, F> { Engine { fetch, permits: Semaphore::new(AT_ONCE), who, urls, } } /// A record as data, its blob refs given proxy URLs. fn rec(&self, mut r: Record) -> Value { if let Some(did) = did_of(&r.uri).map(str::to_string) { // Joined records are read from their repositories, as anyone can. blobs::rewrite(&mut r.value, self.urls, &did, blobs::Visibility::Public); } rec(&r) } /// Fetch the record an at-uri names, a permit held. async fn fetch(&self, uri: &AtUri) -> Option { let _permit = self.permits.acquire().await.ok()?; let r = self.fetch.record(uri).await?; Some(self.rec(r)) } /// The index's counts of records pointing at `target`, a permit held. async fn counts(&self, target: &str) -> Option> { let _permit = self.permits.acquire().await.ok()?; self.fetch.counts(target).await } /// Every record of `collection` in a DID's repository, a permit held. async fn all(&self, did: &str, collection: &str) -> Option>> { let _permit = self.permits.acquire().await.ok()?; self.fetch.all(did, collection).await } /// Run `joins` on every record of `recs`, each result at its /// `joins.`. pub fn run<'b>(&'b self, joins: &'b [Join], mut recs: Vec) -> BoxFuture<'b, Vec> { Box::pin(async move { if joins.is_empty() { return recs; } let results: Vec> = join_all( recs.iter() .map(|r| join_all(joins.iter().map(|j| self.one(j, r)))), ) .await; for (r, found) in recs.iter_mut().zip(results) { if !r.get("joins").is_some_and(Value::is_object) { r["joins"] = Value::Object(Map::new()); } for (j, v) in joins.iter().zip(found) { r["joins"][&j.name] = v; } } recs }) } /// One join's result for one record. async fn one(&self, j: &Join, r: &Value) -> Value { match &j.source { Source::Ref(path) => { let Some(uri) = target(path.get(r), j.collection.as_deref()) else { return Value::Null; }; match self.fetch(&uri).await { Some(found) => self.run(&j.join, vec![found]).await.remove(0), None => Value::Null, } } Source::Each { each, at } => { let items = each.get(r).and_then(Value::as_array); let total = items.map_or(0, Vec::len); let uris: Vec = items .into_iter() .flatten() .take(j.limit) .filter_map(|item| target(at.get(item), j.collection.as_deref())) .collect(); let found: Vec = join_all(uris.iter().map(|u| self.fetch(u))) .await .into_iter() .flatten() .collect(); let records = self.run(&j.join, found).await; json!({ "total": total, "records": records }) } Source::Counts(names) => { let Some(target) = r.get("uri").and_then(Value::as_str) else { return Value::Null; }; backlinks::named(names, self.counts(target).await.as_deref()) } Source::Backlinks { collection, path } => { let none = json!({ "total": 0, "records": [] }); let Some(of) = r .get("uri") .and_then(Value::as_str) .and_then(|u| AtUri::parse(u).ok()) else { return none; }; let did = of.authority.clone(); let Some(all) = self.all(&did, collection).await else { return none; }; let handle = (did == self.who.did) .then_some(self.who.handle.as_deref()) .flatten(); let found = backlinks(&all, path, &of, &did, handle, &j.order_by); let total = found.len(); let records: Vec = found .into_iter() .take(j.limit) .map(|m| self.rec(m.clone())) .collect(); let records = self.run(&j.join, records).await; json!({ "total": total, "records": records }) } } } } /// How long a query join has to answer. pub const QUERY_WAIT: Duration = Duration::from_secs(3); /// A query join's params, read from the main scope: none when its `when` /// holds nothing there (null, false, 0, "", [] or {}) or its `unless` /// holds something, a binding does not resolve, or an `$each` finds /// nothing. An `$each` holds each value once, in order. fn params_of(j: &QueryJoin, scope: &Value) -> Option> { let holds = |at: &crate::view::Path| match at.get(scope) { None | Some(Value::Null | Value::Bool(false)) => false, Some(Value::Number(n)) => n.as_f64() != Some(0.0), Some(Value::String(s)) => !s.is_empty(), Some(Value::Array(a)) => !a.is_empty(), Some(Value::Object(o)) => !o.is_empty(), Some(_) => true, }; if j.when.as_ref().is_some_and(|w| !holds(w)) || j.unless.as_ref().is_some_and(holds) { return None; } j.params .iter() .map(|(k, p)| { let v = match p { Param::Literal(v) => v.clone(), Param::Bound(b) => b.get(scope)?.clone(), Param::Each { each, at } => { let mut found: Vec = Vec::new(); let values = each .get(scope)? .as_array()? .iter() .filter_map(|e| at.get(e)); for v in values { let scalar = matches!(v, Value::String(_) | Value::Number(_) | Value::Bool(_)); if scalar && !found.contains(v) { found.push(v.clone()); if found.len() == crate::view::MAX_PARAM_ARRAY { break; } } } if found.is_empty() { return None; } Value::Array(found) } }; Some((k.clone(), v)) }) .collect() } /// Each `$each` param cut to the most its method takes, its lexicon's /// `maxLength` (`app.bsky.feed.getPosts` takes 25 uris): the first of its /// values asked, not the call refused. fn cut_to_lexicon(j: &QueryJoin, def: &Value, params: &mut Map) { for (name, _) in j .params .iter() .filter(|(_, p)| matches!(p, Param::Each { .. })) { let max = def["parameters"]["properties"][name]["maxLength"] .as_u64() .and_then(|m| usize::try_from(m).ok()); if let (Some(max), Some(Value::Array(values))) = (max, params.get_mut(name)) { values.truncate(max); } } } /// Run a mode's query joins on its main `scope`, all at once, each within /// `wait`, each through `asker` (the person asking): each one's answer by /// its name, or null. A join with no service of its own calls `own` (a /// query view's), else its method's bundled view's. pub async fn queries<'j, A: Asker>( asker: &A, joins: impl Iterator + Clone, own: Option<&str>, scope: &Value, wait: Duration, ) -> Map { let answers = join_all(joins.clone().map(|j| query(asker, j, own, scope, wait))).await; joins .zip(answers) .map(|(j, a)| (j.name.clone(), a)) .collect() } /// One query join's answer, or null. async fn query( asker: &A, j: &QueryJoin, own: Option<&str>, scope: &Value, wait: Duration, ) -> Value { let (name, method) = (j.name.as_str(), j.method.as_str()); let Some(mut params) = params_of(j, scope) else { tracing::debug!(join = name, method, "a query join's params do not resolve"); return Value::Null; }; let Some(service) = j .service_or(own) .and_then(|s| xrpc::Service::parse(&s).ok()) else { tracing::warn!(join = name, method, "a query join has no service"); return Value::Null; }; let asked = async { let docs = asker.docs(method).await; let def = docs.iter().find(|d| d["id"] == method); if let Some(def) = def.and_then(|d| xrpc::query_def(method, d).ok()) { cut_to_lexicon(j, def, &mut params); } xrpc::ask_with(asker, &docs, &service, method, &mut params, j.fresh).await }; match tokio::time::timeout(wait, asked).await { Ok(Ok(asked)) => asked.response, Ok(Err(Refusal::Error(why))) => { tracing::warn!(join = name, method, why, "a query join failed"); Value::Null } Ok(Err(Refusal::SignIn)) => { tracing::debug!(join = name, method, "a query join needs an account"); Value::Null } Err(_) => { tracing::warn!(join = name, method, "a query join ran out of time"); Value::Null } } } #[cfg(test)] mod tests { use super::*; use crate::view; use std::collections::HashMap; use std::sync::Mutex; /// A join with `when` is called only where its scope holds something /// there: one's own favorites on one's own page, none on another's. #[test] fn a_join_with_when_is_called_where_it_holds() { let j = view::QueryJoin { name: "favorites".into(), at: "page.join[0]".into(), method: "a.b.getFavorites".into(), service: None, params: vec![], say: None, when: view::Path::parse("mine"), unless: None, fresh: false, }; let called = |mine: Value| params_of(&j, &json!({ "mine": mine })).is_some(); for held in [ json!(true), json!(1), json!("a"), json!([1]), json!({ "a": 1 }), ] { assert!(called(held.clone()), "{held}"); } for empty in [ json!(null), json!(false), json!(0), json!(""), json!([]), json!({}), ] { assert!(!called(empty.clone()), "{empty}"); } assert!(params_of(&j, &json!({})).is_none()); // unless, the other way: another's known followers, not one's own. let others = view::QueryJoin { when: None, unless: view::Path::parse("mine"), ..j }; assert!(params_of(&others, &json!({ "mine": false })).is_some()); assert!(params_of(&others, &json!({ "mine": true })).is_none()); } /// Records in memory, a backlink index's counts (none: no index), /// whether its repositories cannot be listed, and a count of the /// fetches made. #[derive(Default)] struct Mem { records: HashMap, counts: Option>, unlisted: bool, fetched: Mutex, } impl Mem { fn put(&mut self, uri: &str, value: Value) { self.records.insert( uri.to_string(), Record { uri: uri.into(), cid: format!("cid-{uri}"), value, }, ); } } impl Fetch for Mem { fn record<'a>(&'a self, uri: &'a AtUri) -> BoxFuture<'a, Option> { *self.fetched.lock().unwrap() += 1; let key = format!("at://{}/{}/{}", uri.authority, uri.collection, uri.rkey); Box::pin(async move { self.records.get(&key).cloned() }) } fn all<'a>( &'a self, did: &'a str, collection: &'a str, ) -> BoxFuture<'a, Option>>> { let prefix = format!("at://{did}/{collection}/"); let mut all: Vec = self .records .iter() .filter(|(k, _)| k.starts_with(&prefix)) .map(|(_, r)| r.clone()) .collect(); // Newest first, as listRecords gives them. all.sort_by(|a, b| b.uri.cmp(&a.uri)); let listed = (!self.unlisted).then(|| Arc::new(all)); Box::pin(async move { listed }) } fn counts<'a>(&'a self, target: &'a str) -> BoxFuture<'a, Option>> { let found = self .counts .as_ref() .map(|c| Arc::new(c.get(target).cloned().unwrap_or_default())); Box::pin(async move { found }) } } #[tokio::test] async fn counts_from_the_index() { let mut m = grain(); let g = "at://did:plc:me/social.grain.gallery/g"; let mut counts = Counts::new(); counts.insert( ("social.grain.favorite".into(), ".subject".into()), backlinks::Count { records: 17, distinct_dids: 17, }, ); m.counts = Some(HashMap::from([(g.to_string(), counts)])); let j = joins(json!([{ "as": "counts", "counts": { "favorites": { "collection": "social.grain.favorite", "path": ".subject" }, "comments": { "collection": "social.grain.comment", "path": ".subject" } } }])); let who = who(); let engine = Engine::new(&m, &who, blobs::Urls::proxy("https://hud.example")); let out = engine.run(&j, vec![rec(&m.records[g])]).await; assert_eq!( out[0]["joins"]["counts"], json!({ "favorites": 17, "comments": 0 }) ); // No index: null, and no error. m.counts = None; let engine = Engine::new(&m, &who, blobs::Urls::proxy("https://hud.example")); let out = engine.run(&j, vec![rec(&m.records[g])]).await; assert_eq!(out[0]["joins"]["counts"], Value::Null); } fn who() -> Identity { Identity { did: "did:plc:me".into(), handle: Some("me.example.com".into()), pds: "https://pds.example".into(), } } /// Joins as a view would give them. fn joins(j: Value) -> Vec { let doc = json!({ "collection": "a.b.c", "description": "d", "list": { "join": j, "layout": { "component": "Badge", "props": { "value": "x" } } } }); view::parse(&doc, false).unwrap().list.unwrap().join } fn uris(v: &Value) -> Vec<&str> { v["records"] .as_array() .unwrap() .iter() .map(|r| r["uri"].as_str().unwrap()) .collect() } /// A gallery, its items (one in handle form, one of another gallery), /// their photos (one missing, one a post). fn grain() -> Mem { let mut m = Mem::default(); let g = "at://did:plc:me/social.grain.gallery/g"; m.put(g, json!({ "title": "Coast" })); let item = |m: &mut Mem, rkey: &str, gallery: &str, photo: &str, position: Option, at: &str| { let mut v = json!({ "gallery": gallery, "item": photo, "createdAt": at }); if let Some(p) = position { v["position"] = json!(p); } m.put( &format!("at://did:plc:me/social.grain.gallery.item/{rkey}"), v, ); }; item( &mut m, "i1", g, "at://did:plc:you/social.grain.photo/p1", Some(2), "2024-01-01", ); item( &mut m, "i2", "at://me.example.com/social.grain.gallery/g", "at://did:plc:you/social.grain.photo/p2", None, "2024-01-03", ); item( &mut m, "i3", g, "at://did:plc:you/social.grain.photo/p3", Some(0), "2024-01-02", ); item( &mut m, "i4", g, "at://did:plc:you/social.grain.photo/gone", Some(1), "2024-01-02", ); item( &mut m, "i5", g, "at://did:plc:you/app.bsky.feed.post/p5", Some(3), "2024-01-02", ); item( &mut m, "i6", "at://did:plc:me/social.grain.gallery/other", "at://did:plc:you/social.grain.photo/p1", Some(0), "2024-01-01", ); for p in ["p1", "p2", "p3"] { m.put( &format!("at://did:plc:you/social.grain.photo/{p}"), json!({ "alt": p }), ); } m.put( "at://did:plc:you/app.bsky.feed.post/p5", json!({ "text": "a post" }), ); m } #[tokio::test] async fn backlinks_ordered_limited_and_nested() { let m = grain(); let who = who(); let engine = Engine::new(&m, &who, blobs::Urls::proxy("https://hud.example")); let gallery = rec(&m.records["at://did:plc:me/social.grain.gallery/g"]); let j = joins(json!([{ "as": "items", "backlinks": { "collection": "social.grain.gallery.item", "path": "value.gallery" }, "orderBy": ["value.position", "value.createdAt"], "limit": 4, "join": [{ "as": "photo", "ref": "value.item", "collection": "social.grain.photo" }] }])); let out = engine.run(&j, vec![gallery]).await; let items = &out[0]["joins"]["items"]; // Five items name it, one by handle; another gallery's is not one. assert_eq!(items["total"], 5); // Position ascending (missing is 0, i2 and i3 then by date), then // the first four alone. assert_eq!( uris(items), [ "at://did:plc:me/social.grain.gallery.item/i3", "at://did:plc:me/social.grain.gallery.item/i2", "at://did:plc:me/social.grain.gallery.item/i4", "at://did:plc:me/social.grain.gallery.item/i1", ] ); let photos: Vec<&Value> = items["records"] .as_array() .unwrap() .iter() .map(|i| &i["joins"]["photo"]) .collect(); assert_eq!(photos[0]["value"]["alt"], "p3"); assert_eq!(photos[1]["value"]["alt"], "p2"); // A photo that cannot be fetched is null. assert_eq!(photos[2], &Value::Null); assert_eq!(photos[3]["value"]["alt"], "p1"); } #[tokio::test] async fn each_and_ref_with_collection_filter() { let mut m = grain(); m.put( "at://did:plc:me/fm.plyr.list/l", json!({ "items": [ { "subject": { "uri": "at://did:plc:you/social.grain.photo/p1", "cid": "x" } }, { "subject": { "uri": "at://did:plc:you/app.bsky.feed.post/p5", "cid": "x" } }, { "subject": { "uri": "at://did:plc:you/social.grain.photo/gone", "cid": "x" } }, { "subject": { "uri": "at://did:plc:you/social.grain.photo/p2", "cid": "x" } }, { "subject": { "uri": "at://did:plc:you/social.grain.photo/p3", "cid": "x" } }, ], "first": "at://did:plc:you/app.bsky.feed.post/p5" }), ); let who = who(); let engine = Engine::new(&m, &who, blobs::Urls::proxy("https://hud.example")); let list = rec(&m.records["at://did:plc:me/fm.plyr.list/l"]); let j = joins(json!([ { "as": "photos", "each": "value.items", "ref": "subject", "collection": "social.grain.photo", "limit": 4 }, { "as": "post", "ref": "value.first", "collection": "social.grain.photo" }, { "as": "any", "ref": "value.first" }, ])); let fetched_before = *m.fetched.lock().unwrap(); let out = engine.run(&j, vec![list]).await; let joined = &out[0]["joins"]; // All five elements count; of the first four, the post is refused // unfetched, the missing photo skipped. assert_eq!(joined["photos"]["total"], 5); assert_eq!( uris(&joined["photos"]), [ "at://did:plc:you/social.grain.photo/p1", "at://did:plc:you/social.grain.photo/p2" ] ); assert_eq!(joined["post"], Value::Null); assert_eq!(joined["any"]["value"]["text"], "a post"); // p1, gone, p2 for photos; the post for `any` alone. assert_eq!(*m.fetched.lock().unwrap() - fetched_before, 4); } #[test] fn ordering() { let r = |rkey: &str, v: Value| Record { uri: format!("at://did:plc:me/a.b.c/{rkey}"), cid: String::new(), value: v, }; let all = [ r("m1", json!({ "position": 2, "createdAt": "2024-01-01" })), r("m2", json!({ "createdAt": "2024-01-03" })), r("m3", json!({ "position": 0, "createdAt": "2024-01-02" })), r("m4", json!({ "position": -1 })), r("m6", json!({ "position": 2, "createdAt": "2023-01-01" })), r("m5", json!({ "position": 2, "createdAt": "2023-01-01" })), r("m7", json!({ "position": 0 })), r("m8", json!({ "position": "high" })), ]; let keys = [ Path::parse("value.position").unwrap(), Path::parse("value.createdAt").unwrap(), ]; let mut of: Vec<&Record> = all.iter().collect(); order(&keys, &mut of); let rkeys: Vec<&str> = of.iter().map(|r| rkey(r)).collect(); assert_eq!(rkeys, ["m4", "m7", "m3", "m2", "m5", "m6", "m1", "m8"]); // No keys: as listed. let mut of: Vec<&Record> = all.iter().collect(); order(&[], &mut of); assert_eq!(rkey(of[0]), "m1"); } /// Order keys read a record's uri and cid, and its value through /// arrays; what a key does not find, or finds no number or string at, /// orders as 0 does, before strings. #[test] fn order_keys_read_the_whole_record() { let r = |rkey: &str, cid: &str, v: Value| Record { uri: format!("at://did:plc:me/a.b.c/{rkey}"), cid: cid.into(), value: v, }; let all = [ r("b", "1", json!({ "tags": ["x"], "n": { "x": -1 } })), r("a", "2", json!({ "tags": ["y"], "n": 5 })), ]; let by = |keys: &[&str]| { let keys: Vec = keys.iter().map(|k| Path::parse(k).unwrap()).collect(); let mut of: Vec<&Record> = all.iter().collect(); order(&keys, &mut of); of.iter().map(|r| rkey(r)).collect::>() }; assert_eq!(by(&["uri"]), ["a", "b"]); assert_eq!(by(&["cid"]), ["b", "a"]); assert_eq!(by(&["value.tags.0"]), ["b", "a"]); // b's n is an object: after a's number. assert_eq!(by(&["value.n"]), ["a", "b"]); // a's 5 has no x: 0, after b's -1. assert_eq!(by(&["value.n.x"]), ["b", "a"]); // Outside uri, cid and value: nothing, so by rkey. assert_eq!(by(&["joins.x"]), ["a", "b"]); // A backlink is a string at its path; a number or object is none. let of = AtUri::parse("at://did:plc:me/a.b.c/a").unwrap(); let n = Path::parse("value.n").unwrap(); assert!(backlinks(&all, &n, &of, "did:plc:me", None, &[]).is_empty()); } /// A record with no uri has no counts and no backlinks, and gets its /// joins all the same; a repository that cannot be listed has no /// backlinks; a fetched record whose uri names no repository keeps its /// blob refs as they are. #[tokio::test] async fn joins_on_records_short_of_what_they_read() { let mut m = grain(); m.counts = Some(HashMap::new()); let blob = json!({ "$type": "blob", "mimeType": "image/jpeg", "size": 1, "ref": { "$link": "bafyreie5737gdxlw5i64vzichcalba3z2v5n6icifvx5xytvske7mr3hpm" } }); m.put( "at://did:plc:me/a.b.pic/ok", json!({ "image": blob.clone() }), ); m.records.insert( "at://did:plc:me/a.b.pic/odd".into(), Record { uri: "odd".into(), cid: "c".into(), value: json!({ "image": blob.clone() }), }, ); let who = who(); let j = joins(json!([ { "as": "counts", "counts": { "likes": { "collection": "a.b.like", "path": ".subject" } } }, { "as": "items", "backlinks": { "collection": "social.grain.gallery.item", "path": "value.gallery" } }, { "as": "ok", "ref": "value.ok" }, { "as": "odd", "ref": "value.odd" }, ])); let engine = Engine::new(&m, &who, blobs::Urls::proxy("https://hud.example")); let bare = json!({ "value": { "ok": "at://did:plc:me/a.b.pic/ok", "odd": "at://did:plc:me/a.b.pic/odd" } }); let out = engine.run(&j, vec![bare]).await; let joined = &out[0]["joins"]; assert_eq!(joined["counts"], Value::Null); assert_eq!(joined["items"], json!({ "total": 0, "records": [] })); assert!(joined["ok"]["value"]["image"]["url"].is_string()); assert_eq!(joined["odd"]["value"]["image"], blob); // Listed, a gallery has its items; unlisted, none. let gallery = rec(&m.records["at://did:plc:me/social.grain.gallery/g"]); let items = &j[1..2]; let out = engine.run(items, vec![gallery.clone()]).await; assert_eq!(out[0]["joins"]["items"]["total"], 5); m.unlisted = true; let engine = Engine::new(&m, &who, blobs::Urls::proxy("https://hud.example")); let out = engine.run(items, vec![gallery]).await; assert_eq!( out[0]["joins"]["items"], json!({ "total": 0, "records": [] }) ); } const ALICE: &str = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; /// A server whose PLC directory, handle resolver and Alice's PDS are /// one fake on a loopback port, `alice.test` her handle: her gallery /// `g`, its items listed two at a time; and how many pages it listed. async fn served() -> (Arc, Arc>) { use axum::extract::{Query, State}; use axum::http::{StatusCode, Uri}; use axum::response::{IntoResponse, Response}; type Shared = (String, Arc>); async fn answer( State((url, listed)): State, uri: Uri, Query(q): Query>, ) -> Response { let arg = |k: &str| q.get(k).cloned().unwrap_or_default(); let item = |rkey: &str| { json!({ "uri": format!("at://{ALICE}/social.grain.gallery.item/{rkey}"), "cid": "c", "value": { "gallery": format!("at://{ALICE}/social.grain.gallery/g"), "item": "at://alice.test/social.grain.photo/p" } }) }; match uri.path() { "/xrpc/com.atproto.repo.getRecord" if arg("rkey") == "g" => axum::Json( json!({ "uri": format!("at://{ALICE}/social.grain.gallery/g"), "cid": "c", "value": { "title": "Coast" } }), ) .into_response(), "/xrpc/com.atproto.repo.getRecord" if arg("rkey") == "p" => { axum::Json(json!({ "uri": format!("at://{ALICE}/social.grain.photo/p"), "cid": "c", "value": { "alt": "a photo" } })) .into_response() } "/xrpc/com.atproto.repo.listRecords" if arg("collection") == "social.grain.gallery.item" => { *listed.lock().unwrap() += 1; let (records, cursor) = match arg("cursor").as_str() { "" => (vec![item("i3"), item("i2")], "c1"), "c1" => (vec![item("i1")], "c2"), _ => (vec![], "c3"), }; axum::Json(json!({ "records": records, "cursor": cursor })).into_response() } "/xrpc/com.atproto.identity.resolveHandle" if arg("handle") == "alice.test" => { axum::Json(json!({ "did": ALICE })).into_response() } path if path == format!("/{ALICE}") => { let pds = json!({ "id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": url }); axum::Json(json!({ "id": ALICE, "service": [pds] })).into_response() } _ => ( StatusCode::BAD_REQUEST, axum::Json(json!({ "error": "NotFound", "message": "no" })), ) .into_response(), } } let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let listed = Arc::new(Mutex::new(0)); let app = axum::Router::new() .fallback(answer) .with_state((url.clone(), listed.clone())); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); let dir = std::env::temp_dir().join(format!("hud-join-{}", atproto::jose::random(6))); let hud = Hud::new(crate::Config { port: 0, bind: "::".parse().unwrap(), public_origin: "https://hud.test".into(), cache_dir: dir.join("cache"), blob_cache_bytes: 0, cache_memory_bytes: 1 << 24, data_dir: dir, oauth: false, oauth_scope: crate::scope::asked(crate::view::views(), true, false), oauth_include: None, workspace: true, sign_up_pds: crate::config::SIGN_UP_PDS.into(), client: Default::default(), operators: Vec::new(), logs: false, logs_keep: crate::logs::KEEP, spaces: false, plc_url: url.clone(), handle_resolver: Some(url), allow_private: true, backlinks: crate::backlinks::Backlinks::None, favicons: None, image_cdn: None, relay: None, }) .unwrap(); (hud, listed) } /// Joins run on the network: a record read from its repository's PDS, /// named by DID or by handle, or null where it cannot be read; a /// repository's records listed page by page to an empty one, and kept, /// or none where they cannot be listed; no counts with no index. #[tokio::test] async fn joins_read_the_network() { let (hud, listed) = served().await; let who = Identity { did: ALICE.into(), handle: Some("alice.test".into()), pds: String::new(), }; let j = joins(json!([ { "as": "items", "backlinks": { "collection": "social.grain.gallery.item", "path": "value.gallery" }, "join": [{ "as": "photo", "ref": "value.item" }] }, { "as": "self", "ref": "value.self" }, { "as": "gone", "ref": "value.gone" }, { "as": "stranger", "ref": "value.stranger" }, { "as": "unlisted", "backlinks": { "collection": "social.grain.comment", "path": "value.subject" } }, { "as": "counts", "counts": { "likes": { "collection": "a.b.like", "path": ".subject" } } }, ])); let gallery = json!({ "uri": format!("at://{ALICE}/social.grain.gallery/g"), "value": { "self": "at://alice.test/social.grain.gallery/g", "gone": format!("at://{ALICE}/social.grain.gallery/nope"), "stranger": "at://nobody.test/social.grain.gallery/g" } }); let engine = Engine::new(&*hud, &who, blobs::Urls::proxy("https://hud.example")); let out = engine.run(&j, vec![gallery.clone()]).await; let joined = &out[0]["joins"]; assert_eq!(joined["items"]["total"], 3); assert_eq!( joined["items"]["records"][0]["joins"]["photo"]["value"]["alt"], "a photo" ); assert_eq!(joined["self"]["value"]["title"], "Coast"); assert_eq!( joined["self"]["uri"], format!("at://{ALICE}/social.grain.gallery/g") ); assert_eq!(joined["gone"], Value::Null); assert_eq!(joined["stranger"], Value::Null); assert_eq!(joined["unlisted"], json!({ "total": 0, "records": [] })); assert_eq!(joined["counts"], Value::Null); // Three pages, the last empty; kept, so listed once. assert_eq!(*listed.lock().unwrap(), 3); engine.run(&j[..1], vec![gallery]).await; assert_eq!(*listed.lock().unwrap(), 3); } /// A call a service had: its method and its query's pairs. type Call = (String, Vec<(String, String)>); /// A service in memory: each method's answer after its delay (one it /// has none for needs an account), its lexicons where they are not the /// usual, the params it was asked with and whether fresh, and the most /// calls it had at once. #[derive(Default)] struct Svc { answers: HashMap, lexicon: Option>>, asked: Mutex>, fresh: Mutex>, now: std::sync::atomic::AtomicUsize, most: std::sync::atomic::AtomicUsize, } impl Svc { fn new(answers: &[(&str, u64, Value)]) -> Svc { Svc { answers: answers .iter() .map(|(m, ms, v)| (format!("com.example.{m}"), (*ms, v.clone()))) .collect(), ..Svc::default() } } fn asked(&self) -> Vec { self.asked .lock() .unwrap() .iter() .map(|(m, _)| m.clone()) .collect() } } impl Asker for Svc { /// Its own lexicon, or every method taking an actor, a limit up to /// 10 and up to 3 uris, and answering strings at `items`. fn docs<'a>(&'a self, method: &'a str) -> BoxFuture<'a, Vec>> { let docs = self .lexicon .clone() .unwrap_or_else(|| vec![Arc::new(lexicon(method, Some(3)))]); Box::pin(async move { docs }) } fn handle_did(&self, handle: String) -> BoxFuture<'_, Result> { Box::pin(async move { Err(format!("no {handle}")) }) } fn call<'a>( &'a self, _service: &'a xrpc::Service, method: &'a str, pairs: Vec<(String, String)>, fresh: bool, ) -> BoxFuture<'a, Result<(Value, String), Refusal>> { use std::sync::atomic::Ordering::SeqCst; Box::pin(async move { self.asked.lock().unwrap().push((method.to_string(), pairs)); self.fresh.lock().unwrap().push(fresh); let Some((ms, answer)) = self.answers.get(method).cloned() else { return Err(Refusal::SignIn); }; let now = self.now.fetch_add(1, SeqCst) + 1; self.most.fetch_max(now, SeqCst); tokio::time::sleep(Duration::from_millis(ms)).await; self.now.fetch_sub(1, SeqCst); Ok((answer, "svc.example".into())) }) } fn rewrite(&self, _service: &xrpc::Service, _answer: &mut Value) {} } /// A method taking an actor, a limit up to 10 and uris, at most `max` /// of them, and answering strings at `items`. fn lexicon(method: &str, max: Option) -> Value { let mut doc = json!({ "lexicon": 1, "id": method, "defs": { "main": { "type": "query", "parameters": { "type": "params", "required": ["actor"], "properties": { "actor": { "type": "string", "format": "at-identifier" }, "limit": { "type": "integer", "minimum": 1, "maximum": 10 }, "uris": { "type": "array", "items": { "type": "string", "format": "at-uri" } } } }, "output": { "encoding": "application/json", "schema": { "type": "object", "required": ["items"], "properties": { "items": { "type": "array", "items": { "type": "string" } } } } } } } }); if let Some(max) = max { doc["defs"]["main"]["parameters"]["properties"]["uris"]["maxLength"] = json!(max); } doc } const OWN: Option<&str> = Some("did:web:svc.example#svc"); const WAIT: Duration = Duration::from_secs(3); /// A query view's query joins, each of `com.example.` with /// its params. fn query_joins(joins: &[(&str, Value)]) -> Vec { let joins: Vec = joins .iter() .map(|(m, params)| json!({ "as": m, "query": format!("com.example.{m}"), "params": params })) .collect(); let doc = json!({ "method": "com.example.main", "service": OWN, "description": "d", "query": { "join": joins, "layout": { "component": "Badge", "props": { "value": "x" } } } }); view::parse(&doc, false).unwrap().query.unwrap().queries } fn scope() -> Value { json!({ "method": "com.example.main", "params": { "actor": "alice.example" }, "response": { "did": "did:plc:alice", "big": 50 } }) } fn by_did() -> Value { json!({ "actor": { "$": "response.did" }, "limit": 2 }) } #[tokio::test] async fn query_joins_run_at_once() { let items = json!({ "items": ["a", "b"] }); let svc = Svc::new(&[ ("one", 200, items.clone()), ("two", 200, items.clone()), ("three", 200, items.clone()), ]); let joins = query_joins(&[("one", by_did()), ("two", by_did()), ("three", by_did())]); let started = std::time::Instant::now(); let got = queries(&svc, joins.iter(), OWN, &scope(), WAIT).await; // Three of 200 ms each, together. assert!( started.elapsed() < Duration::from_millis(500), "{:?}", started.elapsed() ); assert_eq!(svc.most.load(std::sync::atomic::Ordering::SeqCst), 3); assert_eq!( Value::Object(got), json!({ "one": items, "two": items, "three": items }) ); // Each asked with its params read from the main answer. let asked = svc.asked.lock().unwrap(); assert_eq!( asked[0].1, [ ("actor".to_string(), "did:plc:alice".to_string()), ("limit".to_string(), "2".to_string()) ] ); } /// A join that says `fresh` is asked so; the rest may have an answer /// kept. #[tokio::test] async fn a_fresh_join_is_asked_fresh() { let items = json!({ "items": [] }); let svc = Svc::new(&[("kept", 0, items.clone()), ("now", 0, items)]); let doc = json!({ "method": "com.example.main", "service": OWN, "description": "d", "query": { "join": [ { "as": "kept", "query": "com.example.kept", "params": by_did() }, { "as": "now", "query": "com.example.now", "params": by_did(), "fresh": true }], "layout": { "component": "Badge", "props": { "value": "x" } } } }); let joins = view::parse(&doc, false).unwrap().query.unwrap().queries; queries(&svc, joins.iter(), OWN, &scope(), WAIT).await; assert_eq!(svc.asked(), ["com.example.kept", "com.example.now"]); assert_eq!(*svc.fresh.lock().unwrap(), [false, true]); } /// Each join has its own time; one out of it is null, the rest answer. #[tokio::test] async fn a_query_join_out_of_time_is_null() { let svc = Svc::new(&[ ("slow", 5_000, json!({ "items": [] })), ("fast", 0, json!({ "items": ["x"] })), ]); let joins = query_joins(&[("slow", by_did()), ("fast", by_did())]); let started = std::time::Instant::now(); let got = queries( &svc, joins.iter(), OWN, &scope(), Duration::from_millis(100), ) .await; assert!(started.elapsed() < Duration::from_millis(1_000)); assert_eq!( Value::Object(got), json!({ "slow": null, "fast": { "items": ["x"] } }) ); } /// An answer its lexicon does not allow is null. #[tokio::test] async fn an_answer_its_lexicon_refuses_is_null() { let svc = Svc::new(&[ ("bad", 0, json!({ "items": 5 })), ("none", 0, json!({ "other": [] })), ("good", 0, json!({ "items": [], "more": 1 })), ]); let joins = query_joins(&[("bad", by_did()), ("none", by_did()), ("good", by_did())]); let got = queries(&svc, joins.iter(), OWN, &scope(), WAIT).await; assert_eq!( Value::Object(got), json!({ "bad": null, "none": null, "good": { "items": [], "more": 1 } }) ); assert_eq!( svc.asked(), ["com.example.bad", "com.example.none", "com.example.good"] ); } /// A binding that does not resolve skips the join, as do params its /// lexicon refuses: neither is asked. #[tokio::test] async fn unresolved_or_refused_params_skip_the_join() { let svc = Svc::new(&[ ("missing", 0, json!({ "items": [] })), ("refused", 0, json!({ "items": [] })), ("root", 0, json!({ "items": ["r"] })), ]); let joins = query_joins(&[ ("missing", json!({ "actor": { "$": "response.nobody" } })), ( "refused", json!({ "actor": { "$": "response.did" }, "limit": { "$": "response.big" } }), ), ("root", json!({ "actor": { "$root": "params.actor" } })), ]); let got = queries(&svc, joins.iter(), OWN, &scope(), WAIT).await; assert_eq!( Value::Object(got), json!({ "missing": null, "refused": null, "root": { "items": ["r"] } }) ); assert_eq!(svc.asked(), ["com.example.root"]); } /// An `$each` over no array skips the join, as does an at-uri by a /// handle that does not resolve: neither is asked. #[tokio::test] async fn no_array_or_handle_skips_the_join() { let svc = Svc::new(&[ ("each", 0, json!({ "items": [] })), ("handle", 0, json!({ "items": [] })), ]); let joins = query_joins(&[ ( "each", json!({ "actor": { "$": "response.did" }, "uris": { "$each": ["response.did", "x"] } }), ), ( "handle", json!({ "actor": { "$": "response.did" }, "uris": ["at://bob.example/app.bsky.feed.post/1"] }), ), ]); let got = queries(&svc, joins.iter(), OWN, &scope(), WAIT).await; assert_eq!(Value::Object(got), json!({ "each": null, "handle": null })); assert!(svc.asked().is_empty()); } /// A join with no service, of its own, the view's or its method's /// bundled view's, is null unasked; one the service refuses without an /// account is null. #[tokio::test] async fn a_query_join_without_service_or_account_is_null() { let svc = Svc::new(&[("open", 0, json!({ "items": [] }))]); let joins = query_joins(&[("open", by_did()), ("closed", by_did())]); let got = queries(&svc, joins.iter(), None, &scope(), WAIT).await; assert_eq!(Value::Object(got), json!({ "open": null, "closed": null })); assert!(svc.asked().is_empty()); let got = queries(&svc, joins.iter(), OWN, &scope(), WAIT).await; assert_eq!( Value::Object(got), json!({ "open": { "items": [] }, "closed": null }) ); assert_eq!(svc.asked(), ["com.example.open", "com.example.closed"]); } /// `$each` asks with the value at its path in each element, in order, /// leaving out an element without one; with none at all the join is /// skipped. A join reading `records` runs on every page of them. #[tokio::test] async fn each_asks_with_every_elements_value_and_none_skips_the_join() { let svc = Svc::new(&[ ("posts", 0, json!({ "items": ["p"] })), ("none", 0, json!({ "items": [] })), ]); let doc = json!({ "collection": "com.example.post", "description": "d", "list": { "join": [ { "as": "posts", "query": "com.example.posts", "service": OWN, "params": { "actor": { "$": "did" }, "uris": { "$each": ["records", "value.subject.uri"] } } }, { "as": "none", "query": "com.example.none", "service": OWN, "params": { "actor": { "$": "did" }, "uris": { "$each": ["records", "value.nothing"] } } }, { "as": "once", "query": "com.example.once", "service": OWN, "params": { "actor": { "$": "did" } } }, ], "layout": { "component": "Badge", "props": { "value": "x" } } } }); let joins = view::parse(&doc, false).unwrap().list.unwrap().queries; let per_page: Vec = joins.iter().map(QueryJoin::per_page).collect(); assert_eq!(per_page, [true, true, false]); let subject = |uri: &str| json!({ "value": { "subject": { "uri": uri } } }); let scope = json!({ "did": "did:plc:alice", "records": [ subject("at://did:plc:a/app.bsky.feed.post/1"), { "value": {} }, subject("at://did:plc:b/app.bsky.feed.post/2"), ] }); let got = queries(&svc, joins[..2].iter(), OWN, &scope, WAIT).await; assert_eq!( Value::Object(got), json!({ "posts": { "items": ["p"] }, "none": null }) ); let asked = svc.asked.lock().unwrap(); assert_eq!(asked.len(), 1, "the join with nothing to ask is skipped"); assert_eq!( asked[0].1, [ ("actor".to_string(), "did:plc:alice".to_string()), ( "uris".to_string(), "at://did:plc:a/app.bsky.feed.post/1".to_string() ), ( "uris".to_string(), "at://did:plc:b/app.bsky.feed.post/2".to_string() ), ] ); } /// `$each` asks each value once, and no more of them than its param's /// `maxLength` (3 here), the first; a query view's join reading the /// answer runs on every page, one reading its params on the first. #[tokio::test] async fn each_asks_each_value_once_and_no_more_than_its_method_takes() { let svc = Svc::new(&[("posts", 0, json!({ "items": ["p"] }))]); let joins = query_joins(&[ ( "posts", json!({ "actor": { "$": "response.did" }, "uris": { "$each": ["response.items", "subject"] } }), ), ("first", json!({ "actor": { "$root": "params.actor" } })), ]); let per_page: Vec = joins.iter().map(QueryJoin::per_page).collect(); assert_eq!(per_page, [true, false]); let post = |n: u8| json!({ "subject": format!("at://did:plc:a/app.bsky.feed.post/{n}") }); let scope = json!({ "response": { "did": "did:plc:alice", "items": [ post(1), post(1), post(2), {}, post(1), post(3), post(4), ] } }); queries(&svc, joins[..1].iter(), OWN, &scope, WAIT).await; let asked = svc.asked.lock().unwrap(); let uris: Vec<&str> = asked[0] .1 .iter() .filter(|(k, _)| k == "uris") .map(|(_, v)| v.as_str()) .collect(); assert_eq!( uris, [ "at://did:plc:a/app.bsky.feed.post/1", "at://did:plc:a/app.bsky.feed.post/2", "at://did:plc:a/app.bsky.feed.post/3", ] ); } /// Where its method names no most, `$each` asks at most 50 values, the /// first; a method with no lexicon is null, never asked. #[tokio::test] async fn each_asks_at_most_fifty_and_a_method_without_lexicon_nothing() { let answers = [("posts", 0, json!({ "items": ["p"] }))]; let svc = Svc { lexicon: Some(vec![Arc::new(lexicon("com.example.posts", None))]), ..Svc::new(&answers) }; let joins = query_joins(&[( "posts", json!({ "actor": { "$": "response.did" }, "uris": { "$each": ["response.items", "subject"] } }), )]); let post = |n: u8| format!("at://did:plc:a/app.bsky.feed.post/{n}"); let items: Vec = (0..60).map(|n| json!({ "subject": post(n) })).collect(); let scope = json!({ "response": { "did": "did:plc:alice", "items": items } }); let got = queries(&svc, joins.iter(), OWN, &scope, WAIT).await; assert_eq!(got["posts"], json!({ "items": ["p"] })); let uris: Vec = svc.asked.lock().unwrap()[0] .1 .iter() .filter(|(k, _)| k == "uris") .map(|(_, v)| v.clone()) .collect(); assert_eq!(uris, (0..50).map(post).collect::>()); let docless = Svc { lexicon: Some(Vec::new()), ..Svc::new(&answers) }; let got = queries(&docless, joins.iter(), OWN, &scope, WAIT).await; assert_eq!(got["posts"], Value::Null); assert!(docless.asked().is_empty()); } }