//! Records checked against the lexicons of their types, as //! `spec/VALIDATION.md` says: what a write's `validate` asks, the //! `validationStatus` it answers, and the duplicates `createRecord` replaces. //! The lexicons and the check itself are `pds-lexicon`'s. use crate::{Store, XrpcError, invalid}; use pds_repo::data::{V, valid_did}; use serde_json::{Map, Value as Json}; /// What a write's `validate` asks: left out, `true` or `false`. One applies /// to every record an `applyWrites` makes. #[derive(Clone, Copy)] pub(crate) struct Validate(Option); impl Validate { /// A write's `validate`, which its lexicon checked is a boolean if it is there. pub(crate) fn of(b: &Map) -> Self { Validate(b.get("validate").and_then(Json::as_bool)) } /// Check a record a write makes of `collection`, at `rkey` (`None` for a /// TID the server makes), as its JSON input: the `validationStatus` the /// write answers for it, or why it is refused. `true` on a type the /// server knows no lexicon for is refused here, with the record, after /// the write's `repo`. A record whose `$type` is not its collection is /// refused before this, so it is checked as one of its collection. pub(crate) fn check( self, collection: &str, rkey: Option<&str>, record: &Json, ) -> Result, XrpcError> { match self.0 { Some(false) => Ok(None), _ if pds_lexicon::knows_record(collection) => { match pds_lexicon::check_record(collection, rkey, record) { Ok(()) => Ok(Some("valid")), Err(e) => invalid(format!("Invalid {collection} record: {}", e.0)), } } Some(true) => invalid(format!("Lexicon not found: lex:{collection}")), None => Ok(Some("unknown")), } } /// The keys of the account's records of `collection` that link to the /// subject `value` links to, which a `createRecord` of it deletes in its /// commit before it creates it: none with `validate: false`. The record /// at the key it creates is among them if it links there too, so that /// the write deletes it and creates the new one, rather than find the /// key taken. pub(crate) fn duplicates(self, st: &Store, collection: &str, value: &V) -> Vec { if self.0 == Some(false) { return vec![]; } let Some(subject) = subject_of(collection, value) else { return vec![]; }; let lo = format!("{collection}/"); let mut out = Vec::new(); let mut from = lo.clone(); loop { let page = st.records(collection, &from, false, 256); let done = page.len() < 256; for (path, rec) in &page { if subject_of(collection, &rec.value) == Some(subject) { out.push(path[lo.len()..].to_string()); } } match page.last() { Some((last, _)) if !done => from = format!("{last}\0"), _ => return out, } } } } /// A field of a map. fn field<'a>(v: &'a V, name: &str) -> Option<&'a V> { match v { V::Map(m) => m.iter().find(|(k, _)| k == name).map(|(_, x)| x), _ => None, } } /// The subject a record of `collection` links to, whichever write made it: /// a follow's or block's `subject` that is a DID, a like's or repost's /// `subject.uri` that is an AT URI. Compared as strings. fn subject_of<'a>(collection: &str, value: &'a V) -> Option<&'a str> { match collection { "app.bsky.graph.follow" | "app.bsky.graph.block" => match field(value, "subject")? { V::Str(s) if valid_did(s) => Some(s), _ => None, }, "app.bsky.feed.like" | "app.bsky.feed.repost" => { match field(field(value, "subject")?, "uri")? { V::Str(s) if pds_lexicon::valid_at_uri(s) => Some(s), _ => None, } } _ => None, } } #[cfg(test)] mod tests { use crate::Shared; use crate::testing::*; use pds_repo::data::{V, cid_of, cid_string, decode_prefix, from_json}; use serde_json::{Value as Json, json}; const CREATE: &str = "com.atproto.repo.createRecord"; const PUT: &str = "com.atproto.repo.putRecord"; const APPLY: &str = "com.atproto.repo.applyWrites"; const AT: &str = "2024-01-01T00:00:00Z"; fn post(text: &str) -> Json { json!({ "$type": "app.bsky.feed.post", "text": text, "createdAt": AT }) } fn follow(subject: &str) -> Json { json!({ "$type": "app.bsky.graph.follow", "subject": subject, "createdAt": AT }) } fn like(uri: &str) -> Json { let cid = cid_string(&cid_of(0x71, uri.as_bytes())); json!({ "$type": "app.bsky.feed.like", "subject": { "uri": uri, "cid": cid }, "createdAt": AT }) } /// The keys of `did`'s records in `collection`, in order. fn keys(s: &Shared, did: &str, collection: &str) -> Vec { records_of(s, did) .into_iter() .filter_map(|(k, _)| k.strip_prefix(&format!("{collection}/")).map(String::from)) .collect() } /// A field of a map. fn get(v: &V, k: &str) -> Option { match v { V::Map(m) => m.iter().find(|(n, _)| n == k).map(|(_, x)| x.clone()), _ => None, } } /// The ops of the latest firehose event, whole. fn last_event_ops(s: &Shared) -> Vec { let frame = s.events.get(s, 0, u64::MAX - 1, 0).pop().unwrap(); let (_, n) = decode_prefix(&frame).unwrap(); let (body, _) = decode_prefix(&frame[n..]).unwrap(); let Some(V::Arr(ops)) = get(&body, "ops") else { panic!("no ops") }; ops } /// The ops of the latest firehose event, as `(action, path)`. fn last_ops(s: &Shared) -> Vec<(String, String)> { last_event_ops(s) .iter() .map(|op| match (get(op, "action"), get(op, "path")) { (Some(V::Str(a)), Some(V::Str(p))) => (a, p), _ => panic!("an op without action or path"), }) .collect() } /// `validate` left out, `true` or `false`, on a known type and another, /// and the `validationStatus` each answers; a record is stored as sent. #[tokio::test] async fn validate_and_status() { let plc = plc().await; let dir = scratch("validate-status"); let s = open(&dir, &plc); let (did, t) = create(&s, "alice").await; let body = |record: Json, validate: Option| { let mut b = json!({ "repo": did, "collection": record["$type"].as_str().unwrap_or("app.bsky.feed.post"), "record": record }); if let Some(v) = validate { b["validate"] = v; } b }; let status = |got: &Json| got.get("validationStatus").cloned(); let got = call(&s, CREATE, Some(&t), body(post("a"), None)) .await .unwrap(); assert_eq!(status(&got), Some(json!("valid"))); assert_eq!( status( &call(&s, CREATE, Some(&t), body(post("b"), Some(json!(true)))) .await .unwrap() ), Some(json!("valid")) ); assert_eq!( status( &call(&s, CREATE, Some(&t), body(post("c"), Some(json!(false)))) .await .unwrap() ), None ); let other = json!({ "$type": "com.example.note", "n": 1 }); assert_eq!( status( &call(&s, CREATE, Some(&t), body(other.clone(), None)) .await .unwrap() ), Some(json!("unknown")) ); assert_eq!( status( &call( &s, CREATE, Some(&t), body(other.clone(), Some(json!(false))) ) .await .unwrap() ), None ); assert_eq!( call(&s, CREATE, Some(&t), body(other, Some(json!(true)))).await, Err("InvalidRequest".into()) ); for v in [json!(null), json!("true"), json!(1)] { assert_eq!( call(&s, CREATE, Some(&t), body(post("d"), Some(v.clone()))).await, Err("InvalidRequest".into()), "{v}" ); } // Broken, it is refused unless `validate` is `false`, and makes nothing. let before = keys(&s, &did, "app.bsky.feed.post").len(); let broken = json!({ "$type": "app.bsky.feed.post", "text": 5, "createdAt": AT }); assert_eq!( call(&s, CREATE, Some(&t), body(broken.clone(), None)).await, Err("InvalidRequest".into()) ); assert_eq!( call( &s, CREATE, Some(&t), body(broken.clone(), Some(json!(true))) ) .await, Err("InvalidRequest".into()) ); assert_eq!(keys(&s, &did, "app.bsky.feed.post").len(), before); call(&s, CREATE, Some(&t), body(broken, Some(json!(false)))) .await .unwrap(); // Stored exactly as sent, `$type` added where it was left out, with no defaults filled in. let sent = json!({ "text": "e", "createdAt": AT, "unnamed": { "x": [1] } }); let got = call(&s, CREATE, Some(&t), body(sent.clone(), None)) .await .unwrap(); let mut stored = sent; stored["$type"] = json!("app.bsky.feed.post"); let cid = cid_string(&pds_repo::make_record(from_json(&stored).unwrap()).cid); assert_eq!( (status(&got), got["cid"].clone()), (Some(json!("valid")), json!(cid)) ); let _ = std::fs::remove_dir_all(&dir); } /// The record is checked after the write's `repo` and its `$type`, and /// before a swap, a key that exists and the blobs; `putRecord` of exactly /// the record stored is still checked. #[tokio::test] async fn where_in_a_write() { let plc = plc().await; let dir = scratch("validate-order"); let s = open(&dir, &plc); let (did, t) = create(&s, "alice").await; let (bob, _) = create(&s, "bob").await; let broken = json!({ "$type": "app.bsky.feed.post", "createdAt": AT }); let put = |repo: &str, record: &Json, extra: Json| { let mut b = json!({ "repo": repo, "collection": "app.bsky.feed.post", "rkey": "3jzfcijpj2z2a", "record": record }); b.as_object_mut() .unwrap() .extend(extra.as_object().unwrap().clone()); b }; assert_eq!( call(&s, PUT, Some(&t), put(&bob, &broken, json!({}))).await, Err("AuthenticationRequired".into()) ); assert_eq!( call(&s, PUT, Some(&t), put("nobody.test", &broken, json!({}))).await, Err("AccountNotFound".into()) ); // So is `validate: true` on a type with no lexicon, and an invalid key, there: the record stage follows the repo. let unknown = json!({ "repo": bob, "collection": "com.example.note", "rkey": "n", "record": {}, "validate": true }); assert_eq!( call(&s, PUT, Some(&t), unknown).await, Err("AuthenticationRequired".into()) ); let bad_key = json!({ "repo": bob, "collection": "app.bsky.actor.profile", "record": {} }); assert_eq!( call(&s, CREATE, Some(&t), bad_key).await, Err("AuthenticationRequired".into()) ); let writes = json!({ "repo": bob, "writes": [{ "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.feed.post", "value": broken }] }); assert_eq!( call(&s, APPLY, Some(&t), writes).await, Err("AuthenticationRequired".into()) ); let null = json!({ "repo": bob, "collection": "com.example.note", "rkey": "n", "record": {}, "validate": null }); assert_eq!( call(&s, PUT, Some(&t), null).await, Err("InvalidRequest".into()) ); // An unchecked record is held only to the data model: a nested `$type` // that is no string, or a union value without one, reaches the repo check. let odd = json!({ "$type": "app.bsky.feed.post", "text": "x", "createdAt": AT, "reply": { "$type": { "a": 1 } }, "embed": { "x": 1 }, "y": [{ "$type": 5 }] }); assert_eq!( call( &s, PUT, Some(&t), put(&bob, &odd, json!({ "validate": false })) ) .await, Err("AuthenticationRequired".into()) ); let stored = call( &s, PUT, Some(&t), put( &did, &odd, json!({ "validate": false, "rkey": "3jzfcijpj2z2b" }), ), ) .await .unwrap(); assert_eq!( stored["cid"], json!(cid_string( &pds_repo::make_record(from_json(&odd).unwrap()).cid )) ); assert_eq!( call( &s, PUT, Some(&t), put(&did, &odd, json!({ "rkey": "3jzfcijpj2z2c" })) ) .await, Err("InvalidRequest".into()) ); let note = json!({ "repo": did, "collection": "com.example.note", "rkey": "odd", "record": { "x": { "$type": 5 } } }); assert_eq!( call(&s, PUT, Some(&t), note).await.unwrap()["validationStatus"], json!("unknown") ); // A lexicon of anything but a record makes no type known. let images = json!({ "repo": did, "collection": "app.bsky.embed.images", "rkey": "i", "record": { "images": 5 } }); assert_eq!( call(&s, PUT, Some(&t), images.clone()).await.unwrap()["validationStatus"], json!("unknown") ); let mut strict = images; strict["validate"] = json!(true); assert_eq!( call(&s, PUT, Some(&t), strict).await, Err("InvalidRequest".into()) ); assert_eq!( call( &s, PUT, Some(&t), put( &did, &broken, json!({ "swapCommit": cid_string(&cid_of(0x71, b"x")) }) ) ) .await, Err("InvalidRequest".into()) ); let missing_blob = json!({ "$type": "blob", "ref": { "$link": cid_string(&cid_of(0x55, b"x")) }, "mimeType": "image/png", "size": 1 }); let profile = |avatar: Json| json!({ "repo": did, "collection": "app.bsky.actor.profile", "rkey": "self", "record": { "avatar": avatar } }); assert_eq!(call(&s, PUT, Some(&t), profile(json!({ "$type": "blob", "ref": missing_blob["ref"], "mimeType": "image/gif", "size": 1 }))).await, Err("InvalidRequest".into())); assert_eq!( call(&s, PUT, Some(&t), profile(missing_blob)).await, Err("BlobNotFound".into()) ); // Kept unchecked by `validate: false`, then written again exactly as stored. call( &s, PUT, Some(&t), put(&did, &broken, json!({ "validate": false })), ) .await .unwrap(); assert_eq!( call(&s, PUT, Some(&t), put(&did, &broken, json!({}))).await, Err("InvalidRequest".into()) ); let same = call( &s, PUT, Some(&t), put(&did, &broken, json!({ "validate": false })), ) .await .unwrap(); assert!( same.get("commit").is_none() && same.get("validationStatus").is_none(), "{same}" ); // A key that exists is createRecord's 500 only once the record passes. let create_at = |record: &Json| json!({ "repo": did, "collection": "app.bsky.feed.post", "rkey": "3jzfcijpj2z2a", "record": record }); assert_eq!( call(&s, CREATE, Some(&t), create_at(&broken)).await, Err("InvalidRequest".into()) ); assert_eq!( call(&s, CREATE, Some(&t), create_at(&post("x"))).await, Err("InternalServerError".into()) ); let _ = std::fs::remove_dir_all(&dir); } /// Each type's `key`, and the TID a write without a key makes. #[tokio::test] async fn record_keys() { let plc = plc().await; let dir = scratch("validate-keys"); let s = open(&dir, &plc); let (did, t) = create(&s, "alice").await; let write = |nsid: &'static str, collection: &str, rkey: Option<&str>, record: Json, validate: Option| { let mut b = json!({ "repo": did, "collection": collection, "record": record }); if let Some(k) = rkey { b["rkey"] = json!(k); } if let Some(v) = validate { b["validate"] = json!(v); } let s = s.clone(); let t = t.clone(); async move { call(&s, nsid, Some(&t), b).await.map(|_| ()) } }; let bad = Err("InvalidRequest".to_string()); let profile = json!({ "displayName": "a" }); let p = "app.bsky.actor.profile"; assert_eq!( write(CREATE, p, None, profile.clone(), None).await, bad, "a TID is no self" ); assert_eq!( write(PUT, p, Some("Self"), profile.clone(), None).await, bad ); assert_eq!( write(PUT, p, Some("3jzfcijpj2z2a"), profile.clone(), None).await, bad ); assert_eq!( write(PUT, p, Some("self"), profile.clone(), None).await, Ok(()) ); assert_eq!( write(CREATE, p, None, profile.clone(), Some(false)).await, Ok(()), "unchecked with validate: false" ); let post_c = "app.bsky.feed.post"; for k in ["self", "3jzfcijpj2z2", "3JZFCIJPJ2Z2A", "kjzfcijpj2z2a"] { assert_eq!( write(PUT, post_c, Some(k), post("a"), None).await, bad, "{k}" ); } assert_eq!(write(CREATE, post_c, None, post("a"), None).await, Ok(())); let generator = json!({ "did": "did:web:feed.test", "displayName": "g", "createdAt": AT }); assert_eq!( write( PUT, "app.bsky.feed.generator", Some("any-key:at~all"), generator, None ) .await, Ok(()) ); let schema = json!({ "lexicon": 1 }); assert_eq!( write( PUT, "com.atproto.lexicon.schema", Some("not-an-nsid"), schema.clone(), None ) .await, bad ); assert_eq!( write( CREATE, "com.atproto.lexicon.schema", None, schema.clone(), None ) .await, bad ); assert_eq!( write( PUT, "com.atproto.lexicon.schema", Some("com.example.thing"), schema, None ) .await, Ok(()) ); let _ = std::fs::remove_dir_all(&dir); } /// `createRecord` of a follow, block, like or repost deletes, in its own /// commit, the account's others of the same subject, however they were /// made; with `validate: false` it deletes none, and no other write does. #[tokio::test] async fn duplicates() { let plc = plc().await; let dir = scratch("validate-dups"); let s = open(&dir, &plc); let (did, t) = create(&s, "alice").await; let (bob, _) = create(&s, "bob").await; let f = "app.bsky.graph.follow"; let create_rec = |record: Json, validate: Option| { let mut b = json!({ "repo": did, "collection": record["$type"], "record": record }); if let Some(v) = validate { b["validate"] = json!(v); } let (s, t) = (s.clone(), t.clone()); async move { call(&s, CREATE, Some(&t), b).await.unwrap() } }; let first = create_rec(follow(&bob), None).await; let k1 = first["uri"] .as_str() .unwrap() .rsplit('/') .next() .unwrap() .to_string(); let second = create_rec(follow(&bob), None).await; let k2 = second["uri"] .as_str() .unwrap() .rsplit('/') .next() .unwrap() .to_string(); assert_eq!(keys(&s, &did, f), vec![k2.clone()]); assert_eq!( last_ops(&s), vec![ ("delete".into(), format!("{f}/{k1}")), ("create".into(), format!("{f}/{k2}")) ] ); // Kept by validate: false, putRecord and applyWrites. create_rec(follow(&bob), Some(false)).await; let put = json!({ "repo": did, "collection": f, "rkey": "3jzfcijpj2z2a", "record": follow(&bob) }); call(&s, PUT, Some(&t), put).await.unwrap(); let apply = json!({ "repo": did, "writes": [{ "$type": "com.atproto.repo.applyWrites#create", "collection": f, "value": follow(&bob) }] }); call(&s, APPLY, Some(&t), apply).await.unwrap(); assert_eq!(keys(&s, &did, f).len(), 4); // A follow of another DID, or one whose subject is no DID, is no duplicate. create_rec(follow("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"), None).await; create_rec(follow("bob.test"), Some(false)).await; assert_eq!(keys(&s, &did, f).len(), 6); // The next createRecord of a follow of bob deletes all four. let last = create_rec(follow(&bob), None).await; let k = last["uri"] .as_str() .unwrap() .rsplit('/') .next() .unwrap() .to_string(); let ops = last_ops(&s); assert_eq!( ops.iter().filter(|(a, _)| a == "delete").count(), 4, "{ops:?}" ); assert_eq!( ops.last().unwrap(), &("create".to_string(), format!("{f}/{k}")) ); assert_eq!(keys(&s, &did, f).len(), 3); assert_eq!(last["validationStatus"], json!("valid")); // Likes by `subject.uri`, compared as strings; a block's DID is apart from a follow's. let uri = format!("at://{bob}/app.bsky.feed.post/3jzfcijpj2z2a"); let l = "app.bsky.feed.like"; create_rec(like(&uri), None).await; create_rec(like("at://bob.test/app.bsky.feed.post/3jzfcijpj2z2a"), None).await; create_rec(like(&uri), None).await; assert_eq!(keys(&s, &did, l).len(), 2); create_rec( json!({ "$type": "app.bsky.graph.block", "subject": bob, "createdAt": AT }), None, ) .await; assert_eq!(keys(&s, &did, f).len(), 3); assert_eq!(keys(&s, &did, "app.bsky.graph.block").len(), 1); // At a key that holds a follow of the same DID, even the same record, it // deletes that one and creates the new one in one commit, whose create's // `prev` is the record deleted. let at = |record: Json, validate: Option| { let mut b = json!({ "repo": did, "collection": f, "rkey": k, "record": record }); if let Some(v) = validate { b["validate"] = json!(v); } let (s, t) = (s.clone(), t.clone()); async move { call(&s, CREATE, Some(&t), b).await } }; let again = at(follow(&bob), None).await.unwrap(); assert_eq!(again["cid"], last["cid"]); let ops = last_event_ops(&s); assert_eq!( last_ops(&s), vec![ ("delete".into(), format!("{f}/{k}")), ("create".into(), format!("{f}/{k}")) ] ); for op in &ops { let Some(V::Link(prev)) = get(op, "prev") else { panic!("an op without prev") }; assert_eq!(json!(cid_string(&prev)), last["cid"]); } assert_eq!(keys(&s, &did, f).len(), 3); // A key holding a follow of another DID is taken, and so is any with `validate: false`. assert_eq!( at(follow("did:plc:bbbbbbbbbbbbbbbbbbbbbbbb"), None).await, Err("InternalServerError".into()) ); assert_eq!( at(follow(&bob), Some(false)).await, Err("InternalServerError".into()) ); // The deletes survive a restart, as any are. drop(s); let s = open(&dir, &plc); assert_eq!((keys(&s, &did, f).len(), keys(&s, &did, l).len()), (3, 2)); let _ = std::fs::remove_dir_all(&dir); } /// `applyWrites` checks each write in order, answers each result's /// `validationStatus`, and makes nothing if one fails. #[tokio::test] async fn apply_writes() { let plc = plc().await; let dir = scratch("validate-apply"); let s = open(&dir, &plc); let (did, t) = create(&s, "alice").await; let create_w = |collection: &str, rkey: Option<&str>, value: Json| { let mut w = json!({ "$type": "com.atproto.repo.applyWrites#create", "collection": collection, "value": value }); if let Some(k) = rkey { w["rkey"] = json!(k); } w }; let apply = |writes: Vec, validate: Option| { let mut b = json!({ "repo": did, "writes": writes }); if let Some(v) = validate { b["validate"] = json!(v); } let (s, t) = (s.clone(), t.clone()); async move { call(&s, APPLY, Some(&t), b).await } }; let p = "app.bsky.feed.post"; let got = apply( vec![ create_w(p, Some("3jzfcijpj2z2a"), post("a")), create_w("com.example.note", Some("n"), json!({ "n": 1 })), ], None, ) .await .unwrap(); let statuses: Vec = got["results"] .as_array() .unwrap() .iter() .map(|r| r["validationStatus"].clone()) .collect(); assert_eq!(statuses, vec![json!("valid"), json!("unknown")]); let update = json!({ "$type": "com.atproto.repo.applyWrites#update", "collection": p, "rkey": "3jzfcijpj2z2a", "value": post("b") }); let delete = json!({ "$type": "com.atproto.repo.applyWrites#delete", "collection": "com.example.note", "rkey": "n" }); let got = apply(vec![update.clone(), delete.clone()], None) .await .unwrap(); assert_eq!(got["results"][0]["validationStatus"], json!("valid")); assert_eq!( got["results"][0]["$type"], json!("com.atproto.repo.applyWrites#updateResult") ); assert_eq!( got["results"][1], json!({ "$type": "com.atproto.repo.applyWrites#deleteResult" }) ); let got = apply( vec![ create_w(p, None, post("c")), create_w("com.example.note", None, json!({})), ], Some(false), ) .await .unwrap(); assert!( got["results"] .as_array() .unwrap() .iter() .all(|r| r.get("validationStatus").is_none()), "{got}" ); // One broken write, wherever it is, makes nothing. let before = keys(&s, &did, p); let broken = create_w(p, None, json!({ "$type": p, "text": "x" })); for writes in [ vec![broken.clone(), create_w(p, None, post("d"))], vec![create_w(p, None, post("d")), broken.clone()], ] { assert_eq!(apply(writes, None).await, Err("InvalidRequest".into())); } assert_eq!( apply( vec![create_w(p, None, post("d")), broken.clone()], Some(true) ) .await, Err("InvalidRequest".into()) ); assert_eq!( apply( vec![ create_w("com.example.note", None, json!({})), create_w(p, None, post("d")) ], Some(true) ) .await, Err("InvalidRequest".into()) ); let mut bad_update = update; bad_update["value"] = json!({ "$type": p, "text": "x" }); assert_eq!( apply(vec![bad_update], None).await, Err("InvalidRequest".into()) ); assert_eq!(keys(&s, &did, p), before); // A create without a key of a `literal:self` type. assert_eq!( apply( vec![create_w("app.bsky.actor.profile", None, json!({}))], None ) .await, Err("InvalidRequest".into()) ); apply( vec![create_w("app.bsky.actor.profile", None, json!({}))], Some(false), ) .await .unwrap(); apply(vec![broken], Some(false)).await.unwrap(); let _ = std::fs::remove_dir_all(&dir); } }