//! The repository whole: `getRepo` and `getLatestCommit`, as `spec/SYNC.md` //! says, of any account's repository. The firehose, `subscribeRepos`, is in //! `events`. use crate::cache::Ctx; use crate::checkpoint::Snapshot; use crate::{ Call, Handler, Resp, Service, Shown, Store, XrpcError, commit_meta, internal, opt, required, }; use axum::body::{Body, Bytes}; use axum::http::{StatusCode, header}; use axum::response::{IntoResponse, Response}; use http_body::{Body as HttpBody, Frame}; use serde_json::{Map, Value as Json}; use std::io; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; pub(crate) fn handler(nsid: &str) -> Option { Some(match nsid { "com.atproto.sync.getRepo" => Handler::Query(get_repo), "com.atproto.sync.getLatestCommit" => Handler::Query(get_latest_commit), _ => return None, }) } pub(crate) fn not_found() -> XrpcError { XrpcError { status: 400, name: "RepoNotFound", message: "Could not find repo".into(), } } fn get_repo( s: &Service, _: &Call, shown: &mut Shown, p: &Map, ) -> Result { s.with_store( shown, required(p, "did")?, not_found, |_, st| match CarStream::of(st, opt(p, "since")) { Ok(c) => Ok(Resp::CarStream(c)), Err(e) => { crate::ops::failed_for(crate::ops::Level::Error, "getRepo failed", &st.repo.did, e); internal() } }, ) } /// A repository as a CAR file, rooted at its latest commit, read from its /// store's files as it is sent: the commit, then each block of its tree /// that a commit after `since` put there (every one without), by CID. pub(crate) struct CarStream { snapshot: Snapshot, root: Vec, since: Option, ctx: Arc, } /// The bytes a stream sends at a time. pub(crate) const CHUNK: usize = 64 * 1024; impl CarStream { pub(crate) fn of(st: &Store, since: Option<&str>) -> io::Result { Ok(CarStream { snapshot: st.snapshot()?, root: st.repo.head.clone(), since: since.map(str::to_string), ctx: st.ctx.clone(), }) } /// Give `send` the file in chunks of about `CHUNK` bytes, as it is read; /// stop when it says to. fn run(self, mut send: impl FnMut(Bytes) -> bool) -> io::Result<()> { let CarStream { snapshot, root, since, ctx, } = self; let mut buf = pds_repo::car_file(&root, std::iter::empty()); let mut open = true; snapshot.run(&ctx.blocks, since.as_deref(), |cid, bytes| { varint(&mut buf, cid.len() + bytes.len()); buf.extend_from_slice(cid); buf.extend_from_slice(bytes); if buf.len() >= CHUNK { open = send(Bytes::from(std::mem::take(&mut buf))); } open })?; if open && !buf.is_empty() { send(Bytes::from(buf)); } Ok(()) } /// The file whole, read now. #[cfg(test)] pub(crate) fn bytes(self) -> io::Result> { let mut out = Vec::new(); self.run(|b| { out.extend_from_slice(&b); true })?; Ok(out) } /// The answer: the file sent as it is read, a few chunks ahead of the /// reader at most. pub(crate) fn response(self) -> Response { let (tx, chunks) = tokio::sync::mpsc::channel(4); tokio::task::spawn_blocking(move || { let failed = tx.clone(); if let Err(e) = self.run(|b| tx.blocking_send(Ok(b)).is_ok()) { crate::ops::failed("getRepo failed", &e); let _ = failed.blocking_send(Err(e)); } }); ( StatusCode::OK, [(header::CONTENT_TYPE, "application/vnd.ipld.car")], Body::new(Chunks(chunks)), ) .into_response() } } pub(crate) fn varint(out: &mut Vec, mut n: usize) { while n >= 0x80 { out.push((n as u8) | 0x80); n >>= 7; } out.push(n as u8); } /// A body sent as its chunks come, until they end. pub(crate) struct Chunks(pub(crate) tokio::sync::mpsc::Receiver>); impl HttpBody for Chunks { type Data = Bytes; type Error = io::Error; fn poll_frame( self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll, io::Error>>> { self.get_mut() .0 .poll_recv(cx) .map(|c| c.map(|r| r.map(Frame::data))) } } fn get_latest_commit( s: &Service, _: &Call, shown: &mut Shown, p: &Map, ) -> Result { s.with_repo(shown, required(p, "did")?, not_found, |_, r| { Ok(Resp::Json(commit_meta(r))) }) } #[cfg(test)] mod tests { use crate::testing::*; use pds_repo::data::*; use pds_repo::*; use serde_json::json; /// `getRepo since` answers the same after a restart, which rebuilds when /// each block was added from the log. #[tokio::test] async fn since_survives_restart() { let plc = plc().await; let dir = scratch("since"); let s = open(&dir, &plc); let (did, owner) = create(&s, "alice").await; let (bob, other) = create(&s, "bob").await; let rev = |s: &crate::Service| repo_of(s, &did, |r| r.rev.clone()); let mut revs = vec![rev(&s)]; for (k, n) in [("one", 1), ("two", 2), ("one", 3), ("three", 1)] { let body = json!({ "repo": did, "collection": "a.b.c", "rkey": k, "record": { "n": n } }); call(&s, "com.atproto.repo.putRecord", Some(&owner), body) .await .unwrap(); // Another account's commits change nothing of this one's. let body = json!({ "repo": bob, "collection": "a.b.c", "rkey": k, "record": { "n": n + 10 } }); call(&s, "com.atproto.repo.putRecord", Some(&other), body) .await .unwrap(); revs.push(rev(&s)); } call( &s, "com.atproto.repo.deleteRecord", Some(&owner), json!({ "repo": did, "collection": "a.b.c", "rkey": "two" }), ) .await .unwrap(); revs.push(rev(&s)); let cars = |s: &crate::Service| -> Vec> { revs.iter() .map(|r| { store_of(s, &did, |st| { super::CarStream::of(st, Some(r)).unwrap().bytes().unwrap() }) }) .collect() }; let before = cars(&s); // At the latest rev, the CAR is its header alone. let latest = before.last().unwrap(); assert_eq!( latest, &repo_of(&s, &did, |r| car_file(&r.head, std::iter::empty())) ); // A record's block is among them from the commit that last made the // repository hold it: `three` holds again what `one` first held. let holds = |car: &[u8], n: i64| { let r = make_record(from_json(&json!({ "n": n, "$type": "a.b.c" })).unwrap()); car.windows(r.bytes.len()).any(|w| w == r.bytes.as_slice()) }; assert!(holds(&before[3], 1) && !holds(&before[3], 2) && !holds(&before[3], 3)); assert!(!holds(&before[4], 1)); assert!(holds(&before[0], 1) && holds(&before[0], 3) && !holds(&before[0], 2)); assert!(!before.iter().any(|car| holds(car, 11))); drop(s); let s = open(&dir, &plc); assert_eq!(before, cars(&s)); // getRepo and getLatestCommit of each, by DID. let p = |d: &str| [("did".to_string(), d.to_string())]; let latest = call_with( &s, "com.atproto.sync.getLatestCommit", None, &p(&bob), json!({}), ) .await .unwrap(); assert_eq!(latest["rev"], json!(repo_of(&s, &bob, |r| r.rev.clone()))); assert_eq!( call_with(&s, "com.atproto.sync.getRepo", None, &p(&did), json!({})).await, Err("a CAR file".into()) ); let nobody = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; assert_eq!( call_with( &s, "com.atproto.sync.getLatestCommit", None, &p(nobody), json!({}) ) .await, Err("RepoNotFound".into()) ); let _ = std::fs::remove_dir_all(&dir); } }