chadtmiller.com/monolith
An atproto PDS
0
0
//! The repository whole: `getRepo` and `getLatestCommit`, as `spec/SYNC.md`
//! says, of any account's repository. The firehose, `subscribeRepos`, is in
//! `events`.
use crate::cache::Ctx;
use crate::checkpoint::Snapshot;
use crate::{
Call, Handler, Resp, Service, Shown, Store, XrpcError, commit_meta, internal, opt, required,
};
use axum::body::{Body, Bytes};
use axum::http::{StatusCode, header};
use axum::response::{IntoResponse, Response};
use http_body::{Body as HttpBody, Frame};
use serde_json::{Map, Value as Json};
use std::io;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
pub(crate) fn handler(nsid: &str) -> Option<Handler> {
Some(match nsid {
"com.atproto.sync.getRepo" => Handler::Query(get_repo),
"com.atproto.sync.getLatestCommit" => Handler::Query(get_latest_commit),
_ => return None,
})
}
pub(crate) fn not_found() -> XrpcError {
XrpcError {
status: 400,
name: "RepoNotFound",
message: "Could not find repo".into(),
}
}
fn get_repo(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
s.with_store(
shown,
required(p, "did")?,
not_found,
|_, st| match CarStream::of(st, opt(p, "since")) {
Ok(c) => Ok(Resp::CarStream(c)),
Err(e) => {
crate::ops::failed_for(crate::ops::Level::Error, "getRepo failed", &st.repo.did, e);
internal()
}
},
)
}
/// A repository as a CAR file, rooted at its latest commit, read from its
/// store's files as it is sent: the commit, then each block of its tree
/// that a commit after `since` put there (every one without), by CID.
pub(crate) struct CarStream {
snapshot: Snapshot,
root: Vec<u8>,
since: Option<String>,
ctx: Arc<Ctx>,
}
/// The bytes a stream sends at a time.
pub(crate) const CHUNK: usize = 64 * 1024;
impl CarStream {
pub(crate) fn of(st: &Store, since: Option<&str>) -> io::Result<CarStream> {
Ok(CarStream {
snapshot: st.snapshot()?,
root: st.repo.head.clone(),
since: since.map(str::to_string),
ctx: st.ctx.clone(),
})
}
/// Give `send` the file in chunks of about `CHUNK` bytes, as it is read;
/// stop when it says to.
fn run(self, mut send: impl FnMut(Bytes) -> bool) -> io::Result<()> {
let CarStream {
snapshot,
root,
since,
ctx,
} = self;
let mut buf = pds_repo::car_file(&root, std::iter::empty());
let mut open = true;
snapshot.run(&ctx.blocks, since.as_deref(), |cid, bytes| {
varint(&mut buf, cid.len() + bytes.len());
buf.extend_from_slice(cid);
buf.extend_from_slice(bytes);
if buf.len() >= CHUNK {
open = send(Bytes::from(std::mem::take(&mut buf)));
}
open
})?;
if open && !buf.is_empty() {
send(Bytes::from(buf));
}
Ok(())
}
/// The file whole, read now.
#[cfg(test)]
pub(crate) fn bytes(self) -> io::Result<Vec<u8>> {
let mut out = Vec::new();
self.run(|b| {
out.extend_from_slice(&b);
true
})?;
Ok(out)
}
/// The answer: the file sent as it is read, a few chunks ahead of the
/// reader at most.
pub(crate) fn response(self) -> Response {
let (tx, chunks) = tokio::sync::mpsc::channel(4);
tokio::task::spawn_blocking(move || {
let failed = tx.clone();
if let Err(e) = self.run(|b| tx.blocking_send(Ok(b)).is_ok()) {
crate::ops::failed("getRepo failed", &e);
let _ = failed.blocking_send(Err(e));
}
});
(
StatusCode::OK,
[(header::CONTENT_TYPE, "application/vnd.ipld.car")],
Body::new(Chunks(chunks)),
)
.into_response()
}
}
pub(crate) fn varint(out: &mut Vec<u8>, mut n: usize) {
while n >= 0x80 {
out.push((n as u8) | 0x80);
n >>= 7;
}
out.push(n as u8);
}
/// A body sent as its chunks come, until they end.
pub(crate) struct Chunks(pub(crate) tokio::sync::mpsc::Receiver<io::Result<Bytes>>);
impl HttpBody for Chunks {
type Data = Bytes;
type Error = io::Error;
fn poll_frame(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, io::Error>>> {
self.get_mut()
.0
.poll_recv(cx)
.map(|c| c.map(|r| r.map(Frame::data)))
}
}
fn get_latest_commit(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
s.with_repo(shown, required(p, "did")?, not_found, |_, r| {
Ok(Resp::Json(commit_meta(r)))
})
}
#[cfg(test)]
mod tests {
use crate::testing::*;
use pds_repo::data::*;
use pds_repo::*;
use serde_json::json;
/// `getRepo since` answers the same after a restart, which rebuilds when
/// each block was added from the log.
#[tokio::test]
async fn since_survives_restart() {
let plc = plc().await;
let dir = scratch("since");
let s = open(&dir, &plc);
let (did, owner) = create(&s, "alice").await;
let (bob, other) = create(&s, "bob").await;
let rev = |s: &crate::Service| repo_of(s, &did, |r| r.rev.clone());
let mut revs = vec![rev(&s)];
for (k, n) in [("one", 1), ("two", 2), ("one", 3), ("three", 1)] {
let body =
json!({ "repo": did, "collection": "a.b.c", "rkey": k, "record": { "n": n } });
call(&s, "com.atproto.repo.putRecord", Some(&owner), body)
.await
.unwrap();
// Another account's commits change nothing of this one's.
let body =
json!({ "repo": bob, "collection": "a.b.c", "rkey": k, "record": { "n": n + 10 } });
call(&s, "com.atproto.repo.putRecord", Some(&other), body)
.await
.unwrap();
revs.push(rev(&s));
}
call(
&s,
"com.atproto.repo.deleteRecord",
Some(&owner),
json!({ "repo": did, "collection": "a.b.c", "rkey": "two" }),
)
.await
.unwrap();
revs.push(rev(&s));
let cars = |s: &crate::Service| -> Vec<Vec<u8>> {
revs.iter()
.map(|r| {
store_of(s, &did, |st| {
super::CarStream::of(st, Some(r)).unwrap().bytes().unwrap()
})
})
.collect()
};
let before = cars(&s);
// At the latest rev, the CAR is its header alone.
let latest = before.last().unwrap();
assert_eq!(
latest,
&repo_of(&s, &did, |r| car_file(&r.head, std::iter::empty()))
);
// A record's block is among them from the commit that last made the
// repository hold it: `three` holds again what `one` first held.
let holds = |car: &[u8], n: i64| {
let r = make_record(from_json(&json!({ "n": n, "$type": "a.b.c" })).unwrap());
car.windows(r.bytes.len()).any(|w| w == r.bytes.as_slice())
};
assert!(holds(&before[3], 1) && !holds(&before[3], 2) && !holds(&before[3], 3));
assert!(!holds(&before[4], 1));
assert!(holds(&before[0], 1) && holds(&before[0], 3) && !holds(&before[0], 2));
assert!(!before.iter().any(|car| holds(car, 11)));
drop(s);
let s = open(&dir, &plc);
assert_eq!(before, cars(&s));
// getRepo and getLatestCommit of each, by DID.
let p = |d: &str| [("did".to_string(), d.to_string())];
let latest = call_with(
&s,
"com.atproto.sync.getLatestCommit",
None,
&p(&bob),
json!({}),
)
.await
.unwrap();
assert_eq!(latest["rev"], json!(repo_of(&s, &bob, |r| r.rev.clone())));
assert_eq!(
call_with(&s, "com.atproto.sync.getRepo", None, &p(&did), json!({})).await,
Err("a CAR file".into())
);
let nobody = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";
assert_eq!(
call_with(
&s,
"com.atproto.sync.getLatestCommit",
None,
&p(nobody),
json!({})
)
.await,
Err("RepoNotFound".into())
);
let _ = std::fs::remove_dir_all(&dir);
}
}