~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
273 lines · 8.5 KB Raw History
//! The repository whole: `getRepo` and `getLatestCommit`, as `spec/SYNC.md`
//! says, of any account's repository. The firehose, `subscribeRepos`, is in
//! `events`.

use crate::cache::Ctx;
use crate::checkpoint::Snapshot;
use crate::{
    Call, Handler, Resp, Service, Shown, Store, XrpcError, commit_meta, internal, opt, required,
};
use axum::body::{Body, Bytes};
use axum::http::{StatusCode, header};
use axum::response::{IntoResponse, Response};
use http_body::{Body as HttpBody, Frame};
use serde_json::{Map, Value as Json};
use std::io;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};

pub(crate) fn handler(nsid: &str) -> Option<Handler> {
    Some(match nsid {
        "com.atproto.sync.getRepo" => Handler::Query(get_repo),
        "com.atproto.sync.getLatestCommit" => Handler::Query(get_latest_commit),
        _ => return None,
    })
}

pub(crate) fn not_found() -> XrpcError {
    XrpcError {
        status: 400,
        name: "RepoNotFound",
        message: "Could not find repo".into(),
    }
}

fn get_repo(
    s: &Service,
    _: &Call,
    shown: &mut Shown,
    p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
    s.with_store(
        shown,
        required(p, "did")?,
        not_found,
        |_, st| match CarStream::of(st, opt(p, "since")) {
            Ok(c) => Ok(Resp::CarStream(c)),
            Err(e) => {
                crate::ops::failed_for(crate::ops::Level::Error, "getRepo failed", &st.repo.did, e);
                internal()
            }
        },
    )
}

/// A repository as a CAR file, rooted at its latest commit, read from its
/// store's files as it is sent: the commit, then each block of its tree
/// that a commit after `since` put there (every one without), by CID.
pub(crate) struct CarStream {
    snapshot: Snapshot,
    root: Vec<u8>,
    since: Option<String>,
    ctx: Arc<Ctx>,
}

/// The bytes a stream sends at a time.
pub(crate) const CHUNK: usize = 64 * 1024;

impl CarStream {
    pub(crate) fn of(st: &Store, since: Option<&str>) -> io::Result<CarStream> {
        Ok(CarStream {
            snapshot: st.snapshot()?,
            root: st.repo.head.clone(),
            since: since.map(str::to_string),
            ctx: st.ctx.clone(),
        })
    }

    /// Give `send` the file in chunks of about `CHUNK` bytes, as it is read;
    /// stop when it says to.
    fn run(self, mut send: impl FnMut(Bytes) -> bool) -> io::Result<()> {
        let CarStream {
            snapshot,
            root,
            since,
            ctx,
        } = self;
        let mut buf = pds_repo::car_file(&root, std::iter::empty());
        let mut open = true;
        snapshot.run(&ctx.blocks, since.as_deref(), |cid, bytes| {
            varint(&mut buf, cid.len() + bytes.len());
            buf.extend_from_slice(cid);
            buf.extend_from_slice(bytes);
            if buf.len() >= CHUNK {
                open = send(Bytes::from(std::mem::take(&mut buf)));
            }
            open
        })?;
        if open && !buf.is_empty() {
            send(Bytes::from(buf));
        }
        Ok(())
    }

    /// The file whole, read now.
    #[cfg(test)]
    pub(crate) fn bytes(self) -> io::Result<Vec<u8>> {
        let mut out = Vec::new();
        self.run(|b| {
            out.extend_from_slice(&b);
            true
        })?;
        Ok(out)
    }

    /// The answer: the file sent as it is read, a few chunks ahead of the
    /// reader at most.
    pub(crate) fn response(self) -> Response {
        let (tx, chunks) = tokio::sync::mpsc::channel(4);
        tokio::task::spawn_blocking(move || {
            let failed = tx.clone();
            if let Err(e) = self.run(|b| tx.blocking_send(Ok(b)).is_ok()) {
                crate::ops::failed("getRepo failed", &e);
                let _ = failed.blocking_send(Err(e));
            }
        });
        (
            StatusCode::OK,
            [(header::CONTENT_TYPE, "application/vnd.ipld.car")],
            Body::new(Chunks(chunks)),
        )
            .into_response()
    }
}

pub(crate) fn varint(out: &mut Vec<u8>, mut n: usize) {
    while n >= 0x80 {
        out.push((n as u8) | 0x80);
        n >>= 7;
    }
    out.push(n as u8);
}

/// A body sent as its chunks come, until they end.
pub(crate) struct Chunks(pub(crate) tokio::sync::mpsc::Receiver<io::Result<Bytes>>);

impl HttpBody for Chunks {
    type Data = Bytes;
    type Error = io::Error;

    fn poll_frame(
        self: Pin<&mut Self>,
        cx: &mut Context<'_>,
    ) -> Poll<Option<Result<Frame<Bytes>, io::Error>>> {
        self.get_mut()
            .0
            .poll_recv(cx)
            .map(|c| c.map(|r| r.map(Frame::data)))
    }
}

fn get_latest_commit(
    s: &Service,
    _: &Call,
    shown: &mut Shown,
    p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
    s.with_repo(shown, required(p, "did")?, not_found, |_, r| {
        Ok(Resp::Json(commit_meta(r)))
    })
}

#[cfg(test)]
mod tests {
    use crate::testing::*;
    use pds_repo::data::*;
    use pds_repo::*;
    use serde_json::json;

    /// `getRepo since` answers the same after a restart, which rebuilds when
    /// each block was added from the log.
    #[tokio::test]
    async fn since_survives_restart() {
        let plc = plc().await;
        let dir = scratch("since");
        let s = open(&dir, &plc);
        let (did, owner) = create(&s, "alice").await;
        let (bob, other) = create(&s, "bob").await;
        let rev = |s: &crate::Service| repo_of(s, &did, |r| r.rev.clone());
        let mut revs = vec![rev(&s)];
        for (k, n) in [("one", 1), ("two", 2), ("one", 3), ("three", 1)] {
            let body =
                json!({ "repo": did, "collection": "a.b.c", "rkey": k, "record": { "n": n } });
            call(&s, "com.atproto.repo.putRecord", Some(&owner), body)
                .await
                .unwrap();
            // Another account's commits change nothing of this one's.
            let body =
                json!({ "repo": bob, "collection": "a.b.c", "rkey": k, "record": { "n": n + 10 } });
            call(&s, "com.atproto.repo.putRecord", Some(&other), body)
                .await
                .unwrap();
            revs.push(rev(&s));
        }
        call(
            &s,
            "com.atproto.repo.deleteRecord",
            Some(&owner),
            json!({ "repo": did, "collection": "a.b.c", "rkey": "two" }),
        )
        .await
        .unwrap();
        revs.push(rev(&s));
        let cars = |s: &crate::Service| -> Vec<Vec<u8>> {
            revs.iter()
                .map(|r| {
                    store_of(s, &did, |st| {
                        super::CarStream::of(st, Some(r)).unwrap().bytes().unwrap()
                    })
                })
                .collect()
        };
        let before = cars(&s);
        // At the latest rev, the CAR is its header alone.
        let latest = before.last().unwrap();
        assert_eq!(
            latest,
            &repo_of(&s, &did, |r| car_file(&r.head, std::iter::empty()))
        );
        // A record's block is among them from the commit that last made the
        // repository hold it: `three` holds again what `one` first held.
        let holds = |car: &[u8], n: i64| {
            let r = make_record(from_json(&json!({ "n": n, "$type": "a.b.c" })).unwrap());
            car.windows(r.bytes.len()).any(|w| w == r.bytes.as_slice())
        };
        assert!(holds(&before[3], 1) && !holds(&before[3], 2) && !holds(&before[3], 3));
        assert!(!holds(&before[4], 1));
        assert!(holds(&before[0], 1) && holds(&before[0], 3) && !holds(&before[0], 2));
        assert!(!before.iter().any(|car| holds(car, 11)));
        drop(s);
        let s = open(&dir, &plc);
        assert_eq!(before, cars(&s));
        // getRepo and getLatestCommit of each, by DID.
        let p = |d: &str| [("did".to_string(), d.to_string())];
        let latest = call_with(
            &s,
            "com.atproto.sync.getLatestCommit",
            None,
            &p(&bob),
            json!({}),
        )
        .await
        .unwrap();
        assert_eq!(latest["rev"], json!(repo_of(&s, &bob, |r| r.rev.clone())));
        assert_eq!(
            call_with(&s, "com.atproto.sync.getRepo", None, &p(&did), json!({})).await,
            Err("a CAR file".into())
        );
        let nobody = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";
        assert_eq!(
            call_with(
                &s,
                "com.atproto.sync.getLatestCommit",
                None,
                &p(nobody),
                json!({})
            )
            .await,
            Err("RepoNotFound".into())
        );
        let _ = std::fs::remove_dir_all(&dir);
    }
}