~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
902 lines · 32.5 KB Raw History
//! What a relay reads, as `spec/FEDERATION.md` says: `listRepos`,
//! `getRepoStatus`, `getBlocks` and the sync `getRecord`; and the
//! `requestCrawl` the server sends the crawlers `PDS_CRAWLERS` names.

use crate::sync::not_found;
use crate::{
    Account, Call, Handler, Resp, Service, Shared, Shown, XrpcError, invalid, opt, read, required,
};
use pds_repo::data::parse_cid;
use serde_json::{Map, Value as Json, json};
use std::sync::Arc;
use std::time::Duration;

pub(crate) fn handler(nsid: &str) -> Option<Handler> {
    Some(match nsid {
        "com.atproto.sync.listRepos" => Handler::Query(list_repos),
        "com.atproto.sync.getRepoStatus" => Handler::Query(get_repo_status),
        "com.atproto.sync.getBlocks" => Handler::Query(get_blocks),
        "com.atproto.sync.getRecord" => Handler::Query(get_record),
        _ => return None,
    })
}

/// Every account whose repository has its first commit, in the order
/// `accounts.log` has them, each active or not and why
/// (`spec/LIFECYCLE.md`). A cursor is the ordinal of the first account of
/// the next page, its place in the order accounts were made: as each
/// account keeps its ordinal, and a deleted account's is never given
/// again, it stays good. The listing stops before an account still being
/// made, so that no cursor passes over it.
fn list_repos(
    s: &Service,
    _: &Call,
    shown: &mut Shown,
    p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
    let limit = p.get("limit").and_then(Json::as_u64).unwrap_or(500) as usize;
    let mut from = match opt(p, "cursor") {
        None | Some("") => 0,
        Some(c) => match c.parse::<u64>() {
            Ok(n) if n <= read(&s.directory).next_ordinal() => n,
            _ => return invalid("Malformed cursor"),
        },
    };
    // Each account is loaded as it is listed. One deleted since its place
    // was read is no account, and the page goes on past it.
    let mut accounts: Vec<(u64, Arc<Account>)> = Vec::new();
    while accounts.len() < limit {
        let wanted = limit - accounts.len();
        let page = read(&s.directory).page(from, wanted);
        let Some((last, _)) = page.last() else { break };
        from = last + 1;
        let full = page.len() == wanted;
        accounts.extend(
            page.into_iter()
                .filter_map(|(at, did)| Some((at, s.by_did(&did)?))),
        );
        if !full {
            break;
        }
    }
    let repos: Vec<Json> = accounts
        .iter()
        .map(|(_, a)| {
            let st = read(&a.store);
            shown.0.push((st.wal.clone(), st.wal.written()));
            let mut r = json!({ "did": a.did, "head": st.repo.head_string(), "rev": st.repo.rev, "active": true });
            if let Some(status) = crate::lifecycle::status(a) {
                r["active"] = json!(false);
                r["status"] = json!(status);
            }
            r
        })
        .collect();
    let mut out = json!({ "repos": repos });
    if let Some((at, _)) = accounts.last() {
        out["cursor"] = json!((at + 1).to_string());
    }
    Ok(Resp::Json(out))
}

/// An active account's `{did, active, rev}`, or one not active's status instead of its rev.
fn get_repo_status(
    s: &Service,
    _: &Call,
    shown: &mut Shown,
    p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
    s.with_repo(shown, required(p, "did")?, not_found, |a, r| {
        Ok(Resp::Json(match crate::lifecycle::status(a) {
            None => json!({ "did": a.did, "active": true, "rev": r.rev }),
            Some(status) => json!({ "did": a.did, "active": false, "status": status }),
        }))
    })
}

/// The blocks asked for, all of which the repository must hold now.
fn get_blocks(
    s: &Service,
    _: &Call,
    shown: &mut Shown,
    p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
    let cids = p
        .get("cids")
        .and_then(Json::as_array)
        .map_or(&[][..], Vec::as_slice);
    s.with_store(shown, required(p, "did")?, not_found, |_, st| {
        // A CID that is not a SHA-256 one of DAG-CBOR or raw names no block of a repository.
        let parsed: Option<Vec<Vec<u8>>> = cids
            .iter()
            .map(|c| c.as_str().and_then(parse_cid))
            .collect();
        parsed
            .and_then(|c| st.repo.blocks_car(&st.source(), &c))
            .map_or_else(
                || invalid("Could not find all requested blocks"),
                |car| Ok(Resp::Car(car)),
            )
    })
}

/// The proof of the record at `collection/rkey`, or of its absence.
fn get_record(
    s: &Service,
    _: &Call,
    shown: &mut Shown,
    p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
    let path = format!("{}/{}", required(p, "collection")?, required(p, "rkey")?);
    s.with_store(shown, required(p, "did")?, not_found, |_, st| {
        Ok(Resp::Car(st.repo.proof_car(&st.source(), &path)))
    })
}

/// How long after telling the crawlers the server waits before an event tells them again.
const CRAWL_EVERY: Duration = Duration::from_secs(20 * 60);

/// Tell the crawlers `PDS_CRAWLERS` names of the events from now on, in the background.
pub(crate) fn tell_crawlers(svc: &Shared) {
    let c = &svc.config;
    if !c.crawlers.is_empty() {
        tokio::spawn(tell(
            svc.events.subscribe(),
            svc.http.clone(),
            c.crawlers.clone(),
            c.hostname.clone(),
            CRAWL_EVERY,
        ));
    }
}

/// At the first event `events` brings, and then at the first event at least
/// `every` after it last did, `POST` each crawler a `requestCrawl`, each on
/// its own and answered by none, a failure logged (`ops::send`) and tried
/// no more. The events while it waits are passed over, so a burst of them
/// is one request.
async fn tell(
    mut events: tokio::sync::watch::Receiver<u64>,
    http: reqwest::Client,
    crawlers: Vec<String>,
    hostname: String,
    every: Duration,
) {
    let body = json!({ "hostname": hostname }).to_string();
    while events.changed().await.is_ok() {
        for base in &crawlers {
            let url = format!(
                "{}/xrpc/com.atproto.sync.requestCrawl",
                base.trim_end_matches('/')
            );
            let req = http
                .post(&url)
                .header(reqwest::header::CONTENT_TYPE, "application/json")
                .body(body.clone());
            tokio::spawn(async move {
                let _ = crate::ops::send(crate::ops::Target::Crawler, req).await;
            });
        }
        tokio::time::sleep(every).await;
        events.borrow_and_update();
    }
}

#[cfg(test)]
mod tests {
    use crate::testing::*;
    use crate::{Call, Resp, Shared, answer};
    use axum::http::Method;
    use pds_repo::data::*;
    use serde_json::{Value as Json, json};
    use std::collections::{HashMap, HashSet};
    use std::sync::{Arc, Mutex};
    use std::time::Duration;

    const NOBODY: &str = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";

    fn params(p: &[(&str, &str)]) -> Vec<(String, String)> {
        p.iter()
            .map(|(k, v)| (k.to_string(), v.to_string()))
            .collect()
    }

    async fn query(s: &Shared, nsid: &str, p: &[(&str, &str)]) -> Result<Json, String> {
        call_with(s, nsid, None, &params(p), json!({})).await
    }

    /// A query whose answer is a CAR file: its bytes, or the error's name.
    async fn car(s: &Shared, nsid: &str, p: &[(&str, &str)]) -> Result<Vec<u8>, String> {
        let p = params(p);
        let headers = Default::default();
        let call = Call {
            nsid,
            method: &Method::GET,
            params: &p,
            body: b"",
            bearer: crate::sessions::Bearer::None,
            headers: &headers,
            client: String::new(),
            charged: std::sync::OnceLock::new(),
            http: None,
        };
        match answer(s, &call).await {
            Ok(Resp::Car(b)) => Ok(b),
            Ok(Resp::CarStream(c)) => Ok(c.bytes().unwrap()),
            Ok(Resp::Json(j) | Resp::Headed(j, _)) => panic!("{nsid} answered JSON: {j}"),
            Ok(Resp::Nothing) => panic!("{nsid} answered nothing"),
            Ok(Resp::Blob(_)) => panic!("{nsid} answered a blob"),
            Ok(Resp::Upstream(_)) => panic!("{nsid} was forwarded"),
            Ok(Resp::Star(_) | Resp::Archive(..)) => panic!("{nsid} answered an archive"),
            Err(e) => Err(e.name.to_string()),
        }
    }

    fn varint(b: &[u8], at: &mut usize) -> usize {
        let (mut n, mut shift) = (0usize, 0);
        loop {
            let byte = b[*at];
            *at += 1;
            n |= ((byte & 0x7f) as usize) << shift;
            if byte & 0x80 == 0 {
                return n;
            }
            shift += 7;
        }
    }

    /// A CAR file's roots and blocks, each block checked against its CID.
    fn read_car(b: &[u8]) -> (Vec<Vec<u8>>, Vec<pds_repo::mst::Block>) {
        let mut at = 0;
        let n = varint(b, &mut at);
        let header = decode(&b[at..at + n]).unwrap();
        at += n;
        let V::Map(h) = header else {
            panic!("a header that is not a map")
        };
        assert_eq!(
            h.iter().find(|(k, _)| k == "version").map(|(_, v)| v),
            Some(&V::Int(1))
        );
        let Some((_, V::Arr(roots))) = h.iter().find(|(k, _)| k == "roots") else {
            panic!("a header with no roots")
        };
        let roots = roots
            .iter()
            .map(|r| {
                if let V::Link(c) = r {
                    c.clone()
                } else {
                    panic!("a root that is not a CID")
                }
            })
            .collect();
        let mut blocks = Vec::new();
        while at < b.len() {
            let n = varint(b, &mut at);
            let (cid, data) = (b[at..at + 36].to_vec(), b[at + 36..at + n].to_vec());
            assert_eq!(
                cid,
                cid_of(cid[1], &data),
                "a block that is not what its CID names"
            );
            blocks.push((cid, data));
            at += n;
        }
        (roots, blocks)
    }

    fn field<'a>(v: &'a V, k: &str) -> &'a V {
        let V::Map(m) = v else {
            panic!("not a map: {v:?}")
        };
        m.iter()
            .find(|(n, _)| n == k)
            .map(|(_, v)| v)
            .unwrap_or_else(|| panic!("no {k} in {v:?}"))
    }

    fn link(v: &V) -> Option<Vec<u8>> {
        match v {
            V::Link(c) => Some(c.clone()),
            V::Null => None,
            _ => panic!("not a link: {v:?}"),
        }
    }

    /// Walk a `getRecord` CAR from its root, `head`, down the tree to `key`:
    /// the CID of the record it proves is there, or None if it proves there is
    /// none. Every block it holds must be one the walk uses.
    fn walk_proof(car: &[u8], head: &[u8], key: &str) -> Option<Vec<u8>> {
        let (roots, blocks) = read_car(car);
        assert_eq!(roots, vec![head.to_vec()]);
        let held: HashMap<Vec<u8>, Vec<u8>> = blocks.iter().cloned().collect();
        assert_eq!(held.len(), blocks.len(), "a block twice");
        let mut used = HashSet::new();
        let mut get = |cid: &[u8]| {
            used.insert(cid.to_vec());
            decode(
                held.get(cid)
                    .unwrap_or_else(|| panic!("{key}: {} is not in the proof", cid_string(cid))),
            )
            .unwrap()
        };
        let commit = get(head);
        let mut at = link(field(&commit, "data"));
        let found = loop {
            // A search that comes to no subtree has proved the key is not there.
            let Some(cid) = at else { break None };
            let node = get(&cid);
            let V::Arr(entries) = field(&node, "e") else {
                panic!("entries that are not a list")
            };
            let mut next = link(field(&node, "l"));
            let mut prev: Vec<u8> = Vec::new();
            let mut hit = None;
            for e in entries {
                let (V::Int(p), V::Bytes(k)) = (field(e, "p"), field(e, "k")) else {
                    panic!("a bad entry")
                };
                let mut full = prev[..*p as usize].to_vec();
                full.extend(k);
                prev = full.clone();
                match full.as_slice().cmp(key.as_bytes()) {
                    std::cmp::Ordering::Equal => {
                        hit = link(field(e, "v"));
                        break;
                    }
                    std::cmp::Ordering::Greater => break,
                    std::cmp::Ordering::Less => next = link(field(e, "t")),
                }
            }
            if let Some(v) = hit {
                get(&v);
                break Some(v);
            }
            at = next;
        };
        assert_eq!(
            used.len(),
            held.len(),
            "{key}: blocks the proof does not use"
        );
        found
    }

    /// Create `n` records `<collection>/<prefix><i>` in one commit.
    async fn fill(s: &Shared, did: &str, token: &str, collection: &str, prefix: &str, n: usize) {
        let writes: Vec<Json> = (0..n)
            .map(|i| json!({ "$type": "com.atproto.repo.applyWrites#create", "collection": collection, "rkey": format!("{prefix}{i}"), "value": { "n": i } }))
            .collect();
        call(
            s,
            "com.atproto.repo.applyWrites",
            Some(token),
            json!({ "repo": did, "writes": writes }),
        )
        .await
        .unwrap();
    }

    /// Page through `listRepos` from `cursor`, `limit` at a time: every repo
    /// listed, and the last cursor given.
    async fn list_all(
        s: &Shared,
        mut cursor: Option<String>,
        limit: usize,
    ) -> (Vec<Json>, Option<String>) {
        let mut repos = Vec::new();
        loop {
            let limit = limit.to_string();
            let mut p = vec![("limit", limit.as_str())];
            if let Some(c) = &cursor {
                p.push(("cursor", c));
            }
            let page = query(s, "com.atproto.sync.listRepos", &p).await.unwrap();
            let got = page["repos"].as_array().unwrap();
            assert!(got.len() <= limit.parse().unwrap());
            if got.is_empty() {
                assert_eq!(page.get("cursor"), None, "an empty page has no cursor");
                return (repos, cursor);
            }
            repos.extend(got.iter().cloned());
            cursor = Some(
                page["cursor"]
                    .as_str()
                    .expect("a page with repos has a cursor")
                    .to_string(),
            );
        }
    }

    fn dids(repos: &[Json]) -> Vec<&str> {
        repos.iter().map(|r| r["did"].as_str().unwrap()).collect()
    }

    /// Accounts are listed in the order made, each as `getLatestCommit`
    /// answers it, a page at a time; a cursor stays good as accounts are
    /// made, and after a restart.
    #[tokio::test]
    async fn list_repos_pages() {
        let plc = plc().await;
        let dir = scratch("list-repos");
        let s = open(&dir, &plc);
        assert_eq!(
            query(&s, "com.atproto.sync.listRepos", &[]).await.unwrap(),
            json!({ "repos": [] })
        );
        let mut made = Vec::new();
        for name in ["alice", "bob", "carol", "dave", "erin"] {
            made.push(create(&s, name).await);
        }
        call(
            &s,
            "com.atproto.repo.createRecord",
            Some(&made[1].1),
            json!({ "repo": made[1].0, "collection": "a.b.c", "record": record(1) }),
        )
        .await
        .unwrap();
        let (repos, last) = list_all(&s, None, 2).await;
        assert_eq!(
            dids(&repos),
            made.iter().map(|(d, _)| d.as_str()).collect::<Vec<_>>()
        );
        for r in &repos {
            let latest = query(
                &s,
                "com.atproto.sync.getLatestCommit",
                &[("did", r["did"].as_str().unwrap())],
            )
            .await
            .unwrap();
            assert_eq!(
                r,
                &json!({ "did": r["did"], "head": latest["cid"], "rev": latest["rev"], "active": true })
            );
        }
        // The default limit, and an empty cursor, list them all on one page.
        let all = query(&s, "com.atproto.sync.listRepos", &[("cursor", "")])
            .await
            .unwrap();
        assert_eq!(all["repos"], json!(repos));
        // The last page's cursor lists the accounts made since, and only those.
        assert_eq!(list_all(&s, last.clone(), 3).await.0, Vec::<Json>::new());
        let frank = create(&s, "frank").await.0;
        let gina = create(&s, "gina").await.0;
        let (since, last2) = list_all(&s, last.clone(), 1).await;
        assert_eq!(dids(&since), [frank.as_str(), gina.as_str()]);
        // An account being made holds back those made after it, so that no
        // cursor passes over it.
        {
            let mut d = crate::write(&s.directory);
            d.making(NOBODY);
            d.logged(NOBODY);
        }
        let hidden = create(&s, "hilda").await.0;
        assert_eq!(list_all(&s, last2.clone(), 5).await.0, Vec::<Json>::new());
        crate::write(&s.directory).made(NOBODY);
        assert_eq!(
            dids(&list_all(&s, last2.clone(), 5).await.0),
            [hidden.as_str()]
        );
        for (k, v) in [
            ("limit", "0"),
            ("limit", "1001"),
            ("limit", "two"),
            ("cursor", "nonsense"),
            ("cursor", "99"),
        ] {
            assert_eq!(
                query(&s, "com.atproto.sync.listRepos", &[(k, v)]).await,
                Err("InvalidRequest".into()),
                "{k}={v}"
            );
        }
        // A restart lists them in the same order, and the cursors still hold.
        let before = list_all(&s, None, 500).await.0;
        drop(s);
        let s = open(&dir, &plc);
        assert_eq!(list_all(&s, None, 4).await.0, before);
        assert_eq!(
            dids(&list_all(&s, last, 2).await.0),
            [frank.as_str(), gina.as_str(), hidden.as_str()]
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Accounts made at once are listed in the order `accounts.log` has
    /// them, which a restart keeps.
    #[tokio::test]
    async fn list_repos_concurrent() {
        let plc = plc().await;
        let dir = scratch("list-repos-concurrent");
        let s = open(&dir, &plc);
        let names = [
            "anna", "bert", "cleo", "dirk", "emma", "finn", "gwen", "hugo",
        ];
        let made = futures_join(names.into_iter().map(|n| {
            let s = s.clone();
            async move { create(&s, n).await.0 }
        }))
        .await;
        let listed = list_all(&s, None, 3).await.0;
        let mut sorted = dids(&listed)
            .into_iter()
            .map(str::to_string)
            .collect::<Vec<_>>();
        sorted.sort();
        let mut want = made.clone();
        want.sort();
        assert_eq!(sorted, want);
        drop(s);
        let s = open(&dir, &plc);
        assert_eq!(
            list_all(&s, None, 500)
                .await
                .0
                .iter()
                .map(|r| &r["did"])
                .collect::<Vec<_>>(),
            listed.iter().map(|r| &r["did"]).collect::<Vec<_>>()
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Run futures side by side on the runtime, answering their outputs in order.
    async fn futures_join<F: Future<Output = T> + Send + 'static, T: Send + 'static>(
        fs: impl Iterator<Item = F>,
    ) -> Vec<T> {
        let handles: Vec<_> = fs.map(tokio::spawn).collect();
        let mut out = Vec::new();
        for h in handles {
            out.push(h.await.unwrap());
        }
        out
    }

    #[tokio::test]
    async fn repo_status() {
        let plc = plc().await;
        let dir = scratch("repo-status");
        let s = open(&dir, &plc);
        let (did, _) = create(&s, "alice").await;
        let got = query(&s, "com.atproto.sync.getRepoStatus", &[("did", &did)])
            .await
            .unwrap();
        assert_eq!(
            got,
            json!({ "did": did, "active": true, "rev": repo_of(&s, &did, |r| r.rev.clone()) })
        );
        assert_eq!(
            query(&s, "com.atproto.sync.getRepoStatus", &[("did", NOBODY)]).await,
            Err("RepoNotFound".into())
        );
        assert_eq!(
            query(
                &s,
                "com.atproto.sync.getRepoStatus",
                &[("did", "alice.test")]
            )
            .await,
            Err("InvalidRequest".into())
        );
        assert_eq!(
            query(&s, "com.atproto.sync.getRepoStatus", &[("did", "nonsense")]).await,
            Err("InvalidRequest".into())
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// `getBlocks` answers exactly the blocks asked, once each, with no
    /// roots; a CID the repository does not hold, another account's though
    /// the server has it, refuses the whole call.
    #[tokio::test]
    async fn get_blocks() {
        let plc = plc().await;
        let dir = scratch("get-blocks");
        let s = open(&dir, &plc);
        let (alice, ta) = create(&s, "alice").await;
        let (bob, _) = create(&s, "bob").await;
        fill(&s, &alice, &ta, "a.b.c", "k", 40).await;
        let (_, all) = read_car(
            &car(&s, "com.atproto.sync.getRepo", &[("did", &alice)])
                .await
                .unwrap(),
        );
        // The commit, some nodes and records, one named twice.
        let mut asked: Vec<String> = all.iter().step_by(3).map(|(c, _)| cid_string(c)).collect();
        asked.push(asked[1].clone());
        let mut p: Vec<(&str, &str)> = vec![("did", &alice)];
        p.extend(asked.iter().map(|c| ("cids", c.as_str())));
        let (roots, blocks) = read_car(&car(&s, "com.atproto.sync.getBlocks", &p).await.unwrap());
        assert!(roots.is_empty());
        let got: Vec<String> = blocks.iter().map(|(c, _)| cid_string(c)).collect();
        assert_eq!(
            got.iter().collect::<HashSet<_>>().len(),
            got.len(),
            "a block twice"
        );
        assert_eq!(
            got.iter().collect::<HashSet<_>>(),
            asked.iter().collect::<HashSet<_>>()
        );
        // Another account's latest commit, and a block of no repository.
        let bobs = repo_of(&s, &bob, |r| r.head_string());
        let nothing = cid_string(&cid_of(0x55, b"nothing"));
        for foreign in [&bobs, &nothing] {
            let p = [
                ("did", alice.as_str()),
                ("cids", asked[0].as_str()),
                ("cids", foreign.as_str()),
            ];
            assert_eq!(
                car(&s, "com.atproto.sync.getBlocks", &p).await,
                Err("InvalidRequest".into())
            );
        }
        // No cids, or one that does not parse, before the DID is looked up.
        assert_eq!(
            car(&s, "com.atproto.sync.getBlocks", &[("did", NOBODY)]).await,
            Err("InvalidRequest".into())
        );
        assert_eq!(
            car(
                &s,
                "com.atproto.sync.getBlocks",
                &[("did", NOBODY), ("cids", "nonsense")]
            )
            .await,
            Err("InvalidRequest".into())
        );
        assert_eq!(
            car(
                &s,
                "com.atproto.sync.getBlocks",
                &[("did", NOBODY), ("cids", &asked[0])]
            )
            .await,
            Err("RepoNotFound".into())
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// `getRecord` proves a key there, with its record, and a key not there,
    /// by the nodes a search for it passes through and nothing else.
    #[tokio::test]
    async fn get_record_proofs() {
        let plc = plc().await;
        let dir = scratch("get-record");
        let s = open(&dir, &plc);
        let (alice, ta) = create(&s, "alice").await;
        let (bob, _) = create(&s, "bob").await;
        fill(&s, &alice, &ta, "a.b.c", "k", 60).await;
        fill(&s, &alice, &ta, "x.y.z", "r", 20).await;
        let head = repo_of(&s, &alice, |r| r.head.clone());
        let records: Vec<(String, Vec<u8>)> = records_of(&s, &alice)
            .into_iter()
            .map(|(k, v)| (k, v.cid))
            .collect();
        assert!(
            records
                .iter()
                .any(|(k, _)| pds_repo::mst::depth(k.as_bytes()) > 0),
            "a tree of one level proves too little"
        );
        for (path, cid) in &records {
            let (collection, rkey) = path.split_once('/').unwrap();
            let got = car(
                &s,
                "com.atproto.sync.getRecord",
                &[("did", &alice), ("collection", collection), ("rkey", rkey)],
            )
            .await
            .unwrap();
            assert_eq!(walk_proof(&got, &head, path).as_ref(), Some(cid), "{path}");
        }
        // Keys before, between and after those there, in collections there or not.
        for (collection, rkey) in [
            ("a.a.a", "x"),
            ("a.b.c", "k1a"),
            ("a.b.c", "k99"),
            ("a.b.c", "zzz"),
            ("m.n.o", "k1"),
            ("x.y.z", "r100"),
            ("z.z.z", "z"),
        ] {
            let got = car(
                &s,
                "com.atproto.sync.getRecord",
                &[("did", &alice), ("collection", collection), ("rkey", rkey)],
            )
            .await
            .unwrap();
            assert_eq!(
                walk_proof(&got, &head, &format!("{collection}/{rkey}")),
                None,
                "{collection}/{rkey}"
            );
        }
        // An empty repository proves every key absent by its one node.
        let bob_head = repo_of(&s, &bob, |r| r.head.clone());
        let got = car(
            &s,
            "com.atproto.sync.getRecord",
            &[("did", &bob), ("collection", "a.b.c"), ("rkey", "k1")],
        )
        .await
        .unwrap();
        assert_eq!(walk_proof(&got, &bob_head, "a.b.c/k1"), None);
        assert_eq!(read_car(&got).1.len(), 2);
        for (did, collection, rkey, want) in [
            (alice.as_str(), "nonsense", "k1", "InvalidRequest"),
            (alice.as_str(), "a.b.c", "..", "InvalidRequest"),
            ("alice.test", "a.b.c", "k1", "InvalidRequest"),
            (NOBODY, "a.b.c", "k1", "RepoNotFound"),
        ] {
            let p = [("did", did), ("collection", collection), ("rkey", rkey)];
            assert_eq!(
                car(&s, "com.atproto.sync.getRecord", &p).await,
                Err(want.into()),
                "{did} {collection}/{rkey}"
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// What a crawler stub was sent: method, path, content type and body,
    /// each when it came.
    type Seen = Arc<Mutex<Vec<(String, String, String, Json, std::time::Instant)>>>;

    /// A crawler that records every request and answers `status`, or never
    /// answers if `status` is 0. Answers its URL and what it records.
    async fn crawler(status: u16) -> (String, Seen) {
        use axum::extract::{Request, State};
        use axum::http::{StatusCode, header};
        async fn take(State((seen, status)): State<(Seen, u16)>, req: Request) -> StatusCode {
            let (parts, body) = req.into_parts();
            let body = axum::body::to_bytes(body, 1 << 20).await.unwrap();
            let ct = parts
                .headers
                .get(header::CONTENT_TYPE)
                .and_then(|v| v.to_str().ok())
                .unwrap_or("")
                .to_string();
            let body = serde_json::from_slice(&body).unwrap_or(Json::Null);
            seen.lock().unwrap().push((
                parts.method.to_string(),
                parts.uri.path().to_string(),
                ct,
                body,
                std::time::Instant::now(),
            ));
            if status == 0 {
                std::future::pending::<()>().await;
            }
            StatusCode::from_u16(status).unwrap()
        }
        let seen = Seen::default();
        let app = axum::Router::new()
            .fallback(take)
            .with_state((seen.clone(), status));
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        (url, seen)
    }

    /// A URL where nothing listens.
    async fn nowhere() -> String {
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        format!("http://{}", listener.local_addr().unwrap())
    }

    async fn until(what: &str, f: impl Fn() -> bool) {
        let start = std::time::Instant::now();
        while !f() {
            assert!(
                start.elapsed() < Duration::from_secs(5),
                "{what} within 5 seconds"
            );
            tokio::time::sleep(Duration::from_millis(10)).await;
        }
    }

    /// Each crawler is told at the first event, and then at the first event
    /// at least the interval after it last was: not at the start, not while
    /// there are no events, and once for a burst. A crawler that fails, or
    /// never answers, holds up no write and no other crawler.
    #[tokio::test(flavor = "multi_thread")]
    async fn crawlers_told() {
        let every = Duration::from_secs(3);
        let plc = plc().await;
        let dir = scratch("crawlers");
        let s = open(&dir, &plc);
        let (ok, ok_seen) = crawler(200).await;
        let (failing, failing_seen) = crawler(500).await;
        let (hanging, hanging_seen) = crawler(0).await;
        let crawlers = vec![format!("{ok}/"), failing, nowhere().await, hanging];
        let told = |seen: &Seen| seen.lock().unwrap().len();
        let all = [&ok_seen, &failing_seen, &hanging_seen];
        let each = |n: usize| all.iter().all(|s| told(s) == n);
        tokio::spawn(super::tell(
            s.events.subscribe(),
            crate::plc::client().unwrap(),
            crawlers,
            s.config.hostname.clone(),
            every,
        ));
        tokio::time::sleep(Duration::from_millis(500)).await;
        assert!(each(0), "told at the start");
        let (did, token) = create(&s, "alice").await;
        until("the first request", || each(1)).await;
        let first = ok_seen.lock().unwrap()[0].4;
        for seen in all {
            let (method, path, ct, body, _) = seen.lock().unwrap()[0].clone();
            assert_eq!(
                (method.as_str(), path.as_str(), ct.as_str()),
                (
                    "POST",
                    "/xrpc/com.atproto.sync.requestCrawl",
                    "application/json"
                )
            );
            assert_eq!(body, json!({ "hostname": "localhost" }));
        }
        // A burst of events within the interval, none of them held up.
        for n in 0..5 {
            let started = std::time::Instant::now();
            call(
                &s,
                "com.atproto.repo.createRecord",
                Some(&token),
                json!({ "repo": did, "collection": "a.b.c", "record": record(n) }),
            )
            .await
            .unwrap();
            assert!(started.elapsed() < Duration::from_secs(1));
        }
        create(&s, "bob").await;
        // A write that is refused, or makes no commit, is no event.
        let refused = json!({ "repo": did, "collection": "a.b.c", "rkey": "nope", "record": { "$type": "x.y.z" } });
        assert!(
            call(&s, "com.atproto.repo.putRecord", Some(&token), refused)
                .await
                .is_err()
        );
        // The interval passes with no event: nobody is told.
        tokio::time::sleep(every + Duration::from_millis(500)).await;
        assert!(first.elapsed() > every);
        assert!(
            each(1),
            "told without an event, or more than once for a burst"
        );
        call(
            &s,
            "com.atproto.repo.deleteRecord",
            Some(&token),
            json!({ "repo": did, "collection": "a.b.c", "rkey": "nope" }),
        )
        .await
        .unwrap();
        tokio::time::sleep(Duration::from_millis(500)).await;
        assert!(each(1), "told of a delete that made no commit");
        // The first event after it is told.
        call(
            &s,
            "com.atproto.repo.createRecord",
            Some(&token),
            json!({ "repo": did, "collection": "a.b.c", "record": record(9) }),
        )
        .await
        .unwrap();
        until("the second request", || each(2)).await;
        let second = ok_seen.lock().unwrap()[1].4;
        assert!(second.duration_since(first) >= every);
        let _ = std::fs::remove_dir_all(&dir);
    }
}