chadtmiller.com/monolith
An atproto PDS
0
0
//! What a relay reads, as `spec/FEDERATION.md` says: `listRepos`,
//! `getRepoStatus`, `getBlocks` and the sync `getRecord`; and the
//! `requestCrawl` the server sends the crawlers `PDS_CRAWLERS` names.
use crate::sync::not_found;
use crate::{
Account, Call, Handler, Resp, Service, Shared, Shown, XrpcError, invalid, opt, read, required,
};
use pds_repo::data::parse_cid;
use serde_json::{Map, Value as Json, json};
use std::sync::Arc;
use std::time::Duration;
pub(crate) fn handler(nsid: &str) -> Option<Handler> {
Some(match nsid {
"com.atproto.sync.listRepos" => Handler::Query(list_repos),
"com.atproto.sync.getRepoStatus" => Handler::Query(get_repo_status),
"com.atproto.sync.getBlocks" => Handler::Query(get_blocks),
"com.atproto.sync.getRecord" => Handler::Query(get_record),
_ => return None,
})
}
/// Every account whose repository has its first commit, in the order
/// `accounts.log` has them, each active or not and why
/// (`spec/LIFECYCLE.md`). A cursor is the ordinal of the first account of
/// the next page, its place in the order accounts were made: as each
/// account keeps its ordinal, and a deleted account's is never given
/// again, it stays good. The listing stops before an account still being
/// made, so that no cursor passes over it.
fn list_repos(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
let limit = p.get("limit").and_then(Json::as_u64).unwrap_or(500) as usize;
let mut from = match opt(p, "cursor") {
None | Some("") => 0,
Some(c) => match c.parse::<u64>() {
Ok(n) if n <= read(&s.directory).next_ordinal() => n,
_ => return invalid("Malformed cursor"),
},
};
// Each account is loaded as it is listed. One deleted since its place
// was read is no account, and the page goes on past it.
let mut accounts: Vec<(u64, Arc<Account>)> = Vec::new();
while accounts.len() < limit {
let wanted = limit - accounts.len();
let page = read(&s.directory).page(from, wanted);
let Some((last, _)) = page.last() else { break };
from = last + 1;
let full = page.len() == wanted;
accounts.extend(
page.into_iter()
.filter_map(|(at, did)| Some((at, s.by_did(&did)?))),
);
if !full {
break;
}
}
let repos: Vec<Json> = accounts
.iter()
.map(|(_, a)| {
let st = read(&a.store);
shown.0.push((st.wal.clone(), st.wal.written()));
let mut r = json!({ "did": a.did, "head": st.repo.head_string(), "rev": st.repo.rev, "active": true });
if let Some(status) = crate::lifecycle::status(a) {
r["active"] = json!(false);
r["status"] = json!(status);
}
r
})
.collect();
let mut out = json!({ "repos": repos });
if let Some((at, _)) = accounts.last() {
out["cursor"] = json!((at + 1).to_string());
}
Ok(Resp::Json(out))
}
/// An active account's `{did, active, rev}`, or one not active's status instead of its rev.
fn get_repo_status(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
s.with_repo(shown, required(p, "did")?, not_found, |a, r| {
Ok(Resp::Json(match crate::lifecycle::status(a) {
None => json!({ "did": a.did, "active": true, "rev": r.rev }),
Some(status) => json!({ "did": a.did, "active": false, "status": status }),
}))
})
}
/// The blocks asked for, all of which the repository must hold now.
fn get_blocks(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
let cids = p
.get("cids")
.and_then(Json::as_array)
.map_or(&[][..], Vec::as_slice);
s.with_store(shown, required(p, "did")?, not_found, |_, st| {
// A CID that is not a SHA-256 one of DAG-CBOR or raw names no block of a repository.
let parsed: Option<Vec<Vec<u8>>> = cids
.iter()
.map(|c| c.as_str().and_then(parse_cid))
.collect();
parsed
.and_then(|c| st.repo.blocks_car(&st.source(), &c))
.map_or_else(
|| invalid("Could not find all requested blocks"),
|car| Ok(Resp::Car(car)),
)
})
}
/// The proof of the record at `collection/rkey`, or of its absence.
fn get_record(
s: &Service,
_: &Call,
shown: &mut Shown,
p: &Map<String, Json>,
) -> Result<Resp, XrpcError> {
let path = format!("{}/{}", required(p, "collection")?, required(p, "rkey")?);
s.with_store(shown, required(p, "did")?, not_found, |_, st| {
Ok(Resp::Car(st.repo.proof_car(&st.source(), &path)))
})
}
/// How long after telling the crawlers the server waits before an event tells them again.
const CRAWL_EVERY: Duration = Duration::from_secs(20 * 60);
/// Tell the crawlers `PDS_CRAWLERS` names of the events from now on, in the background.
pub(crate) fn tell_crawlers(svc: &Shared) {
let c = &svc.config;
if !c.crawlers.is_empty() {
tokio::spawn(tell(
svc.events.subscribe(),
svc.http.clone(),
c.crawlers.clone(),
c.hostname.clone(),
CRAWL_EVERY,
));
}
}
/// At the first event `events` brings, and then at the first event at least
/// `every` after it last did, `POST` each crawler a `requestCrawl`, each on
/// its own and answered by none, a failure logged (`ops::send`) and tried
/// no more. The events while it waits are passed over, so a burst of them
/// is one request.
async fn tell(
mut events: tokio::sync::watch::Receiver<u64>,
http: reqwest::Client,
crawlers: Vec<String>,
hostname: String,
every: Duration,
) {
let body = json!({ "hostname": hostname }).to_string();
while events.changed().await.is_ok() {
for base in &crawlers {
let url = format!(
"{}/xrpc/com.atproto.sync.requestCrawl",
base.trim_end_matches('/')
);
let req = http
.post(&url)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(body.clone());
tokio::spawn(async move {
let _ = crate::ops::send(crate::ops::Target::Crawler, req).await;
});
}
tokio::time::sleep(every).await;
events.borrow_and_update();
}
}
#[cfg(test)]
mod tests {
use crate::testing::*;
use crate::{Call, Resp, Shared, answer};
use axum::http::Method;
use pds_repo::data::*;
use serde_json::{Value as Json, json};
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex};
use std::time::Duration;
const NOBODY: &str = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa";
fn params(p: &[(&str, &str)]) -> Vec<(String, String)> {
p.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
async fn query(s: &Shared, nsid: &str, p: &[(&str, &str)]) -> Result<Json, String> {
call_with(s, nsid, None, ¶ms(p), json!({})).await
}
/// A query whose answer is a CAR file: its bytes, or the error's name.
async fn car(s: &Shared, nsid: &str, p: &[(&str, &str)]) -> Result<Vec<u8>, String> {
let p = params(p);
let headers = Default::default();
let call = Call {
nsid,
method: &Method::GET,
params: &p,
body: b"",
bearer: crate::sessions::Bearer::None,
headers: &headers,
client: String::new(),
charged: std::sync::OnceLock::new(),
http: None,
};
match answer(s, &call).await {
Ok(Resp::Car(b)) => Ok(b),
Ok(Resp::CarStream(c)) => Ok(c.bytes().unwrap()),
Ok(Resp::Json(j) | Resp::Headed(j, _)) => panic!("{nsid} answered JSON: {j}"),
Ok(Resp::Nothing) => panic!("{nsid} answered nothing"),
Ok(Resp::Blob(_)) => panic!("{nsid} answered a blob"),
Ok(Resp::Upstream(_)) => panic!("{nsid} was forwarded"),
Ok(Resp::Star(_) | Resp::Archive(..)) => panic!("{nsid} answered an archive"),
Err(e) => Err(e.name.to_string()),
}
}
fn varint(b: &[u8], at: &mut usize) -> usize {
let (mut n, mut shift) = (0usize, 0);
loop {
let byte = b[*at];
*at += 1;
n |= ((byte & 0x7f) as usize) << shift;
if byte & 0x80 == 0 {
return n;
}
shift += 7;
}
}
/// A CAR file's roots and blocks, each block checked against its CID.
fn read_car(b: &[u8]) -> (Vec<Vec<u8>>, Vec<pds_repo::mst::Block>) {
let mut at = 0;
let n = varint(b, &mut at);
let header = decode(&b[at..at + n]).unwrap();
at += n;
let V::Map(h) = header else {
panic!("a header that is not a map")
};
assert_eq!(
h.iter().find(|(k, _)| k == "version").map(|(_, v)| v),
Some(&V::Int(1))
);
let Some((_, V::Arr(roots))) = h.iter().find(|(k, _)| k == "roots") else {
panic!("a header with no roots")
};
let roots = roots
.iter()
.map(|r| {
if let V::Link(c) = r {
c.clone()
} else {
panic!("a root that is not a CID")
}
})
.collect();
let mut blocks = Vec::new();
while at < b.len() {
let n = varint(b, &mut at);
let (cid, data) = (b[at..at + 36].to_vec(), b[at + 36..at + n].to_vec());
assert_eq!(
cid,
cid_of(cid[1], &data),
"a block that is not what its CID names"
);
blocks.push((cid, data));
at += n;
}
(roots, blocks)
}
fn field<'a>(v: &'a V, k: &str) -> &'a V {
let V::Map(m) = v else {
panic!("not a map: {v:?}")
};
m.iter()
.find(|(n, _)| n == k)
.map(|(_, v)| v)
.unwrap_or_else(|| panic!("no {k} in {v:?}"))
}
fn link(v: &V) -> Option<Vec<u8>> {
match v {
V::Link(c) => Some(c.clone()),
V::Null => None,
_ => panic!("not a link: {v:?}"),
}
}
/// Walk a `getRecord` CAR from its root, `head`, down the tree to `key`:
/// the CID of the record it proves is there, or None if it proves there is
/// none. Every block it holds must be one the walk uses.
fn walk_proof(car: &[u8], head: &[u8], key: &str) -> Option<Vec<u8>> {
let (roots, blocks) = read_car(car);
assert_eq!(roots, vec![head.to_vec()]);
let held: HashMap<Vec<u8>, Vec<u8>> = blocks.iter().cloned().collect();
assert_eq!(held.len(), blocks.len(), "a block twice");
let mut used = HashSet::new();
let mut get = |cid: &[u8]| {
used.insert(cid.to_vec());
decode(
held.get(cid)
.unwrap_or_else(|| panic!("{key}: {} is not in the proof", cid_string(cid))),
)
.unwrap()
};
let commit = get(head);
let mut at = link(field(&commit, "data"));
let found = loop {
// A search that comes to no subtree has proved the key is not there.
let Some(cid) = at else { break None };
let node = get(&cid);
let V::Arr(entries) = field(&node, "e") else {
panic!("entries that are not a list")
};
let mut next = link(field(&node, "l"));
let mut prev: Vec<u8> = Vec::new();
let mut hit = None;
for e in entries {
let (V::Int(p), V::Bytes(k)) = (field(e, "p"), field(e, "k")) else {
panic!("a bad entry")
};
let mut full = prev[..*p as usize].to_vec();
full.extend(k);
prev = full.clone();
match full.as_slice().cmp(key.as_bytes()) {
std::cmp::Ordering::Equal => {
hit = link(field(e, "v"));
break;
}
std::cmp::Ordering::Greater => break,
std::cmp::Ordering::Less => next = link(field(e, "t")),
}
}
if let Some(v) = hit {
get(&v);
break Some(v);
}
at = next;
};
assert_eq!(
used.len(),
held.len(),
"{key}: blocks the proof does not use"
);
found
}
/// Create `n` records `<collection>/<prefix><i>` in one commit.
async fn fill(s: &Shared, did: &str, token: &str, collection: &str, prefix: &str, n: usize) {
let writes: Vec<Json> = (0..n)
.map(|i| json!({ "$type": "com.atproto.repo.applyWrites#create", "collection": collection, "rkey": format!("{prefix}{i}"), "value": { "n": i } }))
.collect();
call(
s,
"com.atproto.repo.applyWrites",
Some(token),
json!({ "repo": did, "writes": writes }),
)
.await
.unwrap();
}
/// Page through `listRepos` from `cursor`, `limit` at a time: every repo
/// listed, and the last cursor given.
async fn list_all(
s: &Shared,
mut cursor: Option<String>,
limit: usize,
) -> (Vec<Json>, Option<String>) {
let mut repos = Vec::new();
loop {
let limit = limit.to_string();
let mut p = vec![("limit", limit.as_str())];
if let Some(c) = &cursor {
p.push(("cursor", c));
}
let page = query(s, "com.atproto.sync.listRepos", &p).await.unwrap();
let got = page["repos"].as_array().unwrap();
assert!(got.len() <= limit.parse().unwrap());
if got.is_empty() {
assert_eq!(page.get("cursor"), None, "an empty page has no cursor");
return (repos, cursor);
}
repos.extend(got.iter().cloned());
cursor = Some(
page["cursor"]
.as_str()
.expect("a page with repos has a cursor")
.to_string(),
);
}
}
fn dids(repos: &[Json]) -> Vec<&str> {
repos.iter().map(|r| r["did"].as_str().unwrap()).collect()
}
/// Accounts are listed in the order made, each as `getLatestCommit`
/// answers it, a page at a time; a cursor stays good as accounts are
/// made, and after a restart.
#[tokio::test]
async fn list_repos_pages() {
let plc = plc().await;
let dir = scratch("list-repos");
let s = open(&dir, &plc);
assert_eq!(
query(&s, "com.atproto.sync.listRepos", &[]).await.unwrap(),
json!({ "repos": [] })
);
let mut made = Vec::new();
for name in ["alice", "bob", "carol", "dave", "erin"] {
made.push(create(&s, name).await);
}
call(
&s,
"com.atproto.repo.createRecord",
Some(&made[1].1),
json!({ "repo": made[1].0, "collection": "a.b.c", "record": record(1) }),
)
.await
.unwrap();
let (repos, last) = list_all(&s, None, 2).await;
assert_eq!(
dids(&repos),
made.iter().map(|(d, _)| d.as_str()).collect::<Vec<_>>()
);
for r in &repos {
let latest = query(
&s,
"com.atproto.sync.getLatestCommit",
&[("did", r["did"].as_str().unwrap())],
)
.await
.unwrap();
assert_eq!(
r,
&json!({ "did": r["did"], "head": latest["cid"], "rev": latest["rev"], "active": true })
);
}
// The default limit, and an empty cursor, list them all on one page.
let all = query(&s, "com.atproto.sync.listRepos", &[("cursor", "")])
.await
.unwrap();
assert_eq!(all["repos"], json!(repos));
// The last page's cursor lists the accounts made since, and only those.
assert_eq!(list_all(&s, last.clone(), 3).await.0, Vec::<Json>::new());
let frank = create(&s, "frank").await.0;
let gina = create(&s, "gina").await.0;
let (since, last2) = list_all(&s, last.clone(), 1).await;
assert_eq!(dids(&since), [frank.as_str(), gina.as_str()]);
// An account being made holds back those made after it, so that no
// cursor passes over it.
{
let mut d = crate::write(&s.directory);
d.making(NOBODY);
d.logged(NOBODY);
}
let hidden = create(&s, "hilda").await.0;
assert_eq!(list_all(&s, last2.clone(), 5).await.0, Vec::<Json>::new());
crate::write(&s.directory).made(NOBODY);
assert_eq!(
dids(&list_all(&s, last2.clone(), 5).await.0),
[hidden.as_str()]
);
for (k, v) in [
("limit", "0"),
("limit", "1001"),
("limit", "two"),
("cursor", "nonsense"),
("cursor", "99"),
] {
assert_eq!(
query(&s, "com.atproto.sync.listRepos", &[(k, v)]).await,
Err("InvalidRequest".into()),
"{k}={v}"
);
}
// A restart lists them in the same order, and the cursors still hold.
let before = list_all(&s, None, 500).await.0;
drop(s);
let s = open(&dir, &plc);
assert_eq!(list_all(&s, None, 4).await.0, before);
assert_eq!(
dids(&list_all(&s, last, 2).await.0),
[frank.as_str(), gina.as_str(), hidden.as_str()]
);
let _ = std::fs::remove_dir_all(&dir);
}
/// Accounts made at once are listed in the order `accounts.log` has
/// them, which a restart keeps.
#[tokio::test]
async fn list_repos_concurrent() {
let plc = plc().await;
let dir = scratch("list-repos-concurrent");
let s = open(&dir, &plc);
let names = [
"anna", "bert", "cleo", "dirk", "emma", "finn", "gwen", "hugo",
];
let made = futures_join(names.into_iter().map(|n| {
let s = s.clone();
async move { create(&s, n).await.0 }
}))
.await;
let listed = list_all(&s, None, 3).await.0;
let mut sorted = dids(&listed)
.into_iter()
.map(str::to_string)
.collect::<Vec<_>>();
sorted.sort();
let mut want = made.clone();
want.sort();
assert_eq!(sorted, want);
drop(s);
let s = open(&dir, &plc);
assert_eq!(
list_all(&s, None, 500)
.await
.0
.iter()
.map(|r| &r["did"])
.collect::<Vec<_>>(),
listed.iter().map(|r| &r["did"]).collect::<Vec<_>>()
);
let _ = std::fs::remove_dir_all(&dir);
}
/// Run futures side by side on the runtime, answering their outputs in order.
async fn futures_join<F: Future<Output = T> + Send + 'static, T: Send + 'static>(
fs: impl Iterator<Item = F>,
) -> Vec<T> {
let handles: Vec<_> = fs.map(tokio::spawn).collect();
let mut out = Vec::new();
for h in handles {
out.push(h.await.unwrap());
}
out
}
#[tokio::test]
async fn repo_status() {
let plc = plc().await;
let dir = scratch("repo-status");
let s = open(&dir, &plc);
let (did, _) = create(&s, "alice").await;
let got = query(&s, "com.atproto.sync.getRepoStatus", &[("did", &did)])
.await
.unwrap();
assert_eq!(
got,
json!({ "did": did, "active": true, "rev": repo_of(&s, &did, |r| r.rev.clone()) })
);
assert_eq!(
query(&s, "com.atproto.sync.getRepoStatus", &[("did", NOBODY)]).await,
Err("RepoNotFound".into())
);
assert_eq!(
query(
&s,
"com.atproto.sync.getRepoStatus",
&[("did", "alice.test")]
)
.await,
Err("InvalidRequest".into())
);
assert_eq!(
query(&s, "com.atproto.sync.getRepoStatus", &[("did", "nonsense")]).await,
Err("InvalidRequest".into())
);
let _ = std::fs::remove_dir_all(&dir);
}
/// `getBlocks` answers exactly the blocks asked, once each, with no
/// roots; a CID the repository does not hold, another account's though
/// the server has it, refuses the whole call.
#[tokio::test]
async fn get_blocks() {
let plc = plc().await;
let dir = scratch("get-blocks");
let s = open(&dir, &plc);
let (alice, ta) = create(&s, "alice").await;
let (bob, _) = create(&s, "bob").await;
fill(&s, &alice, &ta, "a.b.c", "k", 40).await;
let (_, all) = read_car(
&car(&s, "com.atproto.sync.getRepo", &[("did", &alice)])
.await
.unwrap(),
);
// The commit, some nodes and records, one named twice.
let mut asked: Vec<String> = all.iter().step_by(3).map(|(c, _)| cid_string(c)).collect();
asked.push(asked[1].clone());
let mut p: Vec<(&str, &str)> = vec![("did", &alice)];
p.extend(asked.iter().map(|c| ("cids", c.as_str())));
let (roots, blocks) = read_car(&car(&s, "com.atproto.sync.getBlocks", &p).await.unwrap());
assert!(roots.is_empty());
let got: Vec<String> = blocks.iter().map(|(c, _)| cid_string(c)).collect();
assert_eq!(
got.iter().collect::<HashSet<_>>().len(),
got.len(),
"a block twice"
);
assert_eq!(
got.iter().collect::<HashSet<_>>(),
asked.iter().collect::<HashSet<_>>()
);
// Another account's latest commit, and a block of no repository.
let bobs = repo_of(&s, &bob, |r| r.head_string());
let nothing = cid_string(&cid_of(0x55, b"nothing"));
for foreign in [&bobs, ¬hing] {
let p = [
("did", alice.as_str()),
("cids", asked[0].as_str()),
("cids", foreign.as_str()),
];
assert_eq!(
car(&s, "com.atproto.sync.getBlocks", &p).await,
Err("InvalidRequest".into())
);
}
// No cids, or one that does not parse, before the DID is looked up.
assert_eq!(
car(&s, "com.atproto.sync.getBlocks", &[("did", NOBODY)]).await,
Err("InvalidRequest".into())
);
assert_eq!(
car(
&s,
"com.atproto.sync.getBlocks",
&[("did", NOBODY), ("cids", "nonsense")]
)
.await,
Err("InvalidRequest".into())
);
assert_eq!(
car(
&s,
"com.atproto.sync.getBlocks",
&[("did", NOBODY), ("cids", &asked[0])]
)
.await,
Err("RepoNotFound".into())
);
let _ = std::fs::remove_dir_all(&dir);
}
/// `getRecord` proves a key there, with its record, and a key not there,
/// by the nodes a search for it passes through and nothing else.
#[tokio::test]
async fn get_record_proofs() {
let plc = plc().await;
let dir = scratch("get-record");
let s = open(&dir, &plc);
let (alice, ta) = create(&s, "alice").await;
let (bob, _) = create(&s, "bob").await;
fill(&s, &alice, &ta, "a.b.c", "k", 60).await;
fill(&s, &alice, &ta, "x.y.z", "r", 20).await;
let head = repo_of(&s, &alice, |r| r.head.clone());
let records: Vec<(String, Vec<u8>)> = records_of(&s, &alice)
.into_iter()
.map(|(k, v)| (k, v.cid))
.collect();
assert!(
records
.iter()
.any(|(k, _)| pds_repo::mst::depth(k.as_bytes()) > 0),
"a tree of one level proves too little"
);
for (path, cid) in &records {
let (collection, rkey) = path.split_once('/').unwrap();
let got = car(
&s,
"com.atproto.sync.getRecord",
&[("did", &alice), ("collection", collection), ("rkey", rkey)],
)
.await
.unwrap();
assert_eq!(walk_proof(&got, &head, path).as_ref(), Some(cid), "{path}");
}
// Keys before, between and after those there, in collections there or not.
for (collection, rkey) in [
("a.a.a", "x"),
("a.b.c", "k1a"),
("a.b.c", "k99"),
("a.b.c", "zzz"),
("m.n.o", "k1"),
("x.y.z", "r100"),
("z.z.z", "z"),
] {
let got = car(
&s,
"com.atproto.sync.getRecord",
&[("did", &alice), ("collection", collection), ("rkey", rkey)],
)
.await
.unwrap();
assert_eq!(
walk_proof(&got, &head, &format!("{collection}/{rkey}")),
None,
"{collection}/{rkey}"
);
}
// An empty repository proves every key absent by its one node.
let bob_head = repo_of(&s, &bob, |r| r.head.clone());
let got = car(
&s,
"com.atproto.sync.getRecord",
&[("did", &bob), ("collection", "a.b.c"), ("rkey", "k1")],
)
.await
.unwrap();
assert_eq!(walk_proof(&got, &bob_head, "a.b.c/k1"), None);
assert_eq!(read_car(&got).1.len(), 2);
for (did, collection, rkey, want) in [
(alice.as_str(), "nonsense", "k1", "InvalidRequest"),
(alice.as_str(), "a.b.c", "..", "InvalidRequest"),
("alice.test", "a.b.c", "k1", "InvalidRequest"),
(NOBODY, "a.b.c", "k1", "RepoNotFound"),
] {
let p = [("did", did), ("collection", collection), ("rkey", rkey)];
assert_eq!(
car(&s, "com.atproto.sync.getRecord", &p).await,
Err(want.into()),
"{did} {collection}/{rkey}"
);
}
let _ = std::fs::remove_dir_all(&dir);
}
/// What a crawler stub was sent: method, path, content type and body,
/// each when it came.
type Seen = Arc<Mutex<Vec<(String, String, String, Json, std::time::Instant)>>>;
/// A crawler that records every request and answers `status`, or never
/// answers if `status` is 0. Answers its URL and what it records.
async fn crawler(status: u16) -> (String, Seen) {
use axum::extract::{Request, State};
use axum::http::{StatusCode, header};
async fn take(State((seen, status)): State<(Seen, u16)>, req: Request) -> StatusCode {
let (parts, body) = req.into_parts();
let body = axum::body::to_bytes(body, 1 << 20).await.unwrap();
let ct = parts
.headers
.get(header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_string();
let body = serde_json::from_slice(&body).unwrap_or(Json::Null);
seen.lock().unwrap().push((
parts.method.to_string(),
parts.uri.path().to_string(),
ct,
body,
std::time::Instant::now(),
));
if status == 0 {
std::future::pending::<()>().await;
}
StatusCode::from_u16(status).unwrap()
}
let seen = Seen::default();
let app = axum::Router::new()
.fallback(take)
.with_state((seen.clone(), status));
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(url, seen)
}
/// A URL where nothing listens.
async fn nowhere() -> String {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
format!("http://{}", listener.local_addr().unwrap())
}
async fn until(what: &str, f: impl Fn() -> bool) {
let start = std::time::Instant::now();
while !f() {
assert!(
start.elapsed() < Duration::from_secs(5),
"{what} within 5 seconds"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
/// Each crawler is told at the first event, and then at the first event
/// at least the interval after it last was: not at the start, not while
/// there are no events, and once for a burst. A crawler that fails, or
/// never answers, holds up no write and no other crawler.
#[tokio::test(flavor = "multi_thread")]
async fn crawlers_told() {
let every = Duration::from_secs(3);
let plc = plc().await;
let dir = scratch("crawlers");
let s = open(&dir, &plc);
let (ok, ok_seen) = crawler(200).await;
let (failing, failing_seen) = crawler(500).await;
let (hanging, hanging_seen) = crawler(0).await;
let crawlers = vec![format!("{ok}/"), failing, nowhere().await, hanging];
let told = |seen: &Seen| seen.lock().unwrap().len();
let all = [&ok_seen, &failing_seen, &hanging_seen];
let each = |n: usize| all.iter().all(|s| told(s) == n);
tokio::spawn(super::tell(
s.events.subscribe(),
crate::plc::client().unwrap(),
crawlers,
s.config.hostname.clone(),
every,
));
tokio::time::sleep(Duration::from_millis(500)).await;
assert!(each(0), "told at the start");
let (did, token) = create(&s, "alice").await;
until("the first request", || each(1)).await;
let first = ok_seen.lock().unwrap()[0].4;
for seen in all {
let (method, path, ct, body, _) = seen.lock().unwrap()[0].clone();
assert_eq!(
(method.as_str(), path.as_str(), ct.as_str()),
(
"POST",
"/xrpc/com.atproto.sync.requestCrawl",
"application/json"
)
);
assert_eq!(body, json!({ "hostname": "localhost" }));
}
// A burst of events within the interval, none of them held up.
for n in 0..5 {
let started = std::time::Instant::now();
call(
&s,
"com.atproto.repo.createRecord",
Some(&token),
json!({ "repo": did, "collection": "a.b.c", "record": record(n) }),
)
.await
.unwrap();
assert!(started.elapsed() < Duration::from_secs(1));
}
create(&s, "bob").await;
// A write that is refused, or makes no commit, is no event.
let refused = json!({ "repo": did, "collection": "a.b.c", "rkey": "nope", "record": { "$type": "x.y.z" } });
assert!(
call(&s, "com.atproto.repo.putRecord", Some(&token), refused)
.await
.is_err()
);
// The interval passes with no event: nobody is told.
tokio::time::sleep(every + Duration::from_millis(500)).await;
assert!(first.elapsed() > every);
assert!(
each(1),
"told without an event, or more than once for a burst"
);
call(
&s,
"com.atproto.repo.deleteRecord",
Some(&token),
json!({ "repo": did, "collection": "a.b.c", "rkey": "nope" }),
)
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(500)).await;
assert!(each(1), "told of a delete that made no commit");
// The first event after it is told.
call(
&s,
"com.atproto.repo.createRecord",
Some(&token),
json!({ "repo": did, "collection": "a.b.c", "record": record(9) }),
)
.await
.unwrap();
until("the second request", || each(2)).await;
let second = ok_seen.lock().unwrap()[1].4;
assert!(second.duration_since(first) >= every);
let _ = std::fs::remove_dir_all(&dir);
}
}