~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
2302 lines · 81.8 KB Raw History
//! Write notifications, the writer set and syncers, each rule of
//! `spec/SPACES-SYNC.md` the harness does not settle as well as those it
//! does: the parties a space host and a repo host deal with, syncers too,
//! are played here by one HTTP server, which records what it is sent and
//! answers as a test asks; the clocks are moved on by `Notify::skew` and
//! the sessions' skew.

use super::*;
use crate::Config;
use crate::space_credentials::tests::{
    account_key, authorized, client, counting, credential, delegation_token, high_s, jwt_of, signed,
};
use crate::testing::*;
use axum::Router;
use axum::body::Bytes;
use axum::extract::{Path as AxPath, State};
use axum::http::{HeaderMap, HeaderValue, Method as Http, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use base64::Engine;
use base64::engine::general_purpose::URL_SAFE_NO_PAD as B64;
use k256::ecdsa::SigningKey;
use k256::ecdsa::signature::Signer;
use std::collections::{HashMap, VecDeque};
use std::path::PathBuf;

const T: &str = "com.example.forum";
/// The harness's parties: a space host for an authority of its own, and a
/// writer elsewhere.
const HOST: &str = "did:plc:hosthosthosthosthosthost";
const WRITER: &str = "did:plc:writerwriterwriterwriter";
const SECOND: u64 = 1000;
const REVOKE: &str = "com.atproto.space.notifyCredentialsRevoked";

fn space(authority: &str, key: &str) -> String {
    format!("at://{authority}/space/{T}/{key}")
}

/// A PLC directory taking any operation, which serves the parties'
/// documents as a test gives them.
#[derive(Clone, Default)]
struct Plc {
    ops: Arc<Mutex<HashMap<String, Json>>>,
    docs: Arc<Mutex<HashMap<String, Json>>>,
}

impl Plc {
    async fn start() -> (Plc, String) {
        async fn put(State(d): State<Plc>, AxPath(did): AxPath<String>, body: Bytes) -> StatusCode {
            let Ok(op) = serde_json::from_slice::<Json>(&body) else {
                return StatusCode::BAD_REQUEST;
            };
            lock(&d.ops).insert(did, op);
            StatusCode::OK
        }
        async fn doc(State(d): State<Plc>, AxPath(did): AxPath<String>) -> Response {
            let doc = lock(&d.docs)
                .get(&did)
                .cloned()
                .or_else(|| lock(&d.ops).get(&did).map(|op| did_document(&did, op)));
            match doc {
                Some(doc) => (
                    [(header::CONTENT_TYPE, "application/json")],
                    doc.to_string(),
                )
                    .into_response(),
                None => StatusCode::NOT_FOUND.into_response(),
            }
        }
        // Its current data, as `activateAccount` reads it.
        async fn data(State(d): State<Plc>, AxPath(did): AxPath<String>) -> Response {
            let Some(op) = lock(&d.ops).get(&did).cloned() else {
                return StatusCode::NOT_FOUND.into_response();
            };
            let data = json!({ "did": did, "verificationMethods": op["verificationMethods"], "rotationKeys": op["rotationKeys"], "alsoKnownAs": op["alsoKnownAs"], "services": op["services"] });
            (
                [(header::CONTENT_TYPE, "application/json")],
                data.to_string(),
            )
                .into_response()
        }
        let d = Plc::default();
        let app = Router::new()
            .route("/{did}", get(doc).post(put))
            .route("/{did}/data", get(data))
            .with_state(d.clone());
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        (d, url)
    }

    /// Serve `did`'s document: its `#atproto` key, if it has one, and these services.
    fn serve(&self, did: &str, key: Option<&SigningKey>, services: &[(&str, &str)]) {
        let methods: Vec<Json> = key
            .map(|k| json!({ "id": format!("{did}#atproto"), "type": "Multikey", "controller": did, "publicKeyMultibase": crate::plc::did_key(k.verifying_key()).trim_start_matches("did:key:") }))
            .into_iter()
            .collect();
        let services: Vec<Json> = services
            .iter()
            .map(|(id, url)| json!({ "id": id, "type": "Service", "serviceEndpoint": url }))
            .collect();
        lock(&self.docs).insert(
            did.to_string(),
            json!({ "id": did, "verificationMethod": methods, "service": services }),
        );
    }
}

/// A request a party got.
#[derive(Clone, Debug)]
struct Got {
    nsid: String,
    path: String,
    auth: Option<String>,
    content_type: Option<String>,
    body: Json,
}

/// How a syncer answers a confirmation of a token: echoing the challenge,
/// with another one, or as `text/plain`, or empty.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Confirm {
    Echo,
    Wrong,
    Text,
    Empty,
}

/// The parties' HTTP server: it records each request, and answers each
/// with the next status a test queued for its body's `space`; else a
/// confirmation as `confirming` says for its token (by default echoing its
/// challenge), and any other `200`.
#[derive(Clone, Default)]
struct Party {
    got: Arc<Mutex<Vec<Got>>>,
    answers: Arc<Mutex<HashMap<String, VecDeque<u16>>>>,
    confirming: Arc<Mutex<HashMap<String, Confirm>>>,
}

impl Party {
    async fn start() -> (Party, String) {
        async fn xrpc(
            State(p): State<Party>,
            AxPath(nsid): AxPath<String>,
            uri: axum::http::Uri,
            headers: HeaderMap,
            body: Bytes,
        ) -> Response {
            let body: Json = serde_json::from_slice(&body).unwrap_or(Json::Null);
            let text = |h: header::HeaderName| {
                headers
                    .get(h)
                    .and_then(|v| v.to_str().ok())
                    .map(str::to_string)
            };
            let space = body["space"].as_str().unwrap_or_default().to_string();
            let token = body["token"].as_str().unwrap_or_default().to_string();
            let challenge = body["challenge"].as_str().unwrap_or_default().to_string();
            lock(&p.got).push(Got {
                nsid: nsid.clone(),
                path: uri.path().to_string(),
                auth: text(header::AUTHORIZATION),
                content_type: text(header::CONTENT_TYPE),
                body,
            });
            if let Some(status) = lock(&p.answers)
                .get_mut(&space)
                .and_then(VecDeque::pop_front)
            {
                return StatusCode::from_u16(status).unwrap().into_response();
            }
            if nsid != CONFIRM {
                return StatusCode::OK.into_response();
            }
            let json = |c: String| {
                (
                    [(header::CONTENT_TYPE, "application/json")],
                    json!({ "challenge": c, "also": true }).to_string(),
                )
                    .into_response()
            };
            match lock(&p.confirming)
                .get(&token)
                .copied()
                .unwrap_or(Confirm::Echo)
            {
                Confirm::Echo => json(challenge),
                Confirm::Wrong => json(format!("{challenge}x")),
                Confirm::Text => (
                    [(header::CONTENT_TYPE, "text/plain")],
                    json!({ "challenge": challenge }).to_string(),
                )
                    .into_response(),
                Confirm::Empty => StatusCode::OK.into_response(),
            }
        }
        let p = Party::default();
        let app = Router::new()
            .route("/xrpc/{nsid}", post(xrpc))
            .with_state(p.clone());
        let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
            .await
            .unwrap();
        let url = format!("http://{}", listener.local_addr().unwrap());
        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
        (p, url)
    }

    /// Answer the next requests about `space` with these statuses.
    fn answer(&self, space: &str, statuses: &[u16]) {
        lock(&self.answers)
            .entry(space.to_string())
            .or_default()
            .extend(statuses);
    }

    /// Answer the confirmations of `token` as `how`.
    fn confirm(&self, token: &str, how: Confirm) {
        lock(&self.confirming).insert(token.to_string(), how);
    }

    /// What it got since last asked.
    fn take(&self) -> Vec<Got> {
        std::mem::take(&mut *lock(&self.got))
    }

    async fn wait_for(&self, n: usize) -> Vec<Got> {
        eventually("the party to be sent enough", || lock(&self.got).len() >= n).await;
        self.take()
    }
}

/// A URL whose server takes each connection and closes it at once.
async fn closing() -> String {
    let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
        .await
        .unwrap();
    let url = format!("http://{}", listener.local_addr().unwrap());
    tokio::spawn(async move {
        while let Ok((socket, _)) = listener.accept().await {
            drop(socket);
        }
    });
    url
}

async fn eventually(what: &str, f: impl Fn() -> bool) {
    for _ in 0..500 {
        if f() {
            return;
        }
        tokio::time::sleep(Duration::from_millis(10)).await;
    }
    panic!("never: {what}");
}

/// Wait until no send is being made.
pub(crate) async fn idle(s: &Shared) {
    eventually("the sends to settle", || {
        lock(&s.space_notify.sending).is_empty()
    })
    .await;
}

/// Run what is due now, and wait for its sends.
async fn run(s: &Shared) -> usize {
    let (sends, _) = run_due(s);
    let n = sends.len();
    for send in sends {
        send.await.unwrap();
    }
    n
}

/// Move this part's clock on to `t`.
fn advance_to(s: &Service, t: u64) {
    let now = s.space_notify.now();
    if t > now {
        s.space_notify.skew.fetch_add(t - now, Ordering::SeqCst);
    }
}

/// Move both clocks, this part's and the sessions' (which signs
/// credentials), on by `ms`, so that a registration made before lapses.
fn advance_both(s: &Service, ms: u64) {
    s.space_notify.skew.fetch_add(ms, Ordering::SeqCst);
    s.sessions.skew.fetch_add(ms / SECOND, Ordering::SeqCst);
}

/// Past the expiry of every registration made so far.
fn lapse(s: &Service) {
    advance_both(s, 601 * SECOND);
}

fn tid_at(ms: u64) -> String {
    tid_of((ms * 1000) << 10)
}

struct World {
    s: Shared,
    plc: Plc,
    plc_url: String,
    dir: PathBuf,
    party: Party,
    url: String,
    /// The authority of `space`, a memberspace in which bob and the
    /// harness's writer write and carol reads.
    alice: (String, String),
    bob: (String, String),
    carol: (String, String),
    writer: SigningKey,
    space: String,
}

async fn world(name: &str) -> World {
    let (plc, plc_url) = Plc::start().await;
    let (party, url) = Party::start().await;
    let dir = scratch(name);
    let s = open(&dir, &plc_url);
    let (alice, bob, carol) = (
        create(&s, "alice").await,
        create(&s, "bob").await,
        create(&s, "carol").await,
    );
    let writer = SigningKey::from_slice(&[7; 32]).unwrap();
    plc.serve(WRITER, Some(&writer), &[("#atproto_pds", &url)]);
    plc.serve(
        HOST,
        Some(&SigningKey::from_slice(&[8; 32]).unwrap()),
        &[("#atproto_pds", &url)],
    );
    let space = create_space(&s, &alice.1, "m").await;
    put_members(
        &s,
        &alice.1,
        &space,
        &[
            (bob.0.as_str(), true, true),
            (WRITER, false, true),
            (carol.0.as_str(), true, false),
        ],
    )
    .await;
    World {
        s,
        plc,
        plc_url,
        dir,
        party,
        url,
        alice,
        bob,
        carol,
        writer,
        space,
    }
}

async fn create_space(s: &Shared, token: &str, key: &str) -> String {
    let body = json!({ "spaceType": T, "skey": key, "policy": { "$type": "com.atproto.memberspace.policy" } });
    call(s, "com.atproto.space.createSpace", Some(token), body)
        .await
        .unwrap()["space"]
        .as_str()
        .unwrap()
        .to_string()
}

async fn delete_space(s: &Shared, token: &str, space: &str) {
    call(
        s,
        "com.atproto.space.deleteSpace",
        Some(token),
        json!({ "space": space }),
    )
    .await
    .unwrap();
}

async fn put_members(s: &Shared, token: &str, space: &str, members: &[(&str, bool, bool)]) {
    let members: Vec<Json> = members
        .iter()
        .map(|(did, read, write)| json!({ "did": did, "read": read, "write": write }))
        .collect();
    call(
        s,
        "com.atproto.memberspace.putMembers",
        Some(token),
        json!({ "space": space, "members": members }),
    )
    .await
    .unwrap();
}

/// A write of `who`'s into its repo in `space`: the commit's `rev` and `data`.
async fn put(s: &Shared, who: &(String, String), space: &str, rkey: &str) -> (String, [u8; 32]) {
    let body = json!({ "space": space, "repo": who.0, "collection": "a.b.c", "rkey": rkey, "record": { "n": 1 } });
    call(s, "com.atproto.space.putRecord", Some(&who.1), body)
        .await
        .unwrap();
    let a = s.account(&who.0).unwrap();
    let c = read(&a.store)
        .spaces
        .get(space)
        .unwrap()
        .commit(&who.0, space);
    (c.rev, c.data)
}

/// A call as sent over HTTP; an error as its status and name.
async fn send(
    s: &Shared,
    method: Http,
    nsid: &str,
    params: &[(&str, &str)],
    headers: &HeaderMap,
    body: &str,
) -> Result<Json, String> {
    let authorization = crate::authorization(headers);
    let bearer = crate::sessions::Bearer::of(authorization.as_deref());
    let params: Vec<(String, String)> = params
        .iter()
        .map(|(k, v)| (k.to_string(), v.to_string()))
        .collect();
    let call = Call {
        nsid,
        method: &method,
        params: &params,
        body: body.as_bytes(),
        bearer,
        headers,
        client: "127.0.0.1".into(),
        charged: Default::default(),
        http: None,
    };
    match crate::answer(s, &call).await {
        Ok(Resp::Json(j)) => Ok(j),
        Ok(Resp::Nothing) => Ok(Json::Null),
        Ok(_) => Err("not JSON".into()),
        Err(e) => Err(format!("{} {}", e.status, e.name)),
    }
}

fn authorization(value: &str) -> HeaderMap {
    let mut h = HeaderMap::new();
    h.insert(header::AUTHORIZATION, HeaderValue::from_str(value).unwrap());
    h
}

/// Service-auth claims for `notifyRepoUpdate`, good for a minute.
fn claims(s: &Service, iss: &str, aud: &str) -> Json {
    let now = s.sessions.now();
    json!({ "iss": iss, "aud": aud, "lxm": NOTIFY_REPO_UPDATE, "iat": now, "exp": now + 60, "jti": "j" })
}

fn signer(key: &SigningKey) -> impl FnOnce(&[u8]) -> Vec<u8> + '_ {
    move |m: &[u8]| {
        let sig: k256::ecdsa::Signature = key.sign(m);
        sig.normalize_s().to_bytes().to_vec()
    }
}

fn token(payload: &Json, sign: impl FnOnce(&[u8]) -> Vec<u8>) -> String {
    format!(
        "Bearer {}",
        jwt_of(&json!({ "alg": "ES256K", "typ": "JWT" }), payload, sign)
    )
}

fn note(space: &str, repo: &str, rev: &str, data: &[u8]) -> Json {
    json!({ "space": space, "repo": repo, "repoRev": rev, "data": to_json(&V::Bytes(data.to_vec())) })
}

async fn notify(s: &Shared, auth: Option<&str>, body: &Json) -> Result<Json, String> {
    let headers = auth.map_or_else(HeaderMap::new, authorization);
    send(
        s,
        Http::POST,
        NOTIFY_REPO_UPDATE,
        &[],
        &headers,
        &body.to_string(),
    )
    .await
}

/// A notification of the harness's writer's commit at `ms`, taken.
async fn write_at(w: &World, ms: u64, data: u8) {
    let ok = token(&claims(&w.s, WRITER, &w.alice.0), signer(&w.writer));
    notify(
        &w.s,
        Some(&ok),
        &note(&w.space, WRITER, &tid_at(ms), &[data; 32]),
    )
    .await
    .unwrap();
}

/// The writer set of `space`, removed entries too.
fn writers(s: &Service, authority: &str, space: &str) -> BTreeMap<String, Writer> {
    let a = s.account(authority).unwrap();
    read(&a.store)
        .authority
        .live(space)
        .map(|sp| sp.writers.clone())
        .unwrap_or_default()
}

fn kept(s: &Service, did: &str) -> Kept {
    read(&s.account(did).unwrap().store).notify.clone()
}

/// Headers presenting a credential of `space` from its authority, signed for `audience`.
async fn credentialed(
    s: &Shared,
    authority: &(String, String),
    space: &str,
    audience: &str,
) -> HeaderMap {
    let c = client(1);
    let t = delegation_token(s, &authority.1, space).await.unwrap();
    let cred = credential(
        s,
        json!({ "space": space, "credentialKey": c.kid, "delegationToken": t }),
    )
    .await
    .unwrap();
    signed(&c, &cred, audience)
}

/// `authorizeSpaceSync` of `space` as the account `session` is a session
/// of, registering `endpoint` and `token`.
async fn register_as(
    s: &Shared,
    session: &str,
    space: &str,
    endpoint: &str,
    token: &str,
) -> Result<Json, String> {
    let t = delegation_token(s, session, space).await.unwrap();
    authorized(
        s,
        json!({ "space": space, "credentialKey": client(1).kid, "delegationToken": t, "notify": { "endpoint": endpoint, "token": token } }),
    )
    .await
}

/// The same, as the space's authority, alice.
async fn register(w: &World, space: &str, endpoint: &str, token: &str) -> Result<Json, String> {
    register_as(&w.s, &w.alice.1, space, endpoint, token).await
}

/// The registration of `endpoint` and `token` for `space`, if there is one.
fn registration(
    s: &Service,
    did: &str,
    space: &str,
    endpoint: &str,
    token: &str,
) -> Option<Registration> {
    kept(s, did)
        .registered
        .get(space)
        .and_then(|r| r.get(&(endpoint.to_string(), token.to_string())))
        .copied()
}

/// That `auth` is service auth from `iss` for `aud` and `lxm`, exactly as
/// sent, signed with `iss`'s key; answers its `jti`.
fn service_auth_of(s: &Service, auth: &str, iss: &str, aud: &str, lxm: &str) -> String {
    let parts: Vec<&str> = auth.strip_prefix("Bearer ").unwrap().split('.').collect();
    let part = |p: &str| serde_json::from_slice::<Json>(&B64.decode(p).unwrap()).unwrap();
    assert_eq!(part(parts[0]), json!({ "typ": "JWT", "alg": "ES256K" }));
    let body = part(parts[1]);
    let mut names: Vec<&String> = body.as_object().unwrap().keys().collect();
    names.sort();
    assert_eq!(names, ["aud", "exp", "iat", "iss", "jti", "lxm"]);
    assert_eq!(
        (&body["iss"], &body["aud"], &body["lxm"]),
        (&json!(iss), &json!(aud), &json!(lxm))
    );
    assert_eq!(
        body["exp"].as_u64().unwrap() - body["iat"].as_u64().unwrap(),
        60
    );
    // The signature is deterministic, low-S: the same bytes signed again by the key.
    assert_eq!(
        B64.decode(parts[2]).unwrap(),
        account_key(s, iss)(format!("{}.{}", parts[0], parts[1]).as_bytes())
    );
    body["jti"].as_str().unwrap().to_string()
}

/// That `got` is a call of `nsid` to a syncer, as the spec says: a `POST`
/// of JSON to `/xrpc/<nsid>` with no `Authorization`. Answers its body.
fn to_syncer<'a>(got: &'a Got, nsid: &str) -> &'a Json {
    assert_eq!(got.nsid, nsid);
    assert_eq!(got.path, format!("/xrpc/{nsid}"));
    assert_eq!(got.auth, None);
    assert_eq!(got.content_type.as_deref(), Some("application/json"));
    &got.body
}

/// Each refusal of a notification, in the order the spec gives them, each
/// before the next; then notifications taken.
#[tokio::test]
async fn notifications_are_refused_in_order() {
    let w = world("notify-order").await;
    let s = &w.s;
    let (alice, carol) = (&w.alice.0, &w.carol.0);
    let now = s.space_notify.now();
    let rev = tid_at(now);
    let good = note(&w.space, WRITER, &rev, &[1; 32]);
    let ok = token(&claims(s, WRITER, alice), signer(&w.writer));
    let with = |k: &str, v: Json| {
        let mut c = claims(s, WRITER, alice);
        if v.is_null() {
            c.as_object_mut().unwrap().remove(k);
        } else {
            c[k] = v;
        }
        token(&c, signer(&w.writer))
    };
    // 1. Service auth, before the input.
    let other_key = SigningKey::from_slice(&[9; 32]).unwrap();
    let expired = with("exp", json!(s.sessions.now() - 1));
    for (auth, want) in [
        (None, "401 MissingJwt"),
        (Some("Basic abc".to_string()), "401 MissingJwt"),
        (Some("Bearer abc".to_string()), "401 BadJwt"),
        (
            Some(format!(
                "Bearer {}",
                jwt_of(
                    &json!({ "alg": "ES256K", "typ": "at+jwt" }),
                    &claims(s, WRITER, alice),
                    signer(&w.writer)
                )
            )),
            "401 BadJwtType",
        ),
        (Some(expired), "401 JwtExpired"),
        (Some(with("lxm", Json::Null)), "401 BadJwtLexiconMethod"),
        (Some(with("lxm", json!(REVOKE))), "401 BadJwtLexiconMethod"),
        (
            Some(with("lxm", json!("com.atproto.space.notifyWrite"))),
            "401 BadJwtLexiconMethod",
        ),
        (Some(with("iss", json!("notadid"))), "401 BadJwtIss"),
        (
            Some(with("iss", json!("did:plc:nobodynobodynobodynobody"))),
            "401 AuthenticationRequired",
        ),
        (
            Some(token(&claims(s, WRITER, alice), signer(&other_key))),
            "401 BadJwtSignature",
        ),
    ] {
        assert_eq!(
            notify(s, auth.as_deref(), &good).await,
            Err(want.into()),
            "{want}"
        );
        assert_eq!(
            notify(s, auth.as_deref(), &json!({ "space": "nope" })).await,
            Err(want.into()),
            "{want}, before the input"
        );
    }
    // 2. The input.
    for bad in [
        note(&w.space, WRITER, "x", &[1]),
        note("nope", WRITER, &rev, &[1]),
        json!({ "space": w.space, "repo": WRITER, "repoRev": rev }),
    ] {
        assert_eq!(
            notify(s, Some(&ok), &bad).await,
            Err("400 InvalidRequest".into()),
            "{bad}"
        );
    }
    // 3. iss, as written, the repo notified of.
    assert_eq!(
        notify(s, Some(&ok), &note(&w.space, carol, &rev, &[1])).await,
        Err("403 Forbidden".into())
    );
    let fragment = token(
        &claims(s, &format!("{WRITER}#atproto_pds"), alice),
        signer(&w.writer),
    );
    assert_eq!(
        notify(s, Some(&fragment), &good).await,
        Err("403 Forbidden".into())
    );
    // 4. aud, the space's authority, bare or as its PDS; before 5, an authority elsewhere.
    for aud in [w.bob.0.clone(), format!("{alice}#other"), HOST.to_string()] {
        assert_eq!(
            notify(s, Some(&with("aud", json!(aud))), &good).await,
            Err("403 Forbidden".into()),
            "{aud}"
        );
    }
    let elsewhere = space(HOST, "x");
    assert_eq!(
        notify(s, Some(&ok), &note(&elsewhere, WRITER, &rev, &[1])).await,
        Err("403 Forbidden".into())
    );
    // 5. The authority here; before 6, a rev too far ahead.
    let to_host = with("aud", json!(HOST));
    let ahead = |minutes: u64| tid_at(now + minutes * 60 * SECOND);
    for r in [&rev, &ahead(6)] {
        assert_eq!(
            notify(s, Some(&to_host), &note(&elsewhere, WRITER, r, &[1])).await,
            Err("400 SpaceNotFound".into())
        );
    }
    // 6. Not more than 5 minutes ahead; before 7, a space never created.
    let never = space(alice, "never");
    for sp in [&w.space, &never] {
        assert_eq!(
            notify(s, Some(&ok), &note(sp, WRITER, &ahead(6), &[1])).await,
            Err("400 FutureRev".into())
        );
    }
    // 7. A live space; never created or deleted, before 8, a writer not authorized.
    let gone = create_space(s, &w.alice.1, "gone").await;
    delete_space(s, &w.alice.1, &gone).await;
    let as_carol = token(&claims(s, carol, alice), account_key(s, carol));
    for sp in [&never, &gone] {
        assert_eq!(
            notify(s, Some(&ok), &note(sp, WRITER, &rev, &[1])).await,
            Err("400 SpaceNotFound".into())
        );
        assert_eq!(
            notify(s, Some(&as_carol), &note(sp, carol, &rev, &[1])).await,
            Err("400 SpaceNotFound".into())
        );
    }
    // 8. A writer the policy authorizes: carol only reads.
    assert_eq!(
        notify(s, Some(&as_carol), &note(&w.space, carol, &rev, &[1])).await,
        Err("403 Forbidden".into())
    );
    assert!(writers(s, alice, &w.space).is_empty());
    // Taken: a bare aud, a high-S signature, 31 bytes of data, spaceRev sent and ignored.
    let high = token(&claims(s, WRITER, alice), |m| high_s(signer(&w.writer)(m)));
    let mut body = note(&w.space, WRITER, &rev, &[3; 31]);
    body["spaceRev"] = json!(tid_at(1));
    body["prevSpaceRev"] = json!(tid_at(0));
    assert_eq!(notify(s, Some(&high), &body).await, Ok(Json::Null));
    let got = &writers(s, alice, &w.space)[WRITER];
    assert_eq!(
        (&got.rev, &got.data, got.removed),
        (&rev, &vec![3; 31], false)
    );
    assert!(got.space_rev > tid_at(now - SECOND));
    // As the authority's PDS, and 4 minutes ahead.
    let pds = with("aud", json!(format!("{alice}#atproto_pds")));
    assert_eq!(
        notify(s, Some(&pds), &note(&w.space, WRITER, &ahead(4), &[4; 32])).await,
        Ok(Json::Null)
    );
    assert_eq!(writers(s, alice, &w.space)[WRITER].rev, ahead(4));
    // The authority's account status does not matter.
    crate::space_credentials::tests::deactivate(s, &w.alice.1, true).await;
    assert_eq!(
        notify(s, Some(&pds), &note(&w.space, WRITER, &ahead(5), &[5; 32])).await,
        Ok(Json::Null)
    );
    assert_eq!(writers(s, alice, &w.space)[WRITER].rev, ahead(5));
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// The writer set: monotonic in each writer's `rev`, each change a new
/// `spaceRev` after every other; writes here taken as from elsewhere, a
/// reader's not; removals, marked and kept; and `listWriters`' entries,
/// pages and refusals.
#[tokio::test]
async fn the_writer_set() {
    let w = world("notify-writers").await;
    let s = &w.s;
    let alice = &w.alice.0;
    let now = s.space_notify.now();
    let ok = token(&claims(s, WRITER, alice), signer(&w.writer));
    let (r0, r1, r2) = (tid_at(now - 2 * SECOND), tid_at(now - SECOND), tid_at(now));
    notify(s, Some(&ok), &note(&w.space, WRITER, &r1, &[1; 32]))
        .await
        .unwrap();
    let first = writers(s, alice, &w.space)[WRITER].clone();
    assert_eq!((&first.rev, &first.data), (&r1, &vec![1; 32]));
    // An earlier or equal rev changes nothing.
    for r in [&r0, &r1] {
        notify(s, Some(&ok), &note(&w.space, WRITER, r, &[2; 32]))
            .await
            .unwrap();
        assert_eq!(writers(s, alice, &w.space)[WRITER], first);
    }
    notify(s, Some(&ok), &note(&w.space, WRITER, &r2, &[2; 32]))
        .await
        .unwrap();
    let second = writers(s, alice, &w.space)[WRITER].clone();
    assert_eq!((&second.rev, &second.data), (&r2, &vec![2; 32]));
    assert!(second.space_rev > first.space_rev);
    // Writes here: bob's and the authority's tracked at their commits, carol's (a reader's) not.
    let (bob_rev, bob_data) = put(s, &w.bob, &w.space, "b").await;
    put(s, &w.carol, &w.space, "c").await;
    let (alice_rev, alice_data) = put(s, &w.alice, &w.space, "a").await;
    eventually("bob's and alice's writes to be tracked", || {
        let ws = writers(s, alice, &w.space);
        ws.get(&w.bob.0).is_some_and(|b| b.rev == bob_rev)
            && ws.get(alice).is_some_and(|a| a.rev == alice_rev)
    })
    .await;
    tokio::time::sleep(Duration::from_millis(100)).await;
    let ws = writers(s, alice, &w.space);
    assert_eq!(
        (&ws[&w.bob.0].data, &ws[alice].data),
        (&bob_data.to_vec(), &alice_data.to_vec())
    );
    assert!(!ws.contains_key(&w.carol.0));
    let mut revs: Vec<&String> = ws.values().map(|w| &w.space_rev).collect();
    revs.sort();
    revs.dedup();
    assert_eq!(revs.len(), 3);
    // Taken out: marked removed, with a new spaceRev, its repoRev and data
    // kept; again, or an account never in it, changes nothing.
    let remove = |did: &str| {
        call(
            s,
            "com.atproto.space.removeWriter",
            Some(&w.alice.1),
            json!({ "space": w.space, "repo": did }),
        )
    };
    remove(WRITER).await.unwrap();
    let removed = writers(s, alice, &w.space)[WRITER].clone();
    assert!(removed.removed);
    assert_eq!((&removed.rev, &removed.data), (&r2, &vec![2; 32]));
    assert!(
        writers(s, alice, &w.space)
            .values()
            .all(|x| x.space_rev <= removed.space_rev)
    );
    let before = writers(s, alice, &w.space);
    remove(WRITER).await.unwrap();
    remove("did:plc:nevernevernevernevernever").await.unwrap();
    assert_eq!(writers(s, alice, &w.space), before);
    // A stale notification of a removed writer changes nothing either.
    notify(s, Some(&ok), &note(&w.space, WRITER, &r2, &[3; 32]))
        .await
        .unwrap();
    assert_eq!(writers(s, alice, &w.space), before);
    // Added back by its next write, after every other.
    let r3 = tid_at(s.space_notify.now() + 1);
    notify(s, Some(&ok), &note(&w.space, WRITER, &r3, &[3; 32]))
        .await
        .unwrap();
    let ws = writers(s, alice, &w.space);
    assert!(!ws[WRITER].removed);
    assert!(ws.values().all(|x| x.space_rev <= ws[WRITER].space_rev));
    // A member's write taken away, and a member removed: removals too.
    put_members(s, &w.alice.1, &w.space, &[(w.bob.0.as_str(), true, false)]).await;
    assert!(writers(s, alice, &w.space)[&w.bob.0].removed);
    call(
        s,
        "com.atproto.memberspace.removeMembers",
        Some(&w.alice.1),
        json!({ "space": w.space, "dids": [WRITER] }),
    )
    .await
    .unwrap();
    assert!(writers(s, alice, &w.space)[WRITER].removed);
    // listWriters: in ascending order of spaceRev, a page at a time.
    let h = credentialed(s, &w.alice, &w.space, alice).await;
    let list = |params: Vec<(&'static str, String)>| {
        let h = h.clone();
        async move {
            let params: Vec<(&str, &str)> = params.iter().map(|(k, v)| (*k, v.as_str())).collect();
            send(s, Http::GET, LIST_WRITERS, &params, &h, "").await
        }
    };
    let all = list(vec![("space", w.space.clone())]).await.unwrap();
    let want: Vec<Json> = {
        let mut ws: Vec<(String, Writer)> = writers(s, alice, &w.space).into_iter().collect();
        ws.sort_by(|a, b| a.1.space_rev.cmp(&b.1.space_rev));
        ws.iter()
            .map(|(did, x)| {
                if x.removed {
                    json!({ "$type": REPO_REMOVED, "repo": did, "spaceRev": x.space_rev })
                } else {
                    json!({ "$type": REPO_WRITE, "repo": did, "repoRev": x.rev, "data": to_json(&V::Bytes(x.data.clone())), "spaceRev": x.space_rev })
                }
            })
            .collect()
    };
    assert_eq!(want.len(), 3);
    assert_eq!(all, json!({ "repos": want, "cursor": want[2]["spaceRev"] }));
    let mut cursor: Option<String> = None;
    let mut paged = Vec::new();
    loop {
        let mut p = vec![("space", w.space.clone()), ("limit", "1".to_string())];
        p.extend(cursor.clone().map(|c| ("cursor", c)));
        let page = list(p).await.unwrap();
        let repos = page["repos"].as_array().unwrap().clone();
        if repos.is_empty() {
            assert_eq!(page, json!({ "repos": [] }));
            break;
        }
        assert_eq!(repos.len(), 1);
        assert_eq!(page["cursor"], repos[0]["spaceRev"]);
        cursor = page["cursor"].as_str().map(str::to_string);
        paged.extend(repos);
    }
    assert_eq!(paged, want);
    // Any string is a cursor, compared by bytes.
    assert_eq!(
        list(vec![("space", w.space.clone()), ("cursor", "zzz".into())]).await,
        Ok(json!({ "repos": [] }))
    );
    assert_eq!(
        list(vec![("space", w.space.clone()), ("cursor", "".into())])
            .await
            .unwrap()["repos"],
        json!(want)
    );
    // Its parameters first, then the credential.
    for p in [
        vec![("limit", "1".to_string())],
        vec![("space", w.space.clone()), ("limit", "0".into())],
        vec![("space", w.space.clone()), ("limit", "1001".into())],
        vec![("space", "nope".into())],
    ] {
        assert_eq!(
            send(
                s,
                Http::GET,
                LIST_WRITERS,
                &p.iter().map(|(k, v)| (*k, v.as_str())).collect::<Vec<_>>(),
                &HeaderMap::new(),
                ""
            )
            .await,
            Err("400 InvalidRequest".into())
        );
    }
    let only = [("space", w.space.as_str())];
    assert_eq!(
        send(s, Http::GET, LIST_WRITERS, &only, &HeaderMap::new(), "").await,
        Err("401 MissingJwt".into())
    );
    assert_eq!(
        send(
            s,
            Http::GET,
            LIST_WRITERS,
            &only,
            &authorization(&format!("Bearer {}", w.alice.1)),
            ""
        )
        .await,
        Err("401 MissingJwt".into())
    );
    let for_bob = credentialed(s, &w.alice, &w.space, &w.bob.0).await;
    assert_eq!(
        send(s, Http::GET, LIST_WRITERS, &only, &for_bob, "").await,
        Err("401 BadSpaceAudience".into())
    );
    let other = create_space(s, &w.alice.1, "other").await;
    assert_eq!(
        send(s, Http::GET, LIST_WRITERS, &[("space", &other)], &h, "").await,
        Err("400 InvalidCredential".into())
    );
    // A deleted space; a credential of it serves it created again, empty.
    delete_space(s, &w.alice.1, &w.space).await;
    assert_eq!(
        send(s, Http::GET, LIST_WRITERS, &only, &h, "").await,
        Err("400 SpaceNotFound".into())
    );
    create_space(s, &w.alice.1, "m").await;
    assert_eq!(
        send(s, Http::GET, LIST_WRITERS, &only, &h, "").await,
        Ok(json!({ "repos": [] }))
    );
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// The writer set's removals survive a compaction, into the account's
/// checkpoint, and a restart; `exportSpaceMeta` leaves them out.
#[tokio::test]
async fn removals_are_kept_but_not_exported() {
    let w = world("notify-removals-kept").await;
    let (s, alice) = (w.s.clone(), w.alice.0.clone());
    let now = s.space_notify.now();
    write_at(&w, now - SECOND, 1).await;
    put(&s, &w.bob, &w.space, "b").await;
    eventually("bob's write to be tracked", || {
        writers(&s, &alice, &w.space).contains_key(&w.bob.0)
    })
    .await;
    call(
        &s,
        "com.atproto.space.removeWriter",
        Some(&w.alice.1),
        json!({ "space": w.space, "repo": WRITER }),
    )
    .await
    .unwrap();
    let before = writers(&s, &alice, &w.space);
    assert!(before[WRITER].removed && !before[&w.bob.0].removed);
    let wal = read(&s.account(&alice).unwrap().store).wal.clone();
    wal.wait_compacted();
    wal.compact().unwrap();
    idle(&s).await;
    drop(wal);
    drop(s);
    drop(w.s);
    let s = open(&w.dir, &w.plc_url);
    assert_eq!(writers(&s, &alice, &w.space), before);
    let params = [("space".to_string(), w.space.clone())];
    let headers = HeaderMap::new();
    let call = Call {
        nsid: "com.atproto.space.exportSpaceMeta",
        method: &Http::GET,
        params: &params,
        body: b"",
        bearer: crate::sessions::Bearer::Token(&w.alice.1),
        headers: &headers,
        client: "127.0.0.1".into(),
        charged: Default::default(),
        http: None,
    };
    let Ok(Resp::Star(archive)) = crate::answer(&s, &call).await else {
        panic!("exportSpaceMeta answered no archive");
    };
    let a = pds_space::star::read(&archive, &pds_space::star::REPO_LIMITS).unwrap();
    let keys: Vec<String> = a
        .pairs
        .iter()
        .map(|(k, _)| String::from_utf8(k.to_vec()).unwrap())
        .filter(|k| k.starts_with("writer/"))
        .collect();
    assert_eq!(keys, [format!("writer/{}", w.bob.0)]);
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// A new `spaceRev` is the TID of its time, or one microsecond after the
/// space's revision.
#[test]
fn space_revs_only_grow() {
    let now = 1_700_000_000_000;
    assert_eq!(next_space_rev(now, None), tid_at(now));
    assert_eq!(next_space_rev(now, Some(&tid_at(now - 1))), tid_at(now));
    let later = tid_at(now + 5);
    let after = next_space_rev(now, Some(&later));
    assert_eq!(tid_value(&after), ((((now + 5) * 1000) + 1) << 10));
    assert!(valid_tid(&after) && after > later);
    // Equal to it, one microsecond after it too; a clock identifier in it is not kept.
    assert_eq!(
        tid_value(&next_space_rev(now, Some(&tid_at(now)))),
        ((now * 1000) + 1) << 10
    );
    let odd = tid_of(((now * 1000) << 10) | 7);
    assert_eq!(
        tid_value(&next_space_rev(now, Some(&odd))),
        ((now * 1000) + 1) << 10
    );
}

/// The retry schedules: a notification's, each window capped at an hour;
/// a notice's, its delays from a minute to twelve hours, given up 72 hours
/// after it was kept; and what an answer makes of a send.
#[test]
fn the_schedules() {
    for (tried, lo, hi) in [
        (0, 30, 60),
        (1, 60, 120),
        (2, 120, 240),
        (5, 960, 1920),
        (6, 1800, 3600),
        (20, 1800, 3600),
    ] {
        for _ in 0..50 {
            let at = retry(0, tried, u64::MAX).unwrap();
            assert_eq!(at.n, tried + 1);
            assert!(
                (lo * SECOND..=hi * SECOND).contains(&at.due),
                "retry {} at {}",
                at.n,
                at.due
            );
        }
    }
    assert_eq!(retry(0, 0, 29 * SECOND), None);
    for (tried, lo, hi) in [
        (0, 30, 60),
        (1, 300, 600),
        (2, 1800, 3600),
        (3, 10800, 21600),
        (4, 21600, 43200),
        (9, 21600, 43200),
    ] {
        for _ in 0..50 {
            let at = notice_retry(0, tried, NOTICE_HORIZON).unwrap();
            assert_eq!((at.n, at.until), (tried + 1, NOTICE_HORIZON));
            assert!(
                (lo * SECOND..=hi * SECOND).contains(&at.due),
                "notice retry {} at {}",
                at.n,
                at.due
            );
        }
    }
    // Rescheduled while 72 hours have not passed, past its horizon too; then given up.
    assert!(
        notice_retry(NOTICE_HORIZON - 1, 7, NOTICE_HORIZON)
            .is_some_and(|at| at.due > NOTICE_HORIZON)
    );
    assert_eq!(notice_retry(NOTICE_HORIZON, 7, NOTICE_HORIZON), None);
    assert_eq!(NOTICE_HORIZON, 72 * 3600 * SECOND);
    assert_eq!(HORIZON, 24 * 3600 * SECOND);
    for s in [200, 201, 299] {
        assert_eq!(outcome(s), Outcome::Delivered);
    }
    // A redirect is as no answer.
    for s in FAILING.into_iter().chain([300, 301, 302, 307, 308]) {
        assert_eq!(outcome(s), Outcome::Failed);
    }
    for s in [400, 401, 403, 404, 409, 501, 505, 523] {
        assert_eq!(outcome(s), Outcome::Refused);
    }
}

/// A challenge is 16 random bytes in lower case unpadded base32: 26
/// characters, new each time.
#[test]
fn challenges() {
    let mut seen = HashSet::new();
    for _ in 0..200 {
        let c = challenge();
        assert_eq!(c.len(), 26);
        assert!(
            c.bytes()
                .all(|b| b.is_ascii_lowercase() || (b'2'..=b'7').contains(&b))
        );
        // The last character holds 3 bits, then two of padding.
        assert!(seen.insert(c));
    }
}

/// The repo host's part: each commit sent at once, exactly as the spec
/// says; failures kept, one per space, the latest, and retried on the
/// schedule with fresh service auth; refusals not; the horizon.
#[tokio::test]
async fn the_retry_queue() {
    let w = world("notify-queue").await;
    let (s, party) = (&w.s, &w.party);
    let alice = &w.alice.0;
    let q = space(HOST, "q");
    // Sent at once, exactly so; a 503 kept for its first retry.
    party.answer(&q, &[503]);
    let t0 = s.space_notify.now();
    let (r1, d1) = put(s, &w.alice, &q, "a").await;
    idle(s).await;
    let got = party.take();
    assert_eq!(got.len(), 1);
    assert_eq!(got[0].nsid, NOTIFY_REPO_UPDATE);
    assert_eq!(
        got[0].body,
        json!({ "space": q, "repo": alice, "repoRev": r1, "data": to_json(&V::Bytes(d1.to_vec())) })
    );
    let jti = service_auth_of(
        s,
        got[0].auth.as_deref().unwrap(),
        alice,
        &format!("{HOST}#atproto_pds"),
        NOTIFY_REPO_UPDATE,
    );
    let p = kept(s, alice).queue[&q].clone();
    assert_eq!((&p.rev, &p.data, p.at.n), (&r1, &d1.to_vec(), 1));
    assert!(
        p.at.due >= t0 + 30 * SECOND && p.at.due <= s.space_notify.now() + 60 * SECOND,
        "{:?}",
        p.at
    );
    assert!(p.at.until >= t0 + HORIZON && p.at.until <= s.space_notify.now() + HORIZON);
    // Not before it is due.
    advance_to(s, p.at.due - 2 * SECOND);
    assert_eq!(run(s).await, 0);
    assert!(party.take().is_empty());
    // When due: sent again with fresh service auth, failing again: the second retry 60 to 120 seconds on.
    advance_to(s, p.at.due);
    party.answer(&q, &[503]);
    assert_eq!(run(s).await, 1);
    let got = party.take();
    assert_eq!(got.len(), 1);
    assert_eq!(got[0].body["repoRev"], json!(r1));
    assert_ne!(
        service_auth_of(
            s,
            got[0].auth.as_deref().unwrap(),
            alice,
            &format!("{HOST}#atproto_pds"),
            NOTIFY_REPO_UPDATE
        ),
        jti
    );
    let t1 = s.space_notify.now();
    let p = kept(s, alice).queue[&q].clone();
    assert_eq!(p.at.n, 2);
    assert!(
        p.at.due >= t1 + 60 * SECOND && p.at.due <= t1 + 121 * SECOND,
        "{:?}",
        p.at
    );
    // Later commits replace the entry, whatever its schedule: its first retry, 24 hours from the latest.
    party.answer(&q, &[503, 503]);
    put(s, &w.alice, &q, "b").await;
    let t2 = s.space_notify.now();
    let (r3, d3) = put(s, &w.alice, &q, "c").await;
    idle(s).await;
    assert_eq!(party.take().len(), 2);
    let p = kept(s, alice).queue[&q].clone();
    assert_eq!((&p.rev, &p.data, p.at.n), (&r3, &d3.to_vec(), 1));
    assert!(p.at.until >= t2 + HORIZON);
    // Only the latest is sent again; delivered, it is taken out.
    advance_to(s, p.at.due);
    assert_eq!(run(s).await, 1);
    let got = party.take();
    assert_eq!(
        got.iter()
            .map(|g| g.body["repoRev"].clone())
            .collect::<Vec<_>>(),
        [json!(r3)]
    );
    assert!(kept(s, alice).queue.is_empty());
    assert_eq!(run(s).await, 0);
    // A refusal is not kept, `400 FutureRev` too; a 429 is, and a 502.
    for (status, keeps) in [
        (400, false),
        (401, false),
        (403, false),
        (429, true),
        (500, true),
        (502, true),
    ] {
        party.answer(&q, &[status]);
        let (rev, _) = put(s, &w.alice, &q, "d").await;
        idle(s).await;
        assert_eq!(party.take().len(), 1);
        assert_eq!(
            kept(s, alice).queue.get(&q).map(|p| p.rev.clone()),
            keeps.then_some(rev),
            "{status}"
        );
    }
    // The horizon: not delivered 24 hours after its commit, it is dropped unsent.
    advance_to(s, kept(s, alice).queue[&q].at.until);
    assert_eq!(run(s).await, 0);
    assert!(party.take().is_empty());
    assert!(kept(s, alice).queue.is_empty());
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// What fails a notification for now without an answer: a document that
/// cannot be read or has no `#atproto_pds`, a connection closed, and, with
/// SSRF protection, an `http` endpoint; each kept.
#[tokio::test]
async fn failures_without_an_answer() {
    let w = world("notify-unanswered").await;
    let (s, alice) = (&w.s, &w.alice.0);
    let nobody = "did:plc:nobodynobodynobodynobody";
    let no_pds = "did:plc:nopdsnopdsnopdsnopdsnopd";
    let closes = "did:plc:closesclosesclosesclos";
    w.plc.serve(no_pds, None, &[("#other", &w.url)]);
    w.plc
        .serve(closes, None, &[("#atproto_pds", &closing().await)]);
    for authority in [nobody, no_pds, closes] {
        let sp = space(authority, "x");
        let (rev, _) = put(s, &w.alice, &sp, "a").await;
        idle(s).await;
        assert_eq!(kept(s, alice).queue[&sp].rev, rev, "{authority}");
    }
    assert!(w.party.take().is_empty());
    // With SSRF protection, the harness's `http` host is not sent to.
    let dir = scratch("notify-ssrf");
    let protected: Shared = Arc::new(
        Service::open(
            &dir,
            Config {
                ssrf_protection: true,
                ..config(&w.plc_url)
            },
        )
        .unwrap(),
    );
    let dave = create(&protected, "dave").await;
    let q = space(HOST, "q");
    let (rev, _) = put(&protected, &dave, &q, "a").await;
    idle(&protected).await;
    assert_eq!(kept(&protected, &dave.0).queue[&q].rev, rev);
    assert!(w.party.take().is_empty());
    let _ = std::fs::remove_dir_all(&dir);
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// The queue survives a restart, its schedule too, and is sent when due.
#[tokio::test]
async fn restart_mid_queue() {
    let w = world("notify-restart").await;
    let party = w.party.clone();
    let (s, alice) = (w.s.clone(), w.alice.clone());
    let (q, q2) = (space(HOST, "q"), space(HOST, "q2"));
    party.answer(&q, &[503, 503]);
    party.answer(&q2, &[503]);
    put(&s, &alice, &q, "a").await;
    let (r2, _) = put(&s, &alice, &q, "b").await;
    let (r3, _) = put(&s, &alice, &q2, "a").await;
    idle(&s).await;
    party.take();
    let before = kept(&s, &alice.0);
    assert_eq!(
        (before.queue[&q].rev.clone(), before.queue[&q2].rev.clone()),
        (r2.clone(), r3.clone())
    );
    drop(s);
    drop(w.s);
    let s = open(&w.dir, &w.plc_url);
    assert_eq!(kept(&s, &alice.0), before);
    assert_eq!(run(&s).await, 0);
    let due = before.queue.values().map(|p| p.at.due).max().unwrap();
    advance_to(&s, due);
    assert_eq!(run(&s).await, 2);
    let mut got: Vec<(Json, Json)> = party
        .take()
        .into_iter()
        .map(|g| (g.body["space"].clone(), g.body["repoRev"].clone()))
        .collect();
    got.sort_by_key(|g| g.0.to_string());
    assert_eq!(got, [(json!(q), json!(r2)), (json!(q2), json!(r3))]);
    assert!(kept(&s, &alice.0).queue.is_empty());
    drop(s);
    assert!(kept(&open(&w.dir, &w.plc_url), &alice.0).queue.is_empty());
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// Registrations: each new one confirmed once, exactly as the spec says,
/// at its endpoint's origin; the same pair renewed with no confirmation;
/// another token, or the endpoint written otherwise, a registration of its
/// own; and each refusal to confirm one.
#[tokio::test]
async fn registrations_are_confirmed() {
    let w = world("notify-confirm").await;
    let (s, party, alice) = (&w.s, &w.party, &w.alice.0);
    let host = w.url.strip_prefix("http://").unwrap();
    // Made: confirmed once, `spaceRev` absent while the set is empty.
    let got = register(&w, &w.space, &w.url, "t1").await.unwrap();
    assert_eq!(
        got.as_object().unwrap().keys().collect::<Vec<_>>(),
        ["credential"]
    );
    let asked = party.take();
    assert_eq!(asked.len(), 1);
    let body = to_syncer(&asked[0], CONFIRM);
    let mut names: Vec<&String> = body.as_object().unwrap().keys().collect();
    names.sort();
    assert_eq!(names, ["challenge", "space", "token"]);
    assert_eq!(
        (&body["space"], &body["token"]),
        (&json!(w.space), &json!("t1"))
    );
    let first = body["challenge"].as_str().unwrap().to_string();
    assert_eq!(first.len(), 26);
    // To lapse when its credential expires.
    let exp = |got: &Json| {
        let cred = got["credential"].as_str().unwrap();
        let payload = cred.split('.').nth(1).unwrap();
        serde_json::from_slice::<Json>(&B64.decode(payload).unwrap()).unwrap()["exp"]
            .as_u64()
            .unwrap()
    };
    let r = registration(s, alice, &w.space, &w.url, "t1").unwrap();
    assert_eq!(
        r,
        Registration {
            expires: exp(&got) * 1000,
            noticed: false
        }
    );
    // Renewed, not confirmed again, to lapse when the new credential expires.
    advance_both(s, 5 * SECOND);
    let again = register(&w, &w.space, &w.url, "t1").await.unwrap();
    assert!(party.take().is_empty());
    let renewed = registration(s, alice, &w.space, &w.url, "t1").unwrap();
    assert_eq!(renewed.expires, exp(&again) * 1000);
    assert!(renewed.expires > r.expires);
    // Another token, the endpoint with a `/` after it, and one with a path
    // and a query, each its own, each confirmed at the origin, with a new challenge.
    let based = format!("{}/base/more?x=1", w.url);
    let slashed = format!("{}/", w.url);
    let long = "x".repeat(512);
    for (endpoint, token) in [
        (w.url.as_str(), "t2"),
        (slashed.as_str(), "t1"),
        (based.as_str(), "t3"),
        (w.url.as_str(), ""),
        (w.url.as_str(), long.as_str()),
    ] {
        register(&w, &w.space, endpoint, token).await.unwrap();
        let asked = party.take();
        assert_eq!(asked.len(), 1, "{endpoint} {token}");
        let body = to_syncer(&asked[0], CONFIRM);
        assert_eq!(body["token"], json!(token));
        assert_ne!(body["challenge"], json!(first));
        assert!(registration(s, alice, &w.space, endpoint, token).is_some());
    }
    assert_eq!(kept(s, alice).registered[&w.space].len(), 6);
    // Not confirmed: nothing registered, no credential.
    let not = Err("400 RegistrationNotConfirmed".to_string());
    for (token, how, status) in [
        ("wrong", Confirm::Wrong, None),
        ("text", Confirm::Text, None),
        ("empty", Confirm::Empty, None),
        ("failing", Confirm::Echo, Some(500)),
        ("redirected", Confirm::Echo, Some(302)),
    ] {
        party.confirm(token, how);
        if let Some(status) = status {
            party.answer(&w.space, &[status]);
        }
        assert_eq!(register(&w, &w.space, &w.url, token).await, not, "{token}");
        assert_eq!(party.take().len(), 1, "{token}");
        assert!(registration(s, alice, &w.space, &w.url, token).is_none());
    }
    // A closed connection; and endpoints sent nothing: not http(s), a user
    // and password, a DID.
    assert_eq!(
        register(&w, &w.space, &closing().await, "closed").await,
        not
    );
    for endpoint in [
        format!("ftp://{host}"),
        format!("http://u:p@{host}"),
        "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
    ] {
        assert_eq!(
            register(&w, &w.space, &endpoint, "t").await,
            not,
            "{endpoint}"
        );
    }
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert!(party.take().is_empty());
    // Refused before step 10: nothing confirmed. A space never made, and
    // one whose authority is elsewhere.
    let never = space(alice, "never");
    assert_eq!(
        register(&w, &never, &w.url, "t").await,
        Err("400 SpaceNotFound".into())
    );
    let t = delegation_token(s, &w.alice.1, &space(HOST, "x"))
        .await
        .unwrap();
    let elsewhere = authorized(
        s,
        json!({ "space": space(HOST, "x"), "credentialKey": client(1).kid, "delegationToken": t, "notify": { "endpoint": w.url, "token": "t" } }),
    )
    .await;
    assert_eq!(elsewhere, Err("400 SpaceNotFound".into()));
    assert!(party.take().is_empty());
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// Each change of the writer set sent to each active registration, once,
/// as the spec says: chained by `prevSpaceRev` from none, a removal too;
/// none for a change that is none, or a reader's write; one attempt each;
/// a refusal at step 8 dropping a registration, which is then confirmed
/// anew; `authorizeSpaceSync`'s `spaceRev` the set's revision.
#[tokio::test]
async fn updates_are_sent() {
    let w = world("notify-updates").await;
    let (s, party, alice) = (&w.s, &w.party, &w.alice.0);
    register(&w, &w.space, &w.url, "a").await.unwrap();
    register(&w, &w.space, &format!("{}/x", w.url), "b")
        .await
        .unwrap();
    party.take();
    // The first change has no prevSpaceRev; each, its own spaceRev, sent to both.
    let now = s.space_notify.now();
    write_at(&w, now - SECOND, 1).await;
    let first = writers(s, alice, &w.space)[WRITER].space_rev.clone();
    let got = party.wait_for(2).await;
    let mut tokens: Vec<&Json> = got
        .iter()
        .map(|g| &to_syncer(g, NOTIFY_SPACE_UPDATE)["token"])
        .collect();
    tokens.sort_by_key(|t| t.to_string());
    assert_eq!(tokens, [&json!("a"), &json!("b")]);
    assert_eq!(
        got[0].body,
        json!({ "space": w.space, "token": got[0].body["token"], "update": { "$type": REPO_WRITE, "repo": WRITER, "repoRev": tid_at(now - SECOND), "data": to_json(&V::Bytes(vec![1; 32])), "spaceRev": first } })
    );
    write_at(&w, now, 2).await;
    let second = writers(s, alice, &w.space)[WRITER].space_rev.clone();
    let got = party.wait_for(2).await;
    assert!(
        got.iter()
            .all(|g| g.body["update"]["prevSpaceRev"] == json!(first)
                && g.body["update"]["spaceRev"] == json!(second))
    );
    // A stale notification, and a reader's write, send nothing.
    write_at(&w, now - SECOND, 3).await;
    put(s, &w.carol, &w.space, "c").await;
    tokio::time::sleep(Duration::from_millis(200)).await;
    assert!(party.take().is_empty());
    // A write here, the authority's too.
    let (bob_rev, _) = put(s, &w.bob, &w.space, "b").await;
    let got = party.wait_for(2).await;
    assert_eq!(
        (
            &got[0].body["update"]["repo"],
            &got[0].body["update"]["repoRev"],
            &got[0].body["update"]["prevSpaceRev"]
        ),
        (&json!(w.bob.0), &json!(bob_rev), &json!(second))
    );
    let third = got[0].body["update"]["spaceRev"].clone();
    // A removal: its spaceRev after the space's revision.
    call(
        s,
        "com.atproto.space.removeWriter",
        Some(&w.alice.1),
        json!({ "space": w.space, "repo": w.bob.0 }),
    )
    .await
    .unwrap();
    let got = party.wait_for(2).await;
    let removed = writers(s, alice, &w.space)[&w.bob.0].space_rev.clone();
    assert_eq!(
        got[0].body["update"],
        json!({ "$type": REPO_REMOVED, "repo": w.bob.0, "spaceRev": removed, "prevSpaceRev": third })
    );
    // authorizeSpaceSync answers the space's revision, the removal's.
    let renewed = register(&w, &w.space, &w.url, "a").await.unwrap();
    assert_eq!(renewed["spaceRev"], json!(removed));
    let plain = credential(s, json!({ "space": w.space, "credentialKey": client(1).kid, "delegationToken": delegation_token(s, &w.alice.1, &w.space).await.unwrap() })).await;
    assert!(plain.is_ok());
    // One attempt: a 500 is not sent again, nor kept.
    party.answer(&w.space, &[500, 500]);
    write_at(&w, s.space_notify.now(), 4).await;
    party.wait_for(2).await;
    assert!(kept(s, alice).notices.is_empty());
    assert_eq!(run(s).await, 0);
    tokio::time::sleep(Duration::from_millis(200)).await;
    assert!(party.take().is_empty());
    // A refusal at step 8 drops the registration: carol may read, dave not.
    let dave = create(s, "dave").await;
    assert_eq!(
        register_as(s, &dave.1, &w.space, &w.url, "a").await,
        Err("400 SpaceNotFound".into())
    );
    assert!(registration(s, alice, &w.space, &w.url, "a").is_none());
    write_at(&w, s.space_notify.now(), 5).await;
    let got = party.wait_for(1).await;
    assert_eq!(got[0].body["token"], json!("b"));
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert!(party.take().is_empty());
    // A refusal before step 8 leaves it as it was.
    let b = registration(s, alice, &w.space, &format!("{}/x", w.url), "b");
    let refused = authorized(
        s,
        json!({ "space": w.space, "credentialKey": client(1).kid, "delegationToken": "x.y.z", "notify": { "endpoint": format!("{}/x", w.url), "token": "b" } }),
    )
    .await;
    assert_eq!(refused, Err("400 InvalidDelegationToken".into()));
    assert_eq!(
        registration(s, alice, &w.space, &format!("{}/x", w.url), "b"),
        b
    );
    // Confirmed anew.
    register_as(s, &w.carol.1, &w.space, &w.url, "a")
        .await
        .unwrap();
    assert_eq!(party.take()[0].nsid, CONFIRM);
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// A lapsed registration: sent one expiry notice in place of the first
/// update after it lapses, and nothing more; renewed without a
/// confirmation, and owed one again when it next lapses. An expiry notice
/// refused, or given up, drops its registration unless it was renewed
/// since; one failed for now is kept and retried as "Notices" says.
#[tokio::test]
async fn expiry_notices() {
    let w = world("notify-expiry").await;
    let s0 = w.s.clone();
    let (s, party, alice) = (&s0, &w.party, &w.alice.0);
    register(&w, &w.space, &w.url, "a").await.unwrap();
    party.take();
    let expires = registration(s, alice, &w.space, &w.url, "a")
        .unwrap()
        .expires;
    // Lapsed: the change owes it an expiry notice, at the change's spaceRev.
    lapse(s);
    write_at(&w, s.space_notify.now(), 1).await;
    idle(s).await;
    let got = party.wait_for(1).await;
    let space_rev = writers(s, alice, &w.space)[WRITER].space_rev.clone();
    assert_eq!(
        to_syncer(&got[0], NOTIFY_SPACE_UPDATE),
        &json!({ "space": w.space, "token": "a", "update": { "$type": EXPIRED, "spaceRev": space_rev } })
    );
    assert_eq!(
        registration(s, alice, &w.space, &w.url, "a"),
        Some(Registration {
            expires,
            noticed: true
        })
    );
    // Once: the next change sends nothing.
    write_at(&w, s.space_notify.now() + 1, 2).await;
    idle(s).await;
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert!(party.take().is_empty());
    // Renewed with no confirmation: sent updates again, its spaceRev the set's.
    let renewed = register(&w, &w.space, &w.url, "a").await.unwrap();
    assert!(party.take().is_empty());
    assert_eq!(
        renewed["spaceRev"],
        json!(writers(s, alice, &w.space)[WRITER].space_rev)
    );
    assert!(
        !registration(s, alice, &w.space, &w.url, "a")
            .unwrap()
            .noticed
    );
    write_at(&w, s.space_notify.now() + 2, 3).await;
    let got = party.wait_for(1).await;
    assert_eq!(got[0].body["update"]["$type"], json!(REPO_WRITE));
    // Lapsed again, owed one again; refused, the registration is dropped.
    lapse(s);
    party.answer(&w.space, &[400]);
    write_at(&w, s.space_notify.now(), 4).await;
    idle(s).await;
    assert_eq!(party.take().len(), 1);
    assert!(registration(s, alice, &w.space, &w.url, "a").is_none());
    assert!(kept(s, alice).notices.is_empty());
    // Failed for now: kept, retried 30 to 60 seconds on, until delivered.
    register(&w, &w.space, &w.url, "b").await.unwrap();
    party.take();
    lapse(s);
    party.answer(&w.space, &[503]);
    let t0 = s.space_notify.now();
    write_at(&w, t0, 5).await;
    idle(s).await;
    assert_eq!(party.take().len(), 1);
    let key = (w.space.clone(), (w.url.clone(), "b".to_string()));
    let n = kept(s, alice).notices[&key].clone();
    assert!(matches!(n.owed, Owed::Expired { .. }));
    assert_eq!(n.at.n, 1);
    assert!(n.at.due >= t0 + 30 * SECOND && n.at.due <= s.space_notify.now() + 60 * SECOND);
    assert!(n.at.until >= t0 + NOTICE_HORIZON);
    // Across a restart.
    let before = kept(s, alice);
    drop(s0);
    drop(w.s);
    let s = open(&w.dir, &w.plc_url);
    assert_eq!(kept(&s, alice), before);
    advance_to(&s, n.at.due);
    assert_eq!(run(&s).await, 1);
    let got = party.take();
    assert_eq!(got.len(), 1);
    assert_eq!(got[0].body["update"]["$type"], json!(EXPIRED));
    assert!(kept(&s, alice).notices.is_empty());
    assert!(registration(&s, alice, &w.space, &w.url, "b").is_some());
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// The notice queue's rules: given up 72 hours after it was kept, which
/// drops an expiry notice's registration; dropped unsent once its
/// registration is renewed or dropped; not sent while the authority is not
/// active, and rescheduled; an expiry notice never replacing one kept.
#[tokio::test]
async fn the_notice_queue() {
    let w = world("notify-notice-queue").await;
    let (s, party, alice) = (&w.s, &w.party, &w.alice.0);
    let key = |token: &str| (w.space.clone(), (w.url.clone(), token.to_string()));
    for t in ["a", "b", "c"] {
        register(&w, &w.space, &w.url, t).await.unwrap();
    }
    party.take();
    lapse(s);
    party.answer(&w.space, &[503, 503, 503]);
    write_at(&w, s.space_notify.now(), 1).await;
    idle(s).await;
    assert_eq!(party.take().len(), 3);
    assert_eq!(kept(s, alice).notices.len(), 3);
    // b renewed, c dropped by a refusal: theirs are dropped unsent when due.
    register(&w, &w.space, &w.url, "b").await.unwrap();
    let dave = create(s, "dave").await;
    assert_eq!(
        register_as(s, &dave.1, &w.space, &w.url, "c").await,
        Err("400 SpaceNotFound".into())
    );
    party.take();
    let due = kept(s, alice)
        .notices
        .values()
        .map(|n| n.at.due)
        .max()
        .unwrap();
    // The authority deactivated: not sent, rescheduled as a retry that failed.
    crate::space_credentials::tests::deactivate(s, &w.alice.1, true).await;
    advance_to(s, due);
    assert_eq!(run(s).await, 0);
    assert!(party.take().is_empty());
    let k = kept(s, alice);
    assert_eq!(k.notices.keys().collect::<Vec<_>>(), [&key("a")]);
    assert_eq!(k.notices[&key("a")].at.n, 2);
    crate::space_credentials::tests::deactivate(s, &w.alice.1, false).await;
    // Failing until 72 hours after it was kept: then given up, its
    // registration dropped.
    let until = k.notices[&key("a")].at.until;
    while let Some(n) = kept(s, alice).notices.get(&key("a")).cloned() {
        advance_to(s, n.at.due);
        party.answer(&w.space, &[503]);
        assert_eq!(run(s).await, 1);
        party.take();
        if s.space_notify.now() >= until {
            assert!(kept(s, alice).notices.is_empty());
        }
    }
    assert!(s.space_notify.now() >= until);
    assert!(registration(s, alice, &w.space, &w.url, "a").is_none());
    assert!(registration(s, alice, &w.space, &w.url, "b").is_some());
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// A space deleted: each registration, lapsed ones too, owed a deletion
/// notice, kept in place of any notice of its registration and sent at
/// once; the registrations dropped and the writer set emptied; a notice
/// that failed retried on the schedule, across a restart, until delivered,
/// or dropped unsent once the space is created again. Deleting it again
/// sends nothing.
#[tokio::test]
async fn deletion_notices() {
    let w = world("notify-deleted").await;
    let party = w.party.clone();
    let (s, alice) = (w.s.clone(), w.alice.clone());
    let d1 = create_space(&s, &alice.1, "d1").await;
    let d2 = create_space(&s, &alice.1, "d2").await;
    for sp in [&d1, &d2] {
        for t in ["a", "b"] {
            register(&w, sp, &w.url, t).await.unwrap();
        }
    }
    // d1's "b" lapsed, with an expiry notice kept for it.
    party.take();
    put(&s, &alice, &d1, "x").await;
    party.wait_for(2).await;
    let tracked = |s: &Service| writers(s, &alice.0, &d1).len();
    eventually("alice's write to be tracked", || tracked(&s) == 1).await;
    lapse(&s);
    register(&w, &d1, &w.url, "a").await.unwrap();
    register(&w, &d2, &w.url, "a").await.unwrap();
    party.take();
    // The update to "a" and the expiry notice to "b" both fail.
    party.answer(&d1, &[503, 503]);
    put(&s, &alice, &d1, "y").await;
    party.wait_for(2).await;
    eventually("the notice to be kept", || {
        kept(&s, &alice.0).notices.len() == 1
    })
    .await;
    idle(&s).await;
    // Deleted: every registration told, the lapsed too, at once; none kept.
    party.answer(&d1, &[503]);
    delete_space(&s, &alice.1, &d1).await;
    let k = kept(&s, &alice.0);
    assert!(!k.registered.contains_key(&d1));
    assert!(writers(&s, &alice.0, &d1).is_empty());
    assert_eq!(k.notices.len(), 2);
    assert!(
        k.notices
            .values()
            .all(|n| n.owed == Owed::Deleted && n.at.n == 0)
    );
    assert_eq!(run(&s).await, 2);
    let got = party.take();
    assert_eq!(got.len(), 2);
    for g in &got {
        assert_eq!(
            to_syncer(g, NOTIFY_SPACE_UPDATE)["update"],
            json!({ "$type": SPACE_DELETED })
        );
    }
    let k = kept(&s, &alice.0);
    assert_eq!(k.notices.len(), 1);
    assert!(k.notices.values().all(|n| n.at.n == 1));
    // Deleted again: nothing.
    delete_space(&s, &alice.1, &d1).await;
    assert_eq!(run(&s).await, 0);
    assert_eq!(kept(&s, &alice.0), k);
    // The other space's notices fail too; then it is created again at once.
    party.answer(&d2, &[503, 503]);
    delete_space(&s, &alice.1, &d2).await;
    assert_eq!(run(&s).await, 2);
    assert_eq!(party.take().len(), 2);
    create_space(&s, &alice.1, "d2").await;
    // Across a restart.
    let before = kept(&s, &alice.0);
    assert_eq!(before.notices.len(), 3);
    drop(s);
    drop(w.s);
    let s = open(&w.dir, &w.plc_url);
    assert_eq!(kept(&s, &alice.0), before);
    // When due: d1's delivered, d2's dropped unsent as it is live again.
    advance_to(&s, before.notices.values().map(|n| n.at.due).max().unwrap());
    assert_eq!(run(&s).await, 1);
    let got = party.take();
    assert_eq!(
        got.iter()
            .map(|g| (g.body["space"].clone(), g.body["update"].clone()))
            .collect::<Vec<_>>(),
        [(json!(d1), json!({ "$type": SPACE_DELETED }))]
    );
    assert!(kept(&s, &alice.0).notices.is_empty());
    // A space created again starts with no registration and no writer set:
    // a write is sent to no one, and authorizeSpaceSync has no spaceRev.
    create_space(&s, &alice.1, "d1").await;
    let fresh = authorized(
        &s,
        json!({ "space": d1, "credentialKey": client(1).kid, "delegationToken": delegation_token(&s, &alice.1, &d1).await.unwrap() }),
    )
    .await
    .unwrap();
    assert!(fresh.get("spaceRev").is_none());
    put(&s, &alice, &d1, "a").await;
    eventually("alice's write to be tracked", || {
        writers(&s, &alice.0, &d1).contains_key(&alice.0)
    })
    .await;
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert!(party.take().is_empty());
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// The limit on registrations: 64 active in a space; a 65th refused with
/// nothing sent, while one registered already is renewed; one dropped by a
/// refusal makes room; lapsed ones count for nothing.
#[tokio::test]
async fn registration_limits() {
    let w = world("notify-limits").await;
    let (s, party, alice) = (&w.s, &w.party, &w.alice.0);
    for i in 0..64 {
        register(&w, &w.space, &w.url, &format!("t{i}"))
            .await
            .unwrap();
    }
    assert_eq!(party.take().len(), 64);
    let many = Err("400 TooManyRegistrations".to_string());
    assert_eq!(register(&w, &w.space, &w.url, "t64").await, many);
    assert!(party.take().is_empty());
    register(&w, &w.space, &w.url, "t0").await.unwrap();
    assert!(party.take().is_empty());
    // Another space has room of its own.
    let other = create_space(s, &w.alice.1, "other").await;
    register(&w, &other, &w.url, "t64").await.unwrap();
    party.take();
    // A non-member's refusal drops one: a new one is confirmed and taken.
    let dave = create(s, "dave").await;
    assert_eq!(
        register_as(s, &dave.1, &w.space, &w.url, "t1").await,
        Err("400 SpaceNotFound".into())
    );
    register(&w, &w.space, &w.url, "t64").await.unwrap();
    assert_eq!(party.take().len(), 1);
    assert_eq!(register(&w, &w.space, &w.url, "t65").await, many);
    // Lapsed, they count for nothing; and are not dropped for lapsing.
    lapse(s);
    register(&w, &w.space, &w.url, "t65").await.unwrap();
    assert_eq!(kept(s, alice).registered[&w.space].len(), 65);
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// A place among the updates waiting is had while fewer than 1024 wait.
#[test]
fn updates_waiting_are_bounded() {
    assert_eq!(UPDATES_WAITING, 1024);
    let waiting = Waiting::new(UPDATES_WAITING);
    let mut places: Vec<Place> = (0..UPDATES_WAITING)
        .map(|_| waiting.enter().expect("a place"))
        .collect();
    assert!(waiting.enter().is_none());
    places.pop();
    let again = waiting.enter().expect("a place given back");
    assert!(waiting.enter().is_none());
    drop(again);
    drop(places);
    assert_eq!(
        (0..UPDATES_WAITING).filter_map(|_| waiting.enter()).count(),
        UPDATES_WAITING
    );
}

/// Endpoints (`spec/SPACES-SYNC.md`, "Environment"): one with a path and a
/// query is sent updates at its origin. With 1024 updates waiting, one
/// more is given up at once.
#[tokio::test]
async fn updates_go_to_origins_and_are_bounded() {
    let w = world("notify-origins").await;
    let (s, party) = (&w.s, &w.party);
    let based = format!("{}/base/more?x=1#f", w.url);
    register(&w, &w.space, &based, "t").await.unwrap();
    party.take();
    let t = s.space_notify.now();
    write_at(&w, t, 1).await;
    let got = party.wait_for(1).await;
    assert_eq!(got[0].path, format!("/xrpc/{NOTIFY_SPACE_UPDATE}"));
    // 1024 waiting: the next is given up at once, not sent later.
    let held: Vec<Place> = (0..UPDATES_WAITING)
        .map(|_| s.space_notify.updates.enter().unwrap())
        .collect();
    write_at(&w, t + 1, 2).await;
    tokio::time::sleep(Duration::from_millis(200)).await;
    drop(held);
    tokio::time::sleep(Duration::from_millis(200)).await;
    assert!(party.take().is_empty());
    write_at(&w, t + 2, 3).await;
    assert_eq!(
        party.wait_for(1).await[0].body["update"]["repoRev"],
        json!(tid_at(t + 2))
    );
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// What a `did:web`'s host answers for its document.
enum WebDoc {
    /// This document, padded to so many bytes if not 0.
    Doc(Json, usize),
    /// A `302` to this URL.
    Moved(String),
}

/// A `did:web` on `localhost` whose host answers as `make` says for the
/// DID; the DID, its port, and how often its document was asked for.
async fn web_did(make: impl FnOnce(&str) -> WebDoc) -> (String, u16, Arc<AtomicUsize>) {
    let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
        .await
        .unwrap();
    let port = listener.local_addr().unwrap().port();
    let did = format!("did:web:localhost%3A{port}");
    let (status, location, body) = match make(&did) {
        WebDoc::Doc(mut doc, size) => {
            if size > 0 {
                doc["pad"] = json!("");
                let n = doc.to_string().len();
                doc["pad"] = json!("x".repeat(size - n));
            }
            (StatusCode::OK, None, doc.to_string())
        }
        WebDoc::Moved(to) => (StatusCode::FOUND, Some(to), String::new()),
    };
    let asked = Arc::new(AtomicUsize::new(0));
    let counted = asked.clone();
    let app = Router::new().route(
        "/.well-known/did.json",
        get(move || async move {
            counted.fetch_add(1, Ordering::SeqCst);
            let mut res =
                (status, [(header::CONTENT_TYPE, "application/json")], body).into_response();
            if let Some(to) = location {
                res.headers_mut()
                    .insert(header::LOCATION, HeaderValue::from_str(&to).unwrap());
            }
            res
        }),
    );
    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
    (did, port, asked)
}

/// A `did:web`'s document with its `#atproto` key.
fn web_doc(did: &str, key: &SigningKey) -> Json {
    json!({
        "id": did,
        "verificationMethod": [{ "id": format!("{did}#atproto"), "type": "Multikey", "controller": did, "publicKeyMultibase": crate::plc::did_key(key.verifying_key()).trim_start_matches("did:key:") }],
    })
}

/// With SSRF protection off, a `did:web`'s document is read from its host
/// over `http`, the issuer of service auth's: up to 65536 bytes; a
/// redirect, never followed, and a longer one cannot be read.
#[tokio::test]
async fn did_web_documents() {
    let w = world("notify-did-web").await;
    let (s, alice) = (&w.s, &w.alice.0);
    let key = SigningKey::from_slice(&[11; 32]).unwrap();
    let max = crate::outbound::DID_DOC_MAX;
    let (served, _, _) = web_did(|did| WebDoc::Doc(web_doc(did, &key), 0)).await;
    let (edge, _, _) = web_did(|did| WebDoc::Doc(web_doc(did, &key), max)).await;
    let (big, _, _) = web_did(|did| WebDoc::Doc(web_doc(did, &key), max + 1)).await;
    let (_, port, target_asked) = web_did(|did| WebDoc::Doc(web_doc(did, &key), 0)).await;
    let (moved, _, moved_asked) =
        web_did(|_| WebDoc::Moved(format!("http://localhost:{port}/.well-known/did.json"))).await;
    let t = s.space_notify.now();
    for (i, (did, read)) in [
        (&served, true),
        (&edge, true),
        (&big, false),
        (&moved, false),
    ]
    .into_iter()
    .enumerate()
    {
        // Read, its service auth verifies (and the writer is refused later); else none can.
        let ok = token(&claims(s, did, alice), signer(&key));
        let want = if read {
            "403 Forbidden"
        } else {
            "401 AuthenticationRequired"
        };
        assert_eq!(
            notify(
                s,
                Some(&ok),
                &note(&w.space, did, &tid_at(t + i as u64), &[1; 32])
            )
            .await,
            Err(want.into()),
            "{did}"
        );
    }
    assert!(moved_asked.load(Ordering::SeqCst) > 0);
    assert_eq!(
        target_asked.load(Ordering::SeqCst),
        0,
        "a redirect is not followed"
    );
    let _ = std::fs::remove_dir_all(&w.dir);
}

/// With SSRF protection on, no `did:web` on `localhost` or an IP address is
/// read, the issuer's of service auth or of a delegation token; and no
/// syncer's endpoint there is asked to confirm. No connection is made.
#[tokio::test]
async fn through_the_guard() {
    let (_plc, plc_url) = Plc::start().await;
    let dir = scratch("notify-guard");
    let s: Shared = Arc::new(
        Service::open(
            &dir,
            Config {
                ssrf_protection: true,
                ..config(&plc_url)
            },
        )
        .unwrap(),
    );
    let alice = create(&s, "alice").await;
    let sp = create_space(&s, &alice.1, "m").await;
    let (port, connections) = counting().await;
    let key = SigningKey::from_slice(&[12; 32]).unwrap();
    let now = s.sessions.now();
    for did in [
        format!("did:web:localhost%3A{port}"),
        format!("did:web:127.0.0.1%3A{port}"),
    ] {
        let write = token(&claims(&s, &did, &alice.0), signer(&key));
        let got = notify(
            &s,
            Some(&write),
            &note(&sp, &did, &tid_at(s.space_notify.now()), &[1; 32]),
        )
        .await;
        assert_eq!(got, Err("401 AuthenticationRequired".into()), "{did}");
        let revoke = token(
            &json!({ "iss": did, "aud": alice.0, "lxm": REVOKE, "iat": now, "exp": now + 60 }),
            signer(&key),
        );
        let body = json!({ "space": space(&did, "x"), "credentials": ["j"] }).to_string();
        assert_eq!(
            send(&s, Http::POST, REVOKE, &[], &authorization(&revoke), &body).await,
            Err("401 AuthenticationRequired".into()),
            "{did}"
        );
        let header =
            json!({ "alg": "ES256K", "typ": "atproto-space-delegation+jwt", "kid": "#atproto" });
        let payload = json!({ "iss": did, "sub": sp, "aud": format!("{}#atproto_pds", alice.0), "iat": now, "exp": now + 60, "jti": "j" });
        let delegated = jwt_of(&header, &payload, signer(&key));
        let got = credential(
            &s,
            json!({ "space": sp, "credentialKey": client(1).kid, "delegationToken": delegated }),
        )
        .await;
        assert_eq!(got, Err("400 InvalidDelegationToken".into()), "{did}");
    }
    for endpoint in [
        format!("http://localhost:{port}"),
        format!("https://localhost:{port}"),
        format!("https://127.0.0.1:{port}"),
    ] {
        let t = delegation_token(&s, &alice.1, &sp).await.unwrap();
        let got = authorized(
            &s,
            json!({ "space": sp, "credentialKey": client(1).kid, "delegationToken": t, "notify": { "endpoint": endpoint, "token": "t" } }),
        )
        .await;
        assert_eq!(
            got,
            Err("400 RegistrationNotConfirmed".into()),
            "{endpoint}"
        );
    }
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert_eq!(connections.load(Ordering::SeqCst), 0);
    let _ = std::fs::remove_dir_all(&dir);
}

/// A call as `send` makes it, from client address `client`: its answer,
/// and whether it paid its rate limits.
async fn paying(
    s: &Shared,
    client: &str,
    nsid: &str,
    headers: &HeaderMap,
    body: &str,
) -> (Result<(), String>, bool) {
    let authorization = crate::authorization(headers);
    let bearer = crate::sessions::Bearer::of(authorization.as_deref());
    let call = Call {
        nsid,
        method: &Http::POST,
        params: &[],
        body: body.as_bytes(),
        bearer,
        headers,
        client: client.into(),
        charged: Default::default(),
        http: None,
    };
    let out = crate::answer(s, &call)
        .await
        .map(|_| ())
        .map_err(|e| format!("{} {}", e.status, e.name));
    (out, call.charged.get().is_some())
}

/// Service auth pays its rate limits once the checks before its issuer's
/// document is read have passed, and before it is read: what is refused
/// before costs nothing, and a call over its limit is `429` with nothing
/// fetched (`spec/LIMITS.md`, "Rate limits").
#[tokio::test]
async fn service_auth_pays_before_fetching() {
    let (_plc, plc_url) = Plc::start().await;
    let dir = scratch("notify-pays");
    let s = open_with(&dir, &plc_url, |c| c.rate_limits = true);
    let (port, connections) = counting().await;
    let iss = format!("did:web:localhost%3A{port}");
    let key = SigningKey::from_slice(&[13; 32]).unwrap();
    let now = s.sessions.now();
    let auth = |iss: &str, lxm: &str| {
        authorization(&token(
            &json!({ "iss": iss, "aud": HOST, "lxm": lxm, "iat": now, "exp": now + 60 }),
            signer(&key),
        ))
    };
    let write = note(
        &space(HOST, "x"),
        &iss,
        &tid_at(s.space_notify.now()),
        &[1; 32],
    )
    .to_string();
    let (a, b) = ("198.51.100.1", "198.51.100.2");
    // Refused before the issuer's document is read: nothing paid.
    for (h, want) in [
        (HeaderMap::new(), "401 MissingJwt"),
        (authorization("Bearer abc"), "401 BadJwt"),
        (auth(&iss, REVOKE), "401 BadJwtLexiconMethod"),
        (auth("localhost", NOTIFY_REPO_UPDATE), "401 BadJwtIss"),
    ] {
        assert_eq!(
            paying(&s, a, NOTIFY_REPO_UPDATE, &h, &write).await,
            (Err(want.into()), false),
            "{want}"
        );
    }
    assert_eq!(connections.load(Ordering::SeqCst), 0);
    // Past them it pays, then reads the document.
    assert_eq!(
        paying(
            &s,
            a,
            NOTIFY_REPO_UPDATE,
            &auth(&iss, NOTIFY_REPO_UPDATE),
            &write
        )
        .await,
        (Err("401 AuthenticationRequired".into()), true)
    );
    eventually("the issuer's host to be asked", || {
        connections.load(Ordering::SeqCst) > 0
    })
    .await;
    let before = connections.load(Ordering::SeqCst);
    // Another client, its global limit spent: 429, nothing fetched.
    let method = Http::GET;
    let none = HeaderMap::new();
    for _ in 0..3000 {
        let call = Call {
            nsid: "com.atproto.server.describeServer",
            method: &method,
            params: &[],
            body: b"",
            bearer: crate::sessions::Bearer::None,
            headers: &none,
            client: b.into(),
            charged: Default::default(),
            http: None,
        };
        assert!(crate::limits::charge(&s, &call, None, None).is_ok());
    }
    let revoke = json!({ "space": space(HOST, "x"), "credentials": ["j"] }).to_string();
    for (nsid, body) in [(NOTIFY_REPO_UPDATE, &write), (REVOKE, &revoke)] {
        assert_eq!(
            paying(&s, b, nsid, &auth(&iss, nsid), body).await,
            (Err("429 RateLimitExceeded".into()), true),
            "{nsid}"
        );
    }
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert_eq!(connections.load(Ordering::SeqCst), before);
    let _ = std::fs::remove_dir_all(&dir);
}

/// The two methods are tools: a read, and one that only adds
/// (`spec/COMMANDS.md`, "Hints").
#[test]
fn tools() {
    for (nsid, tool, hints) in [
        (
            NOTIFY_REPO_UPDATE,
            "space_notify_repo_update",
            json!({ "readOnlyHint": false, "destructiveHint": false }),
        ),
        (
            LIST_WRITERS,
            "space_list_writers",
            json!({ "readOnlyHint": true }),
        ),
    ] {
        let m = pds_lexicon::method(nsid).unwrap();
        assert_eq!(m.tool().as_deref(), Some(tool));
        assert_eq!(m.annotations(), Some(hints), "{nsid}");
        assert!(handler(nsid).is_some());
    }
}

/// Each kind of entry this part keeps, a frame each, as the byte-cut test
/// in `log` writes them into `did`'s log: a writer's queue in `q`, an
/// authority's registrations and notices in `m`.
pub(crate) fn frames_of_each_kind(q: &str, m: &str) -> Vec<Vec<Json>> {
    let at = |n: u32| Schedule {
        n,
        due: 1_000 + u64::from(n),
        until: 2_000_000,
    };
    let (r1, r2) = ("3jzfcijpj2z2a", "3jzfcijpj2z2b");
    let a = ("https://a.example".to_string(), "ta".to_string());
    let b = ("https://b.example/".to_string(), "tb".to_string());
    vec![
        vec![queued(q, r1, &[1; 32], at(1))],
        vec![queued(q, r2, &[2; 32], at(2))],
        vec![unqueued(q, r2)],
        vec![registered(m, (a.0.as_str(), a.1.as_str()), 5_000)],
        vec![registered(m, (b.0.as_str(), b.1.as_str()), 6_000)],
        vec![noticed(m, &a)],
        vec![notice(
            m,
            &a,
            &Owed::Expired {
                space_rev: r1.into(),
                lapse: 5_000,
            },
            at(1),
        )],
        vec![dropped(m, &a), notice_done(m, &a)],
        vec![
            json!({ "t": EMPTIED, "space": m }),
            notice(m, &b, &Owed::Deleted, at(0)),
        ],
        vec![notice(m, &b, &Owed::Deleted, at(1))],
    ]
}

/// Log a frame of entries into `did`'s log, as this part does.
pub(crate) fn keep(s: &Service, did: &str, entries: &[Json]) {
    log(&mut write(&s.account(did).unwrap().store), entries);
}