~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
2801 lines · 140.9 KB Raw History
//! The rest of computer.socialweb.test.*: tenant clusters, the k8s and ssh web pages (OAuth by headless Chrome, a
//! cookie the host keeps), the ssh gateway's temporary key, an atirc observer token, the orchestrator's ssh / restart /
//! cluster / bundle / records / events, the page round (0009), root on a staging VM. Secrets (cookies, tokens,
//! passwords) stay in the host.

use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Mutex;
use std::time::{Duration, Instant};

use atlibs_host::run::{Sub, SubCmd};
use atlibs_kube::Cluster;
use atlibs_ssh::Dest;
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use testkit::requester::{exec_line, kept_by_rule, oauth_signin_on, tail, uris_in};
use testkit::target::{check_private, expand};
use tokio::io::AsyncBufReadExt;
use tokio::sync::mpsc;

use super::{Swc, abs_dir_of, exec_words, str_of};

#[derive(Default)]
pub struct Extra {
    /// origin -> Cookie header of a signed-in page session
    jars: Mutex<HashMap<String, String>>,
    /// (ssh web origin, rkey): temporary keys to remove
    keys: Mutex<Vec<(String, String)>>,
    /// observer nick -> its atirc token
    tokens: Mutex<HashMap<String, String>>,
    /// requester name -> its run requester argv (restart)
    pub argv: Mutex<HashMap<String, Vec<String>>>,
    /// owner mesh nodes this run started (`atirc mesh run`): name -> (atirc, its config dir, the process)
    pub nodes: tokio::sync::Mutex<HashMap<String, (PathBuf, PathBuf, tokio::process::Child)>>,
    /// knotless cleanups: "sh <script>" and "record <at-uri>", in the order made
    pub knotless: Mutex<Vec<String>>,
    /// bidder name -> the start (epoch s) of the run bidder process this run restarted first: its log is read from there, so counts
    /// taken before and after a restart are of one window (the journal read starts at the running process otherwise)
    log_since: Mutex<HashMap<String, u64>>,
    /// bidders this run upgraded to a `from` binary (bidder.upgrade), each with the from spec it was given: the cleanup upgrades
    /// each back to the binary under test
    upgraded: Mutex<Vec<(String, String)>>,
}

fn origin(url: &str) -> Result<String, String> {
    let u = reqwest::Url::parse(url).map_err(|e| e.to_string())?;
    Ok(u.origin().ascii_serialization())
}

const STAGING_PKI_MOUNT: &str = "staging-pki";
/// atirc's refusal of a revoked token (error `Unauthorized`, message `atirc token revoked`) as the CLI prints it on the
/// transport the token names: `pub` and `revoke` go to the token's `endpoint` claim, not to ATIRC_ENDPOINT, and the server
/// mints its iroh ticket there when it runs iroh (prod's `atirc-iroh.service`), its http(s) URL otherwise (staging's
/// `atirc serve`). http(s): atproto-client's `Not authorized: {error} {message}`; iroh: atirc's `{error}: {message}`.
const ATIRC_REVOKED_HTTP: &str = "Not authorized: Unauthorized atirc token revoked";
const ATIRC_REVOKED_IROH: &str = "Unauthorized: atirc token revoked";

fn atirc_revoked(token: &str) -> Result<&'static str, String> {
    let payload = token.split('.').nth(1).ok_or("the atirc token is not a JWT")?;
    let (mut bytes, mut acc, mut bits) = (Vec::new(), 0u32, 0u32);
    for c in payload.bytes() {
        let v = match c {
            b'A'..=b'Z' => c - b'A',
            b'a'..=b'z' => c - b'a' + 26,
            b'0'..=b'9' => c - b'0' + 52,
            b'-' => 62,
            b'_' => 63,
            _ => return Err("the atirc token's payload is not base64url".into()),
        };
        acc = (acc << 6) | u32::from(v);
        bits += 6;
        if bits >= 8 {
            bits -= 8;
            bytes.push((acc >> bits) as u8);
            acc &= (1 << bits) - 1;
        }
    }
    let claims: Value = serde_json::from_slice(&bytes).map_err(|e| format!("the atirc token's payload: {e}"))?;
    let endpoint = claims["endpoint"].as_str().ok_or("the atirc token names no endpoint")?;
    let iroh = endpoint
        .strip_prefix("endpoint")
        .is_some_and(|t| !t.is_empty() && t.chars().all(|c| c.is_ascii_lowercase() || ('2'..='7').contains(&c)));
    Ok(if iroh { ATIRC_REVOKED_IROH } else { ATIRC_REVOKED_HTTP })
}

/// Chrome takes no CA file: it accepts a certificate by its public key (`--ignore-certificate-errors-spki-list`). The
/// leaf the network's front presents for each name, its chain and name verified against the network CA by openssl first,
/// as the sha256 of its SubjectPublicKeyInfo, base64; comma-separated, empty without names.
async fn front_spki(front: std::net::SocketAddr, ca: &std::path::Path, names: &[String]) -> Result<String, String> {
    const SPKI: &str = "set -euo pipefail; front=$1 ca=$2; shift 2; for n in \"$@\"; do \
        openssl s_client -connect \"$front\" -servername \"$n\" -CAfile \"$ca\" -verify_return_error -verify_hostname \"$n\" </dev/null 2>/dev/null \
        | openssl x509 -pubkey -noout | openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 -A; echo; done | sort -u | paste -sd, -";
    if names.is_empty() {
        return Ok(String::new());
    }
    let out = tokio::process::Command::new("bash")
        .args(["-c", SPKI, "front-spki", &front.to_string()])
        .arg(ca)
        .args(names)
        .stdin(Stdio::null())
        .output()
        .await
        .map_err(|e| format!("openssl: {e}"))?;
    let spki = String::from_utf8_lossy(&out.stdout).trim().to_string();
    if !out.status.success() || spki.is_empty() {
        return Err(format!(
            "the front {front} presented no certificate the network CA {} verifies for {names:?}: {}",
            ca.display(),
            String::from_utf8_lossy(&out.stderr).trim()
        ));
    }
    Ok(spki)
}

pub(super) fn write_private(path: &PathBuf, text: &str) -> Result<(), String> {
    use std::io::Write;
    use std::os::unix::fs::OpenOptionsExt;
    let mut f = std::fs::OpenOptions::new()
        .create(true)
        .truncate(true)
        .write(true)
        .mode(0o600)
        .open(path)
        .map_err(|e| e.to_string())?;
    f.write_all(text.as_bytes()).map_err(|e| e.to_string())
}

fn find_named(dir: &std::path::Path, name: &str, out: &mut Vec<PathBuf>) {
    for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
        let p = e.path();
        if p.is_dir() {
            find_named(&p, name, out);
        } else if p.file_name().is_some_and(|n| n == name) {
            out.push(p);
        }
    }
}

/// A subscription of the JSON log lines `keep` takes, from the files `logs()` names now (new ones appear), and the
/// child's exit as {msg: "exited", code}.
pub fn tail_sub(
    logs: impl Fn() -> Vec<PathBuf> + Send + 'static,
    mut child: Option<tokio::process::Child>,
    keep: fn(&str) -> bool,
) -> Sub {
    let (tx, messages) = mpsc::unbounded_channel();
    let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
    tokio::spawn(async move {
        let mut offsets: HashMap<PathBuf, usize> = HashMap::new();
        // the same line can land in two files (the run's dev core.log is copied to logs/core.log when it ends)
        let mut seen = std::collections::HashSet::new();
        loop {
            for p in logs() {
                let Ok(text) = std::fs::read_to_string(&p) else {
                    continue;
                };
                let off = offsets.entry(p.clone()).or_default();
                if text.len() < *off {
                    *off = 0;
                }
                let fresh = &text[*off..];
                let Some(end) = fresh.rfind('\n') else {
                    continue;
                };
                for line in fresh[..=end].lines() {
                    let Ok(v) = serde_json::from_str::<Value>(line) else {
                        continue;
                    };
                    if !seen.insert(line.to_string()) {
                        continue;
                    }
                    if keep(v["msg"].as_str().unwrap_or_default()) && tx.send(Ok(v)).is_err() {
                        return;
                    }
                }
                *off += end + 1;
            }
            if let Some(c) = child.as_mut()
                && let Ok(Some(st)) = c.try_wait()
            {
                let _ = tx.send(Ok(json!({ "msg": "exited", "code": st.code() })));
                child = None;
            }
            tokio::select! {
                cmd = cmds.recv() => match cmd {
                    Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("this subscription takes no sends".into())); }
                    Some(SubCmd::Close) | None => return,
                },
                _ = tokio::time::sleep(Duration::from_secs(1)) => {}
            }
        }
    });
    Sub { messages, commands }
}

pub fn market_lines(msg: &str) -> bool {
    msg == "contract"
        || msg == "ssh returned"
        || msg.starts_with("requester_")
        || msg.starts_with("guest_")
        || msg.starts_with("infra_")
}

impl Swc {
    /// A market network target: `run requester --network <instance>` finds the instance under its `--state-dir`, so the
    /// requester's state is the network's (as `run request`'s, request_state); otherwise this run's own dir.
    pub(crate) fn requester_state(&self, name: &str) -> PathBuf {
        match (&self.target.network, self.associated()) {
            (Some(n), _) => expand(&n.state_dir),
            (None, Some(id)) => expand(&id.state_dir),
            (None, None) => self.dir.join(format!("requester-{name}")),
        }
    }

    /// plan 0046: the account's associated host requester identity, outside a market network
    pub(crate) fn associated(&self) -> Option<&testkit::target::AssociatedIdentity> {
        if self.target.network.is_some() {
            return None;
        }
        self.target
            .account(&self.account)
            .ok()
            .and_then(|a| a.identity.as_ref())
    }

    fn atirc(&self) -> Result<PathBuf, String> {
        testkit::atirc::atirc_bin().map_err(|e| e.to_string())
    }

    /// scripts/oauth-signin.ts as the run's account. On a network of its own (the target has `tools` and an XRPC front) it
    /// runs with the suite's pinned deno and Chrome, and Chrome accepts the leaf the network's names present (the sign-in
    /// origin, the PLC, the PDS, the relay: each verified against the network CA first), since Chrome takes no CA file.
    async fn signin(&self, url: &str, extra: &[&str]) -> Result<(), String> {
        let a = self.acct()?;
        let mut args: Vec<String> = extra.iter().map(|s| s.to_string()).collect();
        let mut deno = PathBuf::from("deno");
        if let Some(tools) = self.target.tools.as_deref().map(expand) {
            deno = tools.join("deno/deno");
            args.extend(["--chrome".into(), tools.join("chrome/chrome").display().to_string()]);
        }
        let mut ca = None;
        if let Some(f) = self.target.xrpc_front.as_ref().filter(|_| self.target.name != "prod") {
            let bundle = expand(&f.ca);
            let names: Vec<String> = [Some(&self.target.plc), Some(&self.target.pds), self.target.relay.as_ref()]
                .into_iter()
                .flatten()
                .filter_map(|u| reqwest::Url::parse(u).ok())
                .filter(|u| u.scheme() == "https")
                .filter_map(|u| u.host_str().map(String::from))
                .collect();
            let mut keys: Vec<String> = front_spki(f.address, &bundle, &names).await?.split(',').filter(|k| !k.is_empty()).map(String::from).collect();
            let origin = reqwest::Url::parse(url).map_err(|e| format!("the sign-in url: {e}"))?;
            if let (Some(host), Some(port)) = (origin.host_str(), origin.port_or_known_default()) {
                let at = tokio::net::lookup_host((host, port))
                    .await
                    .map_err(|e| format!("{host}: {e}"))?
                    .next()
                    .ok_or_else(|| format!("{host} resolves to no address"))?;
                for k in front_spki(at, &bundle, &[host.to_string()]).await?.split(',') {
                    if !k.is_empty() && !keys.iter().any(|h| h == k) {
                        keys.push(k.to_string());
                    }
                }
            }
            if !keys.is_empty() {
                args.push(format!("--chrome-arg=--ignore-certificate-errors-spki-list={}", keys.join(",")));
            }
            ca = Some(bundle);
        }
        let refs: Vec<&str> = args.iter().map(String::as_str).collect();
        oauth_signin_on(&deno, ca.as_deref(), a, url, &refs).await.map_err(|e| e.to_string())
    }

    async fn page_signin(&self, input: &Value) -> Result<Value, String> {
        let url = input["url"].as_str().ok_or("url is required")?;
        let jar = self.dir.join(format!("jar-{}.json", atlibs_host::now_ms()));
        let jar_s = jar.display().to_string();
        let mut extra: Vec<&str> = vec!["--cookie-file", &jar_s];
        if input["startApi"] == true {
            extra.push("--start-api");
        }
        let done = input["done"].as_str().map(String::from);
        if let Some(d) = &done {
            extra.extend(["--done", d.as_str()]);
        }
        let r = self.signin(url, &extra).await;
        let cookies: Value = std::fs::read(&jar)
            .ok()
            .and_then(|b| serde_json::from_slice(&b).ok())
            .unwrap_or_default();
        let _ = std::fs::remove_file(&jar);
        let header = cookies
            .as_array()
            .into_iter()
            .flatten()
            .map(|c| {
                format!(
                    "{}={}",
                    c["name"].as_str().unwrap_or_default(),
                    c["value"].as_str().unwrap_or_default()
                )
            })
            .collect::<Vec<_>>()
            .join("; ");
        if header.is_empty() {
            return Err(format!(
                "signing in at {url}: no session cookie ({})",
                r.err().map(|e| e.to_string()).unwrap_or_default()
            ));
        }
        self.extra.jars.lock().unwrap().insert(origin(url)?, header);
        Ok(json!({ "ok": true }))
    }

    /// A request to a signed-in page: its cookie, and Origin = the page's (a cookie POST needs it).
    async fn page_fetch(
        &self,
        url: &str,
        method: &str,
        body: Option<&Value>,
    ) -> Result<Value, String> {
        let o = origin(url)?;
        let cookie = self
            .extra
            .jars
            .lock()
            .unwrap()
            .get(&o)
            .cloned()
            .ok_or_else(|| format!("{o}: not signed in (page.signin first)"))?;
        let mut req = self
            .http
            .request(method.parse().map_err(|_| "bad method")?, url)
            .header("cookie", cookie)
            .header("origin", &o);
        if let Some(b) = body {
            req = req.json(b);
        }
        let r = req.send().await.map_err(|e| e.to_string())?;
        let status = r.status().as_u16();
        let text = r.text().await.unwrap_or_default();
        Ok(
            json!({ "status": status, "body": serde_json::from_str::<Value>(&text).unwrap_or(Value::String(text)) }),
        )
    }

    async fn temp_key(&self) -> Result<Value, String> {
        let host = self.target.gateway.ssh.clone();
        let web = self.target.gateway.ssh_web();
        let key = self.dir.join("st_gateway_key");
        if !key.exists() {
            let ok = tokio::process::Command::new("ssh-keygen")
                .args(["-q", "-t", "ed25519", "-N", "", "-C", "swc-test-host", "-f"])
                .arg(&key)
                .stdout(Stdio::null())
                .status()
                .await
                .map_err(|e| e.to_string())?
                .success();
            if !ok {
                return Err("ssh-keygen failed".into());
            }
        }
        let public = std::fs::read_to_string(key.with_extension("pub"))
            .map_err(|e| e.to_string())?
            .trim()
            .to_string();
        let added = self
            .page_fetch(
                &format!("{web}/keys"),
                "POST",
                Some(&json!({ "publicKey": public })),
            )
            .await?;
        let rkey = added["body"]["rkey"]
            .as_str()
            .ok_or_else(|| format!("POST /keys: {}", added))?
            .to_string();
        self.extra.keys.lock().unwrap().push((web, rkey));
        let dest = Dest {
            dest: format!("{}@{host}", self.acct()?.handle),
            port: Some(self.target.gateway.ssh_port.unwrap_or(22)),
            key: Some(key),
            known_hosts: Some(self.dir.join("known_hosts")),
            user_config: false,
        };
        self.ssh.add_dest("gateway", dest);
        if let Some(cfg) = &self.target.agent_config {
            let p = expand(cfg);
            check_private(&p).map_err(|e| e.to_string())?;
            self.ssh.add_stdin_file("agentConfig", p);
        }
        Ok(json!({ "dest": "gateway" }))
    }

    /// `atirc <args>` as the observer: {ok, refused, error}. refused: the server answered that the token is revoked
    /// (`atirc_revoked`: atirc's text for the transport the token names), the one failure that is the refusal; error: its last
    /// stderr line.
    async fn observer_run(&self, nick: &str, args: &[&str]) -> Result<Value, String> {
        let token = self
            .extra
            .tokens
            .lock()
            .unwrap()
            .get(nick)
            .cloned()
            .ok_or_else(|| format!("no observer {nick}"))?;
        let endpoint = self.target.atirc.clone().ok_or("the target has no atirc")?;
        let revoked = atirc_revoked(&token)?;
        let out = tokio::process::Command::new(self.atirc()?)
            .args(args)
            .env("ATIRC_TOKEN", token)
            .env("ATIRC_ENDPOINT", endpoint)
            .env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
            .stdin(Stdio::null())
            .stdout(Stdio::null())
            .stderr(Stdio::piped())
            .output()
            .await
            .map_err(|e| e.to_string())?;
        let stderr = String::from_utf8_lossy(&out.stderr);
        let error: String = stderr.lines().last().unwrap_or_default().chars().take(300).collect();
        Ok(json!({ "ok": out.status.success(), "refused": !out.status.success() && stderr.contains(revoked), "error": error }))
    }

    async fn observer_sub(&self, input: &Value) -> Result<Sub, String> {
        let nick = str_of(input, "nick")?.to_string();
        let a = self.acct()?;
        let endpoint = self.target.atirc.clone().ok_or("the target has no atirc")?;
        let out = tokio::process::Command::new(self.atirc()?)
            .args(["token", &nick, "--endpoint", &endpoint])
            .env("ATP_USERNAME", &a.handle)
            .env("ATP_PASSWORD", self.password()?)
            .env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
            .stdin(Stdio::null())
            .output()
            .await
            .map_err(|e| e.to_string())?;
        let token = String::from_utf8_lossy(&out.stdout).trim().to_string();
        if !out.status.success() || token.is_empty() {
            return Err(format!("atirc token {nick} failed"));
        }
        self.extra
            .tokens
            .lock()
            .unwrap()
            .insert(nick.clone(), token.clone());
        // no -c: atirc names the account's channel itself (its default_channel keeps a dash, staging's handles have
        // one); a hand-made name with the dash as `_` reads as another handle's channel and the subscribe is a 403
        let mut child = tokio::process::Command::new(self.atirc()?)
            .args(["sub", "--full", "--cursor", "-1"])
            .env("ATIRC_TOKEN", token)
            .env("ATIRC_ENDPOINT", endpoint)
            .env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
            .stdin(Stdio::null())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .kill_on_drop(true)
            .spawn()
            .map_err(|e| e.to_string())?;
        let stdout = child.stdout.take().unwrap();
        let stderr = child.stderr.take().unwrap();
        let last = std::sync::Arc::new(Mutex::new(String::new()));
        let seen = last.clone();
        let errs = tokio::spawn(async move {
            let mut lines = tokio::io::BufReader::new(stderr).lines();
            while let Ok(Some(l)) = lines.next_line().await {
                *seen.lock().unwrap() = l.chars().take(300).collect();
            }
        });
        let (tx, messages) = mpsc::unbounded_channel();
        let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
        tokio::spawn(async move {
            let mut lines = tokio::io::BufReader::new(stdout).lines();
            loop {
                tokio::select! {
                    l = lines.next_line() => match l {
                        Ok(Some(l)) => if let Ok(v) = serde_json::from_str::<Value>(&l) && tx.send(Ok(v)).is_err() { break },
                        // atirc sub ended by itself: its last stderr line is the cause (a refused subscribe)
                        _ => {
                            let _ = tokio::time::timeout(Duration::from_secs(2), errs).await;
                            let _ = tx.send(Err(format!("atirc sub ended: {}", last.lock().unwrap())));
                            break;
                        }
                    },
                    cmd = cmds.recv() => match cmd {
                        Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("no sends".into())); }
                        Some(SubCmd::Close) | None => break,
                    },
                }
            }
            let _ = child.kill().await;
        });
        Ok(Sub { messages, commands })
    }

    async fn requester_restart(&self, name: &str) -> Result<Value, String> {
        let argv = self
            .extra
            .argv
            .lock()
            .unwrap()
            .get(name)
            .cloned()
            .ok_or_else(|| format!("no requester {name}"))?;
        let state = self.requester_state(name);
        if let Some((_, mut c)) = self.requesters.lock().await.remove(name) {
            if let Some(pid) = c.id() {
                let _ = std::process::Command::new("kill")
                    .args(["-INT", &pid.to_string()])
                    .status();
            }
            if tokio::time::timeout(Duration::from_secs(120), c.wait())
                .await
                .is_err()
            {
                let _ = c.kill().await;
                return Err(format!("run requester {name} did not stop in 120 s"));
            }
        }
        let ready = state.join("requesters").join(name).join("ready.json");
        if ready.exists() {
            return Err("the ready file stayed after Ctrl-C".into());
        }
        let t = Instant::now();
        let log = std::fs::OpenOptions::new()
            .append(true)
            .create(true)
            .open(state.join(format!("{name}.log")))
            .map_err(|e| e.to_string())?;
        let child = self
            .bin
            .cmd()
            .args(&argv)
            .env("ATPROTO_PASSWORD", self.password()?)
            .stdout(log.try_clone().map_err(|e| e.to_string())?)
            .stderr(log)
            .spawn()
            .map_err(|e| e.to_string())?;
        self.requesters
            .lock()
            .await
            .insert(name.into(), (state.clone(), child));
        let deadline = Instant::now() + Duration::from_secs(300);
        loop {
            if let Ok(b) = std::fs::read(&ready)
                && let Ok(v) = serde_json::from_slice::<Value>(&b)
            {
                return Ok(
                    json!({ "did": v["did"], "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }),
                );
            }
            if Instant::now() > deadline {
                return Err(format!(
                    "run requester {name}: not ready 5 min after the restart"
                ));
            }
            tokio::time::sleep(Duration::from_millis(500)).await;
        }
    }

    async fn requester_cli(
        &self,
        verb: &str,
        name: &str,
        extra: &[&str],
    ) -> Result<(bool, String), String> {
        let state = self.requester_state(name).display().to_string();
        let mut args = vec!["run", "requester", verb, name, "--state-dir", &state];
        args.extend(extra);
        self.bin.run(&args).await.map_err(|e| e.to_string())
    }

    /// logs.fetchRuns {name, contract?, dir, to, all?}: the run dirs under `dir` on a guest that hold a failure.json (the failed
    /// compositions of a gate run there; `all`: every dir under `dir`) into this run's dump at `logs/<to>/<run>/`: text files
    /// redacted and cut as the dump's own logs, a ticks.ndjson.zst whole (the dump's own form) while the room lasts, the rest
    /// in `skipped.txt`. The guest is a run requester's contract (`contract`) or an invocation (`name` only).
    async fn fetch_runs(&self, input: &Value) -> Result<Value, String> {
        let (name, dir, to) = (str_of(input, "name")?, abs_dir_of(input, "dir")?, str_of(input, "to")?);
        let to = std::path::Path::new(to).file_name().ok_or("to: a directory name")?.to_string_lossy().to_string();
        let pick = if input["all"] == true { "[ -d \"$d\" ]" } else { "[ -f \"$d\"failure.json ]" };
        let exec = format!(
            "cd '{}' 2>/dev/null || exit 0; echo SWC-LOGS-BEGIN; for d in */; do {pick} && echo \"${{d%/}}\"; done | tar -czf - -T - 2>/dev/null | base64 -w0; echo; echo SWC-LOGS-END",
            dir.replace('\'', "")
        );
        let run = async {
            match input["contract"].as_str() {
                Some(c) => {
                    let line = exec_line(&exec);
                    let mut extra = vec!["--contract", c];
                    extra.extend(line.iter().map(String::as_str));
                    self.requester_cli("ssh", name, &extra).await
                }
                None => self.cli(&[self.sub_args("ssh", name, &[]), exec_line(&exec)].concat()).await,
            }
        };
        let (_, text) = tokio::time::timeout(Duration::from_secs(300), run)
            .await
            .map_err(|_| format!("{name}: the run dirs not read in 5 min"))??;
        let tree = self.unpack_logs(&format!("runs-{to}"), &text).await?;
        let out = self.dump.join("logs").join(&to);
        let mut files = vec![];
        super::walk(&tree, &mut files);
        let (mut room, mut skipped, mut copied) = (super::LOGS_MAX / 3 * 2, vec![], 0);
        let mut runs: Vec<String> = vec![];
        for f in super::by_size(files) {
            let rel = f.strip_prefix(&tree).unwrap_or(&f).to_path_buf();
            if let Some(r) = rel.components().next().map(|c| c.as_os_str().to_string_lossy().to_string())
                && !runs.contains(&r)
            {
                runs.push(r);
            }
            let dest = out.join(&rel);
            if f.extension().is_some_and(|e| e == "zst") {
                let len = f.metadata().map(|m| m.len()).unwrap_or(u64::MAX);
                if len > room {
                    skipped.push(dest.display().to_string());
                    continue;
                }
                room -= len;
                if let Some(d) = dest.parent() {
                    let _ = std::fs::create_dir_all(d);
                }
                if std::fs::copy(&f, &dest).is_ok() {
                    copied += 1;
                }
                continue;
            }
            let before = skipped.len();
            super::copy_redacted(&f, &dest, &mut room, &mut skipped);
            if skipped.len() == before {
                copied += 1;
            }
        }
        let _ = std::fs::remove_dir_all(&tree);
        if !skipped.is_empty() {
            let _ = std::fs::create_dir_all(&out);
            let _ = std::fs::write(out.join("skipped.txt"), skipped.join("\n") + "\n");
        }
        Ok(json!({ "runs": runs, "files": copied, "skipped": skipped.len(), "at": out.display().to_string() }))
    }

    /// plan 0050: the run's root CA and one leaf for the staging network's names, from the run requester's own OpenBao. The
    /// host declares a pki mount and a policy for the requester's client on its OpenBaoServer (the kube patch restores the
    /// spec at the run's cleanup), then issues the leaf as that client. The leaf and its key stay in the run's work dir
    /// (0600, `staging-pki/`), never in a result; the root is public.
    async fn staging_pki(&self, input: &Value) -> Result<Value, String> {
        let name = str_of(input, "name")?;
        let names: Vec<String> = input["names"].as_array().into_iter().flatten().filter_map(|n| n.as_str().map(String::from)).collect();
        let ips: Vec<String> = input["ipSans"].as_array().into_iter().flatten().filter_map(|n| n.as_str().map(String::from)).collect();
        let Some(cn) = names.first().cloned() else { return Err("names: at least one".into()) };
        let cluster = format!("requester:{name}");
        let key = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "OpenBaoServer", "name": "requester" });
        let kube = |nsid: &str, input: Value| {
            atlibs_host::run::Call::call(&*self.kube, nsid.into(), atlibs_host::run::Request { input: Some(input), ..Default::default() })
        };
        let obs = match kube(atlibs_kube::GET, key.clone()).await {
            Ok(v) => v,
            // a hand run's associated host requester (plan 0046) was not started by this host: its cluster from its state
            Err(_) => {
                let kubeconfig = self.requester_state(name).join("requesters").join(name).join("dev").join(name).join("kcp/admin.kubeconfig");
                self.kube.add_cluster(&cluster, Cluster { kubeconfig: kubeconfig.display().to_string(), workspace: None, via: Default::default() });
                kube(atlibs_kube::GET, key.clone()).await?
            }
        }
        .unwrap_or_default();
        let spec = &obs["spec"];
        let mut mounts = spec["declared"]["secretMounts"].as_array().cloned().unwrap_or_default();
        if !mounts.iter().any(|m| m["path"] == STAGING_PKI_MOUNT) {
            mounts.push(json!({ "path": STAGING_PKI_MOUNT, "type": "pki", "maxLeaseTTL": "48h" }));
        }
        let mut policies = spec["declared"]["policies"].as_array().cloned().unwrap_or_default();
        if !policies.iter().any(|p| p["name"] == STAGING_PKI_MOUNT) {
            policies.push(json!({ "name": STAGING_PKI_MOUNT, "rules": [{ "path": format!("{STAGING_PKI_MOUNT}/*"), "capabilities": ["create", "read", "update", "list"] }] }));
        }
        let clients: Vec<Value> = spec["clients"].as_array().cloned().unwrap_or_default().into_iter().map(|mut c| {
            if c["name"] == "requester" {
                let mut p = c["policies"].as_array().cloned().unwrap_or_default();
                if !p.iter().any(|x| x == STAGING_PKI_MOUNT) {
                    p.push(json!(STAGING_PKI_MOUNT));
                }
                c["policies"] = json!(p);
            }
            c
        }).collect();
        let mut patch = key.clone();
        patch["patch"] = json!({ "spec": { "declared": { "secretMounts": mounts, "policies": policies }, "clients": clients } });
        kube(atlibs_kube::PATCH, patch).await?;
        let d = self.requester_state(name).join("requesters").join(name).join("dev").join(name).join("openbaoservers/requester/openbao");
        let runtime = PathBuf::from(std::env::var("HOME").unwrap_or_default()).join(".cache/socialweb-computer/runtime");
        let bao = std::fs::read_dir(&runtime).map_err(|e| e.to_string())?.flatten().map(|e| e.path().join("bao")).find(|p| p.is_file()).ok_or("no bao in the runtime cache")?;
        let bao_cmd = |args: &[&str], token: Option<&str>, addr: &str| {
            let mut c = tokio::process::Command::new(&bao);
            c.args(args)
                .env("BAO_ADDR", addr)
                .env("BAO_CACERT", d.join("tls/ca.crt"))
                .env("BAO_CLIENT_CERT", d.join("clients/requester/client.crt"))
                .env("BAO_CLIENT_KEY", d.join("clients/requester/client.key"))
                .stdin(Stdio::null());
            if let Some(t) = token {
                c.env("BAO_TOKEN", t);
            }
            c
        };
        let deadline = Instant::now() + Duration::from_secs(120);
        let run = self.dir.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
        let mut issuer = String::new();
        let (root, issued) = loop {
            // the patched spec makes the server reconcile: its status (address) is rewritten, so read it each try
            let addr = kube(atlibs_kube::GET, key.clone()).await?.unwrap_or_default()["status"]["address"].as_str().unwrap_or_default().to_string();
            if addr.is_empty() {
                if Instant::now() > deadline {
                    return Err("staging pki: the OpenBaoServer reports no address".into());
                }
                tokio::time::sleep(Duration::from_secs(3)).await;
                continue;
            }
            let token = bao_cmd(&["write", "-field=token", "auth/cert/login", "name=requester"], None, &addr).output().await.map_err(|e| e.to_string())?;
            let token = String::from_utf8_lossy(&token.stdout).trim().to_string();
            // one root per run (plan 0050): a fresh issuer, named for the run; the role issues from it. The leaf outlives
            // the network's renew window (24h before its end: EnsureGuestTLS keeps it) and stays inside the root's life
            if issuer.is_empty() {
                let cn = format!("common_name=socialweb staging root {run}");
                let rotated = bao_cmd(&["write", "-format=json", &format!("{STAGING_PKI_MOUNT}/root/rotate/internal"), &cn, "ttl=47h", "key_type=ec", "key_bits=256"], Some(&token), &addr).output().await.map_err(|e| e.to_string())?;
                if rotated.status.success() {
                    let v: Value = serde_json::from_slice(&rotated.stdout).unwrap_or_default();
                    issuer = v["data"]["issuer_id"].as_str().unwrap_or_default().to_string();
                }
            }
            let role = format!("{STAGING_PKI_MOUNT}/roles/net");
            let issuer_ref = format!("issuer_ref={}", if issuer.is_empty() { "default" } else { issuer.as_str() });
            let made = bao_cmd(&["write", &role, "allow_any_name=true", "allow_subdomains=true", "allow_bare_domains=true", "allow_wildcard_certificates=true", "allow_ip_sans=true", "max_ttl=47h", &issuer_ref], Some(&token), &addr).output().await.map_err(|e| e.to_string())?;
            let root = bao_cmd(&["read", "-field=certificate", &format!("{STAGING_PKI_MOUNT}/issuer/{issuer}")], Some(&token), &addr).output().await.map_err(|e| e.to_string())?;
            let issue_path = format!("{STAGING_PKI_MOUNT}/issue/net");
            let (cn_arg, alt_arg, ip_arg) = (format!("common_name={cn}"), format!("alt_names={}", names.join(",")), format!("ip_sans={}", ips.join(",")));
            let issued = if !issuer.is_empty() && made.status.success() && root.status.success() {
                bao_cmd(&["write", "-format=json", &issue_path, &cn_arg, &alt_arg, &ip_arg, "ttl=46h"], Some(&token), &addr).output().await.map_err(|e| e.to_string())?
            } else {
                made
            };
            if issued.status.success() && String::from_utf8_lossy(&root.stdout).contains("BEGIN CERTIFICATE") {
                break (String::from_utf8_lossy(&root.stdout).trim().to_string(), issued.stdout);
            }
            if Instant::now() > deadline {
                return Err(format!("staging pki: {}", tail(&String::from_utf8_lossy(&issued.stderr))));
            }
            tokio::time::sleep(Duration::from_secs(3)).await;
        };
        let leaf: Value = serde_json::from_slice(&issued).map_err(|e| e.to_string())?;
        let dir = self.dir.join("staging-pki");
        std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
        write_private(&dir.join("root.crt"), &format!("{root}\n"))?;
        write_private(&dir.join("leaf.crt"), &format!("{}\n", leaf["data"]["certificate"].as_str().unwrap_or_default().trim()))?;
        write_private(&dir.join("leaf.key"), &format!("{}\n", leaf["data"]["private_key"].as_str().unwrap_or_default().trim()))?;
        // every staging VM's user_data (the requester adds the session's ssh keys): the root in its system store, and the
        // module's boot commands (its resolver: the network's DNS); a variant (`bootcmdFor.<name>`) adds its own, in
        // user-data-<name> (userDataFor.<name>)
        let user_data = |cmds: &[Value]| {
            let mut ud = String::from("#cloud-config\nca_certs:\n  trusted:\n    - |\n");
            for line in root.lines() {
                ud.push_str("      ");
                ud.push_str(line.trim());
                ud.push('\n');
            }
            if !cmds.is_empty() {
                ud.push_str("bootcmd:\n");
                for c in cmds {
                    ud.push_str(&format!("  - {}\n", serde_json::to_string(c.as_str().unwrap_or_default()).unwrap_or_default()));
                }
            }
            ud
        };
        let common: Vec<Value> = input["bootcmd"].as_array().cloned().unwrap_or_default();
        let ud = user_data(&common);
        write_private(&dir.join("user-data"), &ud)?;
        let mut variants = serde_json::Map::new();
        for (name, extra) in input["bootcmdFor"].as_object().cloned().unwrap_or_default() {
            if !name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') {
                return Err(format!("bootcmdFor {name:?}: lowercase letters, digits and dashes"));
            }
            let cmds: Vec<Value> = common.iter().cloned().chain(extra.as_array().cloned().unwrap_or_default()).collect();
            let v = user_data(&cmds);
            write_private(&dir.join(format!("user-data-{name}")), &v)?;
            variants.insert(name, json!(v));
        }
        Ok(json!({ "root": root, "names": names, "userData": ud, "userDataFor": variants, "userDataFile": dir.join("user-data").display().to_string() }))
    }

    async fn requester_bundle(&self, input: &Value) -> Result<Value, String> {
        let name = str_of(input, "name")?;
        let contract = str_of(input, "contract")?;
        let cfg = expand(
            self.target
                .agent_config
                .as_deref()
                .ok_or("the target has no agentConfig")?,
        );
        check_private(&cfg).map_err(|e| e.to_string())?;
        let get = |kind: &str, n: &str| {
            let input = json!({ "cluster": format!("requester:{name}"), "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": n });
            atlibs_host::run::Call::call(
                &*self.kube,
                atlibs_kube::GET.into(),
                atlibs_host::run::Request {
                    input: Some(input),
                    ..Default::default()
                },
            )
        };
        let addr =
            get("OpenBaoServer", "requester").await?.unwrap_or_default()["status"]["address"]
                .as_str()
                .ok_or("no OpenBaoServer address")?
                .to_string();
        // a network target's request is a network child: its instance dir is the network's dev/<i>/apps/dev/<i>-<name>
        let d = match &self.target.network {
            Some(n) => testkit::target::expand(&n.state_dir).join(format!(
                "dev/{i}/apps/dev/{i}-{name}/openbaoservers/requester/openbao",
                i = n.instance
            )),
            None => self
                .requester_state(name)
                .join("requesters")
                .join(name)
                .join("dev")
                .join(name)
                .join("openbaoservers/requester/openbao"),
        };
        let runtime = PathBuf::from(std::env::var("HOME").unwrap_or_default())
            .join(".cache/socialweb-computer/runtime");
        let bao = std::fs::read_dir(&runtime)
            .map_err(|e| e.to_string())?
            .flatten()
            .map(|e| e.path().join("bao"))
            .find(|p| p.is_file())
            .ok_or("no bao in the runtime cache")?;
        let bundle = self.dir.join(format!("{contract}-bundle.json"));
        let v = json!([{ "path": "/etc/atirc/atirc.env", "value": "", "mode": "0600" }, { "path": "/etc/atirc/config.toml", "value": std::fs::read_to_string(&cfg).map_err(|e| e.to_string())?, "mode": "0600" }]);
        write_private(&bundle, &v.to_string())?;
        let env = |c: &mut tokio::process::Command| {
            c.env("BAO_ADDR", &addr)
                .env("BAO_CACERT", d.join("tls/ca.crt"))
                .env("BAO_CLIENT_CERT", d.join("clients/requester/client.crt"))
                .env("BAO_CLIENT_KEY", d.join("clients/requester/client.key"));
        };
        let mut login = tokio::process::Command::new(&bao);
        login.args(["write", "-field=token", "auth/cert/login", "name=requester"]);
        env(&mut login);
        let out = login.output().await.map_err(|e| e.to_string())?;
        let mut put = tokio::process::Command::new(&bao);
        put.args([
            "kv",
            "put",
            "-mount=requester",
            &format!("contracts/{contract}/secrets"),
        ])
        .arg(format!("bundle=@{}", bundle.display()));
        env(&mut put);
        put.env("BAO_TOKEN", String::from_utf8_lossy(&out.stdout).trim())
            .stdout(Stdio::null())
            .stderr(Stdio::null());
        let ok = out.status.success() && put.status().await.map_err(|e| e.to_string())?.success();
        let _ = std::fs::remove_file(&bundle);
        if ok {
            Ok(json!({ "ok": true }))
        } else {
            Err("the agent bundle did not reach the requester's OpenBao".into())
        }
    }

    /// The rest of the calls; None: not one of these.
    pub(crate) async fn more(&self, what: &str, input: &Value) -> Option<Result<Value, String>> {
        Some(match what {
            // undo now what the run applied and patched in its clusters (a module about to remove a cluster's kcp, 0001 s2's
            // temporary bidder: afterwards nothing answers there, so the end's cleanup could not)
            "kube.undo" => {
                let errors = self.kube.cleanup().await;
                if errors.is_empty() { Ok(json!({ "ok": true })) } else { Err(errors.join("; ")) }
            }
            "kube.addCluster" => {
                let name = match str_of(input, "name") {
                    Ok(n) => n.to_string(),
                    Err(e) => return Some(Err(e)),
                };
                let path = self.dir.join(format!("tenant-{name}.kubeconfig"));
                let r = write_private(&path, input["kubeconfig"].as_str().unwrap_or_default());
                // viaRoot: a kcp front on a staging VM's loopback, through an ssh forward to that VM
                let via = match (input["viaRoot"] == true, self.target.root_hosts.first()) {
                    (true, Some(h)) if self.target.name != "prod" => {
                        atlibs_kube::Via::Ssh { host: h.clone() }
                    }
                    _ => Default::default(),
                };
                r.map(|_| {
                    self.kube.add_cluster(
                        &format!("tenant:{name}"),
                        Cluster {
                            kubeconfig: path.display().to_string(),
                            workspace: None,
                            via,
                        },
                    );
                    json!({ "cluster": format!("tenant:{name}") })
                })
            }
            // a tenant kubeconfig minted and kept by the host (the module never holds it): over xrpc with the
            // account's service auth, or on the signed-in k8s page; registered as cluster tenant:<as>
            "gateway.kubeconfig" | "page.kubeconfig" => {
                let (Ok(name), Ok(alias)) = (str_of(input, "name"), str_of(input, "as")) else {
                    return Some(Err("name and as are required".into()));
                };
                let ttl = input["ttlSeconds"].as_u64().unwrap_or(3600);
                async {
                    let g = &self.target.gateway;
                    let kc = if what == "gateway.kubeconfig" {
                        let at = testkit::Atproto::new(&self.target);
                        let sess = at.session(self.acct()?).await.map_err(|e| e.to_string())?;
                        let v = at
                            .service_call(
                                &sess,
                                &g.url,
                                &g.aud,
                                "com.publicdomainrelay.temp.cluster.getKubeconfig",
                                Some(&json!({ "name": name, "ttlSeconds": ttl })),
                                &[],
                            )
                            .await
                            .map_err(|e| e.to_string())?;
                        v["kubeconfig"].as_str().map(String::from)
                    } else {
                        let v = self
                            .page_fetch(
                                &format!("{}/api/clusters/kubeconfig", g.url.trim_end_matches('/')),
                                "POST",
                                Some(&json!({ "name": name, "ttlSeconds": ttl })),
                            )
                            .await?;
                        if v["status"] != 200 {
                            return Err(format!(
                                "the page's /api/clusters/kubeconfig: HTTP {}",
                                v["status"]
                            ));
                        }
                        v["body"]["kubeconfig"].as_str().map(String::from)
                    }
                    .ok_or("no kubeconfig in the answer")?;
                    let path = self.dir.join(format!("tenant-{alias}.kubeconfig"));
                    write_private(&path, &kc)?;
                    self.kube.add_cluster(
                        &format!("tenant:{alias}"),
                        Cluster {
                            kubeconfig: path.display().to_string(),
                            workspace: None,
                            via: Default::default(),
                        },
                    );
                    Ok(json!({ "cluster": format!("tenant:{alias}") }))
                }
                .await
            }
            "page.signin" => self.page_signin(input).await,
            // an OAuth sign-in with no page session of ours (a QR session transfer): the account's consent, Authorize
            "oauth.signin" => match input["url"].as_str() {
                Some(url) => {
                    let mut extra = vec![];
                    if let Some(d) = input["done"].as_str() {
                        extra.extend(["--done", d]);
                    }
                    let r = self.signin(url, &extra).await;
                    Ok(json!({ "ok": r.is_ok(), "note": r.err() }))
                }
                None => Err("url is required".into()),
            },
            // a network bidder's core restarted in place (staging; 0007 restart): `dev restart <network> <bidder>/core` with
            // the network's own binary; its VMs stay up and the new core adopts them
            // the memory cgroup of THIS run's own VM (staging / dev network, VM units under the account's systemd user
            // manager): {receipt, cluster, vm} -> unit, current, high, max, swapMax, events {high, max, oom, oom_kill}. With
            // `setHigh` (bytes, or "max" for no limit) it first sets the unit's MemoryHigh (runtime property: fault injection on
            // the test's own VM only; the receipt is one this run made and names the ComputeVM `vm`)
            "vm.memory" => {
                let (Some(receipt), Ok(cluster), Ok(vm)) = (
                    input["receipt"].as_str().filter(|r| r.starts_with("at://")),
                    str_of(input, "cluster"),
                    str_of(input, "vm"),
                ) else {
                    return Some(Err(
                        "receipt (an at:// URI), cluster and vm are required".into()
                    ));
                };
                if self.target.network.is_none() {
                    return Some(Err(
                        "vm.memory: the target is no market network (staging only)".into(),
                    ));
                }
                if !self.receipts.lock().unwrap().iter().any(|r| r == receipt) {
                    return Some(Err(format!(
                        "vm.memory: {receipt} is not a contract of this run"
                    )));
                }
                if vm.is_empty()
                    || !vm
                        .chars()
                        .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
                {
                    return Some(Err("vm.memory: vm is a ComputeVM name".into()));
                }
                let via_ops = cluster.strip_prefix("bidder:").filter(|n| self.ops.as_ref().is_some_and(|o| o.serves(n)));
                let cv = atlibs_host::run::Call::call(
                    &crate::ops::OpsKube { ops: self.ops.clone(), kube: self.kube.clone() },
                    atlibs_kube::GET.into(),
                    atlibs_host::run::Request { input: Some(json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM", "name": vm })), ..Default::default() },
                ).await;
                match cv {
                    Ok(Some(o)) if o["spec"]["contract"]["receipt"]["uri"] == receipt => {}
                    Ok(_) => {
                        return Some(Err(format!(
                            "vm.memory: the ComputeVM {vm} is not the receipt's"
                        )));
                    }
                    Err(e) => return Some(Err(format!("vm.memory: {e}"))),
                }
                // a bidder behind its read-only ops routes: the figures only, no setHigh
                if let (Some(n), Some(o)) = (via_ops, &self.ops) {
                    if !input["setHigh"].is_null() {
                        return Some(Err(format!("vm.memory: {cluster} is reached through the read-only ops routes: setHigh is not served")));
                    }
                    return Some(o.memory(n, vm).await.map_err(|e| format!("vm.memory: {e}")));
                }
                let set = match input["setHigh"].as_u64() {
                    Some(n) => format!("MemoryHigh={n}"),
                    None if input["setHigh"] == "max" => "MemoryHigh=infinity".to_string(),
                    None => String::new(),
                };
                // the VM unit is in the user manager of the account that runs the bidder: this account's (a dev network), or, as root on a
                // joined bidder's host, the installed account's (swc-bidder)
                let script = format!(
                    r#"unit_of() {{ $as systemctl --user list-units --all --plain --no-legend 'socialweb-vm-*-{vm}.service' 2>/dev/null | awk '{{print $1}}' | head -1; }}
as=""; u=""
if [ "$(id -u)" = 0 ]; then
  for d in /run/user/*; do
    n=$(id -un "${{d##*/}}" 2>/dev/null) || continue
    [ "$n" != root ] || continue
    as="runuser -u $n -- env XDG_RUNTIME_DIR=$d"
    u=$(unit_of)
    [ -z "$u" ] || break
  done
else
  export XDG_RUNTIME_DIR="${{XDG_RUNTIME_DIR:-/run/user/$(id -u)}}"
  u=$(unit_of)
fi
[ -n "$u" ] || {{ echo '{{"error":"no VM unit of {vm} in a user manager of this host'"'"'s bidder account (a proc-backend bidder has no VM cgroup)"}}'; exit 0; }}
{setcmd}
cg=$($as systemctl --user show "$u" -p ControlGroup --value)
d=/sys/fs/cgroup$cg
[ -r "$d/memory.current" ] || {{ echo '{{"error":"no readable memory cgroup for the unit"}}'; exit 0; }}
ev=$(awk '{{printf "\"%s\":%s,", $1, $2}}' "$d/memory.events" | sed 's/,$//')
printf '{{"unit":"%s","current":%s,"high":"%s","max":"%s","swapMax":"%s","events":{{%s}}}}\n' "$u" "$(cat $d/memory.current)" "$(cat $d/memory.high)" "$(cat $d/memory.max)" "$(cat $d/memory.swap.max 2>/dev/null)" "$ev"
"#,
                    setcmd = if set.is_empty() {
                        String::new()
                    } else {
                        format!(
                            "$as systemctl --user set-property --runtime \"$u\" {set} || {{ echo '{{\"error\":\"set-property failed\"}}'; exit 0; }}"
                        )
                    },
                );
                // a joined bidder (staging's multi layout) holds its VMs on its own VM: there, as the target's ssh host of its kcp
                let winner = cluster
                    .strip_prefix("bidder:")
                    .and_then(|n| self.target.bidders.iter().find(|b| b.name == n))
                    .and_then(|b| b.kcp.as_ref())
                    .and_then(|k| match &k.via {
                        testkit::target::Via::Ssh { host } => Some(host.clone()),
                        _ => None,
                    });
                on_host(winner.as_deref(), &script)
                    .await
                    .and_then(|o| {
                        serde_json::from_str::<Value>(o.trim())
                            .map_err(|e| format!("vm.memory: {e}: {}", tail(&o)))
                    })
                    .and_then(|v| match v["error"].as_str() {
                        Some(e) => Err(format!("vm.memory: {e}")),
                        None => Ok(v),
                    })
            }
            // the shipped binary against a tampered copy of what it extracts (acceptance 0018, staging): variant file / fileSidecar
            // flips a byte of a download in the extracted socialweb-guest-downloads tree (and, for fileSidecar, rewrites its
            // sidecar to match) and runs `dev verify-assets` again on the same asset cache; variant record flips one digit of
            // the embedded build record in a copy of the binary and runs it
            "assets.tamper" => self.assets_tamper(input).await,
            // submitAccept of an accept record that does not exist (a requester that deleted its accept before the bidder read it): the
            // account's service auth to the bidder's market endpoint (its offering: https, else iroh), for at://<account>/market.accept/<rkey>
            "bidder.submitAccept" => {
                let (Ok(name), Ok(rkey), Ok(cid)) = (
                    str_of(input, "name"),
                    str_of(input, "rkey"),
                    str_of(input, "cid"),
                ) else {
                    return Some(Err("name, rkey and cid are required".into()));
                };
                let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
                    return Some(Err(format!("no bidder {name} in the target")));
                };
                async {
                    let at = testkit::Atproto::new(&self.target);
                    let sess = at.session(self.acct()?).await.map_err(|e| e.to_string())?;
                    let offering = at
                        .record(&format!(
                            "at://{}/com.publicdomainrelay.temp.market.offering/self",
                            b.did
                        ))
                        .await
                        .map_err(|e| e.to_string())?
                        .ok_or("the bidder has no offering record")?;
                    let endpoint = testkit::irohfetch::market_base(&offering)
                        .ok_or("the offering names no https or iroh market endpoint")?;
                    let uri = format!(
                        "at://{}/com.publicdomainrelay.temp.market.accept/{rkey}",
                        sess.did()
                    );
                    let reply = at
                        .service_call(
                            &sess,
                            &endpoint,
                            &format!("{}#pdr_temp_market", b.did),
                            "com.publicdomainrelay.temp.market.submitAccept",
                            Some(&json!({ "acceptUri": uri, "acceptCid": cid })),
                            &[],
                        )
                        .await
                        .map_err(|e| e.to_string())?;
                    Ok(json!({ "acceptUri": uri, "reply": reply }))
                }
                .await
            }
            "bidder.restart" => {
                let Ok(name) = str_of(input, "name") else {
                    return Some(Err("name is required".into()));
                };
                if !self.target.bidders.iter().any(|b| b.name == name) {
                    return Some(Err(format!("no bidder {name} in the target")));
                }
                let Some(n) = self.target.network.clone() else {
                    return Some(Err(
                        "bidder.restart: the target is no market network (staging only)".into(),
                    ));
                };
                let t = Instant::now();
                // a joined bidder (staging's multi layout) is installed on its own VM (`install bidder`): its core is the user unit
                // socialweb-computer-bidder@<handle> of its account; restarted in place as `upgrade` does (the unit alone: its kcp and VM
                // units stay and the new core adopts them)
                let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
                if let Some(testkit::target::Via::Ssh { host }) = via {
                    let script = BIDDER_RESTART.replace("@NAME@", name).replace("@UNIT_TIMEOUTS@", UNIT_TIMEOUTS);
                    let out = match on_host(Some(&host), &script).await {
                        Ok(o) => o,
                        Err(e) => return Some(Err(format!("bidder.restart {name}: {e}"))),
                    };
                    let Some(since) = out.lines().find_map(|l| l.strip_prefix("since ")).and_then(|n| n.trim().parse::<u64>().ok()) else {
                        return Some(Err(format!("bidder.restart {name}: {}", tail(&out))));
                    };
                    self.extra.log_since.lock().unwrap().entry(name.to_string()).or_insert(since);
                    return Some(Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0, "report": tail(&out) })));
                }
                let bin = self.bin.path.display().to_string();
                let args: Vec<String> = [
                    "dev",
                    "restart",
                    &n.instance,
                    &format!("{name}/core"),
                    "--state-dir",
                    &testkit::target::expand(&n.state_dir).display().to_string(),
                    "--binary",
                    &bin,
                ]
                .map(String::from)
                .to_vec();
                self.cli(&args).await.and_then(|(ok, out)| {
                    if ok {
                        Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0, "report": tail(&out) }))
                    } else {
                        Err(format!("dev restart {name}/core: {}", tail(&out)))
                    }
                })
            }
            // a joined staging bidder upgraded in place to another release binary (`upgrade --binary --sha256 --state-dir` as its
            // account, docs/ci-debug.md step 5d): binary `from` (the input's `from` spec, else SWC_UPGRADE_FROM, else
            // SWC_COMPAT_REQUESTER: the previous rc) or `test` (the binary under test); both are staged on the bidder's host
            // first. stopFirst: the core unit is stopped before the upgrade, so nothing the old core serves outlives the call
            "bidder.upgrade" => {
                let (Ok(name), Ok(which)) = (str_of(input, "name"), str_of(input, "binary")) else {
                    return Some(Err("name and binary (from or test) are required".into()));
                };
                if self.target.network.is_none() {
                    return Some(Err("bidder.upgrade: the target is no market network (staging only)".into()));
                }
                let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
                let Some(testkit::target::Via::Ssh { host }) = via else {
                    return Some(Err(format!("bidder.upgrade {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
                };
                let t = Instant::now();
                self.bidder_upgrade(name, &host, which, input["from"].as_str(), input["stopFirst"] == true).await.map(|mut v| {
                    v["seconds"] = json!((t.elapsed().as_secs_f64() * 10.0).round() / 10.0);
                    v
                })
            }
            // how many lines of a bidder's log hold each text (a text: a string, or a list of strings a line must all
            // hold), counted now (a module diffs two counts): a network bidder's core.log under the network's dev
            // state, an installed one's journal (swc-bidder, read-only, over ssh); the texts never reach a shell
            "bidder.logCount" | "bidder.logLines" => {
                let Ok(name) = str_of(input, "name") else {
                    return Some(Err("name is required".into()));
                };
                let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
                    return Some(Err(format!("no bidder {name} in the target")));
                };
                let texts: Vec<Vec<String>> = input["texts"]
                    .as_array()
                    .into_iter()
                    .flatten()
                    .map(|t| match t {
                        Value::Array(a) => a
                            .iter()
                            .filter_map(|x| x.as_str().map(String::from))
                            .collect(),
                        t => t.as_str().map(|x| vec![x.to_string()]).unwrap_or_default(),
                    })
                    .collect();
                if texts.is_empty()
                    || texts.iter().any(|t| {
                        t.is_empty()
                            || t.iter()
                                .any(|x| x.is_empty() || x.chars().any(char::is_control))
                    })
                {
                    return Some(Err("texts: strings or lists of strings, none empty".into()));
                }
                let lines = what == "bidder.logLines";
                let count = |log: &str| {
                    json!(
                        texts
                            .iter()
                            .map(|t| (
                                t.join(" & "),
                                json!(
                                    log.lines()
                                        .filter(|l| t.iter().all(|x| l.contains(x.as_str())))
                                        .count()
                                )
                            ))
                            .collect::<serde_json::Map<_, _>>()
                    )
                };
                // bidder.logLines: the JSON lines holding every text of the first entry of `texts`, as {time, msg, fields} of the
                // keys named in `keys` (at most 50)
                let pick = |log: &str| {
                    let keys: Vec<String> = input["keys"]
                        .as_array()
                        .into_iter()
                        .flatten()
                        .filter_map(|k| k.as_str().map(String::from))
                        .collect();
                    let rows: Vec<Value> = log
                        .lines()
                        .filter(|l| texts[0].iter().all(|x| l.contains(x.as_str())))
                        .filter_map(|l| serde_json::from_str::<Value>(l).ok())
                        .map(|v| {
                            let mut o = json!({ "time": v["time"], "msg": v["msg"] });
                            for k in &keys {
                                o[k] = v[k].clone();
                            }
                            o
                        })
                        .take(50)
                        .collect();
                    json!({ "lines": rows })
                };
                let out = |log: &str, from: &str| {
                    if lines {
                        let mut v = pick(log);
                        v["log"] = json!(from);
                        v
                    } else {
                        json!({ "counts": count(log), "log": from })
                    }
                };
                // a bidder the host reaches by ssh (prod, a staging joined bidder): its own host's log, from the process this run restarted
                // when it restarted one
                if let Some(o) = self.ops.as_ref().filter(|o| o.serves(name)) {
                    return Some(if lines {
                        let keys: Vec<String> = input["keys"].as_array().into_iter().flatten().filter_map(|k| k.as_str().map(String::from)).collect();
                        o.log_lines(name, &texts, &keys).await
                    } else {
                        o.log_count(name, &texts).await
                    });
                }
                let since = self.extra.log_since.lock().unwrap().get(name).copied();
                let script = format!("since={}\n{BIDDER_LOG}", since.map(|n| n.to_string()).unwrap_or_default());
                if let Some(testkit::target::Via::Ssh { host }) = b.kcp.as_ref().map(|k| &k.via) {
                    return Some(on_host(Some(host), &script).await.map(|l| out(&l, "journal")));
                }
                if let Some(n) = &self.target.network {
                    let i = &n.instance;
                    let p = testkit::target::expand(&n.state_dir)
                        .join(format!("dev/{i}/apps/dev/{i}-{name}/logs/core.log"));
                    return Some(
                        std::fs::read_to_string(&p)
                            .map(|l| out(&l, &p.display().to_string()))
                            .map_err(|e| format!("{}: {e}", p.display())),
                    );
                }
                let Some(testkit::target::Via::Ssh { host }) = b.kcp.as_ref().map(|k| &k.via)
                else {
                    return Some(Err(format!(
                        "{name}: the host reaches its bidder log by ssh or in the network only"
                    )));
                };
                on_host(Some(host), &script)
                    .await
                    .map(|l| out(&l, "journal"))
            }
            // a bidder host's health without sudo (0006, 0008): its `run bidder` unit's ERROR lines since the process
            // started (and in its first 10 min), automatic restarts since the last stop, VM processes, when it verified
            // its pinned guest image (bidder_guest_image_ready) and how often it could not
            "bidder.health" => {
                let Ok(name) = str_of(input, "name") else {
                    return Some(Err("name is required".into()));
                };
                let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
                    return Some(Err(format!("no bidder {name} in the target")));
                };
                if let Some(o) = self.ops.as_ref().filter(|o| o.serves(name)) {
                    return Some(o.health(name).await.map_err(|e| format!("{name}: {e}")));
                }
                let via = match b.kcp.as_ref().map(|k| &k.via) {
                    Some(testkit::target::Via::Ssh { host }) => Some(host.clone()),
                    Some(testkit::target::Via::Direct) => None,
                    _ => {
                        return Some(Err(format!(
                            "{name}: the host reaches its bidder by ssh or locally only"
                        )));
                    }
                };
                on_host(via.as_deref(), BIDDER_HEALTH).await.and_then(|o| {
                    serde_json::from_str(o.trim()).map_err(|e| format!("{name}: {e}: {}", tail(&o)))
                })
            }
            // the bidder host's /proc/meminfo MemTotal and MemAvailable in MiB and, for each of `vms` ({name, owner}), the anon MiB
            // of its unit's cgroup (socialweb-vm-<sha256(owner)[:10]>-<name>.service), as the bidder reads them (vmsize.go): a module
            // computes the memory the bidder can still admit exactly as the bidder does
            "bidder.meminfo" => {
                let Ok(name) = str_of(input, "name") else {
                    return Some(Err("name is required".into()));
                };
                let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
                    return Some(Err(format!("no bidder {name} in the target")));
                };
                let via = match b.kcp.as_ref().map(|k| &k.via) {
                    Some(testkit::target::Via::Ssh { host }) => Some(host.clone()),
                    Some(testkit::target::Via::Direct) => None,
                    _ => return Some(Err(format!("{name}: the host reaches its bidder by ssh or locally only"))),
                };
                let mut units = String::new();
                for v in input["vms"].as_array().into_iter().flatten() {
                    let (Some(vm), Some(owner)) = (v["name"].as_str(), v["owner"].as_str()) else {
                        return Some(Err("vms[] holds {name, owner}".into()));
                    };
                    if vm.is_empty() || !vm.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.') {
                        return Some(Err(format!("{vm}: not a VM name")));
                    }
                    let id: String = Sha256::digest(owner.as_bytes()).iter().map(|b| format!("{b:02x}")).collect();
                    units.push_str(&format!(" {vm}=socialweb-vm-{}-{vm}.service", &id[..10]));
                }
                on_host(via.as_deref(), &format!("units='{}'\n{MEMINFO}", units.trim()))
                    .await
                    .and_then(|o| serde_json::from_str(o.trim()).map_err(|e| format!("{name}: {e}: {}", tail(&o))))
            }
            // a guest's nf_tables tables on a staging bidder host (root over ssh: the run's own throwaway bidder VM), in the network
            // namespace of the VMM of socialweb-vm-<sha256(owner)[:10]>-<vm>.service (nsenter -n nft). op list: the product's
            // table bridge swcguest. probe: a test-only table bridge swcprobe, two counters and no verdict for UDP from the guest's
            // tap to this host's own IPv4 addresses on ports min-max: chain sent before swcguest (priority -301), chain passed
            // after it (-299: what swcguest accepted). counters: that table. unprobe: it deleted. Staging only
            "bidder.guestNft" => {
                let (Ok(name), Ok(vm), Ok(owner), Ok(op)) = (str_of(input, "name"), str_of(input, "vm"), str_of(input, "owner"), str_of(input, "op")) else {
                    return Some(Err("name, vm, owner and op are required".into()));
                };
                if self.target.network.is_none() {
                    return Some(Err("bidder.guestNft: the target is no market network (staging only)".into()));
                }
                if !["list", "probe", "counters", "unprobe"].contains(&op) {
                    return Some(Err(format!("bidder.guestNft: op {op}: list, probe, counters or unprobe")));
                }
                if vm.is_empty() || !vm.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') {
                    return Some(Err(format!("{vm}: not a VM name")));
                }
                let (min, max) = (input["min"].as_u64().unwrap_or(0), input["max"].as_u64().unwrap_or(0));
                if op == "probe" && !(1024..=65535).contains(&min) || op == "probe" && !(min..=65535).contains(&max) {
                    return Some(Err(format!("bidder.guestNft probe: ports {min}-{max}")));
                }
                let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
                let Some(testkit::target::Via::Ssh { host }) = via else {
                    return Some(Err(format!("bidder.guestNft {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
                };
                let id: String = Sha256::digest(owner.as_bytes()).iter().map(|b| format!("{b:02x}")).collect();
                let script = GUEST_NFT
                    .replace("@UNIT@", &format!("socialweb-vm-{}-{vm}.service", &id[..10]))
                    .replace("@OP@", op)
                    .replace("@MIN@", &min.to_string())
                    .replace("@MAX@", &max.to_string());
                on_host(Some(&host), &script).await.map(|o| {
                    let counter = |chain: &str| {
                        o.split(&format!("chain {chain} {{")).nth(1).and_then(|c| c.split("counter packets ").nth(1)).and_then(|c| c.split_whitespace().next()).and_then(|n| n.parse::<u64>().ok())
                    };
                    let pid = o.lines().find_map(|l| l.strip_prefix("vmm-pid ")).unwrap_or_default().to_string();
                    let own = o.lines().find_map(|l| l.strip_prefix("own ")).unwrap_or_default().to_string();
                    let out: String = o.lines().filter(|l| !l.starts_with("vmm-pid ") && !l.starts_with("own ")).collect::<Vec<_>>().join("\n");
                    json!({ "output": out, "vmmPid": pid, "own": own, "sent": counter("sent"), "passed": counter("passed") })
                })
            }
            // the public URL of the binary under test (the bucket's main/<commit>/ or local/<commit>/ build of the commit its
            // `version` names, whose SHA256SUMS lists exactly its sha256), for a guest to fetch at bucket speed
            "binary.published" => {
                async {
                    let sha256 = file_sha256(&self.bin.path).await?;
                    let commit = self.bin.version.split_whitespace().nth(1).filter(|c| c.len() == 40 && c.chars().all(|x| x.is_ascii_hexdigit())).ok_or_else(|| format!("the binary under test's version {:?} names no commit", self.bin.version))?;
                    let mut seen = vec![];
                    for spec in [format!("sha:{commit}"), format!("local:{commit}")] {
                        match testkit::requester::Binary::published(&spec).await {
                            Ok(Some((url, sum))) if sum == sha256 => return Ok(json!({ "url": url, "sha256": sum, "spec": spec })),
                            Ok(Some((_, sum))) => seen.push(format!("{spec}: {sum}")),
                            Ok(None) => {}
                            Err(e) => seen.push(format!("{spec}: {e}")),
                        }
                    }
                    Err(format!("the binary under test ({sha256}, commit {commit}) is not published: {}", seen.join("; ")))
                }
                .await
            }
            // the first line of a bidder's journal (since its run bidder process started) holding every text of `texts`, read
            // as it is written (journalctl -f), or found false after `seconds`; installed staging bidders only
            "bidder.awaitLog" => {
                let Ok(name) = str_of(input, "name") else {
                    return Some(Err("name is required".into()));
                };
                let texts: Vec<String> = input["texts"].as_array().into_iter().flatten().filter_map(|t| t.as_str().map(String::from)).collect();
                if texts.is_empty() || texts.iter().any(|t| t.is_empty() || t.chars().any(char::is_control)) {
                    return Some(Err("texts: strings, none empty".into()));
                }
                let seconds = input["seconds"].as_u64().unwrap_or(300).clamp(1, 3600);
                let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
                let Some(testkit::target::Via::Ssh { host }) = via else {
                    return Some(Err(format!("bidder.awaitLog {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
                };
                let quoted = format!("'{}'", texts.join("\u{1f}").replace('\'', "'\\''"));
                let script = AWAIT_LOG.replace("@TEXTS@", &quoted).replace("@SECS@", &seconds.to_string());
                let t = Instant::now();
                on_host(Some(&host), &script).await.map(|o| {
                    let line = o.lines().find_map(|l| l.strip_prefix("line ")).map(String::from);
                    let parsed = line.as_deref().and_then(|l| serde_json::from_str::<Value>(l).ok());
                    json!({ "found": line.is_some(), "line": parsed.unwrap_or(json!(line)), "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0, "report": tail(&o) })
                })
            }
            // the dumbpipe children of an installed bidder's run bidder process (its iroh ingress listeners: pid, executable,
            // --host address), read-only from /proc over ssh; installed staging bidders only
            "bidder.dumbpipe" => {
                let Ok(name) = str_of(input, "name") else {
                    return Some(Err("name is required".into()));
                };
                let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
                let Some(testkit::target::Via::Ssh { host }) = via else {
                    return Some(Err(format!("bidder.dumbpipe {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
                };
                on_host(Some(&host), BIDDER_DUMBPIPE).await.and_then(|o| {
                    let core = o.lines().find_map(|l| l.strip_prefix("core ")).map(str::trim).unwrap_or_default().to_string();
                    if core.is_empty() {
                        return Err(format!("bidder.dumbpipe {name}: {}", tail(&o)));
                    }
                    let listeners: Vec<Value> = o
                        .lines()
                        .filter_map(|l| l.strip_prefix("listener "))
                        .map(|l| {
                            let w: Vec<&str> = l.split_whitespace().collect();
                            let host = w.iter().position(|x| *x == "--host").and_then(|p| w.get(p + 1)).copied().unwrap_or_default();
                            json!({ "pid": w.first().copied().unwrap_or_default(), "exe": w.get(1).copied().unwrap_or_default(), "host": host, "args": w.get(2..).map(|a| a.join(" ")).unwrap_or_default() })
                        })
                        .collect();
                    Ok(json!({ "core": core, "listeners": listeners }))
                })
            }
            // the ssh gateway's core log lines of a DID since an ISO time (0002: revoke before release), read over ssh
            "gateway.log" => {
                let (Ok(did), Ok(since)) = (str_of(input, "did"), str_of(input, "since")) else {
                    return Some(Err("did and since are required".into()));
                };
                if !did.starts_with("did:") || did.contains(['\'', ' ']) {
                    return Some(Err("did is a DID".into()));
                }
                let msgs: Vec<String> = input["msgs"]
                    .as_array()
                    .into_iter()
                    .flatten()
                    .filter_map(|m| m.as_str().map(String::from))
                    .collect();
                if let Some(o) = self.ops.as_ref().filter(|o| o.serves_gateway()) {
                    return Some(o.gateway_log(did, since, &msgs).await);
                }
                let Some(l) = self.target.gateway.log.clone() else {
                    return Some(Err("the target names no gateway.log".into()));
                };
                on_host(Some(&l.host), &format!("grep -F '\"did\":\"{did}\"' '{}' || true", l.path.replace('\'', ""))).await.map(|o| {
                    let all: Vec<Value> = o
                        .lines()
                        .filter_map(|l| serde_json::from_str::<Value>(l).ok())
                        .filter(|v| v["time"].as_str().is_some_and(|t| t >= since))
                        .collect();
                    crate::ops::gateway_lines(&all, &msgs)
                })
            }
            "page.fetch" => match input["url"].as_str() {
                Some(url) => {
                    self.page_fetch(
                        url,
                        input["method"].as_str().unwrap_or("GET"),
                        input.get("body"),
                    )
                    .await
                }
                None => Err("url is required".into()),
            },
            "ssh.tempKey" => self.temp_key().await,
            "atirc.observerPub" => match (str_of(input, "nick"), input["text"].as_str()) {
                (Ok(n), Some(t)) => self.observer_run(n, &["pub", t]).await,
                _ => Err("nick and text are required".into()),
            },
            "atirc.observerRevoke" => match str_of(input, "nick") {
                Ok(n) => self.observer_run(n, &["revoke"]).await,
                Err(e) => Err(e),
            },
            // the owner's mesh node, as the hand run does it: `atirc mesh run --peer <identity>@<did>` as the account
            // (an AtircNode's spec.peers takes no @did yet)
            "atirc.meshRun" => {
                let (Ok(name), Some(peer)) = (str_of(input, "name"), input["peer"].as_str()) else {
                    return Some(Err("name and peer are required".into()));
                };
                async {
                    let a = self.acct()?;
                    let bin = self.atirc()?;
                    let dir = self.dir.join(format!("atirc-{name}"));
                    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
                    std::fs::set_permissions(
                        &dir,
                        std::os::unix::fs::PermissionsExt::from_mode(0o700),
                    )
                    .map_err(|e| e.to_string())?;
                    let log =
                        std::fs::File::create(dir.join("mesh.log")).map_err(|e| e.to_string())?;
                    let child = tokio::process::Command::new(&bin)
                        .args([
                            // atirc names the owner's channel itself (its default_channels): a hand-made name
                            // differed for a handle with a dash (staging's), so the agent never dialed this node
                            "mesh",
                            "run",
                            "--peer",
                            peer,
                            "--irc-port",
                            "0",
                        ])
                        .env("ATP_USERNAME", &a.handle)
                        .env("ATP_PASSWORD", self.password()?)
                        .env("ATIRC_CONFIG_DIR", &dir)
                        // the target's PLC (staging: the network's, https by name, plan 0050)
                        .env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
                        .stdin(Stdio::null())
                        .stdout(log.try_clone().map_err(|e| e.to_string())?)
                        .stderr(log)
                        .kill_on_drop(true)
                        .spawn()
                        .map_err(|e| e.to_string())?;
                    self.extra
                        .nodes
                        .lock()
                        .await
                        .insert(name.to_string(), (bin.clone(), dir.clone(), child));
                    // ready when `mesh peers` answers
                    let deadline = Instant::now() + Duration::from_secs(90);
                    loop {
                        let ok = tokio::process::Command::new(&bin)
                            .args(["mesh", "peers"])
                            .env("ATIRC_CONFIG_DIR", &dir)
                            .env("ATP_USERNAME", &a.handle)
                            .stdin(Stdio::null())
                            .stdout(Stdio::null())
                            .stderr(Stdio::null())
                            .status()
                            .await
                            .is_ok_and(|s| s.success());
                        if ok {
                            return Ok(json!({ "node": name }));
                        }
                        if Instant::now() > deadline {
                            return Err(
                                "atirc mesh run did not answer in 90 s (mesh.log in the work dir)"
                                    .into(),
                            );
                        }
                        tokio::time::sleep(Duration::from_secs(2)).await;
                    }
                }
                .await
            }
            "logs.fetchRuns" => self.fetch_runs(input).await,
            "requester.ssh" => {
                let (Ok(name), Ok(contract), Ok(command)) =
                    (str_of(input, "name"), str_of(input, "contract"), exec_words(input))
                else {
                    return Some(Err("name, contract, exec or argv are required".into()));
                };
                let mut extra = vec!["--contract", contract];
                if let Some(t) = input["transport"].as_str() {
                    extra.extend(["--transport", t]);
                }
                extra.extend(command.iter().map(String::as_str));
                self.requester_cli("ssh", name, &extra)
                    .await
                    .map(|(ok, out)| json!({ "ok": ok, "output": out }))
            }
            "requester.restart" => match str_of(input, "name") {
                Ok(n) => self.requester_restart(n).await,
                Err(e) => Err(e),
            },
            "requester.cluster" => {
                let (Ok(name), Ok(cluster)) = (str_of(input, "name"), str_of(input, "cluster"))
                else {
                    return Some(Err("name and cluster are required".into()));
                };
                async {
                    let (ok, out) = self
                        .requester_cli("cluster", name, &["--cluster", cluster])
                        .await?;
                    if !ok {
                        return Err(format!("run requester cluster: {}", tail(&out)));
                    }
                    let deadline = Instant::now() + Duration::from_secs(600);
                    let path = loop {
                        let (ok, out) = self
                            .requester_cli("kubeconfig", name, &["--cluster", cluster])
                            .await?;
                        if let Some(p) = out
                            .lines()
                            .map(str::trim)
                            .find(|l| l.starts_with('/'))
                            .filter(|_| ok)
                        {
                            break p.to_string();
                        }
                        if Instant::now() > deadline {
                            return Err(format!("cluster {cluster}: no kubeconfig in 10 min"));
                        }
                        tokio::time::sleep(Duration::from_secs(5)).await;
                    };
                    self.kube.add_cluster(
                        &format!("tenant:{cluster}"),
                        Cluster {
                            kubeconfig: path,
                            workspace: None,
                            via: Default::default(),
                        },
                    );
                    Ok(json!({ "cluster": format!("tenant:{cluster}") }))
                }
                .await
            }
            "requester.bundle" => self.requester_bundle(input).await,
            "staging.pki" => self.staging_pki(input).await,
            "requester.records" => match (str_of(input, "name"), self.acct()) {
                (Ok(name), Ok(a)) => {
                    let mut logs = vec![];
                    let state = self.requester_state(name);
                    match &self.target.network {
                        // a network's state dir holds every requester of the network: only this one's (plan 0044 finding 26)
                        Some(n) => {
                            let i = &n.instance;
                            for d in [state.join("requesters").join(name), state.join(format!("dev/{i}/apps/dev/{i}-{name}"))] {
                                find_named(&d, "core.log", &mut logs);
                            }
                        }
                        None => find_named(&state, "core.log", &mut logs),
                    }
                    logs.push(state.join(format!("{name}.log")));
                    let declined = declined_accepts(&logs);
                    Ok(json!({ "uris": uris_in(&logs, &a.did).into_iter().filter(|u| !kept_by_rule(u) && !declined.contains(u)).collect::<Vec<_>>(),
                        "declinedAccepts": declined.len(), "declined": declined }))
                }
                (Err(e), _) | (_, Err(e)) => Err(e),
            },
            "page.round" => {
                let Some(repo) = self.target.page_repo.clone() else {
                    return Some(Err("the target has no pageRepo".into()));
                };
                let bidder = input["bidder"].as_str().unwrap_or_default().to_string();
                async {
                    let a = self.acct()?;
                    let root = expand(&repo);
                    // the checkout's own pinned node and Chrome (a staging VM's, tests/compositions/staging/page.sh), else the machine's
                    let tool = |p: &str| Some(root.join("tools").join(p)).filter(|p| p.exists());
                    let mut cmd = tokio::process::Command::new(tool("node/bin/node").unwrap_or_else(|| "node".into()));
                    cmd.current_dir(&root)
                        .args(["scripts/page-round.mjs", &bidder, "uname -a; id"])
                        .env("ALICE_ATPROTO_HANDLE", &a.handle)
                        .env("ALICE_ATPROTO_PASSWORD", self.password()?)
                        .stdin(Stdio::null());
                    if let Some(chrome) = tool("chrome/chrome") {
                        cmd.env("CHROME", chrome);
                    }
                    // a network of its own: the page signs in at the target's PDS, resolves DIDs at its PLC and asks its relay; the
                    // script's fetches (node) and the page's (Chrome) trust the network CA
                    if self.target.name != "prod" {
                        cmd.env("PAGE_PDS", &self.target.pds).env("PAGE_PLC", &self.target.plc);
                        if let Some(relay) = &self.target.relay {
                            cmd.env("PAGE_RELAY", relay);
                        }
                        if let Some(f) = &self.target.xrpc_front {
                            let ca = expand(&f.ca);
                            let names: Vec<String> = [Some(&self.target.plc), Some(&self.target.pds), self.target.relay.as_ref()]
                                .into_iter()
                                .flatten()
                                .filter_map(|u| reqwest::Url::parse(u).ok())
                                .filter(|u| u.scheme() == "https")
                                .filter_map(|u| u.host_str().map(String::from))
                                .collect();
                            cmd.env("NODE_EXTRA_CA_CERTS", &ca);
                            let spki = front_spki(f.address, &ca, &names).await?;
                            if !spki.is_empty() {
                                cmd.env("PAGE_CHROME_ARGS", json!([format!("--ignore-certificate-errors-spki-list={spki}")]).to_string());
                            }
                        }
                    }
                    let out = tokio::time::timeout(Duration::from_secs(900), cmd.output())
                        .await
                        .map_err(|_| "page-round did not end in 15 min".to_string())?
                        .map_err(|e| e.to_string())?;
                    let text = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
                    if text.contains(&self.password()?) {
                        return Err("page-round printed the password".into());
                    }
                    Ok(json!({ "ok": out.status.success(), "lines": text.lines().collect::<Vec<_>>() }))
                }
                .await
            }
            "example" => match input["name"]
                .as_str()
                .filter(|n| !n.contains('/') && n.ends_with(".yaml"))
            {
                Some(n) => std::fs::read_to_string(testkit::repo().join("examples/infra").join(n))
                    .map(|t| json!({ "text": t }))
                    .map_err(|e| format!("{n}: {e}")),
                None => Err("name is an examples/infra file".into()),
            },
            "request.r2" => {
                let (Ok(name), Ok(r2)) = (str_of(input, "name"), str_of(input, "r2Name")) else {
                    return Some(Err("name and r2Name are required".into()));
                };
                let mut extra = vec!["--r2-name", r2, "--hold"];
                if input["agent"] == true {
                    extra.push("--agent");
                }
                let args = self.sub_args("r2", name, &extra);
                self.cli(&args).await.and_then(|(ok, out)| {
                    if ok {
                        Ok(json!({}))
                    } else {
                        Err(format!("run request r2: {}", tail(&out)))
                    }
                })
            }
            "request.checkLogs" => match (str_of(input, "name"), self.password()) {
                (Ok(name), Ok(pw)) => {
                    let mut problems = vec![];
                    for p in self.request_logs(name) {
                        let Ok(text) = std::fs::read_to_string(&p) else {
                            continue;
                        };
                        if text.contains(&pw) {
                            problems.push(format!("{}: the app password", p.display()));
                        }
                        for bad in ["a=candidate", "AGE-SECRET-KEY"] {
                            if text.contains(bad) {
                                problems.push(format!("{}: plaintext {bad}", p.display()));
                            }
                        }
                    }
                    Ok(json!({ "ok": problems.is_empty(), "problems": problems }))
                }
                (Err(e), _) | (_, Err(e)) => Err(e),
            },
            "release.put" | "release.delete" => {
                let did = match self.acct() {
                    Ok(a) => a.did.clone(),
                    Err(e) => return Some(Err(e)),
                };
                async {
                    let base = self.target.pds.trim_end_matches('/').to_string();
                    let s: Value = self
                        .http
                        .post(format!("{base}/xrpc/com.atproto.server.createSession"))
                        .json(&json!({ "identifier": did, "password": self.password()? }))
                        .send()
                        .await
                        .map_err(|e| e.to_string())?
                        .json()
                        .await
                        .map_err(|e| e.to_string())?;
                    let jwt = s["accessJwt"].as_str().ok_or_else(|| format!("createSession {did}: {}", s["error"]))?.to_string();
                    let call = |nsid: &'static str, body: Value| {
                        let (http, base, jwt) = (self.http.clone(), base.clone(), jwt.clone());
                        async move {
                            let r = http.post(format!("{base}/xrpc/{nsid}")).bearer_auth(jwt).json(&body).send().await.map_err(|e| e.to_string())?;
                            let status = r.status();
                            let v: Value = r.json().await.unwrap_or_default();
                            if !status.is_success() {
                                return Err(format!("{nsid}: {status} {} {}", v["error"], v["message"]));
                            }
                            Ok::<Value, String>(v)
                        }
                    };
                    let mut out = Vec::new();
                    if what == "release.put" {
                        for r in input["records"].as_array().cloned().unwrap_or_default() {
                            let v = call("com.atproto.repo.putRecord", json!({ "repo": did, "collection": r["collection"], "rkey": r["rkey"], "record": r["record"], "validate": false })).await?;
                            out.push(json!({ "uri": v["uri"], "cid": v["cid"] }));
                        }
                    } else {
                        for uri in input["uris"].as_array().cloned().unwrap_or_default() {
                            let u = uri.as_str().unwrap_or_default();
                            let parts: Vec<&str> = u.trim_start_matches("at://").splitn(3, '/').collect();
                            if parts.len() == 3 {
                                call("com.atproto.repo.deleteRecord", json!({ "repo": did, "collection": parts[1], "rkey": parts[2] })).await?;
                                out.push(json!({ "uri": u }));
                            }
                        }
                    }
                    Ok(json!({ "records": out }))
                }
                .await
            }
            "root.exec" => {
                let Some(host) = (self.target.name != "prod")
                    .then(|| self.target.root_hosts.first().cloned())
                    .flatten()
                else {
                    return Some(Err(
                        "root steps run only on a staging target with rootHosts".into(),
                    ));
                };
                let cmd = input["cmd"].as_str().unwrap_or_default().to_string();
                async {
                    let mut c = tokio::process::Command::new("ssh");
                    c.args(["-o", "BatchMode=yes", &host, &cmd]).stdout(Stdio::piped()).stderr(Stdio::piped()).stdin(Stdio::piped());
                    let mut child = c.spawn().map_err(|e| e.to_string())?;
                    let mut sin = child.stdin.take().unwrap();
                    if input["stdinPassword"] == true {
                        use tokio::io::AsyncWriteExt;
                        sin.write_all(self.password()?.as_bytes()).await.map_err(|e| e.to_string())?;
                    }
                    drop(sin);
                    let out = child.wait_with_output().await.map_err(|e| e.to_string())?;
                    Ok(json!({ "exit": out.status.code(), "stdout": String::from_utf8_lossy(&out.stdout), "stderr": tail(&String::from_utf8_lossy(&out.stderr)) }))
                }
                .await
            }
            "root.putBinary" => {
                let Some(host) = (self.target.name != "prod")
                    .then(|| self.target.root_hosts.first().cloned())
                    .flatten()
                else {
                    return Some(Err(
                        "root steps run only on a staging target with rootHosts".into(),
                    ));
                };
                let remote = "/root/swc-st-socialweb-computer";
                // several entries share this path (0001:s2, 0013:b, 0020): the copy goes beside it and is renamed over
                // it, so an entry running the binary keeps its file and none sees a half-written one
                let part = format!("{remote}.{}.part", std::process::id());
                async {
                    let ok = tokio::process::Command::new("scp")
                        .args(["-q", "-o", "BatchMode=yes"])
                        .arg(&self.bin.path)
                        .arg(format!("{host}:{part}"))
                        .status()
                        .await
                        .map_err(|e| e.to_string())?
                        .success();
                    if !ok {
                        return Err(format!("scp to {host} failed"));
                    }
                    let ok = tokio::process::Command::new("ssh")
                        .args(["-o", "BatchMode=yes", &host, "mv", "-f", &part, remote])
                        .status()
                        .await
                        .map_err(|e| e.to_string())?
                        .success();
                    if ok {
                        Ok(json!({ "path": remote }))
                    } else {
                        Err(format!("mv on {host} failed"))
                    }
                }
                .await
            }
            _ => return None,
        })
    }

    pub(crate) async fn more_sub(&self, what: &str, input: &Value) -> Option<Result<Sub, String>> {
        Some(match what {
            "atirc.observer" => self.observer_sub(input).await,
            "requester.events" => match str_of(input, "name") {
                Ok(n) => {
                    let state = self.requester_state(n);
                    let main = state.join(format!("{n}.log"));
                    Ok(tail_sub(
                        move || {
                            let mut v = vec![main.clone()];
                            find_named(&state, "core.log", &mut v);
                            v
                        },
                        None,
                        market_lines,
                    ))
                }
                Err(e) => Err(e),
            },
            _ => return None,
        })
    }

    pub(crate) async fn more_cleanup(&self) -> Vec<String> {
        let mut errors = vec![];
        let upgraded: Vec<(String, String)> = self.extra.upgraded.lock().unwrap().clone();
        for (name, from) in upgraded {
            let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
            let Some(testkit::target::Via::Ssh { host }) = via else {
                errors.push(format!("bidder {name}: left on the from binary (no ssh host)"));
                continue;
            };
            if let Err(e) = self.bidder_upgrade(&name, &host, "test", Some(&from), false).await {
                errors.push(format!("bidder {name} back to the binary under test: {e}"));
            }
        }
        for (_, (_, _, mut c)) in self.extra.nodes.lock().await.drain() {
            let _ = c.kill().await;
        }
        let nicks: Vec<String> = self.extra.tokens.lock().unwrap().keys().cloned().collect();
        for n in nicks {
            if let Err(e) = self.observer_run(&n, &["revoke"]).await {
                errors.push(format!("revoke observer {n}: {e}"));
            }
        }
        let keys: Vec<_> = std::mem::take(&mut *self.extra.keys.lock().unwrap());
        for (web, rkey) in keys {
            if let Err(e) = self
                .page_fetch(&format!("{web}/keys/{rkey}"), "DELETE", None)
                .await
            {
                errors.push(format!("remove the temporary key: {e}"));
            }
        }
        errors
    }
}

impl Swc {
    /// resources.json: everything this run started or created, how to reach it, no secret (paths only).
    pub async fn resources(&self) -> Value {
        let requesters: Vec<Value> = self
            .requesters
            .lock()
            .await
            .iter()
            .map(|(n, (state, c))| {
                json!({ "name": n, "state": state, "pid": c.id(), "cluster": format!("requester:{n}"), "log": state.join(format!("{n}.log")),
                         "ssh": format!("socialweb-computer run requester ssh {n} --state-dir {} --contract <cc>", state.display()) })
            })
            .collect();
        let requests: Vec<Value> = self
            .requests
            .lock()
            .unwrap()
            .iter()
            .map(|(n, state)| {
                let summary: Value = std::fs::read(state.join("requests").join(n).join("summary.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default();
                json!({ "name": n, "state": state, "receipt": summary["contract"]["receiptUri"], "winner": summary["contract"]["winnerDid"], "logs": self.request_logs(n),
                         "ssh": format!("socialweb-computer run request ssh {n} --state-dir {}", state.display()) })
            })
            .collect();
        json!({
            "account": self.acct().map(|a| a.did.clone()).unwrap_or_default(),
            "requesters": requesters,
            "requests": requests,
            "kube": self.kube.ledger(),
            "clusters": self.kube.clusters(),
            "receipts": *self.receipts.lock().unwrap(),
            "keys": self.extra.keys.lock().unwrap().iter().map(|(w, r)| json!({ "web": w, "rkey": r })).collect::<Vec<_>>(),
            "observers": self.extra.tokens.lock().unwrap().keys().cloned().collect::<Vec<_>>(),
            "envFiles": *self.env_files.lock().unwrap(),
        })
    }

    pub async fn write_resources(&self) {
        let v = self.resources().await;
        let _ = std::fs::write(
            self.dump.join("resources.json"),
            serde_json::to_string_pretty(&v).unwrap_or_default(),
        );
    }
}

/// `swc-test-host cleanup <run>`: from resources.json alone (the host that ran it may be gone): release the
/// invocations, stop the requesters, delete and restore kube objects, remove the temporary keys, revoke observer
/// tokens (a fresh account token revokes the subhandle), remove the env files. The errors.
pub async fn cleanup_from_disk(
    dir: &std::path::Path,
    target: &testkit::target::Target,
    account: &str,
    bin: &testkit::requester::Binary,
) -> Vec<String> {
    let mut errors = vec![];
    let Ok(text) = std::fs::read_to_string(dir.join("resources.json")) else {
        return vec![format!("{}: no resources.json", dir.display())];
    };
    let r: Value = serde_json::from_str(&text).unwrap_or_default();
    for q in r["requests"].as_array().into_iter().flatten() {
        let (n, st) = (
            q["name"].as_str().unwrap_or_default(),
            q["state"].as_str().unwrap_or_default(),
        );
        match bin
            .run(&["run", "request", "release", n, "--state-dir", st])
            .await
        {
            Ok((true, _)) => {}
            Ok((false, out)) => errors.push(format!("release {n}: {}", tail(&out))),
            Err(e) => errors.push(format!("release {n}: {e}")),
        }
    }
    let clusters: HashMap<String, Cluster> =
        serde_json::from_value(r["clusters"].clone()).unwrap_or_default();
    let kube = atlibs_kube::Kube::new(clusters, None);
    if let Err(e) = kube.load_ledger(&r["kube"]) {
        errors.push(format!("kube ledger: {e}"));
    }
    errors.extend(kube.cleanup().await);
    // a deleted Infra's finalizer (it releases the contracts) runs in its requester: wait for it before the stop
    for k in r["kube"]["created"].as_array().into_iter().flatten() {
        let input = json!({ "cluster": k["cluster"], "apiVersion": k["apiVersion"], "kind": k["kind"], "name": k["name"] });
        for _ in 0..60 {
            let got = atlibs_host::run::Call::call(
                &*kube,
                atlibs_kube::GET.into(),
                atlibs_host::run::Request {
                    input: Some(input.clone()),
                    ..Default::default()
                },
            )
            .await;
            if !matches!(&got, Ok(Some(v)) if v.get("metadata").is_some()) {
                break;
            }
            tokio::time::sleep(Duration::from_secs(10)).await;
        }
    }
    for q in r["requesters"].as_array().into_iter().flatten() {
        if let Some(pid) = q["pid"].as_u64()
            && std::path::Path::new(&format!("/proc/{pid}")).exists()
        {
            let _ = std::process::Command::new("kill")
                .args(["-INT", &pid.to_string()])
                .status();
        }
    }
    let Ok(acct) = target.account(account) else {
        return errors;
    };
    let at = testkit::Atproto::new(target);
    let keys = r["keys"].as_array().is_some_and(|k| !k.is_empty());
    if keys && let Ok(s) = at.session(acct).await {
        for k in r["keys"].as_array().into_iter().flatten() {
            if let Err(e) = at
                .delete_record(
                    &s,
                    "com.publicdomainrelay.temp.badgeBlueKeys",
                    k["rkey"].as_str().unwrap_or_default(),
                )
                .await
            {
                errors.push(format!("temporary key: {e}"));
            }
        }
    }
    if let (Some(endpoint), Ok(atirc)) = (target.atirc.clone(), testkit::atirc::atirc_bin()) {
        for nick in r["observers"]
            .as_array()
            .into_iter()
            .flatten()
            .filter_map(Value::as_str)
        {
            let pw = acct
                .password()
                .map(|p| p.expose().to_string())
                .unwrap_or_default();
            let out = tokio::process::Command::new(&atirc)
                .args(["token", "--endpoint", &endpoint])
                .env("ATP_USERNAME", &acct.handle)
                .env("ATP_PASSWORD", pw)
                .env("ATIRC_PLC_DIRECTORY_URL", &target.plc)
                .stdin(Stdio::null())
                .output()
                .await;
            let Ok(out) = out else { continue };
            let token = String::from_utf8_lossy(&out.stdout).trim().to_string();
            let ok = tokio::process::Command::new(&atirc)
                .args(["revoke", &format!("{nick}.{}", acct.handle)])
                .env("ATIRC_TOKEN", token)
                .env("ATIRC_ENDPOINT", &endpoint)
                .env("ATIRC_PLC_DIRECTORY_URL", &target.plc)
                .stdin(Stdio::null())
                .stdout(Stdio::null())
                .stderr(Stdio::null())
                .status()
                .await
                .is_ok_and(|s| s.success());
            if !ok {
                errors.push(format!("revoke observer {nick}"));
            }
        }
    }
    for p in r["envFiles"]
        .as_array()
        .into_iter()
        .flatten()
        .filter_map(Value::as_str)
    {
        let _ = std::fs::remove_file(p);
    }
    errors
}

/// bidder.restart on a joined bidder's host, as root or as the bidder's account: the installed bidder @NAME@'s core unit restarted through
/// its account's user manager (what `upgrade` restarts: install.EnabledCoreUnits). Prints `since <epoch>` (the start of the process it
/// replaced) and the unit's MainPID before and after; exit 1 when @NAME@ has no installed unit here or the restart failed.
const BIDDER_RESTART: &str = r#"unit=""
for p in $(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder'); do
  u=$(grep -o '[^/]*bidder@@NAME@\.service' /proc/$p/cgroup 2>/dev/null | tail -1)
  [ -n "$u" ] || continue
  pid=$p; unit=$u; break
done
[ -n "$unit" ] || { echo 'no installed bidder unit of @NAME@ on this host (install bidder)'; exit 1; }
user=$(ps -o user= -p $pid | tr -d ' ')
U=$(id -u "$user")
since=$(date -d "$(ps -o lstart= -p $pid)" +%s)
sc() { if [ "$(id -u)" = "$U" ]; then XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; else runuser -u "$user" -- env XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; fi; }
@UNIT_TIMEOUTS@
echo "budget $(( stop_s + start_s ))"
before=$(sc show "$unit" -p MainPID --value)
sc restart "$unit" || { echo "systemctl --user restart $unit failed"; sc status "$unit" --no-pager 2>&1 | tail -5; exit 1; }
after=$(sc show "$unit" -p MainPID --value)
[ -n "$after" ] && [ "$after" != 0 ] && [ "$after" != "$before" ] || { echo "$unit: MainPID $before -> $after after the restart"; exit 1; }
echo "since $since"
echo "restarted $unit of $user: MainPID $before -> $after, $(sc show "$unit" -p ActiveState --value)"
"#;

/// the installed unit's own systemd start and stop timeouts in seconds (start_s, stop_s), for a script's budget line
const UNIT_TIMEOUTS: &str = r#"tsec() { systemd-analyze timespan "$(sc show "$unit" -p "$1" --value)" 2>/dev/null | awk 'NF == 2 && $2 ~ /^[0-9]+$/ { print int($2 / 1000000); exit }'; }
start_s=$(tsec TimeoutStartUSec); stop_s=$(tsec TimeoutStopUSec); start_s=${start_s:-0}; stop_s=${stop_s:-0}"#;

/// bidder.logCount on a bidder host: the swc-bidder account's journal since its run bidder process started (`since`, when set: from
/// that epoch instead, the process a restart of this run replaced).
const BIDDER_LOG: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || pid=$(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || exit 0
user=$(ps -o user= -p $pid | tr -d ' ')
if grep -q '\.service' /proc/$pid/cgroup && ! grep -q 'session-[0-9]*\.scope' /proc/$pid/cgroup; then
  journalctl _UID=$(id -u "$user") --since "@${since:-$(date -d "$(ps -o lstart= -p $pid)" +%s)}" -o cat --no-pager 2>/dev/null
else
  cat $(eval echo "~$user")/.local/state/socialweb-computer/dev/b-*/logs/core.log 2>/dev/null
fi
"#;

/// bidder.guestNft on a bidder host, as root: the VMM of @UNIT@ is its process in a network namespace other than pid 1's.
const GUEST_NFT: &str = r#"unit='@UNIT@'; op='@OP@'; min=@MIN@; max=@MAX@
command -v nft >/dev/null || { echo "nft is not installed on $(hostname)"; exit 1; }
pid=
for f in /proc/[0-9]*/cgroup; do
  grep -q "/$unit\$" "$f" 2>/dev/null || continue
  p=${f#/proc/}; p=${p%/cgroup}
  [ "$(readlink /proc/$p/ns/net)" != "$(readlink /proc/1/ns/net)" ] && { pid=$p; break; }
done
[ -n "$pid" ] || { echo "no process of $unit in a guest network namespace on $(hostname)"; exit 1; }
echo "vmm-pid $pid"
own=$(ip -4 -o addr show scope global | awk '{split($4,a,"/"); printf "%s%s", s, a[1]; s=", "}')
echo "own $own"
n() { nsenter -t "$pid" -n nft "$@"; }
case "$op" in
  list) n list table bridge swcguest ;;
  probe)
    n delete table bridge swcprobe 2>/dev/null
    n -f - <<EOF || exit 1
table bridge swcprobe {
  chain sent { type filter hook prerouting priority -301; policy accept; iifname "fctap0" ip daddr { $own } udp dport $min-$max counter; }
  chain passed { type filter hook prerouting priority -299; policy accept; iifname "fctap0" ip daddr { $own } udp dport $min-$max counter; }
}
EOF
    n list table bridge swcprobe ;;
  counters) n list table bridge swcprobe ;;
  unprobe) n delete table bridge swcprobe ;;
esac
"#;

/// bidder.awaitLog on a bidder host: the installed bidder's journal from its run bidder process's start, followed until a
/// line holds every text (one line "line <text>"), at most @SECS@ seconds.
const AWAIT_LOG: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || { echo "no run bidder process of swc-bidder on $(hostname)"; exit 1; }
start=$(date -d "$(ps -o lstart= -p $pid)" +%s)
echo "budget @SECS@"
timeout @SECS@ journalctl _UID=$(id -u swc-bidder) --since "@$start" -o cat -f --no-pager 2>/dev/null |
  T=@TEXTS@ awk 'BEGIN { n = split(ENVIRON["T"], a, "\037") } { for (i = 1; i <= n; i++) if (!index($0, a[i])) next; print "line " $0; fflush(); exit }'
exit 0
"#;

/// bidder.dumbpipe on a bidder host: the run bidder process ("core <pid>") and each dumbpipe child of it ("listener <pid>
/// <exe> <argv>"), from /proc, no sudo.
const BIDDER_DUMBPIPE: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || pid=$(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || { echo "no run bidder process of swc-bidder on $(hostname)"; exit 1; }
echo "core $pid"
for c in $(pgrep -P "$pid" -x dumbpipe); do
  echo "listener $c $(readlink /proc/$c/exe 2>/dev/null) $(tr '\0' ' ' < /proc/$c/cmdline 2>/dev/null)"
done
exit 0
"#;

/// bidder.meminfo on a bidder host: MemTotal and MemAvailable in MiB, and the anon MiB of each named VM unit's cgroup
/// memory.stat (`units` holds vm=unit pairs), as the bidder reads them; a unit without a cgroup is left out.
const MEMINFO: &str = r#"awk '/^MemTotal:/{t=$2} /^MemAvailable:/{a=$2} END{printf "{\"memTotalMiB\":%d,\"memAvailableMiB\":%d,\"anonMiB\":{", t/1024, a/1024}' /proc/meminfo
sep=
for u in $units; do
  vm=${u%%=*}; unit=${u#*=}
  d=$(find /sys/fs/cgroup -type d -name "$unit" -print -quit 2>/dev/null)
  [ -n "$d" ] || continue
  a=$(awk '$1=="anon"{printf "%d", $2/1048576}' "$d/memory.stat" 2>/dev/null)
  [ -n "$a" ] || continue
  printf '%s"%s":%s' "$sep" "$vm" "$a"; sep=,
done
echo '}}'
"#;

/// bidder.health on a bidder host: read-only, no sudo (journal of the account swc-bidder, /proc), one JSON line.
const BIDDER_HEALTH: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || pid=$(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || { echo '{"error":"no run bidder process"}'; exit 0; }
user=$(ps -o user= -p $pid | tr -d ' ')
unit=$(grep -o '[^/]*\.service' /proc/$pid/cgroup | tail -1)
start=$(date -d "$(ps -o lstart= -p $pid)" +%s)
U=$(id -u "$user")
# the lines of this core start: every line of the core unit's current invocation (the core and its forks), and every
# other line of the bidder's account (the units the core starts) logged at or after this process's start instant (its
# /proc/<pid>/stat clock ticks against a process started after the wall clock is read: never late; less 30 ms, two
# ticks and slack; a line by its own `time`, else its journal time); never a core unit line of another invocation (the
# previous core, also when its shutdown overlaps this start); a `run bidder --fg` (staging) writes its instance log,
# whose lines name no pid: those from the same start instant
d=$(date +%s.%N)
rst=$(sh -c 'sed "s/^.*) //" /proc/$$/stat | cut -d" " -f20')
t0=$(awk -v st="$(sed 's/^.*) //' /proc/$pid/stat | cut -d' ' -f20)" -v hz="$(getconf CLK_TCK)" -v d="$d" -v rst="$rst" \
  'BEGIN { printf "%.3f", d + (st - rst) / hz - 0.03 }')
until=$(awk -v t="$t0" 'BEGIN { printf "%.3f", t + 600 }')
since=$(( ${t0%.*} - 60 ))
epoch='def epoch: capture("^(?<d>[0-9-]{10}T[0-9:]{8})(?<f>[.][0-9]+)?(?<z>Z|[+-][0-9]{2}:[0-9]{2})$")
  | (.d + "Z" | fromdateiso8601) + ("0" + (.f // "") | tonumber)
    - (if .z == "Z" then 0 else (if .z[0:1] == "-" then -1 else 1 end) * ((.z[1:3] | tonumber) * 3600 + (.z[4:6] | tonumber) * 60) end);'
if [ -n "$unit" ] && ! grep -q 'session-[0-9]*\.scope' /proc/$pid/cgroup; then
  inv=$( (tr '\0' '\n' < /proc/$pid/environ) 2>/dev/null | sed -n 's/^INVOCATION_ID=//p')
  [ -n "$inv" ] || inv=$(journalctl _PID=$pid --since "@$since" -n 1 -o export --no-pager 2>/dev/null | sed -n 's/^_SYSTEMD_INVOCATION_ID=//p' | head -1)
  log() {
    c=1e12
    [ "${1:-}" = --until ] && c=${2#@}
    journalctl _UID=$U --since "@$since" -o json --no-pager 2>/dev/null | jq -r --argjson s "$t0" --argjson c "$c" --arg unit "$unit" \
      --arg inv "$inv" --arg pid "$pid" "$epoch"'
      select((.MESSAGE | type) == "string")
      | (._SYSTEMD_USER_UNIT == $unit and (._PID == $pid or ($inv != "" and ._SYSTEMD_INVOCATION_ID == $inv))) as $core
      | select($core or ._SYSTEMD_USER_UNIT != $unit or $inv == "")
      | select(((.MESSAGE | fromjson? | .time | strings | epoch) // ((.__REALTIME_TIMESTAMP | tonumber) / 1000000)) as $t
        | ($core or $t >= $s) and $t < $c)
      | .MESSAGE'
  }
else
  unit=""
  log() {
    f=$(ls $(eval echo "~$user")/.local/state/socialweb-computer/dev/b-*/logs/core.log 2>/dev/null | head -1)
    [ -n "$f" ] || return 0
    c=1e12
    [ "${1:-}" = --until ] && c=${2#@}
    jq -cR --argjson s "$t0" --argjson c "$c" "$epoch"'fromjson? | select(((.time // "") | epoch) as $t | $t >= $s and $t < $c)' "$f"
  }
fi
lines=$(log)
errors=$(grep -c '"level":"ERROR"' <<<"$lines")
startup=$(log --until "@$until" | grep -c '"level":"ERROR"\|unsupported type: source.TypedFunc')
sample=$(grep '"level":"ERROR"' <<<"$lines" | tail -3 | jq -c '{time, msg, err: ((.err // .error // "") | tostring | .[0:160])}' | jq -sc .)
restarts=$(journalctl _UID=$U USER_UNIT=$unit --since -14d -o cat --no-pager 2>/dev/null | awk '/Stopping /{n=0} /Scheduled restart job/{n++} END{print n+0}')
procs=$(( $(pgrep -c -u "$user" -x firecracker) + $(pgrep -c -u "$user" -f qemu-system) ))
ready=$(grep -m1 '"msg":"bidder_guest_image_ready"' <<<"$lines" | jq -r '.time // ""')
unready=$(grep -c '"msg":"bidder_guest_image_unready"' <<<"$lines")
jq -nc --arg unit "$unit" --arg started "$(date -u -d @$start +%FT%TZ)" \
  --argjson errors "$errors" --argjson startup "$startup" --argjson restarts "$restarts" --argjson procs "$procs" --argjson sample "${sample:-[]}" \
  --arg ready "$ready" --argjson unready "$unready" \
  '{unit: $unit, started: $started, errors: $errors, startupErrors: $startup, restarts: $restarts, vmProcesses: $procs, errorSample: $sample,
    guestImageReadyAt: $ready, guestImageUnready: $unready}'
"#;

impl Swc {
    async fn bidder_upgrade(&self, name: &str, host: &str, which: &str, from: Option<&str>, stop_first: bool) -> Result<Value, String> {
        let spec = match from {
            Some(f) => f.to_string(),
            None => std::env::var("SWC_UPGRADE_FROM")
                .or_else(|_| std::env::var("SWC_COMPAT_REQUESTER"))
                .map_err(|_| "bidder.upgrade: no from spec in the input and SWC_UPGRADE_FROM and SWC_COMPAT_REQUESTER are unset (the from binary: sha:<commit>, release:<version>, path:<file>)".to_string())?,
        };
        let from = Upgradable::of(&spec, None).await.map_err(|e| format!("bidder.upgrade: the from binary {spec}: {e}"))?;
        let test = Upgradable::of(&self.target.requester, Some(&self.bin.path)).await.map_err(|e| format!("bidder.upgrade: the binary under test {}: {e}", self.target.requester))?;
        let want = match which {
            "from" => &from,
            "test" => &test,
            w => return Err(format!("bidder.upgrade: binary {w}: from or test")),
        };
        let t = Instant::now();
        let (remote, how) = want.stage(host).await.map_err(|e| format!("bidder.upgrade {name}: {e}"))?;
        let staged_s = (t.elapsed().as_secs_f64() * 10.0).round() / 10.0;
        if which == "from" {
            let mut up = self.extra.upgraded.lock().unwrap();
            up.retain(|(n, _)| n != name);
            up.push((name.to_string(), spec.clone()));
        }
        let script = BIDDER_UPGRADE
            .replace("@UNIT_TIMEOUTS@", UNIT_TIMEOUTS)
            .replace("@NAME@", name)
            .replace("@BIN@", &remote)
            .replace("@SUM@", &want.sha256)
            .replace("@STOP@", if stop_first { "1" } else { "0" });
        let out = on_host(Some(host), &script).await.map_err(|e| format!("bidder.upgrade {name}: {e}"))?;
        let Some(running) = out.lines().find_map(|l| l.strip_prefix("running ")) else {
            return Err(format!("bidder.upgrade {name}: {}", tail(&out)));
        };
        if which == "test" {
            self.extra.upgraded.lock().unwrap().retain(|(n, _)| n != name);
        }
        Ok(json!({ "sha256": running.trim(), "from": from.sha256, "test": test.sha256, "fromSpec": spec, "staged": how, "stagedSeconds": staged_s, "report": tail(&out) }))
    }
}

/// A binary bidder.upgrade puts on a bidder host: its sha256, its public URL when it has one (release, sha:, local:),
/// else the local file the scp fallback copies (path:, oci:)
struct Upgradable {
    sha256: String,
    url: Option<String>,
    spec: String,
    local: Option<PathBuf>,
}

/// one download or copy of a binary onto a bidder host: the bound the other installs give the network (20 min)
const STAGE_BUDGET_S: u64 = 1200;

impl Upgradable {
    async fn of(spec: &str, local: Option<&std::path::Path>) -> Result<Self, String> {
        if let Some((url, sha256)) = testkit::requester::Binary::published(spec).await.map_err(|e| e.to_string())? {
            if let Some(l) = local {
                let have = file_sha256(l).await?;
                if have != sha256 {
                    return Err(format!("{} is {have}, but {spec} publishes {sha256}", l.display()));
                }
            }
            return Ok(Self { sha256, url: Some(url), spec: spec.to_string(), local: local.map(PathBuf::from) });
        }
        let path = match local {
            Some(l) => l.to_path_buf(),
            None => testkit::requester::Binary::resolve(spec).await.map_err(|e| e.to_string())?.path,
        };
        Ok(Self { sha256: file_sha256(&path).await?, url: None, spec: spec.to_string(), local: Some(path) })
    }

    /// `/var/tmp/swc-upgrade-<sha256>` on the host, checked against the sha256: kept there, or the install's own
    /// `bin/<sha256>/` copied beside it, or fetched by the host itself from the public URL, else (no URL) copied from
    /// this host over ssh at a rate that must fit the budget; how it got there
    async fn stage(&self, host: &str) -> Result<(String, String), String> {
        let remote = format!("/var/tmp/swc-upgrade-{}", self.sha256);
        let script = STAGE_UPGRADE
            .replace("@SUM@", &self.sha256)
            .replace("@URL@", self.url.as_deref().unwrap_or_default())
            .replace("@BUDGET@", &STAGE_BUDGET_S.to_string());
        let out = on_host(Some(host), &script).await?;
        if let Some(how) = out.lines().find_map(|l| l.strip_prefix("staged ")) {
            return Ok((remote, how.to_string()));
        }
        if !out.lines().any(|l| l == "nofetch") {
            return Err(format!("staging {} ({}) on {host}: {}", self.spec, self.sha256, tail(&out)));
        }
        let Some(local) = &self.local else {
            return Err(format!("staging {} on {host}: no public URL and no local file", self.spec));
        };
        let how = scp_within_budget(host, local, &remote).await?;
        Ok((remote, how))
    }
}

const STAGE_UPGRADE: &str = r#"f=/var/tmp/swc-upgrade-@SUM@
ok() { [ -f "$1" ] && [ "$(sha256sum "$1" | cut -d' ' -f1)" = @SUM@ ]; }
if ok "$f"; then echo "staged kept $f"; exit 0; fi
for p in $(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder'); do
  exe=$(readlink /proc/$p/exe)
  k="$(dirname "$(dirname "$exe")")/@SUM@/$(basename "$exe")"
  if ok "$k"; then cp -f "$k" "$f.part" && chmod 0755 "$f.part" && mv -f "$f.part" "$f" && echo "staged from the install's $k"; exit 0; fi
done
u='@URL@'
[ -n "$u" ] || { echo nofetch; exit 0; }
echo "budget @BUDGET@"
t0=$(date +%s)
curl -fsS --max-time @BUDGET@ -o "$f.part" "$u"; rc=$?
if [ $rc != 0 ]; then
  echo "fetching $u failed (curl exit $rc) after $(( $(date +%s) - t0 )) s with $(stat -c %s "$f.part" 2>/dev/null || echo 0) bytes (bound @BUDGET@ s)"
  rm -f "$f.part"; exit 1
fi
ok "$f.part" || { echo "$u does not hash to @SUM@"; rm -f "$f.part"; exit 1; }
chmod 0755 "$f.part" && mv -f "$f.part" "$f"
s=$(( $(date +%s) - t0 )); b=$(stat -c %s "$f")
echo "staged fetched $b bytes in $s s ($(( b / (s > 0 ? s : 1) / 1048576 )) MiB/s) from $u"
"#;

static FILE_SUMS: std::sync::OnceLock<Mutex<HashMap<PathBuf, String>>> = std::sync::OnceLock::new();

async fn file_sha256(path: &std::path::Path) -> Result<String, String> {
    let sums = FILE_SUMS.get_or_init(Default::default);
    if let Some(s) = sums.lock().unwrap().get(path) {
        return Ok(s.clone());
    }
    let out = tokio::process::Command::new("sha256sum").arg(path).output().await.map_err(|e| format!("sha256sum {}: {e}", path.display()))?;
    let sum = String::from_utf8_lossy(&out.stdout).split_whitespace().next().unwrap_or_default().to_string();
    if !out.status.success() || sum.len() != 64 {
        return Err(format!("sha256sum {}: {}", path.display(), String::from_utf8_lossy(&out.stderr)));
    }
    sums.lock().unwrap().insert(path.to_path_buf(), sum.clone());
    Ok(sum)
}

/// `local` onto a bidder host at `remote` over ssh (a path: binary of a hand run: no public URL), mode 0755. After 30 s
/// the copied bytes give the rate; a copy that would not end inside the budget stops there and says its rate
async fn scp_within_budget(host: &str, local: &std::path::Path, remote: &str) -> Result<String, String> {
    let size = std::fs::metadata(local).map_err(|e| format!("{}: {e}", local.display()))?.len();
    let part = format!("{remote}.{}.part", std::process::id());
    let t = Instant::now();
    let mut child = tokio::process::Command::new("scp")
        .args(["-q", "-o", "BatchMode=yes", "-o", "LogLevel=ERROR"])
        .arg(local)
        .arg(format!("{host}:{part}"))
        .kill_on_drop(true)
        .spawn()
        .map_err(|e| e.to_string())?;
    let status = tokio::select! {
        s = child.wait() => s.map_err(|e| e.to_string())?,
        _ = tokio::time::sleep(Duration::from_secs(30)) => {
            let done: u64 = on_host(Some(host), &format!("stat -c %s {part} 2>/dev/null || echo 0")).await?.trim().parse().unwrap_or(0);
            let rate = done as f64 / t.elapsed().as_secs_f64();
            let eta = if rate > 0.0 { size as f64 / rate } else { f64::INFINITY };
            if eta > STAGE_BUDGET_S as f64 {
                let _ = child.kill().await;
                let _ = on_host(Some(host), &format!("rm -f {part}")).await;
                return Err(format!(
                    "copying {} ({size} bytes) to {host} over ssh runs at {:.2} MiB/s: about {:.0} min, over the {} min budget; name the binary by sha:, local: or release: so the host fetches it",
                    local.display(), rate / 1048576.0, eta / 60.0, STAGE_BUDGET_S / 60
                ));
            }
            match tokio::time::timeout(Duration::from_secs(STAGE_BUDGET_S), child.wait()).await {
                Ok(s) => s.map_err(|e| e.to_string())?,
                Err(_) => {
                    let _ = child.kill().await;
                    return Err(format!("copying {} to {host} over ssh: not done in {} min", local.display(), STAGE_BUDGET_S / 60));
                }
            }
        }
    };
    if !status.success() {
        return Err(format!("scp {} to {host} failed", local.display()));
    }
    let o = on_host(Some(host), &format!("chmod 0755 {part} && mv -f {part} {remote} && echo staged")).await?;
    if !o.contains("staged") {
        return Err(format!("staging {remote} on {host}: {}", tail(&o)));
    }
    let s = t.elapsed().as_secs_f64();
    Ok(format!("copied {size} bytes in {s:.0} s ({:.2} MiB/s) over ssh", size as f64 / s.max(0.1) / 1048576.0))
}

const BIDDER_UPGRADE: &str = r#"unit=""; pid=""
for p in $(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder'); do
  u=$(grep -o '[^/]*bidder@@NAME@\.service' /proc/$p/cgroup 2>/dev/null | tail -1)
  [ -n "$u" ] || continue
  pid=$p; unit=$u; break
done
[ -n "$unit" ] || { echo 'no installed bidder unit of @NAME@ on this host (install bidder)'; exit 1; }
user=$(ps -o user= -p $pid | tr -d ' ')
U=$(id -u "$user")
exe=$(readlink /proc/$pid/exe)
dir=$(dirname "$exe"); bin=$(dirname "$dir")
[ "$(basename "$bin")" = bin ] || { echo "@NAME@ does not run from an install's bin/<sha256>/: $exe"; exit 1; }
ST=$(dirname "$bin")
sc() { if [ "$(id -u)" = "$U" ]; then XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; else runuser -u "$user" -- env XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; fi; }
@UNIT_TIMEOUTS@
recovery_s=60; wait_s=60
cores=$(ls "$(getent passwd "$user" | cut -d: -f6)/.config/systemd/user/default.target.wants" 2>/dev/null | grep -cE '^socialweb.*-computer-(bidder@.*|requester@.*|gateway)\.service$')
echo "budget $(( stop_s + (cores > 0 ? cores : 1) * (stop_s + start_s + recovery_s) + start_s + wait_s ))"
if [ "$(basename "$dir")" = @SUM@ ]; then echo "running @SUM@"; echo "$unit already runs bin/@SUM@"; exit 0; fi
if [ @STOP@ = 1 ]; then sc stop "$unit" || { echo "systemctl --user stop $unit failed"; exit 1; }; fi
out=$(runuser -u "$user" -- env XDG_RUNTIME_DIR=/run/user/$U "$ST/bin/current/socialweb-computer" upgrade --binary @BIN@ --sha256 @SUM@ --state-dir "$ST" 2>&1); rc=$?
sc start "$unit"
[ $rc = 0 ] || { echo "upgrade exit $rc: $(tail -c 1500 <<<"$out")"; exit 1; }
new=""
for i in $(seq 1 $wait_s); do new=$(sc show "$unit" -p MainPID --value); [ -n "$new" ] && [ "$new" != 0 ] && [ "$new" != "$pid" ] && break; sleep 1; done
nexe=$(readlink /proc/$new/exe 2>/dev/null)
case "$nexe" in "$bin/@SUM@/"*) ;; *) echo "$unit runs ${nexe:-nothing} (MainPID ${new:-0}) after the upgrade, not $bin/@SUM@/"; exit 1 ;; esac
echo "running @SUM@"
echo "upgraded $unit of $user: $(basename "$dir") -> @SUM@, MainPID $pid -> $new"
"#;

const HOST_SCRIPT_S: u64 = 120;

/// A script on this host (`bash -s`) or on an ssh alias (`ssh -o BatchMode=yes <host> bash -s`), its stdout. It ends
/// within HOST_SCRIPT_S, or, when it states the budget of its bounded steps (a stdout line `budget <seconds>`, the first
/// one, not returned), within that budget plus HOST_SCRIPT_S.
async fn on_host(host: Option<&str>, script: &str) -> Result<String, String> {
    use tokio::io::AsyncWriteExt;
    let mut cmd = match host {
        Some(h) => {
            let mut c = tokio::process::Command::new("ssh");
            c.args([
                "-o",
                "BatchMode=yes",
                "-o",
                "LogLevel=ERROR",
                h,
                "bash",
                "-s",
            ]);
            c
        }
        None => {
            let mut c = tokio::process::Command::new("bash");
            c.arg("-s");
            c
        }
    };
    let mut child = cmd
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .kill_on_drop(true)
        .spawn()
        .map_err(|e| e.to_string())?;
    child
        .stdin
        .take()
        .ok_or("no stdin")?
        .write_all(script.as_bytes())
        .await
        .map_err(|e| e.to_string())?;
    let mut stdout = tokio::io::BufReader::new(child.stdout.take().ok_or("no stdout")?);
    let mut stderr = child.stderr.take().ok_or("no stderr")?;
    let errs = tokio::spawn(async move {
        let mut b = Vec::new();
        let _ = tokio::io::AsyncReadExt::read_to_end(&mut stderr, &mut b).await;
        b
    });
    let t0 = Instant::now();
    let mut budget: Option<u64> = None;
    let left = |budget: Option<u64>| Duration::from_secs(HOST_SCRIPT_S.saturating_add(budget.unwrap_or(0))).saturating_sub(t0.elapsed());
    let cut = |budget: Option<u64>| match budget {
        Some(b) => format!("the host script did not end in {} s (its stated budget {b} s + {HOST_SCRIPT_S} s)", HOST_SCRIPT_S.saturating_add(b)),
        None => format!("the host script did not end in {HOST_SCRIPT_S} s"),
    };
    let mut out = Vec::new();
    loop {
        let mut line = Vec::new();
        let n = tokio::time::timeout(left(budget), stdout.read_until(b'\n', &mut line))
            .await
            .map_err(|_| cut(budget))?
            .map_err(|e| e.to_string())?;
        if n == 0 {
            break;
        }
        if budget.is_none() {
            if let Some(b) = std::str::from_utf8(&line).ok().and_then(|l| l.trim_end().strip_prefix("budget ")).and_then(|b| b.parse().ok()) {
                budget = Some(b);
                continue;
            }
        }
        out.extend_from_slice(&line);
    }
    let status = tokio::time::timeout(left(budget), child.wait())
        .await
        .map_err(|_| cut(budget))?
        .map_err(|e| e.to_string())?;
    if !status.success() && out.is_empty() {
        let err = errs.await.unwrap_or_default();
        return Err(format!(
            "{}: {}",
            host.unwrap_or("this host"),
            String::from_utf8_lossy(&err).trim()
        ));
    }
    Ok(String::from_utf8_lossy(&out).into_owned())
}

const TAMPERED_DOWNLOAD: &str = "socialweb-guest-agent-x86_64";

async fn sha256_of(p: &std::path::Path) -> Result<String, String> {
    let out = tokio::process::Command::new("sha256sum")
        .arg(p)
        .output()
        .await
        .map_err(|e| e.to_string())?;
    String::from_utf8_lossy(&out.stdout)
        .split_whitespace()
        .next()
        .map(String::from)
        .ok_or_else(|| {
            format!(
                "sha256sum {}: {}",
                p.display(),
                String::from_utf8_lossy(&out.stderr)
            )
        })
}

/// The extracted tree of socialweb-guest-downloads in an asset cache: the directory that holds the tampered download.
fn downloads_tree(cache: &std::path::Path) -> Result<PathBuf, String> {
    std::fs::read_dir(cache)
        .map_err(|e| format!("{}: {e}", cache.display()))?
        .filter_map(Result::ok)
        .map(|e| e.path())
        .find(|p| p.join(format!("{TAMPERED_DOWNLOAD}.sha256")).is_file())
        .ok_or_else(|| {
            format!(
                "no extracted socialweb-guest-downloads tree under {}",
                cache.display()
            )
        })
}

async fn remove_tree(p: &std::path::Path) {
    let _ = tokio::process::Command::new("chmod")
        .args(["-R", "u+w"])
        .arg(p)
        .stderr(Stdio::null())
        .status()
        .await;
    let _ = std::fs::remove_dir_all(p);
}

impl Swc {
    async fn verify_assets(
        &self,
        bin: &std::path::Path,
        cache: &std::path::Path,
    ) -> Result<(bool, String), String> {
        let out = tokio::process::Command::new(bin)
            .args(["dev", "verify-assets", "--asset-cache"])
            .arg(cache)
            .stdin(Stdio::null())
            .output()
            .await
            .map_err(|e| format!("{}: {e}", bin.display()))?;
        let mut text = String::from_utf8_lossy(&out.stderr).into_owned();
        if !out.status.success() {
            text.push_str(&String::from_utf8_lossy(&out.stdout));
        }
        Ok((out.status.success(), text))
    }

    async fn assets_tamper(&self, input: &Value) -> Result<Value, String> {
        let variant = str_of(input, "variant")?;
        let root = expand("~/.tmp/swc-system-tests/assets-tamper")
            .join(format!("{}-{variant}", std::process::id()));
        remove_tree(&root).await;
        std::fs::create_dir_all(&root).map_err(|e| format!("{}: {e}", root.display()))?;
        let r = self.assets_tamper_in(variant, &root).await;
        remove_tree(&root).await;
        r
    }

    async fn assets_tamper_in(
        &self,
        variant: &str,
        root: &std::path::Path,
    ) -> Result<Value, String> {
        let bin = self.bin.path.clone();
        let cache = root.join("cache");
        let (ok, first) = self.verify_assets(&bin, &cache).await?;
        if !ok {
            return Err(format!(
                "dev verify-assets on an untouched cache failed (this binary embeds no verifiable assets?): {}",
                tail(&first)
            ));
        }
        let tree = downloads_tree(&cache)?;
        match variant {
            "file" | "fileSidecar" => {
                let (f, side) = (
                    tree.join(TAMPERED_DOWNLOAD),
                    tree.join(format!("{TAMPERED_DOWNLOAD}.sha256")),
                );
                let (want, want_side) = (
                    sha256_of(&f).await?,
                    std::fs::read(&side).map_err(|e| e.to_string())?,
                );
                let _ = tokio::process::Command::new("chmod")
                    .args(["-R", "u+w"])
                    .arg(&tree)
                    .status()
                    .await;
                let mut b = std::fs::read(&f).map_err(|e| e.to_string())?;
                let n = b.len();
                b[n / 2] ^= 1;
                std::fs::write(&f, &b).map_err(|e| e.to_string())?;
                if variant == "fileSidecar" {
                    let now = sha256_of(&f).await?;
                    std::fs::write(&side, format!("{now}  {TAMPERED_DOWNLOAD}\n"))
                        .map_err(|e| e.to_string())?;
                }
                let tampered = sha256_of(&f).await? != want;
                let (verify_ok, text) = self.verify_assets(&bin, &cache).await?;
                let healed = downloads_tree(&cache).is_ok_and(|t| t == tree)
                    && sha256_of(&f).await.is_ok_and(|h| h == want)
                    && std::fs::read(&side).is_ok_and(|s| s == want_side);
                Ok(
                    json!({ "variant": variant, "tampered": tampered, "verifyOk": verify_ok, "healed": healed, "output": tail(&text) }),
                )
            }
            "record" => {
                let name = tree
                    .file_name()
                    .and_then(|n| n.to_str())
                    .ok_or("tree name")?
                    .to_string();
                let mut bytes =
                    std::fs::read(&bin).map_err(|e| format!("{}: {e}", bin.display()))?;
                let digest = name.as_bytes();
                let mut flipped = 0;
                let mut at = 0;
                while let Some(i) = bytes[at..].windows(digest.len()).position(|w| w == digest) {
                    let i = at + i;
                    bytes[i] = if bytes[i] == b'0' { b'1' } else { b'0' };
                    flipped += 1;
                    at = i + digest.len();
                }
                if flipped == 0 {
                    return Err(format!(
                        "the tree digest {name} of the embedded build record is not in {}",
                        bin.display()
                    ));
                }
                let copy = root.join("swc-tampered");
                std::fs::write(&copy, &bytes).map_err(|e| e.to_string())?;
                let _ = tokio::process::Command::new("chmod")
                    .args(["0755"])
                    .arg(&copy)
                    .status()
                    .await;
                let cache1 = root.join("cache1");
                let (ok, text) = self.verify_assets(&copy, &cache1).await?;
                Ok(
                    json!({ "variant": variant, "recordFlipped": true, "flippedOccurrences": flipped, "refused": !ok && text.contains("unpacks to tree"),
                           "extractedNothing": downloads_tree(&cache1).is_err(), "output": tail(&text) }),
                )
            }
            other => Err(format!("variant {other:?}: file, fileSidecar or record")),
        }
    }
}

/// The accepts a bidder declined, by these log files (`testkit::requester::declined_accepts`).
pub(super) fn declined_accepts(paths: &[PathBuf]) -> Vec<String> {
    testkit::requester::declined_accepts(&log_lines_all(paths, testkit::requester::ACCEPT_DECLINED))
}

/// Every JSON log line of `msg` in these files.
fn log_lines_all(paths: &[PathBuf], msg: &str) -> Vec<Value> {
    paths
        .iter()
        .filter_map(|p| std::fs::read_to_string(p).ok())
        .flat_map(|t| t.lines().filter(|l| l.contains(msg)).filter_map(|l| serde_json::from_str::<Value>(l).ok()).collect::<Vec<_>>())
        .filter(|v| v["msg"] == msg)
        .collect()
}