~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
2018 lines · 89.1 KB Raw History
//! computer.socialweb.test.*: the socialweb.computer side of the test host. The requester under test (a release, a
//! commit or a local build) as the account: `run requester` (the orchestrator, its kcp registered as kube cluster
//! `requester:<name>`), `run request` invocations (a subscription of their log events), the owner's atirc mesh node.
//! Every requester and invocation it starts is stopped or released by `cleanup()`, however the run ended.
//! com.fedfork.exec.identlink.{walk,plc}: identlink against the target's PLC.

use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use atlibs_host::BoxFut;
use atlibs_host::run::{Call, Request, Sub, SubCmd};
use atlibs_host::transport::Transport;
use atlibs_kube::{Cluster, Kube};
use serde_json::{Value, json};
use testkit::requester::{Binary, SignIn, exec_line, identity_args, kept_by_rule, request_line, split_exec, tail, uris_in};
use testkit::target::{Account, Target, check_private, expand};
use tokio::sync::mpsc;

#[path = "knotless.rs"]
mod knotless;
#[path = "more.rs"]
mod more;
pub use more::cleanup_from_disk;

const P: &str = "computer.socialweb.test.";
/// Where the guest requester reads its association ref (the renter writes it), and how long it may take to come.
const R2_REF: &str = "/run/socialweb-r2/association-ref";
const R2_REF_WAIT: Duration = Duration::from_secs(15 * 60);
const PUT_RETRIES: u32 = 3;
const PUT_RETRY_BACKOFF: Duration = Duration::from_secs(10);
pub const NSIDS: &[&str] = &[
    "computer.socialweb.test.requester.start",
    "computer.socialweb.test.request.start",
    "computer.socialweb.test.request.ssh",
    "computer.socialweb.test.request.put",
    "computer.socialweb.test.request.agent",
    "computer.socialweb.test.request.summary",
    "computer.socialweb.test.request.release",
    "computer.socialweb.test.request.kill",
    "computer.socialweb.test.devsup.race",
    "computer.socialweb.test.devsup.samename",
    "computer.socialweb.test.request.state",
    "computer.socialweb.test.request.sshTry",
    "computer.socialweb.test.request.setPhase",
    "computer.socialweb.test.request.releaseTry",
    "computer.socialweb.test.request.removeState",
    "computer.socialweb.test.request.records",
    "computer.socialweb.test.account.envFile",
    "computer.socialweb.test.atirc.meshSub",
    "computer.socialweb.test.atirc.meshRun",
    "computer.socialweb.test.atirc.meshPub",
    "computer.socialweb.test.atirc.meshPeers",
    "computer.socialweb.test.kube.addCluster",
    "computer.socialweb.test.kube.undo",
    "computer.socialweb.test.page.signin",
    "computer.socialweb.test.page.fetch",
    "computer.socialweb.test.page.round",
    "computer.socialweb.test.ssh.tempKey",
    "computer.socialweb.test.atirc.observer",
    "computer.socialweb.test.atirc.observerPub",
    "computer.socialweb.test.atirc.observerRevoke",
    "computer.socialweb.test.requester.ssh",
    "computer.socialweb.test.requester.restart",
    "computer.socialweb.test.requester.cluster",
    "computer.socialweb.test.requester.bundle",
    "computer.socialweb.test.staging.pki",
    "computer.socialweb.test.logs.fetchRuns",
    "computer.socialweb.test.requester.records",
    "computer.socialweb.test.requester.events",
    "computer.socialweb.test.root.exec",
    "computer.socialweb.test.root.putBinary",
    "computer.socialweb.test.release.put",
    "computer.socialweb.test.release.delete",
    "computer.socialweb.test.example",
    "computer.socialweb.test.gateway.kubeconfig",
    "computer.socialweb.test.page.kubeconfig",
    "computer.socialweb.test.oauth.signin",
    "computer.socialweb.test.request.r2",
    "computer.socialweb.test.request.checkLogs",
    "computer.socialweb.test.knotless.seed",
    "computer.socialweb.test.knotless.sh",
    "computer.socialweb.test.bidder.health",
    "computer.socialweb.test.bidder.meminfo",
    "computer.socialweb.test.bidder.logCount",
    "computer.socialweb.test.vm.memory",
    "computer.socialweb.test.bidder.logLines",
    "computer.socialweb.test.bidder.submitAccept",
    "computer.socialweb.test.bidder.restart",
    "computer.socialweb.test.bidder.upgrade",
    "computer.socialweb.test.bidder.guestNft",
    "computer.socialweb.test.bidder.awaitLog",
    "computer.socialweb.test.bidder.dumbpipe",
    "computer.socialweb.test.binary.published",
    "computer.socialweb.test.assets.tamper",
    "computer.socialweb.test.gateway.log",
    "computer.socialweb.test.repo.listCreated",
    "computer.socialweb.test.request.contract",
    "com.fedfork.exec.identlink.walk",
    "com.fedfork.exec.identlink.plc",
];
/// `run request` flags a module may pass, before `--exec -- <command> [args...]` (the command is not checked);
/// `@agentConfig` is the target's agent config (owner-only file).
const FLAGS: &[&str] = &[
    "--hold",
    "--agent",
    "--agent-config",
    "--tunnel",
    "--transport",
    "--exec",
    "--guest-tools",
    "--associate",
    "--expires-in",
    "--nested-virt",
    "--cpus",
    "--mem",
    "--image",
];

pub struct Swc {
    pub target: Target,
    pub account: String,
    pub bin: Binary,
    /// the work dir: requester state, kubeconfigs, keys (secrets; never an artifact)
    pub dir: PathBuf,
    /// the dump dir: resources.json beside the ticks (no secret)
    pub dump: PathBuf,
    pub kube: Arc<Kube>,
    /// receipts of this run's contracts: the kube guard lets a test delete only VMs of these
    pub receipts: Arc<Mutex<Vec<String>>>,
    /// guest requester DIDs this run's own requester associated (its `requester_association_written` subject): the kube
    /// guard lets a test delete their rentals too (owner, 2026-11-18: "Both")
    pub associated: Arc<Mutex<Vec<String>>>,
    requesters: tokio::sync::Mutex<HashMap<String, (PathBuf, tokio::process::Child)>>,
    requests: Mutex<HashMap<String, PathBuf>>,
    env_files: Mutex<Vec<PathBuf>>,
    http: reqwest::Client,
    ssh: Arc<atlibs_ssh::Ssh>,
    /// the target's read-only ops routes (prod in CI), when it names any
    ops: Option<Arc<crate::ops::Ops>>,
    extra: more::Extra,
}

/// A call's input; a subscription's arguments come as params (strings), its structure in the `json` param.
fn input_of(r: &Request) -> Value {
    if let Some(i) = &r.input {
        return i.clone();
    }
    let params = r.params.clone().unwrap_or_default();
    match params
        .get("json")
        .and_then(Value::as_str)
        .and_then(|j| serde_json::from_str::<Value>(j).ok())
    {
        Some(v) => v,
        None => Value::Object(params),
    }
}

/// `v` reduced to the dotted `paths` it has (a.b keeps {a: {b}}), nothing else.
fn project(v: &Value, paths: &[&str]) -> Value {
    let mut out = json!({});
    for p in paths {
        let keys: Vec<&str> = p.split('.').collect();
        let Some(found) = keys.iter().try_fold(v, |at, k| at.get(*k)) else { continue };
        let mut dst = &mut out;
        for k in &keys[..keys.len() - 1] {
            if !dst[*k].is_object() {
                dst[*k] = json!({});
            }
            dst = &mut dst[*k];
        }
        dst[keys[keys.len() - 1]] = found.clone();
    }
    out
}

fn str_of<'a>(v: &'a Value, k: &str) -> Result<&'a str, String> {
    v[k].as_str()
        .filter(|s| !s.is_empty() && !s.contains(['/', ' ', '\n']))
        .ok_or_else(|| format!("{k} is required (a plain name)"))
}

fn abs_dir_of<'a>(v: &'a Value, k: &str) -> Result<&'a str, String> {
    v[k].as_str()
        .filter(|s| {
            s.strip_prefix('/').is_some_and(|rest| rest.split('/').all(|c| !c.is_empty() && c != "." && c != ".."))
                && !s.chars().any(|c| c.is_whitespace() || c.is_control() || "'\"\\$`".contains(c))
        })
        .ok_or_else(|| format!("{k} is required (a clean absolute path)"))
}

impl Swc {
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        target: Target,
        account: &str,
        bin: Binary,
        dir: PathBuf,
        kube: Arc<Kube>,
        receipts: Arc<Mutex<Vec<String>>>,
        associated: Arc<Mutex<Vec<String>>>,
        ssh: Arc<atlibs_ssh::Ssh>,
        dump: PathBuf,
        ops: Option<Arc<crate::ops::Ops>>,
    ) -> Arc<Self> {
        let http = testkit::atproto::http_client(&target);
        Arc::new(Self {
            target,
            account: account.into(),
            bin,
            dir,
            dump,
            kube,
            receipts,
            associated,
            requesters: Default::default(),
            requests: Default::default(),
            env_files: Default::default(),
            http,
            ssh,
            ops,
            extra: Default::default(),
        })
    }

    pub fn transport(self: &Arc<Self>) -> Transport {
        Transport {
            provides: NSIDS,
            call: self.clone(),
        }
    }

    fn acct(&self) -> Result<&Account, String> {
        self.target
            .account(&self.account)
            .map_err(|e| e.to_string())
    }

    fn password(&self) -> Result<String, String> {
        Ok(self
            .acct()?
            .password()
            .map_err(|e| e.to_string())?
            .expose()
            .to_string())
    }

    fn request_state(&self) -> PathBuf {
        // the guest requester's `run request` holds a dev instance (its PDS, its kcp) whose sockets under
        // requests/<name>/run/ must fit sun_path (107 bytes): a short dir of its own, ~/.swc/<run ms>
        if self.target.network.is_none() && self.acct().is_ok_and(|a| a.guest_requester) {
            let name = self
                .dir
                .file_name()
                .unwrap_or_default()
                .to_string_lossy()
                .to_string();
            let ms = name.rsplit('-').next().unwrap_or_default().to_string();
            return PathBuf::from(std::env::var("HOME").unwrap_or_default())
                .join(".swc")
                .join(ms);
        }
        match (&self.target.network, self.associated()) {
            (Some(n), _) => expand(&n.state_dir),
            (None, Some(id)) => expand(&id.state_dir),
            (None, None) => self.dir.join("requests"),
        }
    }

    fn request_logs(&self, name: &str) -> Vec<PathBuf> {
        let base = self.request_state().join("requests").join(name);
        let mut logs = vec![
            self.request_state().join(format!("{name}-request.log")),
            base.join("dev").join(name).join("logs/core.log"),
            base.join("logs/core.log"),
        ];
        // `run request --network`: the invocation's requester is a child core of the network, its log there
        if let Some(n) = &self.target.network {
            logs.push(
                expand(&n.state_dir)
                    .join("dev")
                    .join(&n.instance)
                    .join("apps/dev")
                    .join(format!("{}-{name}", n.instance))
                    .join("logs/core.log"),
            );
        }
        logs
    }

    pub(crate) async fn cli(&self, args: &[String]) -> Result<(bool, String), String> {
        self.bin
            .run(&args.iter().map(String::as_str).collect::<Vec<_>>())
            .await
            .map_err(|e| e.to_string())
    }

    /// Copies one of this host's files to the guest of invocation `name` (stdin of `run request ssh --exec -- bash -c 'cat >
    /// <remote>'`). `what` names the file, never a path: testHost ($SWC_TEST_HOST_STATIC, a static swc-test-host) or
    /// suite (a tar.gz of the repo's examples, scripts, tests/compositions, tests/targets, did-key-associator with the four
    /// typescript-helpers libraries it imports (the network's QR service, qr.sh) and the module wasm and
    /// module.json files; SWC_REPO on the guest; with $SWC_PAGE_REPO, a built checkout of the requester page repo or
    /// release's unpacked page payload (scripts/ci-page.sh), also 0009's page as page/: web/webrtc with its modules' files,
    /// the round's two scripts and COMMIT (the payload's own file, else the checkout's HEAD)).
    async fn request_put(&self, input: &Value) -> Result<Value, String> {
        let name = str_of(input, "name")?;
        let remote = input["remote"].as_str().unwrap_or_default();
        if !remote.starts_with('/') || remote.contains(['\'', ' ', '\n']) {
            return Err(format!(
                "remote {remote:?}: an absolute path without quotes or spaces"
            ));
        }
        let bytes = match str_of(input, "what")? {
            "testHost" => {
                let p = std::env::var("SWC_TEST_HOST_STATIC")
                    .map_err(|_| "SWC_TEST_HOST_STATIC is not set (tests/build-static-host.sh)")?;
                std::fs::read(&p).map_err(|e| format!("{p}: {e}"))?
            }
            "suite" => {
                let tests = std::env::current_dir().map_err(|e| e.to_string())?;
                let out = tokio::process::Command::new("sh")
                    .args(["-c", "set -e; p=; if [ -n \"${SWC_PAGE_REPO:-}\" ]; then p=$(mktemp -d); trap 'rm -rf \"$p\"' EXIT; mkdir -p \"$p/page/web\" \"$p/page/scripts\"; \
                        cp -rL \"$SWC_PAGE_REPO/web/webrtc\" \"$p/page/web/\"; cp \"$SWC_PAGE_REPO/scripts/page-round.mjs\" \"$SWC_PAGE_REPO/scripts/browser.mjs\" \"$p/page/scripts/\"; \
                        if [ -f \"$SWC_PAGE_REPO/COMMIT\" ]; then cp \"$SWC_PAGE_REPO/COMMIT\" \"$p/page/COMMIT\"; else git -C \"$SWC_PAGE_REPO\" rev-parse HEAD > \"$p/page/COMMIT\"; fi; fi; \
                        tar -czf - --exclude=did-key-associator/dist --exclude=did-key-associator/dist-backend tests/gate.sh tests/entries.txt tests/compositions tests/targets did-key-associator typescript-helpers/lib/oauth-scope typescript-helpers/lib/cli-args-env typescript-helpers/lib/logger typescript-helpers/lib/serve tests/modules/*/module.json tests/modules/target/wasm32-unknown-unknown/release/*.wasm deploy/vm-images examples scripts ${p:+-C \"$p\" page}"])
                    .current_dir(tests.join(".."))
                    .output()
                    .await
                    .map_err(|e| e.to_string())?;
                if !out.status.success() {
                    return Err(format!(
                        "tar of the suite in {}: {}",
                        tests.display(),
                        String::from_utf8_lossy(&out.stderr)
                    ));
                }
                out.stdout
            }
            "meshd" => {
                let p = std::env::var("SWC_MESHD").map_err(
                    |_| "SWC_MESHD is not set (a static socialweb-meshd, rust/build.sh)",
                )?;
                std::fs::read(&p).map_err(|e| format!("{p}: {e}"))?
            }
            "binary" => {
                let p = std::env::var("SWC_BINARY")
                    .map_err(|_| "SWC_BINARY is not set (the socialweb-computer under test, a release build of this checkout)")?;
                std::fs::read(&p).map_err(|e| format!("{p}: {e}"))?
            }
            "stagingRoot" | "stagingLeaf" | "stagingKey" => {
                let file = match input["what"].as_str().unwrap_or_default() {
                    "stagingRoot" => "root.crt",
                    "stagingLeaf" => "leaf.crt",
                    _ => "leaf.key",
                };
                let p = self.dir.join("staging-pki").join(file);
                std::fs::read(&p).map_err(|e| format!("{}: {e} (staging.pki first)", p.display()))?
            }
            "agentConfig" => {
                let cfg = testkit::target::expand(
                    self.target
                        .agent_config
                        .as_deref()
                        .ok_or("the target has no agentConfig")?,
                );
                testkit::target::check_private(&cfg).map_err(|e| e.to_string())?;
                std::fs::read(&cfg).map_err(|e| format!("{}: {e}", cfg.display()))?
            }
            other => {
                return Err(format!(
                    "what {other:?}: testHost, suite, meshd, binary or agentConfig"
                ));
            }
        };
        let umask = if input["what"] == "agentConfig" || input["what"] == "stagingKey" || input["what"] == "stagingLeaf" {
            "umask 077 && "
        } else {
            ""
        };
        // a file the bucket publishes (the test host, meshd by tree): the VM downloads it at full speed, checked against
        // this file's sha256, instead of a stream over its ssh (minutes at a relayed path's ~50 KB/s, plan 0044 finding 35)
        let url = match input["what"].as_str().unwrap_or_default() {
            "testHost" => std::env::var("SWC_TEST_HOST_URL").ok(),
            "meshd" => std::env::var("SWC_MESHD_URL").ok(),
            _ => None,
        }
        .filter(|u| u.starts_with("https://") && !u.contains(['\'', ' ', '\n']));
        let sha: String = {
            use sha2::Digest;
            sha2::Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect()
        };
        if let Some(url) = url {
            let exec = format!(
                "curl -fsS --retry 3 -o '{remote}.part' '{url}' && echo '{sha}  {remote}.part' | sha256sum -c --quiet && mv '{remote}.part' '{remote}'"
            );
            if self.put_exec(name, input, exec, None).await.is_ok() {
                return Ok(json!({ "remote": remote, "bytes": bytes.len(), "from": url }));
            }
        }
        let exec = format!(
            "{umask}cat > '{remote}.part' && echo '{sha}  {remote}.part' | sha256sum -c --quiet && mv '{remote}.part' '{remote}'"
        );
        let mut tries: u32 = 0;
        loop {
            tries += 1;
            match self.put_exec(name, input, exec.clone(), Some(&bytes)).await {
                Ok(()) => break,
                Err(e) if e.contains("connection lost") && tries <= PUT_RETRIES => {
                    tokio::time::sleep(PUT_RETRY_BACKOFF * tries).await;
                }
                Err(e) => return Err(format!("put {remote} (try {tries}): {e}")),
            }
        }
        Ok(json!({ "remote": remote, "bytes": bytes.len(), "sha256": sha, "tries": tries }))
    }

    // an exec on the VM of a put: a Held contract of the run requester `name` (an Infra's), else `name` is a run request
    // invocation; stdin is the file when one is streamed
    async fn put_exec(&self, name: &str, input: &Value, exec: String, stdin: Option<&[u8]>) -> Result<(), String> {
        let args: Vec<String> = match input["contract"].as_str() {
            Some(c) => ["run", "requester", "ssh", name, "--state-dir"]
                .map(String::from)
                .into_iter()
                .chain([self.requester_state(name).display().to_string(), "--contract".into(), c.into()])
                .chain(exec_line(&exec))
                .collect(),
            None => {
                let mut v = self.sub_args("ssh", name, &[]);
                v.extend(exec_line(&exec));
                v
            }
        };
        let mut child = self
            .bin
            .cmd()
            .args(args)
            .stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() })
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()
            .map_err(|e| e.to_string())?;
        let mut wrote = Ok(());
        if let Some(bytes) = stdin {
            let mut sin = child.stdin.take().unwrap();
            use tokio::io::AsyncWriteExt;
            wrote = sin.write_all(bytes).await;
            drop(sin);
        }
        let out = child.wait_with_output().await.map_err(|e| e.to_string())?;
        if !out.status.success() {
            return Err(tail(&String::from_utf8_lossy(&out.stderr)));
        }
        if let Err(e) = wrote {
            return Err(format!("stdin: {e}"));
        }
        Ok(())
    }

    pub(crate) fn sub_args(&self, verb: &str, name: &str, extra: &[&str]) -> Vec<String> {
        let mut v: Vec<String> = ["run", "request", verb, name, "--state-dir"]
            .map(String::from)
            .to_vec();
        v.push(self.request_state().display().to_string());
        v.extend(extra.iter().map(|s| s.to_string()));
        v
    }

    async fn requester_start(&self, input: &Value) -> Result<Value, String> {
        let name = str_of(input, "name")?.to_string();
        let state = self.requester_state(&name);
        std::fs::create_dir_all(&state).map_err(|e| e.to_string())?;
        std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))
            .map_err(|e| e.to_string())?;
        let mut args: Vec<String> = vec![
            "run".into(),
            "requester".into(),
            "--name".into(),
            name.clone(),
            "--state-dir".into(),
            state.display().to_string(),
        ];
        if input["withGateway"] == true {
            args.push("--with-gateway".into());
        }
        // a market network target: the requester is a child of the network (its relays, PLC and bidders), as request_start's
        if let Some(n) = &self.target.network {
            args.extend(["--network".to_string(), n.instance.clone()]);
        }
        // R2 (a guest requester): its identity is the renter's association, through the guest's /run/socialweb-r2 files
        let guest = self.acct()?.guest_requester;
        let associated = self.associated().map(|id| id.name.clone());
        if guest {
            args.push("--guest-requester".into());
        } else if let Some(id) = &associated {
            args.extend(["--identity".to_string(), id.clone()]);
        } else {
            args.extend(identity_args(
                &self.target,
                self.acct()?,
                SignIn::AppPassword,
            ));
        }
        // plan 0040 step 5: the target's bidders filter its discovery (every rental, a --with-gateway node's too); a market
        // network sets its own bidders for its child applications, so only a target outside one passes them
        let only: Vec<&str> = input["bidderDids"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
        if !only.is_empty() && self.target.network.is_none() {
            args.extend(["--bidder-dids".to_string(), only.join(",")]);
        }
        self.extra
            .argv
            .lock()
            .unwrap()
            .insert(name.clone(), args.clone());
        let log =
            std::fs::File::create(state.join(format!("{name}.log"))).map_err(|e| e.to_string())?;
        let child = self
            .bin
            .cmd()
            .args(&args)
            .env(
                "ATPROTO_PASSWORD",
                if guest || associated.is_some() {
                    String::new()
                } else {
                    self.password()?
                },
            )
            .stdout(log.try_clone().map_err(|e| e.to_string())?)
            .stderr(log)
            .spawn()
            .map_err(|e| e.to_string())?;
        self.requesters
            .lock()
            .await
            .insert(name.clone(), (state.clone(), child));
        let ready = state.join("requesters").join(&name).join("ready.json");
        let deadline = Instant::now() + Duration::from_secs(600);
        let r: Value = loop {
            if let Ok(b) = std::fs::read(&ready)
                && let Ok(v) = serde_json::from_slice(&b)
            {
                break v;
            }
            if let Some((_, c)) = self.requesters.lock().await.get_mut(&name)
                && let Ok(Some(st)) = c.try_wait()
            {
                // the requester's own last lines (its log, redacted as the run's logs are), so the failure reads from the
                // gate's RESULT line alone (a CI run keeps no artifacts)
                let said = std::fs::read_to_string(state.join(format!("{name}.log"))).unwrap_or_default();
                let lines: Vec<&str> = said.lines().filter(|l| !l.trim().is_empty()).collect();
                let last = redact_log(&lines[lines.len().saturating_sub(3)..].join("\n"));
                let last: Vec<String> = last.lines().map(|l| l.chars().take(400).collect()).collect();
                return Err(format!(
                    "run requester {name} exited {st} before ready: {}",
                    last.join(" | ")
                ));
            }
            if Instant::now() > deadline {
                return Err(format!("run requester {name}: no ready file in 10 min"));
            }
            tokio::time::sleep(Duration::from_secs(1)).await;
        };
        let kubeconfig = r["kubeconfig"]
            .as_str()
            .ok_or("ready file without kubeconfig")?
            .to_string();
        self.kube.add_cluster(
            &format!("requester:{name}"),
            Cluster {
                kubeconfig,
                workspace: None,
                via: Default::default(),
            },
        );
        Ok(
            json!({ "did": r["did"], "handle": r["handle"], "identityMode": r["identityMode"], "cluster": format!("requester:{name}"),
                    "gateway": r["gateway"], "sshKey": r["sshPublicKey"].as_str().and_then(|p| std::fs::read_to_string(p).ok()).map(|k| k.trim().to_string()) }),
        )
    }

    fn request_args(&self, input: &Value) -> Result<(Vec<String>, Option<Vec<String>>), String> {
        let words: Vec<String> = input["args"]
            .as_array()
            .into_iter()
            .flatten()
            .map(|w| w.as_str().map(String::from).ok_or("args are strings"))
            .collect::<Result<_, _>>()?;
        let (opts, command) = split_exec(&words).map_err(|e| e.to_string())?;
        let mut out = vec![];
        for w in &opts {
            let w = w.as_str();
            if w == "@agentConfig" {
                let p = expand(
                    self.target
                        .agent_config
                        .as_deref()
                        .ok_or("the target has no agentConfig")?,
                );
                check_private(&p).map_err(|e| e.to_string())?;
                out.push(p.display().to_string());
            } else if w.starts_with("--") && !FLAGS.contains(&w.split('=').next().unwrap()) {
                return Err(format!("run request flag {w} is not one a test may pass"));
            } else {
                out.push(w.to_string());
            }
        }
        Ok((out, command))
    }

    /// `run request --name <name> <args> <identity> --exec -- <command>`, its log events as messages until the
    /// subscription closes.
    async fn request_start(&self, input: &Value) -> Result<Sub, String> {
        let name = str_of(input, "name")?.to_string();
        let (mut args, command) = self.request_args(input)?;
        // secrets: the module's bundle [{path, value, mode?}] (test values only), written 0600 in the work dir
        if let Some(bundle) = input["secrets"].as_array() {
            let p = self.dir.join(format!("{name}-secrets.json"));
            more::write_private(&p, &Value::Array(bundle.clone()).to_string())?;
            args.extend(["--secrets".to_string(), p.display().to_string()]);
        }
        if let Some(n) = &self.target.network {
            args.extend(["--network".to_string(), n.instance.clone()]);
        }
        let state = self.request_state();
        std::fs::create_dir_all(&state).map_err(|e| e.to_string())?;
        std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))
            .map_err(|e| e.to_string())?;
        let logs = self.request_logs(&name);
        let f = std::fs::File::create(&logs[0]).map_err(|e| e.to_string())?;
        // compat: the unchanged client (SWC_COMPAT_REQUESTER: release:<version>, sha:..., path:...) against new servers
        let bin = match (
            input["compat"] == true,
            std::env::var("SWC_COMPAT_REQUESTER"),
        ) {
            (true, Ok(spec)) => Binary::resolve(&spec).await.map_err(|e| e.to_string())?,
            _ => self.bin.clone(),
        };
        let mut cmd = bin.cmd();
        cmd.args(["run", "request", "--state-dir"])
            .arg(&state)
            .args(["--name", &name]);
        // R2 rents as the renter's agent: its identity is the association, no password
        let mut pass: Vec<String> = vec![];
        if self.acct()?.guest_requester {
            // its own key in the VM it rents, for request.ssh
            args.extend(["--guest-requester".to_string(), "--skip-ssh=false".to_string()]);
        } else if let Some(id) = self.associated() {
            args.extend(["--identity".to_string(), id.name.clone()]);
        } else {
            pass.extend(identity_args(&self.target, self.acct()?, SignIn::AppPassword));
            cmd.env("ATPROTO_PASSWORD", self.password()?);
        }
        // the RFP's compute.vm role (e.g. the staging layer the watch page shows), and the bidders it may (bidderDids)
        // or may not (denyBidderDids) go to: MarketRequester (contract) options
        if let Some(v) = input["vmName"].as_str() {
            if v.is_empty()
                || !v
                    .chars()
                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
            {
                return Err(format!(
                    "vmName {v:?}: lowercase letters, digits and dashes"
                ));
            }
            pass.extend(["--vm-name".to_string(), v.to_string()]);
        }
        // the requester's comms mode and secrets fallback (which channel the guest's secrets take): requester options
        for (k, flag) in [("commsMode", "--comms-mode"), ("secretsFallback", "--secrets-fallback")] {
            if let Some(v) = input[k].as_str() {
                if v.is_empty() || !v.chars().all(|c| c.is_ascii_alphabetic()) {
                    return Err(format!("{k} {v:?}: a mode name"));
                }
                pass.extend([flag.to_string(), v.to_string()]);
            }
        }
        for (k, flag) in [
            ("bidderDids", "--allow-bidder-dids"),
            ("denyBidderDids", "--deny-bidder-dids"),
        ] {
            let Some(dids) = input[k].as_array() else {
                continue;
            };
            let dids: Vec<&str> = dids.iter().filter_map(Value::as_str).collect();
            if dids
                .iter()
                .any(|d| !d.starts_with("did:") || d.contains(','))
            {
                return Err(format!("{k}: DIDs"));
            }
            if !dids.is_empty() {
                pass.extend([flag.to_string(), dids.join(",")]);
            }
        }
        // the bidder endpoint kinds the requester may use, in order (SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS): iroh, webrtc, https
        if let Some(kinds) = input["bidderEndpoints"].as_array() {
            let kinds: Vec<&str> = kinds.iter().filter_map(Value::as_str).collect();
            if kinds.is_empty() || kinds.iter().any(|k| !["iroh", "webrtc", "https"].contains(k)) {
                return Err("bidderEndpoints: iroh, webrtc or https".into());
            }
            cmd.env("SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS", kinds.join(","));
        }
        // plan 0050: the staging run's user_data (its root CA), made by staging.pki
        if input["stagingUserData"] == true || input["stagingUserData"].is_string() {
            let p = match input["stagingUserData"].as_str() {
                Some(v) if v.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') => {
                    self.dir.join("staging-pki").join(format!("user-data-{v}"))
                }
                Some(v) => return Err(format!("stagingUserData {v:?}: a variant name")),
                None => self.dir.join("staging-pki").join("user-data"),
            };
            if !p.is_file() {
                return Err(format!("{}: staging.pki first", p.display()));
            }
            pass.extend(["--user-data".to_string(), p.display().to_string()]);
        }
        // runcmd: shell lines the guest runs from the request's own user_data (e.g. 0007:webrtc turns a channel off)
        if let Some(lines) = input["runcmd"].as_array() {
            if pass.iter().any(|w| w == "--user-data") {
                return Err("runcmd and stagingUserData: one --user-data".into());
            }
            let mut ud = String::from("#cloud-config\nruncmd:\n");
            for l in lines {
                let l = l.as_str().ok_or("runcmd: shell lines")?;
                ud.push_str(&format!("  - [sh, -c, {}]\n", serde_json::to_string(l).map_err(|e| e.to_string())?));
            }
            let p = self.dir.join(format!("{name}-user-data.yaml"));
            more::write_private(&p, &ud)?;
            pass.extend(["--user-data".to_string(), p.display().to_string()]);
        }
        // a CLI before plan 0043 (0006:compat's previous rc) takes them after `--` and the command as one `--exec` line
        cmd.args(request_line(bin.exec_argv().await, &args, &pass, command.as_deref()));
        // pty: the invocation runs on a pseudo-terminal (socat SYSTEM,pty) whose stdin stays open, so `run request`
        // without `--exec` opens its interactive ssh session (a terminal on stdin) and holds it
        let pty = input["pty"] == true;
        let mut spawn_cmd = if pty {
            let std_cmd = cmd.as_std();
            let line = std::iter::once(std_cmd.get_program())
                .chain(std_cmd.get_args())
                .map(|w| format!("'{}'", w.to_string_lossy().replace('\'', r#"'"'"'"#)))
                .collect::<Vec<_>>()
                .join(" ");
            let envs: Vec<_> = std_cmd
                .get_envs()
                .filter_map(|(k, v)| v.map(|v| (k.to_os_string(), v.to_os_string())))
                .collect();
            // socat's SYSTEM address runs the line with sh -c on a pty; its `,` `:` `\\` are escaped
            let esc = line
                .replace('\\', "\\\\")
                .replace(',', "\\,")
                .replace(':', "\\:");
            let mut script = tokio::process::Command::new("socat");
            script
                .args(["-", &format!("SYSTEM:{esc},pty,setsid,ctty,stderr")])
                .envs(envs);
            script.stdin(Stdio::piped());
            script
        } else {
            cmd
        };
        let mut child = spawn_cmd
            .stdout(f.try_clone().map_err(|e| e.to_string())?)
            .stderr(f)
            .spawn()
            .map_err(|e| e.to_string())?;
        // held open until the invocation is over
        let pty_stdin = child.stdin.take();
        self.requests.lock().unwrap().insert(name.clone(), state);
        let (tx, messages) = mpsc::unbounded_channel();
        // which binary runs the invocation (0006 compat: the unchanged CLI's)
        let _ = tx.send(Ok(json!({ "msg": "requester_binary", "version": bin.version, "compat": input["compat"] == true })));
        let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
        let receipts = self.receipts.clone();
        let associated = self.associated.clone();
        // dev nets only: when the contract fails with "did not come on network", the tail of the console of the guest(s)
        // that never reported (the cause is in the guest: PID 1, cloud-final), in the event and in the dump
        let console_net = self
            .target
            .network
            .as_ref()
            .map(|n| (expand(&n.state_dir), n.instance.clone()));
        let console_dump = self.dump.join("logs").join(&name);
        // R2 waits for its association ref, which the renter writes once it verified R2: one that never comes is an
        // error of its own, not a request that waits until the run's deadline
        let ref_by = self
            .acct()?
            .guest_requester
            .then(|| Instant::now() + R2_REF_WAIT);
        tokio::spawn(async move {
            let _pty_stdin = pty_stdin;
            let mut offsets = vec![0usize; logs.len()];
            let mut exited = false;
            // the run's dev core.log is copied to logs/core.log when the process ends: each line once
            let mut seen = std::collections::HashSet::new();
            loop {
                for (i, p) in logs.iter().enumerate() {
                    let Ok(text) = std::fs::read_to_string(p) else {
                        continue;
                    };
                    if text.len() < offsets[i] {
                        offsets[i] = 0;
                    }
                    let fresh = &text[offsets[i]..];
                    let Some(end) = fresh.rfind('\n') else {
                        continue;
                    };
                    for line in fresh[..=end].lines() {
                        let Ok(v) = serde_json::from_str::<Value>(line) else {
                            continue;
                        };
                        if !seen.insert(line.to_string()) {
                            continue;
                        }
                        // the receipt is known (and the run's VM ours to fault) from its first event, before any summary
                        if let Some(r) = v["receipt"].as_str() {
                            let mut rs = receipts.lock().unwrap();
                            if !rs.iter().any(|x| x == r) {
                                rs.push(r.into());
                            }
                        }
                        if v["msg"] == "requester_association_written"
                            && let Some(d) = v["subject"].as_str().filter(|d| d.starts_with("did:"))
                        {
                            let mut a = associated.lock().unwrap();
                            if !a.iter().any(|x| x == d) {
                                a.push(d.into());
                            }
                        }
                        let mut v = v;
                        if v["msg"] == "contract"
                            && v["phase"] == "Failed"
                            && v["message"]
                                .as_str()
                                .is_some_and(|m| m.contains("did not come on network"))
                            && let Some((state_dir, instance)) = &console_net
                        {
                            let tails = silent_console_tails(state_dir, instance, &console_dump);
                            if !tails.is_empty() {
                                let m = v["message"].as_str().unwrap_or_default().to_string();
                                v["message"] = json!(format!(
                                    "{m}\nconsole tail of the guest(s) that never reported (dev net):\n{tails}"
                                ));
                            }
                        }
                        let msg = v["msg"].as_str().unwrap_or_default().to_string();
                        let msg = msg.as_str();
                        if (msg == "contract"
                            || msg == "ssh returned"
                            || msg.starts_with("requester_")
                            || msg.starts_with("guest_")
                            || msg.starts_with("ssh to the guest over")
                            || msg.starts_with("no --exec and stdin is not a terminal")
                            || msg.starts_with("ssh not ready yet"))
                            && tx.send(Ok(v)).is_err()
                        {
                            return;
                        }
                    }
                    offsets[i] += end + 1;
                }
                if !exited && let Ok(Some(st)) = child.try_wait() {
                    exited = true;
                    // the CLI's own last lines (a refusal before any kcp is plain stderr)
                    let tail = std::fs::read_to_string(&logs[0]).map(|t| testkit::requester::tail(&t)).unwrap_or_default();
                    let _ = tx.send(Ok(json!({ "msg": "exited", "code": st.code(), "tail": tail })));
                }
                let referred =
                    || std::fs::read_to_string(R2_REF).is_ok_and(|r| !r.trim().is_empty());
                if !exited && ref_by.is_some_and(|by| Instant::now() > by) && !referred() {
                    exited = true;
                    let _ = child.start_kill();
                    let why = format!(
                        "no association ref in {R2_REF} after {} min: the renter's associate failed or never ran (the knotless check's log says why)",
                        R2_REF_WAIT.as_secs() / 60
                    );
                    let _ = tx.send(Ok(json!({ "msg": "exited", "code": null, "error": why })));
                }
                tokio::select! {
                    cmd = cmds.recv() => match cmd {
                        Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("request.start takes no sends".into())); }
                        Some(SubCmd::Close) | None => return,
                    },
                    _ = tokio::time::sleep(Duration::from_secs(1)) => {}
                }
            }
        });
        Ok(Sub { messages, commands })
    }

    fn summary(&self, name: &str) -> Result<Value, String> {
        let p = self
            .request_state()
            .join("requests")
            .join(name)
            .join("summary.json");
        let v: Value = serde_json::from_slice(
            &std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?,
        )
        .map_err(|e| e.to_string())?;
        if let Some(r) = v["contract"]["receiptUri"].as_str() {
            let mut rs = self.receipts.lock().unwrap();
            if !rs.iter().any(|x| x == r) {
                rs.push(r.into());
            }
        }
        Ok(v)
    }

    /// The restart race of a real devsup with no VM (docs/acceptance/0017): `dev up --profile quick` on the proc backend in an
    /// instance of this run, then per delay `iterations` times: `dev down` SIGKILLed `delay` ms after it started (devsup keeps
    /// stopping its units in order and exits), `dev up` at once, and 6 s later the kcp must answer /readyz 200. The binary is the
    /// target's requester (the build under test). Skipped when it has no `dev up`.
    // plan 0046 step 6: two instances of one name in two state dirs (two `run requester --name stg`) are two
    // instances: each its own devsup (its runtime dir keyed by its state dir), and a down of one leaves the other up
    async fn devsup_samename(&self) -> Result<Value, String> {
        let inst = format!("same{}", std::process::id() % 100000);
        let dirs = [self.dir.join("samename-a"), self.dir.join("samename-b")];
        let cmd = |verb: &[&str], dir: &std::path::Path| {
            let mut c = self.bin.cmd();
            c.args(["dev", verb[0]]).args(&verb[1..]).args(["--instance", &inst, "--state-dir"]).arg(dir);
            c
        };
        let up = |dir: std::path::PathBuf| async move {
            let _ = std::fs::create_dir_all(&dir);
            let mut c = cmd(&["up", "--profile", "quick", "--host-backend", "proc"], &dir);
            let out = tokio::time::timeout(Duration::from_secs(200), c.output()).await;
            let text = match out {
                Ok(Ok(o)) => format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)),
                Ok(Err(e)) => e.to_string(),
                Err(_) => "dev up did not return in 200 s".to_string(),
            };
            let url = text.lines().find(|l| l.contains("\"kcpURL\"")).and_then(|l| l.split('"').nth(3)).unwrap_or_default().to_string();
            (url, text)
        };
        let ready = |url: String| async move {
            if url.is_empty() {
                return "none".to_string();
            }
            let o = tokio::process::Command::new("curl").args(["-sk", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}"]).arg(format!("{url}/readyz")).output().await;
            o.map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default()
        };
        let down = |dir: std::path::PathBuf| async move {
            let _ = cmd(&["down"], &dir).output().await;
        };
        let (a, at) = up(dirs[0].clone()).await;
        let (b, bt) = up(dirs[1].clone()).await;
        let (ra, rb) = (ready(a.clone()).await, ready(b.clone()).await);
        down(dirs[0].clone()).await;
        tokio::time::sleep(Duration::from_secs(3)).await;
        let rb_after = ready(b.clone()).await;
        down(dirs[1].clone()).await;
        if a.is_empty() && (at.contains("unknown command") || at.contains("flag provided but not defined")) {
            return Ok(json!({ "skipped": "the build has no `dev up --profile quick` on the proc backend" }));
        }
        let fail = |what: &str, text: &str| json!({ "ok": false, "why": what, "log": tail(&redact_log(text)) });
        Ok(match () {
            _ if ra != "200" => fail("instance a's kcp is not ready", &at),
            _ if rb != "200" => fail("instance b's kcp is not ready (it took a's devsup?)", &bt),
            _ if a == b => fail("a and b answer on one kcp URL", &bt),
            _ if rb_after != "200" => fail("dev down of a took b's kcp down", &bt),
            _ => json!({ "ok": true, "a": a, "b": b, "bAfterDownA": rb_after, "binary": self.bin.version }),
        })
    }

    async fn devsup_race(&self, input: &Value) -> Result<Value, String> {
        let delays: Vec<u64> = input["delaysMs"]
            .as_array()
            .map(|a| a.iter().filter_map(Value::as_u64).collect())
            .unwrap_or_else(|| vec![500, 1000, 2000]);
        let per = input["perDelay"].as_u64().unwrap_or(4);
        let inst = format!("race{}", std::process::id() % 100000);
        let state = self.dir.join("devsup-race");
        let logs = self.dump.join("logs").join("devsup-race");
        let _ = std::fs::create_dir_all(&state);
        let _ = std::fs::create_dir_all(&logs);
        let base = |verb: &[&str]| {
            let mut c = self.bin.cmd();
            c.args(["dev", verb[0]])
                .args(&verb[1..])
                .args(["--instance", &inst, "--state-dir"])
                .arg(&state);
            c
        };
        let up = |label: String| {
            let mut c = base(&["up", "--profile", "quick"]);
            c.args(["--host-backend", "proc"]);
            (c, label)
        };
        let down = || async {
            let _ = base(&["down"]).output().await;
        };
        let run_up = |c: tokio::process::Command, label: String| async move {
            let mut c = c;
            let t = Instant::now();
            let out = tokio::time::timeout(Duration::from_secs(200), c.output()).await;
            let secs = (t.elapsed().as_secs_f64() * 10.0).round() / 10.0;
            let (rc, text) = match out {
                Ok(Ok(o)) => (
                    o.status.code().unwrap_or(-1),
                    format!(
                        "{}{}",
                        String::from_utf8_lossy(&o.stdout),
                        String::from_utf8_lossy(&o.stderr)
                    ),
                ),
                Ok(Err(e)) => (-1, e.to_string()),
                Err(_) => (-2, "dev up did not return in 200 s".to_string()),
            };
            (rc, secs, text, label)
        };
        // the first up: a build without `dev up` (a release one) skips
        let (c, l) = up("first".into());
        let (rc, _, text, _) = run_up(c, l).await;
        if rc != 0 {
            down().await;
            if text.contains("unknown command") || text.contains("flag provided but not defined") {
                return Ok(
                    json!({ "skipped": "the requester build has no `dev up --profile quick` on the proc backend" }),
                );
            }
            return Err(format!(
                "the first dev up of the race instance failed (exit {rc}): {}",
                tail(&text)
            ));
        }
        let mut iterations = vec![];
        let (mut ok, mut bad) = (0u64, 0u64);
        for d in &delays {
            for i in 0..per {
                let mut dc = base(&["down"])
                    .kill_on_drop(true)
                    .spawn()
                    .map_err(|e| e.to_string())?;
                tokio::time::sleep(Duration::from_millis(*d)).await;
                let _ = dc.start_kill();
                let _ = dc.wait().await;
                let (c, l) = up(format!("{d}ms-{i}"));
                let (rc, secs, text, label) = run_up(c, l).await;
                tokio::time::sleep(Duration::from_secs(6)).await;
                let url = text
                    .lines()
                    .find(|l| l.contains("\"kcpURL\""))
                    .and_then(|l| l.split('"').nth(3))
                    .unwrap_or_default()
                    .to_string();
                let code = if url.is_empty() {
                    "none".to_string()
                } else {
                    let o = tokio::process::Command::new("curl")
                        .args(["-sk", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}"])
                        .arg(format!("{url}/readyz"))
                        .output()
                        .await;
                    o.map(|o| String::from_utf8_lossy(&o.stdout).to_string())
                        .unwrap_or_default()
                };
                let good = rc == 0 && code == "200";
                if good {
                    ok += 1;
                } else {
                    bad += 1;
                    let _ = std::fs::write(logs.join(format!("up-{label}.log")), redact_log(&text));
                }
                let why = text
                    .lines()
                    .rev()
                    .find(|l| l.contains("dev up:"))
                    .unwrap_or_default()
                    .to_string();
                iterations.push(json!({ "delayMs": d, "exit": rc, "upSeconds": secs, "readyz": code, "ok": good, "error": if good { Value::Null } else { json!(why) } }));
            }
        }
        down().await;
        Ok(json!({ "ok": ok, "bad": bad, "iterations": iterations, "binary": self.bin.version }))
    }

    fn invocation_dir(&self, name: &str) -> PathBuf {
        self.request_state().join("requests").join(name)
    }

    /// The invocation's ComputeContract `contract`, read from its kcp (its instance record: a network child's under the
    /// network's state, a standalone one's under the invocation dir): {phase, secretsFetchedAt, secretsFetchedVia,
    /// conditions}.
    async fn request_contract(&self, name: &str) -> Result<Value, String> {
        let dir = self.invocation_dir(name);
        let inv: Value = serde_json::from_slice(
            &std::fs::read(dir.join("invocation.json")).map_err(|e| format!("{name}: invocation.json: {e}"))?,
        )
        .map_err(|e| e.to_string())?;
        let instance = inv["instance"].as_str().filter(|i| !i.is_empty()).ok_or("the invocation names no instance yet")?;
        let record = match (inv["mode"].as_str(), &self.target.network) {
            (Some("network"), Some(n)) => expand(&n.state_dir).join("dev").join(&n.instance).join("apps").join("dev").join(instance),
            (Some("network"), None) => return Err("a network invocation on a target without a network".into()),
            _ => dir.join("dev").join(instance),
        }
        .join("instance.json");
        let rec: Value = serde_json::from_slice(&std::fs::read(&record).map_err(|e| format!("{}: {e}", record.display()))?)
            .map_err(|e| e.to_string())?;
        let kubeconfig = rec["adminKubeconfig"].as_str().filter(|k| !k.is_empty()).ok_or("the instance record has no adminKubeconfig")?;
        let cluster = format!("request:{name}");
        self.kube.add_cluster(
            &cluster,
            Cluster {
                kubeconfig: kubeconfig.to_string(),
                workspace: Some(rec["baseWorkspace"].as_str().filter(|w| !w.is_empty()).unwrap_or("root").to_string()),
                via: Default::default(),
            },
        );
        let input = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeContract", "name": "contract" });
        let c = self
            .kube
            .call(atlibs_kube::GET.into(), Request { input: Some(input), ..Default::default() })
            .await?
            .unwrap_or_default();
        let st = &c["status"];
        Ok(json!({ "phase": st["phase"], "receipt": st["contract"]["receipt"]["uri"], "secretsFetchedAt": st["secretsFetchedAt"], "secretsFetchedVia": st["secretsFetchedVia"], "conditions": st["conditions"] }))
    }

    /// The test's own invocation directory: whether it exists and what its summary says.
    fn invocation_state(&self, name: &str) -> Result<Value, String> {
        let dir = self.invocation_dir(name);
        let summary = std::fs::read(dir.join("summary.json"))
            .ok()
            .and_then(|b| serde_json::from_slice::<Value>(&b).ok());
        // the instance state of a standalone invocation (dev/<name> of its dir) and any live process whose command line names the
        // invocation's dir (its devsup, kcp, kine, front, core)
        let instance_state = std::fs::read_dir(dir.join("dev"))
            .map(|mut d| d.next().is_some())
            .unwrap_or(false);
        let needle = dir.display().to_string();
        let live = std::fs::read_dir("/proc")
            .map(|d| {
                d.flatten()
                    .filter(|e| {
                        e.file_name()
                            .to_str()
                            .is_some_and(|n| n.parse::<u32>().is_ok())
                    })
                    .filter(|e| {
                        std::fs::read(e.path().join("cmdline")).is_ok_and(|raw| {
                            String::from_utf8_lossy(&raw)
                                .replace('\0', " ")
                                .contains(&needle)
                        })
                    })
                    .count()
            })
            .unwrap_or(0);
        Ok(
            json!({ "dir": needle, "exists": dir.exists(), "instanceState": instance_state, "liveProcesses": live, "summary": summary }),
        )
    }

    /// Rewrites the phase of the test's own invocation summary (top level and contract): the state of a half-created
    /// invocation whose network application is gone.
    fn set_phase(&self, name: &str, phase: &str) -> Result<Value, String> {
        let p = self.invocation_dir(name).join("summary.json");
        let mut v: Value = serde_json::from_slice(
            &std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?,
        )
        .map_err(|e| e.to_string())?;
        v["phase"] = json!(phase);
        if v["contract"].is_object() {
            v["contract"]["phase"] = json!(phase);
        }
        std::fs::write(
            &p,
            serde_json::to_vec_pretty(&v).map_err(|e| e.to_string())?,
        )
        .map_err(|e| e.to_string())?;
        Ok(json!({ "phase": phase }))
    }

    /// SIGKILL (or `signal`) to the process that owns the invocation `name`: `<bin> run request ... --name <name>` of this
    /// run (not a release, proxy or ssh verb), found by its command line; the kcp child it started keeps running.
    fn kill_owner(&self, name: &str, signal: &str) -> Result<Value, String> {
        if !["KILL", "TERM", "INT"].contains(&signal) {
            return Err(format!("signal {signal}: KILL, TERM or INT"));
        }
        let state = self.request_state().display().to_string();
        let mut pids = vec![];
        for e in std::fs::read_dir("/proc")
            .map_err(|e| e.to_string())?
            .flatten()
        {
            let Some(pid) = e.file_name().to_str().and_then(|n| n.parse::<u32>().ok()) else {
                continue;
            };
            let Ok(raw) = std::fs::read(e.path().join("cmdline")) else {
                continue;
            };
            let argv: Vec<&str> = raw
                .split(|b| *b == 0)
                .filter_map(|a| std::str::from_utf8(a).ok())
                .collect();
            let named = argv.windows(2).any(|w| w[0] == "--name" && w[1] == name);
            let owner = argv.len() > 3
                && argv[0]
                    .rsplit('/')
                    .next()
                    .is_some_and(|b| b.starts_with("socialweb-computer"))
                && argv[1] == "run"
                && argv[2] == "request"
                && argv[3].starts_with("--")
                && argv.contains(&state.as_str());
            if named && owner {
                pids.push(pid);
            }
        }
        if pids.is_empty() {
            return Err(format!("no process owns the invocation {name}"));
        }
        for p in &pids {
            let _ = std::process::Command::new("kill")
                .arg(format!("-{signal}"))
                .arg(p.to_string())
                .status();
        }
        Ok(json!({ "killed": pids }))
    }

    async fn release(&self, name: &str) -> Result<Value, String> {
        let t = Instant::now();
        let (ok, out) = self.cli(&self.sub_args("release", name, &[])).await?;
        // what each release did and how long, in the dump (an artifact: where a slow release waited)
        let d = self.dump.join("logs").join(name);
        if std::fs::create_dir_all(&d).is_ok() {
            use std::io::Write;
            if let Ok(mut f) = std::fs::OpenOptions::new()
                .create(true)
                .append(true)
                .open(d.join("release.log"))
            {
                let _ = writeln!(
                    f,
                    "== run request release {name}: ok {ok} in {:.0} s",
                    t.elapsed().as_secs_f64()
                );
                let _ = f.write_all(redact_log(&out).as_bytes());
            }
        }
        if !ok {
            return Err(format!("run request release {name}: {}", tail(&out)));
        }
        Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }))
    }

    /// The owner node's atirc and config dir: the AtircNode's HostProcess (same host).
    async fn mesh_node(&self, input: &Value) -> Result<(PathBuf, String), String> {
        let node = str_of(input, "node")?;
        if let Some((bin, dir, _)) = self.extra.nodes.lock().await.get(node) {
            return Ok((bin.clone(), dir.display().to_string()));
        }
        let cluster = input["cluster"].as_str().ok_or("cluster is required")?;
        let get = |kind: &str, name: &str| {
            let input = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": name });
            self.kube.call(
                atlibs_kube::GET.into(),
                Request {
                    input: Some(input),
                    ..Default::default()
                },
            )
        };
        let deadline = Instant::now() + Duration::from_secs(180);
        loop {
            let o = get("AtircNode", node).await?.unwrap_or_default();
            if o["status"]["phase"] == "Running" {
                let dir = o["status"]["configDir"]
                    .as_str()
                    .ok_or("AtircNode without configDir")?
                    .to_string();
                let pid = match o["status"]["runtime"]["pid"].as_i64() {
                    Some(p) => p,
                    None => {
                        let hp = o["status"]["hostProcess"]
                            .as_str()
                            .ok_or("AtircNode without hostProcess")?;
                        get("HostProcess", hp).await?.unwrap_or_default()["status"]["runtime"]["pid"].as_i64().ok_or("HostProcess without pid")?
                    }
                };
                let bin = std::fs::read_link(format!("/proc/{pid}/exe"))
                    .map_err(|e| format!("the AtircNode runs on this host: {e}"))?;
                return Ok((bin, dir));
            }
            if Instant::now() > deadline {
                return Err(format!(
                    "AtircNode {node} not Running in 3 min: {}",
                    o["status"]
                ));
            }
            tokio::time::sleep(Duration::from_secs(2)).await;
        }
    }

    async fn mesh(&self, input: &Value, args: &[&str]) -> Result<Value, String> {
        let (bin, dir) = self.mesh_node(input).await?;
        let out = tokio::process::Command::new(bin)
            .arg("mesh")
            .args(args)
            .env("ATIRC_CONFIG_DIR", dir)
            .env("ATP_USERNAME", &self.acct()?.handle)
            .stdin(Stdio::null())
            .output()
            .await
            .map_err(|e| e.to_string())?;
        Ok(
            json!({ "ok": out.status.success(), "output": format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)) }),
        )
    }

    async fn plc(&self, did: &str) -> Result<Value, String> {
        let base = self.target.plc.trim_end_matches('/');
        let doc = self
            .http
            .get(format!("{base}/{did}"))
            .send()
            .await
            .map_err(|e| e.to_string())?;
        let status = doc.status().as_u16();
        let doc: Value = doc.json().await.unwrap_or_default();
        let last: Value = self
            .http
            .get(format!("{base}/{did}/log/last"))
            .send()
            .await
            .map_err(|e| e.to_string())?
            .json()
            .await
            .unwrap_or_default();
        let data: Value = self
            .http
            .get(format!("{base}/{did}/data"))
            .send()
            .await
            .map_err(|e| e.to_string())?
            .json()
            .await
            .unwrap_or_default();
        Ok(
            json!({ "docStatus": status, "lastType": last["type"], "alsoKnownAs": doc["alsoKnownAs"], "rotationKeys": data["rotationKeys"] }),
        )
    }

    /// One com.atproto.repo.listRecords page of the repo's PDS, each record as {uri, createdAt} and, with `fields` (dotted
    /// paths into the value, e.g. receipt.uri), `value` holding only those paths; and the cursor. Never the whole value: a
    /// compute.vm's user_data makes a page of 100 pass the module input cap (1 MiB).
    async fn list_created(&self, input: &Value) -> Result<Value, String> {
        let repo = input["repo"].as_str().ok_or("repo is required")?;
        let collection = input["collection"].as_str().ok_or("collection is required")?;
        let at = testkit::Atproto::new(&self.target);
        let pds = at.net.pds(repo).await?;
        let mut q = vec![
            ("repo", repo.to_string()),
            ("collection", collection.to_string()),
            ("limit", input["limit"].as_u64().unwrap_or(100).to_string()),
        ];
        if let Some(c) = input["cursor"].as_str() {
            q.push(("cursor", c.to_string()));
        }
        let r = at
            .http
            .get(format!("{}/xrpc/com.atproto.repo.listRecords", pds.trim_end_matches('/')))
            .query(&q)
            .send()
            .await
            .map_err(|e| e.to_string())?;
        let status = r.status();
        let body = r.text().await.map_err(|e| e.to_string())?;
        if !status.is_success() {
            return Err(format!("listRecords {repo} {collection}: {status}: {}", tail(&body)));
        }
        let o: Value = serde_json::from_str(&body).map_err(|e| format!("listRecords {repo} {collection}: {e}"))?;
        let fields: Vec<&str> = input["fields"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
        let records: Vec<Value> = o["records"]
            .as_array()
            .into_iter()
            .flatten()
            .map(|r| {
                let mut out = json!({ "uri": r["uri"], "createdAt": r["value"]["createdAt"] });
                if !fields.is_empty() {
                    out["value"] = project(&r["value"], &fields);
                }
                out
            })
            .collect();
        Ok(json!({ "records": records, "cursor": o["cursor"] }))
    }

    async fn handle(&self, nsid: &str, input: Value) -> Result<Value, String> {
        let what = nsid.strip_prefix(P).unwrap_or(nsid);
        match what {
            "requester.start" => self.requester_start(&input).await,
            "request.put" => self.request_put(&input).await,
            "request.ssh" => {
                let name = str_of(&input, "name")?;
                let mut args = self.sub_args("ssh", name, &[]);
                if let Some(t) = input["transport"].as_str() {
                    args.extend(["--transport".to_string(), t.to_string()]);
                }
                args.extend(exec_words(&input)?);
                let (ok, out) = self.cli(&args).await?;
                Ok(json!({ "ok": ok, "output": out }))
            }
            "request.agent" => {
                let name = str_of(&input, "name")?;
                let (ok, out) = self.cli(&self.sub_args("agent", name, &[])).await?;
                if ok {
                    Ok(json!({}))
                } else {
                    Err(format!("run request agent: {}", tail(&out)))
                }
            }
            "request.summary" => self.summary(str_of(&input, "name")?),
            "request.release" => self.release(str_of(&input, "name")?).await,
            "request.state" => self.invocation_state(str_of(&input, "name")?),
            "request.contract" => self.request_contract(str_of(&input, "name")?).await,
            "request.sshTry" => {
                let name = str_of(&input, "name")?;
                let (ok, out) = self
                    .cli(&self.sub_args("ssh", name, &["--exec", "--", "id"]))
                    .await?;
                Ok(
                    json!({ "ok": ok, "output": tail(&out), "restarted": out.contains("starting the invocation's kcp again") || out.contains("starting it again from its state dir") }),
                )
            }
            "request.setPhase" => self.set_phase(str_of(&input, "name")?, str_of(&input, "phase")?),
            "request.removeState" => {
                let dir = self.invocation_dir(str_of(&input, "name")?);
                std::fs::remove_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
                Ok(json!({ "removed": dir.display().to_string() }))
            }
            "request.releaseTry" => {
                let name = str_of(&input, "name")?;
                let t = Instant::now();
                let (ok, out) = self.cli(&self.sub_args("release", name, &[])).await?;
                Ok(
                    json!({ "ok": ok, "output": tail(&out), "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }),
                )
            }
            "devsup.samename" => self.devsup_samename().await,
            "devsup.race" => self.devsup_race(&input).await,
            "request.kill" => self.kill_owner(
                str_of(&input, "name")?,
                input["signal"].as_str().unwrap_or("KILL"),
            ),
            "request.records" => {
                let name = str_of(&input, "name")?;
                let did = self.acct()?.did_now();
                let logs = self.request_logs(name);
                let declined = more::declined_accepts(&logs);
                let mut uris: Vec<String> = uris_in(&logs, &did)
                    .into_iter()
                    .filter(|u| !kept_by_rule(u) && !declined.contains(u))
                    .collect();
                if let Ok(s) = self.summary(name)
                    && let Some(r) = s["contract"]["receiptUri"].as_str()
                {
                    uris.push(format!(
                        "at://{did}/{}/{}",
                        identlink::market::VM_DELETE_NSID,
                        identlink::market::release_rkey(r)
                    ));
                }
                Ok(json!({ "uris": uris, "declined": declined }))
            }
            "repo.listCreated" => self.list_created(&input).await,
            "account.envFile" => {
                use std::io::Write;
                use std::os::unix::fs::OpenOptionsExt;
                let p = self
                    .dir
                    .join(format!("atp-{}.env", self.env_files.lock().unwrap().len()));
                let mut f = std::fs::OpenOptions::new()
                    .create(true)
                    .truncate(true)
                    .write(true)
                    .mode(0o600)
                    .open(&p)
                    .map_err(|e| e.to_string())?;
                writeln!(f, "ATP_PASSWORD={}", self.password()?).map_err(|e| e.to_string())?;
                self.env_files.lock().unwrap().push(p.clone());
                Ok(json!({ "path": p }))
            }
            "atirc.meshPub" => {
                let text = input["text"]
                    .as_str()
                    .ok_or("text is required")?
                    .to_string();
                self.mesh(&input, &["pub", &text]).await
            }
            "atirc.meshPeers" => self.mesh(&input, &["peers"]).await,
            "com.fedfork.exec.identlink.walk" => {
                let agent = input["agent"].as_str().ok_or("agent is required")?;
                // the host's client (the target's names through its front, trusting its CA): a staging network's PLC is
                // https by name under the run's root, which identlink's default client (built-in roots) does not trust
                let io = identlink::HttpIo {
                    client: self.http.clone(),
                    plc: self.target.plc.trim_end_matches('/').into(),
                    ..Default::default()
                };
                Ok(
                    serde_json::to_value(identlink::walk(&io, agent, Default::default()).await)
                        .unwrap(),
                )
            }
            "com.fedfork.exec.identlink.plc" => {
                self.plc(input["did"].as_str().ok_or("did is required")?)
                    .await
            }
            other => match self.more(other, &input).await {
                Some(r) => r,
                None => match self.knotless(other, &input).await {
                    Some(r) => r,
                    None => Err(format!("{nsid}: not a call of this host")),
                },
            },
        }
    }

    /// A failed run's logs into `out` (the dump's logs/), before the freeze or the cleanup, each redacted: per
    /// invocation `<name>/local/` (this host's request log, its requester's dev logs) and `<name>/vm/` (over
    /// request.ssh: the VM's journal and every *.log under /home and /root: core.log, dev logs, step logs), and per
    /// requester its log. What could not be read goes into `<name>/errors.txt`. Each the last 512 KiB, 6 MiB in all,
    /// smallest first (an artifact reaches the runner through the guest agent's 16 MiB job output ring), the rest
    /// named in `skipped.txt`.
    pub async fn collect_logs(&self, out: &Path) {
        let mut room = LOGS_MAX;
        let mut skipped = vec![];
        let requests: Vec<(String, PathBuf)> = self
            .requests
            .lock()
            .unwrap()
            .iter()
            .map(|(n, s)| (n.clone(), s.clone()))
            .collect();
        for (name, state) in requests {
            let o = out.join(&name);
            let mut files = vec![state.join(format!("{name}-request.log"))];
            walk(&state.join("requests").join(&name), &mut files);
            for f in by_size(files) {
                let to = o.join("local").join(f.strip_prefix(&state).unwrap_or(&f));
                copy_redacted(&f, &to, &mut room, &mut skipped);
            }
            // only a VM the invocation held answers ssh (an invocation still renting would make it wait)
            let held = std::fs::read_to_string(state.join(format!("{name}-request.log")))
                .is_ok_and(|l| l.contains(r#""phase":"Held""#));
            let vm = match held {
                true => self.vm_logs(&name).await,
                false => Err(format!("{name}: its VM was never Held: no VM logs")),
            };
            match vm {
                Ok(tree) => {
                    let mut files = vec![];
                    walk(&tree, &mut files);
                    for f in by_size(files) {
                        let to = o.join("vm").join(f.strip_prefix(&tree).unwrap_or(&f));
                        copy_redacted(&f, &to, &mut room, &mut skipped);
                    }
                    let _ = std::fs::remove_dir_all(&tree);
                }
                Err(e) => {
                    let _ = std::fs::create_dir_all(&o);
                    let _ = std::fs::write(o.join("errors.txt"), redact_log(&e));
                }
            }
        }
        for (name, (state, _)) in self.requesters.lock().await.iter() {
            let f = state.join(format!("{name}.log"));
            let to = out
                .join(format!("requester-{name}"))
                .join(format!("{name}.log"));
            copy_redacted(&f, &to, &mut room, &mut skipped);
        }
        if !skipped.is_empty() {
            let _ = std::fs::write(out.join("skipped.txt"), skipped.join("\n") + "\n");
        }
    }

    /// The VM of invocation `name`: its logs as a tar.gz (base64 between markers: the CLI's own lines share the
    /// output), unpacked into a dir of the work dir.
    async fn vm_logs(&self, name: &str) -> Result<PathBuf, String> {
        const LOGS: &str = "cd / && journalctl --no-pager -o short-iso -n 20000 > root/swc-journal.log 2>&1; \
            find home root -xdev -type f -name '*.log' -size -64M > root/swc-logs.list 2>/dev/null; echo SWC-LOGS-BEGIN; \
            tar -czf - -T root/swc-logs.list 2>/dev/null | base64 -w0; echo; echo SWC-LOGS-END; rm -f root/swc-journal.log root/swc-logs.list";
        let (_, text) = tokio::time::timeout(
            Duration::from_secs(300),
            self.cli(&[self.sub_args("ssh", name, &[]), exec_line(LOGS)].concat()),
        )
        .await
        .map_err(|_| format!("{name}: the VM's logs not read in 5 min"))??;
        self.unpack_logs(&format!("logs-{name}"), &text).await
    }

    /// A tar.gz the VM printed as base64 between SWC-LOGS-BEGIN and SWC-LOGS-END, unpacked into `<work dir>/<key>`.
    pub(crate) async fn unpack_logs(&self, key: &str, text: &str) -> Result<PathBuf, String> {
        let name = key;
        let b64 = text
            .split_once("SWC-LOGS-BEGIN\n")
            .and_then(|(_, r)| r.split_once("\nSWC-LOGS-END"))
            .map(|(b, _)| b.trim().to_string())
            .ok_or_else(|| format!("{name}: no logs from the VM: {}", tail(text)))?;
        let (file, tree) = (
            self.dir.join(format!("{key}.b64")),
            self.dir.join(key),
        );
        std::fs::write(&file, b64).map_err(|e| e.to_string())?;
        std::fs::create_dir_all(&tree).map_err(|e| e.to_string())?;
        let out = tokio::process::Command::new("sh")
            .args(["-c", "base64 -d < \"$1\" | tar -xzf - -C \"$2\"", "sh"])
            .arg(&file)
            .arg(&tree)
            .output()
            .await
            .map_err(|e| e.to_string())?;
        let _ = std::fs::remove_file(&file);
        if !out.status.success() {
            return Err(format!(
                "{name}: unpacking the VM's logs: {}",
                String::from_utf8_lossy(&out.stderr)
            ));
        }
        Ok(tree)
    }

    /// Wait (5 min) until the run requester `name` holds no ComputeContract that is not Released or Failed.
    async fn requester_contracts_released(&self, name: &str) -> Result<(), String> {
        let input = json!({ "cluster": format!("requester:{name}"), "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeContract" });
        let deadline = Instant::now() + Duration::from_secs(300);
        loop {
            let listed = self
                .kube
                .call(
                    atlibs_kube::LIST.into(),
                    Request {
                        input: Some(input.clone()),
                        ..Default::default()
                    },
                )
                .await;
            let left: Vec<String> = match &listed {
                Ok(o) => o
                    .as_ref()
                    .and_then(|o| o["items"].as_array())
                    .into_iter()
                    .flatten()
                    .filter(|c| {
                        !matches!(c["status"]["phase"].as_str(), Some("Released" | "Failed"))
                    })
                    .filter_map(|c| c["metadata"]["name"].as_str().map(String::from))
                    .collect(),
                Err(_) => vec![],
            };
            if left.is_empty() {
                return Ok(());
            }
            if Instant::now() > deadline {
                return Err(format!(
                    "run requester {name}: contracts not released 5 min after cleanup: {}",
                    left.join(", ")
                ));
            }
            tokio::time::sleep(Duration::from_secs(5)).await;
        }
    }

    /// Release every invocation, wait for each run requester's contracts, stop every requester (Ctrl-C), remove the
    /// password env files. The errors.
    pub async fn cleanup(&self) -> Vec<String> {
        let mut errors = self.knotless_cleanup().await;
        errors.extend(self.more_cleanup().await);
        let names: Vec<String> = self.requests.lock().unwrap().keys().cloned().collect();
        // every invocation at once (a multi-VM run's four, a guest requester's slow ones), each said with its time
        let released = futures::future::join_all(names.iter().map(|n| async move {
            let t = Instant::now();
            // an owner that is settling a release of its own (0005 B's out-of-band loss) stops its kcp under our ask:
            // ask again once it is gone
            let mut r = self.release(n).await;
            // a Failed invocation the run released already leaves no invocation.json: nothing is left to release
            // an invocation a release process took over (its owner was killed) is released and its kcp application gone
            if r.as_ref().is_err_and(|e| {
                e.contains(&format!("no invocation {n} under"))
                    || e.contains(&format!("the market network has no application {n}"))
            }) {
                r = Ok(json!({ "gone": true }));
            }
            for _ in 0..3 {
                if r.is_ok() {
                    break;
                }
                tokio::time::sleep(Duration::from_secs(20)).await;
                r = self.release(n).await;
            }
            eprintln!(
                "cleanup: {n} released in {:.0} s{}",
                t.elapsed().as_secs_f64(),
                r.as_ref()
                    .err()
                    .map(|e| format!(": {e}"))
                    .unwrap_or_default()
            );
            r
        }))
        .await;
        errors.extend(released.into_iter().filter_map(Result::err));
        // a run requester releases the contracts of the Infra/EphemeralInfra objects kube.cleanup just deleted only
        // while it runs: wait for them before the Ctrl-C (it leaves Held VMs to adopt), bounded
        let live: Vec<String> = self.requesters.lock().await.keys().cloned().collect();
        let settled =
            futures::future::join_all(live.iter().map(|n| self.requester_contracts_released(n)))
                .await;
        errors.extend(settled.into_iter().filter_map(Result::err));
        for (name, (_, mut c)) in self.requesters.lock().await.drain() {
            if let Ok(None) = c.try_wait() {
                if let Some(pid) = c.id() {
                    let _ = std::process::Command::new("kill")
                        .args(["-INT", &pid.to_string()])
                        .status();
                }
                if tokio::time::timeout(Duration::from_secs(120), c.wait())
                    .await
                    .is_err()
                {
                    let _ = c.kill().await;
                    errors.push(format!("run requester {name} did not stop in 120 s"));
                }
            }
        }
        for p in self.env_files.lock().unwrap().drain(..) {
            let _ = std::fs::remove_file(p);
        }
        errors
    }
}

impl Call for Swc {
    fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Option<Value>, String>> {
        Box::pin(async move {
            let r = self.handle(&nsid, input_of(&request)).await.map(Some);
            self.write_resources().await;
            r
        })
    }

    fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Sub, String>> {
        Box::pin(async move {
            let input = input_of(&request);
            match nsid.strip_prefix(P) {
                Some("request.start") => {
                    let r = self.request_start(&input).await;
                    self.write_resources().await;
                    r
                }
                Some("atirc.meshSub") => {
                    let (bin, dir) = self.mesh_node(&input).await?;
                    let mut child = tokio::process::Command::new(bin)
                        .args(["mesh", "sub", "--cursor", "0"])
                        .env("ATIRC_CONFIG_DIR", dir)
                        .env("ATP_USERNAME", &self.acct()?.handle)
                        .stdin(Stdio::null())
                        .stdout(Stdio::piped())
                        .stderr(Stdio::null())
                        .kill_on_drop(true)
                        .spawn()
                        .map_err(|e| e.to_string())?;
                    let out = child.stdout.take().unwrap();
                    let (tx, messages) = mpsc::unbounded_channel();
                    let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
                    tokio::spawn(async move {
                        use tokio::io::AsyncBufReadExt;
                        let mut lines = tokio::io::BufReader::new(out).lines();
                        loop {
                            tokio::select! {
                                l = lines.next_line() => match l {
                                    Ok(Some(l)) => if let Ok(v) = serde_json::from_str::<Value>(&l) && tx.send(Ok(v)).is_err() { break },
                                    _ => break,
                                },
                                cmd = cmds.recv() => match cmd {
                                    Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("meshSub takes no sends".into())); }
                                    Some(SubCmd::Close) | None => break,
                                },
                            }
                        }
                        let _ = child.kill().await;
                    });
                    Ok(Sub { messages, commands })
                }
                Some(other) => match self.more_sub(other, &input).await {
                    Some(r) => r,
                    None => Err(format!("{nsid} is not a subscription of this host")),
                },
                None => Err(format!("{nsid} is not a subscription of this host")),
            }
        })
    }
}

/// Every regular *.log file under `dir` (never a symlink, a kubeconfig, a key or a state file).
fn walk(dir: &Path, out: &mut Vec<PathBuf>) {
    for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
        let (p, Ok(t)) = (e.path(), e.file_type()) else {
            continue;
        };
        if t.is_dir() {
            walk(&p, out);
        } else if t.is_file() && p.extension().is_some_and(|x| x == "log") {
            out.push(p);
        }
    }
}

/// A failed run's logs/ in all, and each log's last bytes kept.
const LOGS_MAX: u64 = 6 << 20;
const LOG_MAX: usize = 512 << 10;

fn by_size(mut files: Vec<PathBuf>) -> Vec<PathBuf> {
    files.sort_by_key(|f| f.metadata().map(|m| m.len()).unwrap_or(0));
    files
}

/// The last LOG_MAX bytes of a log, redacted, at `to` (0600), while `room` lasts; else `to` goes into `skipped`.
fn copy_redacted(from: &Path, to: &Path, room: &mut u64, skipped: &mut Vec<String>) {
    let Ok(bytes) = std::fs::read(from) else {
        return;
    };
    let tail = &bytes[bytes.len().saturating_sub(LOG_MAX)..];
    if tail.len() as u64 > *room {
        skipped.push(to.display().to_string());
        return;
    }
    *room -= tail.len() as u64;
    let text = String::from_utf8_lossy(tail);
    if let Some(d) = to.parent() {
        let _ = std::fs::create_dir_all(d);
    }
    use std::io::Write;
    use std::os::unix::fs::OpenOptionsExt;
    if let Ok(mut f) = std::fs::OpenOptions::new()
        .create(true)
        .truncate(true)
        .write(true)
        .mode(0o600)
        .open(to)
    {
        let _ = f.write_all(redact_log(&text).as_bytes());
    }
}

/// Words that name a secret in a log: the dump's (atlibs ticks) and a kubeconfig's, a header's, a PEM's.
const SECRET_WORDS: &[&str] = &[
    "password",
    "passwd",
    "token",
    "secret",
    "kubeconfig",
    "cookie",
    "authorization",
    "bearer",
    "jwt",
    "apikey",
    "api_key",
    "privatekey",
    "private_key",
    "private-key",
    "key-data",
    "credential",
    "seed",
];

/// A log redacted line by line: JSON (whole or after a prefix) as the dump is (secret-named keys) plus every string
/// value scrubbed; other text cut after the first secret word's `=` or `:`; JWT-like words and PEM blocks dropped.
/// The last lines of the console of every VM of the dev net `instance` started in the last 15 minutes whose console
/// never printed its "reporting iroh://" line (the guest never came on network): at most 3 VMs, 40 lines each, redacted;
/// the last 300 lines of each go to `dump_dir/console-<vm>.log`.
fn silent_console_tails(state_dir: &Path, instance: &str, dump_dir: &Path) -> String {
    let apps = state_dir.join("dev").join(instance).join("apps/dev");
    let mut found: Vec<(std::time::SystemTime, String, String)> = vec![];
    let Ok(bidders) = std::fs::read_dir(&apps) else {
        return String::new();
    };
    for b in bidders.flatten() {
        let Ok(handles) = std::fs::read_dir(b.path().join("handles")) else {
            continue;
        };
        for h in handles.flatten() {
            let Ok(vms) = std::fs::read_dir(h.path().join("vms")) else {
                continue;
            };
            for vm in vms.flatten() {
                let console = vm.path().join("console.log");
                let Ok(meta) = std::fs::metadata(&console) else {
                    continue;
                };
                let Ok(modified) = meta.modified() else {
                    continue;
                };
                if modified
                    .elapsed()
                    .map_or(true, |e| e > Duration::from_secs(900))
                {
                    continue;
                }
                let Ok(raw) = std::fs::read(&console) else {
                    continue;
                };
                let text = String::from_utf8_lossy(&raw).to_string();
                if text.contains("reporting iroh://") {
                    continue;
                }
                found.push((modified, vm.file_name().to_string_lossy().to_string(), text));
            }
        }
    }
    found.sort_by(|a, b| b.0.cmp(&a.0));
    let mut out = String::new();
    for (_, name, text) in found.into_iter().take(3) {
        let lines: Vec<&str> = text.lines().collect();
        let _ = std::fs::create_dir_all(dump_dir);
        let from = lines.len().saturating_sub(300);
        let _ = std::fs::write(
            dump_dir.join(format!("console-{name}.log")),
            redact_log(&lines[from..].join("\n")),
        );
        let from = lines.len().saturating_sub(40);
        out.push_str(&format!(
            "== {name} (last 40 of {} lines)\n{}\n",
            lines.len(),
            redact_log(&lines[from..].join("\n"))
        ));
    }
    out
}

pub fn redact_log(text: &str) -> String {
    let mut out = String::with_capacity(text.len());
    let mut pem = false;
    for l in text.lines() {
        if l.contains("-----BEGIN") {
            pem = !l.contains("-----END");
            out.push_str("<redacted pem>\n");
            continue;
        }
        if pem {
            pem = !l.contains("-----END");
            continue;
        }
        let line = match l
            .find('{')
            .map(|i| (i, serde_json::from_str::<Value>(&l[i..])))
        {
            Some((i, Ok(v))) => format!(
                "{}{}",
                scrub(&l[..i]),
                scrub_strings(atlibs_host::ticks::redact(&v))
            ),
            _ => scrub(l),
        };
        out.push_str(&line);
        out.push('\n');
    }
    out
}

fn scrub_strings(v: Value) -> Value {
    match v {
        Value::String(s) => Value::String(scrub(&s)),
        Value::Array(a) => Value::Array(a.into_iter().map(scrub_strings).collect()),
        Value::Object(o) => {
            Value::Object(o.into_iter().map(|(k, v)| (k, scrub_strings(v))).collect())
        }
        v => v,
    }
}

fn scrub(s: &str) -> String {
    let lower = s.to_ascii_lowercase();
    let cut = SECRET_WORDS
        .iter()
        .filter_map(|w| lower.find(w).map(|i| i + w.len()))
        .filter_map(|i| {
            lower[i..]
                .find(['=', ':'])
                .filter(|j| *j < 40)
                .map(|j| i + j)
        })
        .min();
    let mut s = match cut {
        Some(j) => format!("{}<redacted>", &s[..=j]),
        None => s.to_string(),
    };
    for w in s
        .clone()
        .split(|c: char| c.is_whitespace() || "\"'=,;()[]{}".contains(c))
    {
        if w.len() > 20 && w.starts_with("eyJ") && w.matches('.').count() >= 2 {
            s = s.replace(w, "<redacted>");
        }
    }
    s
}

/// The command of a host call: `argv` (an array: that exact argv on the guest) or `exec` (a shell line, run as
/// `bash -c <line>`, as the guest's root login shell ran a `--exec <line>` before plan 0043).
pub(crate) fn exec_words(input: &Value) -> Result<Vec<String>, String> {
    if let Some(argv) = input["argv"].as_array() {
        let argv: Vec<String> = argv
            .iter()
            .map(|w| w.as_str().map(String::from).ok_or("argv: strings"))
            .collect::<Result<_, _>>()?;
        if argv.is_empty() {
            return Err("argv: a command".into());
        }
        return Ok(["--exec".to_string(), "--".to_string()].into_iter().chain(argv).collect());
    }
    Ok(exec_line(input["exec"].as_str().ok_or("exec or argv is required")?))
}