~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
953 lines · 35.4 KB Raw History
//! The orchestrator: a real Go requester (`socialweb-computer run requester` / `run request`) of a release,
//! a commit or a local build, signed in as a target account. Tests drive it through its kcp and its CLI.

use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::{Duration, Instant};

use anyhow::{Context, Result, anyhow, bail, ensure};
use serde_json::Value;
use sha2::{Digest, Sha256};

use crate::kcp::Kcp;
use crate::target::{Account, Target, expand};
use crate::{Ctx, eventually, home_tmp, repo};

const BUCKET: &str = "https://socialweb-computer-kcp.sfo3.digitaloceanspaces.com";

#[derive(Clone, Debug)]
pub struct Binary {
    pub path: PathBuf,
    /// `socialweb-computer version` (commit, dirty state)
    pub version: String,
}

fn arch() -> &'static str {
    if cfg!(target_arch = "aarch64") {
        "linux-arm64"
    } else {
        "linux-amd64"
    }
}

/// The release account (socialweb.computer) and project key of the release records: `software.release.channel/<key>:stable`
/// names a `software.release.version/<key>:<version>`, which names a `software.release.manifest` whose artifacts carry
/// each binary's url and its raw CIDv1 (sha2-256), the same records the bidders follow (`internal/releasefollow`).
const RELEASE_DID: &str = "did:plc:gmlkrhimoueejcertevr4hl3";
const RELEASE_KEY: &str = "socialweb-computer";

const B32: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";

/// The sha256 a raw CIDv1 (base32, sha2-256) names, as release-workload's `sha_of`.
fn sha_of(cid: &str) -> Option<String> {
    let (mut out, mut buf, mut bits) = (vec![], 0u32, 0u32);
    for c in cid.strip_prefix('b')?.bytes() {
        buf = (buf << 5) | B32.iter().position(|x| *x == c)? as u32;
        bits += 5;
        if bits >= 8 {
            bits -= 8;
            out.push(((buf >> bits) & 0xff) as u8);
        }
    }
    (out.len() == 36 && out[..4] == [0x01, 0x55, 0x12, 0x20])
        .then(|| out[4..].iter().map(|b| format!("{b:02x}")).collect())
}

/// The release account's public records, read without a credential (plc.directory, then its PDS's getRecord).
struct Records {
    http: reqwest::Client,
    pds: String,
}

impl Records {
    async fn new() -> Result<Self> {
        let http = reqwest::Client::new();
        let doc: Value = http
            .get(format!("https://plc.directory/{RELEASE_DID}"))
            .send()
            .await?
            .error_for_status()?
            .json()
            .await?;
        let pds = doc["service"]
            .as_array()
            .into_iter()
            .flatten()
            .find(|s| s["id"] == "#atproto_pds")
            .and_then(|s| s["serviceEndpoint"].as_str())
            .with_context(|| format!("{RELEASE_DID}: no #atproto_pds in its DID document"))?
            .trim_end_matches('/')
            .to_string();
        Ok(Self { http, pds })
    }

    /// None when the record is not there.
    async fn get(&self, collection: &str, rkey: &str) -> Result<Option<Value>> {
        let url = format!(
            "{}/xrpc/com.atproto.repo.getRecord?repo={RELEASE_DID}&collection={collection}&rkey={rkey}",
            self.pds
        );
        let r = self.http.get(&url).send().await?;
        if r.status() == reqwest::StatusCode::BAD_REQUEST {
            let body: Value = r.json().await.unwrap_or_default();
            if body["error"] == "RecordNotFound" {
                return Ok(None);
            }
            bail!("{url}: {body}");
        }
        Ok(Some(r.error_for_status()?.json().await?))
    }

    /// The version a channel points at (`stable`, `rc`); None when the channel has no record.
    async fn channel(&self, name: &str) -> Result<Option<String>> {
        let Some(c) = self.get("software.release.channel", &format!("{RELEASE_KEY}:{name}")).await? else {
            return Ok(None);
        };
        let uri = c["value"]["version"]["uri"].as_str().with_context(|| format!("channel {name}: no version.uri: {c}"))?;
        let rkey = uri.rsplit('/').next().unwrap_or_default();
        Ok(Some(rkey.strip_prefix(&format!("{RELEASE_KEY}:")).unwrap_or(rkey).to_string()))
    }

    /// The url and sha256 of `file` in that version's manifest.
    async fn artifact(&self, version: &str, file: &str) -> Result<(String, String)> {
        let v = self
            .get("software.release.version", &format!("{RELEASE_KEY}:{version}"))
            .await?
            .with_context(|| format!("{RELEASE_DID}: no software.release.version {RELEASE_KEY}:{version}"))?;
        let uri = v["value"]["manifest"]["uri"].as_str().with_context(|| format!("version {version}: no manifest.uri: {v}"))?;
        let m = self
            .get("software.release.manifest", uri.rsplit('/').next().unwrap_or_default())
            .await?
            .with_context(|| format!("version {version}: its manifest {uri} is not there"))?;
        let a = m["value"]["artifacts"]
            .as_array()
            .into_iter()
            .flatten()
            .find(|a| a["name"] == file)
            .with_context(|| format!("version {version}: its manifest {uri} lists no {file}"))?;
        let url = a["url"].as_str().with_context(|| format!("{uri} {file}: no url"))?;
        let cid = a["cid"].as_str().unwrap_or_default();
        let sha = sha_of(cid).with_context(|| format!("{uri} {file}: cid {cid:?} is not a raw sha2-256 CIDv1"))?;
        Ok((url.to_string(), sha))
    }
}

/// `release`: the version channel stable points at; `release:<version>`: that version (an rc or a stable).
async fn release(version: Option<&str>, file: &str) -> Result<(String, String, String)> {
    let r = Records::new().await?;
    let v = match version {
        Some(v) => v.to_string(),
        None => match r.channel("stable").await? {
            Some(v) => v,
            None => bail!(
                "requester \"release\": {RELEASE_DID} has no software.release.channel {RELEASE_KEY}:stable yet (no promote has gone green; channel rc: {}): name a version, release:<version>, or sha:<commit> / local:<commit>",
                r.channel("rc").await?.unwrap_or_else(|| "none".into())
            ),
        },
    };
    let (url, sha) = r.artifact(&v, file).await?;
    Ok((v, url, sha))
}

/// `url` into `path` once, checked against `want` (sha256).
async fn fetch(path: &Path, url: &str, want: &str) -> Result<()> {
    if path.exists() {
        return Ok(());
    }
    let bytes = reqwest::Client::new()
        .get(url)
        .send()
        .await?
        .error_for_status()?
        .bytes()
        .await?;
    let got = format!("{:x}", Sha256::digest(&bytes));
    ensure!(got == want, "{url}: sha256 {got}, want {want}");
    std::fs::create_dir_all(path.parent().unwrap())?;
    let tmp = path.with_extension("tmp");
    std::fs::write(&tmp, &bytes)?;
    std::fs::set_permissions(&tmp, std::os::unix::fs::PermissionsExt::from_mode(0o755))?;
    std::fs::rename(&tmp, &path)?;
    Ok(())
}

/// `<BUCKET>/<prefix>/binaries/<file>` and its sha256 from the SHA256SUMS beside it.
async fn bucket_artifact(prefix: &str, file: &str) -> Result<(String, String)> {
    let base = format!("{BUCKET}/{prefix}/binaries");
    let sums = reqwest::Client::new()
        .get(format!("{base}/SHA256SUMS"))
        .send()
        .await?
        .error_for_status()?
        .text()
        .await?;
    let want = sums
        .lines()
        .find_map(|l| l.strip_suffix(&format!("  {file}")))
        .with_context(|| format!("{base}/SHA256SUMS lists no {file}"))?;
    Ok((format!("{base}/{file}"), want.to_string()))
}

impl Binary {
    /// "release" (the version channel stable points at) | "release:<version>" (that version's record) | "sha:<full commit>" | "local:<full commit>" (a hand build, local/<commit>) | "oci:<host/name>@sha256:<manifest digest>" (a
    /// CI build pushed to ttl.sh, pulled with oras, which checks it against the digest) | "path:<file>"; downloads are checked against SHA256SUMS
    /// and cached under ~/.tmp/swc-system-tests/bin.
    /// `<host/name>@sha256:<manifest digest>`: pulled with oras (the ORAS env, else `oras` on PATH) into ~/.tmp/swc-system-tests/bin/oci-<digest>
    async fn oci(reference: &str) -> Result<Self> {
        let (_, digest) = reference
            .split_once("@sha256:")
            .filter(|(_, d)| d.len() == 64 && d.bytes().all(|b| b.is_ascii_hexdigit()))
            .with_context(|| format!("oci:{reference}: want <host/name>@sha256:<64 hex>"))?;
        let dir = home_tmp("bin").join(format!("oci-{digest}"));
        let path = dir.join(format!("socialweb-computer-{}", arch()));
        if !path.exists() {
            std::fs::create_dir_all(&dir)?;
            let oras = std::env::var("ORAS").unwrap_or_else(|_| "oras".into());
            let out = tokio::process::Command::new(&oras)
                .args(["pull", "--no-tty", reference, "-o"])
                .arg(&dir)
                .output()
                .await
                .with_context(|| format!("run {oras}"))?;
            ensure!(
                out.status.success(),
                "oras pull {reference}: {}",
                String::from_utf8_lossy(&out.stderr)
            );
            // a file pushed in parts (scripts/oci-push.sh): <file>.part-0000, .part-0001, ... put back together in order
            let name = path
                .file_name()
                .and_then(|n| n.to_str())
                .unwrap_or_default()
                .to_string();
            let mut parts: Vec<_> = std::fs::read_dir(&dir)?
                .flatten()
                .map(|e| e.path())
                .filter(|p| {
                    p.file_name()
                        .and_then(|n| n.to_str())
                        .is_some_and(|n| n.starts_with(&format!("{name}.part-")))
                })
                .collect();
            parts.sort();
            if !parts.is_empty() {
                let mut whole = std::fs::File::create(&path)?;
                for part in &parts {
                    std::io::copy(&mut std::fs::File::open(part)?, &mut whole)?;
                }
                for part in &parts {
                    std::fs::remove_file(part)?;
                }
            }
            std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o755))?;
        }
        Self::at(path)
    }

    pub async fn resolve(spec: &str) -> Result<Self> {
        let file = format!("socialweb-computer-{}", arch());
        let prefix = match spec.split_once(':') {
            _ if spec == "release" || spec.starts_with("release:") => {
                let (_, url, sha) = release(spec.strip_prefix("release:"), &file).await?;
                let path = home_tmp("bin").join(format!("release-{sha}")).join(&file);
                fetch(&path, &url, &sha).await?;
                return Self::at(path);
            }
            Some(("sha", sha)) => format!("main/{sha}"),
            Some(("local", sha)) => format!("local/{sha}"),
            Some(("path", p)) => return Self::at(expand(p)),
            Some(("oci", r)) => return Self::oci(r).await,
            _ => bail!(
                "requester {spec:?}: want release, release:<version>, sha:<commit>, local:<commit>, oci:<ref>@sha256:<digest> or path:<file>"
            ),
        };
        let path = home_tmp("bin").join(prefix.replace('/', "-")).join(&file);
        if !path.exists() {
            let (url, want) = bucket_artifact(&prefix, &file).await?;
            fetch(&path, &url, &want).await?;
        }
        Self::at(path)
    }

    /// The public URL and sha256 of a spec's build for this host's arch, read from its records or SHA256SUMS without
    /// downloading it: release, release:<version>, sha:<commit>, local:<commit>; None for path: and oci: (no public URL).
    pub async fn published(spec: &str) -> Result<Option<(String, String)>> {
        let file = format!("socialweb-computer-{}", arch());
        let prefix = match spec.split_once(':') {
            _ if spec == "release" || spec.starts_with("release:") => {
                let (_, url, sha) = release(spec.strip_prefix("release:"), &file).await?;
                return Ok(Some((url, sha)));
            }
            Some(("sha", sha)) => format!("main/{sha}"),
            Some(("local", sha)) => format!("local/{sha}"),
            _ => return Ok(None),
        };
        Ok(Some(bucket_artifact(&prefix, &file).await?))
    }

    fn at(path: PathBuf) -> Result<Self> {
        let out = std::process::Command::new(&path)
            .arg("version")
            .output()
            .with_context(|| path.display().to_string())?;
        ensure!(out.status.success(), "{} version failed", path.display());
        let v = String::from_utf8_lossy(&out.stdout)
            .lines()
            .next()
            .unwrap_or_default()
            .trim_start_matches("socialweb-computer ")
            .to_string();
        Ok(Self { path, version: v })
    }

    pub fn cmd(&self) -> tokio::process::Command {
        let mut c = tokio::process::Command::new(&self.path);
        c.stdin(Stdio::null());
        c
    }

    /// Run to completion; (exit ok, stdout + stderr). The output never holds a secret of ours (no secret goes in).
    pub async fn run(&self, args: &[&str]) -> Result<(bool, String)> {
        let out = self.cmd().args(args).output().await?;
        let mut text = String::from_utf8_lossy(&out.stdout).to_string();
        text.push_str(&String::from_utf8_lossy(&out.stderr));
        Ok((out.status.success(), text))
    }

    /// Whether this CLI takes every option before `--exec -- <command> [args...]` (plan 0043). A CLI before it (the
    /// previous rc of 0006:compat) takes the requester options after `--` and `--exec <one shell line>`.
    pub async fn exec_argv(&self) -> bool {
        match self.run(&["run", "request", "--help"]).await {
            Ok((_, out)) => out.contains(EXEC_USAGE),
            Err(_) => true,
        }
    }
}

const EXEC_USAGE: &str = "--exec -- <command> [args...]";

/// `--exec -- bash -c <line>`: a shell line as the guest's root login shell (bash) ran a `--exec <line>` before plan 0043.
pub fn exec_line(line: &str) -> Vec<String> {
    ["--exec", "--", "bash", "-c", line].map(String::from).to_vec()
}

/// The argv as one line for the guest's shell, each word single-quoted (what the CLI sends since plan 0043).
pub fn shell_words(argv: &[String]) -> String {
    argv.iter()
        .map(|w| format!("'{}'", w.replace('\'', r"'\''")))
        .collect::<Vec<_>>()
        .join(" ")
}

/// `args` cut at `--exec -- <command> [args...]`: the options before it and the command, if any.
pub fn split_exec(args: &[String]) -> Result<(Vec<String>, Option<Vec<String>>)> {
    match args.iter().position(|a| a == "--exec") {
        None => {
            ensure!(!args.iter().any(|a| a == "--"), "args: -- only after --exec ({EXEC_USAGE})");
            Ok((args.to_vec(), None))
        }
        Some(i) => {
            ensure!(
                args.get(i + 1).map(String::as_str) == Some("--") && args.len() > i + 2,
                "args: --exec takes the command after --: {EXEC_USAGE}"
            );
            Ok((args[..i].to_vec(), Some(args[i + 2..].to_vec())))
        }
    }
}

/// A `run request` or `run requester` command line from its parts: `opts`, the requester (application) options
/// `app`, then `--exec -- <command>`; for a CLI before plan 0043 (`exec_argv` false) the command as one `--exec`
/// line among the options and `app` after `--`.
pub fn request_line(exec_argv: bool, opts: &[String], app: &[String], command: Option<&[String]>) -> Vec<String> {
    let mut out = opts.to_vec();
    if exec_argv {
        out.extend(app.iter().cloned());
        if let Some(c) = command {
            out.extend(["--exec".to_string(), "--".to_string()]);
            out.extend(c.iter().cloned());
        }
        return out;
    }
    if let Some(c) = command {
        out.extend(["--exec".to_string(), shell_words(c)]);
    }
    if !app.is_empty() {
        out.push("--".into());
        out.extend(app.iter().cloned());
    }
    out
}

/// How the requester signs in as its account.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum SignIn {
    /// pdsAccount: the app password in ATPROTO_PASSWORD of the one command
    AppPassword,
    /// OAuth QR, driven by scripts/oauth-signin.ts (headless Chrome) from the creds file
    OAuthQr,
}

impl SignIn {
    /// SWC_SIGNIN=oauth-qr: OAuth QR (headless Chrome); default: the app password.
    pub fn from_env() -> Self {
        if std::env::var("SWC_SIGNIN").is_ok_and(|v| v == "oauth-qr") {
            SignIn::OAuthQr
        } else {
            SignIn::AppPassword
        }
    }
}

pub fn identity_args(t: &Target, a: &Account, mode: SignIn) -> Vec<String> {
    match mode {
        SignIn::AppPassword => vec![
            "--atproto-identity-mode".into(),
            "pdsAccount".into(),
            "--atproto-handle".into(),
            a.handle.clone(),
            "--atproto-pds-url".into(),
            t.pds.clone(),
        ],
        SignIn::OAuthQr => vec!["--atproto-oauth-qr".into()],
    }
}

/// Sign a QR sign-in URL in with the creds file (the URL never reaches a log of ours).
pub async fn oauth_signin(a: &Account, url: &str) -> Result<()> {
    oauth_signin_with(a, url, &[]).await
}

/// `oauth_signin` with more oauth-signin.ts options (`--start-api`, `--cookie-file <owner-only file>`).
pub async fn oauth_signin_with(a: &Account, url: &str, extra: &[&str]) -> Result<()> {
    oauth_signin_on(Path::new("deno"), None, a, url, extra).await
}

/// `oauth_signin_with` run by the deno at `deno`, whose own fetches (`--start-api`) also trust the CA bundle `ca`.
pub async fn oauth_signin_on(deno: &Path, ca: Option<&Path>, a: &Account, url: &str, extra: &[&str]) -> Result<()> {
    let prefix = a
        .oauth_account
        .as_deref()
        .context("the account has no oauthAccount (creds prefix) for OAuth")?;
    let mut cmd = tokio::process::Command::new(deno);
    if let Some(ca) = ca {
        cmd.env("DENO_CERT", ca);
    }
    let out = cmd
        .current_dir(repo())
        .args([
            "run",
            "-A",
            "--no-lock",
            "scripts/oauth-signin.ts",
            "--creds",
        ])
        .arg(expand(&a.creds))
        .args(["--account", prefix, "--url", url])
        .args(extra)
        .stdout(Stdio::null())
        .stdin(Stdio::null())
        .output()
        .await?;
    ensure!(
        out.status.success(),
        "oauth-signin.ts exited {}: {}",
        out.status,
        tail(&without_url(&String::from_utf8_lossy(&out.stderr), url))
    );
    Ok(())
}

/// `text` without the sign-in URL: whole, and its query and fragment (the transfer's secret) wherever they appear alone.
fn without_url(text: &str, url: &str) -> String {
    let mut out = text.replace(url, "<sign-in url>");
    for part in url.split(['?', '#']).skip(1).filter(|p| !p.is_empty()) {
        out = out.replace(part, "<redacted>");
    }
    out
}

/// The JSON lines of a log whose msg is `msg`.
pub fn log_lines(path: &Path, msg: &str) -> Vec<Value> {
    std::fs::read_to_string(path)
        .unwrap_or_default()
        .lines()
        .filter(|l| l.contains(msg))
        .filter_map(|l| serde_json::from_str::<Value>(l).ok())
        .filter(|v| v["msg"] == msg)
        .collect()
}

pub async fn wait_log(paths: &[PathBuf], msg: &str, timeout: Duration) -> Result<Value> {
    eventually(
        &format!("log line {msg}"),
        timeout,
        Duration::from_secs(1),
        || async {
            Ok(paths
                .iter()
                .find_map(|p| log_lines(p, msg).into_iter().next()))
        },
    )
    .await
}

/// Every at://<did>/... URI the logs name (the records a run wrote in that repo).
pub fn uris_in(paths: &[PathBuf], did: &str) -> Vec<String> {
    let needle = format!("at://{did}/");
    let mut out = vec![];
    for p in paths {
        let text = std::fs::read_to_string(p).unwrap_or_default();
        for (i, _) in text.match_indices(&needle) {
            let uri: String = text[i..]
                .chars()
                .take_while(|c| !matches!(c, '"' | ' ' | '\\' | ',' | ')' | '\n'))
                .collect();
            if uri.split('/').count() == 5 && !out.contains(&uri) {
                out.push(uri);
            }
        }
    }
    out
}

/// Records that stay by rule after a release: vouches (Retain), the signer key, the operator associations of bidders.
pub fn kept_by_rule(uri: &str) -> bool {
    ["/sh.tangled.graph.vouch/", "badgeBlueKeys", "/app.bsky."]
        .iter()
        .any(|k| uri.contains(k))
}

/// The requester's log message of an accept a bidder refused (409 CapacityExceeded, no receipt).
pub const ACCEPT_DECLINED: &str = "requester_submit_accept_declined";

/// The accepts a bidder refused, by a requester's log lines: the requester keeps them by design (docs/acceptance/0014:
/// "the requester keeps that accept (a bidder may still act on it)"), so they are no leftover of a release. The one
/// rule of every record check (request.records, requester.records, gateway.log for assert.noneSince).
pub fn declined_accepts<'a>(lines: impl IntoIterator<Item = &'a Value>) -> Vec<String> {
    let mut out: Vec<String> = lines
        .into_iter()
        .filter(|l| l["msg"] == ACCEPT_DECLINED)
        .filter_map(|l| l["accept"].as_str().map(String::from))
        .collect();
    out.sort();
    out.dedup();
    out
}

/// A long-lived `run requester` (the orchestrator): its kcp, its ready file, its CLI helpers.
pub struct Requester {
    pub bin: Binary,
    pub name: String,
    pub state: PathBuf,
    pub log: PathBuf,
    args: Vec<String>,
    password: Option<String>,
    child: tokio::sync::Mutex<Option<tokio::process::Child>>,
}

impl Requester {
    /// Start `run requester --name <name>` in a new owner-only state dir under the run's scratch dir, sign in,
    /// wait for the ready file. Stopped (Ctrl-C) by a cleanup.
    pub async fn start(
        ctx: &std::sync::Arc<Ctx>,
        role: &str,
        name: &str,
        mode: SignIn,
        extra: &[&str],
    ) -> Result<std::sync::Arc<Self>> {
        let a = ctx.role(role)?;
        let bin = Binary::resolve(&ctx.target.requester).await?;
        let state = ctx.dir.join(format!("requester-{name}"));
        std::fs::create_dir_all(&state)?;
        std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))?;
        let mut args: Vec<String> = ["run", "requester", "--name", name, "--state-dir"]
            .iter()
            .map(|s| s.to_string())
            .collect();
        args.push(state.display().to_string());
        args.extend(extra.iter().map(|s| s.to_string()));
        args.extend(identity_args(&ctx.target, a, mode));
        let r = std::sync::Arc::new(Self {
            bin,
            name: name.into(),
            log: state.join(format!("{name}.log")),
            state,
            args,
            password: (mode == SignIn::AppPassword)
                .then(|| a.password().map(|p| p.expose().to_string()))
                .transpose()?,
            child: tokio::sync::Mutex::new(None),
        });
        let stop = r.clone();
        ctx.defer(&format!("stop requester {name}"), async move {
            stop.stop().await
        });
        r.spawn().await?;
        if mode == SignIn::OAuthQr {
            let url = eventually(
                "the sign-in URL",
                Duration::from_secs(300),
                Duration::from_secs(3),
                || async {
                    let (ok, out) = r
                        .bin
                        .run(&[
                            "run",
                            "requester",
                            "signin",
                            &r.name,
                            "--state-dir",
                            &r.state.display().to_string(),
                        ])
                        .await?;
                    Ok(ok
                        .then(|| {
                            out.lines()
                                .find(|l| l.starts_with("https://"))
                                .map(str::to_string)
                        })
                        .flatten())
                },
            )
            .await?;
            oauth_signin(a, &url).await?;
        }
        r.wait_ready(Duration::from_secs(600)).await?;
        Ok(r)
    }

    async fn spawn(&self) -> Result<()> {
        let log = std::fs::OpenOptions::new()
            .create(true)
            .append(true)
            .open(&self.log)?;
        let mut c = self.bin.cmd();
        c.args(&self.args).stdout(log.try_clone()?).stderr(log);
        if let Some(p) = &self.password {
            c.env("ATPROTO_PASSWORD", p);
        }
        *self.child.lock().await = Some(c.spawn()?);
        Ok(())
    }

    pub fn ready_path(&self) -> PathBuf {
        self.state
            .join("requesters")
            .join(&self.name)
            .join("ready.json")
    }

    pub fn ready(&self) -> Result<Value> {
        Ok(serde_json::from_slice(&std::fs::read(self.ready_path())?)?)
    }

    pub async fn wait_ready(&self, timeout: Duration) -> Result<Value> {
        eventually(
            "the requester's ready file",
            timeout,
            Duration::from_secs(1),
            || async {
                if let Some(c) = self.child.lock().await.as_mut()
                    && let Some(st) = c.try_wait()?
                {
                    bail!(
                        "run requester {} exited {st} before ready; see {}",
                        self.name,
                        self.log.display()
                    );
                }
                Ok(self.ready().ok())
            },
        )
        .await
    }

    /// The requester workspace's kcp (kube-rs over the ready file's kubeconfig).
    pub async fn kcp(&self) -> Result<Kcp> {
        Kcp::kubeconfig(
            self.ready()?["kubeconfig"]
                .as_str()
                .context("ready file without kubeconfig")?,
        )
        .await
    }

    pub fn did(&self) -> Result<String> {
        Ok(self.ready()?["did"]
            .as_str()
            .context("ready file without did")?
            .into())
    }

    /// The core logs of the requester (its own log and the core.log files under the state dir).
    pub fn logs(&self) -> Vec<PathBuf> {
        let mut out = vec![self.log.clone()];
        find_named(&self.state, "core.log", &mut out);
        out
    }

    /// Ctrl-C: the kcp stops, the ready file goes, the state stays.
    pub async fn stop(&self) -> Result<()> {
        let mut g = self.child.lock().await;
        if let Some(c) = g.as_mut()
            && c.try_wait()?.is_none()
        {
            if let Some(pid) = c.id() {
                std::process::Command::new("kill")
                    .args(["-INT", &pid.to_string()])
                    .status()?;
            }
            if tokio::time::timeout(Duration::from_secs(120), c.wait())
                .await
                .is_err()
            {
                c.kill().await?;
                bail!(
                    "run requester {} did not stop in 120 s after SIGINT",
                    self.name
                );
            }
        }
        *g = None;
        Ok(())
    }

    /// Ctrl-C and the same command again (0013 A9).
    pub async fn restart(&self) -> Result<Value> {
        self.stop().await?;
        ensure!(
            !self.ready_path().exists(),
            "the ready file stayed after Ctrl-C"
        );
        self.spawn().await?;
        self.wait_ready(Duration::from_secs(300)).await
    }

    /// The WebRTC transport to a port a Held contract's guest tunnels as `tag` (a kcp in the guest): this requester's
    /// `run requester proxy` dials the datachannel (signalling through its own kcp) once per connection.
    pub fn webrtc_proxy(&self, contract: &str, tag: &str) -> crate::target::Via {
        let mut proxy = vec![self.bin.path.display().to_string()];
        proxy.extend(["run", "requester", "proxy", &self.name, "--state-dir"].map(String::from));
        proxy.push(self.state.display().to_string());
        proxy.extend(["--contract", contract, "--tag", tag].map(String::from));
        crate::target::Via::WebRtc { proxy }
    }

    /// `run requester ssh <name> --contract <cc> [--transport webrtc] --exec -- bash -c <line>`.
    pub async fn ssh(&self, contract: &str, transport: Option<&str>, exec: &str) -> Result<String> {
        let state = self.state.display().to_string();
        let mut args = vec!["run", "requester", "ssh", &self.name, "--state-dir", &state, "--contract", contract];
        if let Some(t) = transport {
            args.extend(["--transport", t]);
        }
        let line = exec_line(exec);
        args.extend(line.iter().map(String::as_str));
        let (ok, out) = self.bin.run(&args).await?;
        ensure!(
            ok,
            "run requester ssh {contract} ({transport:?}) failed: {}",
            tail(&out)
        );
        Ok(out)
    }
}

fn find_named(dir: &Path, name: &str, out: &mut Vec<PathBuf>) {
    for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
        let p = e.path();
        if p.is_dir() {
            find_named(&p, name, out);
        } else if p.file_name().is_some_and(|n| n == name) {
            out.push(p);
        }
    }
}

pub fn tail(s: &str) -> String {
    let lines: Vec<_> = s.lines().collect();
    lines[lines.len().saturating_sub(15)..].join("\n")
}

/// One `run request` invocation (its own kcp; the CLI path of the runbooks).
pub struct RunRequest {
    pub bin: Binary,
    pub name: String,
    pub state: PathBuf,
    pub log: PathBuf,
    child: tokio::sync::Mutex<Option<tokio::process::Child>>,
}

impl RunRequest {
    /// `run request --state-dir <dir> --name <name> <args> <identity> [--exec -- <command>]` in the background (app
    /// password); an older CLI gets its own layout (`request_line`).
    /// A cleanup releases it.
    pub async fn start(
        ctx: &std::sync::Arc<Ctx>,
        role: &str,
        name: &str,
        args: &[&str],
    ) -> Result<std::sync::Arc<Self>> {
        let bin = Binary::resolve(&ctx.target.requester).await?;
        Self::start_with(ctx, bin, role, name, args).await
    }

    /// `start` with another binary (0006 compat: the previous release's CLI against new servers).
    pub async fn start_with(
        ctx: &std::sync::Arc<Ctx>,
        bin: Binary,
        role: &str,
        name: &str,
        args: &[&str],
    ) -> Result<std::sync::Arc<Self>> {
        let a = ctx.role(role)?;
        let state = ctx.dir.join("requests");
        std::fs::create_dir_all(&state)?;
        std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))?;
        let log = state.join(format!("{name}-request.log"));
        let f = std::fs::File::create(&log)?;
        let (opts, command) = split_exec(&args.iter().map(|s| s.to_string()).collect::<Vec<_>>())?;
        let line = request_line(
            bin.exec_argv().await,
            &opts,
            &identity_args(&ctx.target, a, SignIn::AppPassword),
            command.as_deref(),
        );
        let mut c = bin.cmd();
        c.args(["run", "request", "--state-dir"])
            .arg(&state)
            .args(["--name", name])
            .args(&line);
        c.env("ATPROTO_PASSWORD", a.password()?.expose());
        c.stdout(f.try_clone()?).stderr(f);
        let r = std::sync::Arc::new(Self {
            bin,
            name: name.into(),
            state,
            log,
            child: tokio::sync::Mutex::new(Some(c.spawn()?)),
        });
        let rel = r.clone();
        ctx.defer(&format!("release {name}"), async move {
            rel.release().await.map(|_| ())
        });
        Ok(r)
    }

    /// The CLI log plus the invocation's core.log (while it runs, and after release).
    pub fn logs(&self) -> Vec<PathBuf> {
        let base = self.state.join("requests").join(&self.name);
        vec![
            self.log.clone(),
            base.join("dev").join(&self.name).join("logs/core.log"),
            base.join("logs/core.log"),
        ]
    }

    /// The contract phase lines of the CLI log; waits for `phase`.
    pub async fn wait_phase(&self, phase: &str, timeout: Duration) -> Result<Value> {
        eventually(
            &format!("{} {phase}", self.name),
            timeout,
            Duration::from_secs(1),
            || async {
                if let Some(c) = self.child.lock().await.as_mut()
                    && let Some(st) = c.try_wait()?
                    && !st.success()
                {
                    bail!(
                        "run request {} exited {st}: {}",
                        self.name,
                        tail(&std::fs::read_to_string(&self.log).unwrap_or_default())
                    );
                }
                Ok(log_lines(&self.log, "contract")
                    .into_iter()
                    .find(|l| l["phase"] == phase))
            },
        )
        .await
    }

    pub async fn wait_log(&self, msg: &str, timeout: Duration) -> Result<Value> {
        wait_log(&self.logs(), msg, timeout).await
    }

    fn sub(&self, verb: &str, extra: &[&str]) -> Vec<String> {
        let mut v: Vec<String> = ["run", "request", verb, &self.name, "--state-dir"]
            .iter()
            .map(|s| s.to_string())
            .collect();
        v.push(self.state.display().to_string());
        v.extend(extra.iter().map(|s| s.to_string()));
        v
    }

    /// `run request agent <name>`: publishes guest.agentStart.
    pub async fn agent(&self) -> Result<()> {
        let args = self.sub("agent", &[]);
        let (ok, out) = self
            .bin
            .run(&args.iter().map(String::as_str).collect::<Vec<_>>())
            .await?;
        ensure!(ok, "run request agent: {}", tail(&out));
        Ok(())
    }

    /// `run request ssh <name> [--transport webrtc] --exec -- bash -c <line>` (iroh first by default).
    pub async fn ssh(&self, transport: Option<&str>, exec: &str) -> Result<String> {
        let mut extra = vec![];
        if let Some(t) = transport {
            extra.extend(["--transport", t]);
        }
        let line = exec_line(exec);
        extra.extend(line.iter().map(String::as_str));
        let args = self.sub("ssh", &extra);
        let (ok, out) = self
            .bin
            .run(&args.iter().map(String::as_str).collect::<Vec<_>>())
            .await?;
        ensure!(ok, "run request ssh: {}", tail(&out));
        Ok(out)
    }

    /// Wait for the --hold process to exit (it exits after its first ssh unless held).
    pub async fn wait_exit(&self, timeout: Duration) -> Result<()> {
        if let Some(c) = self.child.lock().await.as_mut() {
            tokio::time::timeout(timeout, c.wait())
                .await
                .map_err(|_| {
                    anyhow!("run request {} still running after {timeout:?}", self.name)
                })??;
        }
        Ok(())
    }

    /// `run request release`; idempotent. Returns the seconds it took.
    pub async fn release(&self) -> Result<f64> {
        let t = Instant::now();
        let args = self.sub("release", &[]);
        let (ok, out) = self
            .bin
            .run(&args.iter().map(String::as_str).collect::<Vec<_>>())
            .await?;
        if let Some(mut c) = self.child.lock().await.take()
            && tokio::time::timeout(Duration::from_secs(300), c.wait())
                .await
                .is_err()
        {
            c.kill().await?;
        }
        ensure!(ok, "run request release {}: {}", self.name, tail(&out));
        Ok(t.elapsed().as_secs_f64())
    }

    pub fn summary(&self) -> Result<Value> {
        Ok(serde_json::from_slice(&std::fs::read(
            self.state
                .join("requests")
                .join(&self.name)
                .join("summary.json"),
        )?)?)
    }
}