//! The orchestrator: a real Go requester (`socialweb-computer run requester` / `run request`) of a release, //! a commit or a local build, signed in as a target account. Tests drive it through its kcp and its CLI. use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail, ensure}; use serde_json::Value; use sha2::{Digest, Sha256}; use crate::kcp::Kcp; use crate::target::{Account, Target, expand}; use crate::{Ctx, eventually, home_tmp, repo}; const BUCKET: &str = "https://socialweb-computer-kcp.sfo3.digitaloceanspaces.com"; #[derive(Clone, Debug)] pub struct Binary { pub path: PathBuf, /// `socialweb-computer version` (commit, dirty state) pub version: String, } fn arch() -> &'static str { if cfg!(target_arch = "aarch64") { "linux-arm64" } else { "linux-amd64" } } /// The release account (socialweb.computer) and project key of the release records: `software.release.channel/:stable` /// names a `software.release.version/:`, which names a `software.release.manifest` whose artifacts carry /// each binary's url and its raw CIDv1 (sha2-256), the same records the bidders follow (`internal/releasefollow`). const RELEASE_DID: &str = "did:plc:gmlkrhimoueejcertevr4hl3"; const RELEASE_KEY: &str = "socialweb-computer"; const B32: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567"; /// The sha256 a raw CIDv1 (base32, sha2-256) names, as release-workload's `sha_of`. fn sha_of(cid: &str) -> Option { let (mut out, mut buf, mut bits) = (vec![], 0u32, 0u32); for c in cid.strip_prefix('b')?.bytes() { buf = (buf << 5) | B32.iter().position(|x| *x == c)? as u32; bits += 5; if bits >= 8 { bits -= 8; out.push(((buf >> bits) & 0xff) as u8); } } (out.len() == 36 && out[..4] == [0x01, 0x55, 0x12, 0x20]) .then(|| out[4..].iter().map(|b| format!("{b:02x}")).collect()) } /// The release account's public records, read without a credential (plc.directory, then its PDS's getRecord). struct Records { http: reqwest::Client, pds: String, } impl Records { async fn new() -> Result { let http = reqwest::Client::new(); let doc: Value = http .get(format!("https://plc.directory/{RELEASE_DID}")) .send() .await? .error_for_status()? .json() .await?; let pds = doc["service"] .as_array() .into_iter() .flatten() .find(|s| s["id"] == "#atproto_pds") .and_then(|s| s["serviceEndpoint"].as_str()) .with_context(|| format!("{RELEASE_DID}: no #atproto_pds in its DID document"))? .trim_end_matches('/') .to_string(); Ok(Self { http, pds }) } /// None when the record is not there. async fn get(&self, collection: &str, rkey: &str) -> Result> { let url = format!( "{}/xrpc/com.atproto.repo.getRecord?repo={RELEASE_DID}&collection={collection}&rkey={rkey}", self.pds ); let r = self.http.get(&url).send().await?; if r.status() == reqwest::StatusCode::BAD_REQUEST { let body: Value = r.json().await.unwrap_or_default(); if body["error"] == "RecordNotFound" { return Ok(None); } bail!("{url}: {body}"); } Ok(Some(r.error_for_status()?.json().await?)) } /// The version a channel points at (`stable`, `rc`); None when the channel has no record. async fn channel(&self, name: &str) -> Result> { let Some(c) = self.get("software.release.channel", &format!("{RELEASE_KEY}:{name}")).await? else { return Ok(None); }; let uri = c["value"]["version"]["uri"].as_str().with_context(|| format!("channel {name}: no version.uri: {c}"))?; let rkey = uri.rsplit('/').next().unwrap_or_default(); Ok(Some(rkey.strip_prefix(&format!("{RELEASE_KEY}:")).unwrap_or(rkey).to_string())) } /// The url and sha256 of `file` in that version's manifest. async fn artifact(&self, version: &str, file: &str) -> Result<(String, String)> { let v = self .get("software.release.version", &format!("{RELEASE_KEY}:{version}")) .await? .with_context(|| format!("{RELEASE_DID}: no software.release.version {RELEASE_KEY}:{version}"))?; let uri = v["value"]["manifest"]["uri"].as_str().with_context(|| format!("version {version}: no manifest.uri: {v}"))?; let m = self .get("software.release.manifest", uri.rsplit('/').next().unwrap_or_default()) .await? .with_context(|| format!("version {version}: its manifest {uri} is not there"))?; let a = m["value"]["artifacts"] .as_array() .into_iter() .flatten() .find(|a| a["name"] == file) .with_context(|| format!("version {version}: its manifest {uri} lists no {file}"))?; let url = a["url"].as_str().with_context(|| format!("{uri} {file}: no url"))?; let cid = a["cid"].as_str().unwrap_or_default(); let sha = sha_of(cid).with_context(|| format!("{uri} {file}: cid {cid:?} is not a raw sha2-256 CIDv1"))?; Ok((url.to_string(), sha)) } } /// `release`: the version channel stable points at; `release:`: that version (an rc or a stable). async fn release(version: Option<&str>, file: &str) -> Result<(String, String, String)> { let r = Records::new().await?; let v = match version { Some(v) => v.to_string(), None => match r.channel("stable").await? { Some(v) => v, None => bail!( "requester \"release\": {RELEASE_DID} has no software.release.channel {RELEASE_KEY}:stable yet (no promote has gone green; channel rc: {}): name a version, release:, or sha: / local:", r.channel("rc").await?.unwrap_or_else(|| "none".into()) ), }, }; let (url, sha) = r.artifact(&v, file).await?; Ok((v, url, sha)) } /// `url` into `path` once, checked against `want` (sha256). async fn fetch(path: &Path, url: &str, want: &str) -> Result<()> { if path.exists() { return Ok(()); } let bytes = reqwest::Client::new() .get(url) .send() .await? .error_for_status()? .bytes() .await?; let got = format!("{:x}", Sha256::digest(&bytes)); ensure!(got == want, "{url}: sha256 {got}, want {want}"); std::fs::create_dir_all(path.parent().unwrap())?; let tmp = path.with_extension("tmp"); std::fs::write(&tmp, &bytes)?; std::fs::set_permissions(&tmp, std::os::unix::fs::PermissionsExt::from_mode(0o755))?; std::fs::rename(&tmp, &path)?; Ok(()) } /// `//binaries/` and its sha256 from the SHA256SUMS beside it. async fn bucket_artifact(prefix: &str, file: &str) -> Result<(String, String)> { let base = format!("{BUCKET}/{prefix}/binaries"); let sums = reqwest::Client::new() .get(format!("{base}/SHA256SUMS")) .send() .await? .error_for_status()? .text() .await?; let want = sums .lines() .find_map(|l| l.strip_suffix(&format!(" {file}"))) .with_context(|| format!("{base}/SHA256SUMS lists no {file}"))?; Ok((format!("{base}/{file}"), want.to_string())) } impl Binary { /// "release" (the version channel stable points at) | "release:" (that version's record) | "sha:" | "local:" (a hand build, local/) | "oci:@sha256:" (a /// CI build pushed to ttl.sh, pulled with oras, which checks it against the digest) | "path:"; downloads are checked against SHA256SUMS /// and cached under ~/.tmp/swc-system-tests/bin. /// `@sha256:`: pulled with oras (the ORAS env, else `oras` on PATH) into ~/.tmp/swc-system-tests/bin/oci- async fn oci(reference: &str) -> Result { let (_, digest) = reference .split_once("@sha256:") .filter(|(_, d)| d.len() == 64 && d.bytes().all(|b| b.is_ascii_hexdigit())) .with_context(|| format!("oci:{reference}: want @sha256:<64 hex>"))?; let dir = home_tmp("bin").join(format!("oci-{digest}")); let path = dir.join(format!("socialweb-computer-{}", arch())); if !path.exists() { std::fs::create_dir_all(&dir)?; let oras = std::env::var("ORAS").unwrap_or_else(|_| "oras".into()); let out = tokio::process::Command::new(&oras) .args(["pull", "--no-tty", reference, "-o"]) .arg(&dir) .output() .await .with_context(|| format!("run {oras}"))?; ensure!( out.status.success(), "oras pull {reference}: {}", String::from_utf8_lossy(&out.stderr) ); // a file pushed in parts (scripts/oci-push.sh): .part-0000, .part-0001, ... put back together in order let name = path .file_name() .and_then(|n| n.to_str()) .unwrap_or_default() .to_string(); let mut parts: Vec<_> = std::fs::read_dir(&dir)? .flatten() .map(|e| e.path()) .filter(|p| { p.file_name() .and_then(|n| n.to_str()) .is_some_and(|n| n.starts_with(&format!("{name}.part-"))) }) .collect(); parts.sort(); if !parts.is_empty() { let mut whole = std::fs::File::create(&path)?; for part in &parts { std::io::copy(&mut std::fs::File::open(part)?, &mut whole)?; } for part in &parts { std::fs::remove_file(part)?; } } std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o755))?; } Self::at(path) } pub async fn resolve(spec: &str) -> Result { let file = format!("socialweb-computer-{}", arch()); let prefix = match spec.split_once(':') { _ if spec == "release" || spec.starts_with("release:") => { let (_, url, sha) = release(spec.strip_prefix("release:"), &file).await?; let path = home_tmp("bin").join(format!("release-{sha}")).join(&file); fetch(&path, &url, &sha).await?; return Self::at(path); } Some(("sha", sha)) => format!("main/{sha}"), Some(("local", sha)) => format!("local/{sha}"), Some(("path", p)) => return Self::at(expand(p)), Some(("oci", r)) => return Self::oci(r).await, _ => bail!( "requester {spec:?}: want release, release:, sha:, local:, oci:@sha256: or path:" ), }; let path = home_tmp("bin").join(prefix.replace('/', "-")).join(&file); if !path.exists() { let (url, want) = bucket_artifact(&prefix, &file).await?; fetch(&path, &url, &want).await?; } Self::at(path) } /// The public URL and sha256 of a spec's build for this host's arch, read from its records or SHA256SUMS without /// downloading it: release, release:, sha:, local:; None for path: and oci: (no public URL). pub async fn published(spec: &str) -> Result> { let file = format!("socialweb-computer-{}", arch()); let prefix = match spec.split_once(':') { _ if spec == "release" || spec.starts_with("release:") => { let (_, url, sha) = release(spec.strip_prefix("release:"), &file).await?; return Ok(Some((url, sha))); } Some(("sha", sha)) => format!("main/{sha}"), Some(("local", sha)) => format!("local/{sha}"), _ => return Ok(None), }; Ok(Some(bucket_artifact(&prefix, &file).await?)) } fn at(path: PathBuf) -> Result { let out = std::process::Command::new(&path) .arg("version") .output() .with_context(|| path.display().to_string())?; ensure!(out.status.success(), "{} version failed", path.display()); let v = String::from_utf8_lossy(&out.stdout) .lines() .next() .unwrap_or_default() .trim_start_matches("socialweb-computer ") .to_string(); Ok(Self { path, version: v }) } pub fn cmd(&self) -> tokio::process::Command { let mut c = tokio::process::Command::new(&self.path); c.stdin(Stdio::null()); c } /// Run to completion; (exit ok, stdout + stderr). The output never holds a secret of ours (no secret goes in). pub async fn run(&self, args: &[&str]) -> Result<(bool, String)> { let out = self.cmd().args(args).output().await?; let mut text = String::from_utf8_lossy(&out.stdout).to_string(); text.push_str(&String::from_utf8_lossy(&out.stderr)); Ok((out.status.success(), text)) } /// Whether this CLI takes every option before `--exec -- [args...]` (plan 0043). A CLI before it (the /// previous rc of 0006:compat) takes the requester options after `--` and `--exec `. pub async fn exec_argv(&self) -> bool { match self.run(&["run", "request", "--help"]).await { Ok((_, out)) => out.contains(EXEC_USAGE), Err(_) => true, } } } const EXEC_USAGE: &str = "--exec -- [args...]"; /// `--exec -- bash -c `: a shell line as the guest's root login shell (bash) ran a `--exec ` before plan 0043. pub fn exec_line(line: &str) -> Vec { ["--exec", "--", "bash", "-c", line].map(String::from).to_vec() } /// The argv as one line for the guest's shell, each word single-quoted (what the CLI sends since plan 0043). pub fn shell_words(argv: &[String]) -> String { argv.iter() .map(|w| format!("'{}'", w.replace('\'', r"'\''"))) .collect::>() .join(" ") } /// `args` cut at `--exec -- [args...]`: the options before it and the command, if any. pub fn split_exec(args: &[String]) -> Result<(Vec, Option>)> { match args.iter().position(|a| a == "--exec") { None => { ensure!(!args.iter().any(|a| a == "--"), "args: -- only after --exec ({EXEC_USAGE})"); Ok((args.to_vec(), None)) } Some(i) => { ensure!( args.get(i + 1).map(String::as_str) == Some("--") && args.len() > i + 2, "args: --exec takes the command after --: {EXEC_USAGE}" ); Ok((args[..i].to_vec(), Some(args[i + 2..].to_vec()))) } } } /// A `run request` or `run requester` command line from its parts: `opts`, the requester (application) options /// `app`, then `--exec -- `; for a CLI before plan 0043 (`exec_argv` false) the command as one `--exec` /// line among the options and `app` after `--`. pub fn request_line(exec_argv: bool, opts: &[String], app: &[String], command: Option<&[String]>) -> Vec { let mut out = opts.to_vec(); if exec_argv { out.extend(app.iter().cloned()); if let Some(c) = command { out.extend(["--exec".to_string(), "--".to_string()]); out.extend(c.iter().cloned()); } return out; } if let Some(c) = command { out.extend(["--exec".to_string(), shell_words(c)]); } if !app.is_empty() { out.push("--".into()); out.extend(app.iter().cloned()); } out } /// How the requester signs in as its account. #[derive(Clone, Copy, Debug, PartialEq)] pub enum SignIn { /// pdsAccount: the app password in ATPROTO_PASSWORD of the one command AppPassword, /// OAuth QR, driven by scripts/oauth-signin.ts (headless Chrome) from the creds file OAuthQr, } impl SignIn { /// SWC_SIGNIN=oauth-qr: OAuth QR (headless Chrome); default: the app password. pub fn from_env() -> Self { if std::env::var("SWC_SIGNIN").is_ok_and(|v| v == "oauth-qr") { SignIn::OAuthQr } else { SignIn::AppPassword } } } pub fn identity_args(t: &Target, a: &Account, mode: SignIn) -> Vec { match mode { SignIn::AppPassword => vec![ "--atproto-identity-mode".into(), "pdsAccount".into(), "--atproto-handle".into(), a.handle.clone(), "--atproto-pds-url".into(), t.pds.clone(), ], SignIn::OAuthQr => vec!["--atproto-oauth-qr".into()], } } /// Sign a QR sign-in URL in with the creds file (the URL never reaches a log of ours). pub async fn oauth_signin(a: &Account, url: &str) -> Result<()> { oauth_signin_with(a, url, &[]).await } /// `oauth_signin` with more oauth-signin.ts options (`--start-api`, `--cookie-file `). pub async fn oauth_signin_with(a: &Account, url: &str, extra: &[&str]) -> Result<()> { oauth_signin_on(Path::new("deno"), None, a, url, extra).await } /// `oauth_signin_with` run by the deno at `deno`, whose own fetches (`--start-api`) also trust the CA bundle `ca`. pub async fn oauth_signin_on(deno: &Path, ca: Option<&Path>, a: &Account, url: &str, extra: &[&str]) -> Result<()> { let prefix = a .oauth_account .as_deref() .context("the account has no oauthAccount (creds prefix) for OAuth")?; let mut cmd = tokio::process::Command::new(deno); if let Some(ca) = ca { cmd.env("DENO_CERT", ca); } let out = cmd .current_dir(repo()) .args([ "run", "-A", "--no-lock", "scripts/oauth-signin.ts", "--creds", ]) .arg(expand(&a.creds)) .args(["--account", prefix, "--url", url]) .args(extra) .stdout(Stdio::null()) .stdin(Stdio::null()) .output() .await?; ensure!( out.status.success(), "oauth-signin.ts exited {}: {}", out.status, tail(&without_url(&String::from_utf8_lossy(&out.stderr), url)) ); Ok(()) } /// `text` without the sign-in URL: whole, and its query and fragment (the transfer's secret) wherever they appear alone. fn without_url(text: &str, url: &str) -> String { let mut out = text.replace(url, ""); for part in url.split(['?', '#']).skip(1).filter(|p| !p.is_empty()) { out = out.replace(part, ""); } out } /// The JSON lines of a log whose msg is `msg`. pub fn log_lines(path: &Path, msg: &str) -> Vec { std::fs::read_to_string(path) .unwrap_or_default() .lines() .filter(|l| l.contains(msg)) .filter_map(|l| serde_json::from_str::(l).ok()) .filter(|v| v["msg"] == msg) .collect() } pub async fn wait_log(paths: &[PathBuf], msg: &str, timeout: Duration) -> Result { eventually( &format!("log line {msg}"), timeout, Duration::from_secs(1), || async { Ok(paths .iter() .find_map(|p| log_lines(p, msg).into_iter().next())) }, ) .await } /// Every at:///... URI the logs name (the records a run wrote in that repo). pub fn uris_in(paths: &[PathBuf], did: &str) -> Vec { let needle = format!("at://{did}/"); let mut out = vec![]; for p in paths { let text = std::fs::read_to_string(p).unwrap_or_default(); for (i, _) in text.match_indices(&needle) { let uri: String = text[i..] .chars() .take_while(|c| !matches!(c, '"' | ' ' | '\\' | ',' | ')' | '\n')) .collect(); if uri.split('/').count() == 5 && !out.contains(&uri) { out.push(uri); } } } out } /// Records that stay by rule after a release: vouches (Retain), the signer key, the operator associations of bidders. pub fn kept_by_rule(uri: &str) -> bool { ["/sh.tangled.graph.vouch/", "badgeBlueKeys", "/app.bsky."] .iter() .any(|k| uri.contains(k)) } /// The requester's log message of an accept a bidder refused (409 CapacityExceeded, no receipt). pub const ACCEPT_DECLINED: &str = "requester_submit_accept_declined"; /// The accepts a bidder refused, by a requester's log lines: the requester keeps them by design (docs/acceptance/0014: /// "the requester keeps that accept (a bidder may still act on it)"), so they are no leftover of a release. The one /// rule of every record check (request.records, requester.records, gateway.log for assert.noneSince). pub fn declined_accepts<'a>(lines: impl IntoIterator) -> Vec { let mut out: Vec = lines .into_iter() .filter(|l| l["msg"] == ACCEPT_DECLINED) .filter_map(|l| l["accept"].as_str().map(String::from)) .collect(); out.sort(); out.dedup(); out } /// A long-lived `run requester` (the orchestrator): its kcp, its ready file, its CLI helpers. pub struct Requester { pub bin: Binary, pub name: String, pub state: PathBuf, pub log: PathBuf, args: Vec, password: Option, child: tokio::sync::Mutex>, } impl Requester { /// Start `run requester --name ` in a new owner-only state dir under the run's scratch dir, sign in, /// wait for the ready file. Stopped (Ctrl-C) by a cleanup. pub async fn start( ctx: &std::sync::Arc, role: &str, name: &str, mode: SignIn, extra: &[&str], ) -> Result> { let a = ctx.role(role)?; let bin = Binary::resolve(&ctx.target.requester).await?; let state = ctx.dir.join(format!("requester-{name}")); std::fs::create_dir_all(&state)?; std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))?; let mut args: Vec = ["run", "requester", "--name", name, "--state-dir"] .iter() .map(|s| s.to_string()) .collect(); args.push(state.display().to_string()); args.extend(extra.iter().map(|s| s.to_string())); args.extend(identity_args(&ctx.target, a, mode)); let r = std::sync::Arc::new(Self { bin, name: name.into(), log: state.join(format!("{name}.log")), state, args, password: (mode == SignIn::AppPassword) .then(|| a.password().map(|p| p.expose().to_string())) .transpose()?, child: tokio::sync::Mutex::new(None), }); let stop = r.clone(); ctx.defer(&format!("stop requester {name}"), async move { stop.stop().await }); r.spawn().await?; if mode == SignIn::OAuthQr { let url = eventually( "the sign-in URL", Duration::from_secs(300), Duration::from_secs(3), || async { let (ok, out) = r .bin .run(&[ "run", "requester", "signin", &r.name, "--state-dir", &r.state.display().to_string(), ]) .await?; Ok(ok .then(|| { out.lines() .find(|l| l.starts_with("https://")) .map(str::to_string) }) .flatten()) }, ) .await?; oauth_signin(a, &url).await?; } r.wait_ready(Duration::from_secs(600)).await?; Ok(r) } async fn spawn(&self) -> Result<()> { let log = std::fs::OpenOptions::new() .create(true) .append(true) .open(&self.log)?; let mut c = self.bin.cmd(); c.args(&self.args).stdout(log.try_clone()?).stderr(log); if let Some(p) = &self.password { c.env("ATPROTO_PASSWORD", p); } *self.child.lock().await = Some(c.spawn()?); Ok(()) } pub fn ready_path(&self) -> PathBuf { self.state .join("requesters") .join(&self.name) .join("ready.json") } pub fn ready(&self) -> Result { Ok(serde_json::from_slice(&std::fs::read(self.ready_path())?)?) } pub async fn wait_ready(&self, timeout: Duration) -> Result { eventually( "the requester's ready file", timeout, Duration::from_secs(1), || async { if let Some(c) = self.child.lock().await.as_mut() && let Some(st) = c.try_wait()? { bail!( "run requester {} exited {st} before ready; see {}", self.name, self.log.display() ); } Ok(self.ready().ok()) }, ) .await } /// The requester workspace's kcp (kube-rs over the ready file's kubeconfig). pub async fn kcp(&self) -> Result { Kcp::kubeconfig( self.ready()?["kubeconfig"] .as_str() .context("ready file without kubeconfig")?, ) .await } pub fn did(&self) -> Result { Ok(self.ready()?["did"] .as_str() .context("ready file without did")? .into()) } /// The core logs of the requester (its own log and the core.log files under the state dir). pub fn logs(&self) -> Vec { let mut out = vec![self.log.clone()]; find_named(&self.state, "core.log", &mut out); out } /// Ctrl-C: the kcp stops, the ready file goes, the state stays. pub async fn stop(&self) -> Result<()> { let mut g = self.child.lock().await; if let Some(c) = g.as_mut() && c.try_wait()?.is_none() { if let Some(pid) = c.id() { std::process::Command::new("kill") .args(["-INT", &pid.to_string()]) .status()?; } if tokio::time::timeout(Duration::from_secs(120), c.wait()) .await .is_err() { c.kill().await?; bail!( "run requester {} did not stop in 120 s after SIGINT", self.name ); } } *g = None; Ok(()) } /// Ctrl-C and the same command again (0013 A9). pub async fn restart(&self) -> Result { self.stop().await?; ensure!( !self.ready_path().exists(), "the ready file stayed after Ctrl-C" ); self.spawn().await?; self.wait_ready(Duration::from_secs(300)).await } /// The WebRTC transport to a port a Held contract's guest tunnels as `tag` (a kcp in the guest): this requester's /// `run requester proxy` dials the datachannel (signalling through its own kcp) once per connection. pub fn webrtc_proxy(&self, contract: &str, tag: &str) -> crate::target::Via { let mut proxy = vec![self.bin.path.display().to_string()]; proxy.extend(["run", "requester", "proxy", &self.name, "--state-dir"].map(String::from)); proxy.push(self.state.display().to_string()); proxy.extend(["--contract", contract, "--tag", tag].map(String::from)); crate::target::Via::WebRtc { proxy } } /// `run requester ssh --contract [--transport webrtc] --exec -- bash -c `. pub async fn ssh(&self, contract: &str, transport: Option<&str>, exec: &str) -> Result { let state = self.state.display().to_string(); let mut args = vec!["run", "requester", "ssh", &self.name, "--state-dir", &state, "--contract", contract]; if let Some(t) = transport { args.extend(["--transport", t]); } let line = exec_line(exec); args.extend(line.iter().map(String::as_str)); let (ok, out) = self.bin.run(&args).await?; ensure!( ok, "run requester ssh {contract} ({transport:?}) failed: {}", tail(&out) ); Ok(out) } } fn find_named(dir: &Path, name: &str, out: &mut Vec) { for e in std::fs::read_dir(dir).into_iter().flatten().flatten() { let p = e.path(); if p.is_dir() { find_named(&p, name, out); } else if p.file_name().is_some_and(|n| n == name) { out.push(p); } } } pub fn tail(s: &str) -> String { let lines: Vec<_> = s.lines().collect(); lines[lines.len().saturating_sub(15)..].join("\n") } /// One `run request` invocation (its own kcp; the CLI path of the runbooks). pub struct RunRequest { pub bin: Binary, pub name: String, pub state: PathBuf, pub log: PathBuf, child: tokio::sync::Mutex>, } impl RunRequest { /// `run request --state-dir --name [--exec -- ]` in the background (app /// password); an older CLI gets its own layout (`request_line`). /// A cleanup releases it. pub async fn start( ctx: &std::sync::Arc, role: &str, name: &str, args: &[&str], ) -> Result> { let bin = Binary::resolve(&ctx.target.requester).await?; Self::start_with(ctx, bin, role, name, args).await } /// `start` with another binary (0006 compat: the previous release's CLI against new servers). pub async fn start_with( ctx: &std::sync::Arc, bin: Binary, role: &str, name: &str, args: &[&str], ) -> Result> { let a = ctx.role(role)?; let state = ctx.dir.join("requests"); std::fs::create_dir_all(&state)?; std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))?; let log = state.join(format!("{name}-request.log")); let f = std::fs::File::create(&log)?; let (opts, command) = split_exec(&args.iter().map(|s| s.to_string()).collect::>())?; let line = request_line( bin.exec_argv().await, &opts, &identity_args(&ctx.target, a, SignIn::AppPassword), command.as_deref(), ); let mut c = bin.cmd(); c.args(["run", "request", "--state-dir"]) .arg(&state) .args(["--name", name]) .args(&line); c.env("ATPROTO_PASSWORD", a.password()?.expose()); c.stdout(f.try_clone()?).stderr(f); let r = std::sync::Arc::new(Self { bin, name: name.into(), state, log, child: tokio::sync::Mutex::new(Some(c.spawn()?)), }); let rel = r.clone(); ctx.defer(&format!("release {name}"), async move { rel.release().await.map(|_| ()) }); Ok(r) } /// The CLI log plus the invocation's core.log (while it runs, and after release). pub fn logs(&self) -> Vec { let base = self.state.join("requests").join(&self.name); vec![ self.log.clone(), base.join("dev").join(&self.name).join("logs/core.log"), base.join("logs/core.log"), ] } /// The contract phase lines of the CLI log; waits for `phase`. pub async fn wait_phase(&self, phase: &str, timeout: Duration) -> Result { eventually( &format!("{} {phase}", self.name), timeout, Duration::from_secs(1), || async { if let Some(c) = self.child.lock().await.as_mut() && let Some(st) = c.try_wait()? && !st.success() { bail!( "run request {} exited {st}: {}", self.name, tail(&std::fs::read_to_string(&self.log).unwrap_or_default()) ); } Ok(log_lines(&self.log, "contract") .into_iter() .find(|l| l["phase"] == phase)) }, ) .await } pub async fn wait_log(&self, msg: &str, timeout: Duration) -> Result { wait_log(&self.logs(), msg, timeout).await } fn sub(&self, verb: &str, extra: &[&str]) -> Vec { let mut v: Vec = ["run", "request", verb, &self.name, "--state-dir"] .iter() .map(|s| s.to_string()) .collect(); v.push(self.state.display().to_string()); v.extend(extra.iter().map(|s| s.to_string())); v } /// `run request agent `: publishes guest.agentStart. pub async fn agent(&self) -> Result<()> { let args = self.sub("agent", &[]); let (ok, out) = self .bin .run(&args.iter().map(String::as_str).collect::>()) .await?; ensure!(ok, "run request agent: {}", tail(&out)); Ok(()) } /// `run request ssh [--transport webrtc] --exec -- bash -c ` (iroh first by default). pub async fn ssh(&self, transport: Option<&str>, exec: &str) -> Result { let mut extra = vec![]; if let Some(t) = transport { extra.extend(["--transport", t]); } let line = exec_line(exec); extra.extend(line.iter().map(String::as_str)); let args = self.sub("ssh", &extra); let (ok, out) = self .bin .run(&args.iter().map(String::as_str).collect::>()) .await?; ensure!(ok, "run request ssh: {}", tail(&out)); Ok(out) } /// Wait for the --hold process to exit (it exits after its first ssh unless held). pub async fn wait_exit(&self, timeout: Duration) -> Result<()> { if let Some(c) = self.child.lock().await.as_mut() { tokio::time::timeout(timeout, c.wait()) .await .map_err(|_| { anyhow!("run request {} still running after {timeout:?}", self.name) })??; } Ok(()) } /// `run request release`; idempotent. Returns the seconds it took. pub async fn release(&self) -> Result { let t = Instant::now(); let args = self.sub("release", &[]); let (ok, out) = self .bin .run(&args.iter().map(String::as_str).collect::>()) .await?; if let Some(mut c) = self.child.lock().await.take() && tokio::time::timeout(Duration::from_secs(300), c.wait()) .await .is_err() { c.kill().await?; } ensure!(ok, "run request release {}: {}", self.name, tail(&out)); Ok(t.elapsed().as_secs_f64()) } pub fn summary(&self) -> Result { Ok(serde_json::from_slice(&std::fs::read( self.state .join("requests") .join(&self.name) .join("summary.json"), )?)?) } }