0
0
//! The orchestrator: a real Go requester (`socialweb-computer run requester` / `run request`) of a release,
//! a commit or a local build, signed in as a target account. Tests drive it through its kcp and its CLI.
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::time::{Duration, Instant};
use anyhow::{Context, Result, anyhow, bail, ensure};
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::kcp::Kcp;
use crate::target::{Account, Target, expand};
use crate::{Ctx, eventually, home_tmp, repo};
const BUCKET: &str = "https://socialweb-computer-kcp.sfo3.digitaloceanspaces.com";
#[derive(Clone, Debug)]
pub struct Binary {
pub path: PathBuf,
/// `socialweb-computer version` (commit, dirty state)
pub version: String,
}
fn arch() -> &'static str {
if cfg!(target_arch = "aarch64") {
"linux-arm64"
} else {
"linux-amd64"
}
}
/// The release account (socialweb.computer) and project key of the release records: `software.release.channel/<key>:stable`
/// names a `software.release.version/<key>:<version>`, which names a `software.release.manifest` whose artifacts carry
/// each binary's url and its raw CIDv1 (sha2-256), the same records the bidders follow (`internal/releasefollow`).
const RELEASE_DID: &str = "did:plc:gmlkrhimoueejcertevr4hl3";
const RELEASE_KEY: &str = "socialweb-computer";
const B32: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";
/// The sha256 a raw CIDv1 (base32, sha2-256) names, as release-workload's `sha_of`.
fn sha_of(cid: &str) -> Option<String> {
let (mut out, mut buf, mut bits) = (vec![], 0u32, 0u32);
for c in cid.strip_prefix('b')?.bytes() {
buf = (buf << 5) | B32.iter().position(|x| *x == c)? as u32;
bits += 5;
if bits >= 8 {
bits -= 8;
out.push(((buf >> bits) & 0xff) as u8);
}
}
(out.len() == 36 && out[..4] == [0x01, 0x55, 0x12, 0x20])
.then(|| out[4..].iter().map(|b| format!("{b:02x}")).collect())
}
/// The release account's public records, read without a credential (plc.directory, then its PDS's getRecord).
struct Records {
http: reqwest::Client,
pds: String,
}
impl Records {
async fn new() -> Result<Self> {
let http = reqwest::Client::new();
let doc: Value = http
.get(format!("https://plc.directory/{RELEASE_DID}"))
.send()
.await?
.error_for_status()?
.json()
.await?;
let pds = doc["service"]
.as_array()
.into_iter()
.flatten()
.find(|s| s["id"] == "#atproto_pds")
.and_then(|s| s["serviceEndpoint"].as_str())
.with_context(|| format!("{RELEASE_DID}: no #atproto_pds in its DID document"))?
.trim_end_matches('/')
.to_string();
Ok(Self { http, pds })
}
/// None when the record is not there.
async fn get(&self, collection: &str, rkey: &str) -> Result<Option<Value>> {
let url = format!(
"{}/xrpc/com.atproto.repo.getRecord?repo={RELEASE_DID}&collection={collection}&rkey={rkey}",
self.pds
);
let r = self.http.get(&url).send().await?;
if r.status() == reqwest::StatusCode::BAD_REQUEST {
let body: Value = r.json().await.unwrap_or_default();
if body["error"] == "RecordNotFound" {
return Ok(None);
}
bail!("{url}: {body}");
}
Ok(Some(r.error_for_status()?.json().await?))
}
/// The version a channel points at (`stable`, `rc`); None when the channel has no record.
async fn channel(&self, name: &str) -> Result<Option<String>> {
let Some(c) = self.get("software.release.channel", &format!("{RELEASE_KEY}:{name}")).await? else {
return Ok(None);
};
let uri = c["value"]["version"]["uri"].as_str().with_context(|| format!("channel {name}: no version.uri: {c}"))?;
let rkey = uri.rsplit('/').next().unwrap_or_default();
Ok(Some(rkey.strip_prefix(&format!("{RELEASE_KEY}:")).unwrap_or(rkey).to_string()))
}
/// The url and sha256 of `file` in that version's manifest.
async fn artifact(&self, version: &str, file: &str) -> Result<(String, String)> {
let v = self
.get("software.release.version", &format!("{RELEASE_KEY}:{version}"))
.await?
.with_context(|| format!("{RELEASE_DID}: no software.release.version {RELEASE_KEY}:{version}"))?;
let uri = v["value"]["manifest"]["uri"].as_str().with_context(|| format!("version {version}: no manifest.uri: {v}"))?;
let m = self
.get("software.release.manifest", uri.rsplit('/').next().unwrap_or_default())
.await?
.with_context(|| format!("version {version}: its manifest {uri} is not there"))?;
let a = m["value"]["artifacts"]
.as_array()
.into_iter()
.flatten()
.find(|a| a["name"] == file)
.with_context(|| format!("version {version}: its manifest {uri} lists no {file}"))?;
let url = a["url"].as_str().with_context(|| format!("{uri} {file}: no url"))?;
let cid = a["cid"].as_str().unwrap_or_default();
let sha = sha_of(cid).with_context(|| format!("{uri} {file}: cid {cid:?} is not a raw sha2-256 CIDv1"))?;
Ok((url.to_string(), sha))
}
}
/// `release`: the version channel stable points at; `release:<version>`: that version (an rc or a stable).
async fn release(version: Option<&str>, file: &str) -> Result<(String, String, String)> {
let r = Records::new().await?;
let v = match version {
Some(v) => v.to_string(),
None => match r.channel("stable").await? {
Some(v) => v,
None => bail!(
"requester \"release\": {RELEASE_DID} has no software.release.channel {RELEASE_KEY}:stable yet (no promote has gone green; channel rc: {}): name a version, release:<version>, or sha:<commit> / local:<commit>",
r.channel("rc").await?.unwrap_or_else(|| "none".into())
),
},
};
let (url, sha) = r.artifact(&v, file).await?;
Ok((v, url, sha))
}
/// `url` into `path` once, checked against `want` (sha256).
async fn fetch(path: &Path, url: &str, want: &str) -> Result<()> {
if path.exists() {
return Ok(());
}
let bytes = reqwest::Client::new()
.get(url)
.send()
.await?
.error_for_status()?
.bytes()
.await?;
let got = format!("{:x}", Sha256::digest(&bytes));
ensure!(got == want, "{url}: sha256 {got}, want {want}");
std::fs::create_dir_all(path.parent().unwrap())?;
let tmp = path.with_extension("tmp");
std::fs::write(&tmp, &bytes)?;
std::fs::set_permissions(&tmp, std::os::unix::fs::PermissionsExt::from_mode(0o755))?;
std::fs::rename(&tmp, &path)?;
Ok(())
}
/// `<BUCKET>/<prefix>/binaries/<file>` and its sha256 from the SHA256SUMS beside it.
async fn bucket_artifact(prefix: &str, file: &str) -> Result<(String, String)> {
let base = format!("{BUCKET}/{prefix}/binaries");
let sums = reqwest::Client::new()
.get(format!("{base}/SHA256SUMS"))
.send()
.await?
.error_for_status()?
.text()
.await?;
let want = sums
.lines()
.find_map(|l| l.strip_suffix(&format!(" {file}")))
.with_context(|| format!("{base}/SHA256SUMS lists no {file}"))?;
Ok((format!("{base}/{file}"), want.to_string()))
}
impl Binary {
/// "release" (the version channel stable points at) | "release:<version>" (that version's record) | "sha:<full commit>" | "local:<full commit>" (a hand build, local/<commit>) | "oci:<host/name>@sha256:<manifest digest>" (a
/// CI build pushed to ttl.sh, pulled with oras, which checks it against the digest) | "path:<file>"; downloads are checked against SHA256SUMS
/// and cached under ~/.tmp/swc-system-tests/bin.
/// `<host/name>@sha256:<manifest digest>`: pulled with oras (the ORAS env, else `oras` on PATH) into ~/.tmp/swc-system-tests/bin/oci-<digest>
async fn oci(reference: &str) -> Result<Self> {
let (_, digest) = reference
.split_once("@sha256:")
.filter(|(_, d)| d.len() == 64 && d.bytes().all(|b| b.is_ascii_hexdigit()))
.with_context(|| format!("oci:{reference}: want <host/name>@sha256:<64 hex>"))?;
let dir = home_tmp("bin").join(format!("oci-{digest}"));
let path = dir.join(format!("socialweb-computer-{}", arch()));
if !path.exists() {
std::fs::create_dir_all(&dir)?;
let oras = std::env::var("ORAS").unwrap_or_else(|_| "oras".into());
let out = tokio::process::Command::new(&oras)
.args(["pull", "--no-tty", reference, "-o"])
.arg(&dir)
.output()
.await
.with_context(|| format!("run {oras}"))?;
ensure!(
out.status.success(),
"oras pull {reference}: {}",
String::from_utf8_lossy(&out.stderr)
);
// a file pushed in parts (scripts/oci-push.sh): <file>.part-0000, .part-0001, ... put back together in order
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or_default()
.to_string();
let mut parts: Vec<_> = std::fs::read_dir(&dir)?
.flatten()
.map(|e| e.path())
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with(&format!("{name}.part-")))
})
.collect();
parts.sort();
if !parts.is_empty() {
let mut whole = std::fs::File::create(&path)?;
for part in &parts {
std::io::copy(&mut std::fs::File::open(part)?, &mut whole)?;
}
for part in &parts {
std::fs::remove_file(part)?;
}
}
std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o755))?;
}
Self::at(path)
}
pub async fn resolve(spec: &str) -> Result<Self> {
let file = format!("socialweb-computer-{}", arch());
let prefix = match spec.split_once(':') {
_ if spec == "release" || spec.starts_with("release:") => {
let (_, url, sha) = release(spec.strip_prefix("release:"), &file).await?;
let path = home_tmp("bin").join(format!("release-{sha}")).join(&file);
fetch(&path, &url, &sha).await?;
return Self::at(path);
}
Some(("sha", sha)) => format!("main/{sha}"),
Some(("local", sha)) => format!("local/{sha}"),
Some(("path", p)) => return Self::at(expand(p)),
Some(("oci", r)) => return Self::oci(r).await,
_ => bail!(
"requester {spec:?}: want release, release:<version>, sha:<commit>, local:<commit>, oci:<ref>@sha256:<digest> or path:<file>"
),
};
let path = home_tmp("bin").join(prefix.replace('/', "-")).join(&file);
if !path.exists() {
let (url, want) = bucket_artifact(&prefix, &file).await?;
fetch(&path, &url, &want).await?;
}
Self::at(path)
}
/// The public URL and sha256 of a spec's build for this host's arch, read from its records or SHA256SUMS without
/// downloading it: release, release:<version>, sha:<commit>, local:<commit>; None for path: and oci: (no public URL).
pub async fn published(spec: &str) -> Result<Option<(String, String)>> {
let file = format!("socialweb-computer-{}", arch());
let prefix = match spec.split_once(':') {
_ if spec == "release" || spec.starts_with("release:") => {
let (_, url, sha) = release(spec.strip_prefix("release:"), &file).await?;
return Ok(Some((url, sha)));
}
Some(("sha", sha)) => format!("main/{sha}"),
Some(("local", sha)) => format!("local/{sha}"),
_ => return Ok(None),
};
Ok(Some(bucket_artifact(&prefix, &file).await?))
}
fn at(path: PathBuf) -> Result<Self> {
let out = std::process::Command::new(&path)
.arg("version")
.output()
.with_context(|| path.display().to_string())?;
ensure!(out.status.success(), "{} version failed", path.display());
let v = String::from_utf8_lossy(&out.stdout)
.lines()
.next()
.unwrap_or_default()
.trim_start_matches("socialweb-computer ")
.to_string();
Ok(Self { path, version: v })
}
pub fn cmd(&self) -> tokio::process::Command {
let mut c = tokio::process::Command::new(&self.path);
c.stdin(Stdio::null());
c
}
/// Run to completion; (exit ok, stdout + stderr). The output never holds a secret of ours (no secret goes in).
pub async fn run(&self, args: &[&str]) -> Result<(bool, String)> {
let out = self.cmd().args(args).output().await?;
let mut text = String::from_utf8_lossy(&out.stdout).to_string();
text.push_str(&String::from_utf8_lossy(&out.stderr));
Ok((out.status.success(), text))
}
/// Whether this CLI takes every option before `--exec -- <command> [args...]` (plan 0043). A CLI before it (the
/// previous rc of 0006:compat) takes the requester options after `--` and `--exec <one shell line>`.
pub async fn exec_argv(&self) -> bool {
match self.run(&["run", "request", "--help"]).await {
Ok((_, out)) => out.contains(EXEC_USAGE),
Err(_) => true,
}
}
}
const EXEC_USAGE: &str = "--exec -- <command> [args...]";
/// `--exec -- bash -c <line>`: a shell line as the guest's root login shell (bash) ran a `--exec <line>` before plan 0043.
pub fn exec_line(line: &str) -> Vec<String> {
["--exec", "--", "bash", "-c", line].map(String::from).to_vec()
}
/// The argv as one line for the guest's shell, each word single-quoted (what the CLI sends since plan 0043).
pub fn shell_words(argv: &[String]) -> String {
argv.iter()
.map(|w| format!("'{}'", w.replace('\'', r"'\''")))
.collect::<Vec<_>>()
.join(" ")
}
/// `args` cut at `--exec -- <command> [args...]`: the options before it and the command, if any.
pub fn split_exec(args: &[String]) -> Result<(Vec<String>, Option<Vec<String>>)> {
match args.iter().position(|a| a == "--exec") {
None => {
ensure!(!args.iter().any(|a| a == "--"), "args: -- only after --exec ({EXEC_USAGE})");
Ok((args.to_vec(), None))
}
Some(i) => {
ensure!(
args.get(i + 1).map(String::as_str) == Some("--") && args.len() > i + 2,
"args: --exec takes the command after --: {EXEC_USAGE}"
);
Ok((args[..i].to_vec(), Some(args[i + 2..].to_vec())))
}
}
}
/// A `run request` or `run requester` command line from its parts: `opts`, the requester (application) options
/// `app`, then `--exec -- <command>`; for a CLI before plan 0043 (`exec_argv` false) the command as one `--exec`
/// line among the options and `app` after `--`.
pub fn request_line(exec_argv: bool, opts: &[String], app: &[String], command: Option<&[String]>) -> Vec<String> {
let mut out = opts.to_vec();
if exec_argv {
out.extend(app.iter().cloned());
if let Some(c) = command {
out.extend(["--exec".to_string(), "--".to_string()]);
out.extend(c.iter().cloned());
}
return out;
}
if let Some(c) = command {
out.extend(["--exec".to_string(), shell_words(c)]);
}
if !app.is_empty() {
out.push("--".into());
out.extend(app.iter().cloned());
}
out
}
/// How the requester signs in as its account.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum SignIn {
/// pdsAccount: the app password in ATPROTO_PASSWORD of the one command
AppPassword,
/// OAuth QR, driven by scripts/oauth-signin.ts (headless Chrome) from the creds file
OAuthQr,
}
impl SignIn {
/// SWC_SIGNIN=oauth-qr: OAuth QR (headless Chrome); default: the app password.
pub fn from_env() -> Self {
if std::env::var("SWC_SIGNIN").is_ok_and(|v| v == "oauth-qr") {
SignIn::OAuthQr
} else {
SignIn::AppPassword
}
}
}
pub fn identity_args(t: &Target, a: &Account, mode: SignIn) -> Vec<String> {
match mode {
SignIn::AppPassword => vec![
"--atproto-identity-mode".into(),
"pdsAccount".into(),
"--atproto-handle".into(),
a.handle.clone(),
"--atproto-pds-url".into(),
t.pds.clone(),
],
SignIn::OAuthQr => vec!["--atproto-oauth-qr".into()],
}
}
/// Sign a QR sign-in URL in with the creds file (the URL never reaches a log of ours).
pub async fn oauth_signin(a: &Account, url: &str) -> Result<()> {
oauth_signin_with(a, url, &[]).await
}
/// `oauth_signin` with more oauth-signin.ts options (`--start-api`, `--cookie-file <owner-only file>`).
pub async fn oauth_signin_with(a: &Account, url: &str, extra: &[&str]) -> Result<()> {
oauth_signin_on(Path::new("deno"), None, a, url, extra).await
}
/// `oauth_signin_with` run by the deno at `deno`, whose own fetches (`--start-api`) also trust the CA bundle `ca`.
pub async fn oauth_signin_on(deno: &Path, ca: Option<&Path>, a: &Account, url: &str, extra: &[&str]) -> Result<()> {
let prefix = a
.oauth_account
.as_deref()
.context("the account has no oauthAccount (creds prefix) for OAuth")?;
let mut cmd = tokio::process::Command::new(deno);
if let Some(ca) = ca {
cmd.env("DENO_CERT", ca);
}
let out = cmd
.current_dir(repo())
.args([
"run",
"-A",
"--no-lock",
"scripts/oauth-signin.ts",
"--creds",
])
.arg(expand(&a.creds))
.args(["--account", prefix, "--url", url])
.args(extra)
.stdout(Stdio::null())
.stdin(Stdio::null())
.output()
.await?;
ensure!(
out.status.success(),
"oauth-signin.ts exited {}: {}",
out.status,
tail(&without_url(&String::from_utf8_lossy(&out.stderr), url))
);
Ok(())
}
/// `text` without the sign-in URL: whole, and its query and fragment (the transfer's secret) wherever they appear alone.
fn without_url(text: &str, url: &str) -> String {
let mut out = text.replace(url, "<sign-in url>");
for part in url.split(['?', '#']).skip(1).filter(|p| !p.is_empty()) {
out = out.replace(part, "<redacted>");
}
out
}
/// The JSON lines of a log whose msg is `msg`.
pub fn log_lines(path: &Path, msg: &str) -> Vec<Value> {
std::fs::read_to_string(path)
.unwrap_or_default()
.lines()
.filter(|l| l.contains(msg))
.filter_map(|l| serde_json::from_str::<Value>(l).ok())
.filter(|v| v["msg"] == msg)
.collect()
}
pub async fn wait_log(paths: &[PathBuf], msg: &str, timeout: Duration) -> Result<Value> {
eventually(
&format!("log line {msg}"),
timeout,
Duration::from_secs(1),
|| async {
Ok(paths
.iter()
.find_map(|p| log_lines(p, msg).into_iter().next()))
},
)
.await
}
/// Every at://<did>/... URI the logs name (the records a run wrote in that repo).
pub fn uris_in(paths: &[PathBuf], did: &str) -> Vec<String> {
let needle = format!("at://{did}/");
let mut out = vec![];
for p in paths {
let text = std::fs::read_to_string(p).unwrap_or_default();
for (i, _) in text.match_indices(&needle) {
let uri: String = text[i..]
.chars()
.take_while(|c| !matches!(c, '"' | ' ' | '\\' | ',' | ')' | '\n'))
.collect();
if uri.split('/').count() == 5 && !out.contains(&uri) {
out.push(uri);
}
}
}
out
}
/// Records that stay by rule after a release: vouches (Retain), the signer key, the operator associations of bidders.
pub fn kept_by_rule(uri: &str) -> bool {
["/sh.tangled.graph.vouch/", "badgeBlueKeys", "/app.bsky."]
.iter()
.any(|k| uri.contains(k))
}
/// The requester's log message of an accept a bidder refused (409 CapacityExceeded, no receipt).
pub const ACCEPT_DECLINED: &str = "requester_submit_accept_declined";
/// The accepts a bidder refused, by a requester's log lines: the requester keeps them by design (docs/acceptance/0014:
/// "the requester keeps that accept (a bidder may still act on it)"), so they are no leftover of a release. The one
/// rule of every record check (request.records, requester.records, gateway.log for assert.noneSince).
pub fn declined_accepts<'a>(lines: impl IntoIterator<Item = &'a Value>) -> Vec<String> {
let mut out: Vec<String> = lines
.into_iter()
.filter(|l| l["msg"] == ACCEPT_DECLINED)
.filter_map(|l| l["accept"].as_str().map(String::from))
.collect();
out.sort();
out.dedup();
out
}
/// A long-lived `run requester` (the orchestrator): its kcp, its ready file, its CLI helpers.
pub struct Requester {
pub bin: Binary,
pub name: String,
pub state: PathBuf,
pub log: PathBuf,
args: Vec<String>,
password: Option<String>,
child: tokio::sync::Mutex<Option<tokio::process::Child>>,
}
impl Requester {
/// Start `run requester --name <name>` in a new owner-only state dir under the run's scratch dir, sign in,
/// wait for the ready file. Stopped (Ctrl-C) by a cleanup.
pub async fn start(
ctx: &std::sync::Arc<Ctx>,
role: &str,
name: &str,
mode: SignIn,
extra: &[&str],
) -> Result<std::sync::Arc<Self>> {
let a = ctx.role(role)?;
let bin = Binary::resolve(&ctx.target.requester).await?;
let state = ctx.dir.join(format!("requester-{name}"));
std::fs::create_dir_all(&state)?;
std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))?;
let mut args: Vec<String> = ["run", "requester", "--name", name, "--state-dir"]
.iter()
.map(|s| s.to_string())
.collect();
args.push(state.display().to_string());
args.extend(extra.iter().map(|s| s.to_string()));
args.extend(identity_args(&ctx.target, a, mode));
let r = std::sync::Arc::new(Self {
bin,
name: name.into(),
log: state.join(format!("{name}.log")),
state,
args,
password: (mode == SignIn::AppPassword)
.then(|| a.password().map(|p| p.expose().to_string()))
.transpose()?,
child: tokio::sync::Mutex::new(None),
});
let stop = r.clone();
ctx.defer(&format!("stop requester {name}"), async move {
stop.stop().await
});
r.spawn().await?;
if mode == SignIn::OAuthQr {
let url = eventually(
"the sign-in URL",
Duration::from_secs(300),
Duration::from_secs(3),
|| async {
let (ok, out) = r
.bin
.run(&[
"run",
"requester",
"signin",
&r.name,
"--state-dir",
&r.state.display().to_string(),
])
.await?;
Ok(ok
.then(|| {
out.lines()
.find(|l| l.starts_with("https://"))
.map(str::to_string)
})
.flatten())
},
)
.await?;
oauth_signin(a, &url).await?;
}
r.wait_ready(Duration::from_secs(600)).await?;
Ok(r)
}
async fn spawn(&self) -> Result<()> {
let log = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&self.log)?;
let mut c = self.bin.cmd();
c.args(&self.args).stdout(log.try_clone()?).stderr(log);
if let Some(p) = &self.password {
c.env("ATPROTO_PASSWORD", p);
}
*self.child.lock().await = Some(c.spawn()?);
Ok(())
}
pub fn ready_path(&self) -> PathBuf {
self.state
.join("requesters")
.join(&self.name)
.join("ready.json")
}
pub fn ready(&self) -> Result<Value> {
Ok(serde_json::from_slice(&std::fs::read(self.ready_path())?)?)
}
pub async fn wait_ready(&self, timeout: Duration) -> Result<Value> {
eventually(
"the requester's ready file",
timeout,
Duration::from_secs(1),
|| async {
if let Some(c) = self.child.lock().await.as_mut()
&& let Some(st) = c.try_wait()?
{
bail!(
"run requester {} exited {st} before ready; see {}",
self.name,
self.log.display()
);
}
Ok(self.ready().ok())
},
)
.await
}
/// The requester workspace's kcp (kube-rs over the ready file's kubeconfig).
pub async fn kcp(&self) -> Result<Kcp> {
Kcp::kubeconfig(
self.ready()?["kubeconfig"]
.as_str()
.context("ready file without kubeconfig")?,
)
.await
}
pub fn did(&self) -> Result<String> {
Ok(self.ready()?["did"]
.as_str()
.context("ready file without did")?
.into())
}
/// The core logs of the requester (its own log and the core.log files under the state dir).
pub fn logs(&self) -> Vec<PathBuf> {
let mut out = vec![self.log.clone()];
find_named(&self.state, "core.log", &mut out);
out
}
/// Ctrl-C: the kcp stops, the ready file goes, the state stays.
pub async fn stop(&self) -> Result<()> {
let mut g = self.child.lock().await;
if let Some(c) = g.as_mut()
&& c.try_wait()?.is_none()
{
if let Some(pid) = c.id() {
std::process::Command::new("kill")
.args(["-INT", &pid.to_string()])
.status()?;
}
if tokio::time::timeout(Duration::from_secs(120), c.wait())
.await
.is_err()
{
c.kill().await?;
bail!(
"run requester {} did not stop in 120 s after SIGINT",
self.name
);
}
}
*g = None;
Ok(())
}
/// Ctrl-C and the same command again (0013 A9).
pub async fn restart(&self) -> Result<Value> {
self.stop().await?;
ensure!(
!self.ready_path().exists(),
"the ready file stayed after Ctrl-C"
);
self.spawn().await?;
self.wait_ready(Duration::from_secs(300)).await
}
/// The WebRTC transport to a port a Held contract's guest tunnels as `tag` (a kcp in the guest): this requester's
/// `run requester proxy` dials the datachannel (signalling through its own kcp) once per connection.
pub fn webrtc_proxy(&self, contract: &str, tag: &str) -> crate::target::Via {
let mut proxy = vec![self.bin.path.display().to_string()];
proxy.extend(["run", "requester", "proxy", &self.name, "--state-dir"].map(String::from));
proxy.push(self.state.display().to_string());
proxy.extend(["--contract", contract, "--tag", tag].map(String::from));
crate::target::Via::WebRtc { proxy }
}
/// `run requester ssh <name> --contract <cc> [--transport webrtc] --exec -- bash -c <line>`.
pub async fn ssh(&self, contract: &str, transport: Option<&str>, exec: &str) -> Result<String> {
let state = self.state.display().to_string();
let mut args = vec!["run", "requester", "ssh", &self.name, "--state-dir", &state, "--contract", contract];
if let Some(t) = transport {
args.extend(["--transport", t]);
}
let line = exec_line(exec);
args.extend(line.iter().map(String::as_str));
let (ok, out) = self.bin.run(&args).await?;
ensure!(
ok,
"run requester ssh {contract} ({transport:?}) failed: {}",
tail(&out)
);
Ok(out)
}
}
fn find_named(dir: &Path, name: &str, out: &mut Vec<PathBuf>) {
for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
let p = e.path();
if p.is_dir() {
find_named(&p, name, out);
} else if p.file_name().is_some_and(|n| n == name) {
out.push(p);
}
}
}
pub fn tail(s: &str) -> String {
let lines: Vec<_> = s.lines().collect();
lines[lines.len().saturating_sub(15)..].join("\n")
}
/// One `run request` invocation (its own kcp; the CLI path of the runbooks).
pub struct RunRequest {
pub bin: Binary,
pub name: String,
pub state: PathBuf,
pub log: PathBuf,
child: tokio::sync::Mutex<Option<tokio::process::Child>>,
}
impl RunRequest {
/// `run request --state-dir <dir> --name <name> <args> <identity> [--exec -- <command>]` in the background (app
/// password); an older CLI gets its own layout (`request_line`).
/// A cleanup releases it.
pub async fn start(
ctx: &std::sync::Arc<Ctx>,
role: &str,
name: &str,
args: &[&str],
) -> Result<std::sync::Arc<Self>> {
let bin = Binary::resolve(&ctx.target.requester).await?;
Self::start_with(ctx, bin, role, name, args).await
}
/// `start` with another binary (0006 compat: the previous release's CLI against new servers).
pub async fn start_with(
ctx: &std::sync::Arc<Ctx>,
bin: Binary,
role: &str,
name: &str,
args: &[&str],
) -> Result<std::sync::Arc<Self>> {
let a = ctx.role(role)?;
let state = ctx.dir.join("requests");
std::fs::create_dir_all(&state)?;
std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))?;
let log = state.join(format!("{name}-request.log"));
let f = std::fs::File::create(&log)?;
let (opts, command) = split_exec(&args.iter().map(|s| s.to_string()).collect::<Vec<_>>())?;
let line = request_line(
bin.exec_argv().await,
&opts,
&identity_args(&ctx.target, a, SignIn::AppPassword),
command.as_deref(),
);
let mut c = bin.cmd();
c.args(["run", "request", "--state-dir"])
.arg(&state)
.args(["--name", name])
.args(&line);
c.env("ATPROTO_PASSWORD", a.password()?.expose());
c.stdout(f.try_clone()?).stderr(f);
let r = std::sync::Arc::new(Self {
bin,
name: name.into(),
state,
log,
child: tokio::sync::Mutex::new(Some(c.spawn()?)),
});
let rel = r.clone();
ctx.defer(&format!("release {name}"), async move {
rel.release().await.map(|_| ())
});
Ok(r)
}
/// The CLI log plus the invocation's core.log (while it runs, and after release).
pub fn logs(&self) -> Vec<PathBuf> {
let base = self.state.join("requests").join(&self.name);
vec![
self.log.clone(),
base.join("dev").join(&self.name).join("logs/core.log"),
base.join("logs/core.log"),
]
}
/// The contract phase lines of the CLI log; waits for `phase`.
pub async fn wait_phase(&self, phase: &str, timeout: Duration) -> Result<Value> {
eventually(
&format!("{} {phase}", self.name),
timeout,
Duration::from_secs(1),
|| async {
if let Some(c) = self.child.lock().await.as_mut()
&& let Some(st) = c.try_wait()?
&& !st.success()
{
bail!(
"run request {} exited {st}: {}",
self.name,
tail(&std::fs::read_to_string(&self.log).unwrap_or_default())
);
}
Ok(log_lines(&self.log, "contract")
.into_iter()
.find(|l| l["phase"] == phase))
},
)
.await
}
pub async fn wait_log(&self, msg: &str, timeout: Duration) -> Result<Value> {
wait_log(&self.logs(), msg, timeout).await
}
fn sub(&self, verb: &str, extra: &[&str]) -> Vec<String> {
let mut v: Vec<String> = ["run", "request", verb, &self.name, "--state-dir"]
.iter()
.map(|s| s.to_string())
.collect();
v.push(self.state.display().to_string());
v.extend(extra.iter().map(|s| s.to_string()));
v
}
/// `run request agent <name>`: publishes guest.agentStart.
pub async fn agent(&self) -> Result<()> {
let args = self.sub("agent", &[]);
let (ok, out) = self
.bin
.run(&args.iter().map(String::as_str).collect::<Vec<_>>())
.await?;
ensure!(ok, "run request agent: {}", tail(&out));
Ok(())
}
/// `run request ssh <name> [--transport webrtc] --exec -- bash -c <line>` (iroh first by default).
pub async fn ssh(&self, transport: Option<&str>, exec: &str) -> Result<String> {
let mut extra = vec![];
if let Some(t) = transport {
extra.extend(["--transport", t]);
}
let line = exec_line(exec);
extra.extend(line.iter().map(String::as_str));
let args = self.sub("ssh", &extra);
let (ok, out) = self
.bin
.run(&args.iter().map(String::as_str).collect::<Vec<_>>())
.await?;
ensure!(ok, "run request ssh: {}", tail(&out));
Ok(out)
}
/// Wait for the --hold process to exit (it exits after its first ssh unless held).
pub async fn wait_exit(&self, timeout: Duration) -> Result<()> {
if let Some(c) = self.child.lock().await.as_mut() {
tokio::time::timeout(timeout, c.wait())
.await
.map_err(|_| {
anyhow!("run request {} still running after {timeout:?}", self.name)
})??;
}
Ok(())
}
/// `run request release`; idempotent. Returns the seconds it took.
pub async fn release(&self) -> Result<f64> {
let t = Instant::now();
let args = self.sub("release", &[]);
let (ok, out) = self
.bin
.run(&args.iter().map(String::as_str).collect::<Vec<_>>())
.await?;
if let Some(mut c) = self.child.lock().await.take()
&& tokio::time::timeout(Duration::from_secs(300), c.wait())
.await
.is_err()
{
c.kill().await?;
}
ensure!(ok, "run request release {}: {}", self.name, tail(&out));
Ok(t.elapsed().as_secs_f64())
}
pub fn summary(&self) -> Result<Value> {
Ok(serde_json::from_slice(&std::fs::read(
self.state
.join("requests")
.join(&self.name)
.join("summary.json"),
)?)?)
}
}