//! computer.socialweb.test.*: the socialweb.computer side of the test host. The requester under test (a release, a //! commit or a local build) as the account: `run requester` (the orchestrator, its kcp registered as kube cluster //! `requester:`), `run request` invocations (a subscription of their log events), the owner's atirc mesh node. //! Every requester and invocation it starts is stopped or released by `cleanup()`, however the run ended. //! com.fedfork.exec.identlink.{walk,plc}: identlink against the target's PLC. use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use atlibs_host::BoxFut; use atlibs_host::run::{Call, Request, Sub, SubCmd}; use atlibs_host::transport::Transport; use atlibs_kube::{Cluster, Kube}; use serde_json::{Value, json}; use testkit::requester::{Binary, SignIn, exec_line, identity_args, kept_by_rule, request_line, split_exec, tail, uris_in}; use testkit::target::{Account, Target, check_private, expand}; use tokio::sync::mpsc; #[path = "knotless.rs"] mod knotless; #[path = "more.rs"] mod more; pub use more::cleanup_from_disk; const P: &str = "computer.socialweb.test."; /// Where the guest requester reads its association ref (the renter writes it), and how long it may take to come. const R2_REF: &str = "/run/socialweb-r2/association-ref"; const R2_REF_WAIT: Duration = Duration::from_secs(15 * 60); const PUT_RETRIES: u32 = 3; const PUT_RETRY_BACKOFF: Duration = Duration::from_secs(10); pub const NSIDS: &[&str] = &[ "computer.socialweb.test.requester.start", "computer.socialweb.test.request.start", "computer.socialweb.test.request.ssh", "computer.socialweb.test.request.put", "computer.socialweb.test.request.agent", "computer.socialweb.test.request.summary", "computer.socialweb.test.request.release", "computer.socialweb.test.request.kill", "computer.socialweb.test.devsup.race", "computer.socialweb.test.devsup.samename", "computer.socialweb.test.request.state", "computer.socialweb.test.request.sshTry", "computer.socialweb.test.request.setPhase", "computer.socialweb.test.request.releaseTry", "computer.socialweb.test.request.removeState", "computer.socialweb.test.request.records", "computer.socialweb.test.account.envFile", "computer.socialweb.test.atirc.meshSub", "computer.socialweb.test.atirc.meshRun", "computer.socialweb.test.atirc.meshPub", "computer.socialweb.test.atirc.meshPeers", "computer.socialweb.test.kube.addCluster", "computer.socialweb.test.kube.undo", "computer.socialweb.test.page.signin", "computer.socialweb.test.page.fetch", "computer.socialweb.test.page.round", "computer.socialweb.test.ssh.tempKey", "computer.socialweb.test.atirc.observer", "computer.socialweb.test.atirc.observerPub", "computer.socialweb.test.atirc.observerRevoke", "computer.socialweb.test.requester.ssh", "computer.socialweb.test.requester.restart", "computer.socialweb.test.requester.cluster", "computer.socialweb.test.requester.bundle", "computer.socialweb.test.staging.pki", "computer.socialweb.test.logs.fetchRuns", "computer.socialweb.test.requester.records", "computer.socialweb.test.requester.events", "computer.socialweb.test.root.exec", "computer.socialweb.test.root.putBinary", "computer.socialweb.test.release.put", "computer.socialweb.test.release.delete", "computer.socialweb.test.example", "computer.socialweb.test.gateway.kubeconfig", "computer.socialweb.test.page.kubeconfig", "computer.socialweb.test.oauth.signin", "computer.socialweb.test.request.r2", "computer.socialweb.test.request.checkLogs", "computer.socialweb.test.knotless.seed", "computer.socialweb.test.knotless.sh", "computer.socialweb.test.bidder.health", "computer.socialweb.test.bidder.meminfo", "computer.socialweb.test.bidder.logCount", "computer.socialweb.test.vm.memory", "computer.socialweb.test.bidder.logLines", "computer.socialweb.test.bidder.submitAccept", "computer.socialweb.test.bidder.restart", "computer.socialweb.test.bidder.upgrade", "computer.socialweb.test.bidder.guestNft", "computer.socialweb.test.bidder.awaitLog", "computer.socialweb.test.bidder.dumbpipe", "computer.socialweb.test.binary.published", "computer.socialweb.test.assets.tamper", "computer.socialweb.test.gateway.log", "computer.socialweb.test.repo.listCreated", "computer.socialweb.test.request.contract", "com.fedfork.exec.identlink.walk", "com.fedfork.exec.identlink.plc", ]; /// `run request` flags a module may pass, before `--exec -- [args...]` (the command is not checked); /// `@agentConfig` is the target's agent config (owner-only file). const FLAGS: &[&str] = &[ "--hold", "--agent", "--agent-config", "--tunnel", "--transport", "--exec", "--guest-tools", "--associate", "--expires-in", "--nested-virt", "--cpus", "--mem", "--image", ]; pub struct Swc { pub target: Target, pub account: String, pub bin: Binary, /// the work dir: requester state, kubeconfigs, keys (secrets; never an artifact) pub dir: PathBuf, /// the dump dir: resources.json beside the ticks (no secret) pub dump: PathBuf, pub kube: Arc, /// receipts of this run's contracts: the kube guard lets a test delete only VMs of these pub receipts: Arc>>, /// guest requester DIDs this run's own requester associated (its `requester_association_written` subject): the kube /// guard lets a test delete their rentals too (owner, 2026-11-18: "Both") pub associated: Arc>>, requesters: tokio::sync::Mutex>, requests: Mutex>, env_files: Mutex>, http: reqwest::Client, ssh: Arc, /// the target's read-only ops routes (prod in CI), when it names any ops: Option>, extra: more::Extra, } /// A call's input; a subscription's arguments come as params (strings), its structure in the `json` param. fn input_of(r: &Request) -> Value { if let Some(i) = &r.input { return i.clone(); } let params = r.params.clone().unwrap_or_default(); match params .get("json") .and_then(Value::as_str) .and_then(|j| serde_json::from_str::(j).ok()) { Some(v) => v, None => Value::Object(params), } } /// `v` reduced to the dotted `paths` it has (a.b keeps {a: {b}}), nothing else. fn project(v: &Value, paths: &[&str]) -> Value { let mut out = json!({}); for p in paths { let keys: Vec<&str> = p.split('.').collect(); let Some(found) = keys.iter().try_fold(v, |at, k| at.get(*k)) else { continue }; let mut dst = &mut out; for k in &keys[..keys.len() - 1] { if !dst[*k].is_object() { dst[*k] = json!({}); } dst = &mut dst[*k]; } dst[keys[keys.len() - 1]] = found.clone(); } out } fn str_of<'a>(v: &'a Value, k: &str) -> Result<&'a str, String> { v[k].as_str() .filter(|s| !s.is_empty() && !s.contains(['/', ' ', '\n'])) .ok_or_else(|| format!("{k} is required (a plain name)")) } fn abs_dir_of<'a>(v: &'a Value, k: &str) -> Result<&'a str, String> { v[k].as_str() .filter(|s| { s.strip_prefix('/').is_some_and(|rest| rest.split('/').all(|c| !c.is_empty() && c != "." && c != "..")) && !s.chars().any(|c| c.is_whitespace() || c.is_control() || "'\"\\$`".contains(c)) }) .ok_or_else(|| format!("{k} is required (a clean absolute path)")) } impl Swc { #[allow(clippy::too_many_arguments)] pub fn new( target: Target, account: &str, bin: Binary, dir: PathBuf, kube: Arc, receipts: Arc>>, associated: Arc>>, ssh: Arc, dump: PathBuf, ops: Option>, ) -> Arc { let http = testkit::atproto::http_client(&target); Arc::new(Self { target, account: account.into(), bin, dir, dump, kube, receipts, associated, requesters: Default::default(), requests: Default::default(), env_files: Default::default(), http, ssh, ops, extra: Default::default(), }) } pub fn transport(self: &Arc) -> Transport { Transport { provides: NSIDS, call: self.clone(), } } fn acct(&self) -> Result<&Account, String> { self.target .account(&self.account) .map_err(|e| e.to_string()) } fn password(&self) -> Result { Ok(self .acct()? .password() .map_err(|e| e.to_string())? .expose() .to_string()) } fn request_state(&self) -> PathBuf { // the guest requester's `run request` holds a dev instance (its PDS, its kcp) whose sockets under // requests//run/ must fit sun_path (107 bytes): a short dir of its own, ~/.swc/ if self.target.network.is_none() && self.acct().is_ok_and(|a| a.guest_requester) { let name = self .dir .file_name() .unwrap_or_default() .to_string_lossy() .to_string(); let ms = name.rsplit('-').next().unwrap_or_default().to_string(); return PathBuf::from(std::env::var("HOME").unwrap_or_default()) .join(".swc") .join(ms); } match (&self.target.network, self.associated()) { (Some(n), _) => expand(&n.state_dir), (None, Some(id)) => expand(&id.state_dir), (None, None) => self.dir.join("requests"), } } fn request_logs(&self, name: &str) -> Vec { let base = self.request_state().join("requests").join(name); let mut logs = vec![ self.request_state().join(format!("{name}-request.log")), base.join("dev").join(name).join("logs/core.log"), base.join("logs/core.log"), ]; // `run request --network`: the invocation's requester is a child core of the network, its log there if let Some(n) = &self.target.network { logs.push( expand(&n.state_dir) .join("dev") .join(&n.instance) .join("apps/dev") .join(format!("{}-{name}", n.instance)) .join("logs/core.log"), ); } logs } pub(crate) async fn cli(&self, args: &[String]) -> Result<(bool, String), String> { self.bin .run(&args.iter().map(String::as_str).collect::>()) .await .map_err(|e| e.to_string()) } /// Copies one of this host's files to the guest of invocation `name` (stdin of `run request ssh --exec -- bash -c 'cat > /// '`). `what` names the file, never a path: testHost ($SWC_TEST_HOST_STATIC, a static swc-test-host) or /// suite (a tar.gz of the repo's examples, scripts, tests/compositions, tests/targets, did-key-associator with the four /// typescript-helpers libraries it imports (the network's QR service, qr.sh) and the module wasm and /// module.json files; SWC_REPO on the guest; with $SWC_PAGE_REPO, a built checkout of the requester page repo or /// release's unpacked page payload (scripts/ci-page.sh), also 0009's page as page/: web/webrtc with its modules' files, /// the round's two scripts and COMMIT (the payload's own file, else the checkout's HEAD)). async fn request_put(&self, input: &Value) -> Result { let name = str_of(input, "name")?; let remote = input["remote"].as_str().unwrap_or_default(); if !remote.starts_with('/') || remote.contains(['\'', ' ', '\n']) { return Err(format!( "remote {remote:?}: an absolute path without quotes or spaces" )); } let bytes = match str_of(input, "what")? { "testHost" => { let p = std::env::var("SWC_TEST_HOST_STATIC") .map_err(|_| "SWC_TEST_HOST_STATIC is not set (tests/build-static-host.sh)")?; std::fs::read(&p).map_err(|e| format!("{p}: {e}"))? } "suite" => { let tests = std::env::current_dir().map_err(|e| e.to_string())?; let out = tokio::process::Command::new("sh") .args(["-c", "set -e; p=; if [ -n \"${SWC_PAGE_REPO:-}\" ]; then p=$(mktemp -d); trap 'rm -rf \"$p\"' EXIT; mkdir -p \"$p/page/web\" \"$p/page/scripts\"; \ cp -rL \"$SWC_PAGE_REPO/web/webrtc\" \"$p/page/web/\"; cp \"$SWC_PAGE_REPO/scripts/page-round.mjs\" \"$SWC_PAGE_REPO/scripts/browser.mjs\" \"$p/page/scripts/\"; \ if [ -f \"$SWC_PAGE_REPO/COMMIT\" ]; then cp \"$SWC_PAGE_REPO/COMMIT\" \"$p/page/COMMIT\"; else git -C \"$SWC_PAGE_REPO\" rev-parse HEAD > \"$p/page/COMMIT\"; fi; fi; \ tar -czf - --exclude=did-key-associator/dist --exclude=did-key-associator/dist-backend tests/gate.sh tests/entries.txt tests/compositions tests/targets did-key-associator typescript-helpers/lib/oauth-scope typescript-helpers/lib/cli-args-env typescript-helpers/lib/logger typescript-helpers/lib/serve tests/modules/*/module.json tests/modules/target/wasm32-unknown-unknown/release/*.wasm deploy/vm-images examples scripts ${p:+-C \"$p\" page}"]) .current_dir(tests.join("..")) .output() .await .map_err(|e| e.to_string())?; if !out.status.success() { return Err(format!( "tar of the suite in {}: {}", tests.display(), String::from_utf8_lossy(&out.stderr) )); } out.stdout } "meshd" => { let p = std::env::var("SWC_MESHD").map_err( |_| "SWC_MESHD is not set (a static socialweb-meshd, rust/build.sh)", )?; std::fs::read(&p).map_err(|e| format!("{p}: {e}"))? } "binary" => { let p = std::env::var("SWC_BINARY") .map_err(|_| "SWC_BINARY is not set (the socialweb-computer under test, a release build of this checkout)")?; std::fs::read(&p).map_err(|e| format!("{p}: {e}"))? } "stagingRoot" | "stagingLeaf" | "stagingKey" => { let file = match input["what"].as_str().unwrap_or_default() { "stagingRoot" => "root.crt", "stagingLeaf" => "leaf.crt", _ => "leaf.key", }; let p = self.dir.join("staging-pki").join(file); std::fs::read(&p).map_err(|e| format!("{}: {e} (staging.pki first)", p.display()))? } "agentConfig" => { let cfg = testkit::target::expand( self.target .agent_config .as_deref() .ok_or("the target has no agentConfig")?, ); testkit::target::check_private(&cfg).map_err(|e| e.to_string())?; std::fs::read(&cfg).map_err(|e| format!("{}: {e}", cfg.display()))? } other => { return Err(format!( "what {other:?}: testHost, suite, meshd, binary or agentConfig" )); } }; let umask = if input["what"] == "agentConfig" || input["what"] == "stagingKey" || input["what"] == "stagingLeaf" { "umask 077 && " } else { "" }; // a file the bucket publishes (the test host, meshd by tree): the VM downloads it at full speed, checked against // this file's sha256, instead of a stream over its ssh (minutes at a relayed path's ~50 KB/s, plan 0044 finding 35) let url = match input["what"].as_str().unwrap_or_default() { "testHost" => std::env::var("SWC_TEST_HOST_URL").ok(), "meshd" => std::env::var("SWC_MESHD_URL").ok(), _ => None, } .filter(|u| u.starts_with("https://") && !u.contains(['\'', ' ', '\n'])); let sha: String = { use sha2::Digest; sha2::Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect() }; if let Some(url) = url { let exec = format!( "curl -fsS --retry 3 -o '{remote}.part' '{url}' && echo '{sha} {remote}.part' | sha256sum -c --quiet && mv '{remote}.part' '{remote}'" ); if self.put_exec(name, input, exec, None).await.is_ok() { return Ok(json!({ "remote": remote, "bytes": bytes.len(), "from": url })); } } let exec = format!( "{umask}cat > '{remote}.part' && echo '{sha} {remote}.part' | sha256sum -c --quiet && mv '{remote}.part' '{remote}'" ); let mut tries: u32 = 0; loop { tries += 1; match self.put_exec(name, input, exec.clone(), Some(&bytes)).await { Ok(()) => break, Err(e) if e.contains("connection lost") && tries <= PUT_RETRIES => { tokio::time::sleep(PUT_RETRY_BACKOFF * tries).await; } Err(e) => return Err(format!("put {remote} (try {tries}): {e}")), } } Ok(json!({ "remote": remote, "bytes": bytes.len(), "sha256": sha, "tries": tries })) } // an exec on the VM of a put: a Held contract of the run requester `name` (an Infra's), else `name` is a run request // invocation; stdin is the file when one is streamed async fn put_exec(&self, name: &str, input: &Value, exec: String, stdin: Option<&[u8]>) -> Result<(), String> { let args: Vec = match input["contract"].as_str() { Some(c) => ["run", "requester", "ssh", name, "--state-dir"] .map(String::from) .into_iter() .chain([self.requester_state(name).display().to_string(), "--contract".into(), c.into()]) .chain(exec_line(&exec)) .collect(), None => { let mut v = self.sub_args("ssh", name, &[]); v.extend(exec_line(&exec)); v } }; let mut child = self .bin .cmd() .args(args) .stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() }) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .map_err(|e| e.to_string())?; let mut wrote = Ok(()); if let Some(bytes) = stdin { let mut sin = child.stdin.take().unwrap(); use tokio::io::AsyncWriteExt; wrote = sin.write_all(bytes).await; drop(sin); } let out = child.wait_with_output().await.map_err(|e| e.to_string())?; if !out.status.success() { return Err(tail(&String::from_utf8_lossy(&out.stderr))); } if let Err(e) = wrote { return Err(format!("stdin: {e}")); } Ok(()) } pub(crate) fn sub_args(&self, verb: &str, name: &str, extra: &[&str]) -> Vec { let mut v: Vec = ["run", "request", verb, name, "--state-dir"] .map(String::from) .to_vec(); v.push(self.request_state().display().to_string()); v.extend(extra.iter().map(|s| s.to_string())); v } async fn requester_start(&self, input: &Value) -> Result { let name = str_of(input, "name")?.to_string(); let state = self.requester_state(&name); std::fs::create_dir_all(&state).map_err(|e| e.to_string())?; std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700)) .map_err(|e| e.to_string())?; let mut args: Vec = vec![ "run".into(), "requester".into(), "--name".into(), name.clone(), "--state-dir".into(), state.display().to_string(), ]; if input["withGateway"] == true { args.push("--with-gateway".into()); } // a market network target: the requester is a child of the network (its relays, PLC and bidders), as request_start's if let Some(n) = &self.target.network { args.extend(["--network".to_string(), n.instance.clone()]); } // R2 (a guest requester): its identity is the renter's association, through the guest's /run/socialweb-r2 files let guest = self.acct()?.guest_requester; let associated = self.associated().map(|id| id.name.clone()); if guest { args.push("--guest-requester".into()); } else if let Some(id) = &associated { args.extend(["--identity".to_string(), id.clone()]); } else { args.extend(identity_args( &self.target, self.acct()?, SignIn::AppPassword, )); } // plan 0040 step 5: the target's bidders filter its discovery (every rental, a --with-gateway node's too); a market // network sets its own bidders for its child applications, so only a target outside one passes them let only: Vec<&str> = input["bidderDids"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); if !only.is_empty() && self.target.network.is_none() { args.extend(["--bidder-dids".to_string(), only.join(",")]); } self.extra .argv .lock() .unwrap() .insert(name.clone(), args.clone()); let log = std::fs::File::create(state.join(format!("{name}.log"))).map_err(|e| e.to_string())?; let child = self .bin .cmd() .args(&args) .env( "ATPROTO_PASSWORD", if guest || associated.is_some() { String::new() } else { self.password()? }, ) .stdout(log.try_clone().map_err(|e| e.to_string())?) .stderr(log) .spawn() .map_err(|e| e.to_string())?; self.requesters .lock() .await .insert(name.clone(), (state.clone(), child)); let ready = state.join("requesters").join(&name).join("ready.json"); let deadline = Instant::now() + Duration::from_secs(600); let r: Value = loop { if let Ok(b) = std::fs::read(&ready) && let Ok(v) = serde_json::from_slice(&b) { break v; } if let Some((_, c)) = self.requesters.lock().await.get_mut(&name) && let Ok(Some(st)) = c.try_wait() { // the requester's own last lines (its log, redacted as the run's logs are), so the failure reads from the // gate's RESULT line alone (a CI run keeps no artifacts) let said = std::fs::read_to_string(state.join(format!("{name}.log"))).unwrap_or_default(); let lines: Vec<&str> = said.lines().filter(|l| !l.trim().is_empty()).collect(); let last = redact_log(&lines[lines.len().saturating_sub(3)..].join("\n")); let last: Vec = last.lines().map(|l| l.chars().take(400).collect()).collect(); return Err(format!( "run requester {name} exited {st} before ready: {}", last.join(" | ") )); } if Instant::now() > deadline { return Err(format!("run requester {name}: no ready file in 10 min")); } tokio::time::sleep(Duration::from_secs(1)).await; }; let kubeconfig = r["kubeconfig"] .as_str() .ok_or("ready file without kubeconfig")? .to_string(); self.kube.add_cluster( &format!("requester:{name}"), Cluster { kubeconfig, workspace: None, via: Default::default(), }, ); Ok( json!({ "did": r["did"], "handle": r["handle"], "identityMode": r["identityMode"], "cluster": format!("requester:{name}"), "gateway": r["gateway"], "sshKey": r["sshPublicKey"].as_str().and_then(|p| std::fs::read_to_string(p).ok()).map(|k| k.trim().to_string()) }), ) } fn request_args(&self, input: &Value) -> Result<(Vec, Option>), String> { let words: Vec = input["args"] .as_array() .into_iter() .flatten() .map(|w| w.as_str().map(String::from).ok_or("args are strings")) .collect::>()?; let (opts, command) = split_exec(&words).map_err(|e| e.to_string())?; let mut out = vec![]; for w in &opts { let w = w.as_str(); if w == "@agentConfig" { let p = expand( self.target .agent_config .as_deref() .ok_or("the target has no agentConfig")?, ); check_private(&p).map_err(|e| e.to_string())?; out.push(p.display().to_string()); } else if w.starts_with("--") && !FLAGS.contains(&w.split('=').next().unwrap()) { return Err(format!("run request flag {w} is not one a test may pass")); } else { out.push(w.to_string()); } } Ok((out, command)) } /// `run request --name --exec -- `, its log events as messages until the /// subscription closes. async fn request_start(&self, input: &Value) -> Result { let name = str_of(input, "name")?.to_string(); let (mut args, command) = self.request_args(input)?; // secrets: the module's bundle [{path, value, mode?}] (test values only), written 0600 in the work dir if let Some(bundle) = input["secrets"].as_array() { let p = self.dir.join(format!("{name}-secrets.json")); more::write_private(&p, &Value::Array(bundle.clone()).to_string())?; args.extend(["--secrets".to_string(), p.display().to_string()]); } if let Some(n) = &self.target.network { args.extend(["--network".to_string(), n.instance.clone()]); } let state = self.request_state(); std::fs::create_dir_all(&state).map_err(|e| e.to_string())?; std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700)) .map_err(|e| e.to_string())?; let logs = self.request_logs(&name); let f = std::fs::File::create(&logs[0]).map_err(|e| e.to_string())?; // compat: the unchanged client (SWC_COMPAT_REQUESTER: release:, sha:..., path:...) against new servers let bin = match ( input["compat"] == true, std::env::var("SWC_COMPAT_REQUESTER"), ) { (true, Ok(spec)) => Binary::resolve(&spec).await.map_err(|e| e.to_string())?, _ => self.bin.clone(), }; let mut cmd = bin.cmd(); cmd.args(["run", "request", "--state-dir"]) .arg(&state) .args(["--name", &name]); // R2 rents as the renter's agent: its identity is the association, no password let mut pass: Vec = vec![]; if self.acct()?.guest_requester { // its own key in the VM it rents, for request.ssh args.extend(["--guest-requester".to_string(), "--skip-ssh=false".to_string()]); } else if let Some(id) = self.associated() { args.extend(["--identity".to_string(), id.name.clone()]); } else { pass.extend(identity_args(&self.target, self.acct()?, SignIn::AppPassword)); cmd.env("ATPROTO_PASSWORD", self.password()?); } // the RFP's compute.vm role (e.g. the staging layer the watch page shows), and the bidders it may (bidderDids) // or may not (denyBidderDids) go to: MarketRequester (contract) options if let Some(v) = input["vmName"].as_str() { if v.is_empty() || !v .chars() .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') { return Err(format!( "vmName {v:?}: lowercase letters, digits and dashes" )); } pass.extend(["--vm-name".to_string(), v.to_string()]); } // the requester's comms mode and secrets fallback (which channel the guest's secrets take): requester options for (k, flag) in [("commsMode", "--comms-mode"), ("secretsFallback", "--secrets-fallback")] { if let Some(v) = input[k].as_str() { if v.is_empty() || !v.chars().all(|c| c.is_ascii_alphabetic()) { return Err(format!("{k} {v:?}: a mode name")); } pass.extend([flag.to_string(), v.to_string()]); } } for (k, flag) in [ ("bidderDids", "--allow-bidder-dids"), ("denyBidderDids", "--deny-bidder-dids"), ] { let Some(dids) = input[k].as_array() else { continue; }; let dids: Vec<&str> = dids.iter().filter_map(Value::as_str).collect(); if dids .iter() .any(|d| !d.starts_with("did:") || d.contains(',')) { return Err(format!("{k}: DIDs")); } if !dids.is_empty() { pass.extend([flag.to_string(), dids.join(",")]); } } // the bidder endpoint kinds the requester may use, in order (SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS): iroh, webrtc, https if let Some(kinds) = input["bidderEndpoints"].as_array() { let kinds: Vec<&str> = kinds.iter().filter_map(Value::as_str).collect(); if kinds.is_empty() || kinds.iter().any(|k| !["iroh", "webrtc", "https"].contains(k)) { return Err("bidderEndpoints: iroh, webrtc or https".into()); } cmd.env("SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS", kinds.join(",")); } // plan 0050: the staging run's user_data (its root CA), made by staging.pki if input["stagingUserData"] == true || input["stagingUserData"].is_string() { let p = match input["stagingUserData"].as_str() { Some(v) if v.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') => { self.dir.join("staging-pki").join(format!("user-data-{v}")) } Some(v) => return Err(format!("stagingUserData {v:?}: a variant name")), None => self.dir.join("staging-pki").join("user-data"), }; if !p.is_file() { return Err(format!("{}: staging.pki first", p.display())); } pass.extend(["--user-data".to_string(), p.display().to_string()]); } // runcmd: shell lines the guest runs from the request's own user_data (e.g. 0007:webrtc turns a channel off) if let Some(lines) = input["runcmd"].as_array() { if pass.iter().any(|w| w == "--user-data") { return Err("runcmd and stagingUserData: one --user-data".into()); } let mut ud = String::from("#cloud-config\nruncmd:\n"); for l in lines { let l = l.as_str().ok_or("runcmd: shell lines")?; ud.push_str(&format!(" - [sh, -c, {}]\n", serde_json::to_string(l).map_err(|e| e.to_string())?)); } let p = self.dir.join(format!("{name}-user-data.yaml")); more::write_private(&p, &ud)?; pass.extend(["--user-data".to_string(), p.display().to_string()]); } // a CLI before plan 0043 (0006:compat's previous rc) takes them after `--` and the command as one `--exec` line cmd.args(request_line(bin.exec_argv().await, &args, &pass, command.as_deref())); // pty: the invocation runs on a pseudo-terminal (socat SYSTEM,pty) whose stdin stays open, so `run request` // without `--exec` opens its interactive ssh session (a terminal on stdin) and holds it let pty = input["pty"] == true; let mut spawn_cmd = if pty { let std_cmd = cmd.as_std(); let line = std::iter::once(std_cmd.get_program()) .chain(std_cmd.get_args()) .map(|w| format!("'{}'", w.to_string_lossy().replace('\'', r#"'"'"'"#))) .collect::>() .join(" "); let envs: Vec<_> = std_cmd .get_envs() .filter_map(|(k, v)| v.map(|v| (k.to_os_string(), v.to_os_string()))) .collect(); // socat's SYSTEM address runs the line with sh -c on a pty; its `,` `:` `\\` are escaped let esc = line .replace('\\', "\\\\") .replace(',', "\\,") .replace(':', "\\:"); let mut script = tokio::process::Command::new("socat"); script .args(["-", &format!("SYSTEM:{esc},pty,setsid,ctty,stderr")]) .envs(envs); script.stdin(Stdio::piped()); script } else { cmd }; let mut child = spawn_cmd .stdout(f.try_clone().map_err(|e| e.to_string())?) .stderr(f) .spawn() .map_err(|e| e.to_string())?; // held open until the invocation is over let pty_stdin = child.stdin.take(); self.requests.lock().unwrap().insert(name.clone(), state); let (tx, messages) = mpsc::unbounded_channel(); // which binary runs the invocation (0006 compat: the unchanged CLI's) let _ = tx.send(Ok(json!({ "msg": "requester_binary", "version": bin.version, "compat": input["compat"] == true }))); let (commands, mut cmds) = mpsc::unbounded_channel::(); let receipts = self.receipts.clone(); let associated = self.associated.clone(); // dev nets only: when the contract fails with "did not come on network", the tail of the console of the guest(s) // that never reported (the cause is in the guest: PID 1, cloud-final), in the event and in the dump let console_net = self .target .network .as_ref() .map(|n| (expand(&n.state_dir), n.instance.clone())); let console_dump = self.dump.join("logs").join(&name); // R2 waits for its association ref, which the renter writes once it verified R2: one that never comes is an // error of its own, not a request that waits until the run's deadline let ref_by = self .acct()? .guest_requester .then(|| Instant::now() + R2_REF_WAIT); tokio::spawn(async move { let _pty_stdin = pty_stdin; let mut offsets = vec![0usize; logs.len()]; let mut exited = false; // the run's dev core.log is copied to logs/core.log when the process ends: each line once let mut seen = std::collections::HashSet::new(); loop { for (i, p) in logs.iter().enumerate() { let Ok(text) = std::fs::read_to_string(p) else { continue; }; if text.len() < offsets[i] { offsets[i] = 0; } let fresh = &text[offsets[i]..]; let Some(end) = fresh.rfind('\n') else { continue; }; for line in fresh[..=end].lines() { let Ok(v) = serde_json::from_str::(line) else { continue; }; if !seen.insert(line.to_string()) { continue; } // the receipt is known (and the run's VM ours to fault) from its first event, before any summary if let Some(r) = v["receipt"].as_str() { let mut rs = receipts.lock().unwrap(); if !rs.iter().any(|x| x == r) { rs.push(r.into()); } } if v["msg"] == "requester_association_written" && let Some(d) = v["subject"].as_str().filter(|d| d.starts_with("did:")) { let mut a = associated.lock().unwrap(); if !a.iter().any(|x| x == d) { a.push(d.into()); } } let mut v = v; if v["msg"] == "contract" && v["phase"] == "Failed" && v["message"] .as_str() .is_some_and(|m| m.contains("did not come on network")) && let Some((state_dir, instance)) = &console_net { let tails = silent_console_tails(state_dir, instance, &console_dump); if !tails.is_empty() { let m = v["message"].as_str().unwrap_or_default().to_string(); v["message"] = json!(format!( "{m}\nconsole tail of the guest(s) that never reported (dev net):\n{tails}" )); } } let msg = v["msg"].as_str().unwrap_or_default().to_string(); let msg = msg.as_str(); if (msg == "contract" || msg == "ssh returned" || msg.starts_with("requester_") || msg.starts_with("guest_") || msg.starts_with("ssh to the guest over") || msg.starts_with("no --exec and stdin is not a terminal") || msg.starts_with("ssh not ready yet")) && tx.send(Ok(v)).is_err() { return; } } offsets[i] += end + 1; } if !exited && let Ok(Some(st)) = child.try_wait() { exited = true; // the CLI's own last lines (a refusal before any kcp is plain stderr) let tail = std::fs::read_to_string(&logs[0]).map(|t| testkit::requester::tail(&t)).unwrap_or_default(); let _ = tx.send(Ok(json!({ "msg": "exited", "code": st.code(), "tail": tail }))); } let referred = || std::fs::read_to_string(R2_REF).is_ok_and(|r| !r.trim().is_empty()); if !exited && ref_by.is_some_and(|by| Instant::now() > by) && !referred() { exited = true; let _ = child.start_kill(); let why = format!( "no association ref in {R2_REF} after {} min: the renter's associate failed or never ran (the knotless check's log says why)", R2_REF_WAIT.as_secs() / 60 ); let _ = tx.send(Ok(json!({ "msg": "exited", "code": null, "error": why }))); } tokio::select! { cmd = cmds.recv() => match cmd { Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("request.start takes no sends".into())); } Some(SubCmd::Close) | None => return, }, _ = tokio::time::sleep(Duration::from_secs(1)) => {} } } }); Ok(Sub { messages, commands }) } fn summary(&self, name: &str) -> Result { let p = self .request_state() .join("requests") .join(name) .join("summary.json"); let v: Value = serde_json::from_slice( &std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?, ) .map_err(|e| e.to_string())?; if let Some(r) = v["contract"]["receiptUri"].as_str() { let mut rs = self.receipts.lock().unwrap(); if !rs.iter().any(|x| x == r) { rs.push(r.into()); } } Ok(v) } /// The restart race of a real devsup with no VM (docs/acceptance/0017): `dev up --profile quick` on the proc backend in an /// instance of this run, then per delay `iterations` times: `dev down` SIGKILLed `delay` ms after it started (devsup keeps /// stopping its units in order and exits), `dev up` at once, and 6 s later the kcp must answer /readyz 200. The binary is the /// target's requester (the build under test). Skipped when it has no `dev up`. // plan 0046 step 6: two instances of one name in two state dirs (two `run requester --name stg`) are two // instances: each its own devsup (its runtime dir keyed by its state dir), and a down of one leaves the other up async fn devsup_samename(&self) -> Result { let inst = format!("same{}", std::process::id() % 100000); let dirs = [self.dir.join("samename-a"), self.dir.join("samename-b")]; let cmd = |verb: &[&str], dir: &std::path::Path| { let mut c = self.bin.cmd(); c.args(["dev", verb[0]]).args(&verb[1..]).args(["--instance", &inst, "--state-dir"]).arg(dir); c }; let up = |dir: std::path::PathBuf| async move { let _ = std::fs::create_dir_all(&dir); let mut c = cmd(&["up", "--profile", "quick", "--host-backend", "proc"], &dir); let out = tokio::time::timeout(Duration::from_secs(200), c.output()).await; let text = match out { Ok(Ok(o)) => format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)), Ok(Err(e)) => e.to_string(), Err(_) => "dev up did not return in 200 s".to_string(), }; let url = text.lines().find(|l| l.contains("\"kcpURL\"")).and_then(|l| l.split('"').nth(3)).unwrap_or_default().to_string(); (url, text) }; let ready = |url: String| async move { if url.is_empty() { return "none".to_string(); } let o = tokio::process::Command::new("curl").args(["-sk", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}"]).arg(format!("{url}/readyz")).output().await; o.map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default() }; let down = |dir: std::path::PathBuf| async move { let _ = cmd(&["down"], &dir).output().await; }; let (a, at) = up(dirs[0].clone()).await; let (b, bt) = up(dirs[1].clone()).await; let (ra, rb) = (ready(a.clone()).await, ready(b.clone()).await); down(dirs[0].clone()).await; tokio::time::sleep(Duration::from_secs(3)).await; let rb_after = ready(b.clone()).await; down(dirs[1].clone()).await; if a.is_empty() && (at.contains("unknown command") || at.contains("flag provided but not defined")) { return Ok(json!({ "skipped": "the build has no `dev up --profile quick` on the proc backend" })); } let fail = |what: &str, text: &str| json!({ "ok": false, "why": what, "log": tail(&redact_log(text)) }); Ok(match () { _ if ra != "200" => fail("instance a's kcp is not ready", &at), _ if rb != "200" => fail("instance b's kcp is not ready (it took a's devsup?)", &bt), _ if a == b => fail("a and b answer on one kcp URL", &bt), _ if rb_after != "200" => fail("dev down of a took b's kcp down", &bt), _ => json!({ "ok": true, "a": a, "b": b, "bAfterDownA": rb_after, "binary": self.bin.version }), }) } async fn devsup_race(&self, input: &Value) -> Result { let delays: Vec = input["delaysMs"] .as_array() .map(|a| a.iter().filter_map(Value::as_u64).collect()) .unwrap_or_else(|| vec![500, 1000, 2000]); let per = input["perDelay"].as_u64().unwrap_or(4); let inst = format!("race{}", std::process::id() % 100000); let state = self.dir.join("devsup-race"); let logs = self.dump.join("logs").join("devsup-race"); let _ = std::fs::create_dir_all(&state); let _ = std::fs::create_dir_all(&logs); let base = |verb: &[&str]| { let mut c = self.bin.cmd(); c.args(["dev", verb[0]]) .args(&verb[1..]) .args(["--instance", &inst, "--state-dir"]) .arg(&state); c }; let up = |label: String| { let mut c = base(&["up", "--profile", "quick"]); c.args(["--host-backend", "proc"]); (c, label) }; let down = || async { let _ = base(&["down"]).output().await; }; let run_up = |c: tokio::process::Command, label: String| async move { let mut c = c; let t = Instant::now(); let out = tokio::time::timeout(Duration::from_secs(200), c.output()).await; let secs = (t.elapsed().as_secs_f64() * 10.0).round() / 10.0; let (rc, text) = match out { Ok(Ok(o)) => ( o.status.code().unwrap_or(-1), format!( "{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr) ), ), Ok(Err(e)) => (-1, e.to_string()), Err(_) => (-2, "dev up did not return in 200 s".to_string()), }; (rc, secs, text, label) }; // the first up: a build without `dev up` (a release one) skips let (c, l) = up("first".into()); let (rc, _, text, _) = run_up(c, l).await; if rc != 0 { down().await; if text.contains("unknown command") || text.contains("flag provided but not defined") { return Ok( json!({ "skipped": "the requester build has no `dev up --profile quick` on the proc backend" }), ); } return Err(format!( "the first dev up of the race instance failed (exit {rc}): {}", tail(&text) )); } let mut iterations = vec![]; let (mut ok, mut bad) = (0u64, 0u64); for d in &delays { for i in 0..per { let mut dc = base(&["down"]) .kill_on_drop(true) .spawn() .map_err(|e| e.to_string())?; tokio::time::sleep(Duration::from_millis(*d)).await; let _ = dc.start_kill(); let _ = dc.wait().await; let (c, l) = up(format!("{d}ms-{i}")); let (rc, secs, text, label) = run_up(c, l).await; tokio::time::sleep(Duration::from_secs(6)).await; let url = text .lines() .find(|l| l.contains("\"kcpURL\"")) .and_then(|l| l.split('"').nth(3)) .unwrap_or_default() .to_string(); let code = if url.is_empty() { "none".to_string() } else { let o = tokio::process::Command::new("curl") .args(["-sk", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}"]) .arg(format!("{url}/readyz")) .output() .await; o.map(|o| String::from_utf8_lossy(&o.stdout).to_string()) .unwrap_or_default() }; let good = rc == 0 && code == "200"; if good { ok += 1; } else { bad += 1; let _ = std::fs::write(logs.join(format!("up-{label}.log")), redact_log(&text)); } let why = text .lines() .rev() .find(|l| l.contains("dev up:")) .unwrap_or_default() .to_string(); iterations.push(json!({ "delayMs": d, "exit": rc, "upSeconds": secs, "readyz": code, "ok": good, "error": if good { Value::Null } else { json!(why) } })); } } down().await; Ok(json!({ "ok": ok, "bad": bad, "iterations": iterations, "binary": self.bin.version })) } fn invocation_dir(&self, name: &str) -> PathBuf { self.request_state().join("requests").join(name) } /// The invocation's ComputeContract `contract`, read from its kcp (its instance record: a network child's under the /// network's state, a standalone one's under the invocation dir): {phase, secretsFetchedAt, secretsFetchedVia, /// conditions}. async fn request_contract(&self, name: &str) -> Result { let dir = self.invocation_dir(name); let inv: Value = serde_json::from_slice( &std::fs::read(dir.join("invocation.json")).map_err(|e| format!("{name}: invocation.json: {e}"))?, ) .map_err(|e| e.to_string())?; let instance = inv["instance"].as_str().filter(|i| !i.is_empty()).ok_or("the invocation names no instance yet")?; let record = match (inv["mode"].as_str(), &self.target.network) { (Some("network"), Some(n)) => expand(&n.state_dir).join("dev").join(&n.instance).join("apps").join("dev").join(instance), (Some("network"), None) => return Err("a network invocation on a target without a network".into()), _ => dir.join("dev").join(instance), } .join("instance.json"); let rec: Value = serde_json::from_slice(&std::fs::read(&record).map_err(|e| format!("{}: {e}", record.display()))?) .map_err(|e| e.to_string())?; let kubeconfig = rec["adminKubeconfig"].as_str().filter(|k| !k.is_empty()).ok_or("the instance record has no adminKubeconfig")?; let cluster = format!("request:{name}"); self.kube.add_cluster( &cluster, Cluster { kubeconfig: kubeconfig.to_string(), workspace: Some(rec["baseWorkspace"].as_str().filter(|w| !w.is_empty()).unwrap_or("root").to_string()), via: Default::default(), }, ); let input = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeContract", "name": "contract" }); let c = self .kube .call(atlibs_kube::GET.into(), Request { input: Some(input), ..Default::default() }) .await? .unwrap_or_default(); let st = &c["status"]; Ok(json!({ "phase": st["phase"], "receipt": st["contract"]["receipt"]["uri"], "secretsFetchedAt": st["secretsFetchedAt"], "secretsFetchedVia": st["secretsFetchedVia"], "conditions": st["conditions"] })) } /// The test's own invocation directory: whether it exists and what its summary says. fn invocation_state(&self, name: &str) -> Result { let dir = self.invocation_dir(name); let summary = std::fs::read(dir.join("summary.json")) .ok() .and_then(|b| serde_json::from_slice::(&b).ok()); // the instance state of a standalone invocation (dev/ of its dir) and any live process whose command line names the // invocation's dir (its devsup, kcp, kine, front, core) let instance_state = std::fs::read_dir(dir.join("dev")) .map(|mut d| d.next().is_some()) .unwrap_or(false); let needle = dir.display().to_string(); let live = std::fs::read_dir("/proc") .map(|d| { d.flatten() .filter(|e| { e.file_name() .to_str() .is_some_and(|n| n.parse::().is_ok()) }) .filter(|e| { std::fs::read(e.path().join("cmdline")).is_ok_and(|raw| { String::from_utf8_lossy(&raw) .replace('\0', " ") .contains(&needle) }) }) .count() }) .unwrap_or(0); Ok( json!({ "dir": needle, "exists": dir.exists(), "instanceState": instance_state, "liveProcesses": live, "summary": summary }), ) } /// Rewrites the phase of the test's own invocation summary (top level and contract): the state of a half-created /// invocation whose network application is gone. fn set_phase(&self, name: &str, phase: &str) -> Result { let p = self.invocation_dir(name).join("summary.json"); let mut v: Value = serde_json::from_slice( &std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?, ) .map_err(|e| e.to_string())?; v["phase"] = json!(phase); if v["contract"].is_object() { v["contract"]["phase"] = json!(phase); } std::fs::write( &p, serde_json::to_vec_pretty(&v).map_err(|e| e.to_string())?, ) .map_err(|e| e.to_string())?; Ok(json!({ "phase": phase })) } /// SIGKILL (or `signal`) to the process that owns the invocation `name`: ` run request ... --name ` of this /// run (not a release, proxy or ssh verb), found by its command line; the kcp child it started keeps running. fn kill_owner(&self, name: &str, signal: &str) -> Result { if !["KILL", "TERM", "INT"].contains(&signal) { return Err(format!("signal {signal}: KILL, TERM or INT")); } let state = self.request_state().display().to_string(); let mut pids = vec![]; for e in std::fs::read_dir("/proc") .map_err(|e| e.to_string())? .flatten() { let Some(pid) = e.file_name().to_str().and_then(|n| n.parse::().ok()) else { continue; }; let Ok(raw) = std::fs::read(e.path().join("cmdline")) else { continue; }; let argv: Vec<&str> = raw .split(|b| *b == 0) .filter_map(|a| std::str::from_utf8(a).ok()) .collect(); let named = argv.windows(2).any(|w| w[0] == "--name" && w[1] == name); let owner = argv.len() > 3 && argv[0] .rsplit('/') .next() .is_some_and(|b| b.starts_with("socialweb-computer")) && argv[1] == "run" && argv[2] == "request" && argv[3].starts_with("--") && argv.contains(&state.as_str()); if named && owner { pids.push(pid); } } if pids.is_empty() { return Err(format!("no process owns the invocation {name}")); } for p in &pids { let _ = std::process::Command::new("kill") .arg(format!("-{signal}")) .arg(p.to_string()) .status(); } Ok(json!({ "killed": pids })) } async fn release(&self, name: &str) -> Result { let t = Instant::now(); let (ok, out) = self.cli(&self.sub_args("release", name, &[])).await?; // what each release did and how long, in the dump (an artifact: where a slow release waited) let d = self.dump.join("logs").join(name); if std::fs::create_dir_all(&d).is_ok() { use std::io::Write; if let Ok(mut f) = std::fs::OpenOptions::new() .create(true) .append(true) .open(d.join("release.log")) { let _ = writeln!( f, "== run request release {name}: ok {ok} in {:.0} s", t.elapsed().as_secs_f64() ); let _ = f.write_all(redact_log(&out).as_bytes()); } } if !ok { return Err(format!("run request release {name}: {}", tail(&out))); } Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 })) } /// The owner node's atirc and config dir: the AtircNode's HostProcess (same host). async fn mesh_node(&self, input: &Value) -> Result<(PathBuf, String), String> { let node = str_of(input, "node")?; if let Some((bin, dir, _)) = self.extra.nodes.lock().await.get(node) { return Ok((bin.clone(), dir.display().to_string())); } let cluster = input["cluster"].as_str().ok_or("cluster is required")?; let get = |kind: &str, name: &str| { let input = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": name }); self.kube.call( atlibs_kube::GET.into(), Request { input: Some(input), ..Default::default() }, ) }; let deadline = Instant::now() + Duration::from_secs(180); loop { let o = get("AtircNode", node).await?.unwrap_or_default(); if o["status"]["phase"] == "Running" { let dir = o["status"]["configDir"] .as_str() .ok_or("AtircNode without configDir")? .to_string(); let pid = match o["status"]["runtime"]["pid"].as_i64() { Some(p) => p, None => { let hp = o["status"]["hostProcess"] .as_str() .ok_or("AtircNode without hostProcess")?; get("HostProcess", hp).await?.unwrap_or_default()["status"]["runtime"]["pid"].as_i64().ok_or("HostProcess without pid")? } }; let bin = std::fs::read_link(format!("/proc/{pid}/exe")) .map_err(|e| format!("the AtircNode runs on this host: {e}"))?; return Ok((bin, dir)); } if Instant::now() > deadline { return Err(format!( "AtircNode {node} not Running in 3 min: {}", o["status"] )); } tokio::time::sleep(Duration::from_secs(2)).await; } } async fn mesh(&self, input: &Value, args: &[&str]) -> Result { let (bin, dir) = self.mesh_node(input).await?; let out = tokio::process::Command::new(bin) .arg("mesh") .args(args) .env("ATIRC_CONFIG_DIR", dir) .env("ATP_USERNAME", &self.acct()?.handle) .stdin(Stdio::null()) .output() .await .map_err(|e| e.to_string())?; Ok( json!({ "ok": out.status.success(), "output": format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)) }), ) } async fn plc(&self, did: &str) -> Result { let base = self.target.plc.trim_end_matches('/'); let doc = self .http .get(format!("{base}/{did}")) .send() .await .map_err(|e| e.to_string())?; let status = doc.status().as_u16(); let doc: Value = doc.json().await.unwrap_or_default(); let last: Value = self .http .get(format!("{base}/{did}/log/last")) .send() .await .map_err(|e| e.to_string())? .json() .await .unwrap_or_default(); let data: Value = self .http .get(format!("{base}/{did}/data")) .send() .await .map_err(|e| e.to_string())? .json() .await .unwrap_or_default(); Ok( json!({ "docStatus": status, "lastType": last["type"], "alsoKnownAs": doc["alsoKnownAs"], "rotationKeys": data["rotationKeys"] }), ) } /// One com.atproto.repo.listRecords page of the repo's PDS, each record as {uri, createdAt} and, with `fields` (dotted /// paths into the value, e.g. receipt.uri), `value` holding only those paths; and the cursor. Never the whole value: a /// compute.vm's user_data makes a page of 100 pass the module input cap (1 MiB). async fn list_created(&self, input: &Value) -> Result { let repo = input["repo"].as_str().ok_or("repo is required")?; let collection = input["collection"].as_str().ok_or("collection is required")?; let at = testkit::Atproto::new(&self.target); let pds = at.net.pds(repo).await?; let mut q = vec![ ("repo", repo.to_string()), ("collection", collection.to_string()), ("limit", input["limit"].as_u64().unwrap_or(100).to_string()), ]; if let Some(c) = input["cursor"].as_str() { q.push(("cursor", c.to_string())); } let r = at .http .get(format!("{}/xrpc/com.atproto.repo.listRecords", pds.trim_end_matches('/'))) .query(&q) .send() .await .map_err(|e| e.to_string())?; let status = r.status(); let body = r.text().await.map_err(|e| e.to_string())?; if !status.is_success() { return Err(format!("listRecords {repo} {collection}: {status}: {}", tail(&body))); } let o: Value = serde_json::from_str(&body).map_err(|e| format!("listRecords {repo} {collection}: {e}"))?; let fields: Vec<&str> = input["fields"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); let records: Vec = o["records"] .as_array() .into_iter() .flatten() .map(|r| { let mut out = json!({ "uri": r["uri"], "createdAt": r["value"]["createdAt"] }); if !fields.is_empty() { out["value"] = project(&r["value"], &fields); } out }) .collect(); Ok(json!({ "records": records, "cursor": o["cursor"] })) } async fn handle(&self, nsid: &str, input: Value) -> Result { let what = nsid.strip_prefix(P).unwrap_or(nsid); match what { "requester.start" => self.requester_start(&input).await, "request.put" => self.request_put(&input).await, "request.ssh" => { let name = str_of(&input, "name")?; let mut args = self.sub_args("ssh", name, &[]); if let Some(t) = input["transport"].as_str() { args.extend(["--transport".to_string(), t.to_string()]); } args.extend(exec_words(&input)?); let (ok, out) = self.cli(&args).await?; Ok(json!({ "ok": ok, "output": out })) } "request.agent" => { let name = str_of(&input, "name")?; let (ok, out) = self.cli(&self.sub_args("agent", name, &[])).await?; if ok { Ok(json!({})) } else { Err(format!("run request agent: {}", tail(&out))) } } "request.summary" => self.summary(str_of(&input, "name")?), "request.release" => self.release(str_of(&input, "name")?).await, "request.state" => self.invocation_state(str_of(&input, "name")?), "request.contract" => self.request_contract(str_of(&input, "name")?).await, "request.sshTry" => { let name = str_of(&input, "name")?; let (ok, out) = self .cli(&self.sub_args("ssh", name, &["--exec", "--", "id"])) .await?; Ok( json!({ "ok": ok, "output": tail(&out), "restarted": out.contains("starting the invocation's kcp again") || out.contains("starting it again from its state dir") }), ) } "request.setPhase" => self.set_phase(str_of(&input, "name")?, str_of(&input, "phase")?), "request.removeState" => { let dir = self.invocation_dir(str_of(&input, "name")?); std::fs::remove_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; Ok(json!({ "removed": dir.display().to_string() })) } "request.releaseTry" => { let name = str_of(&input, "name")?; let t = Instant::now(); let (ok, out) = self.cli(&self.sub_args("release", name, &[])).await?; Ok( json!({ "ok": ok, "output": tail(&out), "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }), ) } "devsup.samename" => self.devsup_samename().await, "devsup.race" => self.devsup_race(&input).await, "request.kill" => self.kill_owner( str_of(&input, "name")?, input["signal"].as_str().unwrap_or("KILL"), ), "request.records" => { let name = str_of(&input, "name")?; let did = self.acct()?.did_now(); let logs = self.request_logs(name); let declined = more::declined_accepts(&logs); let mut uris: Vec = uris_in(&logs, &did) .into_iter() .filter(|u| !kept_by_rule(u) && !declined.contains(u)) .collect(); if let Ok(s) = self.summary(name) && let Some(r) = s["contract"]["receiptUri"].as_str() { uris.push(format!( "at://{did}/{}/{}", identlink::market::VM_DELETE_NSID, identlink::market::release_rkey(r) )); } Ok(json!({ "uris": uris, "declined": declined })) } "repo.listCreated" => self.list_created(&input).await, "account.envFile" => { use std::io::Write; use std::os::unix::fs::OpenOptionsExt; let p = self .dir .join(format!("atp-{}.env", self.env_files.lock().unwrap().len())); let mut f = std::fs::OpenOptions::new() .create(true) .truncate(true) .write(true) .mode(0o600) .open(&p) .map_err(|e| e.to_string())?; writeln!(f, "ATP_PASSWORD={}", self.password()?).map_err(|e| e.to_string())?; self.env_files.lock().unwrap().push(p.clone()); Ok(json!({ "path": p })) } "atirc.meshPub" => { let text = input["text"] .as_str() .ok_or("text is required")? .to_string(); self.mesh(&input, &["pub", &text]).await } "atirc.meshPeers" => self.mesh(&input, &["peers"]).await, "com.fedfork.exec.identlink.walk" => { let agent = input["agent"].as_str().ok_or("agent is required")?; // the host's client (the target's names through its front, trusting its CA): a staging network's PLC is // https by name under the run's root, which identlink's default client (built-in roots) does not trust let io = identlink::HttpIo { client: self.http.clone(), plc: self.target.plc.trim_end_matches('/').into(), ..Default::default() }; Ok( serde_json::to_value(identlink::walk(&io, agent, Default::default()).await) .unwrap(), ) } "com.fedfork.exec.identlink.plc" => { self.plc(input["did"].as_str().ok_or("did is required")?) .await } other => match self.more(other, &input).await { Some(r) => r, None => match self.knotless(other, &input).await { Some(r) => r, None => Err(format!("{nsid}: not a call of this host")), }, }, } } /// A failed run's logs into `out` (the dump's logs/), before the freeze or the cleanup, each redacted: per /// invocation `/local/` (this host's request log, its requester's dev logs) and `/vm/` (over /// request.ssh: the VM's journal and every *.log under /home and /root: core.log, dev logs, step logs), and per /// requester its log. What could not be read goes into `/errors.txt`. Each the last 512 KiB, 6 MiB in all, /// smallest first (an artifact reaches the runner through the guest agent's 16 MiB job output ring), the rest /// named in `skipped.txt`. pub async fn collect_logs(&self, out: &Path) { let mut room = LOGS_MAX; let mut skipped = vec![]; let requests: Vec<(String, PathBuf)> = self .requests .lock() .unwrap() .iter() .map(|(n, s)| (n.clone(), s.clone())) .collect(); for (name, state) in requests { let o = out.join(&name); let mut files = vec![state.join(format!("{name}-request.log"))]; walk(&state.join("requests").join(&name), &mut files); for f in by_size(files) { let to = o.join("local").join(f.strip_prefix(&state).unwrap_or(&f)); copy_redacted(&f, &to, &mut room, &mut skipped); } // only a VM the invocation held answers ssh (an invocation still renting would make it wait) let held = std::fs::read_to_string(state.join(format!("{name}-request.log"))) .is_ok_and(|l| l.contains(r#""phase":"Held""#)); let vm = match held { true => self.vm_logs(&name).await, false => Err(format!("{name}: its VM was never Held: no VM logs")), }; match vm { Ok(tree) => { let mut files = vec![]; walk(&tree, &mut files); for f in by_size(files) { let to = o.join("vm").join(f.strip_prefix(&tree).unwrap_or(&f)); copy_redacted(&f, &to, &mut room, &mut skipped); } let _ = std::fs::remove_dir_all(&tree); } Err(e) => { let _ = std::fs::create_dir_all(&o); let _ = std::fs::write(o.join("errors.txt"), redact_log(&e)); } } } for (name, (state, _)) in self.requesters.lock().await.iter() { let f = state.join(format!("{name}.log")); let to = out .join(format!("requester-{name}")) .join(format!("{name}.log")); copy_redacted(&f, &to, &mut room, &mut skipped); } if !skipped.is_empty() { let _ = std::fs::write(out.join("skipped.txt"), skipped.join("\n") + "\n"); } } /// The VM of invocation `name`: its logs as a tar.gz (base64 between markers: the CLI's own lines share the /// output), unpacked into a dir of the work dir. async fn vm_logs(&self, name: &str) -> Result { const LOGS: &str = "cd / && journalctl --no-pager -o short-iso -n 20000 > root/swc-journal.log 2>&1; \ find home root -xdev -type f -name '*.log' -size -64M > root/swc-logs.list 2>/dev/null; echo SWC-LOGS-BEGIN; \ tar -czf - -T root/swc-logs.list 2>/dev/null | base64 -w0; echo; echo SWC-LOGS-END; rm -f root/swc-journal.log root/swc-logs.list"; let (_, text) = tokio::time::timeout( Duration::from_secs(300), self.cli(&[self.sub_args("ssh", name, &[]), exec_line(LOGS)].concat()), ) .await .map_err(|_| format!("{name}: the VM's logs not read in 5 min"))??; self.unpack_logs(&format!("logs-{name}"), &text).await } /// A tar.gz the VM printed as base64 between SWC-LOGS-BEGIN and SWC-LOGS-END, unpacked into `/`. pub(crate) async fn unpack_logs(&self, key: &str, text: &str) -> Result { let name = key; let b64 = text .split_once("SWC-LOGS-BEGIN\n") .and_then(|(_, r)| r.split_once("\nSWC-LOGS-END")) .map(|(b, _)| b.trim().to_string()) .ok_or_else(|| format!("{name}: no logs from the VM: {}", tail(text)))?; let (file, tree) = ( self.dir.join(format!("{key}.b64")), self.dir.join(key), ); std::fs::write(&file, b64).map_err(|e| e.to_string())?; std::fs::create_dir_all(&tree).map_err(|e| e.to_string())?; let out = tokio::process::Command::new("sh") .args(["-c", "base64 -d < \"$1\" | tar -xzf - -C \"$2\"", "sh"]) .arg(&file) .arg(&tree) .output() .await .map_err(|e| e.to_string())?; let _ = std::fs::remove_file(&file); if !out.status.success() { return Err(format!( "{name}: unpacking the VM's logs: {}", String::from_utf8_lossy(&out.stderr) )); } Ok(tree) } /// Wait (5 min) until the run requester `name` holds no ComputeContract that is not Released or Failed. async fn requester_contracts_released(&self, name: &str) -> Result<(), String> { let input = json!({ "cluster": format!("requester:{name}"), "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeContract" }); let deadline = Instant::now() + Duration::from_secs(300); loop { let listed = self .kube .call( atlibs_kube::LIST.into(), Request { input: Some(input.clone()), ..Default::default() }, ) .await; let left: Vec = match &listed { Ok(o) => o .as_ref() .and_then(|o| o["items"].as_array()) .into_iter() .flatten() .filter(|c| { !matches!(c["status"]["phase"].as_str(), Some("Released" | "Failed")) }) .filter_map(|c| c["metadata"]["name"].as_str().map(String::from)) .collect(), Err(_) => vec![], }; if left.is_empty() { return Ok(()); } if Instant::now() > deadline { return Err(format!( "run requester {name}: contracts not released 5 min after cleanup: {}", left.join(", ") )); } tokio::time::sleep(Duration::from_secs(5)).await; } } /// Release every invocation, wait for each run requester's contracts, stop every requester (Ctrl-C), remove the /// password env files. The errors. pub async fn cleanup(&self) -> Vec { let mut errors = self.knotless_cleanup().await; errors.extend(self.more_cleanup().await); let names: Vec = self.requests.lock().unwrap().keys().cloned().collect(); // every invocation at once (a multi-VM run's four, a guest requester's slow ones), each said with its time let released = futures::future::join_all(names.iter().map(|n| async move { let t = Instant::now(); // an owner that is settling a release of its own (0005 B's out-of-band loss) stops its kcp under our ask: // ask again once it is gone let mut r = self.release(n).await; // a Failed invocation the run released already leaves no invocation.json: nothing is left to release // an invocation a release process took over (its owner was killed) is released and its kcp application gone if r.as_ref().is_err_and(|e| { e.contains(&format!("no invocation {n} under")) || e.contains(&format!("the market network has no application {n}")) }) { r = Ok(json!({ "gone": true })); } for _ in 0..3 { if r.is_ok() { break; } tokio::time::sleep(Duration::from_secs(20)).await; r = self.release(n).await; } eprintln!( "cleanup: {n} released in {:.0} s{}", t.elapsed().as_secs_f64(), r.as_ref() .err() .map(|e| format!(": {e}")) .unwrap_or_default() ); r })) .await; errors.extend(released.into_iter().filter_map(Result::err)); // a run requester releases the contracts of the Infra/EphemeralInfra objects kube.cleanup just deleted only // while it runs: wait for them before the Ctrl-C (it leaves Held VMs to adopt), bounded let live: Vec = self.requesters.lock().await.keys().cloned().collect(); let settled = futures::future::join_all(live.iter().map(|n| self.requester_contracts_released(n))) .await; errors.extend(settled.into_iter().filter_map(Result::err)); for (name, (_, mut c)) in self.requesters.lock().await.drain() { if let Ok(None) = c.try_wait() { if let Some(pid) = c.id() { let _ = std::process::Command::new("kill") .args(["-INT", &pid.to_string()]) .status(); } if tokio::time::timeout(Duration::from_secs(120), c.wait()) .await .is_err() { let _ = c.kill().await; errors.push(format!("run requester {name} did not stop in 120 s")); } } } for p in self.env_files.lock().unwrap().drain(..) { let _ = std::fs::remove_file(p); } errors } } impl Call for Swc { fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result, String>> { Box::pin(async move { let r = self.handle(&nsid, input_of(&request)).await.map(Some); self.write_resources().await; r }) } fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result> { Box::pin(async move { let input = input_of(&request); match nsid.strip_prefix(P) { Some("request.start") => { let r = self.request_start(&input).await; self.write_resources().await; r } Some("atirc.meshSub") => { let (bin, dir) = self.mesh_node(&input).await?; let mut child = tokio::process::Command::new(bin) .args(["mesh", "sub", "--cursor", "0"]) .env("ATIRC_CONFIG_DIR", dir) .env("ATP_USERNAME", &self.acct()?.handle) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .kill_on_drop(true) .spawn() .map_err(|e| e.to_string())?; let out = child.stdout.take().unwrap(); let (tx, messages) = mpsc::unbounded_channel(); let (commands, mut cmds) = mpsc::unbounded_channel::(); tokio::spawn(async move { use tokio::io::AsyncBufReadExt; let mut lines = tokio::io::BufReader::new(out).lines(); loop { tokio::select! { l = lines.next_line() => match l { Ok(Some(l)) => if let Ok(v) = serde_json::from_str::(&l) && tx.send(Ok(v)).is_err() { break }, _ => break, }, cmd = cmds.recv() => match cmd { Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("meshSub takes no sends".into())); } Some(SubCmd::Close) | None => break, }, } } let _ = child.kill().await; }); Ok(Sub { messages, commands }) } Some(other) => match self.more_sub(other, &input).await { Some(r) => r, None => Err(format!("{nsid} is not a subscription of this host")), }, None => Err(format!("{nsid} is not a subscription of this host")), } }) } } /// Every regular *.log file under `dir` (never a symlink, a kubeconfig, a key or a state file). fn walk(dir: &Path, out: &mut Vec) { for e in std::fs::read_dir(dir).into_iter().flatten().flatten() { let (p, Ok(t)) = (e.path(), e.file_type()) else { continue; }; if t.is_dir() { walk(&p, out); } else if t.is_file() && p.extension().is_some_and(|x| x == "log") { out.push(p); } } } /// A failed run's logs/ in all, and each log's last bytes kept. const LOGS_MAX: u64 = 6 << 20; const LOG_MAX: usize = 512 << 10; fn by_size(mut files: Vec) -> Vec { files.sort_by_key(|f| f.metadata().map(|m| m.len()).unwrap_or(0)); files } /// The last LOG_MAX bytes of a log, redacted, at `to` (0600), while `room` lasts; else `to` goes into `skipped`. fn copy_redacted(from: &Path, to: &Path, room: &mut u64, skipped: &mut Vec) { let Ok(bytes) = std::fs::read(from) else { return; }; let tail = &bytes[bytes.len().saturating_sub(LOG_MAX)..]; if tail.len() as u64 > *room { skipped.push(to.display().to_string()); return; } *room -= tail.len() as u64; let text = String::from_utf8_lossy(tail); if let Some(d) = to.parent() { let _ = std::fs::create_dir_all(d); } use std::io::Write; use std::os::unix::fs::OpenOptionsExt; if let Ok(mut f) = std::fs::OpenOptions::new() .create(true) .truncate(true) .write(true) .mode(0o600) .open(to) { let _ = f.write_all(redact_log(&text).as_bytes()); } } /// Words that name a secret in a log: the dump's (atlibs ticks) and a kubeconfig's, a header's, a PEM's. const SECRET_WORDS: &[&str] = &[ "password", "passwd", "token", "secret", "kubeconfig", "cookie", "authorization", "bearer", "jwt", "apikey", "api_key", "privatekey", "private_key", "private-key", "key-data", "credential", "seed", ]; /// A log redacted line by line: JSON (whole or after a prefix) as the dump is (secret-named keys) plus every string /// value scrubbed; other text cut after the first secret word's `=` or `:`; JWT-like words and PEM blocks dropped. /// The last lines of the console of every VM of the dev net `instance` started in the last 15 minutes whose console /// never printed its "reporting iroh://" line (the guest never came on network): at most 3 VMs, 40 lines each, redacted; /// the last 300 lines of each go to `dump_dir/console-.log`. fn silent_console_tails(state_dir: &Path, instance: &str, dump_dir: &Path) -> String { let apps = state_dir.join("dev").join(instance).join("apps/dev"); let mut found: Vec<(std::time::SystemTime, String, String)> = vec![]; let Ok(bidders) = std::fs::read_dir(&apps) else { return String::new(); }; for b in bidders.flatten() { let Ok(handles) = std::fs::read_dir(b.path().join("handles")) else { continue; }; for h in handles.flatten() { let Ok(vms) = std::fs::read_dir(h.path().join("vms")) else { continue; }; for vm in vms.flatten() { let console = vm.path().join("console.log"); let Ok(meta) = std::fs::metadata(&console) else { continue; }; let Ok(modified) = meta.modified() else { continue; }; if modified .elapsed() .map_or(true, |e| e > Duration::from_secs(900)) { continue; } let Ok(raw) = std::fs::read(&console) else { continue; }; let text = String::from_utf8_lossy(&raw).to_string(); if text.contains("reporting iroh://") { continue; } found.push((modified, vm.file_name().to_string_lossy().to_string(), text)); } } } found.sort_by(|a, b| b.0.cmp(&a.0)); let mut out = String::new(); for (_, name, text) in found.into_iter().take(3) { let lines: Vec<&str> = text.lines().collect(); let _ = std::fs::create_dir_all(dump_dir); let from = lines.len().saturating_sub(300); let _ = std::fs::write( dump_dir.join(format!("console-{name}.log")), redact_log(&lines[from..].join("\n")), ); let from = lines.len().saturating_sub(40); out.push_str(&format!( "== {name} (last 40 of {} lines)\n{}\n", lines.len(), redact_log(&lines[from..].join("\n")) )); } out } pub fn redact_log(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut pem = false; for l in text.lines() { if l.contains("-----BEGIN") { pem = !l.contains("-----END"); out.push_str("\n"); continue; } if pem { pem = !l.contains("-----END"); continue; } let line = match l .find('{') .map(|i| (i, serde_json::from_str::(&l[i..]))) { Some((i, Ok(v))) => format!( "{}{}", scrub(&l[..i]), scrub_strings(atlibs_host::ticks::redact(&v)) ), _ => scrub(l), }; out.push_str(&line); out.push('\n'); } out } fn scrub_strings(v: Value) -> Value { match v { Value::String(s) => Value::String(scrub(&s)), Value::Array(a) => Value::Array(a.into_iter().map(scrub_strings).collect()), Value::Object(o) => { Value::Object(o.into_iter().map(|(k, v)| (k, scrub_strings(v))).collect()) } v => v, } } fn scrub(s: &str) -> String { let lower = s.to_ascii_lowercase(); let cut = SECRET_WORDS .iter() .filter_map(|w| lower.find(w).map(|i| i + w.len())) .filter_map(|i| { lower[i..] .find(['=', ':']) .filter(|j| *j < 40) .map(|j| i + j) }) .min(); let mut s = match cut { Some(j) => format!("{}", &s[..=j]), None => s.to_string(), }; for w in s .clone() .split(|c: char| c.is_whitespace() || "\"'=,;()[]{}".contains(c)) { if w.len() > 20 && w.starts_with("eyJ") && w.matches('.').count() >= 2 { s = s.replace(w, ""); } } s } /// The command of a host call: `argv` (an array: that exact argv on the guest) or `exec` (a shell line, run as /// `bash -c `, as the guest's root login shell ran a `--exec ` before plan 0043). pub(crate) fn exec_words(input: &Value) -> Result, String> { if let Some(argv) = input["argv"].as_array() { let argv: Vec = argv .iter() .map(|w| w.as_str().map(String::from).ok_or("argv: strings")) .collect::>()?; if argv.is_empty() { return Err("argv: a command".into()); } return Ok(["--exec".to_string(), "--".to_string()].into_iter().chain(argv).collect()); } Ok(exec_line(input["exec"].as_str().ok_or("exec or argv is required")?)) }