//! swc-test-host: runs a test composition (tests/compositions/) against SWC_TARGET as an account, with //! atlibs-kube, atlibs-ssh, atlibs-iroh, identlink and the socialweb handler; the root instance `test` runs `run`. //! //! swc-test-host [--account a] [--on-fail freeze|cleanup] [--freeze-at ]... //! [--freeze-ttl ] [--dry-run] [--instance i --entry e --input ''] //! swc-test-host attach stdio MCP to a live run's control socket (a runs dir: its newest live run) //! swc-test-host cleanup from resources.json (the host may be gone) //! swc-test-host attach|cleanup --ci --key //! a frozen CI run, over the handle `knotless run attach` printed //! swc-test-host replay [--upto ] //! //! A failure (pass false, an error) freezes the run when --on-fail freeze (the default outside CI): no more events, //! every resource kept, the dump in the run dir (state.json, events.ndjson, resources.json, failure.json), the //! control socket served (MCP), until `continue`/`cleanup` or the TTL (default 2 h), then cleanup. SIGUSR1 freezes a //! live run. The readings land in tests/readings as the Rust tests' do. mod ops; mod swc; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; use anyhow::{Context, Result, anyhow}; use atlibs_host::compose::{self, Composition, ModuleLoader}; use atlibs_host::freeze::{Break, Freezer, Tool}; use atlibs_host::host::Fetch; use atlibs_host::run::{Call, Module, Options, Request}; use atlibs_host::store::Store; use atlibs_host::wasm::{Bound, Wasm}; use atlibs_host::{BoxFut, Signal}; use atlibs_kube::{Cluster, Kube, Via}; use serde_json::{Value, json}; use testkit::requester::Binary; use testkit::target::{KcpEndpoint, Target}; fn cluster_of(ep: &KcpEndpoint) -> Cluster { let via = match &ep.via { testkit::target::Via::Direct => Via::Direct, testkit::target::Via::Overlay { name } => Via::Overlay { name: name.clone() }, testkit::target::Via::Ssh { host } => Via::Ssh { host: host.clone() }, testkit::target::Via::Iroh { ticket } => Via::Iroh { ticket: ticket.clone(), }, testkit::target::Via::WebRtc { proxy } => Via::Proxy { command: proxy.clone(), }, }; Cluster { kubeconfig: testkit::target::expand(&ep.kubeconfig) .display() .to_string(), workspace: ep.workspace.clone(), via, } } fn wasm_path(modules: &Path, key: &str) -> PathBuf { modules .join("target/wasm32-unknown-unknown/release") .join(format!("{}.wasm", key.replace('-', "_"))) } /// A module of tests/modules by its key: module.json + the release wasm. fn loader(modules: PathBuf) -> ModuleLoader { Arc::new(move |key: String, _pin: Option| { let modules = modules.clone(); Box::pin(async move { let dir = modules.join(&key); let manifest: Value = serde_json::from_slice( &std::fs::read(dir.join("module.json")).map_err(|e| format!("{key}: {e}"))?, ) .map_err(|e| e.to_string())?; let wasm = wasm_path(&modules, &key); let bytes = std::fs::read(&wasm) .map_err(|e| format!("{}: {e} (cargo wasm in tests/modules)", wasm.display()))?; let mut m = Module::from_manifest(bytes, &manifest)?; m.id = Some(dir.display().to_string()); // a local module has no published session record: module.json `scope` stands in for its client scope m.scope = manifest["scope"] .as_str() .map(|s| (key.clone(), s.to_string())); m.key = Some(key); Ok(m) }) }) } /// One line of what a tick saw and did: the event (its id; a request's message and phase), the calls and timers it /// made (ids and NSIDs), and its end. Never a call's params or a result's output. fn progress(event: &Value, out: &Value) { let at = atlibs_host::iso_now(); let data = &event["data"]; let said = [ data["msg"].as_str(), data["phase"].as_str(), data["contractPhase"].as_str(), ] .into_iter() .flatten() .filter(|s| !s.is_empty()) .collect::>() .join(" "); let woke = format!( "{} {}", event["type"].as_str().unwrap_or("?"), event["id"].as_str().unwrap_or_default() ); let calls: Vec = (out["calls"].as_array().into_iter().flatten()) .map(|c| { format!( "{}={}", c["id"].as_str().unwrap_or("?"), c["nsid"] .as_str() .unwrap_or("?") .trim_start_matches("computer.socialweb.test.") ) }) .collect(); let timers: Vec<&str> = (out["timers"].as_array().into_iter().flatten()) .filter_map(|t| t["id"].as_str()) .collect(); let mut line = format!("{at} {}", woke.trim()); if !said.is_empty() { line += &format!(" ({said})"); } if !calls.is_empty() { line += &format!(" -> {}", calls.join(" ")); } if !timers.is_empty() { line += &format!(" timers {}", timers.join(" ")); } if out["done"] == true { line += &format!(" done pass={}", out["result"]["pass"]); if let Some(e) = out["result"]["error"].as_str() { line += &format!(" error: {}", swc::redact_log(e).trim_end()); } } println!("{line}"); } fn flag(args: &[String], f: &str) -> Option { args.iter() .position(|a| a == f) .and_then(|i| args.get(i + 1)) .cloned() } fn flags(args: &[String], f: &str) -> Vec { args.windows(2) .filter(|w| w[0] == f) .map(|w| w[1].clone()) .collect() } fn tool( name: &str, description: &str, schema: Value, f: impl Fn(Value) -> BoxFut<'static, Result> + Send + Sync + 'static, ) -> Tool { Tool { name: name.into(), description: description.into(), schema, call: Arc::new(f), } } /// A frozen CI run, from the attach handle `knotless run attach` printed ({ticket, hostKeys}): ssh over iroh with /// the private `key` whose public half the runner gave the CI VM, which runs only the workflow's `attach` (this /// host's attach to the frozen run there). `cleanup`: its MCP `cleanup` tool, the run's cleanup and release. async fn ci_session(handle: &str, key: &str, cleanup: bool) -> Result { use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; let h: Value = serde_json::from_slice( &std::fs::read(handle).context("the attach handle (knotless run attach > )")?, )?; let ticket = h["ticket"].as_str().context("the handle has no ticket")?; let dir = testkit::home_tmp(&format!("attach-{}", std::process::id())); std::fs::create_dir_all(&dir)?; let kh = dir.join("known_hosts"); std::fs::write( &kh, atlibs_iroh::known_hosts("guest", h.get("hostKeys")).map_err(|e| anyhow!(e))?, )?; let mut ssh = tokio::process::Command::new("ssh"); ssh.args(atlibs_iroh::ssh_args(Path::new(key), ticket, &kh, "attach")); let ok = if !cleanup { ssh.status().await?.success() } else { let mut c = ssh .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .spawn()?; let mut sin = c.stdin.take().context("stdin")?; let call = json!({ "jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": { "name": "cleanup", "arguments": {} } }); let init = json!({ "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} }); sin.write_all(format!("{init}\n{call}\n").as_bytes()) .await?; let mut lines = tokio::io::BufReader::new(c.stdout.take().context("stdout")?).lines(); let mut ok = false; while let Some(l) = lines.next_line().await? { let v: Value = serde_json::from_str(&l).unwrap_or_default(); if v["id"] == 2 { println!("{l}"); ok = v["result"]["isError"] == false; break; } } drop(sin); let _ = c.wait().await; ok }; let _ = std::fs::remove_dir_all(&dir); Ok(ok) } /// The control socket of `dir`: a run dir's, or the newest run's under a runs dir (SWC_RUNS) that serves one. fn control_socket(dir: &Path) -> Result { if dir.join("control.sock").exists() { return Ok(dir.join("control.sock")); } std::fs::read_dir(dir)? .flatten() .map(|e| e.path().join("control.sock")) .filter(|s| s.exists()) .max_by_key(|s| s.metadata().and_then(|m| m.modified()).ok()) .with_context(|| format!("{}: no run serving a control socket", dir.display())) } async fn run_dir_cmd(cmd: &str, args: &[String]) -> Result { if let Some(handle) = flag(args, "--ci") { let key = flag(args, "--key").context("--key ")?; return ci_session(&handle, &key, cmd == "cleanup").await; } let dir = PathBuf::from(args.first().context("a run dir")?); match cmd { "attach" => { atlibs_host::freeze::attach(&control_socket(&dir)?) .await .map_err(|e| anyhow!(e))?; Ok(true) } "cleanup" => { let meta: Value = serde_json::from_slice(&std::fs::read(dir.join("run.json")).context("run.json")?)?; let target = Target::load()?; network_env(&target); let bin = Binary::resolve(meta["requester"].as_str().unwrap_or(&target.requester)).await?; let account = match meta["account"].as_str() { Some(a) => a, None => target.role_name(testkit::role::REQUESTER)?, }; let errors = swc::cleanup_from_disk(&dir, &target, account, &bin).await; println!("{}", json!({ "cleanupErrors": errors })); Ok(errors.is_empty()) } "replay" => { // every tick again from ticks.ndjson.zst, the whole composition in seq order; --to stops there and // prints every run's state; --modules : the wasm of another build (tests/modules by default) let meta: Value = serde_json::from_slice(&std::fs::read(dir.join("run.json")).context("run.json")?)?; // the composition where the run had it, else this checkout's of the same name (a CI run's artifacts // name the CI VM's checkout) let recorded = PathBuf::from( meta["composition"] .as_str() .context("run.json composition")?, ); let comp = match recorded.join("composition.json").exists() { true => recorded.join("composition.json"), false => testkit::repo() .join("tests/compositions") .join(recorded.file_name().context("run.json composition")?) .join("composition.json"), }; let manifest: Value = serde_json::from_slice( &std::fs::read(&comp).with_context(|| comp.display().to_string())?, )?; let modules = flag(args, "--modules") .map(PathBuf::from) .unwrap_or_else(|| testkit::repo().join("tests/modules")); let wasm = Wasm::new(5000); let mut runners: HashMap> = HashMap::new(); for inst in manifest["instances"].as_array().into_iter().flatten() { let (name, key) = ( inst["name"].as_str().unwrap_or_default(), inst["module"].as_str().unwrap_or_default(), ); let bytes = std::fs::read(wasm_path(&modules, key)) .with_context(|| format!("{key}: the wasm (cargo wasm)"))?; runners.insert( name.to_string(), Arc::new(Bound { wasm: wasm.clone(), bytes: Arc::new(bytes), sha256: None, }), ); } let records = atlibs_host::ticks::read(&dir).map_err(|e| anyhow!(e))?; let to = flag(args, "--to").and_then(|s| s.parse().ok()); let mut r = atlibs_host::ticks::replay(&records, to, |i| runners.get(i).cloned(), 1 << 20) .await .map_err(|e| anyhow!(e))?; if let Some(seq) = to { r["statesAt"] = Value::Object(atlibs_host::ticks::states_at(&records, seq)); } println!("{}", serde_json::to_string_pretty(&r)?); Ok(r["firstDiverge"].is_null()) } _ => unreachable!(), } } /// A module's atproto calls (atlibs-host) read DIDs at ATLIBS_PLC_URL; a staging or dev network's names are private /// (`*.relay.localhost` behind its front), so a non-prod target sets ATLIBS_ALLOW_PRIVATE=1 too. Before any thread /// reads the environment. fn network_env(t: &Target) { unsafe { std::env::set_var("ATLIBS_PLC_URL", &t.plc); if t.name != "prod" { std::env::set_var("ATLIBS_ALLOW_PRIVATE", "1"); } } } async fn main_() -> Result { let args: Vec = std::env::args().skip(1).collect(); if let Some(cmd) = args .first() .filter(|c| ["attach", "cleanup", "replay"].contains(&c.as_str())) { return run_dir_cmd(cmd, &args[1..]).await; } let comp_dir = PathBuf::from( args.first() .context("usage: swc-test-host | attach|cleanup|replay ")?, ); let dry = args.iter().any(|a| a == "--dry-run"); let freeze_on_fail = flag(&args, "--on-fail").map_or(std::env::var("CI").is_err(), |v| v == "freeze"); let ttl = Duration::from_secs( flag(&args, "--freeze-ttl") .and_then(|s| s.parse().ok()) .unwrap_or(7200), ); let breaks: Vec = flags(&args, "--freeze-at") .iter() .map(|s| Break::parse(s)) .collect(); let tests = testkit::repo().join("tests"); let manifest: Value = serde_json::from_slice( &std::fs::read(comp_dir.join("composition.json")).context("composition.json")?, )?; let runbook = comp_dir.file_name().unwrap().to_string_lossy().to_string(); let target = Target::load()?; network_env(&target); // no --account: the target's requester (a composition runs as the requester under test) let account = match flag(&args, "--account") { Some(a) => a, None => target.role_name(testkit::role::REQUESTER)?.to_string(), }; let acc = target.account(&account)?; let bin = Binary::resolve(&target.requester).await?; let comp = compose::load(&manifest, &loader(tests.join("modules"))) .await .map_err(|e| anyhow!(e))?; // two dirs per run: the dump (SWC_RUNS: ticks, state, failure, resources, run.json; no secret, a CI artifact) and // the work dir (requester state, kubeconfigs, keys, the store: secrets, never uploaded) std::fs::create_dir_all(testkit::runs_dir())?; let mut ms = atlibs_host::now_ms(); while std::fs::create_dir(testkit::runs_dir().join(format!("{runbook}-composition-{ms}"))).is_err() { ms += 1; } let run_name = format!("{runbook}-composition-{ms}"); let dump = testkit::runs_dir().join(&run_name); let dir = testkit::home_tmp("work").join(&run_name); for d in [&dump, &dir] { std::fs::create_dir_all(d)?; std::fs::set_permissions(d, std::os::unix::fs::PermissionsExt::from_mode(0o700))?; } // transports: kube (bidder and gateway kcps; requesters add theirs), ssh, iroh, the socialweb handler let receipts: Arc>> = Default::default(); let associated: Arc>> = Default::default(); let mut clusters = HashMap::new(); for b in &target.bidders { if let Some(ep) = &b.kcp { clusters.insert(format!("bidder:{}", b.name), cluster_of(ep)); } } if let Some(ep) = &target.gateway.kcp { clusters.insert("gateway".into(), cluster_of(ep)); } let rs = receipts.clone(); let assoc = associated.clone(); // fault injection reaches only this run's own VMs; a bidder root's spec is patched only to be restored, and so is a // run requester's OpenBaoServer (staging.pki declares the run's pki mount there, plan 0050) let guard: atlibs_kube::Guard = Arc::new(move |cluster: &str, verb: &str, obj: &Value| { match (verb, obj["kind"].as_str()) { ("delete", Some("ComputeVM" | "FirecrackerVM" | "QemuVM")) => { obj["spec"]["contract"]["receipt"]["uri"] .as_str() .is_some_and(|r| rs.lock().unwrap().iter().any(|x| x == r)) || obj["kind"] == "ComputeVM" && obj["spec"]["requester"]["did"] .as_str() .is_some_and(|d| assoc.lock().unwrap().iter().any(|x| x == d)) } ("patch", Some("MarketBidder")) => true, ("patch", Some("OpenBaoServer")) => cluster.starts_with("requester:"), _ => false, } }); let kube = Kube::new(clusters, Some(guard)); // a bidder the target reaches only through its read-only ops routes is the kube cluster bidder: all the same let ops = ops::Ops::load(&target).map_err(|e| anyhow!(e))?; let kube_call: Arc = Arc::new(ops::OpsKube { ops: ops.clone(), kube: kube.clone() }); let ssh = atlibs_ssh::Ssh::new(HashMap::new(), HashMap::new()); let swc = swc::Swc::new( Target::load()?, &account, bin.clone(), dir.clone(), kube.clone(), receipts, associated, ssh.clone(), dump.clone(), ops.clone(), ); let transports = vec![ atlibs_host::transport::Transport { provides: atlibs_kube::NSIDS, call: kube_call.clone() }, ssh.transport(), // SWC_IROH_KEY: a key of its own per concurrent run (gate.sh's slots); two runs on one key are one iroh node atlibs_iroh::transport( std::env::var_os("SWC_IROH_KEY") .map(PathBuf::from) .unwrap_or_else(|| testkit::home_tmp("iroh_ed25519")), ), swc.transport(), ]; let provides = Arc::new(atlibs_host::transport::provides(&transports)); compose::check(&comp, &provides).map_err(|e| anyhow!(e))?; // a required hook outside com.fedfork.exec.* needs an edge at run time (host transports serve only that prefix) for (name, m) in &comp.modules { for hk in &m.hooks { let routed = comp.manifest["edges"] .as_array() .into_iter() .flatten() .any(|e| e["from"] == name.as_str() && e["nsid"] == hk.nsid.as_str()); if hk.required && !hk.nsid.starts_with("com.fedfork.exec.") && !routed { anyhow::bail!( "instance {name}: required hook {} has no edge (a host call is a call, not a hook)", hk.nsid ); } } } // a dry run touches no account, and the guest requester has no creds: they are read only to run as one if !dry && !acc.guest_requester { acc.password()?; } if dry { println!( "dry run {runbook}: composition ok ({} instances), requester {}", comp.modules.len(), bin.version ); return Ok(true); } let instance = flag(&args, "--instance").unwrap_or_else(|| "test".into()); let entry = flag(&args, "--entry").unwrap_or_else(|| "run".into()); let signed_in = flag(&args, "--instance").is_none() && !acc.guest_requester; let root_key = manifest["instances"] .as_array() .into_iter() .flatten() .find(|i| i["name"] == instance.as_str()) .and_then(|i| i["module"].as_str()) .unwrap_or_default() .to_string(); std::fs::write( dump.join("run.json"), serde_json::to_string_pretty( &json!({ "composition": comp_dir.canonicalize()?, "account": account, "requester": target.requester, "rootModule": root_key, "instance": instance, "entry": entry, "work": dir }), )?, )?; let _lock = if signed_in { Some(testkit::lock_account(&acc.handle).await?) } else { None }; let store = Store::open(&dir.join("store"), "swc-test-host").map_err(|e| anyhow!(e))?; let (session, fetch): (_, Arc) = if signed_in { let at = testkit::Atproto::new(&target); (Some(at.session(acc).await?), Arc::new(at.http.clone())) } else { (None, Arc::new(testkit::atproto::http_client(&target))) }; let mut calls = HashMap::new(); for (name, module) in &comp.modules { calls.insert( name.clone(), atlibs_host::registry::module_call( &store, fetch.clone(), session.clone(), &transports, module, &format!("{runbook}#{name}"), ) .await .map_err(|e| anyhow!(e))?, ); } let call_for: compose::CallFor = Arc::new(move |n: &str| calls.get(n).cloned()); let comp = Arc::new(Composition { manifest: comp.manifest.clone(), modules: comp.modules.clone(), }); let wasm = Wasm::new(5000); // freeze: the gate, the recorder, SIGUSR1, the control socket (MCP) with this host's tools // every tick of every instance (ticks.ndjson.zst), a full state each 50 ticks of a run let ticks = atlibs_host::ticks::Ticks::new(&dump, 50); let freezer = Freezer::new(&dump, breaks, Some(ticks.clone())); let signal = Signal::new(); { let f = freezer.clone(); tokio::spawn(async move { if let Ok(mut usr1) = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::user_defined1()) { while usr1.recv().await.is_some() { f.freeze("SIGUSR1"); } } }); } let (cleaned, clean_now) = (Arc::new(tokio::sync::Notify::new()), signal.clone()); // SIGTERM or SIGINT (a CI run cancelled or out of time: its workflow's `stop`): abort the run, no freeze, and // the cleanup releases everything it started before the process exits { let (f, sig, c) = (freezer.clone(), signal.clone(), cleaned.clone()); tokio::spawn(async move { use tokio::signal::unix::{SignalKind, signal}; let (Ok(mut term), Ok(mut int)) = ( signal(SignalKind::terminate()), signal(SignalKind::interrupt()), ) else { return; }; tokio::select! { _ = term.recv() => {}, _ = int.recv() => {} } eprintln!( "swc-test-host: stopping: the run is aborted and everything it started released" ); sig.abort("stopped (SIGTERM)"); f.thaw(); c.notify_waiters(); }); } let host_tools = { let (s, k, c1, cf, comp2, w2, cl, sig) = ( swc.clone(), kube.clone(), cleaned.clone(), call_for.clone(), comp.clone(), wasm.clone(), clean_now.clone(), signal.clone(), ); let fz = freezer.clone(); let pv2 = provides.clone(); let kget = kube_call.clone(); let klist = kube_call.clone(); let klogs = k.clone(); let s2 = swc.clone(); let s3 = swc.clone(); let s4 = swc.clone(); let obj = json!({ "type": "object", "properties": { "cluster": { "type": "string" }, "apiVersion": { "type": "string" }, "kind": { "type": "string" }, "name": { "type": "string" }, "namespace": { "type": "string" } }, "required": ["cluster", "apiVersion", "kind"] }); vec![ tool("resources", "What the run started and created: requesters, invocations (receipt, winner, ssh), kube objects, clusters, records, keys.", json!({ "type": "object", "properties": {} }), move |_| { let s = s.clone(); Box::pin(async move { Ok(s.resources().await) }) }), tool("kube_get", "Get one object of any kcp or cluster of the run.", obj.clone(), move |a| { let k = kget.clone(); Box::pin(async move { k.call(atlibs_kube::GET.into(), Request { input: Some(a), ..Default::default() }).await.map(Option::unwrap_or_default) }) }), tool("kube_list", "List a kind on a kcp or cluster of the run.", obj.clone(), move |a| { let k = klist.clone(); Box::pin(async move { k.call(atlibs_kube::LIST.into(), Request { input: Some(a), ..Default::default() }).await.map(Option::unwrap_or_default) }) }), tool("kube_logs", "A pod's log on a tenant cluster: {cluster, namespace, pod}.", json!({ "type": "object", "properties": { "cluster": {}, "namespace": {}, "pod": {} }, "required": ["cluster", "pod"] }), move |a| { let k = klogs.clone(); Box::pin(async move { let path = format!("/api/v1/namespaces/{}/pods/{}/log", a["namespace"].as_str().unwrap_or("default"), a["pod"].as_str().unwrap_or_default()); k.call(atlibs_kube::RAW.into(), Request { input: Some(json!({ "cluster": a["cluster"], "path": path })), ..Default::default() }).await.map(Option::unwrap_or_default) }) }), tool("ssh_exec", "Run a command in a run's VM: {name (the run request invocation), exec, transport?}.", json!({ "type": "object", "properties": { "name": {}, "exec": {}, "transport": {} }, "required": ["name", "exec"] }), move |a| { let s = s2.clone(); Box::pin(async move { s.call("computer.socialweb.test.request.ssh".into(), Request { input: Some(a), ..Default::default() }).await.map(Option::unwrap_or_default) }) }), tool("requester_ssh", "Run a command in a Held contract's VM of a run requester: {name, contract, exec, transport?}.", json!({ "type": "object", "properties": { "name": {}, "contract": {}, "exec": {}, "transport": {} }, "required": ["name", "contract", "exec"] }), move |a| { let s = s3.clone(); Box::pin(async move { s.call("computer.socialweb.test.requester.ssh".into(), Request { input: Some(a), ..Default::default() }).await.map(Option::unwrap_or_default) }) }), tool("record_get", "Read a record {uri} (public, the PLC the target names).", json!({ "type": "object", "properties": { "uri": { "type": "string" } }, "required": ["uri"] }), move |a| { let s = s4.clone(); Box::pin(async move { let at = testkit::Atproto::new(&s.target); at.record(a["uri"].as_str().unwrap_or_default()).await.map(|v| json!({ "value": v })).map_err(|e| e.to_string()) }) }), tool("run_assert", "Run an entry of the composition's assert instance: {entry, input}.", json!({ "type": "object", "properties": { "entry": {}, "input": {} }, "required": ["entry"] }), move |a| { let (cf, comp, w, pv) = (cf.clone(), comp2.clone(), w2.clone(), pv2.clone()); Box::pin(async move { let input = a["input"].as_object().cloned().unwrap_or_default(); compose::run_compose(comp, w, None, "assert", a["entry"].as_str().unwrap_or_default(), input, cf, HashMap::new(), None, Options { provides: Some(pv), ..Default::default() }).await }) }), tool("cleanup", "End the run now: abort it if it runs, release and delete everything, stop serving.", json!({ "type": "object", "properties": {} }), move |_| { let (c1, cl, fz) = (c1.clone(), cl.clone(), fz.clone()); Box::pin(async move { cl.abort("cleanup from the control socket"); fz.thaw(); c1.notify_waiters(); Ok(json!({ "cleaning": true })) }) }), ] .into_iter() .chain(std::iter::once(tool("thaw", "Same as continue.", json!({ "type": "object", "properties": {} }), { let f = freezer.clone(); move |_| { f.thaw(); Box::pin(async { Ok(json!({ "thawed": true })) }) } }))) .chain(std::iter::once(tool("abort", "Abort the run (cleanup follows).", json!({ "type": "object", "properties": {} }), move |_| { sig.abort("aborted from the control socket"); Box::pin(async { Ok(json!({ "aborted": true })) }) }))) .collect::>() }; let server = tokio::spawn(freezer.clone().serve(dump.join("control.sock"), host_tools)); // resources.json every few seconds (kube writes do not pass the socialweb handler); a breakpoint freeze past // its TTL aborts the run let ticker = { let (s, f, sig) = (swc.clone(), freezer.clone(), signal.clone()); tokio::spawn(async move { let mut frozen_for = Duration::ZERO; loop { tokio::time::sleep(Duration::from_secs(5)).await; s.write_resources().await; frozen_for = if f.frozen().is_some() { frozen_for + Duration::from_secs(5) } else { Duration::ZERO }; if frozen_for >= ttl { sig.abort("the freeze TTL ran out"); f.thaw(); } } }) }; eprintln!( "run dir: {} (attach: swc-test-host attach {})", dump.display(), dump.display() ); let options = Options { signal: Some(signal.clone()), inputs: freezer.inputs(), budget: Some( json!({ "calls": "unlimited", "events": "unlimited", "wallMs": "unlimited", "subscriptions": 8, "inFlight": 12 }), ), provides: Some(provides), gate: Some(freezer.gate()), tick_for: Some(ticks.tick_for()), on_event: { // and a progress line per tick on stdout, as the run goes (a CI step's log): names only, never params let frozen = freezer.on_event(); Some(Arc::new(move |event: &Value, out: &Value| { frozen(event, out); progress(event, out); })) }, http: { // com.fedfork.exec.http.fetch: the target's own origins only let mut allow = serde_json::Map::new(); for o in [ target.atirc.clone(), Some(target.gateway.url.clone()), Some(target.plc.clone()), Some(target.gateway.ssh_web()), ] .into_iter() .flatten() { allow.insert(o.trim_end_matches('/').to_string(), json!({})); } Some(Arc::new( atlibs_host::run::Http::from_json(fetch.clone(), &Value::Object(allow)) .map_err(|e| anyhow!(e))?, )) }, ..Default::default() }; let started = atlibs_host::iso_now(); let name = format!( "st-{runbook}-{}-{}", atlibs_host::now_ms() / 1000, std::process::id() % 10000 ); // what a test module knows of the target: no credential, no path let bidders: Vec = target .bidders .iter() .map(|b| { // write: the host has the bidder's kcp (operator access); false: only its read-only ops routes (reads as the same cluster) let reads = b.kcp.is_some() || ops.as_ref().is_some_and(|o| o.serves(&b.name)); json!({ "name": b.name, "did": b.did, "operator": b.operator, "cluster": reads.then(|| format!("bidder:{}", b.name)), "write": b.kcp.is_some() }) }) .collect(); let mut input: serde_json::Map = json!({ "did": acc.did, "handle": acc.handle, "name": name, "target": target.name, "bidders": bidders, "gateway": { "url": target.gateway.url, "aud": target.gateway.aud, "ssh": target.gateway.ssh, "sshWeb": target.gateway.ssh_web(), "requesterDid": target.gateway.requester_did, "cluster": target.gateway.kcp.as_ref().map(|_| "gateway") }, "atirc": target.atirc, "plc": target.plc, "pds": target.pds, "requester": target.requester, "restartSshMaxS": target.restart_ssh_max_s, "perHost": args.iter().any(|a| a == "--per-host"), "guestRequester": acc.guest_requester, }) .as_object() .unwrap() .clone(); if let Some(i) = flag(&args, "--input") { // the root run's --input adds to what it knows of the target (`{"suite": false}`); another entry's is all of it let given: serde_json::Map = serde_json::from_str(&i)?; match signed_in || acc.guest_requester { true if flag(&args, "--instance").is_none() => input.extend(given), _ => input = given, } } let ran = compose::run_compose( comp, wasm, Some(store.ledger("managed_records")), &instance, &entry, input, call_for, HashMap::new(), None, options, ) .await; let result = match &ran { // the root run's result says pass, whoever runs it (a guest requester has no sign-in) Ok(v) if flag(&args, "--instance").is_none() => { let r = v.get("result").unwrap_or(v); if r.get("pass").is_some() { r.clone() } else { // a trapped or crashed module leaves no result: its error (else the whole run doc) is the failure let error = v.get("error").filter(|e| !e.is_null()).cloned(); json!({ "pass": false, "error": error.unwrap_or_else(|| json!(format!("the module returned no result: {v}"))), "result": v }) } } Ok(v) => { json!({ "pass": v.get("error").is_none_or(Value::is_null), "error": v.get("error"), "result": v }) } Err(e) => json!({ "pass": false, "error": e }), }; let failed = result["pass"] != true; ticks.flush(); swc.write_resources().await; if failed { swc.collect_logs(&dump.join("logs")).await; eprintln!("logs: {}", dump.join("logs").display()); } if failed && freeze_on_fail && signal.reason().is_none() { freezer.set_failure( json!({ "error": result.get("error"), "phase": result.get("phase"), "result": result }), ); freezer.freeze("failure"); eprintln!( "FROZEN: {} (attach: swc-test-host attach {}; thaws in {:?})", result["error"], dump.display(), ttl ); tokio::select! { _ = freezer.wait_thawed(ttl) => {} _ = cleaned.notified() => {} } } // the run's kube objects first (a tenant cluster's pods, so its nodes scale down), then invocations and requesters let mut cleanup = kube.cleanup().await; cleanup.extend(swc.cleanup().await); swc.write_resources().await; ticker.abort(); server.abort(); let _ = std::fs::remove_file(dump.join("control.sock")); let pass = !failed && cleanup.is_empty(); let mut readings = result["readings"].clone(); if !readings.is_object() { readings = json!({}); } readings["requester"] = json!({ "spec": target.requester, "version": bin.version }); let doc = json!({ "runbook": runbook, "test": "composition", "target": target.name, "startedAt": started, "endedAt": atlibs_host::iso_now(), "pass": pass, "error": result.get("error").map(|e| e.as_str().map_or_else(|| e.clone(), |s| json!(swc::redact_log(s).trim_end()))), "phase": result.get("phase"), "cleanupErrors": cleanup, "readings": readings, "dump": dump, "work": dir, }); if signed_in { let out = std::env::var("SWC_READINGS") .map(PathBuf::from) .unwrap_or_else(|_| tests.join("readings")); std::fs::create_dir_all(&out)?; let file = out.join(format!( "{runbook}-composition-{}.json", started.replace([':', '.'], "") )); std::fs::write(&file, serde_json::to_string_pretty(&doc)?)?; eprintln!("readings: {}", file.display()); } println!("{}", serde_json::to_string_pretty(&doc)?); Ok(pass) } #[tokio::main] async fn main() -> std::process::ExitCode { atlibs_host::tls(); match main_().await { Ok(true) => std::process::ExitCode::SUCCESS, Ok(false) => std::process::ExitCode::FAILURE, Err(e) => { eprintln!("swc-test-host: {e:#}"); std::process::ExitCode::from(2) } } }