0
0
//! computer.socialweb.test.*: the socialweb.computer side of the test host. The requester under test (a release, a
//! commit or a local build) as the account: `run requester` (the orchestrator, its kcp registered as kube cluster
//! `requester:<name>`), `run request` invocations (a subscription of their log events), the owner's atirc mesh node.
//! Every requester and invocation it starts is stopped or released by `cleanup()`, however the run ended.
//! com.fedfork.exec.identlink.{walk,plc}: identlink against the target's PLC.
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use atlibs_host::BoxFut;
use atlibs_host::run::{Call, Request, Sub, SubCmd};
use atlibs_host::transport::Transport;
use atlibs_kube::{Cluster, Kube};
use serde_json::{Value, json};
use testkit::requester::{Binary, SignIn, exec_line, identity_args, kept_by_rule, request_line, split_exec, tail, uris_in};
use testkit::target::{Account, Target, check_private, expand};
use tokio::sync::mpsc;
#[path = "knotless.rs"]
mod knotless;
#[path = "more.rs"]
mod more;
pub use more::cleanup_from_disk;
const P: &str = "computer.socialweb.test.";
/// Where the guest requester reads its association ref (the renter writes it), and how long it may take to come.
const R2_REF: &str = "/run/socialweb-r2/association-ref";
const R2_REF_WAIT: Duration = Duration::from_secs(15 * 60);
const PUT_RETRIES: u32 = 3;
const PUT_RETRY_BACKOFF: Duration = Duration::from_secs(10);
pub const NSIDS: &[&str] = &[
"computer.socialweb.test.requester.start",
"computer.socialweb.test.request.start",
"computer.socialweb.test.request.ssh",
"computer.socialweb.test.request.put",
"computer.socialweb.test.request.agent",
"computer.socialweb.test.request.summary",
"computer.socialweb.test.request.release",
"computer.socialweb.test.request.kill",
"computer.socialweb.test.devsup.race",
"computer.socialweb.test.devsup.samename",
"computer.socialweb.test.request.state",
"computer.socialweb.test.request.sshTry",
"computer.socialweb.test.request.setPhase",
"computer.socialweb.test.request.releaseTry",
"computer.socialweb.test.request.removeState",
"computer.socialweb.test.request.records",
"computer.socialweb.test.account.envFile",
"computer.socialweb.test.atirc.meshSub",
"computer.socialweb.test.atirc.meshRun",
"computer.socialweb.test.atirc.meshPub",
"computer.socialweb.test.atirc.meshPeers",
"computer.socialweb.test.kube.addCluster",
"computer.socialweb.test.kube.undo",
"computer.socialweb.test.page.signin",
"computer.socialweb.test.page.fetch",
"computer.socialweb.test.page.round",
"computer.socialweb.test.ssh.tempKey",
"computer.socialweb.test.atirc.observer",
"computer.socialweb.test.atirc.observerPub",
"computer.socialweb.test.atirc.observerRevoke",
"computer.socialweb.test.requester.ssh",
"computer.socialweb.test.requester.restart",
"computer.socialweb.test.requester.cluster",
"computer.socialweb.test.requester.bundle",
"computer.socialweb.test.staging.pki",
"computer.socialweb.test.logs.fetchRuns",
"computer.socialweb.test.requester.records",
"computer.socialweb.test.requester.events",
"computer.socialweb.test.root.exec",
"computer.socialweb.test.root.putBinary",
"computer.socialweb.test.release.put",
"computer.socialweb.test.release.delete",
"computer.socialweb.test.example",
"computer.socialweb.test.gateway.kubeconfig",
"computer.socialweb.test.page.kubeconfig",
"computer.socialweb.test.oauth.signin",
"computer.socialweb.test.request.r2",
"computer.socialweb.test.request.checkLogs",
"computer.socialweb.test.knotless.seed",
"computer.socialweb.test.knotless.sh",
"computer.socialweb.test.bidder.health",
"computer.socialweb.test.bidder.meminfo",
"computer.socialweb.test.bidder.logCount",
"computer.socialweb.test.vm.memory",
"computer.socialweb.test.bidder.logLines",
"computer.socialweb.test.bidder.submitAccept",
"computer.socialweb.test.bidder.restart",
"computer.socialweb.test.bidder.upgrade",
"computer.socialweb.test.bidder.guestNft",
"computer.socialweb.test.bidder.awaitLog",
"computer.socialweb.test.bidder.dumbpipe",
"computer.socialweb.test.binary.published",
"computer.socialweb.test.assets.tamper",
"computer.socialweb.test.gateway.log",
"computer.socialweb.test.repo.listCreated",
"computer.socialweb.test.request.contract",
"com.fedfork.exec.identlink.walk",
"com.fedfork.exec.identlink.plc",
];
/// `run request` flags a module may pass, before `--exec -- <command> [args...]` (the command is not checked);
/// `@agentConfig` is the target's agent config (owner-only file).
const FLAGS: &[&str] = &[
"--hold",
"--agent",
"--agent-config",
"--tunnel",
"--transport",
"--exec",
"--guest-tools",
"--associate",
"--expires-in",
"--nested-virt",
"--cpus",
"--mem",
"--image",
];
pub struct Swc {
pub target: Target,
pub account: String,
pub bin: Binary,
/// the work dir: requester state, kubeconfigs, keys (secrets; never an artifact)
pub dir: PathBuf,
/// the dump dir: resources.json beside the ticks (no secret)
pub dump: PathBuf,
pub kube: Arc<Kube>,
/// receipts of this run's contracts: the kube guard lets a test delete only VMs of these
pub receipts: Arc<Mutex<Vec<String>>>,
/// guest requester DIDs this run's own requester associated (its `requester_association_written` subject): the kube
/// guard lets a test delete their rentals too (owner, 2026-11-18: "Both")
pub associated: Arc<Mutex<Vec<String>>>,
requesters: tokio::sync::Mutex<HashMap<String, (PathBuf, tokio::process::Child)>>,
requests: Mutex<HashMap<String, PathBuf>>,
env_files: Mutex<Vec<PathBuf>>,
http: reqwest::Client,
ssh: Arc<atlibs_ssh::Ssh>,
/// the target's read-only ops routes (prod in CI), when it names any
ops: Option<Arc<crate::ops::Ops>>,
extra: more::Extra,
}
/// A call's input; a subscription's arguments come as params (strings), its structure in the `json` param.
fn input_of(r: &Request) -> Value {
if let Some(i) = &r.input {
return i.clone();
}
let params = r.params.clone().unwrap_or_default();
match params
.get("json")
.and_then(Value::as_str)
.and_then(|j| serde_json::from_str::<Value>(j).ok())
{
Some(v) => v,
None => Value::Object(params),
}
}
/// `v` reduced to the dotted `paths` it has (a.b keeps {a: {b}}), nothing else.
fn project(v: &Value, paths: &[&str]) -> Value {
let mut out = json!({});
for p in paths {
let keys: Vec<&str> = p.split('.').collect();
let Some(found) = keys.iter().try_fold(v, |at, k| at.get(*k)) else { continue };
let mut dst = &mut out;
for k in &keys[..keys.len() - 1] {
if !dst[*k].is_object() {
dst[*k] = json!({});
}
dst = &mut dst[*k];
}
dst[keys[keys.len() - 1]] = found.clone();
}
out
}
fn str_of<'a>(v: &'a Value, k: &str) -> Result<&'a str, String> {
v[k].as_str()
.filter(|s| !s.is_empty() && !s.contains(['/', ' ', '\n']))
.ok_or_else(|| format!("{k} is required (a plain name)"))
}
fn abs_dir_of<'a>(v: &'a Value, k: &str) -> Result<&'a str, String> {
v[k].as_str()
.filter(|s| {
s.strip_prefix('/').is_some_and(|rest| rest.split('/').all(|c| !c.is_empty() && c != "." && c != ".."))
&& !s.chars().any(|c| c.is_whitespace() || c.is_control() || "'\"\\$`".contains(c))
})
.ok_or_else(|| format!("{k} is required (a clean absolute path)"))
}
impl Swc {
#[allow(clippy::too_many_arguments)]
pub fn new(
target: Target,
account: &str,
bin: Binary,
dir: PathBuf,
kube: Arc<Kube>,
receipts: Arc<Mutex<Vec<String>>>,
associated: Arc<Mutex<Vec<String>>>,
ssh: Arc<atlibs_ssh::Ssh>,
dump: PathBuf,
ops: Option<Arc<crate::ops::Ops>>,
) -> Arc<Self> {
let http = testkit::atproto::http_client(&target);
Arc::new(Self {
target,
account: account.into(),
bin,
dir,
dump,
kube,
receipts,
associated,
requesters: Default::default(),
requests: Default::default(),
env_files: Default::default(),
http,
ssh,
ops,
extra: Default::default(),
})
}
pub fn transport(self: &Arc<Self>) -> Transport {
Transport {
provides: NSIDS,
call: self.clone(),
}
}
fn acct(&self) -> Result<&Account, String> {
self.target
.account(&self.account)
.map_err(|e| e.to_string())
}
fn password(&self) -> Result<String, String> {
Ok(self
.acct()?
.password()
.map_err(|e| e.to_string())?
.expose()
.to_string())
}
fn request_state(&self) -> PathBuf {
// the guest requester's `run request` holds a dev instance (its PDS, its kcp) whose sockets under
// requests/<name>/run/ must fit sun_path (107 bytes): a short dir of its own, ~/.swc/<run ms>
if self.target.network.is_none() && self.acct().is_ok_and(|a| a.guest_requester) {
let name = self
.dir
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_string();
let ms = name.rsplit('-').next().unwrap_or_default().to_string();
return PathBuf::from(std::env::var("HOME").unwrap_or_default())
.join(".swc")
.join(ms);
}
match (&self.target.network, self.associated()) {
(Some(n), _) => expand(&n.state_dir),
(None, Some(id)) => expand(&id.state_dir),
(None, None) => self.dir.join("requests"),
}
}
fn request_logs(&self, name: &str) -> Vec<PathBuf> {
let base = self.request_state().join("requests").join(name);
let mut logs = vec![
self.request_state().join(format!("{name}-request.log")),
base.join("dev").join(name).join("logs/core.log"),
base.join("logs/core.log"),
];
// `run request --network`: the invocation's requester is a child core of the network, its log there
if let Some(n) = &self.target.network {
logs.push(
expand(&n.state_dir)
.join("dev")
.join(&n.instance)
.join("apps/dev")
.join(format!("{}-{name}", n.instance))
.join("logs/core.log"),
);
}
logs
}
pub(crate) async fn cli(&self, args: &[String]) -> Result<(bool, String), String> {
self.bin
.run(&args.iter().map(String::as_str).collect::<Vec<_>>())
.await
.map_err(|e| e.to_string())
}
/// Copies one of this host's files to the guest of invocation `name` (stdin of `run request ssh --exec -- bash -c 'cat >
/// <remote>'`). `what` names the file, never a path: testHost ($SWC_TEST_HOST_STATIC, a static swc-test-host) or
/// suite (a tar.gz of the repo's examples, scripts, tests/compositions, tests/targets, did-key-associator with the four
/// typescript-helpers libraries it imports (the network's QR service, qr.sh) and the module wasm and
/// module.json files; SWC_REPO on the guest; with $SWC_PAGE_REPO, a built checkout of the requester page repo or
/// release's unpacked page payload (scripts/ci-page.sh), also 0009's page as page/: web/webrtc with its modules' files,
/// the round's two scripts and COMMIT (the payload's own file, else the checkout's HEAD)).
async fn request_put(&self, input: &Value) -> Result<Value, String> {
let name = str_of(input, "name")?;
let remote = input["remote"].as_str().unwrap_or_default();
if !remote.starts_with('/') || remote.contains(['\'', ' ', '\n']) {
return Err(format!(
"remote {remote:?}: an absolute path without quotes or spaces"
));
}
let bytes = match str_of(input, "what")? {
"testHost" => {
let p = std::env::var("SWC_TEST_HOST_STATIC")
.map_err(|_| "SWC_TEST_HOST_STATIC is not set (tests/build-static-host.sh)")?;
std::fs::read(&p).map_err(|e| format!("{p}: {e}"))?
}
"suite" => {
let tests = std::env::current_dir().map_err(|e| e.to_string())?;
let out = tokio::process::Command::new("sh")
.args(["-c", "set -e; p=; if [ -n \"${SWC_PAGE_REPO:-}\" ]; then p=$(mktemp -d); trap 'rm -rf \"$p\"' EXIT; mkdir -p \"$p/page/web\" \"$p/page/scripts\"; \
cp -rL \"$SWC_PAGE_REPO/web/webrtc\" \"$p/page/web/\"; cp \"$SWC_PAGE_REPO/scripts/page-round.mjs\" \"$SWC_PAGE_REPO/scripts/browser.mjs\" \"$p/page/scripts/\"; \
if [ -f \"$SWC_PAGE_REPO/COMMIT\" ]; then cp \"$SWC_PAGE_REPO/COMMIT\" \"$p/page/COMMIT\"; else git -C \"$SWC_PAGE_REPO\" rev-parse HEAD > \"$p/page/COMMIT\"; fi; fi; \
tar -czf - --exclude=did-key-associator/dist --exclude=did-key-associator/dist-backend tests/gate.sh tests/entries.txt tests/compositions tests/targets did-key-associator typescript-helpers/lib/oauth-scope typescript-helpers/lib/cli-args-env typescript-helpers/lib/logger typescript-helpers/lib/serve tests/modules/*/module.json tests/modules/target/wasm32-unknown-unknown/release/*.wasm deploy/vm-images examples scripts ${p:+-C \"$p\" page}"])
.current_dir(tests.join(".."))
.output()
.await
.map_err(|e| e.to_string())?;
if !out.status.success() {
return Err(format!(
"tar of the suite in {}: {}",
tests.display(),
String::from_utf8_lossy(&out.stderr)
));
}
out.stdout
}
"meshd" => {
let p = std::env::var("SWC_MESHD").map_err(
|_| "SWC_MESHD is not set (a static socialweb-meshd, rust/build.sh)",
)?;
std::fs::read(&p).map_err(|e| format!("{p}: {e}"))?
}
"binary" => {
let p = std::env::var("SWC_BINARY")
.map_err(|_| "SWC_BINARY is not set (the socialweb-computer under test, a release build of this checkout)")?;
std::fs::read(&p).map_err(|e| format!("{p}: {e}"))?
}
"stagingRoot" | "stagingLeaf" | "stagingKey" => {
let file = match input["what"].as_str().unwrap_or_default() {
"stagingRoot" => "root.crt",
"stagingLeaf" => "leaf.crt",
_ => "leaf.key",
};
let p = self.dir.join("staging-pki").join(file);
std::fs::read(&p).map_err(|e| format!("{}: {e} (staging.pki first)", p.display()))?
}
"agentConfig" => {
let cfg = testkit::target::expand(
self.target
.agent_config
.as_deref()
.ok_or("the target has no agentConfig")?,
);
testkit::target::check_private(&cfg).map_err(|e| e.to_string())?;
std::fs::read(&cfg).map_err(|e| format!("{}: {e}", cfg.display()))?
}
other => {
return Err(format!(
"what {other:?}: testHost, suite, meshd, binary or agentConfig"
));
}
};
let umask = if input["what"] == "agentConfig" || input["what"] == "stagingKey" || input["what"] == "stagingLeaf" {
"umask 077 && "
} else {
""
};
// a file the bucket publishes (the test host, meshd by tree): the VM downloads it at full speed, checked against
// this file's sha256, instead of a stream over its ssh (minutes at a relayed path's ~50 KB/s, plan 0044 finding 35)
let url = match input["what"].as_str().unwrap_or_default() {
"testHost" => std::env::var("SWC_TEST_HOST_URL").ok(),
"meshd" => std::env::var("SWC_MESHD_URL").ok(),
_ => None,
}
.filter(|u| u.starts_with("https://") && !u.contains(['\'', ' ', '\n']));
let sha: String = {
use sha2::Digest;
sha2::Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect()
};
if let Some(url) = url {
let exec = format!(
"curl -fsS --retry 3 -o '{remote}.part' '{url}' && echo '{sha} {remote}.part' | sha256sum -c --quiet && mv '{remote}.part' '{remote}'"
);
if self.put_exec(name, input, exec, None).await.is_ok() {
return Ok(json!({ "remote": remote, "bytes": bytes.len(), "from": url }));
}
}
let exec = format!(
"{umask}cat > '{remote}.part' && echo '{sha} {remote}.part' | sha256sum -c --quiet && mv '{remote}.part' '{remote}'"
);
let mut tries: u32 = 0;
loop {
tries += 1;
match self.put_exec(name, input, exec.clone(), Some(&bytes)).await {
Ok(()) => break,
Err(e) if e.contains("connection lost") && tries <= PUT_RETRIES => {
tokio::time::sleep(PUT_RETRY_BACKOFF * tries).await;
}
Err(e) => return Err(format!("put {remote} (try {tries}): {e}")),
}
}
Ok(json!({ "remote": remote, "bytes": bytes.len(), "sha256": sha, "tries": tries }))
}
// an exec on the VM of a put: a Held contract of the run requester `name` (an Infra's), else `name` is a run request
// invocation; stdin is the file when one is streamed
async fn put_exec(&self, name: &str, input: &Value, exec: String, stdin: Option<&[u8]>) -> Result<(), String> {
let args: Vec<String> = match input["contract"].as_str() {
Some(c) => ["run", "requester", "ssh", name, "--state-dir"]
.map(String::from)
.into_iter()
.chain([self.requester_state(name).display().to_string(), "--contract".into(), c.into()])
.chain(exec_line(&exec))
.collect(),
None => {
let mut v = self.sub_args("ssh", name, &[]);
v.extend(exec_line(&exec));
v
}
};
let mut child = self
.bin
.cmd()
.args(args)
.stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() })
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| e.to_string())?;
let mut wrote = Ok(());
if let Some(bytes) = stdin {
let mut sin = child.stdin.take().unwrap();
use tokio::io::AsyncWriteExt;
wrote = sin.write_all(bytes).await;
drop(sin);
}
let out = child.wait_with_output().await.map_err(|e| e.to_string())?;
if !out.status.success() {
return Err(tail(&String::from_utf8_lossy(&out.stderr)));
}
if let Err(e) = wrote {
return Err(format!("stdin: {e}"));
}
Ok(())
}
pub(crate) fn sub_args(&self, verb: &str, name: &str, extra: &[&str]) -> Vec<String> {
let mut v: Vec<String> = ["run", "request", verb, name, "--state-dir"]
.map(String::from)
.to_vec();
v.push(self.request_state().display().to_string());
v.extend(extra.iter().map(|s| s.to_string()));
v
}
async fn requester_start(&self, input: &Value) -> Result<Value, String> {
let name = str_of(input, "name")?.to_string();
let state = self.requester_state(&name);
std::fs::create_dir_all(&state).map_err(|e| e.to_string())?;
std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))
.map_err(|e| e.to_string())?;
let mut args: Vec<String> = vec![
"run".into(),
"requester".into(),
"--name".into(),
name.clone(),
"--state-dir".into(),
state.display().to_string(),
];
if input["withGateway"] == true {
args.push("--with-gateway".into());
}
// a market network target: the requester is a child of the network (its relays, PLC and bidders), as request_start's
if let Some(n) = &self.target.network {
args.extend(["--network".to_string(), n.instance.clone()]);
}
// R2 (a guest requester): its identity is the renter's association, through the guest's /run/socialweb-r2 files
let guest = self.acct()?.guest_requester;
let associated = self.associated().map(|id| id.name.clone());
if guest {
args.push("--guest-requester".into());
} else if let Some(id) = &associated {
args.extend(["--identity".to_string(), id.clone()]);
} else {
args.extend(identity_args(
&self.target,
self.acct()?,
SignIn::AppPassword,
));
}
// plan 0040 step 5: the target's bidders filter its discovery (every rental, a --with-gateway node's too); a market
// network sets its own bidders for its child applications, so only a target outside one passes them
let only: Vec<&str> = input["bidderDids"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
if !only.is_empty() && self.target.network.is_none() {
args.extend(["--bidder-dids".to_string(), only.join(",")]);
}
self.extra
.argv
.lock()
.unwrap()
.insert(name.clone(), args.clone());
let log =
std::fs::File::create(state.join(format!("{name}.log"))).map_err(|e| e.to_string())?;
let child = self
.bin
.cmd()
.args(&args)
.env(
"ATPROTO_PASSWORD",
if guest || associated.is_some() {
String::new()
} else {
self.password()?
},
)
.stdout(log.try_clone().map_err(|e| e.to_string())?)
.stderr(log)
.spawn()
.map_err(|e| e.to_string())?;
self.requesters
.lock()
.await
.insert(name.clone(), (state.clone(), child));
let ready = state.join("requesters").join(&name).join("ready.json");
let deadline = Instant::now() + Duration::from_secs(600);
let r: Value = loop {
if let Ok(b) = std::fs::read(&ready)
&& let Ok(v) = serde_json::from_slice(&b)
{
break v;
}
if let Some((_, c)) = self.requesters.lock().await.get_mut(&name)
&& let Ok(Some(st)) = c.try_wait()
{
// the requester's own last lines (its log, redacted as the run's logs are), so the failure reads from the
// gate's RESULT line alone (a CI run keeps no artifacts)
let said = std::fs::read_to_string(state.join(format!("{name}.log"))).unwrap_or_default();
let lines: Vec<&str> = said.lines().filter(|l| !l.trim().is_empty()).collect();
let last = redact_log(&lines[lines.len().saturating_sub(3)..].join("\n"));
let last: Vec<String> = last.lines().map(|l| l.chars().take(400).collect()).collect();
return Err(format!(
"run requester {name} exited {st} before ready: {}",
last.join(" | ")
));
}
if Instant::now() > deadline {
return Err(format!("run requester {name}: no ready file in 10 min"));
}
tokio::time::sleep(Duration::from_secs(1)).await;
};
let kubeconfig = r["kubeconfig"]
.as_str()
.ok_or("ready file without kubeconfig")?
.to_string();
self.kube.add_cluster(
&format!("requester:{name}"),
Cluster {
kubeconfig,
workspace: None,
via: Default::default(),
},
);
Ok(
json!({ "did": r["did"], "handle": r["handle"], "identityMode": r["identityMode"], "cluster": format!("requester:{name}"),
"gateway": r["gateway"], "sshKey": r["sshPublicKey"].as_str().and_then(|p| std::fs::read_to_string(p).ok()).map(|k| k.trim().to_string()) }),
)
}
fn request_args(&self, input: &Value) -> Result<(Vec<String>, Option<Vec<String>>), String> {
let words: Vec<String> = input["args"]
.as_array()
.into_iter()
.flatten()
.map(|w| w.as_str().map(String::from).ok_or("args are strings"))
.collect::<Result<_, _>>()?;
let (opts, command) = split_exec(&words).map_err(|e| e.to_string())?;
let mut out = vec![];
for w in &opts {
let w = w.as_str();
if w == "@agentConfig" {
let p = expand(
self.target
.agent_config
.as_deref()
.ok_or("the target has no agentConfig")?,
);
check_private(&p).map_err(|e| e.to_string())?;
out.push(p.display().to_string());
} else if w.starts_with("--") && !FLAGS.contains(&w.split('=').next().unwrap()) {
return Err(format!("run request flag {w} is not one a test may pass"));
} else {
out.push(w.to_string());
}
}
Ok((out, command))
}
/// `run request --name <name> <args> <identity> --exec -- <command>`, its log events as messages until the
/// subscription closes.
async fn request_start(&self, input: &Value) -> Result<Sub, String> {
let name = str_of(input, "name")?.to_string();
let (mut args, command) = self.request_args(input)?;
// secrets: the module's bundle [{path, value, mode?}] (test values only), written 0600 in the work dir
if let Some(bundle) = input["secrets"].as_array() {
let p = self.dir.join(format!("{name}-secrets.json"));
more::write_private(&p, &Value::Array(bundle.clone()).to_string())?;
args.extend(["--secrets".to_string(), p.display().to_string()]);
}
if let Some(n) = &self.target.network {
args.extend(["--network".to_string(), n.instance.clone()]);
}
let state = self.request_state();
std::fs::create_dir_all(&state).map_err(|e| e.to_string())?;
std::fs::set_permissions(&state, std::os::unix::fs::PermissionsExt::from_mode(0o700))
.map_err(|e| e.to_string())?;
let logs = self.request_logs(&name);
let f = std::fs::File::create(&logs[0]).map_err(|e| e.to_string())?;
// compat: the unchanged client (SWC_COMPAT_REQUESTER: release:<version>, sha:..., path:...) against new servers
let bin = match (
input["compat"] == true,
std::env::var("SWC_COMPAT_REQUESTER"),
) {
(true, Ok(spec)) => Binary::resolve(&spec).await.map_err(|e| e.to_string())?,
_ => self.bin.clone(),
};
let mut cmd = bin.cmd();
cmd.args(["run", "request", "--state-dir"])
.arg(&state)
.args(["--name", &name]);
// R2 rents as the renter's agent: its identity is the association, no password
let mut pass: Vec<String> = vec![];
if self.acct()?.guest_requester {
// its own key in the VM it rents, for request.ssh
args.extend(["--guest-requester".to_string(), "--skip-ssh=false".to_string()]);
} else if let Some(id) = self.associated() {
args.extend(["--identity".to_string(), id.name.clone()]);
} else {
pass.extend(identity_args(&self.target, self.acct()?, SignIn::AppPassword));
cmd.env("ATPROTO_PASSWORD", self.password()?);
}
// the RFP's compute.vm role (e.g. the staging layer the watch page shows), and the bidders it may (bidderDids)
// or may not (denyBidderDids) go to: MarketRequester (contract) options
if let Some(v) = input["vmName"].as_str() {
if v.is_empty()
|| !v
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
{
return Err(format!(
"vmName {v:?}: lowercase letters, digits and dashes"
));
}
pass.extend(["--vm-name".to_string(), v.to_string()]);
}
// the requester's comms mode and secrets fallback (which channel the guest's secrets take): requester options
for (k, flag) in [("commsMode", "--comms-mode"), ("secretsFallback", "--secrets-fallback")] {
if let Some(v) = input[k].as_str() {
if v.is_empty() || !v.chars().all(|c| c.is_ascii_alphabetic()) {
return Err(format!("{k} {v:?}: a mode name"));
}
pass.extend([flag.to_string(), v.to_string()]);
}
}
for (k, flag) in [
("bidderDids", "--allow-bidder-dids"),
("denyBidderDids", "--deny-bidder-dids"),
] {
let Some(dids) = input[k].as_array() else {
continue;
};
let dids: Vec<&str> = dids.iter().filter_map(Value::as_str).collect();
if dids
.iter()
.any(|d| !d.starts_with("did:") || d.contains(','))
{
return Err(format!("{k}: DIDs"));
}
if !dids.is_empty() {
pass.extend([flag.to_string(), dids.join(",")]);
}
}
// the bidder endpoint kinds the requester may use, in order (SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS): iroh, webrtc, https
if let Some(kinds) = input["bidderEndpoints"].as_array() {
let kinds: Vec<&str> = kinds.iter().filter_map(Value::as_str).collect();
if kinds.is_empty() || kinds.iter().any(|k| !["iroh", "webrtc", "https"].contains(k)) {
return Err("bidderEndpoints: iroh, webrtc or https".into());
}
cmd.env("SOCIALWEB_REQUESTER_BIDDER_ENDPOINTS", kinds.join(","));
}
// plan 0050: the staging run's user_data (its root CA), made by staging.pki
if input["stagingUserData"] == true || input["stagingUserData"].is_string() {
let p = match input["stagingUserData"].as_str() {
Some(v) if v.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') => {
self.dir.join("staging-pki").join(format!("user-data-{v}"))
}
Some(v) => return Err(format!("stagingUserData {v:?}: a variant name")),
None => self.dir.join("staging-pki").join("user-data"),
};
if !p.is_file() {
return Err(format!("{}: staging.pki first", p.display()));
}
pass.extend(["--user-data".to_string(), p.display().to_string()]);
}
// runcmd: shell lines the guest runs from the request's own user_data (e.g. 0007:webrtc turns a channel off)
if let Some(lines) = input["runcmd"].as_array() {
if pass.iter().any(|w| w == "--user-data") {
return Err("runcmd and stagingUserData: one --user-data".into());
}
let mut ud = String::from("#cloud-config\nruncmd:\n");
for l in lines {
let l = l.as_str().ok_or("runcmd: shell lines")?;
ud.push_str(&format!(" - [sh, -c, {}]\n", serde_json::to_string(l).map_err(|e| e.to_string())?));
}
let p = self.dir.join(format!("{name}-user-data.yaml"));
more::write_private(&p, &ud)?;
pass.extend(["--user-data".to_string(), p.display().to_string()]);
}
// a CLI before plan 0043 (0006:compat's previous rc) takes them after `--` and the command as one `--exec` line
cmd.args(request_line(bin.exec_argv().await, &args, &pass, command.as_deref()));
// pty: the invocation runs on a pseudo-terminal (socat SYSTEM,pty) whose stdin stays open, so `run request`
// without `--exec` opens its interactive ssh session (a terminal on stdin) and holds it
let pty = input["pty"] == true;
let mut spawn_cmd = if pty {
let std_cmd = cmd.as_std();
let line = std::iter::once(std_cmd.get_program())
.chain(std_cmd.get_args())
.map(|w| format!("'{}'", w.to_string_lossy().replace('\'', r#"'"'"'"#)))
.collect::<Vec<_>>()
.join(" ");
let envs: Vec<_> = std_cmd
.get_envs()
.filter_map(|(k, v)| v.map(|v| (k.to_os_string(), v.to_os_string())))
.collect();
// socat's SYSTEM address runs the line with sh -c on a pty; its `,` `:` `\\` are escaped
let esc = line
.replace('\\', "\\\\")
.replace(',', "\\,")
.replace(':', "\\:");
let mut script = tokio::process::Command::new("socat");
script
.args(["-", &format!("SYSTEM:{esc},pty,setsid,ctty,stderr")])
.envs(envs);
script.stdin(Stdio::piped());
script
} else {
cmd
};
let mut child = spawn_cmd
.stdout(f.try_clone().map_err(|e| e.to_string())?)
.stderr(f)
.spawn()
.map_err(|e| e.to_string())?;
// held open until the invocation is over
let pty_stdin = child.stdin.take();
self.requests.lock().unwrap().insert(name.clone(), state);
let (tx, messages) = mpsc::unbounded_channel();
// which binary runs the invocation (0006 compat: the unchanged CLI's)
let _ = tx.send(Ok(json!({ "msg": "requester_binary", "version": bin.version, "compat": input["compat"] == true })));
let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
let receipts = self.receipts.clone();
let associated = self.associated.clone();
// dev nets only: when the contract fails with "did not come on network", the tail of the console of the guest(s)
// that never reported (the cause is in the guest: PID 1, cloud-final), in the event and in the dump
let console_net = self
.target
.network
.as_ref()
.map(|n| (expand(&n.state_dir), n.instance.clone()));
let console_dump = self.dump.join("logs").join(&name);
// R2 waits for its association ref, which the renter writes once it verified R2: one that never comes is an
// error of its own, not a request that waits until the run's deadline
let ref_by = self
.acct()?
.guest_requester
.then(|| Instant::now() + R2_REF_WAIT);
tokio::spawn(async move {
let _pty_stdin = pty_stdin;
let mut offsets = vec![0usize; logs.len()];
let mut exited = false;
// the run's dev core.log is copied to logs/core.log when the process ends: each line once
let mut seen = std::collections::HashSet::new();
loop {
for (i, p) in logs.iter().enumerate() {
let Ok(text) = std::fs::read_to_string(p) else {
continue;
};
if text.len() < offsets[i] {
offsets[i] = 0;
}
let fresh = &text[offsets[i]..];
let Some(end) = fresh.rfind('\n') else {
continue;
};
for line in fresh[..=end].lines() {
let Ok(v) = serde_json::from_str::<Value>(line) else {
continue;
};
if !seen.insert(line.to_string()) {
continue;
}
// the receipt is known (and the run's VM ours to fault) from its first event, before any summary
if let Some(r) = v["receipt"].as_str() {
let mut rs = receipts.lock().unwrap();
if !rs.iter().any(|x| x == r) {
rs.push(r.into());
}
}
if v["msg"] == "requester_association_written"
&& let Some(d) = v["subject"].as_str().filter(|d| d.starts_with("did:"))
{
let mut a = associated.lock().unwrap();
if !a.iter().any(|x| x == d) {
a.push(d.into());
}
}
let mut v = v;
if v["msg"] == "contract"
&& v["phase"] == "Failed"
&& v["message"]
.as_str()
.is_some_and(|m| m.contains("did not come on network"))
&& let Some((state_dir, instance)) = &console_net
{
let tails = silent_console_tails(state_dir, instance, &console_dump);
if !tails.is_empty() {
let m = v["message"].as_str().unwrap_or_default().to_string();
v["message"] = json!(format!(
"{m}\nconsole tail of the guest(s) that never reported (dev net):\n{tails}"
));
}
}
let msg = v["msg"].as_str().unwrap_or_default().to_string();
let msg = msg.as_str();
if (msg == "contract"
|| msg == "ssh returned"
|| msg.starts_with("requester_")
|| msg.starts_with("guest_")
|| msg.starts_with("ssh to the guest over")
|| msg.starts_with("no --exec and stdin is not a terminal")
|| msg.starts_with("ssh not ready yet"))
&& tx.send(Ok(v)).is_err()
{
return;
}
}
offsets[i] += end + 1;
}
if !exited && let Ok(Some(st)) = child.try_wait() {
exited = true;
// the CLI's own last lines (a refusal before any kcp is plain stderr)
let tail = std::fs::read_to_string(&logs[0]).map(|t| testkit::requester::tail(&t)).unwrap_or_default();
let _ = tx.send(Ok(json!({ "msg": "exited", "code": st.code(), "tail": tail })));
}
let referred =
|| std::fs::read_to_string(R2_REF).is_ok_and(|r| !r.trim().is_empty());
if !exited && ref_by.is_some_and(|by| Instant::now() > by) && !referred() {
exited = true;
let _ = child.start_kill();
let why = format!(
"no association ref in {R2_REF} after {} min: the renter's associate failed or never ran (the knotless check's log says why)",
R2_REF_WAIT.as_secs() / 60
);
let _ = tx.send(Ok(json!({ "msg": "exited", "code": null, "error": why })));
}
tokio::select! {
cmd = cmds.recv() => match cmd {
Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("request.start takes no sends".into())); }
Some(SubCmd::Close) | None => return,
},
_ = tokio::time::sleep(Duration::from_secs(1)) => {}
}
}
});
Ok(Sub { messages, commands })
}
fn summary(&self, name: &str) -> Result<Value, String> {
let p = self
.request_state()
.join("requests")
.join(name)
.join("summary.json");
let v: Value = serde_json::from_slice(
&std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?,
)
.map_err(|e| e.to_string())?;
if let Some(r) = v["contract"]["receiptUri"].as_str() {
let mut rs = self.receipts.lock().unwrap();
if !rs.iter().any(|x| x == r) {
rs.push(r.into());
}
}
Ok(v)
}
/// The restart race of a real devsup with no VM (docs/acceptance/0017): `dev up --profile quick` on the proc backend in an
/// instance of this run, then per delay `iterations` times: `dev down` SIGKILLed `delay` ms after it started (devsup keeps
/// stopping its units in order and exits), `dev up` at once, and 6 s later the kcp must answer /readyz 200. The binary is the
/// target's requester (the build under test). Skipped when it has no `dev up`.
// plan 0046 step 6: two instances of one name in two state dirs (two `run requester --name stg`) are two
// instances: each its own devsup (its runtime dir keyed by its state dir), and a down of one leaves the other up
async fn devsup_samename(&self) -> Result<Value, String> {
let inst = format!("same{}", std::process::id() % 100000);
let dirs = [self.dir.join("samename-a"), self.dir.join("samename-b")];
let cmd = |verb: &[&str], dir: &std::path::Path| {
let mut c = self.bin.cmd();
c.args(["dev", verb[0]]).args(&verb[1..]).args(["--instance", &inst, "--state-dir"]).arg(dir);
c
};
let up = |dir: std::path::PathBuf| async move {
let _ = std::fs::create_dir_all(&dir);
let mut c = cmd(&["up", "--profile", "quick", "--host-backend", "proc"], &dir);
let out = tokio::time::timeout(Duration::from_secs(200), c.output()).await;
let text = match out {
Ok(Ok(o)) => format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)),
Ok(Err(e)) => e.to_string(),
Err(_) => "dev up did not return in 200 s".to_string(),
};
let url = text.lines().find(|l| l.contains("\"kcpURL\"")).and_then(|l| l.split('"').nth(3)).unwrap_or_default().to_string();
(url, text)
};
let ready = |url: String| async move {
if url.is_empty() {
return "none".to_string();
}
let o = tokio::process::Command::new("curl").args(["-sk", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}"]).arg(format!("{url}/readyz")).output().await;
o.map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default()
};
let down = |dir: std::path::PathBuf| async move {
let _ = cmd(&["down"], &dir).output().await;
};
let (a, at) = up(dirs[0].clone()).await;
let (b, bt) = up(dirs[1].clone()).await;
let (ra, rb) = (ready(a.clone()).await, ready(b.clone()).await);
down(dirs[0].clone()).await;
tokio::time::sleep(Duration::from_secs(3)).await;
let rb_after = ready(b.clone()).await;
down(dirs[1].clone()).await;
if a.is_empty() && (at.contains("unknown command") || at.contains("flag provided but not defined")) {
return Ok(json!({ "skipped": "the build has no `dev up --profile quick` on the proc backend" }));
}
let fail = |what: &str, text: &str| json!({ "ok": false, "why": what, "log": tail(&redact_log(text)) });
Ok(match () {
_ if ra != "200" => fail("instance a's kcp is not ready", &at),
_ if rb != "200" => fail("instance b's kcp is not ready (it took a's devsup?)", &bt),
_ if a == b => fail("a and b answer on one kcp URL", &bt),
_ if rb_after != "200" => fail("dev down of a took b's kcp down", &bt),
_ => json!({ "ok": true, "a": a, "b": b, "bAfterDownA": rb_after, "binary": self.bin.version }),
})
}
async fn devsup_race(&self, input: &Value) -> Result<Value, String> {
let delays: Vec<u64> = input["delaysMs"]
.as_array()
.map(|a| a.iter().filter_map(Value::as_u64).collect())
.unwrap_or_else(|| vec![500, 1000, 2000]);
let per = input["perDelay"].as_u64().unwrap_or(4);
let inst = format!("race{}", std::process::id() % 100000);
let state = self.dir.join("devsup-race");
let logs = self.dump.join("logs").join("devsup-race");
let _ = std::fs::create_dir_all(&state);
let _ = std::fs::create_dir_all(&logs);
let base = |verb: &[&str]| {
let mut c = self.bin.cmd();
c.args(["dev", verb[0]])
.args(&verb[1..])
.args(["--instance", &inst, "--state-dir"])
.arg(&state);
c
};
let up = |label: String| {
let mut c = base(&["up", "--profile", "quick"]);
c.args(["--host-backend", "proc"]);
(c, label)
};
let down = || async {
let _ = base(&["down"]).output().await;
};
let run_up = |c: tokio::process::Command, label: String| async move {
let mut c = c;
let t = Instant::now();
let out = tokio::time::timeout(Duration::from_secs(200), c.output()).await;
let secs = (t.elapsed().as_secs_f64() * 10.0).round() / 10.0;
let (rc, text) = match out {
Ok(Ok(o)) => (
o.status.code().unwrap_or(-1),
format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
),
),
Ok(Err(e)) => (-1, e.to_string()),
Err(_) => (-2, "dev up did not return in 200 s".to_string()),
};
(rc, secs, text, label)
};
// the first up: a build without `dev up` (a release one) skips
let (c, l) = up("first".into());
let (rc, _, text, _) = run_up(c, l).await;
if rc != 0 {
down().await;
if text.contains("unknown command") || text.contains("flag provided but not defined") {
return Ok(
json!({ "skipped": "the requester build has no `dev up --profile quick` on the proc backend" }),
);
}
return Err(format!(
"the first dev up of the race instance failed (exit {rc}): {}",
tail(&text)
));
}
let mut iterations = vec![];
let (mut ok, mut bad) = (0u64, 0u64);
for d in &delays {
for i in 0..per {
let mut dc = base(&["down"])
.kill_on_drop(true)
.spawn()
.map_err(|e| e.to_string())?;
tokio::time::sleep(Duration::from_millis(*d)).await;
let _ = dc.start_kill();
let _ = dc.wait().await;
let (c, l) = up(format!("{d}ms-{i}"));
let (rc, secs, text, label) = run_up(c, l).await;
tokio::time::sleep(Duration::from_secs(6)).await;
let url = text
.lines()
.find(|l| l.contains("\"kcpURL\""))
.and_then(|l| l.split('"').nth(3))
.unwrap_or_default()
.to_string();
let code = if url.is_empty() {
"none".to_string()
} else {
let o = tokio::process::Command::new("curl")
.args(["-sk", "-m", "5", "-o", "/dev/null", "-w", "%{http_code}"])
.arg(format!("{url}/readyz"))
.output()
.await;
o.map(|o| String::from_utf8_lossy(&o.stdout).to_string())
.unwrap_or_default()
};
let good = rc == 0 && code == "200";
if good {
ok += 1;
} else {
bad += 1;
let _ = std::fs::write(logs.join(format!("up-{label}.log")), redact_log(&text));
}
let why = text
.lines()
.rev()
.find(|l| l.contains("dev up:"))
.unwrap_or_default()
.to_string();
iterations.push(json!({ "delayMs": d, "exit": rc, "upSeconds": secs, "readyz": code, "ok": good, "error": if good { Value::Null } else { json!(why) } }));
}
}
down().await;
Ok(json!({ "ok": ok, "bad": bad, "iterations": iterations, "binary": self.bin.version }))
}
fn invocation_dir(&self, name: &str) -> PathBuf {
self.request_state().join("requests").join(name)
}
/// The invocation's ComputeContract `contract`, read from its kcp (its instance record: a network child's under the
/// network's state, a standalone one's under the invocation dir): {phase, secretsFetchedAt, secretsFetchedVia,
/// conditions}.
async fn request_contract(&self, name: &str) -> Result<Value, String> {
let dir = self.invocation_dir(name);
let inv: Value = serde_json::from_slice(
&std::fs::read(dir.join("invocation.json")).map_err(|e| format!("{name}: invocation.json: {e}"))?,
)
.map_err(|e| e.to_string())?;
let instance = inv["instance"].as_str().filter(|i| !i.is_empty()).ok_or("the invocation names no instance yet")?;
let record = match (inv["mode"].as_str(), &self.target.network) {
(Some("network"), Some(n)) => expand(&n.state_dir).join("dev").join(&n.instance).join("apps").join("dev").join(instance),
(Some("network"), None) => return Err("a network invocation on a target without a network".into()),
_ => dir.join("dev").join(instance),
}
.join("instance.json");
let rec: Value = serde_json::from_slice(&std::fs::read(&record).map_err(|e| format!("{}: {e}", record.display()))?)
.map_err(|e| e.to_string())?;
let kubeconfig = rec["adminKubeconfig"].as_str().filter(|k| !k.is_empty()).ok_or("the instance record has no adminKubeconfig")?;
let cluster = format!("request:{name}");
self.kube.add_cluster(
&cluster,
Cluster {
kubeconfig: kubeconfig.to_string(),
workspace: Some(rec["baseWorkspace"].as_str().filter(|w| !w.is_empty()).unwrap_or("root").to_string()),
via: Default::default(),
},
);
let input = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeContract", "name": "contract" });
let c = self
.kube
.call(atlibs_kube::GET.into(), Request { input: Some(input), ..Default::default() })
.await?
.unwrap_or_default();
let st = &c["status"];
Ok(json!({ "phase": st["phase"], "receipt": st["contract"]["receipt"]["uri"], "secretsFetchedAt": st["secretsFetchedAt"], "secretsFetchedVia": st["secretsFetchedVia"], "conditions": st["conditions"] }))
}
/// The test's own invocation directory: whether it exists and what its summary says.
fn invocation_state(&self, name: &str) -> Result<Value, String> {
let dir = self.invocation_dir(name);
let summary = std::fs::read(dir.join("summary.json"))
.ok()
.and_then(|b| serde_json::from_slice::<Value>(&b).ok());
// the instance state of a standalone invocation (dev/<name> of its dir) and any live process whose command line names the
// invocation's dir (its devsup, kcp, kine, front, core)
let instance_state = std::fs::read_dir(dir.join("dev"))
.map(|mut d| d.next().is_some())
.unwrap_or(false);
let needle = dir.display().to_string();
let live = std::fs::read_dir("/proc")
.map(|d| {
d.flatten()
.filter(|e| {
e.file_name()
.to_str()
.is_some_and(|n| n.parse::<u32>().is_ok())
})
.filter(|e| {
std::fs::read(e.path().join("cmdline")).is_ok_and(|raw| {
String::from_utf8_lossy(&raw)
.replace('\0', " ")
.contains(&needle)
})
})
.count()
})
.unwrap_or(0);
Ok(
json!({ "dir": needle, "exists": dir.exists(), "instanceState": instance_state, "liveProcesses": live, "summary": summary }),
)
}
/// Rewrites the phase of the test's own invocation summary (top level and contract): the state of a half-created
/// invocation whose network application is gone.
fn set_phase(&self, name: &str, phase: &str) -> Result<Value, String> {
let p = self.invocation_dir(name).join("summary.json");
let mut v: Value = serde_json::from_slice(
&std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?,
)
.map_err(|e| e.to_string())?;
v["phase"] = json!(phase);
if v["contract"].is_object() {
v["contract"]["phase"] = json!(phase);
}
std::fs::write(
&p,
serde_json::to_vec_pretty(&v).map_err(|e| e.to_string())?,
)
.map_err(|e| e.to_string())?;
Ok(json!({ "phase": phase }))
}
/// SIGKILL (or `signal`) to the process that owns the invocation `name`: `<bin> run request ... --name <name>` of this
/// run (not a release, proxy or ssh verb), found by its command line; the kcp child it started keeps running.
fn kill_owner(&self, name: &str, signal: &str) -> Result<Value, String> {
if !["KILL", "TERM", "INT"].contains(&signal) {
return Err(format!("signal {signal}: KILL, TERM or INT"));
}
let state = self.request_state().display().to_string();
let mut pids = vec![];
for e in std::fs::read_dir("/proc")
.map_err(|e| e.to_string())?
.flatten()
{
let Some(pid) = e.file_name().to_str().and_then(|n| n.parse::<u32>().ok()) else {
continue;
};
let Ok(raw) = std::fs::read(e.path().join("cmdline")) else {
continue;
};
let argv: Vec<&str> = raw
.split(|b| *b == 0)
.filter_map(|a| std::str::from_utf8(a).ok())
.collect();
let named = argv.windows(2).any(|w| w[0] == "--name" && w[1] == name);
let owner = argv.len() > 3
&& argv[0]
.rsplit('/')
.next()
.is_some_and(|b| b.starts_with("socialweb-computer"))
&& argv[1] == "run"
&& argv[2] == "request"
&& argv[3].starts_with("--")
&& argv.contains(&state.as_str());
if named && owner {
pids.push(pid);
}
}
if pids.is_empty() {
return Err(format!("no process owns the invocation {name}"));
}
for p in &pids {
let _ = std::process::Command::new("kill")
.arg(format!("-{signal}"))
.arg(p.to_string())
.status();
}
Ok(json!({ "killed": pids }))
}
async fn release(&self, name: &str) -> Result<Value, String> {
let t = Instant::now();
let (ok, out) = self.cli(&self.sub_args("release", name, &[])).await?;
// what each release did and how long, in the dump (an artifact: where a slow release waited)
let d = self.dump.join("logs").join(name);
if std::fs::create_dir_all(&d).is_ok() {
use std::io::Write;
if let Ok(mut f) = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(d.join("release.log"))
{
let _ = writeln!(
f,
"== run request release {name}: ok {ok} in {:.0} s",
t.elapsed().as_secs_f64()
);
let _ = f.write_all(redact_log(&out).as_bytes());
}
}
if !ok {
return Err(format!("run request release {name}: {}", tail(&out)));
}
Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }))
}
/// The owner node's atirc and config dir: the AtircNode's HostProcess (same host).
async fn mesh_node(&self, input: &Value) -> Result<(PathBuf, String), String> {
let node = str_of(input, "node")?;
if let Some((bin, dir, _)) = self.extra.nodes.lock().await.get(node) {
return Ok((bin.clone(), dir.display().to_string()));
}
let cluster = input["cluster"].as_str().ok_or("cluster is required")?;
let get = |kind: &str, name: &str| {
let input = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": name });
self.kube.call(
atlibs_kube::GET.into(),
Request {
input: Some(input),
..Default::default()
},
)
};
let deadline = Instant::now() + Duration::from_secs(180);
loop {
let o = get("AtircNode", node).await?.unwrap_or_default();
if o["status"]["phase"] == "Running" {
let dir = o["status"]["configDir"]
.as_str()
.ok_or("AtircNode without configDir")?
.to_string();
let pid = match o["status"]["runtime"]["pid"].as_i64() {
Some(p) => p,
None => {
let hp = o["status"]["hostProcess"]
.as_str()
.ok_or("AtircNode without hostProcess")?;
get("HostProcess", hp).await?.unwrap_or_default()["status"]["runtime"]["pid"].as_i64().ok_or("HostProcess without pid")?
}
};
let bin = std::fs::read_link(format!("/proc/{pid}/exe"))
.map_err(|e| format!("the AtircNode runs on this host: {e}"))?;
return Ok((bin, dir));
}
if Instant::now() > deadline {
return Err(format!(
"AtircNode {node} not Running in 3 min: {}",
o["status"]
));
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
async fn mesh(&self, input: &Value, args: &[&str]) -> Result<Value, String> {
let (bin, dir) = self.mesh_node(input).await?;
let out = tokio::process::Command::new(bin)
.arg("mesh")
.args(args)
.env("ATIRC_CONFIG_DIR", dir)
.env("ATP_USERNAME", &self.acct()?.handle)
.stdin(Stdio::null())
.output()
.await
.map_err(|e| e.to_string())?;
Ok(
json!({ "ok": out.status.success(), "output": format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)) }),
)
}
async fn plc(&self, did: &str) -> Result<Value, String> {
let base = self.target.plc.trim_end_matches('/');
let doc = self
.http
.get(format!("{base}/{did}"))
.send()
.await
.map_err(|e| e.to_string())?;
let status = doc.status().as_u16();
let doc: Value = doc.json().await.unwrap_or_default();
let last: Value = self
.http
.get(format!("{base}/{did}/log/last"))
.send()
.await
.map_err(|e| e.to_string())?
.json()
.await
.unwrap_or_default();
let data: Value = self
.http
.get(format!("{base}/{did}/data"))
.send()
.await
.map_err(|e| e.to_string())?
.json()
.await
.unwrap_or_default();
Ok(
json!({ "docStatus": status, "lastType": last["type"], "alsoKnownAs": doc["alsoKnownAs"], "rotationKeys": data["rotationKeys"] }),
)
}
/// One com.atproto.repo.listRecords page of the repo's PDS, each record as {uri, createdAt} and, with `fields` (dotted
/// paths into the value, e.g. receipt.uri), `value` holding only those paths; and the cursor. Never the whole value: a
/// compute.vm's user_data makes a page of 100 pass the module input cap (1 MiB).
async fn list_created(&self, input: &Value) -> Result<Value, String> {
let repo = input["repo"].as_str().ok_or("repo is required")?;
let collection = input["collection"].as_str().ok_or("collection is required")?;
let at = testkit::Atproto::new(&self.target);
let pds = at.net.pds(repo).await?;
let mut q = vec![
("repo", repo.to_string()),
("collection", collection.to_string()),
("limit", input["limit"].as_u64().unwrap_or(100).to_string()),
];
if let Some(c) = input["cursor"].as_str() {
q.push(("cursor", c.to_string()));
}
let r = at
.http
.get(format!("{}/xrpc/com.atproto.repo.listRecords", pds.trim_end_matches('/')))
.query(&q)
.send()
.await
.map_err(|e| e.to_string())?;
let status = r.status();
let body = r.text().await.map_err(|e| e.to_string())?;
if !status.is_success() {
return Err(format!("listRecords {repo} {collection}: {status}: {}", tail(&body)));
}
let o: Value = serde_json::from_str(&body).map_err(|e| format!("listRecords {repo} {collection}: {e}"))?;
let fields: Vec<&str> = input["fields"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
let records: Vec<Value> = o["records"]
.as_array()
.into_iter()
.flatten()
.map(|r| {
let mut out = json!({ "uri": r["uri"], "createdAt": r["value"]["createdAt"] });
if !fields.is_empty() {
out["value"] = project(&r["value"], &fields);
}
out
})
.collect();
Ok(json!({ "records": records, "cursor": o["cursor"] }))
}
async fn handle(&self, nsid: &str, input: Value) -> Result<Value, String> {
let what = nsid.strip_prefix(P).unwrap_or(nsid);
match what {
"requester.start" => self.requester_start(&input).await,
"request.put" => self.request_put(&input).await,
"request.ssh" => {
let name = str_of(&input, "name")?;
let mut args = self.sub_args("ssh", name, &[]);
if let Some(t) = input["transport"].as_str() {
args.extend(["--transport".to_string(), t.to_string()]);
}
args.extend(exec_words(&input)?);
let (ok, out) = self.cli(&args).await?;
Ok(json!({ "ok": ok, "output": out }))
}
"request.agent" => {
let name = str_of(&input, "name")?;
let (ok, out) = self.cli(&self.sub_args("agent", name, &[])).await?;
if ok {
Ok(json!({}))
} else {
Err(format!("run request agent: {}", tail(&out)))
}
}
"request.summary" => self.summary(str_of(&input, "name")?),
"request.release" => self.release(str_of(&input, "name")?).await,
"request.state" => self.invocation_state(str_of(&input, "name")?),
"request.contract" => self.request_contract(str_of(&input, "name")?).await,
"request.sshTry" => {
let name = str_of(&input, "name")?;
let (ok, out) = self
.cli(&self.sub_args("ssh", name, &["--exec", "--", "id"]))
.await?;
Ok(
json!({ "ok": ok, "output": tail(&out), "restarted": out.contains("starting the invocation's kcp again") || out.contains("starting it again from its state dir") }),
)
}
"request.setPhase" => self.set_phase(str_of(&input, "name")?, str_of(&input, "phase")?),
"request.removeState" => {
let dir = self.invocation_dir(str_of(&input, "name")?);
std::fs::remove_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
Ok(json!({ "removed": dir.display().to_string() }))
}
"request.releaseTry" => {
let name = str_of(&input, "name")?;
let t = Instant::now();
let (ok, out) = self.cli(&self.sub_args("release", name, &[])).await?;
Ok(
json!({ "ok": ok, "output": tail(&out), "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }),
)
}
"devsup.samename" => self.devsup_samename().await,
"devsup.race" => self.devsup_race(&input).await,
"request.kill" => self.kill_owner(
str_of(&input, "name")?,
input["signal"].as_str().unwrap_or("KILL"),
),
"request.records" => {
let name = str_of(&input, "name")?;
let did = self.acct()?.did_now();
let logs = self.request_logs(name);
let declined = more::declined_accepts(&logs);
let mut uris: Vec<String> = uris_in(&logs, &did)
.into_iter()
.filter(|u| !kept_by_rule(u) && !declined.contains(u))
.collect();
if let Ok(s) = self.summary(name)
&& let Some(r) = s["contract"]["receiptUri"].as_str()
{
uris.push(format!(
"at://{did}/{}/{}",
identlink::market::VM_DELETE_NSID,
identlink::market::release_rkey(r)
));
}
Ok(json!({ "uris": uris, "declined": declined }))
}
"repo.listCreated" => self.list_created(&input).await,
"account.envFile" => {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let p = self
.dir
.join(format!("atp-{}.env", self.env_files.lock().unwrap().len()));
let mut f = std::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.mode(0o600)
.open(&p)
.map_err(|e| e.to_string())?;
writeln!(f, "ATP_PASSWORD={}", self.password()?).map_err(|e| e.to_string())?;
self.env_files.lock().unwrap().push(p.clone());
Ok(json!({ "path": p }))
}
"atirc.meshPub" => {
let text = input["text"]
.as_str()
.ok_or("text is required")?
.to_string();
self.mesh(&input, &["pub", &text]).await
}
"atirc.meshPeers" => self.mesh(&input, &["peers"]).await,
"com.fedfork.exec.identlink.walk" => {
let agent = input["agent"].as_str().ok_or("agent is required")?;
// the host's client (the target's names through its front, trusting its CA): a staging network's PLC is
// https by name under the run's root, which identlink's default client (built-in roots) does not trust
let io = identlink::HttpIo {
client: self.http.clone(),
plc: self.target.plc.trim_end_matches('/').into(),
..Default::default()
};
Ok(
serde_json::to_value(identlink::walk(&io, agent, Default::default()).await)
.unwrap(),
)
}
"com.fedfork.exec.identlink.plc" => {
self.plc(input["did"].as_str().ok_or("did is required")?)
.await
}
other => match self.more(other, &input).await {
Some(r) => r,
None => match self.knotless(other, &input).await {
Some(r) => r,
None => Err(format!("{nsid}: not a call of this host")),
},
},
}
}
/// A failed run's logs into `out` (the dump's logs/), before the freeze or the cleanup, each redacted: per
/// invocation `<name>/local/` (this host's request log, its requester's dev logs) and `<name>/vm/` (over
/// request.ssh: the VM's journal and every *.log under /home and /root: core.log, dev logs, step logs), and per
/// requester its log. What could not be read goes into `<name>/errors.txt`. Each the last 512 KiB, 6 MiB in all,
/// smallest first (an artifact reaches the runner through the guest agent's 16 MiB job output ring), the rest
/// named in `skipped.txt`.
pub async fn collect_logs(&self, out: &Path) {
let mut room = LOGS_MAX;
let mut skipped = vec![];
let requests: Vec<(String, PathBuf)> = self
.requests
.lock()
.unwrap()
.iter()
.map(|(n, s)| (n.clone(), s.clone()))
.collect();
for (name, state) in requests {
let o = out.join(&name);
let mut files = vec![state.join(format!("{name}-request.log"))];
walk(&state.join("requests").join(&name), &mut files);
for f in by_size(files) {
let to = o.join("local").join(f.strip_prefix(&state).unwrap_or(&f));
copy_redacted(&f, &to, &mut room, &mut skipped);
}
// only a VM the invocation held answers ssh (an invocation still renting would make it wait)
let held = std::fs::read_to_string(state.join(format!("{name}-request.log")))
.is_ok_and(|l| l.contains(r#""phase":"Held""#));
let vm = match held {
true => self.vm_logs(&name).await,
false => Err(format!("{name}: its VM was never Held: no VM logs")),
};
match vm {
Ok(tree) => {
let mut files = vec![];
walk(&tree, &mut files);
for f in by_size(files) {
let to = o.join("vm").join(f.strip_prefix(&tree).unwrap_or(&f));
copy_redacted(&f, &to, &mut room, &mut skipped);
}
let _ = std::fs::remove_dir_all(&tree);
}
Err(e) => {
let _ = std::fs::create_dir_all(&o);
let _ = std::fs::write(o.join("errors.txt"), redact_log(&e));
}
}
}
for (name, (state, _)) in self.requesters.lock().await.iter() {
let f = state.join(format!("{name}.log"));
let to = out
.join(format!("requester-{name}"))
.join(format!("{name}.log"));
copy_redacted(&f, &to, &mut room, &mut skipped);
}
if !skipped.is_empty() {
let _ = std::fs::write(out.join("skipped.txt"), skipped.join("\n") + "\n");
}
}
/// The VM of invocation `name`: its logs as a tar.gz (base64 between markers: the CLI's own lines share the
/// output), unpacked into a dir of the work dir.
async fn vm_logs(&self, name: &str) -> Result<PathBuf, String> {
const LOGS: &str = "cd / && journalctl --no-pager -o short-iso -n 20000 > root/swc-journal.log 2>&1; \
find home root -xdev -type f -name '*.log' -size -64M > root/swc-logs.list 2>/dev/null; echo SWC-LOGS-BEGIN; \
tar -czf - -T root/swc-logs.list 2>/dev/null | base64 -w0; echo; echo SWC-LOGS-END; rm -f root/swc-journal.log root/swc-logs.list";
let (_, text) = tokio::time::timeout(
Duration::from_secs(300),
self.cli(&[self.sub_args("ssh", name, &[]), exec_line(LOGS)].concat()),
)
.await
.map_err(|_| format!("{name}: the VM's logs not read in 5 min"))??;
self.unpack_logs(&format!("logs-{name}"), &text).await
}
/// A tar.gz the VM printed as base64 between SWC-LOGS-BEGIN and SWC-LOGS-END, unpacked into `<work dir>/<key>`.
pub(crate) async fn unpack_logs(&self, key: &str, text: &str) -> Result<PathBuf, String> {
let name = key;
let b64 = text
.split_once("SWC-LOGS-BEGIN\n")
.and_then(|(_, r)| r.split_once("\nSWC-LOGS-END"))
.map(|(b, _)| b.trim().to_string())
.ok_or_else(|| format!("{name}: no logs from the VM: {}", tail(text)))?;
let (file, tree) = (
self.dir.join(format!("{key}.b64")),
self.dir.join(key),
);
std::fs::write(&file, b64).map_err(|e| e.to_string())?;
std::fs::create_dir_all(&tree).map_err(|e| e.to_string())?;
let out = tokio::process::Command::new("sh")
.args(["-c", "base64 -d < \"$1\" | tar -xzf - -C \"$2\"", "sh"])
.arg(&file)
.arg(&tree)
.output()
.await
.map_err(|e| e.to_string())?;
let _ = std::fs::remove_file(&file);
if !out.status.success() {
return Err(format!(
"{name}: unpacking the VM's logs: {}",
String::from_utf8_lossy(&out.stderr)
));
}
Ok(tree)
}
/// Wait (5 min) until the run requester `name` holds no ComputeContract that is not Released or Failed.
async fn requester_contracts_released(&self, name: &str) -> Result<(), String> {
let input = json!({ "cluster": format!("requester:{name}"), "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeContract" });
let deadline = Instant::now() + Duration::from_secs(300);
loop {
let listed = self
.kube
.call(
atlibs_kube::LIST.into(),
Request {
input: Some(input.clone()),
..Default::default()
},
)
.await;
let left: Vec<String> = match &listed {
Ok(o) => o
.as_ref()
.and_then(|o| o["items"].as_array())
.into_iter()
.flatten()
.filter(|c| {
!matches!(c["status"]["phase"].as_str(), Some("Released" | "Failed"))
})
.filter_map(|c| c["metadata"]["name"].as_str().map(String::from))
.collect(),
Err(_) => vec![],
};
if left.is_empty() {
return Ok(());
}
if Instant::now() > deadline {
return Err(format!(
"run requester {name}: contracts not released 5 min after cleanup: {}",
left.join(", ")
));
}
tokio::time::sleep(Duration::from_secs(5)).await;
}
}
/// Release every invocation, wait for each run requester's contracts, stop every requester (Ctrl-C), remove the
/// password env files. The errors.
pub async fn cleanup(&self) -> Vec<String> {
let mut errors = self.knotless_cleanup().await;
errors.extend(self.more_cleanup().await);
let names: Vec<String> = self.requests.lock().unwrap().keys().cloned().collect();
// every invocation at once (a multi-VM run's four, a guest requester's slow ones), each said with its time
let released = futures::future::join_all(names.iter().map(|n| async move {
let t = Instant::now();
// an owner that is settling a release of its own (0005 B's out-of-band loss) stops its kcp under our ask:
// ask again once it is gone
let mut r = self.release(n).await;
// a Failed invocation the run released already leaves no invocation.json: nothing is left to release
// an invocation a release process took over (its owner was killed) is released and its kcp application gone
if r.as_ref().is_err_and(|e| {
e.contains(&format!("no invocation {n} under"))
|| e.contains(&format!("the market network has no application {n}"))
}) {
r = Ok(json!({ "gone": true }));
}
for _ in 0..3 {
if r.is_ok() {
break;
}
tokio::time::sleep(Duration::from_secs(20)).await;
r = self.release(n).await;
}
eprintln!(
"cleanup: {n} released in {:.0} s{}",
t.elapsed().as_secs_f64(),
r.as_ref()
.err()
.map(|e| format!(": {e}"))
.unwrap_or_default()
);
r
}))
.await;
errors.extend(released.into_iter().filter_map(Result::err));
// a run requester releases the contracts of the Infra/EphemeralInfra objects kube.cleanup just deleted only
// while it runs: wait for them before the Ctrl-C (it leaves Held VMs to adopt), bounded
let live: Vec<String> = self.requesters.lock().await.keys().cloned().collect();
let settled =
futures::future::join_all(live.iter().map(|n| self.requester_contracts_released(n)))
.await;
errors.extend(settled.into_iter().filter_map(Result::err));
for (name, (_, mut c)) in self.requesters.lock().await.drain() {
if let Ok(None) = c.try_wait() {
if let Some(pid) = c.id() {
let _ = std::process::Command::new("kill")
.args(["-INT", &pid.to_string()])
.status();
}
if tokio::time::timeout(Duration::from_secs(120), c.wait())
.await
.is_err()
{
let _ = c.kill().await;
errors.push(format!("run requester {name} did not stop in 120 s"));
}
}
}
for p in self.env_files.lock().unwrap().drain(..) {
let _ = std::fs::remove_file(p);
}
errors
}
}
impl Call for Swc {
fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Option<Value>, String>> {
Box::pin(async move {
let r = self.handle(&nsid, input_of(&request)).await.map(Some);
self.write_resources().await;
r
})
}
fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Sub, String>> {
Box::pin(async move {
let input = input_of(&request);
match nsid.strip_prefix(P) {
Some("request.start") => {
let r = self.request_start(&input).await;
self.write_resources().await;
r
}
Some("atirc.meshSub") => {
let (bin, dir) = self.mesh_node(&input).await?;
let mut child = tokio::process::Command::new(bin)
.args(["mesh", "sub", "--cursor", "0"])
.env("ATIRC_CONFIG_DIR", dir)
.env("ATP_USERNAME", &self.acct()?.handle)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.kill_on_drop(true)
.spawn()
.map_err(|e| e.to_string())?;
let out = child.stdout.take().unwrap();
let (tx, messages) = mpsc::unbounded_channel();
let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
tokio::spawn(async move {
use tokio::io::AsyncBufReadExt;
let mut lines = tokio::io::BufReader::new(out).lines();
loop {
tokio::select! {
l = lines.next_line() => match l {
Ok(Some(l)) => if let Ok(v) = serde_json::from_str::<Value>(&l) && tx.send(Ok(v)).is_err() { break },
_ => break,
},
cmd = cmds.recv() => match cmd {
Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("meshSub takes no sends".into())); }
Some(SubCmd::Close) | None => break,
},
}
}
let _ = child.kill().await;
});
Ok(Sub { messages, commands })
}
Some(other) => match self.more_sub(other, &input).await {
Some(r) => r,
None => Err(format!("{nsid} is not a subscription of this host")),
},
None => Err(format!("{nsid} is not a subscription of this host")),
}
})
}
}
/// Every regular *.log file under `dir` (never a symlink, a kubeconfig, a key or a state file).
fn walk(dir: &Path, out: &mut Vec<PathBuf>) {
for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
let (p, Ok(t)) = (e.path(), e.file_type()) else {
continue;
};
if t.is_dir() {
walk(&p, out);
} else if t.is_file() && p.extension().is_some_and(|x| x == "log") {
out.push(p);
}
}
}
/// A failed run's logs/ in all, and each log's last bytes kept.
const LOGS_MAX: u64 = 6 << 20;
const LOG_MAX: usize = 512 << 10;
fn by_size(mut files: Vec<PathBuf>) -> Vec<PathBuf> {
files.sort_by_key(|f| f.metadata().map(|m| m.len()).unwrap_or(0));
files
}
/// The last LOG_MAX bytes of a log, redacted, at `to` (0600), while `room` lasts; else `to` goes into `skipped`.
fn copy_redacted(from: &Path, to: &Path, room: &mut u64, skipped: &mut Vec<String>) {
let Ok(bytes) = std::fs::read(from) else {
return;
};
let tail = &bytes[bytes.len().saturating_sub(LOG_MAX)..];
if tail.len() as u64 > *room {
skipped.push(to.display().to_string());
return;
}
*room -= tail.len() as u64;
let text = String::from_utf8_lossy(tail);
if let Some(d) = to.parent() {
let _ = std::fs::create_dir_all(d);
}
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
if let Ok(mut f) = std::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.mode(0o600)
.open(to)
{
let _ = f.write_all(redact_log(&text).as_bytes());
}
}
/// Words that name a secret in a log: the dump's (atlibs ticks) and a kubeconfig's, a header's, a PEM's.
const SECRET_WORDS: &[&str] = &[
"password",
"passwd",
"token",
"secret",
"kubeconfig",
"cookie",
"authorization",
"bearer",
"jwt",
"apikey",
"api_key",
"privatekey",
"private_key",
"private-key",
"key-data",
"credential",
"seed",
];
/// A log redacted line by line: JSON (whole or after a prefix) as the dump is (secret-named keys) plus every string
/// value scrubbed; other text cut after the first secret word's `=` or `:`; JWT-like words and PEM blocks dropped.
/// The last lines of the console of every VM of the dev net `instance` started in the last 15 minutes whose console
/// never printed its "reporting iroh://" line (the guest never came on network): at most 3 VMs, 40 lines each, redacted;
/// the last 300 lines of each go to `dump_dir/console-<vm>.log`.
fn silent_console_tails(state_dir: &Path, instance: &str, dump_dir: &Path) -> String {
let apps = state_dir.join("dev").join(instance).join("apps/dev");
let mut found: Vec<(std::time::SystemTime, String, String)> = vec![];
let Ok(bidders) = std::fs::read_dir(&apps) else {
return String::new();
};
for b in bidders.flatten() {
let Ok(handles) = std::fs::read_dir(b.path().join("handles")) else {
continue;
};
for h in handles.flatten() {
let Ok(vms) = std::fs::read_dir(h.path().join("vms")) else {
continue;
};
for vm in vms.flatten() {
let console = vm.path().join("console.log");
let Ok(meta) = std::fs::metadata(&console) else {
continue;
};
let Ok(modified) = meta.modified() else {
continue;
};
if modified
.elapsed()
.map_or(true, |e| e > Duration::from_secs(900))
{
continue;
}
let Ok(raw) = std::fs::read(&console) else {
continue;
};
let text = String::from_utf8_lossy(&raw).to_string();
if text.contains("reporting iroh://") {
continue;
}
found.push((modified, vm.file_name().to_string_lossy().to_string(), text));
}
}
}
found.sort_by(|a, b| b.0.cmp(&a.0));
let mut out = String::new();
for (_, name, text) in found.into_iter().take(3) {
let lines: Vec<&str> = text.lines().collect();
let _ = std::fs::create_dir_all(dump_dir);
let from = lines.len().saturating_sub(300);
let _ = std::fs::write(
dump_dir.join(format!("console-{name}.log")),
redact_log(&lines[from..].join("\n")),
);
let from = lines.len().saturating_sub(40);
out.push_str(&format!(
"== {name} (last 40 of {} lines)\n{}\n",
lines.len(),
redact_log(&lines[from..].join("\n"))
));
}
out
}
pub fn redact_log(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut pem = false;
for l in text.lines() {
if l.contains("-----BEGIN") {
pem = !l.contains("-----END");
out.push_str("<redacted pem>\n");
continue;
}
if pem {
pem = !l.contains("-----END");
continue;
}
let line = match l
.find('{')
.map(|i| (i, serde_json::from_str::<Value>(&l[i..])))
{
Some((i, Ok(v))) => format!(
"{}{}",
scrub(&l[..i]),
scrub_strings(atlibs_host::ticks::redact(&v))
),
_ => scrub(l),
};
out.push_str(&line);
out.push('\n');
}
out
}
fn scrub_strings(v: Value) -> Value {
match v {
Value::String(s) => Value::String(scrub(&s)),
Value::Array(a) => Value::Array(a.into_iter().map(scrub_strings).collect()),
Value::Object(o) => {
Value::Object(o.into_iter().map(|(k, v)| (k, scrub_strings(v))).collect())
}
v => v,
}
}
fn scrub(s: &str) -> String {
let lower = s.to_ascii_lowercase();
let cut = SECRET_WORDS
.iter()
.filter_map(|w| lower.find(w).map(|i| i + w.len()))
.filter_map(|i| {
lower[i..]
.find(['=', ':'])
.filter(|j| *j < 40)
.map(|j| i + j)
})
.min();
let mut s = match cut {
Some(j) => format!("{}<redacted>", &s[..=j]),
None => s.to_string(),
};
for w in s
.clone()
.split(|c: char| c.is_whitespace() || "\"'=,;()[]{}".contains(c))
{
if w.len() > 20 && w.starts_with("eyJ") && w.matches('.').count() >= 2 {
s = s.replace(w, "<redacted>");
}
}
s
}
/// The command of a host call: `argv` (an array: that exact argv on the guest) or `exec` (a shell line, run as
/// `bash -c <line>`, as the guest's root login shell ran a `--exec <line>` before plan 0043).
pub(crate) fn exec_words(input: &Value) -> Result<Vec<String>, String> {
if let Some(argv) = input["argv"].as_array() {
let argv: Vec<String> = argv
.iter()
.map(|w| w.as_str().map(String::from).ok_or("argv: strings"))
.collect::<Result<_, _>>()?;
if argv.is_empty() {
return Err("argv: a command".into());
}
return Ok(["--exec".to_string(), "--".to_string()].into_iter().chain(argv).collect());
}
Ok(exec_line(input["exec"].as_str().ok_or("exec or argv is required")?))
}