0
0
//! The rest of computer.socialweb.test.*: tenant clusters, the k8s and ssh web pages (OAuth by headless Chrome, a
//! cookie the host keeps), the ssh gateway's temporary key, an atirc observer token, the orchestrator's ssh / restart /
//! cluster / bundle / records / events, the page round (0009), root on a staging VM. Secrets (cookies, tokens,
//! passwords) stay in the host.
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use atlibs_host::run::{Sub, SubCmd};
use atlibs_kube::Cluster;
use atlibs_ssh::Dest;
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use testkit::requester::{exec_line, kept_by_rule, oauth_signin_on, tail, uris_in};
use testkit::target::{check_private, expand};
use tokio::io::AsyncBufReadExt;
use tokio::sync::mpsc;
use super::{Swc, abs_dir_of, exec_words, str_of};
#[derive(Default)]
pub struct Extra {
/// origin -> Cookie header of a signed-in page session
jars: Mutex<HashMap<String, String>>,
/// (ssh web origin, rkey): temporary keys to remove
keys: Mutex<Vec<(String, String)>>,
/// observer nick -> its atirc token
tokens: Mutex<HashMap<String, String>>,
/// requester name -> its run requester argv (restart)
pub argv: Mutex<HashMap<String, Vec<String>>>,
/// owner mesh nodes this run started (`atirc mesh run`): name -> (atirc, its config dir, the process)
pub nodes: tokio::sync::Mutex<HashMap<String, (PathBuf, PathBuf, tokio::process::Child)>>,
/// knotless cleanups: "sh <script>" and "record <at-uri>", in the order made
pub knotless: Mutex<Vec<String>>,
/// bidder name -> the start (epoch s) of the run bidder process this run restarted first: its log is read from there, so counts
/// taken before and after a restart are of one window (the journal read starts at the running process otherwise)
log_since: Mutex<HashMap<String, u64>>,
/// bidders this run upgraded to a `from` binary (bidder.upgrade), each with the from spec it was given: the cleanup upgrades
/// each back to the binary under test
upgraded: Mutex<Vec<(String, String)>>,
}
fn origin(url: &str) -> Result<String, String> {
let u = reqwest::Url::parse(url).map_err(|e| e.to_string())?;
Ok(u.origin().ascii_serialization())
}
const STAGING_PKI_MOUNT: &str = "staging-pki";
/// atirc's refusal of a revoked token (error `Unauthorized`, message `atirc token revoked`) as the CLI prints it on the
/// transport the token names: `pub` and `revoke` go to the token's `endpoint` claim, not to ATIRC_ENDPOINT, and the server
/// mints its iroh ticket there when it runs iroh (prod's `atirc-iroh.service`), its http(s) URL otherwise (staging's
/// `atirc serve`). http(s): atproto-client's `Not authorized: {error} {message}`; iroh: atirc's `{error}: {message}`.
const ATIRC_REVOKED_HTTP: &str = "Not authorized: Unauthorized atirc token revoked";
const ATIRC_REVOKED_IROH: &str = "Unauthorized: atirc token revoked";
fn atirc_revoked(token: &str) -> Result<&'static str, String> {
let payload = token.split('.').nth(1).ok_or("the atirc token is not a JWT")?;
let (mut bytes, mut acc, mut bits) = (Vec::new(), 0u32, 0u32);
for c in payload.bytes() {
let v = match c {
b'A'..=b'Z' => c - b'A',
b'a'..=b'z' => c - b'a' + 26,
b'0'..=b'9' => c - b'0' + 52,
b'-' => 62,
b'_' => 63,
_ => return Err("the atirc token's payload is not base64url".into()),
};
acc = (acc << 6) | u32::from(v);
bits += 6;
if bits >= 8 {
bits -= 8;
bytes.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
let claims: Value = serde_json::from_slice(&bytes).map_err(|e| format!("the atirc token's payload: {e}"))?;
let endpoint = claims["endpoint"].as_str().ok_or("the atirc token names no endpoint")?;
let iroh = endpoint
.strip_prefix("endpoint")
.is_some_and(|t| !t.is_empty() && t.chars().all(|c| c.is_ascii_lowercase() || ('2'..='7').contains(&c)));
Ok(if iroh { ATIRC_REVOKED_IROH } else { ATIRC_REVOKED_HTTP })
}
/// Chrome takes no CA file: it accepts a certificate by its public key (`--ignore-certificate-errors-spki-list`). The
/// leaf the network's front presents for each name, its chain and name verified against the network CA by openssl first,
/// as the sha256 of its SubjectPublicKeyInfo, base64; comma-separated, empty without names.
async fn front_spki(front: std::net::SocketAddr, ca: &std::path::Path, names: &[String]) -> Result<String, String> {
const SPKI: &str = "set -euo pipefail; front=$1 ca=$2; shift 2; for n in \"$@\"; do \
openssl s_client -connect \"$front\" -servername \"$n\" -CAfile \"$ca\" -verify_return_error -verify_hostname \"$n\" </dev/null 2>/dev/null \
| openssl x509 -pubkey -noout | openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 -A; echo; done | sort -u | paste -sd, -";
if names.is_empty() {
return Ok(String::new());
}
let out = tokio::process::Command::new("bash")
.args(["-c", SPKI, "front-spki", &front.to_string()])
.arg(ca)
.args(names)
.stdin(Stdio::null())
.output()
.await
.map_err(|e| format!("openssl: {e}"))?;
let spki = String::from_utf8_lossy(&out.stdout).trim().to_string();
if !out.status.success() || spki.is_empty() {
return Err(format!(
"the front {front} presented no certificate the network CA {} verifies for {names:?}: {}",
ca.display(),
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(spki)
}
pub(super) fn write_private(path: &PathBuf, text: &str) -> Result<(), String> {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut f = std::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.mode(0o600)
.open(path)
.map_err(|e| e.to_string())?;
f.write_all(text.as_bytes()).map_err(|e| e.to_string())
}
fn find_named(dir: &std::path::Path, name: &str, out: &mut Vec<PathBuf>) {
for e in std::fs::read_dir(dir).into_iter().flatten().flatten() {
let p = e.path();
if p.is_dir() {
find_named(&p, name, out);
} else if p.file_name().is_some_and(|n| n == name) {
out.push(p);
}
}
}
/// A subscription of the JSON log lines `keep` takes, from the files `logs()` names now (new ones appear), and the
/// child's exit as {msg: "exited", code}.
pub fn tail_sub(
logs: impl Fn() -> Vec<PathBuf> + Send + 'static,
mut child: Option<tokio::process::Child>,
keep: fn(&str) -> bool,
) -> Sub {
let (tx, messages) = mpsc::unbounded_channel();
let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
tokio::spawn(async move {
let mut offsets: HashMap<PathBuf, usize> = HashMap::new();
// the same line can land in two files (the run's dev core.log is copied to logs/core.log when it ends)
let mut seen = std::collections::HashSet::new();
loop {
for p in logs() {
let Ok(text) = std::fs::read_to_string(&p) else {
continue;
};
let off = offsets.entry(p.clone()).or_default();
if text.len() < *off {
*off = 0;
}
let fresh = &text[*off..];
let Some(end) = fresh.rfind('\n') else {
continue;
};
for line in fresh[..=end].lines() {
let Ok(v) = serde_json::from_str::<Value>(line) else {
continue;
};
if !seen.insert(line.to_string()) {
continue;
}
if keep(v["msg"].as_str().unwrap_or_default()) && tx.send(Ok(v)).is_err() {
return;
}
}
*off += end + 1;
}
if let Some(c) = child.as_mut()
&& let Ok(Some(st)) = c.try_wait()
{
let _ = tx.send(Ok(json!({ "msg": "exited", "code": st.code() })));
child = None;
}
tokio::select! {
cmd = cmds.recv() => match cmd {
Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("this subscription takes no sends".into())); }
Some(SubCmd::Close) | None => return,
},
_ = tokio::time::sleep(Duration::from_secs(1)) => {}
}
}
});
Sub { messages, commands }
}
pub fn market_lines(msg: &str) -> bool {
msg == "contract"
|| msg == "ssh returned"
|| msg.starts_with("requester_")
|| msg.starts_with("guest_")
|| msg.starts_with("infra_")
}
impl Swc {
/// A market network target: `run requester --network <instance>` finds the instance under its `--state-dir`, so the
/// requester's state is the network's (as `run request`'s, request_state); otherwise this run's own dir.
pub(crate) fn requester_state(&self, name: &str) -> PathBuf {
match (&self.target.network, self.associated()) {
(Some(n), _) => expand(&n.state_dir),
(None, Some(id)) => expand(&id.state_dir),
(None, None) => self.dir.join(format!("requester-{name}")),
}
}
/// plan 0046: the account's associated host requester identity, outside a market network
pub(crate) fn associated(&self) -> Option<&testkit::target::AssociatedIdentity> {
if self.target.network.is_some() {
return None;
}
self.target
.account(&self.account)
.ok()
.and_then(|a| a.identity.as_ref())
}
fn atirc(&self) -> Result<PathBuf, String> {
testkit::atirc::atirc_bin().map_err(|e| e.to_string())
}
/// scripts/oauth-signin.ts as the run's account. On a network of its own (the target has `tools` and an XRPC front) it
/// runs with the suite's pinned deno and Chrome, and Chrome accepts the leaf the network's names present (the sign-in
/// origin, the PLC, the PDS, the relay: each verified against the network CA first), since Chrome takes no CA file.
async fn signin(&self, url: &str, extra: &[&str]) -> Result<(), String> {
let a = self.acct()?;
let mut args: Vec<String> = extra.iter().map(|s| s.to_string()).collect();
let mut deno = PathBuf::from("deno");
if let Some(tools) = self.target.tools.as_deref().map(expand) {
deno = tools.join("deno/deno");
args.extend(["--chrome".into(), tools.join("chrome/chrome").display().to_string()]);
}
let mut ca = None;
if let Some(f) = self.target.xrpc_front.as_ref().filter(|_| self.target.name != "prod") {
let bundle = expand(&f.ca);
let names: Vec<String> = [Some(&self.target.plc), Some(&self.target.pds), self.target.relay.as_ref()]
.into_iter()
.flatten()
.filter_map(|u| reqwest::Url::parse(u).ok())
.filter(|u| u.scheme() == "https")
.filter_map(|u| u.host_str().map(String::from))
.collect();
let mut keys: Vec<String> = front_spki(f.address, &bundle, &names).await?.split(',').filter(|k| !k.is_empty()).map(String::from).collect();
let origin = reqwest::Url::parse(url).map_err(|e| format!("the sign-in url: {e}"))?;
if let (Some(host), Some(port)) = (origin.host_str(), origin.port_or_known_default()) {
let at = tokio::net::lookup_host((host, port))
.await
.map_err(|e| format!("{host}: {e}"))?
.next()
.ok_or_else(|| format!("{host} resolves to no address"))?;
for k in front_spki(at, &bundle, &[host.to_string()]).await?.split(',') {
if !k.is_empty() && !keys.iter().any(|h| h == k) {
keys.push(k.to_string());
}
}
}
if !keys.is_empty() {
args.push(format!("--chrome-arg=--ignore-certificate-errors-spki-list={}", keys.join(",")));
}
ca = Some(bundle);
}
let refs: Vec<&str> = args.iter().map(String::as_str).collect();
oauth_signin_on(&deno, ca.as_deref(), a, url, &refs).await.map_err(|e| e.to_string())
}
async fn page_signin(&self, input: &Value) -> Result<Value, String> {
let url = input["url"].as_str().ok_or("url is required")?;
let jar = self.dir.join(format!("jar-{}.json", atlibs_host::now_ms()));
let jar_s = jar.display().to_string();
let mut extra: Vec<&str> = vec!["--cookie-file", &jar_s];
if input["startApi"] == true {
extra.push("--start-api");
}
let done = input["done"].as_str().map(String::from);
if let Some(d) = &done {
extra.extend(["--done", d.as_str()]);
}
let r = self.signin(url, &extra).await;
let cookies: Value = std::fs::read(&jar)
.ok()
.and_then(|b| serde_json::from_slice(&b).ok())
.unwrap_or_default();
let _ = std::fs::remove_file(&jar);
let header = cookies
.as_array()
.into_iter()
.flatten()
.map(|c| {
format!(
"{}={}",
c["name"].as_str().unwrap_or_default(),
c["value"].as_str().unwrap_or_default()
)
})
.collect::<Vec<_>>()
.join("; ");
if header.is_empty() {
return Err(format!(
"signing in at {url}: no session cookie ({})",
r.err().map(|e| e.to_string()).unwrap_or_default()
));
}
self.extra.jars.lock().unwrap().insert(origin(url)?, header);
Ok(json!({ "ok": true }))
}
/// A request to a signed-in page: its cookie, and Origin = the page's (a cookie POST needs it).
async fn page_fetch(
&self,
url: &str,
method: &str,
body: Option<&Value>,
) -> Result<Value, String> {
let o = origin(url)?;
let cookie = self
.extra
.jars
.lock()
.unwrap()
.get(&o)
.cloned()
.ok_or_else(|| format!("{o}: not signed in (page.signin first)"))?;
let mut req = self
.http
.request(method.parse().map_err(|_| "bad method")?, url)
.header("cookie", cookie)
.header("origin", &o);
if let Some(b) = body {
req = req.json(b);
}
let r = req.send().await.map_err(|e| e.to_string())?;
let status = r.status().as_u16();
let text = r.text().await.unwrap_or_default();
Ok(
json!({ "status": status, "body": serde_json::from_str::<Value>(&text).unwrap_or(Value::String(text)) }),
)
}
async fn temp_key(&self) -> Result<Value, String> {
let host = self.target.gateway.ssh.clone();
let web = self.target.gateway.ssh_web();
let key = self.dir.join("st_gateway_key");
if !key.exists() {
let ok = tokio::process::Command::new("ssh-keygen")
.args(["-q", "-t", "ed25519", "-N", "", "-C", "swc-test-host", "-f"])
.arg(&key)
.stdout(Stdio::null())
.status()
.await
.map_err(|e| e.to_string())?
.success();
if !ok {
return Err("ssh-keygen failed".into());
}
}
let public = std::fs::read_to_string(key.with_extension("pub"))
.map_err(|e| e.to_string())?
.trim()
.to_string();
let added = self
.page_fetch(
&format!("{web}/keys"),
"POST",
Some(&json!({ "publicKey": public })),
)
.await?;
let rkey = added["body"]["rkey"]
.as_str()
.ok_or_else(|| format!("POST /keys: {}", added))?
.to_string();
self.extra.keys.lock().unwrap().push((web, rkey));
let dest = Dest {
dest: format!("{}@{host}", self.acct()?.handle),
port: Some(self.target.gateway.ssh_port.unwrap_or(22)),
key: Some(key),
known_hosts: Some(self.dir.join("known_hosts")),
user_config: false,
};
self.ssh.add_dest("gateway", dest);
if let Some(cfg) = &self.target.agent_config {
let p = expand(cfg);
check_private(&p).map_err(|e| e.to_string())?;
self.ssh.add_stdin_file("agentConfig", p);
}
Ok(json!({ "dest": "gateway" }))
}
/// `atirc <args>` as the observer: {ok, refused, error}. refused: the server answered that the token is revoked
/// (`atirc_revoked`: atirc's text for the transport the token names), the one failure that is the refusal; error: its last
/// stderr line.
async fn observer_run(&self, nick: &str, args: &[&str]) -> Result<Value, String> {
let token = self
.extra
.tokens
.lock()
.unwrap()
.get(nick)
.cloned()
.ok_or_else(|| format!("no observer {nick}"))?;
let endpoint = self.target.atirc.clone().ok_or("the target has no atirc")?;
let revoked = atirc_revoked(&token)?;
let out = tokio::process::Command::new(self.atirc()?)
.args(args)
.env("ATIRC_TOKEN", token)
.env("ATIRC_ENDPOINT", endpoint)
.env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output()
.await
.map_err(|e| e.to_string())?;
let stderr = String::from_utf8_lossy(&out.stderr);
let error: String = stderr.lines().last().unwrap_or_default().chars().take(300).collect();
Ok(json!({ "ok": out.status.success(), "refused": !out.status.success() && stderr.contains(revoked), "error": error }))
}
async fn observer_sub(&self, input: &Value) -> Result<Sub, String> {
let nick = str_of(input, "nick")?.to_string();
let a = self.acct()?;
let endpoint = self.target.atirc.clone().ok_or("the target has no atirc")?;
let out = tokio::process::Command::new(self.atirc()?)
.args(["token", &nick, "--endpoint", &endpoint])
.env("ATP_USERNAME", &a.handle)
.env("ATP_PASSWORD", self.password()?)
.env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
.stdin(Stdio::null())
.output()
.await
.map_err(|e| e.to_string())?;
let token = String::from_utf8_lossy(&out.stdout).trim().to_string();
if !out.status.success() || token.is_empty() {
return Err(format!("atirc token {nick} failed"));
}
self.extra
.tokens
.lock()
.unwrap()
.insert(nick.clone(), token.clone());
// no -c: atirc names the account's channel itself (its default_channel keeps a dash, staging's handles have
// one); a hand-made name with the dash as `_` reads as another handle's channel and the subscribe is a 403
let mut child = tokio::process::Command::new(self.atirc()?)
.args(["sub", "--full", "--cursor", "-1"])
.env("ATIRC_TOKEN", token)
.env("ATIRC_ENDPOINT", endpoint)
.env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
.map_err(|e| e.to_string())?;
let stdout = child.stdout.take().unwrap();
let stderr = child.stderr.take().unwrap();
let last = std::sync::Arc::new(Mutex::new(String::new()));
let seen = last.clone();
let errs = tokio::spawn(async move {
let mut lines = tokio::io::BufReader::new(stderr).lines();
while let Ok(Some(l)) = lines.next_line().await {
*seen.lock().unwrap() = l.chars().take(300).collect();
}
});
let (tx, messages) = mpsc::unbounded_channel();
let (commands, mut cmds) = mpsc::unbounded_channel::<SubCmd>();
tokio::spawn(async move {
let mut lines = tokio::io::BufReader::new(stdout).lines();
loop {
tokio::select! {
l = lines.next_line() => match l {
Ok(Some(l)) => if let Ok(v) = serde_json::from_str::<Value>(&l) && tx.send(Ok(v)).is_err() { break },
// atirc sub ended by itself: its last stderr line is the cause (a refused subscribe)
_ => {
let _ = tokio::time::timeout(Duration::from_secs(2), errs).await;
let _ = tx.send(Err(format!("atirc sub ended: {}", last.lock().unwrap())));
break;
}
},
cmd = cmds.recv() => match cmd {
Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("no sends".into())); }
Some(SubCmd::Close) | None => break,
},
}
}
let _ = child.kill().await;
});
Ok(Sub { messages, commands })
}
async fn requester_restart(&self, name: &str) -> Result<Value, String> {
let argv = self
.extra
.argv
.lock()
.unwrap()
.get(name)
.cloned()
.ok_or_else(|| format!("no requester {name}"))?;
let state = self.requester_state(name);
if let Some((_, mut c)) = self.requesters.lock().await.remove(name) {
if let Some(pid) = c.id() {
let _ = std::process::Command::new("kill")
.args(["-INT", &pid.to_string()])
.status();
}
if tokio::time::timeout(Duration::from_secs(120), c.wait())
.await
.is_err()
{
let _ = c.kill().await;
return Err(format!("run requester {name} did not stop in 120 s"));
}
}
let ready = state.join("requesters").join(name).join("ready.json");
if ready.exists() {
return Err("the ready file stayed after Ctrl-C".into());
}
let t = Instant::now();
let log = std::fs::OpenOptions::new()
.append(true)
.create(true)
.open(state.join(format!("{name}.log")))
.map_err(|e| e.to_string())?;
let child = self
.bin
.cmd()
.args(&argv)
.env("ATPROTO_PASSWORD", self.password()?)
.stdout(log.try_clone().map_err(|e| e.to_string())?)
.stderr(log)
.spawn()
.map_err(|e| e.to_string())?;
self.requesters
.lock()
.await
.insert(name.into(), (state.clone(), child));
let deadline = Instant::now() + Duration::from_secs(300);
loop {
if let Ok(b) = std::fs::read(&ready)
&& let Ok(v) = serde_json::from_slice::<Value>(&b)
{
return Ok(
json!({ "did": v["did"], "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0 }),
);
}
if Instant::now() > deadline {
return Err(format!(
"run requester {name}: not ready 5 min after the restart"
));
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
}
async fn requester_cli(
&self,
verb: &str,
name: &str,
extra: &[&str],
) -> Result<(bool, String), String> {
let state = self.requester_state(name).display().to_string();
let mut args = vec!["run", "requester", verb, name, "--state-dir", &state];
args.extend(extra);
self.bin.run(&args).await.map_err(|e| e.to_string())
}
/// logs.fetchRuns {name, contract?, dir, to, all?}: the run dirs under `dir` on a guest that hold a failure.json (the failed
/// compositions of a gate run there; `all`: every dir under `dir`) into this run's dump at `logs/<to>/<run>/`: text files
/// redacted and cut as the dump's own logs, a ticks.ndjson.zst whole (the dump's own form) while the room lasts, the rest
/// in `skipped.txt`. The guest is a run requester's contract (`contract`) or an invocation (`name` only).
async fn fetch_runs(&self, input: &Value) -> Result<Value, String> {
let (name, dir, to) = (str_of(input, "name")?, abs_dir_of(input, "dir")?, str_of(input, "to")?);
let to = std::path::Path::new(to).file_name().ok_or("to: a directory name")?.to_string_lossy().to_string();
let pick = if input["all"] == true { "[ -d \"$d\" ]" } else { "[ -f \"$d\"failure.json ]" };
let exec = format!(
"cd '{}' 2>/dev/null || exit 0; echo SWC-LOGS-BEGIN; for d in */; do {pick} && echo \"${{d%/}}\"; done | tar -czf - -T - 2>/dev/null | base64 -w0; echo; echo SWC-LOGS-END",
dir.replace('\'', "")
);
let run = async {
match input["contract"].as_str() {
Some(c) => {
let line = exec_line(&exec);
let mut extra = vec!["--contract", c];
extra.extend(line.iter().map(String::as_str));
self.requester_cli("ssh", name, &extra).await
}
None => self.cli(&[self.sub_args("ssh", name, &[]), exec_line(&exec)].concat()).await,
}
};
let (_, text) = tokio::time::timeout(Duration::from_secs(300), run)
.await
.map_err(|_| format!("{name}: the run dirs not read in 5 min"))??;
let tree = self.unpack_logs(&format!("runs-{to}"), &text).await?;
let out = self.dump.join("logs").join(&to);
let mut files = vec![];
super::walk(&tree, &mut files);
let (mut room, mut skipped, mut copied) = (super::LOGS_MAX / 3 * 2, vec![], 0);
let mut runs: Vec<String> = vec![];
for f in super::by_size(files) {
let rel = f.strip_prefix(&tree).unwrap_or(&f).to_path_buf();
if let Some(r) = rel.components().next().map(|c| c.as_os_str().to_string_lossy().to_string())
&& !runs.contains(&r)
{
runs.push(r);
}
let dest = out.join(&rel);
if f.extension().is_some_and(|e| e == "zst") {
let len = f.metadata().map(|m| m.len()).unwrap_or(u64::MAX);
if len > room {
skipped.push(dest.display().to_string());
continue;
}
room -= len;
if let Some(d) = dest.parent() {
let _ = std::fs::create_dir_all(d);
}
if std::fs::copy(&f, &dest).is_ok() {
copied += 1;
}
continue;
}
let before = skipped.len();
super::copy_redacted(&f, &dest, &mut room, &mut skipped);
if skipped.len() == before {
copied += 1;
}
}
let _ = std::fs::remove_dir_all(&tree);
if !skipped.is_empty() {
let _ = std::fs::create_dir_all(&out);
let _ = std::fs::write(out.join("skipped.txt"), skipped.join("\n") + "\n");
}
Ok(json!({ "runs": runs, "files": copied, "skipped": skipped.len(), "at": out.display().to_string() }))
}
/// plan 0050: the run's root CA and one leaf for the staging network's names, from the run requester's own OpenBao. The
/// host declares a pki mount and a policy for the requester's client on its OpenBaoServer (the kube patch restores the
/// spec at the run's cleanup), then issues the leaf as that client. The leaf and its key stay in the run's work dir
/// (0600, `staging-pki/`), never in a result; the root is public.
async fn staging_pki(&self, input: &Value) -> Result<Value, String> {
let name = str_of(input, "name")?;
let names: Vec<String> = input["names"].as_array().into_iter().flatten().filter_map(|n| n.as_str().map(String::from)).collect();
let ips: Vec<String> = input["ipSans"].as_array().into_iter().flatten().filter_map(|n| n.as_str().map(String::from)).collect();
let Some(cn) = names.first().cloned() else { return Err("names: at least one".into()) };
let cluster = format!("requester:{name}");
let key = json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "OpenBaoServer", "name": "requester" });
let kube = |nsid: &str, input: Value| {
atlibs_host::run::Call::call(&*self.kube, nsid.into(), atlibs_host::run::Request { input: Some(input), ..Default::default() })
};
let obs = match kube(atlibs_kube::GET, key.clone()).await {
Ok(v) => v,
// a hand run's associated host requester (plan 0046) was not started by this host: its cluster from its state
Err(_) => {
let kubeconfig = self.requester_state(name).join("requesters").join(name).join("dev").join(name).join("kcp/admin.kubeconfig");
self.kube.add_cluster(&cluster, Cluster { kubeconfig: kubeconfig.display().to_string(), workspace: None, via: Default::default() });
kube(atlibs_kube::GET, key.clone()).await?
}
}
.unwrap_or_default();
let spec = &obs["spec"];
let mut mounts = spec["declared"]["secretMounts"].as_array().cloned().unwrap_or_default();
if !mounts.iter().any(|m| m["path"] == STAGING_PKI_MOUNT) {
mounts.push(json!({ "path": STAGING_PKI_MOUNT, "type": "pki", "maxLeaseTTL": "48h" }));
}
let mut policies = spec["declared"]["policies"].as_array().cloned().unwrap_or_default();
if !policies.iter().any(|p| p["name"] == STAGING_PKI_MOUNT) {
policies.push(json!({ "name": STAGING_PKI_MOUNT, "rules": [{ "path": format!("{STAGING_PKI_MOUNT}/*"), "capabilities": ["create", "read", "update", "list"] }] }));
}
let clients: Vec<Value> = spec["clients"].as_array().cloned().unwrap_or_default().into_iter().map(|mut c| {
if c["name"] == "requester" {
let mut p = c["policies"].as_array().cloned().unwrap_or_default();
if !p.iter().any(|x| x == STAGING_PKI_MOUNT) {
p.push(json!(STAGING_PKI_MOUNT));
}
c["policies"] = json!(p);
}
c
}).collect();
let mut patch = key.clone();
patch["patch"] = json!({ "spec": { "declared": { "secretMounts": mounts, "policies": policies }, "clients": clients } });
kube(atlibs_kube::PATCH, patch).await?;
let d = self.requester_state(name).join("requesters").join(name).join("dev").join(name).join("openbaoservers/requester/openbao");
let runtime = PathBuf::from(std::env::var("HOME").unwrap_or_default()).join(".cache/socialweb-computer/runtime");
let bao = std::fs::read_dir(&runtime).map_err(|e| e.to_string())?.flatten().map(|e| e.path().join("bao")).find(|p| p.is_file()).ok_or("no bao in the runtime cache")?;
let bao_cmd = |args: &[&str], token: Option<&str>, addr: &str| {
let mut c = tokio::process::Command::new(&bao);
c.args(args)
.env("BAO_ADDR", addr)
.env("BAO_CACERT", d.join("tls/ca.crt"))
.env("BAO_CLIENT_CERT", d.join("clients/requester/client.crt"))
.env("BAO_CLIENT_KEY", d.join("clients/requester/client.key"))
.stdin(Stdio::null());
if let Some(t) = token {
c.env("BAO_TOKEN", t);
}
c
};
let deadline = Instant::now() + Duration::from_secs(120);
let run = self.dir.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let mut issuer = String::new();
let (root, issued) = loop {
// the patched spec makes the server reconcile: its status (address) is rewritten, so read it each try
let addr = kube(atlibs_kube::GET, key.clone()).await?.unwrap_or_default()["status"]["address"].as_str().unwrap_or_default().to_string();
if addr.is_empty() {
if Instant::now() > deadline {
return Err("staging pki: the OpenBaoServer reports no address".into());
}
tokio::time::sleep(Duration::from_secs(3)).await;
continue;
}
let token = bao_cmd(&["write", "-field=token", "auth/cert/login", "name=requester"], None, &addr).output().await.map_err(|e| e.to_string())?;
let token = String::from_utf8_lossy(&token.stdout).trim().to_string();
// one root per run (plan 0050): a fresh issuer, named for the run; the role issues from it. The leaf outlives
// the network's renew window (24h before its end: EnsureGuestTLS keeps it) and stays inside the root's life
if issuer.is_empty() {
let cn = format!("common_name=socialweb staging root {run}");
let rotated = bao_cmd(&["write", "-format=json", &format!("{STAGING_PKI_MOUNT}/root/rotate/internal"), &cn, "ttl=47h", "key_type=ec", "key_bits=256"], Some(&token), &addr).output().await.map_err(|e| e.to_string())?;
if rotated.status.success() {
let v: Value = serde_json::from_slice(&rotated.stdout).unwrap_or_default();
issuer = v["data"]["issuer_id"].as_str().unwrap_or_default().to_string();
}
}
let role = format!("{STAGING_PKI_MOUNT}/roles/net");
let issuer_ref = format!("issuer_ref={}", if issuer.is_empty() { "default" } else { issuer.as_str() });
let made = bao_cmd(&["write", &role, "allow_any_name=true", "allow_subdomains=true", "allow_bare_domains=true", "allow_wildcard_certificates=true", "allow_ip_sans=true", "max_ttl=47h", &issuer_ref], Some(&token), &addr).output().await.map_err(|e| e.to_string())?;
let root = bao_cmd(&["read", "-field=certificate", &format!("{STAGING_PKI_MOUNT}/issuer/{issuer}")], Some(&token), &addr).output().await.map_err(|e| e.to_string())?;
let issue_path = format!("{STAGING_PKI_MOUNT}/issue/net");
let (cn_arg, alt_arg, ip_arg) = (format!("common_name={cn}"), format!("alt_names={}", names.join(",")), format!("ip_sans={}", ips.join(",")));
let issued = if !issuer.is_empty() && made.status.success() && root.status.success() {
bao_cmd(&["write", "-format=json", &issue_path, &cn_arg, &alt_arg, &ip_arg, "ttl=46h"], Some(&token), &addr).output().await.map_err(|e| e.to_string())?
} else {
made
};
if issued.status.success() && String::from_utf8_lossy(&root.stdout).contains("BEGIN CERTIFICATE") {
break (String::from_utf8_lossy(&root.stdout).trim().to_string(), issued.stdout);
}
if Instant::now() > deadline {
return Err(format!("staging pki: {}", tail(&String::from_utf8_lossy(&issued.stderr))));
}
tokio::time::sleep(Duration::from_secs(3)).await;
};
let leaf: Value = serde_json::from_slice(&issued).map_err(|e| e.to_string())?;
let dir = self.dir.join("staging-pki");
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
write_private(&dir.join("root.crt"), &format!("{root}\n"))?;
write_private(&dir.join("leaf.crt"), &format!("{}\n", leaf["data"]["certificate"].as_str().unwrap_or_default().trim()))?;
write_private(&dir.join("leaf.key"), &format!("{}\n", leaf["data"]["private_key"].as_str().unwrap_or_default().trim()))?;
// every staging VM's user_data (the requester adds the session's ssh keys): the root in its system store, and the
// module's boot commands (its resolver: the network's DNS); a variant (`bootcmdFor.<name>`) adds its own, in
// user-data-<name> (userDataFor.<name>)
let user_data = |cmds: &[Value]| {
let mut ud = String::from("#cloud-config\nca_certs:\n trusted:\n - |\n");
for line in root.lines() {
ud.push_str(" ");
ud.push_str(line.trim());
ud.push('\n');
}
if !cmds.is_empty() {
ud.push_str("bootcmd:\n");
for c in cmds {
ud.push_str(&format!(" - {}\n", serde_json::to_string(c.as_str().unwrap_or_default()).unwrap_or_default()));
}
}
ud
};
let common: Vec<Value> = input["bootcmd"].as_array().cloned().unwrap_or_default();
let ud = user_data(&common);
write_private(&dir.join("user-data"), &ud)?;
let mut variants = serde_json::Map::new();
for (name, extra) in input["bootcmdFor"].as_object().cloned().unwrap_or_default() {
if !name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') {
return Err(format!("bootcmdFor {name:?}: lowercase letters, digits and dashes"));
}
let cmds: Vec<Value> = common.iter().cloned().chain(extra.as_array().cloned().unwrap_or_default()).collect();
let v = user_data(&cmds);
write_private(&dir.join(format!("user-data-{name}")), &v)?;
variants.insert(name, json!(v));
}
Ok(json!({ "root": root, "names": names, "userData": ud, "userDataFor": variants, "userDataFile": dir.join("user-data").display().to_string() }))
}
async fn requester_bundle(&self, input: &Value) -> Result<Value, String> {
let name = str_of(input, "name")?;
let contract = str_of(input, "contract")?;
let cfg = expand(
self.target
.agent_config
.as_deref()
.ok_or("the target has no agentConfig")?,
);
check_private(&cfg).map_err(|e| e.to_string())?;
let get = |kind: &str, n: &str| {
let input = json!({ "cluster": format!("requester:{name}"), "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": n });
atlibs_host::run::Call::call(
&*self.kube,
atlibs_kube::GET.into(),
atlibs_host::run::Request {
input: Some(input),
..Default::default()
},
)
};
let addr =
get("OpenBaoServer", "requester").await?.unwrap_or_default()["status"]["address"]
.as_str()
.ok_or("no OpenBaoServer address")?
.to_string();
// a network target's request is a network child: its instance dir is the network's dev/<i>/apps/dev/<i>-<name>
let d = match &self.target.network {
Some(n) => testkit::target::expand(&n.state_dir).join(format!(
"dev/{i}/apps/dev/{i}-{name}/openbaoservers/requester/openbao",
i = n.instance
)),
None => self
.requester_state(name)
.join("requesters")
.join(name)
.join("dev")
.join(name)
.join("openbaoservers/requester/openbao"),
};
let runtime = PathBuf::from(std::env::var("HOME").unwrap_or_default())
.join(".cache/socialweb-computer/runtime");
let bao = std::fs::read_dir(&runtime)
.map_err(|e| e.to_string())?
.flatten()
.map(|e| e.path().join("bao"))
.find(|p| p.is_file())
.ok_or("no bao in the runtime cache")?;
let bundle = self.dir.join(format!("{contract}-bundle.json"));
let v = json!([{ "path": "/etc/atirc/atirc.env", "value": "", "mode": "0600" }, { "path": "/etc/atirc/config.toml", "value": std::fs::read_to_string(&cfg).map_err(|e| e.to_string())?, "mode": "0600" }]);
write_private(&bundle, &v.to_string())?;
let env = |c: &mut tokio::process::Command| {
c.env("BAO_ADDR", &addr)
.env("BAO_CACERT", d.join("tls/ca.crt"))
.env("BAO_CLIENT_CERT", d.join("clients/requester/client.crt"))
.env("BAO_CLIENT_KEY", d.join("clients/requester/client.key"));
};
let mut login = tokio::process::Command::new(&bao);
login.args(["write", "-field=token", "auth/cert/login", "name=requester"]);
env(&mut login);
let out = login.output().await.map_err(|e| e.to_string())?;
let mut put = tokio::process::Command::new(&bao);
put.args([
"kv",
"put",
"-mount=requester",
&format!("contracts/{contract}/secrets"),
])
.arg(format!("bundle=@{}", bundle.display()));
env(&mut put);
put.env("BAO_TOKEN", String::from_utf8_lossy(&out.stdout).trim())
.stdout(Stdio::null())
.stderr(Stdio::null());
let ok = out.status.success() && put.status().await.map_err(|e| e.to_string())?.success();
let _ = std::fs::remove_file(&bundle);
if ok {
Ok(json!({ "ok": true }))
} else {
Err("the agent bundle did not reach the requester's OpenBao".into())
}
}
/// The rest of the calls; None: not one of these.
pub(crate) async fn more(&self, what: &str, input: &Value) -> Option<Result<Value, String>> {
Some(match what {
// undo now what the run applied and patched in its clusters (a module about to remove a cluster's kcp, 0001 s2's
// temporary bidder: afterwards nothing answers there, so the end's cleanup could not)
"kube.undo" => {
let errors = self.kube.cleanup().await;
if errors.is_empty() { Ok(json!({ "ok": true })) } else { Err(errors.join("; ")) }
}
"kube.addCluster" => {
let name = match str_of(input, "name") {
Ok(n) => n.to_string(),
Err(e) => return Some(Err(e)),
};
let path = self.dir.join(format!("tenant-{name}.kubeconfig"));
let r = write_private(&path, input["kubeconfig"].as_str().unwrap_or_default());
// viaRoot: a kcp front on a staging VM's loopback, through an ssh forward to that VM
let via = match (input["viaRoot"] == true, self.target.root_hosts.first()) {
(true, Some(h)) if self.target.name != "prod" => {
atlibs_kube::Via::Ssh { host: h.clone() }
}
_ => Default::default(),
};
r.map(|_| {
self.kube.add_cluster(
&format!("tenant:{name}"),
Cluster {
kubeconfig: path.display().to_string(),
workspace: None,
via,
},
);
json!({ "cluster": format!("tenant:{name}") })
})
}
// a tenant kubeconfig minted and kept by the host (the module never holds it): over xrpc with the
// account's service auth, or on the signed-in k8s page; registered as cluster tenant:<as>
"gateway.kubeconfig" | "page.kubeconfig" => {
let (Ok(name), Ok(alias)) = (str_of(input, "name"), str_of(input, "as")) else {
return Some(Err("name and as are required".into()));
};
let ttl = input["ttlSeconds"].as_u64().unwrap_or(3600);
async {
let g = &self.target.gateway;
let kc = if what == "gateway.kubeconfig" {
let at = testkit::Atproto::new(&self.target);
let sess = at.session(self.acct()?).await.map_err(|e| e.to_string())?;
let v = at
.service_call(
&sess,
&g.url,
&g.aud,
"com.publicdomainrelay.temp.cluster.getKubeconfig",
Some(&json!({ "name": name, "ttlSeconds": ttl })),
&[],
)
.await
.map_err(|e| e.to_string())?;
v["kubeconfig"].as_str().map(String::from)
} else {
let v = self
.page_fetch(
&format!("{}/api/clusters/kubeconfig", g.url.trim_end_matches('/')),
"POST",
Some(&json!({ "name": name, "ttlSeconds": ttl })),
)
.await?;
if v["status"] != 200 {
return Err(format!(
"the page's /api/clusters/kubeconfig: HTTP {}",
v["status"]
));
}
v["body"]["kubeconfig"].as_str().map(String::from)
}
.ok_or("no kubeconfig in the answer")?;
let path = self.dir.join(format!("tenant-{alias}.kubeconfig"));
write_private(&path, &kc)?;
self.kube.add_cluster(
&format!("tenant:{alias}"),
Cluster {
kubeconfig: path.display().to_string(),
workspace: None,
via: Default::default(),
},
);
Ok(json!({ "cluster": format!("tenant:{alias}") }))
}
.await
}
"page.signin" => self.page_signin(input).await,
// an OAuth sign-in with no page session of ours (a QR session transfer): the account's consent, Authorize
"oauth.signin" => match input["url"].as_str() {
Some(url) => {
let mut extra = vec![];
if let Some(d) = input["done"].as_str() {
extra.extend(["--done", d]);
}
let r = self.signin(url, &extra).await;
Ok(json!({ "ok": r.is_ok(), "note": r.err() }))
}
None => Err("url is required".into()),
},
// a network bidder's core restarted in place (staging; 0007 restart): `dev restart <network> <bidder>/core` with
// the network's own binary; its VMs stay up and the new core adopts them
// the memory cgroup of THIS run's own VM (staging / dev network, VM units under the account's systemd user
// manager): {receipt, cluster, vm} -> unit, current, high, max, swapMax, events {high, max, oom, oom_kill}. With
// `setHigh` (bytes, or "max" for no limit) it first sets the unit's MemoryHigh (runtime property: fault injection on
// the test's own VM only; the receipt is one this run made and names the ComputeVM `vm`)
"vm.memory" => {
let (Some(receipt), Ok(cluster), Ok(vm)) = (
input["receipt"].as_str().filter(|r| r.starts_with("at://")),
str_of(input, "cluster"),
str_of(input, "vm"),
) else {
return Some(Err(
"receipt (an at:// URI), cluster and vm are required".into()
));
};
if self.target.network.is_none() {
return Some(Err(
"vm.memory: the target is no market network (staging only)".into(),
));
}
if !self.receipts.lock().unwrap().iter().any(|r| r == receipt) {
return Some(Err(format!(
"vm.memory: {receipt} is not a contract of this run"
)));
}
if vm.is_empty()
|| !vm
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
{
return Some(Err("vm.memory: vm is a ComputeVM name".into()));
}
let via_ops = cluster.strip_prefix("bidder:").filter(|n| self.ops.as_ref().is_some_and(|o| o.serves(n)));
let cv = atlibs_host::run::Call::call(
&crate::ops::OpsKube { ops: self.ops.clone(), kube: self.kube.clone() },
atlibs_kube::GET.into(),
atlibs_host::run::Request { input: Some(json!({ "cluster": cluster, "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM", "name": vm })), ..Default::default() },
).await;
match cv {
Ok(Some(o)) if o["spec"]["contract"]["receipt"]["uri"] == receipt => {}
Ok(_) => {
return Some(Err(format!(
"vm.memory: the ComputeVM {vm} is not the receipt's"
)));
}
Err(e) => return Some(Err(format!("vm.memory: {e}"))),
}
// a bidder behind its read-only ops routes: the figures only, no setHigh
if let (Some(n), Some(o)) = (via_ops, &self.ops) {
if !input["setHigh"].is_null() {
return Some(Err(format!("vm.memory: {cluster} is reached through the read-only ops routes: setHigh is not served")));
}
return Some(o.memory(n, vm).await.map_err(|e| format!("vm.memory: {e}")));
}
let set = match input["setHigh"].as_u64() {
Some(n) => format!("MemoryHigh={n}"),
None if input["setHigh"] == "max" => "MemoryHigh=infinity".to_string(),
None => String::new(),
};
// the VM unit is in the user manager of the account that runs the bidder: this account's (a dev network), or, as root on a
// joined bidder's host, the installed account's (swc-bidder)
let script = format!(
r#"unit_of() {{ $as systemctl --user list-units --all --plain --no-legend 'socialweb-vm-*-{vm}.service' 2>/dev/null | awk '{{print $1}}' | head -1; }}
as=""; u=""
if [ "$(id -u)" = 0 ]; then
for d in /run/user/*; do
n=$(id -un "${{d##*/}}" 2>/dev/null) || continue
[ "$n" != root ] || continue
as="runuser -u $n -- env XDG_RUNTIME_DIR=$d"
u=$(unit_of)
[ -z "$u" ] || break
done
else
export XDG_RUNTIME_DIR="${{XDG_RUNTIME_DIR:-/run/user/$(id -u)}}"
u=$(unit_of)
fi
[ -n "$u" ] || {{ echo '{{"error":"no VM unit of {vm} in a user manager of this host'"'"'s bidder account (a proc-backend bidder has no VM cgroup)"}}'; exit 0; }}
{setcmd}
cg=$($as systemctl --user show "$u" -p ControlGroup --value)
d=/sys/fs/cgroup$cg
[ -r "$d/memory.current" ] || {{ echo '{{"error":"no readable memory cgroup for the unit"}}'; exit 0; }}
ev=$(awk '{{printf "\"%s\":%s,", $1, $2}}' "$d/memory.events" | sed 's/,$//')
printf '{{"unit":"%s","current":%s,"high":"%s","max":"%s","swapMax":"%s","events":{{%s}}}}\n' "$u" "$(cat $d/memory.current)" "$(cat $d/memory.high)" "$(cat $d/memory.max)" "$(cat $d/memory.swap.max 2>/dev/null)" "$ev"
"#,
setcmd = if set.is_empty() {
String::new()
} else {
format!(
"$as systemctl --user set-property --runtime \"$u\" {set} || {{ echo '{{\"error\":\"set-property failed\"}}'; exit 0; }}"
)
},
);
// a joined bidder (staging's multi layout) holds its VMs on its own VM: there, as the target's ssh host of its kcp
let winner = cluster
.strip_prefix("bidder:")
.and_then(|n| self.target.bidders.iter().find(|b| b.name == n))
.and_then(|b| b.kcp.as_ref())
.and_then(|k| match &k.via {
testkit::target::Via::Ssh { host } => Some(host.clone()),
_ => None,
});
on_host(winner.as_deref(), &script)
.await
.and_then(|o| {
serde_json::from_str::<Value>(o.trim())
.map_err(|e| format!("vm.memory: {e}: {}", tail(&o)))
})
.and_then(|v| match v["error"].as_str() {
Some(e) => Err(format!("vm.memory: {e}")),
None => Ok(v),
})
}
// the shipped binary against a tampered copy of what it extracts (acceptance 0018, staging): variant file / fileSidecar
// flips a byte of a download in the extracted socialweb-guest-downloads tree (and, for fileSidecar, rewrites its
// sidecar to match) and runs `dev verify-assets` again on the same asset cache; variant record flips one digit of
// the embedded build record in a copy of the binary and runs it
"assets.tamper" => self.assets_tamper(input).await,
// submitAccept of an accept record that does not exist (a requester that deleted its accept before the bidder read it): the
// account's service auth to the bidder's market endpoint (its offering: https, else iroh), for at://<account>/market.accept/<rkey>
"bidder.submitAccept" => {
let (Ok(name), Ok(rkey), Ok(cid)) = (
str_of(input, "name"),
str_of(input, "rkey"),
str_of(input, "cid"),
) else {
return Some(Err("name, rkey and cid are required".into()));
};
let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
return Some(Err(format!("no bidder {name} in the target")));
};
async {
let at = testkit::Atproto::new(&self.target);
let sess = at.session(self.acct()?).await.map_err(|e| e.to_string())?;
let offering = at
.record(&format!(
"at://{}/com.publicdomainrelay.temp.market.offering/self",
b.did
))
.await
.map_err(|e| e.to_string())?
.ok_or("the bidder has no offering record")?;
let endpoint = testkit::irohfetch::market_base(&offering)
.ok_or("the offering names no https or iroh market endpoint")?;
let uri = format!(
"at://{}/com.publicdomainrelay.temp.market.accept/{rkey}",
sess.did()
);
let reply = at
.service_call(
&sess,
&endpoint,
&format!("{}#pdr_temp_market", b.did),
"com.publicdomainrelay.temp.market.submitAccept",
Some(&json!({ "acceptUri": uri, "acceptCid": cid })),
&[],
)
.await
.map_err(|e| e.to_string())?;
Ok(json!({ "acceptUri": uri, "reply": reply }))
}
.await
}
"bidder.restart" => {
let Ok(name) = str_of(input, "name") else {
return Some(Err("name is required".into()));
};
if !self.target.bidders.iter().any(|b| b.name == name) {
return Some(Err(format!("no bidder {name} in the target")));
}
let Some(n) = self.target.network.clone() else {
return Some(Err(
"bidder.restart: the target is no market network (staging only)".into(),
));
};
let t = Instant::now();
// a joined bidder (staging's multi layout) is installed on its own VM (`install bidder`): its core is the user unit
// socialweb-computer-bidder@<handle> of its account; restarted in place as `upgrade` does (the unit alone: its kcp and VM
// units stay and the new core adopts them)
let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
if let Some(testkit::target::Via::Ssh { host }) = via {
let script = BIDDER_RESTART.replace("@NAME@", name).replace("@UNIT_TIMEOUTS@", UNIT_TIMEOUTS);
let out = match on_host(Some(&host), &script).await {
Ok(o) => o,
Err(e) => return Some(Err(format!("bidder.restart {name}: {e}"))),
};
let Some(since) = out.lines().find_map(|l| l.strip_prefix("since ")).and_then(|n| n.trim().parse::<u64>().ok()) else {
return Some(Err(format!("bidder.restart {name}: {}", tail(&out))));
};
self.extra.log_since.lock().unwrap().entry(name.to_string()).or_insert(since);
return Some(Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0, "report": tail(&out) })));
}
let bin = self.bin.path.display().to_string();
let args: Vec<String> = [
"dev",
"restart",
&n.instance,
&format!("{name}/core"),
"--state-dir",
&testkit::target::expand(&n.state_dir).display().to_string(),
"--binary",
&bin,
]
.map(String::from)
.to_vec();
self.cli(&args).await.and_then(|(ok, out)| {
if ok {
Ok(json!({ "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0, "report": tail(&out) }))
} else {
Err(format!("dev restart {name}/core: {}", tail(&out)))
}
})
}
// a joined staging bidder upgraded in place to another release binary (`upgrade --binary --sha256 --state-dir` as its
// account, docs/ci-debug.md step 5d): binary `from` (the input's `from` spec, else SWC_UPGRADE_FROM, else
// SWC_COMPAT_REQUESTER: the previous rc) or `test` (the binary under test); both are staged on the bidder's host
// first. stopFirst: the core unit is stopped before the upgrade, so nothing the old core serves outlives the call
"bidder.upgrade" => {
let (Ok(name), Ok(which)) = (str_of(input, "name"), str_of(input, "binary")) else {
return Some(Err("name and binary (from or test) are required".into()));
};
if self.target.network.is_none() {
return Some(Err("bidder.upgrade: the target is no market network (staging only)".into()));
}
let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
let Some(testkit::target::Via::Ssh { host }) = via else {
return Some(Err(format!("bidder.upgrade {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
};
let t = Instant::now();
self.bidder_upgrade(name, &host, which, input["from"].as_str(), input["stopFirst"] == true).await.map(|mut v| {
v["seconds"] = json!((t.elapsed().as_secs_f64() * 10.0).round() / 10.0);
v
})
}
// how many lines of a bidder's log hold each text (a text: a string, or a list of strings a line must all
// hold), counted now (a module diffs two counts): a network bidder's core.log under the network's dev
// state, an installed one's journal (swc-bidder, read-only, over ssh); the texts never reach a shell
"bidder.logCount" | "bidder.logLines" => {
let Ok(name) = str_of(input, "name") else {
return Some(Err("name is required".into()));
};
let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
return Some(Err(format!("no bidder {name} in the target")));
};
let texts: Vec<Vec<String>> = input["texts"]
.as_array()
.into_iter()
.flatten()
.map(|t| match t {
Value::Array(a) => a
.iter()
.filter_map(|x| x.as_str().map(String::from))
.collect(),
t => t.as_str().map(|x| vec![x.to_string()]).unwrap_or_default(),
})
.collect();
if texts.is_empty()
|| texts.iter().any(|t| {
t.is_empty()
|| t.iter()
.any(|x| x.is_empty() || x.chars().any(char::is_control))
})
{
return Some(Err("texts: strings or lists of strings, none empty".into()));
}
let lines = what == "bidder.logLines";
let count = |log: &str| {
json!(
texts
.iter()
.map(|t| (
t.join(" & "),
json!(
log.lines()
.filter(|l| t.iter().all(|x| l.contains(x.as_str())))
.count()
)
))
.collect::<serde_json::Map<_, _>>()
)
};
// bidder.logLines: the JSON lines holding every text of the first entry of `texts`, as {time, msg, fields} of the
// keys named in `keys` (at most 50)
let pick = |log: &str| {
let keys: Vec<String> = input["keys"]
.as_array()
.into_iter()
.flatten()
.filter_map(|k| k.as_str().map(String::from))
.collect();
let rows: Vec<Value> = log
.lines()
.filter(|l| texts[0].iter().all(|x| l.contains(x.as_str())))
.filter_map(|l| serde_json::from_str::<Value>(l).ok())
.map(|v| {
let mut o = json!({ "time": v["time"], "msg": v["msg"] });
for k in &keys {
o[k] = v[k].clone();
}
o
})
.take(50)
.collect();
json!({ "lines": rows })
};
let out = |log: &str, from: &str| {
if lines {
let mut v = pick(log);
v["log"] = json!(from);
v
} else {
json!({ "counts": count(log), "log": from })
}
};
// a bidder the host reaches by ssh (prod, a staging joined bidder): its own host's log, from the process this run restarted
// when it restarted one
if let Some(o) = self.ops.as_ref().filter(|o| o.serves(name)) {
return Some(if lines {
let keys: Vec<String> = input["keys"].as_array().into_iter().flatten().filter_map(|k| k.as_str().map(String::from)).collect();
o.log_lines(name, &texts, &keys).await
} else {
o.log_count(name, &texts).await
});
}
let since = self.extra.log_since.lock().unwrap().get(name).copied();
let script = format!("since={}\n{BIDDER_LOG}", since.map(|n| n.to_string()).unwrap_or_default());
if let Some(testkit::target::Via::Ssh { host }) = b.kcp.as_ref().map(|k| &k.via) {
return Some(on_host(Some(host), &script).await.map(|l| out(&l, "journal")));
}
if let Some(n) = &self.target.network {
let i = &n.instance;
let p = testkit::target::expand(&n.state_dir)
.join(format!("dev/{i}/apps/dev/{i}-{name}/logs/core.log"));
return Some(
std::fs::read_to_string(&p)
.map(|l| out(&l, &p.display().to_string()))
.map_err(|e| format!("{}: {e}", p.display())),
);
}
let Some(testkit::target::Via::Ssh { host }) = b.kcp.as_ref().map(|k| &k.via)
else {
return Some(Err(format!(
"{name}: the host reaches its bidder log by ssh or in the network only"
)));
};
on_host(Some(host), &script)
.await
.map(|l| out(&l, "journal"))
}
// a bidder host's health without sudo (0006, 0008): its `run bidder` unit's ERROR lines since the process
// started (and in its first 10 min), automatic restarts since the last stop, VM processes, when it verified
// its pinned guest image (bidder_guest_image_ready) and how often it could not
"bidder.health" => {
let Ok(name) = str_of(input, "name") else {
return Some(Err("name is required".into()));
};
let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
return Some(Err(format!("no bidder {name} in the target")));
};
if let Some(o) = self.ops.as_ref().filter(|o| o.serves(name)) {
return Some(o.health(name).await.map_err(|e| format!("{name}: {e}")));
}
let via = match b.kcp.as_ref().map(|k| &k.via) {
Some(testkit::target::Via::Ssh { host }) => Some(host.clone()),
Some(testkit::target::Via::Direct) => None,
_ => {
return Some(Err(format!(
"{name}: the host reaches its bidder by ssh or locally only"
)));
}
};
on_host(via.as_deref(), BIDDER_HEALTH).await.and_then(|o| {
serde_json::from_str(o.trim()).map_err(|e| format!("{name}: {e}: {}", tail(&o)))
})
}
// the bidder host's /proc/meminfo MemTotal and MemAvailable in MiB and, for each of `vms` ({name, owner}), the anon MiB
// of its unit's cgroup (socialweb-vm-<sha256(owner)[:10]>-<name>.service), as the bidder reads them (vmsize.go): a module
// computes the memory the bidder can still admit exactly as the bidder does
"bidder.meminfo" => {
let Ok(name) = str_of(input, "name") else {
return Some(Err("name is required".into()));
};
let Some(b) = self.target.bidders.iter().find(|b| b.name == name) else {
return Some(Err(format!("no bidder {name} in the target")));
};
let via = match b.kcp.as_ref().map(|k| &k.via) {
Some(testkit::target::Via::Ssh { host }) => Some(host.clone()),
Some(testkit::target::Via::Direct) => None,
_ => return Some(Err(format!("{name}: the host reaches its bidder by ssh or locally only"))),
};
let mut units = String::new();
for v in input["vms"].as_array().into_iter().flatten() {
let (Some(vm), Some(owner)) = (v["name"].as_str(), v["owner"].as_str()) else {
return Some(Err("vms[] holds {name, owner}".into()));
};
if vm.is_empty() || !vm.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.') {
return Some(Err(format!("{vm}: not a VM name")));
}
let id: String = Sha256::digest(owner.as_bytes()).iter().map(|b| format!("{b:02x}")).collect();
units.push_str(&format!(" {vm}=socialweb-vm-{}-{vm}.service", &id[..10]));
}
on_host(via.as_deref(), &format!("units='{}'\n{MEMINFO}", units.trim()))
.await
.and_then(|o| serde_json::from_str(o.trim()).map_err(|e| format!("{name}: {e}: {}", tail(&o))))
}
// a guest's nf_tables tables on a staging bidder host (root over ssh: the run's own throwaway bidder VM), in the network
// namespace of the VMM of socialweb-vm-<sha256(owner)[:10]>-<vm>.service (nsenter -n nft). op list: the product's
// table bridge swcguest. probe: a test-only table bridge swcprobe, two counters and no verdict for UDP from the guest's
// tap to this host's own IPv4 addresses on ports min-max: chain sent before swcguest (priority -301), chain passed
// after it (-299: what swcguest accepted). counters: that table. unprobe: it deleted. Staging only
"bidder.guestNft" => {
let (Ok(name), Ok(vm), Ok(owner), Ok(op)) = (str_of(input, "name"), str_of(input, "vm"), str_of(input, "owner"), str_of(input, "op")) else {
return Some(Err("name, vm, owner and op are required".into()));
};
if self.target.network.is_none() {
return Some(Err("bidder.guestNft: the target is no market network (staging only)".into()));
}
if !["list", "probe", "counters", "unprobe"].contains(&op) {
return Some(Err(format!("bidder.guestNft: op {op}: list, probe, counters or unprobe")));
}
if vm.is_empty() || !vm.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') {
return Some(Err(format!("{vm}: not a VM name")));
}
let (min, max) = (input["min"].as_u64().unwrap_or(0), input["max"].as_u64().unwrap_or(0));
if op == "probe" && !(1024..=65535).contains(&min) || op == "probe" && !(min..=65535).contains(&max) {
return Some(Err(format!("bidder.guestNft probe: ports {min}-{max}")));
}
let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
let Some(testkit::target::Via::Ssh { host }) = via else {
return Some(Err(format!("bidder.guestNft {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
};
let id: String = Sha256::digest(owner.as_bytes()).iter().map(|b| format!("{b:02x}")).collect();
let script = GUEST_NFT
.replace("@UNIT@", &format!("socialweb-vm-{}-{vm}.service", &id[..10]))
.replace("@OP@", op)
.replace("@MIN@", &min.to_string())
.replace("@MAX@", &max.to_string());
on_host(Some(&host), &script).await.map(|o| {
let counter = |chain: &str| {
o.split(&format!("chain {chain} {{")).nth(1).and_then(|c| c.split("counter packets ").nth(1)).and_then(|c| c.split_whitespace().next()).and_then(|n| n.parse::<u64>().ok())
};
let pid = o.lines().find_map(|l| l.strip_prefix("vmm-pid ")).unwrap_or_default().to_string();
let own = o.lines().find_map(|l| l.strip_prefix("own ")).unwrap_or_default().to_string();
let out: String = o.lines().filter(|l| !l.starts_with("vmm-pid ") && !l.starts_with("own ")).collect::<Vec<_>>().join("\n");
json!({ "output": out, "vmmPid": pid, "own": own, "sent": counter("sent"), "passed": counter("passed") })
})
}
// the public URL of the binary under test (the bucket's main/<commit>/ or local/<commit>/ build of the commit its
// `version` names, whose SHA256SUMS lists exactly its sha256), for a guest to fetch at bucket speed
"binary.published" => {
async {
let sha256 = file_sha256(&self.bin.path).await?;
let commit = self.bin.version.split_whitespace().nth(1).filter(|c| c.len() == 40 && c.chars().all(|x| x.is_ascii_hexdigit())).ok_or_else(|| format!("the binary under test's version {:?} names no commit", self.bin.version))?;
let mut seen = vec![];
for spec in [format!("sha:{commit}"), format!("local:{commit}")] {
match testkit::requester::Binary::published(&spec).await {
Ok(Some((url, sum))) if sum == sha256 => return Ok(json!({ "url": url, "sha256": sum, "spec": spec })),
Ok(Some((_, sum))) => seen.push(format!("{spec}: {sum}")),
Ok(None) => {}
Err(e) => seen.push(format!("{spec}: {e}")),
}
}
Err(format!("the binary under test ({sha256}, commit {commit}) is not published: {}", seen.join("; ")))
}
.await
}
// the first line of a bidder's journal (since its run bidder process started) holding every text of `texts`, read
// as it is written (journalctl -f), or found false after `seconds`; installed staging bidders only
"bidder.awaitLog" => {
let Ok(name) = str_of(input, "name") else {
return Some(Err("name is required".into()));
};
let texts: Vec<String> = input["texts"].as_array().into_iter().flatten().filter_map(|t| t.as_str().map(String::from)).collect();
if texts.is_empty() || texts.iter().any(|t| t.is_empty() || t.chars().any(char::is_control)) {
return Some(Err("texts: strings, none empty".into()));
}
let seconds = input["seconds"].as_u64().unwrap_or(300).clamp(1, 3600);
let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
let Some(testkit::target::Via::Ssh { host }) = via else {
return Some(Err(format!("bidder.awaitLog {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
};
let quoted = format!("'{}'", texts.join("\u{1f}").replace('\'', "'\\''"));
let script = AWAIT_LOG.replace("@TEXTS@", "ed).replace("@SECS@", &seconds.to_string());
let t = Instant::now();
on_host(Some(&host), &script).await.map(|o| {
let line = o.lines().find_map(|l| l.strip_prefix("line ")).map(String::from);
let parsed = line.as_deref().and_then(|l| serde_json::from_str::<Value>(l).ok());
json!({ "found": line.is_some(), "line": parsed.unwrap_or(json!(line)), "seconds": (t.elapsed().as_secs_f64() * 10.0).round() / 10.0, "report": tail(&o) })
})
}
// the dumbpipe children of an installed bidder's run bidder process (its iroh ingress listeners: pid, executable,
// --host address), read-only from /proc over ssh; installed staging bidders only
"bidder.dumbpipe" => {
let Ok(name) = str_of(input, "name") else {
return Some(Err("name is required".into()));
};
let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
let Some(testkit::target::Via::Ssh { host }) = via else {
return Some(Err(format!("bidder.dumbpipe {name}: not an installed bidder this host reaches by ssh (staging's multi layout)")));
};
on_host(Some(&host), BIDDER_DUMBPIPE).await.and_then(|o| {
let core = o.lines().find_map(|l| l.strip_prefix("core ")).map(str::trim).unwrap_or_default().to_string();
if core.is_empty() {
return Err(format!("bidder.dumbpipe {name}: {}", tail(&o)));
}
let listeners: Vec<Value> = o
.lines()
.filter_map(|l| l.strip_prefix("listener "))
.map(|l| {
let w: Vec<&str> = l.split_whitespace().collect();
let host = w.iter().position(|x| *x == "--host").and_then(|p| w.get(p + 1)).copied().unwrap_or_default();
json!({ "pid": w.first().copied().unwrap_or_default(), "exe": w.get(1).copied().unwrap_or_default(), "host": host, "args": w.get(2..).map(|a| a.join(" ")).unwrap_or_default() })
})
.collect();
Ok(json!({ "core": core, "listeners": listeners }))
})
}
// the ssh gateway's core log lines of a DID since an ISO time (0002: revoke before release), read over ssh
"gateway.log" => {
let (Ok(did), Ok(since)) = (str_of(input, "did"), str_of(input, "since")) else {
return Some(Err("did and since are required".into()));
};
if !did.starts_with("did:") || did.contains(['\'', ' ']) {
return Some(Err("did is a DID".into()));
}
let msgs: Vec<String> = input["msgs"]
.as_array()
.into_iter()
.flatten()
.filter_map(|m| m.as_str().map(String::from))
.collect();
if let Some(o) = self.ops.as_ref().filter(|o| o.serves_gateway()) {
return Some(o.gateway_log(did, since, &msgs).await);
}
let Some(l) = self.target.gateway.log.clone() else {
return Some(Err("the target names no gateway.log".into()));
};
on_host(Some(&l.host), &format!("grep -F '\"did\":\"{did}\"' '{}' || true", l.path.replace('\'', ""))).await.map(|o| {
let all: Vec<Value> = o
.lines()
.filter_map(|l| serde_json::from_str::<Value>(l).ok())
.filter(|v| v["time"].as_str().is_some_and(|t| t >= since))
.collect();
crate::ops::gateway_lines(&all, &msgs)
})
}
"page.fetch" => match input["url"].as_str() {
Some(url) => {
self.page_fetch(
url,
input["method"].as_str().unwrap_or("GET"),
input.get("body"),
)
.await
}
None => Err("url is required".into()),
},
"ssh.tempKey" => self.temp_key().await,
"atirc.observerPub" => match (str_of(input, "nick"), input["text"].as_str()) {
(Ok(n), Some(t)) => self.observer_run(n, &["pub", t]).await,
_ => Err("nick and text are required".into()),
},
"atirc.observerRevoke" => match str_of(input, "nick") {
Ok(n) => self.observer_run(n, &["revoke"]).await,
Err(e) => Err(e),
},
// the owner's mesh node, as the hand run does it: `atirc mesh run --peer <identity>@<did>` as the account
// (an AtircNode's spec.peers takes no @did yet)
"atirc.meshRun" => {
let (Ok(name), Some(peer)) = (str_of(input, "name"), input["peer"].as_str()) else {
return Some(Err("name and peer are required".into()));
};
async {
let a = self.acct()?;
let bin = self.atirc()?;
let dir = self.dir.join(format!("atirc-{name}"));
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
std::fs::set_permissions(
&dir,
std::os::unix::fs::PermissionsExt::from_mode(0o700),
)
.map_err(|e| e.to_string())?;
let log =
std::fs::File::create(dir.join("mesh.log")).map_err(|e| e.to_string())?;
let child = tokio::process::Command::new(&bin)
.args([
// atirc names the owner's channel itself (its default_channels): a hand-made name
// differed for a handle with a dash (staging's), so the agent never dialed this node
"mesh",
"run",
"--peer",
peer,
"--irc-port",
"0",
])
.env("ATP_USERNAME", &a.handle)
.env("ATP_PASSWORD", self.password()?)
.env("ATIRC_CONFIG_DIR", &dir)
// the target's PLC (staging: the network's, https by name, plan 0050)
.env("ATIRC_PLC_DIRECTORY_URL", &self.target.plc)
.stdin(Stdio::null())
.stdout(log.try_clone().map_err(|e| e.to_string())?)
.stderr(log)
.kill_on_drop(true)
.spawn()
.map_err(|e| e.to_string())?;
self.extra
.nodes
.lock()
.await
.insert(name.to_string(), (bin.clone(), dir.clone(), child));
// ready when `mesh peers` answers
let deadline = Instant::now() + Duration::from_secs(90);
loop {
let ok = tokio::process::Command::new(&bin)
.args(["mesh", "peers"])
.env("ATIRC_CONFIG_DIR", &dir)
.env("ATP_USERNAME", &a.handle)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.await
.is_ok_and(|s| s.success());
if ok {
return Ok(json!({ "node": name }));
}
if Instant::now() > deadline {
return Err(
"atirc mesh run did not answer in 90 s (mesh.log in the work dir)"
.into(),
);
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
.await
}
"logs.fetchRuns" => self.fetch_runs(input).await,
"requester.ssh" => {
let (Ok(name), Ok(contract), Ok(command)) =
(str_of(input, "name"), str_of(input, "contract"), exec_words(input))
else {
return Some(Err("name, contract, exec or argv are required".into()));
};
let mut extra = vec!["--contract", contract];
if let Some(t) = input["transport"].as_str() {
extra.extend(["--transport", t]);
}
extra.extend(command.iter().map(String::as_str));
self.requester_cli("ssh", name, &extra)
.await
.map(|(ok, out)| json!({ "ok": ok, "output": out }))
}
"requester.restart" => match str_of(input, "name") {
Ok(n) => self.requester_restart(n).await,
Err(e) => Err(e),
},
"requester.cluster" => {
let (Ok(name), Ok(cluster)) = (str_of(input, "name"), str_of(input, "cluster"))
else {
return Some(Err("name and cluster are required".into()));
};
async {
let (ok, out) = self
.requester_cli("cluster", name, &["--cluster", cluster])
.await?;
if !ok {
return Err(format!("run requester cluster: {}", tail(&out)));
}
let deadline = Instant::now() + Duration::from_secs(600);
let path = loop {
let (ok, out) = self
.requester_cli("kubeconfig", name, &["--cluster", cluster])
.await?;
if let Some(p) = out
.lines()
.map(str::trim)
.find(|l| l.starts_with('/'))
.filter(|_| ok)
{
break p.to_string();
}
if Instant::now() > deadline {
return Err(format!("cluster {cluster}: no kubeconfig in 10 min"));
}
tokio::time::sleep(Duration::from_secs(5)).await;
};
self.kube.add_cluster(
&format!("tenant:{cluster}"),
Cluster {
kubeconfig: path,
workspace: None,
via: Default::default(),
},
);
Ok(json!({ "cluster": format!("tenant:{cluster}") }))
}
.await
}
"requester.bundle" => self.requester_bundle(input).await,
"staging.pki" => self.staging_pki(input).await,
"requester.records" => match (str_of(input, "name"), self.acct()) {
(Ok(name), Ok(a)) => {
let mut logs = vec![];
let state = self.requester_state(name);
match &self.target.network {
// a network's state dir holds every requester of the network: only this one's (plan 0044 finding 26)
Some(n) => {
let i = &n.instance;
for d in [state.join("requesters").join(name), state.join(format!("dev/{i}/apps/dev/{i}-{name}"))] {
find_named(&d, "core.log", &mut logs);
}
}
None => find_named(&state, "core.log", &mut logs),
}
logs.push(state.join(format!("{name}.log")));
let declined = declined_accepts(&logs);
Ok(json!({ "uris": uris_in(&logs, &a.did).into_iter().filter(|u| !kept_by_rule(u) && !declined.contains(u)).collect::<Vec<_>>(),
"declinedAccepts": declined.len(), "declined": declined }))
}
(Err(e), _) | (_, Err(e)) => Err(e),
},
"page.round" => {
let Some(repo) = self.target.page_repo.clone() else {
return Some(Err("the target has no pageRepo".into()));
};
let bidder = input["bidder"].as_str().unwrap_or_default().to_string();
async {
let a = self.acct()?;
let root = expand(&repo);
// the checkout's own pinned node and Chrome (a staging VM's, tests/compositions/staging/page.sh), else the machine's
let tool = |p: &str| Some(root.join("tools").join(p)).filter(|p| p.exists());
let mut cmd = tokio::process::Command::new(tool("node/bin/node").unwrap_or_else(|| "node".into()));
cmd.current_dir(&root)
.args(["scripts/page-round.mjs", &bidder, "uname -a; id"])
.env("ALICE_ATPROTO_HANDLE", &a.handle)
.env("ALICE_ATPROTO_PASSWORD", self.password()?)
.stdin(Stdio::null());
if let Some(chrome) = tool("chrome/chrome") {
cmd.env("CHROME", chrome);
}
// a network of its own: the page signs in at the target's PDS, resolves DIDs at its PLC and asks its relay; the
// script's fetches (node) and the page's (Chrome) trust the network CA
if self.target.name != "prod" {
cmd.env("PAGE_PDS", &self.target.pds).env("PAGE_PLC", &self.target.plc);
if let Some(relay) = &self.target.relay {
cmd.env("PAGE_RELAY", relay);
}
if let Some(f) = &self.target.xrpc_front {
let ca = expand(&f.ca);
let names: Vec<String> = [Some(&self.target.plc), Some(&self.target.pds), self.target.relay.as_ref()]
.into_iter()
.flatten()
.filter_map(|u| reqwest::Url::parse(u).ok())
.filter(|u| u.scheme() == "https")
.filter_map(|u| u.host_str().map(String::from))
.collect();
cmd.env("NODE_EXTRA_CA_CERTS", &ca);
let spki = front_spki(f.address, &ca, &names).await?;
if !spki.is_empty() {
cmd.env("PAGE_CHROME_ARGS", json!([format!("--ignore-certificate-errors-spki-list={spki}")]).to_string());
}
}
}
let out = tokio::time::timeout(Duration::from_secs(900), cmd.output())
.await
.map_err(|_| "page-round did not end in 15 min".to_string())?
.map_err(|e| e.to_string())?;
let text = format!("{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
if text.contains(&self.password()?) {
return Err("page-round printed the password".into());
}
Ok(json!({ "ok": out.status.success(), "lines": text.lines().collect::<Vec<_>>() }))
}
.await
}
"example" => match input["name"]
.as_str()
.filter(|n| !n.contains('/') && n.ends_with(".yaml"))
{
Some(n) => std::fs::read_to_string(testkit::repo().join("examples/infra").join(n))
.map(|t| json!({ "text": t }))
.map_err(|e| format!("{n}: {e}")),
None => Err("name is an examples/infra file".into()),
},
"request.r2" => {
let (Ok(name), Ok(r2)) = (str_of(input, "name"), str_of(input, "r2Name")) else {
return Some(Err("name and r2Name are required".into()));
};
let mut extra = vec!["--r2-name", r2, "--hold"];
if input["agent"] == true {
extra.push("--agent");
}
let args = self.sub_args("r2", name, &extra);
self.cli(&args).await.and_then(|(ok, out)| {
if ok {
Ok(json!({}))
} else {
Err(format!("run request r2: {}", tail(&out)))
}
})
}
"request.checkLogs" => match (str_of(input, "name"), self.password()) {
(Ok(name), Ok(pw)) => {
let mut problems = vec![];
for p in self.request_logs(name) {
let Ok(text) = std::fs::read_to_string(&p) else {
continue;
};
if text.contains(&pw) {
problems.push(format!("{}: the app password", p.display()));
}
for bad in ["a=candidate", "AGE-SECRET-KEY"] {
if text.contains(bad) {
problems.push(format!("{}: plaintext {bad}", p.display()));
}
}
}
Ok(json!({ "ok": problems.is_empty(), "problems": problems }))
}
(Err(e), _) | (_, Err(e)) => Err(e),
},
"release.put" | "release.delete" => {
let did = match self.acct() {
Ok(a) => a.did.clone(),
Err(e) => return Some(Err(e)),
};
async {
let base = self.target.pds.trim_end_matches('/').to_string();
let s: Value = self
.http
.post(format!("{base}/xrpc/com.atproto.server.createSession"))
.json(&json!({ "identifier": did, "password": self.password()? }))
.send()
.await
.map_err(|e| e.to_string())?
.json()
.await
.map_err(|e| e.to_string())?;
let jwt = s["accessJwt"].as_str().ok_or_else(|| format!("createSession {did}: {}", s["error"]))?.to_string();
let call = |nsid: &'static str, body: Value| {
let (http, base, jwt) = (self.http.clone(), base.clone(), jwt.clone());
async move {
let r = http.post(format!("{base}/xrpc/{nsid}")).bearer_auth(jwt).json(&body).send().await.map_err(|e| e.to_string())?;
let status = r.status();
let v: Value = r.json().await.unwrap_or_default();
if !status.is_success() {
return Err(format!("{nsid}: {status} {} {}", v["error"], v["message"]));
}
Ok::<Value, String>(v)
}
};
let mut out = Vec::new();
if what == "release.put" {
for r in input["records"].as_array().cloned().unwrap_or_default() {
let v = call("com.atproto.repo.putRecord", json!({ "repo": did, "collection": r["collection"], "rkey": r["rkey"], "record": r["record"], "validate": false })).await?;
out.push(json!({ "uri": v["uri"], "cid": v["cid"] }));
}
} else {
for uri in input["uris"].as_array().cloned().unwrap_or_default() {
let u = uri.as_str().unwrap_or_default();
let parts: Vec<&str> = u.trim_start_matches("at://").splitn(3, '/').collect();
if parts.len() == 3 {
call("com.atproto.repo.deleteRecord", json!({ "repo": did, "collection": parts[1], "rkey": parts[2] })).await?;
out.push(json!({ "uri": u }));
}
}
}
Ok(json!({ "records": out }))
}
.await
}
"root.exec" => {
let Some(host) = (self.target.name != "prod")
.then(|| self.target.root_hosts.first().cloned())
.flatten()
else {
return Some(Err(
"root steps run only on a staging target with rootHosts".into(),
));
};
let cmd = input["cmd"].as_str().unwrap_or_default().to_string();
async {
let mut c = tokio::process::Command::new("ssh");
c.args(["-o", "BatchMode=yes", &host, &cmd]).stdout(Stdio::piped()).stderr(Stdio::piped()).stdin(Stdio::piped());
let mut child = c.spawn().map_err(|e| e.to_string())?;
let mut sin = child.stdin.take().unwrap();
if input["stdinPassword"] == true {
use tokio::io::AsyncWriteExt;
sin.write_all(self.password()?.as_bytes()).await.map_err(|e| e.to_string())?;
}
drop(sin);
let out = child.wait_with_output().await.map_err(|e| e.to_string())?;
Ok(json!({ "exit": out.status.code(), "stdout": String::from_utf8_lossy(&out.stdout), "stderr": tail(&String::from_utf8_lossy(&out.stderr)) }))
}
.await
}
"root.putBinary" => {
let Some(host) = (self.target.name != "prod")
.then(|| self.target.root_hosts.first().cloned())
.flatten()
else {
return Some(Err(
"root steps run only on a staging target with rootHosts".into(),
));
};
let remote = "/root/swc-st-socialweb-computer";
// several entries share this path (0001:s2, 0013:b, 0020): the copy goes beside it and is renamed over
// it, so an entry running the binary keeps its file and none sees a half-written one
let part = format!("{remote}.{}.part", std::process::id());
async {
let ok = tokio::process::Command::new("scp")
.args(["-q", "-o", "BatchMode=yes"])
.arg(&self.bin.path)
.arg(format!("{host}:{part}"))
.status()
.await
.map_err(|e| e.to_string())?
.success();
if !ok {
return Err(format!("scp to {host} failed"));
}
let ok = tokio::process::Command::new("ssh")
.args(["-o", "BatchMode=yes", &host, "mv", "-f", &part, remote])
.status()
.await
.map_err(|e| e.to_string())?
.success();
if ok {
Ok(json!({ "path": remote }))
} else {
Err(format!("mv on {host} failed"))
}
}
.await
}
_ => return None,
})
}
pub(crate) async fn more_sub(&self, what: &str, input: &Value) -> Option<Result<Sub, String>> {
Some(match what {
"atirc.observer" => self.observer_sub(input).await,
"requester.events" => match str_of(input, "name") {
Ok(n) => {
let state = self.requester_state(n);
let main = state.join(format!("{n}.log"));
Ok(tail_sub(
move || {
let mut v = vec![main.clone()];
find_named(&state, "core.log", &mut v);
v
},
None,
market_lines,
))
}
Err(e) => Err(e),
},
_ => return None,
})
}
pub(crate) async fn more_cleanup(&self) -> Vec<String> {
let mut errors = vec![];
let upgraded: Vec<(String, String)> = self.extra.upgraded.lock().unwrap().clone();
for (name, from) in upgraded {
let via = self.target.bidders.iter().find(|b| b.name == name).and_then(|b| b.kcp.as_ref()).map(|k| k.via.clone());
let Some(testkit::target::Via::Ssh { host }) = via else {
errors.push(format!("bidder {name}: left on the from binary (no ssh host)"));
continue;
};
if let Err(e) = self.bidder_upgrade(&name, &host, "test", Some(&from), false).await {
errors.push(format!("bidder {name} back to the binary under test: {e}"));
}
}
for (_, (_, _, mut c)) in self.extra.nodes.lock().await.drain() {
let _ = c.kill().await;
}
let nicks: Vec<String> = self.extra.tokens.lock().unwrap().keys().cloned().collect();
for n in nicks {
if let Err(e) = self.observer_run(&n, &["revoke"]).await {
errors.push(format!("revoke observer {n}: {e}"));
}
}
let keys: Vec<_> = std::mem::take(&mut *self.extra.keys.lock().unwrap());
for (web, rkey) in keys {
if let Err(e) = self
.page_fetch(&format!("{web}/keys/{rkey}"), "DELETE", None)
.await
{
errors.push(format!("remove the temporary key: {e}"));
}
}
errors
}
}
impl Swc {
/// resources.json: everything this run started or created, how to reach it, no secret (paths only).
pub async fn resources(&self) -> Value {
let requesters: Vec<Value> = self
.requesters
.lock()
.await
.iter()
.map(|(n, (state, c))| {
json!({ "name": n, "state": state, "pid": c.id(), "cluster": format!("requester:{n}"), "log": state.join(format!("{n}.log")),
"ssh": format!("socialweb-computer run requester ssh {n} --state-dir {} --contract <cc>", state.display()) })
})
.collect();
let requests: Vec<Value> = self
.requests
.lock()
.unwrap()
.iter()
.map(|(n, state)| {
let summary: Value = std::fs::read(state.join("requests").join(n).join("summary.json")).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default();
json!({ "name": n, "state": state, "receipt": summary["contract"]["receiptUri"], "winner": summary["contract"]["winnerDid"], "logs": self.request_logs(n),
"ssh": format!("socialweb-computer run request ssh {n} --state-dir {}", state.display()) })
})
.collect();
json!({
"account": self.acct().map(|a| a.did.clone()).unwrap_or_default(),
"requesters": requesters,
"requests": requests,
"kube": self.kube.ledger(),
"clusters": self.kube.clusters(),
"receipts": *self.receipts.lock().unwrap(),
"keys": self.extra.keys.lock().unwrap().iter().map(|(w, r)| json!({ "web": w, "rkey": r })).collect::<Vec<_>>(),
"observers": self.extra.tokens.lock().unwrap().keys().cloned().collect::<Vec<_>>(),
"envFiles": *self.env_files.lock().unwrap(),
})
}
pub async fn write_resources(&self) {
let v = self.resources().await;
let _ = std::fs::write(
self.dump.join("resources.json"),
serde_json::to_string_pretty(&v).unwrap_or_default(),
);
}
}
/// `swc-test-host cleanup <run>`: from resources.json alone (the host that ran it may be gone): release the
/// invocations, stop the requesters, delete and restore kube objects, remove the temporary keys, revoke observer
/// tokens (a fresh account token revokes the subhandle), remove the env files. The errors.
pub async fn cleanup_from_disk(
dir: &std::path::Path,
target: &testkit::target::Target,
account: &str,
bin: &testkit::requester::Binary,
) -> Vec<String> {
let mut errors = vec![];
let Ok(text) = std::fs::read_to_string(dir.join("resources.json")) else {
return vec![format!("{}: no resources.json", dir.display())];
};
let r: Value = serde_json::from_str(&text).unwrap_or_default();
for q in r["requests"].as_array().into_iter().flatten() {
let (n, st) = (
q["name"].as_str().unwrap_or_default(),
q["state"].as_str().unwrap_or_default(),
);
match bin
.run(&["run", "request", "release", n, "--state-dir", st])
.await
{
Ok((true, _)) => {}
Ok((false, out)) => errors.push(format!("release {n}: {}", tail(&out))),
Err(e) => errors.push(format!("release {n}: {e}")),
}
}
let clusters: HashMap<String, Cluster> =
serde_json::from_value(r["clusters"].clone()).unwrap_or_default();
let kube = atlibs_kube::Kube::new(clusters, None);
if let Err(e) = kube.load_ledger(&r["kube"]) {
errors.push(format!("kube ledger: {e}"));
}
errors.extend(kube.cleanup().await);
// a deleted Infra's finalizer (it releases the contracts) runs in its requester: wait for it before the stop
for k in r["kube"]["created"].as_array().into_iter().flatten() {
let input = json!({ "cluster": k["cluster"], "apiVersion": k["apiVersion"], "kind": k["kind"], "name": k["name"] });
for _ in 0..60 {
let got = atlibs_host::run::Call::call(
&*kube,
atlibs_kube::GET.into(),
atlibs_host::run::Request {
input: Some(input.clone()),
..Default::default()
},
)
.await;
if !matches!(&got, Ok(Some(v)) if v.get("metadata").is_some()) {
break;
}
tokio::time::sleep(Duration::from_secs(10)).await;
}
}
for q in r["requesters"].as_array().into_iter().flatten() {
if let Some(pid) = q["pid"].as_u64()
&& std::path::Path::new(&format!("/proc/{pid}")).exists()
{
let _ = std::process::Command::new("kill")
.args(["-INT", &pid.to_string()])
.status();
}
}
let Ok(acct) = target.account(account) else {
return errors;
};
let at = testkit::Atproto::new(target);
let keys = r["keys"].as_array().is_some_and(|k| !k.is_empty());
if keys && let Ok(s) = at.session(acct).await {
for k in r["keys"].as_array().into_iter().flatten() {
if let Err(e) = at
.delete_record(
&s,
"com.publicdomainrelay.temp.badgeBlueKeys",
k["rkey"].as_str().unwrap_or_default(),
)
.await
{
errors.push(format!("temporary key: {e}"));
}
}
}
if let (Some(endpoint), Ok(atirc)) = (target.atirc.clone(), testkit::atirc::atirc_bin()) {
for nick in r["observers"]
.as_array()
.into_iter()
.flatten()
.filter_map(Value::as_str)
{
let pw = acct
.password()
.map(|p| p.expose().to_string())
.unwrap_or_default();
let out = tokio::process::Command::new(&atirc)
.args(["token", "--endpoint", &endpoint])
.env("ATP_USERNAME", &acct.handle)
.env("ATP_PASSWORD", pw)
.env("ATIRC_PLC_DIRECTORY_URL", &target.plc)
.stdin(Stdio::null())
.output()
.await;
let Ok(out) = out else { continue };
let token = String::from_utf8_lossy(&out.stdout).trim().to_string();
let ok = tokio::process::Command::new(&atirc)
.args(["revoke", &format!("{nick}.{}", acct.handle)])
.env("ATIRC_TOKEN", token)
.env("ATIRC_ENDPOINT", &endpoint)
.env("ATIRC_PLC_DIRECTORY_URL", &target.plc)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.await
.is_ok_and(|s| s.success());
if !ok {
errors.push(format!("revoke observer {nick}"));
}
}
}
for p in r["envFiles"]
.as_array()
.into_iter()
.flatten()
.filter_map(Value::as_str)
{
let _ = std::fs::remove_file(p);
}
errors
}
/// bidder.restart on a joined bidder's host, as root or as the bidder's account: the installed bidder @NAME@'s core unit restarted through
/// its account's user manager (what `upgrade` restarts: install.EnabledCoreUnits). Prints `since <epoch>` (the start of the process it
/// replaced) and the unit's MainPID before and after; exit 1 when @NAME@ has no installed unit here or the restart failed.
const BIDDER_RESTART: &str = r#"unit=""
for p in $(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder'); do
u=$(grep -o '[^/]*bidder@@NAME@\.service' /proc/$p/cgroup 2>/dev/null | tail -1)
[ -n "$u" ] || continue
pid=$p; unit=$u; break
done
[ -n "$unit" ] || { echo 'no installed bidder unit of @NAME@ on this host (install bidder)'; exit 1; }
user=$(ps -o user= -p $pid | tr -d ' ')
U=$(id -u "$user")
since=$(date -d "$(ps -o lstart= -p $pid)" +%s)
sc() { if [ "$(id -u)" = "$U" ]; then XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; else runuser -u "$user" -- env XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; fi; }
@UNIT_TIMEOUTS@
echo "budget $(( stop_s + start_s ))"
before=$(sc show "$unit" -p MainPID --value)
sc restart "$unit" || { echo "systemctl --user restart $unit failed"; sc status "$unit" --no-pager 2>&1 | tail -5; exit 1; }
after=$(sc show "$unit" -p MainPID --value)
[ -n "$after" ] && [ "$after" != 0 ] && [ "$after" != "$before" ] || { echo "$unit: MainPID $before -> $after after the restart"; exit 1; }
echo "since $since"
echo "restarted $unit of $user: MainPID $before -> $after, $(sc show "$unit" -p ActiveState --value)"
"#;
/// the installed unit's own systemd start and stop timeouts in seconds (start_s, stop_s), for a script's budget line
const UNIT_TIMEOUTS: &str = r#"tsec() { systemd-analyze timespan "$(sc show "$unit" -p "$1" --value)" 2>/dev/null | awk 'NF == 2 && $2 ~ /^[0-9]+$/ { print int($2 / 1000000); exit }'; }
start_s=$(tsec TimeoutStartUSec); stop_s=$(tsec TimeoutStopUSec); start_s=${start_s:-0}; stop_s=${stop_s:-0}"#;
/// bidder.logCount on a bidder host: the swc-bidder account's journal since its run bidder process started (`since`, when set: from
/// that epoch instead, the process a restart of this run replaced).
const BIDDER_LOG: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || pid=$(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || exit 0
user=$(ps -o user= -p $pid | tr -d ' ')
if grep -q '\.service' /proc/$pid/cgroup && ! grep -q 'session-[0-9]*\.scope' /proc/$pid/cgroup; then
journalctl _UID=$(id -u "$user") --since "@${since:-$(date -d "$(ps -o lstart= -p $pid)" +%s)}" -o cat --no-pager 2>/dev/null
else
cat $(eval echo "~$user")/.local/state/socialweb-computer/dev/b-*/logs/core.log 2>/dev/null
fi
"#;
/// bidder.guestNft on a bidder host, as root: the VMM of @UNIT@ is its process in a network namespace other than pid 1's.
const GUEST_NFT: &str = r#"unit='@UNIT@'; op='@OP@'; min=@MIN@; max=@MAX@
command -v nft >/dev/null || { echo "nft is not installed on $(hostname)"; exit 1; }
pid=
for f in /proc/[0-9]*/cgroup; do
grep -q "/$unit\$" "$f" 2>/dev/null || continue
p=${f#/proc/}; p=${p%/cgroup}
[ "$(readlink /proc/$p/ns/net)" != "$(readlink /proc/1/ns/net)" ] && { pid=$p; break; }
done
[ -n "$pid" ] || { echo "no process of $unit in a guest network namespace on $(hostname)"; exit 1; }
echo "vmm-pid $pid"
own=$(ip -4 -o addr show scope global | awk '{split($4,a,"/"); printf "%s%s", s, a[1]; s=", "}')
echo "own $own"
n() { nsenter -t "$pid" -n nft "$@"; }
case "$op" in
list) n list table bridge swcguest ;;
probe)
n delete table bridge swcprobe 2>/dev/null
n -f - <<EOF || exit 1
table bridge swcprobe {
chain sent { type filter hook prerouting priority -301; policy accept; iifname "fctap0" ip daddr { $own } udp dport $min-$max counter; }
chain passed { type filter hook prerouting priority -299; policy accept; iifname "fctap0" ip daddr { $own } udp dport $min-$max counter; }
}
EOF
n list table bridge swcprobe ;;
counters) n list table bridge swcprobe ;;
unprobe) n delete table bridge swcprobe ;;
esac
"#;
/// bidder.awaitLog on a bidder host: the installed bidder's journal from its run bidder process's start, followed until a
/// line holds every text (one line "line <text>"), at most @SECS@ seconds.
const AWAIT_LOG: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || { echo "no run bidder process of swc-bidder on $(hostname)"; exit 1; }
start=$(date -d "$(ps -o lstart= -p $pid)" +%s)
echo "budget @SECS@"
timeout @SECS@ journalctl _UID=$(id -u swc-bidder) --since "@$start" -o cat -f --no-pager 2>/dev/null |
T=@TEXTS@ awk 'BEGIN { n = split(ENVIRON["T"], a, "\037") } { for (i = 1; i <= n; i++) if (!index($0, a[i])) next; print "line " $0; fflush(); exit }'
exit 0
"#;
/// bidder.dumbpipe on a bidder host: the run bidder process ("core <pid>") and each dumbpipe child of it ("listener <pid>
/// <exe> <argv>"), from /proc, no sudo.
const BIDDER_DUMBPIPE: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || pid=$(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || { echo "no run bidder process of swc-bidder on $(hostname)"; exit 1; }
echo "core $pid"
for c in $(pgrep -P "$pid" -x dumbpipe); do
echo "listener $c $(readlink /proc/$c/exe 2>/dev/null) $(tr '\0' ' ' < /proc/$c/cmdline 2>/dev/null)"
done
exit 0
"#;
/// bidder.meminfo on a bidder host: MemTotal and MemAvailable in MiB, and the anon MiB of each named VM unit's cgroup
/// memory.stat (`units` holds vm=unit pairs), as the bidder reads them; a unit without a cgroup is left out.
const MEMINFO: &str = r#"awk '/^MemTotal:/{t=$2} /^MemAvailable:/{a=$2} END{printf "{\"memTotalMiB\":%d,\"memAvailableMiB\":%d,\"anonMiB\":{", t/1024, a/1024}' /proc/meminfo
sep=
for u in $units; do
vm=${u%%=*}; unit=${u#*=}
d=$(find /sys/fs/cgroup -type d -name "$unit" -print -quit 2>/dev/null)
[ -n "$d" ] || continue
a=$(awk '$1=="anon"{printf "%d", $2/1048576}' "$d/memory.stat" 2>/dev/null)
[ -n "$a" ] || continue
printf '%s"%s":%s' "$sep" "$vm" "$a"; sep=,
done
echo '}}'
"#;
/// bidder.health on a bidder host: read-only, no sudo (journal of the account swc-bidder, /proc), one JSON line.
const BIDDER_HEALTH: &str = r#"pid=$(pgrep -u swc-bidder -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || pid=$(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder' | head -1)
[ -n "$pid" ] || { echo '{"error":"no run bidder process"}'; exit 0; }
user=$(ps -o user= -p $pid | tr -d ' ')
unit=$(grep -o '[^/]*\.service' /proc/$pid/cgroup | tail -1)
start=$(date -d "$(ps -o lstart= -p $pid)" +%s)
U=$(id -u "$user")
# the lines of this core start: every line of the core unit's current invocation (the core and its forks), and every
# other line of the bidder's account (the units the core starts) logged at or after this process's start instant (its
# /proc/<pid>/stat clock ticks against a process started after the wall clock is read: never late; less 30 ms, two
# ticks and slack; a line by its own `time`, else its journal time); never a core unit line of another invocation (the
# previous core, also when its shutdown overlaps this start); a `run bidder --fg` (staging) writes its instance log,
# whose lines name no pid: those from the same start instant
d=$(date +%s.%N)
rst=$(sh -c 'sed "s/^.*) //" /proc/$$/stat | cut -d" " -f20')
t0=$(awk -v st="$(sed 's/^.*) //' /proc/$pid/stat | cut -d' ' -f20)" -v hz="$(getconf CLK_TCK)" -v d="$d" -v rst="$rst" \
'BEGIN { printf "%.3f", d + (st - rst) / hz - 0.03 }')
until=$(awk -v t="$t0" 'BEGIN { printf "%.3f", t + 600 }')
since=$(( ${t0%.*} - 60 ))
epoch='def epoch: capture("^(?<d>[0-9-]{10}T[0-9:]{8})(?<f>[.][0-9]+)?(?<z>Z|[+-][0-9]{2}:[0-9]{2})$")
| (.d + "Z" | fromdateiso8601) + ("0" + (.f // "") | tonumber)
- (if .z == "Z" then 0 else (if .z[0:1] == "-" then -1 else 1 end) * ((.z[1:3] | tonumber) * 3600 + (.z[4:6] | tonumber) * 60) end);'
if [ -n "$unit" ] && ! grep -q 'session-[0-9]*\.scope' /proc/$pid/cgroup; then
inv=$( (tr '\0' '\n' < /proc/$pid/environ) 2>/dev/null | sed -n 's/^INVOCATION_ID=//p')
[ -n "$inv" ] || inv=$(journalctl _PID=$pid --since "@$since" -n 1 -o export --no-pager 2>/dev/null | sed -n 's/^_SYSTEMD_INVOCATION_ID=//p' | head -1)
log() {
c=1e12
[ "${1:-}" = --until ] && c=${2#@}
journalctl _UID=$U --since "@$since" -o json --no-pager 2>/dev/null | jq -r --argjson s "$t0" --argjson c "$c" --arg unit "$unit" \
--arg inv "$inv" --arg pid "$pid" "$epoch"'
select((.MESSAGE | type) == "string")
| (._SYSTEMD_USER_UNIT == $unit and (._PID == $pid or ($inv != "" and ._SYSTEMD_INVOCATION_ID == $inv))) as $core
| select($core or ._SYSTEMD_USER_UNIT != $unit or $inv == "")
| select(((.MESSAGE | fromjson? | .time | strings | epoch) // ((.__REALTIME_TIMESTAMP | tonumber) / 1000000)) as $t
| ($core or $t >= $s) and $t < $c)
| .MESSAGE'
}
else
unit=""
log() {
f=$(ls $(eval echo "~$user")/.local/state/socialweb-computer/dev/b-*/logs/core.log 2>/dev/null | head -1)
[ -n "$f" ] || return 0
c=1e12
[ "${1:-}" = --until ] && c=${2#@}
jq -cR --argjson s "$t0" --argjson c "$c" "$epoch"'fromjson? | select(((.time // "") | epoch) as $t | $t >= $s and $t < $c)' "$f"
}
fi
lines=$(log)
errors=$(grep -c '"level":"ERROR"' <<<"$lines")
startup=$(log --until "@$until" | grep -c '"level":"ERROR"\|unsupported type: source.TypedFunc')
sample=$(grep '"level":"ERROR"' <<<"$lines" | tail -3 | jq -c '{time, msg, err: ((.err // .error // "") | tostring | .[0:160])}' | jq -sc .)
restarts=$(journalctl _UID=$U USER_UNIT=$unit --since -14d -o cat --no-pager 2>/dev/null | awk '/Stopping /{n=0} /Scheduled restart job/{n++} END{print n+0}')
procs=$(( $(pgrep -c -u "$user" -x firecracker) + $(pgrep -c -u "$user" -f qemu-system) ))
ready=$(grep -m1 '"msg":"bidder_guest_image_ready"' <<<"$lines" | jq -r '.time // ""')
unready=$(grep -c '"msg":"bidder_guest_image_unready"' <<<"$lines")
jq -nc --arg unit "$unit" --arg started "$(date -u -d @$start +%FT%TZ)" \
--argjson errors "$errors" --argjson startup "$startup" --argjson restarts "$restarts" --argjson procs "$procs" --argjson sample "${sample:-[]}" \
--arg ready "$ready" --argjson unready "$unready" \
'{unit: $unit, started: $started, errors: $errors, startupErrors: $startup, restarts: $restarts, vmProcesses: $procs, errorSample: $sample,
guestImageReadyAt: $ready, guestImageUnready: $unready}'
"#;
impl Swc {
async fn bidder_upgrade(&self, name: &str, host: &str, which: &str, from: Option<&str>, stop_first: bool) -> Result<Value, String> {
let spec = match from {
Some(f) => f.to_string(),
None => std::env::var("SWC_UPGRADE_FROM")
.or_else(|_| std::env::var("SWC_COMPAT_REQUESTER"))
.map_err(|_| "bidder.upgrade: no from spec in the input and SWC_UPGRADE_FROM and SWC_COMPAT_REQUESTER are unset (the from binary: sha:<commit>, release:<version>, path:<file>)".to_string())?,
};
let from = Upgradable::of(&spec, None).await.map_err(|e| format!("bidder.upgrade: the from binary {spec}: {e}"))?;
let test = Upgradable::of(&self.target.requester, Some(&self.bin.path)).await.map_err(|e| format!("bidder.upgrade: the binary under test {}: {e}", self.target.requester))?;
let want = match which {
"from" => &from,
"test" => &test,
w => return Err(format!("bidder.upgrade: binary {w}: from or test")),
};
let t = Instant::now();
let (remote, how) = want.stage(host).await.map_err(|e| format!("bidder.upgrade {name}: {e}"))?;
let staged_s = (t.elapsed().as_secs_f64() * 10.0).round() / 10.0;
if which == "from" {
let mut up = self.extra.upgraded.lock().unwrap();
up.retain(|(n, _)| n != name);
up.push((name.to_string(), spec.clone()));
}
let script = BIDDER_UPGRADE
.replace("@UNIT_TIMEOUTS@", UNIT_TIMEOUTS)
.replace("@NAME@", name)
.replace("@BIN@", &remote)
.replace("@SUM@", &want.sha256)
.replace("@STOP@", if stop_first { "1" } else { "0" });
let out = on_host(Some(host), &script).await.map_err(|e| format!("bidder.upgrade {name}: {e}"))?;
let Some(running) = out.lines().find_map(|l| l.strip_prefix("running ")) else {
return Err(format!("bidder.upgrade {name}: {}", tail(&out)));
};
if which == "test" {
self.extra.upgraded.lock().unwrap().retain(|(n, _)| n != name);
}
Ok(json!({ "sha256": running.trim(), "from": from.sha256, "test": test.sha256, "fromSpec": spec, "staged": how, "stagedSeconds": staged_s, "report": tail(&out) }))
}
}
/// A binary bidder.upgrade puts on a bidder host: its sha256, its public URL when it has one (release, sha:, local:),
/// else the local file the scp fallback copies (path:, oci:)
struct Upgradable {
sha256: String,
url: Option<String>,
spec: String,
local: Option<PathBuf>,
}
/// one download or copy of a binary onto a bidder host: the bound the other installs give the network (20 min)
const STAGE_BUDGET_S: u64 = 1200;
impl Upgradable {
async fn of(spec: &str, local: Option<&std::path::Path>) -> Result<Self, String> {
if let Some((url, sha256)) = testkit::requester::Binary::published(spec).await.map_err(|e| e.to_string())? {
if let Some(l) = local {
let have = file_sha256(l).await?;
if have != sha256 {
return Err(format!("{} is {have}, but {spec} publishes {sha256}", l.display()));
}
}
return Ok(Self { sha256, url: Some(url), spec: spec.to_string(), local: local.map(PathBuf::from) });
}
let path = match local {
Some(l) => l.to_path_buf(),
None => testkit::requester::Binary::resolve(spec).await.map_err(|e| e.to_string())?.path,
};
Ok(Self { sha256: file_sha256(&path).await?, url: None, spec: spec.to_string(), local: Some(path) })
}
/// `/var/tmp/swc-upgrade-<sha256>` on the host, checked against the sha256: kept there, or the install's own
/// `bin/<sha256>/` copied beside it, or fetched by the host itself from the public URL, else (no URL) copied from
/// this host over ssh at a rate that must fit the budget; how it got there
async fn stage(&self, host: &str) -> Result<(String, String), String> {
let remote = format!("/var/tmp/swc-upgrade-{}", self.sha256);
let script = STAGE_UPGRADE
.replace("@SUM@", &self.sha256)
.replace("@URL@", self.url.as_deref().unwrap_or_default())
.replace("@BUDGET@", &STAGE_BUDGET_S.to_string());
let out = on_host(Some(host), &script).await?;
if let Some(how) = out.lines().find_map(|l| l.strip_prefix("staged ")) {
return Ok((remote, how.to_string()));
}
if !out.lines().any(|l| l == "nofetch") {
return Err(format!("staging {} ({}) on {host}: {}", self.spec, self.sha256, tail(&out)));
}
let Some(local) = &self.local else {
return Err(format!("staging {} on {host}: no public URL and no local file", self.spec));
};
let how = scp_within_budget(host, local, &remote).await?;
Ok((remote, how))
}
}
const STAGE_UPGRADE: &str = r#"f=/var/tmp/swc-upgrade-@SUM@
ok() { [ -f "$1" ] && [ "$(sha256sum "$1" | cut -d' ' -f1)" = @SUM@ ]; }
if ok "$f"; then echo "staged kept $f"; exit 0; fi
for p in $(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder'); do
exe=$(readlink /proc/$p/exe)
k="$(dirname "$(dirname "$exe")")/@SUM@/$(basename "$exe")"
if ok "$k"; then cp -f "$k" "$f.part" && chmod 0755 "$f.part" && mv -f "$f.part" "$f" && echo "staged from the install's $k"; exit 0; fi
done
u='@URL@'
[ -n "$u" ] || { echo nofetch; exit 0; }
echo "budget @BUDGET@"
t0=$(date +%s)
curl -fsS --max-time @BUDGET@ -o "$f.part" "$u"; rc=$?
if [ $rc != 0 ]; then
echo "fetching $u failed (curl exit $rc) after $(( $(date +%s) - t0 )) s with $(stat -c %s "$f.part" 2>/dev/null || echo 0) bytes (bound @BUDGET@ s)"
rm -f "$f.part"; exit 1
fi
ok "$f.part" || { echo "$u does not hash to @SUM@"; rm -f "$f.part"; exit 1; }
chmod 0755 "$f.part" && mv -f "$f.part" "$f"
s=$(( $(date +%s) - t0 )); b=$(stat -c %s "$f")
echo "staged fetched $b bytes in $s s ($(( b / (s > 0 ? s : 1) / 1048576 )) MiB/s) from $u"
"#;
static FILE_SUMS: std::sync::OnceLock<Mutex<HashMap<PathBuf, String>>> = std::sync::OnceLock::new();
async fn file_sha256(path: &std::path::Path) -> Result<String, String> {
let sums = FILE_SUMS.get_or_init(Default::default);
if let Some(s) = sums.lock().unwrap().get(path) {
return Ok(s.clone());
}
let out = tokio::process::Command::new("sha256sum").arg(path).output().await.map_err(|e| format!("sha256sum {}: {e}", path.display()))?;
let sum = String::from_utf8_lossy(&out.stdout).split_whitespace().next().unwrap_or_default().to_string();
if !out.status.success() || sum.len() != 64 {
return Err(format!("sha256sum {}: {}", path.display(), String::from_utf8_lossy(&out.stderr)));
}
sums.lock().unwrap().insert(path.to_path_buf(), sum.clone());
Ok(sum)
}
/// `local` onto a bidder host at `remote` over ssh (a path: binary of a hand run: no public URL), mode 0755. After 30 s
/// the copied bytes give the rate; a copy that would not end inside the budget stops there and says its rate
async fn scp_within_budget(host: &str, local: &std::path::Path, remote: &str) -> Result<String, String> {
let size = std::fs::metadata(local).map_err(|e| format!("{}: {e}", local.display()))?.len();
let part = format!("{remote}.{}.part", std::process::id());
let t = Instant::now();
let mut child = tokio::process::Command::new("scp")
.args(["-q", "-o", "BatchMode=yes", "-o", "LogLevel=ERROR"])
.arg(local)
.arg(format!("{host}:{part}"))
.kill_on_drop(true)
.spawn()
.map_err(|e| e.to_string())?;
let status = tokio::select! {
s = child.wait() => s.map_err(|e| e.to_string())?,
_ = tokio::time::sleep(Duration::from_secs(30)) => {
let done: u64 = on_host(Some(host), &format!("stat -c %s {part} 2>/dev/null || echo 0")).await?.trim().parse().unwrap_or(0);
let rate = done as f64 / t.elapsed().as_secs_f64();
let eta = if rate > 0.0 { size as f64 / rate } else { f64::INFINITY };
if eta > STAGE_BUDGET_S as f64 {
let _ = child.kill().await;
let _ = on_host(Some(host), &format!("rm -f {part}")).await;
return Err(format!(
"copying {} ({size} bytes) to {host} over ssh runs at {:.2} MiB/s: about {:.0} min, over the {} min budget; name the binary by sha:, local: or release: so the host fetches it",
local.display(), rate / 1048576.0, eta / 60.0, STAGE_BUDGET_S / 60
));
}
match tokio::time::timeout(Duration::from_secs(STAGE_BUDGET_S), child.wait()).await {
Ok(s) => s.map_err(|e| e.to_string())?,
Err(_) => {
let _ = child.kill().await;
return Err(format!("copying {} to {host} over ssh: not done in {} min", local.display(), STAGE_BUDGET_S / 60));
}
}
}
};
if !status.success() {
return Err(format!("scp {} to {host} failed", local.display()));
}
let o = on_host(Some(host), &format!("chmod 0755 {part} && mv -f {part} {remote} && echo staged")).await?;
if !o.contains("staged") {
return Err(format!("staging {remote} on {host}: {}", tail(&o)));
}
let s = t.elapsed().as_secs_f64();
Ok(format!("copied {size} bytes in {s:.0} s ({:.2} MiB/s) over ssh", size as f64 / s.max(0.1) / 1048576.0))
}
const BIDDER_UPGRADE: &str = r#"unit=""; pid=""
for p in $(pgrep -f '^[^ ]*socialweb-computer[^ ]* run bidder'); do
u=$(grep -o '[^/]*bidder@@NAME@\.service' /proc/$p/cgroup 2>/dev/null | tail -1)
[ -n "$u" ] || continue
pid=$p; unit=$u; break
done
[ -n "$unit" ] || { echo 'no installed bidder unit of @NAME@ on this host (install bidder)'; exit 1; }
user=$(ps -o user= -p $pid | tr -d ' ')
U=$(id -u "$user")
exe=$(readlink /proc/$pid/exe)
dir=$(dirname "$exe"); bin=$(dirname "$dir")
[ "$(basename "$bin")" = bin ] || { echo "@NAME@ does not run from an install's bin/<sha256>/: $exe"; exit 1; }
ST=$(dirname "$bin")
sc() { if [ "$(id -u)" = "$U" ]; then XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; else runuser -u "$user" -- env XDG_RUNTIME_DIR=/run/user/$U systemctl --user "$@"; fi; }
@UNIT_TIMEOUTS@
recovery_s=60; wait_s=60
cores=$(ls "$(getent passwd "$user" | cut -d: -f6)/.config/systemd/user/default.target.wants" 2>/dev/null | grep -cE '^socialweb.*-computer-(bidder@.*|requester@.*|gateway)\.service$')
echo "budget $(( stop_s + (cores > 0 ? cores : 1) * (stop_s + start_s + recovery_s) + start_s + wait_s ))"
if [ "$(basename "$dir")" = @SUM@ ]; then echo "running @SUM@"; echo "$unit already runs bin/@SUM@"; exit 0; fi
if [ @STOP@ = 1 ]; then sc stop "$unit" || { echo "systemctl --user stop $unit failed"; exit 1; }; fi
out=$(runuser -u "$user" -- env XDG_RUNTIME_DIR=/run/user/$U "$ST/bin/current/socialweb-computer" upgrade --binary @BIN@ --sha256 @SUM@ --state-dir "$ST" 2>&1); rc=$?
sc start "$unit"
[ $rc = 0 ] || { echo "upgrade exit $rc: $(tail -c 1500 <<<"$out")"; exit 1; }
new=""
for i in $(seq 1 $wait_s); do new=$(sc show "$unit" -p MainPID --value); [ -n "$new" ] && [ "$new" != 0 ] && [ "$new" != "$pid" ] && break; sleep 1; done
nexe=$(readlink /proc/$new/exe 2>/dev/null)
case "$nexe" in "$bin/@SUM@/"*) ;; *) echo "$unit runs ${nexe:-nothing} (MainPID ${new:-0}) after the upgrade, not $bin/@SUM@/"; exit 1 ;; esac
echo "running @SUM@"
echo "upgraded $unit of $user: $(basename "$dir") -> @SUM@, MainPID $pid -> $new"
"#;
const HOST_SCRIPT_S: u64 = 120;
/// A script on this host (`bash -s`) or on an ssh alias (`ssh -o BatchMode=yes <host> bash -s`), its stdout. It ends
/// within HOST_SCRIPT_S, or, when it states the budget of its bounded steps (a stdout line `budget <seconds>`, the first
/// one, not returned), within that budget plus HOST_SCRIPT_S.
async fn on_host(host: Option<&str>, script: &str) -> Result<String, String> {
use tokio::io::AsyncWriteExt;
let mut cmd = match host {
Some(h) => {
let mut c = tokio::process::Command::new("ssh");
c.args([
"-o",
"BatchMode=yes",
"-o",
"LogLevel=ERROR",
h,
"bash",
"-s",
]);
c
}
None => {
let mut c = tokio::process::Command::new("bash");
c.arg("-s");
c
}
};
let mut child = cmd
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
.map_err(|e| e.to_string())?;
child
.stdin
.take()
.ok_or("no stdin")?
.write_all(script.as_bytes())
.await
.map_err(|e| e.to_string())?;
let mut stdout = tokio::io::BufReader::new(child.stdout.take().ok_or("no stdout")?);
let mut stderr = child.stderr.take().ok_or("no stderr")?;
let errs = tokio::spawn(async move {
let mut b = Vec::new();
let _ = tokio::io::AsyncReadExt::read_to_end(&mut stderr, &mut b).await;
b
});
let t0 = Instant::now();
let mut budget: Option<u64> = None;
let left = |budget: Option<u64>| Duration::from_secs(HOST_SCRIPT_S.saturating_add(budget.unwrap_or(0))).saturating_sub(t0.elapsed());
let cut = |budget: Option<u64>| match budget {
Some(b) => format!("the host script did not end in {} s (its stated budget {b} s + {HOST_SCRIPT_S} s)", HOST_SCRIPT_S.saturating_add(b)),
None => format!("the host script did not end in {HOST_SCRIPT_S} s"),
};
let mut out = Vec::new();
loop {
let mut line = Vec::new();
let n = tokio::time::timeout(left(budget), stdout.read_until(b'\n', &mut line))
.await
.map_err(|_| cut(budget))?
.map_err(|e| e.to_string())?;
if n == 0 {
break;
}
if budget.is_none() {
if let Some(b) = std::str::from_utf8(&line).ok().and_then(|l| l.trim_end().strip_prefix("budget ")).and_then(|b| b.parse().ok()) {
budget = Some(b);
continue;
}
}
out.extend_from_slice(&line);
}
let status = tokio::time::timeout(left(budget), child.wait())
.await
.map_err(|_| cut(budget))?
.map_err(|e| e.to_string())?;
if !status.success() && out.is_empty() {
let err = errs.await.unwrap_or_default();
return Err(format!(
"{}: {}",
host.unwrap_or("this host"),
String::from_utf8_lossy(&err).trim()
));
}
Ok(String::from_utf8_lossy(&out).into_owned())
}
const TAMPERED_DOWNLOAD: &str = "socialweb-guest-agent-x86_64";
async fn sha256_of(p: &std::path::Path) -> Result<String, String> {
let out = tokio::process::Command::new("sha256sum")
.arg(p)
.output()
.await
.map_err(|e| e.to_string())?;
String::from_utf8_lossy(&out.stdout)
.split_whitespace()
.next()
.map(String::from)
.ok_or_else(|| {
format!(
"sha256sum {}: {}",
p.display(),
String::from_utf8_lossy(&out.stderr)
)
})
}
/// The extracted tree of socialweb-guest-downloads in an asset cache: the directory that holds the tampered download.
fn downloads_tree(cache: &std::path::Path) -> Result<PathBuf, String> {
std::fs::read_dir(cache)
.map_err(|e| format!("{}: {e}", cache.display()))?
.filter_map(Result::ok)
.map(|e| e.path())
.find(|p| p.join(format!("{TAMPERED_DOWNLOAD}.sha256")).is_file())
.ok_or_else(|| {
format!(
"no extracted socialweb-guest-downloads tree under {}",
cache.display()
)
})
}
async fn remove_tree(p: &std::path::Path) {
let _ = tokio::process::Command::new("chmod")
.args(["-R", "u+w"])
.arg(p)
.stderr(Stdio::null())
.status()
.await;
let _ = std::fs::remove_dir_all(p);
}
impl Swc {
async fn verify_assets(
&self,
bin: &std::path::Path,
cache: &std::path::Path,
) -> Result<(bool, String), String> {
let out = tokio::process::Command::new(bin)
.args(["dev", "verify-assets", "--asset-cache"])
.arg(cache)
.stdin(Stdio::null())
.output()
.await
.map_err(|e| format!("{}: {e}", bin.display()))?;
let mut text = String::from_utf8_lossy(&out.stderr).into_owned();
if !out.status.success() {
text.push_str(&String::from_utf8_lossy(&out.stdout));
}
Ok((out.status.success(), text))
}
async fn assets_tamper(&self, input: &Value) -> Result<Value, String> {
let variant = str_of(input, "variant")?;
let root = expand("~/.tmp/swc-system-tests/assets-tamper")
.join(format!("{}-{variant}", std::process::id()));
remove_tree(&root).await;
std::fs::create_dir_all(&root).map_err(|e| format!("{}: {e}", root.display()))?;
let r = self.assets_tamper_in(variant, &root).await;
remove_tree(&root).await;
r
}
async fn assets_tamper_in(
&self,
variant: &str,
root: &std::path::Path,
) -> Result<Value, String> {
let bin = self.bin.path.clone();
let cache = root.join("cache");
let (ok, first) = self.verify_assets(&bin, &cache).await?;
if !ok {
return Err(format!(
"dev verify-assets on an untouched cache failed (this binary embeds no verifiable assets?): {}",
tail(&first)
));
}
let tree = downloads_tree(&cache)?;
match variant {
"file" | "fileSidecar" => {
let (f, side) = (
tree.join(TAMPERED_DOWNLOAD),
tree.join(format!("{TAMPERED_DOWNLOAD}.sha256")),
);
let (want, want_side) = (
sha256_of(&f).await?,
std::fs::read(&side).map_err(|e| e.to_string())?,
);
let _ = tokio::process::Command::new("chmod")
.args(["-R", "u+w"])
.arg(&tree)
.status()
.await;
let mut b = std::fs::read(&f).map_err(|e| e.to_string())?;
let n = b.len();
b[n / 2] ^= 1;
std::fs::write(&f, &b).map_err(|e| e.to_string())?;
if variant == "fileSidecar" {
let now = sha256_of(&f).await?;
std::fs::write(&side, format!("{now} {TAMPERED_DOWNLOAD}\n"))
.map_err(|e| e.to_string())?;
}
let tampered = sha256_of(&f).await? != want;
let (verify_ok, text) = self.verify_assets(&bin, &cache).await?;
let healed = downloads_tree(&cache).is_ok_and(|t| t == tree)
&& sha256_of(&f).await.is_ok_and(|h| h == want)
&& std::fs::read(&side).is_ok_and(|s| s == want_side);
Ok(
json!({ "variant": variant, "tampered": tampered, "verifyOk": verify_ok, "healed": healed, "output": tail(&text) }),
)
}
"record" => {
let name = tree
.file_name()
.and_then(|n| n.to_str())
.ok_or("tree name")?
.to_string();
let mut bytes =
std::fs::read(&bin).map_err(|e| format!("{}: {e}", bin.display()))?;
let digest = name.as_bytes();
let mut flipped = 0;
let mut at = 0;
while let Some(i) = bytes[at..].windows(digest.len()).position(|w| w == digest) {
let i = at + i;
bytes[i] = if bytes[i] == b'0' { b'1' } else { b'0' };
flipped += 1;
at = i + digest.len();
}
if flipped == 0 {
return Err(format!(
"the tree digest {name} of the embedded build record is not in {}",
bin.display()
));
}
let copy = root.join("swc-tampered");
std::fs::write(©, &bytes).map_err(|e| e.to_string())?;
let _ = tokio::process::Command::new("chmod")
.args(["0755"])
.arg(©)
.status()
.await;
let cache1 = root.join("cache1");
let (ok, text) = self.verify_assets(©, &cache1).await?;
Ok(
json!({ "variant": variant, "recordFlipped": true, "flippedOccurrences": flipped, "refused": !ok && text.contains("unpacks to tree"),
"extractedNothing": downloads_tree(&cache1).is_err(), "output": tail(&text) }),
)
}
other => Err(format!("variant {other:?}: file, fileSidecar or record")),
}
}
}
/// The accepts a bidder declined, by these log files (`testkit::requester::declined_accepts`).
pub(super) fn declined_accepts(paths: &[PathBuf]) -> Vec<String> {
testkit::requester::declined_accepts(&log_lines_all(paths, testkit::requester::ACCEPT_DECLINED))
}
/// Every JSON log line of `msg` in these files.
fn log_lines_all(paths: &[PathBuf], msg: &str) -> Vec<Value> {
paths
.iter()
.filter_map(|p| std::fs::read_to_string(p).ok())
.flat_map(|t| t.lines().filter(|l| l.contains(msg)).filter_map(|l| serde_json::from_str::<Value>(l).ok()).collect::<Vec<_>>())
.filter(|v| v["msg"] == msg)
.collect()
}