0
0
//! The bookkeeping of a runbook's state machine: `S` holds the state (phase, marks, readings, flags), the clock of
//! this event, and ends the run as pass or fail; calls to the test host and to kube by name; the market's rkeys.
use abi::{Call, Event, Handle};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
pub const T: &str = "computer.socialweb.test.";
pub const MIN: u64 = 60_000;
pub const ASSOC: &str = "com.publicdomainrelay.temp.market.association";
pub const ENDORSE: &str = "link.ident.endorsement";
pub const VM_TERMINATED: &str = "com.publicdomainrelay.temp.compute.events.vm.terminated";
/// Seconds of an ISO time (UTC, `Z`).
pub fn secs(iso: &str) -> f64 {
let n = |a: usize, b: usize| {
iso.get(a..b)
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or(0)
};
let (y, m, d) = (n(0, 4), n(5, 7), n(8, 10));
let (y2, m2) = if m <= 2 { (y - 1, m + 9) } else { (y, m - 3) };
let days = 365 * y2 + y2 / 4 - y2 / 100 + y2 / 400 + (153 * m2 + 2) / 5 + d;
let frac = iso
.get(19..)
.and_then(|f| f.trim_end_matches('Z').parse::<f64>().ok())
.unwrap_or(0.0);
(days * 86400 + n(11, 13) * 3600 + n(14, 16) * 60 + n(17, 19)) as f64 + frac
}
pub const GUEST_SECRETS: &str = "com.publicdomainrelay.temp.compute.events.guest.secrets";
pub const MARKET_EVENT: &str = "com.publicdomainrelay.temp.market.event";
/// The run's `--secrets` bundle (test values): a string value; an object value the guest gets as its compact JSON text;
/// modes 0644 and 0400; a file in /etc (an existing directory: its mode and owner must not change); all in a directory
/// the guest does not have.
pub fn secrets_bundle() -> Value {
json!([
{ "path": "/root/swc-test-secrets/a.txt", "value": "swc-test-string-0001" },
{ "path": "/root/swc-test-secrets/b.json", "value": { "k": "v", "n": 1 } },
{ "path": "/root/swc-test-secrets/c.txt", "value": "c", "mode": "0644" },
{ "path": "/root/swc-test-secrets/d.txt", "value": "d", "mode": "0400" },
{ "path": "/etc/swc-test-secret", "value": "e" }
])
}
pub const SECRETS_BAD_PATH: &str = "entry 0: the path is not a clean absolute path";
pub const SECRETS_BAD_MODE: &str = "entry 0: the mode is not an octal string of at most 0777";
/// Bundles every requester refuses before the RFP (the CLI's `--secrets` read, exit 2, nothing rented), each with the
/// message it must print: a `..` path and a relative one; a path with a control character (a trailing newline, which
/// a shell's `$(...)` would strip); a `..` hidden by a trailing newline; a mode with a trailing newline.
pub fn secrets_bundles_refused() -> Vec<(&'static str, Value, &'static str)> {
vec![
("dots", json!([{ "path": "/root/swc-test-secrets/../escape", "value": "x" }, { "path": "relative/x", "value": "y" }]), SECRETS_BAD_PATH),
("ctl", json!([{ "path": "/root/swc-test-secrets/x\n", "value": "x" }]), SECRETS_BAD_PATH),
("ctldots", json!([{ "path": "/root/swc-test-secrets/..\n", "value": "x" }]), SECRETS_BAD_PATH),
("ctlmode", json!([{ "path": "/root/swc-test-secrets/m.txt", "value": "x", "mode": "0600\n" }]), SECRETS_BAD_MODE),
]
}
/// listRecords pages as repo.listCreated returns them ({uri, createdAt}): the uris created at or after `since`.
pub fn created_since(o: &Value, since: f64) -> Vec<Value> {
o["records"].as_array().into_iter().flatten().filter(|r| r["createdAt"].as_str().is_some_and(|c| secs(c) >= since)).cloned().collect()
}
/// request.contract's answer: (status.secretsFetchedVia, the SecretsApplied condition's reason), empty when unset.
pub fn secrets_status(o: &Value) -> (String, String) {
let via = o["secretsFetchedVia"].as_str().unwrap_or_default().to_string();
let reason = o["conditions"].as_array().into_iter().flatten().find(|c| c["type"] == "SecretsApplied").and_then(|c| c["reason"].as_str()).unwrap_or_default().to_string();
(via, reason)
}
/// The SecretsApplied reason of a getSecrets fetch over `channel` (requester secretsFetchedReason).
pub fn fetched_reason(channel: &str) -> &'static str {
match channel {
"iroh" => "GetSecretsIroh",
"webrtc" => "GetSecretsWebRTC",
"https" => "GetSecretsHTTPS",
_ => "GetSecrets",
}
}
/// The user_data runcmd line that turns the iroh channel off for one contract: requester.json without its ticket, so
/// the guest agent forwards getSecrets over WebRTC.
pub const SECRETS_NO_IROH: &str = "jq -c 'del(.iroh)' /etc/socialweb-guest-agent/requester.json > /etc/socialweb-guest-agent/requester.json.t && mv /etc/socialweb-guest-agent/requester.json.t /etc/socialweb-guest-agent/requester.json";
/// request.ssh exec: one line with the files, the modes of the directory and the files, and /etc's mode and owner.
pub const SECRETS_READ: &str = "D=/root/swc-test-secrets; printf 'SECRETS<%s|%s|%s|%s|%s>\\n' \"$(cat $D/a.txt)\" \"$(cat $D/b.json)\" \"$(stat -c %a $D $D/a.txt $D/b.json $D/c.txt $D/d.txt /etc/swc-test-secret | tr '\\n' ' ')\" \"$(stat -c '%a %U:%G' /etc)\" \"$(cat /etc/swc-test-secret)\"";
pub const SECRETS_WANT: &str = "SECRETS<swc-test-string-0001|{\"k\":\"v\",\"n\":1}|700 600 600 644 400 600 |755 root:root|e>";
/// The guest wrote the bundle of secrets_bundle(), as SECRETS_READ printed it.
pub fn secrets_written(output: &str) -> Result<(), String> {
if output.lines().any(|l| l.trim_end() == SECRETS_WANT) {
return Ok(());
}
Err(format!("the guest's secrets files: want {SECRETS_WANT}, got {}", output.lines().find(|l| l.starts_with("SECRETS<")).unwrap_or("no SECRETS line")))
}
/// request.ssh exec: a symlink at a target, then a refetch: the file replaces the link and the link's target is untouched.
pub const SECRETS_SYMLINK: &str = "D=/root/swc-test-secrets; echo keep > $D/target && ln -sfn $D/target $D/a.txt && SECRETS_REFETCH=1 /usr/local/bin/setup-secrets.sh >/dev/null 2>&1; printf 'LINK<%s|%s|%s>\\n' \"$(stat -c %F $D/a.txt)\" \"$(cat $D/target)\" \"$(cat $D/a.txt)\"";
pub const SECRETS_SYMLINK_WANT: &str = "LINK<regular file|keep|swc-test-string-0001>";
/// request.ssh exec: the guest asks for a sealed bundle as a guest agent of comms mode Firehose would (its marker, no
/// ready file, a restart).
pub const SECRETS_FORCE_SEALED: &str = "printf '{\"fallback\":\"sealed\"}\\n' > /etc/socialweb-guest-agent/secrets.json && rm -f /run/socialweb-guest-agent/secrets.ready && systemctl restart socialweb-guest-agent.service && echo FORCED";
/// termination-payload-<first 32 hex of sha256("terminated:<receipt uri>")>: the receipt's vm.terminated rkey.
pub fn termination_rkey(receipt: &str) -> String {
format!(
"termination-payload-{}",
key_hash(&format!("terminated:{receipt}"))
)
}
/// The first 32 hex of sha256(key): a bidder event record's rkey ends `-payload-<this>` of its StateSync key
/// (market.EventRKeys); a report's key is `<nsid>:<tag>:<receipt uri>`.
pub fn key_hash(key: &str) -> String {
Sha256::digest(key)[..16]
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
/// A call to the test host: computer.socialweb.test.<what>.
pub fn host(id: &str, what: &str, input: Value) -> Call {
Call::new(id, format!("{T}{what}")).input(input)
}
/// A subscription of the test host (its log events, an observer): atlibs takes a non-exec subscription's arguments as
/// params, never input, and makes every param a string (an array would be comma-joined): the arguments go as one
/// `json` param, which the host parses.
pub fn host_sub(id: &str, what: &str, args: Value) -> Call {
Call::new(id, format!("{T}{what}"))
.params(json!({ "json": args.to_string() }))
.subscribe()
}
/// A kube call: com.fedfork.exec.kube.<verb> on `cluster`.
pub fn kube(id: &str, verb: &str, cluster: &str, mut input: Value) -> Call {
input["cluster"] = json!(cluster);
Call::new(id, format!("com.fedfork.exec.kube.{verb}")).input(input)
}
/// A getRecord of an at:// URI, a public read of its repo.
pub fn get_record(id: &str, uri: &str) -> Call {
let mut p = uri.trim_start_matches("at://").splitn(3, '/');
let (repo, collection, rkey) = (
p.next().unwrap_or_default(),
p.next().unwrap_or_default(),
p.next().unwrap_or_default(),
);
Call::new(id, "com.atproto.repo.getRecord")
.params(json!({ "repo": repo, "collection": collection, "rkey": rkey }))
.service(repo)
}
pub fn sw(kind: &str, name: &str) -> Value {
json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": name })
}
/// Before an apply: every ssh public key in a templated spec is THIS run's (from the host), and there is one. A key
/// baked into a file (an earlier run's) or a placeholder left in is refused: the guest would refuse our ssh.
pub fn keys_are(spec: &str, key: &str) -> Result<(), String> {
let key = key.trim();
if key.is_empty() || !key.starts_with("ssh-") {
return Err("the run has no ssh public key from the host".into());
}
let mut n = 0;
for l in spec.lines() {
let t = l
.trim()
.trim_start_matches("- ")
.trim()
.trim_matches(|c| c == '"' || c == '\'');
if ["ssh-ed25519 ", "ssh-rsa ", "ecdsa-sha2-", "sk-ssh-"]
.iter()
.any(|p| t.starts_with(p))
{
if t != key {
return Err(format!(
"the spec carries an ssh key that is not this run's: {}",
t.chars().take(40).collect::<String>()
));
}
n += 1;
}
}
if n == 0 {
return Err("the spec carries no ssh key".into());
}
Ok(())
}
pub struct S {
pub st: Value,
pub now: f64,
}
impl S {
pub fn new(state: Value, now_iso: &str) -> Self {
let now = secs(now_iso);
let st = if state.is_null() {
json!({ "phase": "start", "t0": now, "readings": {}, "f": {} })
} else {
state
};
Self { st, now }
}
pub fn phase(&self) -> String {
self.st["phase"].as_str().unwrap_or_default().to_string()
}
pub fn is(&self, p: &str) -> bool {
self.st["phase"] == p
}
pub fn set(&mut self, p: &str) {
self.st["phase"] = json!(p);
}
/// Seconds since the run started.
pub fn t(&self) -> f64 {
((self.now - self.st["t0"].as_f64().unwrap_or(self.now)) * 10.0).round() / 10.0
}
pub fn mark(&mut self, k: &str) {
self.st["marks"][k] = json!(self.now);
}
/// Seconds since mark `k`.
pub fn since(&self, k: &str) -> f64 {
((self.now - self.st["marks"][k].as_f64().unwrap_or(self.now)) * 10.0).round() / 10.0
}
pub fn reading(&mut self, k: &str, v: impl Into<Value>) {
self.st["readings"][k] = v.into();
}
pub fn flag(&mut self, k: &str) {
self.st["f"][k] = json!(true);
}
pub fn has(&self, k: &str) -> bool {
self.st["f"][k] == true
}
pub fn fail(&self, error: impl Into<String>) -> Handle {
Handle::new().result(json!({ "pass": false, "error": error.into(), "phase": self.st["phase"], "readings": self.st["readings"] })).done()
}
pub fn pass(&mut self) -> Handle {
let t = self.t();
self.reading("seconds", t);
Handle::new()
.result(json!({ "pass": true, "readings": self.st["readings"] }))
.done()
}
/// The run cannot do what its entry proves here (a target field, an input or a capability is missing): a FAIL.
/// There is no skipped result: the gate picks the entries a target runs (tests/entries.txt).
pub fn unmet(&self, why: &str) -> Handle {
self.fail(format!("unmet: {why}"))
}
pub fn keep(&self, h: Handle) -> Handle {
h.state(self.st.clone())
}
}
/// A `requester_market_event` message: the requester's line for a market event of a contract this run made, which the
/// host forwards like Held. An OOM kill (guest.oomKill, vm.oomKilled, vm.terminated reason oom) fails the run naming
/// the VM; memory pressure (guest.memoryPressure, vm.memoryPressure) is a warning in readings.memory. Call it first.
/// The target's bidder DIDs (plan 0040 built 5: a rental's allowlist, filtering discovery).
pub fn bidder_dids(args: &Value) -> Vec<String> {
args["bidders"].as_array().into_iter().flatten().filter_map(|b| b["did"].as_str()).filter(|d| d.starts_with("did:")).map(String::from).collect()
}
pub fn market_event(s: &mut S, event: &Event) -> Option<Handle> {
let Event::Message { id, data, .. } = event else {
return None;
};
if let Some(h) = repeated_failure(s, data) {
return Some(h);
}
let e = data["event"].as_str().unwrap_or_default();
let kill = data["kill"] == true;
if data["msg"] != "requester_market_event" || (!kill && !e.ends_with("memoryPressure")) {
return None;
}
// the invocation's instance names the VM's run (a contract is "contract" in its own workspace)
let vm = [&data["instance"], &data["contract"]]
.iter()
.find_map(|v| v.as_str())
.unwrap_or(id)
.to_string();
let entry = json!({ "event": e, "vm": vm, "kill": kill, "payload": data["payload"] });
match s.st["readings"]["memory"].as_array_mut() {
Some(a) => a.push(entry),
None => s.st["readings"]["memory"] = json!([entry]),
}
kill.then(|| s.fail(format!("OOM: {e} on VM {vm}: {}", data["payload"])))
}
/// A step the requester retries (`retry: true`) that fails the same way REPEAT_FAILS times over REPEAT_FAIL_S has failed:
/// fail the entry now rather than wait out its deadline (plan 0044, "Timing and watching"). A refusal (`retry: false`) is
/// left to the entry, which may expect it.
pub const REPEAT_FAILS: u64 = 5;
pub const REPEAT_FAIL_S: f64 = 180.0;
fn repeated_failure(s: &mut S, data: &Value) -> Option<Handle> {
let msg = data["msg"].as_str().unwrap_or_default();
if msg != "requester_association_failed" || data["retry"] != true {
return None;
}
let err = data["err"].as_str().unwrap_or_default();
let key = format!("{msg} {} {err}", data["step"].as_str().unwrap_or_default());
let t = s.t();
let seen = &mut s.st["repeats"][&key];
let n = seen["n"].as_u64().unwrap_or(0) + 1;
let first = seen["first"].as_f64().unwrap_or(t);
*seen = json!({ "n": n, "first": first });
(n >= REPEAT_FAILS && t - first >= REPEAT_FAIL_S).then(|| {
s.fail(format!(
"{msg} {n} times in {:.0} s, the same error each time: {err}",
t - first
))
})
}
/// A call's output, or the run's failure naming the call.
pub fn out(s: &S, id: &str, r: Result<&Value, String>) -> Result<Value, Handle> {
r.map(Value::clone)
.map_err(|e| s.fail(format!("{id}: {e}")))
}
/// Ready pods (name, node) of a pods list.
pub fn ready_pods(list: &Value) -> Vec<(String, String)> {
list["items"]
.as_array()
.into_iter()
.flatten()
.filter(|p| {
p["status"]["conditions"].as_array().is_some_and(|c| {
c.iter()
.any(|c| c["type"] == "Ready" && c["status"] == "True")
})
})
.map(|p| {
(
p["metadata"]["name"]
.as_str()
.unwrap_or_default()
.to_string(),
p["spec"]["nodeName"]
.as_str()
.unwrap_or_default()
.to_string(),
)
})
.collect()
}
pub fn names(list: &Value) -> Vec<String> {
list["items"]
.as_array()
.into_iter()
.flatten()
.filter_map(|o| o["metadata"]["name"].as_str().map(String::from))
.collect()
}