//! The bookkeeping of a runbook's state machine: `S` holds the state (phase, marks, readings, flags), the clock of //! this event, and ends the run as pass or fail; calls to the test host and to kube by name; the market's rkeys. use abi::{Call, Event, Handle}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; pub const T: &str = "computer.socialweb.test."; pub const MIN: u64 = 60_000; pub const ASSOC: &str = "com.publicdomainrelay.temp.market.association"; pub const ENDORSE: &str = "link.ident.endorsement"; pub const VM_TERMINATED: &str = "com.publicdomainrelay.temp.compute.events.vm.terminated"; /// Seconds of an ISO time (UTC, `Z`). pub fn secs(iso: &str) -> f64 { let n = |a: usize, b: usize| { iso.get(a..b) .and_then(|s| s.parse::().ok()) .unwrap_or(0) }; let (y, m, d) = (n(0, 4), n(5, 7), n(8, 10)); let (y2, m2) = if m <= 2 { (y - 1, m + 9) } else { (y, m - 3) }; let days = 365 * y2 + y2 / 4 - y2 / 100 + y2 / 400 + (153 * m2 + 2) / 5 + d; let frac = iso .get(19..) .and_then(|f| f.trim_end_matches('Z').parse::().ok()) .unwrap_or(0.0); (days * 86400 + n(11, 13) * 3600 + n(14, 16) * 60 + n(17, 19)) as f64 + frac } pub const GUEST_SECRETS: &str = "com.publicdomainrelay.temp.compute.events.guest.secrets"; pub const MARKET_EVENT: &str = "com.publicdomainrelay.temp.market.event"; /// The run's `--secrets` bundle (test values): a string value; an object value the guest gets as its compact JSON text; /// modes 0644 and 0400; a file in /etc (an existing directory: its mode and owner must not change); all in a directory /// the guest does not have. pub fn secrets_bundle() -> Value { json!([ { "path": "/root/swc-test-secrets/a.txt", "value": "swc-test-string-0001" }, { "path": "/root/swc-test-secrets/b.json", "value": { "k": "v", "n": 1 } }, { "path": "/root/swc-test-secrets/c.txt", "value": "c", "mode": "0644" }, { "path": "/root/swc-test-secrets/d.txt", "value": "d", "mode": "0400" }, { "path": "/etc/swc-test-secret", "value": "e" } ]) } pub const SECRETS_BAD_PATH: &str = "entry 0: the path is not a clean absolute path"; pub const SECRETS_BAD_MODE: &str = "entry 0: the mode is not an octal string of at most 0777"; /// Bundles every requester refuses before the RFP (the CLI's `--secrets` read, exit 2, nothing rented), each with the /// message it must print: a `..` path and a relative one; a path with a control character (a trailing newline, which /// a shell's `$(...)` would strip); a `..` hidden by a trailing newline; a mode with a trailing newline. pub fn secrets_bundles_refused() -> Vec<(&'static str, Value, &'static str)> { vec![ ("dots", json!([{ "path": "/root/swc-test-secrets/../escape", "value": "x" }, { "path": "relative/x", "value": "y" }]), SECRETS_BAD_PATH), ("ctl", json!([{ "path": "/root/swc-test-secrets/x\n", "value": "x" }]), SECRETS_BAD_PATH), ("ctldots", json!([{ "path": "/root/swc-test-secrets/..\n", "value": "x" }]), SECRETS_BAD_PATH), ("ctlmode", json!([{ "path": "/root/swc-test-secrets/m.txt", "value": "x", "mode": "0600\n" }]), SECRETS_BAD_MODE), ] } /// listRecords pages as repo.listCreated returns them ({uri, createdAt}): the uris created at or after `since`. pub fn created_since(o: &Value, since: f64) -> Vec { o["records"].as_array().into_iter().flatten().filter(|r| r["createdAt"].as_str().is_some_and(|c| secs(c) >= since)).cloned().collect() } /// request.contract's answer: (status.secretsFetchedVia, the SecretsApplied condition's reason), empty when unset. pub fn secrets_status(o: &Value) -> (String, String) { let via = o["secretsFetchedVia"].as_str().unwrap_or_default().to_string(); let reason = o["conditions"].as_array().into_iter().flatten().find(|c| c["type"] == "SecretsApplied").and_then(|c| c["reason"].as_str()).unwrap_or_default().to_string(); (via, reason) } /// The SecretsApplied reason of a getSecrets fetch over `channel` (requester secretsFetchedReason). pub fn fetched_reason(channel: &str) -> &'static str { match channel { "iroh" => "GetSecretsIroh", "webrtc" => "GetSecretsWebRTC", "https" => "GetSecretsHTTPS", _ => "GetSecrets", } } /// The user_data runcmd line that turns the iroh channel off for one contract: requester.json without its ticket, so /// the guest agent forwards getSecrets over WebRTC. pub const SECRETS_NO_IROH: &str = "jq -c 'del(.iroh)' /etc/socialweb-guest-agent/requester.json > /etc/socialweb-guest-agent/requester.json.t && mv /etc/socialweb-guest-agent/requester.json.t /etc/socialweb-guest-agent/requester.json"; /// request.ssh exec: one line with the files, the modes of the directory and the files, and /etc's mode and owner. pub const SECRETS_READ: &str = "D=/root/swc-test-secrets; printf 'SECRETS<%s|%s|%s|%s|%s>\\n' \"$(cat $D/a.txt)\" \"$(cat $D/b.json)\" \"$(stat -c %a $D $D/a.txt $D/b.json $D/c.txt $D/d.txt /etc/swc-test-secret | tr '\\n' ' ')\" \"$(stat -c '%a %U:%G' /etc)\" \"$(cat /etc/swc-test-secret)\""; pub const SECRETS_WANT: &str = "SECRETS"; /// The guest wrote the bundle of secrets_bundle(), as SECRETS_READ printed it. pub fn secrets_written(output: &str) -> Result<(), String> { if output.lines().any(|l| l.trim_end() == SECRETS_WANT) { return Ok(()); } Err(format!("the guest's secrets files: want {SECRETS_WANT}, got {}", output.lines().find(|l| l.starts_with("SECRETS<")).unwrap_or("no SECRETS line"))) } /// request.ssh exec: a symlink at a target, then a refetch: the file replaces the link and the link's target is untouched. pub const SECRETS_SYMLINK: &str = "D=/root/swc-test-secrets; echo keep > $D/target && ln -sfn $D/target $D/a.txt && SECRETS_REFETCH=1 /usr/local/bin/setup-secrets.sh >/dev/null 2>&1; printf 'LINK<%s|%s|%s>\\n' \"$(stat -c %F $D/a.txt)\" \"$(cat $D/target)\" \"$(cat $D/a.txt)\""; pub const SECRETS_SYMLINK_WANT: &str = "LINK"; /// request.ssh exec: the guest asks for a sealed bundle as a guest agent of comms mode Firehose would (its marker, no /// ready file, a restart). pub const SECRETS_FORCE_SEALED: &str = "printf '{\"fallback\":\"sealed\"}\\n' > /etc/socialweb-guest-agent/secrets.json && rm -f /run/socialweb-guest-agent/secrets.ready && systemctl restart socialweb-guest-agent.service && echo FORCED"; /// termination-payload-")>: the receipt's vm.terminated rkey. pub fn termination_rkey(receipt: &str) -> String { format!( "termination-payload-{}", key_hash(&format!("terminated:{receipt}")) ) } /// The first 32 hex of sha256(key): a bidder event record's rkey ends `-payload-` of its StateSync key /// (market.EventRKeys); a report's key is `::`. pub fn key_hash(key: &str) -> String { Sha256::digest(key)[..16] .iter() .map(|b| format!("{b:02x}")) .collect() } /// A call to the test host: computer.socialweb.test.. pub fn host(id: &str, what: &str, input: Value) -> Call { Call::new(id, format!("{T}{what}")).input(input) } /// A subscription of the test host (its log events, an observer): atlibs takes a non-exec subscription's arguments as /// params, never input, and makes every param a string (an array would be comma-joined): the arguments go as one /// `json` param, which the host parses. pub fn host_sub(id: &str, what: &str, args: Value) -> Call { Call::new(id, format!("{T}{what}")) .params(json!({ "json": args.to_string() })) .subscribe() } /// A kube call: com.fedfork.exec.kube. on `cluster`. pub fn kube(id: &str, verb: &str, cluster: &str, mut input: Value) -> Call { input["cluster"] = json!(cluster); Call::new(id, format!("com.fedfork.exec.kube.{verb}")).input(input) } /// A getRecord of an at:// URI, a public read of its repo. pub fn get_record(id: &str, uri: &str) -> Call { let mut p = uri.trim_start_matches("at://").splitn(3, '/'); let (repo, collection, rkey) = ( p.next().unwrap_or_default(), p.next().unwrap_or_default(), p.next().unwrap_or_default(), ); Call::new(id, "com.atproto.repo.getRecord") .params(json!({ "repo": repo, "collection": collection, "rkey": rkey })) .service(repo) } pub fn sw(kind: &str, name: &str) -> Value { json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": name }) } /// Before an apply: every ssh public key in a templated spec is THIS run's (from the host), and there is one. A key /// baked into a file (an earlier run's) or a placeholder left in is refused: the guest would refuse our ssh. pub fn keys_are(spec: &str, key: &str) -> Result<(), String> { let key = key.trim(); if key.is_empty() || !key.starts_with("ssh-") { return Err("the run has no ssh public key from the host".into()); } let mut n = 0; for l in spec.lines() { let t = l .trim() .trim_start_matches("- ") .trim() .trim_matches(|c| c == '"' || c == '\''); if ["ssh-ed25519 ", "ssh-rsa ", "ecdsa-sha2-", "sk-ssh-"] .iter() .any(|p| t.starts_with(p)) { if t != key { return Err(format!( "the spec carries an ssh key that is not this run's: {}", t.chars().take(40).collect::() )); } n += 1; } } if n == 0 { return Err("the spec carries no ssh key".into()); } Ok(()) } pub struct S { pub st: Value, pub now: f64, } impl S { pub fn new(state: Value, now_iso: &str) -> Self { let now = secs(now_iso); let st = if state.is_null() { json!({ "phase": "start", "t0": now, "readings": {}, "f": {} }) } else { state }; Self { st, now } } pub fn phase(&self) -> String { self.st["phase"].as_str().unwrap_or_default().to_string() } pub fn is(&self, p: &str) -> bool { self.st["phase"] == p } pub fn set(&mut self, p: &str) { self.st["phase"] = json!(p); } /// Seconds since the run started. pub fn t(&self) -> f64 { ((self.now - self.st["t0"].as_f64().unwrap_or(self.now)) * 10.0).round() / 10.0 } pub fn mark(&mut self, k: &str) { self.st["marks"][k] = json!(self.now); } /// Seconds since mark `k`. pub fn since(&self, k: &str) -> f64 { ((self.now - self.st["marks"][k].as_f64().unwrap_or(self.now)) * 10.0).round() / 10.0 } pub fn reading(&mut self, k: &str, v: impl Into) { self.st["readings"][k] = v.into(); } pub fn flag(&mut self, k: &str) { self.st["f"][k] = json!(true); } pub fn has(&self, k: &str) -> bool { self.st["f"][k] == true } pub fn fail(&self, error: impl Into) -> Handle { Handle::new().result(json!({ "pass": false, "error": error.into(), "phase": self.st["phase"], "readings": self.st["readings"] })).done() } pub fn pass(&mut self) -> Handle { let t = self.t(); self.reading("seconds", t); Handle::new() .result(json!({ "pass": true, "readings": self.st["readings"] })) .done() } /// The run cannot do what its entry proves here (a target field, an input or a capability is missing): a FAIL. /// There is no skipped result: the gate picks the entries a target runs (tests/entries.txt). pub fn unmet(&self, why: &str) -> Handle { self.fail(format!("unmet: {why}")) } pub fn keep(&self, h: Handle) -> Handle { h.state(self.st.clone()) } } /// A `requester_market_event` message: the requester's line for a market event of a contract this run made, which the /// host forwards like Held. An OOM kill (guest.oomKill, vm.oomKilled, vm.terminated reason oom) fails the run naming /// the VM; memory pressure (guest.memoryPressure, vm.memoryPressure) is a warning in readings.memory. Call it first. /// The target's bidder DIDs (plan 0040 built 5: a rental's allowlist, filtering discovery). pub fn bidder_dids(args: &Value) -> Vec { args["bidders"].as_array().into_iter().flatten().filter_map(|b| b["did"].as_str()).filter(|d| d.starts_with("did:")).map(String::from).collect() } pub fn market_event(s: &mut S, event: &Event) -> Option { let Event::Message { id, data, .. } = event else { return None; }; if let Some(h) = repeated_failure(s, data) { return Some(h); } let e = data["event"].as_str().unwrap_or_default(); let kill = data["kill"] == true; if data["msg"] != "requester_market_event" || (!kill && !e.ends_with("memoryPressure")) { return None; } // the invocation's instance names the VM's run (a contract is "contract" in its own workspace) let vm = [&data["instance"], &data["contract"]] .iter() .find_map(|v| v.as_str()) .unwrap_or(id) .to_string(); let entry = json!({ "event": e, "vm": vm, "kill": kill, "payload": data["payload"] }); match s.st["readings"]["memory"].as_array_mut() { Some(a) => a.push(entry), None => s.st["readings"]["memory"] = json!([entry]), } kill.then(|| s.fail(format!("OOM: {e} on VM {vm}: {}", data["payload"]))) } /// A step the requester retries (`retry: true`) that fails the same way REPEAT_FAILS times over REPEAT_FAIL_S has failed: /// fail the entry now rather than wait out its deadline (plan 0044, "Timing and watching"). A refusal (`retry: false`) is /// left to the entry, which may expect it. pub const REPEAT_FAILS: u64 = 5; pub const REPEAT_FAIL_S: f64 = 180.0; fn repeated_failure(s: &mut S, data: &Value) -> Option { let msg = data["msg"].as_str().unwrap_or_default(); if msg != "requester_association_failed" || data["retry"] != true { return None; } let err = data["err"].as_str().unwrap_or_default(); let key = format!("{msg} {} {err}", data["step"].as_str().unwrap_or_default()); let t = s.t(); let seen = &mut s.st["repeats"][&key]; let n = seen["n"].as_u64().unwrap_or(0) + 1; let first = seen["first"].as_f64().unwrap_or(t); *seen = json!({ "n": n, "first": first }); (n >= REPEAT_FAILS && t - first >= REPEAT_FAIL_S).then(|| { s.fail(format!( "{msg} {n} times in {:.0} s, the same error each time: {err}", t - first )) }) } /// A call's output, or the run's failure naming the call. pub fn out(s: &S, id: &str, r: Result<&Value, String>) -> Result { r.map(Value::clone) .map_err(|e| s.fail(format!("{id}: {e}"))) } /// Ready pods (name, node) of a pods list. pub fn ready_pods(list: &Value) -> Vec<(String, String)> { list["items"] .as_array() .into_iter() .flatten() .filter(|p| { p["status"]["conditions"].as_array().is_some_and(|c| { c.iter() .any(|c| c["type"] == "Ready" && c["status"] == "True") }) }) .map(|p| { ( p["metadata"]["name"] .as_str() .unwrap_or_default() .to_string(), p["spec"]["nodeName"] .as_str() .unwrap_or_default() .to_string(), ) }) .collect() } pub fn names(list: &Value) -> Vec { list["items"] .as_array() .into_iter() .flatten() .filter_map(|o| o["metadata"]["name"].as_str().map(String::from)) .collect() }