~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
372 lines · 15.5 KB Raw History
//! The bookkeeping of a runbook's state machine: `S` holds the state (phase, marks, readings, flags), the clock of
//! this event, and ends the run as pass or fail; calls to the test host and to kube by name; the market's rkeys.
use abi::{Call, Event, Handle};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};

pub const T: &str = "computer.socialweb.test.";
pub const MIN: u64 = 60_000;
pub const ASSOC: &str = "com.publicdomainrelay.temp.market.association";
pub const ENDORSE: &str = "link.ident.endorsement";
pub const VM_TERMINATED: &str = "com.publicdomainrelay.temp.compute.events.vm.terminated";

/// Seconds of an ISO time (UTC, `Z`).
pub fn secs(iso: &str) -> f64 {
    let n = |a: usize, b: usize| {
        iso.get(a..b)
            .and_then(|s| s.parse::<i64>().ok())
            .unwrap_or(0)
    };
    let (y, m, d) = (n(0, 4), n(5, 7), n(8, 10));
    let (y2, m2) = if m <= 2 { (y - 1, m + 9) } else { (y, m - 3) };
    let days = 365 * y2 + y2 / 4 - y2 / 100 + y2 / 400 + (153 * m2 + 2) / 5 + d;
    let frac = iso
        .get(19..)
        .and_then(|f| f.trim_end_matches('Z').parse::<f64>().ok())
        .unwrap_or(0.0);
    (days * 86400 + n(11, 13) * 3600 + n(14, 16) * 60 + n(17, 19)) as f64 + frac
}

pub const GUEST_SECRETS: &str = "com.publicdomainrelay.temp.compute.events.guest.secrets";
pub const MARKET_EVENT: &str = "com.publicdomainrelay.temp.market.event";

/// The run's `--secrets` bundle (test values): a string value; an object value the guest gets as its compact JSON text;
/// modes 0644 and 0400; a file in /etc (an existing directory: its mode and owner must not change); all in a directory
/// the guest does not have.
pub fn secrets_bundle() -> Value {
    json!([
        { "path": "/root/swc-test-secrets/a.txt", "value": "swc-test-string-0001" },
        { "path": "/root/swc-test-secrets/b.json", "value": { "k": "v", "n": 1 } },
        { "path": "/root/swc-test-secrets/c.txt", "value": "c", "mode": "0644" },
        { "path": "/root/swc-test-secrets/d.txt", "value": "d", "mode": "0400" },
        { "path": "/etc/swc-test-secret", "value": "e" }
    ])
}

pub const SECRETS_BAD_PATH: &str = "entry 0: the path is not a clean absolute path";
pub const SECRETS_BAD_MODE: &str = "entry 0: the mode is not an octal string of at most 0777";

/// Bundles every requester refuses before the RFP (the CLI's `--secrets` read, exit 2, nothing rented), each with the
/// message it must print: a `..` path and a relative one; a path with a control character (a trailing newline, which
/// a shell's `$(...)` would strip); a `..` hidden by a trailing newline; a mode with a trailing newline.
pub fn secrets_bundles_refused() -> Vec<(&'static str, Value, &'static str)> {
    vec![
        ("dots", json!([{ "path": "/root/swc-test-secrets/../escape", "value": "x" }, { "path": "relative/x", "value": "y" }]), SECRETS_BAD_PATH),
        ("ctl", json!([{ "path": "/root/swc-test-secrets/x\n", "value": "x" }]), SECRETS_BAD_PATH),
        ("ctldots", json!([{ "path": "/root/swc-test-secrets/..\n", "value": "x" }]), SECRETS_BAD_PATH),
        ("ctlmode", json!([{ "path": "/root/swc-test-secrets/m.txt", "value": "x", "mode": "0600\n" }]), SECRETS_BAD_MODE),
    ]
}

/// listRecords pages as repo.listCreated returns them ({uri, createdAt}): the uris created at or after `since`.
pub fn created_since(o: &Value, since: f64) -> Vec<Value> {
    o["records"].as_array().into_iter().flatten().filter(|r| r["createdAt"].as_str().is_some_and(|c| secs(c) >= since)).cloned().collect()
}

/// request.contract's answer: (status.secretsFetchedVia, the SecretsApplied condition's reason), empty when unset.
pub fn secrets_status(o: &Value) -> (String, String) {
    let via = o["secretsFetchedVia"].as_str().unwrap_or_default().to_string();
    let reason = o["conditions"].as_array().into_iter().flatten().find(|c| c["type"] == "SecretsApplied").and_then(|c| c["reason"].as_str()).unwrap_or_default().to_string();
    (via, reason)
}

/// The SecretsApplied reason of a getSecrets fetch over `channel` (requester secretsFetchedReason).
pub fn fetched_reason(channel: &str) -> &'static str {
    match channel {
        "iroh" => "GetSecretsIroh",
        "webrtc" => "GetSecretsWebRTC",
        "https" => "GetSecretsHTTPS",
        _ => "GetSecrets",
    }
}

/// The user_data runcmd line that turns the iroh channel off for one contract: requester.json without its ticket, so
/// the guest agent forwards getSecrets over WebRTC.
pub const SECRETS_NO_IROH: &str = "jq -c 'del(.iroh)' /etc/socialweb-guest-agent/requester.json > /etc/socialweb-guest-agent/requester.json.t && mv /etc/socialweb-guest-agent/requester.json.t /etc/socialweb-guest-agent/requester.json";

/// request.ssh exec: one line with the files, the modes of the directory and the files, and /etc's mode and owner.
pub const SECRETS_READ: &str = "D=/root/swc-test-secrets; printf 'SECRETS<%s|%s|%s|%s|%s>\\n' \"$(cat $D/a.txt)\" \"$(cat $D/b.json)\" \"$(stat -c %a $D $D/a.txt $D/b.json $D/c.txt $D/d.txt /etc/swc-test-secret | tr '\\n' ' ')\" \"$(stat -c '%a %U:%G' /etc)\" \"$(cat /etc/swc-test-secret)\"";
pub const SECRETS_WANT: &str = "SECRETS<swc-test-string-0001|{\"k\":\"v\",\"n\":1}|700 600 600 644 400 600 |755 root:root|e>";

/// The guest wrote the bundle of secrets_bundle(), as SECRETS_READ printed it.
pub fn secrets_written(output: &str) -> Result<(), String> {
    if output.lines().any(|l| l.trim_end() == SECRETS_WANT) {
        return Ok(());
    }
    Err(format!("the guest's secrets files: want {SECRETS_WANT}, got {}", output.lines().find(|l| l.starts_with("SECRETS<")).unwrap_or("no SECRETS line")))
}

/// request.ssh exec: a symlink at a target, then a refetch: the file replaces the link and the link's target is untouched.
pub const SECRETS_SYMLINK: &str = "D=/root/swc-test-secrets; echo keep > $D/target && ln -sfn $D/target $D/a.txt && SECRETS_REFETCH=1 /usr/local/bin/setup-secrets.sh >/dev/null 2>&1; printf 'LINK<%s|%s|%s>\\n' \"$(stat -c %F $D/a.txt)\" \"$(cat $D/target)\" \"$(cat $D/a.txt)\"";
pub const SECRETS_SYMLINK_WANT: &str = "LINK<regular file|keep|swc-test-string-0001>";

/// request.ssh exec: the guest asks for a sealed bundle as a guest agent of comms mode Firehose would (its marker, no
/// ready file, a restart).
pub const SECRETS_FORCE_SEALED: &str = "printf '{\"fallback\":\"sealed\"}\\n' > /etc/socialweb-guest-agent/secrets.json && rm -f /run/socialweb-guest-agent/secrets.ready && systemctl restart socialweb-guest-agent.service && echo FORCED";


/// termination-payload-<first 32 hex of sha256("terminated:<receipt uri>")>: the receipt's vm.terminated rkey.
pub fn termination_rkey(receipt: &str) -> String {
    format!(
        "termination-payload-{}",
        key_hash(&format!("terminated:{receipt}"))
    )
}

/// The first 32 hex of sha256(key): a bidder event record's rkey ends `-payload-<this>` of its StateSync key
/// (market.EventRKeys); a report's key is `<nsid>:<tag>:<receipt uri>`.
pub fn key_hash(key: &str) -> String {
    Sha256::digest(key)[..16]
        .iter()
        .map(|b| format!("{b:02x}"))
        .collect()
}

/// A call to the test host: computer.socialweb.test.<what>.
pub fn host(id: &str, what: &str, input: Value) -> Call {
    Call::new(id, format!("{T}{what}")).input(input)
}

/// A subscription of the test host (its log events, an observer): atlibs takes a non-exec subscription's arguments as
/// params, never input, and makes every param a string (an array would be comma-joined): the arguments go as one
/// `json` param, which the host parses.
pub fn host_sub(id: &str, what: &str, args: Value) -> Call {
    Call::new(id, format!("{T}{what}"))
        .params(json!({ "json": args.to_string() }))
        .subscribe()
}

/// A kube call: com.fedfork.exec.kube.<verb> on `cluster`.
pub fn kube(id: &str, verb: &str, cluster: &str, mut input: Value) -> Call {
    input["cluster"] = json!(cluster);
    Call::new(id, format!("com.fedfork.exec.kube.{verb}")).input(input)
}

/// A getRecord of an at:// URI, a public read of its repo.
pub fn get_record(id: &str, uri: &str) -> Call {
    let mut p = uri.trim_start_matches("at://").splitn(3, '/');
    let (repo, collection, rkey) = (
        p.next().unwrap_or_default(),
        p.next().unwrap_or_default(),
        p.next().unwrap_or_default(),
    );
    Call::new(id, "com.atproto.repo.getRecord")
        .params(json!({ "repo": repo, "collection": collection, "rkey": rkey }))
        .service(repo)
}

pub fn sw(kind: &str, name: &str) -> Value {
    json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": kind, "name": name })
}

/// Before an apply: every ssh public key in a templated spec is THIS run's (from the host), and there is one. A key
/// baked into a file (an earlier run's) or a placeholder left in is refused: the guest would refuse our ssh.
pub fn keys_are(spec: &str, key: &str) -> Result<(), String> {
    let key = key.trim();
    if key.is_empty() || !key.starts_with("ssh-") {
        return Err("the run has no ssh public key from the host".into());
    }
    let mut n = 0;
    for l in spec.lines() {
        let t = l
            .trim()
            .trim_start_matches("- ")
            .trim()
            .trim_matches(|c| c == '"' || c == '\'');
        if ["ssh-ed25519 ", "ssh-rsa ", "ecdsa-sha2-", "sk-ssh-"]
            .iter()
            .any(|p| t.starts_with(p))
        {
            if t != key {
                return Err(format!(
                    "the spec carries an ssh key that is not this run's: {}",
                    t.chars().take(40).collect::<String>()
                ));
            }
            n += 1;
        }
    }
    if n == 0 {
        return Err("the spec carries no ssh key".into());
    }
    Ok(())
}

pub struct S {
    pub st: Value,
    pub now: f64,
}

impl S {
    pub fn new(state: Value, now_iso: &str) -> Self {
        let now = secs(now_iso);
        let st = if state.is_null() {
            json!({ "phase": "start", "t0": now, "readings": {}, "f": {} })
        } else {
            state
        };
        Self { st, now }
    }

    pub fn phase(&self) -> String {
        self.st["phase"].as_str().unwrap_or_default().to_string()
    }

    pub fn is(&self, p: &str) -> bool {
        self.st["phase"] == p
    }

    pub fn set(&mut self, p: &str) {
        self.st["phase"] = json!(p);
    }

    /// Seconds since the run started.
    pub fn t(&self) -> f64 {
        ((self.now - self.st["t0"].as_f64().unwrap_or(self.now)) * 10.0).round() / 10.0
    }

    pub fn mark(&mut self, k: &str) {
        self.st["marks"][k] = json!(self.now);
    }

    /// Seconds since mark `k`.
    pub fn since(&self, k: &str) -> f64 {
        ((self.now - self.st["marks"][k].as_f64().unwrap_or(self.now)) * 10.0).round() / 10.0
    }

    pub fn reading(&mut self, k: &str, v: impl Into<Value>) {
        self.st["readings"][k] = v.into();
    }

    pub fn flag(&mut self, k: &str) {
        self.st["f"][k] = json!(true);
    }

    pub fn has(&self, k: &str) -> bool {
        self.st["f"][k] == true
    }

    pub fn fail(&self, error: impl Into<String>) -> Handle {
        Handle::new().result(json!({ "pass": false, "error": error.into(), "phase": self.st["phase"], "readings": self.st["readings"] })).done()
    }

    pub fn pass(&mut self) -> Handle {
        let t = self.t();
        self.reading("seconds", t);
        Handle::new()
            .result(json!({ "pass": true, "readings": self.st["readings"] }))
            .done()
    }

    /// The run cannot do what its entry proves here (a target field, an input or a capability is missing): a FAIL.
    /// There is no skipped result: the gate picks the entries a target runs (tests/entries.txt).
    pub fn unmet(&self, why: &str) -> Handle {
        self.fail(format!("unmet: {why}"))
    }

    pub fn keep(&self, h: Handle) -> Handle {
        h.state(self.st.clone())
    }
}

/// A `requester_market_event` message: the requester's line for a market event of a contract this run made, which the
/// host forwards like Held. An OOM kill (guest.oomKill, vm.oomKilled, vm.terminated reason oom) fails the run naming
/// the VM; memory pressure (guest.memoryPressure, vm.memoryPressure) is a warning in readings.memory. Call it first.
/// The target's bidder DIDs (plan 0040 built 5: a rental's allowlist, filtering discovery).
pub fn bidder_dids(args: &Value) -> Vec<String> {
    args["bidders"].as_array().into_iter().flatten().filter_map(|b| b["did"].as_str()).filter(|d| d.starts_with("did:")).map(String::from).collect()
}

pub fn market_event(s: &mut S, event: &Event) -> Option<Handle> {
    let Event::Message { id, data, .. } = event else {
        return None;
    };
    if let Some(h) = repeated_failure(s, data) {
        return Some(h);
    }
    let e = data["event"].as_str().unwrap_or_default();
    let kill = data["kill"] == true;
    if data["msg"] != "requester_market_event" || (!kill && !e.ends_with("memoryPressure")) {
        return None;
    }
    // the invocation's instance names the VM's run (a contract is "contract" in its own workspace)
    let vm = [&data["instance"], &data["contract"]]
        .iter()
        .find_map(|v| v.as_str())
        .unwrap_or(id)
        .to_string();
    let entry = json!({ "event": e, "vm": vm, "kill": kill, "payload": data["payload"] });
    match s.st["readings"]["memory"].as_array_mut() {
        Some(a) => a.push(entry),
        None => s.st["readings"]["memory"] = json!([entry]),
    }
    kill.then(|| s.fail(format!("OOM: {e} on VM {vm}: {}", data["payload"])))
}

/// A step the requester retries (`retry: true`) that fails the same way REPEAT_FAILS times over REPEAT_FAIL_S has failed:
/// fail the entry now rather than wait out its deadline (plan 0044, "Timing and watching"). A refusal (`retry: false`) is
/// left to the entry, which may expect it.
pub const REPEAT_FAILS: u64 = 5;
pub const REPEAT_FAIL_S: f64 = 180.0;

fn repeated_failure(s: &mut S, data: &Value) -> Option<Handle> {
    let msg = data["msg"].as_str().unwrap_or_default();
    if msg != "requester_association_failed" || data["retry"] != true {
        return None;
    }
    let err = data["err"].as_str().unwrap_or_default();
    let key = format!("{msg} {} {err}", data["step"].as_str().unwrap_or_default());
    let t = s.t();
    let seen = &mut s.st["repeats"][&key];
    let n = seen["n"].as_u64().unwrap_or(0) + 1;
    let first = seen["first"].as_f64().unwrap_or(t);
    *seen = json!({ "n": n, "first": first });
    (n >= REPEAT_FAILS && t - first >= REPEAT_FAIL_S).then(|| {
        s.fail(format!(
            "{msg} {n} times in {:.0} s, the same error each time: {err}",
            t - first
        ))
    })
}

/// A call's output, or the run's failure naming the call.
pub fn out(s: &S, id: &str, r: Result<&Value, String>) -> Result<Value, Handle> {
    r.map(Value::clone)
        .map_err(|e| s.fail(format!("{id}: {e}")))
}

/// Ready pods (name, node) of a pods list.
pub fn ready_pods(list: &Value) -> Vec<(String, String)> {
    list["items"]
        .as_array()
        .into_iter()
        .flatten()
        .filter(|p| {
            p["status"]["conditions"].as_array().is_some_and(|c| {
                c.iter()
                    .any(|c| c["type"] == "Ready" && c["status"] == "True")
            })
        })
        .map(|p| {
            (
                p["metadata"]["name"]
                    .as_str()
                    .unwrap_or_default()
                    .to_string(),
                p["spec"]["nodeName"]
                    .as_str()
                    .unwrap_or_default()
                    .to_string(),
            )
        })
        .collect()
}

pub fn names(list: &Value) -> Vec<String> {
    list["items"]
        .as_array()
        .into_iter()
        .flatten()
        .filter_map(|o| o["metadata"]["name"].as_str().map(String::from))
        .collect()
}