~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
242 lines · 12.4 KB Raw History
// docs/acceptance/0003-cli-webrtc-ssh.md (and 0001 section 9, ACC34): `run request --hold` with ssh over WebRTC,
// fresh and from a restarted invocation kcp (<= 10 s since rc165's fix), no plaintext signaling or password in any
// log, Released, every record of the run gone (the release's vm.delete too), the winner's host empty. Discovery; with
// --per-host and every bidder's kcp, each host in turn by pausing the others (restored after); where a bidder's kcp is read
// only (`write` false: prod through the ops routes) nothing is paused and each round names its host in the request
// (bidderDids) instead.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, host, kube, out, sw};

// an argv with a space, a quote and a leading dash reaches the guest word for word (`--exec -- <command> [args...]`, plan 0043)
const ARGV: [&str; 7] = ["sh", "-c", "uname -a; id; printf '%s|' \"$@\"", "x", "a b", "-c", "d'e"];
const ARGV_OUT: &str = "a b|-c|d'e|";

fn bidder_patch(id: &str, cluster: &str, suspend: bool) -> abi::Call {
    let mut q = sw("MarketBidder", "app");
    q["patch"] = json!({ "spec": { "suspend": suspend } });
    kube(id, "patch", cluster, q)
}

/// Without the pause: the read of every bidder's MarketBidder phase that follows a pause, at once.
fn phases(s: &mut S, bidders: &[Value], h: Handle) -> Handle {
    s.mark("arrange");
    s.st["ok"] = json!(true);
    bidders.iter().enumerate().fold(h, |h, (j, b)| h.call(kube(&format!("phase-{j}"), "get", b["cluster"].as_str().unwrap_or_default(), sw("MarketBidder", "app"))))
}

fn run_0003(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let base = args["name"].as_str().ok_or("name")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let per_host = args["perHost"] == true && !bidders.is_empty() && bidders.iter().all(|b| b["cluster"].is_string());
    if args["perHost"] == true && !per_host {
        return Ok(s.unmet("perHost: the target names no kcp for one of its bidders"));
    }
    // operator write access to every bidder's kcp: the rounds pause the others by patch; without it they name the round's bidder
    let pause = bidders.iter().all(|b| b["write"] != false);
    let rounds = if per_host { bidders.len() } else { 1 };
    let i = s.st["i"].as_u64().unwrap_or(0) as usize;
    let name = format!("{base}-{i}");
    let mut h = Handle::new();
    // pause every bidder but round i's (per host), or nothing
    let arrange = |h: Handle, i: Option<usize>| bidders.iter().enumerate().fold(h, |h, (j, b)| h.call(bidder_patch(&format!("pause-{j}"), b["cluster"].as_str().unwrap_or_default(), i.is_some_and(|i| i != j))));
    let start = |s: &mut S, h: Handle| {
        s.set("held");
        s.st["f"] = json!({});
        let a = json!(["--hold", "--tunnel", "ssh:22:webrtc", "--transport", "webrtc", "--exec", "--", "sh", "-c", "uname -a; id"]);
        let mut q = json!({ "name": name, "args": a });
        if per_host && !pause {
            q["bidderDids"] = json!([bidders[i]["did"]]);
        }
        h.call(steps::host_sub("req", "request.start", q))
    };
    match run.event {
        Event::Start => {
            h = h.timer("deadline", (25 * rounds as u64) * MIN);
            if per_host {
                s.set("arrange");
                s.st["fan"] = json!(bidders.len());
                h = if pause { arrange(h, Some(0)) } else { phases(&mut s, &bidders, h) };
            } else {
                h = start(&mut s, h);
            }
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done (phase {}, round {i})", s.phase()))),
        Event::Timer { id } if id == "poll" => {
            for (j, b) in bidders.iter().enumerate() {
                h = h.call(kube(&format!("phase-{j}"), "get", b["cluster"].as_str().unwrap_or_default(), sw("MarketBidder", "app")));
            }
            s.st["fan"] = json!(bidders.len());
            s.st["ok"] = json!(true);
        }
        Event::Message { id, data, .. } if id == "req" => {
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
                "contract" if data["phase"] == "Held" && !s.has("held") => {
                    s.flag("held");
                    s.mark("held");
                }
                "ssh returned" if !s.has("ssh") => {
                    if data["transport"] != "webrtc" || data["exit"] != 0 {
                        return Ok(s.fail(format!("the first ssh: {data}")));
                    }
                    s.flag("ssh");
                    s.st["fresh"] = data["seconds"].clone();
                }
                "exited" => s.flag("exited"),
                _ => {}
            }
            if s.is("held") && s.has("held") && s.has("ssh") && s.has("exited") {
                s.set("restarted");
                s.mark("ssh2");
                h = h.cancel("req").call(host("ssh2", "request.ssh", json!({ "name": name, "transport": "webrtc", "argv": ARGV })));
            }
        }
        Event::Result { id, result, .. } => {
            if id.starts_with("pause-") || id.starts_with("phase-") {
                let o = match out(&s, &id, result.output()) {
                    Ok(o) => o,
                    Err(f) => return Ok(f),
                };
                if id.starts_with("phase-") {
                    let j: usize = id[6..].parse().unwrap_or(0);
                    let want = if s.is("resume") || j == i || !pause { "Ready" } else { "Suspended" };
                    if o["status"]["phase"] != want {
                        s.st["ok"] = json!(false);
                    }
                }
                let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
                s.st["fan"] = json!(left);
                if left == 0 {
                    if id.starts_with("pause-") {
                        s.mark("arrange");
                        return Ok(s.keep(h.timer("poll", 5_000)));
                    }
                    if s.st["ok"] != true {
                        if s.since("arrange") > 300.0 {
                            return Ok(s.fail("the bidders did not reach their phases in 5 min"));
                        }
                        return Ok(s.keep(h.timer("poll", 5_000)));
                    }
                    if s.is("resume") {
                        return Ok(s.pass());
                    }
                    h = start(&mut s, h);
                }
                return Ok(s.keep(h));
            }
            let o = match out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "ssh2" => {
                    let wall = s.since("ssh2");
                    let txt = o["output"].as_str().unwrap_or_default();
                    if o["ok"] != true || !txt.contains("Linux") || !txt.contains("uid=") || !txt.contains(ARGV_OUT) {
                        return Ok(s.fail(format!("restarted ssh: {txt}")));
                    }
                    // the runbook's number is the CLI's own `ssh returned` seconds (the wall time adds the kcp's restart
                    // from its state dir and its stop)
                    let t = txt
                        .lines()
                        .filter_map(|l| serde_json::from_str::<Value>(l).ok())
                        .find(|l| l["msg"] == "ssh returned")
                        .and_then(|l| l["seconds"].as_f64())
                        .map_or(wall, |x| (x * 10.0).round() / 10.0);
                    let max = args["restartSshMaxS"].as_f64().unwrap_or(10.0);
                    if t > max {
                        return Ok(s.fail(format!("the restarted invocation's ssh took {t} s (want <= {max} s)")));
                    }
                    s.st["restarted"] = json!(t);
                    s.st["restarted_wall"] = json!(wall);
                    h = h.call(host("summary", "request.summary", json!({ "name": name })));
                }
                "summary" => {
                    let w = o["contract"]["winnerDid"].clone();
                    let Some(b) = bidders.iter().find(|b| b["did"] == w) else { return Ok(s.fail(format!("winner {w} is not a target bidder"))) };
                    if per_host && bidders[i]["did"] != w {
                        return Ok(s.fail(format!("{} {}, the winner is {w}", bidders[i]["name"], if pause { "paused all but itself" } else { "was the only bidder named" })));
                    }
                    s.st["winner"] = json!({ "did": w, "name": b["name"], "cluster": b["cluster"] });
                    s.st["receipt"] = o["contract"]["receiptUri"].clone();
                    h = h.call(host("release", "request.release", json!({ "name": name })));
                }
                "release" => {
                    s.st["release_s"] = o["seconds"].clone();
                    h = h.call(host("logs", "request.checkLogs", json!({ "name": name })));
                }
                "logs" => {
                    if o["ok"] != true {
                        return Ok(s.fail(format!("logs: {}", o["problems"])));
                    }
                    h = h.call(host("records", "request.records", json!({ "name": name })));
                }
                "records" => {
                    s.st["records_checked"] = json!(o["uris"].as_array().map_or(0, Vec::len));
                    h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
                }
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left after the release: {} {}", o["left"], o["errors"])));
                    }
                    match s.st["winner"]["cluster"].as_str() {
                        Some(c) => h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))),
                        None => return Ok(s.unmet("the target names no kcp for the winner: its host is not checked empty")),
                    }
                }
                "vms" => {
                    if o["items"].as_array().into_iter().flatten().any(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]) {
                        return Ok(s.fail("the winner's host still runs this run's VM"));
                    }
                    { let h = next(&mut s, h, i, rounds, &bidders, per_host); return Ok(s.keep(h)); };
                }
                _ => {}
            }
        }
        _ => {}
    }
    Ok(s.keep(h))
}

/// Record the round, then the next one (re-arranged), or resume every bidder / pass.
fn next(s: &mut S, mut h: Handle, i: usize, rounds: usize, bidders: &[Value], per_host: bool) -> Handle {
    let r = json!({ "winner": s.st["winner"]["name"], "fresh_s": s.st["fresh"], "restarted_s": s.st["restarted"], "restarted_wall_s": s.st["restarted_wall"], "release_s": s.st["release_s"], "records_checked": s.st["records_checked"] });
    let key = if per_host { bidders[i]["name"].as_str().unwrap_or("?").to_string() } else { "discovery".into() };
    s.st["readings"][key] = r;
    s.st["i"] = json!(i + 1);
    let pause = bidders.iter().all(|b| b["write"] != false);
    if i + 1 < rounds {
        s.set("arrange");
        s.st["fan"] = json!(bidders.len());
        if !pause {
            return phases(s, bidders, h);
        }
        for (j, b) in bidders.iter().enumerate() {
            h = h.call(bidder_patch(&format!("pause-{j}"), b["cluster"].as_str().unwrap_or_default(), j != i + 1));
        }
        return h;
    }
    if per_host {
        s.set("resume");
        s.st["fan"] = json!(bidders.len());
        if !pause {
            return phases(s, bidders, h);
        }
        for (j, b) in bidders.iter().enumerate() {
            h = h.call(bidder_patch(&format!("pause-{j}"), b["cluster"].as_str().unwrap_or_default(), false));
        }
        return h;
    }
    let t = s.t();
    s.reading("seconds", t);
    abi::Handle::new().result(json!({ "pass": true, "readings": s.st["readings"] })).done()
}

entries! { run => run_0003 }