~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
154 lines · 9.1 KB Raw History
// docs/acceptance/0008-host-observer.md: a VM's timeline on its host (Running there while Held), the release, the
// vm.terminated pinned by one getRecord at its computed key (deleted_on_request), the release's records gone, the
// orphan check on every host the target reaches: CONFIRMED (identity Ready, 0 startup ERROR, 0 automatic restarts),
// nothing of the receipt, no VM process without a Running ComputeVM. Without the winner's kcp its timeline is skipped.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, VM_TERMINATED, get_record, host, kube, out, termination_rkey};

fn run_0008(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let vms = |id: &str, c: &str| kube(id, "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }));
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            s.set("held");
            h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--hold"] }))).timer("deadline", 30 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "contract" => {
            let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default());
            s.st["winner"] = w.clone();
            s.reading("winner", w.clone());
            if !bidders.iter().any(|b| b["did"] == w) {
                return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
            }
            match bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) {
                Some(c) => {
                    s.st["cluster"] = json!(c);
                    h = h.call(vms("running", c));
                }
                None => return Ok(s.unmet(&format!("the target names no kcp for the winner {w}: its host timeline is not read"))),
            }
        }
        Event::Timer { id } if id == "poll" => {
            h = h.call(get_record("term", &format!("at://{}/{VM_TERMINATED}/{}", s.st["winner"].as_str().unwrap_or_default(), termination_rkey(s.st["receipt"].as_str().unwrap_or_default()))));
        }
        Event::Message { id, data, .. } if id == "req" => {
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
                "contract" if data["phase"] == "Held" => s.flag("held"),
                _ => {}
            }
            // the --hold process stays (it settles the end): the receipt comes from its log, the winner is its author
            if s.st["receipt"].is_null() && data["receipt"].is_string() {
                s.st["receipt"] = data["receipt"].clone();
            }
            if s.is("held") && s.has("held") && s.st["receipt"].is_string() {
                s.set("summary");
                h = h.timer("contract", 1);
            }
        }
        Event::Result { id, result, .. } => {
            let r = result.output();
            if id == "term" && r.is_err() {
                if s.since("release") > 120.0 {
                    return Ok(s.fail("no vm.terminated 2 min after the release"));
                }
                return Ok(s.keep(h.timer("poll", 3_000)));
            }
            let o = match out(&s, &id, r) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "running" => {
                    let mine: Vec<Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).map(|v| json!({ "name": v["metadata"]["name"], "phase": v["status"]["phase"] })).collect();
                    if !mine.iter().any(|v| v["phase"] == "Running") {
                        return Ok(s.fail(format!("no Running VM of the receipt on the winner: {mine:?}")));
                    }
                    s.reading("running", json!(mine));
                    s.mark("release");
                    h = h.call(host("release", "request.release", json!({ "name": name })));
                }
                "release" => {
                    s.reading("release_s", o["seconds"].clone());
                    h = h.timer("poll", 1_000);
                }
                "term" => {
                    if o["value"]["reason"] != "deleted_on_request" {
                        return Ok(s.fail(format!("vm.terminated {}", o["value"])));
                    }
                    s.reading("terminated", json!({ "reason": o["value"]["reason"], "request": o["value"]["request"]["uri"], "createdAt": o["value"]["createdAt"] }));
                    h = h.call(host("records", "request.records", json!({ "name": name })));
                }
                "records" => h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))),
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                    }
                    // every host the target reaches: CONFIRMED (identity Ready, no pending action, 0 startup ERROR,
                    // 0 automatic restarts) and empty of this receipt, no VM process without a Running ComputeVM
                    let reach: Vec<&Value> = bidders.iter().filter(|b| b["cluster"].is_string()).collect();
                    s.set("hosts");
                    s.st["fan"] = json!(reach.len() * 3);
                    if reach.len() != bidders.len() || reach.is_empty() {
                        return Ok(s.unmet(&format!("the target names a kcp for {} of its {} bidders: every host is checked", reach.len(), bidders.len())));
                    }
                    for (j, b) in bidders.iter().enumerate().filter(|(_, b)| b["cluster"].is_string()) {
                        let c = b["cluster"].as_str().unwrap_or_default();
                        h = h.call(kube(&format!("mi-{j}"), "get", c, steps::sw("MarketIdentity", "bidder"))).call(vms(&format!("vm-{j}"), c)).call(host(&format!("hl-{j}"), "bidder.health", json!({ "name": b["name"] })));
                    }
                }
                x if s.is("hosts") && x.len() > 3 && ["mi-", "vm-", "hl-"].contains(&&x[..3]) => {
                    let j: usize = x[3..].parse().unwrap_or(0);
                    let name = bidders[j]["name"].as_str().unwrap_or_default().to_string();
                    match &x[..2] {
                        "mi" => {
                            if o["status"]["phase"] != "Ready" || !o["status"]["pendingAction"].is_null() {
                                return Ok(s.fail(format!("{name} MarketIdentity {}", o["status"])));
                            }
                        }
                        "vm" => {
                            let items = o["items"].as_array().cloned().unwrap_or_default();
                            if items.iter().any(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]) {
                                return Ok(s.fail(format!("{name} still runs a VM of the receipt")));
                            }
                            s.st["readings"]["hosts"][&name]["runningVMs"] = json!(items.iter().filter(|v| v["status"]["phase"] == "Running").count());
                        }
                        _ => {
                            if o["error"].is_string() || o["startupErrors"] != 0 || o["restarts"] != 0 {
                                return Ok(s.fail(format!("{name} not CONFIRMED: {o}")));
                            }
                            for k in ["started", "errors", "vmProcesses"] {
                                s.st["readings"]["hosts"][&name][k] = o[k].clone();
                            }
                        }
                    }
                    let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
                    s.st["fan"] = json!(left);
                    if left == 0 {
                        for (n, r) in s.st["readings"]["hosts"].as_object().cloned().unwrap_or_default() {
                            if r["vmProcesses"].as_u64().unwrap_or(0) > r["runningVMs"].as_u64().unwrap_or(0) {
                                return Ok(s.fail(format!("{n}: {} VM processes, {} Running ComputeVMs (orphans)", r["vmProcesses"], r["runningVMs"])));
                            }
                        }
                        return Ok(s.pass());
                    }
                }
                _ => {}
            }
        }
        _ => {}
    }
    Ok(s.keep(h))
}

entries! { run => run_0008 }