// docs/acceptance/0008-host-observer.md: a VM's timeline on its host (Running there while Held), the release, the // vm.terminated pinned by one getRecord at its computed key (deleted_on_request), the release's records gone, the // orphan check on every host the target reaches: CONFIRMED (identity Ready, 0 startup ERROR, 0 automatic restarts), // nothing of the receipt, no VM process without a Running ComputeVM. Without the winner's kcp its timeline is skipped. use abi::{Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{MIN, S, VM_TERMINATED, get_record, host, kube, out, termination_rkey}; fn run_0008(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let vms = |id: &str, c: &str| kube(id, "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--hold"] }))).timer("deadline", 30 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))), Event::Timer { id } if id == "contract" => { let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default()); s.st["winner"] = w.clone(); s.reading("winner", w.clone()); if !bidders.iter().any(|b| b["did"] == w) { return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target"))); } match bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) { Some(c) => { s.st["cluster"] = json!(c); h = h.call(vms("running", c)); } None => return Ok(s.unmet(&format!("the target names no kcp for the winner {w}: its host timeline is not read"))), } } Event::Timer { id } if id == "poll" => { h = h.call(get_record("term", &format!("at://{}/{VM_TERMINATED}/{}", s.st["winner"].as_str().unwrap_or_default(), termination_rkey(s.st["receipt"].as_str().unwrap_or_default())))); } Event::Message { id, data, .. } if id == "req" => { match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")), "contract" if data["phase"] == "Held" => s.flag("held"), _ => {} } // the --hold process stays (it settles the end): the receipt comes from its log, the winner is its author if s.st["receipt"].is_null() && data["receipt"].is_string() { s.st["receipt"] = data["receipt"].clone(); } if s.is("held") && s.has("held") && s.st["receipt"].is_string() { s.set("summary"); h = h.timer("contract", 1); } } Event::Result { id, result, .. } => { let r = result.output(); if id == "term" && r.is_err() { if s.since("release") > 120.0 { return Ok(s.fail("no vm.terminated 2 min after the release")); } return Ok(s.keep(h.timer("poll", 3_000))); } let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "running" => { let mine: Vec = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).map(|v| json!({ "name": v["metadata"]["name"], "phase": v["status"]["phase"] })).collect(); if !mine.iter().any(|v| v["phase"] == "Running") { return Ok(s.fail(format!("no Running VM of the receipt on the winner: {mine:?}"))); } s.reading("running", json!(mine)); s.mark("release"); h = h.call(host("release", "request.release", json!({ "name": name }))); } "release" => { s.reading("release_s", o["seconds"].clone()); h = h.timer("poll", 1_000); } "term" => { if o["value"]["reason"] != "deleted_on_request" { return Ok(s.fail(format!("vm.terminated {}", o["value"]))); } s.reading("terminated", json!({ "reason": o["value"]["reason"], "request": o["value"]["request"]["uri"], "createdAt": o["value"]["createdAt"] })); h = h.call(host("records", "request.records", json!({ "name": name }))); } "records" => h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))), "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } // every host the target reaches: CONFIRMED (identity Ready, no pending action, 0 startup ERROR, // 0 automatic restarts) and empty of this receipt, no VM process without a Running ComputeVM let reach: Vec<&Value> = bidders.iter().filter(|b| b["cluster"].is_string()).collect(); s.set("hosts"); s.st["fan"] = json!(reach.len() * 3); if reach.len() != bidders.len() || reach.is_empty() { return Ok(s.unmet(&format!("the target names a kcp for {} of its {} bidders: every host is checked", reach.len(), bidders.len()))); } for (j, b) in bidders.iter().enumerate().filter(|(_, b)| b["cluster"].is_string()) { let c = b["cluster"].as_str().unwrap_or_default(); h = h.call(kube(&format!("mi-{j}"), "get", c, steps::sw("MarketIdentity", "bidder"))).call(vms(&format!("vm-{j}"), c)).call(host(&format!("hl-{j}"), "bidder.health", json!({ "name": b["name"] }))); } } x if s.is("hosts") && x.len() > 3 && ["mi-", "vm-", "hl-"].contains(&&x[..3]) => { let j: usize = x[3..].parse().unwrap_or(0); let name = bidders[j]["name"].as_str().unwrap_or_default().to_string(); match &x[..2] { "mi" => { if o["status"]["phase"] != "Ready" || !o["status"]["pendingAction"].is_null() { return Ok(s.fail(format!("{name} MarketIdentity {}", o["status"]))); } } "vm" => { let items = o["items"].as_array().cloned().unwrap_or_default(); if items.iter().any(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]) { return Ok(s.fail(format!("{name} still runs a VM of the receipt"))); } s.st["readings"]["hosts"][&name]["runningVMs"] = json!(items.iter().filter(|v| v["status"]["phase"] == "Running").count()); } _ => { if o["error"].is_string() || o["startupErrors"] != 0 || o["restarts"] != 0 { return Ok(s.fail(format!("{name} not CONFIRMED: {o}"))); } for k in ["started", "errors", "vmProcesses"] { s.st["readings"]["hosts"][&name][k] = o[k].clone(); } } } let left = s.st["fan"].as_u64().unwrap_or(1) - 1; s.st["fan"] = json!(left); if left == 0 { for (n, r) in s.st["readings"]["hosts"].as_object().cloned().unwrap_or_default() { if r["vmProcesses"].as_u64().unwrap_or(0) > r["runningVMs"].as_u64().unwrap_or(0) { return Ok(s.fail(format!("{n}: {} VM processes, {} Running ComputeVMs (orphans)", r["vmProcesses"], r["runningVMs"]))); } } return Ok(s.pass()); } } _ => {} } } _ => {} } Ok(s.keep(h)) } entries! { run => run_0008 }