0
0
// docs/acceptance/0006-rollout-compat.md. run: on each bidder kcp the host reaches, the MarketIdentity is Ready and
// signed in with no pending action, the Bidder runs bidder-tangled-vouch, the DID is the target's, and its host (bidder.health,
// no sudo) has 0 ERROR in the bidder's first 10 min, 0 automatic restarts, bidder_guest_image_ready since its start, no VM
// process without a Running ComputeVM. compat: the
// unchanged CLI (SWC_COMPAT_REQUESTER) rents and releases against the new servers: Held, `id` ran, Released, every
// record of the run gone (the release's vm.delete too: no release leftovers since rc165), none created in the repo during
// the run left (the release process's own records), the CLI's version a reading.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, host, kube, out, sw};
fn bidders(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let bs: Vec<Value> = args["bidders"].as_array().into_iter().flatten().filter(|b| b["cluster"].is_string()).cloned().collect();
let mut h = Handle::new();
match run.event {
Event::Start => {
let all = args["bidders"].as_array().map_or(0, |a| a.len());
if bs.is_empty() || bs.len() != all {
return Ok(s.unmet(&format!("the target names a kcp for {} of its {all} bidders: every bidder is checked", bs.len())));
}
s.st["fan"] = json!(bs.len() * 5);
for (j, b) in bs.iter().enumerate() {
let c = b["cluster"].as_str().unwrap_or_default();
h = h
.call(kube(&format!("mi-{j}"), "get", c, sw("MarketIdentity", "bidder")))
.call(kube(&format!("bd-{j}"), "get", c, sw("Bidder", "bidder")))
.call(kube(&format!("mb-{j}"), "get", c, sw("MarketBidder", "app")))
.call(kube(&format!("vm-{j}"), "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })))
.call(host(&format!("hl-{j}"), "bidder.health", json!({ "name": b["name"] })));
}
}
Event::Result { id, result, .. } => {
let o = match out(&s, &id, result.output()) {
Ok(o) => o,
Err(f) => return Ok(f),
};
let j: usize = id[3..].parse().unwrap_or(0);
let name = bs[j]["name"].as_str().unwrap_or_default().to_string();
match &id[..2] {
"mi" => {
let signed = o["status"]["conditions"].as_array().into_iter().flatten().find(|c| c["type"] == "SignedIn").map(|c| c["status"].clone());
if o["status"]["phase"] != "Ready" || !o["status"]["pendingAction"].is_null() || signed.as_ref().is_some_and(|s| s != "True") {
return Ok(s.fail(format!("{name} MarketIdentity {}", o["status"])));
}
}
"bd" => {
let p = &o["spec"]["policy"];
if !(p.is_null() || p == "bidder-tangled-vouch") {
return Ok(s.fail(format!("{name} policy {p}")));
}
s.st["readings"][&name]["policy"] = p.clone();
}
"mb" => {
if o["status"]["did"] != bs[j]["did"] {
return Ok(s.fail(format!("{name} DID {} is not the target's", o["status"]["did"])));
}
s.st["readings"][&name]["phase"] = o["status"]["phase"].clone();
}
"vm" => {
let running = o["items"].as_array().into_iter().flatten().filter(|v| v["status"]["phase"] == "Running").count();
s.st["readings"][&name]["runningVMs"] = json!(running);
}
// the runbook's startup and restart checks, read from the host (no sudo): 0 ERROR in the bidder's first
// 10 min, 0 automatic restarts since its last stop; later ERROR lines are anomalies, kept as readings
_ => {
if o["error"].is_string() {
return Ok(s.fail(format!("{name} health: {}", o["error"])));
}
for k in ["started", "errors", "startupErrors", "restarts", "vmProcesses", "guestImageReadyAt", "guestImageUnready"] {
s.st["readings"][&name][k] = o[k].clone();
}
if o["errors"].as_u64().unwrap_or(0) > 0 {
s.st["readings"][&name]["errorSample"] = o["errorSample"].clone();
}
if o["startupErrors"] != 0 || o["restarts"] != 0 {
return Ok(s.fail(format!("{name}: {} startup ERROR lines, {} automatic restarts since {}", o["startupErrors"], o["restarts"], o["started"])));
}
// the bidder verified its pinned guest image since it started (until then it offers nothing)
if o["guestImageReadyAt"].as_str().unwrap_or_default().is_empty() {
return Ok(s.fail(format!("{name}: no bidder_guest_image_ready since it started {} ({} bidder_guest_image_unready)", o["started"], o["guestImageUnready"])));
}
}
}
let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
s.st["fan"] = json!(left);
if left == 0 {
// the host is empty but for live guests: no VM process without a Running ComputeVM
for b in &bs {
let r = &s.st["readings"][b["name"].as_str().unwrap_or_default()];
if r["vmProcesses"].as_u64().unwrap_or(0) > r["runningVMs"].as_u64().unwrap_or(0) {
return Ok(s.fail(format!("{}: {} VM processes, {} Running ComputeVMs (orphans)", b["name"], r["vmProcesses"], r["runningVMs"])));
}
}
return Ok(s.pass());
}
}
_ => {}
}
Ok(s.keep(h))
}
fn compat_handler(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let mut h = Handle::new();
match run.event {
Event::Start => {
s.set("held");
s.st["since"] = args["now"].clone();
h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "compat": true, "args": ["--hold", "--exec", "--", "id"] }))).timer("deadline", 20 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 20 min (phase {})", s.phase()))),
Event::Message { id, data, .. } if id == "req" => {
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
"exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
"contract" if data["phase"] == "Held" && !s.has("held") => {
s.flag("held");
let t = s.t();
s.reading("held_s", t);
}
"ssh returned" => {
if data["exit"] != 0 {
return Ok(s.fail(format!("id over ssh: {data}")));
}
s.flag("ssh");
}
"exited" => s.flag("exited"),
"requester_binary" => s.reading("cli", data["version"].clone()),
_ => {}
}
if s.is("held") && s.has("held") && s.has("ssh") && s.has("exited") {
s.set("release");
h = h.cancel("req").call(host("release", "request.release", json!({ "name": name })));
}
}
Event::Result { id, result, .. } => {
let o = match out(&s, &id, result.output()) {
Ok(o) => o,
Err(f) => return Ok(f),
};
match id.as_str() {
"release" => {
s.reading("release_s", o["seconds"].clone());
h = h.call(host("logs", "request.checkLogs", json!({ "name": name })));
}
"logs" => {
if o["ok"] != true {
return Ok(s.fail(format!("logs: {}", o["problems"])));
}
h = h.call(host("records", "request.records", json!({ "name": name })));
}
"records" => {
s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
s.st["declined"] = o["declined"].clone();
h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
}
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left after the release: {} {}", o["left"], o["errors"])));
}
// the logs name only the run's records: the release process's own (market.event, vm.delete) show in a
// scan of the repo since the run started
h = h.call(host("since", "assert.noneSince", json!({ "did": args["did"], "since": s.st["since"], "declined": s.st["declined"] })));
}
"since" => {
s.reading("declined_accepts", o["declined"].clone());
if o["left"].as_array().is_none_or(|l| !l.is_empty()) || o["errors"].as_array().is_some_and(|e| !e.is_empty()) {
return Ok(s.fail(format!("records created during the run left: {} {}", o["left"], o["errors"])));
}
return Ok(s.pass());
}
_ => {}
}
}
_ => {}
}
Ok(s.keep(h))
}
entries! { run => bidders, compat => compat_handler }