~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
788 lines · 45.9 KB Raw History
// docs/acceptance/0007-bidder-termination.md: fault injection on the winner's kcp, scoped to this run's own receipt:
// delete its ComputeVM; it goes within 2 min, its vm.terminated is pinned `deleted_on_request`, the requester deletes
// its contract records (the release's vm.delete too), the host runs nothing of the receipt. Needs the winner's kcp.
// A winner whose kcp is read only (`write` false) gets no fault: its one ComputeVM is read, a release ends it, then
// the records are gone and the host runs nothing of the receipt.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{
    GUEST_SECRETS, MARKET_EVENT, MIN, S, SECRETS_FORCE_SEALED, SECRETS_NO_IROH, SECRETS_READ, SECRETS_SYMLINK, SECRETS_SYMLINK_WANT, VM_TERMINATED, created_since,
    fetched_reason, get_record, host, kube, out, secrets_status, secrets_written, sw, termination_rkey,
};

/// The refused bundles' answers: each `bad-<name>` request exited non-zero with its message in the CLI's last lines.
fn bad_exit(s: &mut S, id: &str, data: &Value) -> Option<Handle> {
    let name = id.strip_prefix("bad-")?;
    let (_, _, want) = steps::secrets_bundles_refused().into_iter().find(|(n, _, _)| *n == name)?;
    match data["msg"].as_str().unwrap_or_default() {
        "exited" if data["code"].as_i64().unwrap_or(0) != 0 && data["tail"].as_str().unwrap_or_default().contains(want) => {
            s.st["badOk"][name] = data["code"].clone();
            if steps::secrets_bundles_refused().iter().all(|(n, _, _)| !s.st["badOk"][*n].is_null()) {
                s.flag("badRefused");
                s.reading("refusedBundles", s.st["badOk"].clone());
            }
            None
        }
        "exited" => Some(s.fail(format!("the refused bundle {name} did not exit non-zero with {want:?}: {data}"))),
        "contract" if !data["phase"].as_str().unwrap_or_default().is_empty() => Some(s.fail(format!("the refused bundle {name} reached a contract: {}", data["phase"]))),
        _ => None,
    }
}

/// A page of the requester's market.events, each with the two value fields the sealed check reads.
fn list_events(renter: &Value, cursor: Option<&str>) -> abi::Call {
    host("events", "repo.listCreated", json!({ "repo": renter, "collection": MARKET_EVENT, "limit": 100, "cursor": cursor, "fields": ["receipt.uri", "payload.uri"] }))
}

/// The repo-wide sealed-record check: every market.event of the requester's repo created since the run started, as
/// listed (its receipt.uri and payload.uri included, so a record deleted after the listing is never read again); one
/// whose receipt is this run's with a guest.secrets payload is a sealed bundle for this contract.
fn scan_events(s: &mut S, h: Handle, o: &Value) -> Result<Handle, String> {
    let since = s.st["start"].as_f64().unwrap_or(s.now) - 30.0;
    for r in created_since(o, since) {
        let v = &r["value"];
        if v["receipt"]["uri"] == s.st["receipt"] && v["payload"]["uri"].as_str().unwrap_or_default().contains(GUEST_SECRETS) {
            return Err(format!("the requester's repo holds a sealed bundle for this contract while it has an inbound endpoint: {} -> {}", r["uri"], v["payload"]["uri"]));
        }
        s.st["evCount"] = json!(s.st["evCount"].as_u64().unwrap_or(0) + 1);
    }
    s.st["pages"] = json!(s.st["pages"].as_u64().unwrap_or(0) + 1);
    if let Some(c) = o["cursor"].as_str().filter(|c| !c.is_empty()) {
        return Ok(h.call(list_events(&s.st["renter"], Some(c))));
    }
    s.reading("sealedRecords", json!({ "since": "run start", "events": s.st["evCount"], "pages": s.st["pages"], "forThisContract": 0 }));
    s.set("summary");
    Ok(h.timer("contract", 1))
}

fn run_0007(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let cluster = s.st["cluster"].as_str().unwrap_or_default().to_string();
    let vms = |id: &str| kube(id, "list", &cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }));
    let write = !bidders.iter().any(|b| b["cluster"] == cluster.as_str() && b["write"] == false);
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            s.set("held");
            s.st["start"] = json!(s.now);
            s.st["badOk"] = json!({});
            h = h
                .call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "secretsFallback": "Sealed" })))
                .timer("deadline", 30 * MIN);
            for (i, (n, bundle, _)) in steps::secrets_bundles_refused().into_iter().enumerate() {
                h = h.call(steps::host_sub(&format!("bad-{n}"), "request.start", json!({ "name": format!("{name}-b{i}"), "args": [], "secrets": bundle })));
            }
        }
        Event::Message { id, data, .. } if id.starts_with("bad-") => {
            if let Some(f) = bad_exit(&mut s, &id, &data) {
                return Ok(f);
            }
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "fetchwait" && s.is("held") => {
            return Ok(s.fail("Held, and no requester_guest_secrets_fetched 60 s later: the guest did not fetch over getSecrets"));
        }
        Event::Timer { id } if id == "forcewait" && s.is("forcewait") => {
            return Ok(s.fail("no requester_guest_secrets_not_sealed 2 min after the guest asked for a sealed bundle"));
        }
        Event::Timer { id } if id == "cc" && s.is("status") => h = h.call(host("cc", "request.contract", json!({ "name": name }))),
        Event::Timer { id } if id == "contract" => {
            let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default());
            s.st["winner"] = w.clone();
            s.reading("winner", w.clone());
            if !bidders.iter().any(|b| b["did"] == w) {
                return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
            }
            let Some(c) = bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) else {
                return Ok(s.unmet(&format!("the target names no kcp for the winner {w}")));
            };
            s.st["cluster"] = json!(c);
            s.set("find");
            h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
        }
        Event::Timer { id } if id == "poll" => {
            h = match s.phase().as_str() {
                "gone" | "empty" => h.call(vms("q")),
                "pinned" => h.call(get_record("term", &format!("at://{}/{VM_TERMINATED}/{}", s.st["winner"].as_str().unwrap_or_default(), termination_rkey(s.st["receipt"].as_str().unwrap_or_default())))),
                _ => h,
            };
        }
        Event::Message { id, data, .. } if id == "req" => {
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
                "contract" if data["phase"] == "Held" && !s.has("held") => {
                    s.flag("held");
                    h = h.timer("fetchwait", 60_000);
                }
                "requester_contract_records_deleted" => s.flag("recordsDeleted"),
                "requester_guest_secrets_fetched" if !s.has("fetched") => {
                    let channel = data["channel"].as_str().unwrap_or_default().to_string();
                    if channel != "iroh" && channel != "webrtc" {
                        return Ok(s.fail(format!("the guest fetched its secrets over {channel:?}, not a peer transport")));
                    }
                    s.reading("secretsChannel", channel.clone());
                    s.st["channel"] = json!(channel);
                    s.flag("fetched");
                }
                "requester_guest_secrets_sealed" => return Ok(s.fail(format!("the requester sealed the bundle while it has an inbound endpoint: {data}"))),
                "requester_contract_event_published" if data["nsid"] == GUEST_SECRETS => {
                    return Ok(s.fail(format!("the requester published a guest.secrets record while it has an inbound endpoint: {}", data["record"])));
                }
                // only once the guest was made to ask (forcewait is set before the force ssh goes out)
                "requester_guest_secrets_not_sealed" if s.is("forcewait") => {
                    s.reading("notSealed", data["reason"].clone());
                    s.set("nosealcheck");
                    h = h.call(host("recs", "request.records", json!({ "name": name })));
                }
                _ => {}
            }
            // the --hold process stays (it settles the end): the receipt comes from its log, the winner is its author
            if s.st["receipt"].is_null() && data["receipt"].is_string() {
                s.st["receipt"] = data["receipt"].clone();
            }
            if s.is("held") && s.has("held") && s.has("fetched") && s.st["receipt"].is_string() {
                s.set("status");
                s.mark("status");
                h = h.call(host("cc", "request.contract", json!({ "name": name })));
            }
            if s.is("records") && s.has("recordsDeleted") {
                s.set("check");
                h = h.call(host("records", "request.records", json!({ "name": name })));
            }
        }
        Event::Result { id, result, .. } => {
            let r = result.output();
            if id == "term" && r.is_err() {
                if s.since("delete") > 120.0 {
                    return Ok(s.fail("no vm.terminated 2 min after the delete"));
                }
                return Ok(s.keep(h.timer("poll", 3_000)));
            }
            let o = match out(&s, &id, r) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "cc" => {
                    let (via, reason) = secrets_status(&o);
                    let channel = s.st["channel"].as_str().unwrap_or_default().to_string();
                    if via.is_empty() || reason.is_empty() {
                        if s.since("status") > 30.0 {
                            return Ok(s.fail(format!("the contract records no secrets channel 30 s after the fetch: secretsFetchedVia {via:?}, SecretsApplied {reason:?}")));
                        }
                        return Ok(s.keep(h.timer("cc", 2_000)));
                    }
                    if via != channel || reason != fetched_reason(&channel) {
                        return Ok(s.fail(format!("the log says {channel}, the contract secretsFetchedVia {via:?} and SecretsApplied {reason:?}")));
                    }
                    s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason }));
                    s.set("secrets");
                    h = h.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ })));
                }
                "sec" => {
                    if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) {
                        return Ok(s.fail(format!("getSecrets: {e}")));
                    }
                    s.reading("secrets", "the string, the object as JSON text, modes 0600 0644 0400, a 0700 dir made, /etc unchanged");
                    h = h.call(host("link", "request.ssh", json!({ "name": name, "exec": SECRETS_SYMLINK })));
                }
                "link" => {
                    if !o["output"].as_str().unwrap_or_default().lines().any(|l| l.trim_end() == SECRETS_SYMLINK_WANT) {
                        return Ok(s.fail(format!("a symlink at a secret's path: want {SECRETS_SYMLINK_WANT}, got {}", o["output"])));
                    }
                    s.set("forcewait");
                    h = h.timer("forcewait", 2 * MIN).call(host("force", "request.ssh", json!({ "name": name, "exec": SECRETS_FORCE_SEALED })));
                }
                "force" => {
                    if !o["output"].as_str().unwrap_or_default().lines().any(|l| l.trim() == "FORCED") {
                        return Ok(s.fail(format!("the guest could not ask for a sealed bundle: {}", o["output"])));
                    }
                }
                "recs" => {
                    let uris: Vec<String> = o["uris"].as_array().into_iter().flatten().filter_map(|u| u.as_str().map(String::from)).collect();
                    if let Some(u) = uris.iter().find(|u| u.contains(GUEST_SECRETS)) {
                        return Ok(s.fail(format!("the requester published a sealed bundle while it has an inbound endpoint: {u}")));
                    }
                    let Some(renter) = uris.first().and_then(|u| u.trim_start_matches("at://").split('/').next()).map(String::from) else {
                        return Ok(s.fail("request.records names no record of the requester"));
                    };
                    s.st["renter"] = json!(renter);
                    s.set("scan");
                    h = h.call(list_events(&json!(renter), None));
                }
                "events" => match scan_events(&mut s, h, &o) {
                    Ok(next) => h = next,
                    Err(why) => return Ok(s.fail(why)),
                },
                "vms" => {
                    let mine: Vec<String> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).filter_map(|v| v["metadata"]["name"].as_str().map(String::from)).collect();
                    if mine.len() != 1 {
                        return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", mine.len())));
                    }
                    s.st["vm"] = json!(mine[0]);
                    if write {
                        s.mark("delete");
                        h = h.call(kube("del", "delete", &cluster, sw("ComputeVM", &mine[0])));
                    } else {
                        s.set("release");
                        s.reading("fault", "none: the winner's kcp is read only");
                        h = h.call(host("release", "request.release", json!({ "name": name })));
                    }
                }
                "release" => {
                    s.reading("release_s", o["seconds"].clone());
                    s.mark("delete");
                    s.set("check");
                    h = h.call(host("records", "request.records", json!({ "name": name })));
                }
                "del" => {
                    s.set("gone");
                    h = h.timer("poll", 2_000);
                }
                "q" => {
                    let there = o["items"].as_array().into_iter().flatten().any(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]);
                    if there && s.since("delete") > 120.0 {
                        return Ok(s.fail(format!("the ComputeVM is still there 2 min after the {}", if write { "delete" } else { "release" })));
                    }
                    if there {
                        h = h.timer("poll", 2_000);
                    } else if s.is("gone") {
                        let t = s.since("delete");
                        s.reading("gone_s", t);
                        s.set("pinned");
                        h = h.timer("poll", 1_000);
                    } else if !s.has("badRefused") {
                        return Ok(s.fail(format!("not every refused bundle exited non-zero with its message: {}", s.st["badOk"])));
                    } else {
                        return Ok(s.pass());
                    }
                }
                "term" => {
                    if o["value"]["reason"] != "deleted_on_request" {
                        return Ok(s.fail(format!("vm.terminated {}", o["value"])));
                    }
                    s.reading("terminated", json!({ "reason": o["value"]["reason"], "request": o["value"]["request"]["uri"] }));
                    s.set("records");
                    if s.has("recordsDeleted") {
                        s.set("check");
                        h = h.call(host("records", "request.records", json!({ "name": name })));
                    }
                }
                "records" => {
                    s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
                    h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
                }
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                    }
                    s.set("empty");
                    h = h.call(vms("q"));
                }
                _ => {}
            }
        }
        _ => {}
    }
    Ok(s.keep(h))
}

/// webrtc: getSecrets over the WebRTC channel. The contract's user_data drops the iroh ticket from requester.json, so the
/// guest agent forwards over WebRTC (the requester's guest WebRTC ingress offers over the sealed signalling); the log and
/// the contract (status.secretsFetchedVia, SecretsApplied GetSecretsWebRTC) must say webrtc, the guest holds the files,
/// and the release leaves no record.
fn run_webrtc(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            s.set("held");
            let start = json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "runcmd": [SECRETS_NO_IROH] });
            h = h.call(steps::host_sub("req", "request.start", start)).timer("deadline", 20 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 20 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "fetchwait" && s.is("held") => {
            return Ok(s.fail("Held, and no requester_guest_secrets_fetched 2 min later: the guest did not fetch over WebRTC"));
        }
        Event::Timer { id } if id == "cc" && s.is("status") => h = h.call(host("cc", "request.contract", json!({ "name": name }))),
        Event::Message { id, data, .. } if id == "req" => {
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))),
                "contract" if data["phase"] == "Held" && !s.has("held") => {
                    s.flag("held");
                    h = h.timer("fetchwait", 2 * MIN);
                }
                "requester_guest_secrets_fetched" if !s.has("fetched") => {
                    if data["channel"] != "webrtc" {
                        return Ok(s.fail(format!("iroh off: the guest fetched its secrets over {}, not webrtc", data["channel"])));
                    }
                    s.reading("secretsChannel", "webrtc");
                    s.flag("fetched");
                }
                "requester_guest_webrtc_offered" | "requester_guest_webrtc_answered" => s.reading(data["msg"].as_str().unwrap_or_default(), json!(s.now)),
                _ => {}
            }
            if s.is("held") && s.has("held") && s.has("fetched") {
                s.set("status");
                s.mark("status");
                h = h.call(host("cc", "request.contract", json!({ "name": name })));
            }
        }
        Event::Result { id, result, .. } => {
            let o = match out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "cc" => {
                    let (via, reason) = secrets_status(&o);
                    if via.is_empty() || reason.is_empty() {
                        if s.since("status") > 30.0 {
                            return Ok(s.fail(format!("the contract records no secrets channel 30 s after the fetch: secretsFetchedVia {via:?}, SecretsApplied {reason:?}")));
                        }
                        return Ok(s.keep(h.timer("cc", 2_000)));
                    }
                    if via != "webrtc" || reason != fetched_reason("webrtc") {
                        return Ok(s.fail(format!("iroh off: the contract says secretsFetchedVia {via:?}, SecretsApplied {reason:?}")));
                    }
                    s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason }));
                    s.set("secrets");
                    h = h.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ })));
                }
                "sec" => {
                    if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) {
                        return Ok(s.fail(format!("getSecrets over WebRTC: {e}")));
                    }
                    s.set("release");
                    s.flag("releasing");
                    h = h.call(host("release", "request.release", json!({ "name": name })));
                }
                "release" => {
                    s.reading("release_s", o["seconds"].clone());
                    h = h.call(host("records", "request.records", json!({ "name": name })));
                }
                "records" => {
                    s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
                    h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
                }
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                    }
                    return Ok(s.pass());
                }
                _ => {}
            }
        }
        Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
        _ => {}
    }
    Ok(s.keep(h))
}

/// Log lines of the bidder's core that mean the restarted core started or stopped THIS VM's unit: a VM unit that is
/// Active is adopted silently (hostprocess supervisor markAdopted), so any of these after the restart is a restart
/// of the VM. [text, text...]: a line holds all of them.
fn vm_lines(vm: &str) -> Value {
    json!([
        ["\"msg\":\"placed\"", vm],
        ["\"msg\":\"stopped\"", vm],
        ["work dir and API socket removed", vm],
        ["\"msg\":\"shutdown\"", vm],
        ["the process wait saw the VMM exit", vm],
        ["bidder_vm_terminated", vm],
        ["a released bid is marked terminated", vm],
    ])
}

/// The positive controls: the restarted core really started (a dev network's core logs `core holds the instance`
/// with a new generation; an installed bidder's logs `adopted` for each of its infrastructure units, role kine-a,
/// front, kcp-a). Index 0 and 1 of the counts.
fn control_lines() -> Value {
    json!(["core holds the instance", ["\"msg\":\"adopted\"", "\"role\":"]])
}

fn counts(o: &Value, texts: &Value) -> Vec<u64> {
    texts
        .as_array()
        .into_iter()
        .flatten()
        .map(|t| {
            let k = match t {
                Value::Array(a) => a.iter().filter_map(Value::as_str).collect::<Vec<_>>().join(" & "),
                t => t.as_str().unwrap_or_default().to_string(),
            };
            o["counts"][k].as_u64().unwrap_or(0)
        })
        .collect()
}

fn fcvm(id: &str, cluster: &str, name: &str) -> abi::Call {
    kube(id, "get", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "FirecrackerVM", "name": name }))
}

/// The VM's HostProcess `vm-<name>`: status.runtime is the unit as the supervisor last read it (MainPID, InvocationID,
/// ActiveEnterTimestamp).
fn hproc(id: &str, cluster: &str, vm: &str) -> abi::Call {
    kube(id, "get", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "HostProcess", "plural": "hostprocesses", "name": format!("vm-{vm}") }))
}

fn cvms(id: &str, cluster: &str) -> abi::Call {
    kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))
}

/// restart (staging): the winner's core restarted in place (`dev restart <network> <bidder>/core`, the upgrade path, not a
/// drain) under this run's Held VM. The VM's own unit is identified by its FirecrackerVM runtime (pid, invocationID,
/// startTicks, unit) before and after and must be unchanged; its ComputeVM is read every 2 s from before the restart
/// to 30 s after it and is Running every time (never Lost or Releasing); the guest's boot id is unchanged and ssh
/// answers; the new core's log has none of vm_lines() for it (it adopts an Active unit silently); no vm.terminated; the
/// contract never leaves Held. The restart itself must show (control_lines: a new core generation, or the infra
/// units adopted). Then the release leaves no record.
fn run_restart(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let cluster = s.st["cluster"].as_str().unwrap_or_default().to_string();
    let vm = s.st["vm"].as_str().unwrap_or_default().to_string();
    let boot = |id: &str| host(id, "request.ssh", json!({ "name": name, "exec": "echo BOOT $(cat /proc/sys/kernel/random/boot_id)" }));
    let logs = |id: &str, s: &S| {
        let mut texts = control_lines().as_array().cloned().unwrap_or_default();
        texts.extend(vm_lines(&vm).as_array().cloned().unwrap_or_default());
        host(id, "bidder.logCount", json!({ "name": s.st["bidder"], "texts": texts }))
    };
    let begin = |s: &mut S, h: Handle| -> Result<Handle, Handle> {
        let mut h = h;
        let w = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
        s.reading("winner", w.clone());
        let Some(b) = bidders.iter().find(|b| b["did"] == w.as_str()).cloned() else {
            return Err(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
        };
        let Some(c) = b["cluster"].as_str() else {
            return Err(s.unmet(&format!("the target names no kcp for the winner {w}")));
        };
        s.st["cluster"] = json!(c);
        s.st["bidder"] = b["name"].clone();
        s.set("before");
        h = h
            .call(boot("boot0"))
            .call(cvms("vms", c))
            .call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ })))
            .call(host("recs", "request.records", json!({ "name": name })))
            .call(host("cc", "request.contract", json!({ "name": name })));
        s.mark("status");
        Ok(h)
    };
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            if args["target"] == "prod" {
                return Ok(s.unmet("staging only: it restarts a bidder's core (prod: the rollout's own upgrade)"));
            }
            s.set("held");
            let start = json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "commsMode": "Firehose", "secretsFallback": "Sealed" });
            h = h.call(steps::host_sub("req", "request.start", start)).timer("deadline", 30 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "sealwait" && s.is("held") => {
            return Ok(s.fail("Held, and no secretsApplied via sealed in the requester's log 5 min later"));
        }
        // the watch: the ComputeVM every 2 s, from before the restart to 30 s after it returns
        Event::Timer { id } if id == "watch" => {
            if s.is("restarting") || s.is("watching") {
                h = h.call(cvms("w", &cluster)).timer("watch", 2_000);
            }
        }
        Event::Message { data, .. } => {
            if s.st["receipt"].is_null() && data["receipt"].is_string() {
                s.st["receipt"] = data["receipt"].clone();
            }
            match (data["msg"].as_str().unwrap_or_default(), data["phase"].as_str().unwrap_or_default()) {
                ("contract", "Held") if !s.has("held") => {
                    s.flag("held");
                    h = h.timer("sealwait", 5 * MIN);
                }
                ("contract", "Held") => {}
                ("contract", p) if s.has("held") && !s.has("releasing") && !p.is_empty() => {
                    return Ok(s.fail(format!("the contract left Held around the restart: {p} {}", data["message"])));
                }
                ("contract", "Failed") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                ("exited", _) if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))),
                ("requester_guest_secrets_fetched", _) => return Ok(s.fail("comms mode Firehose: the guest fetched over getSecrets")),
                ("requester_guest_secrets_not_sealed", _) => return Ok(s.fail(format!("comms mode Firehose: the requester did not seal: {}", data["reason"]))),
                ("requester_guest_secrets", _) if data["event"] == "secretsApplied" && data["via"] == "sealed" => s.flag("sealedWritten"),
                _ => {}
            }
            if s.is("held") && s.has("held") && s.has("sealedWritten") {
                if s.st["receipt"].is_string() {
                    match begin(&mut s, h) {
                        Ok(next) => h = next,
                        Err(f) => return Ok(f),
                    }
                } else if !s.has("rcptAsked") {
                    s.flag("rcptAsked");
                    h = h.call(host("rcpt", "request.contract", json!({ "name": name })));
                }
            }
        }
        Event::Timer { id } if id == "cc" && s.is("before") => h = h.call(host("cc", "request.contract", json!({ "name": name }))),
        Event::Timer { id } if id == "rcpt" && s.is("held") => h = h.call(host("rcpt", "request.contract", json!({ "name": name }))),
        Event::Result { id, result, .. } if id == "rcpt" => {
            if !s.is("held") {
                return Ok(s.keep(h));
            }
            match result.output().ok().and_then(|o| o["receipt"].as_str().map(String::from)) {
                Some(rc) => {
                    s.st["receipt"] = json!(rc);
                    match begin(&mut s, h) {
                        Ok(next) => h = next,
                        Err(f) => return Ok(f),
                    }
                }
                None => h = h.timer("rcpt", 2_000),
            }
        }
        Event::Result { id, result, .. } => {
            let r = result.output();
            if id == "term" {
                if let Ok(o) = r {
                    return Ok(s.fail(format!("vm.terminated after the restart: {}", o["value"])));
                }
                s.flag("noTerm");
            } else if id == "w" && !(s.is("restarting") || s.is("watching")) {
                // a list that came back after the watch ended (the release has begun): not read
            } else if id == "w" {
                // an unreadable list during the restart is a reading, not a verdict; a VM that is not Running is
                let Ok(o) = r else {
                    s.st["unreadable"] = json!(s.st["unreadable"].as_u64().unwrap_or(0) + 1);
                    return Ok(s.keep(h));
                };
                let mine = o["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]);
                s.st["watched"] = json!(s.st["watched"].as_u64().unwrap_or(0) + 1);
                match mine {
                    Some(v) if v["status"]["phase"] == "Running" && v["metadata"]["deletionTimestamp"].is_null() && !v["status"]["lostReason"].is_string() => {}
                    other => {
                        return Ok(s.fail(format!("the ComputeVM left Running during the restart: {}", other.map(|v| json!({ "phase": v["status"]["phase"], "lostReason": v["status"]["lostReason"], "deleting": v["metadata"]["deletionTimestamp"] })).unwrap_or(json!("gone")))));
                    }
                }
                if s.is("watching") && s.since("restarted") > 30.0 {
                    s.set("verify");
                    s.reading("watched", s.st["watched"].clone());
                    s.reading("watchUnreadable", s.st["unreadable"].clone());
                    h = h.call(fcvm("fc1", &cluster, &vm)).call(hproc("hp1", &cluster, &vm)).call(boot("boot1")).call(logs("log1", &s)).call(host(
                        "adopt",
                        "bidder.logLines",
                        json!({ "name": s.st["bidder"], "texts": [["hostprocess_unit_adopted", format!("vm-{vm}")]], "keys": ["hostprocess", "unit", "pid", "invocation", "activeSince", "state"] }),
                    ));
                    let rcpt = s.st["receipt"].as_str().unwrap_or_default().to_string();
                    let wd = rcpt.trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
                    h = h.call(get_record("term", &format!("at://{wd}/{VM_TERMINATED}/{}", termination_rkey(&rcpt))));
                }
            } else {
                let o = match out(&s, &id, r) {
                    Ok(o) => o,
                    Err(f) => return Ok(f),
                };
                match id.as_str() {
                    "sec" => {
                        if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) {
                            return Ok(s.fail(format!("the sealed record: {e}")));
                        }
                        s.flag("secOk");
                    }
                    "cc" => {
                        let (via, reason) = secrets_status(&o);
                        if !via.is_empty() {
                            return Ok(s.fail(format!("comms mode Firehose: the contract records a getSecrets channel {via:?}")));
                        }
                        if reason != "SealedFallback" {
                            if s.since("status") > 30.0 {
                                return Ok(s.fail(format!("comms mode Firehose: SecretsApplied {reason:?} 30 s after secretsApplied via sealed, want SealedFallback")));
                            }
                            return Ok(s.keep(h.timer("cc", 2_000)));
                        }
                        s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason }));
                        s.flag("ccOk");
                    }
                    "recs" => {
                        let Some(u) = o["uris"].as_array().into_iter().flatten().filter_map(Value::as_str).find(|u| u.contains(GUEST_SECRETS)).map(String::from) else {
                            return Ok(s.fail(format!("comms mode Firehose: no {GUEST_SECRETS} record among the requester's records")));
                        };
                        h = h.call(get_record("sealrec", &u));
                    }
                    "sealrec" => {
                        if !o["value"]["ciphertext"].as_str().unwrap_or_default().starts_with("-----BEGIN AGE ENCRYPTED FILE-----") {
                            return Ok(s.fail(format!("the {GUEST_SECRETS} record holds no age armor: {}", o["uri"])));
                        }
                        s.reading("sealedRecord", o["uri"].clone());
                        s.flag("sealedSeen");
                    }
                    "boot0" | "boot1" => {
                        let Some(b) = o["output"].as_str().unwrap_or_default().lines().find_map(|l| l.strip_prefix("BOOT ")) else {
                            return Ok(s.fail(format!("{id}: ssh into the guest: {}", o["output"])));
                        };
                        s.st[id.as_str()] = json!(b.trim());
                    }
                    "vms" => {
                        let mine: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).collect();
                        let [v] = mine.as_slice() else {
                            return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", mine.len())));
                        };
                        s.st["vm"] = v["metadata"]["name"].clone();
                        let n = v["metadata"]["name"].as_str().unwrap_or_default();
                        h = h.call(fcvm("fc0", &cluster, n)).call(hproc("hp0", &cluster, n));
                    }
                    "hp0" | "hp1" => {
                        let rt = &o["status"]["runtime"];
                        if rt["pid"].as_i64().unwrap_or(0) == 0 || rt["invocationID"].as_str().unwrap_or_default().is_empty() {
                            return Ok(s.fail(format!("the HostProcess vm-{vm} has no runtime pid and invocationID ({id}): {}", o["status"])));
                        }
                        s.st[id.as_str()] = json!({ "pid": rt["pid"], "invocationID": rt["invocationID"], "unit": rt["unit"], "startTicks": rt["startTicks"], "activeSince": rt["activeSince"], "phase": o["status"]["phase"] });
                    }
                    "fc0" | "fc1" => {
                        let rt = &o["status"]["runtime"];
                        if rt["pid"].as_i64().unwrap_or(0) == 0 || rt["invocationID"].as_str().unwrap_or_default().is_empty() {
                            return Ok(s.fail(format!("the FirecrackerVM has no runtime pid and invocationID ({id}): {}", o["status"])));
                        }
                        let k = if id == "fc0" { "rt0" } else { "rt1" };
                        s.st[k] = json!({ "pid": rt["pid"], "invocationID": rt["invocationID"], "unit": rt["unit"], "startTicks": rt["startTicks"], "startTime": rt["startTime"], "phase": o["status"]["phase"], "lostReason": o["status"]["lostReason"] });
                    }
                    "restart" => {
                        s.reading("restart_s", o["seconds"].clone());
                        s.set("watching");
                        s.mark("restarted");
                    }
                    "adopt" => s.st["adopt"] = o["lines"].clone(),
                    "release" => {
                        s.reading("release_s", o["seconds"].clone());
                        h = h.call(host("records", "request.records", json!({ "name": name })));
                    }
                    "records" => {
                        s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
                        h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
                    }
                    "gone" => {
                        if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                            return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                        }
                        return Ok(s.pass());
                    }
                    _ => {}
                }
                if id == "log0" || id == "log1" {
                    let mut texts = control_lines().as_array().cloned().unwrap_or_default();
                    texts.extend(vm_lines(&vm).as_array().cloned().unwrap_or_default());
                    s.st[if id == "log0" { "c0" } else { "c1" }] = json!(counts(&o, &json!(texts)));
                }
            }
            // before: the guest's boot id, the VM's runtime and the log counts, then the restart with the watch
            if s.is("before") && s.st["boot0"].is_string() && s.st["rt0"].is_object() && s.st["hp0"].is_object() && s.has("secOk") && s.has("sealedSeen") && s.has("ccOk") && !s.has("logAsked") {
                s.flag("logAsked");
                h = h.call(logs("log0", &s));
            } else if s.is("before") && s.st["c0"].is_array() {
                s.reading("before", json!({ "firecrackervm": s.st["rt0"], "hostprocess": s.st["hp0"] }));
                s.set("restarting");
                s.mark("restarted");
                h = h.call(host("restart", "bidder.restart", json!({ "name": s.st["bidder"] }))).call(cvms("w", &cluster)).timer("watch", 2_000);
            }
            // verify: the three answers in, then the verdict, then the release
            if s.is("verify") && ["rt1", "hp1", "boot1", "c1", "adopt"].iter().all(|k| !s.st[*k].is_null()) && s.has("noTerm") {
                let (a, b) = (s.st["rt0"].clone(), s.st["rt1"].clone());
                let (c0, c1): (Vec<u64>, Vec<u64>) = (
                    s.st["c0"].as_array().into_iter().flatten().map(|v| v.as_u64().unwrap_or(0)).collect(),
                    s.st["c1"].as_array().into_iter().flatten().map(|v| v.as_u64().unwrap_or(0)).collect(),
                );
                let mut p = vec![];
                for k in ["pid", "invocationID", "unit", "startTicks", "startTime"] {
                    if a[k] != b[k] {
                        p.push(format!("the VM's VMM changed: {k} {} -> {}", a[k], b[k]));
                    }
                }
                // the unit as the supervisor reads it: MainPID, InvocationID, ActiveEnterTimestamp, before and after
                let (u, v) = (s.st["hp0"].clone(), s.st["hp1"].clone());
                for k in ["pid", "invocationID", "unit", "startTicks", "activeSince"] {
                    if u[k] != v[k] {
                        p.push(format!("the VM's unit changed: {k} {} -> {}", u[k], v[k]));
                    }
                }
                if b["phase"] != "Running" || b["lostReason"].is_string() {
                    p.push(format!("the VM is {} ({}) after the restart", b["phase"], b["lostReason"]));
                }
                if s.st["boot0"] != s.st["boot1"] {
                    p.push(format!("the guest rebooted: {} -> {}", s.st["boot0"], s.st["boot1"]));
                }
                // the restart really happened: a new core generation, or the infra units adopted (>= 3 role lines)
                let newgen = c1.first().copied().unwrap_or(0).saturating_sub(c0.first().copied().unwrap_or(0));
                let infra = c1.get(1).copied().unwrap_or(0).saturating_sub(c0.get(1).copied().unwrap_or(0));
                s.reading("controls", json!({ "newCoreGenerations": newgen, "infraAdopted": infra }));
                if newgen == 0 && infra < 3 {
                    p.push("the bidder's log shows no restart (no new `core holds the instance`, fewer than 3 adopted infra units)".to_string());
                }
                // the new core's supervisor adopted exactly this VM's unit (hostprocess_unit_adopted: pid, invocation and
                // activeSince of the unit as the supervisor read it), the one the FirecrackerVM named before
                let ad = s.st["adopt"].as_array().cloned().unwrap_or_default();
                s.reading("adopted", json!(ad));
                match ad.as_slice() {
                    [l] => {
                        if l["pid"].as_i64() != u["pid"].as_i64() || l["invocation"] != u["invocationID"] {
                            p.push(format!("the adopted unit is not the VM's: pid {} invocation {} (before: {} {})", l["pid"], l["invocation"], u["pid"], u["invocationID"]));
                        }
                        // activeSince: the unit's ActiveEnterTimestamp, as the supervisor read it before (to the second)
                        if l["activeSince"].as_str().zip(u["activeSince"].as_str()).is_none_or(|(x, y)| (steps::secs(x) - steps::secs(y)).abs() > 1.0) {
                            p.push(format!("activeSince {} is not the unit's before the restart {}", l["activeSince"], u["activeSince"]));
                        }
                        if !l["state"].as_str().unwrap_or_default().eq_ignore_ascii_case("active") {
                            p.push(format!("the adopted unit's state: {}", l["state"]));
                        }
                    }
                    _ => p.push(format!("{} hostprocess_unit_adopted lines for vm-{vm}, one expected", ad.len())),
                }
                // the VM's own lines: none
                let names = vm_lines(&vm);
                let mut touched = vec![];
                for (i, t) in names.as_array().into_iter().flatten().enumerate() {
                    let n = c1.get(i + 2).copied().unwrap_or(0).saturating_sub(c0.get(i + 2).copied().unwrap_or(0));
                    if n > 0 {
                        touched.push(format!("{t} x{n}"));
                    }
                }
                s.reading("vmLinesAfterRestart", json!(touched));
                if !touched.is_empty() {
                    p.push(format!("the restarted core touched the VM's unit: {}", touched.join(", ")));
                }
                s.reading("after", b);
                if !p.is_empty() {
                    return Ok(s.fail(p.join("; ")));
                }
                s.set("release");
                s.flag("releasing");
                h = h.call(host("release", "request.release", json!({ "name": name })));
            }
        }
        Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
        _ => {}
    }
    Ok(s.keep(h))
}

mod fence;
mod ingress;
mod upgrade;

entries! { run => run_0007, restart => run_restart, webrtc => run_webrtc, upgrade => upgrade::run_upgrade, fence => fence::run_fence, ingress => ingress::run_ingress }