// docs/acceptance/0007-bidder-termination.md: fault injection on the winner's kcp, scoped to this run's own receipt: // delete its ComputeVM; it goes within 2 min, its vm.terminated is pinned `deleted_on_request`, the requester deletes // its contract records (the release's vm.delete too), the host runs nothing of the receipt. Needs the winner's kcp. // A winner whose kcp is read only (`write` false) gets no fault: its one ComputeVM is read, a release ends it, then // the records are gone and the host runs nothing of the receipt. use abi::{Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{ GUEST_SECRETS, MARKET_EVENT, MIN, S, SECRETS_FORCE_SEALED, SECRETS_NO_IROH, SECRETS_READ, SECRETS_SYMLINK, SECRETS_SYMLINK_WANT, VM_TERMINATED, created_since, fetched_reason, get_record, host, kube, out, secrets_status, secrets_written, sw, termination_rkey, }; /// The refused bundles' answers: each `bad-` request exited non-zero with its message in the CLI's last lines. fn bad_exit(s: &mut S, id: &str, data: &Value) -> Option { let name = id.strip_prefix("bad-")?; let (_, _, want) = steps::secrets_bundles_refused().into_iter().find(|(n, _, _)| *n == name)?; match data["msg"].as_str().unwrap_or_default() { "exited" if data["code"].as_i64().unwrap_or(0) != 0 && data["tail"].as_str().unwrap_or_default().contains(want) => { s.st["badOk"][name] = data["code"].clone(); if steps::secrets_bundles_refused().iter().all(|(n, _, _)| !s.st["badOk"][*n].is_null()) { s.flag("badRefused"); s.reading("refusedBundles", s.st["badOk"].clone()); } None } "exited" => Some(s.fail(format!("the refused bundle {name} did not exit non-zero with {want:?}: {data}"))), "contract" if !data["phase"].as_str().unwrap_or_default().is_empty() => Some(s.fail(format!("the refused bundle {name} reached a contract: {}", data["phase"]))), _ => None, } } /// A page of the requester's market.events, each with the two value fields the sealed check reads. fn list_events(renter: &Value, cursor: Option<&str>) -> abi::Call { host("events", "repo.listCreated", json!({ "repo": renter, "collection": MARKET_EVENT, "limit": 100, "cursor": cursor, "fields": ["receipt.uri", "payload.uri"] })) } /// The repo-wide sealed-record check: every market.event of the requester's repo created since the run started, as /// listed (its receipt.uri and payload.uri included, so a record deleted after the listing is never read again); one /// whose receipt is this run's with a guest.secrets payload is a sealed bundle for this contract. fn scan_events(s: &mut S, h: Handle, o: &Value) -> Result { let since = s.st["start"].as_f64().unwrap_or(s.now) - 30.0; for r in created_since(o, since) { let v = &r["value"]; if v["receipt"]["uri"] == s.st["receipt"] && v["payload"]["uri"].as_str().unwrap_or_default().contains(GUEST_SECRETS) { return Err(format!("the requester's repo holds a sealed bundle for this contract while it has an inbound endpoint: {} -> {}", r["uri"], v["payload"]["uri"])); } s.st["evCount"] = json!(s.st["evCount"].as_u64().unwrap_or(0) + 1); } s.st["pages"] = json!(s.st["pages"].as_u64().unwrap_or(0) + 1); if let Some(c) = o["cursor"].as_str().filter(|c| !c.is_empty()) { return Ok(h.call(list_events(&s.st["renter"], Some(c)))); } s.reading("sealedRecords", json!({ "since": "run start", "events": s.st["evCount"], "pages": s.st["pages"], "forThisContract": 0 })); s.set("summary"); Ok(h.timer("contract", 1)) } fn run_0007(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let cluster = s.st["cluster"].as_str().unwrap_or_default().to_string(); let vms = |id: &str| kube(id, "list", &cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })); let write = !bidders.iter().any(|b| b["cluster"] == cluster.as_str() && b["write"] == false); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); s.st["start"] = json!(s.now); s.st["badOk"] = json!({}); h = h .call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "secretsFallback": "Sealed" }))) .timer("deadline", 30 * MIN); for (i, (n, bundle, _)) in steps::secrets_bundles_refused().into_iter().enumerate() { h = h.call(steps::host_sub(&format!("bad-{n}"), "request.start", json!({ "name": format!("{name}-b{i}"), "args": [], "secrets": bundle }))); } } Event::Message { id, data, .. } if id.starts_with("bad-") => { if let Some(f) = bad_exit(&mut s, &id, &data) { return Ok(f); } } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))), Event::Timer { id } if id == "fetchwait" && s.is("held") => { return Ok(s.fail("Held, and no requester_guest_secrets_fetched 60 s later: the guest did not fetch over getSecrets")); } Event::Timer { id } if id == "forcewait" && s.is("forcewait") => { return Ok(s.fail("no requester_guest_secrets_not_sealed 2 min after the guest asked for a sealed bundle")); } Event::Timer { id } if id == "cc" && s.is("status") => h = h.call(host("cc", "request.contract", json!({ "name": name }))), Event::Timer { id } if id == "contract" => { let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default()); s.st["winner"] = w.clone(); s.reading("winner", w.clone()); if !bidders.iter().any(|b| b["did"] == w) { return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target"))); } let Some(c) = bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) else { return Ok(s.unmet(&format!("the target names no kcp for the winner {w}"))); }; s.st["cluster"] = json!(c); s.set("find"); h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))); } Event::Timer { id } if id == "poll" => { h = match s.phase().as_str() { "gone" | "empty" => h.call(vms("q")), "pinned" => h.call(get_record("term", &format!("at://{}/{VM_TERMINATED}/{}", s.st["winner"].as_str().unwrap_or_default(), termination_rkey(s.st["receipt"].as_str().unwrap_or_default())))), _ => h, }; } Event::Message { id, data, .. } if id == "req" => { match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")), "contract" if data["phase"] == "Held" && !s.has("held") => { s.flag("held"); h = h.timer("fetchwait", 60_000); } "requester_contract_records_deleted" => s.flag("recordsDeleted"), "requester_guest_secrets_fetched" if !s.has("fetched") => { let channel = data["channel"].as_str().unwrap_or_default().to_string(); if channel != "iroh" && channel != "webrtc" { return Ok(s.fail(format!("the guest fetched its secrets over {channel:?}, not a peer transport"))); } s.reading("secretsChannel", channel.clone()); s.st["channel"] = json!(channel); s.flag("fetched"); } "requester_guest_secrets_sealed" => return Ok(s.fail(format!("the requester sealed the bundle while it has an inbound endpoint: {data}"))), "requester_contract_event_published" if data["nsid"] == GUEST_SECRETS => { return Ok(s.fail(format!("the requester published a guest.secrets record while it has an inbound endpoint: {}", data["record"]))); } // only once the guest was made to ask (forcewait is set before the force ssh goes out) "requester_guest_secrets_not_sealed" if s.is("forcewait") => { s.reading("notSealed", data["reason"].clone()); s.set("nosealcheck"); h = h.call(host("recs", "request.records", json!({ "name": name }))); } _ => {} } // the --hold process stays (it settles the end): the receipt comes from its log, the winner is its author if s.st["receipt"].is_null() && data["receipt"].is_string() { s.st["receipt"] = data["receipt"].clone(); } if s.is("held") && s.has("held") && s.has("fetched") && s.st["receipt"].is_string() { s.set("status"); s.mark("status"); h = h.call(host("cc", "request.contract", json!({ "name": name }))); } if s.is("records") && s.has("recordsDeleted") { s.set("check"); h = h.call(host("records", "request.records", json!({ "name": name }))); } } Event::Result { id, result, .. } => { let r = result.output(); if id == "term" && r.is_err() { if s.since("delete") > 120.0 { return Ok(s.fail("no vm.terminated 2 min after the delete")); } return Ok(s.keep(h.timer("poll", 3_000))); } let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "cc" => { let (via, reason) = secrets_status(&o); let channel = s.st["channel"].as_str().unwrap_or_default().to_string(); if via.is_empty() || reason.is_empty() { if s.since("status") > 30.0 { return Ok(s.fail(format!("the contract records no secrets channel 30 s after the fetch: secretsFetchedVia {via:?}, SecretsApplied {reason:?}"))); } return Ok(s.keep(h.timer("cc", 2_000))); } if via != channel || reason != fetched_reason(&channel) { return Ok(s.fail(format!("the log says {channel}, the contract secretsFetchedVia {via:?} and SecretsApplied {reason:?}"))); } s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason })); s.set("secrets"); h = h.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ }))); } "sec" => { if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) { return Ok(s.fail(format!("getSecrets: {e}"))); } s.reading("secrets", "the string, the object as JSON text, modes 0600 0644 0400, a 0700 dir made, /etc unchanged"); h = h.call(host("link", "request.ssh", json!({ "name": name, "exec": SECRETS_SYMLINK }))); } "link" => { if !o["output"].as_str().unwrap_or_default().lines().any(|l| l.trim_end() == SECRETS_SYMLINK_WANT) { return Ok(s.fail(format!("a symlink at a secret's path: want {SECRETS_SYMLINK_WANT}, got {}", o["output"]))); } s.set("forcewait"); h = h.timer("forcewait", 2 * MIN).call(host("force", "request.ssh", json!({ "name": name, "exec": SECRETS_FORCE_SEALED }))); } "force" => { if !o["output"].as_str().unwrap_or_default().lines().any(|l| l.trim() == "FORCED") { return Ok(s.fail(format!("the guest could not ask for a sealed bundle: {}", o["output"]))); } } "recs" => { let uris: Vec = o["uris"].as_array().into_iter().flatten().filter_map(|u| u.as_str().map(String::from)).collect(); if let Some(u) = uris.iter().find(|u| u.contains(GUEST_SECRETS)) { return Ok(s.fail(format!("the requester published a sealed bundle while it has an inbound endpoint: {u}"))); } let Some(renter) = uris.first().and_then(|u| u.trim_start_matches("at://").split('/').next()).map(String::from) else { return Ok(s.fail("request.records names no record of the requester")); }; s.st["renter"] = json!(renter); s.set("scan"); h = h.call(list_events(&json!(renter), None)); } "events" => match scan_events(&mut s, h, &o) { Ok(next) => h = next, Err(why) => return Ok(s.fail(why)), }, "vms" => { let mine: Vec = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).filter_map(|v| v["metadata"]["name"].as_str().map(String::from)).collect(); if mine.len() != 1 { return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", mine.len()))); } s.st["vm"] = json!(mine[0]); if write { s.mark("delete"); h = h.call(kube("del", "delete", &cluster, sw("ComputeVM", &mine[0]))); } else { s.set("release"); s.reading("fault", "none: the winner's kcp is read only"); h = h.call(host("release", "request.release", json!({ "name": name }))); } } "release" => { s.reading("release_s", o["seconds"].clone()); s.mark("delete"); s.set("check"); h = h.call(host("records", "request.records", json!({ "name": name }))); } "del" => { s.set("gone"); h = h.timer("poll", 2_000); } "q" => { let there = o["items"].as_array().into_iter().flatten().any(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]); if there && s.since("delete") > 120.0 { return Ok(s.fail(format!("the ComputeVM is still there 2 min after the {}", if write { "delete" } else { "release" }))); } if there { h = h.timer("poll", 2_000); } else if s.is("gone") { let t = s.since("delete"); s.reading("gone_s", t); s.set("pinned"); h = h.timer("poll", 1_000); } else if !s.has("badRefused") { return Ok(s.fail(format!("not every refused bundle exited non-zero with its message: {}", s.st["badOk"]))); } else { return Ok(s.pass()); } } "term" => { if o["value"]["reason"] != "deleted_on_request" { return Ok(s.fail(format!("vm.terminated {}", o["value"]))); } s.reading("terminated", json!({ "reason": o["value"]["reason"], "request": o["value"]["request"]["uri"] })); s.set("records"); if s.has("recordsDeleted") { s.set("check"); h = h.call(host("records", "request.records", json!({ "name": name }))); } } "records" => { s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } s.set("empty"); h = h.call(vms("q")); } _ => {} } } _ => {} } Ok(s.keep(h)) } /// webrtc: getSecrets over the WebRTC channel. The contract's user_data drops the iroh ticket from requester.json, so the /// guest agent forwards over WebRTC (the requester's guest WebRTC ingress offers over the sealed signalling); the log and /// the contract (status.secretsFetchedVia, SecretsApplied GetSecretsWebRTC) must say webrtc, the guest holds the files, /// and the release leaves no record. fn run_webrtc(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); let start = json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "runcmd": [SECRETS_NO_IROH] }); h = h.call(steps::host_sub("req", "request.start", start)).timer("deadline", 20 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 20 min (phase {})", s.phase()))), Event::Timer { id } if id == "fetchwait" && s.is("held") => { return Ok(s.fail("Held, and no requester_guest_secrets_fetched 2 min later: the guest did not fetch over WebRTC")); } Event::Timer { id } if id == "cc" && s.is("status") => h = h.call(host("cc", "request.contract", json!({ "name": name }))), Event::Message { id, data, .. } if id == "req" => { match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "exited" if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))), "contract" if data["phase"] == "Held" && !s.has("held") => { s.flag("held"); h = h.timer("fetchwait", 2 * MIN); } "requester_guest_secrets_fetched" if !s.has("fetched") => { if data["channel"] != "webrtc" { return Ok(s.fail(format!("iroh off: the guest fetched its secrets over {}, not webrtc", data["channel"]))); } s.reading("secretsChannel", "webrtc"); s.flag("fetched"); } "requester_guest_webrtc_offered" | "requester_guest_webrtc_answered" => s.reading(data["msg"].as_str().unwrap_or_default(), json!(s.now)), _ => {} } if s.is("held") && s.has("held") && s.has("fetched") { s.set("status"); s.mark("status"); h = h.call(host("cc", "request.contract", json!({ "name": name }))); } } Event::Result { id, result, .. } => { let o = match out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "cc" => { let (via, reason) = secrets_status(&o); if via.is_empty() || reason.is_empty() { if s.since("status") > 30.0 { return Ok(s.fail(format!("the contract records no secrets channel 30 s after the fetch: secretsFetchedVia {via:?}, SecretsApplied {reason:?}"))); } return Ok(s.keep(h.timer("cc", 2_000))); } if via != "webrtc" || reason != fetched_reason("webrtc") { return Ok(s.fail(format!("iroh off: the contract says secretsFetchedVia {via:?}, SecretsApplied {reason:?}"))); } s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason })); s.set("secrets"); h = h.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ }))); } "sec" => { if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) { return Ok(s.fail(format!("getSecrets over WebRTC: {e}"))); } s.set("release"); s.flag("releasing"); h = h.call(host("release", "request.release", json!({ "name": name }))); } "release" => { s.reading("release_s", o["seconds"].clone()); h = h.call(host("records", "request.records", json!({ "name": name }))); } "records" => { s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } return Ok(s.pass()); } _ => {} } } Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))), _ => {} } Ok(s.keep(h)) } /// Log lines of the bidder's core that mean the restarted core started or stopped THIS VM's unit: a VM unit that is /// Active is adopted silently (hostprocess supervisor markAdopted), so any of these after the restart is a restart /// of the VM. [text, text...]: a line holds all of them. fn vm_lines(vm: &str) -> Value { json!([ ["\"msg\":\"placed\"", vm], ["\"msg\":\"stopped\"", vm], ["work dir and API socket removed", vm], ["\"msg\":\"shutdown\"", vm], ["the process wait saw the VMM exit", vm], ["bidder_vm_terminated", vm], ["a released bid is marked terminated", vm], ]) } /// The positive controls: the restarted core really started (a dev network's core logs `core holds the instance` /// with a new generation; an installed bidder's logs `adopted` for each of its infrastructure units, role kine-a, /// front, kcp-a). Index 0 and 1 of the counts. fn control_lines() -> Value { json!(["core holds the instance", ["\"msg\":\"adopted\"", "\"role\":"]]) } fn counts(o: &Value, texts: &Value) -> Vec { texts .as_array() .into_iter() .flatten() .map(|t| { let k = match t { Value::Array(a) => a.iter().filter_map(Value::as_str).collect::>().join(" & "), t => t.as_str().unwrap_or_default().to_string(), }; o["counts"][k].as_u64().unwrap_or(0) }) .collect() } fn fcvm(id: &str, cluster: &str, name: &str) -> abi::Call { kube(id, "get", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "FirecrackerVM", "name": name })) } /// The VM's HostProcess `vm-`: status.runtime is the unit as the supervisor last read it (MainPID, InvocationID, /// ActiveEnterTimestamp). fn hproc(id: &str, cluster: &str, vm: &str) -> abi::Call { kube(id, "get", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "HostProcess", "plural": "hostprocesses", "name": format!("vm-{vm}") })) } fn cvms(id: &str, cluster: &str) -> abi::Call { kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })) } /// restart (staging): the winner's core restarted in place (`dev restart /core`, the upgrade path, not a /// drain) under this run's Held VM. The VM's own unit is identified by its FirecrackerVM runtime (pid, invocationID, /// startTicks, unit) before and after and must be unchanged; its ComputeVM is read every 2 s from before the restart /// to 30 s after it and is Running every time (never Lost or Releasing); the guest's boot id is unchanged and ssh /// answers; the new core's log has none of vm_lines() for it (it adopts an Active unit silently); no vm.terminated; the /// contract never leaves Held. The restart itself must show (control_lines: a new core generation, or the infra /// units adopted). Then the release leaves no record. fn run_restart(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let cluster = s.st["cluster"].as_str().unwrap_or_default().to_string(); let vm = s.st["vm"].as_str().unwrap_or_default().to_string(); let boot = |id: &str| host(id, "request.ssh", json!({ "name": name, "exec": "echo BOOT $(cat /proc/sys/kernel/random/boot_id)" })); let logs = |id: &str, s: &S| { let mut texts = control_lines().as_array().cloned().unwrap_or_default(); texts.extend(vm_lines(&vm).as_array().cloned().unwrap_or_default()); host(id, "bidder.logCount", json!({ "name": s.st["bidder"], "texts": texts })) }; let begin = |s: &mut S, h: Handle| -> Result { let mut h = h; let w = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string(); s.reading("winner", w.clone()); let Some(b) = bidders.iter().find(|b| b["did"] == w.as_str()).cloned() else { return Err(s.fail(format!("unknown bidder {w}: the winner is not in the target"))); }; let Some(c) = b["cluster"].as_str() else { return Err(s.unmet(&format!("the target names no kcp for the winner {w}"))); }; s.st["cluster"] = json!(c); s.st["bidder"] = b["name"].clone(); s.set("before"); h = h .call(boot("boot0")) .call(cvms("vms", c)) .call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ }))) .call(host("recs", "request.records", json!({ "name": name }))) .call(host("cc", "request.contract", json!({ "name": name }))); s.mark("status"); Ok(h) }; let mut h = Handle::new(); match run.event { Event::Start => { if args["target"] == "prod" { return Ok(s.unmet("staging only: it restarts a bidder's core (prod: the rollout's own upgrade)")); } s.set("held"); let start = json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "commsMode": "Firehose", "secretsFallback": "Sealed" }); h = h.call(steps::host_sub("req", "request.start", start)).timer("deadline", 30 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))), Event::Timer { id } if id == "sealwait" && s.is("held") => { return Ok(s.fail("Held, and no secretsApplied via sealed in the requester's log 5 min later")); } // the watch: the ComputeVM every 2 s, from before the restart to 30 s after it returns Event::Timer { id } if id == "watch" => { if s.is("restarting") || s.is("watching") { h = h.call(cvms("w", &cluster)).timer("watch", 2_000); } } Event::Message { data, .. } => { if s.st["receipt"].is_null() && data["receipt"].is_string() { s.st["receipt"] = data["receipt"].clone(); } match (data["msg"].as_str().unwrap_or_default(), data["phase"].as_str().unwrap_or_default()) { ("contract", "Held") if !s.has("held") => { s.flag("held"); h = h.timer("sealwait", 5 * MIN); } ("contract", "Held") => {} ("contract", p) if s.has("held") && !s.has("releasing") && !p.is_empty() => { return Ok(s.fail(format!("the contract left Held around the restart: {p} {}", data["message"]))); } ("contract", "Failed") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), ("exited", _) if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))), ("requester_guest_secrets_fetched", _) => return Ok(s.fail("comms mode Firehose: the guest fetched over getSecrets")), ("requester_guest_secrets_not_sealed", _) => return Ok(s.fail(format!("comms mode Firehose: the requester did not seal: {}", data["reason"]))), ("requester_guest_secrets", _) if data["event"] == "secretsApplied" && data["via"] == "sealed" => s.flag("sealedWritten"), _ => {} } if s.is("held") && s.has("held") && s.has("sealedWritten") { if s.st["receipt"].is_string() { match begin(&mut s, h) { Ok(next) => h = next, Err(f) => return Ok(f), } } else if !s.has("rcptAsked") { s.flag("rcptAsked"); h = h.call(host("rcpt", "request.contract", json!({ "name": name }))); } } } Event::Timer { id } if id == "cc" && s.is("before") => h = h.call(host("cc", "request.contract", json!({ "name": name }))), Event::Timer { id } if id == "rcpt" && s.is("held") => h = h.call(host("rcpt", "request.contract", json!({ "name": name }))), Event::Result { id, result, .. } if id == "rcpt" => { if !s.is("held") { return Ok(s.keep(h)); } match result.output().ok().and_then(|o| o["receipt"].as_str().map(String::from)) { Some(rc) => { s.st["receipt"] = json!(rc); match begin(&mut s, h) { Ok(next) => h = next, Err(f) => return Ok(f), } } None => h = h.timer("rcpt", 2_000), } } Event::Result { id, result, .. } => { let r = result.output(); if id == "term" { if let Ok(o) = r { return Ok(s.fail(format!("vm.terminated after the restart: {}", o["value"]))); } s.flag("noTerm"); } else if id == "w" && !(s.is("restarting") || s.is("watching")) { // a list that came back after the watch ended (the release has begun): not read } else if id == "w" { // an unreadable list during the restart is a reading, not a verdict; a VM that is not Running is let Ok(o) = r else { s.st["unreadable"] = json!(s.st["unreadable"].as_u64().unwrap_or(0) + 1); return Ok(s.keep(h)); }; let mine = o["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]); s.st["watched"] = json!(s.st["watched"].as_u64().unwrap_or(0) + 1); match mine { Some(v) if v["status"]["phase"] == "Running" && v["metadata"]["deletionTimestamp"].is_null() && !v["status"]["lostReason"].is_string() => {} other => { return Ok(s.fail(format!("the ComputeVM left Running during the restart: {}", other.map(|v| json!({ "phase": v["status"]["phase"], "lostReason": v["status"]["lostReason"], "deleting": v["metadata"]["deletionTimestamp"] })).unwrap_or(json!("gone"))))); } } if s.is("watching") && s.since("restarted") > 30.0 { s.set("verify"); s.reading("watched", s.st["watched"].clone()); s.reading("watchUnreadable", s.st["unreadable"].clone()); h = h.call(fcvm("fc1", &cluster, &vm)).call(hproc("hp1", &cluster, &vm)).call(boot("boot1")).call(logs("log1", &s)).call(host( "adopt", "bidder.logLines", json!({ "name": s.st["bidder"], "texts": [["hostprocess_unit_adopted", format!("vm-{vm}")]], "keys": ["hostprocess", "unit", "pid", "invocation", "activeSince", "state"] }), )); let rcpt = s.st["receipt"].as_str().unwrap_or_default().to_string(); let wd = rcpt.trim_start_matches("at://").split('/').next().unwrap_or_default().to_string(); h = h.call(get_record("term", &format!("at://{wd}/{VM_TERMINATED}/{}", termination_rkey(&rcpt)))); } } else { let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "sec" => { if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) { return Ok(s.fail(format!("the sealed record: {e}"))); } s.flag("secOk"); } "cc" => { let (via, reason) = secrets_status(&o); if !via.is_empty() { return Ok(s.fail(format!("comms mode Firehose: the contract records a getSecrets channel {via:?}"))); } if reason != "SealedFallback" { if s.since("status") > 30.0 { return Ok(s.fail(format!("comms mode Firehose: SecretsApplied {reason:?} 30 s after secretsApplied via sealed, want SealedFallback"))); } return Ok(s.keep(h.timer("cc", 2_000))); } s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason })); s.flag("ccOk"); } "recs" => { let Some(u) = o["uris"].as_array().into_iter().flatten().filter_map(Value::as_str).find(|u| u.contains(GUEST_SECRETS)).map(String::from) else { return Ok(s.fail(format!("comms mode Firehose: no {GUEST_SECRETS} record among the requester's records"))); }; h = h.call(get_record("sealrec", &u)); } "sealrec" => { if !o["value"]["ciphertext"].as_str().unwrap_or_default().starts_with("-----BEGIN AGE ENCRYPTED FILE-----") { return Ok(s.fail(format!("the {GUEST_SECRETS} record holds no age armor: {}", o["uri"]))); } s.reading("sealedRecord", o["uri"].clone()); s.flag("sealedSeen"); } "boot0" | "boot1" => { let Some(b) = o["output"].as_str().unwrap_or_default().lines().find_map(|l| l.strip_prefix("BOOT ")) else { return Ok(s.fail(format!("{id}: ssh into the guest: {}", o["output"]))); }; s.st[id.as_str()] = json!(b.trim()); } "vms" => { let mine: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).collect(); let [v] = mine.as_slice() else { return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", mine.len()))); }; s.st["vm"] = v["metadata"]["name"].clone(); let n = v["metadata"]["name"].as_str().unwrap_or_default(); h = h.call(fcvm("fc0", &cluster, n)).call(hproc("hp0", &cluster, n)); } "hp0" | "hp1" => { let rt = &o["status"]["runtime"]; if rt["pid"].as_i64().unwrap_or(0) == 0 || rt["invocationID"].as_str().unwrap_or_default().is_empty() { return Ok(s.fail(format!("the HostProcess vm-{vm} has no runtime pid and invocationID ({id}): {}", o["status"]))); } s.st[id.as_str()] = json!({ "pid": rt["pid"], "invocationID": rt["invocationID"], "unit": rt["unit"], "startTicks": rt["startTicks"], "activeSince": rt["activeSince"], "phase": o["status"]["phase"] }); } "fc0" | "fc1" => { let rt = &o["status"]["runtime"]; if rt["pid"].as_i64().unwrap_or(0) == 0 || rt["invocationID"].as_str().unwrap_or_default().is_empty() { return Ok(s.fail(format!("the FirecrackerVM has no runtime pid and invocationID ({id}): {}", o["status"]))); } let k = if id == "fc0" { "rt0" } else { "rt1" }; s.st[k] = json!({ "pid": rt["pid"], "invocationID": rt["invocationID"], "unit": rt["unit"], "startTicks": rt["startTicks"], "startTime": rt["startTime"], "phase": o["status"]["phase"], "lostReason": o["status"]["lostReason"] }); } "restart" => { s.reading("restart_s", o["seconds"].clone()); s.set("watching"); s.mark("restarted"); } "adopt" => s.st["adopt"] = o["lines"].clone(), "release" => { s.reading("release_s", o["seconds"].clone()); h = h.call(host("records", "request.records", json!({ "name": name }))); } "records" => { s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } return Ok(s.pass()); } _ => {} } if id == "log0" || id == "log1" { let mut texts = control_lines().as_array().cloned().unwrap_or_default(); texts.extend(vm_lines(&vm).as_array().cloned().unwrap_or_default()); s.st[if id == "log0" { "c0" } else { "c1" }] = json!(counts(&o, &json!(texts))); } } // before: the guest's boot id, the VM's runtime and the log counts, then the restart with the watch if s.is("before") && s.st["boot0"].is_string() && s.st["rt0"].is_object() && s.st["hp0"].is_object() && s.has("secOk") && s.has("sealedSeen") && s.has("ccOk") && !s.has("logAsked") { s.flag("logAsked"); h = h.call(logs("log0", &s)); } else if s.is("before") && s.st["c0"].is_array() { s.reading("before", json!({ "firecrackervm": s.st["rt0"], "hostprocess": s.st["hp0"] })); s.set("restarting"); s.mark("restarted"); h = h.call(host("restart", "bidder.restart", json!({ "name": s.st["bidder"] }))).call(cvms("w", &cluster)).timer("watch", 2_000); } // verify: the three answers in, then the verdict, then the release if s.is("verify") && ["rt1", "hp1", "boot1", "c1", "adopt"].iter().all(|k| !s.st[*k].is_null()) && s.has("noTerm") { let (a, b) = (s.st["rt0"].clone(), s.st["rt1"].clone()); let (c0, c1): (Vec, Vec) = ( s.st["c0"].as_array().into_iter().flatten().map(|v| v.as_u64().unwrap_or(0)).collect(), s.st["c1"].as_array().into_iter().flatten().map(|v| v.as_u64().unwrap_or(0)).collect(), ); let mut p = vec![]; for k in ["pid", "invocationID", "unit", "startTicks", "startTime"] { if a[k] != b[k] { p.push(format!("the VM's VMM changed: {k} {} -> {}", a[k], b[k])); } } // the unit as the supervisor reads it: MainPID, InvocationID, ActiveEnterTimestamp, before and after let (u, v) = (s.st["hp0"].clone(), s.st["hp1"].clone()); for k in ["pid", "invocationID", "unit", "startTicks", "activeSince"] { if u[k] != v[k] { p.push(format!("the VM's unit changed: {k} {} -> {}", u[k], v[k])); } } if b["phase"] != "Running" || b["lostReason"].is_string() { p.push(format!("the VM is {} ({}) after the restart", b["phase"], b["lostReason"])); } if s.st["boot0"] != s.st["boot1"] { p.push(format!("the guest rebooted: {} -> {}", s.st["boot0"], s.st["boot1"])); } // the restart really happened: a new core generation, or the infra units adopted (>= 3 role lines) let newgen = c1.first().copied().unwrap_or(0).saturating_sub(c0.first().copied().unwrap_or(0)); let infra = c1.get(1).copied().unwrap_or(0).saturating_sub(c0.get(1).copied().unwrap_or(0)); s.reading("controls", json!({ "newCoreGenerations": newgen, "infraAdopted": infra })); if newgen == 0 && infra < 3 { p.push("the bidder's log shows no restart (no new `core holds the instance`, fewer than 3 adopted infra units)".to_string()); } // the new core's supervisor adopted exactly this VM's unit (hostprocess_unit_adopted: pid, invocation and // activeSince of the unit as the supervisor read it), the one the FirecrackerVM named before let ad = s.st["adopt"].as_array().cloned().unwrap_or_default(); s.reading("adopted", json!(ad)); match ad.as_slice() { [l] => { if l["pid"].as_i64() != u["pid"].as_i64() || l["invocation"] != u["invocationID"] { p.push(format!("the adopted unit is not the VM's: pid {} invocation {} (before: {} {})", l["pid"], l["invocation"], u["pid"], u["invocationID"])); } // activeSince: the unit's ActiveEnterTimestamp, as the supervisor read it before (to the second) if l["activeSince"].as_str().zip(u["activeSince"].as_str()).is_none_or(|(x, y)| (steps::secs(x) - steps::secs(y)).abs() > 1.0) { p.push(format!("activeSince {} is not the unit's before the restart {}", l["activeSince"], u["activeSince"])); } if !l["state"].as_str().unwrap_or_default().eq_ignore_ascii_case("active") { p.push(format!("the adopted unit's state: {}", l["state"])); } } _ => p.push(format!("{} hostprocess_unit_adopted lines for vm-{vm}, one expected", ad.len())), } // the VM's own lines: none let names = vm_lines(&vm); let mut touched = vec![]; for (i, t) in names.as_array().into_iter().flatten().enumerate() { let n = c1.get(i + 2).copied().unwrap_or(0).saturating_sub(c0.get(i + 2).copied().unwrap_or(0)); if n > 0 { touched.push(format!("{t} x{n}")); } } s.reading("vmLinesAfterRestart", json!(touched)); if !touched.is_empty() { p.push(format!("the restarted core touched the VM's unit: {}", touched.join(", "))); } s.reading("after", b); if !p.is_empty() { return Ok(s.fail(p.join("; "))); } s.set("release"); s.flag("releasing"); h = h.call(host("release", "request.release", json!({ "name": name }))); } } Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))), _ => {} } Ok(s.keep(h)) } mod fence; mod ingress; mod upgrade; entries! { run => run_0007, restart => run_restart, webrtc => run_webrtc, upgrade => upgrade::run_upgrade, fence => fence::run_fence, ingress => ingress::run_ingress }