0
0
// docs/acceptance/0007-bidder-termination.md: fault injection on the winner's kcp, scoped to this run's own receipt:
// delete its ComputeVM; it goes within 2 min, its vm.terminated is pinned `deleted_on_request`, the requester deletes
// its contract records (the release's vm.delete too), the host runs nothing of the receipt. Needs the winner's kcp.
// A winner whose kcp is read only (`write` false) gets no fault: its one ComputeVM is read, a release ends it, then
// the records are gone and the host runs nothing of the receipt.
use abi::{Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{
GUEST_SECRETS, MARKET_EVENT, MIN, S, SECRETS_FORCE_SEALED, SECRETS_NO_IROH, SECRETS_READ, SECRETS_SYMLINK, SECRETS_SYMLINK_WANT, VM_TERMINATED, created_since,
fetched_reason, get_record, host, kube, out, secrets_status, secrets_written, sw, termination_rkey,
};
/// The refused bundles' answers: each `bad-<name>` request exited non-zero with its message in the CLI's last lines.
fn bad_exit(s: &mut S, id: &str, data: &Value) -> Option<Handle> {
let name = id.strip_prefix("bad-")?;
let (_, _, want) = steps::secrets_bundles_refused().into_iter().find(|(n, _, _)| *n == name)?;
match data["msg"].as_str().unwrap_or_default() {
"exited" if data["code"].as_i64().unwrap_or(0) != 0 && data["tail"].as_str().unwrap_or_default().contains(want) => {
s.st["badOk"][name] = data["code"].clone();
if steps::secrets_bundles_refused().iter().all(|(n, _, _)| !s.st["badOk"][*n].is_null()) {
s.flag("badRefused");
s.reading("refusedBundles", s.st["badOk"].clone());
}
None
}
"exited" => Some(s.fail(format!("the refused bundle {name} did not exit non-zero with {want:?}: {data}"))),
"contract" if !data["phase"].as_str().unwrap_or_default().is_empty() => Some(s.fail(format!("the refused bundle {name} reached a contract: {}", data["phase"]))),
_ => None,
}
}
/// A page of the requester's market.events, each with the two value fields the sealed check reads.
fn list_events(renter: &Value, cursor: Option<&str>) -> abi::Call {
host("events", "repo.listCreated", json!({ "repo": renter, "collection": MARKET_EVENT, "limit": 100, "cursor": cursor, "fields": ["receipt.uri", "payload.uri"] }))
}
/// The repo-wide sealed-record check: every market.event of the requester's repo created since the run started, as
/// listed (its receipt.uri and payload.uri included, so a record deleted after the listing is never read again); one
/// whose receipt is this run's with a guest.secrets payload is a sealed bundle for this contract.
fn scan_events(s: &mut S, h: Handle, o: &Value) -> Result<Handle, String> {
let since = s.st["start"].as_f64().unwrap_or(s.now) - 30.0;
for r in created_since(o, since) {
let v = &r["value"];
if v["receipt"]["uri"] == s.st["receipt"] && v["payload"]["uri"].as_str().unwrap_or_default().contains(GUEST_SECRETS) {
return Err(format!("the requester's repo holds a sealed bundle for this contract while it has an inbound endpoint: {} -> {}", r["uri"], v["payload"]["uri"]));
}
s.st["evCount"] = json!(s.st["evCount"].as_u64().unwrap_or(0) + 1);
}
s.st["pages"] = json!(s.st["pages"].as_u64().unwrap_or(0) + 1);
if let Some(c) = o["cursor"].as_str().filter(|c| !c.is_empty()) {
return Ok(h.call(list_events(&s.st["renter"], Some(c))));
}
s.reading("sealedRecords", json!({ "since": "run start", "events": s.st["evCount"], "pages": s.st["pages"], "forThisContract": 0 }));
s.set("summary");
Ok(h.timer("contract", 1))
}
fn run_0007(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
let cluster = s.st["cluster"].as_str().unwrap_or_default().to_string();
let vms = |id: &str| kube(id, "list", &cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }));
let write = !bidders.iter().any(|b| b["cluster"] == cluster.as_str() && b["write"] == false);
let mut h = Handle::new();
match run.event {
Event::Start => {
s.set("held");
s.st["start"] = json!(s.now);
s.st["badOk"] = json!({});
h = h
.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "secretsFallback": "Sealed" })))
.timer("deadline", 30 * MIN);
for (i, (n, bundle, _)) in steps::secrets_bundles_refused().into_iter().enumerate() {
h = h.call(steps::host_sub(&format!("bad-{n}"), "request.start", json!({ "name": format!("{name}-b{i}"), "args": [], "secrets": bundle })));
}
}
Event::Message { id, data, .. } if id.starts_with("bad-") => {
if let Some(f) = bad_exit(&mut s, &id, &data) {
return Ok(f);
}
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))),
Event::Timer { id } if id == "fetchwait" && s.is("held") => {
return Ok(s.fail("Held, and no requester_guest_secrets_fetched 60 s later: the guest did not fetch over getSecrets"));
}
Event::Timer { id } if id == "forcewait" && s.is("forcewait") => {
return Ok(s.fail("no requester_guest_secrets_not_sealed 2 min after the guest asked for a sealed bundle"));
}
Event::Timer { id } if id == "cc" && s.is("status") => h = h.call(host("cc", "request.contract", json!({ "name": name }))),
Event::Timer { id } if id == "contract" => {
let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default());
s.st["winner"] = w.clone();
s.reading("winner", w.clone());
if !bidders.iter().any(|b| b["did"] == w) {
return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
}
let Some(c) = bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) else {
return Ok(s.unmet(&format!("the target names no kcp for the winner {w}")));
};
s.st["cluster"] = json!(c);
s.set("find");
h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
}
Event::Timer { id } if id == "poll" => {
h = match s.phase().as_str() {
"gone" | "empty" => h.call(vms("q")),
"pinned" => h.call(get_record("term", &format!("at://{}/{VM_TERMINATED}/{}", s.st["winner"].as_str().unwrap_or_default(), termination_rkey(s.st["receipt"].as_str().unwrap_or_default())))),
_ => h,
};
}
Event::Message { id, data, .. } if id == "req" => {
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
"exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
"contract" if data["phase"] == "Held" && !s.has("held") => {
s.flag("held");
h = h.timer("fetchwait", 60_000);
}
"requester_contract_records_deleted" => s.flag("recordsDeleted"),
"requester_guest_secrets_fetched" if !s.has("fetched") => {
let channel = data["channel"].as_str().unwrap_or_default().to_string();
if channel != "iroh" && channel != "webrtc" {
return Ok(s.fail(format!("the guest fetched its secrets over {channel:?}, not a peer transport")));
}
s.reading("secretsChannel", channel.clone());
s.st["channel"] = json!(channel);
s.flag("fetched");
}
"requester_guest_secrets_sealed" => return Ok(s.fail(format!("the requester sealed the bundle while it has an inbound endpoint: {data}"))),
"requester_contract_event_published" if data["nsid"] == GUEST_SECRETS => {
return Ok(s.fail(format!("the requester published a guest.secrets record while it has an inbound endpoint: {}", data["record"])));
}
// only once the guest was made to ask (forcewait is set before the force ssh goes out)
"requester_guest_secrets_not_sealed" if s.is("forcewait") => {
s.reading("notSealed", data["reason"].clone());
s.set("nosealcheck");
h = h.call(host("recs", "request.records", json!({ "name": name })));
}
_ => {}
}
// the --hold process stays (it settles the end): the receipt comes from its log, the winner is its author
if s.st["receipt"].is_null() && data["receipt"].is_string() {
s.st["receipt"] = data["receipt"].clone();
}
if s.is("held") && s.has("held") && s.has("fetched") && s.st["receipt"].is_string() {
s.set("status");
s.mark("status");
h = h.call(host("cc", "request.contract", json!({ "name": name })));
}
if s.is("records") && s.has("recordsDeleted") {
s.set("check");
h = h.call(host("records", "request.records", json!({ "name": name })));
}
}
Event::Result { id, result, .. } => {
let r = result.output();
if id == "term" && r.is_err() {
if s.since("delete") > 120.0 {
return Ok(s.fail("no vm.terminated 2 min after the delete"));
}
return Ok(s.keep(h.timer("poll", 3_000)));
}
let o = match out(&s, &id, r) {
Ok(o) => o,
Err(f) => return Ok(f),
};
match id.as_str() {
"cc" => {
let (via, reason) = secrets_status(&o);
let channel = s.st["channel"].as_str().unwrap_or_default().to_string();
if via.is_empty() || reason.is_empty() {
if s.since("status") > 30.0 {
return Ok(s.fail(format!("the contract records no secrets channel 30 s after the fetch: secretsFetchedVia {via:?}, SecretsApplied {reason:?}")));
}
return Ok(s.keep(h.timer("cc", 2_000)));
}
if via != channel || reason != fetched_reason(&channel) {
return Ok(s.fail(format!("the log says {channel}, the contract secretsFetchedVia {via:?} and SecretsApplied {reason:?}")));
}
s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason }));
s.set("secrets");
h = h.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ })));
}
"sec" => {
if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) {
return Ok(s.fail(format!("getSecrets: {e}")));
}
s.reading("secrets", "the string, the object as JSON text, modes 0600 0644 0400, a 0700 dir made, /etc unchanged");
h = h.call(host("link", "request.ssh", json!({ "name": name, "exec": SECRETS_SYMLINK })));
}
"link" => {
if !o["output"].as_str().unwrap_or_default().lines().any(|l| l.trim_end() == SECRETS_SYMLINK_WANT) {
return Ok(s.fail(format!("a symlink at a secret's path: want {SECRETS_SYMLINK_WANT}, got {}", o["output"])));
}
s.set("forcewait");
h = h.timer("forcewait", 2 * MIN).call(host("force", "request.ssh", json!({ "name": name, "exec": SECRETS_FORCE_SEALED })));
}
"force" => {
if !o["output"].as_str().unwrap_or_default().lines().any(|l| l.trim() == "FORCED") {
return Ok(s.fail(format!("the guest could not ask for a sealed bundle: {}", o["output"])));
}
}
"recs" => {
let uris: Vec<String> = o["uris"].as_array().into_iter().flatten().filter_map(|u| u.as_str().map(String::from)).collect();
if let Some(u) = uris.iter().find(|u| u.contains(GUEST_SECRETS)) {
return Ok(s.fail(format!("the requester published a sealed bundle while it has an inbound endpoint: {u}")));
}
let Some(renter) = uris.first().and_then(|u| u.trim_start_matches("at://").split('/').next()).map(String::from) else {
return Ok(s.fail("request.records names no record of the requester"));
};
s.st["renter"] = json!(renter);
s.set("scan");
h = h.call(list_events(&json!(renter), None));
}
"events" => match scan_events(&mut s, h, &o) {
Ok(next) => h = next,
Err(why) => return Ok(s.fail(why)),
},
"vms" => {
let mine: Vec<String> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).filter_map(|v| v["metadata"]["name"].as_str().map(String::from)).collect();
if mine.len() != 1 {
return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", mine.len())));
}
s.st["vm"] = json!(mine[0]);
if write {
s.mark("delete");
h = h.call(kube("del", "delete", &cluster, sw("ComputeVM", &mine[0])));
} else {
s.set("release");
s.reading("fault", "none: the winner's kcp is read only");
h = h.call(host("release", "request.release", json!({ "name": name })));
}
}
"release" => {
s.reading("release_s", o["seconds"].clone());
s.mark("delete");
s.set("check");
h = h.call(host("records", "request.records", json!({ "name": name })));
}
"del" => {
s.set("gone");
h = h.timer("poll", 2_000);
}
"q" => {
let there = o["items"].as_array().into_iter().flatten().any(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]);
if there && s.since("delete") > 120.0 {
return Ok(s.fail(format!("the ComputeVM is still there 2 min after the {}", if write { "delete" } else { "release" })));
}
if there {
h = h.timer("poll", 2_000);
} else if s.is("gone") {
let t = s.since("delete");
s.reading("gone_s", t);
s.set("pinned");
h = h.timer("poll", 1_000);
} else if !s.has("badRefused") {
return Ok(s.fail(format!("not every refused bundle exited non-zero with its message: {}", s.st["badOk"])));
} else {
return Ok(s.pass());
}
}
"term" => {
if o["value"]["reason"] != "deleted_on_request" {
return Ok(s.fail(format!("vm.terminated {}", o["value"])));
}
s.reading("terminated", json!({ "reason": o["value"]["reason"], "request": o["value"]["request"]["uri"] }));
s.set("records");
if s.has("recordsDeleted") {
s.set("check");
h = h.call(host("records", "request.records", json!({ "name": name })));
}
}
"records" => {
s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
}
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
}
s.set("empty");
h = h.call(vms("q"));
}
_ => {}
}
}
_ => {}
}
Ok(s.keep(h))
}
/// webrtc: getSecrets over the WebRTC channel. The contract's user_data drops the iroh ticket from requester.json, so the
/// guest agent forwards over WebRTC (the requester's guest WebRTC ingress offers over the sealed signalling); the log and
/// the contract (status.secretsFetchedVia, SecretsApplied GetSecretsWebRTC) must say webrtc, the guest holds the files,
/// and the release leaves no record.
fn run_webrtc(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let mut h = Handle::new();
match run.event {
Event::Start => {
s.set("held");
let start = json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "runcmd": [SECRETS_NO_IROH] });
h = h.call(steps::host_sub("req", "request.start", start)).timer("deadline", 20 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 20 min (phase {})", s.phase()))),
Event::Timer { id } if id == "fetchwait" && s.is("held") => {
return Ok(s.fail("Held, and no requester_guest_secrets_fetched 2 min later: the guest did not fetch over WebRTC"));
}
Event::Timer { id } if id == "cc" && s.is("status") => h = h.call(host("cc", "request.contract", json!({ "name": name }))),
Event::Message { id, data, .. } if id == "req" => {
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
"exited" if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))),
"contract" if data["phase"] == "Held" && !s.has("held") => {
s.flag("held");
h = h.timer("fetchwait", 2 * MIN);
}
"requester_guest_secrets_fetched" if !s.has("fetched") => {
if data["channel"] != "webrtc" {
return Ok(s.fail(format!("iroh off: the guest fetched its secrets over {}, not webrtc", data["channel"])));
}
s.reading("secretsChannel", "webrtc");
s.flag("fetched");
}
"requester_guest_webrtc_offered" | "requester_guest_webrtc_answered" => s.reading(data["msg"].as_str().unwrap_or_default(), json!(s.now)),
_ => {}
}
if s.is("held") && s.has("held") && s.has("fetched") {
s.set("status");
s.mark("status");
h = h.call(host("cc", "request.contract", json!({ "name": name })));
}
}
Event::Result { id, result, .. } => {
let o = match out(&s, &id, result.output()) {
Ok(o) => o,
Err(f) => return Ok(f),
};
match id.as_str() {
"cc" => {
let (via, reason) = secrets_status(&o);
if via.is_empty() || reason.is_empty() {
if s.since("status") > 30.0 {
return Ok(s.fail(format!("the contract records no secrets channel 30 s after the fetch: secretsFetchedVia {via:?}, SecretsApplied {reason:?}")));
}
return Ok(s.keep(h.timer("cc", 2_000)));
}
if via != "webrtc" || reason != fetched_reason("webrtc") {
return Ok(s.fail(format!("iroh off: the contract says secretsFetchedVia {via:?}, SecretsApplied {reason:?}")));
}
s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason }));
s.set("secrets");
h = h.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ })));
}
"sec" => {
if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) {
return Ok(s.fail(format!("getSecrets over WebRTC: {e}")));
}
s.set("release");
s.flag("releasing");
h = h.call(host("release", "request.release", json!({ "name": name })));
}
"release" => {
s.reading("release_s", o["seconds"].clone());
h = h.call(host("records", "request.records", json!({ "name": name })));
}
"records" => {
s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
}
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
}
return Ok(s.pass());
}
_ => {}
}
}
Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
_ => {}
}
Ok(s.keep(h))
}
/// Log lines of the bidder's core that mean the restarted core started or stopped THIS VM's unit: a VM unit that is
/// Active is adopted silently (hostprocess supervisor markAdopted), so any of these after the restart is a restart
/// of the VM. [text, text...]: a line holds all of them.
fn vm_lines(vm: &str) -> Value {
json!([
["\"msg\":\"placed\"", vm],
["\"msg\":\"stopped\"", vm],
["work dir and API socket removed", vm],
["\"msg\":\"shutdown\"", vm],
["the process wait saw the VMM exit", vm],
["bidder_vm_terminated", vm],
["a released bid is marked terminated", vm],
])
}
/// The positive controls: the restarted core really started (a dev network's core logs `core holds the instance`
/// with a new generation; an installed bidder's logs `adopted` for each of its infrastructure units, role kine-a,
/// front, kcp-a). Index 0 and 1 of the counts.
fn control_lines() -> Value {
json!(["core holds the instance", ["\"msg\":\"adopted\"", "\"role\":"]])
}
fn counts(o: &Value, texts: &Value) -> Vec<u64> {
texts
.as_array()
.into_iter()
.flatten()
.map(|t| {
let k = match t {
Value::Array(a) => a.iter().filter_map(Value::as_str).collect::<Vec<_>>().join(" & "),
t => t.as_str().unwrap_or_default().to_string(),
};
o["counts"][k].as_u64().unwrap_or(0)
})
.collect()
}
fn fcvm(id: &str, cluster: &str, name: &str) -> abi::Call {
kube(id, "get", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "FirecrackerVM", "name": name }))
}
/// The VM's HostProcess `vm-<name>`: status.runtime is the unit as the supervisor last read it (MainPID, InvocationID,
/// ActiveEnterTimestamp).
fn hproc(id: &str, cluster: &str, vm: &str) -> abi::Call {
kube(id, "get", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "HostProcess", "plural": "hostprocesses", "name": format!("vm-{vm}") }))
}
fn cvms(id: &str, cluster: &str) -> abi::Call {
kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))
}
/// restart (staging): the winner's core restarted in place (`dev restart <network> <bidder>/core`, the upgrade path, not a
/// drain) under this run's Held VM. The VM's own unit is identified by its FirecrackerVM runtime (pid, invocationID,
/// startTicks, unit) before and after and must be unchanged; its ComputeVM is read every 2 s from before the restart
/// to 30 s after it and is Running every time (never Lost or Releasing); the guest's boot id is unchanged and ssh
/// answers; the new core's log has none of vm_lines() for it (it adopts an Active unit silently); no vm.terminated; the
/// contract never leaves Held. The restart itself must show (control_lines: a new core generation, or the infra
/// units adopted). Then the release leaves no record.
fn run_restart(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
let cluster = s.st["cluster"].as_str().unwrap_or_default().to_string();
let vm = s.st["vm"].as_str().unwrap_or_default().to_string();
let boot = |id: &str| host(id, "request.ssh", json!({ "name": name, "exec": "echo BOOT $(cat /proc/sys/kernel/random/boot_id)" }));
let logs = |id: &str, s: &S| {
let mut texts = control_lines().as_array().cloned().unwrap_or_default();
texts.extend(vm_lines(&vm).as_array().cloned().unwrap_or_default());
host(id, "bidder.logCount", json!({ "name": s.st["bidder"], "texts": texts }))
};
let begin = |s: &mut S, h: Handle| -> Result<Handle, Handle> {
let mut h = h;
let w = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
s.reading("winner", w.clone());
let Some(b) = bidders.iter().find(|b| b["did"] == w.as_str()).cloned() else {
return Err(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
};
let Some(c) = b["cluster"].as_str() else {
return Err(s.unmet(&format!("the target names no kcp for the winner {w}")));
};
s.st["cluster"] = json!(c);
s.st["bidder"] = b["name"].clone();
s.set("before");
h = h
.call(boot("boot0"))
.call(cvms("vms", c))
.call(host("sec", "request.ssh", json!({ "name": name, "exec": SECRETS_READ })))
.call(host("recs", "request.records", json!({ "name": name })))
.call(host("cc", "request.contract", json!({ "name": name })));
s.mark("status");
Ok(h)
};
let mut h = Handle::new();
match run.event {
Event::Start => {
if args["target"] == "prod" {
return Ok(s.unmet("staging only: it restarts a bidder's core (prod: the rollout's own upgrade)"));
}
s.set("held");
let start = json!({ "name": name, "args": ["--hold"], "secrets": steps::secrets_bundle(), "commsMode": "Firehose", "secretsFallback": "Sealed" });
h = h.call(steps::host_sub("req", "request.start", start)).timer("deadline", 30 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 30 min (phase {})", s.phase()))),
Event::Timer { id } if id == "sealwait" && s.is("held") => {
return Ok(s.fail("Held, and no secretsApplied via sealed in the requester's log 5 min later"));
}
// the watch: the ComputeVM every 2 s, from before the restart to 30 s after it returns
Event::Timer { id } if id == "watch" => {
if s.is("restarting") || s.is("watching") {
h = h.call(cvms("w", &cluster)).timer("watch", 2_000);
}
}
Event::Message { data, .. } => {
if s.st["receipt"].is_null() && data["receipt"].is_string() {
s.st["receipt"] = data["receipt"].clone();
}
match (data["msg"].as_str().unwrap_or_default(), data["phase"].as_str().unwrap_or_default()) {
("contract", "Held") if !s.has("held") => {
s.flag("held");
h = h.timer("sealwait", 5 * MIN);
}
("contract", "Held") => {}
("contract", p) if s.has("held") && !s.has("releasing") && !p.is_empty() => {
return Ok(s.fail(format!("the contract left Held around the restart: {p} {}", data["message"])));
}
("contract", "Failed") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
("exited", _) if !s.has("releasing") => return Ok(s.fail(format!("run request exited before the release: {data}"))),
("requester_guest_secrets_fetched", _) => return Ok(s.fail("comms mode Firehose: the guest fetched over getSecrets")),
("requester_guest_secrets_not_sealed", _) => return Ok(s.fail(format!("comms mode Firehose: the requester did not seal: {}", data["reason"]))),
("requester_guest_secrets", _) if data["event"] == "secretsApplied" && data["via"] == "sealed" => s.flag("sealedWritten"),
_ => {}
}
if s.is("held") && s.has("held") && s.has("sealedWritten") {
if s.st["receipt"].is_string() {
match begin(&mut s, h) {
Ok(next) => h = next,
Err(f) => return Ok(f),
}
} else if !s.has("rcptAsked") {
s.flag("rcptAsked");
h = h.call(host("rcpt", "request.contract", json!({ "name": name })));
}
}
}
Event::Timer { id } if id == "cc" && s.is("before") => h = h.call(host("cc", "request.contract", json!({ "name": name }))),
Event::Timer { id } if id == "rcpt" && s.is("held") => h = h.call(host("rcpt", "request.contract", json!({ "name": name }))),
Event::Result { id, result, .. } if id == "rcpt" => {
if !s.is("held") {
return Ok(s.keep(h));
}
match result.output().ok().and_then(|o| o["receipt"].as_str().map(String::from)) {
Some(rc) => {
s.st["receipt"] = json!(rc);
match begin(&mut s, h) {
Ok(next) => h = next,
Err(f) => return Ok(f),
}
}
None => h = h.timer("rcpt", 2_000),
}
}
Event::Result { id, result, .. } => {
let r = result.output();
if id == "term" {
if let Ok(o) = r {
return Ok(s.fail(format!("vm.terminated after the restart: {}", o["value"])));
}
s.flag("noTerm");
} else if id == "w" && !(s.is("restarting") || s.is("watching")) {
// a list that came back after the watch ended (the release has begun): not read
} else if id == "w" {
// an unreadable list during the restart is a reading, not a verdict; a VM that is not Running is
let Ok(o) = r else {
s.st["unreadable"] = json!(s.st["unreadable"].as_u64().unwrap_or(0) + 1);
return Ok(s.keep(h));
};
let mine = o["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]);
s.st["watched"] = json!(s.st["watched"].as_u64().unwrap_or(0) + 1);
match mine {
Some(v) if v["status"]["phase"] == "Running" && v["metadata"]["deletionTimestamp"].is_null() && !v["status"]["lostReason"].is_string() => {}
other => {
return Ok(s.fail(format!("the ComputeVM left Running during the restart: {}", other.map(|v| json!({ "phase": v["status"]["phase"], "lostReason": v["status"]["lostReason"], "deleting": v["metadata"]["deletionTimestamp"] })).unwrap_or(json!("gone")))));
}
}
if s.is("watching") && s.since("restarted") > 30.0 {
s.set("verify");
s.reading("watched", s.st["watched"].clone());
s.reading("watchUnreadable", s.st["unreadable"].clone());
h = h.call(fcvm("fc1", &cluster, &vm)).call(hproc("hp1", &cluster, &vm)).call(boot("boot1")).call(logs("log1", &s)).call(host(
"adopt",
"bidder.logLines",
json!({ "name": s.st["bidder"], "texts": [["hostprocess_unit_adopted", format!("vm-{vm}")]], "keys": ["hostprocess", "unit", "pid", "invocation", "activeSince", "state"] }),
));
let rcpt = s.st["receipt"].as_str().unwrap_or_default().to_string();
let wd = rcpt.trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
h = h.call(get_record("term", &format!("at://{wd}/{VM_TERMINATED}/{}", termination_rkey(&rcpt))));
}
} else {
let o = match out(&s, &id, r) {
Ok(o) => o,
Err(f) => return Ok(f),
};
match id.as_str() {
"sec" => {
if let Err(e) = secrets_written(o["output"].as_str().unwrap_or_default()) {
return Ok(s.fail(format!("the sealed record: {e}")));
}
s.flag("secOk");
}
"cc" => {
let (via, reason) = secrets_status(&o);
if !via.is_empty() {
return Ok(s.fail(format!("comms mode Firehose: the contract records a getSecrets channel {via:?}")));
}
if reason != "SealedFallback" {
if s.since("status") > 30.0 {
return Ok(s.fail(format!("comms mode Firehose: SecretsApplied {reason:?} 30 s after secretsApplied via sealed, want SealedFallback")));
}
return Ok(s.keep(h.timer("cc", 2_000)));
}
s.reading("secretsStatus", json!({ "secretsFetchedVia": via, "secretsApplied": reason }));
s.flag("ccOk");
}
"recs" => {
let Some(u) = o["uris"].as_array().into_iter().flatten().filter_map(Value::as_str).find(|u| u.contains(GUEST_SECRETS)).map(String::from) else {
return Ok(s.fail(format!("comms mode Firehose: no {GUEST_SECRETS} record among the requester's records")));
};
h = h.call(get_record("sealrec", &u));
}
"sealrec" => {
if !o["value"]["ciphertext"].as_str().unwrap_or_default().starts_with("-----BEGIN AGE ENCRYPTED FILE-----") {
return Ok(s.fail(format!("the {GUEST_SECRETS} record holds no age armor: {}", o["uri"])));
}
s.reading("sealedRecord", o["uri"].clone());
s.flag("sealedSeen");
}
"boot0" | "boot1" => {
let Some(b) = o["output"].as_str().unwrap_or_default().lines().find_map(|l| l.strip_prefix("BOOT ")) else {
return Ok(s.fail(format!("{id}: ssh into the guest: {}", o["output"])));
};
s.st[id.as_str()] = json!(b.trim());
}
"vms" => {
let mine: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).collect();
let [v] = mine.as_slice() else {
return Ok(s.fail(format!("{} ComputeVMs of this run's receipt on the winner", mine.len())));
};
s.st["vm"] = v["metadata"]["name"].clone();
let n = v["metadata"]["name"].as_str().unwrap_or_default();
h = h.call(fcvm("fc0", &cluster, n)).call(hproc("hp0", &cluster, n));
}
"hp0" | "hp1" => {
let rt = &o["status"]["runtime"];
if rt["pid"].as_i64().unwrap_or(0) == 0 || rt["invocationID"].as_str().unwrap_or_default().is_empty() {
return Ok(s.fail(format!("the HostProcess vm-{vm} has no runtime pid and invocationID ({id}): {}", o["status"])));
}
s.st[id.as_str()] = json!({ "pid": rt["pid"], "invocationID": rt["invocationID"], "unit": rt["unit"], "startTicks": rt["startTicks"], "activeSince": rt["activeSince"], "phase": o["status"]["phase"] });
}
"fc0" | "fc1" => {
let rt = &o["status"]["runtime"];
if rt["pid"].as_i64().unwrap_or(0) == 0 || rt["invocationID"].as_str().unwrap_or_default().is_empty() {
return Ok(s.fail(format!("the FirecrackerVM has no runtime pid and invocationID ({id}): {}", o["status"])));
}
let k = if id == "fc0" { "rt0" } else { "rt1" };
s.st[k] = json!({ "pid": rt["pid"], "invocationID": rt["invocationID"], "unit": rt["unit"], "startTicks": rt["startTicks"], "startTime": rt["startTime"], "phase": o["status"]["phase"], "lostReason": o["status"]["lostReason"] });
}
"restart" => {
s.reading("restart_s", o["seconds"].clone());
s.set("watching");
s.mark("restarted");
}
"adopt" => s.st["adopt"] = o["lines"].clone(),
"release" => {
s.reading("release_s", o["seconds"].clone());
h = h.call(host("records", "request.records", json!({ "name": name })));
}
"records" => {
s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
}
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
}
return Ok(s.pass());
}
_ => {}
}
if id == "log0" || id == "log1" {
let mut texts = control_lines().as_array().cloned().unwrap_or_default();
texts.extend(vm_lines(&vm).as_array().cloned().unwrap_or_default());
s.st[if id == "log0" { "c0" } else { "c1" }] = json!(counts(&o, &json!(texts)));
}
}
// before: the guest's boot id, the VM's runtime and the log counts, then the restart with the watch
if s.is("before") && s.st["boot0"].is_string() && s.st["rt0"].is_object() && s.st["hp0"].is_object() && s.has("secOk") && s.has("sealedSeen") && s.has("ccOk") && !s.has("logAsked") {
s.flag("logAsked");
h = h.call(logs("log0", &s));
} else if s.is("before") && s.st["c0"].is_array() {
s.reading("before", json!({ "firecrackervm": s.st["rt0"], "hostprocess": s.st["hp0"] }));
s.set("restarting");
s.mark("restarted");
h = h.call(host("restart", "bidder.restart", json!({ "name": s.st["bidder"] }))).call(cvms("w", &cluster)).timer("watch", 2_000);
}
// verify: the three answers in, then the verdict, then the release
if s.is("verify") && ["rt1", "hp1", "boot1", "c1", "adopt"].iter().all(|k| !s.st[*k].is_null()) && s.has("noTerm") {
let (a, b) = (s.st["rt0"].clone(), s.st["rt1"].clone());
let (c0, c1): (Vec<u64>, Vec<u64>) = (
s.st["c0"].as_array().into_iter().flatten().map(|v| v.as_u64().unwrap_or(0)).collect(),
s.st["c1"].as_array().into_iter().flatten().map(|v| v.as_u64().unwrap_or(0)).collect(),
);
let mut p = vec![];
for k in ["pid", "invocationID", "unit", "startTicks", "startTime"] {
if a[k] != b[k] {
p.push(format!("the VM's VMM changed: {k} {} -> {}", a[k], b[k]));
}
}
// the unit as the supervisor reads it: MainPID, InvocationID, ActiveEnterTimestamp, before and after
let (u, v) = (s.st["hp0"].clone(), s.st["hp1"].clone());
for k in ["pid", "invocationID", "unit", "startTicks", "activeSince"] {
if u[k] != v[k] {
p.push(format!("the VM's unit changed: {k} {} -> {}", u[k], v[k]));
}
}
if b["phase"] != "Running" || b["lostReason"].is_string() {
p.push(format!("the VM is {} ({}) after the restart", b["phase"], b["lostReason"]));
}
if s.st["boot0"] != s.st["boot1"] {
p.push(format!("the guest rebooted: {} -> {}", s.st["boot0"], s.st["boot1"]));
}
// the restart really happened: a new core generation, or the infra units adopted (>= 3 role lines)
let newgen = c1.first().copied().unwrap_or(0).saturating_sub(c0.first().copied().unwrap_or(0));
let infra = c1.get(1).copied().unwrap_or(0).saturating_sub(c0.get(1).copied().unwrap_or(0));
s.reading("controls", json!({ "newCoreGenerations": newgen, "infraAdopted": infra }));
if newgen == 0 && infra < 3 {
p.push("the bidder's log shows no restart (no new `core holds the instance`, fewer than 3 adopted infra units)".to_string());
}
// the new core's supervisor adopted exactly this VM's unit (hostprocess_unit_adopted: pid, invocation and
// activeSince of the unit as the supervisor read it), the one the FirecrackerVM named before
let ad = s.st["adopt"].as_array().cloned().unwrap_or_default();
s.reading("adopted", json!(ad));
match ad.as_slice() {
[l] => {
if l["pid"].as_i64() != u["pid"].as_i64() || l["invocation"] != u["invocationID"] {
p.push(format!("the adopted unit is not the VM's: pid {} invocation {} (before: {} {})", l["pid"], l["invocation"], u["pid"], u["invocationID"]));
}
// activeSince: the unit's ActiveEnterTimestamp, as the supervisor read it before (to the second)
if l["activeSince"].as_str().zip(u["activeSince"].as_str()).is_none_or(|(x, y)| (steps::secs(x) - steps::secs(y)).abs() > 1.0) {
p.push(format!("activeSince {} is not the unit's before the restart {}", l["activeSince"], u["activeSince"]));
}
if !l["state"].as_str().unwrap_or_default().eq_ignore_ascii_case("active") {
p.push(format!("the adopted unit's state: {}", l["state"]));
}
}
_ => p.push(format!("{} hostprocess_unit_adopted lines for vm-{vm}, one expected", ad.len())),
}
// the VM's own lines: none
let names = vm_lines(&vm);
let mut touched = vec![];
for (i, t) in names.as_array().into_iter().flatten().enumerate() {
let n = c1.get(i + 2).copied().unwrap_or(0).saturating_sub(c0.get(i + 2).copied().unwrap_or(0));
if n > 0 {
touched.push(format!("{t} x{n}"));
}
}
s.reading("vmLinesAfterRestart", json!(touched));
if !touched.is_empty() {
p.push(format!("the restarted core touched the VM's unit: {}", touched.join(", ")));
}
s.reading("after", b);
if !p.is_empty() {
return Ok(s.fail(p.join("; ")));
}
s.set("release");
s.flag("releasing");
h = h.call(host("release", "request.release", json!({ "name": name })));
}
}
Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
_ => {}
}
Ok(s.keep(h))
}
mod fence;
mod ingress;
mod upgrade;
entries! { run => run_0007, restart => run_restart, webrtc => run_webrtc, upgrade => upgrade::run_upgrade, fence => fence::run_fence, ingress => ingress::run_ingress }