// docs/acceptance/0017-owner-release.md: the owner `run request --hold` (on a pseudo-terminal, so it opens and holds an // interactive ssh session into its guest, no --exec) and a SEPARATE `run request release ` process. run: the release // returns Released within 60 s, and the owner exits only after the contract is Released (its summary says Released and // kcpRunning false at its exit). killed: the owner is SIGKILLed 2 s after the release was asked; the release client takes // the invocation over and still settles within 60 s. A release that waits out its timeout on an owner that stopped its // kcp (main before the fix: 27 minutes) fails here at 90 s. use abi::{Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{MIN, S, host, host_sub}; const RELEASE_MAX_S: f64 = 60.0; fn phase_of(summary: &Value) -> String { summary["contract"]["phase"].as_str().or(summary["phase"].as_str()).unwrap_or_default().to_lowercase() } fn run_with(run: Run, killed: bool, kill_ms: u64) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); h = h .call(host_sub("req", "request.start", json!({ "name": name, "pty": true, "args": ["--hold", "--expires-in", "1h"] }))) .timer("deadline", 25 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 25 min (phase {})", s.phase()))), Event::Timer { id } if id == "settle" && s.is("held") && s.has("held") && s.has("sshing") && !s.has("exited") => { s.set("release"); s.mark("release"); h = h.call(host("release", "request.release", json!({ "name": name }))).timer("relguard", 90_000); if killed { h = h.timer("kill", kill_ms); } } Event::Timer { id } if id == "settle" => return Ok(s.fail(format!("the owner is not holding an interactive ssh session (held {}, ssh {}, exited {})", s.has("held"), s.has("sshing"), s.has("exited")))), Event::Timer { id } if id == "kill" => h = h.call(host("kill", "request.kill", json!({ "name": name }))), Event::Timer { id } if id == "relguard" && !s.has("released") => { return Ok(s.fail(format!("the release did not return in 90 s after it was asked (owner exited {}, killed {}); main waited out the release timeout on an owner that had stopped its kcp", s.has("exited"), s.has("killed")))); } Event::Timer { id } if id == "exitguard" && !s.has("exited") => return Ok(s.fail("the owner did not exit within 60 s after the contract was Released")), Event::Message { id, data, .. } if id == "req" => match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "contract" if data["phase"] == "Held" && !s.has("held") => { s.flag("held"); let t = s.t(); s.reading("held_s", t); } m if m.starts_with("no --exec and stdin is not a terminal") => return Ok(s.fail("the owner got no terminal on stdin: no interactive session (the pty wrapper did not work)")), m if m.starts_with("ssh to the guest over") && !s.has("sshing") => { s.flag("sshing"); h = h.timer("settle", 30_000); } "ssh returned" if !s.is("release") => return Ok(s.fail(format!("the interactive ssh session ended before the release was asked: {data}"))), "exited" => { s.flag("exited"); s.reading("owner_exit", data["code"].clone()); if !s.is("release") && !s.is("records") { return Ok(s.fail(format!("the owner exited before the release was asked: {data}"))); } if !killed { // the owner exits only after Released: its summary at its exit h = h.call(host("exit_summary", "request.summary", json!({ "name": name }))); } } _ => {} }, Event::Closed { id, error } if id == "req" && !s.has("exited") => return Ok(s.fail(format!("the owner's subscription closed before it exited: {}", error.unwrap_or_default()))), Event::Result { id, result, .. } => { if id == "summary" { if let Err(e) = result.output() { return Ok(s.fail(format!("the release returned but the invocation was left unsettled (no summary: {e}): the owner stopped mid-release and the release client did not take the invocation over"))); } } let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "kill" => { s.flag("killed"); s.reading("killed_pids", o["killed"].clone()); } "release" => { let secs = s.since("release"); s.flag("released"); s.reading("release_s", o["seconds"].clone()); s.reading("release_asked_to_returned_s", secs); if o["seconds"].as_f64().unwrap_or(f64::MAX) > RELEASE_MAX_S { return Ok(s.fail(format!("the release took {} s (more than {RELEASE_MAX_S})", o["seconds"]))); } h = h.call(host("summary", "request.summary", json!({ "name": name }))).timer("exitguard", 60_000); } "exit_summary" => { let phase = phase_of(&o); s.reading("owner_exit_phase", phase.clone()); s.reading("owner_exit_kcp_running", o["kcpRunning"].clone()); if phase != "released" || o["kcpRunning"] != false { return Ok(s.fail(format!("the owner exited with the contract in phase {phase:?} and kcpRunning {}: it stopped its kcp before Released", o["kcpRunning"]))); } s.flag("exitSummaryOk"); } "summary" => { let phase = phase_of(&o); s.reading("summary_phase", phase.clone()); s.reading("summary_kcp_running", o["kcpRunning"].clone()); // killedlate: the owner died after the release was acked, while stopping its kcp: its summary is its last word // (kcpRunning true); the second release below is what settles the invocation if phase != "released" || (kill_ms < 1000 && o["kcpRunning"] != false) { return Ok(s.fail(format!("after the release the summary says phase {phase:?}, kcpRunning {}", o["kcpRunning"]))); } s.flag("summaryOk"); if killed { // a second release of the taken-over invocation only removes it h = h.call(host("release2", "request.release", json!({ "name": name }))); } else { h = h.call(host("sshafter", "request.sshTry", json!({ "name": name }))); } } "release2" => { s.reading("release2_s", o["seconds"].clone()); h = h.call(host("state", "request.state", json!({ "name": name }))); } "state" => { // a second release that finds the invocation's state gone only removes the directory; one that has to // restart the kcp (the dying owner left its state) finishes like any release: the record (summary // Released, kcpRunning false) may stay, the instance state and every process of it may not let summary_ok = phase_of(&o["summary"]) == "released" && o["summary"]["kcpRunning"] == false; s.reading("dirKept", o["exists"].clone()); if o["instanceState"] == true || o["liveProcesses"].as_u64().unwrap_or(0) > 0 { return Ok(s.fail(format!("after the second release the invocation {} still has instance state ({}) or live processes ({})", o["dir"], o["instanceState"], o["liveProcesses"]))); } if o["exists"] != false && !summary_ok { return Ok(s.fail(format!("after the second release the invocation directory {} is still there and its summary is not Released with kcpRunning false: {}", o["dir"], o["summary"]))); } s.flag("removedOk"); h = h.call(host("sshafter", "request.sshTry", json!({ "name": name }))); } "sshafter" => { // ssh on a finished invocation must not start its kcp again let out = o["output"].as_str().unwrap_or_default().to_string(); s.reading("sshAfter", json!({ "ok": o["ok"], "restarted": o["restarted"], "output": out.clone() })); if o["ok"] == true || o["restarted"] == true { return Ok(s.fail(format!("run request ssh on the released invocation succeeded or restarted its kcp: {out}"))); } h = h.call(host("state3", "request.state", json!({ "name": name }))); } "state3" => { if o["instanceState"] == true || o["liveProcesses"].as_u64().unwrap_or(0) > 0 { return Ok(s.fail(format!("run request ssh on the released invocation left instance state ({}) or processes ({})", o["instanceState"], o["liveProcesses"]))); } s.flag("sshGuardOk"); } "records" => { s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left after the release: {} {}", o["left"], o["errors"]))); } return Ok(s.pass()); } _ => {} } } _ => {} } if s.is("release") && s.has("released") && s.has("summaryOk") && s.has("exited") && (killed || s.has("exitSummaryOk")) && (!killed || s.has("removedOk")) && s.has("sshGuardOk") { s.set("records"); h = h.call(host("records", "request.records", json!({ "name": name }))); } Ok(s.keep(h)) } // half: an invocation whose network application is gone but whose own summary does NOT say its contract ended must not be // "released" by a release command: it fails naming the phase and keeps the state. Made from the test's own invocation: a // normal release (application removed, summary Released), then its summary phase rewritten to Held. fn half_0017(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); h = h.call(host_sub("req", "request.start", json!({ "name": name, "args": ["--hold", "--exec", "--", "id"] }))).timer("deadline", 20 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 20 min (phase {})", s.phase()))), Event::Message { id, data, .. } if id == "req" => match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "contract" if data["phase"] == "Held" => s.flag("held"), "ssh returned" => { if data["exit"] != 0 { return Ok(s.fail(format!("id over ssh: {data}"))); } s.flag("ssh"); } "exited" => { s.flag("exited"); if !s.is("held") && !s.is("released") { return Ok(s.fail(format!("the owner exited before the release was asked: {data}"))); } } _ => {} }, Event::Result { id, result, .. } => { let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "release" => { s.flag("released"); h = h.call(host("state1", "request.state", json!({ "name": name }))); } "state1" => { let phase = phase_of(&o["summary"]); if o["exists"] != true || phase != "released" { return Ok(s.fail(format!("after the normal release the invocation directory exists {} with phase {phase:?}", o["exists"]))); } s.flag("state1Ok"); } "setphase" => h = h.call(host("try", "request.releaseTry", json!({ "name": name }))), "try" => { let out = o["output"].as_str().unwrap_or_default().to_string(); s.reading("half_release_output", out.clone()); if o["ok"] == true { return Ok(s.fail(format!("a release of an invocation whose state says Held and whose application is gone exited OK: {out}"))); } if !out.contains("Held") { return Ok(s.fail(format!("the failed release does not name the phase (Held): {out}"))); } h = h.call(host("state2", "request.state", json!({ "name": name }))); } "state2" => { if o["exists"] != true { return Ok(s.fail("the failed release removed the invocation directory: a VM may still be held")); } h = h.call(host("remove", "request.removeState", json!({ "name": name }))); } "remove" => return Ok(s.pass()), _ => {} } } Event::Closed { id, error } if id == "req" && !s.has("exited") => return Ok(s.fail(format!("the owner's subscription closed before it exited: {}", error.unwrap_or_default()))), _ => {} } if s.is("held") && s.has("held") && s.has("ssh") && !s.has("released") && !s.has("releasing") { s.flag("releasing"); h = h.call(host("release", "request.release", json!({ "name": name }))); } if s.has("released") && s.has("exited") && s.has("state1Ok") && !s.has("edited") { s.flag("edited"); h = h.call(host("setphase", "request.setPhase", json!({ "name": name, "phase": "Held" }))); } Ok(s.keep(h)) } fn run_0017(run: Run) -> Result { run_with(run, false, 0) } fn killed_0017(run: Run) -> Result { run_with(run, true, 100) } // killedlate: the owner is killed 2 s after the release was asked, while it is stopping its kcp after Released (the release // client then sees the owner go): the invocation it leaves must still end with the summary Released, kcpRunning false fn killedlate_0017(run: Run) -> Result { run_with(run, true, 2000) } // devsup: the restart race of a real devsup, no VM (docs/acceptance/0017): the host runs `dev up --profile quick` in an instance of // its own, then 12 times (delays 0.5, 1.0, 2.0 s) `dev down` SIGKILLed mid-stop followed at once by `dev up`: every up must succeed // and its kcp be ready. Without the devsup `stopping` handling about half of them fail (a unit "deactivating", or a kcp whose kine // was stopped under it). fn devsup_0017(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); let mut h = Handle::new(); match run.event { Event::Start => { h = h .call(host("race", "devsup.race", json!({ "delaysMs": [500, 1000, 2000], "perDelay": 4 }))) .timer("deadline", 25 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail("the devsup race did not finish in 25 min")), Event::Result { id, result, .. } if id == "race" => { let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; if let Some(why) = o["skipped"].as_str() { return Ok(s.unmet(why)); } s.reading("iterations", o["iterations"].clone()); s.reading("ok", o["ok"].clone()); s.reading("bad", o["bad"].clone()); s.reading("binary", o["binary"].clone()); if o["bad"].as_u64().unwrap_or(1) != 0 { let bad: Vec<&Value> = o["iterations"].as_array().into_iter().flatten().filter(|i| i["ok"] != true).collect(); return Ok(s.fail(format!("{} of {} restarts after a SIGKILLed dev down failed: {}", o["bad"], o["ok"].as_u64().unwrap_or(0) + o["bad"].as_u64().unwrap_or(0), json!(bad)))); } return Ok(s.pass()); } _ => {} } Ok(s.keep(h)) } // samename (plan 0046 step 6, plan 0044 finding 7): two instances of one name in two state dirs, no VM: both kcps ready on two // URLs, and a down of the first leaves the second ready. fn samename_0017(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); let mut h = Handle::new(); match run.event { Event::Start => h = h.call(host("same", "devsup.samename", json!({}))).timer("deadline", 10 * MIN), Event::Timer { id } if id == "deadline" => return Ok(s.fail("the two instances did not come up and down in 10 min")), Event::Result { id, result, .. } if id == "same" => { let o = match steps::out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; if let Some(why) = o["skipped"].as_str() { return Ok(s.unmet(why)); } s.reading("binary", o["binary"].clone()); if o["ok"] != true { return Ok(s.fail(format!("{}: {}", o["why"].as_str().unwrap_or_default(), o["log"].as_str().unwrap_or_default()))); } s.reading("kcps", json!([o["a"], o["b"]])); return Ok(s.pass()); } _ => {} } Ok(s.keep(h)) } entries! { run => run_0017, killed => killed_0017, killedlate => killedlate_0017, half => half_0017, devsup => devsup_0017, samename => samename_0017 }