0
0
// docs/acceptance/0005-guest-requester-association.md. run (A): an outer VM with a guest requester R2 (`run request r2`),
// R2 Held on its nested VM, the renter's contract-scoped grant + endorsement, R2's associationRequest, the walk; the
// outer-only release cascades (r2Released), records gone, R2 tombstoned. b (B): R2 with a co-located account mesh1
// (peer records, R2's grant, agent-of R2, the walk), then the out-of-band loss: the outer ComputeVM deleted on its
// bidder's kcp (only this run's receipt), the renter tombstones both DIDs; without that kcp, a release. A winner whose
// kcp is read only (`write` false) keeps its ComputeVM: the VM is read there, then a release ends it.
use abi::{Call, Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{ASSOC, ENDORSE, MIN, S, get_record, host, kube, out};
fn walk(id: &str, agent: &str) -> Call {
Call::new(id, "com.fedfork.exec.identlink.walk").input(json!({ "agent": agent }))
}
fn plc(id: &str, did: &str) -> Call {
Call::new(id, "com.fedfork.exec.identlink.plc").input(json!({ "did": did }))
}
/// A contract as its accept's URI: the associationRequest names it by URI, the grant by strongRef.
fn uri_of(c: &Value) -> &Value {
if c.is_string() { c } else { &c["uri"] }
}
fn run_a(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state.clone(), args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let account = args["did"].as_str().ok_or("did")?.to_string();
let renter = s.st["renter"].as_str().unwrap_or_default().to_string();
let mut h = Handle::new();
match &run.event {
Event::Start => {
s.set("held");
let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--agent"]);
h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 60 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
// the chain reads, again every 10 s for up to 3 min (R2 writes its side seconds after it reports)
Event::Timer { id } if id == "chain" => {
let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
s.st["got"] = json!({});
s.st["fan"] = json!(5);
h = h
.call(get_record("grant", &format!("at://{renter}/{ASSOC}/{r2}")))
.call(get_record("endorse", &format!("at://{renter}/{ENDORSE}/{r2}")))
.call(get_record("areq", &format!("at://{r2}/com.publicdomainrelay.temp.market.associationRequest/{renter}")))
.call(plc("plc", &r2))
.call(walk("walk", &r2));
}
Event::Timer { id } if id == "poll" => {
h = h.call(host("held2", "request.ssh", json!({ "name": name, "exec": "journalctl -u socialweb-r2.service -o cat --no-pager | grep -c '\"phase\":\"Held\"' || true" })));
}
Event::Message { id, data, .. } if id == "req" => {
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
"exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
"contract" if data["phase"] == "Held" => s.flag("held"),
"exited" => s.flag("exited"),
"requester_association_written" => {
s.st["r2"] = data["subject"].clone();
s.st["renter"] = data["did"].clone();
}
"requester_guest_requester_released" => s.flag("r2released"),
"requester_association_deleted" => s.flag("assocDeleted"),
_ => {}
}
if s.is("held") && s.has("held") {
s.set("r2");
h = h.call(host("r2", "request.r2", json!({ "name": name, "r2Name": "r2-st", "agent": true })));
}
if s.is("assoc") && s.st["r2"].is_string() {
s.set("r2held");
s.mark("r2");
h = h.timer("poll", 10_000);
}
if s.is("released") && s.has("r2released") && s.has("assocDeleted") {
s.set("records");
h = h.call(host("records", "request.records", json!({ "name": name })));
}
}
Event::Result { id, result, .. } => {
let r = result.output();
let o = match out(&s, id, r.clone()) {
Err(_) if ["grant", "endorse", "areq", "plc", "walk"].contains(&id.as_str()) => json!({ "error": r.unwrap_err() }),
Ok(o) => o,
Err(f) => return Ok(f),
};
let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
match id.as_str() {
"r2" => {
s.set("assoc");
if s.st["r2"].is_string() {
s.set("r2held");
s.mark("r2");
h = h.timer("poll", 10_000);
}
}
"held2" => {
let n: u64 = o["output"].as_str().unwrap_or_default().lines().filter_map(|l| l.trim().parse().ok()).next().unwrap_or(0);
if n > 0 {
let t = s.t();
s.reading("r2_held_s", t);
s.set("chain");
s.mark("chain");
h = h.timer("chain", 1);
} else if s.since("r2") > 600.0 {
return Ok(s.fail("R2 not Held in 10 min"));
} else {
h = h.timer("poll", 10_000);
}
}
"grant" | "endorse" | "areq" | "plc" | "walk" => {
s.st["got"][id.as_str()] = o;
let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
s.st["fan"] = json!(left);
if left == 0 {
let g = s.st["got"].clone();
let mut p = vec![];
if g["grant"]["value"]["agent"] != r2.as_str() || !g["grant"]["value"]["expiresAt"].is_string() {
p.push(format!("renter grant {}", g["grant"]));
}
if g["endorse"]["value"]["agent"] != r2.as_str() {
p.push(format!("renter endorsement {}", g["endorse"]));
}
if g["areq"]["value"]["principal"] != renter.as_str() || uri_of(&g["areq"]["value"]["contract"]) != uri_of(&g["grant"]["value"]["contract"]) {
p.push(format!("associationRequest {}", g["areq"]));
}
if !g["plc"]["alsoKnownAs"].as_array().is_some_and(|a| a.contains(&json!(format!("https://ident.link/agent-of/{renter}")))) {
p.push(format!("R2 alsoKnownAs {}", g["plc"]["alsoKnownAs"]));
}
if g["walk"]["roots"] != json!([account]) {
p.push(format!("walk {}", g["walk"]));
}
if !p.is_empty() {
if s.since("chain") < 180.0 {
return Ok(s.keep(h.timer("chain", 10_000)));
}
return Ok(s.fail(format!("after 3 min: {}", p.join("; "))));
}
s.reading("chain_s", s.since("chain"));
s.set("released");
h = h.call(host("release", "request.release", json!({ "name": name })));
}
}
"release" => {
s.reading("release_s", o["seconds"].clone());
// the log may have reported both before the release returned: then the records are asked already
if s.is("released") && s.has("r2released") && s.has("assocDeleted") {
s.set("records");
h = h.call(host("records", "request.records", json!({ "name": name })));
}
}
"records" => {
let mut uris = o["uris"].as_array().cloned().unwrap_or_default();
uris.push(json!(format!("at://{renter}/{ENDORSE}/{r2}")));
uris.push(json!(format!("at://{renter}/{ASSOC}/{r2}")));
s.reading("records_checked", uris.len());
h = h.call(host("gone", "assert.recordsGone", json!({ "uris": uris })));
}
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
}
h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [r2] })));
}
"tomb" => {
if o["alive"].as_array().is_none_or(|x| !x.is_empty()) {
return Ok(s.fail(format!("not tombstoned: {}", o["alive"])));
}
return Ok(s.pass());
}
_ => {}
}
}
Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.clone().unwrap_or_default()))),
_ => {}
}
Ok(s.keep(h))
}
/// The --hold process exits when its ssh session drops (rc165: 255 some 55 s after R2 came up): then nothing of the
/// renter runs to settle the out-of-band loss, and a release (its kcp started again from the state dir) does.
fn settle(s: &mut S, name: &str) -> Option<Call> {
if !(s.is("lost") && s.has("deleted") && s.has("exited") && !s.has("renterTombstoned") && !s.has("settling")) {
return None;
}
s.flag("settling");
s.reading("settled_by", "run request release: the --hold process exited when its ssh dropped");
Some(host("release", "request.release", json!({ "name": name })))
}
/// Both hosts empty (the runbook's pass criterion): R2's own rentals, by `spec.requester.did`, on every bidder the target
/// names a kcp for, gone within R2_VMS_WINDOW_S: after a release the cascade releases them; after the out-of-band loss
/// their bidder ends them (`association_ended`: the outer contract's vm.terminated is there). Past the window the test
/// deletes what is left on a bidder it may write to (the runbook's cleanup) and fails; on one it may not, it only watches
/// 3 min more (`bidder_late` or `left`). None when no bidder kcp is named.
const R2_VMS_WINDOW_S: f64 = 360.0;
fn r2_vms(s: &mut S, mut h: Handle, bidders: &[Value]) -> Option<Handle> {
let with: Vec<(usize, String)> = bidders.iter().enumerate().filter_map(|(j, b)| b["cluster"].as_str().map(|c| (j, c.to_string()))).collect();
if with.is_empty() {
s.reading("r2_vms_checked", false);
return None;
}
s.st["wait"] = json!(with.len());
s.st["left"] = json!([]);
for (j, c) in &with {
h = h.call(kube(&format!("nv:{j}"), "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
}
Some(h)
}
fn run_b(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let account = args["did"].as_str().ok_or("did")?.to_string();
let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
let mut h = Handle::new();
match run.event {
Event::Start => {
s.set("held");
// the runbook's outer request (part A's): with --agent R2 gets no atirc token of its own, whose mint answers 403
// "association not verified" right after the grant and holds R2's association ref (rc165)
let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--agent"]);
h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 60 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
Event::Timer { id } if id == "contract" => {
let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default());
s.reading("winner", w.clone());
if !bidders.iter().any(|b| b["did"] == w) {
return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
}
match bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) {
Some(c) => {
s.st["cluster"] = json!(c);
s.set("lost");
s.reading("out_of_band", true);
h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
}
None => return Ok(s.unmet("the target names no kcp for the winner: the VM cannot be deleted out of band")),
}
}
Event::Timer { id } if id == "m-again" => {
let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
s.st["got"] = json!({});
h = h.call(plc("plc-r2", &r2));
}
Event::Timer { id } if id == "poll" => {
let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
h = h.call(Call::new("list", "com.atproto.repo.listRecords").params(json!({ "repo": r2, "collection": ENDORSE })).service(r2.clone()));
}
Event::Message { id, data, .. } if id == "req" => {
if s.st["receipt"].is_null() && data["receipt"].is_string() {
s.st["receipt"] = data["receipt"].clone();
}
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
"exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
"contract" if data["phase"] == "Held" => s.flag("held"),
"exited" => s.flag("exited"),
"requester_association_written" => {
s.st["r2"] = data["subject"].clone();
s.st["renter"] = data["did"].clone();
}
"requester_tombstoned" => s.flag("renterTombstoned"),
"requester_contract_records_deleted" => s.flag("recordsDeleted"),
_ => {}
}
if s.is("held") && s.has("held") {
s.set("start-r2");
// the runbook's start, exactly: never `install -d /run/socialweb-r2` (it resets the mode the guest agent set,
// and R2 cannot write requester-did)
// and the guest agent makes the dir once the bundle is in, seconds after Held: wait for it
let cmd = "for i in $(seq 120); do [ -d /run/socialweb-r2 ] && break; sleep 1; done; printf 'R2_NAME=r2\\nR2_HOLD=1\\nR2_COLOCATED=mesh1\\n' > /run/socialweb-r2/r2-start.env && chown root:socialweb-r2 /run/socialweb-r2/r2-start.env && chmod 0640 /run/socialweb-r2/r2-start.env && systemctl start --no-block socialweb-r2.service";
h = h.call(host("r2", "request.ssh", json!({ "name": name, "exec": cmd })));
}
if s.is("assoc") && s.st["r2"].is_string() {
s.set("mesh1");
s.mark("r2");
h = h.timer("poll", 10_000);
}
if s.is("lost") && s.has("renterTombstoned") && s.has("recordsDeleted") {
s.set("tomb");
h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"], s.st["mesh1"]] })));
} else if let Some(h2) = settle(&mut s, &name) {
h = h.call(h2);
}
}
Event::Result { id, .. } if id.starts_with("nd:") => {}
Event::Result { id, result, .. } => {
let r = result.output();
let o = match out(&s, &id, r.clone()) {
Err(_) if ["peer", "g2", "plc-m", "walk"].contains(&id.as_str()) => json!({ "error": r.unwrap_err() }),
Ok(o) => o,
Err(f) => return Ok(f),
};
let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
match id.as_str() {
"r2" => {
if o["ok"] != true {
return Ok(s.fail(format!("starting R2: {}", o["output"].as_str().unwrap_or_default().lines().rev().take(3).collect::<Vec<_>>().join(" | "))));
}
s.set("assoc");
if s.st["r2"].is_string() {
s.set("mesh1");
s.mark("r2");
h = h.timer("poll", 10_000);
}
}
"list" => match o["records"].as_array().and_then(|r| r.first()).and_then(|r| r["value"]["agent"].as_str()) {
Some(m) => {
s.st["mesh1"] = json!(m);
s.reading("mesh1", m);
h = h.call(plc("plc-r2", &r2));
}
None if s.since("r2") > 300.0 => return Ok(s.fail("no endorsement of mesh1 in R2's repo in 5 min")),
None => h = h.timer("poll", 10_000),
},
"plc-r2" => {
if s.st["t_m"].is_null() {
s.mark("m");
s.st["t_m"] = json!(true);
}
let r2h = o["alsoKnownAs"].as_array().into_iter().flatten().find_map(|a| a.as_str()?.strip_prefix("at://").map(String::from)).unwrap_or_default();
let m = s.st["mesh1"].as_str().unwrap_or_default().to_string();
s.st["fan"] = json!(4);
h = h
.call(get_record("peer", &format!("at://{m}/computer.socialweb.peer/mesh1.{r2h}")))
.call(get_record("g2", &format!("at://{r2}/{ASSOC}/{m}")))
.call(plc("plc-m", &m))
.call(walk("walk", &m));
}
"peer" | "g2" | "plc-m" | "walk" => {
s.st["got"][id.as_str()] = o;
let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
s.st["fan"] = json!(left);
if left == 0 {
let g = s.st["got"].clone();
let mut p = vec![];
if !g["peer"]["value"]["iroh"].is_string() || g["peer"]["value"].get("lease").is_some() {
p.push(format!("mesh1 peer {}", g["peer"]));
}
if !g["plc-m"]["alsoKnownAs"].as_array().is_some_and(|a| a.contains(&json!(format!("https://ident.link/agent-of/{r2}")))) {
p.push(format!("mesh1 alsoKnownAs {}", g["plc-m"]["alsoKnownAs"]));
}
if g["walk"]["roots"] != json!([account]) || !g["walk"]["ancestors"].as_array().is_some_and(|a| a.iter().any(|x| x["did"] == r2.as_str() && x["depth"] == 1)) {
p.push(format!("walk {}", g["walk"]));
}
if !p.is_empty() {
if s.since("m") < 180.0 {
return Ok(s.keep(h.timer("m-again", 10_000)));
}
return Ok(s.fail(format!("after 3 min: {}", p.join("; "))));
}
s.reading("mesh1_chain_s", s.since("m"));
// the --hold process stays: the receipt is from its log, the winner its author
h = h.timer("contract", 1);
}
}
"vms" => {
let vm = o["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).and_then(|v| v["metadata"]["name"].as_str()).map(String::from);
let Some(vm) = vm else { return Ok(s.fail("the outer ComputeVM of this run's receipt is not on its bidder")) };
let c = s.st["cluster"].as_str().unwrap_or_default().to_string();
if bidders.iter().any(|b| b["cluster"] == c.as_str() && b["write"] == false) {
s.set("released");
s.reading("out_of_band", false);
h = h.call(host("release", "request.release", json!({ "name": name })));
} else {
h = h.call(kube("delvm", "delete", &c, steps::sw("ComputeVM", &vm)));
}
}
"delvm" => {
s.flag("deleted");
if let Some(c) = settle(&mut s, &name) {
h = h.call(c);
}
}
"release" => {
s.set("tomb");
h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"], s.st["mesh1"]] })));
}
"tomb" => {
if o["alive"].as_array().is_none_or(|x| !x.is_empty()) {
return Ok(s.fail(format!("not tombstoned: {}", o["alive"])));
}
h = h.call(host("records", "request.records", json!({ "name": name })));
}
"records" => h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))),
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left: {}", o["left"])));
}
s.set("hosts");
s.mark("hosts");
return Ok(match r2_vms(&mut s, h, &bidders) {
Some(h) => s.keep(h),
None => s.pass(),
});
}
nv if nv.starts_with("nv:") => {
let j: usize = nv[3..].parse().unwrap_or(0);
let c = bidders.get(j).and_then(|b| b["cluster"].as_str()).unwrap_or_default().to_string();
let writable = bidders.get(j).is_some_and(|b| b["write"] != false);
let late = s.has("late");
for v in o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["requester"]["did"] == r2.as_str()) {
let vm = v["metadata"]["name"].as_str().unwrap_or_default().to_string();
if let Some(reason) = v["spec"]["release"]["reason"].as_str() {
s.st["readings"]["r2_vm_release_reason"][&vm] = json!(reason);
}
s.st["left"].as_array_mut().map(|a| a.push(json!({ "vm": vm, "cluster": c, "phase": v["status"]["phase"],
"release": v["spec"]["release"], "deleting": !v["metadata"]["deletionTimestamp"].is_null() })));
if late && writable && s.st["asked"][&vm] != true && v["metadata"]["deletionTimestamp"].is_null() {
s.st["asked"][&vm] = json!(true);
s.flag("testDeleted");
h = h.call(kube(&format!("nd:{vm}"), "delete", &c, steps::sw("ComputeVM", &vm)));
}
}
let left = s.st["wait"].as_u64().unwrap_or(1) - 1;
s.st["wait"] = json!(left);
if left == 0 {
let vms = s.st["left"].clone();
let gone = vms.as_array().is_some_and(Vec::is_empty);
let t = s.since("hosts");
if gone {
s.reading("r2_vms_gone_s", t);
if !late {
s.reading("r2_vms_ended_by", if s.has("deleted") { "bidder" } else { "release" });
return Ok(s.pass());
}
let by = if s.has("testDeleted") { "test" } else { "bidder_late" };
s.reading("r2_vms_ended_by", by);
return Ok(s.fail(format!("R2's own rentals not ended in {R2_VMS_WINDOW_S} s ({by}, gone at {t:.0} s): {}", s.st["stuck"])));
}
if late && t > R2_VMS_WINDOW_S + 180.0 {
s.reading("r2_vms_ended_by", "left");
return Ok(s.fail(format!("R2's own rentals still on the bidders {t:.0} s after the release: {vms}")));
}
if !late && t > R2_VMS_WINDOW_S {
s.flag("late");
s.st["stuck"] = vms;
}
h = h.timer("hosts", 5_000);
}
}
_ => {}
}
}
Event::Timer { id } if id == "hosts" => {
if let Some(h2) = r2_vms(&mut s, h, &bidders) {
return Ok(s.keep(h2));
}
return Ok(s.pass());
}
Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
_ => {}
}
Ok(s.keep(h))
}
/// r2fail: an R2 that fails after it reported its DID (its unit killed) leaves no live DID at PLC once the outer
/// contract is released (the renter's tombstone fallback). KNOWN GAP for rc166: a failed attempt left
/// did:plc:7f2a7u2jjxft6fngc4yzmxyr live, so this fails on rc165.
fn run_r2fail(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let mut h = Handle::new();
match run.event {
Event::Start => {
s.set("held");
let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest"]);
h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 40 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 40 min (phase {})", s.phase()))),
Event::Timer { id } if id == "poll" => h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"]] }))),
Event::Message { id, data, .. } if id == "req" => {
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Held" => s.flag("held"),
"exited" => s.flag("exited"),
"requester_guest_requester_reported" if s.st["r2"].is_null() => s.st["r2"] = data["did"].clone(),
_ => {}
}
if s.is("held") && s.has("held") {
s.set("r2");
h = h.call(host("r2", "request.r2", json!({ "name": name, "r2Name": "r2-fail" })));
}
if s.is("reported") && s.st["r2"].is_string() {
s.set("kill");
s.reading("r2", s.st["r2"].clone());
h = h.call(host("kill", "request.ssh", json!({ "name": name, "exec": "systemctl kill --signal=KILL socialweb-r2.service; systemctl stop socialweb-r2.service; true" })));
}
}
Event::Result { id, result, .. } => {
let o = match out(&s, &id, result.output()) {
Ok(o) => o,
Err(f) => return Ok(f),
};
match id.as_str() {
"r2" => {
s.set("reported");
if s.st["r2"].is_string() {
s.set("kill");
s.reading("r2", s.st["r2"].clone());
h = h.call(host("kill", "request.ssh", json!({ "name": name, "exec": "systemctl kill --signal=KILL socialweb-r2.service; systemctl stop socialweb-r2.service; true" })));
}
}
"kill" => h = h.call(host("release", "request.release", json!({ "name": name }))),
"release" => {
s.mark("release");
h = h.timer("poll", 5_000);
}
"tomb" => {
if o["alive"].as_array().is_some_and(Vec::is_empty) {
return Ok(s.pass());
}
if s.since("release") > 300.0 {
return Ok(s.fail(format!("KNOWN GAP rc166: the failed R2 left {} live at PLC 5 min after the release", o["alive"])));
}
h = h.timer("poll", 10_000);
}
_ => {}
}
}
Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
_ => {}
}
Ok(s.keep(h))
}
entries! { run => run_a, b => run_b, r2fail => run_r2fail }