~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
542 lines · 29.5 KB Raw History
// docs/acceptance/0005-guest-requester-association.md. run (A): an outer VM with a guest requester R2 (`run request r2`),
// R2 Held on its nested VM, the renter's contract-scoped grant + endorsement, R2's associationRequest, the walk; the
// outer-only release cascades (r2Released), records gone, R2 tombstoned. b (B): R2 with a co-located account mesh1
// (peer records, R2's grant, agent-of R2, the walk), then the out-of-band loss: the outer ComputeVM deleted on its
// bidder's kcp (only this run's receipt), the renter tombstones both DIDs; without that kcp, a release. A winner whose
// kcp is read only (`write` false) keeps its ComputeVM: the VM is read there, then a release ends it.
use abi::{Call, Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{ASSOC, ENDORSE, MIN, S, get_record, host, kube, out};

fn walk(id: &str, agent: &str) -> Call {
    Call::new(id, "com.fedfork.exec.identlink.walk").input(json!({ "agent": agent }))
}

fn plc(id: &str, did: &str) -> Call {
    Call::new(id, "com.fedfork.exec.identlink.plc").input(json!({ "did": did }))
}

/// A contract as its accept's URI: the associationRequest names it by URI, the grant by strongRef.
fn uri_of(c: &Value) -> &Value {
    if c.is_string() { c } else { &c["uri"] }
}

fn run_a(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state.clone(), args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let account = args["did"].as_str().ok_or("did")?.to_string();
    let renter = s.st["renter"].as_str().unwrap_or_default().to_string();
    let mut h = Handle::new();
    match &run.event {
        Event::Start => {
            s.set("held");
            let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--agent"]);
            h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 60 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
        // the chain reads, again every 10 s for up to 3 min (R2 writes its side seconds after it reports)
        Event::Timer { id } if id == "chain" => {
            let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
            s.st["got"] = json!({});
            s.st["fan"] = json!(5);
            h = h
                .call(get_record("grant", &format!("at://{renter}/{ASSOC}/{r2}")))
                .call(get_record("endorse", &format!("at://{renter}/{ENDORSE}/{r2}")))
                .call(get_record("areq", &format!("at://{r2}/com.publicdomainrelay.temp.market.associationRequest/{renter}")))
                .call(plc("plc", &r2))
                .call(walk("walk", &r2));
        }
        Event::Timer { id } if id == "poll" => {
            h = h.call(host("held2", "request.ssh", json!({ "name": name, "exec": "journalctl -u socialweb-r2.service -o cat --no-pager | grep -c '\"phase\":\"Held\"' || true" })));
        }
        Event::Message { id, data, .. } if id == "req" => {
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
                "contract" if data["phase"] == "Held" => s.flag("held"),
                "exited" => s.flag("exited"),
                "requester_association_written" => {
                    s.st["r2"] = data["subject"].clone();
                    s.st["renter"] = data["did"].clone();
                }
                "requester_guest_requester_released" => s.flag("r2released"),
                "requester_association_deleted" => s.flag("assocDeleted"),
                _ => {}
            }
            if s.is("held") && s.has("held") {
                s.set("r2");
                h = h.call(host("r2", "request.r2", json!({ "name": name, "r2Name": "r2-st", "agent": true })));
            }
            if s.is("assoc") && s.st["r2"].is_string() {
                s.set("r2held");
                s.mark("r2");
                h = h.timer("poll", 10_000);
            }
            if s.is("released") && s.has("r2released") && s.has("assocDeleted") {
                s.set("records");
                h = h.call(host("records", "request.records", json!({ "name": name })));
            }
        }
        Event::Result { id, result, .. } => {
            let r = result.output();
            let o = match out(&s, id, r.clone()) {
                Err(_) if ["grant", "endorse", "areq", "plc", "walk"].contains(&id.as_str()) => json!({ "error": r.unwrap_err() }),
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
            match id.as_str() {
                "r2" => {
                    s.set("assoc");
                    if s.st["r2"].is_string() {
                        s.set("r2held");
                        s.mark("r2");
                        h = h.timer("poll", 10_000);
                    }
                }
                "held2" => {
                    let n: u64 = o["output"].as_str().unwrap_or_default().lines().filter_map(|l| l.trim().parse().ok()).next().unwrap_or(0);
                    if n > 0 {
                        let t = s.t();
                        s.reading("r2_held_s", t);
                        s.set("chain");
                        s.mark("chain");
                        h = h.timer("chain", 1);
                    } else if s.since("r2") > 600.0 {
                        return Ok(s.fail("R2 not Held in 10 min"));
                    } else {
                        h = h.timer("poll", 10_000);
                    }
                }
                "grant" | "endorse" | "areq" | "plc" | "walk" => {
                    s.st["got"][id.as_str()] = o;
                    let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
                    s.st["fan"] = json!(left);
                    if left == 0 {
                        let g = s.st["got"].clone();
                        let mut p = vec![];
                        if g["grant"]["value"]["agent"] != r2.as_str() || !g["grant"]["value"]["expiresAt"].is_string() {
                            p.push(format!("renter grant {}", g["grant"]));
                        }
                        if g["endorse"]["value"]["agent"] != r2.as_str() {
                            p.push(format!("renter endorsement {}", g["endorse"]));
                        }
                        if g["areq"]["value"]["principal"] != renter.as_str() || uri_of(&g["areq"]["value"]["contract"]) != uri_of(&g["grant"]["value"]["contract"]) {
                            p.push(format!("associationRequest {}", g["areq"]));
                        }
                        if !g["plc"]["alsoKnownAs"].as_array().is_some_and(|a| a.contains(&json!(format!("https://ident.link/agent-of/{renter}")))) {
                            p.push(format!("R2 alsoKnownAs {}", g["plc"]["alsoKnownAs"]));
                        }
                        if g["walk"]["roots"] != json!([account]) {
                            p.push(format!("walk {}", g["walk"]));
                        }
                        if !p.is_empty() {
                            if s.since("chain") < 180.0 {
                                return Ok(s.keep(h.timer("chain", 10_000)));
                            }
                            return Ok(s.fail(format!("after 3 min: {}", p.join("; "))));
                        }
                        s.reading("chain_s", s.since("chain"));
                        s.set("released");
                        h = h.call(host("release", "request.release", json!({ "name": name })));
                    }
                }
                "release" => {
                    s.reading("release_s", o["seconds"].clone());
                    // the log may have reported both before the release returned: then the records are asked already
                    if s.is("released") && s.has("r2released") && s.has("assocDeleted") {
                        s.set("records");
                        h = h.call(host("records", "request.records", json!({ "name": name })));
                    }
                }
                "records" => {
                    let mut uris = o["uris"].as_array().cloned().unwrap_or_default();
                    uris.push(json!(format!("at://{renter}/{ENDORSE}/{r2}")));
                    uris.push(json!(format!("at://{renter}/{ASSOC}/{r2}")));
                    s.reading("records_checked", uris.len());
                    h = h.call(host("gone", "assert.recordsGone", json!({ "uris": uris })));
                }
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                    }
                    h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [r2] })));
                }
                "tomb" => {
                    if o["alive"].as_array().is_none_or(|x| !x.is_empty()) {
                        return Ok(s.fail(format!("not tombstoned: {}", o["alive"])));
                    }
                    return Ok(s.pass());
                }
                _ => {}
            }
        }
        Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.clone().unwrap_or_default()))),
        _ => {}
    }
    Ok(s.keep(h))
}

/// The --hold process exits when its ssh session drops (rc165: 255 some 55 s after R2 came up): then nothing of the
/// renter runs to settle the out-of-band loss, and a release (its kcp started again from the state dir) does.
fn settle(s: &mut S, name: &str) -> Option<Call> {
    if !(s.is("lost") && s.has("deleted") && s.has("exited") && !s.has("renterTombstoned") && !s.has("settling")) {
        return None;
    }
    s.flag("settling");
    s.reading("settled_by", "run request release: the --hold process exited when its ssh dropped");
    Some(host("release", "request.release", json!({ "name": name })))
}

/// Both hosts empty (the runbook's pass criterion): R2's own rentals, by `spec.requester.did`, on every bidder the target
/// names a kcp for, gone within R2_VMS_WINDOW_S: after a release the cascade releases them; after the out-of-band loss
/// their bidder ends them (`association_ended`: the outer contract's vm.terminated is there). Past the window the test
/// deletes what is left on a bidder it may write to (the runbook's cleanup) and fails; on one it may not, it only watches
/// 3 min more (`bidder_late` or `left`). None when no bidder kcp is named.
const R2_VMS_WINDOW_S: f64 = 360.0;

fn r2_vms(s: &mut S, mut h: Handle, bidders: &[Value]) -> Option<Handle> {
    let with: Vec<(usize, String)> = bidders.iter().enumerate().filter_map(|(j, b)| b["cluster"].as_str().map(|c| (j, c.to_string()))).collect();
    if with.is_empty() {
        s.reading("r2_vms_checked", false);
        return None;
    }
    s.st["wait"] = json!(with.len());
    s.st["left"] = json!([]);
    for (j, c) in &with {
        h = h.call(kube(&format!("nv:{j}"), "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
    }
    Some(h)
}

fn run_b(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let account = args["did"].as_str().ok_or("did")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            s.set("held");
            // the runbook's outer request (part A's): with --agent R2 gets no atirc token of its own, whose mint answers 403
            // "association not verified" right after the grant and holds R2's association ref (rc165)
            let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--agent"]);
            h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 60 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "contract" => {
            let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default());
            s.reading("winner", w.clone());
            if !bidders.iter().any(|b| b["did"] == w) {
                return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target")));
            }
            match bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) {
                Some(c) => {
                    s.st["cluster"] = json!(c);
                    s.set("lost");
                    s.reading("out_of_band", true);
                    h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })));
                }
                None => return Ok(s.unmet("the target names no kcp for the winner: the VM cannot be deleted out of band")),
            }
        }
        Event::Timer { id } if id == "m-again" => {
            let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
            s.st["got"] = json!({});
            h = h.call(plc("plc-r2", &r2));
        }
        Event::Timer { id } if id == "poll" => {
            let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
            h = h.call(Call::new("list", "com.atproto.repo.listRecords").params(json!({ "repo": r2, "collection": ENDORSE })).service(r2.clone()));
        }
        Event::Message { id, data, .. } if id == "req" => {
            if s.st["receipt"].is_null() && data["receipt"].is_string() {
                s.st["receipt"] = data["receipt"].clone();
            }
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
                "contract" if data["phase"] == "Held" => s.flag("held"),
                "exited" => s.flag("exited"),
                "requester_association_written" => {
                    s.st["r2"] = data["subject"].clone();
                    s.st["renter"] = data["did"].clone();
                }
                "requester_tombstoned" => s.flag("renterTombstoned"),
                "requester_contract_records_deleted" => s.flag("recordsDeleted"),
                _ => {}
            }
            if s.is("held") && s.has("held") {
                s.set("start-r2");
                // the runbook's start, exactly: never `install -d /run/socialweb-r2` (it resets the mode the guest agent set,
                // and R2 cannot write requester-did)
                // and the guest agent makes the dir once the bundle is in, seconds after Held: wait for it
                let cmd = "for i in $(seq 120); do [ -d /run/socialweb-r2 ] && break; sleep 1; done; printf 'R2_NAME=r2\\nR2_HOLD=1\\nR2_COLOCATED=mesh1\\n' > /run/socialweb-r2/r2-start.env && chown root:socialweb-r2 /run/socialweb-r2/r2-start.env && chmod 0640 /run/socialweb-r2/r2-start.env && systemctl start --no-block socialweb-r2.service";
                h = h.call(host("r2", "request.ssh", json!({ "name": name, "exec": cmd })));
            }
            if s.is("assoc") && s.st["r2"].is_string() {
                s.set("mesh1");
                s.mark("r2");
                h = h.timer("poll", 10_000);
            }
            if s.is("lost") && s.has("renterTombstoned") && s.has("recordsDeleted") {
                s.set("tomb");
                h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"], s.st["mesh1"]] })));
            } else if let Some(h2) = settle(&mut s, &name) {
                h = h.call(h2);
            }
        }
        Event::Result { id, .. } if id.starts_with("nd:") => {}
        Event::Result { id, result, .. } => {
            let r = result.output();
            let o = match out(&s, &id, r.clone()) {
                Err(_) if ["peer", "g2", "plc-m", "walk"].contains(&id.as_str()) => json!({ "error": r.unwrap_err() }),
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            let r2 = s.st["r2"].as_str().unwrap_or_default().to_string();
            match id.as_str() {
                "r2" => {
                    if o["ok"] != true {
                        return Ok(s.fail(format!("starting R2: {}", o["output"].as_str().unwrap_or_default().lines().rev().take(3).collect::<Vec<_>>().join(" | "))));
                    }
                    s.set("assoc");
                    if s.st["r2"].is_string() {
                        s.set("mesh1");
                        s.mark("r2");
                        h = h.timer("poll", 10_000);
                    }
                }
                "list" => match o["records"].as_array().and_then(|r| r.first()).and_then(|r| r["value"]["agent"].as_str()) {
                    Some(m) => {
                        s.st["mesh1"] = json!(m);
                        s.reading("mesh1", m);
                        h = h.call(plc("plc-r2", &r2));
                    }
                    None if s.since("r2") > 300.0 => return Ok(s.fail("no endorsement of mesh1 in R2's repo in 5 min")),
                    None => h = h.timer("poll", 10_000),
                },
                "plc-r2" => {
                    if s.st["t_m"].is_null() {
                        s.mark("m");
                        s.st["t_m"] = json!(true);
                    }
                    let r2h = o["alsoKnownAs"].as_array().into_iter().flatten().find_map(|a| a.as_str()?.strip_prefix("at://").map(String::from)).unwrap_or_default();
                    let m = s.st["mesh1"].as_str().unwrap_or_default().to_string();
                    s.st["fan"] = json!(4);
                    h = h
                        .call(get_record("peer", &format!("at://{m}/computer.socialweb.peer/mesh1.{r2h}")))
                        .call(get_record("g2", &format!("at://{r2}/{ASSOC}/{m}")))
                        .call(plc("plc-m", &m))
                        .call(walk("walk", &m));
                }
                "peer" | "g2" | "plc-m" | "walk" => {
                    s.st["got"][id.as_str()] = o;
                    let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
                    s.st["fan"] = json!(left);
                    if left == 0 {
                        let g = s.st["got"].clone();
                        let mut p = vec![];
                        if !g["peer"]["value"]["iroh"].is_string() || g["peer"]["value"].get("lease").is_some() {
                            p.push(format!("mesh1 peer {}", g["peer"]));
                        }
                        if !g["plc-m"]["alsoKnownAs"].as_array().is_some_and(|a| a.contains(&json!(format!("https://ident.link/agent-of/{r2}")))) {
                            p.push(format!("mesh1 alsoKnownAs {}", g["plc-m"]["alsoKnownAs"]));
                        }
                        if g["walk"]["roots"] != json!([account]) || !g["walk"]["ancestors"].as_array().is_some_and(|a| a.iter().any(|x| x["did"] == r2.as_str() && x["depth"] == 1)) {
                            p.push(format!("walk {}", g["walk"]));
                        }
                        if !p.is_empty() {
                            if s.since("m") < 180.0 {
                                return Ok(s.keep(h.timer("m-again", 10_000)));
                            }
                            return Ok(s.fail(format!("after 3 min: {}", p.join("; "))));
                        }
                        s.reading("mesh1_chain_s", s.since("m"));
                        // the --hold process stays: the receipt is from its log, the winner its author
                        h = h.timer("contract", 1);
                    }
                }
                "vms" => {
                    let vm = o["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).and_then(|v| v["metadata"]["name"].as_str()).map(String::from);
                    let Some(vm) = vm else { return Ok(s.fail("the outer ComputeVM of this run's receipt is not on its bidder")) };
                    let c = s.st["cluster"].as_str().unwrap_or_default().to_string();
                    if bidders.iter().any(|b| b["cluster"] == c.as_str() && b["write"] == false) {
                        s.set("released");
                        s.reading("out_of_band", false);
                        h = h.call(host("release", "request.release", json!({ "name": name })));
                    } else {
                        h = h.call(kube("delvm", "delete", &c, steps::sw("ComputeVM", &vm)));
                    }
                }
                "delvm" => {
                    s.flag("deleted");
                    if let Some(c) = settle(&mut s, &name) {
                        h = h.call(c);
                    }
                }
                "release" => {
                    s.set("tomb");
                    h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"], s.st["mesh1"]] })));
                }
                "tomb" => {
                    if o["alive"].as_array().is_none_or(|x| !x.is_empty()) {
                        return Ok(s.fail(format!("not tombstoned: {}", o["alive"])));
                    }
                    h = h.call(host("records", "request.records", json!({ "name": name })));
                }
                "records" => h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))),
                "gone" => {
                    if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                        return Ok(s.fail(format!("records left: {}", o["left"])));
                    }
                    s.set("hosts");
                    s.mark("hosts");
                    return Ok(match r2_vms(&mut s, h, &bidders) {
                        Some(h) => s.keep(h),
                        None => s.pass(),
                    });
                }
                nv if nv.starts_with("nv:") => {
                    let j: usize = nv[3..].parse().unwrap_or(0);
                    let c = bidders.get(j).and_then(|b| b["cluster"].as_str()).unwrap_or_default().to_string();
                    let writable = bidders.get(j).is_some_and(|b| b["write"] != false);
                    let late = s.has("late");
                    for v in o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["requester"]["did"] == r2.as_str()) {
                        let vm = v["metadata"]["name"].as_str().unwrap_or_default().to_string();
                        if let Some(reason) = v["spec"]["release"]["reason"].as_str() {
                            s.st["readings"]["r2_vm_release_reason"][&vm] = json!(reason);
                        }
                        s.st["left"].as_array_mut().map(|a| a.push(json!({ "vm": vm, "cluster": c, "phase": v["status"]["phase"],
                            "release": v["spec"]["release"], "deleting": !v["metadata"]["deletionTimestamp"].is_null() })));
                        if late && writable && s.st["asked"][&vm] != true && v["metadata"]["deletionTimestamp"].is_null() {
                            s.st["asked"][&vm] = json!(true);
                            s.flag("testDeleted");
                            h = h.call(kube(&format!("nd:{vm}"), "delete", &c, steps::sw("ComputeVM", &vm)));
                        }
                    }
                    let left = s.st["wait"].as_u64().unwrap_or(1) - 1;
                    s.st["wait"] = json!(left);
                    if left == 0 {
                        let vms = s.st["left"].clone();
                        let gone = vms.as_array().is_some_and(Vec::is_empty);
                        let t = s.since("hosts");
                        if gone {
                            s.reading("r2_vms_gone_s", t);
                            if !late {
                                s.reading("r2_vms_ended_by", if s.has("deleted") { "bidder" } else { "release" });
                                return Ok(s.pass());
                            }
                            let by = if s.has("testDeleted") { "test" } else { "bidder_late" };
                            s.reading("r2_vms_ended_by", by);
                            return Ok(s.fail(format!("R2's own rentals not ended in {R2_VMS_WINDOW_S} s ({by}, gone at {t:.0} s): {}", s.st["stuck"])));
                        }
                        if late && t > R2_VMS_WINDOW_S + 180.0 {
                            s.reading("r2_vms_ended_by", "left");
                            return Ok(s.fail(format!("R2's own rentals still on the bidders {t:.0} s after the release: {vms}")));
                        }
                        if !late && t > R2_VMS_WINDOW_S {
                            s.flag("late");
                            s.st["stuck"] = vms;
                        }
                        h = h.timer("hosts", 5_000);
                    }
                }
                _ => {}
            }
        }
        Event::Timer { id } if id == "hosts" => {
            if let Some(h2) = r2_vms(&mut s, h, &bidders) {
                return Ok(s.keep(h2));
            }
            return Ok(s.pass());
        }
        Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
        _ => {}
    }
    Ok(s.keep(h))
}


/// r2fail: an R2 that fails after it reported its DID (its unit killed) leaves no live DID at PLC once the outer
/// contract is released (the renter's tombstone fallback). KNOWN GAP for rc166: a failed attempt left
/// did:plc:7f2a7u2jjxft6fngc4yzmxyr live, so this fails on rc165.
fn run_r2fail(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            s.set("held");
            let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest"]);
            h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 40 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 40 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "poll" => h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"]] }))),
        Event::Message { id, data, .. } if id == "req" => {
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Held" => s.flag("held"),
                "exited" => s.flag("exited"),
                "requester_guest_requester_reported" if s.st["r2"].is_null() => s.st["r2"] = data["did"].clone(),
                _ => {}
            }
            if s.is("held") && s.has("held") {
                s.set("r2");
                h = h.call(host("r2", "request.r2", json!({ "name": name, "r2Name": "r2-fail" })));
            }
            if s.is("reported") && s.st["r2"].is_string() {
                s.set("kill");
                s.reading("r2", s.st["r2"].clone());
                h = h.call(host("kill", "request.ssh", json!({ "name": name, "exec": "systemctl kill --signal=KILL socialweb-r2.service; systemctl stop socialweb-r2.service; true" })));
            }
        }
        Event::Result { id, result, .. } => {
            let o = match out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "r2" => {
                    s.set("reported");
                    if s.st["r2"].is_string() {
                        s.set("kill");
                        s.reading("r2", s.st["r2"].clone());
                        h = h.call(host("kill", "request.ssh", json!({ "name": name, "exec": "systemctl kill --signal=KILL socialweb-r2.service; systemctl stop socialweb-r2.service; true" })));
                    }
                }
                "kill" => h = h.call(host("release", "request.release", json!({ "name": name }))),
                "release" => {
                    s.mark("release");
                    h = h.timer("poll", 5_000);
                }
                "tomb" => {
                    if o["alive"].as_array().is_some_and(Vec::is_empty) {
                        return Ok(s.pass());
                    }
                    if s.since("release") > 300.0 {
                        return Ok(s.fail(format!("KNOWN GAP rc166: the failed R2 left {} live at PLC 5 min after the release", o["alive"])));
                    }
                    h = h.timer("poll", 10_000);
                }
                _ => {}
            }
        }
        Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
        _ => {}
    }
    Ok(s.keep(h))
}

entries! { run => run_a, b => run_b, r2fail => run_r2fail }