// docs/acceptance/0005-guest-requester-association.md. run (A): an outer VM with a guest requester R2 (`run request r2`), // R2 Held on its nested VM, the renter's contract-scoped grant + endorsement, R2's associationRequest, the walk; the // outer-only release cascades (r2Released), records gone, R2 tombstoned. b (B): R2 with a co-located account mesh1 // (peer records, R2's grant, agent-of R2, the walk), then the out-of-band loss: the outer ComputeVM deleted on its // bidder's kcp (only this run's receipt), the renter tombstones both DIDs; without that kcp, a release. A winner whose // kcp is read only (`write` false) keeps its ComputeVM: the VM is read there, then a release ends it. use abi::{Call, Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{ASSOC, ENDORSE, MIN, S, get_record, host, kube, out}; fn walk(id: &str, agent: &str) -> Call { Call::new(id, "com.fedfork.exec.identlink.walk").input(json!({ "agent": agent })) } fn plc(id: &str, did: &str) -> Call { Call::new(id, "com.fedfork.exec.identlink.plc").input(json!({ "did": did })) } /// A contract as its accept's URI: the associationRequest names it by URI, the grant by strongRef. fn uri_of(c: &Value) -> &Value { if c.is_string() { c } else { &c["uri"] } } fn run_a(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state.clone(), args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let account = args["did"].as_str().ok_or("did")?.to_string(); let renter = s.st["renter"].as_str().unwrap_or_default().to_string(); let mut h = Handle::new(); match &run.event { Event::Start => { s.set("held"); let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--agent"]); h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 60 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))), // the chain reads, again every 10 s for up to 3 min (R2 writes its side seconds after it reports) Event::Timer { id } if id == "chain" => { let r2 = s.st["r2"].as_str().unwrap_or_default().to_string(); s.st["got"] = json!({}); s.st["fan"] = json!(5); h = h .call(get_record("grant", &format!("at://{renter}/{ASSOC}/{r2}"))) .call(get_record("endorse", &format!("at://{renter}/{ENDORSE}/{r2}"))) .call(get_record("areq", &format!("at://{r2}/com.publicdomainrelay.temp.market.associationRequest/{renter}"))) .call(plc("plc", &r2)) .call(walk("walk", &r2)); } Event::Timer { id } if id == "poll" => { h = h.call(host("held2", "request.ssh", json!({ "name": name, "exec": "journalctl -u socialweb-r2.service -o cat --no-pager | grep -c '\"phase\":\"Held\"' || true" }))); } Event::Message { id, data, .. } if id == "req" => { match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")), "contract" if data["phase"] == "Held" => s.flag("held"), "exited" => s.flag("exited"), "requester_association_written" => { s.st["r2"] = data["subject"].clone(); s.st["renter"] = data["did"].clone(); } "requester_guest_requester_released" => s.flag("r2released"), "requester_association_deleted" => s.flag("assocDeleted"), _ => {} } if s.is("held") && s.has("held") { s.set("r2"); h = h.call(host("r2", "request.r2", json!({ "name": name, "r2Name": "r2-st", "agent": true }))); } if s.is("assoc") && s.st["r2"].is_string() { s.set("r2held"); s.mark("r2"); h = h.timer("poll", 10_000); } if s.is("released") && s.has("r2released") && s.has("assocDeleted") { s.set("records"); h = h.call(host("records", "request.records", json!({ "name": name }))); } } Event::Result { id, result, .. } => { let r = result.output(); let o = match out(&s, id, r.clone()) { Err(_) if ["grant", "endorse", "areq", "plc", "walk"].contains(&id.as_str()) => json!({ "error": r.unwrap_err() }), Ok(o) => o, Err(f) => return Ok(f), }; let r2 = s.st["r2"].as_str().unwrap_or_default().to_string(); match id.as_str() { "r2" => { s.set("assoc"); if s.st["r2"].is_string() { s.set("r2held"); s.mark("r2"); h = h.timer("poll", 10_000); } } "held2" => { let n: u64 = o["output"].as_str().unwrap_or_default().lines().filter_map(|l| l.trim().parse().ok()).next().unwrap_or(0); if n > 0 { let t = s.t(); s.reading("r2_held_s", t); s.set("chain"); s.mark("chain"); h = h.timer("chain", 1); } else if s.since("r2") > 600.0 { return Ok(s.fail("R2 not Held in 10 min")); } else { h = h.timer("poll", 10_000); } } "grant" | "endorse" | "areq" | "plc" | "walk" => { s.st["got"][id.as_str()] = o; let left = s.st["fan"].as_u64().unwrap_or(1) - 1; s.st["fan"] = json!(left); if left == 0 { let g = s.st["got"].clone(); let mut p = vec![]; if g["grant"]["value"]["agent"] != r2.as_str() || !g["grant"]["value"]["expiresAt"].is_string() { p.push(format!("renter grant {}", g["grant"])); } if g["endorse"]["value"]["agent"] != r2.as_str() { p.push(format!("renter endorsement {}", g["endorse"])); } if g["areq"]["value"]["principal"] != renter.as_str() || uri_of(&g["areq"]["value"]["contract"]) != uri_of(&g["grant"]["value"]["contract"]) { p.push(format!("associationRequest {}", g["areq"])); } if !g["plc"]["alsoKnownAs"].as_array().is_some_and(|a| a.contains(&json!(format!("https://ident.link/agent-of/{renter}")))) { p.push(format!("R2 alsoKnownAs {}", g["plc"]["alsoKnownAs"])); } if g["walk"]["roots"] != json!([account]) { p.push(format!("walk {}", g["walk"])); } if !p.is_empty() { if s.since("chain") < 180.0 { return Ok(s.keep(h.timer("chain", 10_000))); } return Ok(s.fail(format!("after 3 min: {}", p.join("; ")))); } s.reading("chain_s", s.since("chain")); s.set("released"); h = h.call(host("release", "request.release", json!({ "name": name }))); } } "release" => { s.reading("release_s", o["seconds"].clone()); // the log may have reported both before the release returned: then the records are asked already if s.is("released") && s.has("r2released") && s.has("assocDeleted") { s.set("records"); h = h.call(host("records", "request.records", json!({ "name": name }))); } } "records" => { let mut uris = o["uris"].as_array().cloned().unwrap_or_default(); uris.push(json!(format!("at://{renter}/{ENDORSE}/{r2}"))); uris.push(json!(format!("at://{renter}/{ASSOC}/{r2}"))); s.reading("records_checked", uris.len()); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": uris }))); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [r2] }))); } "tomb" => { if o["alive"].as_array().is_none_or(|x| !x.is_empty()) { return Ok(s.fail(format!("not tombstoned: {}", o["alive"]))); } return Ok(s.pass()); } _ => {} } } Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.clone().unwrap_or_default()))), _ => {} } Ok(s.keep(h)) } /// The --hold process exits when its ssh session drops (rc165: 255 some 55 s after R2 came up): then nothing of the /// renter runs to settle the out-of-band loss, and a release (its kcp started again from the state dir) does. fn settle(s: &mut S, name: &str) -> Option { if !(s.is("lost") && s.has("deleted") && s.has("exited") && !s.has("renterTombstoned") && !s.has("settling")) { return None; } s.flag("settling"); s.reading("settled_by", "run request release: the --hold process exited when its ssh dropped"); Some(host("release", "request.release", json!({ "name": name }))) } /// Both hosts empty (the runbook's pass criterion): R2's own rentals, by `spec.requester.did`, on every bidder the target /// names a kcp for, gone within R2_VMS_WINDOW_S: after a release the cascade releases them; after the out-of-band loss /// their bidder ends them (`association_ended`: the outer contract's vm.terminated is there). Past the window the test /// deletes what is left on a bidder it may write to (the runbook's cleanup) and fails; on one it may not, it only watches /// 3 min more (`bidder_late` or `left`). None when no bidder kcp is named. const R2_VMS_WINDOW_S: f64 = 360.0; fn r2_vms(s: &mut S, mut h: Handle, bidders: &[Value]) -> Option { let with: Vec<(usize, String)> = bidders.iter().enumerate().filter_map(|(j, b)| b["cluster"].as_str().map(|c| (j, c.to_string()))).collect(); if with.is_empty() { s.reading("r2_vms_checked", false); return None; } s.st["wait"] = json!(with.len()); s.st["left"] = json!([]); for (j, c) in &with { h = h.call(kube(&format!("nv:{j}"), "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))); } Some(h) } fn run_b(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let account = args["did"].as_str().ok_or("did")?.to_string(); let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); // the runbook's outer request (part A's): with --agent R2 gets no atirc token of its own, whose mint answers 403 // "association not verified" right after the grant and holds R2's association ref (rc165) let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--agent"]); h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 60 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))), Event::Timer { id } if id == "contract" => { let w = json!(s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default()); s.reading("winner", w.clone()); if !bidders.iter().any(|b| b["did"] == w) { return Ok(s.fail(format!("unknown bidder {w}: the winner is not in the target"))); } match bidders.iter().find(|b| b["did"] == w).and_then(|b| b["cluster"].as_str()) { Some(c) => { s.st["cluster"] = json!(c); s.set("lost"); s.reading("out_of_band", true); h = h.call(kube("vms", "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))); } None => return Ok(s.unmet("the target names no kcp for the winner: the VM cannot be deleted out of band")), } } Event::Timer { id } if id == "m-again" => { let r2 = s.st["r2"].as_str().unwrap_or_default().to_string(); s.st["got"] = json!({}); h = h.call(plc("plc-r2", &r2)); } Event::Timer { id } if id == "poll" => { let r2 = s.st["r2"].as_str().unwrap_or_default().to_string(); h = h.call(Call::new("list", "com.atproto.repo.listRecords").params(json!({ "repo": r2, "collection": ENDORSE })).service(r2.clone())); } Event::Message { id, data, .. } if id == "req" => { if s.st["receipt"].is_null() && data["receipt"].is_string() { s.st["receipt"] = data["receipt"].clone(); } match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")), "contract" if data["phase"] == "Held" => s.flag("held"), "exited" => s.flag("exited"), "requester_association_written" => { s.st["r2"] = data["subject"].clone(); s.st["renter"] = data["did"].clone(); } "requester_tombstoned" => s.flag("renterTombstoned"), "requester_contract_records_deleted" => s.flag("recordsDeleted"), _ => {} } if s.is("held") && s.has("held") { s.set("start-r2"); // the runbook's start, exactly: never `install -d /run/socialweb-r2` (it resets the mode the guest agent set, // and R2 cannot write requester-did) // and the guest agent makes the dir once the bundle is in, seconds after Held: wait for it let cmd = "for i in $(seq 120); do [ -d /run/socialweb-r2 ] && break; sleep 1; done; printf 'R2_NAME=r2\\nR2_HOLD=1\\nR2_COLOCATED=mesh1\\n' > /run/socialweb-r2/r2-start.env && chown root:socialweb-r2 /run/socialweb-r2/r2-start.env && chmod 0640 /run/socialweb-r2/r2-start.env && systemctl start --no-block socialweb-r2.service"; h = h.call(host("r2", "request.ssh", json!({ "name": name, "exec": cmd }))); } if s.is("assoc") && s.st["r2"].is_string() { s.set("mesh1"); s.mark("r2"); h = h.timer("poll", 10_000); } if s.is("lost") && s.has("renterTombstoned") && s.has("recordsDeleted") { s.set("tomb"); h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"], s.st["mesh1"]] }))); } else if let Some(h2) = settle(&mut s, &name) { h = h.call(h2); } } Event::Result { id, .. } if id.starts_with("nd:") => {} Event::Result { id, result, .. } => { let r = result.output(); let o = match out(&s, &id, r.clone()) { Err(_) if ["peer", "g2", "plc-m", "walk"].contains(&id.as_str()) => json!({ "error": r.unwrap_err() }), Ok(o) => o, Err(f) => return Ok(f), }; let r2 = s.st["r2"].as_str().unwrap_or_default().to_string(); match id.as_str() { "r2" => { if o["ok"] != true { return Ok(s.fail(format!("starting R2: {}", o["output"].as_str().unwrap_or_default().lines().rev().take(3).collect::>().join(" | ")))); } s.set("assoc"); if s.st["r2"].is_string() { s.set("mesh1"); s.mark("r2"); h = h.timer("poll", 10_000); } } "list" => match o["records"].as_array().and_then(|r| r.first()).and_then(|r| r["value"]["agent"].as_str()) { Some(m) => { s.st["mesh1"] = json!(m); s.reading("mesh1", m); h = h.call(plc("plc-r2", &r2)); } None if s.since("r2") > 300.0 => return Ok(s.fail("no endorsement of mesh1 in R2's repo in 5 min")), None => h = h.timer("poll", 10_000), }, "plc-r2" => { if s.st["t_m"].is_null() { s.mark("m"); s.st["t_m"] = json!(true); } let r2h = o["alsoKnownAs"].as_array().into_iter().flatten().find_map(|a| a.as_str()?.strip_prefix("at://").map(String::from)).unwrap_or_default(); let m = s.st["mesh1"].as_str().unwrap_or_default().to_string(); s.st["fan"] = json!(4); h = h .call(get_record("peer", &format!("at://{m}/computer.socialweb.peer/mesh1.{r2h}"))) .call(get_record("g2", &format!("at://{r2}/{ASSOC}/{m}"))) .call(plc("plc-m", &m)) .call(walk("walk", &m)); } "peer" | "g2" | "plc-m" | "walk" => { s.st["got"][id.as_str()] = o; let left = s.st["fan"].as_u64().unwrap_or(1) - 1; s.st["fan"] = json!(left); if left == 0 { let g = s.st["got"].clone(); let mut p = vec![]; if !g["peer"]["value"]["iroh"].is_string() || g["peer"]["value"].get("lease").is_some() { p.push(format!("mesh1 peer {}", g["peer"])); } if !g["plc-m"]["alsoKnownAs"].as_array().is_some_and(|a| a.contains(&json!(format!("https://ident.link/agent-of/{r2}")))) { p.push(format!("mesh1 alsoKnownAs {}", g["plc-m"]["alsoKnownAs"])); } if g["walk"]["roots"] != json!([account]) || !g["walk"]["ancestors"].as_array().is_some_and(|a| a.iter().any(|x| x["did"] == r2.as_str() && x["depth"] == 1)) { p.push(format!("walk {}", g["walk"])); } if !p.is_empty() { if s.since("m") < 180.0 { return Ok(s.keep(h.timer("m-again", 10_000))); } return Ok(s.fail(format!("after 3 min: {}", p.join("; ")))); } s.reading("mesh1_chain_s", s.since("m")); // the --hold process stays: the receipt is from its log, the winner its author h = h.timer("contract", 1); } } "vms" => { let vm = o["items"].as_array().into_iter().flatten().find(|v| v["spec"]["contract"]["receipt"]["uri"] == s.st["receipt"]).and_then(|v| v["metadata"]["name"].as_str()).map(String::from); let Some(vm) = vm else { return Ok(s.fail("the outer ComputeVM of this run's receipt is not on its bidder")) }; let c = s.st["cluster"].as_str().unwrap_or_default().to_string(); if bidders.iter().any(|b| b["cluster"] == c.as_str() && b["write"] == false) { s.set("released"); s.reading("out_of_band", false); h = h.call(host("release", "request.release", json!({ "name": name }))); } else { h = h.call(kube("delvm", "delete", &c, steps::sw("ComputeVM", &vm))); } } "delvm" => { s.flag("deleted"); if let Some(c) = settle(&mut s, &name) { h = h.call(c); } } "release" => { s.set("tomb"); h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"], s.st["mesh1"]] }))); } "tomb" => { if o["alive"].as_array().is_none_or(|x| !x.is_empty()) { return Ok(s.fail(format!("not tombstoned: {}", o["alive"]))); } h = h.call(host("records", "request.records", json!({ "name": name }))); } "records" => h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))), "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {}", o["left"]))); } s.set("hosts"); s.mark("hosts"); return Ok(match r2_vms(&mut s, h, &bidders) { Some(h) => s.keep(h), None => s.pass(), }); } nv if nv.starts_with("nv:") => { let j: usize = nv[3..].parse().unwrap_or(0); let c = bidders.get(j).and_then(|b| b["cluster"].as_str()).unwrap_or_default().to_string(); let writable = bidders.get(j).is_some_and(|b| b["write"] != false); let late = s.has("late"); for v in o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["requester"]["did"] == r2.as_str()) { let vm = v["metadata"]["name"].as_str().unwrap_or_default().to_string(); if let Some(reason) = v["spec"]["release"]["reason"].as_str() { s.st["readings"]["r2_vm_release_reason"][&vm] = json!(reason); } s.st["left"].as_array_mut().map(|a| a.push(json!({ "vm": vm, "cluster": c, "phase": v["status"]["phase"], "release": v["spec"]["release"], "deleting": !v["metadata"]["deletionTimestamp"].is_null() }))); if late && writable && s.st["asked"][&vm] != true && v["metadata"]["deletionTimestamp"].is_null() { s.st["asked"][&vm] = json!(true); s.flag("testDeleted"); h = h.call(kube(&format!("nd:{vm}"), "delete", &c, steps::sw("ComputeVM", &vm))); } } let left = s.st["wait"].as_u64().unwrap_or(1) - 1; s.st["wait"] = json!(left); if left == 0 { let vms = s.st["left"].clone(); let gone = vms.as_array().is_some_and(Vec::is_empty); let t = s.since("hosts"); if gone { s.reading("r2_vms_gone_s", t); if !late { s.reading("r2_vms_ended_by", if s.has("deleted") { "bidder" } else { "release" }); return Ok(s.pass()); } let by = if s.has("testDeleted") { "test" } else { "bidder_late" }; s.reading("r2_vms_ended_by", by); return Ok(s.fail(format!("R2's own rentals not ended in {R2_VMS_WINDOW_S} s ({by}, gone at {t:.0} s): {}", s.st["stuck"]))); } if late && t > R2_VMS_WINDOW_S + 180.0 { s.reading("r2_vms_ended_by", "left"); return Ok(s.fail(format!("R2's own rentals still on the bidders {t:.0} s after the release: {vms}"))); } if !late && t > R2_VMS_WINDOW_S { s.flag("late"); s.st["stuck"] = vms; } h = h.timer("hosts", 5_000); } } _ => {} } } Event::Timer { id } if id == "hosts" => { if let Some(h2) = r2_vms(&mut s, h, &bidders) { return Ok(s.keep(h2)); } return Ok(s.pass()); } Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))), _ => {} } Ok(s.keep(h)) } /// r2fail: an R2 that fails after it reported its DID (its unit killed) leaves no live DID at PLC once the outer /// contract is released (the renter's tombstone fallback). KNOWN GAP for rc166: a failed attempt left /// did:plc:7f2a7u2jjxft6fngc4yzmxyr live, so this fails on rc165. fn run_r2fail(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let mut h = Handle::new(); match run.event { Event::Start => { s.set("held"); let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest"]); h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a }))).timer("deadline", 40 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 40 min (phase {})", s.phase()))), Event::Timer { id } if id == "poll" => h = h.call(host("tomb", "assert.tombstoned", json!({ "dids": [s.st["r2"]] }))), Event::Message { id, data, .. } if id == "req" => { match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Held" => s.flag("held"), "exited" => s.flag("exited"), "requester_guest_requester_reported" if s.st["r2"].is_null() => s.st["r2"] = data["did"].clone(), _ => {} } if s.is("held") && s.has("held") { s.set("r2"); h = h.call(host("r2", "request.r2", json!({ "name": name, "r2Name": "r2-fail" }))); } if s.is("reported") && s.st["r2"].is_string() { s.set("kill"); s.reading("r2", s.st["r2"].clone()); h = h.call(host("kill", "request.ssh", json!({ "name": name, "exec": "systemctl kill --signal=KILL socialweb-r2.service; systemctl stop socialweb-r2.service; true" }))); } } Event::Result { id, result, .. } => { let o = match out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "r2" => { s.set("reported"); if s.st["r2"].is_string() { s.set("kill"); s.reading("r2", s.st["r2"].clone()); h = h.call(host("kill", "request.ssh", json!({ "name": name, "exec": "systemctl kill --signal=KILL socialweb-r2.service; systemctl stop socialweb-r2.service; true" }))); } } "kill" => h = h.call(host("release", "request.release", json!({ "name": name }))), "release" => { s.mark("release"); h = h.timer("poll", 5_000); } "tomb" => { if o["alive"].as_array().is_some_and(Vec::is_empty) { return Ok(s.pass()); } if s.since("release") > 300.0 { return Ok(s.fail(format!("KNOWN GAP rc166: the failed R2 left {} live at PLC 5 min after the release", o["alive"]))); } h = h.timer("poll", 10_000); } _ => {} } } Event::Closed { id, error } if id == "req" => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))), _ => {} } Ok(s.keep(h)) } entries! { run => run_a, b => run_b, r2fail => run_r2fail }