// fence (staging): a guest's nftables rules follow its bidder through an upgrade (plan 0040 decision 75; Findings "PROMOTE // 93fd8c11 ROOT CAUSE"). The target's first bidder B is upgraded to the `from` binary FROM (v0.0.0-rc186, main 688c2105: // the last release without decision 54's rule; the entry's own, whatever rc the network follows), which must load no // decision 54 rule (its guest's table bridge swcguest holds no accept of the WebRTC port range, else unmet). G, `--associate guest --guest-tools socialweb-computer --hold`, pinned to B, // opens a long ssh session (its iroh ssh) that stays open across the upgrade. A probe table counts G's UDP to B's own // addresses on the range before swcguest (sent) and after it (passed). G fetches the binary under test from the bucket // (`binary.published`; its guest tools are the from binary, which cannot pin a bidder), as the promote's requester ran the // binary under test in a guest of the previous rc. From inside G its guest requester (that binary, user r2; one // invocation: a guest's association serves one) holds T pinned to B (`--tunnel ssh:22:webrtc`) and reaches it over WebRTC // only (`run request ssh --transport webrtc --exec -- uname -a`): with the from binary's rules it must fail (the red // promote: guest tap to its host's own address:316xx dropped; sent > 0, passed 0). B is then upgraded to the binary under // test with G running: its journal shows guest_rules_reloaded for G (webrtcUDPPorts the range), G's swcguest now holds the // range accept, the same ssh from inside G exits 0 with `uname -a`, passed > 0 (decision 54's rule, packet by packet). B is rolled back to the from binary (it has no reload: // G's table stays as it is) and upgraded again (a new core of the same binary reloads it byte for byte the same). G's // ssh session opened before the first upgrade is still open and ends cleanly. Then the probe goes, G is released and B // runs the binary under test. use abi::{Event, Handle, Run}; use serde_json::{Value, json}; use steps::{MIN, S, host, kube, out}; const RANGE: (u64, u64) = (31000, 31999); const FROM: &str = "release:v0.0.0-rc186"; const HOLD: &str = "f=/root/swc-fence-hold; rm -f $f; mkfifo $f; echo HOLD-OPEN; cat $f; echo HOLD-END"; const RELEASE_HOLD: &str = "echo released > /root/swc-fence-hold"; const CLI: &str = "/usr/local/lib/swc-fence/socialweb-computer"; const TNAME: &str = "fence-t"; const FETCH: &str = r#"cloud-init status --wait >/dev/null 2>&1 t=@CLI@; mkdir -p ${t%/*} cat > ${t%/*}/r2.sh <<'EOF' d=$(dirname "$0"); uid=$(id -u r2) cd /home/r2 op=$1; n=$2; log=/root/swc-fence-$n-$op.log as_r2() { runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus "$@" } show() { grep -v '^{' $log | tail -c 1500 grep -o '"msg":"[^"]*"\|"phase":"[A-Za-z]*"' $log | uniq | tail -12 | tr '\n' ' '; echo echo EXIT=$1 } case $op in rent) R2_PLC= R2_INGRESS= [ -r /etc/socialweb-guest-agent/r2-network.env ] && . /etc/socialweb-guest-agent/r2-network.env pass=() [ -n "$R2_PLC" ] && pass+=(--plc-directory-url "$R2_PLC") [ -n "$R2_INGRESS" ] && pass+=(--ingress-proxy-host "$R2_INGRESS") : > $log as_r2 setsid $d/socialweb-computer run request --name "$n" --guest-requester --allow-bidder-dids "$3" --cpus 1 --mem 1G --hold --skip-ssh=false \ --tunnel ssh:22:webrtc "${pass[@]}" > $log 2>&1 < /dev/null & rp=$!; echo $rp > /root/swc-fence-$n.pid exec 3< <(timeout 900 tail -n +1 -F --pid=$rp $log 2>/dev/null); tp=$! rc=1 while read -r l <&3; do case "$l" in *'"phase":"Held"'*) rc=0; break ;; *'"phase":"Failed"'*) break ;; esac done kill $tp 2>/dev/null show $rc ;; ssh) as_r2 timeout 180 $d/socialweb-computer run request ssh "$n" --transport webrtc --exec -- uname -a > $log 2>&1 show $? ;; release) as_r2 timeout 300 $d/socialweb-computer run request release "$n" > $log 2>&1 rc=$? rp=$(cat /root/swc-fence-$n.pid 2>/dev/null); [ -n "$rp" ] && kill -INT $rp 2>/dev/null show $rc ;; esac EOF curl -fsS -m 900 -o $t.part '@URL@' && [ "$(sha256sum $t.part | cut -d' ' -f1)" = @SHA@ ] && chmod 0755 $t.part && mv -f $t.part $t && echo "FETCHED $(sha256sum $t | cut -d' ' -f1)""#; fn r2(op: &str, name: &str, bidder: &str) -> String { format!("bash {}/r2.sh {op} {name} {bidder}", CLI.trim_end_matches("/socialweb-computer")) } fn fvms(id: &str, cluster: &str) -> abi::Call { kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "FirecrackerVM" })) } fn range_accepts(rules: &str) -> usize { rules.lines().filter(|l| l.contains(&RANGE.0.to_string()) && l.contains(&RANGE.1.to_string()) && l.trim_end().ends_with("accept")).count() } fn exit_of(o: &Value) -> i64 { o["output"].as_str().unwrap_or_default().lines().rev().find_map(|l| l.strip_prefix("EXIT=")).and_then(|n| n.trim().parse().ok()).unwrap_or(-1) } pub fn run_fence(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); // the guest requester's own atirc token: its mint answers 403 "association not verified" for minutes right after the // grant (0005's note, rc165; 0005 avoids it with --agent, whose co-located identity-only R2 would take G's one // association here); the requester retries it and the rental does not wait on it: counted in readings, not failed fast if let Event::Message { id, data, .. } = &run.event { if id == "req" && data["msg"] == "requester_association_failed" && data["step"].as_str().is_some_and(|x| x.starts_with("mint the guest requester's atirc token")) { let n = s.st["readings"]["atirc_mint_403"].as_u64().unwrap_or(0) + 1; s.reading("atirc_mint_403", n); return Ok(s.keep(Handle::new())); } } if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let b = args["bidders"][0].clone(); let (bname, bdid, bcluster) = (b["name"].clone(), b["did"].as_str().unwrap_or_default().to_string(), b["cluster"].as_str().unwrap_or_default().to_string()); let vm = s.st["vm"].as_str().unwrap_or_default().to_string(); let owner = s.st["owner"].as_str().unwrap_or_default().to_string(); let nft = |id: &str, op: &str| host(id, "bidder.guestNft", json!({ "name": bname, "vm": vm, "owner": owner, "op": op, "min": RANGE.0, "max": RANGE.1 })); let mut h = Handle::new(); match run.event { Event::Start => { if args["target"] == "prod" { return Ok(s.unmet("staging only: it upgrades a bidder to another binary")); } if !bname.is_string() || !bdid.starts_with("did:") || bcluster.is_empty() { return Ok(s.unmet("the target's first bidder has no name, DID and kcp")); } s.set("pre"); h = h.call(host("pre", "bidder.upgrade", json!({ "name": bname, "binary": "from", "from": FROM }))).timer("deadline", 60 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))), Event::Message { id, data, .. } if id == "req" => match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("G's contract failed before Held: {}", data["message"]))), "exited" if !s.has("held") => return Ok(s.fail(format!("G's run request exited before Held: {data}"))), "contract" if data["phase"] == "Held" && !s.has("held") => { s.flag("held"); let t = s.since("requested"); s.reading("g_held_s", t); h = h.call(fvms("after", &bcluster)); } _ => {} }, Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("G's run request closed: {}", error.unwrap_or_default()))), Event::Result { id, result, .. } => { let o = match out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "pre" => { if o["from"] == o["test"] { return Ok(s.unmet(&format!("the from binary {} ({}) is the binary under test: there is no upgrade to show", o["fromSpec"], o["from"]))); } if o["sha256"] != o["from"] { return Ok(s.fail(format!("{bname} runs {} after the upgrade to the from binary {}", o["sha256"], o["from"]))); } s.st["from"] = o["from"].clone(); s.st["test"] = o["test"].clone(); s.reading("from", json!({ "spec": o["fromSpec"], "sha256": o["from"] })); s.reading("test", o["test"].clone()); h = h.call(fvms("base", &bcluster)); } "base" => { s.st["base"] = json!(steps::names(&o)); s.set("g"); s.mark("requested"); let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--expires-in", "1h"]); h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a, "bidderDids": [bdid] }))); } "after" => { let base = s.st["base"].as_array().cloned().unwrap_or_default(); let new: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["metadata"]["deletionTimestamp"].is_null() && !base.contains(&v["metadata"]["name"])).collect(); let [g] = new.as_slice() else { return Ok(s.fail(format!("{} new FirecrackerVMs on {bname} after G was Held, not one", new.len()))); }; s.st["vm"] = g["metadata"]["name"].clone(); s.st["owner"] = g["spec"]["owner"]["handle"].clone(); s.reading("g", json!({ "bidder": bname, "firecrackervm": g["metadata"]["name"] })); let (vm, owner) = (g["metadata"]["name"].as_str().unwrap_or_default(), g["spec"]["owner"]["handle"].as_str().unwrap_or_default()); h = h.call(host("rules0", "bidder.guestNft", json!({ "name": bname, "vm": vm, "owner": owner, "op": "list" }))); } "rules0" => { let rules = o["output"].as_str().unwrap_or_default().to_string(); s.reading("rules_before", rules.clone()); if !rules.contains("table bridge swcguest") { return Ok(s.fail(format!("G's namespace holds no table bridge swcguest: {rules}"))); } if range_accepts(&rules) != 0 { return Ok(s.unmet(&format!("the from binary {FROM} ({}) already loads decision 54's accept of {}-{} into G: it cannot show the reload", s.st["from"], RANGE.0, RANGE.1))); } h = h.call(nft("probe", "probe")); } "probe" => { s.reading("own", o["own"].clone()); h = h.call(host("url", "binary.published", json!({}))); } "url" => { let (Some(url), Some(sha)) = (o["url"].as_str(), o["sha256"].as_str()) else { return Ok(s.unmet(&format!("the binary under test has no public URL for G to fetch: {o}"))); }; s.st["cliSha"] = json!(sha); s.reading("cli", json!({ "url": url, "sha256": sha })); s.mark("fetch"); let fetch = FETCH.replace("@CLI@", CLI).replace("@URL@", url).replace("@SHA@", sha); h = h.call(host("fetch", "request.ssh", json!({ "name": name, "exec": fetch }))); } "fetch" => { if !o["output"].as_str().unwrap_or_default().contains(&format!("FETCHED {}", s.st["cliSha"].as_str().unwrap_or_default())) { return Ok(s.fail(format!("G did not fetch the binary under test (its requester for the rentals from inside it): {}", o["output"]))); } s.reading("fetch_s", s.since("fetch")); s.mark("rent"); h = h.call(host("rent", "request.ssh", json!({ "name": name, "exec": r2("rent", TNAME, &bdid) }))); } "rent" => { s.reading("rent", o["output"].clone()); if exit_of(&o) != 0 { return Ok(s.fail(format!("G's guest requester did not hold T on {bname}: {}", o["output"]))); } s.reading("t_held_s", s.since("rent")); s.flag("tHeld"); s.set("dial0"); s.mark("hold"); h = h .call(host("hold", "request.ssh", json!({ "name": name, "exec": HOLD }))) .call(host("dial0", "request.ssh", json!({ "name": name, "exec": r2("ssh", TNAME, "") }))); } "hold" => { let output = o["output"].as_str().unwrap_or_default(); if !s.has("holdReleased") || o["ok"] != true || !output.contains("HOLD-OPEN") || !output.contains("released") || !output.contains("HOLD-END") { return Ok(s.fail(format!("G's ssh session opened before the upgrade did not stay open until it was released: {o}"))); } s.reading("hold_session_s", s.since("hold")); s.flag("holdOk"); } "dial0" => { s.reading("dial_before", o["output"].clone()); if exit_of(&o) == 0 { return Ok(s.fail(format!("ssh over WebRTC from G to T on {bname} passed under the from binary's rules: the red promote is not reproduced here: {}", o["output"]))); } h = h.call(nft("cnt0", "counters")); } "cnt0" => { s.reading("probe_before", json!({ "sent": o["sent"], "passed": o["passed"] })); if o["sent"].as_u64().unwrap_or(0) == 0 || o["passed"].as_u64() != Some(0) { return Ok(s.fail(format!("before the upgrade G's UDP to {bname}'s own addresses on {}-{}: sent {} passed {} (want sent > 0, passed 0)", RANGE.0, RANGE.1, o["sent"], o["passed"]))); } s.set("up"); s.mark("upgrade"); h = h.call(host("up", "bidder.upgrade", json!({ "name": bname, "binary": "test", "from": FROM }))); } "up" => { if o["sha256"] != s.st["test"] { return Ok(s.fail(format!("{bname} runs {} after the upgrade, not the binary under test {}", o["sha256"], s.st["test"]))); } s.reading("upgrade_s", o["seconds"].clone()); h = h.call(host("reload", "bidder.awaitLog", json!({ "name": bname, "texts": ["\"msg\":\"guest_rules_reloaded\"", format!("\"firecrackervm\":\"{vm}\"")], "seconds": 600 }))); } "reload" => { if o["found"] != true { return Ok(s.fail(format!("{bname} logged no guest_rules_reloaded for {vm} within 600 s of its upgrade: {}", o["report"]))); } let l = &o["line"]; s.reading("reloaded", l.clone()); s.reading("reload_after_upgrade_s", s.since("upgrade")); if l["webrtcUDPPorts"] != format!("{}-{}", RANGE.0, RANGE.1) { return Ok(s.fail(format!("guest_rules_reloaded for {vm} names webrtcUDPPorts {}, not {}-{}", l["webrtcUDPPorts"], RANGE.0, RANGE.1))); } h = h.call(nft("rules1", "list")); } "rules1" => { let rules = o["output"].as_str().unwrap_or_default().to_string(); s.reading("rules_after", rules.clone()); if range_accepts(&rules) == 0 { return Ok(s.fail(format!("after the reload G's swcguest holds no accept of {}-{}: {rules}", RANGE.0, RANGE.1))); } s.set("dial1"); h = h.call(host("dial1", "request.ssh", json!({ "name": name, "exec": r2("ssh", TNAME, "") }))); } "dial1" => { s.reading("dial_after", o["output"].clone()); if exit_of(&o) != 0 || !o["output"].as_str().unwrap_or_default().contains("Linux") { return Ok(s.fail(format!("ssh over WebRTC from G to T on {bname} after the reload: {}", o["output"]))); } h = h.call(nft("cnt1", "counters")); } "cnt1" => { s.reading("probe_after", json!({ "sent": o["sent"], "passed": o["passed"] })); if o["passed"].as_u64().unwrap_or(0) == 0 { return Ok(s.fail(format!("after the reload no UDP of G to {bname}'s own addresses on {}-{} passed swcguest: sent {} passed {}", RANGE.0, RANGE.1, o["sent"], o["passed"]))); } h = h.call(host("down", "bidder.upgrade", json!({ "name": bname, "binary": "from", "from": FROM }))); } "down" => { if o["sha256"] != s.st["from"] { return Ok(s.fail(format!("{bname} runs {} after the rollback, not the from binary {}", o["sha256"], s.st["from"]))); } h = h.call(nft("rules2", "list")); } "rules2" => { let rules = o["output"].as_str().unwrap_or_default(); s.reading("rules_after_rollback", rules); if s.st["readings"]["rules_after"] != rules { return Ok(s.fail(format!("after the rollback to the from binary (no reload in it) G's swcguest changed: {rules}"))); } h = h.call(host("up2", "bidder.upgrade", json!({ "name": bname, "binary": "test", "from": FROM }))); } "up2" => { if o["sha256"] != s.st["test"] { return Ok(s.fail(format!("{bname} runs {} after the second upgrade, not the binary under test {}", o["sha256"], s.st["test"]))); } h = h.call(host("reload2", "bidder.awaitLog", json!({ "name": bname, "texts": ["\"msg\":\"guest_rules_reloaded\"", format!("\"firecrackervm\":\"{vm}\"")], "seconds": 600 }))); } "reload2" => { if o["found"] != true { return Ok(s.fail(format!("{bname} logged no guest_rules_reloaded for {vm} after the second upgrade: {}", o["report"]))); } h = h.call(nft("rules3", "list")); } "rules3" => { let rules = o["output"].as_str().unwrap_or_default(); if s.st["readings"]["rules_after"] != rules { return Ok(s.fail(format!("a second reload by the same binary changed G's swcguest: {rules}"))); } s.reading("second_reload_identical", true); s.flag("holdReleased"); h = h.call(host("free", "request.ssh", json!({ "name": name, "exec": RELEASE_HOLD }))); } "free" => {} "unprobe" => { h = h.call(host("tfree", "request.ssh", json!({ "name": name, "exec": r2("release", TNAME, "") }))); } "tfree" => { if exit_of(&o) != 0 { return Ok(s.fail(format!("G's guest requester did not release T: {}", o["output"]))); } s.set("release"); s.flag("releasing"); h = h.cancel("req").call(host("release", "request.release", json!({ "name": name }))); } "release" => { s.reading("release_s", o["seconds"].clone()); return Ok(s.pass()); } _ => {} } if s.has("holdOk") && s.is("dial1") && s.has("holdReleased") && !s.has("unprobed") { s.flag("unprobed"); h = h.call(nft("unprobe", "unprobe")); } } _ => {} } Ok(s.keep(h)) }