0
0
// fence (staging): a guest's nftables rules follow its bidder through an upgrade (plan 0040 decision 75; Findings "PROMOTE
// 93fd8c11 ROOT CAUSE"). The target's first bidder B is upgraded to the `from` binary FROM (v0.0.0-rc186, main 688c2105:
// the last release without decision 54's rule; the entry's own, whatever rc the network follows), which must load no
// decision 54 rule (its guest's table bridge swcguest holds no accept of the WebRTC port range, else unmet). G, `--associate guest --guest-tools socialweb-computer --hold`, pinned to B,
// opens a long ssh session (its iroh ssh) that stays open across the upgrade. A probe table counts G's UDP to B's own
// addresses on the range before swcguest (sent) and after it (passed). G fetches the binary under test from the bucket
// (`binary.published`; its guest tools are the from binary, which cannot pin a bidder), as the promote's requester ran the
// binary under test in a guest of the previous rc. From inside G its guest requester (that binary, user r2; one
// invocation: a guest's association serves one) holds T pinned to B (`--tunnel ssh:22:webrtc`) and reaches it over WebRTC
// only (`run request ssh --transport webrtc --exec -- uname -a`): with the from binary's rules it must fail (the red
// promote: guest tap to its host's own address:316xx dropped; sent > 0, passed 0). B is then upgraded to the binary under
// test with G running: its journal shows guest_rules_reloaded for G (webrtcUDPPorts the range), G's swcguest now holds the
// range accept, the same ssh from inside G exits 0 with `uname -a`, passed > 0 (decision 54's rule, packet by packet). B is rolled back to the from binary (it has no reload:
// G's table stays as it is) and upgraded again (a new core of the same binary reloads it byte for byte the same). G's
// ssh session opened before the first upgrade is still open and ends cleanly. Then the probe goes, G is released and B
// runs the binary under test.
use abi::{Event, Handle, Run};
use serde_json::{Value, json};
use steps::{MIN, S, host, kube, out};
const RANGE: (u64, u64) = (31000, 31999);
const FROM: &str = "release:v0.0.0-rc186";
const HOLD: &str = "f=/root/swc-fence-hold; rm -f $f; mkfifo $f; echo HOLD-OPEN; cat $f; echo HOLD-END";
const RELEASE_HOLD: &str = "echo released > /root/swc-fence-hold";
const CLI: &str = "/usr/local/lib/swc-fence/socialweb-computer";
const TNAME: &str = "fence-t";
const FETCH: &str = r#"cloud-init status --wait >/dev/null 2>&1
t=@CLI@; mkdir -p ${t%/*}
cat > ${t%/*}/r2.sh <<'EOF'
d=$(dirname "$0"); uid=$(id -u r2)
cd /home/r2
op=$1; n=$2; log=/root/swc-fence-$n-$op.log
as_r2() {
runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus "$@"
}
show() {
grep -v '^{' $log | tail -c 1500
grep -o '"msg":"[^"]*"\|"phase":"[A-Za-z]*"' $log | uniq | tail -12 | tr '\n' ' '; echo
echo EXIT=$1
}
case $op in
rent)
R2_PLC= R2_INGRESS=
[ -r /etc/socialweb-guest-agent/r2-network.env ] && . /etc/socialweb-guest-agent/r2-network.env
pass=()
[ -n "$R2_PLC" ] && pass+=(--plc-directory-url "$R2_PLC")
[ -n "$R2_INGRESS" ] && pass+=(--ingress-proxy-host "$R2_INGRESS")
: > $log
as_r2 setsid $d/socialweb-computer run request --name "$n" --guest-requester --allow-bidder-dids "$3" --cpus 1 --mem 1G --hold --skip-ssh=false \
--tunnel ssh:22:webrtc "${pass[@]}" > $log 2>&1 < /dev/null &
rp=$!; echo $rp > /root/swc-fence-$n.pid
exec 3< <(timeout 900 tail -n +1 -F --pid=$rp $log 2>/dev/null); tp=$!
rc=1
while read -r l <&3; do
case "$l" in *'"phase":"Held"'*) rc=0; break ;; *'"phase":"Failed"'*) break ;; esac
done
kill $tp 2>/dev/null
show $rc ;;
ssh)
as_r2 timeout 180 $d/socialweb-computer run request ssh "$n" --transport webrtc --exec -- uname -a > $log 2>&1
show $? ;;
release)
as_r2 timeout 300 $d/socialweb-computer run request release "$n" > $log 2>&1
rc=$?
rp=$(cat /root/swc-fence-$n.pid 2>/dev/null); [ -n "$rp" ] && kill -INT $rp 2>/dev/null
show $rc ;;
esac
EOF
curl -fsS -m 900 -o $t.part '@URL@' && [ "$(sha256sum $t.part | cut -d' ' -f1)" = @SHA@ ] && chmod 0755 $t.part && mv -f $t.part $t && echo "FETCHED $(sha256sum $t | cut -d' ' -f1)""#;
fn r2(op: &str, name: &str, bidder: &str) -> String {
format!("bash {}/r2.sh {op} {name} {bidder}", CLI.trim_end_matches("/socialweb-computer"))
}
fn fvms(id: &str, cluster: &str) -> abi::Call {
kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "FirecrackerVM" }))
}
fn range_accepts(rules: &str) -> usize {
rules.lines().filter(|l| l.contains(&RANGE.0.to_string()) && l.contains(&RANGE.1.to_string()) && l.trim_end().ends_with("accept")).count()
}
fn exit_of(o: &Value) -> i64 {
o["output"].as_str().unwrap_or_default().lines().rev().find_map(|l| l.strip_prefix("EXIT=")).and_then(|n| n.trim().parse().ok()).unwrap_or(-1)
}
pub fn run_fence(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
// the guest requester's own atirc token: its mint answers 403 "association not verified" for minutes right after the
// grant (0005's note, rc165; 0005 avoids it with --agent, whose co-located identity-only R2 would take G's one
// association here); the requester retries it and the rental does not wait on it: counted in readings, not failed fast
if let Event::Message { id, data, .. } = &run.event {
if id == "req" && data["msg"] == "requester_association_failed" && data["step"].as_str().is_some_and(|x| x.starts_with("mint the guest requester's atirc token")) {
let n = s.st["readings"]["atirc_mint_403"].as_u64().unwrap_or(0) + 1;
s.reading("atirc_mint_403", n);
return Ok(s.keep(Handle::new()));
}
}
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let b = args["bidders"][0].clone();
let (bname, bdid, bcluster) = (b["name"].clone(), b["did"].as_str().unwrap_or_default().to_string(), b["cluster"].as_str().unwrap_or_default().to_string());
let vm = s.st["vm"].as_str().unwrap_or_default().to_string();
let owner = s.st["owner"].as_str().unwrap_or_default().to_string();
let nft = |id: &str, op: &str| host(id, "bidder.guestNft", json!({ "name": bname, "vm": vm, "owner": owner, "op": op, "min": RANGE.0, "max": RANGE.1 }));
let mut h = Handle::new();
match run.event {
Event::Start => {
if args["target"] == "prod" {
return Ok(s.unmet("staging only: it upgrades a bidder to another binary"));
}
if !bname.is_string() || !bdid.starts_with("did:") || bcluster.is_empty() {
return Ok(s.unmet("the target's first bidder has no name, DID and kcp"));
}
s.set("pre");
h = h.call(host("pre", "bidder.upgrade", json!({ "name": bname, "binary": "from", "from": FROM }))).timer("deadline", 60 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
Event::Message { id, data, .. } if id == "req" => match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("G's contract failed before Held: {}", data["message"]))),
"exited" if !s.has("held") => return Ok(s.fail(format!("G's run request exited before Held: {data}"))),
"contract" if data["phase"] == "Held" && !s.has("held") => {
s.flag("held");
let t = s.since("requested");
s.reading("g_held_s", t);
h = h.call(fvms("after", &bcluster));
}
_ => {}
},
Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("G's run request closed: {}", error.unwrap_or_default()))),
Event::Result { id, result, .. } => {
let o = match out(&s, &id, result.output()) {
Ok(o) => o,
Err(f) => return Ok(f),
};
match id.as_str() {
"pre" => {
if o["from"] == o["test"] {
return Ok(s.unmet(&format!("the from binary {} ({}) is the binary under test: there is no upgrade to show", o["fromSpec"], o["from"])));
}
if o["sha256"] != o["from"] {
return Ok(s.fail(format!("{bname} runs {} after the upgrade to the from binary {}", o["sha256"], o["from"])));
}
s.st["from"] = o["from"].clone();
s.st["test"] = o["test"].clone();
s.reading("from", json!({ "spec": o["fromSpec"], "sha256": o["from"] }));
s.reading("test", o["test"].clone());
h = h.call(fvms("base", &bcluster));
}
"base" => {
s.st["base"] = json!(steps::names(&o));
s.set("g");
s.mark("requested");
let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--expires-in", "1h"]);
h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a, "bidderDids": [bdid] })));
}
"after" => {
let base = s.st["base"].as_array().cloned().unwrap_or_default();
let new: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["metadata"]["deletionTimestamp"].is_null() && !base.contains(&v["metadata"]["name"])).collect();
let [g] = new.as_slice() else {
return Ok(s.fail(format!("{} new FirecrackerVMs on {bname} after G was Held, not one", new.len())));
};
s.st["vm"] = g["metadata"]["name"].clone();
s.st["owner"] = g["spec"]["owner"]["handle"].clone();
s.reading("g", json!({ "bidder": bname, "firecrackervm": g["metadata"]["name"] }));
let (vm, owner) = (g["metadata"]["name"].as_str().unwrap_or_default(), g["spec"]["owner"]["handle"].as_str().unwrap_or_default());
h = h.call(host("rules0", "bidder.guestNft", json!({ "name": bname, "vm": vm, "owner": owner, "op": "list" })));
}
"rules0" => {
let rules = o["output"].as_str().unwrap_or_default().to_string();
s.reading("rules_before", rules.clone());
if !rules.contains("table bridge swcguest") {
return Ok(s.fail(format!("G's namespace holds no table bridge swcguest: {rules}")));
}
if range_accepts(&rules) != 0 {
return Ok(s.unmet(&format!("the from binary {FROM} ({}) already loads decision 54's accept of {}-{} into G: it cannot show the reload", s.st["from"], RANGE.0, RANGE.1)));
}
h = h.call(nft("probe", "probe"));
}
"probe" => {
s.reading("own", o["own"].clone());
h = h.call(host("url", "binary.published", json!({})));
}
"url" => {
let (Some(url), Some(sha)) = (o["url"].as_str(), o["sha256"].as_str()) else {
return Ok(s.unmet(&format!("the binary under test has no public URL for G to fetch: {o}")));
};
s.st["cliSha"] = json!(sha);
s.reading("cli", json!({ "url": url, "sha256": sha }));
s.mark("fetch");
let fetch = FETCH.replace("@CLI@", CLI).replace("@URL@", url).replace("@SHA@", sha);
h = h.call(host("fetch", "request.ssh", json!({ "name": name, "exec": fetch })));
}
"fetch" => {
if !o["output"].as_str().unwrap_or_default().contains(&format!("FETCHED {}", s.st["cliSha"].as_str().unwrap_or_default())) {
return Ok(s.fail(format!("G did not fetch the binary under test (its requester for the rentals from inside it): {}", o["output"])));
}
s.reading("fetch_s", s.since("fetch"));
s.mark("rent");
h = h.call(host("rent", "request.ssh", json!({ "name": name, "exec": r2("rent", TNAME, &bdid) })));
}
"rent" => {
s.reading("rent", o["output"].clone());
if exit_of(&o) != 0 {
return Ok(s.fail(format!("G's guest requester did not hold T on {bname}: {}", o["output"])));
}
s.reading("t_held_s", s.since("rent"));
s.flag("tHeld");
s.set("dial0");
s.mark("hold");
h = h
.call(host("hold", "request.ssh", json!({ "name": name, "exec": HOLD })))
.call(host("dial0", "request.ssh", json!({ "name": name, "exec": r2("ssh", TNAME, "") })));
}
"hold" => {
let output = o["output"].as_str().unwrap_or_default();
if !s.has("holdReleased") || o["ok"] != true || !output.contains("HOLD-OPEN") || !output.contains("released") || !output.contains("HOLD-END") {
return Ok(s.fail(format!("G's ssh session opened before the upgrade did not stay open until it was released: {o}")));
}
s.reading("hold_session_s", s.since("hold"));
s.flag("holdOk");
}
"dial0" => {
s.reading("dial_before", o["output"].clone());
if exit_of(&o) == 0 {
return Ok(s.fail(format!("ssh over WebRTC from G to T on {bname} passed under the from binary's rules: the red promote is not reproduced here: {}", o["output"])));
}
h = h.call(nft("cnt0", "counters"));
}
"cnt0" => {
s.reading("probe_before", json!({ "sent": o["sent"], "passed": o["passed"] }));
if o["sent"].as_u64().unwrap_or(0) == 0 || o["passed"].as_u64() != Some(0) {
return Ok(s.fail(format!("before the upgrade G's UDP to {bname}'s own addresses on {}-{}: sent {} passed {} (want sent > 0, passed 0)", RANGE.0, RANGE.1, o["sent"], o["passed"])));
}
s.set("up");
s.mark("upgrade");
h = h.call(host("up", "bidder.upgrade", json!({ "name": bname, "binary": "test", "from": FROM })));
}
"up" => {
if o["sha256"] != s.st["test"] {
return Ok(s.fail(format!("{bname} runs {} after the upgrade, not the binary under test {}", o["sha256"], s.st["test"])));
}
s.reading("upgrade_s", o["seconds"].clone());
h = h.call(host("reload", "bidder.awaitLog", json!({ "name": bname, "texts": ["\"msg\":\"guest_rules_reloaded\"", format!("\"firecrackervm\":\"{vm}\"")], "seconds": 600 })));
}
"reload" => {
if o["found"] != true {
return Ok(s.fail(format!("{bname} logged no guest_rules_reloaded for {vm} within 600 s of its upgrade: {}", o["report"])));
}
let l = &o["line"];
s.reading("reloaded", l.clone());
s.reading("reload_after_upgrade_s", s.since("upgrade"));
if l["webrtcUDPPorts"] != format!("{}-{}", RANGE.0, RANGE.1) {
return Ok(s.fail(format!("guest_rules_reloaded for {vm} names webrtcUDPPorts {}, not {}-{}", l["webrtcUDPPorts"], RANGE.0, RANGE.1)));
}
h = h.call(nft("rules1", "list"));
}
"rules1" => {
let rules = o["output"].as_str().unwrap_or_default().to_string();
s.reading("rules_after", rules.clone());
if range_accepts(&rules) == 0 {
return Ok(s.fail(format!("after the reload G's swcguest holds no accept of {}-{}: {rules}", RANGE.0, RANGE.1)));
}
s.set("dial1");
h = h.call(host("dial1", "request.ssh", json!({ "name": name, "exec": r2("ssh", TNAME, "") })));
}
"dial1" => {
s.reading("dial_after", o["output"].clone());
if exit_of(&o) != 0 || !o["output"].as_str().unwrap_or_default().contains("Linux") {
return Ok(s.fail(format!("ssh over WebRTC from G to T on {bname} after the reload: {}", o["output"])));
}
h = h.call(nft("cnt1", "counters"));
}
"cnt1" => {
s.reading("probe_after", json!({ "sent": o["sent"], "passed": o["passed"] }));
if o["passed"].as_u64().unwrap_or(0) == 0 {
return Ok(s.fail(format!("after the reload no UDP of G to {bname}'s own addresses on {}-{} passed swcguest: sent {} passed {}", RANGE.0, RANGE.1, o["sent"], o["passed"])));
}
h = h.call(host("down", "bidder.upgrade", json!({ "name": bname, "binary": "from", "from": FROM })));
}
"down" => {
if o["sha256"] != s.st["from"] {
return Ok(s.fail(format!("{bname} runs {} after the rollback, not the from binary {}", o["sha256"], s.st["from"])));
}
h = h.call(nft("rules2", "list"));
}
"rules2" => {
let rules = o["output"].as_str().unwrap_or_default();
s.reading("rules_after_rollback", rules);
if s.st["readings"]["rules_after"] != rules {
return Ok(s.fail(format!("after the rollback to the from binary (no reload in it) G's swcguest changed: {rules}")));
}
h = h.call(host("up2", "bidder.upgrade", json!({ "name": bname, "binary": "test", "from": FROM })));
}
"up2" => {
if o["sha256"] != s.st["test"] {
return Ok(s.fail(format!("{bname} runs {} after the second upgrade, not the binary under test {}", o["sha256"], s.st["test"])));
}
h = h.call(host("reload2", "bidder.awaitLog", json!({ "name": bname, "texts": ["\"msg\":\"guest_rules_reloaded\"", format!("\"firecrackervm\":\"{vm}\"")], "seconds": 600 })));
}
"reload2" => {
if o["found"] != true {
return Ok(s.fail(format!("{bname} logged no guest_rules_reloaded for {vm} after the second upgrade: {}", o["report"])));
}
h = h.call(nft("rules3", "list"));
}
"rules3" => {
let rules = o["output"].as_str().unwrap_or_default();
if s.st["readings"]["rules_after"] != rules {
return Ok(s.fail(format!("a second reload by the same binary changed G's swcguest: {rules}")));
}
s.reading("second_reload_identical", true);
s.flag("holdReleased");
h = h.call(host("free", "request.ssh", json!({ "name": name, "exec": RELEASE_HOLD })));
}
"free" => {}
"unprobe" => {
h = h.call(host("tfree", "request.ssh", json!({ "name": name, "exec": r2("release", TNAME, "") })));
}
"tfree" => {
if exit_of(&o) != 0 {
return Ok(s.fail(format!("G's guest requester did not release T: {}", o["output"])));
}
s.set("release");
s.flag("releasing");
h = h.cancel("req").call(host("release", "request.release", json!({ "name": name })));
}
"release" => {
s.reading("release_s", o["seconds"].clone());
return Ok(s.pass());
}
_ => {}
}
if s.has("holdOk") && s.is("dial1") && s.has("holdReleased") && !s.has("unprobed") {
s.flag("unprobed");
h = h.call(nft("unprobe", "unprobe"));
}
}
_ => {}
}
Ok(s.keep(h))
}