~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
351 lines · 20.6 KB Raw History
// fence (staging): a guest's nftables rules follow its bidder through an upgrade (plan 0040 decision 75; Findings "PROMOTE
// 93fd8c11 ROOT CAUSE"). The target's first bidder B is upgraded to the `from` binary FROM (v0.0.0-rc186, main 688c2105:
// the last release without decision 54's rule; the entry's own, whatever rc the network follows), which must load no
// decision 54 rule (its guest's table bridge swcguest holds no accept of the WebRTC port range, else unmet). G, `--associate guest --guest-tools socialweb-computer --hold`, pinned to B,
// opens a long ssh session (its iroh ssh) that stays open across the upgrade. A probe table counts G's UDP to B's own
// addresses on the range before swcguest (sent) and after it (passed). G fetches the binary under test from the bucket
// (`binary.published`; its guest tools are the from binary, which cannot pin a bidder), as the promote's requester ran the
// binary under test in a guest of the previous rc. From inside G its guest requester (that binary, user r2; one
// invocation: a guest's association serves one) holds T pinned to B (`--tunnel ssh:22:webrtc`) and reaches it over WebRTC
// only (`run request ssh --transport webrtc --exec -- uname -a`): with the from binary's rules it must fail (the red
// promote: guest tap to its host's own address:316xx dropped; sent > 0, passed 0). B is then upgraded to the binary under
// test with G running: its journal shows guest_rules_reloaded for G (webrtcUDPPorts the range), G's swcguest now holds the
// range accept, the same ssh from inside G exits 0 with `uname -a`, passed > 0 (decision 54's rule, packet by packet). B is rolled back to the from binary (it has no reload:
// G's table stays as it is) and upgraded again (a new core of the same binary reloads it byte for byte the same). G's
// ssh session opened before the first upgrade is still open and ends cleanly. Then the probe goes, G is released and B
// runs the binary under test.
use abi::{Event, Handle, Run};
use serde_json::{Value, json};
use steps::{MIN, S, host, kube, out};

const RANGE: (u64, u64) = (31000, 31999);

const FROM: &str = "release:v0.0.0-rc186";

const HOLD: &str = "f=/root/swc-fence-hold; rm -f $f; mkfifo $f; echo HOLD-OPEN; cat $f; echo HOLD-END";

const RELEASE_HOLD: &str = "echo released > /root/swc-fence-hold";

const CLI: &str = "/usr/local/lib/swc-fence/socialweb-computer";

const TNAME: &str = "fence-t";

const FETCH: &str = r#"cloud-init status --wait >/dev/null 2>&1
t=@CLI@; mkdir -p ${t%/*}
cat > ${t%/*}/r2.sh <<'EOF'
d=$(dirname "$0"); uid=$(id -u r2)
cd /home/r2
op=$1; n=$2; log=/root/swc-fence-$n-$op.log
as_r2() {
  runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR=/run/user/$uid DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$uid/bus "$@"
}
show() {
  grep -v '^{' $log | tail -c 1500
  grep -o '"msg":"[^"]*"\|"phase":"[A-Za-z]*"' $log | uniq | tail -12 | tr '\n' ' '; echo
  echo EXIT=$1
}
case $op in
  rent)
    R2_PLC= R2_INGRESS=
    [ -r /etc/socialweb-guest-agent/r2-network.env ] && . /etc/socialweb-guest-agent/r2-network.env
    pass=()
    [ -n "$R2_PLC" ] && pass+=(--plc-directory-url "$R2_PLC")
    [ -n "$R2_INGRESS" ] && pass+=(--ingress-proxy-host "$R2_INGRESS")
    : > $log
    as_r2 setsid $d/socialweb-computer run request --name "$n" --guest-requester --allow-bidder-dids "$3" --cpus 1 --mem 1G --hold --skip-ssh=false \
      --tunnel ssh:22:webrtc "${pass[@]}" > $log 2>&1 < /dev/null &
    rp=$!; echo $rp > /root/swc-fence-$n.pid
    exec 3< <(timeout 900 tail -n +1 -F --pid=$rp $log 2>/dev/null); tp=$!
    rc=1
    while read -r l <&3; do
      case "$l" in *'"phase":"Held"'*) rc=0; break ;; *'"phase":"Failed"'*) break ;; esac
    done
    kill $tp 2>/dev/null
    show $rc ;;
  ssh)
    as_r2 timeout 180 $d/socialweb-computer run request ssh "$n" --transport webrtc --exec -- uname -a > $log 2>&1
    show $? ;;
  release)
    as_r2 timeout 300 $d/socialweb-computer run request release "$n" > $log 2>&1
    rc=$?
    rp=$(cat /root/swc-fence-$n.pid 2>/dev/null); [ -n "$rp" ] && kill -INT $rp 2>/dev/null
    show $rc ;;
esac
EOF
curl -fsS -m 900 -o $t.part '@URL@' && [ "$(sha256sum $t.part | cut -d' ' -f1)" = @SHA@ ] && chmod 0755 $t.part && mv -f $t.part $t && echo "FETCHED $(sha256sum $t | cut -d' ' -f1)""#;

fn r2(op: &str, name: &str, bidder: &str) -> String {
    format!("bash {}/r2.sh {op} {name} {bidder}", CLI.trim_end_matches("/socialweb-computer"))
}

fn fvms(id: &str, cluster: &str) -> abi::Call {
    kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "FirecrackerVM" }))
}

fn range_accepts(rules: &str) -> usize {
    rules.lines().filter(|l| l.contains(&RANGE.0.to_string()) && l.contains(&RANGE.1.to_string()) && l.trim_end().ends_with("accept")).count()
}

fn exit_of(o: &Value) -> i64 {
    o["output"].as_str().unwrap_or_default().lines().rev().find_map(|l| l.strip_prefix("EXIT=")).and_then(|n| n.trim().parse().ok()).unwrap_or(-1)
}

pub fn run_fence(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    // the guest requester's own atirc token: its mint answers 403 "association not verified" for minutes right after the
    // grant (0005's note, rc165; 0005 avoids it with --agent, whose co-located identity-only R2 would take G's one
    // association here); the requester retries it and the rental does not wait on it: counted in readings, not failed fast
    if let Event::Message { id, data, .. } = &run.event {
        if id == "req" && data["msg"] == "requester_association_failed" && data["step"].as_str().is_some_and(|x| x.starts_with("mint the guest requester's atirc token")) {
            let n = s.st["readings"]["atirc_mint_403"].as_u64().unwrap_or(0) + 1;
            s.reading("atirc_mint_403", n);
            return Ok(s.keep(Handle::new()));
        }
    }
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let b = args["bidders"][0].clone();
    let (bname, bdid, bcluster) = (b["name"].clone(), b["did"].as_str().unwrap_or_default().to_string(), b["cluster"].as_str().unwrap_or_default().to_string());
    let vm = s.st["vm"].as_str().unwrap_or_default().to_string();
    let owner = s.st["owner"].as_str().unwrap_or_default().to_string();
    let nft = |id: &str, op: &str| host(id, "bidder.guestNft", json!({ "name": bname, "vm": vm, "owner": owner, "op": op, "min": RANGE.0, "max": RANGE.1 }));
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            if args["target"] == "prod" {
                return Ok(s.unmet("staging only: it upgrades a bidder to another binary"));
            }
            if !bname.is_string() || !bdid.starts_with("did:") || bcluster.is_empty() {
                return Ok(s.unmet("the target's first bidder has no name, DID and kcp"));
            }
            s.set("pre");
            h = h.call(host("pre", "bidder.upgrade", json!({ "name": bname, "binary": "from", "from": FROM }))).timer("deadline", 60 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
        Event::Message { id, data, .. } if id == "req" => match data["msg"].as_str().unwrap_or_default() {
            "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("G's contract failed before Held: {}", data["message"]))),
            "exited" if !s.has("held") => return Ok(s.fail(format!("G's run request exited before Held: {data}"))),
            "contract" if data["phase"] == "Held" && !s.has("held") => {
                s.flag("held");
                let t = s.since("requested");
                s.reading("g_held_s", t);
                h = h.call(fvms("after", &bcluster));
            }
            _ => {}
        },
        Event::Closed { id, error } if id == "req" && !s.has("releasing") => return Ok(s.fail(format!("G's run request closed: {}", error.unwrap_or_default()))),
        Event::Result { id, result, .. } => {
            let o = match out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "pre" => {
                    if o["from"] == o["test"] {
                        return Ok(s.unmet(&format!("the from binary {} ({}) is the binary under test: there is no upgrade to show", o["fromSpec"], o["from"])));
                    }
                    if o["sha256"] != o["from"] {
                        return Ok(s.fail(format!("{bname} runs {} after the upgrade to the from binary {}", o["sha256"], o["from"])));
                    }
                    s.st["from"] = o["from"].clone();
                    s.st["test"] = o["test"].clone();
                    s.reading("from", json!({ "spec": o["fromSpec"], "sha256": o["from"] }));
                    s.reading("test", o["test"].clone());
                    h = h.call(fvms("base", &bcluster));
                }
                "base" => {
                    s.st["base"] = json!(steps::names(&o));
                    s.set("g");
                    s.mark("requested");
                    let a = json!(["--hold", "--guest-tools", "socialweb-computer", "--associate", "guest", "--expires-in", "1h"]);
                    h = h.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": a, "bidderDids": [bdid] })));
                }
                "after" => {
                    let base = s.st["base"].as_array().cloned().unwrap_or_default();
                    let new: Vec<&Value> = o["items"].as_array().into_iter().flatten().filter(|v| v["metadata"]["deletionTimestamp"].is_null() && !base.contains(&v["metadata"]["name"])).collect();
                    let [g] = new.as_slice() else {
                        return Ok(s.fail(format!("{} new FirecrackerVMs on {bname} after G was Held, not one", new.len())));
                    };
                    s.st["vm"] = g["metadata"]["name"].clone();
                    s.st["owner"] = g["spec"]["owner"]["handle"].clone();
                    s.reading("g", json!({ "bidder": bname, "firecrackervm": g["metadata"]["name"] }));
                    let (vm, owner) = (g["metadata"]["name"].as_str().unwrap_or_default(), g["spec"]["owner"]["handle"].as_str().unwrap_or_default());
                    h = h.call(host("rules0", "bidder.guestNft", json!({ "name": bname, "vm": vm, "owner": owner, "op": "list" })));
                }
                "rules0" => {
                    let rules = o["output"].as_str().unwrap_or_default().to_string();
                    s.reading("rules_before", rules.clone());
                    if !rules.contains("table bridge swcguest") {
                        return Ok(s.fail(format!("G's namespace holds no table bridge swcguest: {rules}")));
                    }
                    if range_accepts(&rules) != 0 {
                        return Ok(s.unmet(&format!("the from binary {FROM} ({}) already loads decision 54's accept of {}-{} into G: it cannot show the reload", s.st["from"], RANGE.0, RANGE.1)));
                    }
                    h = h.call(nft("probe", "probe"));
                }
                "probe" => {
                    s.reading("own", o["own"].clone());
                    h = h.call(host("url", "binary.published", json!({})));
                }
                "url" => {
                    let (Some(url), Some(sha)) = (o["url"].as_str(), o["sha256"].as_str()) else {
                        return Ok(s.unmet(&format!("the binary under test has no public URL for G to fetch: {o}")));
                    };
                    s.st["cliSha"] = json!(sha);
                    s.reading("cli", json!({ "url": url, "sha256": sha }));
                    s.mark("fetch");
                    let fetch = FETCH.replace("@CLI@", CLI).replace("@URL@", url).replace("@SHA@", sha);
                    h = h.call(host("fetch", "request.ssh", json!({ "name": name, "exec": fetch })));
                }
                "fetch" => {
                    if !o["output"].as_str().unwrap_or_default().contains(&format!("FETCHED {}", s.st["cliSha"].as_str().unwrap_or_default())) {
                        return Ok(s.fail(format!("G did not fetch the binary under test (its requester for the rentals from inside it): {}", o["output"])));
                    }
                    s.reading("fetch_s", s.since("fetch"));
                    s.mark("rent");
                    h = h.call(host("rent", "request.ssh", json!({ "name": name, "exec": r2("rent", TNAME, &bdid) })));
                }
                "rent" => {
                    s.reading("rent", o["output"].clone());
                    if exit_of(&o) != 0 {
                        return Ok(s.fail(format!("G's guest requester did not hold T on {bname}: {}", o["output"])));
                    }
                    s.reading("t_held_s", s.since("rent"));
                    s.flag("tHeld");
                    s.set("dial0");
                    s.mark("hold");
                    h = h
                        .call(host("hold", "request.ssh", json!({ "name": name, "exec": HOLD })))
                        .call(host("dial0", "request.ssh", json!({ "name": name, "exec": r2("ssh", TNAME, "") })));
                }
                "hold" => {
                    let output = o["output"].as_str().unwrap_or_default();
                    if !s.has("holdReleased") || o["ok"] != true || !output.contains("HOLD-OPEN") || !output.contains("released") || !output.contains("HOLD-END") {
                        return Ok(s.fail(format!("G's ssh session opened before the upgrade did not stay open until it was released: {o}")));
                    }
                    s.reading("hold_session_s", s.since("hold"));
                    s.flag("holdOk");
                }
                "dial0" => {
                    s.reading("dial_before", o["output"].clone());
                    if exit_of(&o) == 0 {
                        return Ok(s.fail(format!("ssh over WebRTC from G to T on {bname} passed under the from binary's rules: the red promote is not reproduced here: {}", o["output"])));
                    }
                    h = h.call(nft("cnt0", "counters"));
                }
                "cnt0" => {
                    s.reading("probe_before", json!({ "sent": o["sent"], "passed": o["passed"] }));
                    if o["sent"].as_u64().unwrap_or(0) == 0 || o["passed"].as_u64() != Some(0) {
                        return Ok(s.fail(format!("before the upgrade G's UDP to {bname}'s own addresses on {}-{}: sent {} passed {} (want sent > 0, passed 0)", RANGE.0, RANGE.1, o["sent"], o["passed"])));
                    }
                    s.set("up");
                    s.mark("upgrade");
                    h = h.call(host("up", "bidder.upgrade", json!({ "name": bname, "binary": "test", "from": FROM })));
                }
                "up" => {
                    if o["sha256"] != s.st["test"] {
                        return Ok(s.fail(format!("{bname} runs {} after the upgrade, not the binary under test {}", o["sha256"], s.st["test"])));
                    }
                    s.reading("upgrade_s", o["seconds"].clone());
                    h = h.call(host("reload", "bidder.awaitLog", json!({ "name": bname, "texts": ["\"msg\":\"guest_rules_reloaded\"", format!("\"firecrackervm\":\"{vm}\"")], "seconds": 600 })));
                }
                "reload" => {
                    if o["found"] != true {
                        return Ok(s.fail(format!("{bname} logged no guest_rules_reloaded for {vm} within 600 s of its upgrade: {}", o["report"])));
                    }
                    let l = &o["line"];
                    s.reading("reloaded", l.clone());
                    s.reading("reload_after_upgrade_s", s.since("upgrade"));
                    if l["webrtcUDPPorts"] != format!("{}-{}", RANGE.0, RANGE.1) {
                        return Ok(s.fail(format!("guest_rules_reloaded for {vm} names webrtcUDPPorts {}, not {}-{}", l["webrtcUDPPorts"], RANGE.0, RANGE.1)));
                    }
                    h = h.call(nft("rules1", "list"));
                }
                "rules1" => {
                    let rules = o["output"].as_str().unwrap_or_default().to_string();
                    s.reading("rules_after", rules.clone());
                    if range_accepts(&rules) == 0 {
                        return Ok(s.fail(format!("after the reload G's swcguest holds no accept of {}-{}: {rules}", RANGE.0, RANGE.1)));
                    }
                    s.set("dial1");
                    h = h.call(host("dial1", "request.ssh", json!({ "name": name, "exec": r2("ssh", TNAME, "") })));
                }
                "dial1" => {
                    s.reading("dial_after", o["output"].clone());
                    if exit_of(&o) != 0 || !o["output"].as_str().unwrap_or_default().contains("Linux") {
                        return Ok(s.fail(format!("ssh over WebRTC from G to T on {bname} after the reload: {}", o["output"])));
                    }
                    h = h.call(nft("cnt1", "counters"));
                }
                "cnt1" => {
                    s.reading("probe_after", json!({ "sent": o["sent"], "passed": o["passed"] }));
                    if o["passed"].as_u64().unwrap_or(0) == 0 {
                        return Ok(s.fail(format!("after the reload no UDP of G to {bname}'s own addresses on {}-{} passed swcguest: sent {} passed {}", RANGE.0, RANGE.1, o["sent"], o["passed"])));
                    }
                    h = h.call(host("down", "bidder.upgrade", json!({ "name": bname, "binary": "from", "from": FROM })));
                }
                "down" => {
                    if o["sha256"] != s.st["from"] {
                        return Ok(s.fail(format!("{bname} runs {} after the rollback, not the from binary {}", o["sha256"], s.st["from"])));
                    }
                    h = h.call(nft("rules2", "list"));
                }
                "rules2" => {
                    let rules = o["output"].as_str().unwrap_or_default();
                    s.reading("rules_after_rollback", rules);
                    if s.st["readings"]["rules_after"] != rules {
                        return Ok(s.fail(format!("after the rollback to the from binary (no reload in it) G's swcguest changed: {rules}")));
                    }
                    h = h.call(host("up2", "bidder.upgrade", json!({ "name": bname, "binary": "test", "from": FROM })));
                }
                "up2" => {
                    if o["sha256"] != s.st["test"] {
                        return Ok(s.fail(format!("{bname} runs {} after the second upgrade, not the binary under test {}", o["sha256"], s.st["test"])));
                    }
                    h = h.call(host("reload2", "bidder.awaitLog", json!({ "name": bname, "texts": ["\"msg\":\"guest_rules_reloaded\"", format!("\"firecrackervm\":\"{vm}\"")], "seconds": 600 })));
                }
                "reload2" => {
                    if o["found"] != true {
                        return Ok(s.fail(format!("{bname} logged no guest_rules_reloaded for {vm} after the second upgrade: {}", o["report"])));
                    }
                    h = h.call(nft("rules3", "list"));
                }
                "rules3" => {
                    let rules = o["output"].as_str().unwrap_or_default();
                    if s.st["readings"]["rules_after"] != rules {
                        return Ok(s.fail(format!("a second reload by the same binary changed G's swcguest: {rules}")));
                    }
                    s.reading("second_reload_identical", true);
                    s.flag("holdReleased");
                    h = h.call(host("free", "request.ssh", json!({ "name": name, "exec": RELEASE_HOLD })));
                }
                "free" => {}
                "unprobe" => {
                    h = h.call(host("tfree", "request.ssh", json!({ "name": name, "exec": r2("release", TNAME, "") })));
                }
                "tfree" => {
                    if exit_of(&o) != 0 {
                        return Ok(s.fail(format!("G's guest requester did not release T: {}", o["output"])));
                    }
                    s.set("release");
                    s.flag("releasing");
                    h = h.cancel("req").call(host("release", "request.release", json!({ "name": name })));
                }
                "release" => {
                    s.reading("release_s", o["seconds"].clone());
                    return Ok(s.pass());
                }
                _ => {}
            }
            if s.has("holdOk") && s.is("dial1") && s.has("holdReleased") && !s.has("unprobed") {
                s.flag("unprobed");
                h = h.call(nft("unprobe", "unprobe"));
            }
        }
        _ => {}
    }
    Ok(s.keep(h))
}