~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
306 lines · 17.1 KB Raw History
// upgrade (staging, exclusive): a guest placed before its bidder's self-update still gets its binaries
// (docs/plans/0040-findings.md, "A SELF-UPDATE TOOK AWAY THE BINARY A LIVE GUEST WAS STILL FETCHING"). Every bidder of the
// target is upgraded to the `from` binary (SWC_UPGRADE_FROM, else SWC_COMPAT_REQUESTER: the previous rc) and is ready; a
// `--guest-tools socialweb-computer --hold --exec -- id` request places its VM; the moment its ComputeVM shows on a bidder, that
// bidder's core is stopped and upgraded to the binary under test, before the guest fetches. The from binary records
// nothing for it: the user_data names the from binary's shas, and the new core serves each one it does not pin from
// what the install keeps (its own binary from bin/<sha256>/, the guest agent from the from binary's asset tree). The guest
// is Held, `id` is uid 0 and, once its setup-guest-binaries.service is active or failed (300 s at most), the unit is
// active and its /usr/local/bin/socialweb-computer is the from binary; the bidder's log, read after that, shows the new
// core served socialweb-computer-x86_64 at the from binary's sha256, every download it served is what the guest
// installed (the guest agent either at the core's own pin or from a kept older downloads tree: reading agentFrom), at
// least one `serves a previous binary` and no `pinned_download_not_pinned`. Then the release leaves no
// record, every bidder is back on the binary under test, ready, with no restart and no startup ERROR.
use abi::{Event, Handle, Run};
use serde_json::{Value, json};
use steps::{MIN, S, host, kube, out};

fn cvms(id: &str, cluster: &str) -> abi::Call {
    kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))
}

fn names(o: &Value) -> Vec<String> {
    o["items"].as_array().into_iter().flatten().filter_map(|v| v["metadata"]["name"].as_str().map(String::from)).collect()
}

fn idx(id: &str, prefix: &str) -> Option<usize> {
    id.strip_prefix(prefix).and_then(|n| n.parse().ok())
}

pub fn run_upgrade(run: Run<Value>) -> Result<Handle, String> {
    let args = &run.args;
    let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let name = args["name"].as_str().ok_or("name")?.to_string();
    let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
    let n = bidders.len();
    let bname = |i: usize| bidders[i]["name"].clone();
    let bcluster = |i: usize| bidders[i]["cluster"].as_str().unwrap_or_default().to_string();
    let health_result = matches!(&run.event, Event::Result { id, .. } if idx(id, "h").is_some());
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            if args["target"] == "prod" {
                return Ok(s.unmet("staging only: it upgrades the bidders to another binary"));
            }
            if n == 0 || bidders.iter().any(|b| !b["name"].is_string() || !b["cluster"].is_string()) {
                return Ok(s.unmet("the target names no bidder with a name and a kcp"));
            }
            s.set("pre");
            for i in 0..n {
                h = h.call(host(&format!("pre{i}"), "bidder.upgrade", json!({ "name": bname(i), "binary": "from" })));
            }
            h = h.timer("deadline", 60 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "health" => {
            let mut k = 0;
            for i in 0..n {
                if !s.has(&format!("ok{i}")) {
                    k += 1;
                    h = h.call(host(&format!("h{i}"), "bidder.health", json!({ "name": bname(i) })));
                }
            }
            s.st["hpending"] = json!(k);
        }
        Event::Timer { id } if id == "scan" && s.is("held") && !s.has("seen") => {
            s.st["scans"] = json!(n);
            for i in 0..n {
                h = h.call(cvms(&format!("scan{i}"), &bcluster(i)));
            }
        }
        Event::Message { id, data, .. } if id == "req" => {
            if s.st["receipt"].is_null() && data["receipt"].is_string() {
                s.st["receipt"] = data["receipt"].clone();
            }
            match data["msg"].as_str().unwrap_or_default() {
                "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
                "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
                "contract" if data["phase"] == "Held" && !s.has("held") => {
                    s.flag("held");
                    let t = s.since("requested");
                    s.reading("held_s", t);
                }
                "ssh returned" => {
                    if data["exit"] != 0 {
                        return Ok(s.fail(format!("id over ssh: {data}")));
                    }
                    s.flag("ssh");
                }
                "exited" => s.flag("exited"),
                _ => {}
            }
        }
        Event::Closed { id, error } if id == "req" && !s.has("exited") && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
        Event::Result { id, result, .. } => {
            let r = result.output();
            if let Some(i) = idx(&id, "scan") {
                if s.has("seen") {
                    return Ok(s.keep(h));
                }
                let o = match out(&s, &id, r) {
                    Ok(o) => o,
                    Err(f) => return Ok(f),
                };
                let base = s.st[format!("base{i}")].clone();
                let new = o["items"].as_array().into_iter().flatten().find(|v| v["metadata"]["deletionTimestamp"].is_null() && !base.as_array().into_iter().flatten().any(|b| *b == v["metadata"]["name"])).cloned();
                if let Some(v) = new {
                    s.flag("seen");
                    s.st["vm"] = v["metadata"]["name"].clone();
                    s.st["winner"] = json!(i);
                    s.reading("computevm", json!({ "bidder": bname(i), "name": v["metadata"]["name"] }));
                    let t = s.since("requested");
                    s.reading("computevm_seen_s", t);
                    s.mark("upgrade");
                    h = h.call(host("up", "bidder.upgrade", json!({ "name": bname(i), "binary": "test", "stopFirst": true })));
                    return Ok(s.keep(h));
                }
                let left = s.st["scans"].as_u64().unwrap_or(1).saturating_sub(1);
                s.st["scans"] = json!(left);
                if left == 0 {
                    h = h.timer("scan", 1_000);
                }
                return Ok(s.keep(h));
            }
            let o = match out(&s, &id, r) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            if let Some(i) = idx(&id, "pre") {
                if o["from"] == o["test"] {
                    return Ok(s.unmet(&format!("the from binary ({}) is the binary under test: channel rc already points at this commit, so `test` ran again after a green promote moved rc (a normal cycle runs `test` before promote moves rc); set SWC_UPGRADE_FROM to an earlier release", o["fromSpec"])));
                }
                if o["sha256"] != o["from"] {
                    return Ok(s.fail(format!("{} runs {} after the upgrade to the from binary {}", bname(i), o["sha256"], o["from"])));
                }
                s.st["from"] = o["from"].clone();
                s.st["test"] = o["test"].clone();
                s.reading("from", json!({ "spec": o["fromSpec"], "sha256": o["from"] }));
                s.reading("test", o["test"].clone());
                s.flag(&format!("pre{i}"));
            } else if let Some(i) = idx(&id, "base") {
                s.st[format!("base{i}")] = json!(names(&o));
            } else if let Some(i) = idx(&id, "h") {
                let ready = o["guestImageReadyAt"].as_str().is_some_and(|t| !t.is_empty());
                if s.is("back") && ready && (o["restarts"] != 0 || o["startupErrors"] != 0) {
                    return Ok(s.fail(format!("{} after the upgrade back: {o}", bname(i))));
                }
                if ready {
                    s.flag(&format!("ok{i}"));
                    s.st["health"][bname(i).as_str().unwrap_or_default()] = o.clone();
                }
            } else if let Some(i) = idx(&id, "b") {
                if o["sha256"] != s.st["test"] {
                    return Ok(s.fail(format!("{} runs {} after the upgrade back, not the binary under test {}", bname(i), o["sha256"], s.st["test"])));
                }
                s.flag(&format!("b{i}"));
            } else {
                match id.as_str() {
                    "up" => {
                        if o["sha256"] != s.st["test"] {
                            return Ok(s.fail(format!("the winner runs {} after the upgrade, not the binary under test {}", o["sha256"], s.st["test"])));
                        }
                        s.reading("upgrade_s", o["seconds"].clone());
                        s.flag("upgraded");
                    }
                    "id" => {
                        s.st["idOut"] = o["output"].clone();
                        let w = s.st["winner"].as_u64().unwrap_or(0) as usize;
                        h = h
                            .call(host("served", "bidder.logLines", json!({ "name": bname(w), "texts": [["\"msg\":\"pinned_download_served\""]], "keys": ["name", "sha256", "path"] })))
                            .call(host("counts", "bidder.logCount", json!({ "name": bname(w), "texts": ["serves a previous binary", "pinned_download_not_pinned"] })));
                    }
                    "served" => s.st["served"] = o["lines"].clone(),
                    "counts" => s.st["counts"] = o["counts"].clone(),
                    "release" => {
                        s.reading("release_s", o["seconds"].clone());
                        h = h.call(host("records", "request.records", json!({ "name": name })));
                    }
                    "records" => {
                        s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
                        h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
                    }
                    "gone" => {
                        if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
                            return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
                        }
                        s.set("back");
                        for i in 0..n {
                            h = h.call(host(&format!("b{i}"), "bidder.upgrade", json!({ "name": bname(i), "binary": "test" })));
                        }
                    }
                    _ => {}
                }
            }
        }
        _ => {}
    }
    if s.is("pre") && (0..n).all(|i| s.has(&format!("pre{i}"))) {
        s.set("ready");
        s.mark("ready");
        h = h.timer("health", 1);
    } else if (s.is("ready") || s.is("back")) && health_result {
        let left = s.st["hpending"].as_u64().unwrap_or(1).saturating_sub(1);
        s.st["hpending"] = json!(left);
        if left == 0 && (0..n).all(|i| s.has(&format!("ok{i}"))) {
            if s.is("back") {
                s.reading("health", s.st["health"].clone());
                return Ok(s.pass());
            }
            // each bidder's ComputeVMs once it is ready on the from binary (a list beside the upgrade meets its core mid-restart)
            s.set("base");
            for i in 0..n {
                s.st["f"][format!("ok{i}")] = json!(false);
                h = h.call(cvms(&format!("base{i}"), &bcluster(i)));
            }
        } else if left == 0 && s.since(if s.is("back") { "back" } else { "ready" }) > 600.0 {
            return Ok(s.fail(format!("bidders not ready 10 min after the upgrade: {}", s.st["health"])));
        } else if left == 0 {
            h = h.timer("health", 5_000);
        }
    } else if s.is("base") && (0..n).all(|i| s.st[format!("base{i}")].is_array()) {
        s.set("held");
        s.mark("requested");
        h = h
            .call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--guest-tools", "socialweb-computer", "--hold", "--exec", "--", "id"] })))
            .timer("scan", 1_000);
    } else if s.is("back") && (0..n).all(|i| s.has(&format!("b{i}"))) && !s.has("backAsked") {
        s.flag("backAsked");
        s.mark("back");
        h = h.timer("health", 1);
    }
    if s.is("held") && s.has("held") && s.has("ssh") && s.has("exited") && s.has("upgraded") {
        let w = s.st["winner"].as_u64().unwrap_or(0) as usize;
        let author = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
        if bidders[w]["did"].as_str() != Some(author.as_str()) {
            return Ok(s.fail(format!("the receipt's author {author} is not {} whose ComputeVM was seen first", bname(w))));
        }
        s.set("verify");
        h = h
            .cancel("req")
            .call(host("id", "request.ssh", json!({ "name": name, "exec": "id; timeout 300 sh -c 'until systemctl is-active -q setup-guest-binaries.service || systemctl is-failed -q setup-guest-binaries.service; do sleep 2; done'; systemctl is-active setup-guest-binaries.service; sha256sum /usr/local/bin/socialweb-computer /usr/local/bin/socialweb-guest-agent" })));
    }
    if s.is("verify") && !s.st["idOut"].is_null() && !s.st["served"].is_null() && !s.st["counts"].is_null() {
        let mut p = vec![];
        let id_out = s.st["idOut"].as_str().unwrap_or_default().to_string();
        s.reading("guest", id_out.clone());
        let installed = |file: &str| id_out.lines().find(|l| l.ends_with(&format!("/usr/local/bin/{file}"))).and_then(|l| l.split_whitespace().next()).unwrap_or_default().to_string();
        let served = s.st["served"].as_array().cloned().unwrap_or_default();
        s.reading("served", json!(served));
        if !served.iter().any(|l| l["name"] == "socialweb-computer-x86_64" && l["sha256"] == s.st["from"]) {
            p.push(format!("the upgraded core did not serve socialweb-computer-x86_64 {} (the guest fetched it before the upgrade, or never)", s.st["from"]));
        }
        for l in &served {
            let Some(file) = l["name"].as_str().and_then(|n| n.strip_suffix("-x86_64")) else { continue };
            let got = installed(file);
            if json!(got) != l["sha256"] {
                p.push(format!("the upgraded core served {} {}, the guest installed {got:?}", l["name"], l["sha256"]));
            }
        }
        // the guest agent: a line with no path is the upgraded core's own pin (its downloads tree, held in memory); one
        // with a path is a kept file, which for the agent must be an older downloads tree, never bin/
        let agent = installed("socialweb-guest-agent");
        let agent_line = served.iter().find(|l| l["name"] == "socialweb-guest-agent-x86_64" && json!(agent) == l["sha256"]);
        let agent_from = match agent_line.map(|l| l["path"].as_str().unwrap_or_default()) {
            None => {
                p.push(format!("the upgraded core did not serve the guest agent the guest installed ({agent:?}): it differs from the core's own pin and no kept downloads tree served it, or the guest fetched it before the upgrade"));
                json!(null)
            }
            Some("") => json!("pin"),
            Some(path) if path.ends_with("/socialweb-guest-agent-x86_64") && !path.contains("/bin/") => json!({ "tree": path }),
            Some(path) => {
                p.push(format!("the upgraded core served the guest agent {agent} from {path}, not from a kept downloads tree"));
                json!({ "other": path })
            }
        };
        s.reading("agentFrom", agent_from);
        let previous = s.st["counts"]["serves a previous binary"].as_u64().unwrap_or(0);
        let unpinned = s.st["counts"]["pinned_download_not_pinned"].as_u64().unwrap_or(0);
        s.reading("counts", json!({ "servesPrevious": previous, "notPinned": unpinned }));
        if previous == 0 {
            p.push("the upgraded core logged no `serves a previous binary`".to_string());
        }
        if unpinned != 0 {
            p.push(format!("the upgraded core logged {unpinned} pinned_download_not_pinned"));
        }
        if !id_out.contains("uid=0(") {
            p.push(format!("id in the guest: {id_out}"));
        }
        if !id_out.lines().any(|l| l.trim() == "active") {
            p.push(format!("setup-guest-binaries.service is not active in the guest: {id_out}"));
        }
        let computer = installed("socialweb-computer");
        if json!(computer) != s.st["from"] {
            p.push(format!("the guest's /usr/local/bin/socialweb-computer is {computer:?}, not the from binary {}", s.st["from"]));
        }
        if !p.is_empty() {
            return Ok(s.fail(p.join("; ")));
        }
        s.set("release");
        s.flag("releasing");
        h = h.call(host("release", "request.release", json!({ "name": name })));
    }
    Ok(s.keep(h))
}