// upgrade (staging, exclusive): a guest placed before its bidder's self-update still gets its binaries // (docs/plans/0040-findings.md, "A SELF-UPDATE TOOK AWAY THE BINARY A LIVE GUEST WAS STILL FETCHING"). Every bidder of the // target is upgraded to the `from` binary (SWC_UPGRADE_FROM, else SWC_COMPAT_REQUESTER: the previous rc) and is ready; a // `--guest-tools socialweb-computer --hold --exec -- id` request places its VM; the moment its ComputeVM shows on a bidder, that // bidder's core is stopped and upgraded to the binary under test, before the guest fetches. The from binary records // nothing for it: the user_data names the from binary's shas, and the new core serves each one it does not pin from // what the install keeps (its own binary from bin//, the guest agent from the from binary's asset tree). The guest // is Held, `id` is uid 0 and, once its setup-guest-binaries.service is active or failed (300 s at most), the unit is // active and its /usr/local/bin/socialweb-computer is the from binary; the bidder's log, read after that, shows the new // core served socialweb-computer-x86_64 at the from binary's sha256, every download it served is what the guest // installed (the guest agent either at the core's own pin or from a kept older downloads tree: reading agentFrom), at // least one `serves a previous binary` and no `pinned_download_not_pinned`. Then the release leaves no // record, every bidder is back on the binary under test, ready, with no restart and no startup ERROR. use abi::{Event, Handle, Run}; use serde_json::{Value, json}; use steps::{MIN, S, host, kube, out}; fn cvms(id: &str, cluster: &str) -> abi::Call { kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })) } fn names(o: &Value) -> Vec { o["items"].as_array().into_iter().flatten().filter_map(|v| v["metadata"]["name"].as_str().map(String::from)).collect() } fn idx(id: &str, prefix: &str) -> Option { id.strip_prefix(prefix).and_then(|n| n.parse().ok()) } pub fn run_upgrade(run: Run) -> Result { let args = &run.args; let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let name = args["name"].as_str().ok_or("name")?.to_string(); let bidders: Vec = args["bidders"].as_array().cloned().unwrap_or_default(); let n = bidders.len(); let bname = |i: usize| bidders[i]["name"].clone(); let bcluster = |i: usize| bidders[i]["cluster"].as_str().unwrap_or_default().to_string(); let health_result = matches!(&run.event, Event::Result { id, .. } if idx(id, "h").is_some()); let mut h = Handle::new(); match run.event { Event::Start => { if args["target"] == "prod" { return Ok(s.unmet("staging only: it upgrades the bidders to another binary")); } if n == 0 || bidders.iter().any(|b| !b["name"].is_string() || !b["cluster"].is_string()) { return Ok(s.unmet("the target names no bidder with a name and a kcp")); } s.set("pre"); for i in 0..n { h = h.call(host(&format!("pre{i}"), "bidder.upgrade", json!({ "name": bname(i), "binary": "from" }))); } h = h.timer("deadline", 60 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))), Event::Timer { id } if id == "health" => { let mut k = 0; for i in 0..n { if !s.has(&format!("ok{i}")) { k += 1; h = h.call(host(&format!("h{i}"), "bidder.health", json!({ "name": bname(i) }))); } } s.st["hpending"] = json!(k); } Event::Timer { id } if id == "scan" && s.is("held") && !s.has("seen") => { s.st["scans"] = json!(n); for i in 0..n { h = h.call(cvms(&format!("scan{i}"), &bcluster(i))); } } Event::Message { id, data, .. } if id == "req" => { if s.st["receipt"].is_null() && data["receipt"].is_string() { s.st["receipt"] = data["receipt"].clone(); } match data["msg"].as_str().unwrap_or_default() { "contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))), "exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")), "contract" if data["phase"] == "Held" && !s.has("held") => { s.flag("held"); let t = s.since("requested"); s.reading("held_s", t); } "ssh returned" => { if data["exit"] != 0 { return Ok(s.fail(format!("id over ssh: {data}"))); } s.flag("ssh"); } "exited" => s.flag("exited"), _ => {} } } Event::Closed { id, error } if id == "req" && !s.has("exited") && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))), Event::Result { id, result, .. } => { let r = result.output(); if let Some(i) = idx(&id, "scan") { if s.has("seen") { return Ok(s.keep(h)); } let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; let base = s.st[format!("base{i}")].clone(); let new = o["items"].as_array().into_iter().flatten().find(|v| v["metadata"]["deletionTimestamp"].is_null() && !base.as_array().into_iter().flatten().any(|b| *b == v["metadata"]["name"])).cloned(); if let Some(v) = new { s.flag("seen"); s.st["vm"] = v["metadata"]["name"].clone(); s.st["winner"] = json!(i); s.reading("computevm", json!({ "bidder": bname(i), "name": v["metadata"]["name"] })); let t = s.since("requested"); s.reading("computevm_seen_s", t); s.mark("upgrade"); h = h.call(host("up", "bidder.upgrade", json!({ "name": bname(i), "binary": "test", "stopFirst": true }))); return Ok(s.keep(h)); } let left = s.st["scans"].as_u64().unwrap_or(1).saturating_sub(1); s.st["scans"] = json!(left); if left == 0 { h = h.timer("scan", 1_000); } return Ok(s.keep(h)); } let o = match out(&s, &id, r) { Ok(o) => o, Err(f) => return Ok(f), }; if let Some(i) = idx(&id, "pre") { if o["from"] == o["test"] { return Ok(s.unmet(&format!("the from binary ({}) is the binary under test: channel rc already points at this commit, so `test` ran again after a green promote moved rc (a normal cycle runs `test` before promote moves rc); set SWC_UPGRADE_FROM to an earlier release", o["fromSpec"]))); } if o["sha256"] != o["from"] { return Ok(s.fail(format!("{} runs {} after the upgrade to the from binary {}", bname(i), o["sha256"], o["from"]))); } s.st["from"] = o["from"].clone(); s.st["test"] = o["test"].clone(); s.reading("from", json!({ "spec": o["fromSpec"], "sha256": o["from"] })); s.reading("test", o["test"].clone()); s.flag(&format!("pre{i}")); } else if let Some(i) = idx(&id, "base") { s.st[format!("base{i}")] = json!(names(&o)); } else if let Some(i) = idx(&id, "h") { let ready = o["guestImageReadyAt"].as_str().is_some_and(|t| !t.is_empty()); if s.is("back") && ready && (o["restarts"] != 0 || o["startupErrors"] != 0) { return Ok(s.fail(format!("{} after the upgrade back: {o}", bname(i)))); } if ready { s.flag(&format!("ok{i}")); s.st["health"][bname(i).as_str().unwrap_or_default()] = o.clone(); } } else if let Some(i) = idx(&id, "b") { if o["sha256"] != s.st["test"] { return Ok(s.fail(format!("{} runs {} after the upgrade back, not the binary under test {}", bname(i), o["sha256"], s.st["test"]))); } s.flag(&format!("b{i}")); } else { match id.as_str() { "up" => { if o["sha256"] != s.st["test"] { return Ok(s.fail(format!("the winner runs {} after the upgrade, not the binary under test {}", o["sha256"], s.st["test"]))); } s.reading("upgrade_s", o["seconds"].clone()); s.flag("upgraded"); } "id" => { s.st["idOut"] = o["output"].clone(); let w = s.st["winner"].as_u64().unwrap_or(0) as usize; h = h .call(host("served", "bidder.logLines", json!({ "name": bname(w), "texts": [["\"msg\":\"pinned_download_served\""]], "keys": ["name", "sha256", "path"] }))) .call(host("counts", "bidder.logCount", json!({ "name": bname(w), "texts": ["serves a previous binary", "pinned_download_not_pinned"] }))); } "served" => s.st["served"] = o["lines"].clone(), "counts" => s.st["counts"] = o["counts"].clone(), "release" => { s.reading("release_s", o["seconds"].clone()); h = h.call(host("records", "request.records", json!({ "name": name }))); } "records" => { s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len)); h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] }))); } "gone" => { if o["left"].as_array().is_none_or(|l| !l.is_empty()) { return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"]))); } s.set("back"); for i in 0..n { h = h.call(host(&format!("b{i}"), "bidder.upgrade", json!({ "name": bname(i), "binary": "test" }))); } } _ => {} } } } _ => {} } if s.is("pre") && (0..n).all(|i| s.has(&format!("pre{i}"))) { s.set("ready"); s.mark("ready"); h = h.timer("health", 1); } else if (s.is("ready") || s.is("back")) && health_result { let left = s.st["hpending"].as_u64().unwrap_or(1).saturating_sub(1); s.st["hpending"] = json!(left); if left == 0 && (0..n).all(|i| s.has(&format!("ok{i}"))) { if s.is("back") { s.reading("health", s.st["health"].clone()); return Ok(s.pass()); } // each bidder's ComputeVMs once it is ready on the from binary (a list beside the upgrade meets its core mid-restart) s.set("base"); for i in 0..n { s.st["f"][format!("ok{i}")] = json!(false); h = h.call(cvms(&format!("base{i}"), &bcluster(i))); } } else if left == 0 && s.since(if s.is("back") { "back" } else { "ready" }) > 600.0 { return Ok(s.fail(format!("bidders not ready 10 min after the upgrade: {}", s.st["health"]))); } else if left == 0 { h = h.timer("health", 5_000); } } else if s.is("base") && (0..n).all(|i| s.st[format!("base{i}")].is_array()) { s.set("held"); s.mark("requested"); h = h .call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--guest-tools", "socialweb-computer", "--hold", "--exec", "--", "id"] }))) .timer("scan", 1_000); } else if s.is("back") && (0..n).all(|i| s.has(&format!("b{i}"))) && !s.has("backAsked") { s.flag("backAsked"); s.mark("back"); h = h.timer("health", 1); } if s.is("held") && s.has("held") && s.has("ssh") && s.has("exited") && s.has("upgraded") { let w = s.st["winner"].as_u64().unwrap_or(0) as usize; let author = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string(); if bidders[w]["did"].as_str() != Some(author.as_str()) { return Ok(s.fail(format!("the receipt's author {author} is not {} whose ComputeVM was seen first", bname(w)))); } s.set("verify"); h = h .cancel("req") .call(host("id", "request.ssh", json!({ "name": name, "exec": "id; timeout 300 sh -c 'until systemctl is-active -q setup-guest-binaries.service || systemctl is-failed -q setup-guest-binaries.service; do sleep 2; done'; systemctl is-active setup-guest-binaries.service; sha256sum /usr/local/bin/socialweb-computer /usr/local/bin/socialweb-guest-agent" }))); } if s.is("verify") && !s.st["idOut"].is_null() && !s.st["served"].is_null() && !s.st["counts"].is_null() { let mut p = vec![]; let id_out = s.st["idOut"].as_str().unwrap_or_default().to_string(); s.reading("guest", id_out.clone()); let installed = |file: &str| id_out.lines().find(|l| l.ends_with(&format!("/usr/local/bin/{file}"))).and_then(|l| l.split_whitespace().next()).unwrap_or_default().to_string(); let served = s.st["served"].as_array().cloned().unwrap_or_default(); s.reading("served", json!(served)); if !served.iter().any(|l| l["name"] == "socialweb-computer-x86_64" && l["sha256"] == s.st["from"]) { p.push(format!("the upgraded core did not serve socialweb-computer-x86_64 {} (the guest fetched it before the upgrade, or never)", s.st["from"])); } for l in &served { let Some(file) = l["name"].as_str().and_then(|n| n.strip_suffix("-x86_64")) else { continue }; let got = installed(file); if json!(got) != l["sha256"] { p.push(format!("the upgraded core served {} {}, the guest installed {got:?}", l["name"], l["sha256"])); } } // the guest agent: a line with no path is the upgraded core's own pin (its downloads tree, held in memory); one // with a path is a kept file, which for the agent must be an older downloads tree, never bin/ let agent = installed("socialweb-guest-agent"); let agent_line = served.iter().find(|l| l["name"] == "socialweb-guest-agent-x86_64" && json!(agent) == l["sha256"]); let agent_from = match agent_line.map(|l| l["path"].as_str().unwrap_or_default()) { None => { p.push(format!("the upgraded core did not serve the guest agent the guest installed ({agent:?}): it differs from the core's own pin and no kept downloads tree served it, or the guest fetched it before the upgrade")); json!(null) } Some("") => json!("pin"), Some(path) if path.ends_with("/socialweb-guest-agent-x86_64") && !path.contains("/bin/") => json!({ "tree": path }), Some(path) => { p.push(format!("the upgraded core served the guest agent {agent} from {path}, not from a kept downloads tree")); json!({ "other": path }) } }; s.reading("agentFrom", agent_from); let previous = s.st["counts"]["serves a previous binary"].as_u64().unwrap_or(0); let unpinned = s.st["counts"]["pinned_download_not_pinned"].as_u64().unwrap_or(0); s.reading("counts", json!({ "servesPrevious": previous, "notPinned": unpinned })); if previous == 0 { p.push("the upgraded core logged no `serves a previous binary`".to_string()); } if unpinned != 0 { p.push(format!("the upgraded core logged {unpinned} pinned_download_not_pinned")); } if !id_out.contains("uid=0(") { p.push(format!("id in the guest: {id_out}")); } if !id_out.lines().any(|l| l.trim() == "active") { p.push(format!("setup-guest-binaries.service is not active in the guest: {id_out}")); } let computer = installed("socialweb-computer"); if json!(computer) != s.st["from"] { p.push(format!("the guest's /usr/local/bin/socialweb-computer is {computer:?}, not the from binary {}", s.st["from"])); } if !p.is_empty() { return Ok(s.fail(p.join("; "))); } s.set("release"); s.flag("releasing"); h = h.call(host("release", "request.release", json!({ "name": name }))); } Ok(s.keep(h)) }