0
0
// upgrade (staging, exclusive): a guest placed before its bidder's self-update still gets its binaries
// (docs/plans/0040-findings.md, "A SELF-UPDATE TOOK AWAY THE BINARY A LIVE GUEST WAS STILL FETCHING"). Every bidder of the
// target is upgraded to the `from` binary (SWC_UPGRADE_FROM, else SWC_COMPAT_REQUESTER: the previous rc) and is ready; a
// `--guest-tools socialweb-computer --hold --exec -- id` request places its VM; the moment its ComputeVM shows on a bidder, that
// bidder's core is stopped and upgraded to the binary under test, before the guest fetches. The from binary records
// nothing for it: the user_data names the from binary's shas, and the new core serves each one it does not pin from
// what the install keeps (its own binary from bin/<sha256>/, the guest agent from the from binary's asset tree). The guest
// is Held, `id` is uid 0 and, once its setup-guest-binaries.service is active or failed (300 s at most), the unit is
// active and its /usr/local/bin/socialweb-computer is the from binary; the bidder's log, read after that, shows the new
// core served socialweb-computer-x86_64 at the from binary's sha256, every download it served is what the guest
// installed (the guest agent either at the core's own pin or from a kept older downloads tree: reading agentFrom), at
// least one `serves a previous binary` and no `pinned_download_not_pinned`. Then the release leaves no
// record, every bidder is back on the binary under test, ready, with no restart and no startup ERROR.
use abi::{Event, Handle, Run};
use serde_json::{Value, json};
use steps::{MIN, S, host, kube, out};
fn cvms(id: &str, cluster: &str) -> abi::Call {
kube(id, "list", cluster, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))
}
fn names(o: &Value) -> Vec<String> {
o["items"].as_array().into_iter().flatten().filter_map(|v| v["metadata"]["name"].as_str().map(String::from)).collect()
}
fn idx(id: &str, prefix: &str) -> Option<usize> {
id.strip_prefix(prefix).and_then(|n| n.parse().ok())
}
pub fn run_upgrade(run: Run<Value>) -> Result<Handle, String> {
let args = &run.args;
let mut s = S::new(run.state, args["now"].as_str().unwrap_or_default());
if let Some(h) = steps::market_event(&mut s, &run.event) {
return Ok(h);
}
let name = args["name"].as_str().ok_or("name")?.to_string();
let bidders: Vec<Value> = args["bidders"].as_array().cloned().unwrap_or_default();
let n = bidders.len();
let bname = |i: usize| bidders[i]["name"].clone();
let bcluster = |i: usize| bidders[i]["cluster"].as_str().unwrap_or_default().to_string();
let health_result = matches!(&run.event, Event::Result { id, .. } if idx(id, "h").is_some());
let mut h = Handle::new();
match run.event {
Event::Start => {
if args["target"] == "prod" {
return Ok(s.unmet("staging only: it upgrades the bidders to another binary"));
}
if n == 0 || bidders.iter().any(|b| !b["name"].is_string() || !b["cluster"].is_string()) {
return Ok(s.unmet("the target names no bidder with a name and a kcp"));
}
s.set("pre");
for i in 0..n {
h = h.call(host(&format!("pre{i}"), "bidder.upgrade", json!({ "name": bname(i), "binary": "from" })));
}
h = h.timer("deadline", 60 * MIN);
}
Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 60 min (phase {})", s.phase()))),
Event::Timer { id } if id == "health" => {
let mut k = 0;
for i in 0..n {
if !s.has(&format!("ok{i}")) {
k += 1;
h = h.call(host(&format!("h{i}"), "bidder.health", json!({ "name": bname(i) })));
}
}
s.st["hpending"] = json!(k);
}
Event::Timer { id } if id == "scan" && s.is("held") && !s.has("seen") => {
s.st["scans"] = json!(n);
for i in 0..n {
h = h.call(cvms(&format!("scan{i}"), &bcluster(i)));
}
}
Event::Message { id, data, .. } if id == "req" => {
if s.st["receipt"].is_null() && data["receipt"].is_string() {
s.st["receipt"] = data["receipt"].clone();
}
match data["msg"].as_str().unwrap_or_default() {
"contract" if data["phase"] == "Failed" && !s.has("held") => return Ok(s.fail(format!("the contract failed before Held: {}", data["message"]))),
"exited" if !s.has("held") => return Ok(s.fail("run request exited before the VM was Held")),
"contract" if data["phase"] == "Held" && !s.has("held") => {
s.flag("held");
let t = s.since("requested");
s.reading("held_s", t);
}
"ssh returned" => {
if data["exit"] != 0 {
return Ok(s.fail(format!("id over ssh: {data}")));
}
s.flag("ssh");
}
"exited" => s.flag("exited"),
_ => {}
}
}
Event::Closed { id, error } if id == "req" && !s.has("exited") && !s.has("releasing") => return Ok(s.fail(format!("req closed: {}", error.unwrap_or_default()))),
Event::Result { id, result, .. } => {
let r = result.output();
if let Some(i) = idx(&id, "scan") {
if s.has("seen") {
return Ok(s.keep(h));
}
let o = match out(&s, &id, r) {
Ok(o) => o,
Err(f) => return Ok(f),
};
let base = s.st[format!("base{i}")].clone();
let new = o["items"].as_array().into_iter().flatten().find(|v| v["metadata"]["deletionTimestamp"].is_null() && !base.as_array().into_iter().flatten().any(|b| *b == v["metadata"]["name"])).cloned();
if let Some(v) = new {
s.flag("seen");
s.st["vm"] = v["metadata"]["name"].clone();
s.st["winner"] = json!(i);
s.reading("computevm", json!({ "bidder": bname(i), "name": v["metadata"]["name"] }));
let t = s.since("requested");
s.reading("computevm_seen_s", t);
s.mark("upgrade");
h = h.call(host("up", "bidder.upgrade", json!({ "name": bname(i), "binary": "test", "stopFirst": true })));
return Ok(s.keep(h));
}
let left = s.st["scans"].as_u64().unwrap_or(1).saturating_sub(1);
s.st["scans"] = json!(left);
if left == 0 {
h = h.timer("scan", 1_000);
}
return Ok(s.keep(h));
}
let o = match out(&s, &id, r) {
Ok(o) => o,
Err(f) => return Ok(f),
};
if let Some(i) = idx(&id, "pre") {
if o["from"] == o["test"] {
return Ok(s.unmet(&format!("the from binary ({}) is the binary under test: channel rc already points at this commit, so `test` ran again after a green promote moved rc (a normal cycle runs `test` before promote moves rc); set SWC_UPGRADE_FROM to an earlier release", o["fromSpec"])));
}
if o["sha256"] != o["from"] {
return Ok(s.fail(format!("{} runs {} after the upgrade to the from binary {}", bname(i), o["sha256"], o["from"])));
}
s.st["from"] = o["from"].clone();
s.st["test"] = o["test"].clone();
s.reading("from", json!({ "spec": o["fromSpec"], "sha256": o["from"] }));
s.reading("test", o["test"].clone());
s.flag(&format!("pre{i}"));
} else if let Some(i) = idx(&id, "base") {
s.st[format!("base{i}")] = json!(names(&o));
} else if let Some(i) = idx(&id, "h") {
let ready = o["guestImageReadyAt"].as_str().is_some_and(|t| !t.is_empty());
if s.is("back") && ready && (o["restarts"] != 0 || o["startupErrors"] != 0) {
return Ok(s.fail(format!("{} after the upgrade back: {o}", bname(i))));
}
if ready {
s.flag(&format!("ok{i}"));
s.st["health"][bname(i).as_str().unwrap_or_default()] = o.clone();
}
} else if let Some(i) = idx(&id, "b") {
if o["sha256"] != s.st["test"] {
return Ok(s.fail(format!("{} runs {} after the upgrade back, not the binary under test {}", bname(i), o["sha256"], s.st["test"])));
}
s.flag(&format!("b{i}"));
} else {
match id.as_str() {
"up" => {
if o["sha256"] != s.st["test"] {
return Ok(s.fail(format!("the winner runs {} after the upgrade, not the binary under test {}", o["sha256"], s.st["test"])));
}
s.reading("upgrade_s", o["seconds"].clone());
s.flag("upgraded");
}
"id" => {
s.st["idOut"] = o["output"].clone();
let w = s.st["winner"].as_u64().unwrap_or(0) as usize;
h = h
.call(host("served", "bidder.logLines", json!({ "name": bname(w), "texts": [["\"msg\":\"pinned_download_served\""]], "keys": ["name", "sha256", "path"] })))
.call(host("counts", "bidder.logCount", json!({ "name": bname(w), "texts": ["serves a previous binary", "pinned_download_not_pinned"] })));
}
"served" => s.st["served"] = o["lines"].clone(),
"counts" => s.st["counts"] = o["counts"].clone(),
"release" => {
s.reading("release_s", o["seconds"].clone());
h = h.call(host("records", "request.records", json!({ "name": name })));
}
"records" => {
s.reading("records_checked", o["uris"].as_array().map_or(0, Vec::len));
h = h.call(host("gone", "assert.recordsGone", json!({ "uris": o["uris"] })));
}
"gone" => {
if o["left"].as_array().is_none_or(|l| !l.is_empty()) {
return Ok(s.fail(format!("records left: {} {}", o["left"], o["errors"])));
}
s.set("back");
for i in 0..n {
h = h.call(host(&format!("b{i}"), "bidder.upgrade", json!({ "name": bname(i), "binary": "test" })));
}
}
_ => {}
}
}
}
_ => {}
}
if s.is("pre") && (0..n).all(|i| s.has(&format!("pre{i}"))) {
s.set("ready");
s.mark("ready");
h = h.timer("health", 1);
} else if (s.is("ready") || s.is("back")) && health_result {
let left = s.st["hpending"].as_u64().unwrap_or(1).saturating_sub(1);
s.st["hpending"] = json!(left);
if left == 0 && (0..n).all(|i| s.has(&format!("ok{i}"))) {
if s.is("back") {
s.reading("health", s.st["health"].clone());
return Ok(s.pass());
}
// each bidder's ComputeVMs once it is ready on the from binary (a list beside the upgrade meets its core mid-restart)
s.set("base");
for i in 0..n {
s.st["f"][format!("ok{i}")] = json!(false);
h = h.call(cvms(&format!("base{i}"), &bcluster(i)));
}
} else if left == 0 && s.since(if s.is("back") { "back" } else { "ready" }) > 600.0 {
return Ok(s.fail(format!("bidders not ready 10 min after the upgrade: {}", s.st["health"])));
} else if left == 0 {
h = h.timer("health", 5_000);
}
} else if s.is("base") && (0..n).all(|i| s.st[format!("base{i}")].is_array()) {
s.set("held");
s.mark("requested");
h = h
.call(steps::host_sub("req", "request.start", json!({ "name": name, "args": ["--guest-tools", "socialweb-computer", "--hold", "--exec", "--", "id"] })))
.timer("scan", 1_000);
} else if s.is("back") && (0..n).all(|i| s.has(&format!("b{i}"))) && !s.has("backAsked") {
s.flag("backAsked");
s.mark("back");
h = h.timer("health", 1);
}
if s.is("held") && s.has("held") && s.has("ssh") && s.has("exited") && s.has("upgraded") {
let w = s.st["winner"].as_u64().unwrap_or(0) as usize;
let author = s.st["receipt"].as_str().unwrap_or_default().trim_start_matches("at://").split('/').next().unwrap_or_default().to_string();
if bidders[w]["did"].as_str() != Some(author.as_str()) {
return Ok(s.fail(format!("the receipt's author {author} is not {} whose ComputeVM was seen first", bname(w))));
}
s.set("verify");
h = h
.cancel("req")
.call(host("id", "request.ssh", json!({ "name": name, "exec": "id; timeout 300 sh -c 'until systemctl is-active -q setup-guest-binaries.service || systemctl is-failed -q setup-guest-binaries.service; do sleep 2; done'; systemctl is-active setup-guest-binaries.service; sha256sum /usr/local/bin/socialweb-computer /usr/local/bin/socialweb-guest-agent" })));
}
if s.is("verify") && !s.st["idOut"].is_null() && !s.st["served"].is_null() && !s.st["counts"].is_null() {
let mut p = vec![];
let id_out = s.st["idOut"].as_str().unwrap_or_default().to_string();
s.reading("guest", id_out.clone());
let installed = |file: &str| id_out.lines().find(|l| l.ends_with(&format!("/usr/local/bin/{file}"))).and_then(|l| l.split_whitespace().next()).unwrap_or_default().to_string();
let served = s.st["served"].as_array().cloned().unwrap_or_default();
s.reading("served", json!(served));
if !served.iter().any(|l| l["name"] == "socialweb-computer-x86_64" && l["sha256"] == s.st["from"]) {
p.push(format!("the upgraded core did not serve socialweb-computer-x86_64 {} (the guest fetched it before the upgrade, or never)", s.st["from"]));
}
for l in &served {
let Some(file) = l["name"].as_str().and_then(|n| n.strip_suffix("-x86_64")) else { continue };
let got = installed(file);
if json!(got) != l["sha256"] {
p.push(format!("the upgraded core served {} {}, the guest installed {got:?}", l["name"], l["sha256"]));
}
}
// the guest agent: a line with no path is the upgraded core's own pin (its downloads tree, held in memory); one
// with a path is a kept file, which for the agent must be an older downloads tree, never bin/
let agent = installed("socialweb-guest-agent");
let agent_line = served.iter().find(|l| l["name"] == "socialweb-guest-agent-x86_64" && json!(agent) == l["sha256"]);
let agent_from = match agent_line.map(|l| l["path"].as_str().unwrap_or_default()) {
None => {
p.push(format!("the upgraded core did not serve the guest agent the guest installed ({agent:?}): it differs from the core's own pin and no kept downloads tree served it, or the guest fetched it before the upgrade"));
json!(null)
}
Some("") => json!("pin"),
Some(path) if path.ends_with("/socialweb-guest-agent-x86_64") && !path.contains("/bin/") => json!({ "tree": path }),
Some(path) => {
p.push(format!("the upgraded core served the guest agent {agent} from {path}, not from a kept downloads tree"));
json!({ "other": path })
}
};
s.reading("agentFrom", agent_from);
let previous = s.st["counts"]["serves a previous binary"].as_u64().unwrap_or(0);
let unpinned = s.st["counts"]["pinned_download_not_pinned"].as_u64().unwrap_or(0);
s.reading("counts", json!({ "servesPrevious": previous, "notPinned": unpinned }));
if previous == 0 {
p.push("the upgraded core logged no `serves a previous binary`".to_string());
}
if unpinned != 0 {
p.push(format!("the upgraded core logged {unpinned} pinned_download_not_pinned"));
}
if !id_out.contains("uid=0(") {
p.push(format!("id in the guest: {id_out}"));
}
if !id_out.lines().any(|l| l.trim() == "active") {
p.push(format!("setup-guest-binaries.service is not active in the guest: {id_out}"));
}
let computer = installed("socialweb-computer");
if json!(computer) != s.st["from"] {
p.push(format!("the guest's /usr/local/bin/socialweb-computer is {computer:?}, not the from binary {}", s.st["from"]));
}
if !p.is_empty() {
return Ok(s.fail(p.join("; ")));
}
s.set("release");
s.flag("releasing");
h = h.call(host("release", "request.release", json!({ "name": name })));
}
Ok(s.keep(h))
}