// docs/acceptance/0001-auth-modes.md sections 3-6 on the gateway: a tenant cluster, /readyz through the kubeconfig's // /clusters/... path, a no-expiry kubeconfig and its revoke over xrpc and on the k8s page (/api with Origin), then a // deployment with required pod anti-affinity scaled 1 -> 2 -> 1 -> 0 (at most two replicas: a staging network has two bidders or more), each replica on its own node; per step // the gateway requester's VMs on the bidders the host reaches and each released VM's vm.terminated. Discovery only. use abi::{Call, Event, Handle, Run, entries}; use serde_json::{Value, json}; use steps::{MIN, S, VM_TERMINATED, get_record, host, kube, names, out, ready_pods, termination_rkey}; const N: &str = "com.publicdomainrelay.temp.cluster."; const STEPS: [usize; 4] = [1, 2, 1, 0]; const DEPLOY: &str = r#"apiVersion: apps/v1 kind: Deployment metadata: name: scale-check namespace: default spec: replicas: 1 selector: matchLabels: app: scale-check template: metadata: labels: app: scale-check spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app: scale-check topologyKey: kubernetes.io/hostname containers: - name: run image: docker.io/library/busybox:1.36.1 command: ["sh", "-c", "sleep 3600"] "#; fn ns() -> Value { json!({ "apiVersion": "v1", "kind": "Namespace" }) } fn run_0001(run: Run) -> Result { let a = &run.args; let mut s = S::new(run.state, a["now"].as_str().unwrap_or_default()); if let Some(h) = steps::market_event(&mut s, &run.event) { return Ok(h); } let gw = a["gateway"]["url"].as_str().ok_or("gateway.url")?.trim_end_matches('/').to_string(); let cl = a["name"].as_str().ok_or("name")?.to_string(); let tenant = format!("tenant:{cl}"); let xrpc = |id: &str, m: &str, input: Value| Call::new(id, format!("{N}{m}")).input(input).service(gw.clone()); let query = |id: &str, m: &str, params: Value| Call::new(id, format!("{N}{m}")).params(params).service(gw.clone()); let page = |id: &str, path: &str, body: Value| host(id, "page.fetch", json!({ "url": format!("{gw}{path}"), "method": "POST", "body": body })); let bidders: Vec<(String, String)> = a["bidders"].as_array().into_iter().flatten().filter_map(|b| Some((b["did"].as_str()?.to_string(), b["cluster"].as_str()?.to_string()))).collect(); let gw_did = a["gateway"]["requesterDid"].as_str().unwrap_or_default().to_string(); let poll = |s: &S, h: Handle| s.keep(h.timer("poll", 5_000)); // the gateway requester's VMs: a list per bidder cluster let list_vms = |h: Handle| bidders.iter().enumerate().fold(h, |h, (j, (_, c))| h.call(kube(&format!("vm-{j}"), "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" })))); let mut h = Handle::new(); match run.event { Event::Start => { if gw_did.is_empty() { return Ok(s.unmet("the target names no gateway.requesterDid: section 6 reads the gateway requester's VMs on the bidders (orphans, vm.terminated)")); } s.set("create"); h = h.call(xrpc("create", "createCluster", json!({ "name": cl }))).timer("deadline", 90 * MIN); } Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 90 min (phase {})", s.phase()))), Event::Timer { id } if id == "poll" => { h = match s.phase().as_str() { "create" | "gone" => h.call(query("list", "listClusters", json!({}))), "refused-ne" => h.call(kube("q-ne", "list", &format!("{tenant}-ne"), ns())), "refused-pg" => h.call(kube("q-pg", "list", &format!("{tenant}-pg"), ns())), "settle" => h.call(kube("nodes", "list", &tenant, json!({ "apiVersion": "v1", "kind": "Node" }))), _ => h, }; } Event::Timer { id } if id == "relist" => { s.st["fan"] = json!(bidders.len()); s.st["vms"] = json!([]); h = list_vms(h); } // the bidder pins vm.terminated seconds after its VM is gone: read it again every 3 s for up to 2 min Event::Timer { id } if id.starts_with("term-") => { let uri = s.st["term"][&id[5..]].as_str().unwrap_or_default().to_string(); h = h.call(get_record(&id, &uri)); } Event::Result { id, result, .. } => { let r = result.output().map(Value::clone); if id.starts_with("term-") && r.is_err() && s.since("term") < 120.0 { return Ok(s.keep(h.timer(&id, 3_000))); } // refusals the run waits for: an error is the answer if id == "q-ne" || id == "q-pg" || id == "final" { let refused = r.is_err(); match id.as_str() { "q-ne" if refused => { s.set("revoked-ok"); return Ok(s.keep(h.call(kube("ns2", "list", &tenant, ns())))); } "q-pg" if refused => { s.set("page-ok"); return Ok(s.keep(h.call(kube("ns3", "list", &tenant, ns())))); } "final" if refused => return Ok(s.pass()), "final" => return Ok(s.fail("the 8 h kubeconfig still works after deleteCluster")), _ if s.since("revoke") > 30.0 => return Ok(s.fail(format!("{id}: a revoked kubeconfig still works after 30 s"))), _ => return Ok(poll(&s, h)), } } let o = match out(&s, &id, result.output()) { Ok(o) => o, Err(f) => return Ok(f), }; match id.as_str() { "create" => { s.mark("create"); return Ok(poll(&s, h)); } "list" => { let found = o["clusters"].as_array().into_iter().flatten().find(|c| c["name"] == cl.as_str()).cloned(); match (s.phase().as_str(), found) { ("create", Some(c)) if c["phase"] == "Ready" => { let t = s.since("create"); s.reading("cluster_ready_s", t); s.set("kubeconfig"); h = h.call(host("kc", "gateway.kubeconfig", json!({ "name": cl, "ttlSeconds": 28800, "as": cl }))); } ("create", _) if s.since("create") > 300.0 => return Ok(s.fail("the cluster is not Ready 5 min after createCluster")), ("gone", None) => { let t = s.since("delete"); s.reading("cluster_deleted_s", t); h = h.call(kube("final", "list", &tenant, ns())); } ("gone", Some(_)) if s.since("delete") > 300.0 => return Ok(s.fail("the cluster is listed 5 min after deleteCluster")), _ => return Ok(poll(&s, h)), } } // the host keeps the kubeconfig (no secret reaches the module or its ticks) "kc" => h = h.call(kube("readyz", "raw", &tenant, json!({ "path": "/readyz" }))), "readyz" => { if o["text"].as_str().map(str::trim) != Some("ok") { return Ok(s.fail(format!("/readyz through the /clusters/... path: {o}"))); } h = h.call(kube("ns", "list", &tenant, ns())); } "ns" => { s.reading("namespaces", names(&o).len()); h = h.call(host("kcne", "gateway.kubeconfig", json!({ "name": cl, "ttlSeconds": 0, "as": format!("{cl}-ne") }))); } "kcne" => h = h.call(kube("nsne", "list", &format!("{tenant}-ne"), ns())), "nsne" => h = h.call(query("lk", "listKubeconfigs", json!({ "name": cl }))), "lk" | "pks" => { let body = if id == "pks" { &o["body"] } else { &o }; let serial = body["kubeconfigs"] .as_array() .into_iter() .flatten() .filter(|k| k["noExpiry"] == true && k["revoked"] != true) .max_by_key(|k| k["createdAt"].as_str().unwrap_or_default().to_string()) .and_then(|k| k["serial"].as_str()) .map(String::from); let Some(serial) = serial else { return Ok(s.fail(format!("{id}: no live noExpiry kubeconfig: {o}"))) }; h = if id == "lk" { h.call(xrpc("rv", "revokeKubeconfig", json!({ "name": cl, "serial": serial }))) } else { h.call(page("prv", "/api/clusters/kubeconfigs/revoke", json!({ "name": cl, "serial": serial }))) }; } "rv" | "prv" => { let revoked = if id == "rv" { o["revoked"] == true } else { o["status"] == 200 && o["body"]["revoked"] == true }; if !revoked { return Ok(s.fail(format!("{id}: not revoked: {o}"))); } s.mark("revoke"); s.set(if id == "rv" { "refused-ne" } else { "refused-pg" }); return Ok(poll(&s, h)); } "ns2" => { s.set("page"); h = h.call(host("ps", "page.signin", json!({ "url": format!("{gw}/"), "done": "Signed in as" }))); } "ps" => h = h.call(host("pk", "page.kubeconfig", json!({ "name": cl, "ttlSeconds": 0, "as": format!("{cl}-pg") }))), "pk" => h = h.call(kube("nspg", "list", &format!("{tenant}-pg"), ns())), "nspg" => h = h.call(page("pks", "/api/clusters/kubeconfigs", json!({ "name": cl }))), "ns3" => { s.set("deploy"); h = h.call(kube("dep", "apply", &tenant, json!({ "yaml": DEPLOY }))); } "dep" => { s.st["i"] = json!(0); s.st["fan"] = json!(bidders.len()); s.st["vms"] = json!([]); s.set("vms0"); h = list_vms(h); if bidders.is_empty() || bidders.len() != a["bidders"].as_array().map_or(0, |b| b.len()) { return Ok(s.unmet(&format!("the target names a kcp for {} of its bidders: one replica per bidder is read on every host", bidders.len()))); } if bidders.is_empty() { s.set("scale"); h = h.call(kube("scale", "patch", &tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check", "patch": { "spec": { "replicas": STEPS[0] } } }))); s.mark("step"); } } v if v.starts_with("vm-") => { for vm in o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["requester"]["did"] == gw_did.as_str()) { let j: usize = v[3..].parse().unwrap_or(0); s.st["vms"].as_array_mut().unwrap().push(json!([bidders[j].0, vm["spec"]["contract"]["receipt"]["uri"]])); } let left = s.st["fan"].as_u64().unwrap_or(1) - 1; s.st["fan"] = json!(left); if left > 0 { return Ok(s.keep(h)); } let now: Vec = s.st["vms"].as_array().cloned().unwrap_or_default(); let before: Vec = s.st["prev"].as_array().cloned().unwrap_or_default(); let i = s.st["i"].as_u64().unwrap_or(0) as usize; if s.is("orphans") { let n = STEPS[i]; if now.len() != n { // a node leaves the cluster seconds before its VM is released: ask again for up to 2 min if s.since("orphans") < 120.0 { return Ok(s.keep(h.timer("relist", 5_000))); } return Ok(s.fail(format!("step {n}: {} gateway VMs on the bidders, want {n} (2 min after the step)", now.len()))); } let gone: Vec = before.iter().filter(|v| !now.contains(v)).cloned().collect(); s.st["prev"] = json!(now); if !gone.is_empty() { s.st["fan"] = json!(gone.len()); s.set("terminated"); s.mark("term"); for (k, g) in gone.iter().enumerate() { let uri = format!("at://{}/{VM_TERMINATED}/{}", g[0].as_str().unwrap_or_default(), termination_rkey(g[1].as_str().unwrap_or_default())); s.st["term"][k.to_string()] = json!(uri); h = h.call(get_record(&format!("term-{k}"), &uri)); } return Ok(s.keep(h)); } { let h = next_step(&mut s, h, &tenant); return Ok(s.keep(h)); } } // vms0: the baseline, then the first step s.st["prev"] = json!(now); s.set("scale"); s.mark("step"); h = h.call(kube("scale", "patch", &tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check", "patch": { "spec": { "replicas": STEPS[0] } } }))); } "scale" => { s.set("settle"); return Ok(poll(&s, h)); } "nodes" => { s.st["nodes"] = json!(names(&o)); return Ok(s.keep(h.call(kube("pods", "list", &tenant, json!({ "apiVersion": "v1", "kind": "Pod", "namespace": "default", "labelSelector": "app=scale-check" }))))); } "pods" => { let i = s.st["i"].as_u64().unwrap_or(0) as usize; let n = STEPS[i]; let nodes: Vec = serde_json::from_value(s.st["nodes"].clone()).unwrap_or_default(); let pods = ready_pods(&o); let mut on: Vec<&String> = pods.iter().map(|(_, n)| n).collect(); on.sort(); on.dedup(); if nodes.len() == n && pods.len() == n && on.len() == n && on.iter().all(|x| nodes.contains(x)) { let t = s.since("step"); if !s.st["readings"]["steps"].is_array() { s.st["readings"]["steps"] = json!([]); } s.st["readings"]["steps"].as_array_mut().unwrap().push(json!({ "replicas": n, "seconds": t, "nodes": nodes })); s.set("orphans"); s.mark("orphans"); s.st["fan"] = json!(bidders.len()); s.st["vms"] = json!([]); if bidders.is_empty() { { let h = next_step(&mut s, h, &tenant); return Ok(s.keep(h)); } } return Ok(s.keep(list_vms(h))); } if s.since("step") > 900.0 { return Ok(s.fail(format!("step to {n} replicas: {} nodes, {} Ready pods on {} nodes after 15 min", nodes.len(), pods.len(), on.len()))); } return Ok(poll(&s, h)); } t if t.starts_with("term-") => { if o["value"]["reason"] != "deleted_on_request" || !o["value"]["request"]["uri"].is_string() { return Ok(s.fail(format!("vm.terminated {o}"))); } let left = s.st["fan"].as_u64().unwrap_or(1) - 1; s.st["fan"] = json!(left); if left == 0 { { let h = next_step(&mut s, h, &tenant); return Ok(s.keep(h)); } } } "deld" => h = h.call(xrpc("delc", "deleteCluster", json!({ "name": cl }))), "delc" => { s.mark("delete"); s.set("gone"); return Ok(poll(&s, h)); } _ => {} } } _ => {} } Ok(s.keep(h)) } /// After a step's checks: the next replica count, or the end (delete the deployment, then the cluster). fn next_step(s: &mut S, h: Handle, tenant: &str) -> Handle { let i = s.st["i"].as_u64().unwrap_or(0) as usize + 1; s.st["i"] = json!(i); if i == STEPS.len() { s.set("delete"); return h.call(kube("deld", "delete", tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check" }))); } s.set("scale"); s.mark("step"); h.call(kube("scale", "patch", tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check", "patch": { "spec": { "replicas": STEPS[i] } } }))) } mod s2; entries! { run => run_0001, s2 => s2::s2 }