~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
324 lines · 17.5 KB Raw History
// docs/acceptance/0001-auth-modes.md sections 3-6 on the gateway: a tenant cluster, /readyz through the kubeconfig's
// /clusters/... path, a no-expiry kubeconfig and its revoke over xrpc and on the k8s page (/api with Origin), then a
// deployment with required pod anti-affinity scaled 1 -> 2 -> 1 -> 0 (at most two replicas: a staging network has two bidders or more), each replica on its own node; per step
// the gateway requester's VMs on the bidders the host reaches and each released VM's vm.terminated. Discovery only.
use abi::{Call, Event, Handle, Run, entries};
use serde_json::{Value, json};
use steps::{MIN, S, VM_TERMINATED, get_record, host, kube, names, out, ready_pods, termination_rkey};

const N: &str = "com.publicdomainrelay.temp.cluster.";
const STEPS: [usize; 4] = [1, 2, 1, 0];
const DEPLOY: &str = r#"apiVersion: apps/v1
kind: Deployment
metadata:
  name: scale-check
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: scale-check
  template:
    metadata:
      labels:
        app: scale-check
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  app: scale-check
              topologyKey: kubernetes.io/hostname
      containers:
        - name: run
          image: docker.io/library/busybox:1.36.1
          command: ["sh", "-c", "sleep 3600"]
"#;

fn ns() -> Value {
    json!({ "apiVersion": "v1", "kind": "Namespace" })
}

fn run_0001(run: Run<Value>) -> Result<Handle, String> {
    let a = &run.args;
    let mut s = S::new(run.state, a["now"].as_str().unwrap_or_default());
    if let Some(h) = steps::market_event(&mut s, &run.event) {
        return Ok(h);
    }
    let gw = a["gateway"]["url"].as_str().ok_or("gateway.url")?.trim_end_matches('/').to_string();
    let cl = a["name"].as_str().ok_or("name")?.to_string();
    let tenant = format!("tenant:{cl}");
    let xrpc = |id: &str, m: &str, input: Value| Call::new(id, format!("{N}{m}")).input(input).service(gw.clone());
    let query = |id: &str, m: &str, params: Value| Call::new(id, format!("{N}{m}")).params(params).service(gw.clone());
    let page = |id: &str, path: &str, body: Value| host(id, "page.fetch", json!({ "url": format!("{gw}{path}"), "method": "POST", "body": body }));
    let bidders: Vec<(String, String)> = a["bidders"].as_array().into_iter().flatten().filter_map(|b| Some((b["did"].as_str()?.to_string(), b["cluster"].as_str()?.to_string()))).collect();
    let gw_did = a["gateway"]["requesterDid"].as_str().unwrap_or_default().to_string();
    let poll = |s: &S, h: Handle| s.keep(h.timer("poll", 5_000));
    // the gateway requester's VMs: a list per bidder cluster
    let list_vms = |h: Handle| bidders.iter().enumerate().fold(h, |h, (j, (_, c))| h.call(kube(&format!("vm-{j}"), "list", c, json!({ "apiVersion": "socialweb.computer/v1alpha1", "kind": "ComputeVM" }))));
    let mut h = Handle::new();
    match run.event {
        Event::Start => {
            if gw_did.is_empty() {
                return Ok(s.unmet("the target names no gateway.requesterDid: section 6 reads the gateway requester's VMs on the bidders (orphans, vm.terminated)"));
            }
            s.set("create");
            h = h.call(xrpc("create", "createCluster", json!({ "name": cl }))).timer("deadline", 90 * MIN);
        }
        Event::Timer { id } if id == "deadline" => return Ok(s.fail(format!("not done in 90 min (phase {})", s.phase()))),
        Event::Timer { id } if id == "poll" => {
            h = match s.phase().as_str() {
                "create" | "gone" => h.call(query("list", "listClusters", json!({}))),
                "refused-ne" => h.call(kube("q-ne", "list", &format!("{tenant}-ne"), ns())),
                "refused-pg" => h.call(kube("q-pg", "list", &format!("{tenant}-pg"), ns())),
                "settle" => h.call(kube("nodes", "list", &tenant, json!({ "apiVersion": "v1", "kind": "Node" }))),
                _ => h,
            };
        }
        Event::Timer { id } if id == "relist" => {
            s.st["fan"] = json!(bidders.len());
            s.st["vms"] = json!([]);
            h = list_vms(h);
        }
        // the bidder pins vm.terminated seconds after its VM is gone: read it again every 3 s for up to 2 min
        Event::Timer { id } if id.starts_with("term-") => {
            let uri = s.st["term"][&id[5..]].as_str().unwrap_or_default().to_string();
            h = h.call(get_record(&id, &uri));
        }
        Event::Result { id, result, .. } => {
            let r = result.output().map(Value::clone);
            if id.starts_with("term-") && r.is_err() && s.since("term") < 120.0 {
                return Ok(s.keep(h.timer(&id, 3_000)));
            }
            // refusals the run waits for: an error is the answer
            if id == "q-ne" || id == "q-pg" || id == "final" {
                let refused = r.is_err();
                match id.as_str() {
                    "q-ne" if refused => {
                        s.set("revoked-ok");
                        return Ok(s.keep(h.call(kube("ns2", "list", &tenant, ns()))));
                    }
                    "q-pg" if refused => {
                        s.set("page-ok");
                        return Ok(s.keep(h.call(kube("ns3", "list", &tenant, ns()))));
                    }
                    "final" if refused => return Ok(s.pass()),
                    "final" => return Ok(s.fail("the 8 h kubeconfig still works after deleteCluster")),
                    _ if s.since("revoke") > 30.0 => return Ok(s.fail(format!("{id}: a revoked kubeconfig still works after 30 s"))),
                    _ => return Ok(poll(&s, h)),
                }
            }
            let o = match out(&s, &id, result.output()) {
                Ok(o) => o,
                Err(f) => return Ok(f),
            };
            match id.as_str() {
                "create" => {
                    s.mark("create");
                    return Ok(poll(&s, h));
                }
                "list" => {
                    let found = o["clusters"].as_array().into_iter().flatten().find(|c| c["name"] == cl.as_str()).cloned();
                    match (s.phase().as_str(), found) {
                        ("create", Some(c)) if c["phase"] == "Ready" => {
                            let t = s.since("create");
                            s.reading("cluster_ready_s", t);
                            s.set("kubeconfig");
                            h = h.call(host("kc", "gateway.kubeconfig", json!({ "name": cl, "ttlSeconds": 28800, "as": cl })));
                        }
                        ("create", _) if s.since("create") > 300.0 => return Ok(s.fail("the cluster is not Ready 5 min after createCluster")),
                        ("gone", None) => {
                            let t = s.since("delete");
                            s.reading("cluster_deleted_s", t);
                            h = h.call(kube("final", "list", &tenant, ns()));
                        }
                        ("gone", Some(_)) if s.since("delete") > 300.0 => return Ok(s.fail("the cluster is listed 5 min after deleteCluster")),
                        _ => return Ok(poll(&s, h)),
                    }
                }
                // the host keeps the kubeconfig (no secret reaches the module or its ticks)
                "kc" => h = h.call(kube("readyz", "raw", &tenant, json!({ "path": "/readyz" }))),
                "readyz" => {
                    if o["text"].as_str().map(str::trim) != Some("ok") {
                        return Ok(s.fail(format!("/readyz through the /clusters/... path: {o}")));
                    }
                    h = h.call(kube("ns", "list", &tenant, ns()));
                }
                "ns" => {
                    s.reading("namespaces", names(&o).len());
                    h = h.call(host("kcne", "gateway.kubeconfig", json!({ "name": cl, "ttlSeconds": 0, "as": format!("{cl}-ne") })));
                }
                "kcne" => h = h.call(kube("nsne", "list", &format!("{tenant}-ne"), ns())),
                "nsne" => h = h.call(query("lk", "listKubeconfigs", json!({ "name": cl }))),
                "lk" | "pks" => {
                    let body = if id == "pks" { &o["body"] } else { &o };
                    let serial = body["kubeconfigs"]
                        .as_array()
                        .into_iter()
                        .flatten()
                        .filter(|k| k["noExpiry"] == true && k["revoked"] != true)
                        .max_by_key(|k| k["createdAt"].as_str().unwrap_or_default().to_string())
                        .and_then(|k| k["serial"].as_str())
                        .map(String::from);
                    let Some(serial) = serial else { return Ok(s.fail(format!("{id}: no live noExpiry kubeconfig: {o}"))) };
                    h = if id == "lk" {
                        h.call(xrpc("rv", "revokeKubeconfig", json!({ "name": cl, "serial": serial })))
                    } else {
                        h.call(page("prv", "/api/clusters/kubeconfigs/revoke", json!({ "name": cl, "serial": serial })))
                    };
                }
                "rv" | "prv" => {
                    let revoked = if id == "rv" { o["revoked"] == true } else { o["status"] == 200 && o["body"]["revoked"] == true };
                    if !revoked {
                        return Ok(s.fail(format!("{id}: not revoked: {o}")));
                    }
                    s.mark("revoke");
                    s.set(if id == "rv" { "refused-ne" } else { "refused-pg" });
                    return Ok(poll(&s, h));
                }
                "ns2" => {
                    s.set("page");
                    h = h.call(host("ps", "page.signin", json!({ "url": format!("{gw}/"), "done": "Signed in as" })));
                }
                "ps" => h = h.call(host("pk", "page.kubeconfig", json!({ "name": cl, "ttlSeconds": 0, "as": format!("{cl}-pg") }))),
                "pk" => h = h.call(kube("nspg", "list", &format!("{tenant}-pg"), ns())),
                "nspg" => h = h.call(page("pks", "/api/clusters/kubeconfigs", json!({ "name": cl }))),
                "ns3" => {
                    s.set("deploy");
                    h = h.call(kube("dep", "apply", &tenant, json!({ "yaml": DEPLOY })));
                }
                "dep" => {
                    s.st["i"] = json!(0);
                    s.st["fan"] = json!(bidders.len());
                    s.st["vms"] = json!([]);
                    s.set("vms0");
                    h = list_vms(h);
                    if bidders.is_empty() || bidders.len() != a["bidders"].as_array().map_or(0, |b| b.len()) {
                        return Ok(s.unmet(&format!("the target names a kcp for {} of its bidders: one replica per bidder is read on every host", bidders.len())));
                    }
                    if bidders.is_empty() {
                        s.set("scale");
                        h = h.call(kube("scale", "patch", &tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check", "patch": { "spec": { "replicas": STEPS[0] } } })));
                        s.mark("step");
                    }
                }
                v if v.starts_with("vm-") => {
                    for vm in o["items"].as_array().into_iter().flatten().filter(|v| v["spec"]["requester"]["did"] == gw_did.as_str()) {
                        let j: usize = v[3..].parse().unwrap_or(0);
                        s.st["vms"].as_array_mut().unwrap().push(json!([bidders[j].0, vm["spec"]["contract"]["receipt"]["uri"]]));
                    }
                    let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
                    s.st["fan"] = json!(left);
                    if left > 0 {
                        return Ok(s.keep(h));
                    }
                    let now: Vec<Value> = s.st["vms"].as_array().cloned().unwrap_or_default();
                    let before: Vec<Value> = s.st["prev"].as_array().cloned().unwrap_or_default();
                    let i = s.st["i"].as_u64().unwrap_or(0) as usize;
                    if s.is("orphans") {
                        let n = STEPS[i];
                        if now.len() != n {
                            // a node leaves the cluster seconds before its VM is released: ask again for up to 2 min
                            if s.since("orphans") < 120.0 {
                                return Ok(s.keep(h.timer("relist", 5_000)));
                            }
                            return Ok(s.fail(format!("step {n}: {} gateway VMs on the bidders, want {n} (2 min after the step)", now.len())));
                        }
                        let gone: Vec<Value> = before.iter().filter(|v| !now.contains(v)).cloned().collect();
                        s.st["prev"] = json!(now);
                        if !gone.is_empty() {
                            s.st["fan"] = json!(gone.len());
                            s.set("terminated");
                            s.mark("term");
                            for (k, g) in gone.iter().enumerate() {
                                let uri = format!("at://{}/{VM_TERMINATED}/{}", g[0].as_str().unwrap_or_default(), termination_rkey(g[1].as_str().unwrap_or_default()));
                                s.st["term"][k.to_string()] = json!(uri);
                                h = h.call(get_record(&format!("term-{k}"), &uri));
                            }
                            return Ok(s.keep(h));
                        }
                        { let h = next_step(&mut s, h, &tenant); return Ok(s.keep(h)); }
                    }
                    // vms0: the baseline, then the first step
                    s.st["prev"] = json!(now);
                    s.set("scale");
                    s.mark("step");
                    h = h.call(kube("scale", "patch", &tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check", "patch": { "spec": { "replicas": STEPS[0] } } })));
                }
                "scale" => {
                    s.set("settle");
                    return Ok(poll(&s, h));
                }
                "nodes" => {
                    s.st["nodes"] = json!(names(&o));
                    return Ok(s.keep(h.call(kube("pods", "list", &tenant, json!({ "apiVersion": "v1", "kind": "Pod", "namespace": "default", "labelSelector": "app=scale-check" })))));
                }
                "pods" => {
                    let i = s.st["i"].as_u64().unwrap_or(0) as usize;
                    let n = STEPS[i];
                    let nodes: Vec<String> = serde_json::from_value(s.st["nodes"].clone()).unwrap_or_default();
                    let pods = ready_pods(&o);
                    let mut on: Vec<&String> = pods.iter().map(|(_, n)| n).collect();
                    on.sort();
                    on.dedup();
                    if nodes.len() == n && pods.len() == n && on.len() == n && on.iter().all(|x| nodes.contains(x)) {
                        let t = s.since("step");
                        if !s.st["readings"]["steps"].is_array() {
                            s.st["readings"]["steps"] = json!([]);
                        }
                        s.st["readings"]["steps"].as_array_mut().unwrap().push(json!({ "replicas": n, "seconds": t, "nodes": nodes }));
                        s.set("orphans");
                        s.mark("orphans");
                        s.st["fan"] = json!(bidders.len());
                        s.st["vms"] = json!([]);
                        if bidders.is_empty() {
                            { let h = next_step(&mut s, h, &tenant); return Ok(s.keep(h)); }
                        }
                        return Ok(s.keep(list_vms(h)));
                    }
                    if s.since("step") > 900.0 {
                        return Ok(s.fail(format!("step to {n} replicas: {} nodes, {} Ready pods on {} nodes after 15 min", nodes.len(), pods.len(), on.len())));
                    }
                    return Ok(poll(&s, h));
                }
                t if t.starts_with("term-") => {
                    if o["value"]["reason"] != "deleted_on_request" || !o["value"]["request"]["uri"].is_string() {
                        return Ok(s.fail(format!("vm.terminated {o}")));
                    }
                    let left = s.st["fan"].as_u64().unwrap_or(1) - 1;
                    s.st["fan"] = json!(left);
                    if left == 0 {
                        { let h = next_step(&mut s, h, &tenant); return Ok(s.keep(h)); }
                    }
                }
                "deld" => h = h.call(xrpc("delc", "deleteCluster", json!({ "name": cl }))),
                "delc" => {
                    s.mark("delete");
                    s.set("gone");
                    return Ok(poll(&s, h));
                }
                _ => {}
            }
        }
        _ => {}
    }
    Ok(s.keep(h))
}

/// After a step's checks: the next replica count, or the end (delete the deployment, then the cluster).
fn next_step(s: &mut S, h: Handle, tenant: &str) -> Handle {
    let i = s.st["i"].as_u64().unwrap_or(0) as usize + 1;
    s.st["i"] = json!(i);
    if i == STEPS.len() {
        s.set("delete");
        return h.call(kube("deld", "delete", tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check" })));
    }
    s.set("scale");
    s.mark("step");
    h.call(kube("scale", "patch", tenant, json!({ "apiVersion": "apps/v1", "kind": "Deployment", "namespace": "default", "name": "scale-check", "patch": { "spec": { "replicas": STEPS[i] } } })))
}

mod s2;

entries! { run => run_0001, s2 => s2::s2 }