~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/atlibs-kube

Kubernetes (and kcp) API calls and watches for [atlibs](https://knotless.dev/johnandersen777.bsky.social/atlibs) hosts: a transport on atlibs-host's seam.

0
0
main
485 lines · 21.4 KB Raw History
//! com.fedfork.exec.kube.{get,list,apply,patch,delete,watch}: a transport on atlibs-host's seam. The host names its
//! clusters (`Cluster`: a kubeconfig and how to reach it); a module names a cluster, never a credential.
//!
//! Writes are scoped to the run: `apply` records what it created (the ledger); `patch` and `delete` reach only those,
//! or what the host's `Guard` allows (a test's own VM on a bidder). A guarded patch of an object the run did not create
//! saves its spec first; `cleanup()` (the host calls it after the run, whatever ended it) deletes what the run created
//! and puts every patched spec back.

use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use atlibs_host::BoxFut;
use atlibs_host::run::{Call, Request, Sub, SubCmd};
use atlibs_host::transport::Transport;
use futures::StreamExt;
use kube::api::{Api, ApiResource, DeleteParams, DynamicObject, GroupVersionKind, ListParams, Patch, PatchParams};
use kube::config::{KubeConfigOptions, Kubeconfig};
use serde::Deserialize;
use serde_json::{Map, Value, json};
use tokio::sync::mpsc;

pub const GET: &str = "com.fedfork.exec.kube.get";
pub const LIST: &str = "com.fedfork.exec.kube.list";
pub const APPLY: &str = "com.fedfork.exec.kube.apply";
pub const PATCH: &str = "com.fedfork.exec.kube.patch";
pub const DELETE: &str = "com.fedfork.exec.kube.delete";
pub const WATCH: &str = "com.fedfork.exec.kube.watch";
pub const RAW: &str = "com.fedfork.exec.kube.raw";
pub const NSIDS: &[&str] = &[GET, LIST, APPLY, PATCH, DELETE, WATCH, RAW];
const MANAGER: &str = "atlibs-kube";

/// One cluster the host can reach: its kubeconfig (credentials, CA), the `/clusters/<workspace>` path a kcp needs
/// when the kubeconfig names its root, and a transport.
#[derive(Clone, Debug, Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Cluster {
    pub kubeconfig: String,
    #[serde(default)]
    pub workspace: Option<String>,
    #[serde(default)]
    pub via: Via,
}

#[derive(Clone, Debug, Default, Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase", tag = "kind")]
pub enum Via {
    /// the kubeconfig's server URL
    #[default]
    Direct,
    /// the same port on another name (a meshd overlay name), TLS name kept
    Overlay { name: String },
    /// an `ssh -L` forward through an ssh destination to the server's host:port (a front on its host's loopback)
    Ssh { host: String },
    /// atlibs-iroh's raw form: `dumbpipe connect <ticket>` per connection
    Iroh { ticket: String },
    /// any command piping stdin/stdout to the server's port, one run per connection (a WebRTC datachannel proxy)
    Proxy { command: Vec<String> },
}

/// May the run patch or delete this object it did not create? (cluster, verb "patch" | "delete", the object now)
pub type Guard = Arc<dyn Fn(&str, &str, &Value) -> bool + Send + Sync>;

#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct Key {
    cluster: String,
    api_version: String,
    kind: String,
    plural: String,
    namespace: Option<String>,
    name: String,
}

struct Conn {
    client: kube::Client,
    _forward: Option<Forward>,
}

enum Forward {
    Child(#[allow(dead_code)] tokio::process::Child),
    Bridge(tokio::task::JoinHandle<()>),
}

impl Drop for Forward {
    fn drop(&mut self) {
        if let Forward::Bridge(h) = self {
            h.abort();
        }
    }
}

pub struct Kube {
    clusters: Mutex<HashMap<String, Cluster>>,
    guard: Option<Guard>,
    conns: tokio::sync::Mutex<HashMap<String, Arc<Conn>>>,
    created: Mutex<Vec<Key>>,
    /// objects the run patched without creating them: their spec before the first patch
    saved: Mutex<Vec<(Key, Value)>>,
}

fn plural(kind: &str) -> String {
    let k = kind.to_lowercase();
    match k.strip_suffix('y') {
        Some(stem) => format!("{stem}ies"),
        None => format!("{k}s"),
    }
}

fn s(v: &Value, k: &str) -> Option<String> {
    v.get(k).and_then(Value::as_str).map(String::from)
}

/// The merge patch that turns `cur` back into `orig` (keys the run added become null).
pub fn restore_patch(orig: &Value, cur: &Value) -> Value {
    match (orig, cur) {
        (Value::Object(o), Value::Object(c)) => {
            let mut p = Map::new();
            for (k, cv) in c {
                match o.get(k) {
                    None => {
                        p.insert(k.clone(), Value::Null);
                    }
                    Some(ov) if ov != cv => {
                        p.insert(k.clone(), restore_patch(ov, cv));
                    }
                    _ => {}
                }
            }
            for (k, ov) in o {
                if !c.contains_key(k) {
                    p.insert(k.clone(), ov.clone());
                }
            }
            Value::Object(p)
        }
        _ => orig.clone(),
    }
}

async fn free_port() -> Result<u16, String> {
    let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.map_err(|e| e.to_string())?;
    Ok(l.local_addr().map_err(|e| e.to_string())?.port())
}

/// Listen on 127.0.0.1; each connection runs `argv` and pipes the stream through its stdin/stdout.
async fn stdio_bridge(argv: Vec<String>) -> Result<(u16, Forward), String> {
    if argv.is_empty() {
        return Err("an empty proxy command".into());
    }
    let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.map_err(|e| e.to_string())?;
    let port = l.local_addr().map_err(|e| e.to_string())?.port();
    let task = tokio::spawn(async move {
        while let Ok((mut tcp, _)) = l.accept().await {
            let argv = argv.clone();
            tokio::spawn(async move {
                let Ok(mut child) = tokio::process::Command::new(&argv[0])
                    .args(&argv[1..])
                    .stdin(std::process::Stdio::piped())
                    .stdout(std::process::Stdio::piped())
                    .stderr(std::process::Stdio::null())
                    .kill_on_drop(true)
                    .spawn()
                else {
                    return;
                };
                let mut pipe = tokio::io::join(child.stdout.take().unwrap(), child.stdin.take().unwrap());
                let _ = tokio::io::copy_bidirectional(&mut tcp, &mut pipe).await;
            });
        }
    });
    Ok((port, Forward::Bridge(task)))
}

/// A kube client of one cluster over its transport.
async fn connect(c: &Cluster) -> Result<Conn, String> {
    let kc = Kubeconfig::read_from(&c.kubeconfig).map_err(|e| format!("{}: {e}", c.kubeconfig))?;
    let mut cfg = kube::Config::from_custom_kubeconfig(kc, &KubeConfigOptions::default()).await.map_err(|e| e.to_string())?;
    let url = cfg.cluster_url.clone();
    let host = url.host().ok_or("the kubeconfig's server has no host")?.trim_matches(['[', ']']).to_string();
    let port = url.port_u16().unwrap_or(443);
    let path = match &c.workspace {
        Some(ws) => format!("/clusters/{ws}"),
        None => url.path().trim_end_matches('/').to_string(),
    };
    let mut forward = None;
    let authority = match &c.via {
        Via::Direct => format!("{host}:{port}"),
        Via::Overlay { name } => format!("{name}:{port}"),
        Via::Ssh { host: through } => {
            let local = free_port().await?;
            let child = tokio::process::Command::new("ssh")
                .args(["-N", "-o", "BatchMode=yes", "-o", "ExitOnForwardFailure=yes", "-L"])
                .arg(format!("127.0.0.1:{local}:{host}:{port}"))
                .arg(through)
                .stdin(std::process::Stdio::null())
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .kill_on_drop(true)
                .spawn()
                .map_err(|e| format!("ssh: {e}"))?;
            forward = Some(Forward::Child(child));
            let deadline = Instant::now() + Duration::from_secs(30);
            while tokio::net::TcpStream::connect(("127.0.0.1", local)).await.is_err() {
                if Instant::now() > deadline {
                    return Err(format!("the ssh forward through {through} did not open in 30 s"));
                }
                tokio::time::sleep(Duration::from_millis(200)).await;
            }
            format!("127.0.0.1:{local}")
        }
        Via::Iroh { ticket } => {
            let ticket = atlibs_iroh::ticket(Some(&Value::String(ticket.clone())))?;
            let (local, f) = stdio_bridge(vec!["dumbpipe".into(), "connect".into(), ticket]).await?;
            forward = Some(f);
            format!("127.0.0.1:{local}")
        }
        Via::Proxy { command } => {
            let (local, f) = stdio_bridge(command.clone()).await?;
            forward = Some(f);
            format!("127.0.0.1:{local}")
        }
    };
    cfg.cluster_url = format!("https://{authority}{path}").parse().map_err(|e| format!("{e}"))?;
    if !matches!(c.via, Via::Direct) {
        cfg.tls_server_name = Some(host);
    }
    Ok(Conn { client: kube::Client::try_from(cfg).map_err(|e| e.to_string())?, _forward: forward })
}

fn not_found(e: &kube::Error) -> bool {
    matches!(e, kube::Error::Api(a) if a.code == 404)
}

impl Kube {
    pub fn new(clusters: HashMap<String, Cluster>, guard: Option<Guard>) -> Arc<Self> {
        Arc::new(Self { clusters: Mutex::new(clusters), guard, conns: Default::default(), created: Default::default(), saved: Default::default() })
    }

    /// What the run created and what it patched (with the spec before): a host keeps it on disk, so a cleanup
    /// works after the host died (`load_ledger` in a new process).
    pub fn ledger(&self) -> Value {
        json!({ "created": *self.created.lock().unwrap(), "saved": self.saved.lock().unwrap().iter().map(|(k, s)| json!({ "key": k, "spec": s })).collect::<Vec<_>>() })
    }

    pub fn load_ledger(&self, v: &Value) -> Result<(), String> {
        let created: Vec<Key> = serde_json::from_value(v["created"].clone()).map_err(|e| e.to_string())?;
        let saved: Vec<(Key, Value)> = v["saved"].as_array().into_iter().flatten().filter_map(|x| Some((serde_json::from_value(x["key"].clone()).ok()?, x["spec"].clone()))).collect();
        self.created.lock().unwrap().extend(created);
        self.saved.lock().unwrap().extend(saved);
        Ok(())
    }

    /// The clusters it knows (kubeconfig paths and transports, no credential): a dump's.
    pub fn clusters(&self) -> Value {
        json!(*self.clusters.lock().unwrap())
    }

    /// A cluster the host learned during the run (a requester it started: its kcp).
    pub fn add_cluster(&self, name: &str, c: Cluster) {
        self.clusters.lock().unwrap().insert(name.into(), c);
    }

    pub fn transport(self: &Arc<Self>) -> Transport {
        Transport { provides: NSIDS, call: self.clone() }
    }

    async fn conn(&self, cluster: &str) -> Result<Arc<Conn>, String> {
        let mut conns = self.conns.lock().await;
        if let Some(c) = conns.get(cluster) {
            return Ok(c.clone());
        }
        let c = self.clusters.lock().unwrap().get(cluster).cloned().ok_or_else(|| format!("no cluster {cluster} on this host"))?;
        let conn = Arc::new(connect(&c).await.map_err(|e| format!("cluster {cluster}: {e}"))?);
        conns.insert(cluster.into(), conn.clone());
        Ok(conn)
    }

    fn key(input: &Value, obj: Option<&Value>) -> Result<Key, String> {
        let from = |k: &str| obj.and_then(|o| s(o, k)).or_else(|| s(input, k));
        let api_version = from("apiVersion").ok_or("apiVersion is required")?;
        let kind = from("kind").ok_or("kind is required")?;
        let name = obj.and_then(|o| s(&o["metadata"], "name")).or_else(|| s(input, "name")).unwrap_or_default();
        let namespace = obj.and_then(|o| s(&o["metadata"], "namespace")).or_else(|| s(input, "namespace"));
        Ok(Key { cluster: s(input, "cluster").ok_or("cluster is required")?, plural: s(input, "plural").unwrap_or_else(|| plural(&kind)), api_version, kind, namespace, name })
    }

    async fn api(&self, k: &Key) -> Result<Api<DynamicObject>, String> {
        let conn = self.conn(&k.cluster).await?;
        let (group, version) = k.api_version.split_once('/').unwrap_or(("", &k.api_version));
        let ar = ApiResource::from_gvk_with_plural(&GroupVersionKind::gvk(group, version, &k.kind), &k.plural);
        Ok(match &k.namespace {
            Some(ns) => Api::namespaced_with(conn.client.clone(), ns, &ar),
            None => Api::all_with(conn.client.clone(), &ar),
        })
    }

    /// May the run write `k` (created by it, or the guard allows)? Returns the object now (None: gone).
    async fn writable(&self, k: &Key, verb: &str) -> Result<Option<Value>, String> {
        let api = self.api(k).await?;
        let now = api.get_opt(&k.name).await.map_err(|e| e.to_string())?.map(|o| serde_json::to_value(o).unwrap());
        if self.created.lock().unwrap().contains(k) {
            return Ok(now);
        }
        let Some(obj) = &now else { return Ok(None) };
        match &self.guard {
            Some(g) if g(&k.cluster, verb, obj) => Ok(now),
            _ => Err(format!("{verb} {}/{}: the run did not create it", k.kind, k.name)),
        }
    }

    async fn handle(&self, nsid: &str, input: Value) -> Result<Value, String> {
        match nsid {
            GET => {
                let k = Self::key(&input, None)?;
                Ok(self.api(&k).await?.get_opt(&k.name).await.map_err(|e| e.to_string())?.map_or(Value::Null, |o| serde_json::to_value(o).unwrap()))
            }
            LIST => {
                let k = Self::key(&input, None)?;
                let mut lp = ListParams::default();
                if let Some(sel) = s(&input, "labelSelector") {
                    lp = lp.labels(&sel);
                }
                let items = self.api(&k).await?.list(&lp).await.map_err(|e| e.to_string())?.items;
                Ok(json!({ "items": items }))
            }
            APPLY => {
                let objects: Vec<Value> = match (input.get("object"), s(&input, "yaml")) {
                    (Some(o), _) => vec![o.clone()],
                    (None, Some(y)) => serde_yaml::Deserializer::from_str(&y)
                        .map(|d| Value::deserialize(d).map_err(|e| e.to_string()))
                        .filter(|v| !matches!(v, Ok(Value::Null)))
                        .collect::<Result<_, _>>()?,
                    _ => return Err("apply needs object or yaml".into()),
                };
                let dry = input["dryRun"] == true;
                let mut applied = vec![];
                for obj in &objects {
                    let k = Self::key(&input, Some(obj))?;
                    let mut pp = PatchParams::apply(MANAGER).force();
                    pp.dry_run = dry;
                    let existed = !dry && self.api(&k).await?.get_opt(&k.name).await.map_err(|e| e.to_string())?.is_some();
                    let out = self.api(&k).await?.patch(&k.name, &pp, &Patch::Apply(obj)).await.map_err(|e| format!("apply {}/{}: {e}", k.kind, k.name))?;
                    if !dry && !existed {
                        let mut c = self.created.lock().unwrap();
                        if !c.contains(&k) {
                            c.push(k);
                        }
                    }
                    applied.push(serde_json::to_value(out).unwrap());
                }
                Ok(json!({ "applied": applied }))
            }
            PATCH => {
                let k = Self::key(&input, None)?;
                let patch = input.get("patch").ok_or("patch is required")?;
                let now = self.writable(&k, "patch").await?.ok_or_else(|| format!("{}/{} is gone", k.kind, k.name))?;
                if !self.created.lock().unwrap().contains(&k) {
                    let mut saved = self.saved.lock().unwrap();
                    if !saved.iter().any(|(sk, _)| *sk == k) {
                        saved.push((k.clone(), now["spec"].clone()));
                    }
                }
                let out = self.api(&k).await?.patch(&k.name, &PatchParams::default(), &Patch::Merge(patch)).await.map_err(|e| e.to_string())?;
                Ok(serde_json::to_value(out).unwrap())
            }
            DELETE => {
                let k = Self::key(&input, None)?;
                let gone = |deleted: bool| {
                    self.created.lock().unwrap().retain(|c| *c != k);
                    Ok(json!({ "deleted": deleted }))
                };
                if self.writable(&k, "delete").await?.is_none() {
                    return gone(false);
                }
                match self.api(&k).await?.delete(&k.name, &DeleteParams::default()).await {
                    Ok(_) => gone(true),
                    Err(e) if not_found(&e) => gone(false),
                    Err(e) => Err(e.to_string()),
                }
            }
            RAW => {
                let cluster = s(&input, "cluster").ok_or("cluster is required")?;
                let path = s(&input, "path").filter(|p| p.starts_with('/')).ok_or("path is required (/readyz)")?;
                let conn = self.conn(&cluster).await?;
                let req = http::Request::get(path.as_str()).body(vec![]).map_err(|e| e.to_string())?;
                match conn.client.request_text(req).await {
                    Ok(text) => Ok(json!({ "status": 200, "text": text })),
                    Err(kube::Error::Api(e)) => Ok(json!({ "status": e.code, "text": e.message })),
                    Err(e) => Err(e.to_string()),
                }
            }
            _ => Err(format!("{nsid} is not a kube call")),
        }
    }

    /// After the run, whatever ended it: put back every spec the run patched, then delete what it created (last
    /// first). The errors, if any.
    pub async fn cleanup(&self) -> Vec<String> {
        let mut errors = vec![];
        let saved: Vec<_> = std::mem::take(&mut *self.saved.lock().unwrap());
        for (k, orig) in saved.into_iter().rev() {
            let r = async {
                let api = self.api(&k).await?;
                let Some(now) = api.get_opt(&k.name).await.map_err(|e| e.to_string())? else { return Ok(()) };
                let now = serde_json::to_value(now).unwrap();
                let p = json!({ "spec": restore_patch(&orig, &now["spec"]) });
                api.patch(&k.name, &PatchParams::default(), &Patch::Merge(&p)).await.map_err(|e| e.to_string()).map(|_| ())
            }
            .await;
            if let Err(e) = r {
                errors.push(format!("restore {}/{}: {e}", k.kind, k.name));
            }
        }
        let created: Vec<_> = std::mem::take(&mut *self.created.lock().unwrap());
        for k in created.into_iter().rev() {
            let r = async {
                match self.api(&k).await?.delete(&k.name, &DeleteParams::default()).await {
                    Ok(_) => Ok(()),
                    Err(e) if not_found(&e) => Ok(()),
                    Err(e) => Err(e.to_string()),
                }
            }
            .await;
            if let Err(e) = r {
                errors.push(format!("delete {}/{}: {e}", k.kind, k.name));
            }
        }
        errors
    }
}

fn input_of(request: &Request) -> Value {
    request.input.clone().unwrap_or_else(|| Value::Object(request.params.clone().unwrap_or_default()))
}

impl Call for Kube {
    fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Option<Value>, String>> {
        Box::pin(async move { self.handle(&nsid, input_of(&request)).await.map(Some) })
    }

    /// kube.watch: each change of the kind (optionally one name) as a message {type: apply|delete|synced, object}.
    fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Sub, String>> {
        Box::pin(async move {
            if nsid != WATCH {
                return Err(format!("{nsid} is not a subscription"));
            }
            let input = input_of(&request);
            let k = Self::key(&input, None)?;
            let api = self.api(&k).await?;
            let mut cfg = kube::runtime::watcher::Config::default();
            if !k.name.is_empty() {
                cfg = cfg.fields(&format!("metadata.name={}", k.name));
            }
            let (tx, messages) = mpsc::unbounded_channel();
            let (commands, mut cmds) = mpsc::unbounded_channel();
            tokio::spawn(async move {
                let mut stream = kube::runtime::watcher(api, cfg).boxed();
                loop {
                    tokio::select! {
                        cmd = cmds.recv() => match cmd {
                            Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("kube.watch takes no sends".into())); }
                            Some(SubCmd::Close) | None => break,
                        },
                        ev = stream.next() => {
                            use kube::runtime::watcher::Event;
                            let m = match ev {
                                None => break,
                                Some(Err(e)) => Err(e.to_string()),
                                Some(Ok(Event::Apply(o) | Event::InitApply(o))) => Ok(json!({ "type": "apply", "object": o })),
                                Some(Ok(Event::Delete(o))) => Ok(json!({ "type": "delete", "object": o })),
                                Some(Ok(Event::InitDone)) => Ok(json!({ "type": "synced" })),
                                Some(Ok(Event::Init)) => continue,
                            };
                            if tx.send(m).is_err() {
                                break;
                            }
                        }
                    }
                }
            });
            Ok(Sub { messages, commands })
        })
    }
}