//! com.fedfork.exec.kube.{get,list,apply,patch,delete,watch}: a transport on atlibs-host's seam. The host names its //! clusters (`Cluster`: a kubeconfig and how to reach it); a module names a cluster, never a credential. //! //! Writes are scoped to the run: `apply` records what it created (the ledger); `patch` and `delete` reach only those, //! or what the host's `Guard` allows (a test's own VM on a bidder). A guarded patch of an object the run did not create //! saves its spec first; `cleanup()` (the host calls it after the run, whatever ended it) deletes what the run created //! and puts every patched spec back. use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use atlibs_host::BoxFut; use atlibs_host::run::{Call, Request, Sub, SubCmd}; use atlibs_host::transport::Transport; use futures::StreamExt; use kube::api::{Api, ApiResource, DeleteParams, DynamicObject, GroupVersionKind, ListParams, Patch, PatchParams}; use kube::config::{KubeConfigOptions, Kubeconfig}; use serde::Deserialize; use serde_json::{Map, Value, json}; use tokio::sync::mpsc; pub const GET: &str = "com.fedfork.exec.kube.get"; pub const LIST: &str = "com.fedfork.exec.kube.list"; pub const APPLY: &str = "com.fedfork.exec.kube.apply"; pub const PATCH: &str = "com.fedfork.exec.kube.patch"; pub const DELETE: &str = "com.fedfork.exec.kube.delete"; pub const WATCH: &str = "com.fedfork.exec.kube.watch"; pub const RAW: &str = "com.fedfork.exec.kube.raw"; pub const NSIDS: &[&str] = &[GET, LIST, APPLY, PATCH, DELETE, WATCH, RAW]; const MANAGER: &str = "atlibs-kube"; /// One cluster the host can reach: its kubeconfig (credentials, CA), the `/clusters/` path a kcp needs /// when the kubeconfig names its root, and a transport. #[derive(Clone, Debug, Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct Cluster { pub kubeconfig: String, #[serde(default)] pub workspace: Option, #[serde(default)] pub via: Via, } #[derive(Clone, Debug, Default, Deserialize, serde::Serialize)] #[serde(rename_all = "camelCase", tag = "kind")] pub enum Via { /// the kubeconfig's server URL #[default] Direct, /// the same port on another name (a meshd overlay name), TLS name kept Overlay { name: String }, /// an `ssh -L` forward through an ssh destination to the server's host:port (a front on its host's loopback) Ssh { host: String }, /// atlibs-iroh's raw form: `dumbpipe connect ` per connection Iroh { ticket: String }, /// any command piping stdin/stdout to the server's port, one run per connection (a WebRTC datachannel proxy) Proxy { command: Vec }, } /// May the run patch or delete this object it did not create? (cluster, verb "patch" | "delete", the object now) pub type Guard = Arc bool + Send + Sync>; #[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] struct Key { cluster: String, api_version: String, kind: String, plural: String, namespace: Option, name: String, } struct Conn { client: kube::Client, _forward: Option, } enum Forward { Child(#[allow(dead_code)] tokio::process::Child), Bridge(tokio::task::JoinHandle<()>), } impl Drop for Forward { fn drop(&mut self) { if let Forward::Bridge(h) = self { h.abort(); } } } pub struct Kube { clusters: Mutex>, guard: Option, conns: tokio::sync::Mutex>>, created: Mutex>, /// objects the run patched without creating them: their spec before the first patch saved: Mutex>, } fn plural(kind: &str) -> String { let k = kind.to_lowercase(); match k.strip_suffix('y') { Some(stem) => format!("{stem}ies"), None => format!("{k}s"), } } fn s(v: &Value, k: &str) -> Option { v.get(k).and_then(Value::as_str).map(String::from) } /// The merge patch that turns `cur` back into `orig` (keys the run added become null). pub fn restore_patch(orig: &Value, cur: &Value) -> Value { match (orig, cur) { (Value::Object(o), Value::Object(c)) => { let mut p = Map::new(); for (k, cv) in c { match o.get(k) { None => { p.insert(k.clone(), Value::Null); } Some(ov) if ov != cv => { p.insert(k.clone(), restore_patch(ov, cv)); } _ => {} } } for (k, ov) in o { if !c.contains_key(k) { p.insert(k.clone(), ov.clone()); } } Value::Object(p) } _ => orig.clone(), } } async fn free_port() -> Result { let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.map_err(|e| e.to_string())?; Ok(l.local_addr().map_err(|e| e.to_string())?.port()) } /// Listen on 127.0.0.1; each connection runs `argv` and pipes the stream through its stdin/stdout. async fn stdio_bridge(argv: Vec) -> Result<(u16, Forward), String> { if argv.is_empty() { return Err("an empty proxy command".into()); } let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.map_err(|e| e.to_string())?; let port = l.local_addr().map_err(|e| e.to_string())?.port(); let task = tokio::spawn(async move { while let Ok((mut tcp, _)) = l.accept().await { let argv = argv.clone(); tokio::spawn(async move { let Ok(mut child) = tokio::process::Command::new(&argv[0]) .args(&argv[1..]) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::null()) .kill_on_drop(true) .spawn() else { return; }; let mut pipe = tokio::io::join(child.stdout.take().unwrap(), child.stdin.take().unwrap()); let _ = tokio::io::copy_bidirectional(&mut tcp, &mut pipe).await; }); } }); Ok((port, Forward::Bridge(task))) } /// A kube client of one cluster over its transport. async fn connect(c: &Cluster) -> Result { let kc = Kubeconfig::read_from(&c.kubeconfig).map_err(|e| format!("{}: {e}", c.kubeconfig))?; let mut cfg = kube::Config::from_custom_kubeconfig(kc, &KubeConfigOptions::default()).await.map_err(|e| e.to_string())?; let url = cfg.cluster_url.clone(); let host = url.host().ok_or("the kubeconfig's server has no host")?.trim_matches(['[', ']']).to_string(); let port = url.port_u16().unwrap_or(443); let path = match &c.workspace { Some(ws) => format!("/clusters/{ws}"), None => url.path().trim_end_matches('/').to_string(), }; let mut forward = None; let authority = match &c.via { Via::Direct => format!("{host}:{port}"), Via::Overlay { name } => format!("{name}:{port}"), Via::Ssh { host: through } => { let local = free_port().await?; let child = tokio::process::Command::new("ssh") .args(["-N", "-o", "BatchMode=yes", "-o", "ExitOnForwardFailure=yes", "-L"]) .arg(format!("127.0.0.1:{local}:{host}:{port}")) .arg(through) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .kill_on_drop(true) .spawn() .map_err(|e| format!("ssh: {e}"))?; forward = Some(Forward::Child(child)); let deadline = Instant::now() + Duration::from_secs(30); while tokio::net::TcpStream::connect(("127.0.0.1", local)).await.is_err() { if Instant::now() > deadline { return Err(format!("the ssh forward through {through} did not open in 30 s")); } tokio::time::sleep(Duration::from_millis(200)).await; } format!("127.0.0.1:{local}") } Via::Iroh { ticket } => { let ticket = atlibs_iroh::ticket(Some(&Value::String(ticket.clone())))?; let (local, f) = stdio_bridge(vec!["dumbpipe".into(), "connect".into(), ticket]).await?; forward = Some(f); format!("127.0.0.1:{local}") } Via::Proxy { command } => { let (local, f) = stdio_bridge(command.clone()).await?; forward = Some(f); format!("127.0.0.1:{local}") } }; cfg.cluster_url = format!("https://{authority}{path}").parse().map_err(|e| format!("{e}"))?; if !matches!(c.via, Via::Direct) { cfg.tls_server_name = Some(host); } Ok(Conn { client: kube::Client::try_from(cfg).map_err(|e| e.to_string())?, _forward: forward }) } fn not_found(e: &kube::Error) -> bool { matches!(e, kube::Error::Api(a) if a.code == 404) } impl Kube { pub fn new(clusters: HashMap, guard: Option) -> Arc { Arc::new(Self { clusters: Mutex::new(clusters), guard, conns: Default::default(), created: Default::default(), saved: Default::default() }) } /// What the run created and what it patched (with the spec before): a host keeps it on disk, so a cleanup /// works after the host died (`load_ledger` in a new process). pub fn ledger(&self) -> Value { json!({ "created": *self.created.lock().unwrap(), "saved": self.saved.lock().unwrap().iter().map(|(k, s)| json!({ "key": k, "spec": s })).collect::>() }) } pub fn load_ledger(&self, v: &Value) -> Result<(), String> { let created: Vec = serde_json::from_value(v["created"].clone()).map_err(|e| e.to_string())?; let saved: Vec<(Key, Value)> = v["saved"].as_array().into_iter().flatten().filter_map(|x| Some((serde_json::from_value(x["key"].clone()).ok()?, x["spec"].clone()))).collect(); self.created.lock().unwrap().extend(created); self.saved.lock().unwrap().extend(saved); Ok(()) } /// The clusters it knows (kubeconfig paths and transports, no credential): a dump's. pub fn clusters(&self) -> Value { json!(*self.clusters.lock().unwrap()) } /// A cluster the host learned during the run (a requester it started: its kcp). pub fn add_cluster(&self, name: &str, c: Cluster) { self.clusters.lock().unwrap().insert(name.into(), c); } pub fn transport(self: &Arc) -> Transport { Transport { provides: NSIDS, call: self.clone() } } async fn conn(&self, cluster: &str) -> Result, String> { let mut conns = self.conns.lock().await; if let Some(c) = conns.get(cluster) { return Ok(c.clone()); } let c = self.clusters.lock().unwrap().get(cluster).cloned().ok_or_else(|| format!("no cluster {cluster} on this host"))?; let conn = Arc::new(connect(&c).await.map_err(|e| format!("cluster {cluster}: {e}"))?); conns.insert(cluster.into(), conn.clone()); Ok(conn) } fn key(input: &Value, obj: Option<&Value>) -> Result { let from = |k: &str| obj.and_then(|o| s(o, k)).or_else(|| s(input, k)); let api_version = from("apiVersion").ok_or("apiVersion is required")?; let kind = from("kind").ok_or("kind is required")?; let name = obj.and_then(|o| s(&o["metadata"], "name")).or_else(|| s(input, "name")).unwrap_or_default(); let namespace = obj.and_then(|o| s(&o["metadata"], "namespace")).or_else(|| s(input, "namespace")); Ok(Key { cluster: s(input, "cluster").ok_or("cluster is required")?, plural: s(input, "plural").unwrap_or_else(|| plural(&kind)), api_version, kind, namespace, name }) } async fn api(&self, k: &Key) -> Result, String> { let conn = self.conn(&k.cluster).await?; let (group, version) = k.api_version.split_once('/').unwrap_or(("", &k.api_version)); let ar = ApiResource::from_gvk_with_plural(&GroupVersionKind::gvk(group, version, &k.kind), &k.plural); Ok(match &k.namespace { Some(ns) => Api::namespaced_with(conn.client.clone(), ns, &ar), None => Api::all_with(conn.client.clone(), &ar), }) } /// May the run write `k` (created by it, or the guard allows)? Returns the object now (None: gone). async fn writable(&self, k: &Key, verb: &str) -> Result, String> { let api = self.api(k).await?; let now = api.get_opt(&k.name).await.map_err(|e| e.to_string())?.map(|o| serde_json::to_value(o).unwrap()); if self.created.lock().unwrap().contains(k) { return Ok(now); } let Some(obj) = &now else { return Ok(None) }; match &self.guard { Some(g) if g(&k.cluster, verb, obj) => Ok(now), _ => Err(format!("{verb} {}/{}: the run did not create it", k.kind, k.name)), } } async fn handle(&self, nsid: &str, input: Value) -> Result { match nsid { GET => { let k = Self::key(&input, None)?; Ok(self.api(&k).await?.get_opt(&k.name).await.map_err(|e| e.to_string())?.map_or(Value::Null, |o| serde_json::to_value(o).unwrap())) } LIST => { let k = Self::key(&input, None)?; let mut lp = ListParams::default(); if let Some(sel) = s(&input, "labelSelector") { lp = lp.labels(&sel); } let items = self.api(&k).await?.list(&lp).await.map_err(|e| e.to_string())?.items; Ok(json!({ "items": items })) } APPLY => { let objects: Vec = match (input.get("object"), s(&input, "yaml")) { (Some(o), _) => vec![o.clone()], (None, Some(y)) => serde_yaml::Deserializer::from_str(&y) .map(|d| Value::deserialize(d).map_err(|e| e.to_string())) .filter(|v| !matches!(v, Ok(Value::Null))) .collect::>()?, _ => return Err("apply needs object or yaml".into()), }; let dry = input["dryRun"] == true; let mut applied = vec![]; for obj in &objects { let k = Self::key(&input, Some(obj))?; let mut pp = PatchParams::apply(MANAGER).force(); pp.dry_run = dry; let existed = !dry && self.api(&k).await?.get_opt(&k.name).await.map_err(|e| e.to_string())?.is_some(); let out = self.api(&k).await?.patch(&k.name, &pp, &Patch::Apply(obj)).await.map_err(|e| format!("apply {}/{}: {e}", k.kind, k.name))?; if !dry && !existed { let mut c = self.created.lock().unwrap(); if !c.contains(&k) { c.push(k); } } applied.push(serde_json::to_value(out).unwrap()); } Ok(json!({ "applied": applied })) } PATCH => { let k = Self::key(&input, None)?; let patch = input.get("patch").ok_or("patch is required")?; let now = self.writable(&k, "patch").await?.ok_or_else(|| format!("{}/{} is gone", k.kind, k.name))?; if !self.created.lock().unwrap().contains(&k) { let mut saved = self.saved.lock().unwrap(); if !saved.iter().any(|(sk, _)| *sk == k) { saved.push((k.clone(), now["spec"].clone())); } } let out = self.api(&k).await?.patch(&k.name, &PatchParams::default(), &Patch::Merge(patch)).await.map_err(|e| e.to_string())?; Ok(serde_json::to_value(out).unwrap()) } DELETE => { let k = Self::key(&input, None)?; let gone = |deleted: bool| { self.created.lock().unwrap().retain(|c| *c != k); Ok(json!({ "deleted": deleted })) }; if self.writable(&k, "delete").await?.is_none() { return gone(false); } match self.api(&k).await?.delete(&k.name, &DeleteParams::default()).await { Ok(_) => gone(true), Err(e) if not_found(&e) => gone(false), Err(e) => Err(e.to_string()), } } RAW => { let cluster = s(&input, "cluster").ok_or("cluster is required")?; let path = s(&input, "path").filter(|p| p.starts_with('/')).ok_or("path is required (/readyz)")?; let conn = self.conn(&cluster).await?; let req = http::Request::get(path.as_str()).body(vec![]).map_err(|e| e.to_string())?; match conn.client.request_text(req).await { Ok(text) => Ok(json!({ "status": 200, "text": text })), Err(kube::Error::Api(e)) => Ok(json!({ "status": e.code, "text": e.message })), Err(e) => Err(e.to_string()), } } _ => Err(format!("{nsid} is not a kube call")), } } /// After the run, whatever ended it: put back every spec the run patched, then delete what it created (last /// first). The errors, if any. pub async fn cleanup(&self) -> Vec { let mut errors = vec![]; let saved: Vec<_> = std::mem::take(&mut *self.saved.lock().unwrap()); for (k, orig) in saved.into_iter().rev() { let r = async { let api = self.api(&k).await?; let Some(now) = api.get_opt(&k.name).await.map_err(|e| e.to_string())? else { return Ok(()) }; let now = serde_json::to_value(now).unwrap(); let p = json!({ "spec": restore_patch(&orig, &now["spec"]) }); api.patch(&k.name, &PatchParams::default(), &Patch::Merge(&p)).await.map_err(|e| e.to_string()).map(|_| ()) } .await; if let Err(e) = r { errors.push(format!("restore {}/{}: {e}", k.kind, k.name)); } } let created: Vec<_> = std::mem::take(&mut *self.created.lock().unwrap()); for k in created.into_iter().rev() { let r = async { match self.api(&k).await?.delete(&k.name, &DeleteParams::default()).await { Ok(_) => Ok(()), Err(e) if not_found(&e) => Ok(()), Err(e) => Err(e.to_string()), } } .await; if let Err(e) = r { errors.push(format!("delete {}/{}: {e}", k.kind, k.name)); } } errors } } fn input_of(request: &Request) -> Value { request.input.clone().unwrap_or_else(|| Value::Object(request.params.clone().unwrap_or_default())) } impl Call for Kube { fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result, String>> { Box::pin(async move { self.handle(&nsid, input_of(&request)).await.map(Some) }) } /// kube.watch: each change of the kind (optionally one name) as a message {type: apply|delete|synced, object}. fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result> { Box::pin(async move { if nsid != WATCH { return Err(format!("{nsid} is not a subscription")); } let input = input_of(&request); let k = Self::key(&input, None)?; let api = self.api(&k).await?; let mut cfg = kube::runtime::watcher::Config::default(); if !k.name.is_empty() { cfg = cfg.fields(&format!("metadata.name={}", k.name)); } let (tx, messages) = mpsc::unbounded_channel(); let (commands, mut cmds) = mpsc::unbounded_channel(); tokio::spawn(async move { let mut stream = kube::runtime::watcher(api, cfg).boxed(); loop { tokio::select! { cmd = cmds.recv() => match cmd { Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("kube.watch takes no sends".into())); } Some(SubCmd::Close) | None => break, }, ev = stream.next() => { use kube::runtime::watcher::Event; let m = match ev { None => break, Some(Err(e)) => Err(e.to_string()), Some(Ok(Event::Apply(o) | Event::InitApply(o))) => Ok(json!({ "type": "apply", "object": o })), Some(Ok(Event::Delete(o))) => Ok(json!({ "type": "delete", "object": o })), Some(Ok(Event::InitDone)) => Ok(json!({ "type": "synced" })), Some(Ok(Event::Init)) => continue, }; if tx.send(m).is_err() { break; } } } } }); Ok(Sub { messages, commands }) }) } }