johnandersen777.bsky.social/atlibs-kube
Kubernetes (and kcp) API calls and watches for [atlibs](https://knotless.dev/johnandersen777.bsky.social/atlibs) hosts: a transport on atlibs-host's seam.
0
0
//! com.fedfork.exec.kube.{get,list,apply,patch,delete,watch}: a transport on atlibs-host's seam. The host names its
//! clusters (`Cluster`: a kubeconfig and how to reach it); a module names a cluster, never a credential.
//!
//! Writes are scoped to the run: `apply` records what it created (the ledger); `patch` and `delete` reach only those,
//! or what the host's `Guard` allows (a test's own VM on a bidder). A guarded patch of an object the run did not create
//! saves its spec first; `cleanup()` (the host calls it after the run, whatever ended it) deletes what the run created
//! and puts every patched spec back.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use atlibs_host::BoxFut;
use atlibs_host::run::{Call, Request, Sub, SubCmd};
use atlibs_host::transport::Transport;
use futures::StreamExt;
use kube::api::{Api, ApiResource, DeleteParams, DynamicObject, GroupVersionKind, ListParams, Patch, PatchParams};
use kube::config::{KubeConfigOptions, Kubeconfig};
use serde::Deserialize;
use serde_json::{Map, Value, json};
use tokio::sync::mpsc;
pub const GET: &str = "com.fedfork.exec.kube.get";
pub const LIST: &str = "com.fedfork.exec.kube.list";
pub const APPLY: &str = "com.fedfork.exec.kube.apply";
pub const PATCH: &str = "com.fedfork.exec.kube.patch";
pub const DELETE: &str = "com.fedfork.exec.kube.delete";
pub const WATCH: &str = "com.fedfork.exec.kube.watch";
pub const RAW: &str = "com.fedfork.exec.kube.raw";
pub const NSIDS: &[&str] = &[GET, LIST, APPLY, PATCH, DELETE, WATCH, RAW];
const MANAGER: &str = "atlibs-kube";
/// One cluster the host can reach: its kubeconfig (credentials, CA), the `/clusters/<workspace>` path a kcp needs
/// when the kubeconfig names its root, and a transport.
#[derive(Clone, Debug, Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Cluster {
pub kubeconfig: String,
#[serde(default)]
pub workspace: Option<String>,
#[serde(default)]
pub via: Via,
}
#[derive(Clone, Debug, Default, Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase", tag = "kind")]
pub enum Via {
/// the kubeconfig's server URL
#[default]
Direct,
/// the same port on another name (a meshd overlay name), TLS name kept
Overlay { name: String },
/// an `ssh -L` forward through an ssh destination to the server's host:port (a front on its host's loopback)
Ssh { host: String },
/// atlibs-iroh's raw form: `dumbpipe connect <ticket>` per connection
Iroh { ticket: String },
/// any command piping stdin/stdout to the server's port, one run per connection (a WebRTC datachannel proxy)
Proxy { command: Vec<String> },
}
/// May the run patch or delete this object it did not create? (cluster, verb "patch" | "delete", the object now)
pub type Guard = Arc<dyn Fn(&str, &str, &Value) -> bool + Send + Sync>;
#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct Key {
cluster: String,
api_version: String,
kind: String,
plural: String,
namespace: Option<String>,
name: String,
}
struct Conn {
client: kube::Client,
_forward: Option<Forward>,
}
enum Forward {
Child(#[allow(dead_code)] tokio::process::Child),
Bridge(tokio::task::JoinHandle<()>),
}
impl Drop for Forward {
fn drop(&mut self) {
if let Forward::Bridge(h) = self {
h.abort();
}
}
}
pub struct Kube {
clusters: Mutex<HashMap<String, Cluster>>,
guard: Option<Guard>,
conns: tokio::sync::Mutex<HashMap<String, Arc<Conn>>>,
created: Mutex<Vec<Key>>,
/// objects the run patched without creating them: their spec before the first patch
saved: Mutex<Vec<(Key, Value)>>,
}
fn plural(kind: &str) -> String {
let k = kind.to_lowercase();
match k.strip_suffix('y') {
Some(stem) => format!("{stem}ies"),
None => format!("{k}s"),
}
}
fn s(v: &Value, k: &str) -> Option<String> {
v.get(k).and_then(Value::as_str).map(String::from)
}
/// The merge patch that turns `cur` back into `orig` (keys the run added become null).
pub fn restore_patch(orig: &Value, cur: &Value) -> Value {
match (orig, cur) {
(Value::Object(o), Value::Object(c)) => {
let mut p = Map::new();
for (k, cv) in c {
match o.get(k) {
None => {
p.insert(k.clone(), Value::Null);
}
Some(ov) if ov != cv => {
p.insert(k.clone(), restore_patch(ov, cv));
}
_ => {}
}
}
for (k, ov) in o {
if !c.contains_key(k) {
p.insert(k.clone(), ov.clone());
}
}
Value::Object(p)
}
_ => orig.clone(),
}
}
async fn free_port() -> Result<u16, String> {
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.map_err(|e| e.to_string())?;
Ok(l.local_addr().map_err(|e| e.to_string())?.port())
}
/// Listen on 127.0.0.1; each connection runs `argv` and pipes the stream through its stdin/stdout.
async fn stdio_bridge(argv: Vec<String>) -> Result<(u16, Forward), String> {
if argv.is_empty() {
return Err("an empty proxy command".into());
}
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.map_err(|e| e.to_string())?;
let port = l.local_addr().map_err(|e| e.to_string())?.port();
let task = tokio::spawn(async move {
while let Ok((mut tcp, _)) = l.accept().await {
let argv = argv.clone();
tokio::spawn(async move {
let Ok(mut child) = tokio::process::Command::new(&argv[0])
.args(&argv[1..])
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.kill_on_drop(true)
.spawn()
else {
return;
};
let mut pipe = tokio::io::join(child.stdout.take().unwrap(), child.stdin.take().unwrap());
let _ = tokio::io::copy_bidirectional(&mut tcp, &mut pipe).await;
});
}
});
Ok((port, Forward::Bridge(task)))
}
/// A kube client of one cluster over its transport.
async fn connect(c: &Cluster) -> Result<Conn, String> {
let kc = Kubeconfig::read_from(&c.kubeconfig).map_err(|e| format!("{}: {e}", c.kubeconfig))?;
let mut cfg = kube::Config::from_custom_kubeconfig(kc, &KubeConfigOptions::default()).await.map_err(|e| e.to_string())?;
let url = cfg.cluster_url.clone();
let host = url.host().ok_or("the kubeconfig's server has no host")?.trim_matches(['[', ']']).to_string();
let port = url.port_u16().unwrap_or(443);
let path = match &c.workspace {
Some(ws) => format!("/clusters/{ws}"),
None => url.path().trim_end_matches('/').to_string(),
};
let mut forward = None;
let authority = match &c.via {
Via::Direct => format!("{host}:{port}"),
Via::Overlay { name } => format!("{name}:{port}"),
Via::Ssh { host: through } => {
let local = free_port().await?;
let child = tokio::process::Command::new("ssh")
.args(["-N", "-o", "BatchMode=yes", "-o", "ExitOnForwardFailure=yes", "-L"])
.arg(format!("127.0.0.1:{local}:{host}:{port}"))
.arg(through)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.kill_on_drop(true)
.spawn()
.map_err(|e| format!("ssh: {e}"))?;
forward = Some(Forward::Child(child));
let deadline = Instant::now() + Duration::from_secs(30);
while tokio::net::TcpStream::connect(("127.0.0.1", local)).await.is_err() {
if Instant::now() > deadline {
return Err(format!("the ssh forward through {through} did not open in 30 s"));
}
tokio::time::sleep(Duration::from_millis(200)).await;
}
format!("127.0.0.1:{local}")
}
Via::Iroh { ticket } => {
let ticket = atlibs_iroh::ticket(Some(&Value::String(ticket.clone())))?;
let (local, f) = stdio_bridge(vec!["dumbpipe".into(), "connect".into(), ticket]).await?;
forward = Some(f);
format!("127.0.0.1:{local}")
}
Via::Proxy { command } => {
let (local, f) = stdio_bridge(command.clone()).await?;
forward = Some(f);
format!("127.0.0.1:{local}")
}
};
cfg.cluster_url = format!("https://{authority}{path}").parse().map_err(|e| format!("{e}"))?;
if !matches!(c.via, Via::Direct) {
cfg.tls_server_name = Some(host);
}
Ok(Conn { client: kube::Client::try_from(cfg).map_err(|e| e.to_string())?, _forward: forward })
}
fn not_found(e: &kube::Error) -> bool {
matches!(e, kube::Error::Api(a) if a.code == 404)
}
impl Kube {
pub fn new(clusters: HashMap<String, Cluster>, guard: Option<Guard>) -> Arc<Self> {
Arc::new(Self { clusters: Mutex::new(clusters), guard, conns: Default::default(), created: Default::default(), saved: Default::default() })
}
/// What the run created and what it patched (with the spec before): a host keeps it on disk, so a cleanup
/// works after the host died (`load_ledger` in a new process).
pub fn ledger(&self) -> Value {
json!({ "created": *self.created.lock().unwrap(), "saved": self.saved.lock().unwrap().iter().map(|(k, s)| json!({ "key": k, "spec": s })).collect::<Vec<_>>() })
}
pub fn load_ledger(&self, v: &Value) -> Result<(), String> {
let created: Vec<Key> = serde_json::from_value(v["created"].clone()).map_err(|e| e.to_string())?;
let saved: Vec<(Key, Value)> = v["saved"].as_array().into_iter().flatten().filter_map(|x| Some((serde_json::from_value(x["key"].clone()).ok()?, x["spec"].clone()))).collect();
self.created.lock().unwrap().extend(created);
self.saved.lock().unwrap().extend(saved);
Ok(())
}
/// The clusters it knows (kubeconfig paths and transports, no credential): a dump's.
pub fn clusters(&self) -> Value {
json!(*self.clusters.lock().unwrap())
}
/// A cluster the host learned during the run (a requester it started: its kcp).
pub fn add_cluster(&self, name: &str, c: Cluster) {
self.clusters.lock().unwrap().insert(name.into(), c);
}
pub fn transport(self: &Arc<Self>) -> Transport {
Transport { provides: NSIDS, call: self.clone() }
}
async fn conn(&self, cluster: &str) -> Result<Arc<Conn>, String> {
let mut conns = self.conns.lock().await;
if let Some(c) = conns.get(cluster) {
return Ok(c.clone());
}
let c = self.clusters.lock().unwrap().get(cluster).cloned().ok_or_else(|| format!("no cluster {cluster} on this host"))?;
let conn = Arc::new(connect(&c).await.map_err(|e| format!("cluster {cluster}: {e}"))?);
conns.insert(cluster.into(), conn.clone());
Ok(conn)
}
fn key(input: &Value, obj: Option<&Value>) -> Result<Key, String> {
let from = |k: &str| obj.and_then(|o| s(o, k)).or_else(|| s(input, k));
let api_version = from("apiVersion").ok_or("apiVersion is required")?;
let kind = from("kind").ok_or("kind is required")?;
let name = obj.and_then(|o| s(&o["metadata"], "name")).or_else(|| s(input, "name")).unwrap_or_default();
let namespace = obj.and_then(|o| s(&o["metadata"], "namespace")).or_else(|| s(input, "namespace"));
Ok(Key { cluster: s(input, "cluster").ok_or("cluster is required")?, plural: s(input, "plural").unwrap_or_else(|| plural(&kind)), api_version, kind, namespace, name })
}
async fn api(&self, k: &Key) -> Result<Api<DynamicObject>, String> {
let conn = self.conn(&k.cluster).await?;
let (group, version) = k.api_version.split_once('/').unwrap_or(("", &k.api_version));
let ar = ApiResource::from_gvk_with_plural(&GroupVersionKind::gvk(group, version, &k.kind), &k.plural);
Ok(match &k.namespace {
Some(ns) => Api::namespaced_with(conn.client.clone(), ns, &ar),
None => Api::all_with(conn.client.clone(), &ar),
})
}
/// May the run write `k` (created by it, or the guard allows)? Returns the object now (None: gone).
async fn writable(&self, k: &Key, verb: &str) -> Result<Option<Value>, String> {
let api = self.api(k).await?;
let now = api.get_opt(&k.name).await.map_err(|e| e.to_string())?.map(|o| serde_json::to_value(o).unwrap());
if self.created.lock().unwrap().contains(k) {
return Ok(now);
}
let Some(obj) = &now else { return Ok(None) };
match &self.guard {
Some(g) if g(&k.cluster, verb, obj) => Ok(now),
_ => Err(format!("{verb} {}/{}: the run did not create it", k.kind, k.name)),
}
}
async fn handle(&self, nsid: &str, input: Value) -> Result<Value, String> {
match nsid {
GET => {
let k = Self::key(&input, None)?;
Ok(self.api(&k).await?.get_opt(&k.name).await.map_err(|e| e.to_string())?.map_or(Value::Null, |o| serde_json::to_value(o).unwrap()))
}
LIST => {
let k = Self::key(&input, None)?;
let mut lp = ListParams::default();
if let Some(sel) = s(&input, "labelSelector") {
lp = lp.labels(&sel);
}
let items = self.api(&k).await?.list(&lp).await.map_err(|e| e.to_string())?.items;
Ok(json!({ "items": items }))
}
APPLY => {
let objects: Vec<Value> = match (input.get("object"), s(&input, "yaml")) {
(Some(o), _) => vec![o.clone()],
(None, Some(y)) => serde_yaml::Deserializer::from_str(&y)
.map(|d| Value::deserialize(d).map_err(|e| e.to_string()))
.filter(|v| !matches!(v, Ok(Value::Null)))
.collect::<Result<_, _>>()?,
_ => return Err("apply needs object or yaml".into()),
};
let dry = input["dryRun"] == true;
let mut applied = vec![];
for obj in &objects {
let k = Self::key(&input, Some(obj))?;
let mut pp = PatchParams::apply(MANAGER).force();
pp.dry_run = dry;
let existed = !dry && self.api(&k).await?.get_opt(&k.name).await.map_err(|e| e.to_string())?.is_some();
let out = self.api(&k).await?.patch(&k.name, &pp, &Patch::Apply(obj)).await.map_err(|e| format!("apply {}/{}: {e}", k.kind, k.name))?;
if !dry && !existed {
let mut c = self.created.lock().unwrap();
if !c.contains(&k) {
c.push(k);
}
}
applied.push(serde_json::to_value(out).unwrap());
}
Ok(json!({ "applied": applied }))
}
PATCH => {
let k = Self::key(&input, None)?;
let patch = input.get("patch").ok_or("patch is required")?;
let now = self.writable(&k, "patch").await?.ok_or_else(|| format!("{}/{} is gone", k.kind, k.name))?;
if !self.created.lock().unwrap().contains(&k) {
let mut saved = self.saved.lock().unwrap();
if !saved.iter().any(|(sk, _)| *sk == k) {
saved.push((k.clone(), now["spec"].clone()));
}
}
let out = self.api(&k).await?.patch(&k.name, &PatchParams::default(), &Patch::Merge(patch)).await.map_err(|e| e.to_string())?;
Ok(serde_json::to_value(out).unwrap())
}
DELETE => {
let k = Self::key(&input, None)?;
let gone = |deleted: bool| {
self.created.lock().unwrap().retain(|c| *c != k);
Ok(json!({ "deleted": deleted }))
};
if self.writable(&k, "delete").await?.is_none() {
return gone(false);
}
match self.api(&k).await?.delete(&k.name, &DeleteParams::default()).await {
Ok(_) => gone(true),
Err(e) if not_found(&e) => gone(false),
Err(e) => Err(e.to_string()),
}
}
RAW => {
let cluster = s(&input, "cluster").ok_or("cluster is required")?;
let path = s(&input, "path").filter(|p| p.starts_with('/')).ok_or("path is required (/readyz)")?;
let conn = self.conn(&cluster).await?;
let req = http::Request::get(path.as_str()).body(vec![]).map_err(|e| e.to_string())?;
match conn.client.request_text(req).await {
Ok(text) => Ok(json!({ "status": 200, "text": text })),
Err(kube::Error::Api(e)) => Ok(json!({ "status": e.code, "text": e.message })),
Err(e) => Err(e.to_string()),
}
}
_ => Err(format!("{nsid} is not a kube call")),
}
}
/// After the run, whatever ended it: put back every spec the run patched, then delete what it created (last
/// first). The errors, if any.
pub async fn cleanup(&self) -> Vec<String> {
let mut errors = vec![];
let saved: Vec<_> = std::mem::take(&mut *self.saved.lock().unwrap());
for (k, orig) in saved.into_iter().rev() {
let r = async {
let api = self.api(&k).await?;
let Some(now) = api.get_opt(&k.name).await.map_err(|e| e.to_string())? else { return Ok(()) };
let now = serde_json::to_value(now).unwrap();
let p = json!({ "spec": restore_patch(&orig, &now["spec"]) });
api.patch(&k.name, &PatchParams::default(), &Patch::Merge(&p)).await.map_err(|e| e.to_string()).map(|_| ())
}
.await;
if let Err(e) = r {
errors.push(format!("restore {}/{}: {e}", k.kind, k.name));
}
}
let created: Vec<_> = std::mem::take(&mut *self.created.lock().unwrap());
for k in created.into_iter().rev() {
let r = async {
match self.api(&k).await?.delete(&k.name, &DeleteParams::default()).await {
Ok(_) => Ok(()),
Err(e) if not_found(&e) => Ok(()),
Err(e) => Err(e.to_string()),
}
}
.await;
if let Err(e) = r {
errors.push(format!("delete {}/{}: {e}", k.kind, k.name));
}
}
errors
}
}
fn input_of(request: &Request) -> Value {
request.input.clone().unwrap_or_else(|| Value::Object(request.params.clone().unwrap_or_default()))
}
impl Call for Kube {
fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Option<Value>, String>> {
Box::pin(async move { self.handle(&nsid, input_of(&request)).await.map(Some) })
}
/// kube.watch: each change of the kind (optionally one name) as a message {type: apply|delete|synced, object}.
fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Sub, String>> {
Box::pin(async move {
if nsid != WATCH {
return Err(format!("{nsid} is not a subscription"));
}
let input = input_of(&request);
let k = Self::key(&input, None)?;
let api = self.api(&k).await?;
let mut cfg = kube::runtime::watcher::Config::default();
if !k.name.is_empty() {
cfg = cfg.fields(&format!("metadata.name={}", k.name));
}
let (tx, messages) = mpsc::unbounded_channel();
let (commands, mut cmds) = mpsc::unbounded_channel();
tokio::spawn(async move {
let mut stream = kube::runtime::watcher(api, cfg).boxed();
loop {
tokio::select! {
cmd = cmds.recv() => match cmd {
Some(SubCmd::Send(_, reply)) => { let _ = reply.send(Err("kube.watch takes no sends".into())); }
Some(SubCmd::Close) | None => break,
},
ev = stream.next() => {
use kube::runtime::watcher::Event;
let m = match ev {
None => break,
Some(Err(e)) => Err(e.to_string()),
Some(Ok(Event::Apply(o) | Event::InitApply(o))) => Ok(json!({ "type": "apply", "object": o })),
Some(Ok(Event::Delete(o))) => Ok(json!({ "type": "delete", "object": o })),
Some(Ok(Event::InitDone)) => Ok(json!({ "type": "synced" })),
Some(Ok(Event::Init)) => continue,
};
if tx.send(m).is_err() {
break;
}
}
}
}
});
Ok(Sub { messages, commands })
})
}
}