~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
350 lines · 12.0 KB Raw History
//! One kcp (or a tenant k3s) as a kube-rs client: the kubeconfig's credentials over a transport (target::Via).
//! Every kind of ours is cluster-scoped; read and written as dynamic objects.

use std::time::{Duration, Instant};

use anyhow::{Context, Result, anyhow, bail, ensure};
use kube::api::{
    Api, ApiResource, DeleteParams, DynamicObject, GroupVersionKind, ListParams, Patch, PatchParams,
};
use kube::config::{KubeConfigOptions, Kubeconfig};
use serde_json::Value;

use crate::target::{KcpEndpoint, Via, expand};

pub const GROUP: &str = "socialweb.computer";
const MANAGER: &str = "swc-system-tests";

pub struct Kcp {
    pub client: kube::Client,
    /// the forward of a non-direct transport (a child, or a bridge task); ends on drop
    _forward: Option<Forward>,
}

pub enum Forward {
    Child(#[allow(dead_code)] tokio::process::Child),
    Bridge(tokio::task::JoinHandle<()>),
}

impl Drop for Forward {
    fn drop(&mut self) {
        if let Forward::Bridge(h) = self {
            h.abort();
        }
    }
}

fn plural(kind: &str) -> String {
    let k = kind.to_lowercase();
    match k.strip_suffix('y') {
        Some(stem) => format!("{stem}ies"),
        None => format!("{k}s"),
    }
}

/// socialweb.computer/v1alpha1 `kind`.
pub fn resource(kind: &str) -> ApiResource {
    resource_in(GROUP, kind)
}

/// `kind` of another group of ours (events.socialweb.computer, exposure.socialweb.computer), v1alpha1.
pub fn resource_in(group: &str, kind: &str) -> ApiResource {
    ApiResource::from_gvk_with_plural(
        &GroupVersionKind::gvk(group, "v1alpha1", kind),
        &plural(kind),
    )
}

async fn free_port() -> Result<u16> {
    Ok(tokio::net::TcpListener::bind("127.0.0.1:0")
        .await?
        .local_addr()?
        .port())
}

/// A child that forwards 127.0.0.1:<local> to the kcp; waits until the port accepts.
async fn forward_child(
    mut cmd: tokio::process::Command,
    local: u16,
    what: &str,
) -> Result<Forward> {
    let child = cmd
        .stdin(std::process::Stdio::null())
        .stdout(std::process::Stdio::null())
        .stderr(std::process::Stdio::null())
        .kill_on_drop(true)
        .spawn()
        .with_context(|| format!("{what} on PATH"))?;
    let deadline = Instant::now() + Duration::from_secs(30);
    while tokio::net::TcpStream::connect(("127.0.0.1", local))
        .await
        .is_err()
    {
        if Instant::now() > deadline {
            bail!("the {what} forward to the kcp did not open in 30 s");
        }
        tokio::time::sleep(Duration::from_millis(200)).await;
    }
    Ok(Forward::Child(child))
}

/// Listen on 127.0.0.1; each connection runs `argv` and pipes the TCP stream through its stdin/stdout.
pub async fn stdio_bridge(argv: Vec<String>) -> Result<(u16, Forward)> {
    let l = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
    let port = l.local_addr()?.port();
    let task = tokio::spawn(async move {
        while let Ok((mut tcp, _)) = l.accept().await {
            let argv = argv.clone();
            tokio::spawn(async move {
                let Ok(mut child) = tokio::process::Command::new(&argv[0])
                    .args(&argv[1..])
                    .stdin(std::process::Stdio::piped())
                    .stdout(std::process::Stdio::piped())
                    .stderr(std::process::Stdio::null())
                    .kill_on_drop(true)
                    .spawn()
                else {
                    return;
                };
                let mut pipe =
                    tokio::io::join(child.stdout.take().unwrap(), child.stdin.take().unwrap());
                let _ = tokio::io::copy_bidirectional(&mut tcp, &mut pipe).await;
            });
        }
    });
    Ok((port, Forward::Bridge(task)))
}

impl Kcp {
    pub async fn kubeconfig(path: &str) -> Result<Self> {
        Self::connect(&KcpEndpoint {
            kubeconfig: path.into(),
            workspace: None,
            via: Via::Direct,
        })
        .await
    }

    pub async fn connect(ep: &KcpEndpoint) -> Result<Self> {
        let path = expand(&ep.kubeconfig);
        let kc = Kubeconfig::read_from(&path).with_context(|| path.display().to_string())?;
        let mut cfg =
            kube::Config::from_custom_kubeconfig(kc, &KubeConfigOptions::default()).await?;
        let url = cfg.cluster_url.clone();
        let host = url
            .host()
            .context("kubeconfig server has no host")?
            .trim_matches(|c| c == '[' || c == ']')
            .to_string();
        let port = url.port_u16().unwrap_or(443);
        let mut path_q = url.path().trim_end_matches('/').to_string();
        if let Some(ws) = &ep.workspace {
            path_q = format!("/clusters/{ws}");
        }
        let mut forward = None;
        let authority = match &ep.via {
            Via::Direct => format!("{host}:{port}"),
            Via::Overlay { name } => format!("{name}:{port}"),
            Via::Ssh { host: through } => {
                let local = free_port().await?;
                let mut c = tokio::process::Command::new("ssh");
                c.args([
                    "-N",
                    "-o",
                    "BatchMode=yes",
                    "-o",
                    "ExitOnForwardFailure=yes",
                    "-L",
                ])
                .arg(format!("127.0.0.1:{local}:{host}:{port}"))
                .arg(through);
                forward = Some(forward_child(c, local, "ssh").await?);
                format!("127.0.0.1:{local}")
            }
            // atlibs-iroh's raw form: its ticket check, then `dumbpipe connect <ticket>` as the stream (one per
            // connection). ponytail: a child per connection; in-process QUIC belongs in atlibs-iroh (see README)
            Via::Iroh { ticket } => {
                let ticket = atlibs_iroh::ticket(Some(&Value::String(ticket.clone())))
                    .map_err(|e| anyhow!(e))?;
                let (local, f) =
                    stdio_bridge(vec!["dumbpipe".into(), "connect".into(), ticket]).await?;
                forward = Some(f);
                format!("127.0.0.1:{local}")
            }
            Via::WebRtc { proxy } => {
                ensure!(!proxy.is_empty(), "via webRtc: an empty proxy command");
                let (local, f) = stdio_bridge(proxy.clone()).await?;
                forward = Some(f);
                format!("127.0.0.1:{local}")
            }
        };
        cfg.cluster_url = format!("https://{authority}{path_q}").parse()?;
        if !matches!(ep.via, Via::Direct) {
            cfg.tls_server_name = Some(host);
        }
        Ok(Self {
            client: kube::Client::try_from(cfg)?,
            _forward: forward,
        })
    }

    /// GET a raw path under the server URL's /clusters/... prefix (`/readyz`).
    pub async fn raw(&self, path: &str) -> Result<String> {
        Ok(self
            .client
            .request_text(http::Request::get(path).body(vec![])?)
            .await?)
    }

    pub fn api(&self, kind: &str) -> Api<DynamicObject> {
        Api::all_with(self.client.clone(), &resource(kind))
    }

    pub fn api_in(&self, group: &str, kind: &str) -> Api<DynamicObject> {
        Api::all_with(self.client.clone(), &resource_in(group, kind))
    }

    /// The api of an object by its apiVersion (`<group>/v1alpha1`).
    fn api_of(&self, v: &Value, kind: &str) -> Api<DynamicObject> {
        match v["apiVersion"].as_str().and_then(|a| a.split_once('/')) {
            Some((group, _)) if group != GROUP => self.api_in(group, kind),
            _ => self.api(kind),
        }
    }

    pub async fn get(&self, kind: &str, name: &str) -> Result<Option<Value>> {
        Ok(self
            .api(kind)
            .get_opt(name)
            .await?
            .map(|o| serde_json::to_value(o).unwrap()))
    }

    pub async fn list(&self, kind: &str) -> Result<Vec<Value>> {
        Ok(self
            .api(kind)
            .list(&ListParams::default())
            .await?
            .items
            .into_iter()
            .map(|o| serde_json::to_value(o).unwrap())
            .collect())
    }

    /// Server-side apply of one object of ours (apiVersion socialweb.computer/v1alpha1).
    pub async fn apply(&self, obj: &Value) -> Result<Value> {
        let kind = obj["kind"].as_str().context("object without kind")?;
        let name = obj["metadata"]["name"]
            .as_str()
            .context("object without metadata.name")?;
        let out = self
            .api_of(obj, kind)
            .patch(
                name,
                &PatchParams::apply(MANAGER).force(),
                &Patch::Apply(obj),
            )
            .await
            .with_context(|| format!("apply {kind}/{name}"))?;
        Ok(serde_json::to_value(out)?)
    }

    /// Apply every document of a YAML stream (an examples/ file after its substitutions).
    pub async fn apply_yaml(&self, yaml: &str) -> Result<Vec<Value>> {
        self.apply_docs(yaml, false).await
    }

    /// `kubectl apply --dry-run=server` of a YAML stream.
    pub async fn apply_yaml_dry(&self, yaml: &str) -> Result<Vec<Value>> {
        self.apply_docs(yaml, true).await
    }

    async fn apply_docs(&self, yaml: &str, dry: bool) -> Result<Vec<Value>> {
        let mut out = vec![];
        for doc in serde_yaml::Deserializer::from_str(yaml) {
            let v: Value = serde::Deserialize::deserialize(doc)?;
            if v.is_null() {
                continue;
            }
            let kind = v["kind"].as_str().context("object without kind")?;
            let name = v["metadata"]["name"]
                .as_str()
                .context("object without metadata.name")?;
            let mut pp = PatchParams::apply(MANAGER).force();
            pp.dry_run = dry;
            let o = self
                .api(kind)
                .patch(name, &pp, &Patch::Apply(&v))
                .await
                .with_context(|| format!("apply {kind}/{name}"))?;
            out.push(serde_json::to_value(o)?);
        }
        Ok(out)
    }

    pub async fn merge(&self, kind: &str, name: &str, patch: &Value) -> Result<Value> {
        Ok(serde_json::to_value(
            self.api(kind)
                .patch(name, &PatchParams::default(), &Patch::Merge(patch))
                .await?,
        )?)
    }

    /// Delete; Ok(false) when it was already gone.
    pub async fn delete(&self, kind: &str, name: &str) -> Result<bool> {
        match self.api(kind).delete(name, &DeleteParams::default()).await {
            Ok(_) => Ok(true),
            Err(kube::Error::Api(e)) if e.code == 404 => Ok(false),
            Err(e) => Err(e.into()),
        }
    }

    /// Poll until `pred` holds for the object (None: it does not exist). ponytail: 2 s polling, a watch if it matters.
    pub async fn wait(
        &self,
        kind: &str,
        name: &str,
        timeout: Duration,
        what: &str,
        pred: impl Fn(Option<&Value>) -> bool,
    ) -> Result<Option<Value>> {
        let deadline = Instant::now() + timeout;
        loop {
            let o = self.get(kind, name).await?;
            if pred(o.as_ref()) {
                return Ok(o);
            }
            if Instant::now() > deadline {
                let phase = o
                    .as_ref()
                    .map(|o| format!("{} {}", o["status"]["phase"], o["status"]["message"]))
                    .unwrap_or_else(|| "absent".into());
                return Err(anyhow!(
                    "{kind}/{name}: not {what} after {timeout:?} (now {phase})"
                ));
            }
            tokio::time::sleep(Duration::from_secs(2)).await;
        }
    }

    pub async fn wait_phase(
        &self,
        kind: &str,
        name: &str,
        phase: &str,
        timeout: Duration,
    ) -> Result<Value> {
        let o = self
            .wait(kind, name, timeout, phase, |o| {
                o.is_some_and(|o| o["status"]["phase"] == phase)
            })
            .await?;
        Ok(o.unwrap())
    }

    pub async fn wait_gone(&self, kind: &str, name: &str, timeout: Duration) -> Result<()> {
        self.wait(kind, name, timeout, "gone", |o| o.is_none())
            .await
            .map(|_| ())
    }
}