~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
300 lines · 14.9 KB Raw History
//! The read-only operations routes of a target (GET /xrpc/com.fedfork.exec.ops.*, docs/SYSTEM.md): what the host reads
//! of a bidder or the ssh gateway it has no kcp or shell on (prod in CI). A bidder with `ops` in the target is the
//! kube cluster `bidder:<name>` served here: get and list of the kinds the routes answer, in the routes' fixed projection
//! (the fields the modules read), every write refused. One shared secret for every bidder and the gateway (owner): it is the target's `opsSecretFile` (0600), sent as the bearer and never logged.
//! A bidder's routes are on its market mux, reached over its market endpoint: https (`ops.url`) where its offering has
//! one, else iroh (`ops.iroh`, the `endpoint...` ticket of its market.offering record): the same GETs and bearer as
//! HTTP/1.1 over atlibs-iroh's raw fetch (`testkit::irohfetch`: `dumbpipe connect <ticket>`, the stream a requester's
//! market calls take), end-to-end encrypted to the iroh node the ticket names. The gateway's route is its https web origin.

use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

use atlibs_host::BoxFut;
use atlibs_host::run::{Call, Request, Sub};
use atlibs_kube::Kube;
use serde_json::{Value, json};
use testkit::target::{OpsRead, Target};

const NS: &str = "com.fedfork.exec.ops.";

/// One ops read, every try and wait included, ends within this bound: below the smallest deadline of an entry that
/// reads ops routes less its latest read in a passing run (0014:abandon: 15 min; its last poll starts by about 700 s), so
/// a lost transport is what the entry reports, not its deadline.
const READ_BUDGET: Duration = Duration::from_secs(150);
/// One try's answer time, within what is left of READ_BUDGET.
const READ_TRY_TIMEOUT: Duration = Duration::from_secs(60);
const READ_RETRY_BACKOFF: Duration = Duration::from_secs(5);

/// The known messages of com.fedfork.exec.ops.getLog (internal/contract/opsread Messages): name, the texts a line holds.
const MESSAGES: &[(&str, &[&str])] = &[
    ("core_holds_instance", &["core holds the instance"]),
    ("adopted_role", &["\"msg\":\"adopted\"", "\"role\":"]),
    ("placed", &["\"msg\":\"placed\""]),
    ("stopped", &["\"msg\":\"stopped\""]),
    ("work_dir_removed", &["work dir and API socket removed"]),
    ("shutdown", &["\"msg\":\"shutdown\""]),
    ("vmm_exit", &["the process wait saw the VMM exit"]),
    ("bidder_vm_terminated", &["bidder_vm_terminated"]),
    ("released_bid_terminated", &["a released bid is marked terminated"]),
    ("hostprocess_unit_adopted", &["hostprocess_unit_adopted"]),
    ("early_kill", &["before its VMM answered"]),
    ("bidder_accept_abandoned", &["bidder_accept_abandoned"]),
    ("bidder_accept_unresolved", &["bidder_accept_unresolved"]),
    ("accept_provision_failed", &["the provision of an accept failed"]),
];

/// Where a bidder's ops routes are: an https origin, or an iroh market endpoint ticket.
enum Base {
    Https(String),
    Iroh(String),
}

impl Base {
    fn of(o: &OpsRead) -> Result<Self, String> {
        match (o.url.as_deref(), o.iroh.as_deref()) {
            // the bearer goes to https origins only
            (Some(u), None) if u.starts_with("https://") => Ok(Self::Https(u.trim_end_matches('/').to_string())),
            (Some(u), None) => Err(format!("the ops route {u} is not https: the ops read secret is sent to https origins only")),
            (None, Some(t)) => {
                testkit::irohfetch::ticket(t).map(Self::Iroh).ok_or(format!("the ops iroh endpoint {t} is not an iroh endpoint ticket"))
            }
            _ => Err("an ops endpoint is exactly one of url (https) and iroh".into()),
        }
    }
}

pub struct Ops {
    http: reqwest::Client,
    secret: String,
    bidders: HashMap<String, Base>,
    gateway: Option<Base>,
}

impl Ops {
    /// None: the target names no ops route.
    pub fn load(target: &Target) -> Result<Option<Arc<Self>>, String> {
        let mut bidders = HashMap::new();
        for b in target.bidders.iter().filter(|b| b.kcp.is_none()) {
            if let Some(o) = &b.ops {
                bidders.insert(b.name.clone(), Base::of(o).map_err(|e| format!("bidder {}: {e}", b.name))?);
            }
        }
        let gateway = match target.gateway.ops.as_ref().map(Base::of).transpose()? {
            Some(Base::Iroh(_)) => return Err("the gateway's ops route is its https web origin, not iroh".into()),
            g => g,
        };
        if bidders.is_empty() && gateway.is_none() {
            return Ok(None);
        }
        let file = target.ops_secret_file.as_deref().ok_or("the target names ops routes and no opsSecretFile")?;
        let path = testkit::target::expand(file);
        let secret = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?.trim().to_string();
        if secret.is_empty() {
            return Err(format!("{} is empty", path.display()));
        }
        let http = reqwest::Client::builder().timeout(Duration::from_secs(120)).build().map_err(|e| e.to_string())?;
        Ok(Some(Arc::new(Self { http, secret, bidders, gateway })))
    }

    /// The bidder `name` is read through its ops routes.
    pub fn serves(&self, name: &str) -> bool {
        self.bidders.contains_key(name)
    }

    pub fn serves_gateway(&self) -> bool {
        self.gateway.is_some()
    }

    /// One GET of an ops route. The routes are read-only, so a request whose transport failed (a lost iroh session or
    /// relay, an https connection error, no answer in READ_TRY_TIMEOUT) is sent again, 5 s, 10 s, 20 s... apart, until
    /// READ_BUDGET is spent; an answer, whatever its status, is never retried. The error names every failed try.
    async fn get(&self, base: &Base, method: &str, params: &[(&str, &str)]) -> Result<Value, String> {
        let until = tokio::time::Instant::now() + READ_BUDGET;
        let mut failed: Vec<String> = vec![];
        let (status, body) = loop {
            let left = until.saturating_duration_since(tokio::time::Instant::now());
            match self.send(base, method, params, left.min(READ_TRY_TIMEOUT)).await {
                Ok(answer) => break answer,
                Err(e) => failed.push(format!("try {}: {e}", failed.len() + 1)),
            }
            let wait = READ_RETRY_BACKOFF * 2u32.pow(failed.len() as u32 - 1);
            if tokio::time::Instant::now() + wait >= until {
                return Err(failed.join("; "));
            }
            tokio::time::sleep(wait).await;
        };
        if !(200..300).contains(&status) {
            return Err(format!("{NS}{method}: {status} {} {}", body["error"].as_str().unwrap_or_default(), body["message"].as_str().unwrap_or_default()));
        }
        Ok(body)
    }

    async fn send(&self, base: &Base, method: &str, params: &[(&str, &str)], timeout: Duration) -> Result<(u16, Value), String> {
        Ok(match base {
            Base::Https(base) => {
                let url = format!("{base}/xrpc/{NS}{method}");
                let res = self.http.get(&url).query(params).bearer_auth(&self.secret).timeout(timeout).send().await.map_err(|e| format!("{NS}{method}: {}", e.without_url()))?;
                let status = res.status().as_u16();
                (status, res.json::<Value>().await.unwrap_or(Value::Null))
            }
            Base::Iroh(ticket) => {
                let target = format!("/xrpc/{NS}{method}{}", testkit::irohfetch::query(params));
                let headers = vec![("authorization".to_string(), format!("Bearer {}", self.secret))];
                let (status, body) = testkit::irohfetch::request(ticket, "GET", &target, headers, vec![], timeout)
                    .await
                    .map_err(|e| format!("{NS}{method}: {e}"))?;
                (status, serde_json::from_slice(&body).unwrap_or(Value::Null))
            }
        })
    }

    fn bidder(&self, name: &str) -> Result<&Base, String> {
        self.bidders.get(name).ok_or(format!("no ops route for the bidder {name}"))
    }

    pub async fn health(&self, name: &str) -> Result<Value, String> {
        self.get(self.bidder(name)?, "getHealth", &[]).await
    }

    pub async fn memory(&self, name: &str, vm: &str) -> Result<Value, String> {
        self.get(self.bidder(name)?, "getVmMemory", &[("name", vm)]).await
    }

    async fn log(&self, name: &str, text: &[String]) -> Result<Value, String> {
        let (msg, subject) = message_of(text).ok_or(format!("the ops routes serve no log line of {text:?}"))?;
        let mut params = vec![("msg", msg)];
        if let Some(s) = subject.as_deref() {
            params.push(("subject", s));
        }
        self.get(self.bidder(name)?, "getLog", &params).await
    }

    /// bidder.logCount: {counts: {<texts joined by " & ">: n}, log}.
    pub async fn log_count(&self, name: &str, texts: &[Vec<String>]) -> Result<Value, String> {
        let mut counts = serde_json::Map::new();
        let mut from = Value::Null;
        for t in texts {
            let o = self.log(name, t).await?;
            counts.insert(t.join(" & "), o["count"].clone());
            from = o["log"].clone();
        }
        Ok(json!({ "counts": counts, "log": from }))
    }

    /// bidder.logLines: {lines: [{time, msg, <keys>}], log} of the first entry of `texts`.
    pub async fn log_lines(&self, name: &str, texts: &[Vec<String>], keys: &[String]) -> Result<Value, String> {
        let o = self.log(name, &texts[0]).await?;
        let rows: Vec<Value> = (o["lines"].as_array().into_iter().flatten())
            .map(|v| {
                let mut row = json!({ "time": v["time"], "msg": v["msg"] });
                for k in keys {
                    row[k] = v[k].clone();
                }
                row
            })
            .collect();
        Ok(json!({ "lines": rows, "log": o["log"] }))
    }

    /// gateway.log: the ssh gateway's lines of `did` since `since` whose msg is one of `msgs` (none: every known one).
    pub async fn gateway_log(&self, did: &str, since: &str, msgs: &[String]) -> Result<Value, String> {
        let base = self.gateway.as_ref().ok_or("the target names no gateway ops route")?;
        let o = self.get(base, "getRequesterLog", &[("did", did), ("since", since)]).await?;
        Ok(gateway_lines(o["lines"].as_array().map(Vec::as_slice).unwrap_or_default(), msgs))
    }

    /// A kube call on a cluster served here; None: the cluster is not one of these.
    async fn kube(&self, verb: &str, input: &Value) -> Option<Result<Value, String>> {
        let cluster = input["cluster"].as_str()?;
        let name = cluster.strip_prefix("bidder:")?;
        let base = self.bidders.get(name)?;
        let kind = input["kind"].as_str().unwrap_or_default();
        let object = input["name"].as_str().unwrap_or_default();
        let refuse = || Err(format!("{cluster} is reached through the read-only ops routes: {verb} {kind} is not served"));
        let typed = |kind: &str, o: &Value| match o {
            Value::Null => Value::Null,
            o => {
                let mut v = o.clone();
                v["apiVersion"] = json!("socialweb.computer/v1alpha1");
                v["kind"] = json!(kind);
                v
            }
        };
        let named = |kind: &str, o: &Value| if o["metadata"]["name"] == object { typed(kind, o) } else { Value::Null };
        Some(match (verb, kind) {
            ("list", "ComputeVM") if input.get("labelSelector").is_none() && input.get("namespace").is_none() => {
                self.get(base, "listVms", &[]).await.map(|o| {
                    let items: Vec<Value> = o["items"].as_array().into_iter().flatten().map(|v| typed("ComputeVM", v)).collect();
                    json!({ "items": items })
                })
            }
            ("get", "ComputeVM") => self.get(base, "getVm", &[("name", object)]).await.map(|o| typed(kind, &o["computeVm"])),
            ("get", "FirecrackerVM") => self.get(base, "getVm", &[("name", object)]).await.map(|o| typed(kind, &o["firecrackerVm"])),
            ("get", "MarketIdentity") => self.get(base, "getBidder", &[]).await.map(|o| named(kind, &o["identity"])),
            ("get", "Bidder") => self.get(base, "getBidder", &[]).await.map(|o| named(kind, &o["bidder"])),
            ("get", "MarketBidder") => self.get(base, "getBidder", &[]).await.map(|o| named(kind, &o["marketBidder"])),
            _ => refuse(),
        })
    }
}

/// gateway.log's answer from the ssh gateway's lines of a DID (over ssh or getRequesterLog): those whose msg is one of
/// `msgs` (none: all), and `declined`, the accepts a bidder refused among all of them (kept by design, not leftovers).
pub fn gateway_lines(all: &[Value], msgs: &[String]) -> Value {
    let lines: Vec<Value> = all
        .iter()
        .filter(|v| msgs.is_empty() || v["msg"].as_str().is_some_and(|m| msgs.iter().any(|x| x == m)))
        .map(|v| json!({ "time": v["time"], "msg": v["msg"], "subhandle": v["subhandle"], "contract": v["contract"], "receipt": v["receipt"], "deleted": v["deleted"], "of": v["of"], "reason": v["reason"], "accept": v["accept"] }))
        .collect();
    json!({ "lines": lines, "declined": testkit::requester::declined_accepts(all) })
}

/// The getLog message and subject of one bidder.logCount text group: the known message whose texts the group holds, and
/// the one text left (a ComputeVM name, `vm-<name>` or an at:// URI), if any.
fn message_of(text: &[String]) -> Option<(&'static str, Option<String>)> {
    let (name, parts) = MESSAGES
        .iter()
        .filter(|(_, parts)| parts.iter().all(|p| text.iter().any(|t| t == p)))
        .max_by_key(|(_, parts)| parts.len())?;
    let rest: Vec<&String> = text.iter().filter(|t| !parts.contains(&t.as_str())).collect();
    match rest.as_slice() {
        [] => Some((name, None)),
        [s] => Some((name, Some((*s).clone()))),
        _ => None,
    }
}

/// The kube transport's call with the ops clusters in front: a call on a cluster served by ops routes goes there, every
/// other one to atlibs-kube.
pub struct OpsKube {
    pub ops: Option<Arc<Ops>>,
    pub kube: Arc<Kube>,
}

impl Call for OpsKube {
    fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Option<Value>, String>> {
        Box::pin(async move {
            if let (Some(ops), Some(input)) = (&self.ops, &request.input) {
                let verb = nsid.rsplit('.').next().unwrap_or_default();
                if let Some(r) = ops.kube(verb, input).await {
                    return r.map(Some);
                }
            }
            self.kube.call(nsid, request).await
        })
    }

    fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result<Sub, String>> {
        Box::pin(async move {
            let served = self.ops.as_ref().zip(request.input.as_ref()).is_some_and(|(ops, i)| {
                i["cluster"].as_str().and_then(|c| c.strip_prefix("bidder:")).is_some_and(|n| ops.serves(n))
            });
            if served {
                return Err(format!("{nsid}: a cluster reached through the read-only ops routes has no watch"));
            }
            self.kube.subscribe(nsid, request).await
        })
    }
}