//! The read-only operations routes of a target (GET /xrpc/com.fedfork.exec.ops.*, docs/SYSTEM.md): what the host reads //! of a bidder or the ssh gateway it has no kcp or shell on (prod in CI). A bidder with `ops` in the target is the //! kube cluster `bidder:` served here: get and list of the kinds the routes answer, in the routes' fixed projection //! (the fields the modules read), every write refused. One shared secret for every bidder and the gateway (owner): it is the target's `opsSecretFile` (0600), sent as the bearer and never logged. //! A bidder's routes are on its market mux, reached over its market endpoint: https (`ops.url`) where its offering has //! one, else iroh (`ops.iroh`, the `endpoint...` ticket of its market.offering record): the same GETs and bearer as //! HTTP/1.1 over atlibs-iroh's raw fetch (`testkit::irohfetch`: `dumbpipe connect `, the stream a requester's //! market calls take), end-to-end encrypted to the iroh node the ticket names. The gateway's route is its https web origin. use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use atlibs_host::BoxFut; use atlibs_host::run::{Call, Request, Sub}; use atlibs_kube::Kube; use serde_json::{Value, json}; use testkit::target::{OpsRead, Target}; const NS: &str = "com.fedfork.exec.ops."; /// One ops read, every try and wait included, ends within this bound: below the smallest deadline of an entry that /// reads ops routes less its latest read in a passing run (0014:abandon: 15 min; its last poll starts by about 700 s), so /// a lost transport is what the entry reports, not its deadline. const READ_BUDGET: Duration = Duration::from_secs(150); /// One try's answer time, within what is left of READ_BUDGET. const READ_TRY_TIMEOUT: Duration = Duration::from_secs(60); const READ_RETRY_BACKOFF: Duration = Duration::from_secs(5); /// The known messages of com.fedfork.exec.ops.getLog (internal/contract/opsread Messages): name, the texts a line holds. const MESSAGES: &[(&str, &[&str])] = &[ ("core_holds_instance", &["core holds the instance"]), ("adopted_role", &["\"msg\":\"adopted\"", "\"role\":"]), ("placed", &["\"msg\":\"placed\""]), ("stopped", &["\"msg\":\"stopped\""]), ("work_dir_removed", &["work dir and API socket removed"]), ("shutdown", &["\"msg\":\"shutdown\""]), ("vmm_exit", &["the process wait saw the VMM exit"]), ("bidder_vm_terminated", &["bidder_vm_terminated"]), ("released_bid_terminated", &["a released bid is marked terminated"]), ("hostprocess_unit_adopted", &["hostprocess_unit_adopted"]), ("early_kill", &["before its VMM answered"]), ("bidder_accept_abandoned", &["bidder_accept_abandoned"]), ("bidder_accept_unresolved", &["bidder_accept_unresolved"]), ("accept_provision_failed", &["the provision of an accept failed"]), ]; /// Where a bidder's ops routes are: an https origin, or an iroh market endpoint ticket. enum Base { Https(String), Iroh(String), } impl Base { fn of(o: &OpsRead) -> Result { match (o.url.as_deref(), o.iroh.as_deref()) { // the bearer goes to https origins only (Some(u), None) if u.starts_with("https://") => Ok(Self::Https(u.trim_end_matches('/').to_string())), (Some(u), None) => Err(format!("the ops route {u} is not https: the ops read secret is sent to https origins only")), (None, Some(t)) => { testkit::irohfetch::ticket(t).map(Self::Iroh).ok_or(format!("the ops iroh endpoint {t} is not an iroh endpoint ticket")) } _ => Err("an ops endpoint is exactly one of url (https) and iroh".into()), } } } pub struct Ops { http: reqwest::Client, secret: String, bidders: HashMap, gateway: Option, } impl Ops { /// None: the target names no ops route. pub fn load(target: &Target) -> Result>, String> { let mut bidders = HashMap::new(); for b in target.bidders.iter().filter(|b| b.kcp.is_none()) { if let Some(o) = &b.ops { bidders.insert(b.name.clone(), Base::of(o).map_err(|e| format!("bidder {}: {e}", b.name))?); } } let gateway = match target.gateway.ops.as_ref().map(Base::of).transpose()? { Some(Base::Iroh(_)) => return Err("the gateway's ops route is its https web origin, not iroh".into()), g => g, }; if bidders.is_empty() && gateway.is_none() { return Ok(None); } let file = target.ops_secret_file.as_deref().ok_or("the target names ops routes and no opsSecretFile")?; let path = testkit::target::expand(file); let secret = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?.trim().to_string(); if secret.is_empty() { return Err(format!("{} is empty", path.display())); } let http = reqwest::Client::builder().timeout(Duration::from_secs(120)).build().map_err(|e| e.to_string())?; Ok(Some(Arc::new(Self { http, secret, bidders, gateway }))) } /// The bidder `name` is read through its ops routes. pub fn serves(&self, name: &str) -> bool { self.bidders.contains_key(name) } pub fn serves_gateway(&self) -> bool { self.gateway.is_some() } /// One GET of an ops route. The routes are read-only, so a request whose transport failed (a lost iroh session or /// relay, an https connection error, no answer in READ_TRY_TIMEOUT) is sent again, 5 s, 10 s, 20 s... apart, until /// READ_BUDGET is spent; an answer, whatever its status, is never retried. The error names every failed try. async fn get(&self, base: &Base, method: &str, params: &[(&str, &str)]) -> Result { let until = tokio::time::Instant::now() + READ_BUDGET; let mut failed: Vec = vec![]; let (status, body) = loop { let left = until.saturating_duration_since(tokio::time::Instant::now()); match self.send(base, method, params, left.min(READ_TRY_TIMEOUT)).await { Ok(answer) => break answer, Err(e) => failed.push(format!("try {}: {e}", failed.len() + 1)), } let wait = READ_RETRY_BACKOFF * 2u32.pow(failed.len() as u32 - 1); if tokio::time::Instant::now() + wait >= until { return Err(failed.join("; ")); } tokio::time::sleep(wait).await; }; if !(200..300).contains(&status) { return Err(format!("{NS}{method}: {status} {} {}", body["error"].as_str().unwrap_or_default(), body["message"].as_str().unwrap_or_default())); } Ok(body) } async fn send(&self, base: &Base, method: &str, params: &[(&str, &str)], timeout: Duration) -> Result<(u16, Value), String> { Ok(match base { Base::Https(base) => { let url = format!("{base}/xrpc/{NS}{method}"); let res = self.http.get(&url).query(params).bearer_auth(&self.secret).timeout(timeout).send().await.map_err(|e| format!("{NS}{method}: {}", e.without_url()))?; let status = res.status().as_u16(); (status, res.json::().await.unwrap_or(Value::Null)) } Base::Iroh(ticket) => { let target = format!("/xrpc/{NS}{method}{}", testkit::irohfetch::query(params)); let headers = vec![("authorization".to_string(), format!("Bearer {}", self.secret))]; let (status, body) = testkit::irohfetch::request(ticket, "GET", &target, headers, vec![], timeout) .await .map_err(|e| format!("{NS}{method}: {e}"))?; (status, serde_json::from_slice(&body).unwrap_or(Value::Null)) } }) } fn bidder(&self, name: &str) -> Result<&Base, String> { self.bidders.get(name).ok_or(format!("no ops route for the bidder {name}")) } pub async fn health(&self, name: &str) -> Result { self.get(self.bidder(name)?, "getHealth", &[]).await } pub async fn memory(&self, name: &str, vm: &str) -> Result { self.get(self.bidder(name)?, "getVmMemory", &[("name", vm)]).await } async fn log(&self, name: &str, text: &[String]) -> Result { let (msg, subject) = message_of(text).ok_or(format!("the ops routes serve no log line of {text:?}"))?; let mut params = vec![("msg", msg)]; if let Some(s) = subject.as_deref() { params.push(("subject", s)); } self.get(self.bidder(name)?, "getLog", ¶ms).await } /// bidder.logCount: {counts: {: n}, log}. pub async fn log_count(&self, name: &str, texts: &[Vec]) -> Result { let mut counts = serde_json::Map::new(); let mut from = Value::Null; for t in texts { let o = self.log(name, t).await?; counts.insert(t.join(" & "), o["count"].clone()); from = o["log"].clone(); } Ok(json!({ "counts": counts, "log": from })) } /// bidder.logLines: {lines: [{time, msg, }], log} of the first entry of `texts`. pub async fn log_lines(&self, name: &str, texts: &[Vec], keys: &[String]) -> Result { let o = self.log(name, &texts[0]).await?; let rows: Vec = (o["lines"].as_array().into_iter().flatten()) .map(|v| { let mut row = json!({ "time": v["time"], "msg": v["msg"] }); for k in keys { row[k] = v[k].clone(); } row }) .collect(); Ok(json!({ "lines": rows, "log": o["log"] })) } /// gateway.log: the ssh gateway's lines of `did` since `since` whose msg is one of `msgs` (none: every known one). pub async fn gateway_log(&self, did: &str, since: &str, msgs: &[String]) -> Result { let base = self.gateway.as_ref().ok_or("the target names no gateway ops route")?; let o = self.get(base, "getRequesterLog", &[("did", did), ("since", since)]).await?; Ok(gateway_lines(o["lines"].as_array().map(Vec::as_slice).unwrap_or_default(), msgs)) } /// A kube call on a cluster served here; None: the cluster is not one of these. async fn kube(&self, verb: &str, input: &Value) -> Option> { let cluster = input["cluster"].as_str()?; let name = cluster.strip_prefix("bidder:")?; let base = self.bidders.get(name)?; let kind = input["kind"].as_str().unwrap_or_default(); let object = input["name"].as_str().unwrap_or_default(); let refuse = || Err(format!("{cluster} is reached through the read-only ops routes: {verb} {kind} is not served")); let typed = |kind: &str, o: &Value| match o { Value::Null => Value::Null, o => { let mut v = o.clone(); v["apiVersion"] = json!("socialweb.computer/v1alpha1"); v["kind"] = json!(kind); v } }; let named = |kind: &str, o: &Value| if o["metadata"]["name"] == object { typed(kind, o) } else { Value::Null }; Some(match (verb, kind) { ("list", "ComputeVM") if input.get("labelSelector").is_none() && input.get("namespace").is_none() => { self.get(base, "listVms", &[]).await.map(|o| { let items: Vec = o["items"].as_array().into_iter().flatten().map(|v| typed("ComputeVM", v)).collect(); json!({ "items": items }) }) } ("get", "ComputeVM") => self.get(base, "getVm", &[("name", object)]).await.map(|o| typed(kind, &o["computeVm"])), ("get", "FirecrackerVM") => self.get(base, "getVm", &[("name", object)]).await.map(|o| typed(kind, &o["firecrackerVm"])), ("get", "MarketIdentity") => self.get(base, "getBidder", &[]).await.map(|o| named(kind, &o["identity"])), ("get", "Bidder") => self.get(base, "getBidder", &[]).await.map(|o| named(kind, &o["bidder"])), ("get", "MarketBidder") => self.get(base, "getBidder", &[]).await.map(|o| named(kind, &o["marketBidder"])), _ => refuse(), }) } } /// gateway.log's answer from the ssh gateway's lines of a DID (over ssh or getRequesterLog): those whose msg is one of /// `msgs` (none: all), and `declined`, the accepts a bidder refused among all of them (kept by design, not leftovers). pub fn gateway_lines(all: &[Value], msgs: &[String]) -> Value { let lines: Vec = all .iter() .filter(|v| msgs.is_empty() || v["msg"].as_str().is_some_and(|m| msgs.iter().any(|x| x == m))) .map(|v| json!({ "time": v["time"], "msg": v["msg"], "subhandle": v["subhandle"], "contract": v["contract"], "receipt": v["receipt"], "deleted": v["deleted"], "of": v["of"], "reason": v["reason"], "accept": v["accept"] })) .collect(); json!({ "lines": lines, "declined": testkit::requester::declined_accepts(all) }) } /// The getLog message and subject of one bidder.logCount text group: the known message whose texts the group holds, and /// the one text left (a ComputeVM name, `vm-` or an at:// URI), if any. fn message_of(text: &[String]) -> Option<(&'static str, Option)> { let (name, parts) = MESSAGES .iter() .filter(|(_, parts)| parts.iter().all(|p| text.iter().any(|t| t == p))) .max_by_key(|(_, parts)| parts.len())?; let rest: Vec<&String> = text.iter().filter(|t| !parts.contains(&t.as_str())).collect(); match rest.as_slice() { [] => Some((name, None)), [s] => Some((name, Some((*s).clone()))), _ => None, } } /// The kube transport's call with the ops clusters in front: a call on a cluster served by ops routes goes there, every /// other one to atlibs-kube. pub struct OpsKube { pub ops: Option>, pub kube: Arc, } impl Call for OpsKube { fn call(&self, nsid: String, request: Request) -> BoxFut<'_, Result, String>> { Box::pin(async move { if let (Some(ops), Some(input)) = (&self.ops, &request.input) { let verb = nsid.rsplit('.').next().unwrap_or_default(); if let Some(r) = ops.kube(verb, input).await { return r.map(Some); } } self.kube.call(nsid, request).await }) } fn subscribe(&self, nsid: String, request: Request) -> BoxFut<'_, Result> { Box::pin(async move { let served = self.ops.as_ref().zip(request.input.as_ref()).is_some_and(|(ops, i)| { i["cluster"].as_str().and_then(|c| c.strip_prefix("bidder:")).is_some_and(|n| ops.serves(n)) }); if served { return Err(format!("{nsid}: a cluster reached through the read-only ops routes has no watch")); } self.kube.subscribe(nsid, request).await }) } }