//! kcp event-log resources (ADR 0005): //! `kcp-events://watch/{workspace}/{resource}{?namespace,labelSelector,fieldSelector,kcp}`. //! //! One watcher per URI, shared by all subscribers. It seeds known objects //! with one list, then streams `kubectl get -w --output-watch-events -o json` //! (pushed by the API server; no polling), dedupes, keeps a 256-event ring //! with `seq`, and wakes subscribers. When the stream ends it backs off, //! re-lists to reconcile changes missed in the gap, and watches again. use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use serde_json::{json, Value}; use tokio::io::AsyncReadExt; use tokio::sync::broadcast; use tokio::task::JoinHandle; use rmcp::model::{ ListResourceTemplatesResult, ReadResourceResponse, ReadResourceResult, ResourceContents, ResourceTemplate, ServerCapabilities, ServerConfig, SubscriptionFilter, }; use rmcp::service::SubscriptionContext; use rmcp::ErrorData; use super::service::KcpServiceHandle; use super::tools::append_namespace; use crate::resources::EVENT_LOG_MIME; pub const SCHEME: &str = "kcp-events://watch/"; pub const TEMPLATE: &str = "kcp-events://watch/{workspace}/{resource}{?namespace,labelSelector,fieldSelector,kcp}"; pub const DESCRIPTION: &str = "Kubernetes watch events for one resource in a workspace (event-log resource; pushed, no polling). {resource} is a plural name from get_api_resources (e.g. computevms). namespace=* watches every namespace. Events: {\"seq\", \"type\": ADDED|MODIFIED|DELETED|ERROR, \"object\", \"old\" (MODIFIED: previous object)}.\nExample: await watch_start(name=\"cm\", uri=\"kcp-events://watch/root:app/configmaps?namespace=*\")"; pub const API_SCHEME: &str = "kcp-api://"; pub const API_TEMPLATE: &str = "kcp-api://{workspace}/resources{?kcp}"; pub const API_DESCRIPTION: &str = "The workspace's API resources (same JSON as get_api_resources): names to use in kcp-events URIs and tools. Read-only; not watchable.\nExample: kinds = await read_resource(uri=\"kcp-api://root/resources\")"; /// `kcp-api://{workspace}/resources{?kcp}` → (workspace, kcp). pub fn parse_api_uri(uri: &str) -> Result<(String, Option), String> { let rest = uri .strip_prefix(API_SCHEME) .ok_or_else(|| format!("not a kcp api URI (expected {API_TEMPLATE}): {uri}"))?; let (path, query) = rest.split_once('?').unwrap_or((rest, "")); let Some(workspace) = path.strip_suffix("/resources").filter(|w| !w.is_empty()) else { return Err(format!("expected {API_TEMPLATE}, got {uri}")); }; let mut kcp = None; for pair in query.split('&').filter(|pair| !pair.is_empty()) { match pair.split_once('=') { Some(("kcp", value)) => kcp = Some(decode(value)), _ => return Err(format!("unknown kcp api parameter {pair:?}")), } } Ok((decode(workspace), kcp)) } const RING: usize = 256; const MAX_BACKOFF: Duration = Duration::from_secs(30); const HEALTHY_STREAM: Duration = Duration::from_secs(30); const MAX_FAILURES: u32 = 5; /// Longest a first read waits for the watcher's seed list. const SEED_WAIT: Duration = Duration::from_secs(30); /// A parsed watch URI. #[derive(Clone, Debug, PartialEq, Eq)] pub struct WatchTarget { pub workspace: String, pub resource: String, pub namespace: Option, pub label_selector: Option, pub field_selector: Option, pub kcp: Option, } impl WatchTarget { pub fn parse(uri: &str) -> Result { let rest = uri .strip_prefix(SCHEME) .ok_or_else(|| format!("not a kcp watch URI (expected {TEMPLATE}): {uri}"))?; let (path, query) = rest.split_once('?').unwrap_or((rest, "")); let mut segments = path.split('/').filter(|segment| !segment.is_empty()); let (Some(workspace), Some(resource), None) = (segments.next(), segments.next(), segments.next()) else { return Err(format!("expected {TEMPLATE}, got {uri}")); }; let mut target = Self { workspace: decode(workspace), resource: decode(resource), namespace: None, label_selector: None, field_selector: None, kcp: None, }; for pair in query.split('&').filter(|pair| !pair.is_empty()) { let (key, value) = pair.split_once('=').unwrap_or((pair, "")); let value = Some(decode(value)); match key { "namespace" => target.namespace = value, "labelSelector" => target.label_selector = value, "fieldSelector" => target.field_selector = value, "kcp" => target.kcp = value, other => return Err(format!("unknown kcp watch parameter {other:?}")), } } Ok(target) } /// `kubectl get` arguments shared by the seed list and the watch. fn selection(&self) -> Vec { let mut args = vec!["get".to_string(), self.resource.clone()]; append_namespace(&mut args, self.namespace.as_deref(), false); if let Some(selector) = &self.label_selector { args.push("-l".to_string()); args.push(selector.clone()); } if let Some(selector) = &self.field_selector { args.push("--field-selector".to_string()); args.push(selector.clone()); } args } } /// Minimal percent-decoding for URI components. fn decode(text: &str) -> String { let bytes = text.as_bytes(); let mut out = Vec::with_capacity(bytes.len()); let mut index = 0; while index < bytes.len() { if bytes[index] == b'%' && index + 2 < bytes.len() { if let Ok(byte) = u8::from_str_radix(&text[index + 1..index + 3], 16) { out.push(byte); index += 3; continue; } } out.push(if bytes[index] == b'+' { b' ' } else { bytes[index] }); index += 1; } String::from_utf8_lossy(&out).into_owned() } /// Incremental parser for concatenated JSON values (kubectl watch output). #[derive(Default)] pub struct JsonStream { buffer: Vec, } impl JsonStream { pub fn feed(&mut self, bytes: &[u8]) -> Vec { self.buffer.extend_from_slice(bytes); let mut values = Vec::new(); let mut consumed = 0; loop { let mut stream = serde_json::Deserializer::from_slice(&self.buffer[consumed..]).into_iter::(); match stream.next() { Some(Ok(value)) => { consumed += stream.byte_offset(); values.push(value); } Some(Err(error)) if error.is_eof() => break, Some(Err(_)) => { // Not JSON (e.g. a warning line): skip to the next object. match self.buffer[consumed + 1..] .iter() .position(|byte| *byte == b'{') { Some(offset) => consumed += offset + 1, None => { consumed = self.buffer.len(); break; } } } None => break, } } self.buffer.drain(..consumed); values } } /// Known objects by uid, for dedupe and gap reconciliation. #[derive(Default)] pub struct KnownObjects { objects: HashMap, } fn identity(object: &Value) -> Option<(String, String)> { let metadata = object.get("metadata")?; let uid = metadata .get("uid") .and_then(Value::as_str) .map(str::to_string) .or_else(|| { let name = metadata.get("name")?.as_str()?; let namespace = metadata .get("namespace") .and_then(Value::as_str) .unwrap_or(""); Some(format!("{namespace}/{name}")) })?; let version = metadata .get("resourceVersion") .and_then(Value::as_str) .unwrap_or("") .to_string(); Some((uid, version)) } /// Drops `metadata.managedFields`, which is large and rarely useful. pub fn slim(mut object: Value) -> Value { if let Some(metadata) = object.get_mut("metadata").and_then(Value::as_object_mut) { metadata.remove("managedFields"); } object } impl KnownObjects { pub fn seed(&mut self, items: &[Value]) { self.objects = items .iter() .filter_map(|item| { identity(item).map(|(uid, version)| (uid, (version, slim(item.clone())))) }) .collect(); } /// A watch event to deliver, or `None` for duplicates and bookmarks. /// `MODIFIED` carries the previously known object as `old`. pub fn apply(&mut self, event: &Value) -> Option { let kind = event.get("type").and_then(Value::as_str)?.to_string(); let object = slim(event.get("object")?.clone()); match kind.as_str() { "ADDED" | "MODIFIED" => { let (uid, version) = identity(&object)?; if self .objects .get(&uid) .is_some_and(|(known, _)| *known == version) { return None; } let old = self .objects .insert(uid, (version, object.clone())) .map(|(_, old)| old) .filter(|_| kind == "MODIFIED"); Some(Change { kind, object, old }) } "DELETED" => { let (uid, _) = identity(&object)?; self.objects.remove(&uid); Some(Change::new(kind, object)) } "ERROR" => Some(Change::new(kind, object)), _ => None, } } /// Synthetic events for what changed while no stream was open. pub fn reconcile(&mut self, items: &[Value]) -> Vec { let mut events = Vec::new(); let mut current = HashMap::new(); for item in items { let Some((uid, version)) = identity(item) else { continue; }; let object = slim(item.clone()); match self.objects.get(&uid) { None => events.push(Change::new("ADDED".to_string(), object.clone())), Some((known, old)) if *known != version => events.push(Change { kind: "MODIFIED".to_string(), object: object.clone(), old: Some(old.clone()), }), Some(_) => {} } current.insert(uid, (version, object)); } for (uid, (_, object)) in &self.objects { if !current.contains_key(uid) { events.push(Change::new("DELETED".to_string(), object.clone())); } } self.objects = current; events } } /// One event for the ring. #[derive(Clone, Debug, PartialEq)] pub struct Change { pub kind: String, pub object: Value, /// Previous object (`MODIFIED` only). pub old: Option, } impl Change { fn new(kind: String, object: Value) -> Self { Self { kind, object, old: None, } } } /// Ring of recent events with monotonically increasing `seq`. struct Ring { seq: u64, events: VecDeque, } impl Ring { fn push(&mut self, change: Change) { self.seq += 1; let mut event = json!({"seq": self.seq, "type": change.kind, "object": change.object}); if let Some(old) = change.old { event["old"] = old; } self.events.push_back(event); while self.events.len() > RING { self.events.pop_front(); } } } struct Watcher { ring: Mutex, updated: broadcast::Sender<()>, subscribers: AtomicUsize, task: Mutex>>, /// Outcome of the first list: `None` while pending. seeded: tokio::sync::watch::Sender>>, } impl Watcher { fn push(&self, change: Change) { self.ring.lock().expect("ring lock").push(change); let _ = self.updated.send(()); } fn body(&self) -> String { let ring = self.ring.lock().expect("ring lock"); json!({"events": ring.events}).to_string() } } /// All watchers of one server process. #[derive(Default)] pub struct WatchHub { watchers: Mutex>>, } impl WatchHub { /// The event-log body for `uri`: `{"events": [...]}` (empty when no one /// watches it). A watched URI's first read waits for the watcher's seed /// list (a signal, not a poll), so a bad resource fails the read and the /// client's `watch_start` instead of surfacing later as an `ERROR` event. pub async fn read(&self, uri: &str) -> Result<(String, &'static str), String> { WatchTarget::parse(uri)?; let watcher = self.watchers.lock().expect("hub lock").get(uri).cloned(); let Some(watcher) = watcher else { return Ok((json!({"events": []}).to_string(), EVENT_LOG_MIME)); }; let mut seeded = watcher.seeded.subscribe(); let outcome = tokio::time::timeout(SEED_WAIT, seeded.wait_for(Option::is_some)) .await .ok() .and_then(Result::ok) .and_then(|state| state.clone()); if let Some(Err(error)) = outcome { return Err(error); } Ok((watcher.body(), EVENT_LOG_MIME)) } /// Joins (or starts) the watcher for `uri`; returns its update channel. pub fn acquire( &self, service: &KcpServiceHandle, uri: &str, ) -> Result, String> { let target = WatchTarget::parse(uri)?; let mut watchers = self.watchers.lock().expect("hub lock"); let watcher = watchers .entry(uri.to_string()) .or_insert_with(|| { let watcher = Arc::new(Watcher { ring: Mutex::new(Ring { seq: 0, events: VecDeque::new(), }), updated: broadcast::channel(64).0, subscribers: AtomicUsize::new(0), task: Mutex::new(None), seeded: tokio::sync::watch::channel(None).0, }); let task = tokio::spawn(run_watcher(service.clone(), target, watcher.clone())); *watcher.task.lock().expect("task lock") = Some(task); watcher }) .clone(); watcher.subscribers.fetch_add(1, Ordering::SeqCst); Ok(watcher.updated.subscribe()) } /// Leaves `uri`; the last subscriber stops the watcher. pub fn release(&self, uri: &str) { let mut watchers = self.watchers.lock().expect("hub lock"); let last = watchers .get(uri) .is_some_and(|watcher| watcher.subscribers.fetch_sub(1, Ordering::SeqCst) == 1); if last { if let Some(watcher) = watchers.remove(uri) { if let Some(task) = watcher.task.lock().expect("task lock").take() { task.abort(); } } } } } /// Server capabilities: tools plus subscribable resources. pub fn server_info() -> ServerConfig { ServerConfig::new( ServerCapabilities::builder() .enable_tools() .enable_resources() .enable_resources_subscribe() .build(), ) } pub fn resource_templates() -> ListResourceTemplatesResult { ListResourceTemplatesResult::with_all_items(vec![ ResourceTemplate::new(TEMPLATE, "kcp-events") .with_description(DESCRIPTION) .with_mime_type(EVENT_LOG_MIME), ResourceTemplate::new(API_TEMPLATE, "kcp-api") .with_description(API_DESCRIPTION) .with_mime_type("application/json"), ]) } pub async fn read_resource( hub: &WatchHub, service: &KcpServiceHandle, uri: &str, ) -> Result { let (text, mime) = if uri.starts_with(API_SCHEME) { read_api(service, uri) .await .map(|text| (text, "application/json")) } else { hub.read(uri).await } .map_err(|error| ErrorData::resource_not_found(error, None))?; Ok(ReadResourceResponse::Complete(ReadResourceResult::new( vec![ResourceContents::TextResourceContents { uri: uri.to_string(), mime_type: Some(mime.to_string()), text, meta: None, }], ))) } async fn read_api(service: &KcpServiceHandle, uri: &str) -> Result { let (workspace, kcp) = parse_api_uri(uri)?; let kcp = service.resolve_kcp(kcp.as_deref()).await?; let workspace = service.workspace_or_default(&kcp, Some(&workspace)); super::tools::api_resources(service, &kcp, &workspace) .await .map(|value| value.to_string()) } /// Accepts every requested `kcp-events://` URI that parses. pub fn accepted_subscriptions(requested: &SubscriptionFilter) -> Option { let uris: Vec = requested .resource_subscriptions .iter() .flatten() .filter(|uri| WatchTarget::parse(uri).is_ok()) .cloned() .collect(); Some( SubscriptionFilter::builder() .resource_subscriptions(uris) .build(), ) } /// Serves one `subscriptions/listen` stream until the client cancels. pub async fn listen( hub: &WatchHub, service: &KcpServiceHandle, context: SubscriptionContext, ) -> Result<(), ErrorData> { let uris = context .accepted() .resource_subscriptions .clone() .unwrap_or_default(); let (updated_tx, mut updated) = tokio::sync::mpsc::channel::(64); let mut forwarders = Vec::new(); let mut acquired = Vec::new(); for uri in &uris { let mut receiver = match hub.acquire(service, uri) { Ok(receiver) => receiver, Err(_) => continue, }; acquired.push(uri.clone()); let tx = updated_tx.clone(); let uri = uri.clone(); forwarders.push(tokio::spawn(async move { loop { match receiver.recv().await { Ok(()) | Err(broadcast::error::RecvError::Lagged(_)) => { if tx.send(uri.clone()).await.is_err() { return; } } Err(broadcast::error::RecvError::Closed) => return, } } })); } drop(updated_tx); loop { tokio::select! { _ = context.cancelled() => break, uri = updated.recv() => match uri { Some(uri) => { if context.sink().notify_resource_updated(uri).await.is_err() { break; } } None => break, }, } } for forwarder in forwarders { forwarder.abort(); } for uri in acquired { hub.release(&uri); } Ok(()) } async fn list(service: &KcpServiceHandle, target: &WatchTarget) -> Result, String> { let kcp = service.resolve_kcp(target.kcp.as_deref()).await?; let mut args = target.selection(); args.push("-o".to_string()); args.push("json".to_string()); let text = match service.read(&kcp, &target.workspace, &args).await { Ok(text) => text, Err(error) => { return Err( super::tools::with_resource_hint(service, &kcp, &target.workspace, error).await, ) } }; let value: Value = serde_json::from_str(&text).map_err(|error| format!("list is not JSON: {error}"))?; Ok(value .get("items") .and_then(Value::as_array) .cloned() .unwrap_or_default()) } /// Streams one watch until it ends; returns why. async fn stream( service: &KcpServiceHandle, target: &WatchTarget, known: &mut KnownObjects, watcher: &Watcher, ) -> String { let kcp = match service.resolve_kcp(target.kcp.as_deref()).await { Ok(kcp) => kcp, Err(error) => return error, }; let mut args = target.selection(); args.extend( ["-w", "--output-watch-events", "-o", "json"] .into_iter() .map(str::to_string), ); let mut child = match service.read_stream(&kcp, &target.workspace, &args).await { Ok(child) => child, Err(error) => return error, }; let Some(mut stdout) = child.stdout.take() else { return "kubectl watch has no stdout".to_string(); }; let mut parser = JsonStream::default(); let mut chunk = vec![0u8; 64 * 1024]; loop { match stdout.read(&mut chunk).await { Ok(0) => break, Ok(read) => { for event in parser.feed(&chunk[..read]) { if let Some(change) = known.apply(&event) { watcher.push(change); } } } Err(error) => return format!("watch stream read failed: {error}"), } } let mut stderr = String::new(); if let Some(mut pipe) = child.stderr.take() { let _ = pipe.read_to_string(&mut stderr).await; } let status = child.wait().await; format!( "watch stream ended ({}){}", status .map(|status| status.to_string()) .unwrap_or_else(|error| error.to_string()), if stderr.trim().is_empty() { String::new() } else { format!(": {}", stderr.trim()) } ) } async fn run_watcher(service: KcpServiceHandle, target: WatchTarget, watcher: Arc) { let mut known = KnownObjects::default(); let mut seeded = false; let mut backoff = Duration::from_secs(1); let mut failures = 0u32; loop { match list(&service, &target).await { Ok(items) => { if seeded { for change in known.reconcile(&items) { watcher.push(change); } } else { known.seed(&items); seeded = true; } watcher.seeded.send_replace(Some(Ok(()))); } Err(error) => { if !seeded { watcher.seeded.send_replace(Some(Err(error.clone()))); } failures += 1; watcher.push(Change::new("ERROR".to_string(), json!({"message": error}))); if failures >= MAX_FAILURES { watcher.push(Change::new( "ERROR".to_string(), json!({"message": format!("giving up after {MAX_FAILURES} failures")}), )); return; } tokio::time::sleep(backoff).await; backoff = (backoff * 2).min(MAX_BACKOFF); continue; } } let started = Instant::now(); let reason = stream(&service, &target, &mut known, &watcher).await; if started.elapsed() >= HEALTHY_STREAM { backoff = Duration::from_secs(1); failures = 0; } else { failures += 1; if failures >= MAX_FAILURES { watcher.push(Change::new( "ERROR".to_string(), json!({"message": format!("{reason}; giving up after {MAX_FAILURES} failures")}), )); return; } } tokio::time::sleep(backoff).await; backoff = (backoff * 2).min(MAX_BACKOFF); } } #[cfg(test)] mod tests { use super::*; fn object(name: &str, version: &str) -> Value { json!({"metadata": {"name": name, "namespace": "default", "uid": format!("uid-{name}"), "resourceVersion": version, "managedFields": [1]}}) } #[test] fn parses_watch_uris() { let target = WatchTarget::parse( "kcp-events://watch/root:app/configmaps?namespace=*&labelSelector=app%3Dweb&kcp=local", ) .unwrap(); assert_eq!(target.workspace, "root:app"); assert_eq!(target.resource, "configmaps"); assert_eq!(target.namespace.as_deref(), Some("*")); assert_eq!(target.label_selector.as_deref(), Some("app=web")); assert_eq!(target.kcp.as_deref(), Some("local")); assert_eq!( target.selection(), ["get", "configmaps", "--all-namespaces", "-l", "app=web"] ); assert!(WatchTarget::parse("kcp-events://watch/root:app").is_err()); assert!(WatchTarget::parse("kcp-events://watch/root:app/cm?bogus=1").is_err()); assert!(WatchTarget::parse("memo://notes").is_err()); } #[test] fn json_stream_handles_split_and_concatenated_values() { let mut parser = JsonStream::default(); assert!(parser.feed(b"{\"type\": \"ADD").is_empty()); let values = parser.feed(b"ED\"}\n{\n \"type\": \"DELETED\"\n}{\"a\""); assert_eq!( values, vec![json!({"type": "ADDED"}), json!({"type": "DELETED"})] ); assert_eq!(parser.feed(b": 1}"), vec![json!({"a": 1})]); assert_eq!(parser.feed(b"Warning: x\n{\"b\":2}"), vec![json!({"b": 2})]); } #[test] fn known_objects_dedupe_and_reconcile() { let mut known = KnownObjects::default(); known.seed(&[object("a", "1"), object("b", "1")]); assert!(known .apply(&json!({"type": "ADDED", "object": object("a", "1")})) .is_none()); let change = known .apply(&json!({"type": "MODIFIED", "object": object("a", "2")})) .unwrap(); assert_eq!(change.kind, "MODIFIED"); assert!(change.object["metadata"].get("managedFields").is_none()); let old = change.old.expect("MODIFIED carries the previous object"); assert_eq!(old["metadata"]["resourceVersion"], "1"); assert!(old["metadata"].get("managedFields").is_none()); let added = known .apply(&json!({"type": "ADDED", "object": object("n", "1")})) .unwrap(); assert_eq!(added.old, None); assert!(known .apply(&json!({"type": "BOOKMARK", "object": object("a", "2")})) .is_none()); let events = known.reconcile(&[object("a", "3"), object("c", "1"), object("n", "1")]); let mut summary: Vec = events .iter() .map(|change| { format!( "{} {} {}", change.kind, change.object["metadata"]["name"].as_str().unwrap(), change.old.is_some() ) }) .collect(); summary.sort(); assert_eq!( summary, ["ADDED c false", "DELETED b false", "MODIFIED a true"] ); } #[test] fn ring_keeps_bounded_seq_ordered_events() { let mut ring = Ring { seq: 0, events: VecDeque::new(), }; for index in 0..300 { ring.push(Change::new("ADDED".to_string(), json!({"i": index}))); } assert_eq!(ring.events.len(), RING); assert_eq!(ring.events.front().unwrap()["seq"], 45); assert_eq!(ring.events.back().unwrap()["seq"], 300); assert!(ring.events.back().unwrap().get("old").is_none()); ring.push(Change { kind: "MODIFIED".to_string(), object: json!({"v": 2}), old: Some(json!({"v": 1})), }); assert_eq!(ring.events.back().unwrap()["old"], json!({"v": 1})); } }