use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use tokio::sync::watch; use crate::kube::{KubeClient, KubeError, NodeEvent}; use crate::nodes::{Node, NodeSet}; use crate::roster::{ANNOTATION_ROSTER, Roster}; use crate::routes::{ANNOTATION_DID, ANNOTATION_ENDPOINT, EndpointBytes, RouteTable}; use crate::stats::Stats; pub const KUBE_RETRY_MIN: Duration = Duration::from_secs(1); pub const KUBE_RETRY_MAX: Duration = Duration::from_secs(30); pub const ANNOTATE_RETRY: Duration = Duration::from_secs(5); pub const NODE_READER_HINT: &str = "the node authorizer lets a node read only its own Node (AuthorizeNodeWithSelectors); apply rust/socialweb-meshd/deploy/node-reader-rbac.yaml to grant group system:nodes get/list/watch on nodes"; #[derive(Clone)] pub struct SelfIdentity { pub node_name: String, pub endpoint: EndpointBytes, pub endpoint_hex: String, pub did: String, } #[derive(Debug)] pub enum SourceError { Config(String), Forbidden(String), } impl std::fmt::Display for SourceError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { SourceError::Config(msg) => write!(f, "{msg}"), SourceError::Forbidden(msg) => write!(f, "{msg}"), } } } fn publish( self_name: &str, me: &SelfIdentity, nodes: &[Node], tx: &watch::Sender>, stats: &Stats, ) -> Arc { let table = Arc::new(RouteTable::compute(self_name, &me.endpoint, nodes)); Stats::bump(&stats.node_list_updates); let changed = **tx.borrow() != *table; if changed { tracing::info!( event = "routes", own_pod_cidrs = ?table.own_cidrs.iter().map(ToString::to_string).collect::>(), routes = ?table.routes.iter().map(|r| format!("{}->{}", r.cidr, r.node)).collect::>(), rejected = ?table.rejected.iter().map(|r| format!("{}: {}", r.node, r.reason)).collect::>() ); tx.send_replace(Arc::clone(&table)); } table } pub async fn run_routes( me: SelfIdentity, mut kube: Option>>>, mut roster: watch::Receiver>, location: Option>>, tx: watch::Sender>, stats: Arc, ) { loop { let current = Arc::clone(&roster.borrow_and_update()); match kube.as_mut() { Some(kube) => { let nodes = Arc::clone(&kube.borrow_and_update()); let table = publish( &me.node_name, &me, ¤t.beside(&nodes, &me.endpoint_hex), &tx, &stats, ); if !table.self_view.seen { tracing::info!(event = "own_node_waiting", node = %me.node_name); } if let Some(location) = &location { let wanted = nodes .iter() .find(|node| node.metadata.name == me.node_name) .and_then(|node| node.annotation(ANNOTATION_ROSTER)) .map(str::to_string); location.send_if_modified(|old| { let changed = *old != wanted; *old = wanted.clone(); changed }); } } None => match current.self_name(&me.endpoint_hex) { Some(name) => { publish(&name, &me, ¤t.nodes(), &tx, &stats); } None => { publish(&me.node_name, &me, &[], &tx, &stats); } }, } let kube_changed = async { match kube.as_mut() { Some(kube) => kube.changed().await, None => std::future::pending().await, } }; tokio::select! { changed = kube_changed => if changed.is_err() { return }, changed = roster.changed() => if changed.is_err() { return }, } } } fn annotations(me: &SelfIdentity) -> BTreeMap { BTreeMap::from([ (ANNOTATION_ENDPOINT.to_string(), me.endpoint_hex.clone()), (ANNOTATION_DID.to_string(), me.did.clone()), ]) } pub fn kube_client(path: &Path) -> Result, SourceError> { if !path.exists() { return Ok(None); } match KubeClient::from_kubeconfig(path) { Ok(client) => Ok(Some(client)), Err(KubeError::Config(msg)) if msg.starts_with("read ") => Ok(None), Err(err) => Err(SourceError::Config(err.to_string())), } } pub async fn run_annotator( kubeconfig: PathBuf, me: SelfIdentity, mut routes: watch::Receiver>, ) -> Result<(), SourceError> { loop { let needs = { let table = routes.borrow_and_update(); table.self_view.seen && !table.self_annotated_with(&me.endpoint_hex, &me.did) }; if needs { if let Some(client) = kube_client(&kubeconfig)? { match client .patch_node_annotations(&me.node_name, &annotations(&me)) .await { Ok(()) => { tracing::info!(event = "node_annotated", node = %me.node_name, endpoint = %me.endpoint_hex, did = %me.did); } Err(KubeError::Status { code: 403, body }) => { return Err(SourceError::Forbidden(format!( "patching Node {} was forbidden ({body}); --node-name must be the node the kubeconfig's identity is", me.node_name ))); } Err(err) => { tracing::warn!(event = "annotate_failed", node = %me.node_name, error = %err); } } } tokio::select! { changed = routes.changed() => if changed.is_err() { return Ok(()) }, () = tokio::time::sleep(ANNOTATE_RETRY) => {} } } else if routes.changed().await.is_err() { return Ok(()); } } } pub async fn run_kube( kubeconfig: PathBuf, tx: watch::Sender>>, ) -> Result<(), SourceError> { let mut retry = KUBE_RETRY_MIN; loop { let client = match kube_client(&kubeconfig)? { Some(client) => client, None => { tracing::info!(event = "kubeconfig_waiting", path = %kubeconfig.display()); tokio::time::sleep(retry).await; retry = (retry * 2).min(KUBE_RETRY_MAX); continue; } }; let list = match client.list_nodes().await { Ok(list) => list, Err(KubeError::Status { code: 403, body }) => { tracing::error!( event = "list_nodes_forbidden", error = %body, hint = NODE_READER_HINT, retry_in_ms = retry.as_millis() as u64 ); tokio::time::sleep(retry).await; retry = (retry * 2).min(KUBE_RETRY_MAX); continue; } Err(err) => { tracing::warn!(event = "list_nodes_failed", error = %err, retry_in_ms = retry.as_millis() as u64); tokio::time::sleep(retry).await; retry = (retry * 2).min(KUBE_RETRY_MAX); continue; } }; retry = KUBE_RETRY_MIN; let version = list.metadata.resource_version.clone().unwrap_or_default(); let mut nodes = NodeSet::from_list(list); tx.send_replace(Arc::new(nodes.to_vec())); let watch_started = tokio::time::Instant::now(); let watched = client .watch_nodes(&version, |event| { let dirty = match event { NodeEvent::Upsert(node) => { nodes.upsert(*node); true } NodeEvent::Delete(name, _) => { nodes.remove(&name); true } NodeEvent::Bookmark(_) | NodeEvent::Gone => false, }; if dirty { tx.send_replace(Arc::new(nodes.to_vec())); } }) .await; match watched { Ok(_) => tracing::debug!(event = "watch_ended_relisting"), Err(err) if err.is_gone() => tracing::info!(event = "watch_expired_relisting"), Err(err) => tracing::warn!(event = "watch_failed", error = %err), } if watch_started.elapsed() < KUBE_RETRY_MIN { tokio::time::sleep(KUBE_RETRY_MIN).await; } } }