~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
329 lines · 11.7 KB Raw History
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;

use tokio::signal::unix::{SignalKind, signal};
use tokio::sync::watch;

use crate::cli::{
    EXIT_ENDPOINT, EXIT_IDENTITY, EXIT_KUBE, EXIT_NETWORK_DEVICE, EXIT_OK, EXIT_RUNTIME,
    EXIT_STATUS_LISTEN, EXIT_USAGE, IdentityArgs, RunArgs,
};
use crate::dns::{self, DnsOptions};
use crate::identity::{self, Loaded};
use crate::mesh::{self, Connections, EndpointOptions, Mesh};
use crate::netconf::NetConf;
use crate::notify::sd_notify;
use crate::roster::{self, Roster};
use crate::routes::RouteTable;
use crate::source::{self, SelfIdentity, SourceError};
use crate::stats::Stats;
use crate::status::{self, StatusInfo};
use crate::tun::Tun;

pub const SHUTDOWN_GRACE: Duration = Duration::from_secs(3);

pub fn init_logging(filter: &str) {
    let filter = tracing_subscriber::EnvFilter::try_new(filter)
        .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info"));
    let _ = tracing_subscriber::fmt()
        .json()
        .flatten_event(true)
        .with_current_span(false)
        .with_span_list(false)
        .with_target(true)
        .with_env_filter(filter)
        .with_writer(std::io::stderr)
        .try_init();
}

pub fn identity(args: &IdentityArgs) -> i32 {
    match identity::load_or_create(&args.secret_file) {
        Ok(loaded) => {
            let created = matches!(loaded, Loaded::Created(_));
            let id = loaded.identity();
            println!(
                "{}",
                serde_json::json!({
                    "endpoint": id.endpoint_id_hex(),
                    "did": id.did_key(),
                    "secretFile": args.secret_file.display().to_string(),
                    "created": created,
                })
            );
            EXIT_OK
        }
        Err(err) => {
            eprintln!("socialweb-meshd: {err}");
            EXIT_IDENTITY
        }
    }
}

fn fatal(code: i32, event: &str, message: impl std::fmt::Display) -> i32 {
    tracing::error!(event, error = %message, exit_code = code);
    code
}

fn source_exit(result: Result<Result<(), SourceError>, tokio::task::JoinError>, task: &str) -> i32 {
    match result {
        Ok(Ok(())) => fatal(EXIT_RUNTIME, "task_stopped", format!("{task} ended")),
        Ok(Err(SourceError::Config(msg))) => fatal(EXIT_KUBE, "kubeconfig_unusable", msg),
        Ok(Err(SourceError::Forbidden(msg))) => fatal(EXIT_KUBE, "node_patch_forbidden", msg),
        Err(err) => fatal(EXIT_RUNTIME, "task_panicked", format!("{task}: {err}")),
    }
}

fn ready_now(table: &RouteTable, me: &SelfIdentity, kube: bool) -> bool {
    table.self_view.seen
        && !table.own_cidrs.is_empty()
        && (!kube || table.self_annotated_with(&me.endpoint_hex, &me.did))
}

pub async fn run(args: RunArgs) -> i32 {
    let loaded = match identity::load_or_create(&args.secret_file) {
        Ok(loaded) => loaded,
        Err(err) => return fatal(EXIT_IDENTITY, "secret_unusable", err),
    };
    if matches!(loaded, Loaded::Created(_)) {
        tracing::info!(event = "secret_created", path = %args.secret_file.display());
    }
    let id = loaded.identity().clone();
    let me = SelfIdentity {
        node_name: args
            .node_name
            .clone()
            .unwrap_or_else(|| id.endpoint_id_hex()),
        endpoint: id.public(),
        endpoint_hex: id.endpoint_id_hex(),
        did: id.did_key(),
    };
    tracing::info!(event = "identity", node = %me.node_name, endpoint = %me.endpoint_hex, did = %me.did);

    let http = match roster::http_client() {
        Ok(http) => http,
        Err(err) => return fatal(EXIT_RUNTIME, "http_client_failed", err),
    };
    if let Some(Err(err)) = args.dns_zone.as_deref().map(dns::origin) {
        return fatal(EXIT_USAGE, "dns_zone_invalid", err);
    }
    if let Some(Err(err)) = args.kubeconfig.as_deref().map(source::kube_client) {
        return fatal(EXIT_KUBE, "kubeconfig_unusable", err);
    }

    let tun = match Tun::create(&args.tun_name) {
        Ok(tun) => Arc::new(tun),
        Err(err) => {
            return fatal(
                EXIT_NETWORK_DEVICE,
                "tun_create_failed",
                format!("{}: {err}", args.tun_name),
            );
        }
    };
    let index = match tun.index() {
        Ok(index) => index,
        Err(err) => return fatal(EXIT_NETWORK_DEVICE, "tun_index_failed", err),
    };
    let mut netconf = match NetConf::connect(index) {
        Ok(netconf) => netconf,
        Err(err) => return fatal(EXIT_NETWORK_DEVICE, "netlink_failed", err),
    };
    if let Err(err) = netconf.bring_up(args.mtu).await {
        return fatal(EXIT_NETWORK_DEVICE, "tun_up_failed", err);
    }
    tracing::info!(event = "tun_up", tun = %tun.name(), index, mtu = args.mtu);

    let listener = match status::bind(args.metrics_listen).await {
        Ok(listener) => listener,
        Err(err) => {
            return fatal(
                EXIT_STATUS_LISTEN,
                "status_listen_failed",
                format!("{}: {err}", args.metrics_listen),
            );
        }
    };

    if !args.metrics_listen.ip().is_loopback() {
        tracing::warn!(
            event = "status_not_loopback",
            listen = %args.metrics_listen,
            reason = "the status endpoint names every peer endpoint and pod CIDR"
        );
    }

    let options = EndpointOptions {
        relay: args.relay.clone(),
        discovery: args.discovery.into(),
        bind: args.bind.clone(),
    };
    let endpoint = match mesh::bind_endpoint(id.secret.clone(), &options).await {
        Ok(endpoint) => endpoint,
        Err(err) => return fatal(EXIT_ENDPOINT, "endpoint_bind_failed", err),
    };
    tracing::info!(
        event = "endpoint_bound",
        endpoint = %me.endpoint_hex,
        sockets = ?endpoint.bound_sockets(),
        relay = ?options.relay,
        discovery = ?options.discovery
    );

    let stats = Arc::new(Stats::default());
    let connections = Arc::new(Connections::default());
    let initial = RouteTable {
        self_node: me.node_name.clone(),
        ..RouteTable::default()
    };
    let (routes_tx, routes_rx) = watch::channel(Arc::new(initial));

    let info = Arc::new(StatusInfo {
        endpoint_hex: me.endpoint_hex.clone(),
        did: me.did.clone(),
        tun: tun.name().to_string(),
        mtu: args.mtu,
        source: match (&args.kubeconfig, &args.roster) {
            (Some(_), Some(_)) => "kube+roster",
            (Some(_), None) => "kube",
            _ => "roster",
        }
        .to_string(),
        ready: Default::default(),
        routes: routes_rx.clone(),
        connections: Arc::clone(&connections),
        stats: Arc::clone(&stats),
        installed_routes: Default::default(),
    });
    tokio::spawn(status::serve(listener, Arc::clone(&info)));

    let mesh = Arc::new(Mesh {
        endpoint: endpoint.clone(),
        tun: Arc::clone(&tun),
        routes: routes_rx.clone(),
        connections: Arc::clone(&connections),
        stats: Arc::clone(&stats),
    });
    tokio::spawn(Arc::clone(&mesh).run_accept());
    let mut outbound = tokio::spawn(Arc::clone(&mesh).run_outbound());

    let kube_mode = args.kubeconfig.is_some();
    let mut annotator = args.kubeconfig.as_ref().map(|path| {
        tokio::spawn(source::run_annotator(
            path.clone(),
            me.clone(),
            routes_rx.clone(),
        ))
    });
    let (roster_tx, roster_rx) = watch::channel(Arc::new(Roster::default()));
    let (location_tx, location_rx) = watch::channel(args.roster.clone());
    let (kube_tx, kube_rx) = watch::channel(Arc::new(Vec::new()));
    let kubeconfig = args.kubeconfig.clone();
    let routes = source::run_routes(
        me.clone(),
        kube_mode.then_some(kube_rx),
        roster_rx.clone(),
        (kube_mode && args.roster.is_none()).then_some(location_tx),
        routes_tx,
        Arc::clone(&stats),
    );
    let mut source_task = tokio::spawn(async move {
        tokio::select! {
            () = routes => Ok(()),
            () = roster::run(http, location_rx, roster_tx) => Ok(()),
            result = async {
                match kubeconfig {
                    Some(path) => source::run_kube(path, kube_tx).await,
                    None => std::future::pending().await,
                }
            } => result,
        }
    });
    let mut dns_task = args.dns_zone.clone().map(|zone| {
        tokio::spawn(dns::run(
            DnsOptions {
                listen: args.dns_listen.clone(),
                zone,
                upstream: args.dns_upstream.clone(),
            },
            roster_rx.clone(),
        ))
    });

    let sync_info = Arc::clone(&info);
    let sync_stats = Arc::clone(&stats);
    let sync_me = me.clone();
    let mut sync_rx = routes_rx.clone();
    tokio::spawn(async move {
        loop {
            let table = Arc::clone(&sync_rx.borrow_and_update());
            let failures = netconf.sync(&table).await;
            for failure in &failures {
                Stats::bump(&sync_stats.route_sync_failures);
                tracing::warn!(event = "route_sync_failed", error = %failure);
            }
            if let Ok(mut installed) = sync_info.installed_routes.lock() {
                *installed = netconf
                    .installed_routes()
                    .iter()
                    .map(ToString::to_string)
                    .collect();
            }
            let ready = failures.is_empty() && ready_now(&table, &sync_me, kube_mode);
            let was = sync_info.ready.swap(ready, Ordering::Relaxed);
            if ready && !was {
                tracing::info!(
                    event = "ready",
                    own_pod_cidrs = ?table.own_cidrs.iter().map(ToString::to_string).collect::<Vec<_>>(),
                    routes = table.routes.len()
                );
                if let Err(err) = sd_notify("READY=1\nSTATUS=mesh ready") {
                    tracing::warn!(event = "sd_notify_failed", error = %err);
                }
            }
            if sync_rx.changed().await.is_err() {
                return;
            }
        }
    });

    let mut sigterm = match signal(SignalKind::terminate()) {
        Ok(s) => s,
        Err(err) => return fatal(EXIT_RUNTIME, "signal_setup_failed", err),
    };
    let mut sigint = match signal(SignalKind::interrupt()) {
        Ok(s) => s,
        Err(err) => return fatal(EXIT_RUNTIME, "signal_setup_failed", err),
    };

    let code = tokio::select! {
        _ = sigterm.recv() => { tracing::info!(event = "stopping", signal = "SIGTERM"); EXIT_OK }
        _ = sigint.recv() => { tracing::info!(event = "stopping", signal = "SIGINT"); EXIT_OK }
        result = &mut outbound => match result {
            Ok(Ok(())) => fatal(EXIT_RUNTIME, "outbound_stopped", "route source closed"),
            Ok(Err(err)) => fatal(EXIT_NETWORK_DEVICE, "tun_read_failed", err),
            Err(err) => fatal(EXIT_RUNTIME, "outbound_panicked", err),
        },
        result = &mut source_task => source_exit(result, "source"),
        result = async {
            match annotator.as_mut() {
                Some(handle) => handle.await,
                None => std::future::pending().await,
            }
        } => source_exit(result, "annotator"),
        result = async {
            match dns_task.as_mut() {
                Some(handle) => handle.await,
                None => std::future::pending().await,
            }
        } => match result {
            Ok(Ok(())) => fatal(EXIT_RUNTIME, "task_stopped", "dns ended"),
            Ok(Err(err)) => fatal(EXIT_RUNTIME, "dns_failed", err),
            Err(err) => fatal(EXIT_RUNTIME, "task_panicked", format!("dns: {err}")),
        },
    };
    let _ = sd_notify("STOPPING=1");
    connections.close_all();
    let _ = tokio::time::timeout(SHUTDOWN_GRACE, endpoint.close()).await;
    code
}