0
0
package nodeagent
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/childproc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/clock"
"github.com/johnandersen777/socialweb-computer-kcp/internal/nodemarket"
"github.com/johnandersen777/socialweb-computer-kcp/internal/systemdunit"
"github.com/johnandersen777/socialweb-computer-kcp/internal/systemdunit/notify"
"github.com/johnandersen777/socialweb-computer-kcp/internal/vault/openbao"
)
const openBaoLinkName = "openbao"
const apiserverLinkName = "apiserver"
type claimDocument struct {
Node string `json:"node"`
JWTMount string `json:"jwtMount"`
Role string `json:"role"`
Audience string `json:"audience"`
KVMount string `json:"kvMount"`
KVPath string `json:"kvPath"`
}
func LoadConfig(path string) (Config, error) {
if path == "" {
return Config{}, errors.New("nodeagent: no config path was given")
}
raw, err := os.ReadFile(path)
if err != nil {
return Config{}, fmt.Errorf("nodeagent: reading %s: %w", path, err)
}
var cfg Config
if err := json.Unmarshal(raw, &cfg); err != nil {
return Config{}, fmt.Errorf("nodeagent: %s is not a join config: %w", path, err)
}
cfg = cfg.WithDefaults()
cfg.ConfigPath = path
if err := cfg.Validate(); err != nil {
return Config{}, err
}
return cfg, nil
}
func Join(ctx context.Context, cfg Config, log *slog.Logger) error {
if log == nil {
log = slog.Default()
}
cfg = cfg.WithDefaults()
if err := cfg.Validate(); err != nil {
return err
}
if !executable(cfg.DumbpipePath) {
return fmt.Errorf("nodeagent: %s is not installed", cfg.DumbpipePath)
}
if _, err := openBaoLinkArgs(cfg); err != nil {
return err
}
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
wg.Go(func() {
superviseLink(runCtx, cfg, openBaoLinkName, log)
})
if err := prepare(runCtx, cfg, log); err != nil {
cancel()
wg.Wait()
return err
}
if _, err := apiserverLinkArgs(ctx, cfg); err != nil {
cancel()
wg.Wait()
return err
}
wg.Go(func() {
superviseLink(runCtx, cfg, apiserverLinkName, log)
})
if err := notify.Ready("joined the cluster; supervising the OpenBao and apiserver links"); err != nil {
log.Warn("sd_notify READY failed", "err", err.Error())
}
log.Info("the join is complete; the links run under this service", "config", cfg.ConfigPath)
<-ctx.Done()
cancel()
wg.Wait()
return nil
}
func prepare(ctx context.Context, cfg Config, log *slog.Logger) error {
if fileNonEmpty(cfg.DoneStamp) && fileNonEmpty(cfg.APIServerEnvPath) && fileNonEmpty(cfg.MeshdEnvPath) {
log.Info("the join secrets were already fetched; keeping them and starting the links",
"stamp", cfg.DoneStamp)
return nil
}
for _, path := range []string{cfg.ClaimDocPath, cfg.ClaimCAPath} {
if !fileNonEmpty(path) {
return fmt.Errorf("nodeagent: %s is missing", path)
}
}
raw, err := os.ReadFile(cfg.ClaimDocPath)
if err != nil {
return fmt.Errorf("nodeagent: reading %s: %w", cfg.ClaimDocPath, err)
}
var doc claimDocument
if err := json.Unmarshal(raw, &doc); err != nil {
return fmt.Errorf("nodeagent: %s is not a claim document: %w", cfg.ClaimDocPath, err)
}
for name, value := range map[string]string{
"node": doc.Node, "jwtMount": doc.JWTMount, "role": doc.Role,
"kvMount": doc.KVMount, "kvPath": doc.KVPath,
} {
if value == "" {
return fmt.Errorf("nodeagent: %s carries no %s", cfg.ClaimDocPath, name)
}
if !validName(value) {
return fmt.Errorf("nodeagent: %s carries an unsafe name for %s: %q", cfg.ClaimDocPath, name, value)
}
}
if doc.Audience == "" {
return fmt.Errorf("nodeagent: %s carries no audience", cfg.ClaimDocPath)
}
exchanged, err := exchangeWorkloadToken(ctx, cfg, doc.Audience, log)
if err != nil {
return err
}
caPEM, err := os.ReadFile(cfg.ClaimCAPath)
if err != nil {
return fmt.Errorf("nodeagent: reading %s: %w", cfg.ClaimCAPath, err)
}
pool, err := openbao.CertPoolFromPEM(caPEM)
if err != nil {
return fmt.Errorf("nodeagent: %s: %w", cfg.ClaimCAPath, err)
}
client := openbao.New("https://"+cfg.OpenBaoLinkAddress, "", openbao.WithCACertPool(pool))
logged, err := loginOpenBao(ctx, cfg, client, doc, exchanged, log)
if err != nil {
return err
}
data, err := readNodeKV(ctx, cfg, logged, doc, log)
if err != nil {
return err
}
token, ok := firstValue(data, "token")
if !ok {
return fmt.Errorf("nodeagent: %s/%s has no token", doc.KVMount, doc.KVPath)
}
nodePassword, ok := firstValue(data, "nodePassword")
if !ok {
return fmt.Errorf("nodeagent: %s/%s has no nodePassword", doc.KVMount, doc.KVPath)
}
serverCA, ok := firstValue(data, "serverCA")
if !ok {
return fmt.Errorf("nodeagent: %s/%s has no serverCA", doc.KVMount, doc.KVPath)
}
address, port, ticket := "", "", ""
fromBundle := fileNonEmpty(cfg.APIServerEnvPath)
if fromBundle {
env, err := readEnvFile(cfg.APIServerEnvPath)
if err != nil {
return fmt.Errorf("nodeagent: reading %s: %w", cfg.APIServerEnvPath, err)
}
if env["NODE_NAME"] != doc.Node {
return fmt.Errorf("nodeagent: the bundled apiserver link names %q, not %q", env["NODE_NAME"], doc.Node)
}
address, port, ticket = env["ADVERTISE_ADDRESS"], env["APISERVER_PORT"], env["APISERVER_TICKET"]
log.Info("the apiserver link came with the bundle", "path", cfg.APIServerEnvPath)
} else {
for _, key := range []string{"apiserverTicket", "advertiseAddress", "apiserverPort"} {
if _, ok := firstValue(data, key); !ok {
return fmt.Errorf("nodeagent: %s/%s has no %s", doc.KVMount, doc.KVPath, key)
}
}
address, _ = firstValue(data, "advertiseAddress")
port, _ = firstValue(data, "apiserverPort")
ticket, _ = firstValue(data, "apiserverTicket")
}
if err := guardLinkEnv(address, port, ticket); err != nil {
return err
}
joinToken := stripSpace(token)
if !strings.HasPrefix(joinToken, "K10") {
joinToken = nodemarket.PinnedJoinToken(serverCA, joinToken)
}
for _, dir := range []string{cfg.NodeDir, filepath.Dir(cfg.K3sNodePasswordPath)} {
if err := ensureDir(dir, 0o700); err != nil {
return fmt.Errorf("nodeagent: creating %s: %w", dir, err)
}
}
if err := writeAtomic(cfg.ServerCAPath, []byte(serverCA), 0o600); err != nil {
return err
}
if err := writeAtomic(cfg.TokenPath, []byte(joinToken), 0o600); err != nil {
return err
}
password := strings.ReplaceAll(nodePassword, "\n", "")
if err := writeAtomic(cfg.NodePasswordPath, []byte(password), 0o600); err != nil {
return err
}
if err := writeAtomic(cfg.K3sNodePasswordPath, []byte(password), 0o600); err != nil {
return err
}
if !fromBundle {
env := fmt.Sprintf("NODE_NAME=%s\nADVERTISE_ADDRESS=%s\nAPISERVER_PORT=%s\nAPISERVER_TICKET=%s\n",
doc.Node, address, port, ticket)
if err := writeAtomic(cfg.APIServerEnvPath, []byte(env), 0o600); err != nil {
return err
}
}
meshdEnv := fmt.Sprintf("SOCIALWEB_MESHD_NODE_NAME=%s\nSOCIALWEB_MESHD_KUBECONFIG=%s\n", doc.Node, cfg.KubeletKubeconfig)
if err := writeAtomic(cfg.MeshdEnvPath, []byte(meshdEnv), 0o600); err != nil {
return err
}
if err := writeAtomic(cfg.DoneStamp, nil, 0o600); err != nil {
return err
}
log.Info("fetched the join secrets", "node", doc.Node, "kv", doc.KVMount+"/"+doc.KVPath)
return nil
}
func exchangeWorkloadToken(ctx context.Context, cfg Config, audience string, log *slog.Logger) (string, error) {
waited := 0
for !executable(cfg.GuestLoginHelperPath) {
if waited >= cfg.GuestLoginWaitAttempts {
return "", fmt.Errorf("nodeagent: the guest login helper %s never appeared", cfg.GuestLoginHelperPath)
}
waited++
log.Info("waiting for the guest login helper", "path", cfg.GuestLoginHelperPath,
"attempt", waited, "of", cfg.GuestLoginWaitAttempts)
if err := clock.SleepErr(ctx, cfg.GuestLoginWaitInterval()); err != nil {
return "", err
}
}
out, err := run(ctx, cfg.GuestLoginHelperPath, "token", audience)
if err != nil {
return "", fmt.Errorf("nodeagent: exchanging the workload identity token for %s: %w", audience, err)
}
token := strings.TrimSpace(string(out))
if token == "" {
return "", fmt.Errorf("nodeagent: no workload identity token from the bidder's OpenBao for %s", audience)
}
return token, nil
}
func loginOpenBao(ctx context.Context, cfg Config, client *openbao.Client, doc claimDocument, exchanged string, log *slog.Logger) (*openbao.Client, error) {
for attempt := 1; attempt <= cfg.LinkWaitAttempts; attempt++ {
logged, err := client.LoginJWT(ctx, doc.JWTMount, doc.Role, exchanged)
if err == nil {
log.Info("logged in to the cluster OpenBao", "mount", doc.JWTMount, "role", doc.Role)
return logged, nil
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
log.Info("the OpenBao link has not answered yet", "address", cfg.OpenBaoLinkAddress,
"attempt", attempt, "of", cfg.LinkWaitAttempts, "err", errString(err))
if err := clock.SleepErr(ctx, cfg.LinkWaitInterval()); err != nil {
return nil, err
}
}
return nil, fmt.Errorf("nodeagent: OpenBao login at auth/%s as %s failed after %d attempts at the link on %s",
doc.JWTMount, doc.Role, cfg.LinkWaitAttempts, cfg.OpenBaoLinkAddress)
}
func readNodeKV(ctx context.Context, cfg Config, client *openbao.Client, doc claimDocument, log *slog.Logger) (map[string]any, error) {
path := strings.Trim(doc.KVMount, "/") + "/data/" + strings.Trim(doc.KVPath, "/")
for attempt := 1; attempt <= cfg.LinkWaitAttempts; attempt++ {
response, err := client.Read(ctx, path)
if err == nil {
data, dataErr := kvData(response)
if dataErr == nil {
return data, nil
}
err = dataErr
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
log.Info("the OpenBao link has not answered with the node secrets yet", "address", cfg.OpenBaoLinkAddress,
"attempt", attempt, "of", cfg.LinkWaitAttempts, "err", errString(err))
if err := clock.SleepErr(ctx, cfg.LinkWaitInterval()); err != nil {
return nil, err
}
}
return nil, fmt.Errorf("nodeagent: reading %s/%s failed after %d attempts at the link on %s",
doc.KVMount, doc.KVPath, cfg.LinkWaitAttempts, cfg.OpenBaoLinkAddress)
}
func kvData(response map[string]any) (map[string]any, error) {
envelope, ok := response["data"].(map[string]any)
if !ok {
return nil, errors.New("the response carries no data object")
}
if data, ok := envelope["data"].(map[string]any); ok {
return data, nil
}
return envelope, nil
}
func openBaoLinkArgs(cfg Config) ([]string, error) {
raw, err := os.ReadFile(cfg.OpenBaoTicketPath)
if err != nil {
return nil, fmt.Errorf("nodeagent: reading the OpenBao ticket %s: %w", cfg.OpenBaoTicketPath, err)
}
ticket := strings.TrimSpace(string(raw))
if ticket == "" {
return nil, fmt.Errorf("nodeagent: the OpenBao ticket %s is empty", cfg.OpenBaoTicketPath)
}
return []string{"connect-tcp", "--addr", cfg.OpenBaoLinkAddress, ticket}, nil
}
func apiserverLinkArgs(ctx context.Context, cfg Config) ([]string, error) {
env, err := readEnvFile(cfg.APIServerEnvPath)
if err != nil {
return nil, fmt.Errorf("nodeagent: reading %s: %w", cfg.APIServerEnvPath, err)
}
address, port, ticket := env["ADVERTISE_ADDRESS"], env["APISERVER_PORT"], env["APISERVER_TICKET"]
if err := guardLinkEnv(address, port, ticket); err != nil {
return nil, err
}
if err := ensureLoopbackAlias(ctx, address); err != nil {
return nil, err
}
return []string{"connect-tcp", "--addr", address + ":" + port, ticket}, nil
}
func guardLinkEnv(address, port, ticket string) error {
if !validIPv4(address) {
return errors.New("nodeagent: advertiseAddress is not an IPv4 address")
}
if !validPort(port) {
return errors.New("nodeagent: apiserverPort is not a port")
}
if !validTicket(ticket) {
return errors.New("nodeagent: apiserverTicket is not a ticket")
}
return nil
}
func ensureLoopbackAlias(ctx context.Context, address string) error {
iface, err := net.InterfaceByName("lo")
if err == nil {
addrs, err := iface.Addrs()
if err == nil {
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if ip != nil && ip.String() == address {
return nil
}
}
}
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if _, err := run(ctx, "ip", "addr", "add", address+"/32", "dev", "lo"); err != nil {
return fmt.Errorf("nodeagent: adding %s/32 to lo: %w", address, err)
}
return nil
}
func superviseLink(ctx context.Context, cfg Config, name string, log *slog.Logger) {
backoff := systemdunit.Backoff{Min: time.Second, Max: 30 * time.Second}
for {
if ctx.Err() != nil {
return
}
var args []string
var err error
switch name {
case openBaoLinkName:
args, err = openBaoLinkArgs(cfg)
case apiserverLinkName:
args, err = apiserverLinkArgs(ctx, cfg)
default:
err = fmt.Errorf("nodeagent: unknown link %s", name)
}
if err != nil {
log.Error("the link cannot start", "link", name, "err", err.Error())
if clock.SleepErr(ctx, backoff.Next()) != nil {
return
}
continue
}
cmd := exec.CommandContext(ctx, cfg.DumbpipePath, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = childproc.FilterSecrets(os.Stderr)
started := time.Now()
err = cmd.Run()
if ctx.Err() != nil {
return
}
if time.Since(started) > time.Minute {
backoff.Reset()
}
delay := backoff.Next()
log.Warn("the link child exited; restarting it", "link", name, "err", errString(err), "after", delay.String())
if clock.SleepErr(ctx, delay) != nil {
return
}
}
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}