johnandersen777.bsky.social/socialweb-computer-kcp

Switch repository

Loading…

package cloudinit

import (
	"cmp"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/opt"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/sha256sum"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/nodeagent"
	"github.com/johnandersen777/socialweb-computer-kcp/internal/systemdunit"
)

const (
	NodeSecretsDir = "/etc/socialweb/node"

	NodeOpenBaoTicketPath = NodeSecretsDir + "/openbao-ticket"

	NodeOpenBaoCAPath = NodeSecretsDir + "/openbao-ca.pem"

	NodeClaimDocPath = NodeSecretsDir + "/claim.json"

	NodeLinkEnvPath = NodeSecretsDir + "/apiserver.env"

	NodeOpenBaoLinkAddress = "127.0.0.1:18200"

	NodeKVMount = "socialweb-nodes"

	NodeJWTUserClaim = "wid_sub"

	NodeJWTMountPrefix = "jwt-"

	NodeJWTMountHashHex = 12

	NodeUnitK3sAgent = nodeagent.UnitK3sAgent

	NodeUnitDNSLink = nodeagent.UnitDNSLink

	NodeUnitMasquerade = nodeagent.UnitMasquerade

	NodeUnitCNI = nodeagent.UnitCNI

	NodeUnitMeshdInstall = nodeagent.UnitMeshdInstall

	NodeUnitMeshd = nodeagent.UnitMeshd

	NodeUnitDiagnose = nodeagent.UnitDiagnose

	NodeUnitBootstrap = nodeagent.UnitBootstrap

	NodeScriptBootstrap = "/usr/local/bin/socialweb-node-bootstrap"

	NodeSecretsFetchUnit = SecretsFetchUnit

	K3sNodePasswordPath = "/etc/rancher/node/password"

	NodeLocalDNSAddress = nodeagent.NodeLocalDNSAddress

	NodeLocalDNSLabel = "socialweb.computer/node-local-dns"

	NodeLocalDNSLink = nodeagent.NodeLocalDNSLink

	PodClusterCIDR = nodeagent.PodClusterCIDR

	PodMTU = nodeagent.PodMTU

	PodBridge = nodeagent.PodBridge

	PodEgressComment = nodeagent.PodEgressComment

	CNIConflistName = nodeagent.CNIConflistName

	CNIConfDir = nodeagent.DefaultCNIConfDir

	CNIBinDir = nodeagent.DefaultCNIBinDir

	MeshdPath = nodeagent.DefaultMeshdPath

	MeshdEnvPath = NodeSecretsDir + "/meshd.env"

	HostnamePath = nodeagent.DefaultHostnamePath

	SysctlPath = "/etc/sysctl.d/90-socialweb-node.conf"

	K3sJoinModuleID = "k3s-join"

	NodeJoinConfigPath = NodeSecretsDir + "/join.json"
)

const (
	DefaultCNIWaitAttempts = nodeagent.DefaultCNIWaitAttempts

	DefaultDiagnoseAttempts = nodeagent.DefaultDiagnoseAttempts

	DefaultLinkWaitAttempts = nodeagent.DefaultLinkWaitAttempts
)

type MeshdPin = nodeagent.MeshdPin

type NodeAgentPin = nodeagent.NodeAgentPin

type K3sJoinOptions struct {
	DNSNameservers []string

	K3sDataDir string

	AcceptPath string

	DumbpipePath string

	K3sPath string

	NodeDir string

	K3sNodePasswordPath string

	DoneStamp string

	OpenBaoAddr string

	Meshd *MeshdPin

	NodeAgent *NodeAgentPin

	MeshdPath string

	NodeBinaryPath string

	CNIConfDir string

	CNIBinDir string

	KubeletKubeconfig string

	CNIWaitAttempts int

	DiagnoseAttempts int

	LinkWaitAttempts int
}

func (o K3sJoinOptions) defaults() K3sJoinOptions {
	o.NodeDir = cmp.Or(o.NodeDir, NodeSecretsDir)
	o.K3sDataDir = cmp.Or(o.K3sDataDir, nodeagent.DefaultK3sDataDir)
	o.KubeletKubeconfig = cmp.Or(o.KubeletKubeconfig, o.K3sDataDir+"/agent/kubelet.kubeconfig")
	o.DumbpipePath = cmp.Or(o.DumbpipePath, nodeagent.DefaultDumbpipePath)
	o.K3sPath = cmp.Or(o.K3sPath, nodeagent.DefaultK3sPath)
	o.NodeBinaryPath = cmp.Or(o.NodeBinaryPath, nodeagent.DefaultNodeBinaryPath)
	o.K3sNodePasswordPath = cmp.Or(o.K3sNodePasswordPath, K3sNodePasswordPath)
	o.DoneStamp = cmp.Or(o.DoneStamp, nodeagent.DefaultDoneStamp)
	o.OpenBaoAddr = cmp.Or(o.OpenBaoAddr, NodeOpenBaoLinkAddress)
	o.MeshdPath = cmp.Or(o.MeshdPath, MeshdPath)
	o.CNIConfDir = cmp.Or(o.CNIConfDir, CNIConfDir)
	o.CNIBinDir = cmp.Or(o.CNIBinDir, CNIBinDir)
	o.CNIWaitAttempts = opt.Positive(o.CNIWaitAttempts, DefaultCNIWaitAttempts)
	o.DiagnoseAttempts = opt.Positive(o.DiagnoseAttempts, DefaultDiagnoseAttempts)
	o.LinkWaitAttempts = opt.Positive(o.LinkWaitAttempts, DefaultLinkWaitAttempts)
	return o
}

func (o K3sJoinOptions) config() (nodeagent.Config, error) {
	o = o.defaults()
	if err := nodeagent.ValidateNodeAgentPin(o.NodeAgent); err != nil {
		return nodeagent.Config{}, fmt.Errorf("cloudinit: %w", err)
	}
	if err := nodeagent.ValidateMeshdPin(o.Meshd); err != nil {
		return nodeagent.Config{}, fmt.Errorf("cloudinit: %w", err)
	}
	cfg := nodeagent.Config{
		ClaimDocPath:                  o.NodeDir + "/claim.json",
		ClaimCAPath:                   o.NodeDir + "/openbao-ca.pem",
		OpenBaoLinkAddress:            o.OpenBaoAddr,
		OpenBaoTicketPath:             o.NodeDir + "/openbao-ticket",
		GuestLoginHelperPath:          GuestLoginScriptPath,
		K3sDataDir:                    o.K3sDataDir,
		NodeDir:                       o.NodeDir,
		CNIConfDir:                    o.CNIConfDir,
		CNIBinDir:                     o.CNIBinDir,
		KubeletKubeconfig:             o.KubeletKubeconfig,
		APIServerEnvPath:              o.NodeDir + "/apiserver.env",
		TokenPath:                     o.NodeDir + "/token",
		NodePasswordPath:              o.NodeDir + "/node-password",
		ServerCAPath:                  o.NodeDir + "/server-ca.pem",
		K3sNodePasswordPath:           o.K3sNodePasswordPath,
		MeshdEnvPath:                  o.NodeDir + "/meshd.env",
		DoneStamp:                     o.DoneStamp,
		K3sPath:                       o.K3sPath,
		DumbpipePath:                  o.DumbpipePath,
		MeshdPath:                     o.MeshdPath,
		HostnamePath:                  HostnamePath,
		NodeBinaryPath:                o.NodeBinaryPath,
		ConfigPath:                    o.NodeDir + "/join.json",
		LinkWaitAttempts:              o.LinkWaitAttempts,
		CNIWaitAttempts:               o.CNIWaitAttempts,
		DiagnoseAttempts:              o.DiagnoseAttempts,
		GuestLoginWaitAttempts:        nodeagent.DefaultGuestLoginWaitAttempts,
		DiagnoseIntervalSeconds:       int(nodeagent.DefaultDiagnoseInterval.Seconds()),
		CNIWaitIntervalMS:             int(nodeagent.DefaultCNIWaitInterval.Milliseconds()),
		LinkWaitIntervalMS:            int(nodeagent.DefaultLinkWaitInterval.Milliseconds()),
		GuestLoginWaitIntervalSeconds: int(nodeagent.DefaultGuestLoginWaitInterval.Seconds()),
		Meshd:                         o.Meshd,
	}
	cfg = cfg.WithDefaults()
	if err := cfg.Validate(); err != nil {
		return nodeagent.Config{}, fmt.Errorf("cloudinit: %w", err)
	}
	return cfg, nil
}

func NodeAudience(clusterKey, nodeName string) string {
	return "api://socialweb-node?actx=" + clusterKey + "/" + nodeName
}

func NodeJWTMount(issuer string) string {
	sum := sha256sum.Hex([]byte(issuer))
	return NodeJWTMountPrefix + sum[:NodeJWTMountHashHex]
}

func renderResolver(nameservers []string) string {
	var b strings.Builder
	for _, name := range nameservers {
		b.WriteString("nameserver ")
		b.WriteString(name)
		b.WriteString("\n")
	}
	b.WriteString("options timeout:2 attempts:3\n")
	return b.String()
}

func shellQuote(value string) string {
	return "'" + strings.ReplaceAll(value, "'", `'\''`) + "'"
}

func renderBootstrapScript(o K3sJoinOptions, cfg nodeagent.Config) string {
	pin := o.NodeAgent
	return `#!/bin/sh
set -eu
BIN=` + shellQuote(cfg.NodeBinaryPath) + `
CONFIG=` + shellQuote(cfg.ConfigPath) + `
BASE=` + shellQuote(pin.DownloadBase) + `
case "$(uname -m)" in
  x86_64|amd64) ARCH=x86_64; WANT=` + pin.Sha256["x86_64"] + ` ;;
  aarch64|arm64) ARCH=aarch64; WANT=` + pin.Sha256["aarch64"] + ` ;;
  *) echo "socialweb-node-bootstrap: no socialweb-node build for $(uname -m)" >&2; exit 1 ;;
esac
if [ -x "$BIN" ] && [ "$(sha256sum "$BIN" | cut -d' ' -f1)" = "$WANT" ]; then
  exec "$BIN" units start --config "$CONFIG"
fi
TMP="${BIN}.download.$$"
trap 'rm -f "$TMP"' EXIT
curl -fsSL --retry 5 --retry-delay 2 -o "$TMP" "$BASE/downloads/$WANT/socialweb-node-$ARCH"
GOT="$(sha256sum "$TMP" | cut -d' ' -f1)"
[ "$GOT" = "$WANT" ] || { echo "socialweb-node-bootstrap: the door served sha256 $GOT, not the pinned $WANT" >&2; exit 1; }
chmod 0755 "$TMP"
mv -f "$TMP" "$BIN"
trap - EXIT
exec "$BIN" units start --config "$CONFIG"
`
}

func bootstrapUnit(o K3sJoinOptions) systemdunit.Unit {
	return systemdunit.Unit{Name: NodeUnitBootstrap, Sections: []systemdunit.Section{
		{Name: "Unit", Entries: []systemdunit.KeyValue{
			systemdunit.KV("Description", "socialweb node: install the sha256-pinned socialweb-node and start the node units"),
			systemdunit.KV("After", "network-online.target"),
			systemdunit.KV("Wants", "network-online.target"),
			systemdunit.KV("StartLimitIntervalSec", "0"),
		}},
		{Name: "Service", Entries: []systemdunit.KeyValue{
			systemdunit.KV("Type", "oneshot"),
			systemdunit.KV("RemainAfterExit", "yes"),
			systemdunit.KV("ExecStart", o.NodeBinaryPath+"-bootstrap"),
			systemdunit.KV("Restart", "on-failure"),
			systemdunit.KV("RestartSec", "5"),
			systemdunit.KV("StandardOutput", "journal+console"),
			systemdunit.KV("StandardError", "journal+console"),
		}},
		{Name: "Install", Entries: []systemdunit.KeyValue{systemdunit.KV("WantedBy", "multi-user.target")}},
	}}
}

func scriptFile(path, content string) WriteFileEntry {
	return WriteFileEntry{Path: path, Owner: "root:root", Permissions: "0700", Content: content}
}

func unitFileEntry(file systemdunit.File) WriteFileEntry {
	return WriteFileEntry{Path: file.Path, Owner: "root:root", Permissions: fmt.Sprintf("%04o", file.Mode.Perm()), Content: string(file.Content)}
}

func K3sJoinModule(opts K3sJoinOptions) UserDataModule {
	return func(Context) (UserDataPatch, error) {
		opts = opts.defaults()
		cfg, err := opts.config()
		if err != nil {
			return UserDataPatch{}, err
		}
		raw, err := json.MarshalIndent(cfg, "", "  ")
		if err != nil {
			return UserDataPatch{}, fmt.Errorf("cloudinit: encoding the node join config: %w", err)
		}
		raw = append(raw, '\n')
		nameservers := opts.DNSNameservers
		if len(nameservers) == 0 {
			nameservers = []string{DefaultDNSGateway}
		}
		resolver := renderResolver(nameservers)
		files := []WriteFileEntry{
			{Path: ResolvConfPath, Owner: "root:root", Permissions: "0644", Content: resolver},
			{Path: SysctlPath, Owner: "root:root", Permissions: "0644", Content: "net.ipv4.ip_forward = 1\n"},
			{Path: cfg.ConfigPath, Owner: "root:root", Permissions: "0600", Content: string(raw)},
			scriptFile(cfg.NodeBinaryPath+"-bootstrap", renderBootstrapScript(opts, cfg)),
		}
		bootstrap := bootstrapUnit(opts)
		files = append(files, unitFileEntry(systemdunit.Files("/etc/systemd/system", bootstrap)[0]))
		for _, file := range nodeagent.UnitFiles(cfg) {
			files = append(files, unitFileEntry(file))
		}
		writeResolver := "printf '%s' '" + strings.ReplaceAll(resolver, "'", `'\''`) + "' > " + ResolvConfPath
		return UserDataPatch{
			Packages:   []string{"curl", "ca-certificates", "iptables"},
			WriteFiles: files,
			Bootcmd:    []any{writeResolver},
			Runcmd: []any{
				writeResolver,
				"systemctl daemon-reload",
				"systemctl enable " + NodeUnitBootstrap,
				"systemctl start --no-block " + NodeUnitBootstrap,
			},
		}, nil
	}
}