0
0
package portallocator
import (
"fmt"
"net"
"regexp"
"sort"
"strconv"
"strings"
)
const LoopbackAdvertiseAddress = "127.0.0.1"
const MarketAdvertiseAddress = "198.18.255.254"
func FindDuplicatePorts(leases []Lease) []string {
byPort := map[int][]Lease{}
for _, l := range leases {
width := max(l.Width, 1)
for p := l.Port; p < l.Port+width; p++ {
byPort[p] = append(byPort[p], l)
}
}
ports := make([]int, 0, len(byPort))
for p := range byPort {
ports = append(ports, p)
}
sort.Ints(ports)
var out []string
for _, p := range ports {
claimants := byPort[p]
if len(claimants) < 2 {
continue
}
names := make([]string, 0, len(claimants))
for _, c := range claimants {
if c.AppInstance != "" {
names = append(names, fmt.Sprintf("%s (%s, instance %s)", c.Cluster, c.Purpose, c.AppInstance))
continue
}
names = append(names, fmt.Sprintf("%s (%s)", c.Cluster, c.Purpose))
}
out = append(out, fmt.Sprintf(
"port %d is leased to BOTH %s. This is not a port conflict: a colliding advertise-port routes "+
"this node's tunnel to ANOTHER CLUSTER'S apiserver, which surfaces as an x509 name error. "+
"Fix the duplicate LEASE -- do not go looking at certificates",
p, strings.Join(names, " and ")))
}
return out
}
type Endpoint struct {
Address string
Port int
}
var dialLine = regexp.MustCompile(`wss://([A-Za-z0-9._-]+):(\d+)`)
var probeLine = regexp.MustCompile(`KCP_CONN apiserver ([A-Za-z0-9._-]+):(\d+) -> OK`)
func AssertAgentDialledOurPort(line string, want Endpoint) error {
var m []string
for _, re := range []*regexp.Regexp{probeLine, dialLine} {
if found := re.FindStringSubmatch(line); found != nil {
m = found
break
}
}
if m == nil {
return fmt.Errorf("portallocator: no dial line from this node's agent in what it printed, " +
"so which apiserver it reached is unverified. A silent log is a failure here, not a " +
"pass: the agent being connected says nothing about WHICH cluster it reached, and a " +
"guest that was handed the wrong address prints a FAIL line of the same shape")
}
gotPort, err := strconv.Atoi(m[2])
if err != nil {
return fmt.Errorf("portallocator: unparsable dial line %q", line)
}
if m[1] != want.Address || gotPort != want.Port {
return fmt.Errorf("portallocator: ADVERTISE MISMATCH: the agent dialled %s:%d but this cluster's lease is %s:%d. "+
"If the ports differ, a colliding advertise-port routes this node's tunnel to ANOTHER CLUSTER'S apiserver, "+
"which surfaces as an x509 name error -- the certificate is not the fault",
m[1], gotPort, want.Address, want.Port)
}
return nil
}
func AssertAdvertiseAddress(address string) error {
ip := net.ParseIP(address)
if ip == nil {
return fmt.Errorf("portallocator: --advertise-address %q is not an IP address. It is the "+
"address a NODE dials this cluster's apiserver on, so a name here resolves to whatever "+
"the node's resolver says, which is not a value this host owns", address)
}
for _, iface := range hostAddresses() {
if iface.Equal(ip) {
return nil
}
}
return fmt.Errorf("portallocator: --advertise-address %q is not an address this host carries, "+
"so a node that is a LOCAL PROCESS told to dial it dials nothing. This was measured: the "+
"deleted VM substrate's constant 192.168.5.2 has no interface here, nothing answers it, and "+
"the store DSN built from it never connected. A market node is not a local process and does "+
"not answer to this rule: its advertised address is put on the GUEST's own loopback and the "+
"market's door reverse-forwards it here, so it is checked by "+
"AssertMarketAdvertiseAddress instead", address)
}
func AssertMarketAdvertiseAddress(address string) error {
ip := net.ParseIP(address)
if ip == nil {
return fmt.Errorf("portallocator: %q is not an IP address, and it is the address a MARKET "+
"node dials its cluster's apiserver on. A name here resolves to whatever each guest's "+
"resolver says, which is a different answer in every guest and no answer at all in a "+
"guest whose resolver has not been written yet", address)
}
if ip.IsLoopback() {
return fmt.Errorf("portallocator: %q is a loopback address, and a market node's machine is "+
"not this host. k3s refuses a loopback address for the in-cluster `kubernetes` Service "+
"endpoint (measured: subsets[0].addresses[0].ip Invalid value: \"127.0.0.1\": may not be "+
"in the loopback range), so a cluster advertised on one fails to start rather than fails "+
"to be reached", address)
}
if ip.IsUnspecified() {
return fmt.Errorf("portallocator: %q is the unspecified address, which names every "+
"interface and no interface. A market node puts the advertised address on its OWN "+
"loopback and the door reverse-forwards it here, so there is nothing for a wildcard to "+
"be put on and nothing for the forward to name", address)
}
for _, iface := range hostAddresses() {
if iface.Equal(ip) {
return fmt.Errorf("portallocator: %q is an address THIS HOST carries, and a market "+
"node's machine is on the far side of the market's SSH door. The guest puts the "+
"advertised address on its OWN loopback and the door reverse-forwards it to this "+
"host's apiserver, so an address this host already carries is one the guest cannot "+
"take and the forward collides with whatever answers it here", address)
}
}
return nil
}
type Advertisement struct {
Address string
Port int
LinkTarget int
Front Lease
SlotListen []int
}
func AssertAdvertiseInvariant(a Advertisement) error {
return advertiseInvariant(a, dialFromThisHost)
}
func AssertMarketAdvertiseInvariant(a Advertisement) error {
return advertiseInvariant(a, dialFromTheMarket)
}
type dialler uint8
const (
dialFromThisHost dialler = iota
dialFromTheMarket
)
func advertiseInvariant(a Advertisement, dial dialler) error {
check := AssertAdvertiseAddress
if dial == dialFromTheMarket {
check = AssertMarketAdvertiseAddress
}
if err := check(a.Address); err != nil {
return err
}
lease := a.Front
if lease.Purpose != PurposeK3sSecure {
return fmt.Errorf("portallocator: cluster %s holds %s:%d, and that is not the lease whose port "+
"the apiserver is advertised on. The advertised port is the port the cluster's apiserver "+
"front binds -- %s -- and every k3s slot sits behind it on a port of its own",
lease.Cluster, lease.Purpose, lease.Port, PurposeK3sSecure)
}
if a.Port != lease.Port {
return fmt.Errorf("portallocator: refusing to start: --advertise-port is %d but cluster %s's "+
"front lease is %d. The port a cluster advertises is the port everything in the "+
"cluster reaches it on: the `kubernetes` Service endpoints carry it, and the tunnel a "+
"guest's kubelet dials back to the apiserver is dialled at it. Only the front binds it; "+
"announcing any other port leaves coredns, metrics-server and local-path-provisioner "+
"dialling a slot that may be gone after the next control-plane swap",
a.Port, lease.Cluster, lease.Port)
}
if a.LinkTarget != lease.Port {
return fmt.Errorf("portallocator: refusing to start: the apiserver link targets %d but cluster %s's front "+
"lease is %d; a link is never re-pointed for a k3s swap, so it targets the front", a.LinkTarget, lease.Cluster, lease.Port)
}
for _, p := range a.SlotListen {
if p >= lease.Port && p < lease.Port+max(lease.Width, 1) {
return fmt.Errorf("portallocator: refusing to start: a k3s slot of cluster %s listens on %d, inside the "+
"front lease %d. A slot's listen port is never advertised and never the front's", lease.Cluster, p, lease.Port)
}
}
return nil
}
func hostAddresses() []net.IP {
addrs, err := net.InterfaceAddrs()
if err != nil {
return []net.IP{net.ParseIP(LoopbackAdvertiseAddress)}
}
out := make([]net.IP, 0, len(addrs))
for _, a := range addrs {
switch v := a.(type) {
case *net.IPNet:
out = append(out, v.IP)
case *net.IPAddr:
out = append(out, v.IP)
}
}
return out
}