1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
package subnetallocator
import (
"errors"
"fmt"
"net"
"net/netip"
"sort"
"strings"
)
type HostNetwork struct {
Interface string
Address netip.Addr
Prefix netip.Prefix
}
func (h HostNetwork) String() string {
return fmt.Sprintf("%s carried on interface %s as %s", h.Prefix, h.Interface, h.Address)
}
func HostNetworks() ([]HostNetwork, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("listing this host's interfaces: %w", err)
}
out := make([]HostNetwork, 0, len(ifaces))
for _, iface := range ifaces {
addrs, err := iface.Addrs()
if err != nil {
return nil, fmt.Errorf("listing the addresses of interface %s: %w. A range is refused "+
"because it overlaps a network this host carries, so an interface whose addresses "+
"cannot be read is a set of networks no range can be compared against and a range "+
"leased over it is a range nobody checked", iface.Name, err)
}
for _, a := range addrs {
ipnet, ok := a.(*net.IPNet)
if !ok {
return nil, fmt.Errorf("interface %s reports an address of type %T, which is not "+
"an address with a netmask on it. Overlap is refused against the whole network "+
"an interface sits in rather than against the single address it answers on, "+
"because a range sharing any part of that network is a range a guest can be "+
"addressed in alongside this host, so an address without a mask is one this "+
"package cannot refuse anything against", iface.Name, a)
}
addr, ok := netip.AddrFromSlice(ipnet.IP)
if !ok {
return nil, fmt.Errorf("interface %s carries %v, which is not an IP address this "+
"package can compare a range against", iface.Name, ipnet.IP)
}
ones, bits := ipnet.Mask.Size()
if bits == 0 {
return nil, fmt.Errorf("interface %s carries %s under netmask %v, which is not a "+
"contiguous prefix and so cannot be written as a range of addresses. Every "+
"refusal here compares two ranges, and a network that is not a range is one no "+
"comparison covers", iface.Name, ipnet.IP, ipnet.Mask)
}
widest := addr.BitLen()
if addr.Is4In6() {
addr = addr.Unmap()
}
if bits == 128 && widest == 128 && addr.Is4() {
if ones < 96 {
return nil, fmt.Errorf("interface %s carries %s under a %d-bit netmask, which "+
"is shorter than the IPv6 prefix an IPv4-mapped address arrives in, so the "+
"network it sits in cannot be recovered from it", iface.Name, ipnet.IP, ones)
}
ones -= 96
bits = 32
}
if bits != addr.BitLen() {
return nil, fmt.Errorf("interface %s carries %s, an address %d bits wide, under a "+
"%d-bit netmask, so the two do not describe one network and no range can be "+
"compared against them", iface.Name, addr, addr.BitLen(), bits)
}
out = append(out, HostNetwork{
Interface: iface.Name,
Address: addr,
Prefix: netip.PrefixFrom(addr, ones).Masked(),
})
}
}
if len(out) == 0 {
return nil, errors.New("this host reports no interface addresses at all, not even a " +
"loopback address. Every range leased here is non-conflicting because it was compared " +
"against the networks this host carries, and a host carrying none is indistinguishable " +
"from a host whose interfaces could not be read -- so this refuses rather than treating " +
"an unreadable host as an empty one and leasing over the network that made it unreadable")
}
sort.Slice(out, func(i, j int) bool {
if out[i].Prefix != out[j].Prefix {
return out[i].Prefix.String() < out[j].Prefix.String()
}
return out[i].Interface < out[j].Interface
})
return out, nil
}
type HostCollisionError struct {
Block netip.Prefix
Network HostNetwork
}
func (e *HostCollisionError) Error() string {
return fmt.Sprintf("subnetallocator: refusing to lease %s: it overlaps %s. A range that overlaps "+
"a network this host carries is not merely untidy. The guests of a cluster are tapped into "+
"the network namespace pasta builds, and pasta builds that namespace by copying this host's "+
"own network, so an address shared with %s puts a guest and this host's own interface on one "+
"address and which of the two a packet belongs to is decided by whichever answered first. "+
"That is the measured failure this package exists to remove: the host's own LAN address came "+
"back as a node's InternalIP because the node's range was a copy of the host's LAN. This "+
"host's networks are enumerated at every allocation rather than listed here, so this refusal "+
"names the interface it actually collided with instead of a list that was true on one machine",
e.Block, e.Network, e.Network.Prefix)
}
type LeaseCollisionError struct {
Block netip.Prefix
Held Lease
}
func (e *LeaseCollisionError) Error() string {
return fmt.Sprintf("subnetallocator: refusing to lease %s: it overlaps %s, which "+
"cluster %s has held since %s. Two clusters on one range address their nodes identically. "+
"The second cluster's node reports an address the first cluster's node also reports, so the "+
"cluster's view of a node is a node of another cluster; each cluster's flannel is handed the "+
"other's pod CIDR as its own; and a packet addressed to a guest of one arrives at a guest of "+
"the other. Nothing in either cluster reports this, because every address involved is a legal "+
"address -- they are simply not unique. The range is refused here or it is never refused at "+
"all, which is why a range a human picks per cluster is dedicated by convention only",
e.Block, e.Held.Block, e.Held.Cluster, e.Held.AllocatedAt.UTC().Format("2006-01-02T15:04:05Z"))
}
type OutOfPoolError struct {
Block netip.Prefix
Pool Pool
}
func (e *OutOfPoolError) Error() string {
return fmt.Sprintf("subnetallocator: %s is not a range of the pool %s. A range is "+
"non-conflicting only because it was walked out of one pool and compared, block by block, "+
"against one record of what is already leased; a range built anywhere else was compared "+
"against nothing, which is the shape this package exists to take out of the design. The pool "+
"is a constant in this package's code and not a setting: a pool two operators can set "+
"differently is the same collision in a tidier place, and the collision this refuses is "+
"invisible in both clusters when it happens", e.Block, e.Pool)
}
type PoolExhaustedError struct {
Pool Pool
Candidates uint64
HostSkips uint64
LeaseSkips uint64
HostFirst *HostCollisionError
LeaseFirst *LeaseCollisionError
}
func (e *PoolExhaustedError) Error() string {
detail := ""
if e.HostFirst != nil {
detail += fmt.Sprintf(" The first range refused for the host was: %v", e.HostFirst)
}
if e.LeaseFirst != nil {
detail += fmt.Sprintf(" The first range refused for a lease already outstanding was: %v", e.LeaseFirst)
}
return fmt.Sprintf("subnetallocator: the pool %s is exhausted: every one of its %d ranges was "+
"walked and not one is both free and non-conflicting, %d refused for overlapping a network "+
"this host carries and %d for overlapping a range another cluster already holds. "+
"Refusing rather than wrapping around to the first range, because a wrapped range is one a "+
"cluster that is still running addresses its nodes in: the new cluster's nodes would be "+
"addressed exactly as that cluster's nodes, both clusters would disagree about which of them "+
"owns an address, and neither would report it. There is no range left to choose, so this is "+
"not a choice an operator can make differently -- freeing a lease or widening the pool "+
"constant are the two ways out.%s", e.Pool, e.Candidates, e.HostSkips, e.LeaseSkips, detail)
}
func FindDuplicateBlocks(leases []Lease) []string {
byBlock := map[string][]Lease{}
for _, l := range leases {
key := l.Block.Masked().String()
byBlock[key] = append(byBlock[key], l)
}
var out []string
for block, holders := range byBlock {
if len(holders) < 2 {
continue
}
names := make([]string, 0, len(holders))
for _, h := range holders {
names = append(names, h.Cluster)
}
sort.Strings(names)
out = append(out, fmt.Sprintf(
"subnetallocator: range %s is held by BOTH %s. This is not a range conflict: two clusters "+
"in one range address their nodes identically, so the second cluster's node reports "+
"the first cluster's node's address as its InternalIP and a packet addressed to one "+
"guest arrives at the other. Fix the duplicate LEASE -- do not go looking at the "+
"nodes, whose addresses are legal and simply not unique",
block, strings.Join(names, " and ")))
}
sort.Strings(out)
return out
}
func (a *Allocator) checkLocked(block netip.Prefix, hosts []HostNetwork) error {
if !block.IsValid() || !block.Addr().Is4() {
return fmt.Errorf("subnetallocator: %v is not an IPv4 CIDR block, so there is no network to "+
"compare and nothing to compare it with. Overlap is decided between two ranges, and a "+
"value that is not a range is one that can neither conflict nor be proved not to", block)
}
for _, host := range hosts {
if block.Overlaps(host.Prefix) {
return &HostCollisionError{Block: block, Network: host}
}
}
if block.Bits() != a.pool.Bits || !a.pool.Contains(block) {
return &OutOfPoolError{Block: block, Pool: a.pool}
}
for _, held := range a.snapshot() {
if block.Overlaps(held.Block.Masked()) {
return &LeaseCollisionError{Block: block, Held: held}
}
}
return nil
}
