1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
//go:build linux
package apiserverfront
import (
"errors"
"fmt"
"net"
"os"
"strconv"
"strings"
"golang.org/x/sys/unix"
)
func ListenerFromFD(fd int) (net.Listener, error) {
unix.CloseOnExec(fd)
f := os.NewFile(uintptr(fd), FDName)
if f == nil {
return nil, fmt.Errorf("apiserverfront: fd %d is not open", fd)
}
defer f.Close()
ln, err := net.FileListener(f)
if err != nil {
return nil, fmt.Errorf("apiserverfront: the inherited fd %d is no listening socket: %w", fd, err)
}
if _, ok := ln.(*net.TCPListener); !ok {
ln.Close()
return nil, fmt.Errorf("apiserverfront: the inherited fd %d is not a TCP listener", fd)
}
return ln, nil
}
func HeldFromFD(fd int, name string) (*Held, error) {
unix.CloseOnExec(fd)
port, err := boundPort(fd)
if err != nil {
unix.Close(fd)
return nil, fmt.Errorf("apiserverfront: the inherited hold %q (fd %d): %w", name, fd, err)
}
return &Held{Name: name, Port: port, f: os.NewFile(uintptr(fd), HoldFDPrefix+name)}, nil
}
func boundPort(fd int) (int, error) {
sa, err := unix.Getsockname(fd)
if err != nil {
return 0, err
}
switch a := sa.(type) {
case *unix.SockaddrInet6:
return a.Port, nil
case *unix.SockaddrInet4:
return a.Port, nil
}
return 0, errors.New("not an inet socket")
}
func bindHold(name string, port int) (*Held, error) {
h, err := bindHoldOnce(name, port, false)
if err == nil || port == 0 || !errors.Is(err, unix.EADDRINUSE) || !onlyTimeWait(port) {
return h, err
}
return bindHoldOnce(name, port, true)
}
func onlyTimeWait(port int) bool {
waiting, other := 0, 0
for _, table := range []string{"/proc/net/tcp", "/proc/net/tcp6"} {
raw, err := os.ReadFile(table)
if err != nil {
continue
}
lines := strings.Split(string(raw), "\n")
for _, line := range lines[min(1, len(lines)):] {
f := strings.Fields(line)
if len(f) < 4 {
continue
}
i := strings.LastIndexByte(f[1], ':')
if i < 0 {
continue
}
local, err := strconv.ParseUint(f[1][i+1:], 16, 16)
if err != nil || int(local) != port {
continue
}
if f[3] == tcpTimeWait {
waiting++
} else {
other++
}
}
}
return waiting > 0 && other == 0
}
const tcpTimeWait = "06"
func bindHoldOnce(name string, port int, reuseFirst bool) (*Held, error) {
fd, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
var sa unix.Sockaddr = &unix.SockaddrInet6{Port: port}
if err == nil {
if err = unix.SetsockoptInt(fd, unix.IPPROTO_IPV6, unix.IPV6_V6ONLY, 0); err != nil {
unix.Close(fd)
return nil, fmt.Errorf("apiserverfront: hold %q: a dual-stack socket: %w", name, err)
}
} else {
if fd, err = unix.Socket(unix.AF_INET, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0); err != nil {
return nil, fmt.Errorf("apiserverfront: hold %q: %w", name, err)
}
sa = &unix.SockaddrInet4{Port: port}
}
if reuseFirst {
if err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1); err != nil {
unix.Close(fd)
return nil, fmt.Errorf("apiserverfront: hold %q: SO_REUSEADDR before bind over TIME_WAIT: %w", name, err)
}
}
if err := unix.Bind(fd, sa); err != nil {
unix.Close(fd)
return nil, fmt.Errorf("apiserverfront: hold %q on port %d: %w", name, port, err)
}
if err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1); err != nil {
unix.Close(fd)
return nil, fmt.Errorf("apiserverfront: hold %q: SO_REUSEADDR after bind: %w", name, err)
}
got, err := boundPort(fd)
if err != nil {
unix.Close(fd)
return nil, fmt.Errorf("apiserverfront: hold %q: %w", name, err)
}
return &Held{Name: name, Port: got, f: os.NewFile(uintptr(fd), HoldFDPrefix+name)}, nil
}
func storeListener(sock string, ln net.Listener) error {
tl, ok := ln.(*net.TCPListener)
if !ok {
return errors.New("apiserverfront: only a TCP listener goes into the fd store")
}
raw, err := tl.SyscallConn()
if err != nil {
return err
}
var werr error
err = raw.Control(func(fd uintptr) { werr = storeFD(sock, int(fd), "FDNAME="+FDName) })
if err != nil {
return err
}
return werr
}
func storeHold(sock string, h *Held) error {
raw, err := h.f.SyscallConn()
if err != nil {
return err
}
var werr error
err = raw.Control(func(fd uintptr) { werr = storeFD(sock, int(fd), "FDNAME="+HoldFDPrefix+h.Name+"\nFDPOLL=0") })
if err != nil {
return err
}
return werr
}
func storeFD(sock string, fd int, fields string) error {
s, err := unix.Socket(unix.AF_UNIX, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return fmt.Errorf("apiserverfront: a socket for the fd store at %s: %w", sock, err)
}
defer unix.Close(s)
if err := unix.Sendmsg(s, []byte("FDSTORE=1\n"+fields), unix.UnixRights(fd), &unix.SockaddrUnix{Name: sock}, 0); err != nil {
return fmt.Errorf("apiserverfront: handing %q to the fd store at %s: %w", fields, sock, err)
}
return nil
}
