1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
package nodeagent
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"sort"
"strings"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/clock"
)
type crictlCondition struct {
Type string `json:"type"`
Status any `json:"status"`
Reason string `json:"reason"`
Message string `json:"message"`
}
type crictlInfo struct {
Status struct {
Conditions []crictlCondition `json:"conditions"`
} `json:"status"`
LastCNILoadStatus any `json:"lastCNILoadStatus"`
Config struct {
CNI struct {
ConfDir string `json:"confDir"`
BinDirs []string `json:"binDirs"`
BinDir string `json:"binDir"`
} `json:"cni"`
} `json:"config"`
}
func Diagnose(ctx context.Context, cfg Config, log *slog.Logger) error {
if log == nil {
log = slog.Default()
}
for attempt := 1; attempt <= cfg.DiagnoseAttempts; attempt++ {
out, err := run(ctx, cfg.K3sPath, "crictl", "info")
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
log.Warn("crictl info failed", "attempt", attempt, "of", cfg.DiagnoseAttempts, "err", err.Error())
if attempt < cfg.DiagnoseAttempts {
if err := clock.SleepErr(ctx, cfg.DiagnoseInterval()); err != nil {
return err
}
}
continue
}
var info crictlInfo
if err := json.Unmarshal(out, &info); err != nil {
log.Warn("crictl info did not return JSON", "attempt", attempt, "of", cfg.DiagnoseAttempts, "err", err.Error())
if attempt < cfg.DiagnoseAttempts {
if err := clock.SleepErr(ctx, cfg.DiagnoseInterval()); err != nil {
return err
}
}
continue
}
ready := "unknown"
reason := ""
for _, condition := range info.Status.Conditions {
if condition.Type != "NetworkReady" {
continue
}
ready = fmt.Sprint(condition.Status)
reason = strings.TrimSpace(condition.Reason + " " + condition.Message)
break
}
if ready == "true" {
log.Info("NetworkReady=true")
return nil
}
binDirs := info.Config.CNI.BinDirs
if len(binDirs) == 0 && info.Config.CNI.BinDir != "" {
binDirs = []string{info.Config.CNI.BinDir}
}
confDir := info.Config.CNI.ConfDir
log.Info("the node network is not ready", "NetworkReady", ready, "reason", reason,
"lastCNILoadStatus", fmt.Sprint(info.LastCNILoadStatus), "confDir", confDir,
"confDirHolds", dirNames(confDir), "binDirs", strings.Join(binDirs, ","),
"binDirsHold", binDirNames(binDirs), "attempt", attempt, "of", cfg.DiagnoseAttempts)
if attempt < cfg.DiagnoseAttempts {
if err := clock.SleepErr(ctx, cfg.DiagnoseInterval()); err != nil {
return err
}
}
}
log.Info("the node network was still not ready", "readings", cfg.DiagnoseAttempts)
return nil
}
func dirNames(dir string) string {
if dir == "" {
return ""
}
entries, err := os.ReadDir(dir)
if err != nil {
return err.Error()
}
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Name())
}
sort.Strings(names)
return strings.Join(names, " ")
}
func binDirNames(dirs []string) string {
out := make([]string, 0, len(dirs))
for _, dir := range dirs {
out = append(out, dir+"="+dirNames(dir))
}
return strings.Join(out, " ")
}
