0
0
package guestagent
import (
"bufio"
"context"
"encoding/json"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
memorySampleEvery = 10 * time.Second
memoryRearmAfter = 5 * time.Minute
memoryEventEvery = 10 * time.Minute
memoryEventsPerContract = 10
memoryKindPSI = "psi"
memoryKindSwap = "swap"
memoryOOMKey = "oomKill"
psiThreshold = 1000
swapThreshold = 1000
memoryStateFile = "memory-state.json"
psiMemoryPath = "/proc/pressure/memory"
meminfoPath = "/proc/meminfo"
vmstatPath = "/proc/vmstat"
)
type memoryPressureRecord struct {
Type string `json:"$type"`
Kind string `json:"kind"`
Threshold int64 `json:"threshold"`
PSISomeAvg10 *int64 `json:"psiSomeAvg10,omitempty"`
PSIFullAvg10 *int64 `json:"psiFullAvg10,omitempty"`
SwapUsed int64 `json:"swapUsed"`
SwapTotal int64 `json:"swapTotal"`
ObservedAt string `json:"observedAt"`
CreatedAt string `json:"createdAt"`
}
type oomKillRecord struct {
Type string `json:"$type"`
Kills int64 `json:"kills"`
Total int64 `json:"total"`
ObservedAt string `json:"observedAt"`
CreatedAt string `json:"createdAt"`
}
type memoryState struct {
Fired map[string]bool `json:"fired"`
Below map[string]time.Time `json:"below"`
Last map[string]time.Time `json:"last"`
Posted int `json:"posted"`
OOM int64 `json:"oom"`
}
func (a *Agent) runMemory(ctx context.Context) {
path := filepath.Join(a.cfg.StateDir, memoryStateFile)
st := memoryState{Fired: map[string]bool{}, Below: map[string]time.Time{}, Last: map[string]time.Time{}}
if raw, err := os.ReadFile(path); err == nil {
_ = json.Unmarshal(raw, &st)
}
if st.Fired == nil || st.Below == nil || st.Last == nil {
st.Fired, st.Below, st.Last = map[string]bool{}, map[string]time.Time{}, map[string]time.Time{}
}
_, psiErr := os.Stat(psiMemoryPath)
a.log.Info("the memory watch starts", "psi", psiErr == nil, "psiThreshold", psiThreshold, "swapThreshold", swapThreshold, "posted", st.Posted, "oomReported", st.OOM)
ticker := time.NewTicker(memorySampleEvery)
defer ticker.Stop()
for {
a.sampleMemory(ctx, &st, path)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (a *Agent) sampleMemory(ctx context.Context, st *memoryState, path string) {
now := time.Now()
allowed := func(key string) bool {
return st.Posted < memoryEventsPerContract && now.Sub(st.Last[key]) >= memoryEventEvery
}
posted := func(key string) {
st.Last[key] = now
st.Posted++
}
gauges := map[string][2]int64{}
some, full, psi := psiAvg10()
if psi {
gauges[memoryKindPSI] = [2]int64{some, psiThreshold}
}
used, total := swapBytes()
if total > 0 {
gauges[memoryKindSwap] = [2]int64{used * 10000 / total, swapThreshold}
}
for kind, g := range gauges {
value, threshold := g[0], g[1]
switch {
case !st.Fired[kind] && value >= threshold && allowed(kind):
record := memoryPressureRecord{Type: MemoryPressureNSID, Kind: kind, Threshold: threshold, SwapUsed: used, SwapTotal: total, ObservedAt: stamp(now), CreatedAt: createdAt()}
if psi {
record.PSISomeAvg10, record.PSIFullAvg10 = &some, &full
}
a.log.Warn("the guest crossed a memory threshold", "kind", kind, "observed", value, "threshold", threshold)
if err := a.poster.Post(ctx, record.Type, kind, record); err != nil && ctx.Err() != nil {
return
}
st.Fired[kind] = true
delete(st.Below, kind)
posted(kind)
case st.Fired[kind] && value < threshold/2:
if st.Below[kind].IsZero() {
st.Below[kind] = now
} else if now.Sub(st.Below[kind]) >= memoryRearmAfter {
a.log.Info("a memory threshold re-arms", "kind", kind, "observed", value, "threshold", threshold)
st.Fired[kind] = false
delete(st.Below, kind)
}
case st.Fired[kind]:
delete(st.Below, kind)
}
}
if total, ok := vmstat("oom_kill"); ok {
if total < st.OOM {
st.OOM = total
}
if total > st.OOM && allowed(memoryOOMKey) {
record := oomKillRecord{Type: OOMKillNSID, Kills: total - st.OOM, Total: total, ObservedAt: stamp(now), CreatedAt: createdAt()}
a.log.Warn("the guest kernel killed a process for memory", "kills", record.Kills, "total", total)
if err := a.poster.Post(ctx, record.Type, memoryOOMKey, record); err != nil && ctx.Err() != nil {
return
}
st.OOM = total
posted(memoryOOMKey)
}
}
if raw, err := json.Marshal(st); err == nil {
tmp := path + ".tmp"
if os.WriteFile(tmp, raw, 0o600) == nil {
_ = os.Rename(tmp, path)
}
}
}
func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000Z") }
func psiAvg10() (some, full int64, ok bool) {
raw, err := os.ReadFile(psiMemoryPath)
if err != nil {
return 0, 0, false
}
seen := 0
for _, line := range strings.Split(string(raw), "\n") {
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
v, found := strings.CutPrefix(fields[1], "avg10=")
f, err := strconv.ParseFloat(v, 64)
if !found || err != nil {
continue
}
switch fields[0] {
case "some":
some, seen = min(int64(f*100), 10000), seen+1
case "full":
full, seen = min(int64(f*100), 10000), seen+1
}
}
return some, full, seen == 2
}
func swapBytes() (used, total int64) {
t, okT := meminfo("SwapTotal")
f, okF := meminfo("SwapFree")
if !okT || !okF || t == 0 {
return 0, 0
}
return (t - f) * 1024, t * 1024
}
func meminfo(key string) (int64, bool) { return procValue(meminfoPath, key+":") }
func vmstat(key string) (int64, bool) { return procValue(vmstatPath, key) }
func procValue(path, key string) (int64, bool) {
f, err := os.Open(path)
if err != nil {
return 0, false
}
defer f.Close()
s := bufio.NewScanner(f)
for s.Scan() {
fields := strings.Fields(s.Text())
if len(fields) >= 2 && fields[0] == key {
v, err := strconv.ParseInt(fields[1], 10, 64)
return v, err == nil
}
}
return 0, false
}