~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
232 lines · 5.7 KB Raw History
package guestagent

import (
	"bufio"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const (
	memorySampleEvery = 10 * time.Second

	memoryRearmAfter = 5 * time.Minute

	memoryEventEvery = 10 * time.Minute

	memoryEventsPerContract = 10

	memoryKindPSI = "psi"

	memoryKindSwap = "swap"

	memoryOOMKey = "oomKill"

	psiThreshold = 1000

	swapThreshold = 1000

	memoryStateFile = "memory-state.json"

	psiMemoryPath = "/proc/pressure/memory"

	meminfoPath = "/proc/meminfo"

	vmstatPath = "/proc/vmstat"
)

type memoryPressureRecord struct {
	Type string `json:"$type"`

	Kind string `json:"kind"`

	Threshold int64 `json:"threshold"`

	PSISomeAvg10 *int64 `json:"psiSomeAvg10,omitempty"`

	PSIFullAvg10 *int64 `json:"psiFullAvg10,omitempty"`

	SwapUsed int64 `json:"swapUsed"`

	SwapTotal int64 `json:"swapTotal"`

	ObservedAt string `json:"observedAt"`

	CreatedAt string `json:"createdAt"`
}

type oomKillRecord struct {
	Type string `json:"$type"`

	Kills int64 `json:"kills"`

	Total int64 `json:"total"`

	ObservedAt string `json:"observedAt"`

	CreatedAt string `json:"createdAt"`
}

type memoryState struct {
	Fired map[string]bool `json:"fired"`

	Below map[string]time.Time `json:"below"`

	Last map[string]time.Time `json:"last"`

	Posted int `json:"posted"`

	OOM int64 `json:"oom"`
}

func (a *Agent) runMemory(ctx context.Context) {
	path := filepath.Join(a.cfg.StateDir, memoryStateFile)
	st := memoryState{Fired: map[string]bool{}, Below: map[string]time.Time{}, Last: map[string]time.Time{}}
	if raw, err := os.ReadFile(path); err == nil {
		_ = json.Unmarshal(raw, &st)
	}
	if st.Fired == nil || st.Below == nil || st.Last == nil {
		st.Fired, st.Below, st.Last = map[string]bool{}, map[string]time.Time{}, map[string]time.Time{}
	}
	_, psiErr := os.Stat(psiMemoryPath)
	a.log.Info("the memory watch starts", "psi", psiErr == nil, "psiThreshold", psiThreshold, "swapThreshold", swapThreshold, "posted", st.Posted, "oomReported", st.OOM)
	ticker := time.NewTicker(memorySampleEvery)
	defer ticker.Stop()
	for {
		a.sampleMemory(ctx, &st, path)
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
		}
	}
}

func (a *Agent) sampleMemory(ctx context.Context, st *memoryState, path string) {
	now := time.Now()
	allowed := func(key string) bool {
		return st.Posted < memoryEventsPerContract && now.Sub(st.Last[key]) >= memoryEventEvery
	}
	posted := func(key string) {
		st.Last[key] = now
		st.Posted++
	}
	gauges := map[string][2]int64{}
	some, full, psi := psiAvg10()
	if psi {
		gauges[memoryKindPSI] = [2]int64{some, psiThreshold}
	}
	used, total := swapBytes()
	if total > 0 {
		gauges[memoryKindSwap] = [2]int64{used * 10000 / total, swapThreshold}
	}
	for kind, g := range gauges {
		value, threshold := g[0], g[1]
		switch {
		case !st.Fired[kind] && value >= threshold && allowed(kind):
			record := memoryPressureRecord{Type: MemoryPressureNSID, Kind: kind, Threshold: threshold, SwapUsed: used, SwapTotal: total, ObservedAt: stamp(now), CreatedAt: createdAt()}
			if psi {
				record.PSISomeAvg10, record.PSIFullAvg10 = &some, &full
			}
			a.log.Warn("the guest crossed a memory threshold", "kind", kind, "observed", value, "threshold", threshold)
			if err := a.poster.Post(ctx, record.Type, kind, record); err != nil && ctx.Err() != nil {
				return
			}
			st.Fired[kind] = true
			delete(st.Below, kind)
			posted(kind)
		case st.Fired[kind] && value < threshold/2:
			if st.Below[kind].IsZero() {
				st.Below[kind] = now
			} else if now.Sub(st.Below[kind]) >= memoryRearmAfter {
				a.log.Info("a memory threshold re-arms", "kind", kind, "observed", value, "threshold", threshold)
				st.Fired[kind] = false
				delete(st.Below, kind)
			}
		case st.Fired[kind]:
			delete(st.Below, kind)
		}
	}
	if total, ok := vmstat("oom_kill"); ok {
		if total < st.OOM {
			st.OOM = total
		}
		if total > st.OOM && allowed(memoryOOMKey) {
			record := oomKillRecord{Type: OOMKillNSID, Kills: total - st.OOM, Total: total, ObservedAt: stamp(now), CreatedAt: createdAt()}
			a.log.Warn("the guest kernel killed a process for memory", "kills", record.Kills, "total", total)
			if err := a.poster.Post(ctx, record.Type, memoryOOMKey, record); err != nil && ctx.Err() != nil {
				return
			}
			st.OOM = total
			posted(memoryOOMKey)
		}
	}
	if raw, err := json.Marshal(st); err == nil {
		tmp := path + ".tmp"
		if os.WriteFile(tmp, raw, 0o600) == nil {
			_ = os.Rename(tmp, path)
		}
	}
}

func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000Z") }

func psiAvg10() (some, full int64, ok bool) {
	raw, err := os.ReadFile(psiMemoryPath)
	if err != nil {
		return 0, 0, false
	}
	seen := 0
	for _, line := range strings.Split(string(raw), "\n") {
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		v, found := strings.CutPrefix(fields[1], "avg10=")
		f, err := strconv.ParseFloat(v, 64)
		if !found || err != nil {
			continue
		}
		switch fields[0] {
		case "some":
			some, seen = min(int64(f*100), 10000), seen+1
		case "full":
			full, seen = min(int64(f*100), 10000), seen+1
		}
	}
	return some, full, seen == 2
}

func swapBytes() (used, total int64) {
	t, okT := meminfo("SwapTotal")
	f, okF := meminfo("SwapFree")
	if !okT || !okF || t == 0 {
		return 0, 0
	}
	return (t - f) * 1024, t * 1024
}

func meminfo(key string) (int64, bool) { return procValue(meminfoPath, key+":") }

func vmstat(key string) (int64, bool) { return procValue(vmstatPath, key) }

func procValue(path, key string) (int64, bool) {
	f, err := os.Open(path)
	if err != nil {
		return 0, false
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	for s.Scan() {
		fields := strings.Fields(s.Text())
		if len(fields) >= 2 && fields[0] == key {
			v, err := strconv.ParseInt(fields[1], 10, 64)
			return v, err == nil
		}
	}
	return 0, false
}