~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
442 lines · 10.7 KB Raw History
package hostread

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	contract "github.com/johnandersen777/socialweb-computer-kcp/internal/contract/opsread"
)

const (
	SourceJournal = "journal"

	SourceFile = "file"

	cgroupRoot = "/sys/fs/cgroup"

	vmUnitPrefix = "socialweb-vm-"

	unitSuffix = ".service"

	RotatedSuffix = ".1"
)

var (
	started = time.Now()

	serviceUnit = regexp.MustCompile(`[^/]*\.service`)

	sessionScope = regexp.MustCompile(`session-[0-9]*\.scope`)

	userManager = regexp.MustCompile(`^.*/user@[0-9]+\.service`)
)

type Reader struct {
	Started time.Time

	Proc string

	Cgroup string

	Journalctl string

	Systemctl string

	LogFile string
}

var _ contract.Host = (*Reader)(nil)

func New() *Reader {
	return &Reader{Started: started, Proc: "/proc", Cgroup: cgroupRoot, Journalctl: "journalctl", Systemctl: "systemctl"}
}

func (r *Reader) ownCgroup() string {
	raw, err := os.ReadFile(filepath.Join(r.Proc, "self", "cgroup"))
	if err != nil {
		return ""
	}
	for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") {
		if path, ok := strings.CutPrefix(line, "0::"); ok {
			return path
		}
	}
	return ""
}

func (r *Reader) logFile() string {
	if path := StderrFile(r.Proc); path != "" {
		return path
	}
	if info, err := os.Stat(r.LogFile); r.LogFile != "" && err == nil && info.Mode().IsRegular() {
		return r.LogFile
	}
	return ""
}

func StderrFile(proc string) string {
	path, err := os.Readlink(filepath.Join(proc, "self", "fd", "2"))
	if err != nil {
		return ""
	}
	info, err := os.Stat(path)
	if err != nil || !info.Mode().IsRegular() {
		return ""
	}
	return path
}

func (r *Reader) restarts(ctx context.Context, unit string) (int, error) {
	raw, err := exec.CommandContext(ctx, r.Systemctl, "--user", "show", unit, "-p", "NRestarts", "--value").Output()
	if err != nil {
		return 0, fmt.Errorf("hostread: the restart count of %s: %w", unit, err)
	}
	n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
	if err != nil {
		return 0, fmt.Errorf("hostread: the restart count of %s: %w", unit, err)
	}
	return n, nil
}

func (r *Reader) unit() string {
	cgroup := r.ownCgroup()
	if sessionScope.MatchString(cgroup) {
		return ""
	}
	units := serviceUnit.FindAllString(cgroup, -1)
	if len(units) == 0 {
		return ""
	}
	return units[len(units)-1]
}

func lineTime(line []byte) (time.Time, bool) {
	var head struct {
		Time string `json:"time"`
	}
	if json.Unmarshal(line, &head) != nil || head.Time == "" {
		return time.Time{}, false
	}
	t, err := time.Parse(time.RFC3339Nano, head.Time)
	return t, err == nil
}

func each(in io.Reader, visit func(line []byte)) error {
	reader := bufio.NewReader(in)
	for {
		line, err := reader.ReadBytes('\n')
		if line = bytes.TrimRight(line, "\r\n"); len(line) > 0 {
			visit(line)
		}
		if errors.Is(err, io.EOF) {
			return nil
		}
		if err != nil {
			return err
		}
	}
}

func (r *Reader) journal(ctx context.Context, visit func(line []byte), args ...string) error {
	argv := append([]string{"_UID=" + strconv.Itoa(os.Getuid())}, args...)
	argv = append(argv, "-o", "cat", "--no-pager")
	cmd := exec.CommandContext(ctx, r.Journalctl, argv...)
	out, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("hostread: the journal: %w", err)
	}
	scanErr := each(out, visit)
	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("hostread: the journal: %w", err)
	}
	return scanErr
}

func (r *Reader) scan(ctx context.Context, from time.Time, until time.Time, visit func(line []byte)) (string, error) {
	if path := r.logFile(); path != "" {
		window := func(line []byte) {
			t, ok := lineTime(line)
			if !ok || t.Before(from.Truncate(time.Second)) || (!until.IsZero() && !t.Before(until)) {
				return
			}
			visit(line)
		}
		for _, name := range []string{path + RotatedSuffix, path} {
			f, err := os.Open(name)
			if errors.Is(err, os.ErrNotExist) && name != path {
				continue
			}
			if err != nil {
				return SourceFile, err
			}
			err = each(f, window)
			f.Close()
			if err != nil {
				return SourceFile, err
			}
		}
		return SourceFile, nil
	}
	args := []string{"--since", "@" + strconv.FormatInt(from.Unix(), 10)}
	if !until.IsZero() {
		args = append(args, "--until", "@"+strconv.FormatInt(until.Unix(), 10))
	}
	return SourceJournal, r.journal(ctx, visit, args...)
}

func (r *Reader) vmProcesses() int {
	entries, err := os.ReadDir(r.Proc)
	if err != nil {
		return 0
	}
	uid := strconv.Itoa(os.Getuid())
	count := 0
	for _, entry := range entries {
		if _, err := strconv.Atoi(entry.Name()); err != nil {
			continue
		}
		dir := filepath.Join(r.Proc, entry.Name())
		status, err := os.ReadFile(filepath.Join(dir, "status"))
		if err != nil {
			continue
		}
		owned := false
		for line := range strings.SplitSeq(string(status), "\n") {
			if fields := strings.Fields(line); len(fields) > 1 && fields[0] == "Uid:" {
				owned = fields[1] == uid
				break
			}
		}
		if !owned {
			continue
		}
		comm, _ := os.ReadFile(filepath.Join(dir, "comm"))
		cmdline, _ := os.ReadFile(filepath.Join(dir, "cmdline"))
		if strings.TrimSpace(string(comm)) == "firecracker" || bytes.Contains(cmdline, []byte("qemu-system")) {
			count++
		}
	}
	return count
}

func (r *Reader) Health(ctx context.Context) (contract.Health, error) {
	unit := r.unit()
	if StderrFile(r.Proc) != "" {
		unit = ""
	}
	out := contract.Health{Unit: unit, Started: r.Started.UTC().Format("2006-01-02T15:04:05Z"), ErrorSample: []contract.ErrorLine{}}
	isError := []byte(`"level":"ERROR"`)
	var sample [][]byte
	_, err := r.scan(ctx, r.Started, time.Time{}, func(line []byte) {
		if bytes.Contains(line, isError) {
			out.Errors++
			sample = append(sample, bytes.Clone(line))
			if len(sample) > contract.ErrorSampleSize {
				sample = sample[1:]
			}
		}
		if bytes.Contains(line, []byte(`"msg":"bidder_guest_image_unready"`)) {
			out.GuestImageUnready++
		}
		if out.GuestImageReadyAt == "" && bytes.Contains(line, []byte(`"msg":"bidder_guest_image_ready"`)) {
			var v struct {
				Time string `json:"time"`
			}
			if json.Unmarshal(line, &v) == nil {
				out.GuestImageReadyAt = v.Time
			}
		}
	})
	if err != nil {
		return out, err
	}
	if _, err := r.scan(ctx, r.Started, r.Started.Truncate(time.Second).Add(contract.StartupWindow), func(line []byte) {
		if bytes.Contains(line, isError) || bytes.Contains(line, []byte("unsupported type: source.TypedFunc")) {
			out.StartupErrors++
		}
	}); err != nil {
		return out, err
	}
	for _, line := range sample {
		var v map[string]any
		if json.Unmarshal(line, &v) != nil {
			continue
		}
		text := ""
		for _, key := range []string{"err", "error"} {
			if raw, ok := v[key]; ok && raw != nil && raw != "" {
				text = fmt.Sprint(raw)
				break
			}
		}
		if len(text) > contract.ErrorTextLimit {
			text = text[:contract.ErrorTextLimit]
		}
		entry := contract.ErrorLine{Err: text}
		entry.Time, _ = v["time"].(string)
		entry.Msg, _ = v["msg"].(string)
		out.ErrorSample = append(out.ErrorSample, entry)
	}
	if unit != "" {
		n, err := r.restarts(ctx, unit)
		if err != nil {
			return out, err
		}
		out.Restarts = n
	}
	out.VMProcesses = r.vmProcesses()
	return out, nil
}

func pick(line []byte, keys []string) (map[string]any, bool) {
	var v map[string]any
	if json.Unmarshal(line, &v) != nil {
		return nil, false
	}
	row := map[string]any{"time": v["time"], "msg": v["msg"]}
	for _, key := range keys {
		if value, ok := v[key]; ok {
			row[key] = value
		}
	}
	return row, true
}

func (r *Reader) Log(ctx context.Context, query contract.LogQuery) (contract.LogResult, error) {
	out := contract.LogResult{Lines: []map[string]any{}}
	parts, ok := contract.Messages[query.Message]
	if !ok || (query.Subject != "" && !contract.ValidSubject(query.Subject)) {
		return out, contract.ErrInvalid
	}
	if query.Subject != "" {
		parts = append(append([]string(nil), parts...), query.Subject)
	}
	source, err := r.scan(ctx, r.Started, time.Time{}, func(line []byte) {
		for _, part := range parts {
			if !bytes.Contains(line, []byte(part)) {
				return
			}
		}
		out.Count++
		if len(out.Lines) < contract.LogLineLimit {
			if row, ok := pick(line, contract.LineKeys); ok {
				out.Lines = append(out.Lines, row)
			}
		}
	})
	out.Log = source
	return out, err
}

func (r *Reader) RequesterLog(ctx context.Context, did string, since time.Time) ([]map[string]any, error) {
	out := []map[string]any{}
	if !contract.ValidDID(did) {
		return out, contract.ErrInvalid
	}
	needle := []byte(`"did":"` + did + `"`)
	_, err := r.scan(ctx, since, time.Time{}, func(line []byte) {
		if !bytes.Contains(line, needle) {
			return
		}
		t, ok := lineTime(line)
		if !ok || t.Before(since) {
			return
		}
		row, ok := pick(line, contract.RequesterLineKeys)
		if !ok {
			return
		}
		msg, _ := row["msg"].(string)
		for _, known := range contract.RequesterMessages {
			if msg == known {
				out = append(out, row)
				return
			}
		}
	})
	return out, err
}

func isVMUnit(unit, vm string) bool {
	rest, ok := strings.CutPrefix(unit, vmUnitPrefix)
	if !ok {
		return false
	}
	rest, ok = strings.CutSuffix(rest, unitSuffix)
	if !ok {
		return false
	}
	_, name, ok := strings.Cut(rest, "-")
	return ok && name == vm
}

func (r *Reader) vmUnitDir(vm string) (string, error) {
	base := userManager.FindString(r.ownCgroup())
	if base == "" {
		return "", fmt.Errorf("%w: this process runs in no user manager", contract.ErrNotFound)
	}
	root := filepath.Join(r.Cgroup, base)
	found := ""
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil || !d.IsDir() {
			return nil
		}
		if isVMUnit(d.Name(), vm) {
			found = path
			return filepath.SkipAll
		}
		return nil
	})
	if err != nil {
		return "", err
	}
	if found == "" {
		return "", fmt.Errorf("%w: no VM unit of %s in the bidder account's user manager", contract.ErrNotFound, vm)
	}
	return found, nil
}

func (r *Reader) VMMemory(_ context.Context, vm string) (contract.Memory, error) {
	out := contract.Memory{Events: map[string]int64{}}
	if !contract.ValidVMName(vm) {
		return out, contract.ErrInvalid
	}
	dir, err := r.vmUnitDir(vm)
	if err != nil {
		return out, err
	}
	read := func(name string) string {
		raw, _ := os.ReadFile(filepath.Join(dir, name))
		return strings.TrimSpace(string(raw))
	}
	current, err := strconv.ParseInt(read("memory.current"), 10, 64)
	if err != nil {
		return out, fmt.Errorf("%w: no readable memory cgroup for the unit", contract.ErrNotFound)
	}
	out.Unit, out.Current = filepath.Base(dir), current
	out.High, out.Max, out.SwapMax = read("memory.high"), read("memory.max"), read("memory.swap.max")
	for line := range strings.SplitSeq(read("memory.events"), "\n") {
		if fields := strings.Fields(line); len(fields) == 2 {
			if n, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
				out.Events[fields[0]] = n
			}
		}
	}
	return out, nil
}