0
0
//go:build linux
package vmruntime
import (
"bytes"
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"os/user"
"path/filepath"
"slices"
"strconv"
"strings"
"syscall"
"time"
contracthost "github.com/johnandersen777/socialweb-computer-kcp/internal/contract/host"
"github.com/johnandersen777/socialweb-computer-kcp/internal/fcapi"
"github.com/johnandersen777/socialweb-computer-kcp/internal/guestdisk"
"github.com/johnandersen777/socialweb-computer-kcp/internal/guestimage"
"github.com/johnandersen777/socialweb-computer-kcp/internal/guestnet"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostcontract"
"github.com/johnandersen777/socialweb-computer-kcp/internal/hostsvc"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/fswatch"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/fsx"
"golang.org/x/sys/unix"
)
type hostRuntime struct {
cfg Config
subuid hostcontract.SubidEntry
subgid hostcontract.SubidEntry
slots int
target string
}
func ForHost(cfg Config) (Runtime, error) {
if cfg.Manager == nil {
return nil, fmt.Errorf("vmruntime: no placement manager")
}
if !filepath.IsAbs(cfg.StateDir) || !filepath.IsAbs(cfg.RuntimeDir) {
return nil, fmt.Errorf("vmruntime: state dir %q and runtime dir %q must be absolute", cfg.StateDir, cfg.RuntimeDir)
}
vmm, err := ParseVMM(cfg.VMM)
if err != nil {
return nil, err
}
cfg.VMM = vmm
for name, p := range map[string]string{"swc-userns-stage": cfg.Exec.Stage, "pasta": cfg.Exec.Pasta,
"socialweb-vmexec": cfg.Exec.VMExec, VMMExecutable(vmm): cfg.Exec.VMM} {
if !filepath.IsAbs(p) {
return nil, fmt.Errorf("vmruntime: %s path %q is not absolute", name, p)
}
}
cfg.InnerID = cmp.Or(cfg.InnerID, DefaultInnerID)
if cfg.InnerID <= 0 || cfg.InnerID >= hostcontract.VMSlotIDs {
return nil, fmt.Errorf("vmruntime: inner id %d must be inside 1..%d: SELinux refuses pasta as uid 0 of the namespace", cfg.InnerID, hostcontract.VMSlotIDs-1)
}
cfg.BootTimeout = cmp.Or(cfg.BootTimeout, DefaultBootTime)
cfg.StopGrace = cmp.Or(cfg.StopGrace, DefaultStopGrace)
cfg.ConsoleCap = cmp.Or(cfg.ConsoleCap, DefaultConsoleCap)
target := ImageTarget()
if target == "" {
return nil, ErrUnsupported
}
me, err := user.Current()
if err != nil {
return nil, fmt.Errorf("vmruntime: %w", err)
}
uid := os.Getuid()
subuidText, err := os.ReadFile("/etc/subuid")
if err != nil {
return nil, fmt.Errorf("vmruntime: %w", err)
}
subgidText, err := os.ReadFile("/etc/subgid")
if err != nil {
return nil, fmt.Errorf("vmruntime: %w", err)
}
su, okU := hostcontract.AccountRange(string(subuidText), me.Username, uid)
sg, okG := hostcontract.AccountRange(string(subgidText), me.Username, uid)
if !okU || !okG {
return nil, fmt.Errorf("vmruntime: %s has no subordinate id range in /etc/subuid and /etc/subgid; each VM runs in a user namespace mapped to a %d-id slot of it (socialweb-computer install bidder reserves %s:<base>:%d)",
me.Username, hostcontract.VMSlotIDs, me.Username, 256*hostcontract.VMSlotIDs)
}
slots := int(min(su.Count, sg.Count) / hostcontract.VMSlotIDs)
if slots < 1 {
return nil, fmt.Errorf("vmruntime: the subordinate range of %s holds %d ids, under one %d-id slot", me.Username, min(su.Count, sg.Count), hostcontract.VMSlotIDs)
}
return &hostRuntime{cfg: cfg, subuid: su, subgid: sg, slots: slots, target: target}, nil
}
func (r *hostRuntime) Capabilities() Capabilities {
c := r.cfg.Manager.Capabilities()
return Capabilities{Backend: c.Backend, Hypervisor: r.cfg.VMM, EnforcesLimits: c.EnforcesLimits, GracefulStop: GracefulStop(),
NestedVirt: r.cfg.VMM == VMMQEMU && HostNestedVirt(), ImageTarget: r.target}
}
func (r *hostRuntime) imagePin() (guestimage.Pin, error) { return guestimage.PinFor(r.target) }
func (r *hostRuntime) imageStore() guestimage.Store {
return guestimage.Store{Root: filepath.Join(r.cfg.StateDir, "images"), Out: r.cfg.ImageOut, TTY: r.cfg.ImageTTY,
Budget: cmp.Or(r.cfg.RequestedImageBudget, guestimage.DefaultBudget), MinFree: guestimage.DefaultMinFree}
}
func (r *hostRuntime) EnsureImage(ctx context.Context) error {
pin, err := r.imagePin()
if err != nil {
return err
}
_, err = r.imageStore().Ensure(ctx, pin, r.cfg.ImageSource)
return err
}
func (r *hostRuntime) Prepare(ctx context.Context, spec VMSpec) (Prepared, error) {
if err := ValidateSpec(spec); err != nil {
return Prepared{}, err
}
pin, err := r.imagePin()
if err != nil {
return Prepared{}, err
}
store := r.imageStore()
var img guestimage.Result
if spec.Image != nil {
img, err = store.EnsureRequested(ctx, *spec.Image, pin, int64(spec.DiskMiB)<<20)
} else {
img, err = store.Ensure(ctx, pin, r.cfg.ImageSource)
}
if err != nil {
return Prepared{}, err
}
manifest, err := readManifest(img.Dir)
if err != nil {
return Prepared{}, err
}
p := Prepared{
Spec: spec,
Unit: UnitName(spec.Handle, spec.Name),
WorkDir: WorkDir(r.cfg.StateDir, spec.Handle, spec.Name),
SocketDir: SocketDir(r.cfg.RuntimeDir, spec.Handle, spec.Name),
VMMID: VMMID(spec.Handle, spec.Name),
ImageDir: img.Dir,
ImageFetched: img.Fetched,
Vcpu: manifest.Vcpu,
MemMiB: manifest.MemMiB,
}
p.Socket = filepath.Join(p.SocketDir, SocketFile)
if len(p.Socket) >= 108 {
return Prepared{}, fmt.Errorf("vmruntime: the API socket %s is %d bytes, over the 107-byte unix socket limit", p.Socket, len(p.Socket))
}
if spec.Vcpu > 0 {
p.Vcpu = spec.Vcpu
}
if spec.MemMiB > 0 {
p.MemMiB = spec.MemMiB
}
if _, err := os.Stat(p.WorkDir); err == nil {
return Prepared{}, fmt.Errorf("vmruntime: %s already has a work directory %s; clean it up first", spec.Name, p.WorkDir)
}
if err := os.MkdirAll(filepath.Dir(p.WorkDir), 0o700); err != nil {
return Prepared{}, fmt.Errorf("vmruntime: %w", err)
}
if err := os.Mkdir(p.WorkDir, 0o700); err != nil {
return Prepared{}, fmt.Errorf("vmruntime: %w", err)
}
ok := false
defer func() {
if !ok {
os.RemoveAll(p.WorkDir)
}
}()
rootfs := filepath.Join(p.WorkDir, RootfsFile)
if err := guestdisk.Clone(filepath.Join(img.Dir, guestimage.RootfsName), rootfs); err != nil {
return Prepared{}, err
}
if err := guestdisk.GrowTo(rootfs, spec.DiskMiB); err != nil {
return Prepared{}, err
}
meta := spec.MetaData
meta = cmp.Or(meta, "instance-id: "+spec.Name+"\nlocal-hostname: "+spec.Name+"\n")
seed := filepath.Join(p.WorkDir, SeedFile)
if err := guestdisk.WriteSeed(seed, []byte(spec.UserData), []byte(meta), []byte(guestnet.NetworkConfig())); err != nil {
return Prepared{}, err
}
conf := fcapi.Config{
BootSource: fcapi.BootSource{KernelImagePath: filepath.Join(img.Dir, guestimage.KernelName), BootArgs: strings.Join(append(slices.Clone(manifest.BootArgs), devBootArgsExtra()...), " ")},
Drives: []fcapi.Drive{
{DriveID: "rootfs", PathOnHost: rootfs, IsRootDevice: true},
{DriveID: "cidata", PathOnHost: seed, IsReadOnly: true},
},
MachineConfig: fcapi.MachineConfig{VcpuCount: p.Vcpu, MemSizeMiB: p.MemMiB},
NetworkInterfaces: []fcapi.NetworkInterface{{IfaceID: guestnet.Interface, HostDevName: guestnet.Tap, GuestMAC: guestnet.GuestMAC}},
}
if err := fcapi.WriteConfig(filepath.Join(p.WorkDir, ConfigFile), conf); err != nil {
return Prepared{}, fmt.Errorf("vmruntime: %w", err)
}
console := filepath.Join(p.WorkDir, ConsoleFile)
wiring, hostAddrs, err := planFor(spec)
if err != nil {
return Prepared{}, err
}
if spec.WebRTCUDPPort != 0 {
p.WebRTCAddresses = guestnet.LANAddresses(hostAddrs)
}
params := guestnet.VMExec{
Wiring: wiring,
VMM: r.cfg.Exec.VMM,
Argv: fcapi.Argv("firecracker", p.VMMID, filepath.Join(p.WorkDir, ConfigFile), p.Socket),
Console: console,
}
if r.cfg.VMM == VMMQEMU {
if params.Argv, err = qemuArgv(r.cfg.Exec.VMM, p.VMMID, conf, p.Socket); err != nil {
return Prepared{}, err
}
params.Env = qemuEnv(r.cfg.Exec.VMM)
}
if err := fsx.WriteJSON(filepath.Join(p.WorkDir, ParamsFile), params, 0o600); err != nil {
return Prepared{}, err
}
for _, f := range []string{console, filepath.Join(p.WorkDir, VMMLogFile)} {
if err := os.WriteFile(f, nil, 0o600); err != nil {
return Prepared{}, fmt.Errorf("vmruntime: %w", err)
}
}
ok = true
return p, nil
}
func planFor(spec VMSpec) (guestnet.Wiring, []string, error) {
hostAddrs, err := guestnet.HostAddresses()
if err != nil {
return guestnet.Wiring{}, nil, fmt.Errorf("vmruntime: %w", err)
}
return guestnet.Plan(spec.HostPorts, spec.LANUDP, spec.HostUDP, hostAddrs, spec.WebRTCUDPPort, spec.WebRTCUDPPorts, spec.NetworkTCP), hostAddrs, nil
}
func (r *hostRuntime) Rewire(ctx context.Context, ref RuntimeRef, spec VMSpec) (guestnet.Wiring, error) {
w, _, err := planFor(spec)
if err != nil {
return w, err
}
sock, err := r.guestNetfilter(ctx, ref)
if err != nil {
return w, err
}
defer unix.Close(sock)
return w, guestnet.Refence(sock, w)
}
func (r *hostRuntime) guestNetfilter(ctx context.Context, ref RuntimeRef) (int, error) {
obs, err := r.cfg.Manager.Check(ctx, identity(ref))
if err != nil {
return -1, err
}
if obs.Verdict != hostsvc.VerdictRunning {
return -1, fmt.Errorf("vmruntime: %s is not the recorded running VM: %s %s %s", ref.Unit, obs.Verdict, obs.Reason, obs.Detail)
}
pidfd, err := hostsvc.PinProcess(ref.PID, ref.StartTicks)
if err != nil {
return -1, fmt.Errorf("vmruntime: %s: the recorded VMM pid %d: %w", ref.Unit, ref.PID, err)
}
defer pidfd.Close()
pair, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0)
if err != nil {
return -1, fmt.Errorf("vmruntime: socketpair: %w", err)
}
defer unix.Close(pair[0])
theirs := os.NewFile(uintptr(pair[1]), "swc-userns-stage join")
var stderr bytes.Buffer
cmd := exec.CommandContext(ctx, r.cfg.Exec.Stage, "join")
cmd.ExtraFiles = []*os.File{pidfd, theirs}
cmd.Stdout, cmd.Stderr = &stderr, &stderr
err = cmd.Run()
theirs.Close()
if err != nil {
return -1, fmt.Errorf("vmruntime: %s join (pid %d of %s): %w: %s", r.cfg.Exec.Stage, ref.PID, ref.Unit, err, strings.TrimSpace(stderr.String()))
}
oob := make([]byte, unix.CmsgSpace(4))
_, oobn, _, _, err := unix.Recvmsg(pair[0], make([]byte, 1), oob, unix.MSG_DONTWAIT|unix.MSG_CMSG_CLOEXEC)
if err != nil {
return -1, fmt.Errorf("vmruntime: reading the netfilter socket of %s: %w", ref.Unit, err)
}
msgs, err := unix.ParseSocketControlMessage(oob[:oobn])
if err != nil || len(msgs) != 1 {
return -1, fmt.Errorf("vmruntime: %s join passed %d control messages (%v), not one socket", r.cfg.Exec.Stage, len(msgs), err)
}
fds, err := unix.ParseUnixRights(&msgs[0])
if err != nil || len(fds) != 1 {
for _, fd := range fds {
unix.Close(fd)
}
return -1, fmt.Errorf("vmruntime: %s join passed %d descriptors (%v), not one socket", r.cfg.Exec.Stage, len(fds), err)
}
return fds[0], nil
}
func (r *hostRuntime) slotIDs(k int) (uidBase, gidBase, vmmUID int) {
uidBase = int(r.subuid.Start) + k*hostcontract.VMSlotIDs
gidBase = int(r.subgid.Start) + k*hostcontract.VMSlotIDs
return uidBase, gidBase, uidBase + r.cfg.InnerID
}
func (r *hostRuntime) vmmOf(workDir string) string {
var params guestnet.VMExec
raw, err := os.ReadFile(filepath.Join(workDir, ParamsFile))
if err != nil || json.Unmarshal(raw, ¶ms) != nil || params.VMM == "" {
return r.cfg.VMM
}
if filepath.Base(params.VMM) == "firecracker" {
return VMMFirecracker
}
return VMMQEMU
}
func (r *hostRuntime) refVMM(ref RuntimeRef) string {
if ref.Hypervisor != "" {
return ref.Hypervisor
}
return r.vmmOf(cmp.Or(ref.WorkDir, WorkDir(r.cfg.StateDir, ref.Handle, ref.Name)))
}
func (r *hostRuntime) UnitSpec(p Prepared, slot int) hostsvc.UnitSpec {
uidBase, gidBase, _ := r.slotIDs(slot)
memoryMax, memoryHigh, swapMax := contracthost.VMLimits(p.MemMiB)
tasks := uint64(64 + 4*p.Vcpu)
if r.vmmOf(p.WorkDir) == VMMQEMU {
tasks += QEMUExtraTasks
}
exec := []string{r.cfg.Exec.Stage, strconv.Itoa(uidBase), strconv.Itoa(gidBase), strconv.Itoa(hostcontract.VMSlotIDs), strconv.Itoa(r.cfg.InnerID)}
exec = append(exec, guestnet.PastaArgv(r.cfg.Exec.Pasta, guestnet.PastaUDPForwards(p.Spec.WebRTCUDPPort, p.WebRTCAddresses), r.cfg.Exec.VMExec, filepath.Join(p.WorkDir, ParamsFile))...)
return hostsvc.UnitSpec{
Name: p.Unit,
Description: fmt.Sprintf("socialweb vm %s handle %s slot=%d", p.Spec.Name, HandleID(p.Spec.Handle), slot),
Exec: exec,
Env: []string{"TZ=UTC"},
WorkDir: p.WorkDir,
Output: hostsvc.Output{StdoutFile: filepath.Join(p.WorkDir, VMMLogFile), StderrFile: filepath.Join(p.WorkDir, VMMLogFile)},
Limits: hostsvc.Limits{
MemoryMax: memoryMax,
MemoryHigh: memoryHigh,
MemorySwapMax: swapMax,
CPUWeight: 100,
CPUQuotaPercent: 100 * p.Vcpu,
TasksMax: tasks,
},
OOMPolicy: "stop",
KillSignal: syscall.SIGKILL,
StopTimeout: StopTimeout,
Slice: SliceName(p.Spec.Handle),
Restart: hostsvc.RestartNo,
}
}
func (r *hostRuntime) Start(ctx context.Context, p Prepared) (RuntimeRef, error) {
slot, err := r.Reserve(ctx, p, func(ctx context.Context, _ int, spec hostsvc.UnitSpec) error {
_, err := r.cfg.Manager.Start(ctx, spec)
return err
})
ref := RuntimeRef{}
if err == nil {
ref, err = r.Await(ctx, p, slot)
}
if err != nil {
stopCtx := context.WithoutCancel(ctx)
_ = r.cfg.Manager.Stop(stopCtx, p.Unit)
_ = r.cfg.Manager.ResetFailed(stopCtx, p.Unit)
tail := tailOf(filepath.Join(p.WorkDir, VMMLogFile), 3000) + tailOf(filepath.Join(p.WorkDir, ConsoleFile), 3000)
r.Cleanup(stopCtx, p.Spec.Handle, p.Spec.Name)
return RuntimeRef{}, fmt.Errorf("%w\n%s", err, tail)
}
return ref, nil
}
func (r *hostRuntime) Placed(handle, name string) Prepared {
p := Prepared{
Spec: VMSpec{Handle: handle, Name: name},
Unit: UnitName(handle, name),
WorkDir: WorkDir(r.cfg.StateDir, handle, name),
SocketDir: SocketDir(r.cfg.RuntimeDir, handle, name),
VMMID: VMMID(handle, name),
}
p.Socket = filepath.Join(p.SocketDir, SocketFile)
return p
}
func (r *hostRuntime) StopGrace() time.Duration { return r.cfg.StopGrace }
func (r *hostRuntime) awaitDown(ctx context.Context, unit string, grace time.Duration) (bool, error) {
gctx, cancel := context.WithTimeout(ctx, grace)
defer cancel()
_, err := hostsvc.Await(gctx, r.cfg.Manager, unit, func(st hostsvc.UnitStatus) bool { return !hostsvc.Live(st) })
switch {
case err == nil:
return true, nil
case ctx.Err() != nil:
return false, ctx.Err()
case gctx.Err() != nil:
return false, nil
}
return false, err
}
func (r *hostRuntime) Shutdown(ctx context.Context, ref RuntimeRef, grace time.Duration) (bool, error) {
st, err := r.cfg.Manager.Inspect(ctx, ref.Unit)
if err != nil {
return false, err
}
if !hostsvc.Live(st) {
return true, nil
}
if !GracefulStop() || ref.Socket == "" {
return false, nil
}
if err := apiFor(r.refVMM(ref), ref.Socket).SendCtrlAltDel(ctx); err != nil {
return false, nil
}
return r.awaitDown(ctx, ref.Unit, grace)
}
func (r *hostRuntime) Reserve(ctx context.Context, p Prepared, place Placer) (int, error) {
unlock, err := r.lock()
if err != nil {
return 0, err
}
defer unlock()
used, err := r.usedSlots(ctx)
if err != nil {
return 0, err
}
slot := -1
for k := 0; k < r.slots; k++ {
if !used[k] {
slot = k
break
}
}
if slot < 0 {
return 0, fmt.Errorf("vmruntime: all %d VM slots of this account are held by loaded VM units", r.slots)
}
_, _, vmmUID := r.slotIDs(slot)
if err := r.grant(p, vmmUID); err != nil {
return 0, err
}
if err := place(ctx, slot, r.UnitSpec(p, slot)); err != nil {
return 0, err
}
return slot, nil
}
func (r *hostRuntime) grant(p Prepared, vmmUID int) error {
if err := os.MkdirAll(filepath.Dir(p.SocketDir), 0o700); err != nil {
return fmt.Errorf("vmruntime: %w", err)
}
if err := os.Mkdir(p.SocketDir, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
return fmt.Errorf("vmruntime: %w", err)
}
g := newGranter(vmmUID)
files := []struct {
path string
perm uint16
}{
{filepath.Join(p.WorkDir, RootfsFile), 6},
{filepath.Join(p.WorkDir, SeedFile), 4},
{filepath.Join(p.WorkDir, ConfigFile), 4},
{filepath.Join(p.WorkDir, ParamsFile), 4},
{filepath.Join(p.WorkDir, ConsoleFile), 6},
{p.SocketDir, 7},
}
for _, f := range files {
if err := g.need(f.path, f.perm, false); err != nil {
return err
}
}
if err := g.need(p.WorkDir, 1, false); err != nil {
return err
}
if err := g.traverse(p.WorkDir, ""); err != nil {
return err
}
if err := g.traverse(p.SocketDir, ""); err != nil {
return err
}
for _, f := range []string{filepath.Join(p.ImageDir, guestimage.KernelName), r.cfg.Exec.Pasta, r.cfg.Exec.VMExec, r.cfg.Exec.VMM} {
if err := g.traverse(f, ""); err != nil {
return err
}
}
for _, f := range []string{r.cfg.Exec.Pasta, r.cfg.Exec.VMExec, r.cfg.Exec.VMM} {
if err := g.need(f, 5, true); err != nil {
return err
}
}
if r.vmmOf(p.WorkDir) == VMMQEMU {
if err := g.tree(qemuDataDir(r.cfg.Exec.VMM)); err != nil {
return err
}
if lib := qemuLibDir(r.cfg.Exec.VMM); lib != "" {
if err := g.tree(lib); err != nil {
return err
}
}
}
if err := g.need(filepath.Join(p.ImageDir, guestimage.KernelName), 4, true); err != nil {
return err
}
var outside []Grant
for _, gr := range g.grants {
if !strings.HasPrefix(gr.Path, p.WorkDir+string(filepath.Separator)) && gr.Path != p.WorkDir {
outside = append(outside, gr)
}
}
return fsx.WriteJSON(filepath.Join(p.WorkDir, GrantsFile), outside, 0o600)
}
const (
DialRetryMin = 10 * time.Millisecond
DialRetryMax = time.Second
)
func (r *hostRuntime) awaitBoot(ctx context.Context, p Prepared) error {
bctx, cancel := context.WithTimeout(ctx, r.cfg.BootTimeout)
defer cancel()
sock, err := fswatch.Shared().Subscribe(p.SocketDir)
if err != nil {
return fmt.Errorf("vmruntime: watching %s: %w", p.SocketDir, err)
}
defer sock.Close()
units, _ := r.cfg.Manager.Watch(bctx, p.Unit)
exited := func(st hostsvc.UnitStatus) error {
if st.State != hostsvc.Active && st.State != hostsvc.Activating {
return fmt.Errorf("vmruntime: %s went %s (result %s, status %d) before its VMM answered", p.Unit, st.State, st.Result, st.ExecMainStatus)
}
return nil
}
if st, err := r.cfg.Manager.Inspect(bctx, p.Unit); err == nil {
if err := exited(st); err != nil {
return err
}
}
client := apiFor(r.vmmOf(p.WorkDir), p.Socket)
last := fmt.Errorf("the API socket %s is not there", p.Socket)
delay := DialRetryMin
var again <-chan time.Time
check := true
for {
if check && fsx.Exists(p.Socket) {
in, err := client.Describe(bctx)
switch {
case err == nil && in.ID == p.VMMID && in.State == fcapi.StateRunning:
return nil
case err != nil:
last = err
again = time.After(delay)
delay = min(2*delay, DialRetryMax)
default:
last = fmt.Errorf("the VMM answers id %q state %q", in.ID, in.State)
}
}
check = false
select {
case <-bctx.Done():
if ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("vmruntime: %s: the VMM was not Running within %s: %v", p.Unit, r.cfg.BootTimeout, last)
case ev, ok := <-units:
if !ok {
units = nil
continue
}
if ev.Name == p.Unit {
if err := exited(ev.Status); err != nil {
return err
}
}
check = true
case ev, ok := <-sock.Events():
if !ok {
return fmt.Errorf("vmruntime: the watch of %s ended", p.SocketDir)
}
check = filepath.Clean(ev.Name) == filepath.Clean(p.Socket)
case err, ok := <-sock.Errors():
if !ok {
return fmt.Errorf("vmruntime: the watch of %s ended", p.SocketDir)
}
last, check = err, true
case <-again:
again, check = nil, true
}
}
}
func (r *hostRuntime) Await(ctx context.Context, p Prepared, slot int) (RuntimeRef, error) {
if err := r.awaitBoot(ctx, p); err != nil {
return RuntimeRef{}, err
}
st, err := r.cfg.Manager.Inspect(ctx, p.Unit)
if err != nil {
return RuntimeRef{}, err
}
vmm := r.vmmOf(p.WorkDir)
pid, err := vmmPID(st.MainPID, vmm)
if err != nil {
return RuntimeRef{}, err
}
id, err := r.cfg.Manager.Identify(ctx, p.Unit, pid)
if err != nil {
return RuntimeRef{}, err
}
_, _, vmmUID := r.slotIDs(slot)
ref := RuntimeRef{
Backend: id.Backend, Hypervisor: vmm, Handle: p.Spec.Handle, Name: p.Spec.Name,
Unit: p.Unit, InvocationID: id.InvocationID, PID: id.PID, StartTicks: id.StartTicks,
StartTime: time.Now().UTC(), HostBootID: id.BootID, Manager: id.Manager, VMMID: p.VMMID,
Slot: slot, VMMUID: vmmUID, WorkDir: p.WorkDir, Socket: p.Socket,
GuestAddress: guestnet.GuestAddress, HostAddress: guestnet.HostAddress,
ImageSHA256: filepath.Base(p.ImageDir), ImageFetched: p.ImageFetched,
}
if err := fsx.WriteJSON(filepath.Join(p.WorkDir, RefFile), ref, 0o600); err != nil {
return RuntimeRef{}, err
}
return ref, nil
}
func identity(ref RuntimeRef) hostsvc.Identity {
return hostsvc.Identity{Unit: ref.Unit, InvocationID: ref.InvocationID, PID: ref.PID, StartTicks: ref.StartTicks,
BootID: ref.HostBootID, Backend: ref.Backend, Manager: ref.Manager}
}
func (r *hostRuntime) Inspect(ctx context.Context, ref RuntimeRef) (Observation, error) {
obs, err := r.cfg.Manager.Check(ctx, identity(ref))
out := Observation{Verdict: obs.Verdict.String(), Reason: obs.Reason, Detail: obs.Detail, Stopped: obs.Stopped, State: string(obs.Status.State)}
if err != nil || obs.Verdict != hostsvc.VerdictRunning {
return out, err
}
r.rotate(ref.WorkDir)
in, err := apiFor(r.refVMM(ref), ref.Socket).Describe(ctx)
if err != nil {
return Observation{Verdict: hostsvc.VerdictUnknown.String(), Detail: err.Error(), State: out.State}, nil
}
if in.ID != ref.VMMID || in.State != fcapi.StateRunning {
out = Observation{Verdict: hostsvc.VerdictLost.String(), Reason: hostsvc.LostIdentityMismatch, State: out.State,
Detail: fmt.Sprintf("the VMM on %s answers id %q state %q, recorded %q Running", ref.Socket, in.ID, in.State, ref.VMMID)}
if err := r.cfg.Manager.Stop(ctx, ref.Unit); err != nil {
return out, fmt.Errorf("vmruntime: %s does not match its record and stopping it failed: %w", ref.Unit, err)
}
out.Stopped = true
}
return out, nil
}
func (r *hostRuntime) Stop(ctx context.Context, ref RuntimeRef) (StopResult, error) {
start := time.Now()
res := StopResult{}
st, err := r.cfg.Manager.Inspect(ctx, ref.Unit)
if err != nil {
return res, err
}
if GracefulStop() && (st.State == hostsvc.Active || st.State == hostsvc.Activating) {
if err := apiFor(r.refVMM(ref), ref.Socket).SendCtrlAltDel(ctx); err == nil {
if res.Graceful, err = r.awaitDown(ctx, ref.Unit, r.cfg.StopGrace); err != nil {
return res, err
}
}
}
if err := r.cfg.Manager.Stop(ctx, ref.Unit); err != nil {
return res, err
}
if st, err := r.cfg.Manager.Inspect(ctx, ref.Unit); err == nil {
res.Result = st.Result
}
if err := r.cfg.Manager.ResetFailed(ctx, ref.Unit); err != nil {
return res, err
}
res.Elapsed = time.Since(start)
return res, r.Cleanup(ctx, ref.Handle, ref.Name)
}
func (r *hostRuntime) Cleanup(ctx context.Context, handle, name string) error {
unit := UnitName(handle, name)
st, err := r.cfg.Manager.Inspect(ctx, unit)
if err != nil {
return err
}
if st.State == hostsvc.Active || st.State == hostsvc.Activating || st.State == hostsvc.Deactivating {
return fmt.Errorf("vmruntime: %s is %s; stop it before its work directory goes", unit, st.State)
}
work := WorkDir(r.cfg.StateDir, handle, name)
unlock, err := r.lock()
if err != nil {
return err
}
defer unlock()
var errs []error
var grants []Grant
if raw, err := os.ReadFile(filepath.Join(work, GrantsFile)); err == nil {
if err := json.Unmarshal(raw, &grants); err != nil {
errs = append(errs, err)
}
}
if err := revokeAll(grants); err != nil {
errs = append(errs, err)
}
if err := os.RemoveAll(SocketDir(r.cfg.RuntimeDir, handle, name)); err != nil {
errs = append(errs, err)
}
if err := os.RemoveAll(work); err != nil {
errs = append(errs, err)
}
return errors.Join(errs...)
}
func (r *hostRuntime) List(ctx context.Context, handle string) ([]Listed, error) {
units, err := r.cfg.Manager.List(ctx, UnitPrefix(handle))
if err != nil {
return nil, err
}
byName := map[string]*Listed{}
var order []string
for _, u := range units {
name := strings.TrimSuffix(strings.TrimPrefix(u.Name, UnitPrefix(handle)), hostsvc.ServiceSuffix)
byName[name] = &Listed{Name: name, Unit: u.Name, Status: u, Loaded: true}
order = append(order, name)
}
entries, err := os.ReadDir(VMsDir(r.cfg.StateDir, handle))
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
for _, e := range entries {
if !e.IsDir() {
continue
}
l, ok := byName[e.Name()]
if !ok {
l = &Listed{Name: e.Name(), Unit: UnitName(handle, e.Name())}
byName[e.Name()] = l
order = append(order, e.Name())
}
l.WorkDir = filepath.Join(VMsDir(r.cfg.StateDir, handle), e.Name())
}
out := make([]Listed, 0, len(order))
for _, n := range order {
out = append(out, *byName[n])
}
return out, nil
}
func (r *hostRuntime) Watch(ctx context.Context, handle string) (<-chan hostsvc.UnitEvent, error) {
return r.cfg.Manager.Watch(ctx, UnitPrefix(handle))
}
func (r *hostRuntime) Load(handle, name string) (RuntimeRef, error) {
var ref RuntimeRef
raw, err := os.ReadFile(filepath.Join(WorkDir(r.cfg.StateDir, handle, name), RefFile))
if err != nil {
return ref, err
}
return ref, json.Unmarshal(raw, &ref)
}
func (r *hostRuntime) slotLockPath() (string, error) {
account := filepath.Join("/run/user", strconv.Itoa(os.Getuid()))
if st, err := os.Stat(account); err == nil && st.IsDir() {
return filepath.Join(account, "socialweb-vmslots.lock"), nil
}
if err := os.MkdirAll(r.cfg.StateDir, 0o700); err != nil {
return "", fmt.Errorf("vmruntime: %w", err)
}
return filepath.Join(r.cfg.StateDir, "vmslots.lock"), nil
}
func (r *hostRuntime) lock() (func(), error) {
path, err := r.slotLockPath()
if err != nil {
return nil, err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, fmt.Errorf("vmruntime: %w", err)
}
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
f.Close()
return nil, fmt.Errorf("vmruntime: locking the VM slots: %w", err)
}
return func() {
syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
f.Close()
}, nil
}
func (r *hostRuntime) usedSlots(ctx context.Context) (map[int]bool, error) {
units, err := r.cfg.Manager.List(ctx, UnitKind)
if err != nil {
return nil, err
}
used := map[int]bool{}
for _, u := range units {
i := strings.LastIndex(u.Description, "slot=")
if i < 0 {
continue
}
if k, err := strconv.Atoi(u.Description[i+len("slot="):]); err == nil {
used[k] = true
}
}
return used, nil
}
func (r *hostRuntime) rotate(work string) {
for _, name := range []string{ConsoleFile, VMMLogFile} {
path := filepath.Join(work, name)
st, err := os.Stat(path)
if err != nil || st.Size() <= r.cfg.ConsoleCap {
continue
}
in, err := os.Open(path)
if err != nil {
continue
}
if _, err := in.Seek(-r.cfg.ConsoleCap, io.SeekEnd); err == nil {
if out, err := os.OpenFile(path+".1", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600); err == nil {
io.Copy(out, in)
out.Close()
}
}
in.Close()
os.Truncate(path, 0)
}
}
func vmmPID(pastaPID int, vmm string) (int, error) {
comm := "firecracker"
if vmm == VMMQEMU {
comm = QEMUComm
}
if pastaPID <= 0 {
return 0, fmt.Errorf("vmruntime: the unit has no main pid")
}
tasks, _ := os.ReadDir(filepath.Join("/proc", strconv.Itoa(pastaPID), "task"))
for _, t := range tasks {
raw, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pastaPID), "task", t.Name(), "children"))
if err != nil {
continue
}
for f := range strings.FieldsSeq(string(raw)) {
pid, err := strconv.Atoi(f)
if err != nil {
continue
}
c, _ := os.ReadFile(filepath.Join("/proc", f, "comm"))
if strings.TrimSpace(string(c)) == comm {
return pid, nil
}
}
}
return 0, fmt.Errorf("vmruntime: no %s child under pasta pid %d", comm, pastaPID)
}
type imageManifest struct {
BootArgs []string `json:"bootArgs"`
Vcpu int `json:"vcpu"`
MemMiB int `json:"memMiB"`
}
func readManifest(dir string) (imageManifest, error) {
var m imageManifest
raw, err := os.ReadFile(filepath.Join(dir, guestimage.ManifestName))
if err != nil {
return m, fmt.Errorf("vmruntime: %w", err)
}
if err := json.Unmarshal(raw, &m); err != nil {
return m, fmt.Errorf("vmruntime: %s: %w", dir, err)
}
if len(m.BootArgs) == 0 || m.Vcpu <= 0 || m.MemMiB <= 0 {
return m, fmt.Errorf("vmruntime: the image manifest in %s names no boot args or machine size", dir)
}
return m, nil
}
func tailOf(path string, n int64) string {
f, err := os.Open(path)
if err != nil {
return ""
}
defer f.Close()
if st, err := f.Stat(); err == nil && st.Size() > n {
f.Seek(-n, io.SeekEnd)
}
b, _ := io.ReadAll(f)
return "--- " + filepath.Base(path) + " ---\n" + string(b)
}
func devBootArgsExtra() []string {
if v := os.Getenv("SWC_DEV_GUEST_BOOT_ARGS_EXTRA"); v != "" {
return strings.Fields(v)
}
if dir := os.Getenv("XDG_RUNTIME_DIR"); dir != "" {
if raw, err := os.ReadFile(filepath.Join(dir, "swc-dev-guest-boot-args-extra")); err == nil {
return strings.Fields(string(raw))
}
}
return nil
}