package guestrootfs import ( "archive/tar" "bufio" "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "io/fs" "os" "path" "path/filepath" "sort" "strconv" "strings" "time" "github.com/johnandersen777/socialweb-computer-kcp/internal/kit/sha256sum" ) const RootfsLabel = "cloudimg-rootfs" const dpkgStatusPath = "/var/lib/dpkg/status" type File struct { Path string Mode int64 Body []byte } type Spec struct { SourceDateEpoch int64 Files []File Remove []string RequirePackages []string Runs map[string]string Assets map[string]string } type Entry struct { Path string `json:"path"` Type string `json:"type"` Mode string `json:"mode"` UID int `json:"uid"` GID int `json:"gid"` Size int64 `json:"size,omitempty"` SHA256 string `json:"sha256,omitempty"` Link string `json:"link,omitempty"` } type Bake struct { Packages []string `json:"packages"` Files map[string]string `json:"files"` Assets map[string]string `json:"assets"` Runs map[string]string `json:"runs"` Removed []string `json:"removed"` } type Report struct { Entries []Entry `json:"entries"` Bake Bake `json:"bake"` TarSHA256 string `json:"tarSha256"` DataBytes int64 `json:"dataBytes"` SSHHostKeys int `json:"sshHostKeys"` } type Preinstall struct { Packages []string `json:"packages"` Files []struct { Path string `json:"path"` Mode string `json:"mode"` Content string `json:"content"` Source string `json:"source"` } `json:"files"` Runs []string `json:"runs"` } type Build struct { Dir string Preinstall Preinstall Overlay string `json:"overlay"` Remove []string `json:"remove"` Assets map[string]string `json:"assets"` Runs map[string]string `json:"runs"` From string `json:"preinstall"` } type Options struct { SourceDateEpoch int64 Assets map[string]string } func LoadBuild(file string) (Build, error) { raw, err := os.ReadFile(file) if err != nil { return Build{}, fmt.Errorf("guestrootfs: reading %s: %w", file, err) } var b Build if err := json.Unmarshal(raw, &b); err != nil { return Build{}, fmt.Errorf("guestrootfs: %s: %w", file, err) } b.Dir = filepath.Dir(file) if b.From == "" { return Build{}, fmt.Errorf("guestrootfs: %s names no preinstall manifest, and the manifest is "+ "what says which packages and files every guest finds already installed", file) } pre, err := os.ReadFile(filepath.Join(b.Dir, b.From)) if err != nil { return Build{}, fmt.Errorf("guestrootfs: reading the preinstall manifest: %w", err) } if err := json.Unmarshal(pre, &b.Preinstall); err != nil { return Build{}, fmt.Errorf("guestrootfs: %s: %w", b.From, err) } return b, nil } func RunKey(run string) string { sum := sha256.Sum256([]byte(run)) return hex.EncodeToString(sum[:]) } func (b Build) Spec(opts Options) (Spec, error) { if opts.SourceDateEpoch <= 0 { return Spec{}, fmt.Errorf("guestrootfs: no SOURCE_DATE_EPOCH: every timestamp the build " + "writes is clamped to it, and without one two builds of the same inputs differ") } spec := Spec{ SourceDateEpoch: opts.SourceDateEpoch, Remove: append([]string(nil), b.Remove...), RequirePackages: append([]string(nil), b.Preinstall.Packages...), Runs: map[string]string{}, Assets: map[string]string{}, } for _, run := range b.Preinstall.Runs { why, ok := b.Runs[RunKey(run)] if !ok { return Spec{}, fmt.Errorf("guestrootfs: the preinstall manifest runs %q, and the image build "+ "runs nothing inside the rootfs: list its sha256 %s under runs in the build file with "+ "what replaces it, or the image would silently lack what the run installs", run, RunKey(run)) } spec.Runs[RunKey(run)] = why } if b.Overlay != "" { root := filepath.Join(b.Dir, b.Overlay) err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } if !d.Type().IsRegular() { return fmt.Errorf("guestrootfs: the overlay holds %s, which is not a regular file", p) } info, err := d.Info() if err != nil { return err } body, err := os.ReadFile(p) if err != nil { return err } rel, err := filepath.Rel(root, p) if err != nil { return err } mode := int64(0o644) if info.Mode()&0o111 != 0 { mode = 0o755 } spec.Files = append(spec.Files, File{Path: "/" + filepath.ToSlash(rel), Mode: mode, Body: body}) return nil }) if err != nil { return Spec{}, fmt.Errorf("guestrootfs: reading the overlay: %w", err) } } for _, f := range b.Preinstall.Files { if !strings.HasPrefix(f.Path, "/") { return Spec{}, fmt.Errorf("guestrootfs: the preinstall file %q is not an absolute path", f.Path) } body := []byte(f.Content) if f.Source != "" { raw, err := os.ReadFile(filepath.Join(b.Dir, f.Source)) if err != nil { return Spec{}, fmt.Errorf("guestrootfs: the preinstall file %s: %w", f.Path, err) } body = raw } mode := int64(0o644) if f.Mode != "" { m, err := strconv.ParseInt(f.Mode, 8, 64) if err != nil { return Spec{}, fmt.Errorf("guestrootfs: the preinstall file %s has mode %q: %w", f.Path, f.Mode, err) } mode = m } spec.Files = append(spec.Files, File{Path: f.Path, Mode: mode, Body: body}) } for guestPath, name := range b.Assets { local, ok := opts.Assets[name] if !ok { return Spec{}, fmt.Errorf("guestrootfs: the build bakes the asset %s at %s and none was given "+ "for it", name, guestPath) } body, err := os.ReadFile(local) if err != nil { return Spec{}, fmt.Errorf("guestrootfs: the asset %s: %w", name, err) } spec.Files = append(spec.Files, File{Path: guestPath, Mode: 0o755, Body: body}) spec.Assets[guestPath] = name } seen := map[string]bool{} for _, f := range spec.Files { if seen[f.Path] { return Spec{}, fmt.Errorf("guestrootfs: %s is written twice by the build", f.Path) } seen[f.Path] = true } sort.Slice(spec.Files, func(i, j int) bool { return spec.Files[i].Path < spec.Files[j].Path }) return spec, nil } func guestPath(name string) string { p := path.Clean("/" + strings.TrimPrefix(name, "./")) return p } func (s Spec) removed(p string) bool { for _, pattern := range s.Remove { if strings.HasSuffix(pattern, "/**") { if strings.HasPrefix(p, strings.TrimSuffix(pattern, "**")) { return true } continue } if ok, _ := path.Match(pattern, p); ok { return true } } return false } func installedPackages(status []byte) map[string]bool { out := map[string]bool{} var name string scanner := bufio.NewScanner(bytes.NewReader(status)) scanner.Buffer(make([]byte, 1<<20), 1<<20) for scanner.Scan() { line := scanner.Text() switch { case line == "": name = "" case strings.HasPrefix(line, "Package: "): name = strings.TrimSpace(strings.TrimPrefix(line, "Package: ")) case strings.HasPrefix(line, "Status: ") && name != "": if strings.HasSuffix(strings.TrimSpace(line), " installed") && strings.HasPrefix(line, "Status: install ") { out[name] = true } } } return out } type hashingWriter struct { w io.Writer h io.Writer } func (hw hashingWriter) Write(p []byte) (int, error) { n, err := hw.w.Write(p) hw.h.Write(p[:n]) return n, err } func blocks(n int64) int64 { return (n + 4095) / 4096 * 4096 } func Compose(base io.Reader, spec Spec, out io.Writer) (Report, error) { if spec.SourceDateEpoch <= 0 { return Report{}, fmt.Errorf("guestrootfs: no SOURCE_DATE_EPOCH") } epoch := time.Unix(spec.SourceDateEpoch, 0).UTC() sum := sha256.New() w := tar.NewWriter(hashingWriter{w: out, h: sum}) report := Report{Bake: Bake{Files: map[string]string{}, Assets: map[string]string{}, Runs: spec.Runs, Removed: []string{}}} replace := map[string]File{} for _, f := range spec.Files { replace[f.Path] = f } written := map[string]bool{} dirs := map[string]bool{} removed := map[string]bool{} prefix := "" var status []byte emit := func(h *tar.Header, body []byte) error { if h.ModTime.IsZero() || h.ModTime.After(epoch) { h.ModTime = epoch } h.ModTime = h.ModTime.UTC().Truncate(time.Second) h.AccessTime, h.ChangeTime = time.Time{}, time.Time{} var pax map[string]string for k, v := range h.PAXRecords { if strings.HasPrefix(k, "SCHILY.xattr.") { if pax == nil { pax = map[string]string{} } pax[k] = v } } h.PAXRecords = pax h.Format = tar.FormatUnknown if pax != nil { h.Format = tar.FormatPAX } if err := w.WriteHeader(h); err != nil { return fmt.Errorf("guestrootfs: writing %s: %w", h.Name, err) } e := Entry{Path: guestPath(h.Name), Mode: fmt.Sprintf("%04o", h.Mode&0o7777), UID: h.Uid, GID: h.Gid, Link: h.Linkname} switch h.Typeflag { case tar.TypeReg: if _, err := w.Write(body); err != nil { return fmt.Errorf("guestrootfs: writing %s: %w", h.Name, err) } d := sha256.Sum256(body) e.Type, e.Size, e.SHA256 = "file", int64(len(body)), hex.EncodeToString(d[:]) report.DataBytes += blocks(int64(len(body))) case tar.TypeDir: e.Type = "dir" report.DataBytes += 4096 case tar.TypeSymlink: e.Type = "symlink" if len(h.Linkname) > 59 { report.DataBytes += 4096 } case tar.TypeLink: e.Type = "hardlink" default: e.Type = "special" } if strings.HasPrefix(e.Path, "/etc/ssh/ssh_host_") { report.SSHHostKeys++ } report.DataBytes += 256 report.Entries = append(report.Entries, e) written[e.Path] = true return nil } r := tar.NewReader(base) first := true for { h, err := r.Next() if err == io.EOF { break } if err != nil { return Report{}, fmt.Errorf("guestrootfs: reading the base: %w", err) } if first { if strings.HasPrefix(h.Name, "./") || h.Name == "." { prefix = "./" } first = false } p := guestPath(h.Name) if spec.removed(p) { removed[p] = true report.Bake.Removed = append(report.Bake.Removed, p) continue } if h.Typeflag == tar.TypeLink { target := guestPath(h.Linkname) if removed[target] { return Report{}, fmt.Errorf("guestrootfs: %s is a hard link to %s, which the build removes", p, target) } if _, ok := replace[target]; ok { return Report{}, fmt.Errorf("guestrootfs: %s is a hard link to %s, which the build replaces", p, target) } } if h.Typeflag == tar.TypeDir { dirs[p] = true } var body []byte if h.Typeflag == tar.TypeReg { body, err = io.ReadAll(r) if err != nil { return Report{}, fmt.Errorf("guestrootfs: reading %s from the base: %w", p, err) } if p == dpkgStatusPath { status = body } } if f, ok := replace[p]; ok { if h.Typeflag != tar.TypeReg { return Report{}, fmt.Errorf("guestrootfs: the build writes %s and the base holds a "+ "non-file there", p) } body = f.Body h.Size = int64(len(body)) h.Mode = f.Mode h.Uid, h.Gid, h.Uname, h.Gname = 0, 0, "root", "root" h.ModTime = epoch h.PAXRecords = nil delete(replace, p) report.Bake.Files[p] = sha256sum.Hex(body) } if err := emit(h, body); err != nil { return Report{}, err } } if status == nil { return Report{}, fmt.Errorf("guestrootfs: the base carries no %s, so nothing says which "+ "packages it has installed", dpkgStatusPath) } installed := installedPackages(status) var missing []string for _, p := range spec.RequirePackages { if !installed[p] { missing = append(missing, p) continue } report.Bake.Packages = append(report.Bake.Packages, p) } if len(missing) > 0 { return Report{}, fmt.Errorf("guestrootfs: the preinstall manifest needs %s and the base's "+ "dpkg status does not have them installed. The build installs no packages (no apt, no "+ "chroot): pin a base that carries them, or take them out of the manifest", strings.Join(missing, ", ")) } var added []string for p := range replace { added = append(added, p) } sort.Strings(added) for _, p := range added { var parents []string for d := path.Dir(p); d != "/" && !dirs[d]; d = path.Dir(d) { parents = append([]string{d}, parents...) } for _, d := range parents { h := &tar.Header{Name: prefix + strings.TrimPrefix(d, "/") + "/", Typeflag: tar.TypeDir, Mode: 0o755, Uname: "root", Gname: "root", ModTime: epoch} if err := emit(h, nil); err != nil { return Report{}, err } dirs[d] = true } f := replace[p] h := &tar.Header{Name: prefix + strings.TrimPrefix(p, "/"), Typeflag: tar.TypeReg, Mode: f.Mode, Size: int64(len(f.Body)), Uname: "root", Gname: "root", ModTime: epoch} if err := emit(h, f.Body); err != nil { return Report{}, err } if name, ok := spec.Assets[p]; ok { report.Bake.Assets[name] = sha256sum.Hex(f.Body) } else { report.Bake.Files[p] = sha256sum.Hex(f.Body) } } if err := w.Close(); err != nil { return Report{}, fmt.Errorf("guestrootfs: %w", err) } for p, name := range spec.Assets { if _, ok := report.Bake.Assets[name]; !ok { report.Bake.Assets[name] = report.Bake.Files[p] delete(report.Bake.Files, p) } } sort.Slice(report.Entries, func(i, j int) bool { return report.Entries[i].Path < report.Entries[j].Path }) report.TarSHA256 = hex.EncodeToString(sum.Sum(nil)) return report, nil }