0
0
package guestrootfs
import (
"archive/tar"
"bufio"
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/johnandersen777/socialweb-computer-kcp/internal/kit/sha256sum"
)
const RootfsLabel = "cloudimg-rootfs"
const dpkgStatusPath = "/var/lib/dpkg/status"
type File struct {
Path string
Mode int64
Body []byte
}
type Spec struct {
SourceDateEpoch int64
Files []File
Remove []string
RequirePackages []string
Runs map[string]string
Assets map[string]string
}
type Entry struct {
Path string `json:"path"`
Type string `json:"type"`
Mode string `json:"mode"`
UID int `json:"uid"`
GID int `json:"gid"`
Size int64 `json:"size,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Link string `json:"link,omitempty"`
}
type Bake struct {
Packages []string `json:"packages"`
Files map[string]string `json:"files"`
Assets map[string]string `json:"assets"`
Runs map[string]string `json:"runs"`
Removed []string `json:"removed"`
}
type Report struct {
Entries []Entry `json:"entries"`
Bake Bake `json:"bake"`
TarSHA256 string `json:"tarSha256"`
DataBytes int64 `json:"dataBytes"`
SSHHostKeys int `json:"sshHostKeys"`
}
type Preinstall struct {
Packages []string `json:"packages"`
Files []struct {
Path string `json:"path"`
Mode string `json:"mode"`
Content string `json:"content"`
Source string `json:"source"`
} `json:"files"`
Runs []string `json:"runs"`
}
type Build struct {
Dir string
Preinstall Preinstall
Overlay string `json:"overlay"`
Remove []string `json:"remove"`
Assets map[string]string `json:"assets"`
Runs map[string]string `json:"runs"`
From string `json:"preinstall"`
}
type Options struct {
SourceDateEpoch int64
Assets map[string]string
}
func LoadBuild(file string) (Build, error) {
raw, err := os.ReadFile(file)
if err != nil {
return Build{}, fmt.Errorf("guestrootfs: reading %s: %w", file, err)
}
var b Build
if err := json.Unmarshal(raw, &b); err != nil {
return Build{}, fmt.Errorf("guestrootfs: %s: %w", file, err)
}
b.Dir = filepath.Dir(file)
if b.From == "" {
return Build{}, fmt.Errorf("guestrootfs: %s names no preinstall manifest, and the manifest is "+
"what says which packages and files every guest finds already installed", file)
}
pre, err := os.ReadFile(filepath.Join(b.Dir, b.From))
if err != nil {
return Build{}, fmt.Errorf("guestrootfs: reading the preinstall manifest: %w", err)
}
if err := json.Unmarshal(pre, &b.Preinstall); err != nil {
return Build{}, fmt.Errorf("guestrootfs: %s: %w", b.From, err)
}
return b, nil
}
func RunKey(run string) string {
sum := sha256.Sum256([]byte(run))
return hex.EncodeToString(sum[:])
}
func (b Build) Spec(opts Options) (Spec, error) {
if opts.SourceDateEpoch <= 0 {
return Spec{}, fmt.Errorf("guestrootfs: no SOURCE_DATE_EPOCH: every timestamp the build " +
"writes is clamped to it, and without one two builds of the same inputs differ")
}
spec := Spec{
SourceDateEpoch: opts.SourceDateEpoch,
Remove: append([]string(nil), b.Remove...),
RequirePackages: append([]string(nil), b.Preinstall.Packages...),
Runs: map[string]string{},
Assets: map[string]string{},
}
for _, run := range b.Preinstall.Runs {
why, ok := b.Runs[RunKey(run)]
if !ok {
return Spec{}, fmt.Errorf("guestrootfs: the preinstall manifest runs %q, and the image build "+
"runs nothing inside the rootfs: list its sha256 %s under runs in the build file with "+
"what replaces it, or the image would silently lack what the run installs",
run, RunKey(run))
}
spec.Runs[RunKey(run)] = why
}
if b.Overlay != "" {
root := filepath.Join(b.Dir, b.Overlay)
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !d.Type().IsRegular() {
return fmt.Errorf("guestrootfs: the overlay holds %s, which is not a regular file", p)
}
info, err := d.Info()
if err != nil {
return err
}
body, err := os.ReadFile(p)
if err != nil {
return err
}
rel, err := filepath.Rel(root, p)
if err != nil {
return err
}
mode := int64(0o644)
if info.Mode()&0o111 != 0 {
mode = 0o755
}
spec.Files = append(spec.Files, File{Path: "/" + filepath.ToSlash(rel), Mode: mode, Body: body})
return nil
})
if err != nil {
return Spec{}, fmt.Errorf("guestrootfs: reading the overlay: %w", err)
}
}
for _, f := range b.Preinstall.Files {
if !strings.HasPrefix(f.Path, "/") {
return Spec{}, fmt.Errorf("guestrootfs: the preinstall file %q is not an absolute path", f.Path)
}
body := []byte(f.Content)
if f.Source != "" {
raw, err := os.ReadFile(filepath.Join(b.Dir, f.Source))
if err != nil {
return Spec{}, fmt.Errorf("guestrootfs: the preinstall file %s: %w", f.Path, err)
}
body = raw
}
mode := int64(0o644)
if f.Mode != "" {
m, err := strconv.ParseInt(f.Mode, 8, 64)
if err != nil {
return Spec{}, fmt.Errorf("guestrootfs: the preinstall file %s has mode %q: %w", f.Path, f.Mode, err)
}
mode = m
}
spec.Files = append(spec.Files, File{Path: f.Path, Mode: mode, Body: body})
}
for guestPath, name := range b.Assets {
local, ok := opts.Assets[name]
if !ok {
return Spec{}, fmt.Errorf("guestrootfs: the build bakes the asset %s at %s and none was given "+
"for it", name, guestPath)
}
body, err := os.ReadFile(local)
if err != nil {
return Spec{}, fmt.Errorf("guestrootfs: the asset %s: %w", name, err)
}
spec.Files = append(spec.Files, File{Path: guestPath, Mode: 0o755, Body: body})
spec.Assets[guestPath] = name
}
seen := map[string]bool{}
for _, f := range spec.Files {
if seen[f.Path] {
return Spec{}, fmt.Errorf("guestrootfs: %s is written twice by the build", f.Path)
}
seen[f.Path] = true
}
sort.Slice(spec.Files, func(i, j int) bool { return spec.Files[i].Path < spec.Files[j].Path })
return spec, nil
}
func guestPath(name string) string {
p := path.Clean("/" + strings.TrimPrefix(name, "./"))
return p
}
func (s Spec) removed(p string) bool {
for _, pattern := range s.Remove {
if strings.HasSuffix(pattern, "/**") {
if strings.HasPrefix(p, strings.TrimSuffix(pattern, "**")) {
return true
}
continue
}
if ok, _ := path.Match(pattern, p); ok {
return true
}
}
return false
}
func installedPackages(status []byte) map[string]bool {
out := map[string]bool{}
var name string
scanner := bufio.NewScanner(bytes.NewReader(status))
scanner.Buffer(make([]byte, 1<<20), 1<<20)
for scanner.Scan() {
line := scanner.Text()
switch {
case line == "":
name = ""
case strings.HasPrefix(line, "Package: "):
name = strings.TrimSpace(strings.TrimPrefix(line, "Package: "))
case strings.HasPrefix(line, "Status: ") && name != "":
if strings.HasSuffix(strings.TrimSpace(line), " installed") && strings.HasPrefix(line, "Status: install ") {
out[name] = true
}
}
}
return out
}
type hashingWriter struct {
w io.Writer
h io.Writer
}
func (hw hashingWriter) Write(p []byte) (int, error) {
n, err := hw.w.Write(p)
hw.h.Write(p[:n])
return n, err
}
func blocks(n int64) int64 { return (n + 4095) / 4096 * 4096 }
func Compose(base io.Reader, spec Spec, out io.Writer) (Report, error) {
if spec.SourceDateEpoch <= 0 {
return Report{}, fmt.Errorf("guestrootfs: no SOURCE_DATE_EPOCH")
}
epoch := time.Unix(spec.SourceDateEpoch, 0).UTC()
sum := sha256.New()
w := tar.NewWriter(hashingWriter{w: out, h: sum})
report := Report{Bake: Bake{Files: map[string]string{}, Assets: map[string]string{}, Runs: spec.Runs, Removed: []string{}}}
replace := map[string]File{}
for _, f := range spec.Files {
replace[f.Path] = f
}
written := map[string]bool{}
dirs := map[string]bool{}
removed := map[string]bool{}
prefix := ""
var status []byte
emit := func(h *tar.Header, body []byte) error {
if h.ModTime.IsZero() || h.ModTime.After(epoch) {
h.ModTime = epoch
}
h.ModTime = h.ModTime.UTC().Truncate(time.Second)
h.AccessTime, h.ChangeTime = time.Time{}, time.Time{}
var pax map[string]string
for k, v := range h.PAXRecords {
if strings.HasPrefix(k, "SCHILY.xattr.") {
if pax == nil {
pax = map[string]string{}
}
pax[k] = v
}
}
h.PAXRecords = pax
h.Format = tar.FormatUnknown
if pax != nil {
h.Format = tar.FormatPAX
}
if err := w.WriteHeader(h); err != nil {
return fmt.Errorf("guestrootfs: writing %s: %w", h.Name, err)
}
e := Entry{Path: guestPath(h.Name), Mode: fmt.Sprintf("%04o", h.Mode&0o7777), UID: h.Uid, GID: h.Gid, Link: h.Linkname}
switch h.Typeflag {
case tar.TypeReg:
if _, err := w.Write(body); err != nil {
return fmt.Errorf("guestrootfs: writing %s: %w", h.Name, err)
}
d := sha256.Sum256(body)
e.Type, e.Size, e.SHA256 = "file", int64(len(body)), hex.EncodeToString(d[:])
report.DataBytes += blocks(int64(len(body)))
case tar.TypeDir:
e.Type = "dir"
report.DataBytes += 4096
case tar.TypeSymlink:
e.Type = "symlink"
if len(h.Linkname) > 59 {
report.DataBytes += 4096
}
case tar.TypeLink:
e.Type = "hardlink"
default:
e.Type = "special"
}
if strings.HasPrefix(e.Path, "/etc/ssh/ssh_host_") {
report.SSHHostKeys++
}
report.DataBytes += 256
report.Entries = append(report.Entries, e)
written[e.Path] = true
return nil
}
r := tar.NewReader(base)
first := true
for {
h, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
return Report{}, fmt.Errorf("guestrootfs: reading the base: %w", err)
}
if first {
if strings.HasPrefix(h.Name, "./") || h.Name == "." {
prefix = "./"
}
first = false
}
p := guestPath(h.Name)
if spec.removed(p) {
removed[p] = true
report.Bake.Removed = append(report.Bake.Removed, p)
continue
}
if h.Typeflag == tar.TypeLink {
target := guestPath(h.Linkname)
if removed[target] {
return Report{}, fmt.Errorf("guestrootfs: %s is a hard link to %s, which the build removes", p, target)
}
if _, ok := replace[target]; ok {
return Report{}, fmt.Errorf("guestrootfs: %s is a hard link to %s, which the build replaces", p, target)
}
}
if h.Typeflag == tar.TypeDir {
dirs[p] = true
}
var body []byte
if h.Typeflag == tar.TypeReg {
body, err = io.ReadAll(r)
if err != nil {
return Report{}, fmt.Errorf("guestrootfs: reading %s from the base: %w", p, err)
}
if p == dpkgStatusPath {
status = body
}
}
if f, ok := replace[p]; ok {
if h.Typeflag != tar.TypeReg {
return Report{}, fmt.Errorf("guestrootfs: the build writes %s and the base holds a "+
"non-file there", p)
}
body = f.Body
h.Size = int64(len(body))
h.Mode = f.Mode
h.Uid, h.Gid, h.Uname, h.Gname = 0, 0, "root", "root"
h.ModTime = epoch
h.PAXRecords = nil
delete(replace, p)
report.Bake.Files[p] = sha256sum.Hex(body)
}
if err := emit(h, body); err != nil {
return Report{}, err
}
}
if status == nil {
return Report{}, fmt.Errorf("guestrootfs: the base carries no %s, so nothing says which "+
"packages it has installed", dpkgStatusPath)
}
installed := installedPackages(status)
var missing []string
for _, p := range spec.RequirePackages {
if !installed[p] {
missing = append(missing, p)
continue
}
report.Bake.Packages = append(report.Bake.Packages, p)
}
if len(missing) > 0 {
return Report{}, fmt.Errorf("guestrootfs: the preinstall manifest needs %s and the base's "+
"dpkg status does not have them installed. The build installs no packages (no apt, no "+
"chroot): pin a base that carries them, or take them out of the manifest",
strings.Join(missing, ", "))
}
var added []string
for p := range replace {
added = append(added, p)
}
sort.Strings(added)
for _, p := range added {
var parents []string
for d := path.Dir(p); d != "/" && !dirs[d]; d = path.Dir(d) {
parents = append([]string{d}, parents...)
}
for _, d := range parents {
h := &tar.Header{Name: prefix + strings.TrimPrefix(d, "/") + "/", Typeflag: tar.TypeDir, Mode: 0o755,
Uname: "root", Gname: "root", ModTime: epoch}
if err := emit(h, nil); err != nil {
return Report{}, err
}
dirs[d] = true
}
f := replace[p]
h := &tar.Header{Name: prefix + strings.TrimPrefix(p, "/"), Typeflag: tar.TypeReg, Mode: f.Mode,
Size: int64(len(f.Body)), Uname: "root", Gname: "root", ModTime: epoch}
if err := emit(h, f.Body); err != nil {
return Report{}, err
}
if name, ok := spec.Assets[p]; ok {
report.Bake.Assets[name] = sha256sum.Hex(f.Body)
} else {
report.Bake.Files[p] = sha256sum.Hex(f.Body)
}
}
if err := w.Close(); err != nil {
return Report{}, fmt.Errorf("guestrootfs: %w", err)
}
for p, name := range spec.Assets {
if _, ok := report.Bake.Assets[name]; !ok {
report.Bake.Assets[name] = report.Bake.Files[p]
delete(report.Bake.Files, p)
}
}
sort.Slice(report.Entries, func(i, j int) bool { return report.Entries[i].Path < report.Entries[j].Path })
report.TarSHA256 = hex.EncodeToString(sum.Sum(nil))
return report, nil
}