~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
238 lines · 9.6 KB Raw History
#!/usr/bin/env bash
set -euo pipefail

if [ $# -lt 4 ]; then
  echo "usage: scripts/build-socialweb-computer.sh --dev [--platform P]... [--build-root DIR] [--publish DIR [--sync-only]]" >&2
  exit 2
fi
root="$1"
build_root="$2"
publish="$3"
sync_only="$4"
shift 4
platforms=("$@")

die() { echo "build-socialweb-computer --dev: $*" >&2; exit 1; }
say() { echo "build-socialweb-computer --dev: $(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >&2; }
sha_of() {
  local h
  h="$(sha256sum "$1" 2>/dev/null | cut -d' ' -f1)"
  if [ -z "$h" ]; then
    echo "sha_of: cannot hash $1" >&2
    printf 'unhashable:%s\n' "$1"
    return 1
  fi
  printf '%s\n' "$h"
}
started_s="$(date +%s.%N)"
elapsed() { awk -v a="$started_s" -v b="$(date +%s.%N)" 'BEGIN { printf "%.1f", b - a }'; }

command -v rsync >/dev/null 2>&1 || die "rsync is required for --dev"
command -v flock >/dev/null 2>&1 || die "flock is required for --dev"

name="$(basename "$root")"
dev_root="$build_root/dev/$name"
versions="$root/internal/versions/versions.json"

case "$publish" in
  "") [ "$sync_only" = 0 ] || die "--sync-only needs --publish DIR" ;;
  /*) ;;
  *) die "--publish $publish is not an absolute path" ;;
esac

mirror() {
  local dst="$1/src" list
  mkdir -p "$dst" "$dev_root"
  list="$dev_root/mirror.include"
  git -C "$root" ls-files -z -co --exclude-standard \
    | tr '\0' '\n' \
    | awk '{ n = split($0, p, "/"); d = ""; for (i = 1; i < n; i++) { d = d "/" p[i]; if (!(d in seen)) { seen[d] = 1; print d "/" } } print "/" $0 }' \
    >"$list"
  rsync -a --delete --delete-excluded --include-from="$list" --exclude='*' "$root/" "$dst/"
  chmod a+rx "$1"
  chmod -R a+rX "$dst"
  say "mirrored $(grep -vc '/$' "$list") files of $root to $dst ($(elapsed)s)"
}

if [ "$sync_only" = 1 ]; then
  mirror "$publish"
  exit 0
fi

want_go="$(awk '$1=="toolchain"{print $2; exit}' "$root/go.mod")"
[ -n "$want_go" ] || want_go="go$(awk '$1=="go"{print $2; exit}' "$root/go.mod")"
cache="$build_root/cache"
mkdir -p "$cache/assets" "$cache/gomod" "$cache/gocache" "$build_root/tmp" "$dev_root"
export TMPDIR="$build_root/tmp"
export GOTMPDIR="$build_root/tmp"
export GOMODCACHE="$cache/gomod"
export GOCACHE="$cache/gocache"
export GUEST_DOWNLOADS_CACHE="$cache/assets"
export GOFLAGS=-mod=readonly
export CGO_ENABLED=0
unset GOOS GOARCH GOPATH GOROOT SOURCE_DATE_EPOCH
goroot="$(GOTOOLCHAIN="$want_go" go env GOROOT)" || die "cannot get the pinned toolchain $want_go"
export PATH="$goroot/bin:$PATH"
export GOTOOLCHAIN=local
have_go="$(go env GOVERSION)"
[ "$have_go" = "$want_go" ] || die "the toolchain at $goroot is $have_go; go.mod pins $want_go"

hostplat="$(go env GOHOSTOS)/$(go env GOHOSTARCH)"
if [ ${#platforms[@]} -eq 0 ]; then
  platforms=("$hostplat")
fi
for plat in "${platforms[@]}"; do
  [ "$(jq -r --arg p "$plat" '.platforms[$p].status // ""' "$versions")" = "release" ] || die "$plat is not a release platform of versions.json"
done
if [ -n "$publish" ] && [[ " ${platforms[*]} " != *" $hostplat "* ]]; then
  die "--publish copies the host binary; add --platform $hostplat"
fi
stage_plats=("${platforms[@]}")
[[ " ${stage_plats[*]} " == *" $hostplat "* ]] || stage_plats+=("$hostplat")

commit="$(git -C "$root" rev-parse HEAD)"
changed="$(git -C "$root" status --porcelain | grep -c . || true)"
dirty=false
[ "$changed" = 0 ] || dirty=true
say "worktree $root at $commit (dirty $dirty, $changed changed paths), platforms ${platforms[*]}, go $have_go"

shared_blobs="$build_root/dev/assets"
own_blobs="$dev_root/blobs"
mkdir -p "$shared_blobs" "$own_blobs"
from_tree() {
  [ "$(jq -r --arg a "$1" --arg p "$2" '.assets[$a].fetch[$p].kind // ""' "$versions")" = go ]
}
srckeys="$dev_root/srckeys.$$"
rm -rf "$srckeys" && mkdir -p "$srckeys"
trap 'rm -rf "$srckeys"' EXIT
from_script() {
  [ "$(jq -r --arg a "$1" --arg p "$2" '.assets[$a].fetch[$p].kind // ""' "$versions")" = script ]
}
# a script-built asset (k8s-downloads, guest-downloads, swc-userns-stage) has no output pin in versions.json that
# changes with its sources, so its shared blob is keyed by its entry and by stage-assets.sh's source-input key
# (the script, the paths it names, the packages it builds from: the working tree's, uncommitted edits included)
script_key() {
  local plat="$1" asset="$2" k f="$srckeys/${1/\//_}-$2"
  if [ -s "$f" ]; then
    cat "$f"
    return
  fi
  k="$("$root/scripts/stage-assets.sh" --blob-cache "$cache/blobs" --asset "$asset" --platform "$plat" --print-src-key 2>/dev/null)" || k=""
  [ -n "$k" ] || k="nosrckey-$(git -C "$root" rev-parse HEAD 2>/dev/null || echo none)-$(date +%s)"
  printf '%s' "$k" | tee "$f"
}
blob_of() {
  local plat="$1" asset="$2" key
  if from_tree "$asset" "$plat"; then
    echo "$own_blobs/${plat/\//_}/$asset.tar.zst"
    return
  fi
  key="$(jq -cS --arg a "$asset" '.assets[$a]' "$versions")"
  if from_script "$asset" "$plat"; then
    key="$key$(script_key "$plat" "$asset")"
  fi
  echo "$shared_blobs/${plat/\//_}/$asset-$(printf '%s' "$key" | sha256sum | cut -c1-16).tar.zst"
}
rec_of() {
  local b
  b="$(blob_of "$1" "$2")"
  echo "${b%.tar.zst}.built.json"
}
{
  flock 9
  for plat in "${stage_plats[@]}"; do
    key="${plat/\//_}"
    missing=()
    for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do
      from_tree "$asset" "$plat" && continue
      if [ -f "$(blob_of "$plat" "$asset")" ] && { ! from_script "$asset" "$plat" || [ -f "$(rec_of "$plat" "$asset")" ]; }; then
        continue
      fi
      missing+=(--asset "$asset")
    done
    if [ ${#missing[@]} -gt 0 ]; then
      say "stage ${missing[*]} for $plat into $shared_blobs (once per versions.json entry of the asset, and per source inputs of a script-built one)"
      rm -rf "$shared_blobs/.stage"
      "$root/scripts/stage-assets.sh" --cache "$cache/assets" --blob-cache "$cache/blobs" --blobs "$shared_blobs/.stage" --platform "$plat" "${missing[@]}"
      mkdir -p "$shared_blobs/$key"
      for f in "$shared_blobs/.stage/$key"/*.tar.zst; do
        mv -f "$f" "$(blob_of "$plat" "$(basename "$f" .tar.zst)")"
        [ ! -f "${f%.tar.zst}.built.json" ] || mv -f "${f%.tar.zst}.built.json" "$(rec_of "$plat" "$(basename "$f" .tar.zst)")"
      done
      rm -rf "$shared_blobs/.stage"
    fi
  done
} 9>"$shared_blobs/.lock"
for plat in "${platforms[@]}"; do
  tree_assets=()
  built=()
  for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do
    from_tree "$asset" "$plat" || continue
    spec="$(jq -c --arg a "$asset" --arg p "$plat" '.assets[$a].fetch[$p]' "$versions")"
    probe="$dev_root/tree-bin/${plat/\//_}/$asset"
    mkdir -p "$(dirname "$probe")"
    (cd "$root/$(echo "$spec" | jq -r '.module // "."')" && GOOS="${plat%/*}" GOARCH="${plat#*/}" go build -trimpath \
      -buildvcs=false -ldflags='-s -w -buildid=' -o "$probe" "$(echo "$spec" | jq -r .package)")
    sum="$(sha_of "$probe")"
    if [ -f "$(blob_of "$plat" "$asset")" ] && [ -f "$(rec_of "$plat" "$asset")" ] && [ "$(cat "$probe.staged" 2>/dev/null)" = "$sum" ]; then
      continue
    fi
    tree_assets+=(--asset "$asset")
    built+=("$probe:$sum")
  done
  [ ${#tree_assets[@]} -gt 0 ] || continue
  say "stage ${tree_assets[*]} for $plat (built from the working tree, changed)"
  "$root/scripts/stage-assets.sh" --cache "$cache/assets" --blobs "$own_blobs" --platform "$plat" \
    "${tree_assets[@]}" >/dev/null 2>"$dev_root/tree-assets-stage.log" \
    || die "staging ${tree_assets[*]} failed: $(tail -3 "$dev_root/tree-assets-stage.log")"
  for b in "${built[@]}"; do
    echo "${b##*:}" >"${b%:*}.staged"
  done
done
say "assets staged ($(elapsed)s)"

hostkey="${hostplat/\//_}"
source_root="$root"
[ -z "$publish" ] || source_root="$publish/src"
stamp="github.com/johnandersen777/socialweb-computer-kcp/internal/buildstamp"
ldflags="-s -w -X $stamp.Kind=dev -X $stamp.SourceRoot=$source_root -X $stamp.Commit=$commit -X $stamp.Dirty=$dirty -X $stamp.ChangedPaths=$changed -X $stamp.BuiltUTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)"

for plat in "${platforms[@]}"; do
  goos="${plat%/*}"
  goarch="${plat#*/}"
  key="${goos}_${goarch}"
  out="$dev_root/$key"
  mkdir -p "$out"
  overlay="$dev_root/overlay-$key.json"
  {
    for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do
      jq -n --arg k "$root/internal/assets/blobs/$key/$asset.tar.zst" --arg v "$(blob_of "$plat" "$asset")" '{($k): $v}'
      if [ -f "$(rec_of "$plat" "$asset")" ]; then
        jq -n --arg k "$root/internal/assets/blobs/$key/$asset.built.json" --arg v "$(rec_of "$plat" "$asset")" '{($k): $v}'
      elif from_tree "$asset" "$plat" || from_script "$asset" "$plat"; then
        die "$asset ($plat) is built from this commit and has no build record at $(rec_of "$plat" "$asset"): the binary would refuse to start; remove its blob and build again"
      fi
    done
  } | jq -s '{Replace: add}' >"$overlay"
  say "go build $plat"
  (cd "$root" && GOOS="$goos" GOARCH="$goarch" go build -tags release -trimpath -buildvcs=true -overlay "$overlay" \
    -ldflags="$ldflags" -o "$out/socialweb-computer.new" ./cmd/socialweb-computer)
  mv -f "$out/socialweb-computer.new" "$out/socialweb-computer"
  if [ "$plat" = "$hostplat" ]; then
    ln -sfn "$key/socialweb-computer" "$dev_root/socialweb-computer"
  fi
  say "built $out/socialweb-computer ($(elapsed)s)"
done

if [ -n "$publish" ]; then
  mkdir -p "$publish/bin"
  cp -f "$dev_root/$hostkey/socialweb-computer" "$publish/bin/socialweb-computer.new"
  mv -f "$publish/bin/socialweb-computer.new" "$publish/bin/socialweb-computer"
  chmod a+rx "$publish/bin" "$publish/bin/socialweb-computer"
  mirror "$publish"
  say "published $publish/bin/socialweb-computer, source from $publish/src"
fi
if [[ " ${platforms[*]} " == *" $hostplat "* ]]; then
  "$dev_root/$hostkey/socialweb-computer" version | tee "$dev_root/$hostkey/version.txt"
fi
say "done in $(elapsed)s"