~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
209 lines · 9.1 KB Raw History
#!/usr/bin/env bash
# The knotless `release` workflow (.knotless/workflows/release.yml), one function per step; the workflow only calls it (plan 0042).
#   scripts/ci-release.sh <toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify>
# CI: root on the kcp-build VM (/root/..., /opt/ci/env), KNOTLESS_RUN_REF and KNOTLESS_RUN_SHA set. By hand (hat-8, plan 0044):
# not root; CI_HOME (default ~/.tmp/ci-hand) holds what CI puts under /root, the build root is the host's own
# (~/.tmp/socialweb-computer-build); the commit is HEAD and a publish writes local/<sha>/ only, never main/<sha>/ or
# main/latest/. No tag is built: releases/<version>/ is promote's copy of main/<sha>/ (scripts/promote.sh, plan 0040 step 15).
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.."
SPACES_BUCKET=${SPACES_BUCKET:-socialweb-computer-kcp} SPACES_REGION=${SPACES_REGION:-sfo3} TARGET=${TARGET:-firecracker-x86_64}
sha=${KNOTLESS_RUN_SHA:-$(git rev-parse HEAD)}
ref=${KNOTLESS_RUN_REF:-}
if [ "$(id -u)" = 0 ]; then
  work=/root root=/root/release-build
  OUT=${OUT:-/root/ci-out}
else
  work=${CI_HOME:-$HOME/.tmp/ci-hand} root=$HOME/.tmp/socialweb-computer-build
  OUT=${OUT:-$work/ci-out}
  ref=${ref:-hand}
  # never /tmp on hat-8: a tmpfs with a per-user quota (AGENTS.md)
  export TMPDIR=$work/tmp
  mkdir -p "$TMPDIR"
fi
base="https://$SPACES_BUCKET.$SPACES_REGION.digitaloceanspaces.com"
[ -f /opt/ci/env ] && . /opt/ci/env
mkdir -p "$work"

vm_readings() {
  (while sleep 60; do echo "vm: $(free -m | awk '/Mem:/ {print $3 "/" $2 " MiB used, " $7 " available"}'), load $(cut -d' ' -f1-3 /proc/loadavg)"; done) &
  trap "kill $! 2>/dev/null" EXIT
}

pin_of() { jq -er --arg t "$TARGET" --arg f "$1" --arg k "$2" '.guestImages.targets[$t].outputs[$f][$k]' internal/versions/versions.json; }

step_toolchain() {
  if bash scripts/knotless-ci-baked.sh; then :; else bash scripts/knotless-ci-toolchain.sh --wasm --deno --arm64; fi
}

step_checks() {
  vm_readings
  go mod download
  export CHECK_CROSS_TARGETS="linux/arm64 darwin/arm64 windows/amd64"
  cmp scripts/ci-tools.json tests/modules/staging/ci-tools.json
  deploy/check.sh --fast
  test -z "$(git status --porcelain)"
}

step_binaries() {
  vm_readings
  mkdir -p "$root"
  export ASSET_CACHE_URL=$base/asset-cache
  BUILD_CACHE_URL=$base/build-cache bash scripts/ci-build-cache.sh restore "$root" || echo "build-cache: restore failed: cold build"
  local p=() rel tree want got name side
  for x in ${CI_PLATFORMS:-linux/amd64 linux/arm64}; do p+=(--platform "$x"); done
  bash scripts/build-socialweb-computer.sh "${p[@]}" --commit "$sha" --build-root "$root"
  rel="$root/release/$sha"
  ls -la "$rel"
  want="$(jq -r '.assets.deno.version' internal/versions/versions.json)"
  jq -e --arg v "$want" '(.builds | length) > 0 and all(.builds[]; .denoEmbedded.version == $v and (.denoEmbedded.zips | keys | sort) == ["linux/amd64","linux/arm64"])' "$rel/release.json"
  rm -rf "$OUT/binaries" && mkdir -p "$OUT/binaries" && cp -a "$rel"/. "$OUT/binaries/"
  (cd "$OUT/binaries" && sha256sum -c --quiet SHA256SUMS)
  tree="$(mktemp -d)"
  zstd -dc "$root/src/1/internal/assets/blobs/linux_amd64/socialweb-computer-k8s-downloads.tar.zst" | tar -x -C "$tree"
  rm -rf "$work/ci-downloads" && mkdir -p "$work/ci-downloads"
  for side in "$tree"/*.sha256; do
    name="$(basename "$side" .sha256)"
    want="$(cut -d' ' -f1 "$side")"
    got="$(sha256sum "$tree/$name" | cut -d' ' -f1)"
    echo "downloads/$want/$name (sha256 $got)"
    test "$got" = "$want"
    mkdir -p "$work/ci-downloads/$want" && cp "$tree/$name" "$work/ci-downloads/$want/$name"
  done
  test "$(find "$work/ci-downloads" -type f | wc -l)" -gt 0
}

step_staging_binary() {
  mkdir -p oci
  local r
  if r=$(bash scripts/oci-push.sh "$OUT/binaries/socialweb-computer-linux-amd64" 6h); then
    echo "$r" > oci/binary.ref
    echo "staging binary: pushed to $r"
  else
    echo "staging binary: ttl.sh refused the push: no oci/binary.ref (test will fail saying so)"
  fi
}

step_test_host() {
  local tree rtree
  tree="$(git rev-parse HEAD:tests)" rtree="$(git rev-parse HEAD:rust)"
  if curl -sfo /dev/null "$base/tests/$tree/SHA256SUMS" && curl -sfo /dev/null "$base/meshd/$rtree/SHA256SUMS"; then
    echo "tests/$tree/ and meshd/$rtree/ are published: nothing to build"
    return 0
  fi
  BUILD_HOST=local DIST="$PWD/dist" sh tests/publish-test-host.sh
}

step_page() {
  PAGE_WORK="$work/ci-page" DIST="$PWD/dist" bash scripts/ci-page.sh
}

step_guest_image() {
  local pin url img="$work/ci-guest-image" published=true f want bytes head got size
  pin="$(pin_of image.json sha256)"
  url="$base/guest-images/$TARGET/$pin"
  rm -rf "$img" "$img.published"
  for f in image.json vmlinux rootfs.ext4; do
    want="$(pin_of "$f" sha256)" bytes="$(pin_of "$f" bytes)"
    head="$(curl -sSfI --retry 3 --max-time 60 "$url/$f" | tr -d '\r')" || head=
    got="$(printf '%s\n' "$head" | awk 'tolower($1)=="x-amz-meta-sha256:" {print $2}')"
    size="$(printf '%s\n' "$head" | awk 'tolower($1)=="content-length:" {print $2}')"
    echo "$TARGET/$f sha256 ${got:-absent} ${size:-?} bytes (pinned $want, $bytes bytes) at $url/$f"
    [ "$got" = "$want" ] && [ "$size" = "$bytes" ] || published=false
  done
  if [ "$published" = true ]; then
    echo "guest-image: not building $TARGET: every output is published at $url and matches its pin"
    touch "$img.published"
    return 0
  fi
  echo "guest-image: building $TARGET"
  mkdir -p "$img" "$work/guest-image-work"
  scripts/build-guest-image.sh --target "$TARGET" --work "$work/guest-image-work" --out "$img"
  for f in vmlinux rootfs.ext4 image.json; do
    want="$(pin_of "$f" sha256)"
    got="$(sha256sum "$img/$TARGET/$f" | cut -d' ' -f1)"
    echo "$TARGET/$f sha256 $got (pinned $want)"
    test "$got" = "$want"
  done
}

step_asset_cache() {
  scripts/asset-cache-upload.sh "$root/cache/blobs"
}

step_build_cache() {
  bash scripts/ci-build-cache.sh save "$root" || echo "build-cache: save failed: the build is not affected"
}

step_stage() {
  test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY"
  if [ ! -e "$work/ci-guest-image.published" ]; then
    deno run -A --no-lock scripts/publish-spaces.ts --dir "$work/ci-guest-image/$TARGET" --prefix "guest-images/$TARGET/$(pin_of image.json sha256)" --content-addressed
  fi
  deno run -A --no-lock scripts/publish-spaces.ts --dir "$work/ci-downloads" --prefix downloads --content-addressed
  deno run -A --no-lock scripts/publish-spaces.ts --dir "$OUT" --prefix "staging/$sha"
}

step_publish() {
  [ -d /opt/ci/deno ] && export PATH=/opt/ci/deno:$PATH
  test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY"
  case "$ref" in
    refs/heads/main) set -- --to "main/$sha" --to main/latest ;;
    # by hand (no CI ref): local/<sha>/ only; main/<sha>/ is main's CI's, so promote never releases a hand build
    *) set -- --to "local/$sha" ;;
  esac
  deno run -A --no-lock scripts/copy-spaces.ts --from "staging/$sha" "$@"
}

step_verify() {
  local at d sum name pin f want size probe
  case "$ref" in
    refs/heads/main) at="main/$sha" ;;
    *) at="local/$sha" ;;
  esac
  d="$(mktemp -d)"
  curl -sSf -o "$d/STAGED" "$base/staging/$sha/binaries/SHA256SUMS"
  curl -sSf -o "$d/SHA256SUMS" "$base/$at/binaries/SHA256SUMS"
  diff "$d/STAGED" "$d/SHA256SUMS"
  if [ "$ref" = refs/heads/main ]; then
    curl -sSf -o "$d/LATEST" "$base/main/latest/binaries/SHA256SUMS"
    diff "$d/STAGED" "$d/LATEST"
  fi
  while read -r sum name; do
    curl -sSfL -o "$d/$name" "$base/$at/binaries/$name"
    echo "$sum  $d/$name" | sha256sum -c --quiet -
    echo "$at/binaries/$name: sha256 $sum, readable without credentials"
  done < "$d/SHA256SUMS"
  pin="$(pin_of image.json sha256)"
  for f in image.json vmlinux rootfs.ext4; do
    want="$(pin_of "$f" bytes)"
    size="$(curl -sSI --max-time 60 "$base/guest-images/$TARGET/$pin/$f" | tr -d '\r' | awk 'tolower($1)=="content-length:" {print $2}')"
    echo "guest-images/$TARGET/$pin/$f $size bytes (pinned $want)"
    test "$size" = "$want"
  done
  local as=()
  if [ "$(id -u)" = 0 ]; then
    id probe >/dev/null 2>&1 || useradd -m probe
    probe=/home/probe/p
    as=(runuser -u probe -- env HOME=/home/probe)
  else
    probe=$work/probe
    as=(env HOME="$HOME")
  fi
  rm -rf "$probe" && mkdir -p "$probe/run"
  install -m 0755 "$d/socialweb-computer-linux-amd64" "$probe/socialweb-computer"
  [ "$(id -u)" = 0 ] && chown -R probe "$probe"
  "${as[@]}" XDG_RUNTIME_DIR="$probe/run" "$probe/socialweb-computer" run bidder --handle 'NOT A HANDLE' \
    --host-backend proc --profile dev-local --state-dir "$probe/state" > "$probe/out.txt" 2>&1 || true
  cat "$probe/out.txt"
  if grep -qE 'guestimagerelease|guest-image-source is required' "$probe/out.txt"; then
    echo "the published binary cannot fetch the guest image of its own build"
    exit 1
  fi
  echo "the published binary resolved its own guest image"
}

case "${1:-}" in
  toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify) "step_${1//-/_}" ;;
  *) echo "usage: scripts/ci-release.sh <toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify>" >&2; exit 2 ;;
esac