#!/usr/bin/env bash # The knotless `release` workflow (.knotless/workflows/release.yml), one function per step; the workflow only calls it (plan 0042). # scripts/ci-release.sh # CI: root on the kcp-build VM (/root/..., /opt/ci/env), KNOTLESS_RUN_REF and KNOTLESS_RUN_SHA set. By hand (hat-8, plan 0044): # not root; CI_HOME (default ~/.tmp/ci-hand) holds what CI puts under /root, the build root is the host's own # (~/.tmp/socialweb-computer-build); the commit is HEAD and a publish writes local// only, never main// or # main/latest/. No tag is built: releases// is promote's copy of main// (scripts/promote.sh, plan 0040 step 15). set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." SPACES_BUCKET=${SPACES_BUCKET:-socialweb-computer-kcp} SPACES_REGION=${SPACES_REGION:-sfo3} TARGET=${TARGET:-firecracker-x86_64} sha=${KNOTLESS_RUN_SHA:-$(git rev-parse HEAD)} ref=${KNOTLESS_RUN_REF:-} if [ "$(id -u)" = 0 ]; then work=/root root=/root/release-build OUT=${OUT:-/root/ci-out} else work=${CI_HOME:-$HOME/.tmp/ci-hand} root=$HOME/.tmp/socialweb-computer-build OUT=${OUT:-$work/ci-out} ref=${ref:-hand} # never /tmp on hat-8: a tmpfs with a per-user quota (AGENTS.md) export TMPDIR=$work/tmp mkdir -p "$TMPDIR" fi base="https://$SPACES_BUCKET.$SPACES_REGION.digitaloceanspaces.com" [ -f /opt/ci/env ] && . /opt/ci/env mkdir -p "$work" vm_readings() { (while sleep 60; do echo "vm: $(free -m | awk '/Mem:/ {print $3 "/" $2 " MiB used, " $7 " available"}'), load $(cut -d' ' -f1-3 /proc/loadavg)"; done) & trap "kill $! 2>/dev/null" EXIT } pin_of() { jq -er --arg t "$TARGET" --arg f "$1" --arg k "$2" '.guestImages.targets[$t].outputs[$f][$k]' internal/versions/versions.json; } step_toolchain() { if bash scripts/knotless-ci-baked.sh; then :; else bash scripts/knotless-ci-toolchain.sh --wasm --deno --arm64; fi } step_checks() { vm_readings go mod download export CHECK_CROSS_TARGETS="linux/arm64 darwin/arm64 windows/amd64" cmp scripts/ci-tools.json tests/modules/staging/ci-tools.json deploy/check.sh --fast test -z "$(git status --porcelain)" } step_binaries() { vm_readings mkdir -p "$root" export ASSET_CACHE_URL=$base/asset-cache BUILD_CACHE_URL=$base/build-cache bash scripts/ci-build-cache.sh restore "$root" || echo "build-cache: restore failed: cold build" local p=() rel tree want got name side for x in ${CI_PLATFORMS:-linux/amd64 linux/arm64}; do p+=(--platform "$x"); done bash scripts/build-socialweb-computer.sh "${p[@]}" --commit "$sha" --build-root "$root" rel="$root/release/$sha" ls -la "$rel" want="$(jq -r '.assets.deno.version' internal/versions/versions.json)" jq -e --arg v "$want" '(.builds | length) > 0 and all(.builds[]; .denoEmbedded.version == $v and (.denoEmbedded.zips | keys | sort) == ["linux/amd64","linux/arm64"])' "$rel/release.json" rm -rf "$OUT/binaries" && mkdir -p "$OUT/binaries" && cp -a "$rel"/. "$OUT/binaries/" (cd "$OUT/binaries" && sha256sum -c --quiet SHA256SUMS) tree="$(mktemp -d)" zstd -dc "$root/src/1/internal/assets/blobs/linux_amd64/socialweb-computer-k8s-downloads.tar.zst" | tar -x -C "$tree" rm -rf "$work/ci-downloads" && mkdir -p "$work/ci-downloads" for side in "$tree"/*.sha256; do name="$(basename "$side" .sha256)" want="$(cut -d' ' -f1 "$side")" got="$(sha256sum "$tree/$name" | cut -d' ' -f1)" echo "downloads/$want/$name (sha256 $got)" test "$got" = "$want" mkdir -p "$work/ci-downloads/$want" && cp "$tree/$name" "$work/ci-downloads/$want/$name" done test "$(find "$work/ci-downloads" -type f | wc -l)" -gt 0 } step_staging_binary() { mkdir -p oci local r if r=$(bash scripts/oci-push.sh "$OUT/binaries/socialweb-computer-linux-amd64" 6h); then echo "$r" > oci/binary.ref echo "staging binary: pushed to $r" else echo "staging binary: ttl.sh refused the push: no oci/binary.ref (test will fail saying so)" fi } step_test_host() { local tree rtree tree="$(git rev-parse HEAD:tests)" rtree="$(git rev-parse HEAD:rust)" if curl -sfo /dev/null "$base/tests/$tree/SHA256SUMS" && curl -sfo /dev/null "$base/meshd/$rtree/SHA256SUMS"; then echo "tests/$tree/ and meshd/$rtree/ are published: nothing to build" return 0 fi BUILD_HOST=local DIST="$PWD/dist" sh tests/publish-test-host.sh } step_page() { PAGE_WORK="$work/ci-page" DIST="$PWD/dist" bash scripts/ci-page.sh } step_guest_image() { local pin url img="$work/ci-guest-image" published=true f want bytes head got size pin="$(pin_of image.json sha256)" url="$base/guest-images/$TARGET/$pin" rm -rf "$img" "$img.published" for f in image.json vmlinux rootfs.ext4; do want="$(pin_of "$f" sha256)" bytes="$(pin_of "$f" bytes)" head="$(curl -sSfI --retry 3 --max-time 60 "$url/$f" | tr -d '\r')" || head= got="$(printf '%s\n' "$head" | awk 'tolower($1)=="x-amz-meta-sha256:" {print $2}')" size="$(printf '%s\n' "$head" | awk 'tolower($1)=="content-length:" {print $2}')" echo "$TARGET/$f sha256 ${got:-absent} ${size:-?} bytes (pinned $want, $bytes bytes) at $url/$f" [ "$got" = "$want" ] && [ "$size" = "$bytes" ] || published=false done if [ "$published" = true ]; then echo "guest-image: not building $TARGET: every output is published at $url and matches its pin" touch "$img.published" return 0 fi echo "guest-image: building $TARGET" mkdir -p "$img" "$work/guest-image-work" scripts/build-guest-image.sh --target "$TARGET" --work "$work/guest-image-work" --out "$img" for f in vmlinux rootfs.ext4 image.json; do want="$(pin_of "$f" sha256)" got="$(sha256sum "$img/$TARGET/$f" | cut -d' ' -f1)" echo "$TARGET/$f sha256 $got (pinned $want)" test "$got" = "$want" done } step_asset_cache() { scripts/asset-cache-upload.sh "$root/cache/blobs" } step_build_cache() { bash scripts/ci-build-cache.sh save "$root" || echo "build-cache: save failed: the build is not affected" } step_stage() { test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY" if [ ! -e "$work/ci-guest-image.published" ]; then deno run -A --no-lock scripts/publish-spaces.ts --dir "$work/ci-guest-image/$TARGET" --prefix "guest-images/$TARGET/$(pin_of image.json sha256)" --content-addressed fi deno run -A --no-lock scripts/publish-spaces.ts --dir "$work/ci-downloads" --prefix downloads --content-addressed deno run -A --no-lock scripts/publish-spaces.ts --dir "$OUT" --prefix "staging/$sha" } step_publish() { [ -d /opt/ci/deno ] && export PATH=/opt/ci/deno:$PATH test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY" case "$ref" in refs/heads/main) set -- --to "main/$sha" --to main/latest ;; # by hand (no CI ref): local// only; main// is main's CI's, so promote never releases a hand build *) set -- --to "local/$sha" ;; esac deno run -A --no-lock scripts/copy-spaces.ts --from "staging/$sha" "$@" } step_verify() { local at d sum name pin f want size probe case "$ref" in refs/heads/main) at="main/$sha" ;; *) at="local/$sha" ;; esac d="$(mktemp -d)" curl -sSf -o "$d/STAGED" "$base/staging/$sha/binaries/SHA256SUMS" curl -sSf -o "$d/SHA256SUMS" "$base/$at/binaries/SHA256SUMS" diff "$d/STAGED" "$d/SHA256SUMS" if [ "$ref" = refs/heads/main ]; then curl -sSf -o "$d/LATEST" "$base/main/latest/binaries/SHA256SUMS" diff "$d/STAGED" "$d/LATEST" fi while read -r sum name; do curl -sSfL -o "$d/$name" "$base/$at/binaries/$name" echo "$sum $d/$name" | sha256sum -c --quiet - echo "$at/binaries/$name: sha256 $sum, readable without credentials" done < "$d/SHA256SUMS" pin="$(pin_of image.json sha256)" for f in image.json vmlinux rootfs.ext4; do want="$(pin_of "$f" bytes)" size="$(curl -sSI --max-time 60 "$base/guest-images/$TARGET/$pin/$f" | tr -d '\r' | awk 'tolower($1)=="content-length:" {print $2}')" echo "guest-images/$TARGET/$pin/$f $size bytes (pinned $want)" test "$size" = "$want" done local as=() if [ "$(id -u)" = 0 ]; then id probe >/dev/null 2>&1 || useradd -m probe probe=/home/probe/p as=(runuser -u probe -- env HOME=/home/probe) else probe=$work/probe as=(env HOME="$HOME") fi rm -rf "$probe" && mkdir -p "$probe/run" install -m 0755 "$d/socialweb-computer-linux-amd64" "$probe/socialweb-computer" [ "$(id -u)" = 0 ] && chown -R probe "$probe" "${as[@]}" XDG_RUNTIME_DIR="$probe/run" "$probe/socialweb-computer" run bidder --handle 'NOT A HANDLE' \ --host-backend proc --profile dev-local --state-dir "$probe/state" > "$probe/out.txt" 2>&1 || true cat "$probe/out.txt" if grep -qE 'guestimagerelease|guest-image-source is required' "$probe/out.txt"; then echo "the published binary cannot fetch the guest image of its own build" exit 1 fi echo "the published binary resolved its own guest image" } case "${1:-}" in toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify) "step_${1//-/_}" ;; *) echo "usage: scripts/ci-release.sh " >&2; exit 2 ;; esac