0
0
#!/usr/bin/env bash
# The knotless `release` workflow (.knotless/workflows/release.yml), one function per step; the workflow only calls it (plan 0042).
# scripts/ci-release.sh <toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify>
# CI: root on the kcp-build VM (/root/..., /opt/ci/env), KNOTLESS_RUN_REF and KNOTLESS_RUN_SHA set. By hand (hat-8, plan 0044):
# not root; CI_HOME (default ~/.tmp/ci-hand) holds what CI puts under /root, the build root is the host's own
# (~/.tmp/socialweb-computer-build); the commit is HEAD and a publish writes local/<sha>/ only, never main/<sha>/ or
# main/latest/. No tag is built: releases/<version>/ is promote's copy of main/<sha>/ (scripts/promote.sh, plan 0040 step 15).
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.."
SPACES_BUCKET=${SPACES_BUCKET:-socialweb-computer-kcp} SPACES_REGION=${SPACES_REGION:-sfo3} TARGET=${TARGET:-firecracker-x86_64}
sha=${KNOTLESS_RUN_SHA:-$(git rev-parse HEAD)}
ref=${KNOTLESS_RUN_REF:-}
if [ "$(id -u)" = 0 ]; then
work=/root root=/root/release-build
OUT=${OUT:-/root/ci-out}
else
work=${CI_HOME:-$HOME/.tmp/ci-hand} root=$HOME/.tmp/socialweb-computer-build
OUT=${OUT:-$work/ci-out}
ref=${ref:-hand}
# never /tmp on hat-8: a tmpfs with a per-user quota (AGENTS.md)
export TMPDIR=$work/tmp
mkdir -p "$TMPDIR"
fi
base="https://$SPACES_BUCKET.$SPACES_REGION.digitaloceanspaces.com"
[ -f /opt/ci/env ] && . /opt/ci/env
mkdir -p "$work"
vm_readings() {
(while sleep 60; do echo "vm: $(free -m | awk '/Mem:/ {print $3 "/" $2 " MiB used, " $7 " available"}'), load $(cut -d' ' -f1-3 /proc/loadavg)"; done) &
trap "kill $! 2>/dev/null" EXIT
}
pin_of() { jq -er --arg t "$TARGET" --arg f "$1" --arg k "$2" '.guestImages.targets[$t].outputs[$f][$k]' internal/versions/versions.json; }
step_toolchain() {
if bash scripts/knotless-ci-baked.sh; then :; else bash scripts/knotless-ci-toolchain.sh --wasm --deno --arm64; fi
}
step_checks() {
vm_readings
go mod download
export CHECK_CROSS_TARGETS="linux/arm64 darwin/arm64 windows/amd64"
cmp scripts/ci-tools.json tests/modules/staging/ci-tools.json
deploy/check.sh --fast
test -z "$(git status --porcelain)"
}
step_binaries() {
vm_readings
mkdir -p "$root"
export ASSET_CACHE_URL=$base/asset-cache
BUILD_CACHE_URL=$base/build-cache bash scripts/ci-build-cache.sh restore "$root" || echo "build-cache: restore failed: cold build"
local p=() rel tree want got name side
for x in ${CI_PLATFORMS:-linux/amd64 linux/arm64}; do p+=(--platform "$x"); done
bash scripts/build-socialweb-computer.sh "${p[@]}" --commit "$sha" --build-root "$root"
rel="$root/release/$sha"
ls -la "$rel"
want="$(jq -r '.assets.deno.version' internal/versions/versions.json)"
jq -e --arg v "$want" '(.builds | length) > 0 and all(.builds[]; .denoEmbedded.version == $v and (.denoEmbedded.zips | keys | sort) == ["linux/amd64","linux/arm64"])' "$rel/release.json"
rm -rf "$OUT/binaries" && mkdir -p "$OUT/binaries" && cp -a "$rel"/. "$OUT/binaries/"
(cd "$OUT/binaries" && sha256sum -c --quiet SHA256SUMS)
tree="$(mktemp -d)"
zstd -dc "$root/src/1/internal/assets/blobs/linux_amd64/socialweb-computer-k8s-downloads.tar.zst" | tar -x -C "$tree"
rm -rf "$work/ci-downloads" && mkdir -p "$work/ci-downloads"
for side in "$tree"/*.sha256; do
name="$(basename "$side" .sha256)"
want="$(cut -d' ' -f1 "$side")"
got="$(sha256sum "$tree/$name" | cut -d' ' -f1)"
echo "downloads/$want/$name (sha256 $got)"
test "$got" = "$want"
mkdir -p "$work/ci-downloads/$want" && cp "$tree/$name" "$work/ci-downloads/$want/$name"
done
test "$(find "$work/ci-downloads" -type f | wc -l)" -gt 0
}
step_staging_binary() {
mkdir -p oci
local r
if r=$(bash scripts/oci-push.sh "$OUT/binaries/socialweb-computer-linux-amd64" 6h); then
echo "$r" > oci/binary.ref
echo "staging binary: pushed to $r"
else
echo "staging binary: ttl.sh refused the push: no oci/binary.ref (test will fail saying so)"
fi
}
step_test_host() {
local tree rtree
tree="$(git rev-parse HEAD:tests)" rtree="$(git rev-parse HEAD:rust)"
if curl -sfo /dev/null "$base/tests/$tree/SHA256SUMS" && curl -sfo /dev/null "$base/meshd/$rtree/SHA256SUMS"; then
echo "tests/$tree/ and meshd/$rtree/ are published: nothing to build"
return 0
fi
BUILD_HOST=local DIST="$PWD/dist" sh tests/publish-test-host.sh
}
step_page() {
PAGE_WORK="$work/ci-page" DIST="$PWD/dist" bash scripts/ci-page.sh
}
step_guest_image() {
local pin url img="$work/ci-guest-image" published=true f want bytes head got size
pin="$(pin_of image.json sha256)"
url="$base/guest-images/$TARGET/$pin"
rm -rf "$img" "$img.published"
for f in image.json vmlinux rootfs.ext4; do
want="$(pin_of "$f" sha256)" bytes="$(pin_of "$f" bytes)"
head="$(curl -sSfI --retry 3 --max-time 60 "$url/$f" | tr -d '\r')" || head=
got="$(printf '%s\n' "$head" | awk 'tolower($1)=="x-amz-meta-sha256:" {print $2}')"
size="$(printf '%s\n' "$head" | awk 'tolower($1)=="content-length:" {print $2}')"
echo "$TARGET/$f sha256 ${got:-absent} ${size:-?} bytes (pinned $want, $bytes bytes) at $url/$f"
[ "$got" = "$want" ] && [ "$size" = "$bytes" ] || published=false
done
if [ "$published" = true ]; then
echo "guest-image: not building $TARGET: every output is published at $url and matches its pin"
touch "$img.published"
return 0
fi
echo "guest-image: building $TARGET"
mkdir -p "$img" "$work/guest-image-work"
scripts/build-guest-image.sh --target "$TARGET" --work "$work/guest-image-work" --out "$img"
for f in vmlinux rootfs.ext4 image.json; do
want="$(pin_of "$f" sha256)"
got="$(sha256sum "$img/$TARGET/$f" | cut -d' ' -f1)"
echo "$TARGET/$f sha256 $got (pinned $want)"
test "$got" = "$want"
done
}
step_asset_cache() {
scripts/asset-cache-upload.sh "$root/cache/blobs"
}
step_build_cache() {
bash scripts/ci-build-cache.sh save "$root" || echo "build-cache: save failed: the build is not affected"
}
step_stage() {
test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY"
if [ ! -e "$work/ci-guest-image.published" ]; then
deno run -A --no-lock scripts/publish-spaces.ts --dir "$work/ci-guest-image/$TARGET" --prefix "guest-images/$TARGET/$(pin_of image.json sha256)" --content-addressed
fi
deno run -A --no-lock scripts/publish-spaces.ts --dir "$work/ci-downloads" --prefix downloads --content-addressed
deno run -A --no-lock scripts/publish-spaces.ts --dir "$OUT" --prefix "staging/$sha"
}
step_publish() {
[ -d /opt/ci/deno ] && export PATH=/opt/ci/deno:$PATH
test -n "$SPACES_ACCESS_KEY_ID" && test -n "$SPACES_SECRET_ACCESS_KEY"
case "$ref" in
refs/heads/main) set -- --to "main/$sha" --to main/latest ;;
# by hand (no CI ref): local/<sha>/ only; main/<sha>/ is main's CI's, so promote never releases a hand build
*) set -- --to "local/$sha" ;;
esac
deno run -A --no-lock scripts/copy-spaces.ts --from "staging/$sha" "$@"
}
step_verify() {
local at d sum name pin f want size probe
case "$ref" in
refs/heads/main) at="main/$sha" ;;
*) at="local/$sha" ;;
esac
d="$(mktemp -d)"
curl -sSf -o "$d/STAGED" "$base/staging/$sha/binaries/SHA256SUMS"
curl -sSf -o "$d/SHA256SUMS" "$base/$at/binaries/SHA256SUMS"
diff "$d/STAGED" "$d/SHA256SUMS"
if [ "$ref" = refs/heads/main ]; then
curl -sSf -o "$d/LATEST" "$base/main/latest/binaries/SHA256SUMS"
diff "$d/STAGED" "$d/LATEST"
fi
while read -r sum name; do
curl -sSfL -o "$d/$name" "$base/$at/binaries/$name"
echo "$sum $d/$name" | sha256sum -c --quiet -
echo "$at/binaries/$name: sha256 $sum, readable without credentials"
done < "$d/SHA256SUMS"
pin="$(pin_of image.json sha256)"
for f in image.json vmlinux rootfs.ext4; do
want="$(pin_of "$f" bytes)"
size="$(curl -sSI --max-time 60 "$base/guest-images/$TARGET/$pin/$f" | tr -d '\r' | awk 'tolower($1)=="content-length:" {print $2}')"
echo "guest-images/$TARGET/$pin/$f $size bytes (pinned $want)"
test "$size" = "$want"
done
local as=()
if [ "$(id -u)" = 0 ]; then
id probe >/dev/null 2>&1 || useradd -m probe
probe=/home/probe/p
as=(runuser -u probe -- env HOME=/home/probe)
else
probe=$work/probe
as=(env HOME="$HOME")
fi
rm -rf "$probe" && mkdir -p "$probe/run"
install -m 0755 "$d/socialweb-computer-linux-amd64" "$probe/socialweb-computer"
[ "$(id -u)" = 0 ] && chown -R probe "$probe"
"${as[@]}" XDG_RUNTIME_DIR="$probe/run" "$probe/socialweb-computer" run bidder --handle 'NOT A HANDLE' \
--host-backend proc --profile dev-local --state-dir "$probe/state" > "$probe/out.txt" 2>&1 || true
cat "$probe/out.txt"
if grep -qE 'guestimagerelease|guest-image-source is required' "$probe/out.txt"; then
echo "the published binary cannot fetch the guest image of its own build"
exit 1
fi
echo "the published binary resolved its own guest image"
}
case "${1:-}" in
toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify) "step_${1//-/_}" ;;
*) echo "usage: scripts/ci-release.sh <toolchain|checks|binaries|staging-binary|test-host|page|guest-image|asset-cache|build-cache|stage|publish|verify>" >&2; exit 2 ;;
esac