#!/usr/bin/env bash # scripts/ci-build-cache.sh restore|save : the release build's caches (/cache/{gomod,gocache, # cargo-target,meshd}, scripts/build-socialweb-computer.sh) in the bucket's build-cache//.tar.zst with a # .sha256 beside it. Content-addressed: every cache holds only what its inputs name, so a build with one restored # gives the bytes of a cold build (-trimpath; verified by comparing SHA256SUMS). # cargo-target rust/Cargo.lock, rust/rust-toolchain.toml (1.2 GB, 353 MB packed) # meshd the git tree of rust/ (socialweb-meshd per arch, as build.sh keys it; 8 MB packed) # Opt-in with BUILD_CACHE_KINDS="cargo-target meshd gomod gocache" (the default is the first two): measured on hat-8 for # d5f6601e, a fully warm release build is 80 s on the asset cache alone, 12 s with all four kinds, but restoring gomod # (510 MB packed) and gocache (3.1 GB packed, 11 GB unpacked) costs 150 s over the public URL, more than the 68 s they save. # gomod go.sum of the module and of each asset module, and the Go version # gocache the same, and linux_amd64 (the go build cache entries are keyed by compiler, flags and sources) # restore: BUILD_CACHE_URL, the bucket's public HTTPS build-cache/ URL (any run can read it, like ASSET_CACHE_URL); a kind # that is absent, unreadable, does not hash to its record or does not unpack is skipped: a cold build is the fallback and # restore never fails. save: only with the bucket key (SPACES_ACCESS_KEY_ID, SPACES_SECRET_ACCESS_KEY: sealed secrets of # main and v* pushes; a fork, a pull or a branch has none and writes nothing), uploads a kind only where the key is absent, # under BUILD_CACHE_PREFIX (default build-cache). Run from a kcp checkout. set -euo pipefail mode="${1:?usage: ci-build-cache.sh restore|save }" cache="$(cd "${2:?usage: ci-build-cache.sh restore|save }" && pwd -P)/cache" read -ra kinds <<<"${BUILD_CACHE_KINDS:-cargo-target meshd}" sha_of() { local h h="$(sha256sum "$1" 2>/dev/null | cut -d' ' -f1)" if [ -z "$h" ]; then echo "sha_of: cannot hash $1" >&2 printf 'unhashable:%s\n' "$1" return 1 fi printf '%s\n' "$h" } want_go="$(awk '$1 == "toolchain" {print $2; exit}' go.mod)" [ -n "$want_go" ] || want_go="go$(awk '$1 == "go" {print $2; exit}' go.mod)" sums="$(sha_of go.sum)" for m in $(jq -r '[.assets[].fetch // {} | .[] | select(.kind == "go") | .module // empty] | unique[]' internal/versions/versions.json); do sums="$sums $(sha_of "$m/go.sum")" done key_of() { case "$1" in gomod) printf '%s\n' "$want_go $sums" ;; gocache) printf '%s\n' "$want_go linux_amd64 $sums" ;; cargo-target) printf '%s\n' "$(sha_of rust/Cargo.lock) $(sha_of rust/rust-toolchain.toml)" ;; meshd) git rev-parse HEAD:rust ;; esac | sha256sum | cut -d' ' -f1 } work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT case "$mode" in restore) : "${BUILD_CACHE_URL:?restore needs BUILD_CACHE_URL}" mkdir -p "$cache" for k in "${kinds[@]}"; do key="$(key_of "$k")" || { echo "build-cache: $k key: cannot be computed: cold"; continue; } u="$BUILD_CACHE_URL/$k/$key" t0=$SECONDS rm -rf "$work/x" && mkdir -p "$work/x" if curl -sfL --retry 2 --max-time 900 -o "$work/$k.tar.zst" "$u.tar.zst" && curl -sfL --max-time 60 -o "$work/$k.sha256" "$u.sha256" && [ -s "$work/$k.sha256" ] && [ "$(sha_of "$work/$k.tar.zst")" = "$(cat "$work/$k.sha256")" ] && zstd -dcq "$work/$k.tar.zst" | tar -x -C "$work/x" && [ -d "$work/x/$k" ]; then rm -rf "${cache:?}/$k" && mv "$work/x/$k" "$cache/$k" echo "build-cache: $k restored from $u ($(du -sh "$cache/$k" | cut -f1), $((SECONDS - t0)) s)" else echo "build-cache: $k key $key is not in the bucket, unreadable or does not unpack: cold" fi rm -rf "$work/$k.tar.zst" "$work/$k.sha256" "$work/x" done ;; save) if [ -z "${SPACES_ACCESS_KEY_ID:-}" ] || [ -z "${SPACES_SECRET_ACCESS_KEY:-}" ] || [ -z "${SPACES_BUCKET:-}" ] || [ -z "${SPACES_REGION:-}" ]; then echo "build-cache: no bucket key here: nothing is saved" exit 0 fi prefix="${BUILD_CACHE_PREFIX:-build-cache}" base="https://$SPACES_BUCKET.$SPACES_REGION.digitaloceanspaces.com/$prefix" n=0 for k in "${kinds[@]}"; do [ -n "$(ls -A "$cache/$k" 2>/dev/null)" ] || { echo "build-cache: $k is empty here: not saved"; continue; } key="$(key_of "$k")" if curl -sfI "$base/$k/$key.sha256" >/dev/null; then echo "build-cache: $k key $key is already in the bucket" continue fi t0=$SECONDS mkdir -p "$work/up/$k" tar -C "$cache" -cf - "$k" | zstd -q -3 -T0 -o "$work/up/$k/$key.tar.zst" sha_of "$work/up/$k/$key.tar.zst" >"$work/up/$k/$key.sha256" echo "build-cache: $k key $key packed ($(du -sh "$work/up/$k/$key.tar.zst" | cut -f1), $((SECONDS - t0)) s)" n=$((n + 1)) done if [ "$n" -gt 0 ]; then deno run -A --no-lock "$(dirname "$0")/publish-spaces.ts" --dir "$work/up" --prefix "$prefix" --content-addressed fi ;; *) echo "usage: ci-build-cache.sh restore|save " >&2; exit 2 ;; esac