#!/usr/bin/env bash set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" versions="$root/internal/versions/versions.json" blobs="$root/internal/assets/blobs" cache="" print_pins=0 update_pins=0 print_src_key=0 blob_cache="" jobs=6 platforms=() only=() usage() { echo "usage: $0 [--platform linux/amd64|linux/arm64]... [--asset NAME]... [--cache DIR] [--blobs DIR] [--blob-cache DIR] [--jobs N] [--print-pins|--update-pins|--print-src-key]" >&2 echo " downloads every pinned asset of versions.json, verifies its sha256 (and the upstream signature" >&2 echo " where one is published), repacks it as internal/assets/blobs/_/.tar.zst and" >&2 echo " checks the unpacked tree against its treeSha256 pin. An asset built from this repository's own commit" >&2 echo " (fetch kind go or script) has NO pin: its tree digest, sizes and binary sha256 are written beside its blob as" >&2 echo " /_/.built.json (embedded in the release binary, which checks the unpacked tree" >&2 echo " against it)." >&2 echo " --print-pins prints the measured digests and stages even when a tree pin differs. --update-pins writes the" >&2 echo " new pins of the assets built here from pinned upstream source (fetch kind gosrc or cargosrc) into" >&2 echo " versions.json; pins of downloaded assets are never changed. --print-src-key (with --blob-cache, one --asset" >&2 echo " and one --platform) prints the source-input key of a built asset and stages nothing. --cache keeps" >&2 echo " the downloads (keyed by sha256) in DIR instead of internal/assets/blobs/.cache. --blobs writes" >&2 echo " _/.tar.zst under DIR instead of internal/assets/blobs (the --dev build's cache)." >&2 echo " --blob-cache keeps every zstd blob in DIR: an asset of pinned inputs (kind file, oci, debs, gosrc, cargosrc) by" >&2 echo " the sha256 of its versions.json entry, this script, the sources of assetpack and the zstd version and flags (a hit" >&2 echo " skips its download or build, verification and repack); an asset built from this checkout (kind go, script) is rebuilt" >&2 echo " unless its tracked source inputs hash as before (kind go: the in-repo packages it builds from," >&2 echo " go.mod and go.sum; kind script: the script, the paths its '# stage-assets inputs:' line names (go::" >&2 echo " for a package) and versions.json), by those. ASSET_CACHE_URL (the bucket's asset-cache/): a blob the" >&2 echo " --blob-cache misses is fetched from //_/.tar.zst and its tree checked against" >&2 echo " its pin (a built asset: that it unpacks); each blob put in the --blob-cache is listed in /.new for scripts/asset-cache-upload.sh." >&2 echo " --jobs stages up to N assets at once (default 6)." >&2 exit 2 } while [ $# -gt 0 ]; do case "$1" in --platform) platforms+=("$2"); shift 2 ;; --asset) only+=("$2"); shift 2 ;; --cache) cache="$2"; shift 2 ;; --blobs) blobs="$2"; shift 2 ;; --blob-cache) blob_cache="$2"; shift 2 ;; --jobs) jobs="$2"; shift 2 ;; --print-pins) print_pins=1; shift ;; --update-pins) update_pins=1; shift ;; --print-src-key) print_src_key=1; shift ;; -h|--help) usage ;; *) usage ;; esac done [ "$((print_pins + update_pins + print_src_key))" -le 1 ] || { echo "stage-assets: --print-pins, --update-pins and --print-src-key are exclusive" >&2; exit 2; } for tool in curl jq gpg ar tar zstd sha256sum unzip go xz cmp python3; do command -v "$tool" >/dev/null 2>&1 || { echo "stage-assets: $tool is required on the build host" >&2; exit 2; } done if [ ${#platforms[@]} -eq 0 ]; then mapfile -t platforms < <(jq -r '.platforms | to_entries[] | select(.value.status == "release") | .key' "$versions") fi cache="${cache:-$blobs/.cache}" mkdir -p "$cache" "$blobs/.work" work="$(mktemp -d "$blobs/.work/run.XXXXXX")" trap 'chmod -R u+w "$work" 2>/dev/null; rm -rf "$work"' EXIT pins_dir="" if [ "$update_pins" = 1 ]; then pins_dir="$work/pins" mkdir -p "$pins_dir" fi die() { echo "stage-assets: $*" >&2; exit 1; } say() { echo "stage-assets: $*" >&2; } want_go="$(awk '$1=="toolchain"{print $2; exit}' "$root/go.mod")" [ -n "$want_go" ] || want_go="go$(awk '$1=="go"{print $2; exit}' "$root/go.mod")" pinned_goroot="$(GOTOOLCHAIN="$want_go" go env GOROOT)" || die "cannot get the pinned toolchain $want_go" export PATH="$pinned_goroot/bin:$PATH" GOTOOLCHAIN=local [ "$(go env GOVERSION)" = "$want_go" ] || die "the toolchain at $pinned_goroot is $(go env GOVERSION); go.mod pins $want_go" sha_of() { local h h="$(sha256sum "$1" 2>/dev/null | cut -d' ' -f1)" if [ -z "$h" ]; then echo "sha_of: cannot hash $1" >&2 printf 'unhashable:%s\n' "$1" return 1 fi printf '%s\n' "$h" } dl_args=(-fSL --retry 5 --retry-all-errors --retry-delay 3 --connect-timeout 10 --speed-limit 10000 --speed-time 20 --progress-bar) api_args=(-fsSL --retry 3 --connect-timeout 10 --speed-limit 10000 --speed-time 20) fetch_pinned() { local url="$1" want="$2" out="$cache/$2" if [ -f "$out" ] && [ "$(sha_of "$out")" = "$want" ]; then echo "$out" return fi local part="$out.part.$BASHPID" got curl "${dl_args[@]}" -o "$part" "$url" || { rm -f "$part"; die "download failed: $url"; } got="$(sha_of "$part")" if [ "$got" != "$want" ]; then rm -f "$part" die "sha256 mismatch for $url: got $got, pinned $want" fi mv -f "$part" "$out" echo "$out" } fetch_fresh() { local url="$1" out="$2" curl "${dl_args[@]}" -o "$out" "$url" || die "download failed: $url" } key_fpr_ok() { local ring="$1" fpr="$2" gpg --batch --no-default-keyring --keyring "$ring" --with-colons --list-keys 2>/dev/null \ | awk -F: '$1=="pub"{p=1;next} p&&$1=="fpr"{print $10; p=0}' | grep -qx "$fpr" } fetch_key() { local url="$1" fpr="$2" out="$3" candidate try mkdir -p "$(dirname "$out")" "$GNUPGHOME/keys" for candidate in "$url" "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x$fpr" "https://keys.openpgp.org/vks/v1/by-fingerprint/$fpr"; do try="$GNUPGHOME/keys/$fpr.try" rm -f "$try" "$try.gpg" curl "${dl_args[@]}" -o "$try" "$candidate" 2>/dev/null || { say "key $fpr: $candidate unavailable"; continue; } gpg --batch --quiet --no-default-keyring --keyring "$try.gpg" --import "$try" 2>/dev/null || true if key_fpr_ok "$try.gpg" "$fpr"; then mv -f "$try" "$out.part.$$" && mv -f "$out.part.$$" "$out" say "key $fpr: vendored from $candidate into $out" return fi say "key $fpr: $candidate gave no importable key with that primary fingerprint" done die "key $fpr: no source gave it" } key_ring() { local url="$1" fpr="$2" ring="$GNUPGHOME/keys/$2.gpg" vendored="$root/internal/versions/keys/$2.asc" if [ ! -f "$ring" ]; then mkdir -p "$GNUPGHOME/keys" if [ ! -f "$vendored" ]; then [ "$update_pins" = 1 ] || die "key $fpr is not vendored at $vendored; make pins fetches it" fetch_key "$url" "$fpr" "$vendored" fi gpg --batch --quiet --no-default-keyring --keyring "$ring" --import "$vendored" 2>/dev/null key_fpr_ok "$ring" "$fpr" || die "$vendored does not carry primary fingerprint $fpr" fi echo "$ring" } gpg_verify() { local ring="$1" fpr="$2" sig="$3" data="${4:-}" out="${5:-}" status if [ -n "$data" ]; then status="$(gpg --batch --no-default-keyring --keyring "$ring" --trust-model always --status-fd 1 --verify "$sig" "$data" 2>/dev/null || true)" else status="$(gpg --batch --no-default-keyring --keyring "$ring" --trust-model always --status-fd 1 --output "$out" --decrypt "$sig" 2>/dev/null || true)" fi echo "$status" | awk '$2=="VALIDSIG"{print $NF}' | grep -qx "$fpr" || die "signature on $sig does not verify with $fpr" } check_checksums() { local f="$1" name="$2" want="$3" if awk -v n="$name" -v w="$want" '{s=$NF; sub(/^\*/,"",s); sub(/^.*\//,"",s)} $1==w && (s==n || NF==1){ok=1} END{exit !ok}' "$f"; then return fi die "checksum file for $name does not list the pinned $want" } copy_members() { local from="$1" to="$2" members="$3" src dst while IFS=$'\t' read -r src dst; do [ -n "$src" ] || continue if [ ! -e "$from/$src" ]; then (cd "$from" && find . | awk 'NR<=40') >&2 die "member $src is not in the unpacked archive (listing above)" fi mkdir -p "$(dirname "$to/$dst")" cp -L "$from/$src" "$to/$dst" chmod 0755 "$to/$dst" done < <(echo "$members" | jq -r 'to_entries[] | "\(.key)\t\(.value)"') } stage_file() { local spec="$1" tree="$2" x="$3" url want archive file url="$(echo "$spec" | jq -r .url)" want="$(echo "$spec" | jq -r .sha256)" archive="$(echo "$spec" | jq -r .archive)" file="$(fetch_pinned "$url" "$want")" if [ "$(echo "$spec" | jq -r '.checksums // empty')" != "" ]; then local cs name sig cs="$x/checksums" fetch_fresh "$(echo "$spec" | jq -r .checksums.url)" "$cs" name="$(echo "$spec" | jq -r .checksums.name)" sig="$(echo "$spec" | jq -r '.checksums.signature // empty')" if [ -n "$sig" ]; then local fpr ring fpr="$(echo "$spec" | jq -r .checksums.fingerprint)" ring="$(key_ring "$(echo "$spec" | jq -r .checksums.key)" "$fpr")" fetch_fresh "$sig" "$x/checksums.sig" gpg_verify "$ring" "$fpr" "$x/checksums.sig" "$cs" fi check_checksums "$cs" "$name" "$want" fi mkdir -p "$x/u" case "$archive" in tar.gz) tar -xzf "$file" -C "$x/u" ;; zip) unzip -q "$file" -d "$x/u" ;; raw) cp "$file" "$x/u/$(echo "$spec" | jq -r '.members | keys[0]')" ;; *) die "unknown archive kind $archive" ;; esac copy_members "$x/u" "$tree" "$(echo "$spec" | jq -c .members)" } stage_oci() { local spec="$1" tree="$2" x="$3" plat="$4" reg repo index manifest layer tok f reg="$(echo "$spec" | jq -r .registry)" repo="$(echo "$spec" | jq -r .repository)" index="$(echo "$spec" | jq -r .index)" manifest="$(echo "$spec" | jq -r .manifest)" layer="$(echo "$spec" | jq -r .layer)" tok="$(curl "${api_args[@]}" "https://$reg/token?scope=repository:$repo:pull" | jq -r .token)" local accept="application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json" curl "${api_args[@]}" -H "Authorization: Bearer $tok" -H "Accept: $accept" -o "$x/index.json" "https://$reg/v2/$repo/manifests/$index" [ "sha256:$(sha_of "$x/index.json")" = "$index" ] || die "$repo index does not hash to $index" jq -e --arg d "$manifest" --arg os "${plat%/*}" --arg a "${plat#*/}" \ '.manifests[] | select(.digest == $d and .platform.os == $os and .platform.architecture == $a)' "$x/index.json" >/dev/null \ || die "$repo index $index does not name $manifest for $plat" curl "${api_args[@]}" -H "Authorization: Bearer $tok" -H "Accept: $accept" -o "$x/manifest.json" "https://$reg/v2/$repo/manifests/$manifest" [ "sha256:$(sha_of "$x/manifest.json")" = "$manifest" ] || die "$repo manifest does not hash to $manifest" jq -e --arg d "$layer" '.layers[] | select(.digest == $d)' "$x/manifest.json" >/dev/null || die "$manifest does not name layer $layer" f="$cache/${layer#sha256:}" if [ ! -f "$f" ] || [ "sha256:$(sha_of "$f")" != "$layer" ]; then local part="$f.part.$BASHPID" curl "${dl_args[@]}" -H "Authorization: Bearer $tok" -o "$part" "https://$reg/v2/$repo/blobs/$layer" [ "sha256:$(sha_of "$part")" = "$layer" ] || { rm -f "$part"; die "layer does not hash to $layer"; } mv -f "$part" "$f" fi mkdir -p "$x/u" tar -xzf "$f" -C "$x/u" copy_members "$x/u" "$tree" "$(echo "$spec" | jq -c .members)" } stage_script() { local spec="$1" tree="$2" x="$3" script want out script="$root/$(echo "$spec" | jq -r .script)" want="$(echo "$spec" | jq -r '.sha256 // ""')" mkdir -p "$x/u" "$script" --out "$x/u" >&2 out="$x/u/$(echo "$spec" | jq -r '.members | keys[0]')" if [ -n "$want" ] && [ "$(sha_of "$out")" != "$want" ]; then [ "$update_pins" = 1 ] || die "$script built $(sha_of "$out"), pinned $want" fi copy_members "$x/u" "$tree" "$(echo "$spec" | jq -c .members)" } stage_go() { local spec="$1" tree="$2" x="$3" plat="$4" pkg module want out pkg="$(echo "$spec" | jq -r .package)" module="$(echo "$spec" | jq -r '.module // "."')" want="$(echo "$spec" | jq -r '.sha256 // ""')" out="$x/u/$(echo "$spec" | jq -r '.members | keys[0]')" mkdir -p "$x/u" (cd "$root/$module" && CGO_ENABLED=0 GOOS="${plat%/*}" GOARCH="${plat#*/}" go build -trimpath -buildvcs=false \ -ldflags='-s -w -buildid=' -o "$out" "$pkg") say "$module $pkg $plat binary sha256 $(sha_of "$out")" if [ -n "$want" ] && [ "$(sha_of "$out")" != "$want" ]; then echo "stage-assets: $module $pkg $plat built $(sha_of "$out"), pinned $want" >&2 [ "$print_pins" = 1 ] || [ "$update_pins" = 1 ] || die "$module $pkg $plat: sha256 differs from the pin" fi copy_members "$x/u" "$tree" "$(echo "$spec" | jq -c .members)" } gosrc_proxy="https://proxy.golang.org" stage_gosrc() { local asset="$1" spec="$2" tree="$3" x="$4" plat="$5" src url want commit toolchain pkg file comment goroot ldflags p k out src="$(jq -c --arg a "$asset" '.assets[$a].gosrc' "$versions")" [ "$src" != "null" ] || die "$asset: fetch kind gosrc needs an asset-level gosrc object" url="$(echo "$src" | jq -r .url)" want="$(echo "$src" | jq -r .sha256)" commit="$(echo "$src" | jq -r .commit)" toolchain="$(echo "$src" | jq -r .toolchain)" pkg="$(echo "$src" | jq -r .package)" file="$(fetch_pinned "$url" "$want")" comment="$(python3 -I -c 'import sys, tarfile; print(tarfile.open(sys.argv[1]).pax_headers.get("comment", ""))' "$file")" [ "$comment" = "$commit" ] || die "$url: the archive names commit '$comment', pinned $commit" mkdir -p "$x/src" "$x/u" tar -xzf "$file" -C "$x/src" --strip-components=1 ldflags="-s -w -buildid=" for p in $(echo "$src" | jq -r '.versionPackages[]'); do while IFS=$'\t' read -r k v; do ldflags="$ldflags -X $p.$k=$v" done < <(echo "$src" | jq -r '.versionVars | to_entries[] | "\(.key)\t\(.value)"') done out="$x/u/$(echo "$spec" | jq -r '.members | keys[0]')" goroot="$(GOTOOLCHAIN="$toolchain" GOPROXY="$gosrc_proxy" GOSUMDB=sum.golang.org go env GOROOT)" || die "cannot get the toolchain $toolchain" (cd "$x/src" && PATH="$goroot/bin:$PATH" GOTOOLCHAIN=local GOWORK=off GOFLAGS=-mod=readonly GOPROXY="$gosrc_proxy" \ GOSUMDB=sum.golang.org GONOSUMDB= GOPRIVATE= GONOPROXY= GOINSECURE= CGO_ENABLED=0 GOOS="${plat%/*}" GOARCH="${plat#*/}" \ go build -trimpath -buildvcs=false -ldflags="$ldflags" -o "$out" "$pkg") [ "$(PATH="$goroot/bin:$PATH" GOTOOLCHAIN=local go version "$out" | awk '{print $2}')" = "$toolchain" ] || die "$asset $plat was not built by $toolchain" say "$asset $plat built from $commit with $toolchain, binary sha256 $(sha_of "$out")" copy_members "$x/u" "$tree" "$(echo "$spec" | jq -c .members)" } stage_cargosrc() { local asset="$1" spec="$2" tree="$3" x="$4" plat="$5" arch out case "$plat" in linux/amd64) arch=x86_64 ;; linux/arm64) arch=aarch64 ;; *) die "$asset: no cargosrc target for $plat" ;; esac out="$x/u/$(echo "$spec" | jq -r '.members | keys[0]')" "$root/scripts/build-cargosrc.sh" --asset "$asset" --arch "$arch" --out "$out" >&2 copy_members "$x/u" "$tree" "$(echo "$spec" | jq -c .members)" } deb_arch() { case "$1" in linux/amd64) echo amd64 ;; linux/arm64) echo arm64 ;; *) die "no deb arch for $1" ;; esac; } # Each deb by its pinned sha256 from its repo's durable file URL (`files`: {pool} and {file} filled in), never a # moving index: Launchpad keeps every file Ubuntu published, apt-archive.postgresql.org every pgdg version. stage_debs() { local asset="$1" spec="$2" tree="$3" x="$4" plat="$5" arch n i pkg ver repo pool want files deb suffix url arch="$(deb_arch "$plat")" mkdir -p "$x/root" n="$(jq -r --arg a "$asset" '.assets[$a].debs | length' "$versions")" for ((i = 0; i < n; i++)); do read -r pkg ver repo pool want < <(jq -r --arg a "$asset" --arg p "$plat" --argjson i "$i" \ '.assets[$a].debs[$i] | "\(.package) \(.version) \(.repo) \(.pool) \(.sha256[$p])"' "$versions") files="$(jq -r --arg a "$asset" --arg r "$repo" '.assets[$a].repos[$r].files' "$versions")" deb="" for suffix in "$arch" all; do url="${files//\{pool\}/$pool}" url="${url//\{file\}/${pkg}_${ver#*:}_$suffix.deb}" deb="$(fetch_pinned "$url" "$want" 2>/dev/null)" && break deb="" done [ -n "$deb" ] || die "$pkg $ver ($arch): no file with the pinned sha256 $want at $files (tried _$arch.deb and _all.deb)" rm -rf "$x/deb" && mkdir -p "$x/deb" (cd "$x/deb" && ar x "$deb") tar -xf "$x"/deb/data.tar.* -C "$x/root" done case "$(echo "$spec" | jq -r .layout)" in members) copy_members "$x/root" "$tree" "$(echo "$spec" | jq -c .members)" ;; pgdg-tree) local triplet case "$arch" in amd64) triplet=x86_64-linux-gnu ;; arm64) triplet=aarch64-linux-gnu ;; esac mkdir -p "$tree/usr/lib/postgresql/18" "$tree/usr/share/postgresql" "$tree/usr/lib/sys" cp -a "$x/root/usr/lib/postgresql/18/bin" "$x/root/usr/lib/postgresql/18/lib" "$tree/usr/lib/postgresql/18/" cp -a "$x/root/usr/share/postgresql/18" "$tree/usr/share/postgresql/" rm -rf "$tree/usr/share/postgresql/18/man" "$tree/usr/lib/postgresql/18/lib/bitcode" for d in "$x/root/lib/$triplet" "$x/root/usr/lib/$triplet"; do [ -d "$d" ] || continue find "$d" -maxdepth 1 -name '*.so*' \( -type f -o -type l \) -exec cp -a {} "$tree/usr/lib/sys/" \; done find "$tree" -type l | while read -r l; do case "$(readlink "$l")" in /*) die "absolute symlink $l -> $(readlink "$l") in the postgres tree" ;; esac done ;; lib-tree) local triplet case "$arch" in amd64) triplet=x86_64-linux-gnu ;; arm64) triplet=aarch64-linux-gnu ;; esac copy_members "$x/root" "$tree" "$(echo "$spec" | jq -c .members)" mkdir -p "$tree/usr/lib/sys" for d in "$x/root/lib/$triplet" "$x/root/usr/lib/$triplet"; do [ -d "$d" ] || continue find "$d" -maxdepth 1 -name '*.so*' \( -type f -o -type l \) -exec cp -a {} "$tree/usr/lib/sys/" \; done find "$tree" -type l | while read -r l; do case "$(readlink "$l")" in /*) die "absolute symlink $l -> $(readlink "$l") in the $asset tree" ;; esac done ;; *) die "unknown deb layout" ;; esac } (cd "$root" && go build -trimpath -buildvcs=false -o "$work/assetpack" ./internal/assets/assetpack) zstd_args=(-q -19 --long=27 -T0) pack_id="" script_sha="" if [ -n "$blob_cache" ]; then mkdir -p "$blob_cache" blob_cache="$(cd "$blob_cache" && pwd -P)" pack_src="$(cd "$root" && cat internal/assets/assetpack/*.go internal/assets/*.go internal/kit/sha256sum/*.go | sha256sum | cut -d' ' -f1)" pack_id="$(printf '%s\n' "$pack_src" "$(zstd -V)" "${zstd_args[*]}" | sha256sum | cut -d' ' -f1)" script_sha="$(sha_of "${BASH_SOURCE[0]}")" fi place() { ln -f "$1" "$2" 2>/dev/null || cp -f "$1" "$2" } cache_get() { local f="$blob_cache/$1.tar.zst" rec="$blob_cache/$1.sha256" u t pid="$BASHPID" if { [ ! -f "$f" ] || [ ! -f "$rec" ]; } && [ -n "${ASSET_CACHE_URL:-}" ] && [ -n "${3:-}" ]; then u="$ASSET_CACHE_URL/$asset/${plat/\//_}/$1" if curl -sfL --retry 2 --max-time 900 -o "$f.$pid" "$u.tar.zst" && curl -sfL --max-time 60 -o "$rec.$pid" "$u.sha256" && [ -s "$rec.$pid" ] && [ "$(sha_of "$f.$pid")" = "$(cat "$rec.$pid")" ]; then t="$work/remote-$1" mkdir -p "$t" && zstd -dcq "$f.$pid" | tar -x -C "$t" if [ "$3" = - ] && [ -n "$("$work/assetpack" digest "$t")" ] || [ "$("$work/assetpack" digest "$t")" = "$3" ]; then mv -f "$f.$pid" "$f" && mv -f "$rec.$pid" "$rec" say "$plat $asset: $1 from the asset cache, tree $3" else say "warn: $u does not unpack to the pinned tree $3 (or, for a built asset, does not unpack); staging it" fi chmod -R u+w "$t" && rm -rf "$t" fi rm -f "$f.$pid" "$rec.$pid" fi [ -f "$f" ] && [ -f "$rec" ] || return 1 if [ ! -s "$rec" ] || [ "$(sha_of "$f")" != "$(cat "$rec")" ]; then say "warn: cached blob $f does not hash to its record; staging again" rm -f "$f" "$rec" return 1 fi place "$f" "$2" } cache_put() { local tmp="$blob_cache/.$1.$BASHPID" place "$2" "$tmp.tar.zst" sha_of "$tmp.tar.zst" >"$tmp.sha256" mv -f "$tmp.tar.zst" "$blob_cache/$1.tar.zst" mv -f "$tmp.sha256" "$blob_cache/$1.sha256" echo "$asset/${plat/\//_}/$1" >>"$blob_cache/.new" } src_files() { local w m p d for w in "$@"; do case "$w" in go:*) m="${w#go:}" && p="${m#*:}" && m="${m%%:*}" printf '%s\n' "$m/go.mod" "$m/go.sum" (cd "$root/$m" && go list -deps -f '{{if not .Standard}}{{.Dir}}{{end}}' "$p") | while read -r d; do case "$d" in "$root"/*) d="${d#"$root"/}" && git -C "$root" ls-files -- "$d" | awk -F/ -v n="$(echo "$d" | awk -F/ '{print NF}')" 'NF == n + 1' ;; esac done ;; *) git -C "$root" ls-files -- "$w" ;; esac done } src_key() { local spec="$1" kind inputs files git -C "$root" rev-parse --git-dir >/dev/null 2>&1 || return 0 kind="$(echo "$spec" | jq -r .kind)" case "$kind" in go) inputs="go:$(echo "$spec" | jq -r '.module // "."'):$(echo "$spec" | jq -r .package)" ;; script) inputs="$(echo "$spec" | jq -r .script) $(sed -n 's/^# stage-assets inputs: //p' "$root/$(echo "$spec" | jq -r .script)")" ;; esac files="$(cd "$root" && src_files $inputs | sort -u | xargs -r sha256sum | sha256sum | cut -d' ' -f1)" || return 0 printf '%s\n' "$plat" "$asset" "$(jq -cS --arg a "$asset" '.assets[$a]' "$versions")" "$script_sha" "$pack_id" "$(go version)" \ "$([ "$kind" = script ] && sha_of "$versions")" "$files" | sha256sum | cut -d' ' -f1 | sed 's/^/src-/' } unpack_blob() { zstd -dcq "$1" | tar -x -C "$2" } # the sha256 of the primary binary of an asset built from a Go package (the sha256 an OpenBao plugin catalog # registers); empty for a script-built tree built_binary_sha() { local spec="$1" tree="$2" dst [ "$(echo "$spec" | jq -r .kind)" = go ] || return 0 dst="$(echo "$spec" | jq -r '.members | to_entries[0].value')" sha_of "$tree/$dst" } # .built.json beside the blob: what the build produced, which the release binary embeds and checks the unpacked # tree against (internal/assets/built.go); written after the blob so a record never names a blob that is not there write_record() { local pdir="$1" a="$2" p="$3" d="$4" r="$5" z="$6" b="$7" tmp tmp="$pdir/$a.built.json.$BASHPID" jq -cn --arg a "$a" --arg p "$p" --arg d "$d" --argjson r "$r" --argjson z "$z" --arg b "$b" \ '{asset:$a, platform:$p, treeSha256:$d, rawBytes:$r, zstdBytes:$z, binarySha256:$b} | if $b == "" then del(.binarySha256) else . end' \ >"$tmp" mv -f "$tmp" "$pdir/$a.built.json" } stage_one() { local plat="$1" asset="$2" started="$SECONDS" pdir spec kind x tree digest raw zbytes pin in_key="" out_key how owned=0 built=0 member="" binary="" pdir="$blobs/${plat/\//_}" spec="$(jq -c --arg a "$asset" --arg p "$plat" '.assets[$a].fetch[$p]' "$versions")" [ "$spec" != "null" ] || die "$asset has no fetch spec for $plat" kind="$(echo "$spec" | jq -r .kind)" case "$kind" in go|script) built=1 ;; gosrc|cargosrc) owned=1 ;; esac pin="$(jq -r --arg a "$asset" --arg p "$plat" '.assets[$a].platforms[$p].treeSha256 // ""' "$versions")" [ "$built" = 0 ] || [ -z "$pin" ] || die "$asset is built from this repository's own commit (fetch kind $kind) and versions.json must not pin its output: remove platforms.$plat.treeSha256" x="$work/$asset-${plat/\//_}" tree="$x/tree" mkdir -p "$tree" export GNUPGHOME="$x/gnupg" mkdir -m 0700 "$GNUPGHOME" if [ "$print_src_key" = 1 ]; then [ "$built" = 1 ] && [ -n "$blob_cache" ] || die "--print-src-key needs a built asset (fetch kind go or script) and --blob-cache" plat="$plat" asset="$asset" src_key "$spec" return 0 fi if [ -n "$blob_cache" ] && { [ -n "$pin" ] || [ "$built" = 1 ]; } && [ "$print_pins" != 1 ]; then case "$kind" in file|oci|debs|gosrc|cargosrc) in_key="in-$(printf '%s\n' "$plat" "$asset" "$(jq -cS --arg a "$asset" '.assets[$a]' "$versions")" "$script_sha" "$pack_id" | sha256sum | cut -d' ' -f1)" if cache_get "$in_key" "$pdir/$asset.tar.zst" "$pin"; then chmod -R u+w "$x" && rm -rf "$x" say "ok $plat $asset tree $pin cached by its pinned inputs ($((SECONDS - started))s)" return 0 fi ;; go|script) in_key="$(src_key "$spec")" if [ -n "$in_key" ] && cache_get "$in_key" "$pdir/$asset.tar.zst" -; then unpack_blob "$pdir/$asset.tar.zst" "$tree" digest="$("$work/assetpack" digest "$tree")" raw="$(find "$tree" -type f -printf '%s\n' | awk '{s+=$1} END{print s+0}')" binary="$(built_binary_sha "$spec" "$tree")" write_record "$pdir" "$asset" "$plat" "$digest" "$raw" "$(stat -c %s "$pdir/$asset.tar.zst")" "$binary" [ "$print_pins" != 1 ] || jq -cn --arg a "$asset" --arg p "$plat" --arg d "$digest" --argjson r "$raw" --argjson z "$(stat -c %s "$pdir/$asset.tar.zst")" \ '{asset:$a, platform:$p, treeSha256:$d, rawBytes:$r, zstdBytes:$z}' chmod -R u+w "$x" && rm -rf "$x" say "ok $plat $asset tree $digest built record from its cached blob (source inputs hash as before) ($((SECONDS - started))s)" return 0 fi ;; esac fi say "$plat $asset" case "$kind" in file) stage_file "$spec" "$tree" "$x" ;; oci) stage_oci "$spec" "$tree" "$x" "$plat" ;; debs) stage_debs "$asset" "$spec" "$tree" "$x" "$plat" ;; script) stage_script "$spec" "$tree" "$x" ;; go) stage_go "$spec" "$tree" "$x" "$plat" ;; gosrc) stage_gosrc "$asset" "$spec" "$tree" "$x" "$plat" ;; cargosrc) stage_cargosrc "$asset" "$spec" "$tree" "$x" "$plat" ;; *) die "$asset: unknown fetch kind" ;; esac digest="$("$work/assetpack" digest "$tree")" raw="$(find "$tree" -type f -printf '%s\n' | awk '{s+=$1} END{print s+0}')" if [ "$built" = 0 ] && [ "$pin" != "$digest" ]; then echo "stage-assets: $asset $plat unpacks to tree $digest, versions.json pins '${pin}'" >&2 if [ "$print_pins" != 1 ] && ! { [ "$update_pins" = 1 ] && [ "$owned" = 1 ]; }; then return 1 fi fi how="packed" if [ -n "$blob_cache" ]; then "$work/assetpack" tar "$tree" >"$x/tar" out_key="out-$(printf '%s\n' "$(sha_of "$x/tar")" "$pack_id" | sha256sum | cut -d' ' -f1)" if cache_get "$out_key" "$x/blob"; then how="zstd reused by tar sha256" else cat "$x/tar" | zstd "${zstd_args[@]}" -o "$x/blob" cache_put "$out_key" "$x/blob" fi [ -z "$in_key" ] || { [ "$built" = 0 ] && [ "$pin" != "$digest" ]; } || cache_put "$in_key" "$x/blob" else "$work/assetpack" tar "$tree" | zstd "${zstd_args[@]}" -o "$x/blob" fi zbytes="$(stat -c %s "$x/blob")" if [ "$print_pins" = 1 ]; then jq -cn --arg a "$asset" --arg p "$plat" --arg d "$digest" --argjson r "$raw" --argjson z "$zbytes" \ '{asset:$a, platform:$p, treeSha256:$d, rawBytes:$r, zstdBytes:$z}' fi if [ "$built" = 1 ]; then binary="$(built_binary_sha "$spec" "$tree")" fi if [ "$update_pins" = 1 ] && [ "$owned" = 1 ]; then if jq -e --arg a "$asset" --arg p "$plat" '.assets[$a].fetch[$p].sha256 != null' "$versions" >/dev/null; then member="$(echo "$spec" | jq -r '.members | keys[0]')" binary="$(sha_of "$x/u/$member")" fi jq -cn --arg a "$asset" --arg p "$plat" --arg d "$digest" --argjson r "$raw" --argjson z "$zbytes" --arg b "$binary" \ '{asset:$a, platform:$p, treeSha256:$d, rawBytes:$r, zstdBytes:$z, binarySha256:$b} | if $b == "" then del(.binarySha256) else . end' \ >"$pins_dir/${asset}-${plat/\//_}.json" fi rm -f "$pdir/$asset.tar.zst" mv -f "$x/blob" "$pdir/$asset.tar.zst" if [ "$built" = 1 ]; then write_record "$pdir" "$asset" "$plat" "$digest" "$raw" "$zbytes" "$binary" else rm -f "$pdir/$asset.built.json" fi chmod -R u+w "$x" && rm -rf "$x" say "ok $plat $asset tree $digest raw $raw zstd $zbytes, $how ($((SECONDS - started))s)" } failed=0 running=0 stop_all() { local p for p in $(jobs -p); do pkill -TERM -P "$p" 2>/dev/null || true kill -TERM "$p" 2>/dev/null || true done } reap() { local pid="" if ! wait -n -p pid; then failed=1 fi running=$((running - 1)) if [ "$failed" = 1 ] && [ "$print_pins" != 1 ]; then stop_all while [ "$running" -gt 0 ]; do wait -n || true running=$((running - 1)) done die "an asset failed to stage; stopped the others" fi } for plat in "${platforms[@]}"; do mkdir -p "$blobs/${plat/\//_}" for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do if [ ${#only[@]} -gt 0 ] && [[ ! " ${only[*]} " == *" $asset "* ]]; then continue fi [ "$running" -lt "$jobs" ] || reap stage_one "$plat" "$asset" & running=$((running + 1)) done done while [ "$running" -gt 0 ]; do reap done [ "$failed" = 0 ] || exit "$failed" [ "$update_pins" = 0 ] || python3 "$root/scripts/update-pins.py" "$versions" "$pins_dir" exit 0