0
0
#!/usr/bin/env bash
set -euo pipefail
if [ $# -lt 4 ]; then
echo "usage: scripts/build-socialweb-computer.sh --dev [--platform P]... [--build-root DIR] [--publish DIR [--sync-only]]" >&2
exit 2
fi
root="$1"
build_root="$2"
publish="$3"
sync_only="$4"
shift 4
platforms=("$@")
die() { echo "build-socialweb-computer --dev: $*" >&2; exit 1; }
say() { echo "build-socialweb-computer --dev: $(date -u +%Y-%m-%dT%H:%M:%SZ) $*" >&2; }
sha_of() {
local h
h="$(sha256sum "$1" 2>/dev/null | cut -d' ' -f1)"
if [ -z "$h" ]; then
echo "sha_of: cannot hash $1" >&2
printf 'unhashable:%s\n' "$1"
return 1
fi
printf '%s\n' "$h"
}
started_s="$(date +%s.%N)"
elapsed() { awk -v a="$started_s" -v b="$(date +%s.%N)" 'BEGIN { printf "%.1f", b - a }'; }
command -v rsync >/dev/null 2>&1 || die "rsync is required for --dev"
command -v flock >/dev/null 2>&1 || die "flock is required for --dev"
name="$(basename "$root")"
dev_root="$build_root/dev/$name"
versions="$root/internal/versions/versions.json"
case "$publish" in
"") [ "$sync_only" = 0 ] || die "--sync-only needs --publish DIR" ;;
/*) ;;
*) die "--publish $publish is not an absolute path" ;;
esac
mirror() {
local dst="$1/src" list
mkdir -p "$dst" "$dev_root"
list="$dev_root/mirror.include"
git -C "$root" ls-files -z -co --exclude-standard \
| tr '\0' '\n' \
| awk '{ n = split($0, p, "/"); d = ""; for (i = 1; i < n; i++) { d = d "/" p[i]; if (!(d in seen)) { seen[d] = 1; print d "/" } } print "/" $0 }' \
>"$list"
rsync -a --delete --delete-excluded --include-from="$list" --exclude='*' "$root/" "$dst/"
chmod a+rx "$1"
chmod -R a+rX "$dst"
say "mirrored $(grep -vc '/$' "$list") files of $root to $dst ($(elapsed)s)"
}
if [ "$sync_only" = 1 ]; then
mirror "$publish"
exit 0
fi
want_go="$(awk '$1=="toolchain"{print $2; exit}' "$root/go.mod")"
[ -n "$want_go" ] || want_go="go$(awk '$1=="go"{print $2; exit}' "$root/go.mod")"
cache="$build_root/cache"
mkdir -p "$cache/assets" "$cache/gomod" "$cache/gocache" "$build_root/tmp" "$dev_root"
export TMPDIR="$build_root/tmp"
export GOTMPDIR="$build_root/tmp"
export GOMODCACHE="$cache/gomod"
export GOCACHE="$cache/gocache"
export GUEST_DOWNLOADS_CACHE="$cache/assets"
export GOFLAGS=-mod=readonly
export CGO_ENABLED=0
unset GOOS GOARCH GOPATH GOROOT SOURCE_DATE_EPOCH
goroot="$(GOTOOLCHAIN="$want_go" go env GOROOT)" || die "cannot get the pinned toolchain $want_go"
export PATH="$goroot/bin:$PATH"
export GOTOOLCHAIN=local
have_go="$(go env GOVERSION)"
[ "$have_go" = "$want_go" ] || die "the toolchain at $goroot is $have_go; go.mod pins $want_go"
hostplat="$(go env GOHOSTOS)/$(go env GOHOSTARCH)"
if [ ${#platforms[@]} -eq 0 ]; then
platforms=("$hostplat")
fi
for plat in "${platforms[@]}"; do
[ "$(jq -r --arg p "$plat" '.platforms[$p].status // ""' "$versions")" = "release" ] || die "$plat is not a release platform of versions.json"
done
if [ -n "$publish" ] && [[ " ${platforms[*]} " != *" $hostplat "* ]]; then
die "--publish copies the host binary; add --platform $hostplat"
fi
stage_plats=("${platforms[@]}")
[[ " ${stage_plats[*]} " == *" $hostplat "* ]] || stage_plats+=("$hostplat")
commit="$(git -C "$root" rev-parse HEAD)"
changed="$(git -C "$root" status --porcelain | grep -c . || true)"
dirty=false
[ "$changed" = 0 ] || dirty=true
say "worktree $root at $commit (dirty $dirty, $changed changed paths), platforms ${platforms[*]}, go $have_go"
shared_blobs="$build_root/dev/assets"
own_blobs="$dev_root/blobs"
mkdir -p "$shared_blobs" "$own_blobs"
from_tree() {
[ "$(jq -r --arg a "$1" --arg p "$2" '.assets[$a].fetch[$p].kind // ""' "$versions")" = go ]
}
srckeys="$dev_root/srckeys.$$"
rm -rf "$srckeys" && mkdir -p "$srckeys"
trap 'rm -rf "$srckeys"' EXIT
from_script() {
[ "$(jq -r --arg a "$1" --arg p "$2" '.assets[$a].fetch[$p].kind // ""' "$versions")" = script ]
}
# a script-built asset (k8s-downloads, guest-downloads, swc-userns-stage) has no output pin in versions.json that
# changes with its sources, so its shared blob is keyed by its entry and by stage-assets.sh's source-input key
# (the script, the paths it names, the packages it builds from: the working tree's, uncommitted edits included)
script_key() {
local plat="$1" asset="$2" k f="$srckeys/${1/\//_}-$2"
if [ -s "$f" ]; then
cat "$f"
return
fi
k="$("$root/scripts/stage-assets.sh" --blob-cache "$cache/blobs" --asset "$asset" --platform "$plat" --print-src-key 2>/dev/null)" || k=""
[ -n "$k" ] || k="nosrckey-$(git -C "$root" rev-parse HEAD 2>/dev/null || echo none)-$(date +%s)"
printf '%s' "$k" | tee "$f"
}
blob_of() {
local plat="$1" asset="$2" key
if from_tree "$asset" "$plat"; then
echo "$own_blobs/${plat/\//_}/$asset.tar.zst"
return
fi
key="$(jq -cS --arg a "$asset" '.assets[$a]' "$versions")"
if from_script "$asset" "$plat"; then
key="$key$(script_key "$plat" "$asset")"
fi
echo "$shared_blobs/${plat/\//_}/$asset-$(printf '%s' "$key" | sha256sum | cut -c1-16).tar.zst"
}
rec_of() {
local b
b="$(blob_of "$1" "$2")"
echo "${b%.tar.zst}.built.json"
}
{
flock 9
for plat in "${stage_plats[@]}"; do
key="${plat/\//_}"
missing=()
for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do
from_tree "$asset" "$plat" && continue
if [ -f "$(blob_of "$plat" "$asset")" ] && { ! from_script "$asset" "$plat" || [ -f "$(rec_of "$plat" "$asset")" ]; }; then
continue
fi
missing+=(--asset "$asset")
done
if [ ${#missing[@]} -gt 0 ]; then
say "stage ${missing[*]} for $plat into $shared_blobs (once per versions.json entry of the asset, and per source inputs of a script-built one)"
rm -rf "$shared_blobs/.stage"
"$root/scripts/stage-assets.sh" --cache "$cache/assets" --blob-cache "$cache/blobs" --blobs "$shared_blobs/.stage" --platform "$plat" "${missing[@]}"
mkdir -p "$shared_blobs/$key"
for f in "$shared_blobs/.stage/$key"/*.tar.zst; do
mv -f "$f" "$(blob_of "$plat" "$(basename "$f" .tar.zst)")"
[ ! -f "${f%.tar.zst}.built.json" ] || mv -f "${f%.tar.zst}.built.json" "$(rec_of "$plat" "$(basename "$f" .tar.zst)")"
done
rm -rf "$shared_blobs/.stage"
fi
done
} 9>"$shared_blobs/.lock"
for plat in "${platforms[@]}"; do
tree_assets=()
built=()
for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do
from_tree "$asset" "$plat" || continue
spec="$(jq -c --arg a "$asset" --arg p "$plat" '.assets[$a].fetch[$p]' "$versions")"
probe="$dev_root/tree-bin/${plat/\//_}/$asset"
mkdir -p "$(dirname "$probe")"
(cd "$root/$(echo "$spec" | jq -r '.module // "."')" && GOOS="${plat%/*}" GOARCH="${plat#*/}" go build -trimpath \
-buildvcs=false -ldflags='-s -w -buildid=' -o "$probe" "$(echo "$spec" | jq -r .package)")
sum="$(sha_of "$probe")"
if [ -f "$(blob_of "$plat" "$asset")" ] && [ -f "$(rec_of "$plat" "$asset")" ] && [ "$(cat "$probe.staged" 2>/dev/null)" = "$sum" ]; then
continue
fi
tree_assets+=(--asset "$asset")
built+=("$probe:$sum")
done
[ ${#tree_assets[@]} -gt 0 ] || continue
say "stage ${tree_assets[*]} for $plat (built from the working tree, changed)"
"$root/scripts/stage-assets.sh" --cache "$cache/assets" --blobs "$own_blobs" --platform "$plat" \
"${tree_assets[@]}" >/dev/null 2>"$dev_root/tree-assets-stage.log" \
|| die "staging ${tree_assets[*]} failed: $(tail -3 "$dev_root/tree-assets-stage.log")"
for b in "${built[@]}"; do
echo "${b##*:}" >"${b%:*}.staged"
done
done
say "assets staged ($(elapsed)s)"
hostkey="${hostplat/\//_}"
source_root="$root"
[ -z "$publish" ] || source_root="$publish/src"
stamp="github.com/johnandersen777/socialweb-computer-kcp/internal/buildstamp"
ldflags="-s -w -X $stamp.Kind=dev -X $stamp.SourceRoot=$source_root -X $stamp.Commit=$commit -X $stamp.Dirty=$dirty -X $stamp.ChangedPaths=$changed -X $stamp.BuiltUTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
for plat in "${platforms[@]}"; do
goos="${plat%/*}"
goarch="${plat#*/}"
key="${goos}_${goarch}"
out="$dev_root/$key"
mkdir -p "$out"
overlay="$dev_root/overlay-$key.json"
{
for asset in $(jq -r --arg p "$plat" '.platforms[$p].assets[]' "$versions"); do
jq -n --arg k "$root/internal/assets/blobs/$key/$asset.tar.zst" --arg v "$(blob_of "$plat" "$asset")" '{($k): $v}'
if [ -f "$(rec_of "$plat" "$asset")" ]; then
jq -n --arg k "$root/internal/assets/blobs/$key/$asset.built.json" --arg v "$(rec_of "$plat" "$asset")" '{($k): $v}'
elif from_tree "$asset" "$plat" || from_script "$asset" "$plat"; then
die "$asset ($plat) is built from this commit and has no build record at $(rec_of "$plat" "$asset"): the binary would refuse to start; remove its blob and build again"
fi
done
} | jq -s '{Replace: add}' >"$overlay"
say "go build $plat"
(cd "$root" && GOOS="$goos" GOARCH="$goarch" go build -tags release -trimpath -buildvcs=true -overlay "$overlay" \
-ldflags="$ldflags" -o "$out/socialweb-computer.new" ./cmd/socialweb-computer)
mv -f "$out/socialweb-computer.new" "$out/socialweb-computer"
if [ "$plat" = "$hostplat" ]; then
ln -sfn "$key/socialweb-computer" "$dev_root/socialweb-computer"
fi
say "built $out/socialweb-computer ($(elapsed)s)"
done
if [ -n "$publish" ]; then
mkdir -p "$publish/bin"
cp -f "$dev_root/$hostkey/socialweb-computer" "$publish/bin/socialweb-computer.new"
mv -f "$publish/bin/socialweb-computer.new" "$publish/bin/socialweb-computer"
chmod a+rx "$publish/bin" "$publish/bin/socialweb-computer"
mirror "$publish"
say "published $publish/bin/socialweb-computer, source from $publish/src"
fi
if [[ " ${platforms[*]} " == *" $hostplat "* ]]; then
"$dev_root/$hostkey/socialweb-computer" version | tee "$dev_root/$hostkey/version.txt"
fi
say "done in $(elapsed)s"