0
0
#!/usr/bin/env bash
# promote.sh [--dry-run] [test|prepare|rollback]: plan 0040's promote, on the release VM, run by the release-workload module after
# it moved channel rc (test: before, as a plain step).
# test (promote.yml's first step, a plain one: release-workload has done nothing yet; no secret used) the
# `test` check at KNOTLESS_RUN_SHA on KNOTLESS_RUN_REF must have run green (scripts/ci-check-wait.sh, the
# public check record): red, cancelled, skipped by paths, or none at this commit (an asked promote:
# the runner does not wait for a workflow with no check there) exits 1, so the release step never runs,
# channel rc never moves, and the stop (`release-rollback`) finds no rc of this run to roll back. On a
# push `after: [release, test]` already waited for a green test; this is the same rule for every run
# prepare (the release step's first command) the release VM must have no guest requester: the gate signs in as bob and
# alice (their creds files), nothing on this VM runs as R2, so a VM whose binding asked for one
# (`association: true`: the association cloud-init's /etc/systemd/system/socialweb-r2.service) exits 1
# at once, else the runner waits beside the step for an R2 DID nothing reports and stops the step
# mid-gate when the associate gives up (200 polls of 10 s after the rent). Then, as root, the unix user
# r2 the gate runs as (/home/r2), made here since that cloud-init is not there to make it, and socat
# (apt, as the staging install adds it): the test host runs a `pty` request.start (0017:run, killed,
# killedlate) through socat (tests/host/src/swc.rs), which the ci-toolchain image lacks.
# Every mode but rollback runs the same check, --dry-run too
# (no argument) the prod target (scripts/ci-prod-target.sh: the sealed secrets to 0600 files of
# r2, then out of the environment), the socialweb.computer droplets to RELEASE_URL (RELEASE_SHA256), every
# rc operator's bidder heartbeating that binary (deploy/release/rc-operators.json, within updateWithin),
# then the prod gate as tests/gate.sh runs it: every `both` entry of tests/entries.txt as bob,alice
# (GATE_VMS 6), SWC_COMPAT_REQUESTER the previous rc (scripts/release-prev-rc.sh RELEASE_VERSION). Any
# failure exits 1: release-workload
# then moves rc back and rolls the droplets back, and channel stable stays. Both green: before it
# exits 0 it copies main/<KNOTLESS_RUN_SHA>/ inside the bucket to releases/<RELEASE_STABLE_VERSION>/
# (scripts/copy-spaces.ts with the bucket key, as release's publish copies staging/<sha>/ to
# main/<sha>/) and checks the copy from its public URLs (the two SHA256SUMS equal, RELEASE_SHA256 in
# it, every file it lists read without a credential and hashed): the stable version release-workload
# writes on exit 0 has its binaries at releases/<version>/ (plan 0040 step 15). A failed copy or
# check exits 1 like a red gate. The bucket key, RELEASE_STABLE_VERSION and deno are asked for before
# anything is deployed
# rollback (release-workload, after it moved channel rc back: a red promote's own, and promote.yml's `stop` after a
# promote that was cancelled or timed out) a promote still running on this machine is ended first (a step
# the runner cut short leaves its processes: SIGTERM to each, a test host then releases its rentals), then
# the droplets back to PREVIOUS_URL (PREVIOUS_SHA256) and every rc operator's bidder heartbeating that
# binary again (within updateWithin); it returns once the ended promote's processes are gone. Run again
# for the same binary on this machine (the stop after a red promote): nothing to do. No previous: the
# droplets stay
# --dry-run (docs/ci-debug.md "Promote, by hand") no secret, no deploy, no channel move, no gate run. It runs:
# ci-prod-target.sh --dry-run (the target with its file paths; no secret file; the pinned tools when the
# page is there), the binary's download and sha256 check, one read of each rc operator's heartbeat,
# release-prev-rc.sh (public records), and, when the test host is there, its `--dry-run` of every prod
# composition on that target (the target, the composition and the requester binary load; no account
# is touched). It prints the state now of the `test` check the first step requires (KNOTLESS_RUN_SHA, else
# the commit RELEASE_URL names), the droplet deploys and the gate command it would run, and
# the bucket copy it would make, with what main/<sha>/ and releases/<RELEASE_STABLE_VERSION>/ hold
# now (public reads; nothing is copied)
# Needs the secrets of .knotless/workflows/promote.yml, RELEASE_VERSION, RELEASE_STABLE_VERSION, RELEASE_URL and
# RELEASE_SHA256 (release-workload), and before it `promote.sh prepare`, then `scripts/ci-test.sh host` and `binary` (the test host, the page, the
# requester spec).
set -euo pipefail
cd "$(dirname "$0")/.."
dry=0
if [ "${1:-}" = --dry-run ]; then dry=1; shift; fi
would() { echo "==> would run: $*"; }
r2_unit=/etc/systemd/system/socialweb-r2.service
if [ -e "$r2_unit" ] && [ "${1:-}" != rollback ]; then
echo "==> promote runs no guest requester, but this VM has one ($r2_unit: its executor binding asks for \`association: true\`): the runner would stop the gate when its associate gives up; drop \`association\` from socialweb.computer's com.fedfork.exec.binding/release (docs/ci-debug.md \"The release binding\")" >&2
exit 1
fi
if [ "${1:-}" = prepare ]; then
if [ "$(id -u)" = 0 ]; then
id r2 >/dev/null 2>&1 || useradd -m -s /bin/bash r2
# as staging's gate user (tests/modules/staging install): one subordinate id range, uidmap, /etc/rancher
# (0013:c's --with-gateway host check), socat (0017's pty invocations)
sed -i '/^r2:/d' /etc/subuid /etc/subgid
usermod --add-subuids 1000000-2048575 --add-subgids 1000000-2048575 r2
{ command -v socat >/dev/null && command -v newuidmap >/dev/null; } || { apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq socat uidmap >/dev/null; }
install -d -m 0755 /etc/rancher
fi
echo "==> no guest requester on this VM; the gate's unix user r2: $(id r2 2>/dev/null || echo 'none (not root)'); socat: $(command -v socat || echo none); newuidmap: $(command -v newuidmap || echo none)"
exit 0
fi
command -v curl >/dev/null && command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq curl jq >/dev/null; }
if [ "${1:-}" = test ]; then
exec bash scripts/ci-check-wait.sh test "${KNOTLESS_RUN_SHA:?KNOTLESS_RUN_SHA (the commit whose test must have run green)}" "${KNOTLESS_RUN_REF:-refs/heads/main}"
fi
home=${CI_HOME:-/home/r2}
pidfile=$home/promote.pid rolled=$home/promote.rolled-back
# by hand (not root): never /tmp on hat-8, a tmpfs with a per-user quota (AGENTS.md); as scripts/ci-release.sh
if [ "$(id -u)" != 0 ]; then export TMPDIR=$home/tmp; mkdir -p "$TMPDIR"; fi
d=$(mktemp -d); trap 'rm -rf "$d"' EXIT
# the bucket key: this shell's only (the copy at the end), never the gate's or a child's environment
bucket=${SPACES_BUCKET:-socialweb-computer-kcp} region=${SPACES_REGION:-sfo3}
key_id=${SPACES_ACCESS_KEY_ID:-} key_secret=${SPACES_SECRET_ACCESS_KEY:-}
unset SPACES_ACCESS_KEY_ID SPACES_SECRET_ACCESS_KEY
base=https://$bucket.$region.digitaloceanspaces.com
if [ -d /opt/ci/deno ]; then PATH=/opt/ci/deno:$PATH; fi
if [ "$dry" = 0 ]; then (umask 077 && printf '%s\n' "${PROD_DEPLOY_SSH_KEY:?PROD_DEPLOY_SSH_KEY}" > "$d/key"); fi
unset PROD_DEPLOY_SSH_KEY
droplets() { # url sha256
curl -fsSL --retry 3 -o "$d/socialweb-computer" "$1"
echo "$2 $d/socialweb-computer" | sha256sum -c --quiet
chmod 0755 "$d/socialweb-computer"
if [ "$dry" = 1 ]; then
would "BINARY=<$1, sha256 $2 checked> SHA256=$2 SSH_KEY=<PROD_DEPLOY_SSH_KEY> bash scripts/deploy.sh"
would "ssh -p ${SSH_PORT:-1997} root@socialweb.computer systemctl restart swc-ssh-gateway.service"
return
fi
BINARY=$d/socialweb-computer SHA256=$2 SSH_KEY=$d/key bash scripts/deploy.sh
ssh -p "${SSH_PORT:-1997}" -i "$d/key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new root@socialweb.computer \
'systemctl restart swc-ssh-gateway.service && systemctl is-active swc-ssh-gateway.service'
echo "==> droplets on $1"
}
heartbeats() { # sha256: every rc operator's bidder heartbeats that binary, within updateWithin
local within deadline did pds got
within=$(jq -r '.updateWithin' deploy/release/rc-operators.json | sed 's/m$//')
deadline=$(( $(date +%s) + within * 60 ))
for did in $(jq -r '.operators[].bidders[]' deploy/release/rc-operators.json); do
pds=$(curl -fsSL --retry 3 "https://plc.directory/$did" | jq -r '.service[] | select(.id == "#atproto_pds") | .serviceEndpoint')
while :; do
got=$(curl -fsS -m 20 "$pds/xrpc/com.atproto.repo.getRecord?repo=$did&collection=com.publicdomainrelay.temp.market.heartbeat&rkey=self" | jq -r '.value.version.sha256 // ""' || true)
[ "$got" = "$1" ] && { echo "==> $did heartbeats $1"; break; }
if [ "$dry" = 1 ]; then echo "==> $did heartbeats ${got:-nothing} now; would wait ${within}m for $1"; break; fi
[ "$(date +%s)" -lt "$deadline" ] || { echo "==> $did still heartbeats ${got:-nothing} after ${within}m" >&2; return 1; }
sleep 20
done
done
}
stable() { # main/<sha>/ to releases/<RELEASE_STABLE_VERSION>/ inside the bucket, then the copy checked from its public URLs
local from=main/${sha:-<KNOTLESS_RUN_SHA>} to=releases/${RELEASE_STABLE_VERSION:-<RELEASE_STABLE_VERSION>} sum name code
if [ "$dry" = 1 ]; then
would "SPACES_ACCESS_KEY_ID=<sealed> SPACES_SECRET_ACCESS_KEY=<sealed> deno run -A --no-lock scripts/copy-spaces.ts --from $from --to $to"
would "the check of $base/$to/binaries/ against $base/$from/binaries/SHA256SUMS (RELEASE_SHA256 in it; every file read and hashed)"
echo "==> deno: $(command -v deno || echo 'none on PATH (the real promote stops at its start)')"
[ -n "${RELEASE_STABLE_VERSION:-}" ] || echo "==> no RELEASE_STABLE_VERSION (release-workload gives it: the stable version a green promote writes)"
if [ -n "$sha" ]; then
curl -fsSL --retry 3 -o "$d/MAIN" "$base/$from/binaries/SHA256SUMS"
echo "==> $from/binaries/SHA256SUMS now: $(awk '{print $2}' "$d/MAIN" | tr '\n' ' ')"
if [ -n "${RELEASE_SHA256:-}" ]; then
if grep -qx "$RELEASE_SHA256 [ *]socialweb-computer-linux-amd64" "$d/MAIN"; then echo "==> RELEASE_SHA256 is its socialweb-computer-linux-amd64"
else echo "==> RELEASE_SHA256 is NOT its socialweb-computer-linux-amd64: the real promote's check would fail"; fi
fi
fi
if [ -n "${RELEASE_STABLE_VERSION:-}" ]; then
code=$(curl -sS -o /dev/null -w '%{http_code}' -I "$base/$to/binaries/SHA256SUMS" || true)
echo "==> $to/binaries/SHA256SUMS now: HTTP $code"
fi
return 0
fi
SPACES_BUCKET=$bucket SPACES_REGION=$region SPACES_ACCESS_KEY_ID=$key_id SPACES_SECRET_ACCESS_KEY=$key_secret \
deno run -A --no-lock scripts/copy-spaces.ts --from "$from" --to "$to"
curl -fsSL --retry 3 -o "$d/MAIN" "$base/$from/binaries/SHA256SUMS"
curl -fsSL --retry 3 -o "$d/STABLE" "$base/$to/binaries/SHA256SUMS"
diff "$d/MAIN" "$d/STABLE"
grep -qx "$RELEASE_SHA256 [ *]socialweb-computer-linux-amd64" "$d/STABLE" || { echo "==> $to/binaries/SHA256SUMS does not list RELEASE_SHA256 for socialweb-computer-linux-amd64" >&2; return 1; }
while read -r sum name; do
name=${name#\*}
curl -fsSL --retry 3 -o "$d/copy" "$base/$to/binaries/$name"
echo "$sum $d/copy" | sha256sum -c --quiet -
rm -f "$d/copy"
echo "$to/binaries/$name: sha256 $sum, readable without credentials"
done < "$d/STABLE"
echo "==> $to/ holds $from/"
}
tree() { local c; echo "$1"; for c in $(pgrep -P "$1" || true); do tree "$c"; done; }
ended=()
end_promote() {
local p
[ -f "$pidfile" ] || return 0
p=$(<"$pidfile")
if ! grep -qs promote.sh "/proc/$p/cmdline"; then rm -f "$pidfile"; return 0; fi
ended=($(tree "$p"))
if [ "$dry" = 1 ]; then would "kill -TERM ${ended[*]}"; ended=(); return 0; fi
kill -TERM "${ended[@]}" 2>/dev/null || true
while kill -0 "$p" 2>/dev/null; do sleep 1; done
rm -f "$pidfile"
echo "==> ended the promote still running here (pid $p and ${#ended[@]} processes in all)"
}
gone() {
local p
for p in ${ended[@]+"${ended[@]}"}; do while kill -0 "$p" 2>/dev/null; do sleep 2; done; done
[ ${#ended[@]} = 0 ] || echo "==> the ended promote's processes are gone"
}
if [ "${1:-}" = rollback ]; then
end_promote
if [ -z "${PREVIOUS_URL:-}" ]; then echo "==> no previous release: the droplets stay"; gone; exit 0; fi
if [ "$(cat "$rolled" 2>/dev/null)" = "${PREVIOUS_SHA256:?PREVIOUS_SHA256}" ]; then
echo "==> the droplets and the bidders are back on $PREVIOUS_URL already (this machine's earlier rollback)"; gone; exit 0
fi
droplets "$PREVIOUS_URL" "$PREVIOUS_SHA256"
back=0; heartbeats "$PREVIOUS_SHA256" || back=1
if [ "$dry" = 0 ] && [ "$back" = 0 ]; then mkdir -p "$home" && echo "$PREVIOUS_SHA256" > "$rolled"; fi
gone
exit "$back"
fi
sha=${KNOTLESS_RUN_SHA:-}
if [ -z "$sha" ] && [ "$dry" = 1 ] && [[ ${RELEASE_URL:-} =~ /main/([0-9a-f]{40})/ ]]; then sha=${BASH_REMATCH[1]}; fi
ref=${KNOTLESS_RUN_REF:-refs/heads/main}
if [ "$dry" = 0 ]; then
: "${sha:?KNOTLESS_RUN_SHA (the commit whose test this promote waits for)}"
: "${RELEASE_STABLE_VERSION:?RELEASE_STABLE_VERSION (release-workload: the stable version a green promote writes)}"
[ -n "$key_id" ] && [ -n "$key_secret" ] || { echo "promote: SPACES_ACCESS_KEY_ID and SPACES_SECRET_ACCESS_KEY are not set (the bucket key, sealed to refs/heads/main: the copy to releases/$RELEASE_STABLE_VERSION/)" >&2; exit 1; }
command -v deno >/dev/null || { echo "promote: no deno (scripts/copy-spaces.ts: the copy to releases/$RELEASE_STABLE_VERSION/)" >&2; exit 1; }
mkdir -p "$home" && echo $$ > "$pidfile"
trap 'rm -rf "$d" "$pidfile"' EXIT
fi
if [ "$dry" = 1 ]; then
if [ -n "$sha" ]; then bash scripts/ci-check-wait.sh --once test "$sha" "$ref"; fi
would "bash scripts/promote.sh test (promote.yml's first step, before release-workload moves channel rc: exit 1 unless this commit's test ran green)"
fi
compat=$(bash scripts/release-prev-rc.sh ${RELEASE_VERSION:+"$RELEASE_VERSION"})
echo "==> compat requester $compat (the rc before ${RELEASE_VERSION:-the one channel rc points at})"
droplets "${RELEASE_URL:?RELEASE_URL}" "${RELEASE_SHA256:?RELEASE_SHA256}"
heartbeats "$RELEASE_SHA256"
# the target after the roll: each bidder's offering is read from its own PDS, which restarts when the bidder takes the
# release (rc181, 2026-11-18: a 503 from a PDS mid-restart left one bidder without an endpoint)
target=$home/target-prod.json
if [ "$dry" = 1 ]; then
bash scripts/ci-prod-target.sh --dry-run "$target"
else
bash scripts/ci-prod-target.sh "$target"
fi
for v in $(compgen -v | grep -E '^(PROD_.*_ATPROTO_PASSWORD|PROD_OPS_READ_SECRET|ATIRC_AGENT_CONFIG)$' || true); do unset "$v"; done
# as scripts/ci-test.sh runs the staging suite: the checkout as SWC_REPO (a prebuilt test host's own source path is the
# build host's), the binary under test and meshd that ci-test.sh host and binary installed. As r2 when root (the release
# VM); a dry run by hand is the caller's own
host=$PWD/tests/target/release/swc-test-host
as=(env)
if [ "$(id -u)" = 0 ]; then
loginctl enable-linger r2
for _ in $(seq 60); do [ -d "/run/user/$(id -u r2)" ] && break; sleep 1; done
install -d -o r2 gate-runs
as=(runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR="/run/user/$(id -u r2)")
fi
as+=(SWC_REPO="$PWD" SWC_BINARY="$home/swc-under-test" SWC_MESHD=/usr/local/bin/socialweb-meshd SWC_RUNS="$PWD/gate-runs" CI=1
GATE_HOST="$host" SWC_COMPAT_REQUESTER="$compat" GATE_VMS=6)
gate=(./gate.sh "$target" --account bob,alice)
if [ "$dry" = 1 ]; then
would "(cd tests && ${as[*]} ${gate[*]})"
if [ -x "$host" ]; then
for r in $(awk '!/^#/ && $2 == "both" {sub(/:.*/, "", $1); print $1}' tests/entries.txt | sort -u); do
(cd tests && "${as[@]}" SWC_TARGET="$target" "$host" "compositions/$r" --dry-run --account bob)
done
else
echo "==> no test host at $host (scripts/ci-test.sh host installs it): the target is not loaded"
fi
stable
exit 0
fi
(cd tests && "${as[@]}" "${gate[@]}")
echo "==> the staging gate and the prod gate are green at $sha"
echo "==> main/$sha/ to releases/$RELEASE_STABLE_VERSION/ (a failure here is a red promote: channel stable stays and rc goes back)"
stable