#!/usr/bin/env bash # promote.sh [--dry-run] [test|prepare|rollback]: plan 0040's promote, on the release VM, run by the release-workload module after # it moved channel rc (test: before, as a plain step). # test (promote.yml's first step, a plain one: release-workload has done nothing yet; no secret used) the # `test` check at KNOTLESS_RUN_SHA on KNOTLESS_RUN_REF must have run green (scripts/ci-check-wait.sh, the # public check record): red, cancelled, skipped by paths, or none at this commit (an asked promote: # the runner does not wait for a workflow with no check there) exits 1, so the release step never runs, # channel rc never moves, and the stop (`release-rollback`) finds no rc of this run to roll back. On a # push `after: [release, test]` already waited for a green test; this is the same rule for every run # prepare (the release step's first command) the release VM must have no guest requester: the gate signs in as bob and # alice (their creds files), nothing on this VM runs as R2, so a VM whose binding asked for one # (`association: true`: the association cloud-init's /etc/systemd/system/socialweb-r2.service) exits 1 # at once, else the runner waits beside the step for an R2 DID nothing reports and stops the step # mid-gate when the associate gives up (200 polls of 10 s after the rent). Then, as root, the unix user # r2 the gate runs as (/home/r2), made here since that cloud-init is not there to make it, and socat # (apt, as the staging install adds it): the test host runs a `pty` request.start (0017:run, killed, # killedlate) through socat (tests/host/src/swc.rs), which the ci-toolchain image lacks. # Every mode but rollback runs the same check, --dry-run too # (no argument) the prod target (scripts/ci-prod-target.sh: the sealed secrets to 0600 files of # r2, then out of the environment), the socialweb.computer droplets to RELEASE_URL (RELEASE_SHA256), every # rc operator's bidder heartbeating that binary (deploy/release/rc-operators.json, within updateWithin), # then the prod gate as tests/gate.sh runs it: every `both` entry of tests/entries.txt as bob,alice # (GATE_VMS 6), SWC_COMPAT_REQUESTER the previous rc (scripts/release-prev-rc.sh RELEASE_VERSION). Any # failure exits 1: release-workload # then moves rc back and rolls the droplets back, and channel stable stays. Both green: before it # exits 0 it copies main// inside the bucket to releases// # (scripts/copy-spaces.ts with the bucket key, as release's publish copies staging// to # main//) and checks the copy from its public URLs (the two SHA256SUMS equal, RELEASE_SHA256 in # it, every file it lists read without a credential and hashed): the stable version release-workload # writes on exit 0 has its binaries at releases// (plan 0040 step 15). A failed copy or # check exits 1 like a red gate. The bucket key, RELEASE_STABLE_VERSION and deno are asked for before # anything is deployed # rollback (release-workload, after it moved channel rc back: a red promote's own, and promote.yml's `stop` after a # promote that was cancelled or timed out) a promote still running on this machine is ended first (a step # the runner cut short leaves its processes: SIGTERM to each, a test host then releases its rentals), then # the droplets back to PREVIOUS_URL (PREVIOUS_SHA256) and every rc operator's bidder heartbeating that # binary again (within updateWithin); it returns once the ended promote's processes are gone. Run again # for the same binary on this machine (the stop after a red promote): nothing to do. No previous: the # droplets stay # --dry-run (docs/ci-debug.md "Promote, by hand") no secret, no deploy, no channel move, no gate run. It runs: # ci-prod-target.sh --dry-run (the target with its file paths; no secret file; the pinned tools when the # page is there), the binary's download and sha256 check, one read of each rc operator's heartbeat, # release-prev-rc.sh (public records), and, when the test host is there, its `--dry-run` of every prod # composition on that target (the target, the composition and the requester binary load; no account # is touched). It prints the state now of the `test` check the first step requires (KNOTLESS_RUN_SHA, else # the commit RELEASE_URL names), the droplet deploys and the gate command it would run, and # the bucket copy it would make, with what main// and releases// hold # now (public reads; nothing is copied) # Needs the secrets of .knotless/workflows/promote.yml, RELEASE_VERSION, RELEASE_STABLE_VERSION, RELEASE_URL and # RELEASE_SHA256 (release-workload), and before it `promote.sh prepare`, then `scripts/ci-test.sh host` and `binary` (the test host, the page, the # requester spec). set -euo pipefail cd "$(dirname "$0")/.." dry=0 if [ "${1:-}" = --dry-run ]; then dry=1; shift; fi would() { echo "==> would run: $*"; } r2_unit=/etc/systemd/system/socialweb-r2.service if [ -e "$r2_unit" ] && [ "${1:-}" != rollback ]; then echo "==> promote runs no guest requester, but this VM has one ($r2_unit: its executor binding asks for \`association: true\`): the runner would stop the gate when its associate gives up; drop \`association\` from socialweb.computer's com.fedfork.exec.binding/release (docs/ci-debug.md \"The release binding\")" >&2 exit 1 fi if [ "${1:-}" = prepare ]; then if [ "$(id -u)" = 0 ]; then id r2 >/dev/null 2>&1 || useradd -m -s /bin/bash r2 # as staging's gate user (tests/modules/staging install): one subordinate id range, uidmap, /etc/rancher # (0013:c's --with-gateway host check), socat (0017's pty invocations) sed -i '/^r2:/d' /etc/subuid /etc/subgid usermod --add-subuids 1000000-2048575 --add-subgids 1000000-2048575 r2 { command -v socat >/dev/null && command -v newuidmap >/dev/null; } || { apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq socat uidmap >/dev/null; } install -d -m 0755 /etc/rancher fi echo "==> no guest requester on this VM; the gate's unix user r2: $(id r2 2>/dev/null || echo 'none (not root)'); socat: $(command -v socat || echo none); newuidmap: $(command -v newuidmap || echo none)" exit 0 fi command -v curl >/dev/null && command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq curl jq >/dev/null; } if [ "${1:-}" = test ]; then exec bash scripts/ci-check-wait.sh test "${KNOTLESS_RUN_SHA:?KNOTLESS_RUN_SHA (the commit whose test must have run green)}" "${KNOTLESS_RUN_REF:-refs/heads/main}" fi home=${CI_HOME:-/home/r2} pidfile=$home/promote.pid rolled=$home/promote.rolled-back # by hand (not root): never /tmp on hat-8, a tmpfs with a per-user quota (AGENTS.md); as scripts/ci-release.sh if [ "$(id -u)" != 0 ]; then export TMPDIR=$home/tmp; mkdir -p "$TMPDIR"; fi d=$(mktemp -d); trap 'rm -rf "$d"' EXIT # the bucket key: this shell's only (the copy at the end), never the gate's or a child's environment bucket=${SPACES_BUCKET:-socialweb-computer-kcp} region=${SPACES_REGION:-sfo3} key_id=${SPACES_ACCESS_KEY_ID:-} key_secret=${SPACES_SECRET_ACCESS_KEY:-} unset SPACES_ACCESS_KEY_ID SPACES_SECRET_ACCESS_KEY base=https://$bucket.$region.digitaloceanspaces.com if [ -d /opt/ci/deno ]; then PATH=/opt/ci/deno:$PATH; fi if [ "$dry" = 0 ]; then (umask 077 && printf '%s\n' "${PROD_DEPLOY_SSH_KEY:?PROD_DEPLOY_SSH_KEY}" > "$d/key"); fi unset PROD_DEPLOY_SSH_KEY droplets() { # url sha256 curl -fsSL --retry 3 -o "$d/socialweb-computer" "$1" echo "$2 $d/socialweb-computer" | sha256sum -c --quiet chmod 0755 "$d/socialweb-computer" if [ "$dry" = 1 ]; then would "BINARY=<$1, sha256 $2 checked> SHA256=$2 SSH_KEY= bash scripts/deploy.sh" would "ssh -p ${SSH_PORT:-1997} root@socialweb.computer systemctl restart swc-ssh-gateway.service" return fi BINARY=$d/socialweb-computer SHA256=$2 SSH_KEY=$d/key bash scripts/deploy.sh ssh -p "${SSH_PORT:-1997}" -i "$d/key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new root@socialweb.computer \ 'systemctl restart swc-ssh-gateway.service && systemctl is-active swc-ssh-gateway.service' echo "==> droplets on $1" } heartbeats() { # sha256: every rc operator's bidder heartbeats that binary, within updateWithin local within deadline did pds got within=$(jq -r '.updateWithin' deploy/release/rc-operators.json | sed 's/m$//') deadline=$(( $(date +%s) + within * 60 )) for did in $(jq -r '.operators[].bidders[]' deploy/release/rc-operators.json); do pds=$(curl -fsSL --retry 3 "https://plc.directory/$did" | jq -r '.service[] | select(.id == "#atproto_pds") | .serviceEndpoint') while :; do got=$(curl -fsS -m 20 "$pds/xrpc/com.atproto.repo.getRecord?repo=$did&collection=com.publicdomainrelay.temp.market.heartbeat&rkey=self" | jq -r '.value.version.sha256 // ""' || true) [ "$got" = "$1" ] && { echo "==> $did heartbeats $1"; break; } if [ "$dry" = 1 ]; then echo "==> $did heartbeats ${got:-nothing} now; would wait ${within}m for $1"; break; fi [ "$(date +%s)" -lt "$deadline" ] || { echo "==> $did still heartbeats ${got:-nothing} after ${within}m" >&2; return 1; } sleep 20 done done } stable() { # main// to releases// inside the bucket, then the copy checked from its public URLs local from=main/${sha:-} to=releases/${RELEASE_STABLE_VERSION:-} sum name code if [ "$dry" = 1 ]; then would "SPACES_ACCESS_KEY_ID= SPACES_SECRET_ACCESS_KEY= deno run -A --no-lock scripts/copy-spaces.ts --from $from --to $to" would "the check of $base/$to/binaries/ against $base/$from/binaries/SHA256SUMS (RELEASE_SHA256 in it; every file read and hashed)" echo "==> deno: $(command -v deno || echo 'none on PATH (the real promote stops at its start)')" [ -n "${RELEASE_STABLE_VERSION:-}" ] || echo "==> no RELEASE_STABLE_VERSION (release-workload gives it: the stable version a green promote writes)" if [ -n "$sha" ]; then curl -fsSL --retry 3 -o "$d/MAIN" "$base/$from/binaries/SHA256SUMS" echo "==> $from/binaries/SHA256SUMS now: $(awk '{print $2}' "$d/MAIN" | tr '\n' ' ')" if [ -n "${RELEASE_SHA256:-}" ]; then if grep -qx "$RELEASE_SHA256 [ *]socialweb-computer-linux-amd64" "$d/MAIN"; then echo "==> RELEASE_SHA256 is its socialweb-computer-linux-amd64" else echo "==> RELEASE_SHA256 is NOT its socialweb-computer-linux-amd64: the real promote's check would fail"; fi fi fi if [ -n "${RELEASE_STABLE_VERSION:-}" ]; then code=$(curl -sS -o /dev/null -w '%{http_code}' -I "$base/$to/binaries/SHA256SUMS" || true) echo "==> $to/binaries/SHA256SUMS now: HTTP $code" fi return 0 fi SPACES_BUCKET=$bucket SPACES_REGION=$region SPACES_ACCESS_KEY_ID=$key_id SPACES_SECRET_ACCESS_KEY=$key_secret \ deno run -A --no-lock scripts/copy-spaces.ts --from "$from" --to "$to" curl -fsSL --retry 3 -o "$d/MAIN" "$base/$from/binaries/SHA256SUMS" curl -fsSL --retry 3 -o "$d/STABLE" "$base/$to/binaries/SHA256SUMS" diff "$d/MAIN" "$d/STABLE" grep -qx "$RELEASE_SHA256 [ *]socialweb-computer-linux-amd64" "$d/STABLE" || { echo "==> $to/binaries/SHA256SUMS does not list RELEASE_SHA256 for socialweb-computer-linux-amd64" >&2; return 1; } while read -r sum name; do name=${name#\*} curl -fsSL --retry 3 -o "$d/copy" "$base/$to/binaries/$name" echo "$sum $d/copy" | sha256sum -c --quiet - rm -f "$d/copy" echo "$to/binaries/$name: sha256 $sum, readable without credentials" done < "$d/STABLE" echo "==> $to/ holds $from/" } tree() { local c; echo "$1"; for c in $(pgrep -P "$1" || true); do tree "$c"; done; } ended=() end_promote() { local p [ -f "$pidfile" ] || return 0 p=$(<"$pidfile") if ! grep -qs promote.sh "/proc/$p/cmdline"; then rm -f "$pidfile"; return 0; fi ended=($(tree "$p")) if [ "$dry" = 1 ]; then would "kill -TERM ${ended[*]}"; ended=(); return 0; fi kill -TERM "${ended[@]}" 2>/dev/null || true while kill -0 "$p" 2>/dev/null; do sleep 1; done rm -f "$pidfile" echo "==> ended the promote still running here (pid $p and ${#ended[@]} processes in all)" } gone() { local p for p in ${ended[@]+"${ended[@]}"}; do while kill -0 "$p" 2>/dev/null; do sleep 2; done; done [ ${#ended[@]} = 0 ] || echo "==> the ended promote's processes are gone" } if [ "${1:-}" = rollback ]; then end_promote if [ -z "${PREVIOUS_URL:-}" ]; then echo "==> no previous release: the droplets stay"; gone; exit 0; fi if [ "$(cat "$rolled" 2>/dev/null)" = "${PREVIOUS_SHA256:?PREVIOUS_SHA256}" ]; then echo "==> the droplets and the bidders are back on $PREVIOUS_URL already (this machine's earlier rollback)"; gone; exit 0 fi droplets "$PREVIOUS_URL" "$PREVIOUS_SHA256" back=0; heartbeats "$PREVIOUS_SHA256" || back=1 if [ "$dry" = 0 ] && [ "$back" = 0 ]; then mkdir -p "$home" && echo "$PREVIOUS_SHA256" > "$rolled"; fi gone exit "$back" fi sha=${KNOTLESS_RUN_SHA:-} if [ -z "$sha" ] && [ "$dry" = 1 ] && [[ ${RELEASE_URL:-} =~ /main/([0-9a-f]{40})/ ]]; then sha=${BASH_REMATCH[1]}; fi ref=${KNOTLESS_RUN_REF:-refs/heads/main} if [ "$dry" = 0 ]; then : "${sha:?KNOTLESS_RUN_SHA (the commit whose test this promote waits for)}" : "${RELEASE_STABLE_VERSION:?RELEASE_STABLE_VERSION (release-workload: the stable version a green promote writes)}" [ -n "$key_id" ] && [ -n "$key_secret" ] || { echo "promote: SPACES_ACCESS_KEY_ID and SPACES_SECRET_ACCESS_KEY are not set (the bucket key, sealed to refs/heads/main: the copy to releases/$RELEASE_STABLE_VERSION/)" >&2; exit 1; } command -v deno >/dev/null || { echo "promote: no deno (scripts/copy-spaces.ts: the copy to releases/$RELEASE_STABLE_VERSION/)" >&2; exit 1; } mkdir -p "$home" && echo $$ > "$pidfile" trap 'rm -rf "$d" "$pidfile"' EXIT fi if [ "$dry" = 1 ]; then if [ -n "$sha" ]; then bash scripts/ci-check-wait.sh --once test "$sha" "$ref"; fi would "bash scripts/promote.sh test (promote.yml's first step, before release-workload moves channel rc: exit 1 unless this commit's test ran green)" fi compat=$(bash scripts/release-prev-rc.sh ${RELEASE_VERSION:+"$RELEASE_VERSION"}) echo "==> compat requester $compat (the rc before ${RELEASE_VERSION:-the one channel rc points at})" droplets "${RELEASE_URL:?RELEASE_URL}" "${RELEASE_SHA256:?RELEASE_SHA256}" heartbeats "$RELEASE_SHA256" # the target after the roll: each bidder's offering is read from its own PDS, which restarts when the bidder takes the # release (rc181, 2026-11-18: a 503 from a PDS mid-restart left one bidder without an endpoint) target=$home/target-prod.json if [ "$dry" = 1 ]; then bash scripts/ci-prod-target.sh --dry-run "$target" else bash scripts/ci-prod-target.sh "$target" fi for v in $(compgen -v | grep -E '^(PROD_.*_ATPROTO_PASSWORD|PROD_OPS_READ_SECRET|ATIRC_AGENT_CONFIG)$' || true); do unset "$v"; done # as scripts/ci-test.sh runs the staging suite: the checkout as SWC_REPO (a prebuilt test host's own source path is the # build host's), the binary under test and meshd that ci-test.sh host and binary installed. As r2 when root (the release # VM); a dry run by hand is the caller's own host=$PWD/tests/target/release/swc-test-host as=(env) if [ "$(id -u)" = 0 ]; then loginctl enable-linger r2 for _ in $(seq 60); do [ -d "/run/user/$(id -u r2)" ] && break; sleep 1; done install -d -o r2 gate-runs as=(runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR="/run/user/$(id -u r2)") fi as+=(SWC_REPO="$PWD" SWC_BINARY="$home/swc-under-test" SWC_MESHD=/usr/local/bin/socialweb-meshd SWC_RUNS="$PWD/gate-runs" CI=1 GATE_HOST="$host" SWC_COMPAT_REQUESTER="$compat" GATE_VMS=6) gate=(./gate.sh "$target" --account bob,alice) if [ "$dry" = 1 ]; then would "(cd tests && ${as[*]} ${gate[*]})" if [ -x "$host" ]; then for r in $(awk '!/^#/ && $2 == "both" {sub(/:.*/, "", $1); print $1}' tests/entries.txt | sort -u); do (cd tests && "${as[@]}" SWC_TARGET="$target" "$host" "compositions/$r" --dry-run --account bob) done else echo "==> no test host at $host (scripts/ci-test.sh host installs it): the target is not loaded" fi stable exit 0 fi (cd tests && "${as[@]}" "${gate[@]}") echo "==> the staging gate and the prod gate are green at $sha" echo "==> main/$sha/ to releases/$RELEASE_STABLE_VERSION/ (a failure here is a red promote: channel stable stays and rc goes back)" stable