0
0
#!/usr/bin/env bash
# The release gate: every composition against a target, as many at once as there are driver accounts.
# Usage (from tests/): ./gate.sh <target.json> [--account <name>[,<name>...]]
# ./gate.sh --list <staging|prod> the entries it would run, one a line; nothing runs
# The entries are tests/entries.txt's, chosen by the target's kind: a target with `network` is a staging network (every
# entry), any other is prod (the `both` entries). No entry is skipped and none can be named: one entry on a held network is
# swc-test-host itself (docs/ci-debug.md), never a gate result.
# One "RESULT <runbook>:<entry> PASS|FAIL <seconds> <note> [MEMORY <events>]" line each as it ends, then a summary
# table in gate order (with each run's VM memory events from its readings.memory: the test modules fail a run on an OOM
# kill and list pressure); exit 1 on a FAIL.
# Parallel: one run per account at a time (an account has one driver: the test host's account lock), so the accounts
# given (GATE_ACCOUNTS, else --account, default bob) are the slots; each slot has its own iroh key. A run holds the
# VMs of its entry's weight (WEIGHT below, default 1) out of GATE_VMS (4) so the target's bidders are not oversubscribed.
# An exclusive entry (EXCLUSIVE: it fills or counts bidders, or places VMs per host) runs alone: the gate drains first.
# GATE_JOBS=1 runs it serially in gate order.
# GATE_SHARD=i/n: this gate runs share i of the n the entries are dealt into (gate_ids below), for n staging networks at
# once (by hand: scripts/ci-test.sh CI_SHARD; CI's `test` runs every entry on one network); unset, every entry. Its summary
# counts its own share.
# The heavy-job lock: held for the whole run on a target with local networks, for the build only on prod (no `network`).
# A failed run freezes (attach: the FROZEN line of its log) until GATE_FREEZE_TTL (900 s), then its slot goes on.
# SWC_COMPAT_REQUESTER (the previous rc's CLI, sha:<its commit>) is 0006:compat's requester; unset, 0006:compat FAILs.
# 0007:upgrade upgrades the bidders from SWC_UPGRADE_FROM, else SWC_COMPAT_REQUESTER; both unset, it FAILs. 0007:fence
# upgrades from its own release (v0.0.0-rc186, the last without decision 54's rule), neither env.
set -uo pipefail
entries=$(dirname "$0")/entries.txt
# runs alone: k8s nodes one per host and its tenant scale-downs (0001), a bidder core restarted (0007:restart) or upgraded
# (0007:upgrade: every bidder to another binary), its IngressProxy replaced (0007:ingress), bidders
# filled to capacity or booked (0014)
EXCLUSIVE=" ${GATE_EXCLUSIVE:-0001:run 0001:s2 0007:restart 0007:upgrade 0007:fence 0007:ingress 0014:run 0014:concurrent 0014:exhausted 0014:giveup 0014:wait 0014:impossible 0014:backtoback 0014:overcommit 0014:memfit} "
# The entries of a target kind, in gate order. GATE_SHARD=i/n: share i of n, for n networks side by side. The rule (any n up
# to the entry count; every entry in exactly one share, no share empty: scripts/check-shards.sh): the exclusive entries
# first, then the rest, each class longest first by the seconds column of entries.txt (ties: gate order), each entry to
# the share with the least seconds so far (ties: the lowest share), an entry counting its seconds plus one.
gate_ids() {
awk -v k="$1" -v shard="${GATE_SHARD:-1/1}" -v excl="$EXCLUSIVE" '
BEGIN { if (shard !~ /^[1-9][0-9]*\/[1-9][0-9]*$/) err = 1; split(shard, sh, "/"); i = sh[1] + 0; n = sh[2] + 0 }
/^#/ || !NF { next }
!(NF == 3 && ($2 == "both" || $2 == "staging") && $3 ~ /^[0-9]+$/) { err = 1; next }
$2 == "both" || k == "staging" { m++; id[m] = $1; sec[m] = $3 + 0; x[m] = index(excl, " " $1 " ") > 0 }
END {
if (err || !m || i > n || n > m) exit 2
for (c = 1; c >= 0; c--) for (;;) {
b = 0
for (j = 1; j <= m; j++) if (x[j] == c && !(j in to) && (!b || sec[j] > sec[b])) b = j
if (!b) break
s = 1
for (j = 2; j <= n; j++) if (load[j] + 0 < load[s] + 0) s = j
to[b] = s; load[s] += sec[b] + 1
}
for (j = 1; j <= m; j++) if (to[j] == i) print id[j]
}' "$entries"
}
bad_list="gate.sh: $entries: a bad line or no entry, or GATE_SHARD (${GATE_SHARD:-unset}) is not i/n with i at most n and n at most the entries"
if [ "${1:-}" = --list ]; then
case "${2:-}" in staging|prod) ;; *) echo "usage: gate.sh --list <staging|prod>" >&2; exit 2 ;; esac
gate_ids "$2" || { echo "$bad_list" >&2; exit 2; }
exit 0
fi
# The heavy-job lock (hat-8's one heavy job at a time) covers the whole run only for a target with local networks (dev, staging:
# VMs and cores run here); a target without `network` (prod) runs its VMs on the prod bidders and holds the lock only for the build.
if [ -z "${SWC_HEAVY_HELD:-}" ] && jq -e '.network' "${1:?usage: gate.sh <target.json> [--account <name>[,<name>...]]}" > /dev/null 2>&1; then
exec "$(dirname "$0")/../scripts/swc-heavy.sh" "$0" "$@"
fi
target=$(realpath "${1:?usage: gate.sh <target.json> [--account <name>[,<name>...]]}"); shift
accounts=${GATE_ACCOUNTS:-bob}
if [ "${1:-}" = --account ]; then accounts=$2; shift 2; fi
[ $# = 0 ] || { echo "gate.sh: no entry can be named (tests/entries.txt is the list; one entry: swc-test-host, docs/ci-debug.md): $*" >&2; exit 2; }
[ -z "${GATE_SKIP:-}" ] || { echo "gate.sh: GATE_SKIP is gone: an entry passes or fails" >&2; exit 2; }
if jq -e '.network' "$target" > /dev/null 2>&1; then kind=staging; else kind=prod; fi
ids=$(gate_ids $kind) || { echo "$bad_list" >&2; exit 2; }
echo "gate: target kind $kind: $(wc -w <<<"$ids") entries${GATE_SHARD:+ (shard $GATE_SHARD)}"
# VMs an entry holds at its peak (the rest hold one; host checks none)
declare -A WEIGHT=([0002:run]=2 [0003:run]=3 [0012:run]=3 [0013:run]=2 [0013:c]=2 [0013:b]=2 [0006:run]=0 [0011:token]=0 [0016:badsha]=0 [0018:run]=0 [0017:devsup]=0 [0017:samename]=0 [0014:impossible]=0 [0014:abandon]=0 [0020:run]=0)
cd "$(dirname "$0")"
unset ATPROTO_HANDLE ATPROTO_PASSWORD ATP_USERNAME ATP_PASSWORD ATPROTO_PDS
out=${GATE_OUT:-$HOME/.tmp/swc-system-tests/gate/$(date -u +%Y%m%dT%H%M%SZ)}
mkdir -p "$out" && chmod 700 "$out"
# GATE_HOST: a prebuilt test host (its module wasm already under modules/target, as scripts/ci-test.sh host leaves them): no build
if [ -n "${GATE_HOST:-}" ]; then
host=$GATE_HOST
elif [ -n "${SWC_HEAVY_HELD:-}" ]; then
(cd modules && cargo wasm -q) && cargo build -q -p swc-test-host || exit 2
host=$PWD/target/debug/swc-test-host
else
../scripts/swc-heavy.sh sh -c '(cd modules && cargo wasm -q) && cargo build -q -p swc-test-host' || exit 2
host=$PWD/target/debug/swc-test-host
fi
export SWC_TARGET=$target SWC_READINGS=$out
# one entry as <account> on slot <n>: its RESULT line, and <out>/<runbook>-<entry>.tsv for the summary
run_one() {
local id=$1 account=$2 slot=$3 r e t0 res note rc doc s mem input
r=${id%%:*}; e=${id#*:}; [ "$e" = "$id" ] && e=run
t0=$(date +%s)
if [ "$r:$e" = 0006:compat ] && [ -z "${SWC_COMPAT_REQUESTER:-}" ]; then
res=FAIL note="SWC_COMPAT_REQUESTER unset (the previous rc's CLI)"
elif [ "$r:$e" = 0007:upgrade ] && [ -z "${SWC_UPGRADE_FROM:-}${SWC_COMPAT_REQUESTER:-}" ]; then
res=FAIL note="SWC_UPGRADE_FROM and SWC_COMPAT_REQUESTER unset (the binary the bidders are upgraded from)"
elif [ ! -f "compositions/$r/composition.json" ]; then
res=FAIL note="no composition"
else
input=()
case $r in 0016) input=(--input "$(jq -c '{image: {url, sha256}, marker}' ../deploy/vm-images/ci-toolchain.json)") ;; esac
SWC_IROH_KEY=$out/iroh-$slot "$host" "compositions/$r" --account "$account" --entry "$e" --on-fail freeze \
--freeze-ttl "${GATE_FREEZE_TTL:-900}" "${input[@]}" > "$out/$r-$e.json" 2> "$out/$r-$e.log" < /dev/null
rc=$?
doc=$(sed -n '/^{/,$p' "$out/$r-$e.json")
if [ $rc = 0 ] && jq -e '.readings.skipped' <<<"$doc" > /dev/null 2>&1; then
res=FAIL note="skipped: $(jq -r '.readings.skipped' <<<"$doc")"
elif [ $rc = 0 ]; then
res=PASS note=$(jq -r '.dump // ""' <<<"$doc" 2>/dev/null)
else
res=FAIL note=$(jq -r '[.error, (.cleanupErrors // [] | join("; "))] | map(select(. != null and . != "")) | join(" | ") | gsub("[\n\r\t]"; " ")' <<<"$doc" 2>/dev/null)
[ -n "$note" ] || note="exit $rc: $(tail -1 "$out/$r-$e.log" | cut -c1-200)"
fi
fi
s=$(( $(date +%s) - t0 ))
mem=$(jq -r '[.readings.memory // [] | .[] | "\(.event) \(.vm)"] | unique | join(", ")' "$out/$r-$e.json" 2>/dev/null)
echo "RESULT $r:$e $res $s $note${mem:+ MEMORY $mem} (${account})"
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$r:$e" "$res" "$s" "$note" "$mem" "$account" > "$out/$r-$e.tsv"
}
IFS=, read -ra slots <<<"$accounts"
jobs=${GATE_JOBS:-${#slots[@]}}
[ "$jobs" -le "${#slots[@]}" ] || jobs=${#slots[@]}
cap=${GATE_VMS:-4}
declare -A pid_slot=() pid_weight=()
free=()
for ((n = 0; n < jobs; n++)); do free+=("$n"); done
used=0 excl=0 queue=($ids)
g0=$(date +%s)
while [ ${#queue[@]} -gt 0 ] || [ ${#pid_slot[@]} -gt 0 ]; do
left=()
barrier=0
for id in "${queue[@]}"; do
w=${WEIGHT[$id]:-1}
[[ $id != *:* ]] && w=${WEIGHT[$id:run]:-1}
x=0; [[ $EXCLUSIVE == *" $id "* || $EXCLUSIVE == *" $id:run "* ]] && x=1
if [ $barrier = 1 ] || [ $excl = 1 ] || [ ${#free[@]} = 0 ] || { [ $x = 1 ] && [ ${#pid_slot[@]} -gt 0 ]; } || [ $((used + w)) -gt "$cap" -a ${#pid_slot[@]} -gt 0 ]; then
left+=("$id")
# an exclusive entry is a barrier: nothing after it starts before it
[ $x = 1 ] && barrier=1
continue
fi
slot=${free[0]}; free=("${free[@]:1}")
run_one "$id" "${slots[$slot]}" "$slot" &
pid_slot[$!]=$slot pid_weight[$!]=$w
used=$((used + w)); excl=$x
done
queue=("${left[@]}")
[ ${#pid_slot[@]} -gt 0 ] || continue
wait -n -p done_pid
free+=("${pid_slot[$done_pid]}")
used=$((used - pid_weight[$done_pid])); excl=0
unset "pid_slot[$done_pid]" "pid_weight[$done_pid]"
done
failed=0 passed=0 total=0
printf '\n| composition | result | seconds | note | memory | account |\n| --- | --- | --- | --- | --- | --- |\n'
for id in $ids; do
r=${id%%:*}; e=${id#*:}; [ "$e" = "$id" ] && e=run
f=$out/$r-$e.tsv
total=$((total + 1))
[ -f "$f" ] || { printf '| %s | FAIL | 0 | no result | | |\n' "$r:$e"; failed=1; continue; }
awk -F'\t' '{printf "| %s | %s | %s | %s | %s | %s |\n", $1, $2, $3, $4, $5, $6}' "$f"
if [ "$(cut -f2 "$f")" = PASS ]; then passed=$((passed + 1)); else failed=1; fi
cat "$f" >> "$out/results.tsv"
done
echo "gate: $kind: $passed of $total PASS"
echo "wall: $(( $(date +%s) - g0 )) s ($jobs at once, $cap VMs); logs and readings: $out"
exit $failed