~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
168 lines · 10.4 KB Raw History
#!/usr/bin/env bash
# The release gate: every composition against a target, as many at once as there are driver accounts.
# Usage (from tests/): ./gate.sh <target.json> [--account <name>[,<name>...]]
#                       ./gate.sh --list <staging|prod>     the entries it would run, one a line; nothing runs
# The entries are tests/entries.txt's, chosen by the target's kind: a target with `network` is a staging network (every
# entry), any other is prod (the `both` entries). No entry is skipped and none can be named: one entry on a held network is
# swc-test-host itself (docs/ci-debug.md), never a gate result.
# One "RESULT <runbook>:<entry> PASS|FAIL <seconds> <note> [MEMORY <events>]" line each as it ends, then a summary
# table in gate order (with each run's VM memory events from its readings.memory: the test modules fail a run on an OOM
# kill and list pressure); exit 1 on a FAIL.
# Parallel: one run per account at a time (an account has one driver: the test host's account lock), so the accounts
# given (GATE_ACCOUNTS, else --account, default bob) are the slots; each slot has its own iroh key. A run holds the
# VMs of its entry's weight (WEIGHT below, default 1) out of GATE_VMS (4) so the target's bidders are not oversubscribed.
# An exclusive entry (EXCLUSIVE: it fills or counts bidders, or places VMs per host) runs alone: the gate drains first.
# GATE_JOBS=1 runs it serially in gate order.
# GATE_SHARD=i/n: this gate runs share i of the n the entries are dealt into (gate_ids below), for n staging networks at
# once (by hand: scripts/ci-test.sh CI_SHARD; CI's `test` runs every entry on one network); unset, every entry. Its summary
# counts its own share.
# The heavy-job lock: held for the whole run on a target with local networks, for the build only on prod (no `network`).
# A failed run freezes (attach: the FROZEN line of its log) until GATE_FREEZE_TTL (900 s), then its slot goes on.
# SWC_COMPAT_REQUESTER (the previous rc's CLI, sha:<its commit>) is 0006:compat's requester; unset, 0006:compat FAILs.
# 0007:upgrade upgrades the bidders from SWC_UPGRADE_FROM, else SWC_COMPAT_REQUESTER; both unset, it FAILs. 0007:fence
# upgrades from its own release (v0.0.0-rc186, the last without decision 54's rule), neither env.
set -uo pipefail
entries=$(dirname "$0")/entries.txt
# runs alone: k8s nodes one per host and its tenant scale-downs (0001), a bidder core restarted (0007:restart) or upgraded
# (0007:upgrade: every bidder to another binary), its IngressProxy replaced (0007:ingress), bidders
# filled to capacity or booked (0014)
EXCLUSIVE=" ${GATE_EXCLUSIVE:-0001:run 0001:s2 0007:restart 0007:upgrade 0007:fence 0007:ingress 0014:run 0014:concurrent 0014:exhausted 0014:giveup 0014:wait 0014:impossible 0014:backtoback 0014:overcommit 0014:memfit} "
# The entries of a target kind, in gate order. GATE_SHARD=i/n: share i of n, for n networks side by side. The rule (any n up
# to the entry count; every entry in exactly one share, no share empty: scripts/check-shards.sh): the exclusive entries
# first, then the rest, each class longest first by the seconds column of entries.txt (ties: gate order), each entry to
# the share with the least seconds so far (ties: the lowest share), an entry counting its seconds plus one.
gate_ids() {
  awk -v k="$1" -v shard="${GATE_SHARD:-1/1}" -v excl="$EXCLUSIVE" '
    BEGIN { if (shard !~ /^[1-9][0-9]*\/[1-9][0-9]*$/) err = 1; split(shard, sh, "/"); i = sh[1] + 0; n = sh[2] + 0 }
    /^#/ || !NF { next }
    !(NF == 3 && ($2 == "both" || $2 == "staging") && $3 ~ /^[0-9]+$/) { err = 1; next }
    $2 == "both" || k == "staging" { m++; id[m] = $1; sec[m] = $3 + 0; x[m] = index(excl, " " $1 " ") > 0 }
    END {
      if (err || !m || i > n || n > m) exit 2
      for (c = 1; c >= 0; c--) for (;;) {
        b = 0
        for (j = 1; j <= m; j++) if (x[j] == c && !(j in to) && (!b || sec[j] > sec[b])) b = j
        if (!b) break
        s = 1
        for (j = 2; j <= n; j++) if (load[j] + 0 < load[s] + 0) s = j
        to[b] = s; load[s] += sec[b] + 1
      }
      for (j = 1; j <= m; j++) if (to[j] == i) print id[j]
    }' "$entries"
}
bad_list="gate.sh: $entries: a bad line or no entry, or GATE_SHARD (${GATE_SHARD:-unset}) is not i/n with i at most n and n at most the entries"
if [ "${1:-}" = --list ]; then
  case "${2:-}" in staging|prod) ;; *) echo "usage: gate.sh --list <staging|prod>" >&2; exit 2 ;; esac
  gate_ids "$2" || { echo "$bad_list" >&2; exit 2; }
  exit 0
fi
# The heavy-job lock (hat-8's one heavy job at a time) covers the whole run only for a target with local networks (dev, staging:
# VMs and cores run here); a target without `network` (prod) runs its VMs on the prod bidders and holds the lock only for the build.
if [ -z "${SWC_HEAVY_HELD:-}" ] && jq -e '.network' "${1:?usage: gate.sh <target.json> [--account <name>[,<name>...]]}" > /dev/null 2>&1; then
  exec "$(dirname "$0")/../scripts/swc-heavy.sh" "$0" "$@"
fi
target=$(realpath "${1:?usage: gate.sh <target.json> [--account <name>[,<name>...]]}"); shift
accounts=${GATE_ACCOUNTS:-bob}
if [ "${1:-}" = --account ]; then accounts=$2; shift 2; fi
[ $# = 0 ] || { echo "gate.sh: no entry can be named (tests/entries.txt is the list; one entry: swc-test-host, docs/ci-debug.md): $*" >&2; exit 2; }
[ -z "${GATE_SKIP:-}" ] || { echo "gate.sh: GATE_SKIP is gone: an entry passes or fails" >&2; exit 2; }
if jq -e '.network' "$target" > /dev/null 2>&1; then kind=staging; else kind=prod; fi
ids=$(gate_ids $kind) || { echo "$bad_list" >&2; exit 2; }
echo "gate: target kind $kind: $(wc -w <<<"$ids") entries${GATE_SHARD:+ (shard $GATE_SHARD)}"
# VMs an entry holds at its peak (the rest hold one; host checks none)
declare -A WEIGHT=([0002:run]=2 [0003:run]=3 [0012:run]=3 [0013:run]=2 [0013:c]=2 [0013:b]=2 [0006:run]=0 [0011:token]=0 [0016:badsha]=0 [0018:run]=0 [0017:devsup]=0 [0017:samename]=0 [0014:impossible]=0 [0014:abandon]=0 [0020:run]=0)
cd "$(dirname "$0")"
unset ATPROTO_HANDLE ATPROTO_PASSWORD ATP_USERNAME ATP_PASSWORD ATPROTO_PDS
out=${GATE_OUT:-$HOME/.tmp/swc-system-tests/gate/$(date -u +%Y%m%dT%H%M%SZ)}
mkdir -p "$out" && chmod 700 "$out"
# GATE_HOST: a prebuilt test host (its module wasm already under modules/target, as scripts/ci-test.sh host leaves them): no build
if [ -n "${GATE_HOST:-}" ]; then
  host=$GATE_HOST
elif [ -n "${SWC_HEAVY_HELD:-}" ]; then
  (cd modules && cargo wasm -q) && cargo build -q -p swc-test-host || exit 2
  host=$PWD/target/debug/swc-test-host
else
  ../scripts/swc-heavy.sh sh -c '(cd modules && cargo wasm -q) && cargo build -q -p swc-test-host' || exit 2
  host=$PWD/target/debug/swc-test-host
fi
export SWC_TARGET=$target SWC_READINGS=$out
# one entry as <account> on slot <n>: its RESULT line, and <out>/<runbook>-<entry>.tsv for the summary
run_one() {
  local id=$1 account=$2 slot=$3 r e t0 res note rc doc s mem input
  r=${id%%:*}; e=${id#*:}; [ "$e" = "$id" ] && e=run
  t0=$(date +%s)
  if [ "$r:$e" = 0006:compat ] && [ -z "${SWC_COMPAT_REQUESTER:-}" ]; then
    res=FAIL note="SWC_COMPAT_REQUESTER unset (the previous rc's CLI)"
  elif [ "$r:$e" = 0007:upgrade ] && [ -z "${SWC_UPGRADE_FROM:-}${SWC_COMPAT_REQUESTER:-}" ]; then
    res=FAIL note="SWC_UPGRADE_FROM and SWC_COMPAT_REQUESTER unset (the binary the bidders are upgraded from)"
  elif [ ! -f "compositions/$r/composition.json" ]; then
    res=FAIL note="no composition"
  else
    input=()
    case $r in 0016) input=(--input "$(jq -c '{image: {url, sha256}, marker}' ../deploy/vm-images/ci-toolchain.json)") ;; esac
    SWC_IROH_KEY=$out/iroh-$slot "$host" "compositions/$r" --account "$account" --entry "$e" --on-fail freeze \
      --freeze-ttl "${GATE_FREEZE_TTL:-900}" "${input[@]}" > "$out/$r-$e.json" 2> "$out/$r-$e.log" < /dev/null
    rc=$?
    doc=$(sed -n '/^{/,$p' "$out/$r-$e.json")
    if [ $rc = 0 ] && jq -e '.readings.skipped' <<<"$doc" > /dev/null 2>&1; then
      res=FAIL note="skipped: $(jq -r '.readings.skipped' <<<"$doc")"
    elif [ $rc = 0 ]; then
      res=PASS note=$(jq -r '.dump // ""' <<<"$doc" 2>/dev/null)
    else
      res=FAIL note=$(jq -r '[.error, (.cleanupErrors // [] | join("; "))] | map(select(. != null and . != "")) | join(" | ") | gsub("[\n\r\t]"; " ")' <<<"$doc" 2>/dev/null)
      [ -n "$note" ] || note="exit $rc: $(tail -1 "$out/$r-$e.log" | cut -c1-200)"
    fi
  fi
  s=$(( $(date +%s) - t0 ))
  mem=$(jq -r '[.readings.memory // [] | .[] | "\(.event) \(.vm)"] | unique | join(", ")' "$out/$r-$e.json" 2>/dev/null)
  echo "RESULT $r:$e $res $s $note${mem:+ MEMORY $mem} (${account})"
  printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$r:$e" "$res" "$s" "$note" "$mem" "$account" > "$out/$r-$e.tsv"
}
IFS=, read -ra slots <<<"$accounts"
jobs=${GATE_JOBS:-${#slots[@]}}
[ "$jobs" -le "${#slots[@]}" ] || jobs=${#slots[@]}
cap=${GATE_VMS:-4}
declare -A pid_slot=() pid_weight=()
free=()
for ((n = 0; n < jobs; n++)); do free+=("$n"); done
used=0 excl=0 queue=($ids)
g0=$(date +%s)
while [ ${#queue[@]} -gt 0 ] || [ ${#pid_slot[@]} -gt 0 ]; do
  left=()
  barrier=0
  for id in "${queue[@]}"; do
    w=${WEIGHT[$id]:-1}
    [[ $id != *:* ]] && w=${WEIGHT[$id:run]:-1}
    x=0; [[ $EXCLUSIVE == *" $id "* || $EXCLUSIVE == *" $id:run "* ]] && x=1
    if [ $barrier = 1 ] || [ $excl = 1 ] || [ ${#free[@]} = 0 ] || { [ $x = 1 ] && [ ${#pid_slot[@]} -gt 0 ]; } || [ $((used + w)) -gt "$cap" -a ${#pid_slot[@]} -gt 0 ]; then
      left+=("$id")
      # an exclusive entry is a barrier: nothing after it starts before it
      [ $x = 1 ] && barrier=1
      continue
    fi
    slot=${free[0]}; free=("${free[@]:1}")
    run_one "$id" "${slots[$slot]}" "$slot" &
    pid_slot[$!]=$slot pid_weight[$!]=$w
    used=$((used + w)); excl=$x
  done
  queue=("${left[@]}")
  [ ${#pid_slot[@]} -gt 0 ] || continue
  wait -n -p done_pid
  free+=("${pid_slot[$done_pid]}")
  used=$((used - pid_weight[$done_pid])); excl=0
  unset "pid_slot[$done_pid]" "pid_weight[$done_pid]"
done
failed=0 passed=0 total=0
printf '\n| composition | result | seconds | note | memory | account |\n| --- | --- | --- | --- | --- | --- |\n'
for id in $ids; do
  r=${id%%:*}; e=${id#*:}; [ "$e" = "$id" ] && e=run
  f=$out/$r-$e.tsv
  total=$((total + 1))
  [ -f "$f" ] || { printf '| %s | FAIL | 0 | no result |  |  |\n' "$r:$e"; failed=1; continue; }
  awk -F'\t' '{printf "| %s | %s | %s | %s | %s | %s |\n", $1, $2, $3, $4, $5, $6}' "$f"
  if [ "$(cut -f2 "$f")" = PASS ]; then passed=$((passed + 1)); else failed=1; fi
  cat "$f" >> "$out/results.tsv"
done
echo "gate: $kind: $passed of $total PASS"
echo "wall: $(( $(date +%s) - g0 )) s ($jobs at once, $cap VMs); logs and readings: $out"
exit $failed