#!/usr/bin/env bash # The knotless `test` workflow (.knotless/workflows/test.yml), one function per step; a workflow only calls it (plan 0042). By hand # CI_SHARD=i/n: the staging step's network runs share i of the n the gate's entries are dealt into (tests/gate.sh GATE_SHARD). # scripts/ci-test.sh [host|page|binary|staging|stop] no step: host, binary, staging in order (host runs page) # CI: root on the kcp-test VM, the requester is the guest requester r2. By hand (hat-8): CI_ACCOUNT=alice; CI_HOME (default # ~/.tmp/ci-hand) holds what CI puts in /home/r2, /usr/local/bin and /work/runs. CI_TEST_INPUT replaces the staging input; # CI_FREEZE_TTL the freeze; SWC_PAGE_REPO a built checkout of the requester page repo instead of release's page payload; CI_IDENTITY_NAME and CI_IDENTITY_DIR the account's associated host requester (plan 0046). Nothing here builds: `release` (scripts/ci-release.sh) built and published what it downloads. set -euo pipefail [ "${1:-all}" = stop ] || cd "$(dirname "${BASH_SOURCE[0]}")/.." account=${CI_ACCOUNT:-r2} if [ "$account" = r2 ]; then home=/home/r2 bin=/usr/local/bin runs=${SWC_RUNS:-/work/runs} watch=/opt/swc-watch else home=${CI_HOME:-$HOME/.tmp/ci-hand} bin=$home/bin runs=${SWC_RUNS:-$home/runs} watch=$home/swc-watch # never /tmp on hat-8: a tmpfs with a per-user quota (AGENTS.md) export TMPDIR=$home/tmp mkdir -p "$TMPDIR" fi export CI_HOME=$home page=${SWC_PAGE_REPO:-$home/page} base=https://socialweb-computer-kcp.sfo3.digitaloceanspaces.com # the staging input: one staging network: an atproto VM, two bidder VMs (bidders: bidder-1, bidder-2, bob2 and bob3, each # holding one default guest) and a gateway VM, all 4 vCPU / 8G rented from the prod bidders, and the gate on it # (tests/gate.sh from its atproto VM as alice1..alice4), four entries at once (gateJobs: one per driver account), the # gate's VM cap its guest slots (gateVms: unset, the module sets it to bidders), a failed entry frozen 5 s (freezeTtl: the # suite's default 300 s holds one of the network's guest slots per failure). CI runs every entry on that one network # (owner, 2026-11-19: staging and prod in series, for less compute): about 2 h 40 min with the stand-up (plan 0040, # Finding "Gate shards": 9123 s of gate with two bidders, 7492 s with four). deadlineMin (from the rent: stand-up and # gate) 250 and test.yml's staging step timeout 300 agree: 250, then a failed run's 30 min freeze (CI_FREEZE_TTL on main), # then the teardown. Two bidders, not four: 4 staging VMs for 6 (16 vCPU / 32G of the prod bidders, not 24 / 48G) for about # 27 min more. By hand, CI_SHARD=i/n (or `shard` in the input) runs one share of the gate on the network (gate.sh # GATE_SHARD; the shares are disjoint and their union tests/entries.txt: scripts/check-shards.sh). input_default='{"layout": "multi", "provision": "infra", "deadlineMin": 250, "suite": true, "bidders": 2, "gateJobs": 4, "freezeTtl": 5}' art() { [ -n "${KNOTLESS_ARTIFACTS_URL:-}" ] && curl -sfo "$2" -H "Authorization: Bearer $KNOTLESS_ARTIFACTS_TOKEN" "$KNOTLESS_ARTIFACTS_URL?workflow=release&name=dist/$1"; } step_host() { command -v curl >/dev/null && command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq curl jq >/dev/null; } mkdir -p "$bin" local u="$base/tests/$(git rev-parse HEAD:tests)" m="$base/meshd/$(git rev-parse HEAD:rust)" d got= d=$(mktemp -d) if art tests.SHA256SUMS "$d/SHA256SUMS" && art swc-test-host "$d/swc-test-host" && art test-modules.tar.gz "$d/test-modules.tar.gz" && (cd "$d" && sha256sum -c --quiet SHA256SUMS); then got=artifacts; echo "tests: release artifacts of this commit" else rm -f "$d/swc-test-host" "$d/test-modules.tar.gz" if curl -sfo "$d/SHA256SUMS" "$u/SHA256SUMS"; then got=bucket; echo "tests: $u" for f in swc-test-host test-modules.tar.gz; do curl -sfo "$d/$f" "$u/$f"; done fi fi if [ -z "$got" ]; then echo "no test host of this tests/ tree: neither this commit's release artifacts nor $u have it (release builds it)" >&2 exit 1 fi (cd "$d" && sha256sum -c --quiet SHA256SUMS) tar -C tests -xzf "$d/test-modules.tar.gz" install -D -m 0755 "$d/swc-test-host" tests/target/release/swc-test-host install -m 0755 tests/target/release/swc-test-host "$bin/swc-test-host" # the workflow's stop runs this copy (outside the checkout) install -m 0755 scripts/ci-test.sh "$bin/swc-ci-test" if art meshd.SHA256SUMS "$d/meshd.SHA256SUMS" && art socialweb-meshd-x86_64 "$d/socialweb-meshd-x86_64"; then echo "meshd: release artifacts of this commit" elif curl -sfo "$d/meshd.SHA256SUMS" "$m/SHA256SUMS" && curl -sfo "$d/socialweb-meshd-x86_64" "$m/socialweb-meshd-x86_64"; then echo "meshd: $m" else echo "no meshd of this rust/ tree: neither this commit's release artifacts nor $m have it (release builds it)" >&2 exit 1 fi (cd "$d" && sha256sum -c --quiet meshd.SHA256SUMS) install -m 0755 "$d/socialweb-meshd-x86_64" "$bin/socialweb-meshd" rm -rf "$d" step_page } # 0009's page (release's scripts/ci-page.sh built it from the commits pinned in tests/compositions/staging/page-tools.json): # unpacked to $CI_HOME/page, which the staging run gets as SWC_PAGE_REPO. By hand SWC_PAGE_REPO may name a built checkout instead. step_page() { if [ -n "${SWC_PAGE_REPO:-}" ]; then echo "page: SWC_PAGE_REPO $SWC_PAGE_REPO (nothing fetched)" return 0 fi local u d u="$base/page/$(bash scripts/ci-page.sh key)" d=$(mktemp -d) if art page.SHA256SUMS "$d/SHA256SUMS" && art page.tar.gz "$d/page.tar.gz"; then echo "page: release artifacts of this commit" elif curl -sfo "$d/SHA256SUMS" "$u/SHA256SUMS" && curl -sfo "$d/page.tar.gz" "$u/page.tar.gz"; then echo "page: $u" else echo "no page of this pin: neither this commit's release artifacts nor $u have it (release builds it)" >&2 exit 1 fi (cd "$d" && sha256sum -c --quiet SHA256SUMS) mkdir -p "$home" rm -rf "$page" tar -C "$home" --no-same-owner -xzf "$d/page.tar.gz" rm -rf "$d" echo "page: $page at $(cat "$page/COMMIT")" } step_binary() { bash scripts/ci-staging-binary.sh } step_staging() { local spec oras rc=0 freeze_ttl input [ -f "$page/scripts/page-round.mjs" ] || { echo "no page at $page: scripts/ci-test.sh page fetches release's (or SWC_PAGE_REPO names a built checkout)" >&2; exit 1; } if [ "$account" = r2 ]; then id r2 install -d -o r2 "$runs" command -v git >/dev/null && dpkg -s libatomic1 >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq git libatomic1 >/dev/null; } else mkdir -p "$runs" fi [ -d "$watch/.git" ] || git clone -q https://knotless.dev/johnandersen777.bsky.social/socialweb-computer-watch "$watch" git -C "$watch" cat-file -e 9fada6f725fe0bc9f6bed8d21106670977bb659a^{commit} 2>/dev/null || git -C "$watch" fetch -q origin git -C "$watch" checkout -q 9fada6f725fe0bc9f6bed8d21106670977bb659a if [ "${KNOTLESS_RUN_REF:-}" = refs/heads/main ] || [ "$account" != r2 ]; then freeze_ttl=${CI_FREEZE_TTL:-1800}; else freeze_ttl=300; fi input=${CI_TEST_INPUT:-$input_default} # the capture covers the whole run: its deadline (from the rent; the staging module's default 180), a failure's freeze # and 20 min for the rent and the teardown; the end of this step kills it sooner sh "$watch/scripts/ci-capture.sh" "$runs/watch.ndjson" $(( $(jq -r '.deadlineMin // 180' <<<"$input") * 60 + freeze_ttl + 1200 )) spec="$(cat "$home/requester.spec")" echo "requester $spec" if [ "$account" = r2 ]; then jq --arg b "$spec" '.requester = $b | .accounts.r2 = {handle: "", did: "", guestRequester: true}' tests/targets/prod.json > "$home/target-ci.json" chown r2 "$home/target-ci.json" loginctl enable-linger r2 for _ in $(seq 60); do [ -d "/run/user/$(id -u r2)" ] && break; sleep 1; done else # alice through her associated host requester (plan 0046): no sign-in per run jq --arg b "$spec" --arg a "$account" --arg n "${CI_IDENTITY_NAME:-}" --arg d "${CI_IDENTITY_DIR:-}" \ '.requester = $b | if $n != "" then .accounts[$a].identity = {name: $n, stateDir: $d} else . end' tests/targets/prod.json > "$home/target-ci.json" fi # 0013's agent config (owner-only): the target's agentConfig, else none and 0013 fails on it; CI_AGENT_CONFIG names another file. # On CI it is the sealed secret ATIRC_AGENT_CONFIG (the prod target's config): a 0600 file, and the value leaves the environment if [ -n "${ATIRC_AGENT_CONFIG:-}" ] && [ -z "${CI_AGENT_CONFIG:-}" ]; then (umask 077 && printf '%s\n' "$ATIRC_AGENT_CONFIG" > "$home/agent-config.toml") if [ "$account" = r2 ]; then chown r2 "$home/agent-config.toml"; fi CI_AGENT_CONFIG=$home/agent-config.toml fi unset ATIRC_AGENT_CONFIG if [ -n "${CI_AGENT_CONFIG:-}" ]; then jq --arg c "$CI_AGENT_CONFIG" '.agentConfig = $c' "$home/target-ci.json" > "$home/target-ci.json.part" && mv "$home/target-ci.json.part" "$home/target-ci.json" if [ "$account" = r2 ]; then chown r2 "$home/target-ci.json"; fi fi oras=$(bash scripts/ci-oras.sh) if [ "$(jq -r '.suite' <<<"$input")" = true ] && [ "$(jq -r '.compat // ""' <<<"$input")" = "" ]; then input=$(jq -c --arg c "$(bash scripts/release-prev-rc.sh)" '.compat = $c' <<<"$input") fi # CI_SHARD=i/n (by hand): this network runs share i of the n the gate's entries are dealt into (tests/gate.sh # GATE_SHARD); an input that names its own `shard` keeps it if [ -n "${CI_SHARD:-}" ] && [ "$(jq -r '.shard // ""' <<<"$input")" = "" ]; then input=$(jq -c --arg s "$CI_SHARD" '.shard = $s' <<<"$input") fi local shard want shard=$(jq -r '.shard // ""' <<<"$input") want=$(GATE_SHARD=$shard tests/gate.sh --list staging) || { kill "$(cat "$runs/watch.ndjson.pid")" 2>/dev/null; exit 2; } echo "staging input $input" local run=(env HOME="$HOME" SWC_TARGET="$home/target-ci.json" SWC_RUNS="$runs" SWC_REPO="$PWD" SWC_MESHD="$bin/socialweb-meshd" SWC_PAGE_REPO="$page" SWC_BINARY="$home/swc-under-test" ORAS="$oras" SWC_TEST_HOST_STATIC="$bin/swc-test-host" CI=1 SWC_TEST_HOST_URL="$base/tests/$(git rev-parse HEAD:tests)/swc-test-host" SWC_MESHD_URL="$base/meshd/$(git rev-parse HEAD:rust)/socialweb-meshd-x86_64") if [ "$account" = r2 ]; then run=(runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR="/run/user/$(id -u r2)" "${run[@]:2}") fi (cd tests && "${run[@]}" target/release/swc-test-host compositions/staging --account "$account" --on-fail freeze --freeze-ttl "$freeze_ttl" --input "$input" | tee "$runs/staging.json") || rc=$? # the proof lines: the run's RESULT lines and one count over the entries this network was to run (a share's, or all 47); # an entry with no PASS, or no result at all, fails the step whatever the run said if [ "$(jq -r '.suite' <<<"$input")" = true ]; then local total passed missing sed -n '/^{/,$p' "$runs/staging.json" | jq -r '.readings.suite // [] | .[] | "RESULT \(.)"' > "$runs/results.txt" 2>/dev/null || true cat "$runs/results.txt" total=$(wc -l <<<"$want") passed=$(awk -v want="$want" 'BEGIN { split(want, w, "\n"); for (k in w) ok[w[k]] = 1 } $3 == "PASS" && ok[$2] && !seen[$2]++ { p++ } END { print p + 0 }' "$runs/results.txt") missing=$(awk -v want="$want" '{ got[$2] = 1 } END { m = split(want, w, "\n"); for (k = 1; k <= m; k++) if (!got[w[k]]) printf " %s", w[k] }' "$runs/results.txt") [ -z "$missing" ] || echo "gate: staging: no result:$missing" echo "gate: staging: $passed of $total PASS${shard:+ (shard $shard)}" [ "$passed" = "$total" ] || rc=1 fi kill "$(cat "$runs/watch.ndjson.pid")" 2>/dev/null || echo "watch: the capture was not running at the end" echo "watch: $(wc -l < "$runs/watch.ndjson") lines in $runs/watch.ndjson" return $rc } # The workflow's `stop:` (a cancel, a timeout, or the end of the steps): a run still going is frozen as a failure is # (SIGUSR1: no more events, its network kept, `swc-test-host attach` served) for CI_STOP_HOLD seconds (1800) or until # someone ends it from the attach (`cleanup`), then torn down (SIGTERM: its cleanup releases every VM). step_stop() { local p="[s]wc-test-host compositions/" hold=${CI_STOP_HOLD:-1800} end pgrep -f "$p" >/dev/null || return 0 pkill -USR1 -f "$p" || true echo "stop: the run is frozen for $hold s (attach: the run dir's FROZEN line), then cleaned up" end=$(( $(date +%s) + hold )) while pgrep -f "$p" >/dev/null && [ "$(date +%s)" -lt "$end" ]; do sleep 5; done pkill -TERM -f "$p" || return 0 while pgrep -f "$p" >/dev/null; do sleep 5; done } case "${1:-all}" in host) step_host ;; page) step_page ;; binary) step_binary ;; staging) step_staging ;; stop) step_stop ;; all) step_host; step_binary; step_staging ;; *) echo "usage: scripts/ci-test.sh [host|page|binary|staging|stop]" >&2; exit 2 ;; esac