~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
208 lines · 12.6 KB Raw History
#!/usr/bin/env bash
# The knotless `test` workflow (.knotless/workflows/test.yml), one function per step; a workflow only calls it (plan 0042). By hand
# CI_SHARD=i/n: the staging step's network runs share i of the n the gate's entries are dealt into (tests/gate.sh GATE_SHARD).
#   scripts/ci-test.sh [host|page|binary|staging|stop]     no step: host, binary, staging in order (host runs page)
# CI: root on the kcp-test VM, the requester is the guest requester r2. By hand (hat-8): CI_ACCOUNT=alice; CI_HOME (default
# ~/.tmp/ci-hand) holds what CI puts in /home/r2, /usr/local/bin and /work/runs. CI_TEST_INPUT replaces the staging input;
# CI_FREEZE_TTL the freeze; SWC_PAGE_REPO a built checkout of the requester page repo instead of release's page payload; CI_IDENTITY_NAME and CI_IDENTITY_DIR the account's associated host requester (plan 0046). Nothing here builds: `release` (scripts/ci-release.sh) built and published what it downloads.
set -euo pipefail
[ "${1:-all}" = stop ] || cd "$(dirname "${BASH_SOURCE[0]}")/.."
account=${CI_ACCOUNT:-r2}
if [ "$account" = r2 ]; then
  home=/home/r2 bin=/usr/local/bin runs=${SWC_RUNS:-/work/runs} watch=/opt/swc-watch
else
  home=${CI_HOME:-$HOME/.tmp/ci-hand}
  bin=$home/bin runs=${SWC_RUNS:-$home/runs} watch=$home/swc-watch
  # never /tmp on hat-8: a tmpfs with a per-user quota (AGENTS.md)
  export TMPDIR=$home/tmp
  mkdir -p "$TMPDIR"
fi
export CI_HOME=$home
page=${SWC_PAGE_REPO:-$home/page}
base=https://socialweb-computer-kcp.sfo3.digitaloceanspaces.com
# the staging input: one staging network: an atproto VM, two bidder VMs (bidders: bidder-1, bidder-2, bob2 and bob3, each
# holding one default guest) and a gateway VM, all 4 vCPU / 8G rented from the prod bidders, and the gate on it
# (tests/gate.sh from its atproto VM as alice1..alice4), four entries at once (gateJobs: one per driver account), the
# gate's VM cap its guest slots (gateVms: unset, the module sets it to bidders), a failed entry frozen 5 s (freezeTtl: the
# suite's default 300 s holds one of the network's guest slots per failure). CI runs every entry on that one network
# (owner, 2026-11-19: staging and prod in series, for less compute): about 2 h 40 min with the stand-up (plan 0040,
# Finding "Gate shards": 9123 s of gate with two bidders, 7492 s with four). deadlineMin (from the rent: stand-up and
# gate) 250 and test.yml's staging step timeout 300 agree: 250, then a failed run's 30 min freeze (CI_FREEZE_TTL on main),
# then the teardown. Two bidders, not four: 4 staging VMs for 6 (16 vCPU / 32G of the prod bidders, not 24 / 48G) for about
# 27 min more. By hand, CI_SHARD=i/n (or `shard` in the input) runs one share of the gate on the network (gate.sh
# GATE_SHARD; the shares are disjoint and their union tests/entries.txt: scripts/check-shards.sh).
input_default='{"layout": "multi", "provision": "infra", "deadlineMin": 250, "suite": true, "bidders": 2, "gateJobs": 4, "freezeTtl": 5}'

art() { [ -n "${KNOTLESS_ARTIFACTS_URL:-}" ] && curl -sfo "$2" -H "Authorization: Bearer $KNOTLESS_ARTIFACTS_TOKEN" "$KNOTLESS_ARTIFACTS_URL?workflow=release&name=dist/$1"; }

step_host() {
  command -v curl >/dev/null && command -v jq >/dev/null || { apt-get update -qq && apt-get install -y -qq curl jq >/dev/null; }
  mkdir -p "$bin"
  local u="$base/tests/$(git rev-parse HEAD:tests)" m="$base/meshd/$(git rev-parse HEAD:rust)" d got=
  d=$(mktemp -d)
  if art tests.SHA256SUMS "$d/SHA256SUMS" && art swc-test-host "$d/swc-test-host" && art test-modules.tar.gz "$d/test-modules.tar.gz" && (cd "$d" && sha256sum -c --quiet SHA256SUMS); then
    got=artifacts; echo "tests: release artifacts of this commit"
  else
    rm -f "$d/swc-test-host" "$d/test-modules.tar.gz"
    if curl -sfo "$d/SHA256SUMS" "$u/SHA256SUMS"; then
      got=bucket; echo "tests: $u"
      for f in swc-test-host test-modules.tar.gz; do curl -sfo "$d/$f" "$u/$f"; done
    fi
  fi
  if [ -z "$got" ]; then
    echo "no test host of this tests/ tree: neither this commit's release artifacts nor $u have it (release builds it)" >&2
    exit 1
  fi
  (cd "$d" && sha256sum -c --quiet SHA256SUMS)
  tar -C tests -xzf "$d/test-modules.tar.gz"
  install -D -m 0755 "$d/swc-test-host" tests/target/release/swc-test-host
  install -m 0755 tests/target/release/swc-test-host "$bin/swc-test-host"
  # the workflow's stop runs this copy (outside the checkout)
  install -m 0755 scripts/ci-test.sh "$bin/swc-ci-test"
  if art meshd.SHA256SUMS "$d/meshd.SHA256SUMS" && art socialweb-meshd-x86_64 "$d/socialweb-meshd-x86_64"; then
    echo "meshd: release artifacts of this commit"
  elif curl -sfo "$d/meshd.SHA256SUMS" "$m/SHA256SUMS" && curl -sfo "$d/socialweb-meshd-x86_64" "$m/socialweb-meshd-x86_64"; then
    echo "meshd: $m"
  else
    echo "no meshd of this rust/ tree: neither this commit's release artifacts nor $m have it (release builds it)" >&2
    exit 1
  fi
  (cd "$d" && sha256sum -c --quiet meshd.SHA256SUMS)
  install -m 0755 "$d/socialweb-meshd-x86_64" "$bin/socialweb-meshd"
  rm -rf "$d"
  step_page
}

# 0009's page (release's scripts/ci-page.sh built it from the commits pinned in tests/compositions/staging/page-tools.json):
# unpacked to $CI_HOME/page, which the staging run gets as SWC_PAGE_REPO. By hand SWC_PAGE_REPO may name a built checkout instead.
step_page() {
  if [ -n "${SWC_PAGE_REPO:-}" ]; then
    echo "page: SWC_PAGE_REPO $SWC_PAGE_REPO (nothing fetched)"
    return 0
  fi
  local u d
  u="$base/page/$(bash scripts/ci-page.sh key)"
  d=$(mktemp -d)
  if art page.SHA256SUMS "$d/SHA256SUMS" && art page.tar.gz "$d/page.tar.gz"; then
    echo "page: release artifacts of this commit"
  elif curl -sfo "$d/SHA256SUMS" "$u/SHA256SUMS" && curl -sfo "$d/page.tar.gz" "$u/page.tar.gz"; then
    echo "page: $u"
  else
    echo "no page of this pin: neither this commit's release artifacts nor $u have it (release builds it)" >&2
    exit 1
  fi
  (cd "$d" && sha256sum -c --quiet SHA256SUMS)
  mkdir -p "$home"
  rm -rf "$page"
  tar -C "$home" --no-same-owner -xzf "$d/page.tar.gz"
  rm -rf "$d"
  echo "page: $page at $(cat "$page/COMMIT")"
}

step_binary() {
  bash scripts/ci-staging-binary.sh
}

step_staging() {
  local spec oras rc=0 freeze_ttl input
  [ -f "$page/scripts/page-round.mjs" ] || { echo "no page at $page: scripts/ci-test.sh page fetches release's (or SWC_PAGE_REPO names a built checkout)" >&2; exit 1; }
  if [ "$account" = r2 ]; then
    id r2
    install -d -o r2 "$runs"
    command -v git >/dev/null && dpkg -s libatomic1 >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq git libatomic1 >/dev/null; }
  else
    mkdir -p "$runs"
  fi
  [ -d "$watch/.git" ] || git clone -q https://knotless.dev/johnandersen777.bsky.social/socialweb-computer-watch "$watch"
  git -C "$watch" cat-file -e 9fada6f725fe0bc9f6bed8d21106670977bb659a^{commit} 2>/dev/null || git -C "$watch" fetch -q origin
  git -C "$watch" checkout -q 9fada6f725fe0bc9f6bed8d21106670977bb659a
  if [ "${KNOTLESS_RUN_REF:-}" = refs/heads/main ] || [ "$account" != r2 ]; then freeze_ttl=${CI_FREEZE_TTL:-1800}; else freeze_ttl=300; fi
  input=${CI_TEST_INPUT:-$input_default}
  # the capture covers the whole run: its deadline (from the rent; the staging module's default 180), a failure's freeze
  # and 20 min for the rent and the teardown; the end of this step kills it sooner
  sh "$watch/scripts/ci-capture.sh" "$runs/watch.ndjson" $(( $(jq -r '.deadlineMin // 180' <<<"$input") * 60 + freeze_ttl + 1200 ))
  spec="$(cat "$home/requester.spec")"
  echo "requester $spec"
  if [ "$account" = r2 ]; then
    jq --arg b "$spec" '.requester = $b | .accounts.r2 = {handle: "", did: "", guestRequester: true}' tests/targets/prod.json > "$home/target-ci.json"
    chown r2 "$home/target-ci.json"
    loginctl enable-linger r2
    for _ in $(seq 60); do [ -d "/run/user/$(id -u r2)" ] && break; sleep 1; done
  else
    # alice through her associated host requester (plan 0046): no sign-in per run
    jq --arg b "$spec" --arg a "$account" --arg n "${CI_IDENTITY_NAME:-}" --arg d "${CI_IDENTITY_DIR:-}" \
      '.requester = $b | if $n != "" then .accounts[$a].identity = {name: $n, stateDir: $d} else . end' tests/targets/prod.json > "$home/target-ci.json"
  fi
  # 0013's agent config (owner-only): the target's agentConfig, else none and 0013 fails on it; CI_AGENT_CONFIG names another file.
  # On CI it is the sealed secret ATIRC_AGENT_CONFIG (the prod target's config): a 0600 file, and the value leaves the environment
  if [ -n "${ATIRC_AGENT_CONFIG:-}" ] && [ -z "${CI_AGENT_CONFIG:-}" ]; then
    (umask 077 && printf '%s\n' "$ATIRC_AGENT_CONFIG" > "$home/agent-config.toml")
    if [ "$account" = r2 ]; then chown r2 "$home/agent-config.toml"; fi
    CI_AGENT_CONFIG=$home/agent-config.toml
  fi
  unset ATIRC_AGENT_CONFIG
  if [ -n "${CI_AGENT_CONFIG:-}" ]; then
    jq --arg c "$CI_AGENT_CONFIG" '.agentConfig = $c' "$home/target-ci.json" > "$home/target-ci.json.part" && mv "$home/target-ci.json.part" "$home/target-ci.json"
    if [ "$account" = r2 ]; then chown r2 "$home/target-ci.json"; fi
  fi
  oras=$(bash scripts/ci-oras.sh)
  if [ "$(jq -r '.suite' <<<"$input")" = true ] && [ "$(jq -r '.compat // ""' <<<"$input")" = "" ]; then
    input=$(jq -c --arg c "$(bash scripts/release-prev-rc.sh)" '.compat = $c' <<<"$input")
  fi
  # CI_SHARD=i/n (by hand): this network runs share i of the n the gate's entries are dealt into (tests/gate.sh
  # GATE_SHARD); an input that names its own `shard` keeps it
  if [ -n "${CI_SHARD:-}" ] && [ "$(jq -r '.shard // ""' <<<"$input")" = "" ]; then
    input=$(jq -c --arg s "$CI_SHARD" '.shard = $s' <<<"$input")
  fi
  local shard want
  shard=$(jq -r '.shard // ""' <<<"$input")
  want=$(GATE_SHARD=$shard tests/gate.sh --list staging) || { kill "$(cat "$runs/watch.ndjson.pid")" 2>/dev/null; exit 2; }
  echo "staging input $input"
  local run=(env HOME="$HOME" SWC_TARGET="$home/target-ci.json" SWC_RUNS="$runs" SWC_REPO="$PWD" SWC_MESHD="$bin/socialweb-meshd" SWC_PAGE_REPO="$page"
    SWC_BINARY="$home/swc-under-test" ORAS="$oras" SWC_TEST_HOST_STATIC="$bin/swc-test-host" CI=1
    SWC_TEST_HOST_URL="$base/tests/$(git rev-parse HEAD:tests)/swc-test-host" SWC_MESHD_URL="$base/meshd/$(git rev-parse HEAD:rust)/socialweb-meshd-x86_64")
  if [ "$account" = r2 ]; then
    run=(runuser -u r2 -- env HOME=/home/r2 XDG_RUNTIME_DIR="/run/user/$(id -u r2)" "${run[@]:2}")
  fi
  (cd tests && "${run[@]}" target/release/swc-test-host compositions/staging --account "$account" --on-fail freeze --freeze-ttl "$freeze_ttl" --input "$input" | tee "$runs/staging.json") || rc=$?
  # the proof lines: the run's RESULT lines and one count over the entries this network was to run (a share's, or all 47);
  # an entry with no PASS, or no result at all, fails the step whatever the run said
  if [ "$(jq -r '.suite' <<<"$input")" = true ]; then
    local total passed missing
    sed -n '/^{/,$p' "$runs/staging.json" | jq -r '.readings.suite // [] | .[] | "RESULT \(.)"' > "$runs/results.txt" 2>/dev/null || true
    cat "$runs/results.txt"
    total=$(wc -l <<<"$want")
    passed=$(awk -v want="$want" 'BEGIN { split(want, w, "\n"); for (k in w) ok[w[k]] = 1 } $3 == "PASS" && ok[$2] && !seen[$2]++ { p++ } END { print p + 0 }' "$runs/results.txt")
    missing=$(awk -v want="$want" '{ got[$2] = 1 } END { m = split(want, w, "\n"); for (k = 1; k <= m; k++) if (!got[w[k]]) printf " %s", w[k] }' "$runs/results.txt")
    [ -z "$missing" ] || echo "gate: staging: no result:$missing"
    echo "gate: staging: $passed of $total PASS${shard:+ (shard $shard)}"
    [ "$passed" = "$total" ] || rc=1
  fi
  kill "$(cat "$runs/watch.ndjson.pid")" 2>/dev/null || echo "watch: the capture was not running at the end"
  echo "watch: $(wc -l < "$runs/watch.ndjson") lines in $runs/watch.ndjson"
  return $rc
}

# The workflow's `stop:` (a cancel, a timeout, or the end of the steps): a run still going is frozen as a failure is
# (SIGUSR1: no more events, its network kept, `swc-test-host attach` served) for CI_STOP_HOLD seconds (1800) or until
# someone ends it from the attach (`cleanup`), then torn down (SIGTERM: its cleanup releases every VM).
step_stop() {
  local p="[s]wc-test-host compositions/" hold=${CI_STOP_HOLD:-1800} end
  pgrep -f "$p" >/dev/null || return 0
  pkill -USR1 -f "$p" || true
  echo "stop: the run is frozen for $hold s (attach: the run dir's FROZEN line), then cleaned up"
  end=$(( $(date +%s) + hold ))
  while pgrep -f "$p" >/dev/null && [ "$(date +%s)" -lt "$end" ]; do sleep 5; done
  pkill -TERM -f "$p" || return 0
  while pgrep -f "$p" >/dev/null; do sleep 5; done
}

case "${1:-all}" in
  host) step_host ;;
  page) step_page ;;
  binary) step_binary ;;
  staging) step_staging ;;
  stop) step_stop ;;
  all) step_host; step_binary; step_staging ;;
  *) echo "usage: scripts/ci-test.sh [host|page|binary|staging|stop]" >&2; exit 2 ;;
esac