johnandersen777.bsky.social/knotless

Switch repository

Loading…

#!/usr/bin/env bash
# Tries rollout.sh on this machine's docker, with requests flowing the whole
# time, and counts the ones that failed:
#
#   deploy/rollout-test.sh
#
# It builds the image (SKIP_BUILD=1 reuses knotless-rolltest:current), then
# runs docker-compose.yml as its own project, knotless-rolltest, behind a
# Caddy on its own network that proxies to knotless-serve:2590 as the
# proxy in front of knotless.dev does, on 127.0.0.1:$PORT (28480). Four
# clients ask for /healthz (serve alone) and / (serve, the pages, and serve
# again for their data) through Caddy while it:
#
# 1. rolls pages and serve onto a new image, with Caddy as it is today;
# 2. rolls again, with lb_try_duration on the proxy (README.md);
# 3. rolls a serve whose index would be rebuilt, which stops and starts it;
# 4. rolls a serve that never turns healthy, which must leave the old one;
# 5. rolls a serve that cannot check the index, which must too.
#
# Everything it makes is removed at the end, but the image with KEEP_IMAGE=1.

set -euo pipefail
cd "$(dirname "$0")/.."
PORT="${PORT:-28480}"
PROJECT=knotless-rolltest
NETWORK=knotless-rolltest-proxy
IMAGE=knotless-rolltest:current
WORK="$(mktemp -d)"
URL="http://127.0.0.1:$PORT"

cleanup() {
  touch "$WORK/stop"
  wait 2>/dev/null || true
  (cd "$WORK" && docker compose down -v --remove-orphans -t 5 >/dev/null 2>&1) || true
  docker rm -f "$PROJECT-caddy" >/dev/null 2>&1 || true
  docker network rm "$NETWORK" >/dev/null 2>&1 || true
  docker image rm "$PROJECT:broken" >/dev/null 2>&1 || true
  docker tag "$PROJECT:base" "$IMAGE" 2>/dev/null || true
  docker image rm "$PROJECT:base" >/dev/null 2>&1 || true
  for rev in 1 2 3; do docker image rm "$PROJECT:rev$rev" >/dev/null 2>&1 || true; done
  [ -n "${KEEP_IMAGE:-}" ] || docker image rm "$IMAGE" >/dev/null 2>&1 || true
  rm -rf "$WORK"
}
trap cleanup EXIT

[ -n "${SKIP_BUILD:-}" ] || docker build -t "$IMAGE" .
# A tag of its own: a build cannot start FROM an image ID.
BASE="$PROJECT:base"
docker tag "$IMAGE" "$BASE"

# The deploy's compose file and rollout.sh, as deploy.sh puts them on a
# host, with this project's names and paths.
mkdir -p "$WORK/data"
sed -e "s|^name: knotless$|name: $PROJECT|" \
  -e "s|image: knotless:current|image: $IMAGE|" \
  -e "s|driver: journald|driver: json-file|" \
  deploy/docker-compose.yml >"$WORK/docker-compose.yml"
cp deploy/rollout.sh "$WORK/"
cat >"$WORK/.env" <<EOF
KNOTLESS_DOMAIN=localhost
KNOTLESS_PROXY_NETWORK=$NETWORK
KNOTLESS_DATA=$WORK/data
KNOTLESS_JETSTREAM=
KNOTLESS_RELAY=http://127.0.0.1:9
EOF

caddyfile() {
  {
    echo ":80 {"
    echo "  reverse_proxy knotless-serve:2590 {"
    [ "$1" = retry ] && echo "    lb_try_duration 20s" && echo "    lb_try_interval 250ms"
    echo "  }"
    echo "}"
  } >"$WORK/Caddyfile"
}

docker network create "$NETWORK" >/dev/null
caddyfile plain
docker run -d --name "$PROJECT-caddy" --network "$NETWORK" -p "127.0.0.1:$PORT:80" \
  -v "$WORK/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2 >/dev/null

compose() { (cd "$WORK" && docker compose "$@"); }
rollout() { (cd "$WORK" && KNOTLESS_DEPLOY_LOG="$WORK/deploy.log" sh rollout.sh "$@"); }

# A new image of the same build, as a deploy that changed something makes.
retag() {
  printf 'FROM %s\nLABEL rolltest=%s\n' "$BASE" "$1" | docker build -q -t "$PROJECT:$1" - >/dev/null
  docker tag "$PROJECT:$1" "$IMAGE"
}

load() {
  rm -f "$WORK/stop" "$WORK"/codes.*
  for w in 1 2 3 4; do
    path=/healthz
    [ $((w % 2)) -eq 0 ] && path=/
    (while [ ! -f "$WORK/stop" ]; do
      curl -s -o /dev/null -m 30 -w '%{http_code}\n' "$URL$path" >>"$WORK/codes.$w" || true
    done) &
  done
  sleep 3
}

report() {
  sleep 3
  touch "$WORK/stop"
  wait
  local total failed
  total=$(cat "$WORK"/codes.* | wc -l | tr -d ' ')
  failed=$(cat "$WORK"/codes.* | grep -cvE '^[23]' || true)
  echo "== $1: $total requests, $failed failed ($(sort "$WORK"/codes.* | uniq -c | tr -s ' ' | tr '\n' ',' | sed 's/,$//'))"
}

echo "→ first start"
rollout pages serve
for _ in $(seq 60); do curl -sf -o /dev/null "$URL/" && break; sleep 1; done
curl -sf -o /dev/null "$URL/" || { echo "the page never answered" >&2; compose logs --tail 30 >&2; exit 1; }

echo "→ 1. rolling, Caddy as today"
load
retag rev1
rollout pages serve
report "rolling, no lb_try_duration"

echo "→ 2. rolling, with lb_try_duration"
caddyfile retry
docker exec "$PROJECT-caddy" caddy reload --config /etc/caddy/Caddyfile >/dev/null 2>&1
load
retag rev2
rollout pages serve
report "rolling, lb_try_duration 20s"

echo "→ 3. an index the new build would rebuild: stop, then start"
compose exec -T serve node -e \
  "new (require('node:sqlite').DatabaseSync)('/data/index.db').exec('PRAGMA user_version = 1')"
load
retag rev3
rollout serve | tee "$WORK/rollout.out"
report "rebuilt index, stop then start, lb_try_duration 20s"
grep -q "old one stops first" "$WORK/rollout.out" || { echo "it did not stop and start" >&2; exit 1; }

# A build of serve that answers the index check but never listens, and one
# that cannot even check: each must leave the old serve serving.
broken() {
  printf 'FROM %s\nENTRYPOINT %s\n' "$BASE" "$2" | docker build -q -t "$PROJECT:broken" - >/dev/null
  docker tag "$PROJECT:broken" "$IMAGE"
  before=$(compose ps -q serve)
  load
  if KNOTLESS_ROLLOUT_TIMEOUT=15 rollout serve; then
    echo "a broken serve was rolled to" >&2
    exit 1
  fi
  report "$1, kept the old one"
  [ "$(compose ps -q serve)" = "$before" ] || { echo "the old serve is not the one left" >&2; exit 1; }
  echo "→ kept $(echo "$before" | cut -c1-12)"
}

echo "→ 4. a new serve that never turns healthy"
broken "never healthy" '["sh", "-c", "case \"$*\" in *--check-index*) exec \"$@\" ;; *) exec sleep 3600 ;; esac", "--"]'
echo "→ 5. a new serve that cannot check the index"
broken "fails the index check" '["false"]'