johnandersen777.bsky.social/knotless

Switch repository

Loading…

#!/bin/sh
# Moves `run` onto what docker-compose.yml beside this script now says,
# the runner running handing over to the new one rather than stopping
# first:
#
#   /etc/knotless/handover.sh <settings hash, or none>
#
# deploy.sh copies it to the host and runs it there when the runner is to
# be restarted. The runner leading, the newest still to be restarted by
# docker, is sent SIGTERM: it drops the runs waiting that it will not
# run, writes the state file a last time, and from then on a note of its
# own beside it (state.handover/), while it finishes the runs it has
# going (docs/design/runner.md, "A stop"). Once that note is there, the
# new runner starts beside it, reads the state file and takes up the
# rest, and the old one, no longer restarted, is removed once it exits,
# --stop-wait at most after its stop.
#
# A runner that writes no note within KNOTLESS_HANDOVER_WAIT seconds (60),
# one built before runners handed over, is stopped first instead, and the
# new one started once it has, as every deploy once did.
#
# Once the new runner is up, /etc/knotless/run-deployed says
# "<container id> <settings hash>", for the next deploy to compare.

set -eu
cd "$(dirname "$0")"
SETTINGS="${1:-none}"
WAIT="${KNOTLESS_HANDOVER_WAIT:-60}"
LOG="${KNOTLESS_DEPLOY_LOG:-/var/log/knotless-deploy.log}"
DATA=$(sed -n 's/^KNOTLESS_RUN_DATA=//p' .env 2>/dev/null | tail -n 1)
NOTES="${DATA:-/var/lib/knotless-run}/state.handover"

say() {
  line="$(date -u +%Y-%m-%dT%H:%M:%SZ) → run: $*"
  echo "$line"
  echo "$line" >>"$LOG" 2>/dev/null || true
}

# The runners docker restarts, newest last: the one leading, and any whose
# handover a deploy cut short.
leading() {
  docker compose ps -q --status running run |
    xargs -r docker inspect -f '{{.Created}} {{.HostConfig.RestartPolicy.Name}} {{.Id}}' |
    awk '$2 == "always"' | sort | awk '{ print $3 }'
}

# The notes beside the state file now.
notes() {
  ls "$NOTES" 2>/dev/null | grep '\.json$' || true
}

# Notes the new runner as the one deployed with these settings.
record() {
  id=$(leading | tail -n 1)
  [ -z "$id" ] || [ "$SETTINGS" = none ] || printf '%s %s\n' "$id" "$SETTINGS" >run-deployed
}

rm -f run-deployed
# One a recreate cut short leaves never started, and those that handed
# over and have since exited.
docker compose ps -aq --status created run | xargs -r docker rm >/dev/null
docker compose ps -aq --status exited run | xargs -r docker rm >/dev/null

old=$(leading)
if [ -z "$old" ]; then
  say "starting"
  docker compose up -d --no-deps run
  record
  exit
fi

had=$(notes)
for id in $old; do
  docker update --restart=no "$id" >/dev/null
  docker kill -s TERM "$id" >/dev/null
done
waited=0
while :; do
  new=$(notes | grep -vxF "$had" || true)
  [ -z "$new" ] || break
  if [ "$waited" -ge "$WAIT" ]; then
    say "$(echo "$old" | cut -c1-12 | tr '\n' ' ')wrote no handover note in ${WAIT}s: stopping it first"
    for id in $old; do docker update --restart=always "$id" >/dev/null; done
    docker compose up -d --no-deps --force-recreate run
    record
    exit
  fi
  sleep 1
  waited=$((waited + 1))
done

running=$(docker compose ps -q --status running run | wc -l)
say "handed over in ${waited}s; starting the new one beside $(echo "$old" | cut -c1-12 | tr '\n' ' ')"
docker compose up -d --no-deps --no-recreate --scale "run=$((running + 1))" run
record
say "started $(leading | tail -n 1 | cut -c1-12)"

# The old ones exit once their runs end; each is removed then, in the
# background, so a deploy by hand does not wait for them.
for id in $old; do
  setsid nohup sh -c 'docker wait "$1" >/dev/null 2>&1; docker rm "$1" >/dev/null 2>&1' gone "$id" \
    >/dev/null 2>&1 </dev/null &
done