0
0
#!/bin/bash
set -euo pipefail
REPO="$(cd "$(dirname "$0")/.." && pwd)"
SSH_TARGET="${SSH_TARGET:-root@mini-cloud-0002.chadig.com}"
SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519_no_password}"
RELAY_HOSTNAME="${RELAY_HOSTNAME:-relay.mini-cloud-0002.chadig.com}"
LISTEN="${LISTEN:-127.0.0.1:2584}"
PREFLIGHT_LISTEN="${PREFLIGHT_LISTEN:-127.0.0.1:2585}"
BASE="${BASE:-/opt/atproto-relay-go}"
STATE_DIR="${STATE_DIR:-/var/lib/atproto-relay-go}"
RUN_USER="${RUN_USER:-johnandersen777}"
DENO="${DENO:-/home/$RUN_USER/.deno/bin/deno}"
OUT_ROOT="${OUT_ROOT:-$HOME/.tmp/atproto-relay-deploy}"
UNIT=atproto-relay.service
DROPIN_DIR=/etc/systemd/system/atproto-relay.service.d
DROPIN="$DROPIN_DIR/30-go.conf"
BIN_NAME=socialweb-computer
TS_DID="did:web:127.0.0.1"
GO_DID="did:web:$RELAY_HOSTNAME"
START_SEQ_MARGIN=1000
KEEP_VERSIONS=3
SSH_OPTS=(-o IdentitiesOnly=yes -i "$SSH_KEY" -o BatchMode=yes -o ConnectTimeout=20 -o ServerAliveInterval=15)
usage() {
cat <<EOF
usage: scripts/deploy-atproto-relay.sh [--binary PATH [--sha256 HEX]] [--dry-run]
scripts/deploy-atproto-relay.sh --status | --rollback
(no mode) deploy the Go atproto relay (socialweb-computer internal atproto-relay) in place of the
TypeScript relay of $UNIT on $SSH_TARGET:
1. the binary: --binary PATH (a release binary: --sha256 HEX, or a SHA256SUMS beside it that
names it; a dev binary without either is hashed here), or, without --binary, a
linux/amd64 build (CGO off) of this checkout
2. ship it to $BASE/<sha256 prefix>/$BIN_NAME (skipped when it is there) and check its sha256
3. preflight: run it as $RUN_USER on $PREFLIGHT_LISTEN with a scratch state dir and wait for
/xrpc/_health; the TypeScript relay keeps serving
4. while the TypeScript relay serves: save its listHosts to $STATE_DIR/seed-hosts.json (the
Go relay adds those hosts as idle known hosts) and read its newest firehose seq; the Go
relay continues its firehose at that seq plus $START_SEQ_MARGIN ($BASE/start-seq)
5. write $DROPIN (Type=notify, the Go ExecStart on $LISTEN, state in $STATE_DIR, no
EnvironmentFile), restart $UNIT and check http://$LISTEN and https://$RELAY_HOSTNAME:
describeServer must name $GO_DID; a failed check rolls back
caddy is not touched: it keeps proxying https://$RELAY_HOSTNAME to $LISTEN
--dry-run do steps 1 to 4 read-only (no build, no upload, no preflight, no file written) and print the
plan: the binary, the remote state, the TypeScript relay's hosts and seq, the drop-in
--rollback remove $DROPIN and restart $UNIT: the TypeScript relay exactly as before ($STATE_DIR and
$BASE stay for a later deploy)
--status print which relay serves $UNIT and its health; changes nothing
env: SSH_TARGET, SSH_KEY, RELAY_HOSTNAME, LISTEN, PREFLIGHT_LISTEN, BASE, STATE_DIR, RUN_USER, DENO,
OUT_ROOT, START_SEQ (overrides step 4's seq), ALLOWED_HOSTS (--atproto-relay-allowed-hosts),
NO_AUTO_ROLLBACK=1
EOF
}
say() {
echo "deploy-atproto-relay: $*"
}
q() {
printf "%q" "$1"
}
remote() {
ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "$@"
}
unit_state() {
remote "systemctl show $UNIT -p ActiveState -p SubState -p MainPID -p ExecMainStartTimestamp -p DropInPaths -p NRestarts --no-pager
pid=\$(systemctl show $UNIT -p MainPID --value)
if [ \"\$pid\" != 0 ]; then echo \"MainExe=\$(readlink /proc/\$pid/exe)\"; fi
echo \"Current=\$(readlink $(q "$BASE/current") || true)\"
if [ -f $(q "$DROPIN") ]; then echo \"DropIn=\$(grep '^ExecStart=.' $(q "$DROPIN") | cut -c1-400)\"; else echo DropIn=none; fi
if [ -f $(q "$STATE_DIR/relay-state.json") ]; then echo \"State=\$(stat -c '%s bytes %U:%G %a' $(q "$STATE_DIR/relay-state.json"))\"; else echo State=none; fi"
}
fetch() {
local path="$1" body code
body="$(curl -sS --max-time 15 -w '\n%{http_code}' "https://$RELAY_HOSTNAME$path" 2>/dev/null || true)"
code="${body##*$'\n'}"
body="${body%$'\n'*}"
printf '%s %s\n' "$code" "$body"
}
loopback() {
remote "curl -sS --max-time 10 -w ' %{http_code}' http://$LISTEN$1 || true"
}
hosts_summary() {
local body
body="$(curl -sS --max-time 20 "https://$RELAY_HOSTNAME/xrpc/com.atproto.sync.listHosts" 2>/dev/null || true)"
printf '%s' "$body" | jq -c '{hosts: (.hosts | length), states: (.hosts | group_by(.state) | map({(.[0].state): length}) | add)}' 2>/dev/null || echo "unreadable"
}
serving_did() {
fetch /xrpc/com.atproto.server.describeServer | sed -n 's/.*"did":"\([^"]*\)".*/\1/p'
}
report() {
say "unit:"
unit_state | sed 's/^/ /'
say "https://$RELAY_HOSTNAME/xrpc/_health: $(fetch /xrpc/_health)"
say "https://$RELAY_HOSTNAME describeServer did: $(serving_did)"
say "https://$RELAY_HOSTNAME listHosts: $(hosts_summary)"
say "last journal lines:"
remote "journalctl -u $UNIT -n 8 --no-pager -o cat | cut -c1-240" | sed 's/^/ /'
}
restart() {
remote "systemctl daemon-reload && systemctl restart $UNIT"
}
wait_listen() {
remote "for i in \$(seq 1 60); do curl -sf --max-time 2 -o /dev/null http://$LISTEN/xrpc/_health && exit 0; sleep 0.5; done; exit 1"
}
rollback() {
say "removing $DROPIN and restarting $UNIT on the TypeScript relay"
remote "rm -f $(q "$DROPIN") && rmdir --ignore-fail-on-non-empty $(q "$DROPIN_DIR") 2>/dev/null || true"
restart
if ! wait_listen; then
say "rollback: $UNIT does not answer on $LISTEN"
report
exit 1
fi
local did
did="$(serving_did)"
report
if [ "$did" = "$TS_DID" ]; then
say "rolled back: the TypeScript relay serves https://$RELAY_HOSTNAME"
return 0
fi
say "rolled back, but https://$RELAY_HOSTNAME describeServer names '$did', not $TS_DID"
exit 1
}
ts_serving() {
[ "$(remote "test -f $(q "$DROPIN") && echo go || echo ts")" = ts ]
}
ts_head() {
remote "env HOME=/nonexistent DENO_DIR=/proc/nonexistent-deno $(q "$DENO") run --quiet --no-config --allow-net=$LISTEN - <<'JS' 2>/dev/null || echo -1
let max = -1;
let last = Date.now();
const t0 = Date.now();
const ws = new WebSocket('ws://$LISTEN/xrpc/com.atproto.sync.subscribeRepos');
ws.onmessage = (e) => {
last = Date.now();
const m = /\"seq\":(\\d+)/.exec(typeof e.data === 'string' ? e.data : '');
if (m && Number(m[1]) > max) max = Number(m[1]);
};
ws.onerror = () => { console.log(max); Deno.exit(0); };
setInterval(() => {
if (Date.now() - last > 3000 || Date.now() - t0 > 90000) { console.log(max); Deno.exit(0); }
}, 250);
JS"
}
mode=deploy
dry_run=0
binary=""
want_sha=""
while [ $# -gt 0 ]; do
case "$1" in
--rollback) mode=rollback ;;
--status) mode=status ;;
--dry-run) dry_run=1 ;;
--binary)
binary="${2:?--binary wants a path}"
shift
;;
--sha256)
want_sha="${2:?--sha256 wants a hex digest}"
shift
;;
-h | --help)
usage
exit 0
;;
*)
usage >&2
exit 2
;;
esac
shift
done
for tool in curl jq sha256sum; do
command -v "$tool" >/dev/null || {
say "$tool is required"
exit 1
}
done
case "$mode" in
status)
report
exit 0
;;
rollback)
rollback
exit 0
;;
esac
if [ -n "$binary" ]; then
[ -f "$binary" ] || {
say "no binary at $binary"
exit 1
}
sha="$(sha256sum "$binary" | cut -d' ' -f1)"
sums="$(dirname "$binary")/SHA256SUMS"
if [ -z "$want_sha" ] && [ -f "$sums" ]; then
want_sha="$(awk -v n="$(basename "$binary")" '$2 == n || $2 == "*" n {print $1}' "$sums" | head -1)"
fi
if [ -n "$want_sha" ]; then
[ "$sha" = "$want_sha" ] || {
say "$binary is $sha, not the expected $want_sha"
exit 1
}
say "binary $binary sha256 $sha (verified)"
else
say "binary $binary sha256 $sha (a dev binary: no --sha256 and no SHA256SUMS entry)"
fi
elif [ "$dry_run" = 1 ]; then
sha=""
say "would build $BIN_NAME of $(git -C "$REPO" rev-parse --short=12 HEAD) for linux/amd64 (CGO_ENABLED=0)"
else
full_commit="$(git -C "$REPO" rev-parse HEAD)"
version="${full_commit:0:12}"
if [ -n "$(git -C "$REPO" status --porcelain --untracked-files=no)" ]; then
version="$version-dirty-$(date -u +%Y%m%dT%H%M%SZ)"
fi
out="$OUT_ROOT/$version"
mkdir -p "$out"
say "building $BIN_NAME $version for linux/amd64 (CGO_ENABLED=0)"
stamp=github.com/johnandersen777/socialweb-computer-kcp/internal/buildstamp
(cd "$REPO" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false \
-ldflags "-s -w -X $stamp.Version=$version -X $stamp.Commit=$full_commit" -o "$out/$BIN_NAME" ./cmd/socialweb-computer)
binary="$out/$BIN_NAME"
sha="$(sha256sum "$binary" | cut -d' ' -f1)"
say "built $binary sha256 $sha ($(du -h "$binary" | cut -f1))"
fi
if [ -n "$sha" ] && [ "$(uname -s)/$(uname -m)" = "Linux/x86_64" ]; then
"$binary" internal atproto-relay --help >/dev/null 2>&1 || {
say "$binary does not run internal atproto-relay --help"
exit 1
}
fi
vdir="$BASE/${sha:0:16}"
bin="$vdir/$BIN_NAME"
say "remote state:"
unit_state | sed 's/^/ /'
serving=go
if ts_serving; then serving=ts; fi
start_seq="${START_SEQ:-}"
seed_note="kept (the Go relay already serves; its own state is the source)"
if [ "$serving" = ts ]; then
hosts_json="$(remote "curl -sS --max-time 20 http://$LISTEN/xrpc/com.atproto.sync.listHosts" || true)"
seed_note="$(printf '%s' "$hosts_json" | jq -c '{hosts: (.hosts | length), states: (.hosts | group_by(.state) | map({(.[0].state): length}) | add)}' 2>/dev/null || echo unreadable)"
if [ -z "$start_seq" ]; then
stored="$(remote "cat $(q "$BASE/start-seq") 2>/dev/null || true")"
head="$(ts_head | tail -1)"
case "$head" in '' | *[!0-9-]*) head=-1 ;; esac
if [ "$head" -ge 0 ]; then
start_seq=$((head + START_SEQ_MARGIN))
else
start_seq="${stored:-0}"
fi
say "the TypeScript relay's newest firehose seq: $head; the Go relay continues at $start_seq"
fi
else
start_seq="${start_seq:-$(remote "cat $(q "$BASE/start-seq") 2>/dev/null || echo 0")}"
fi
case "$start_seq" in '' | *[!0-9]*) start_seq=0 ;; esac
exec_line="$bin internal atproto-relay --state-dir $STATE_DIR --atproto-relay-listen $LISTEN --atproto-relay-public-url https://$RELAY_HOSTNAME --atproto-relay-start-seq $start_seq"
if [ -n "${ALLOWED_HOSTS:-}" ]; then
exec_line="$exec_line --atproto-relay-allowed-hosts $ALLOWED_HOSTS"
fi
dropin_body="[Service]
Type=notify
NotifyAccess=main
EnvironmentFile=
WorkingDirectory=$STATE_DIR
ExecStart=
ExecStart=$exec_line
TimeoutStartSec=30
TimeoutStopSec=20"
if [ "$dry_run" = 1 ]; then
have="$(remote "if [ -n $(q "$sha") ] && [ -f $(q "$bin") ]; then sha256sum $(q "$bin") | cut -d' ' -f1; fi")"
port_busy="$(remote "ss -ltnH 'sport = :${PREFLIGHT_LISTEN##*:}' | wc -l")"
say "plan (dry run, nothing changed):"
say " serving now: $serving"
if [ -n "$sha" ]; then
if [ "$have" = "$sha" ]; then say " upload: skip, $bin is there"; else say " upload: $binary -> $SSH_TARGET:$bin (sha256 $sha)"; fi
else
say " upload: the binary built from this checkout -> $SSH_TARGET:$BASE/<sha256 prefix>/$BIN_NAME"
fi
say " preflight: as $RUN_USER on $PREFLIGHT_LISTEN ($([ "$port_busy" = 0 ] && echo free || echo "BUSY, the deploy would stop here")), a scratch state dir, GET /xrpc/_health"
say " seed: TypeScript listHosts -> $STATE_DIR/seed-hosts.json: $seed_note"
say " firehose start seq: $start_seq"
say " $DROPIN would be:"
printf '%s\n' "$dropin_body" | sed 's/^/ /'
say " then: systemctl daemon-reload; systemctl restart $UNIT; check http://$LISTEN/xrpc/_health and https://$RELAY_HOSTNAME describeServer did $GO_DID"
say " rollback: scripts/deploy-atproto-relay.sh --rollback (rm $DROPIN; restart $UNIT; describeServer did $TS_DID)"
exit 0
fi
have="$(remote "if [ -f $(q "$bin") ]; then sha256sum $(q "$bin") | cut -d' ' -f1; fi")"
if [ "$have" = "$sha" ]; then
say "$SSH_TARGET:$bin is there with sha256 $sha"
else
say "shipping to $SSH_TARGET:$bin"
remote "install -d -m 0755 $(q "$vdir") && cat > $(q "$bin.partial") && chmod 0755 $(q "$bin.partial") && mv $(q "$bin.partial") $(q "$bin")" <"$binary"
have="$(remote "sha256sum $(q "$bin") | cut -d' ' -f1")"
[ "$have" = "$sha" ] || {
say "the shipped binary is $have, not $sha"
exit 1
}
fi
say "preflight: $bin as $RUN_USER on $PREFLIGHT_LISTEN"
preflight="$(remote "set -u
if [ \"\$(ss -ltnH 'sport = :${PREFLIGHT_LISTEN##*:}' | wc -l)\" != 0 ]; then echo 'busy'; exit 0; fi
dir=\$(mktemp -d /var/tmp/atproto-relay-preflight.XXXXXX)
chown $(q "$RUN_USER") \"\$dir\"
timeout 30 runuser -u $(q "$RUN_USER") -- $(q "$bin") internal atproto-relay --state-dir \"\$dir/state\" --atproto-relay-listen $PREFLIGHT_LISTEN --atproto-relay-public-url https://$RELAY_HOSTNAME --atproto-relay-recrawl-window 0 >\"\$dir/log\" 2>&1 &
pid=\$!
ok=no
for i in \$(seq 1 60); do
if body=\$(curl -sf --max-time 2 http://$PREFLIGHT_LISTEN/xrpc/_health); then ok=\"\$body\"; break; fi
sleep 0.5
done
did=\$(curl -sf --max-time 2 http://$PREFLIGHT_LISTEN/xrpc/com.atproto.server.describeServer || true)
kill -TERM \$pid 2>/dev/null || true
wait \$pid 2>/dev/null || true
if [ \"\$ok\" = no ]; then tail -n 5 \"\$dir/log\" | cut -c1-240; fi
rm -rf \"\$dir\"
echo \"health \$ok\"
echo \"describe \$did\"")"
printf '%s\n' "$preflight" | sed 's/^/ /'
case "$preflight" in
busy*)
say "preflight: $PREFLIGHT_LISTEN is in use; set PREFLIGHT_LISTEN to a free loopback port"
exit 1
;;
*"health no"*)
say "preflight failed; the TypeScript relay still serves"
exit 1
;;
*"$GO_DID"*) ;;
*)
say "preflight: describeServer does not name $GO_DID; the TypeScript relay still serves"
exit 1
;;
esac
say "state dir $STATE_DIR (owner $RUN_USER, 0700)"
remote "install -d -m 0700 -o $(q "$RUN_USER") -g $(q "$RUN_USER") $(q "$STATE_DIR") && printf '%s\n' $(q "$start_seq") > $(q "$BASE/start-seq")"
if [ "$serving" = ts ]; then
say "saving the TypeScript relay's listHosts to $STATE_DIR/seed-hosts.json: $seed_note"
remote "set -e
tmp=\$(mktemp $(q "$STATE_DIR/seed-hosts.json.XXXXXX"))
curl -sSf --max-time 20 http://$LISTEN/xrpc/com.atproto.sync.listHosts > \"\$tmp\"
jq -e '.hosts | type == \"array\"' \"\$tmp\" >/dev/null
chown $(q "$RUN_USER"): \"\$tmp\"
chmod 0600 \"\$tmp\"
mv \"\$tmp\" $(q "$STATE_DIR/seed-hosts.json")"
fi
current_dropin="$(remote "cat $(q "$DROPIN") 2>/dev/null || true")"
active_exe="$(remote "pid=\$(systemctl show $UNIT -p MainPID --value); [ \"\$pid\" != 0 ] && readlink /proc/\$pid/exe || true")"
remote "ln -sfn $(q "${sha:0:16}") $(q "$BASE/current.new") && mv -T $(q "$BASE/current.new") $(q "$BASE/current")"
if [ "$current_dropin" = "$dropin_body" ] && [ "$active_exe" = "$bin" ]; then
say "$UNIT already runs $bin with this drop-in; no restart"
else
say "writing $DROPIN and restarting $UNIT"
remote "set -e
install -d -m 0755 $(q "$DROPIN_DIR")
cat > $(q "$DROPIN.partial") <<'UNIT'
$dropin_body
UNIT
mv $(q "$DROPIN.partial") $(q "$DROPIN")"
if ! restart || ! wait_listen; then
say "$UNIT did not come up on the Go relay"
report
if [ "${NO_AUTO_ROLLBACK:-}" = 1 ]; then
say "NO_AUTO_ROLLBACK=1: left as is; scripts/deploy-atproto-relay.sh --rollback restores the TypeScript relay"
exit 1
fi
rollback
exit 1
fi
fi
health="$(fetch /xrpc/_health)"
did="$(serving_did)"
local_health="$(loopback /xrpc/_health)"
exe="$(remote "readlink /proc/\$(systemctl show $UNIT -p MainPID --value)/exe || true")"
report
say "via http://$LISTEN: $local_health"
if [ "$exe" = "$bin" ] && [ "$did" = "$GO_DID" ]; then
case "$health" in
200*)
remote "cd $(q "$BASE") && ls -1t | grep -E '^[0-9a-f]{16}\$' | grep -vx $(q "${sha:0:16}") | tail -n +$KEEP_VERSIONS | xargs -r rm -rf --" || true
say "deployed: $bin ($sha) serves https://$RELAY_HOSTNAME; rollback with scripts/deploy-atproto-relay.sh --rollback"
exit 0
;;
esac
fi
say "the health check failed: exe $exe, _health $health, describeServer did '$did'"
if [ "${NO_AUTO_ROLLBACK:-}" = 1 ]; then
say "NO_AUTO_ROLLBACK=1: left as is; scripts/deploy-atproto-relay.sh --rollback restores the TypeScript relay"
exit 1
fi
rollback
exit 1