0
0
#!/usr/bin/env bash
# On the atproto VM, as stg, once the bidders joined (bidders.json) and stg's key reaches the gateway VM: the network's
# gateway (plan 0040 step 8d), its own kcp on the gateway VM (`serve gateway kcp --gateway-profile market-network`,
# the network value in its Gateway spec): a gateway child of the network would refuse `run requester --with-gateway`.
# gateway.sh up <url> [stg@<gateway VM>] start it (a live one is kept), wait for Ready, write gateway.json
# gateway.sh ssh-up <web url> <ssh name>:<port> [stg@<gateway VM>]
# the ssh gateway there (step 8e), in prod's shape: standalone `run gateway ssh
# --ssh-gateway-spec-file` (the network value, bidderDids, public-origin the web URL,
# ssh.publicListen, ssh.webListen, ssh.operatorDid the operator account (accounts.sh: bob1), which
# trust.sh vouch has vouch the drivers); adds gateway.json's ssh, sshPort,
# sshWeb and log
# gateway.sh down [stg@<gateway VM>] stop both and their forwards: the ssh gateway by `dev down`, the gateway by
# its supervisor's own stop (SIGTERM to the devsup of state/sup/devsup.json: it
# stops its units in order: the core, the vault, kcp, kine, postgres); fails
# when a process of the gateway's state dir is left
# Names and TLS: the network's DNS answers every name of the zone with the network's address and the run's leaf
# (guest-tls/<domain>.crt, *.<domain>) is on this VM, so <url> is a name of the zone on a port of the network's address
# beside the front's 443, as atirc.sh: socat terminates TLS here and forwards to the gateway VM's web app front
# (spec.webAppFront, plain HTTP on the overlay: caddy's place on prod). A tenant kubeconfig names the client certificate
# front (https://127.0.0.1:<port>, the gateway's own certificate): the same port of this VM's loopback forwards to it,
# TLS untouched (the client certificate is the login).
# The gateway's requester is prod's (docs/acceptance/0001-auth-modes.md "Association"): localRepo, its own Pds and did:plc
# (published through the network's ingress, registered at the network's PLC), the agent of an operator account as prod's is
# alice's: spec.operator is the gateway account of accounts.json (accounts.sh: alice5), the gateway adds its agent-of entry
# to its DID, and once the DID exists `trust.sh associate <DID>` writes the operator's grant and endorsement (prod:
# scripts/associate-bidder.ts). The bidders vouch the operator (trust.sh accounts) and trust.sh vouch has it vouch them.
# The ssh gateway's web origin is forwarded as the gateway's URL is (TLS here, plain HTTP to ssh.webListen on the overlay) and
# its sshd as plain TCP from <port> of the network's address to ssh.publicListen.
# The audience is the did:web of the URL's origin, its port percent-encoded (did:web:<name>%3A<port>): what a module's XRPC
# call to the URL mints its service-auth token for; the name alone is refused as Unauthorized.
# gateway.json: the target's gateway.{url, aud, requesterDid, kcp} and {ssh, sshPort, sshWeb, log} (target.sh).
set -euo pipefail
. "$(dirname "$0")/accounts.sh"
S=${SWC:-/home/stg/stg/socialweb-computer-linux-amd64} ST=${SWC_STATE:-/home/stg/stg/s} I=${SWC_INSTANCE:-stg} C=${SWC_CREDS:-/home/stg}
D=${SWC_DIR:-/home/stg/stg}
L=$D/gateway
GD=${GATEWAY_DIR:-/home/stg/stg/gw} BIN=${GATEWAY_SWC:-/home/stg/stg/socialweb-computer-linux-amd64}
WEB=${GATEWAY_WEB_PORT:-8788} CERT=${GATEWAY_CERT_PORT:-16444} READY_S=${GATEWAY_READY_S:-1200}
SI=${GATEWAY_SSH_INSTANCE:-sg} SWEB=${GATEWAY_SSH_WEB_PORT:-8789} SSHD=${GATEWAY_SSH_PORT:-2222}
usage() { echo "usage: gateway.sh up <url> [stg@<gateway VM>] | ssh-up <web url> <ssh name>:<port> [stg@<gateway VM>] | down [stg@<gateway VM>]" >&2; exit 2; }
# the gateway VM's overlay address is the one after the bidder VMs' (bidders.json; two bidders: 10.44.0.4)
GW=stg@10.44.0.$(( $(jq 'length' "$D/bidders.json" 2>/dev/null || echo 2) + 2 ))
case "${1:-}" in
up) url=${2:-}; [ -n "$url" ] || usage; G=${3:-$GW} ;;
ssh-up) url=${2:-} sshhp=${3:-}; [ -n "$url" ] && [ -n "$sshhp" ] || usage; G=${4:-$GW} ;;
down) G=${2:-$GW} ;;
*) usage ;;
esac
GA=${G#*@}
on_gw() { ssh -o BatchMode=yes -o LogLevel=ERROR "$G" "$@"; }
alive() { [ -f "$L/$1.pid" ] && kill -0 "$(cat "$L/$1.pid")" 2>/dev/null; }
stop_local() {
for p in "$@"; do
[ -f "$L/$p.pid" ] && kill "$(cat "$L/$p.pid")" 2>/dev/null || true
rm -f "$L/$p.pid"
done
}
if [ "$1" = down ]; then
stop_local tls kube sshweb sshd
rc=0
on_gw bash -s <<EOF || rc=$?
export XDG_RUNTIME_DIR=\${XDG_RUNTIME_DIR:-/run/user/\$(id -u)}
[ -d $GD/sg ] && $BIN dev down --instance $SI --state-dir $GD/sg > $GD/ssh-down.log 2>&1
for p in serve cert; do [ -f $GD/\$p.pid ] && kill \$(cat $GD/\$p.pid) 2>/dev/null; rm -f $GD/\$p.pid; done
sup=\$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["pid"])' $GD/state/sup/devsup.json 2>/dev/null || true)
if [ -n "\$sup" ]; then
kill \$sup 2>/dev/null || true
for _ in \$(seq 120); do kill -0 \$sup 2>/dev/null || break; sleep 1; done
fi
left=\$(ps -eo pid=,args= | grep -F -e "$GD/state/" -e "$GD/sg/" | grep -v grep || true)
[ -z "\$left" ] || { echo "gateway.sh: left running on $GA after down:" >&2; echo "\$left" | cut -c1-200 >&2; exit 1; }
EOF
rm -f "$D/gateway.json"
exit $rc
fi
url=${url%/}
scheme=${url%%://*} hp=${url#*://}
host=${hp%%:*} port=${hp##*:}
case "$scheme:$port" in
https:*[!0-9]*|https:) echo "gateway.sh: $url: an https URL with a port" >&2; exit 2 ;;
https:*) ;;
*) echo "gateway.sh: $url: an https URL with a port" >&2; exit 2 ;;
esac
[ "$hp" != "$host" ] || { echo "gateway.sh: $url: an https URL with a port" >&2; exit 2; }
st=$($S dev status --instance "$I" --state-dir "$ST" 2>/dev/null)
pds=$(jq -r '.network.status.bidderPdsUrl // empty' <<<"$st")
front=$(jq -r '.network.status.network.xrpcFront // empty' <<<"$st")
addr=${front%:*}
[ -n "$pds" ] && [ -n "$addr" ] || { echo "gateway.sh: the network's status names no bidder PDS or front (bidderPds=${pds:-<none>} xrpcFront=${front:-<none>})" >&2; exit 1; }
tls=$ST/dev/$I/network/guest-tls leaf=${host#*.}
[ -f "$tls/$leaf.crt" ] && [ -f "$tls/$leaf.key" ] || { echo "gateway.sh: no leaf $tls/$leaf.crt for $host" >&2; exit 1; }
[ -f "$D/net.json" ] && [ -f "$D/bidders.json" ] && [ -f "$D/accounts.json" ] || { echo "gateway.sh: net.json, bidders.json and accounts.json are in $D once the bidders joined" >&2; exit 1; }
if [ "$1" = ssh-up ]; then
shost=${sshhp%%:*} sport=${sshhp##*:}
case "$sport" in
''|*[!0-9]*) echo "gateway.sh: $sshhp: an ssh name with a port" >&2; exit 2 ;;
esac
[ "$sshhp" != "$shost" ] || { echo "gateway.sh: $sshhp: an ssh name with a port" >&2; exit 2; }
op=$(jq -r --arg a "$STG_OPERATOR" '.[$a].did // empty' "$D/accounts.json")
[ -n "$op" ] || { echo "gateway.sh: accounts.json names no account $STG_OPERATOR (trust.sh accounts)" >&2; exit 1; }
install -d -m 0700 "$L"
spec=$(jq -n --slurpfile net "$D/net.json" --slurpfile b "$D/bidders.json" --arg origin "$url" --arg pub "$GA:$SSHD" --arg web "$GA:$SWEB" --arg op "$op" '{
network: $net[0].network, bidderDids: [$b[0][].did], options: [{name: "public-origin", value: $origin}],
ssh: {publicListen: $pub, webListen: $web, operatorDid: $op}}')
[ "$(jq -r '.network.plcDirectoryUrl // empty' <<<"$spec")" ] || { echo "gateway.sh: net.json names no network value" >&2; exit 1; }
on_gw "umask 077; mkdir -p $GD && cat > $GD/ssh-spec.json.part && mv $GD/ssh-spec.json.part $GD/ssh-spec.json" <<<"$spec"
started=$(date +%s)
if ! on_gw bash -s <<EOF
set -euo pipefail
cd $GD
export XDG_RUNTIME_DIR=\${XDG_RUNTIME_DIR:-/run/user/\$(id -u)}
$BIN run gateway ssh --instance $SI --state-dir $GD/sg --host-backend proc --ssh-gateway-spec-file $GD/ssh-spec.json --network-timeout ${READY_S}s < /dev/null > ssh.log 2>&1
EOF
then
echo "gateway.sh: run gateway ssh on $GA failed after $(($(date +%s) - started)) s; ssh.log:" >&2
on_gw "tail -n 30 $GD/ssh.log" >&2 || true
exit 1
fi
stop_local sshweb sshd
setsid nohup socat "OPENSSL-LISTEN:$port,bind=$addr,reuseaddr,fork,cert=$tls/$leaf.crt,key=$tls/$leaf.key,verify=0" \
"TCP:$GA:$SWEB" < /dev/null > "$L/sshweb.log" 2>&1 &
echo $! > "$L/sshweb.pid"
setsid nohup socat "TCP-LISTEN:$sport,bind=$addr,reuseaddr,fork" "TCP:$GA:$SSHD" < /dev/null > "$L/sshd.log" 2>&1 &
echo $! > "$L/sshd.pid"
ok=
for _ in $(seq 60); do
if curl -fsS -o /dev/null --max-time 5 "$url/" 2>/dev/null && ssh-keyscan -T 5 -p "$sport" "$shost" 2>/dev/null | grep -q .; then
ok=1
break
fi
alive sshweb && alive sshd || break
sleep 2
done
if [ -z "$ok" ]; then
echo "gateway.sh: $url/ or the ssh gateway on $shost:$sport did not answer; sshweb.log, sshd.log:" >&2
tail -n 5 "$L/sshweb.log" "$L/sshd.log" >&2 || true
exit 1
fi
have='{}'
[ -f "$D/gateway.json" ] && have=$(cat "$D/gateway.json")
(umask 077 && jq -n --argjson have "$have" --arg ssh "$shost" --argjson port "$sport" --arg web "$url" --arg g "$G" --arg log "$GD/sg/dev/$SI/logs/core.log" \
'$have + {ssh: $ssh, sshPort: $port, sshWeb: $web, log: {host: $g, path: $log}}' > "$D/gateway.json.part" && mv "$D/gateway.json.part" "$D/gateway.json")
echo "gateway.sh: ssh $shost:$sport and $url up (gateway VM $GA, operator $op, ready after $(($(date +%s) - started)) s)"
exit 0
fi
opdid=$(jq -r --arg a "$STG_GATEWAY" '.[$a].did // empty' "$D/accounts.json")
[ -n "$opdid" ] && [ -f "$(jq -r --arg a "$STG_GATEWAY" '.[$a].creds // empty' "$D/accounts.json")" ] || { echo "gateway.sh: accounts.json names no account $STG_GATEWAY (trust.sh accounts)" >&2; exit 1; }
aud=did:web:$host%3A$port
install -d -m 0700 "$L"
spec=$(jq -n --slurpfile net "$D/net.json" --slurpfile b "$D/bidders.json" --arg op "$opdid" \
--arg web "$GA:$WEB" --arg aud "$aud" --arg origin "$url" '{
network: $net[0].network, bidderDids: [$b[0][].did], webAppFront: $web, operator: $op,
options: [{name: "gateway-audience", value: $aud}, {name: "public-origin", value: $origin}]}')
[ "$(jq -r '.network.plcDirectoryUrl // empty' <<<"$spec")" ] && [ "$(jq '.bidderDids | length' <<<"$spec")" -gt 0 ] || { echo "gateway.sh: net.json names no network value or bidders.json no bidder" >&2; exit 1; }
on_gw "umask 077; mkdir -p $GD && cat > $GD/spec.json.part && mv $GD/spec.json.part $GD/spec.json" <<<"$spec"
on_gw bash -s <<EOF
set -euo pipefail
cd $GD
export XDG_RUNTIME_DIR=\${XDG_RUNTIME_DIR:-/run/user/\$(id -u)}
live() { [ -f \$1.pid ] && kill -0 \$(cat \$1.pid) 2>/dev/null; }
if live serve; then
echo "gateway.sh: serve gateway kcp already runs on $GA"
else
rm -f state/gateway.json
(setsid nohup $BIN serve gateway kcp --host-backend proc --state-dir $GD/state --gateway-profile market-network \
--gateway-spec-file $GD/spec.json --kcp-front-address 127.0.0.1:0 --cert-front-address 127.0.0.1:$CERT < /dev/null > serve.log 2>&1 &
echo \$! > serve.pid)
fi
if ! live cert; then
setsid nohup socat TCP-LISTEN:$CERT,bind=$GA,reuseaddr,fork TCP:127.0.0.1:$CERT < /dev/null > cert.log 2>&1 &
echo \$! > cert.pid
fi
EOF
poll() {
on_gw bash -s <<EOF
cd $GD
if ! { [ -f serve.pid ] && kill -0 \$(cat serve.pid) 2>/dev/null; }; then echo exited; exit 0; fi
[ -f state/gateway.json ] || { echo none; exit 0; }
K=\$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("adminKubeconfig",""))' state/gateway.json)
[ -n "\$K" ] && [ -f "\$K" ] || { echo none; exit 0; }
T=\$(awk '/ token:/{print \$2; exit}' \$K); U=\$(awk '/ server:/{print \$2; exit}' \$K)
curl -fsSk --max-time 10 -H "Authorization: Bearer \$T" \${U%%/clusters/*}/clusters/root/apis/socialweb.computer/v1alpha1/gateways/gateway 2>/dev/null \
| python3 -c 'import json,sys; s=json.load(sys.stdin).get("status",{}); print(s.get("phase") or "-", s.get("did") or "-", s.get("message",""))' 2>/dev/null || echo none
EOF
}
started=$(date +%s) last= associated=
while :; do
last=$(poll || echo unreachable)
read -r phase got _ <<<"$last"
[ "$phase" = Ready ] && break
if [ -z "$associated" ] && [ "${got#did:}" != "$got" ]; then
"$(dirname "$0")/trust.sh" associate "$got"
associated=1
fi
if [ "$phase" = exited ] || [ $(($(date +%s) - started)) -ge "$READY_S" ]; then
echo "gateway.sh: the gateway on $GA is not Ready after $(($(date +%s) - started)) s: $last; serve.log:" >&2
on_gw "tail -n 30 $GD/serve.log" >&2 || true
exit 1
fi
sleep 5
done
[ "${got#did:}" != "$got" ] || { echo "gateway.sh: the Ready gateway reports no requester DID ($last)" >&2; exit 1; }
did=$got
kc=$D/kc-gateway.kubeconfig
(umask 077 && on_gw "cat \$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[\"adminKubeconfig\"])' $GD/state/gateway.json)" > "$kc.part" && mv "$kc.part" "$kc")
grep -q ' server:' "$kc" || { echo "gateway.sh: no admin kubeconfig came from $GA" >&2; exit 1; }
stop_local tls kube
setsid nohup socat "OPENSSL-LISTEN:$port,bind=$addr,reuseaddr,fork,cert=$tls/$leaf.crt,key=$tls/$leaf.key,verify=0" \
"TCP:$GA:$WEB" < /dev/null > "$L/tls.log" 2>&1 &
echo $! > "$L/tls.pid"
setsid nohup socat "TCP-LISTEN:$CERT,bind=127.0.0.1,reuseaddr,fork" "TCP:$GA:$CERT" < /dev/null > "$L/kube.log" 2>&1 &
echo $! > "$L/kube.pid"
ok=
for _ in $(seq 60); do
if curl -fsS -o /dev/null --max-time 5 "$url/" 2>/dev/null && curl -sSk -o /dev/null --max-time 5 "https://127.0.0.1:$CERT/" 2>/dev/null; then
ok=1
break
fi
alive tls && alive kube || break
sleep 2
done
if [ -z "$ok" ]; then
echo "gateway.sh: $url/ or the client certificate front on 127.0.0.1:$CERT did not answer; tls.log, kube.log:" >&2
tail -n 5 "$L/tls.log" "$L/kube.log" >&2 || true
exit 1
fi
have='{}'
[ -f "$D/gateway.json" ] && have=$(cat "$D/gateway.json")
(umask 077 && jq -n --argjson have "$have" --arg url "$url" --arg aud "$aud" --arg did "$did" --arg kc "$kc" --arg g "$G" \
'$have + {url: $url, aud: $aud, requesterDid: $did, kcp: {kubeconfig: $kc, via: {kind: "ssh", host: $g}}}' > "$D/gateway.json.part" && mv "$D/gateway.json.part" "$D/gateway.json")
echo "gateway.sh: $url up (gateway VM $GA, aud $aud, requester $did, operator $opdid, ready after $(($(date +%s) - started)) s)"