~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
119 lines · 8.0 KB Raw History
#!/usr/bin/env bash
# ci-prod-target.sh [--dry-run] [<out>]: the prod target of promote's gate (plan 0040 step 11), written on the release VM
# from tests/targets/prod.json and the environment; scripts/promote.sh runs it, as root, after `scripts/ci-test.sh host`
# (the page) and `binary` (the requester spec). No secret is in the repo or in <out> (default $CI_HOME/target-prod.json):
# the target names files, each written 0600 and r2's from a sealed secret of main (.knotless/workflows/promote.yml).
#   requester      $CI_HOME/requester.spec (sha:<the rc's commit>)
#   accounts       each account with an oauthAccount P (alice: ALICE, bob: BOB): creds $CI_HOME/creds/<account>,
#                  exporting P_ATPROTO_HANDLE (prod.json's handle) and P_ATPROTO_PASSWORD from PROD_<P>_ATPROTO_PASSWORD (the
#                  main password: one line, written as is: the readers strip only a quote pair around the whole value)
#   bidders        discovered, replacing prod.json's hand-kept list (its kcp kubeconfigs and ssh hosts are for the gate
#                  by hand on hat-8): every bidder DID of deploy/release/rc-operators.json, named after its handle's first
#                  label, with `ops` its market endpoint (its market.offering/self record on its own PDS), the one
#                  that serves its read-only ops routes (GET /xrpc/com.fedfork.exec.ops.*, docs/SYSTEM.md): `ops.url`
#                  its https endpoint when the offering has one (staging), else `ops.iroh` its iroh endpoint (prod
#                  bidders are behind NAT: iroh and webrtc only). Neither: exit 1. No kcp, no ssh
#   opsSecretFile  $CI_HOME/ops-read-secret from PROD_OPS_READ_SECRET: the bearer of those routes
#   gateway.ops    the ssh gateway's web origin (its getRequesterLog route), in place of gateway.log over ssh: r2 holds
#                  no droplet key
#   agentConfig    $CI_HOME/agent-config.toml from ATIRC_AGENT_CONFIG
#   pageRepo       $CI_HOME/page (release's page payload, scripts/ci-test.sh page)
#   tools          $CI_HOME/page/tools: the pinned node, chrome and deno of tests/compositions/staging/tools.sh
# --dry-run reads no secret and writes none of those files: it writes <out> with the same paths and, when the page is
# there, fetches the tools. Run again it rewrites every file; a tool already unpacked is kept.
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.."
dry=0
if [ "${1:-}" = --dry-run ]; then dry=1; shift; fi
home=${CI_HOME:-/home/r2}
out=${1:-$home/target-prod.json}
src=tests/targets/prod.json
ops=deploy/release/rc-operators.json
own=(); [ "$(id -u)" = 0 ] && own=(-o r2)
need() { [ -n "${!1:-}" ] || { echo "ci-prod-target: $1 is not set (a sealed secret of main: .knotless/workflows/promote.yml)" >&2; exit 1; }; }
# put <file> <secret name> <text>: the file 0600 in a 0700 directory, r2's when root; dry: only says so
put() {
  if [ "$dry" = 1 ]; then echo "==> would write $1 (0600) from $2"; return; fi
  install -d -m 0700 "${own[@]}" "$(dirname "$1")"
  (umask 077 && printf '%s\n' "$3" > "$1.part")
  [ ${#own[@]} = 0 ] || chown r2 "$1.part"
  mv "$1.part" "$1"
}
install -d "${own[@]}" "$home"
t=$(cat "$src")

if [ -f "$home/requester.spec" ]; then
  t=$(jq --arg r "$(cat "$home/requester.spec")" '.requester = $r' <<<"$t")
elif [ "$dry" = 1 ]; then
  echo "==> no $home/requester.spec (scripts/ci-test.sh binary writes it): requester stays $(jq -r .requester <<<"$t")"
else
  echo "ci-prod-target: no $home/requester.spec (scripts/ci-test.sh binary writes it)" >&2
  exit 1
fi

for a in $(jq -r '.accounts | to_entries[] | select(.value.oauthAccount) | .key' "$src"); do
  p=$(jq -r --arg a "$a" '.accounts[$a].oauthAccount' "$src")
  v=PROD_${p}_ATPROTO_PASSWORD
  [ "$dry" = 1 ] || need "$v"
  put "$home/creds/$a" "$v" "export ${p}_ATPROTO_HANDLE=$(jq -r --arg a "$a" '.accounts[$a].handle' "$src")
export ${p}_ATPROTO_PASSWORD=${!v:-}"
  t=$(jq --arg a "$a" --arg f "$home/creds/$a" '.accounts[$a].creds = $f' <<<"$t")
done

# the bidders are discovered, never listed by hand: every bidder DID of the rc operators, its PDS from its DID document, its
# market endpoint from its market.offering record (the mux that serves its read-only ops routes): https, else iroh
[ "$dry" = 1 ] || need PROD_OPS_READ_SECRET
put "$home/ops-read-secret" PROD_OPS_READ_SECRET "${PROD_OPS_READ_SECRET:-}"
plc=$(jq -r '.plc' "$src"); pds=$(jq -r '.pds' "$src")
get() { curl -fsS -m 20 --retry 8 --retry-delay 5 --retry-all-errors "$@"; }
bidders='[]'
for handle in $(jq -r '.operators[].handle' "$ops"); do
  operator=$(get "$pds/xrpc/com.atproto.identity.resolveHandle?handle=$handle" | jq -r '.did // ""') || operator=""
  for did in $(jq -r --arg h "$handle" '.operators[] | select(.handle == $h) | .bidders[]' "$ops"); do
    doc=$(get "$plc/$did") || doc='{}'
    bpds=$(jq -r '[.service[]? | select(.id == "#atproto_pds") | .serviceEndpoint][0] // ""' <<<"$doc")
    label=$(jq -r '(.alsoKnownAs[0] // "") | sub("^at://"; "") | split(".")[0] | ascii_downcase | gsub("[^a-z0-9-]"; "")' <<<"$doc")
    name=${label:-bidder-${did: -8}}
    [ "$(jq --arg n "$name" '[.[] | select(.name == $n)] | length' <<<"$bidders")" = 0 ] || name=$name-${did: -8}
    offering='{}'
    [ -z "$bpds" ] || offering=$(get "$bpds/xrpc/com.atproto.repo.getRecord?repo=$did&collection=com.publicdomainrelay.temp.market.offering&rkey=self") || offering='{}'
    url=$(jq -r '[.value.endpoints[]? | select(.kind == "https") | .address][0] // ""' <<<"$offering")
    case $url in https://*) ;; *) [ -z "$url" ] || echo "ci-prod-target: $did: its https endpoint $url is not https" >&2; url="" ;; esac
    iroh=$(jq -r '[.value.endpoints[]? | select(.kind == "iroh") | .address][0] // "" | sub("^iroh://"; "")' <<<"$offering")
    [[ $iroh =~ ^endpoint[a-z2-7]+$ ]] || { [ -z "$iroh" ] || echo "ci-prod-target: $did: its iroh endpoint is not an endpoint ticket" >&2; iroh=""; }
    if [ -n "$url" ]; then via=$(jq -n --arg u "${url%/}" '{url: $u}'); elif [ -n "$iroh" ]; then via=$(jq -n --arg i "$iroh" '{iroh: $i}'); else via=""; fi
    if [ -z "$operator" ] || [ -z "$via" ]; then
      echo "ci-prod-target: $did of $handle: operator ${operator:-unresolved}, PDS ${bpds:-unresolved}, no https or iroh market endpoint in its offering" >&2
      [ "$dry" = 1 ] && continue
      exit 1
    fi
    bidders=$(jq --arg n "$name" --arg d "$did" --arg o "$operator" --argjson v "$via" '. + [{name: $n, did: $d, operator: $o, ops: $v}]' <<<"$bidders")
  done
done
web=$(jq -r '.gateway.sshWeb // ("https://ssh." + .gateway.ssh)' "$src")
case $web in https://*) ;; *) echo "ci-prod-target: the ssh gateway's web origin $web is not https: the ops read secret goes to https only" >&2; exit 1 ;; esac
t=$(jq --argjson b "$bidders" --arg f "$home/ops-read-secret" --arg w "${web%/}" \
  '.bidders = $b | .opsSecretFile = $f | del(.gateway.log) | .gateway.ops = {url: $w}' <<<"$t")

[ "$dry" = 1 ] || need ATIRC_AGENT_CONFIG
put "$home/agent-config.toml" ATIRC_AGENT_CONFIG "${ATIRC_AGENT_CONFIG:-}"
# every secret is in its file now: none stays in the environment of what runs next (the tools' downloads)
for v in $(compgen -v | grep -E '^(PROD_.*_ATPROTO_PASSWORD|PROD_OPS_READ_SECRET|PROD_DEPLOY_SSH_KEY|ATIRC_AGENT_CONFIG)$' || true); do unset "$v"; done

page=$home/page
if [ -f "$page/scripts/page-round.mjs" ]; then
  tools=$(SWC_DIR=$home SWC_REPO=$PWD bash tests/compositions/staging/tools.sh node chrome deno | tail -1)
  [ ${#own[@]} = 0 ] || chown -R r2 "$page"
  echo "==> page $page at $(cat "$page/COMMIT"); tools $tools"
elif [ "$dry" = 1 ]; then
  tools=$page/tools
  echo "==> no page at $page (scripts/ci-test.sh host fetches release's): would fetch the tools to $tools"
else
  echo "ci-prod-target: no page at $page (scripts/ci-test.sh host fetches release's)" >&2
  exit 1
fi

jq --arg c "$home/agent-config.toml" --arg p "$page" --arg o "$tools" '.agentConfig = $c | .pageRepo = $p | .tools = $o' <<<"$t" > "$out.part"
[ ${#own[@]} = 0 ] || chown r2 "$out.part"
mv "$out.part" "$out"
echo "==> prod target $out: $(jq -c '{requester, pageRepo, tools, agentConfig, creds: (.accounts | map_values(.creds)), opsSecretFile, gateway: .gateway.ops, bidders: [.bidders[] | {name, did, ops}]}' "$out")"