~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
183 lines · 8.6 KB Raw History
#!/usr/bin/env bash
set -u

mode="${1:---fast}"
case "$mode" in
  --fast) run=fast ;;
  --local) run=local ;;
  --full) run=full ;;
  *)
    echo "usage: $0 [--fast|--local|--full]" >&2
    echo "  --fast   the checks, no release binary built (one host go build for the CLI argument checker): gofmt, apigen, codegen drift, go vet (host and every cross-OS target, and pkg/embedded for ios/arm64: a compile error fails there), architecture rules, the docs checker (run=gate blocks), the CLI argument checker, the three pins records, policy engine specification tests (run beside the vets: they wait on timers)" >&2
    echo "  --local  --fast plus cargo test of rust/, the embedded asset check (dev verify-assets) and a --dev build that must start (scripts/check-dev-build.sh); lists what it skipped" >&2
    echo "  --full   --local, and a skip is a failure; needs sudo -n, cargo and the staged assets on this host" >&2
    exit 2
    ;;
esac

fail() {
  echo "check.sh: FAILED $1" >&2
  exit 1
}

unformatted="$(gofmt -l api internal cmd hack lexicons)"
if [ -n "$unformatted" ]; then
  echo "$unformatted" >&2
  fail gofmt
fi
echo "check.sh: ok gofmt"

gen="$(mktemp -d)"
genrepo="$gen/repo"
generatedlist="$(go run ./cmd/socialweb-apigen -root . -list)" || { rm -rf "$gen"; fail "apigen"; }
go run ./cmd/socialweb-apigen -root . -out "$genrepo" || { rm -rf "$gen"; fail "apigen"; }
drift=0
compared=0
for rel in $generatedlist; do
  compared=$((compared + 1))
  if ! diff -u "$rel" "$genrepo/$rel" >&2; then
    drift=1
  fi
done
rm -rf "$gen"
if [ "$compared" -eq 0 ]; then
  fail "apigen generated nothing to compare"
fi
if [ "$drift" -ne 0 ]; then
  echo "check.sh: the files above are generated from api/v1alpha1 and differ from what it generates now; run: go generate ./api/..." >&2
  fail "apigen drift"
fi
echo "check.sh: ok apigen ($compared generated files, no drift from api/v1alpha1)"

generated="api internal/atproto/market internal/atproto/infra deploy"
gensnap() { git ls-files -co --exclude-standard -- $generated | LC_ALL=C sort | xargs sha256sum 2>&1; }
genbefore="$(gensnap)"
go generate ./... >/dev/null || fail "go generate ./..."
if [ "$(gensnap)" != "$genbefore" ]; then
  git status --short -- $generated >&2
  git diff -- $generated >&2
  fail "codegen drift: go generate ./... changed $generated; commit its output"
fi
echo "check.sh: ok codegen (go generate ./... leaves $generated unchanged)"

go vet ./... || fail "go vet (it type-checks every package: a compile error fails here; no binary is built)"
echo "check.sh: ok go vet"

scripts/check-architecture.sh || fail "architecture rules (scripts/check-architecture.sh)"
scripts/check-entries.sh || fail "tests/entries.txt and the plan's Entries table differ (scripts/check-entries.sh)"
scripts/check-shards.sh > /dev/null || fail "the gate's shares are not a partition of tests/entries.txt (scripts/check-shards.sh)"
w=.knotless/workflows
for f in release promote; do
  [ "$(grep -m1 '^    paths:' $w/test.yml)" = "$(grep -m1 '^    paths:' $w/$f.yml)" ] || fail "$w/$f.yml's push paths are not test.yml's (a push that runs one of release, test and promote runs all three)"
done
grep -qx 'after: \[release, test\]' $w/promote.yml || fail "$w/promote.yml does not run after release and test"
[ "$(grep -m1 '^    run: ' $w/promote.yml)" = "    run: bash scripts/promote.sh test" ] || fail "$w/promote.yml's first step is not the plain \`bash scripts/promote.sh test\` (it must end the run before release-workload moves channel rc)"
scripts/check-oauth-scope.sh || fail "the Go oauth scope lists and the QR page's TS ones differ (scripts/check-oauth-scope.sh)"
scripts/check-docs.sh || fail "a run=gate block of the docs did not pass (scripts/check-docs.sh)"
scripts/check-cli-args.sh || fail "a verb, a unit template or a cloud-init text does not parse as the CLI's arguments (scripts/check-cli-args.sh)"
scripts/check-pins-records.sh || fail "the com.fedfork.exec.pins/atlibs records of john, alice and socialweb.computer differ (scripts/check-pins-records.sh)"
echo "check.sh: ok architecture rules"

go run ./hack/identlinkvectors internal/identlink/vectors.json || fail "identlink vectors (go run ./hack/identlinkvectors internal/identlink/vectors.json)"
echo "check.sh: ok identlink vectors"

policyspec="./internal/policyengine/... ./internal/market/bidder/ ./internal/market/requester/"
policylog="$(mktemp)"
go test -count=1 -timeout 15m $policyspec >"$policylog" 2>&1 &
policypid=$!
trap 'kill "$policypid" 2>/dev/null; rm -f "$policylog"' EXIT

selfpinned="$(jq -r '.assets | to_entries[] | select([.value.fetch[]?.kind] | length > 0 and all(. == "go" or . == "script"))
  | select((.value.platforms // {}) != {} or ([.value.fetch[] | has("sha256")] | any)) | .key' internal/versions/versions.json)"
[ -z "$selfpinned" ] || fail "versions.json pins the output of an asset built from this repository's own commit (fetch kind go or script): $(echo $selfpinned); remove its platforms and fetch sha256 (the build records them)"
echo "check.sh: ok versions.json pins no output of an asset built from this commit"

for module in $(jq -r '[.assets[].fetch // {} | .[] | select(.kind == "go") | .module // empty] | unique[]' internal/versions/versions.json); do
  (cd "$module" && go vet ./...) || fail "go vet of the asset module $module"
  echo "check.sh: ok go vet of the asset module $module"
done

crossfailed=""
for target in ${CHECK_CROSS_TARGETS:-darwin/arm64 darwin/amd64 windows/amd64 windows/arm64 linux/arm64}; do
  goos="${target%/*}"
  goarch="${target#*/}"
  if ! GOOS="$goos" GOARCH="$goarch" go vet ./...; then
    crossfailed="$crossfailed vet:$target"
    continue
  fi
  echo "check.sh: ok go vet for $target"
done
if ! GOOS=ios GOARCH=arm64 CGO_ENABLED=1 go vet ./pkg/embedded; then
  crossfailed="$crossfailed vet:ios/arm64"
else
  echo "check.sh: ok go vet of pkg/embedded for ios/arm64"
fi
[ -z "${CHECK_CROSS_TARGETS:-}" ] || echo "check.sh: cross-OS builds limited to CHECK_CROSS_TARGETS=$CHECK_CROSS_TARGETS (the CI gate on a main push; a v* tag and a local check.sh build all)"
if [ -n "$crossfailed" ]; then
  echo "check.sh: the tree must vet (so type-check) for every GOOS/GOARCH above; an OS-specific file needs its build tag and a twin for the other OSes (plans 0011, 0012)" >&2
  fail "cross-OS gate:$crossfailed"
fi

if wait "$policypid"; then
  cat "$policylog"
  echo "check.sh: ok policy engine specification tests"
else
  cat "$policylog" >&2
  fail "policy engine specification tests (go test $policyspec)"
fi

rust_check() {
  if ! command -v cargo >/dev/null 2>&1; then
    if [ "$run" = full ]; then
      fail "cargo not found; --full builds and tests rust/socialweb-meshd"
    fi
    echo "check.sh: skipped cargo test (cargo not found, so rust/socialweb-meshd is not tested on this host)"
    return
  fi
  if cargo fmt --version >/dev/null 2>&1; then
    cargo fmt --check --manifest-path rust/Cargo.toml || fail "cargo fmt"
    echo "check.sh: ok cargo fmt"
  else
    echo "check.sh: skipped cargo fmt (rustfmt not installed)"
  fi
  gates=""
  if [ "$run" != fast ]; then
    if sudo -n true >/dev/null 2>&1; then
      gates="SOCIALWEB_MESHD_NETNS=1 SOCIALWEB_MESHD_K3S=1"
    elif [ "$run" = full ]; then
      fail "sudo -n unavailable; --full runs the meshd netns and k3s proofs, which need CAP_NET_ADMIN"
    fi
  fi
  env $gates cargo test --locked --manifest-path rust/Cargo.toml || fail "cargo test"
  if [ -n "$gates" ]; then
    echo "check.sh: ok cargo test (rust/, with the netns and k3s proofs)"
  else
    echo "check.sh: ok cargo test (rust/; the netns and k3s proofs need SOCIALWEB_MESHD_NETNS=1 / SOCIALWEB_MESHD_K3S=1 and sudo -n)"
  fi
}
if [ "$run" != fast ]; then
  rust_check
fi

if [ "$run" != fast ]; then
  hostplat="$(go env GOOS)_$(go env GOARCH)"
  if [ -n "$(ls internal/assets/blobs/"$hostplat"/*.tar.zst 2>/dev/null)" ]; then
    verifydir="$(mktemp -d)"
    if ! CGO_ENABLED=0 go build -tags release -trimpath -o "$verifydir/socialweb-computer" ./cmd/socialweb-computer; then
      rm -rf "$verifydir"
      fail "release build for the embedded asset check ($hostplat)"
    fi
    if ! "$verifydir/socialweb-computer" dev verify-assets >/dev/null; then
      rm -rf "$verifydir"
      fail "embedded assets extract and verify ($hostplat)"
    fi
    rm -rf "$verifydir"
    echo "check.sh: ok embedded assets extract and verify ($hostplat)"
  elif [ "$run" = full ]; then
    fail "no staged blobs for $hostplat; --full extracts every embedded asset: run scripts/stage-assets.sh"
  else
    echo "check.sh: skipped embedded asset extraction (no staged blobs for $hostplat; run scripts/stage-assets.sh)"
  fi
  scripts/check-dev-build.sh || fail "the --dev build must start (version, dev verify-assets): scripts/check-dev-build.sh"
  echo "check.sh: ok the --dev build starts"
fi