~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
140 lines · 5.0 KB Raw History
#!/usr/bin/env bash
# Verify S3 cache upload/restore with local RustFS + hono-plc + atproto-pds:
#   1. run vuln-notif once, publish 5 CVEs, upload cache.db to RustFS
#   2. delete the local cache.db
#   3. run again, restore the cache from RustFS, publish 5 more, no duplicates
set -euo pipefail

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
cd "$REPO_DIR" || exit 1

PDS_PORT=${PDS_PORT:-4800}
PDS_URL=http://127.0.0.1:$PDS_PORT
PLC_PORT=${PLC_PORT:-2587}
PLC_URL=http://127.0.0.1:$PLC_PORT
S3_PORT=${S3_PORT:-9000}
S3_ENDPOINT_URL=http://127.0.0.1:$S3_PORT
BUCKET=${BUCKET:-vuln-notif-test}
S3_PATH=${S3_PATH:-vuln-notif/cve-cache.db}
ACCESS_KEY=${ACCESS_KEY:-rustfsadmin}
SECRET_KEY=${SECRET_KEY:-rustfsadmin}
S3_REGION=${S3_REGION:-us-east-1}
LIMIT=${LIMIT:-5}
LOOKBACK_DAYS=${LOOKBACK_DAYS:-1}
VULN_NOTIF_BIN=${VULN_NOTIF_BIN:-$REPO_DIR/target/release/vuln-notif}
HANDLE=${HANDLE:-cve.test}
EMAIL=${EMAIL:-cve@example.com}
PASSWORD=${PASSWORD:-test-password}
COLLECTION=com.nahdig.cve.nvdV2

TMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/vuln-notif-s3-e2e.XXXXXX")
RUSTFS_STATE=$TMP_ROOT/rustfs-state
PLC_STATE=$TMP_ROOT/plc-state
PDS_STATE=$TMP_ROOT/pds-state
PDS_DATA=$TMP_ROOT/pds-data
ACCOUNT_FILE=$TMP_ROOT/account.json
CACHE_DB=$TMP_ROOT/cache.db
RUSTFS_STARTED=0
PLC_STARTED=0
PDS_STARTED=0

note() { printf '\n== %s\n' "$*"; }
die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
log() { printf '%s\n' "$*" >&2; }

cleanup() {
  local rc=$?
  set +e
  if [ "$PDS_STARTED" = 1 ]; then
    STATE_DIR=$PDS_STATE "$SCRIPT_DIR/local_atproto_pds.sh" stop >/dev/null 2>&1
  fi
  if [ "$PLC_STARTED" = 1 ]; then
    STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" stop >/dev/null 2>&1
  fi
  if [ "$RUSTFS_STARTED" = 1 ]; then
    STATE_DIR=$RUSTFS_STATE "$SCRIPT_DIR/local_rustfs.sh" stop >/dev/null 2>&1
  fi
  if [ "$rc" -ne 0 ] && [ "${KEEP_TMP:-0}" = 1 ]; then
    printf '\ncleanup: kept %s for inspection\n' "$TMP_ROOT" >&2
  else
    rm -rf "$TMP_ROOT"
  fi
  exit "$rc"
}
trap cleanup EXIT

ndjson_sum() {
  local key=$1 file=$2
  jq -s --arg key "$key" '[.[] | .[$key]? // empty] | add // 0' "$file"
}

[ -x "$VULN_NOTIF_BIN" ] || die "vuln-notif binary not found: $VULN_NOTIF_BIN"

note "starting local RustFS on $S3_ENDPOINT_URL"
STATE_DIR=$RUSTFS_STATE PORT=$S3_PORT BUCKET=$BUCKET \
  ACCESS_KEY=$ACCESS_KEY SECRET_KEY=$SECRET_KEY \
  "$SCRIPT_DIR/local_rustfs.sh" start >/dev/null
RUSTFS_STARTED=1

note "starting hono-plc on $PLC_URL"
PORT=$PLC_PORT STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" start >/dev/null
PLC_STARTED=1

note "starting atproto-pds on $PDS_URL"
PDS_PORT=$PDS_PORT PLC_URL=$PLC_URL STATE_DIR=$PDS_STATE \
  PDS_DATA_DIRECTORY=$PDS_DATA "$SCRIPT_DIR/local_atproto_pds.sh" start >/dev/null
PDS_STARTED=1

note "creating account $HANDLE"
ACCOUNT_FILE=$ACCOUNT_FILE PDS_URL=$PDS_URL PLC_URL=$PLC_URL \
  HANDLE=$HANDLE EMAIL=$EMAIL PASSWORD=$PASSWORD \
  "$SCRIPT_DIR/create_local_account.sh" >"$TMP_ROOT/account.env" ||
  die "create_local_account.sh failed"
DID=$(jq -er '.did' "$ACCOUNT_FILE") || die "account file missing did"
ACCESS_JWT=$(jq -er '.session.accessJwt' "$ACCOUNT_FILE") || die "account file missing session accessJwt"

ATP_APP_PASSWORD=$(jq -er '.app_password' "$ACCOUNT_FILE") || die "account file missing app_password"
export ATP_PDS_URL="$PDS_URL"
export ATP_HANDLE="$HANDLE"
export ATP_APP_PASSWORD
export S3_BUCKET="$BUCKET"
export S3_PATH
export S3_ENDPOINT_URL
export S3_ACCESS_KEY_ID="$ACCESS_KEY"
export S3_SECRET_ACCESS_KEY="$SECRET_KEY"
export S3_REGION

note "run 1: fetch + publish $LIMIT, upload cache to S3"
"$VULN_NOTIF_BIN" --once --limit "$LIMIT" --initial-lookback-days "$LOOKBACK_DAYS" \
  --cache-db "$CACHE_DB" >"$TMP_ROOT/run1.stdout" 2>"$TMP_ROOT/run1.stderr" ||
  die "run 1 failed (see $TMP_ROOT/run1.stderr)"
cat "$TMP_ROOT/run1.stdout"

note "deleting local cache.db and running again"
rm -f "$CACHE_DB" "$CACHE_DB-wal" "$CACHE_DB-shm"
"$VULN_NOTIF_BIN" --once --limit "$LIMIT" --cache-db "$CACHE_DB" \
  >"$TMP_ROOT/run2.stdout" 2>"$TMP_ROOT/run2.stderr" ||
  die "run 2 failed (see $TMP_ROOT/run2.stderr)"
cat "$TMP_ROOT/run2.stdout"

UPLOADED=$(ndjson_sum s3_upload_bytes "$TMP_ROOT/run1.stdout")
DOWNLOADED=$(ndjson_sum s3_download_bytes "$TMP_ROOT/run2.stdout")
LIST_JSON=$(curl -sS --max-time 15 -G \
  -H "Authorization: Bearer $ACCESS_JWT" \
  --data-urlencode "repo=$DID" \
  --data-urlencode "collection=$COLLECTION" \
  --data-urlencode "limit=100" \
  "$PDS_URL/xrpc/com.atproto.repo.listRecords") || die "listRecords request failed"
PDS_RECORDS=$(jq -er '.records | length' <<<"$LIST_JSON") ||
  die "listRecords failed: $LIST_JSON"

printf '\nS3 upload bytes:   %s\n' "$UPLOADED"
printf 'S3 download bytes: %s\n' "$DOWNLOADED"
printf 'PDS records:       %s (expected %s)\n' "$PDS_RECORDS" "$((LIMIT * 2))"

[ "$UPLOADED" -gt 0 ] || die "expected S3 upload bytes > 0, got $UPLOADED"
[ "$DOWNLOADED" -gt 0 ] || die "expected S3 download bytes > 0, got $DOWNLOADED"
[ "$PDS_RECORDS" -eq "$((LIMIT * 2))" ] ||
  die "expected $((LIMIT * 2)) $COLLECTION records, got $PDS_RECORDS"
printf '\nRESULT: PASS\n'