#!/usr/bin/env bash # Verify S3 cache upload/restore with local RustFS + hono-plc + atproto-pds: # 1. run vuln-notif once, publish 5 CVEs, upload cache.db to RustFS # 2. delete the local cache.db # 3. run again, restore the cache from RustFS, publish 5 more, no duplicates set -euo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO_DIR=$(cd "$SCRIPT_DIR/.." && pwd) cd "$REPO_DIR" || exit 1 PDS_PORT=${PDS_PORT:-4800} PDS_URL=http://127.0.0.1:$PDS_PORT PLC_PORT=${PLC_PORT:-2587} PLC_URL=http://127.0.0.1:$PLC_PORT S3_PORT=${S3_PORT:-9000} S3_ENDPOINT_URL=http://127.0.0.1:$S3_PORT BUCKET=${BUCKET:-vuln-notif-test} S3_PATH=${S3_PATH:-vuln-notif/cve-cache.db} ACCESS_KEY=${ACCESS_KEY:-rustfsadmin} SECRET_KEY=${SECRET_KEY:-rustfsadmin} S3_REGION=${S3_REGION:-us-east-1} LIMIT=${LIMIT:-5} LOOKBACK_DAYS=${LOOKBACK_DAYS:-1} VULN_NOTIF_BIN=${VULN_NOTIF_BIN:-$REPO_DIR/target/release/vuln-notif} HANDLE=${HANDLE:-cve.test} EMAIL=${EMAIL:-cve@example.com} PASSWORD=${PASSWORD:-test-password} COLLECTION=com.nahdig.cve.nvdV2 TMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/vuln-notif-s3-e2e.XXXXXX") RUSTFS_STATE=$TMP_ROOT/rustfs-state PLC_STATE=$TMP_ROOT/plc-state PDS_STATE=$TMP_ROOT/pds-state PDS_DATA=$TMP_ROOT/pds-data ACCOUNT_FILE=$TMP_ROOT/account.json CACHE_DB=$TMP_ROOT/cache.db RUSTFS_STARTED=0 PLC_STARTED=0 PDS_STARTED=0 note() { printf '\n== %s\n' "$*"; } die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } log() { printf '%s\n' "$*" >&2; } cleanup() { local rc=$? set +e if [ "$PDS_STARTED" = 1 ]; then STATE_DIR=$PDS_STATE "$SCRIPT_DIR/local_atproto_pds.sh" stop >/dev/null 2>&1 fi if [ "$PLC_STARTED" = 1 ]; then STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" stop >/dev/null 2>&1 fi if [ "$RUSTFS_STARTED" = 1 ]; then STATE_DIR=$RUSTFS_STATE "$SCRIPT_DIR/local_rustfs.sh" stop >/dev/null 2>&1 fi if [ "$rc" -ne 0 ] && [ "${KEEP_TMP:-0}" = 1 ]; then printf '\ncleanup: kept %s for inspection\n' "$TMP_ROOT" >&2 else rm -rf "$TMP_ROOT" fi exit "$rc" } trap cleanup EXIT ndjson_sum() { local key=$1 file=$2 jq -s --arg key "$key" '[.[] | .[$key]? // empty] | add // 0' "$file" } [ -x "$VULN_NOTIF_BIN" ] || die "vuln-notif binary not found: $VULN_NOTIF_BIN" note "starting local RustFS on $S3_ENDPOINT_URL" STATE_DIR=$RUSTFS_STATE PORT=$S3_PORT BUCKET=$BUCKET \ ACCESS_KEY=$ACCESS_KEY SECRET_KEY=$SECRET_KEY \ "$SCRIPT_DIR/local_rustfs.sh" start >/dev/null RUSTFS_STARTED=1 note "starting hono-plc on $PLC_URL" PORT=$PLC_PORT STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" start >/dev/null PLC_STARTED=1 note "starting atproto-pds on $PDS_URL" PDS_PORT=$PDS_PORT PLC_URL=$PLC_URL STATE_DIR=$PDS_STATE \ PDS_DATA_DIRECTORY=$PDS_DATA "$SCRIPT_DIR/local_atproto_pds.sh" start >/dev/null PDS_STARTED=1 note "creating account $HANDLE" ACCOUNT_FILE=$ACCOUNT_FILE PDS_URL=$PDS_URL PLC_URL=$PLC_URL \ HANDLE=$HANDLE EMAIL=$EMAIL PASSWORD=$PASSWORD \ "$SCRIPT_DIR/create_local_account.sh" >"$TMP_ROOT/account.env" || die "create_local_account.sh failed" DID=$(jq -er '.did' "$ACCOUNT_FILE") || die "account file missing did" ACCESS_JWT=$(jq -er '.session.accessJwt' "$ACCOUNT_FILE") || die "account file missing session accessJwt" ATP_APP_PASSWORD=$(jq -er '.app_password' "$ACCOUNT_FILE") || die "account file missing app_password" export ATP_PDS_URL="$PDS_URL" export ATP_HANDLE="$HANDLE" export ATP_APP_PASSWORD export S3_BUCKET="$BUCKET" export S3_PATH export S3_ENDPOINT_URL export S3_ACCESS_KEY_ID="$ACCESS_KEY" export S3_SECRET_ACCESS_KEY="$SECRET_KEY" export S3_REGION note "run 1: fetch + publish $LIMIT, upload cache to S3" "$VULN_NOTIF_BIN" --once --limit "$LIMIT" --initial-lookback-days "$LOOKBACK_DAYS" \ --cache-db "$CACHE_DB" >"$TMP_ROOT/run1.stdout" 2>"$TMP_ROOT/run1.stderr" || die "run 1 failed (see $TMP_ROOT/run1.stderr)" cat "$TMP_ROOT/run1.stdout" note "deleting local cache.db and running again" rm -f "$CACHE_DB" "$CACHE_DB-wal" "$CACHE_DB-shm" "$VULN_NOTIF_BIN" --once --limit "$LIMIT" --cache-db "$CACHE_DB" \ >"$TMP_ROOT/run2.stdout" 2>"$TMP_ROOT/run2.stderr" || die "run 2 failed (see $TMP_ROOT/run2.stderr)" cat "$TMP_ROOT/run2.stdout" UPLOADED=$(ndjson_sum s3_upload_bytes "$TMP_ROOT/run1.stdout") DOWNLOADED=$(ndjson_sum s3_download_bytes "$TMP_ROOT/run2.stdout") LIST_JSON=$(curl -sS --max-time 15 -G \ -H "Authorization: Bearer $ACCESS_JWT" \ --data-urlencode "repo=$DID" \ --data-urlencode "collection=$COLLECTION" \ --data-urlencode "limit=100" \ "$PDS_URL/xrpc/com.atproto.repo.listRecords") || die "listRecords request failed" PDS_RECORDS=$(jq -er '.records | length' <<<"$LIST_JSON") || die "listRecords failed: $LIST_JSON" printf '\nS3 upload bytes: %s\n' "$UPLOADED" printf 'S3 download bytes: %s\n' "$DOWNLOADED" printf 'PDS records: %s (expected %s)\n' "$PDS_RECORDS" "$((LIMIT * 2))" [ "$UPLOADED" -gt 0 ] || die "expected S3 upload bytes > 0, got $UPLOADED" [ "$DOWNLOADED" -gt 0 ] || die "expected S3 download bytes > 0, got $DOWNLOADED" [ "$PDS_RECORDS" -eq "$((LIMIT * 2))" ] || die "expected $((LIMIT * 2)) $COLLECTION records, got $PDS_RECORDS" printf '\nRESULT: PASS\n'