1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#!/usr/bin/env bash
# Verify S3 cache upload/restore with local RustFS + hono-plc + atproto-pds:
# 1. run vuln-notif once, publish 5 CVEs, upload cache.db to RustFS
# 2. delete the local cache.db
# 3. run again, restore the cache from RustFS, publish 5 more, no duplicates
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
cd "$REPO_DIR" || exit 1
PDS_PORT=${PDS_PORT:-4800}
PDS_URL=http://127.0.0.1:$PDS_PORT
PLC_PORT=${PLC_PORT:-2587}
PLC_URL=http://127.0.0.1:$PLC_PORT
S3_PORT=${S3_PORT:-9000}
S3_ENDPOINT_URL=http://127.0.0.1:$S3_PORT
BUCKET=${BUCKET:-vuln-notif-test}
S3_PATH=${S3_PATH:-vuln-notif/cve-cache.db}
ACCESS_KEY=${ACCESS_KEY:-rustfsadmin}
SECRET_KEY=${SECRET_KEY:-rustfsadmin}
S3_REGION=${S3_REGION:-us-east-1}
LIMIT=${LIMIT:-5}
LOOKBACK_DAYS=${LOOKBACK_DAYS:-1}
VULN_NOTIF_BIN=${VULN_NOTIF_BIN:-$REPO_DIR/target/release/vuln-notif}
HANDLE=${HANDLE:-cve.test}
EMAIL=${EMAIL:-cve@example.com}
PASSWORD=${PASSWORD:-test-password}
COLLECTION=com.nahdig.cve.nvdV2
TMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/vuln-notif-s3-e2e.XXXXXX")
RUSTFS_STATE=$TMP_ROOT/rustfs-state
PLC_STATE=$TMP_ROOT/plc-state
PDS_STATE=$TMP_ROOT/pds-state
PDS_DATA=$TMP_ROOT/pds-data
ACCOUNT_FILE=$TMP_ROOT/account.json
CACHE_DB=$TMP_ROOT/cache.db
RUSTFS_STARTED=0
PLC_STARTED=0
PDS_STARTED=0
note() { printf '\n== %s\n' "$*"; }
die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
log() { printf '%s\n' "$*" >&2; }
cleanup() {
local rc=$?
set +e
if [ "$PDS_STARTED" = 1 ]; then
STATE_DIR=$PDS_STATE "$SCRIPT_DIR/local_atproto_pds.sh" stop >/dev/null 2>&1
fi
if [ "$PLC_STARTED" = 1 ]; then
STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" stop >/dev/null 2>&1
fi
if [ "$RUSTFS_STARTED" = 1 ]; then
STATE_DIR=$RUSTFS_STATE "$SCRIPT_DIR/local_rustfs.sh" stop >/dev/null 2>&1
fi
if [ "$rc" -ne 0 ] && [ "${KEEP_TMP:-0}" = 1 ]; then
printf '\ncleanup: kept %s for inspection\n' "$TMP_ROOT" >&2
else
rm -rf "$TMP_ROOT"
fi
exit "$rc"
}
trap cleanup EXIT
ndjson_sum() {
local key=$1 file=$2
jq -s --arg key "$key" '[.[] | .[$key]? // empty] | add // 0' "$file"
}
[ -x "$VULN_NOTIF_BIN" ] || die "vuln-notif binary not found: $VULN_NOTIF_BIN"
note "starting local RustFS on $S3_ENDPOINT_URL"
STATE_DIR=$RUSTFS_STATE PORT=$S3_PORT BUCKET=$BUCKET \
ACCESS_KEY=$ACCESS_KEY SECRET_KEY=$SECRET_KEY \
"$SCRIPT_DIR/local_rustfs.sh" start >/dev/null
RUSTFS_STARTED=1
note "starting hono-plc on $PLC_URL"
PORT=$PLC_PORT STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" start >/dev/null
PLC_STARTED=1
note "starting atproto-pds on $PDS_URL"
PDS_PORT=$PDS_PORT PLC_URL=$PLC_URL STATE_DIR=$PDS_STATE \
PDS_DATA_DIRECTORY=$PDS_DATA "$SCRIPT_DIR/local_atproto_pds.sh" start >/dev/null
PDS_STARTED=1
note "creating account $HANDLE"
ACCOUNT_FILE=$ACCOUNT_FILE PDS_URL=$PDS_URL PLC_URL=$PLC_URL \
HANDLE=$HANDLE EMAIL=$EMAIL PASSWORD=$PASSWORD \
"$SCRIPT_DIR/create_local_account.sh" >"$TMP_ROOT/account.env" ||
die "create_local_account.sh failed"
DID=$(jq -er '.did' "$ACCOUNT_FILE") || die "account file missing did"
ACCESS_JWT=$(jq -er '.session.accessJwt' "$ACCOUNT_FILE") || die "account file missing session accessJwt"
ATP_APP_PASSWORD=$(jq -er '.app_password' "$ACCOUNT_FILE") || die "account file missing app_password"
export ATP_PDS_URL="$PDS_URL"
export ATP_HANDLE="$HANDLE"
export ATP_APP_PASSWORD
export S3_BUCKET="$BUCKET"
export S3_PATH
export S3_ENDPOINT_URL
export S3_ACCESS_KEY_ID="$ACCESS_KEY"
export S3_SECRET_ACCESS_KEY="$SECRET_KEY"
export S3_REGION
note "run 1: fetch + publish $LIMIT, upload cache to S3"
"$VULN_NOTIF_BIN" --once --limit "$LIMIT" --initial-lookback-days "$LOOKBACK_DAYS" \
--cache-db "$CACHE_DB" >"$TMP_ROOT/run1.stdout" 2>"$TMP_ROOT/run1.stderr" ||
die "run 1 failed (see $TMP_ROOT/run1.stderr)"
cat "$TMP_ROOT/run1.stdout"
note "deleting local cache.db and running again"
rm -f "$CACHE_DB" "$CACHE_DB-wal" "$CACHE_DB-shm"
"$VULN_NOTIF_BIN" --once --limit "$LIMIT" --cache-db "$CACHE_DB" \
>"$TMP_ROOT/run2.stdout" 2>"$TMP_ROOT/run2.stderr" ||
die "run 2 failed (see $TMP_ROOT/run2.stderr)"
cat "$TMP_ROOT/run2.stdout"
UPLOADED=$(ndjson_sum s3_upload_bytes "$TMP_ROOT/run1.stdout")
DOWNLOADED=$(ndjson_sum s3_download_bytes "$TMP_ROOT/run2.stdout")
LIST_JSON=$(curl -sS --max-time 15 -G \
-H "Authorization: Bearer $ACCESS_JWT" \
--data-urlencode "repo=$DID" \
--data-urlencode "collection=$COLLECTION" \
--data-urlencode "limit=100" \
"$PDS_URL/xrpc/com.atproto.repo.listRecords") || die "listRecords request failed"
PDS_RECORDS=$(jq -er '.records | length' <<<"$LIST_JSON") ||
die "listRecords failed: $LIST_JSON"
printf '\nS3 upload bytes: %s\n' "$UPLOADED"
printf 'S3 download bytes: %s\n' "$DOWNLOADED"
printf 'PDS records: %s (expected %s)\n' "$PDS_RECORDS" "$((LIMIT * 2))"
[ "$UPLOADED" -gt 0 ] || die "expected S3 upload bytes > 0, got $UPLOADED"
[ "$DOWNLOADED" -gt 0 ] || die "expected S3 download bytes > 0, got $DOWNLOADED"
[ "$PDS_RECORDS" -eq "$((LIMIT * 2))" ] ||
die "expected $((LIMIT * 2)) $COLLECTION records, got $PDS_RECORDS"
printf '\nRESULT: PASS\n'