~ knotless Search/
↑↓ move↵ open⇧↵ all results

johnandersen777.bsky.social/vuln-notif

0
0
main
337 lines · 14.9 KB Raw History
#!/usr/bin/env bash
# Identity migration end-to-end on a temporary local PLC + PDS.
#
# Seeds the legacy RHEL split (three legacy release projects plus software,
# purl, and atprotoV1 records), plans the canonical migration, applies it,
# verifies zero legacy backlinks and stale CIDs, and proves a second apply
# performs zero writes. Localhost only; no live network services are used.
set -euo pipefail

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO_DIR=$(cd "$SCRIPT_DIR/.." && pwd)

PDS_PORT=${PDS_PORT:-4800}
PDS_URL=http://127.0.0.1:$PDS_PORT
PLC_PORT=${PLC_PORT:-2587}
PLC_URL=http://127.0.0.1:$PLC_PORT
HANDLE=${HANDLE:-cve.test}
EMAIL=${EMAIL:-cve@example.com}
PASSWORD=${PASSWORD:-test-password}
BUILD_BINS=${BUILD_BINS:-1}
VULN_NOTIF_BIN=${VULN_NOTIF_BIN:-$REPO_DIR/target/release/vuln-notif}
IDENTITY_MIGRATE_BIN=${IDENTITY_MIGRATE_BIN:-$REPO_DIR/target/release/identity-migrate}
FIXTURES=$REPO_DIR/tests/fixtures/identity-migration
EXPECTED=$FIXTURES/expected_plan.json

TMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/vuln-notif-identity-e2e.XXXXXX")
PLC_STATE=$TMP_ROOT/plc-state
PDS_STATE=$TMP_ROOT/pds-state
PDS_DATA=$TMP_ROOT/pds-data
ACCOUNT_FILE=$TMP_ROOT/account.json
DB=$TMP_ROOT/cache.db
PURLDB=$TMP_ROOT/purl2cpe.db
PLAN=$TMP_ROOT/identity-migration-plan.json
SNAPSHOT_DIR=$TMP_ROOT/snapshot
PLC_STARTED=0
PDS_STARTED=0

note() { printf '\n== %s\n' "$*"; }
log() { printf '%s\n' "$*" >&2; }
fail() { printf 'RESULT: FAIL (%s)\n' "$*" >&2; exit 1; }
die() { fail "$*"; }

cleanup() {
  local rc=$?
  set +e
  if [ "$PDS_STARTED" = 1 ]; then
    STATE_DIR=$PDS_STATE "$SCRIPT_DIR/local_atproto_pds.sh" stop >/dev/null 2>&1
  fi
  if [ "$PLC_STARTED" = 1 ]; then
    STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" stop >/dev/null 2>&1
  fi
  if [ "$rc" -ne 0 ] && [ "${KEEP_TMP:-0}" = 1 ]; then
    printf '\ncleanup: kept %s for inspection\n' "$TMP_ROOT" >&2
  else
    rm -rf "$TMP_ROOT"
  fi
  exit "$rc"
}
trap cleanup EXIT

create_record() {
  local collection=$1 rkey=$2 record=$3 payload response status body
  payload=$(jq -nc \
    --arg repo "$DID" \
    --arg collection "$collection" \
    --arg rkey "$rkey" \
    --argjson record "$record" \
    '{repo:$repo,collection:$collection,rkey:$rkey,record:$record}') || return 1
  response=$(curl -sS --max-time 20 -X POST \
    -H "Authorization: Bearer $ACCESS_JWT" \
    -H 'Content-Type: application/json' \
    -d "$payload" \
    -w '\n%{http_code}' \
    "$PDS_URL/xrpc/com.atproto.repo.createRecord") || return 1
  status=${response##*$'\n'}
  body=${response%$'\n'*}
  if [ "$status" != 200 ]; then
    printf 'createRecord %s HTTP %s: %s\n' "$collection" "$status" "$body" >&2
    return 1
  fi
  printf '%s' "$body"
}

# Sets FETCH_STATUS / FETCH_BODY for one record read.
fetch_record() {
  local response
  FETCH_STATUS=000
  FETCH_BODY=
  response=$(curl -sS --max-time 15 -G \
    -H "Authorization: Bearer $ACCESS_JWT" \
    --data-urlencode "repo=$DID" \
    --data-urlencode "collection=$1" \
    --data-urlencode "rkey=$2" \
    -w '\n%{http_code}' \
    "$PDS_URL/xrpc/com.atproto.repo.getRecord") || return 0
  FETCH_STATUS=${response##*$'\n'}
  FETCH_BODY=${response%$'\n'*}
}

plan_count() {
  jq -r --arg action "$1" '[.entries[] | select(.action == $action)] | length' "$PLAN"
}

note "building release binaries"
if [ "$BUILD_BINS" = 1 ]; then
  (cd "$REPO_DIR" && cargo build --release --locked --bin vuln-notif --bin identity-migrate) ||
    die "cargo build failed"
fi
[ -x "$VULN_NOTIF_BIN" ] || die "missing $VULN_NOTIF_BIN (set BUILD_BINS=1 or VULN_NOTIF_BIN)"
[ -x "$IDENTITY_MIGRATE_BIN" ] || die "missing $IDENTITY_MIGRATE_BIN (set BUILD_BINS=1 or IDENTITY_MIGRATE_BIN)"
[ -f "$FIXTURES/legacy_records.json" ] || die "missing $FIXTURES/legacy_records.json"
[ -f "$FIXTURES/cache_seed.sql" ] || die "missing $FIXTURES/cache_seed.sql"
[ -f "$EXPECTED" ] || die "missing $EXPECTED"

note "starting hono-plc on $PLC_URL"
plc_started_url=$(PORT=$PLC_PORT STATE_DIR=$PLC_STATE "$SCRIPT_DIR/local_plc.sh" start) ||
  die "local_plc.sh start failed"
PLC_STARTED=1
[ "$plc_started_url" = "$PLC_URL" ] || die "local_plc.sh reported unexpected URL: $plc_started_url"

note "starting atproto-pds on $PDS_URL"
pds_started_url=$(PDS_PORT=$PDS_PORT PLC_URL=$PLC_URL STATE_DIR=$PDS_STATE \
  PDS_DATA_DIRECTORY=$PDS_DATA "$SCRIPT_DIR/local_atproto_pds.sh" start) ||
  die "local_atproto_pds.sh start failed"
PDS_STARTED=1
[ "$pds_started_url" = "$PDS_URL" ] || die "local_atproto_pds.sh reported unexpected URL: $pds_started_url"

note "confirming health"
curl -fsS --max-time 5 "$PDS_URL/xrpc/_health" >/dev/null || die "PDS health check failed"

note "creating account $HANDLE"
ACCOUNT_FILE=$ACCOUNT_FILE PDS_URL=$PDS_URL PLC_URL=$PLC_URL \
  HANDLE=$HANDLE EMAIL=$EMAIL PASSWORD=$PASSWORD \
  "$SCRIPT_DIR/create_local_account.sh" >/dev/null || die "create_local_account.sh failed"
DID=$(jq -er '.did' "$ACCOUNT_FILE") || die "account file missing did"
ACCESS_JWT=$(jq -er '.session.accessJwt' "$ACCOUNT_FILE") || die "account file missing session accessJwt"
ATP_APP_PASSWORD=$(jq -er '.app_password' "$ACCOUNT_FILE") || die "account file missing app_password"
export ATP_PDS_URL="$PDS_URL"
export ATP_HANDLE="$HANDLE"
export ATP_APP_PASSWORD
printf 'did: %s\n' "$DID"

note "initializing cache schema"
# `identity-migrate` opens the cache through DbHandle, which applies the full
# schema and rate-budget tables. This is the network-free equivalent of the
# main binary's `--once --dry-run` schema init.
"$IDENTITY_MIGRATE_BIN" normalize-purl-slugs \
  --cache-db "$DB" --purldb "$PURLDB" --handle "did:local:identity-e2e" \
  >"$TMP_ROOT/schema-init.json" || die "cache schema init failed"
[ -f "$DB" ] || die "cache database was not created"

note "seeding legacy cache rows"
sed "s|__DID__|$DID|g" "$FIXTURES/cache_seed.sql" | sqlite3 "$DB" || die "cache seed failed"
seed_counts=$(sqlite3 "$DB" \
  "SELECT (SELECT count(*) FROM software)||','||(SELECT count(*) FROM software_purls)||','||(SELECT count(*) FROM cves);")
[ "$seed_counts" = "3,1,1" ] || die "cache seed row counts $seed_counts != 3,1,1"

note "seeding legacy PDS records"
SEED_CID=
SEEDED=0
while IFS= read -r line; do
  collection=$(jq -r '.collection' <<<"$line")
  rkey=$(jq -r '.rkey' <<<"$line")
  record=$(jq -c '.record' <<<"$line" |
    sed -e "s|__DID__|$DID|g" -e "s|__CID__|$SEED_CID|g")
  body=$(create_record "$collection" "$rkey" "$record") ||
    die "seed createRecord failed for $collection/$rkey"
  if [ -z "$SEED_CID" ]; then
    SEED_CID=$(jq -er '.cid' <<<"$body") || die "seed response missing cid"
  fi
  SEEDED=$((SEEDED + 1))
done < <(jq -c '.records[]' "$FIXTURES/legacy_records.json")
expected_seeded=$(jq -r '.records | length' "$FIXTURES/legacy_records.json")
[ "$SEEDED" = "$expected_seeded" ] ||
  die "seeded $SEEDED records, expected $expected_seeded"
log "seeded $SEEDED records (placeholder cid $SEED_CID)"

note "planning identity migration"
"$IDENTITY_MIGRATE_BIN" plan --cache-db "$DB" --purldb "$PURLDB" --out "$PLAN" \
  >"$TMP_ROOT/plan.jsonl" || die "identity-migrate plan failed"

canonical=$(jq -er '.canonical_slug' "$EXPECTED") || die "expected_plan.json missing canonical_slug"
jq -e --arg did "$DID" '.repo_did == $did' "$PLAN" >/dev/null ||
  die "plan repo_did does not match $DID"
while IFS='=' read -r action expected; do
  actual=$(plan_count "$action")
  [ "$actual" = "$expected" ] ||
    die "plan action $action count $actual != $expected"
done < <(jq -r '.counts | to_entries[] | "\(.key)=\(.value)"' "$EXPECTED")

while IFS= read -r slug; do
  jq -e --arg slug "$slug" --arg canonical "$canonical" '
    any(.entries[];
      .action == "delete_release_record" and
      .legacy_slug == $slug and
      .canonical_slug == $canonical)
  ' "$PLAN" >/dev/null ||
    die "plan does not map legacy project $slug to $canonical (delete)"
done < <(jq -r '.legacy_project_slugs[]' "$EXPECTED")

while IFS= read -r slug; do
  jq -e --arg slug "$slug" --arg canonical "$canonical" '
    any(.entries[];
      .action == "repoint_software" and
      .legacy_slug == $slug and
      .canonical_slug == $canonical)
  ' "$PLAN" >/dev/null ||
    die "plan does not map legacy software $slug to $canonical"
done < <(jq -r '.legacy_software_slugs[]' "$EXPECTED")

jq -e --arg canonical "$canonical" '
  any(.entries[]; .action == "repoint_cve" and .canonical_slug == $canonical)
' "$PLAN" >/dev/null || die "plan does not repoint the CVE to $canonical"
jq -e --arg canonical "$canonical" '
  any(.entries[]; .action == "create_project" and .canonical_slug == $canonical)
' "$PLAN" >/dev/null || die "plan does not create canonical project $canonical"
printf 'plan entries: %s (canonical %s)\n' "$(jq -r '.entries | length' "$PLAN")" "$canonical"

note "applying migration plan (snapshot -> $SNAPSHOT_DIR)"
"$IDENTITY_MIGRATE_BIN" apply --cache-db "$DB" --map "$PLAN" \
  --snapshot-dir "$SNAPSHOT_DIR" >"$TMP_ROOT/apply.jsonl" ||
  die "identity-migrate apply failed"
apply_report=$(jq -s 'map(select(.event == "identity_apply")) | last | .report' "$TMP_ROOT/apply.jsonl")
jq -e --argjson expected "$(jq -c '.apply' "$EXPECTED")" '
  .applied == $expected.applied and
  .skipped_done == $expected.skipped_done and
  .created_projects == $expected.created_projects and
  .software_repoints == $expected.software_repoints and
  .purl_repoints == $expected.purl_repoints and
  .cve_repoints == $expected.cve_repoints and
  .deleted == $expected.deleted and
  .normalized == $expected.normalized
' <<<"$apply_report" >/dev/null || die "apply report mismatch: $apply_report"
snapshot_records=$(jq -s -r 'map(select(.event == "identity_snapshot")) | last | .records' "$TMP_ROOT/apply.jsonl")
[ "$snapshot_records" = "$(jq -r '.snapshot_records' "$EXPECTED")" ] ||
  die "snapshot record count $snapshot_records != $(jq -r '.snapshot_records' "$EXPECTED")"
[ -s "$SNAPSHOT_DIR/software.release.project.jsonl" ] || die "snapshot missing project JSONL"

note "asserting canonical project"
CANONICAL_URI="at://$DID/software.release.project/$canonical"
fetch_record software.release.project "$canonical"
[ "$FETCH_STATUS" = 200 ] || die "canonical project missing (HTTP $FETCH_STATUS: $FETCH_BODY)"
CANONICAL_CID=$(jq -er '.cid' <<<"$FETCH_BODY") || die "canonical project missing cid"
jq -e --arg name "$canonical" '.value.name == $name' <<<"$FETCH_BODY" >/dev/null ||
  die "canonical project name mismatch: $FETCH_BODY"

note "asserting CVE record points at the canonical project"
CVE_ID=CVE-2024-0001
NVD_URI="at://$DID/com.nahdig.cve.nvdV2/$CVE_ID"
fetch_record com.nahdig.cve.atprotoV1 "$CVE_ID"
[ "$FETCH_STATUS" = 200 ] || die "CVE record missing (HTTP $FETCH_STATUS: $FETCH_BODY)"
jq -e --arg project "$CANONICAL_URI" --arg cid "$CANONICAL_CID" --arg nvd "$NVD_URI" '
  (.value.software | length) == 1 and
  .value.software[0].project.uri == $project and
  .value.software[0].project.cid == $cid and
  .value.nvd.uri == $nvd and
  ((.value.software[0].versions // []) | length) == 0
' <<<"$FETCH_BODY" >/dev/null || die "CVE refs not canonical: $FETCH_BODY"

note "asserting software records repointed and purl ref updated"
while IFS= read -r slug; do
  fetch_record com.nahdig.software "$slug"
  [ "$FETCH_STATUS" = 200 ] || die "software record $slug missing after apply"
  jq -e --arg project "$CANONICAL_URI" --arg cid "$CANONICAL_CID" '
    .value.project.uri == $project and .value.project.cid == $cid
  ' <<<"$FETCH_BODY" >/dev/null || die "software $slug did not repoint to $canonical"
done < <(jq -r '.legacy_software_slugs[]' "$EXPECTED")

PURL=pkg:rpm/redhat/enterprise-linux@8
PURL_RKEY=$(jq -r '.records[] | select(.collection == "com.nahdig.purl") | .rkey' \
  "$FIXTURES/legacy_records.json")
fetch_record com.nahdig.purl "$PURL_RKEY"
[ "$FETCH_STATUS" = 200 ] || die "purl record missing after apply"
purl_body=$FETCH_BODY
purl_software_uri=$(jq -er '.value.software[0].uri' <<<"$purl_body") || die "purl has no software ref"
purl_software_slug=${purl_software_uri##*/}
fetch_record com.nahdig.software "$purl_software_slug"
[ "$FETCH_STATUS" = 200 ] || die "purl software record missing after apply"
purl_software_cid=$(jq -er '.cid' <<<"$FETCH_BODY") || die "purl software record missing cid"
jq -e --arg purl "$PURL" --arg uri "$purl_software_uri" --arg cid "$purl_software_cid" '
  .value.purl == $purl and
  .value.software[0].uri == $uri and
  .value.software[0].cid == $cid
' <<<"$purl_body" >/dev/null || die "purl strongRef cid not updated: $purl_body"

note "asserting legacy release records are deleted"
deleted=0
while IFS= read -r uri; do
  rest=${uri#at://}
  rest=${rest#*/}
  collection=${rest%%/*}
  rkey=${rest#*/}
  fetch_record "$collection" "$rkey"
  [ "$FETCH_STATUS" != 200 ] || die "legacy record still exists: $uri"
  jq -e '.error == "RecordNotFound"' <<<"$FETCH_BODY" >/dev/null ||
    log "warning: unexpected delete body for $uri: $FETCH_BODY"
  deleted=$((deleted + 1))
done < <(jq -r '.entries[] | select(.action == "delete_release_record") | .legacy_uri' "$PLAN")
[ "$deleted" = "$(jq -r '.counts.delete_release_record' "$EXPECTED")" ] ||
  die "checked $deleted legacy records, expected $(jq -r '.counts.delete_release_record' "$EXPECTED")"

primary_project=$(jq -r '.legacy_project_slugs[0]' "$EXPECTED")
normalized_slug=$(sqlite3 "$DB" "SELECT software_slug FROM software_purls WHERE purl='$PURL';")
[ "$normalized_slug" = "$primary_project" ] ||
  die "software_purls.software_slug '$normalized_slug' != '$primary_project'"

note "second apply must perform zero writes"
"$IDENTITY_MIGRATE_BIN" apply --cache-db "$DB" --map "$PLAN" \
  >"$TMP_ROOT/apply-second.jsonl" || die "second identity-migrate apply failed"
second_report=$(jq -s 'map(select(.event == "identity_apply")) | last | .report' "$TMP_ROOT/apply-second.jsonl")
jq -e '(.applied == 0) and (.deleted == 0)' <<<"$second_report" >/dev/null ||
  die "second apply wrote: $second_report"
total_entries=$(jq -r '.entries | length' "$PLAN")
skipped=$(jq -r '.skipped_done' <<<"$second_report")
[ "$skipped" = "$total_entries" ] ||
  die "second apply skipped $skipped entries, expected $total_entries"

note "verifying migration"
"$IDENTITY_MIGRATE_BIN" verify --cache-db "$DB" --map "$PLAN" \
  >"$TMP_ROOT/verify.jsonl" || die "identity-migrate verify failed"
verify_report=$(jq -s 'map(select(.event == "identity_verify")) | last | .report' "$TMP_ROOT/verify.jsonl")
jq -e '(.errors | length) == 0 and .stale_refs == 0' <<<"$verify_report" >/dev/null ||
  die "verify errors: $verify_report"
[ "$(jq -r '.deletions' <<<"$verify_report")" = "$(jq -r '.counts.delete_release_record' "$EXPECTED")" ] ||
  die "verify deletion count mismatch: $verify_report"
[ "$(jq -r '.cve_repoints' <<<"$verify_report")" -ge 1 ] ||
  die "verify saw no CVE repoints: $verify_report"

printf '\nRESULT: PASS\n'
printf '  PDS:          %s\n' "$PDS_URL"
printf '  PLC:          %s\n' "$PLC_URL"
printf '  Account:      %s (%s)\n' "$HANDLE" "$DID"
printf '  Plan:         %s entries\n' "$total_entries"
printf '  Apply:        %s deleted, %s applied (first run)\n' \
  "$(jq -r '.deleted' <<<"$apply_report")" "$(jq -r '.applied' <<<"$apply_report")"
printf '  Second apply: %s skipped, 0 written\n' "$skipped"
printf '  Verify:       %s\n' "$verify_report"