chadtmiller.com/pds.js
An atproto PDS in JavaScript, for Cloudflare Workers and Node
0
0
# E2E Local Relay Integration Plan
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
**Goal:** Integrate local PLC and relay from docker-compose into e2e tests for debugging sync issues and comprehensive federation testing.
**Architecture:** Tests use a hybrid docker management approach - check if services are running, start them if not, reset volumes for clean state. The Node.js PDS is configured with `relayUrl` to notify the local relay after commits. New test suite verifies events flow through the relay correctly.
**Tech Stack:** Docker Compose, Vitest, Node.js `child_process` for docker management, WebSocket for firehose tests.
---
## Task 1: Update docker-compose.yml with Health Checks
**Files:**
- Modify: `docker-compose.yml`
**Step 1: Add health check to PLC service**
Add after the `command` line in the `plc` service:
```yaml
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:2582/_health || exit 1"]
interval: 2s
timeout: 5s
retries: 15
```
**Step 2: Update relay service with health check and Linux fix**
Replace the entire `relay` service with:
```yaml
relay:
image: ghcr.io/bluesky-social/indigo:latest
ports:
- "2470:2470"
environment:
- GOLOG_LOG_LEVEL=info
- DATA_DIR=/data
- RELAY_ADMIN_KEY=localdev
- ATP_PLC_HOST=http://plc:2582
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- relay_data:/data
command: ["/bigsky", "--api-listen", ":2470"]
depends_on:
plc:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:2470/xrpc/_health || exit 1"]
interval: 2s
timeout: 5s
retries: 15
```
**Step 3: Verify docker-compose syntax**
Run: `docker-compose config`
Expected: YAML output with no errors
**Step 4: Commit**
```bash
git add docker-compose.yml
git commit -m "feat: add health checks and Linux support to docker-compose"
```
---
## Task 2: Create Docker Services Helper
**Files:**
- Create: `test/helpers/docker-services.js`
**Step 1: Create the docker services helper**
```js
// test/helpers/docker-services.js
import { execSync } from 'node:child_process';
const RELAY_URL = 'http://localhost:2470';
const PLC_URL = 'http://localhost:2582';
/**
* Wait for a service to respond
* @param {string} url
* @param {string} name
* @param {number} maxAttempts
*/
async function waitForService(url, name, maxAttempts = 30) {
for (let i = 0; i < maxAttempts; i++) {
try {
const res = await fetch(url);
if (res.ok) {
console.log(`${name} is ready`);
return;
}
} catch {
// Service not ready yet
}
await new Promise((r) => setTimeout(r, 1000));
}
throw new Error(`${name} failed to start after ${maxAttempts} attempts`);
}
/**
* Ensure docker-compose services are running
* Resets volumes for clean state on first start
*/
export async function ensureDockerServices() {
// Check if relay is already responding
try {
const res = await fetch(RELAY_URL);
if (res.ok) {
console.log('Docker services already running');
return;
}
} catch {
// Services not running, need to start
}
console.log('Starting docker services (this may take a moment)...');
// Reset volumes for clean slate
execSync('docker-compose down -v', { stdio: 'inherit' });
execSync('docker-compose up -d', { stdio: 'inherit' });
// Wait for services to be ready
await waitForService(PLC_URL, 'PLC');
await waitForService(RELAY_URL, 'Relay');
console.log('Docker services are ready');
}
/**
* Request the relay to crawl a PDS
* @param {string} hostname - PDS hostname (e.g., 'host.docker.internal:3000')
*/
export async function requestRelayCrawl(hostname) {
const res = await fetch(`${RELAY_URL}/admin/pds/requestCrawl`, {
method: 'POST',
headers: {
Authorization: 'Bearer localdev',
'Content-Type': 'application/json',
},
body: JSON.stringify({ hostname }),
});
if (!res.ok) {
throw new Error(`Failed to request crawl: ${res.status}`);
}
}
export { RELAY_URL, PLC_URL };
```
**Step 2: Verify file was created**
Run: `ls -la test/helpers/docker-services.js`
Expected: File exists
**Step 3: Commit**
```bash
git add test/helpers/docker-services.js
git commit -m "feat: add docker services helper for e2e tests"
```
---
## Task 3: Update Node Server Helper with Relay Config
**Files:**
- Modify: `test/helpers/node-server.js`
**Step 1: Add USE_LOCAL_INFRA constant and relayUrl config**
Replace the entire file with:
```js
// test/helpers/node-server.js
import { mkdirSync, rmSync } from 'node:fs';
import { createServer } from '@pdsjs/node';
const TEST_DATA_DIR = './test-data';
const TEST_PORT = 3000;
const USE_LOCAL_INFRA = process.env.USE_LOCAL_INFRA !== 'false';
/**
* Start Node.js PDS server for e2e tests
* Cleans test data directory for fresh state each run
* @returns {Promise<{close: () => Promise<void>}>} Server instance with close() method
*/
export async function startNodeServer() {
// Fresh data each run
rmSync(TEST_DATA_DIR, { recursive: true, force: true });
mkdirSync(TEST_DATA_DIR, { recursive: true });
const server = await createServer({
port: TEST_PORT,
dbPath: `${TEST_DATA_DIR}/pds.db`,
blobsDir: `${TEST_DATA_DIR}/blobs`,
jwtSecret: 'test-secret-for-e2e',
hostname: USE_LOCAL_INFRA ? 'host.docker.internal:3000' : `localhost:${TEST_PORT}`,
password: 'test-password',
// Use local relay when docker infrastructure is available
relayUrl: USE_LOCAL_INFRA ? 'localhost:2470' : undefined,
// Keep appview pointing to production (for proxy tests)
appviewUrl: 'https://api.bsky.app',
appviewDid: 'did:web:api.bsky.app',
});
await server.listen();
return server;
}
/**
* Stop Node.js PDS server
* @param {{close: () => Promise<void>}} server - Server instance from startNodeServer
*/
export async function stopNodeServer(server) {
await server.close();
}
export { USE_LOCAL_INFRA, TEST_PORT };
```
**Step 2: Commit**
```bash
git add test/helpers/node-server.js
git commit -m "feat: configure node server helper to use local relay"
```
---
## Task 4: Wire Docker Services into E2E Test Setup
**Files:**
- Modify: `test/e2e.test.js:1-40`
**Step 1: Add docker services import**
Add after the existing imports (around line 11):
```js
import { ensureDockerServices, RELAY_URL } from './helpers/docker-services.js';
import { USE_LOCAL_INFRA } from './helpers/node-server.js';
```
**Step 2: Update beforeAll to start docker services**
Find the `beforeAll` block and update it. Add at the very start of the `beforeAll` callback, before any existing code:
```js
// Start docker services if enabled (PLC, relay)
if (USE_LOCAL_INFRA) {
await ensureDockerServices();
}
```
**Step 3: Increase beforeAll timeout**
Change the beforeAll timeout from whatever it currently is to 120000 (2 minutes) to accommodate docker startup:
```js
}, 120000);
```
**Step 4: Run tests to verify setup works**
Run: `USE_LOCAL_INFRA=false npm run test:e2e:node 2>&1 | tail -5`
Expected: Tests pass (docker services skipped)
**Step 5: Commit**
```bash
git add test/e2e.test.js
git commit -m "feat: wire docker services into e2e test setup"
```
---
## Task 5: Add Relay Sync Test Suite
**Files:**
- Modify: `test/e2e.test.js` (add new describe block before Cleanup)
**Step 1: Add relay sync test suite**
Add this new describe block just before the `describe('Cleanup', ...)` block:
```js
describe('Relay sync (requires docker)', () => {
// Skip entire suite if docker infrastructure is disabled
beforeAll(function () {
if (!USE_LOCAL_INFRA) {
console.log('Skipping relay sync tests (USE_LOCAL_INFRA=false)');
}
});
it('relay receives repo after PDS commits', async () => {
if (!USE_LOCAL_INFRA) return;
// Create a record (triggers notifyRelay internally)
const { status } = await jsonPost(
'/xrpc/com.atproto.repo.createRecord',
{
repo: DID,
collection: 'app.bsky.feed.post',
record: { text: 'relay sync test', createdAt: new Date().toISOString() },
},
{ Authorization: `Bearer ${token}` },
);
expect(status).toBe(200);
// Poll relay until it has synced the repo (or timeout)
let synced = false;
for (let i = 0; i < 30; i++) {
try {
const res = await fetch(
`${RELAY_URL}/xrpc/com.atproto.sync.getLatestCommit?did=${DID}`,
);
if (res.ok) {
const data = await res.json();
if (data.rev && data.rev.length > 0) {
synced = true;
console.log(`Relay synced after ${i + 1} attempts:`, data);
break;
}
}
} catch {
// Relay not ready or error
}
await new Promise((r) => setTimeout(r, 500));
}
expect(synced, 'Relay should have synced the repo').toBe(true);
});
it('relay firehose emits commit events', async () => {
if (!USE_LOCAL_INFRA) return;
// Connect to relay firehose
const wsUrl = RELAY_URL.replace('http', 'ws');
const ws = new WebSocket(
`${wsUrl}/xrpc/com.atproto.sync.subscribeRepos?cursor=0`,
);
const events = [];
await new Promise((resolve, reject) => {
ws.on('message', (data) => {
events.push(data);
if (events.length >= 1) {
ws.close();
resolve();
}
});
ws.on('error', (err) => {
console.error('Relay WebSocket error:', err);
reject(err);
});
ws.on('open', () => {
// Create a record to generate an event
jsonPost(
'/xrpc/com.atproto.repo.createRecord',
{
repo: DID,
collection: 'app.bsky.feed.post',
record: { text: 'firehose test', createdAt: new Date().toISOString() },
},
{ Authorization: `Bearer ${token}` },
);
// Give time for events, then close
setTimeout(() => {
ws.close();
resolve();
}, 5000);
});
setTimeout(() => reject(new Error('Relay firehose timeout')), 15000);
});
console.log(`Received ${events.length} events from relay firehose`);
expect(events.length).toBeGreaterThan(0);
});
});
```
**Step 2: Run tests with docker infrastructure**
Run: `npm run test:e2e:node 2>&1 | grep -E "relay|Relay|sync"`
Expected: Relay sync tests run (may pass or fail - this is debugging!)
**Step 3: Commit**
```bash
git add test/e2e.test.js
git commit -m "test: add relay sync test suite for debugging"
```
---
## Task 6: Test Full Integration
**Step 1: Clean start with docker**
Run: `docker-compose down -v`
Expected: Services stopped, volumes removed
**Step 2: Run full e2e test suite**
Run: `npm run test:e2e:node 2>&1 | tail -20`
Expected: All tests run, docker services started automatically
**Step 3: Check relay logs for any errors**
Run: `docker-compose logs relay 2>&1 | tail -50`
Expected: Log output showing crawl requests and any sync errors
**Step 4: Final commit if any adjustments made**
```bash
git add -A
git commit -m "test: complete local relay integration for e2e tests"
```
---
## Summary
| Task | Description | Commit Message |
|------|-------------|----------------|
| 1 | Docker health checks + Linux fix | `feat: add health checks and Linux support to docker-compose` |
| 2 | Docker services helper | `feat: add docker services helper for e2e tests` |
| 3 | Node server relay config | `feat: configure node server helper to use local relay` |
| 4 | Wire into test setup | `feat: wire docker services into e2e test setup` |
| 5 | Relay sync tests | `test: add relay sync test suite for debugging` |
| 6 | Integration verification | `test: complete local relay integration for e2e tests` |