chadtmiller.com/monolith
An atproto PDS
0
0
// The crawlers each side tells of itself with requestCrawl
// (spec/FEDERATION.md, "requestCrawl"): stubs that record every request
// they get.
import { createServer } from 'node:http'
/// Every request each side's crawlers had, by side name, in the order they
/// came: `{crawler, at, method, path, type, body}`, `at` in ms since the
/// crawlers started.
export const crawls = { reference: [], server: [] }
/// The crawlers that listen, by name, each answering with its status.
export const LISTENING = { 'answers 200': 200, 'answers 500': 500 }
let started = 0
/// How long since the crawlers started, in ms.
export const sinceStart = () => Date.now() - started
/// A crawler for `side` that records each request and answers `status`.
const stub = (side, name, status) =>
new Promise((ok) => {
const s = createServer((req, res) => {
let body = ''
req.on('data', (d) => (body += d))
req.on('end', () => {
let parsed
try {
parsed = JSON.parse(body)
} catch {
parsed = { unparsed: body.slice(0, 200) }
}
crawls[side].push({ crawler: name, at: sinceStart(), method: req.method, path: req.url, type: req.headers['content-type']?.split(';')[0].trim() ?? null, body: parsed })
res.writeHead(status, { 'content-type': 'application/json' })
res.end(status === 200 ? '{}' : '{"error":"InternalServerError","message":"a crawler that fails"}')
})
})
s.listen(0, '127.0.0.1', () => ok(`http://127.0.0.1:${s.address().port}`))
s.unref()
})
/// Start each side's crawlers: those that listen, and a URL where nothing
/// does. Answers each side's list of URLs, by side name, for its
/// `PDS_CRAWLERS` (spec/FEDERATION.md, "Environment").
export async function startCrawlers() {
started = Date.now()
const urls = {}
for (const side of ['reference', 'server']) {
urls[side] = []
for (const [name, status] of Object.entries(LISTENING)) urls[side].push(await stub(side, name, status))
urls[side].push('http://127.0.0.1:1')
}
return urls
}