chadtmiller.com/monolith
An atproto PDS
0
0
// The scale run (spec/STORE.md, "Scale"): the server alone, as the reference
// says nothing of its memory, with `PDS_CACHE_BYTES` small. Its resident
// memory and its starts are measured with few accounts, many, and fifty
// times as many again, with a small repository and a large one, and while a
// large getRepo and a firehose replay are read; each held to its rule. What
// it answers is held to what the harness wrote.
import { execFileSync } from 'node:child_process'
import { isDeepStrictEqual } from 'node:util'
import { readCarWithRoot } from '@atproto/repo'
import { cborDecode, cborDecodeMulti } from '@atproto/common'
import { CACHE_BYTES } from '../lib/args.mjs'
import { cand, metricsUrl, server, wait, waitReady } from '../lib/processes.mjs'
import { fail } from '../lib/plc.mjs'
import { count } from '../lib/report.mjs'
import { xrpcHttp } from '../lib/xrpc.mjs'
/// The accounts made: few, then many, then most, of which some have a
/// password.
const FEW = 100
const MANY = 2000
const MOST = 100000
const WITH_PASSWORDS = 10000
/// How many times over every session with a password is refreshed.
const REFRESHES = 4
/// Of most accounts, every one in this many is looked for after a start.
const ONE_IN = 50
/// How many repositories `listRepos` is asked for at once, and how many times.
const LIST_LIMIT = 100
const LIST_PAGES = 3
/// The large repository: batches of records of about 120 KB, each under
/// the 1000000-byte body (spec/LIMITS.md, "Request size"), to about 64 MiB.
const BIG_RECORD = 120_000
const BIG_BATCH = 7
const BIG_BATCHES = 80
/// What one account holds besides its repository: blobs, each referenced
/// by a record of its own, and a space repo's records, of about 1 KB.
const BLOBS = 4000
const BLOB_BYTES = 512
const SPACE_RECORDS = 40000
/// The bytes of each space record's text.
const SPACE_RECORD_BYTES = 1000
const SPACE_BATCH = 200
const SPACE_TYPE = 'com.example.group'
const SPACE_COLLECTION = 'com.example.scale.space'
/// The rules (spec/STORE.md, "Bounds", "Scale"): what requests under way
/// may take, what an account costs idle, what a just-started server may hold
/// besides, the start's slack, and a cold account's first request's.
const REQUEST_SLACK = 32 * 1048576
const PER_ACCOUNT = 4096
const IDLE_SLACK = 8 * 1048576
const START_SLACK = 250
const COLD_MS = 1000
/// How many requests go at once when making or touching accounts; and when
/// making most of them, which is most of the run's time.
const AT_ONCE = 8
/// How many accounts are first used after a shutdown and a start.
const FIRST_USES = 200
const MOST_AT_ONCE = 32
/// The server's resident memory, in bytes, as `ps` reports it.
function rss() {
const kib = Number(execFileSync('ps', ['-o', 'rss=', '-p', String(server.proc.pid)]).toString().trim())
return kib * 1024
}
/// Run `fn`, sampling the server's resident memory every 20 ms; answers its
/// result and the most memory seen.
async function sampled(fn) {
let peak = rss()
const timer = setInterval(() => {
try {
peak = Math.max(peak, rss())
} catch {}
}, 20)
try {
const result = await fn()
peak = Math.max(peak, rss())
return { result, peak }
} finally {
clearInterval(timer)
}
}
const MiB = (n) => `${(n / 1048576).toFixed(1)} MiB`
/// The memory bounds, given the base: at every moment, and just started.
const state = { r0: 0, t0: 0, accounts: [] }
const bound = () => state.r0 + REQUEST_SLACK + CACHE_BYTES + PER_ACCOUNT * state.accounts.length
const idleBound = () => state.r0 + IDLE_SLACK + PER_ACCOUNT * state.accounts.length
async function holdPeak(what, peak) {
if (peak > bound()) {
await fail(cand, `resident memory, ${what}`, { accounts: state.accounts.length, cache: MiB(CACHE_BYTES), base: MiB(state.r0) }, `at most ${MiB(bound())}: the base, ${MiB(REQUEST_SLACK)} for requests, the cache and 4 KiB an account`, MiB(peak))
}
count(`memory: ${what}`, true)
console.error(` ${what}: peak ${MiB(peak)}, bound ${MiB(bound())}`)
}
/// Stop the server, killed or with the signal given, and start it again;
/// its start held to the rule, and its memory just started to the idle
/// bound. Answers the start's time.
async function restart(what, signal = 'SIGKILL') {
await server.stop(signal)
server.start()
const ms = await waitReady(server, cand.url)
await wait(300)
const r = rss()
console.error(` started ${what}: ${ms} ms, ${MiB(r)}`)
if (ms > 2 * state.t0 + START_SLACK) {
await fail(cand, `start, ${what}`, { accounts: state.accounts.length }, `describeServer within ${2 * state.t0 + START_SLACK} ms: twice the start on an empty data directory, ${state.t0} ms, and ${START_SLACK} ms`, `${ms} ms`)
}
if (r > idleBound()) {
await fail(cand, `resident memory just started, ${what}`, { accounts: state.accounts.length, base: MiB(state.r0) }, `at most ${MiB(idleBound())}: the empty start's, 8 MiB and 4 KiB an account`, MiB(r))
}
count('start', true)
return ms
}
/// Run `fn` on each of `items`, `atOnce` at a time.
async function each(items, fn, atOnce = AT_ONCE) {
let next = 0
await Promise.all(Array.from({ length: atOnce }, async () => {
while (next < items.length) await fn(items[next++])
}))
}
/// Make accounts on the server alone until there are `n`, `atOnce` at a
/// time: without passwords, which costs no password hashing, or each with
/// one.
async function makeAccounts(n, { passwords = false, atOnce = AT_ONCE } = {}) {
const fresh = Array.from({ length: n - state.accounts.length }, (_, i) => state.accounts.length + i)
await each(fresh, async (i) => {
const handle = `scale${i}.test`
const email = `scale${i}@example.com`
const password = passwords ? `scale-password-${i}` : undefined
const res = await xrpcHttp(cand, 'com.atproto.server.createAccount', { body: { handle, email, password } })
if (res.status !== 200) await fail(cand, 'createAccount', { handle, email, password: password && '<password>' }, 'the account', res.body)
state.accounts[i] = { did: res.body.did, handle, email, password, token: res.body.accessJwt, refresh: res.body.refreshJwt }
}, atOnce)
count('accounts made', true)
}
/// The session of every account with a password refreshed, `REFRESHES`
/// times over, each with the token its last refresh answered: what the
/// server keeps of its sessions written many times over.
async function refreshAll() {
const signedIn = state.accounts.filter((a) => a.password)
for (let round = 0; round < REFRESHES; round++) {
await each(signedIn, async (a) => {
const res = await xrpcHttp(cand, 'com.atproto.server.refreshSession', { token: a.refresh, method: 'POST' })
if (res.status !== 200 || typeof res.body?.refreshJwt !== 'string' || typeof res.body?.accessJwt !== 'string') await fail(cand, 'refreshSession', { did: a.did, round }, 'the session, refreshed', res.body)
a.token = res.body.accessJwt
a.refresh = res.body.refreshJwt
})
}
count('sessions refreshed', true)
}
/// The ways an account is found, each a request and whether its answer is
/// the account's: by its DID; by its handle; by the session it had before
/// the start; and, with a password, at a login with its email and with its
/// handle.
function ways(a) {
const login = (identifier) => () => xrpcHttp(cand, 'com.atproto.server.createSession', { body: { identifier, password: a.password } })
return [
['getLatestCommit', () => xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did: a.did } }), (r) => r.status === 200],
['resolveHandle', () => xrpcHttp(cand, 'com.atproto.identity.resolveHandle', { params: { handle: a.handle } }), (r) => r.status === 200 && r.body?.did === a.did],
['getSession, with a token from before the start', () => xrpcHttp(cand, 'com.atproto.server.getSession', { token: a.token }), (r) => r.status === 200 && r.body?.did === a.did && r.body?.handle === a.handle],
...(a.password ? [['createSession, with its email', login(a.email), (r) => r.status === 200 && r.body?.did === a.did], ['createSession, with its handle', login(a.handle), (r) => r.status === 200 && r.body?.did === a.did]] : []),
]
}
/// One account in `ONE_IN`, found after a start in every way an account is
/// found (`ways`), each account by another way first, so that each way
/// finds accounts nothing has asked for since the start; the first within
/// a second more than the same request again. Then the handles and the
/// emails of accounts not asked for yet are taken: no account can be made
/// with one. And `listRepos`' first pages are accounts the harness made,
/// none twice.
async function findSome(what) {
const some = state.accounts.filter((_, i) => i % ONE_IN === 0)
const slow = []
await each(some.map((a, k) => [a, k]), async ([a, k]) => {
const request = { did: a.did, handle: a.handle, after: what }
const all = ways(a)
const turned = [...all.slice(k % all.length), ...all.slice(0, k % all.length)]
const t = Date.now()
for (const [name, ask, found] of turned) {
const res = await ask()
if (!found(res)) await fail(cand, name, { ...request, first: turned[0][0] }, { did: a.did }, res.body)
if (name === turned[0][0]) {
const cold = Date.now() - t
const u = Date.now()
await ask()
if (cold - (Date.now() - u) > COLD_MS) slow.push({ did: a.did, first: name, cold })
}
}
})
if (slow.length) await fail(cand, `a cold account's first request, ${what}`, { accounts: state.accounts.length }, `within ${COLD_MS} ms more than the next`, slow.slice(0, 5))
count('accounts found', true)
// An account's email is one it signs in with: of those with a password.
// Four accounts nothing has asked for since the start, two for each.
const fresh = state.accounts.filter((a, i) => a.password && i % ONE_IN === 1)
const other = { handle: 'scale-other.test', email: 'scale-other@example.com', password: 'scale-other-password' }
for (const body of [{ ...other, handle: fresh[0].handle }, { ...other, email: fresh[1].email }, { ...other, handle: fresh.at(-1).handle }, { ...other, email: fresh.at(-2).email }]) {
const res = await xrpcHttp(cand, 'com.atproto.server.createAccount', { body })
if (res.status !== 400) await fail(cand, "createAccount, its handle or its email an account's", { ...body, password: '<password>', after: what }, 'refused, 400', { status: res.status, body: res.body })
}
count('taken handles and emails refused', true)
const made = new Set(state.accounts.map((a) => a.did))
const listed = new Set()
let cursor
for (let page = 0; page < LIST_PAGES; page++) {
const res = await xrpcHttp(cand, 'com.atproto.sync.listRepos', { params: { limit: LIST_LIMIT, cursor } })
const dids = (res.body?.repos ?? []).map((r) => r.did)
if (res.status !== 200 || dids.length !== LIST_LIMIT || typeof res.body.cursor !== 'string' || dids.some((did) => !made.has(did) || listed.has(did))) {
await fail(cand, 'listRepos', { limit: LIST_LIMIT, page, after: what }, `${LIST_LIMIT} accounts the harness made, none listed before, and a cursor`, res.body)
}
for (const did of dids) listed.add(did)
cursor = res.body.cursor
}
count('listRepos', true)
}
/// Each account's latest commit, all of them touched; and for some, the
/// first request after a start against the second.
async function touchAll(what) {
const slow = []
await each(state.accounts, async (a) => {
const t = Date.now()
const first = await xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did: a.did } })
const cold = Date.now() - t
if (first.status !== 200) await fail(cand, 'getLatestCommit', { did: a.did }, 'the latest commit', first.body)
if (cold > COLD_MS) {
const u = Date.now()
await xrpcHttp(cand, 'com.atproto.sync.getLatestCommit', { params: { did: a.did } })
if (cold - (Date.now() - u) > COLD_MS) slow.push({ did: a.did, cold })
}
})
if (slow.length) await fail(cand, `a cold account's first request, ${what}`, { accounts: state.accounts.length }, `within ${COLD_MS} ms more than the next`, slow.slice(0, 5))
count('cold accounts touched', true)
}
/// The large repository: records of about 120 KB on the first account, by
/// `applyWrites`, each batch under the largest commit (spec/LIMITS.md,
/// "Commits").
async function writeLarge(big) {
const values = new Map()
for (let b = 0; b < BIG_BATCHES; b++) {
const writes = Array.from({ length: BIG_BATCH }, (_, i) => {
const rkey = `big${b * BIG_BATCH + i}`
const value = { $type: 'com.example.scale', n: b * BIG_BATCH + i, text: `${rkey} `.padEnd(BIG_RECORD, 'lorem ipsum dolor sit amet ') }
values.set(rkey, value)
return { $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.scale', rkey, value }
})
const res = await xrpcHttp(cand, 'com.atproto.repo.applyWrites', { body: { repo: big.did, writes }, token: big.token })
if (res.status !== 200) await fail(cand, 'applyWrites of the large repository', { batch: b }, 'the batch written', res.body)
}
count('large repository written', true)
return values
}
/// Much besides a repository, on one account: `BLOBS` blobs uploaded and
/// each referenced by a record, and a space made with `SPACE_RECORDS`
/// records in the account's repo there. Answers the space's URI.
async function writeMuchBesides(rich) {
const call = async (nsid, body, what) => {
const res = await xrpcHttp(cand, nsid, { body, token: rich.token })
if (res.status !== 200) await fail(cand, nsid, { did: rich.did, what }, what, res.body)
return res.body
}
const refs = new Array(BLOBS)
await each(Array.from(refs, (_, i) => i), async (i) => {
const bytes = Buffer.alloc(BLOB_BYTES, `blob ${i} `)
const res = await fetch(`${cand.url}/xrpc/com.atproto.repo.uploadBlob`, { method: 'POST', headers: { authorization: `Bearer ${rich.token}`, 'content-type': 'application/octet-stream' }, body: bytes })
const body = await res.json()
if (res.status !== 200) await fail(cand, 'uploadBlob', { did: rich.did, n: i }, 'the blob', body)
refs[i] = body.blob
})
for (let i = 0; i < BLOBS; i += 100) {
const writes = refs.slice(i, i + 100).map((blob, j) => ({ $type: 'com.atproto.repo.applyWrites#create', collection: 'com.example.scale.blob', rkey: `blob${i + j}`, value: { $type: 'com.example.scale.blob', blob } }))
await call('com.atproto.repo.applyWrites', { repo: rich.did, writes }, 'records referencing the blobs')
}
count('blobs held', true)
const space = (await call('com.atproto.space.createSpace', { spaceType: SPACE_TYPE, skey: 'scale', policy: { $type: 'com.atproto.memberspace.policy' } }, 'the space made')).space
for (let i = 0; i < SPACE_RECORDS; i += SPACE_BATCH) {
const writes = Array.from({ length: SPACE_BATCH }, (_, j) => ({ $type: 'com.atproto.space.applyWrites#create', collection: SPACE_COLLECTION, rkey: `r${i + j}`, value: { $type: SPACE_COLLECTION, n: i + j, text: `${i + j} `.padEnd(SPACE_RECORD_BYTES, 'space ') } }))
await call('com.atproto.space.applyWrites', { space, repo: rich.did, writes }, 'space records')
}
count('space records written', true)
return space
}
/// What the account holds besides its repository, read: a space record,
/// two pages of its space repo and of its blobs, and its space repo whole.
/// Answers how long the first of them took, how long the same request
/// again, and the bytes of the space repo whole.
async function readMuchBesides(rich, space) {
const record = () => xrpcHttp(cand, 'com.atproto.space.getRecord', { params: { space, repo: rich.did, collection: SPACE_COLLECTION, rkey: `r${SPACE_RECORDS - 1}` }, token: rich.token })
const t = Date.now()
const first = await record()
const cold = Date.now() - t
if (first.status !== 200 || first.body?.value?.n !== SPACE_RECORDS - 1) await fail(cand, 'getRecord of the space repo', { did: rich.did }, { n: SPACE_RECORDS - 1 }, first.body)
const u = Date.now()
await record()
const warm = Date.now() - u
for (const [nsid, params, items] of [['com.atproto.space.listRecords', { space, repo: rich.did, collection: SPACE_COLLECTION, limit: 100 }, 'records'], ['com.atproto.sync.listBlobs', { did: rich.did, limit: 500 }, 'cids']]) {
let cursor
for (let page = 0; page < 2; page++) {
const res = await xrpcHttp(cand, nsid, { params: { ...params, cursor }, token: rich.token })
if (res.status !== 200 || res.body?.[items]?.length !== params.limit || typeof res.body.cursor !== 'string') await fail(cand, nsid, { did: rich.did, page }, `${params.limit} ${items} and a cursor`, res.body)
cursor = res.body.cursor
}
}
// The space repo whole, as it is sent: every record's bytes are in it.
const url = new URL(`${cand.url}/xrpc/com.atproto.space.getRepo`)
url.searchParams.set('space', space)
url.searchParams.set('repo', rich.did)
const res = await fetch(url, { headers: { authorization: `Bearer ${rich.token}` } })
let whole = 0
if (res.ok) for await (const part of res.body) whole += part.length
if (!res.ok || whole < SPACE_RECORDS * SPACE_RECORD_BYTES) await fail(cand, 'getRepo of the space repo', { did: rich.did, records: SPACE_RECORDS }, `at least ${SPACE_RECORDS * SPACE_RECORD_BYTES} bytes`, { status: res.status, bytes: whole })
count('much besides read', true)
return { cold, warm, whole }
}
/// A body read as it comes, counting its bytes, and kept.
async function streamed(url) {
const res = await fetch(url)
if (!res.ok) return { status: res.status, bytes: null }
const parts = []
for await (const part of res.body) parts.push(part)
return { status: res.status, bytes: Buffer.concat(parts) }
}
/// Every event from the start, read as fast as they come, until none comes
/// for a second.
function replay() {
return new Promise((done) => {
const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos?cursor=0`)
ws.binaryType = 'arraybuffer'
const seqs = []
let bytes = 0
let quiet
const settle = () => {
clearTimeout(quiet)
quiet = setTimeout(() => (ws.close(), done({ seqs, bytes })), 1000)
}
ws.onmessage = (m) => {
bytes += m.data.byteLength
const [, body] = cborDecodeMulti(new Uint8Array(m.data))
if (body.seq !== undefined) seqs.push(body.seq)
settle()
}
ws.onerror = () => {}
settle()
})
}
/// How long the firehose's first message is waited for before it is given
/// up: long, so that a failure says how late it was.
const FIRST_PATIENCE = 600_000
/// The first message of the firehose from `cursor`: how long it took from
/// asking, and its seq; no time if none came in `FIRST_PATIENCE`.
function firstMessage(cursor) {
return new Promise((done) => {
const at = Date.now()
const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos?cursor=${cursor}`)
ws.binaryType = 'arraybuffer'
const give = (answer) => {
clearTimeout(late)
ws.close()
done(answer)
}
const late = setTimeout(() => give({ ms: null }), FIRST_PATIENCE)
ws.onmessage = (m) => give({ ms: Date.now() - at, seq: cborDecodeMulti(new Uint8Array(m.data))[1]?.seq })
ws.onerror = () => {}
})
}
/// How many syncs to the disk the server has made, as its metrics count
/// them (spec/OPERATIONS.md, `pds_fsync_duration_seconds`).
async function syncs() {
const text = await fetch(metricsUrl).then((r) => r.text())
const m = text.match(/^pds_fsync_duration_seconds_count (\d+)$/m)
if (!m) await fail(cand, 'metrics', { metric: 'pds_fsync_duration_seconds_count' }, 'a count of syncs', 'none')
return Number(m[1])
}
/// A record of each of `accounts` listed, which nothing has asked for since
/// the start that followed `what`, so that each one's state is read from
/// disk: answers how many syncs the server counts meanwhile.
async function firstUses(accounts, what) {
const before = await syncs()
await each(accounts, async (a) => {
const res = await xrpcHttp(cand, 'com.atproto.repo.listRecords', { params: { repo: a.did, collection: 'app.bsky.actor.profile', limit: 1 } })
if (res.status !== 200) await fail(cand, `listRecords, the first request to an account after ${what} and a start`, { did: a.did }, 200, res.body)
})
const made = (await syncs()) - before
console.error(` ${accounts.length} accounts first used after ${what} and a start: ${made} syncs`)
return made
}
/// The seq of the first event `write` makes: a subscription with no cursor
/// is open before it, and its first message is that event's. Answered once
/// the write is answered too: the event may come first, and what follows
/// may kill the server.
function seqOf(write) {
return new Promise((done, failed) => {
const ws = new WebSocket(`${cand.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`)
ws.binaryType = 'arraybuffer'
let written
// Once the event has come the socket is closed, and what it says of its
// closing is nothing to the answer.
let got = false
ws.onopen = () => {
written = write()
written.catch(failed)
}
ws.onmessage = (m) => {
if (got) return
got = true
ws.close()
const seq = cborDecodeMulti(new Uint8Array(m.data))[1]?.seq
written.then(() => done(seq), failed)
}
ws.onerror = () => {
if (!got) failed(new Error('no live subscription'))
}
})
}
/// The run: what spec/STORE.md's "How it is checked" says of it.
export async function scaleRun() {
// The base: just started on an empty data directory.
let t0 = 0
for (let i = 0; i < 3; i++) {
await server.kill()
server.start()
t0 = Math.max(t0, await waitReady(server, cand.url))
}
await wait(300)
state.t0 = t0
state.r0 = rss()
console.error(` started empty: ${t0} ms, ${MiB(state.r0)}`)
// Few accounts, then many: the start, idle memory, and every account touched.
await makeAccounts(FEW)
await restart(`with ${FEW} accounts`)
await makeAccounts(MANY)
await restart(`with ${MANY} accounts`)
await holdPeak(`${MANY} accounts each touched once`, (await sampled(() => touchAll(`${MANY} accounts`))).peak)
// A large repository: written, held, read whole, and replayed.
const big = state.accounts[0]
const { result: values, peak: writing } = await sampled(() => writeLarge(big))
await holdPeak('writing a large repository', writing)
await restart('with a large repository')
const t = Date.now()
const first = await xrpcHttp(cand, 'com.atproto.repo.getRecord', { params: { repo: big.did, collection: 'com.example.scale', rkey: 'big0' } })
const cold = Date.now() - t
const u = Date.now()
await xrpcHttp(cand, 'com.atproto.repo.getRecord', { params: { repo: big.did, collection: 'com.example.scale', rkey: 'big1' } })
if (cold - (Date.now() - u) > COLD_MS) await fail(cand, "the large account's first request after a start", { did: big.did }, `within ${COLD_MS} ms more than the next`, `${cold} ms`)
if (!isDeepStrictEqual(first.body?.value, values.get('big0'))) await fail(cand, 'getRecord of the large repository', { rkey: 'big0' }, values.get('big0').text.slice(0, 40), first.body)
const url = `${cand.url}/xrpc/com.atproto.sync.getRepo?did=${encodeURIComponent(big.did)}`
const { result: repo, peak: reading } = await sampled(() => streamed(url))
if (repo.status !== 200) await fail(cand, 'getRepo of the large repository', { did: big.did }, 'the repository', repo.status)
await holdPeak(`getRepo of ${MiB(repo.bytes.length)}`, reading)
const car = await readCarWithRoot(repo.bytes)
let records = 0
car.blocks.forEach((bytes) => {
const v = cborDecode(bytes)
if (v?.$type === 'com.example.scale') {
records += 1
if (!isDeepStrictEqual(v, values.get(`big${v.n}`))) records = -Infinity
}
})
if (records !== values.size) await fail(cand, 'getRepo of the large repository', { did: big.did }, `every record written, ${values.size}, as written`, records)
const { result: events, peak: replaying } = await sampled(() => replay())
await holdPeak(`the firehose replayed from 0, ${events.seqs.length} events, ${MiB(events.bytes)}`, replaying)
if (events.seqs.length < MANY * 4 + BIG_BATCHES || events.seqs.some((s, i) => i && s <= events.seqs[i - 1])) {
await fail(cand, 'the firehose replayed from 0', {}, `every event, at least ${MANY * 4 + BIG_BATCHES}, in order`, { events: events.seqs.length })
}
count('firehose replayed', true)
// Much besides a repository, on one account: written, held and read
// within the memory bound, and its first request after a start within a
// second of the next.
const rich = state.accounts[2]
const { result: space, peak: holding } = await sampled(() => writeMuchBesides(rich))
await holdPeak(`writing ${BLOBS} blobs and ${SPACE_RECORDS} space records to one account`, holding)
await restart('with an account that holds much besides')
const { result: took, peak: readingBesides } = await sampled(() => readMuchBesides(rich, space))
console.error(` the first request to the account that holds much besides: ${took.cold} ms, and again ${took.warm} ms`)
console.error(` its space repo whole: ${MiB(took.whole)}`)
if (took.cold - took.warm > COLD_MS) await fail(cand, "the first request after a start to an account that holds much besides its repository", { did: rich.did }, `within ${COLD_MS} ms more than the next`, `${took.cold} ms, the next ${took.warm} ms`)
await holdPeak('reading what the account holds besides', readingBesides)
// Most: fifty times as many accounts, some with a password. The start,
// idle memory, and accounts found; then their sessions refreshed, and the
// start and the accounts again.
await makeAccounts(MANY + WITH_PASSWORDS, { passwords: true, atOnce: MOST_AT_ONCE })
await makeAccounts(MOST, { atOnce: MOST_AT_ONCE })
await restart(`with ${MOST} accounts`)
await holdPeak(`one account in ${ONE_IN} of ${MOST} found`, (await sampled(() => findSome(`a start with ${MOST} accounts`))).peak)
await holdPeak(`${WITH_PASSWORDS} accounts' sessions each refreshed ${REFRESHES} times`, (await sampled(refreshAll)).peak)
await restart(`with ${MOST} accounts, sessions refreshed`)
await holdPeak(`one account in ${ONE_IN} of ${MOST} found again`, (await sampled(() => findSome('sessions refreshed and a start'))).peak)
// The firehose after a start, a kill's as a shutdown's: its first
// message within a second, from the first event ever made and from the
// last, a commit made just before. This start followed a kill.
const afterKill = await firstMessage(0)
console.error(` the firehose from 0, after a kill and a start with ${MOST} accounts: its first message ${afterKill.ms === null ? `not in ${FIRST_PATIENCE / 1000} s` : `in ${afterKill.ms} ms`}`)
if (afterKill.ms === null || afterKill.ms > COLD_MS || afterKill.seq !== events.seqs[0]) {
await fail(cand, 'the firehose from 0, after a kill and a start', { accounts: state.accounts.length }, `its first message within ${COLD_MS} ms, the event of seq ${events.seqs[0]}`, afterKill)
}
const writer = state.accounts[1]
const write = (text) => async () => {
const res = await xrpcHttp(cand, 'com.atproto.repo.createRecord', { body: { repo: writer.did, collection: 'com.example.scale', record: { $type: 'com.example.scale', text } }, token: writer.token })
if (res.status !== 200) await fail(cand, `createRecord, ${text}`, { did: writer.did }, 'the record', res.body)
}
const last = await seqOf(write('the last before a shutdown'))
await restart(`with ${MOST} accounts, after a shutdown`, 'SIGTERM')
for (const [what, cursor, seq] of [['0', 0, events.seqs[0]], ['the seq before the last commit', last - 1, last]]) {
const first = await firstMessage(cursor)
console.error(` the firehose from ${what}, after a shutdown and a start with ${MOST} accounts: its first message ${first.ms === null ? `not in ${FIRST_PATIENCE / 1000} s` : `in ${first.ms} ms`}`)
if (first.ms === null || first.ms > COLD_MS || first.seq !== seq) {
await fail(cand, `the firehose from ${what}, after a shutdown and a start`, { accounts: state.accounts.length, cursor }, `its first message within ${COLD_MS} ms, the event of seq ${seq}`, { ms: first.ms, seq: first.seq })
}
}
count('firehose after a shutdown', true)
// An account's first use after a shutdown syncs nothing: accounts nothing
// has asked for since this start, a record of each listed, which reads
// its state from disk, and the server's count of syncs before and after.
// The few allowed are its own housekeeping's.
const clean = await firstUses(state.accounts.slice(-FIRST_USES - 100, -100), 'a shutdown')
if (clean >= FIRST_USES / 10) await fail(cand, 'syncs as accounts are first used after a shutdown and a start', { accounts: FIRST_USES }, `fewer than ${FIRST_USES / 10}: none for an account's first use`, clean)
count('first uses after a shutdown', true)
// A kill loses the firehose nothing of a write: a record written since
// the shutdown, then a kill and a start, and the firehose from the seq
// before that record's commit, the first thing asked of that start, is
// sent it within a second.
const later = await seqOf(write('after the shutdown, before a kill'))
await restart(`with ${MOST} accounts, after a write and a kill`)
const sent = await firstMessage(later - 1)
console.error(` the firehose from the seq before a commit made after the shutdown, after a kill: its first message ${sent.ms === null ? `not in ${FIRST_PATIENCE / 1000} s` : `in ${sent.ms} ms`}`)
if (sent.ms === null || sent.ms > COLD_MS || sent.seq !== later) {
await fail(cand, 'the firehose after a shutdown, a start, a write and a kill', { accounts: state.accounts.length, cursor: later - 1 }, `its first message within ${COLD_MS} ms, the event of seq ${later}`, sent)
}
count('firehose after a kill', true)
// After a kill an account's log is synced as it is first used, and the
// server's count of syncs says so: other accounts nothing has asked for.
const killed = await firstUses(state.accounts.slice(-2 * FIRST_USES - 100, -FIRST_USES - 100), 'a kill')
if (killed < FIRST_USES) await fail(cand, 'syncs as accounts are first used after a kill and a start', { accounts: FIRST_USES }, `at least ${FIRST_USES}: each account's log synced before it is used`, killed)
count('first uses after a kill', true)
return { r0: state.r0, t0, accounts: state.accounts.length, repo: repo.bytes.length, events: events.bytes }
}