~ knotless Search/
↑↓ move↵ open⇧↵ all results
0
0
340 lines · 18.5 KB Raw History
// The performance step (spec/OPERATIONS.md, "Performance"): one fixed
// workload against the reference PDS and against the server, each in a
// process of its own, part by part, each part in three rounds with the sides
// taking turns; it reports each side's best round of each part, its p50, p99
// and throughput, and the server's ratios to the reference's beside the
// baseline's (perf-baseline.json). It fails a part where the server's p99
// ratio is more than 1.5 times the baseline's (or 1, if lower), or its
// throughput ratio below 0.8 times the baseline's (or 1, if higher). Then,
// against the server alone, subscribers come and go, cohort after cohort,
// and it fails if the server then holds more descriptors, threads or memory
// than after the first cohort.
//
//   node perf.mjs -- <server command...>
//
// The server is started as compare.mjs starts it (lib/env.mjs), with no
// rate limits, registering its DIDs with the reference's PLC directory.

import { execFileSync, spawn } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtempSync, openSync, readdirSync, readFileSync } from 'node:fs'
import { createServer } from 'node:net'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { createInterface } from 'node:readline'

/// The factor of the baseline's p99 ratio (or 1, if lower) the server's may
/// be.
const P99_FACTOR = 1.5
/// The factor of the baseline's throughput ratio (or 1, if higher) the
/// server's must reach: run-to-run noise of a few percent, on parts bound by
/// the same work on both sides, would turn a floor of 1 into flakes.
const RATE_FACTOR = 0.8
/// Each part's ratios, the server's best over the reference's, as nightly
/// runs on the runner measured them, the weakest of several.
const BASELINE = JSON.parse(readFileSync(new URL('./perf-baseline.json', import.meta.url), 'utf8')).parts
/// Subscribers come and go: cohorts of this many, after one to warm.
const COHORT = 20
const COHORTS = 20
/// How much more memory the server may hold after the last cohort than
/// after the first: what the allocator keeps of what it freed.
const MEMORY_SLACK = 8 * 1048576
/// How long the server is left alone after the first cohort before what it
/// holds is read: idle threads the parts before used end after 10 s, and
/// what they held goes back with them.
const IDLE_MS = 15000
/// How long the server has, after the last cohort, to give back what it held.
const SETTLE_MS = 20000

// The reference, in a process of its own: started by the run, it says where
// it listens on its first line, and stops when its input closes.
if (process.argv[2] === '--reference') {
  const { TestNetworkNoAppView } = await import('@atproto/dev-env')
  const net = await TestNetworkNoAppView.create({ pds: { hostname: 'localhost', rateLimitsEnabled: false, inviteRequired: false } })
  process.stdout.write(`${JSON.stringify({ url: net.pds.url, plcUrl: net.plc.url, recoveryKey: net.pds.ctx.cfg.identity.recoveryDidKey })}\n`)
  process.stdin.resume()
  process.stdin.on('end', async () => {
    await net.close()
    process.exit(0)
  })
} else await run()

async function run() {
  const { command } = await import('./lib/args.mjs')
  const { below } = await import('./lib/random.mjs')
  const { serverEnv } = await import('./lib/env.mjs')
  const { cborDecodeMulti } = await import('@atproto/common')

  const freePort = () => new Promise((ok) => {
    const s = createServer()
    s.listen(0, '127.0.0.1', () => {
      const { port } = s.address()
      s.close(() => ok(port))
    })
  })

  const refProc = spawn(process.execPath, [new URL(import.meta.url).pathname, '--reference'], { stdio: ['pipe', 'pipe', 'ignore'] })
  const first = await new Promise((ok) => createInterface({ input: refProc.stdout }).once('line', ok))
  const { url: refUrl, plcUrl, recoveryKey } = JSON.parse(first)

  const port = await freePort()
  const dataDir = mkdtempSync(join(tmpdir(), 'pds-perf-'))
  const logPath = join(mkdtempSync(join(tmpdir(), 'pds-perf-log-')), 'server.log')
  const { env } = await serverEnv({ port, dataDir, plc: plcUrl, recoveryKey })
  const log = openSync(logPath, 'a')
  const serverProc = spawn(command[0], command.slice(1), { env: { ...process.env, ...env }, stdio: ['ignore', log, log] })
  const serverUrl = `http://127.0.0.1:${port}`
  const stop = (code) => {
    serverProc.kill('SIGKILL')
    refProc.stdin.end()
    process.exit(code)
  }
  for (const event of ['uncaughtException', 'unhandledRejection'])
    process.on(event, (err) => {
      console.error(err)
      stop(1)
    })
  for (const started = Date.now(); ; ) {
    try {
      if ((await fetch(`${serverUrl}/xrpc/com.atproto.server.describeServer`)).ok) break
    } catch {}
    if (serverProc.exitCode !== null || Date.now() - started > 30000) {
      console.error(`the server did not start; its log is ${logPath}`)
      stop(1)
    }
    await new Promise((ok) => setTimeout(ok, 20))
  }

  const sides = [
    { name: 'reference', url: refUrl, accounts: [], records: [] },
    { name: 'server', url: serverUrl, accounts: [], records: [] },
  ]

  /// One request, timed from sending it to having its answer whole;
  /// anything but a 2xx stops the run.
  async function call(side, nsid, { params, body, bytes, token, type } = {}) {
    const url = new URL(`${side.url}/xrpc/${nsid}`)
    for (const [k, v] of Object.entries(params ?? {})) url.searchParams.set(k, v)
    const headers = {}
    if (token) headers.authorization = `Bearer ${token}`
    if (body !== undefined) headers['content-type'] = 'application/json'
    if (bytes) headers['content-type'] = type
    const t = performance.now()
    const r = await fetch(url, { method: body !== undefined || bytes ? 'POST' : 'GET', headers, body: bytes ?? (body === undefined ? undefined : JSON.stringify(body)) })
    const buf = Buffer.from(await r.arrayBuffer())
    const ms = performance.now() - t
    if (!r.ok) {
      console.error(`${side.name}: ${nsid} answered ${r.status}: ${buf.toString('utf8').slice(0, 300)}${side.name === 'server' ? `\nserver log: ${logPath}` : ''}`)
      stop(1)
    }
    return { ms, json: () => JSON.parse(buf.toString('utf8')) }
  }

  /// Requests `lo` to `hi`, `at` at once, each made by `make(i)`: their
  /// times, and how long they took together.
  async function load(lo, hi, at, make) {
    const times = []
    let next = lo
    const t = performance.now()
    await Promise.all(Array.from({ length: at }, async () => {
      while (next < hi) times.push(await make(next++))
    }))
    return { times, total: performance.now() - t }
  }

  const ACCOUNTS = 40
  /// Each part: how many requests, and a round of them, `lo` to `hi`, on a
  /// side.
  const PARTS = {
    createAccount: [ACCOUNTS, (side, lo, hi) => load(lo, hi, 4, async (i) => {
      const r = await call(side, 'com.atproto.server.createAccount', { body: { handle: `perf${i}.test`, email: `perf${i}@example.com`, password: `perf-password-${i}` } })
      const { did, accessJwt } = r.json()
      side.accounts[i] = { did, token: accessJwt }
      return r.ms
    })],
    createRecord: [2000, (side, lo, hi) => load(lo, hi, 16, async (i) => {
      const a = side.accounts[i % ACCOUNTS]
      const r = await call(side, 'com.atproto.repo.createRecord', { body: { repo: a.did, collection: 'com.example.note', record: { $type: 'com.example.note', text: `a note of about a hundred bytes, number ${i}, to read back later`, n: i } }, token: a.token })
      side.records[i] = { repo: a.did, rkey: r.json().uri.split('/').pop() }
      return r.ms
    })],
    getRecord: [4000, (side, lo, hi, picks) => load(lo, hi, 16, async (i) => (await call(side, 'com.atproto.repo.getRecord', { params: { repo: side.records[picks[i]].repo, collection: 'com.example.note', rkey: side.records[picks[i]].rkey } })).ms)],
    getRepo: [200, (side, lo, hi) => load(lo, hi, 8, async (i) => (await call(side, 'com.atproto.sync.getRepo', { params: { did: side.accounts[i % ACCOUNTS].did } })).ms)],
    firehose: [200, async (side, lo, hi) => {
      // Each from an address of its own, as 20 clients would be: the server
      // holds 16 at most from one (spec/LIMITS.md).
      const subs = await Promise.all(Array.from({ length: 20 }, (_, n) => new Promise((ok, fail) => {
        const ws = new WebSocket(`${side.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`, { headers: { 'x-forwarded-for': `198.18.0.${n + 1}` } })
        ws.binaryType = 'arraybuffer'
        const sub = { ws, got: new Map() }
        ws.onmessage = (m) => {
          const [header, body] = cborDecodeMulti(new Uint8Array(m.data))
          if (header?.t !== '#commit') return
          for (const op of body.ops ?? []) {
            const k = /\/fh(\d+)x$/.exec(op.path)
            if (k && !sub.got.has(Number(k[1]))) sub.got.set(Number(k[1]), performance.now())
          }
        }
        ws.onopen = () => ok(sub)
        ws.onerror = () => fail(new Error(`${side.name}: subscribeRepos`))
      })))
      const sentAt = []
      const a = side.accounts[0]
      const t = performance.now()
      for (let i = lo; i < hi; i++) {
        sentAt[i] = performance.now()
        await call(side, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `fh${i}x`, record: { $type: 'com.example.note', text: 'fan-out', n: i } }, token: a.token })
      }
      for (let w = 0; w < 100 && subs.some((s) => s.got.size < hi - lo); w++) await new Promise((ok) => setTimeout(ok, 100))
      for (const s of subs) s.ws.close()
      const times = []
      let last = t
      for (const s of subs) for (let i = lo; i < hi; i++) {
        const at = s.got.get(i)
        if (at === undefined) {
          console.error(`${side.name}: a subscriber did not get the commit of write ${i} within 10 s`)
          stop(1)
        }
        times.push(at - sentAt[i])
        last = Math.max(last, at)
      }
      return { times, total: last - t }
    }],
    uploadBlob: [200, (side, lo, hi) => {
      const blob = (i) => {
        const out = Buffer.alloc(65536)
        for (let o = 0; o < out.length; o += 32) createHash('sha256').update(`blob ${i} ${o}`).digest().copy(out, o)
        return out
      }
      return load(lo, hi, 8, async (i) => (await call(side, 'com.atproto.repo.uploadBlob', { bytes: blob(i), type: 'application/octet-stream', token: side.accounts[i % ACCOUNTS].token })).ms)
    }],
  }

  const pct = (xs, p) => {
    const s = [...xs].sort((x, y) => x - y)
    return s[Math.min(s.length - 1, Math.ceil((p / 100) * s.length) - 1)]
  }
  // Each part in three rounds, the sides taking turns; each side judged by
  // its best round, as other work on the machine only ever slows a round.
  const ROUNDS = 3
  const rows = []
  const failed = []
  for (const [part, [n, fn]] of Object.entries(PARTS)) {
    const picks = part === 'getRecord' ? Array.from({ length: n }, () => below(2000)) : null
    const rounds = { reference: [], server: [] }
    for (let k = 0; k < ROUNDS; k++) {
      const [lo, hi] = [Math.floor((n * k) / ROUNDS), Math.floor((n * (k + 1)) / ROUNDS)]
      for (const side of sides) {
        const { times, total } = await fn(side, lo, hi, picks)
        rounds[side.name].push({ p50: pct(times, 50), p99: pct(times, 99), rate: (times.length / total) * 1000 })
      }
    }
    const best = (rs) => ({ p50: Math.min(...rs.map((x) => x.p50)), p99: Math.min(...rs.map((x) => x.p99)), rate: Math.max(...rs.map((x) => x.rate)) })
    const [r, s] = [best(rounds.reference), best(rounds.server)]
    // The ratios, held to the baseline's, and never to less than the
    // reference's own.
    const got = { p99: s.p99 / r.p99, rate: s.rate / r.rate }
    const base = BASELINE[part] ?? { p99: 1, rate: 1 }
    const [p99Of, rateOf] = [Math.min(base.p99, 1), Math.max(base.rate, 1)]
    const why = []
    if (got.p99 > P99_FACTOR * p99Of) why.push(`p99 ratio ${sig(got.p99)} (${s.p99.toFixed(2)} ms to the reference's ${r.p99.toFixed(2)} ms), over ${P99_FACTOR} × ${sig(p99Of)}`)
    if (got.rate < RATE_FACTOR * rateOf) why.push(`throughput ratio ${sig(got.rate)} (${s.rate.toFixed(0)}/s to the reference's ${r.rate.toFixed(0)}/s), below ${RATE_FACTOR} × ${sig(rateOf)}`)
    if (why.length) failed.push(`${part}: ${why.join('; ')}`)
    rows.push({ part, r, s, verdict: why.length ? 'FAIL' : 'ok', rounds, got, base })
  }

  const f = (x) => x.toFixed(1).padStart(8)
  console.error(`each side's best of ${ROUNDS} rounds, the sides taking turns, and the server's ratios to the reference's (the baseline's):\n`)
  console.error(`${'part'.padEnd(14)}${'reference p50'.padStart(14)}${'p99'.padStart(9)}${'per s'.padStart(9)}   ${'server p50'.padStart(11)}${'p99'.padStart(9)}${'per s'.padStart(9)}   ${'p99 ratio'.padStart(18)}${'per s ratio'.padStart(18)}`)
  for (const { part, r, s, verdict, got, base } of rows) console.error(`${part.padEnd(14)}${f(r.p50).padStart(14)}${f(r.p99).padStart(9)}${r.rate.toFixed(0).padStart(9)}   ${f(s.p50).padStart(11)}${f(s.p99).padStart(9)}${s.rate.toFixed(0).padStart(9)}   ${`${sig(got.p99)} (${sig(base.p99)})`.padStart(18)}${`${sig(got.rate)} (${sig(base.rate)})`.padStart(18)}   ${verdict}`)
  console.error('\neach round\'s p99, in ms:')
  for (const { part, rounds } of rows) console.error(`${part.padEnd(14)}reference ${rounds.reference.map((x) => x.p99.toFixed(1)).join(', ')}; server ${rounds.server.map((x) => x.p99.toFixed(1)).join(', ')}`)
  console.error(`\nthe ratios, as perf-baseline.json holds them: ${JSON.stringify(Object.fromEntries(rows.map(({ part, got }) => [part, { p99: sig(got.p99), rate: sig(got.rate) }])))}`)

  // Subscribers come and go, against the server alone: what it holds for
  // a cohort of subscribers it gives back once they have left.
  const server = sides[1]
  const sleep = (ms) => new Promise((ok) => setTimeout(ok, ms))
  let cohorts = 0
  async function cohort() {
    const k = cohorts++
    // Each from an address of its own, the same 20 each time: the server
    // holds 16 at most from one (spec/LIMITS.md).
    const subs = await Promise.all(Array.from({ length: COHORT }, (_, n) => new Promise((ok, fail) => {
      const ws = new WebSocket(`${server.url.replace(/^http/, 'ws')}/xrpc/com.atproto.sync.subscribeRepos`, { headers: { 'x-forwarded-for': `198.18.1.${n + 1}` } })
      ws.binaryType = 'arraybuffer'
      const sub = { ws, got: false, closed: new Promise((c) => ws.addEventListener('close', c)) }
      ws.onmessage = (m) => {
        const [header, body] = cborDecodeMulti(new Uint8Array(m.data))
        if (header?.t === '#commit' && (body.ops ?? []).some((op) => op.path === `com.example.note/churn${k}`)) sub.got = true
      }
      ws.onopen = () => ok(sub)
      ws.onerror = () => fail(new Error('server: subscribeRepos'))
    })))
    const a = server.accounts[0]
    await call(server, 'com.atproto.repo.putRecord', { body: { repo: a.did, collection: 'com.example.note', rkey: `churn${k}`, record: { $type: 'com.example.note', text: 'come and go', n: k } }, token: a.token })
    for (const t = Date.now(); subs.some((s) => !s.got); await sleep(5))
      if (Date.now() - t > 10000) {
        console.error(`server: a subscriber of cohort ${k} did not get the commit of its write within 10 s`)
        stop(1)
      }
    for (const s of subs) s.ws.close()
    if (!(await Promise.race([Promise.all(subs.map((s) => s.closed)).then(() => true), sleep(10000).then(() => false)]))) {
      console.error(`server: a subscriber of cohort ${k} did not see its connection closed within 10 s of closing it`)
      stop(1)
    }
  }
  /// What the server process holds: its open file descriptors, threads and
  /// memory, resident or swapped out (Linux), or its footprint, resident or
  /// compressed (macOS), in bytes.
  const pid = String(serverProc.pid)
  const footprintPath = join(mkdtempSync(join(tmpdir(), 'pds-perf-footprint-')), 'footprint.json')
  function holds() {
    if (process.platform === 'linux') {
      const status = readFileSync(`/proc/${pid}/status`, 'utf8')
      const field = (name) => Number(new RegExp(`^${name}:\\s+(\\d+)`, 'm').exec(status)?.[1] ?? 0)
      return { fds: readdirSync(`/proc/${pid}/fd`).length, threads: field('Threads'), memory: (field('VmRSS') + field('VmSwap')) * 1024 }
    }
    const out = (cmd, ...args) => execFileSync(cmd, args, { encoding: 'utf8' })
    out('footprint', '-p', pid, '-j', footprintPath)
    const fp = JSON.parse(readFileSync(footprintPath, 'utf8'))
    return {
      fds: out('lsof', '-n', '-P', '-a', '-p', pid, '-d', '0-1000000', '-F', 'f').split('\n').filter((l) => l.startsWith('f')).length,
      threads: out('ps', '-M', '-p', pid).trim().split('\n').length - 1,
      memory: fp.processes[0].footprint * fp['bytes per unit'],
    }
  }
  await cohort()
  await sleep(IDLE_MS)
  const warmed = holds()
  for (let c = 0; c < COHORTS; c++) await cohort()
  const within = (h) => h.fds <= warmed.fds && h.threads <= warmed.threads && h.memory <= warmed.memory + MEMORY_SLACK
  // The least of each reading: what is held for good is in every one, and
  // what the server starts and ends on its own in only some.
  const least = (a, b) => ({ fds: Math.min(a.fds, b.fds), threads: Math.min(a.threads, b.threads), memory: Math.min(a.memory, b.memory) })
  const settling = Date.now()
  let left = holds()
  while (!within(left) && Date.now() - settling < SETTLE_MS) {
    await sleep(250)
    left = least(left, holds())
  }
  const settled = ((Date.now() - settling) / 1000).toFixed(1)
  const mib = (x) => `${(x / 1048576).toFixed(1)} MiB`
  const held = [
    ['descriptors', warmed.fds, left.fds, left.fds <= warmed.fds],
    ['threads', warmed.threads, left.threads, left.threads <= warmed.threads],
    ['memory', mib(warmed.memory), `${mib(left.memory)} (at most ${mib(MEMORY_SLACK)} more)`, left.memory <= warmed.memory + MEMORY_SLACK],
  ]
  console.error(`\nsubscribers come and go, ${COHORTS} cohorts of ${COHORT} after one to warm; what the server holds ${IDLE_MS / 1000} s after the first, and ${settled} s after the last:\n`)
  for (const [what, a, b, ok] of held) console.error(`${what.padEnd(14)}${String(a).padStart(12)}${String(b).padStart(36)}   ${ok ? 'ok' : 'FAIL'}`)
  for (const [what, a, b, ok] of held) if (!ok) failed.push(`${what}: ${b} after the last cohort, ${a} after the first`)

  if (failed.length) console.error(`\nthe server is slower, or holds more, than spec/OPERATIONS.md, "Performance", allows:\n  ${failed.join('\n  ')}`)
  else console.error(`\nall ${rows.length} parts within the baseline's and the reference's (p99 ratio at most ${P99_FACTOR} ×, throughput ratio at least ${RATE_FACTOR} ×), and what ${COHORTS * COHORT} subscribers held given back`)
  stop(failed.length ? 1 : 0)
}

/// Three significant figures.
function sig(x) {
  return Number(x.toPrecision(3))
}