chadtmiller.com/knotless

// Runs the scenario tracks under harness/scenarios/tracks: each is a day of
// ordinary git work split into steps, and each step is a fresh Claude
// session that can use only the knotless MCP server, logged in as that
// step's account. The steps of a track run in order, each told what the
// earlier ones handed off; tracks run side by side, each with its own
// accounts, against a fresh reference PDS and appview.
//
//   node harness/scenarios.mjs [track ...]
//
// Tracks marked `<!-- needs: spaces -->` run on the spaces alpha build named
// by KNOTLESS_SPACES_DEV_ENV, the rest on the npm reference PDS. Each run
// writes transcripts, report.json and report.md under
// target/scenarios/<time>/. SCENARIO_MODEL picks the model, and
// SCENARIO_TIMEOUT_SECS bounds each step (default 1200).

import { spawn } from 'node:child_process'
import { createServer } from 'node:net'
import { mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { createInterface } from 'node:readline'

const root = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const here = join(root, 'harness', 'scenarios')
const bin = join(root, 'target', 'debug')
const timeout = Number(process.env.SCENARIO_TIMEOUT_SECS ?? 1200) * 1000
const adminPassword = 'scenario-admin'

/** A track: its steps, each a role and what it is asked to do. */
function track(name) {
  const text = readFileSync(join(here, 'tracks', `${name}.md`), 'utf8')
  const [intro, ...parts] = text.split(/^## /m)
  const steps = parts.map((part) => {
    const [role, ...rest] = part.split('\n')
    return { role: role.trim(), task: rest.join('\n').trim() }
  })
  return {
    name,
    spaces: intro.includes('<!-- needs: spaces -->'),
    about: intro.replace(/<!--.*?-->/g, '').trim(),
    steps,
    roles: [...new Set(steps.map((s) => s.role))],
  }
}

const handle = (t, role) => `${role}-${t.name}.test`

/** Fills `{role}` with that role's handle in this track. */
const fill = (t, text) =>
  text.replace(/\{([a-z]+)\}/g, (m, role) =>
    t.roles.includes(role) ? handle(t, role) : m,
  )

const freePort = () =>
  new Promise((ok) => {
    const s = createServer()
    s.listen(0, '127.0.0.1', () => {
      const { port } = s.address()
      s.close(() => ok(port))
    })
  })

/** A reference PDS with every track's accounts, and an appview on it. */
async function world(tracks, spaces, dir) {
  const names = tracks.flatMap((t) =>
    t.roles.map((r) => handle(t, r).replace(/\.test$/, '')),
  )
  const env = { ...process.env }
  delete env.KNOTLESS_DEV_ENV
  if (spaces) {
    env.KNOTLESS_DEV_ENV =
      process.env.KNOTLESS_SPACES_DEV_ENV ?? '../atproto-space-ref/packages/dev-env'
  }
  const pds = spawn('node', [join(root, 'harness', 'pds.mjs'), '--accounts', names.join(',')], {
    cwd: root,
    env,
    stdio: ['pipe', 'pipe', openSync(join(dir, 'pds.log'), 'w')],
  })
  const line = await new Promise((ok, fail) => {
    createInterface({ input: pds.stdout }).once('line', ok)
    pds.once('exit', (code) => fail(new Error(`pds.mjs exited ${code}`)))
  })
  const net = JSON.parse(line)
  if (spaces && !net.spaces) throw new Error('the spaces alpha is not running')
  const port = await freePort()
  const appview = `http://localhost:${port}`
  const serve = spawn(
    join(bin, 'knotless-serve'),
    [
      '--listen', `127.0.0.1:${port}`,
      '--index', join(dir, 'index.db'),
      '--firehose', net.pds.replace('http://', 'ws://'),
      '--public-url', appview,
    ],
    {
      env: {
        ...process.env,
        ...common(net),
        KNOTLESS_CACHE_DIR: join(dir, 'appview-cache'),
        KNOTLESS_CONFIG_DIR: join(dir, 'appview-config'),
        KNOTLESS_ADMIN_PASSWORD: adminPassword,
      },
      stdio: ['ignore', 'ignore', openSync(join(dir, 'appview.log'), 'w')],
    },
  )
  for (let i = 0; ; i++) {
    const up = await fetch(`${appview}/xrpc/dev.knotless.listRepos`).then(
      (r) => r.ok,
      () => false,
    )
    if (up) break
    if (i > 100) throw new Error('the appview did not start')
    await new Promise((ok) => setTimeout(ok, 100))
  }
  return {
    net,
    appview,
    close: () => {
      serve.kill()
      pds.stdin.end()
    },
  }
}

const common = (net) => ({
  KNOTLESS_HANDLE_RESOLVER: net.pds,
  KNOTLESS_PLC_URL: net.plc,
  KNOTLESS_DID_WEB_SCHEME: 'http',
  KNOTLESS_COMPACT: 'off',
})

/** Runs one step as a fresh Claude session; its transcript goes to `out`. */
async function step(t, i, w, dir, handoffs) {
  const { role, task } = t.steps[i]
  const me = handle(t, role)
  const account = w.net.accounts.find((a) => a.handle === me)
  const home = join(dir, 'people', role)
  const config = join(dir, `${i + 1}-${role}.mcp.json`)
  writeFileSync(
    config,
    JSON.stringify({
      mcpServers: {
        knotless: {
          command: join(bin, 'knotless-mcp'),
          env: {
            ...common(w.net),
            PATH: `${bin}:${process.env.PATH}`,
            KNOTLESS_IDENTIFIER: me,
            KNOTLESS_PASSWORD: account.password,
            KNOTLESS_APPVIEW: w.appview,
            KNOTLESS_CONFIG_DIR: join(home, 'config'),
            KNOTLESS_CACHE_DIR: join(home, 'cache'),
            GIT_AUTHOR_NAME: role,
            GIT_AUTHOR_EMAIL: `${role}@example.com`,
            GIT_COMMITTER_NAME: role,
            GIT_COMMITTER_EMAIL: `${role}@example.com`,
            GIT_CONFIG_GLOBAL: '/dev/null',
          },
        },
      },
    }),
  )
  const others = t.roles.filter((r) => r !== role).map((r) => handle(t, r))
  let prompt = readFileSync(join(here, 'brief.md'), 'utf8')
    .replaceAll('{me}', me)
    .replaceAll('{others}', others.length ? others.join(', ') : 'nobody else today')
  prompt += `\n## Your part today\n\n${fill(t, task)}\n`
  if (handoffs.length) {
    prompt += '\n## What the others handed on\n\n'
    prompt += handoffs.map((h) => `- ${h.who}: ${h.handoff}`).join('\n') + '\n'
  }
  const args = [
    '-p',
    '--output-format', 'stream-json',
    '--verbose',
    '--no-session-persistence',
    '--strict-mcp-config',
    '--mcp-config', config,
    '--tools', '',
    '--allowedTools', 'mcp__knotless',
  ]
  if (process.env.SCENARIO_MODEL) args.push('--model', process.env.SCENARIO_MODEL)
  // Outside the repository, so no project instructions reach the session.
  const cwd = mkdtempSync(join(tmpdir(), 'knotless-scenario-'))
  const claude = spawn('claude', args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] })
  claude.stdin.end(prompt)
  const lines = []
  createInterface({ input: claude.stdout }).on('line', (l) => lines.push(l))
  let stderr = ''
  claude.stderr.on('data', (d) => (stderr += d))
  const timer = setTimeout(() => claude.kill(), timeout)
  const code = await new Promise((ok) => claude.on('close', ok))
  clearTimeout(timer)
  writeFileSync(join(dir, `${i + 1}-${role}.jsonl`), lines.join('\n') + '\n')
  writeFileSync(join(dir, `${i + 1}-${role}.prompt.md`), prompt)
  return { role, me, code, stderr, ...read(lines) }
}

/** What a transcript says: the calls made, those that failed, and the
 * report at the end of the reply. */
function read(lines) {
  const events = lines.flatMap((l) => {
    try {
      return [JSON.parse(l)]
    } catch {
      return []
    }
  })
  const calls = []
  const names = new Map()
  for (const e of events) {
    for (const c of e.message?.content ?? []) {
      if (c.type === 'tool_use') names.set(c.id, c.name.replace(/^mcp__knotless__/, ''))
      if (c.type === 'tool_result') {
        const text = [c.content].flat().map((x) => x?.text ?? x).join('')
        calls.push({ tool: names.get(c.tool_use_id), error: !!c.is_error, text })
      }
    }
  }
  const result = events.findLast((e) => e.type === 'result') ?? {}
  const blocks = [...(result.result ?? '').matchAll(/```json\s*([\s\S]*?)```/g)]
  let report = { done: false, summary: 'no report', handoff: '', tripped: [] }
  try {
    report = { ...report, ...JSON.parse(blocks.at(-1)[1]) }
  } catch {}
  return {
    calls,
    errors: calls.filter((c) => c.error),
    turns: result.num_turns,
    cost: result.total_cost_usd ?? 0,
    report,
  }
}

async function run(t, w, dir) {
  const out = join(dir, t.name)
  mkdirSync(out, { recursive: true })
  const steps = []
  for (let i = 0; i < t.steps.length; i++) {
    const handoffs = steps.map((s) => ({ who: s.me, handoff: s.report.handoff }))
    const s = await step(t, i, w, out, handoffs)
    console.log(
      `${t.name} ${i + 1}/${t.steps.length} ${s.role}: ${s.report.done ? 'done' : 'NOT DONE'}, ` +
        `${s.calls.length} calls, ${s.errors.length} errors, ${s.report.tripped.length} tripped`,
    )
    steps.push(s)
  }
  return { track: t.name, about: t.about, steps }
}

function markdown(results, when) {
  let md = `# Scenarios, ${when}\n\n`
  md += '| track | step | done | calls | errors | tripped | cost |\n|---|---|---|---|---|---|---|\n'
  for (const r of results) {
    r.steps.forEach((s, i) => {
      md += `| ${r.track} | ${i + 1} ${s.role} | ${s.report.done ? 'yes' : '**no**'} | ${s.calls.length} | ${s.errors.length} | ${s.report.tripped.length} | $${s.cost.toFixed(2)} |\n`
    })
  }
  for (const r of results) {
    md += `\n## ${r.track}\n\n${r.about}\n`
    r.steps.forEach((s, i) => {
      md += `\n### ${i + 1}. ${s.role} (${s.me})\n\n${s.report.summary}\n`
      if (s.code !== 0) md += `\nThe session exited ${s.code}: ${s.stderr.trim().slice(0, 500)}\n`
      for (const tr of s.report.tripped) {
        md += `\n- **${tr.tool}**: ${tr.what}\n  - expected: ${tr.expected}\n  - workaround: ${tr.workaround}\n`
      }
      for (const e of s.errors) {
        md += `\n- error from \`${e.tool}\`: ${e.text.replace(/\s+/g, ' ').slice(0, 300)}\n`
      }
    })
  }
  return md
}

const all = readdirSync(join(here, 'tracks'))
  .filter((f) => f.endsWith('.md'))
  .map((f) => f.slice(0, -3))
  .sort()
const chosen = process.argv.slice(2).length ? process.argv.slice(2) : all
const tracks = chosen.map(track)
const when = new Date().toISOString().replace(/[:.]/g, '-')
const dir = join(root, 'target', 'scenarios', when)
mkdirSync(dir, { recursive: true })

const groups = [false, true]
  .map((spaces) => ({ spaces, tracks: tracks.filter((t) => t.spaces === spaces) }))
  .filter((g) => g.tracks.length)
const results = (
  await Promise.all(
    groups.map(async (g) => {
      const wdir = join(dir, g.spaces ? 'spaces' : 'stock')
      mkdirSync(wdir, { recursive: true })
      const w = await world(g.tracks, g.spaces, wdir)
      try {
        return await Promise.all(g.tracks.map((t) => run(t, w, dir)))
      } finally {
        w.close()
      }
    }),
  )
).flat()
results.sort((a, b) => a.track.localeCompare(b.track))
writeFileSync(join(dir, 'report.json'), JSON.stringify(results, null, 2))
writeFileSync(join(dir, 'report.md'), markdown(results, when))
console.log(`\n${join(dir, 'report.md')}`)
process.exit(0)