// Coverage over the stock reference PDS and the spaces alpha together,
// failing under 100% of lines or functions: what `just cov` runs.
//
//   node harness/cov.mjs <spaces dev-env> [ignore regex]
//
// What it ignores is the Justfile's `ignore` unless given: tests, the
// harness, examples and the standard library.
//
// The tests are built once, instrumented, and each test binary is run
// against both PDSes, several at a time, rather than one after another
// as `cargo test` would: one harness for each PDS, shared by every binary
// run against it, as `just pds` is. The longest runs, as the last run
// timed them, start first. A binary that fails is run once more: passing
// then, it is reported FLAKY, with its first failure, and passes.
// Failing again only in tests harness/quarantine.json names, until the
// day it names, it is reported QUARANTINED, test by test, and passes too:
// a test known to fail now and then while it is fixed fails no change.
// Each is then run alone until it passes, three times at most, so that
// the lines only it reaches still count.
//
// The report reads this build's binaries alone, and the profiles of their
// runs: never another build's, which a shared target may still hold.
//
// COV_STORE names a directory to keep passing runs in, as CI's test
// workflow does: a run is skipped when one of the same test binary, byte
// for byte, passed against the same harness, with every workspace binary
// it ran the same too, and its profile is reused (see `stored` below).
// Should the gate then fall short, the stored runs are made again, fresh,
// and the gate read again: a line only some runs reach is not lost to a
// store that kept one that missed it.
// COV_STORE_KEEP is how many runs it keeps, newest used first.
//
// A run that goes on past its limit, three times its last time and never
// under ten minutes, or COV_TIMEOUT seconds, is killed and fails without
// a second run: a test that hangs fails the check in minutes, and the log
// names it ("has been running for over 60 seconds"). The floor is wide as
// a busy runner is slow: spaces forge takes 70s on a quiet one and near
// 300s when many runs share it.
//
// COV_NETS=stock runs against the stock PDS alone: the spaces alpha's
// runs count only as stored. Should the gate then fall short, the runs it
// left are made too, against the alpha, and the gate holds. CI runs both
// at once instead: a change that reaches a test binary almost always falls
// short without the alpha, and the two one after the other take longer.
//
// COV_STOCK=a,b runs against the stock PDS only the test binaries named,
// and every one against the alpha. The alpha is the stock PDS with spaces
// added, so the stock PDS reaches nothing more but knotless's handling of
// a PDS without spaces, which only those binaries' tests reach. A line
// that falls short with it names a binary to add.
//
// COV_JOBS sets how many run at once; COV_CLEAN=1 builds every workspace
// crate again first; COV_ONLY=a,b runs only the test binaries named, to
// try something quickly (failing the gate); COV_TOOLCHAIN names the
// toolchain, stable unless it says otherwise, as CI's image's is named for
// its version.

import { execFileSync, spawn, spawnSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import {
  copyFileSync,
  cpSync,
  createReadStream,
  existsSync,
  linkSync,
  mkdirSync,
  openSync,
  readdirSync,
  readFileSync,
  readSync,
  closeSync,
  renameSync,
  rmSync,
  statSync,
  utimesSync,
  writeFileSync,
} from 'node:fs'
import { availableParallelism } from 'node:os'
import { basename, dirname, join, relative, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'

const root = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const [spaces, ignore = '(tests/|harness/|examples/|rustlib/)'] = process.argv.slice(2)
const toolchain = process.env.COV_TOOLCHAIN || 'stable'
const cargo = ['run', toolchain, 'cargo']
const jobs = Number(process.env.COV_JOBS) || Math.max(2, Math.floor(availableParallelism() / 2))
const only = process.env.COV_ONLY?.split(',')
const store = process.env.COV_STORE && resolve(process.env.COV_STORE)
const netsRun = process.env.COV_NETS ? process.env.COV_NETS.split(',') : ['stock', 'spaces']
const partial = netsRun.length < 2
const keep = Number(process.env.COV_STORE_KEEP) || 300
const started = Date.now()
const took = () => `${((Date.now() - started) / 1000).toFixed(0)}s`

/// Runs `rustup run <toolchain> cargo <args>`, its output shown, failing with it.
const run = (args, env = process.env, opts = {}) => {
  const r = spawnSync('rustup', [...cargo, ...args], { cwd: root, env, stdio: 'inherit', ...opts })
  if (r.status !== 0) process.exit(r.status ?? 1)
  return r
}

// What cargo-llvm-cov sets to build and run instrumented, in its own
// target directory, as `cargo llvm-cov` uses.
const target = join(root, 'target/llvm-cov-target')
const shown = spawnSync('rustup', [...cargo, 'llvm-cov', 'show-env', '--sh'], {
  cwd: root,
  env: { ...process.env, CARGO_TARGET_DIR: target },
  encoding: 'utf8',
})
const env = { ...process.env, CARGO_TARGET_DIR: target }
for (const [, k, v] of shown.stdout.matchAll(/^export ([A-Z_0-9]+)=(.*)$/gm))
  env[k] = v.replace(/^'(.*)'$/, '$1')

// cargo builds as many crates at once as the host has cores, even in a
// container held to fewer CPUs and less memory than the host has, as a
// runner's --cpus and --memory hold it: on an 8-core host, in 4 CPUs and
// 7 GiB, eight rustc at once, knotless-serve's near 3 GiB among them,
// were killed for memory, and later two at once, as its lib and its tests
// grew past that. So where the cgroup limits it, the build takes as many
// at once as the CPUs it is given, and one for each 4 GiB at most.
// CARGO_BUILD_JOBS, set, is kept.
const cgroup = (file) => {
  try {
    return readFileSync(join('/sys/fs/cgroup', file), 'utf8').trim()
  } catch {
    return 'max'
  }
}
const buildJobs = () => {
  const [quota, period] = cgroup('cpu.max').split(' ')
  const memory = cgroup('memory.max')
  const caps = []
  if (quota !== 'max') caps.push(Math.floor(Number(quota) / Number(period)))
  if (memory !== 'max') caps.push(Math.floor(Number(memory) / (4 * 2 ** 30)))
  return caps.length ? Math.max(1, Math.min(...caps)) : undefined
}
const limited = buildJobs()
if (!env.CARGO_BUILD_JOBS && limited) {
  env.CARGO_BUILD_JOBS = String(limited)
  console.log(
    `cov: building ${env.CARGO_BUILD_JOBS} crates at once, as the cgroup's CPUs and memory allow`,
  )
}

// The tools beside the toolchain's rustc: llvm-size and llvm-profdata.
const rustc = (...args) =>
  execFileSync('rustup', ['run', toolchain, 'rustc', ...args], { encoding: 'utf8' })
const host = rustc('-vV').match(/^host: (.*)$/m)[1]
const tools = join(rustc('--print', 'sysroot').trim(), 'lib/rustlib', host, 'bin')

// COV_CLEAN=1 builds the workspace again, as after a miss that makes no
// sense. Nothing else needs it: the report never reads another build's
// binaries. It costs more than it looks, as `cargo clean -p` takes any
// dependency named as one of our targets is (the `http` test takes the
// http crate, and all that is built on it).
if (process.env.COV_CLEAN) run(['llvm-cov', 'clean', '--workspace'])
// Each run's profiles go to a directory of its own.
const runsDir = join(target, 'cov-runs')
rmSync(runsDir, { recursive: true, force: true })

// The harnesses start while the tests build, and are stopped unused if
// every run is in the store. COV_HARNESSES sets how many of each net, a
// run going to the one with the fewest at the time: one PDS is one node
// process, which a few busy runs keep from answering the rest quickly.
mkdirSync(join(root, 'target'), { recursive: true })
const harness = (name, devEnv) => {
  const file = join(root, `target/cov-net-${name}.json`)
  // What the PDS logs, which tests provoke on purpose, is kept out of the way.
  const logged = join(root, `target/cov-net-${name}.log`)
  let child
  const start = () =>
    new Promise((ok, fail) => {
      const log = openSync(logged, 'w')
      child = spawn('node', [join(root, 'harness/pds.mjs'), '--accounts', 'seed'], {
        cwd: root,
        env: { ...process.env, KNOTLESS_DEV_ENV: devEnv },
        stdio: ['pipe', 'pipe', log],
      })
      let out = ''
      child.stdout.on('data', (d) => {
        out += d
        if (!out.includes('\n')) return
        writeFileSync(file, out)
        ok(file)
      })
      // Its log's end goes with it, as where it ran, in CI, is gone after.
      child.on('exit', (code) => {
        const end = readFileSync(logged, 'utf8').split('\n').slice(-40).join('\n')
        fail(
          new Error(
            `the ${name} harness exited with ${code}: target/cov-net-${name}.log ends\n${end}`,
          ),
        )
      })
    })
  // The dev-env picks free ports and then binds them, so harnesses
  // starting side by side can collide: it starts again on fresh ones.
  const up = start()
    .catch(() => start())
    .catch(() => start())
  up.catch(() => {})
  return { up, busy: 0, stop: () => child.stdin.end() }
}
const many = Number(process.env.COV_HARNESSES) || 1
const nets = {}
/// The harnesses of `net`, started the first time it is asked for.
const bring = (net) =>
  (nets[net] ??= Array.from({ length: many }, (_, i) =>
    harness(`${net}-${i}`, net === 'spaces' ? spaces : ''),
  ))
for (const net of netsRun) bring(net)
const stop = () =>
  Object.values(nets)
    .flat()
    .forEach((n) => n.stop())
process.on('exit', stop)

// Every test binary, and the package it is in: cargo runs each there.
const built = spawnSync(
  'rustup',
  [...cargo, 'test', '--workspace', '--no-run', '--message-format=json'],
  {
    cwd: root,
    env,
    encoding: 'utf8',
    stdio: ['ignore', 'pipe', 'inherit'],
    maxBuffer: 1 << 28,
  },
)
if (built.status !== 0) process.exit(built.status ?? 1)
const ours = new Set(
  JSON.parse(
    execFileSync('rustup', [...cargo, 'metadata', '--no-deps', '--format-version', '1'], {
      cwd: root,
      encoding: 'utf8',
      maxBuffer: 1 << 26,
    }),
  ).packages.map((p) => p.id),
)
// This build's executables of our own: each test binary, and each binary
// a test may run.
const made = built.stdout
  .split('\n')
  .filter((l) => l.startsWith('{'))
  .map((l) => JSON.parse(l))
  .filter((m) => m.reason === 'compiler-artifact' && m.executable && ours.has(m.package_id))
  .map((m) => ({
    exe: m.executable,
    dir: dirname(m.manifest_path),
    pkg: m.package_id,
    test: !!m.profile.test,
    name: m.profile.test ? `${m.target.name} (${m.target.kind[0]})` : m.target.name,
  }))
const tests = made.filter((m) => m.test && (!only || only.includes(m.name.split(' ')[0])))
const bins = made.filter((m) => !m.test)
console.log(`built ${tests.length} test binaries in ${took()}`)

const sha = (file) =>
  new Promise((ok, fail) => {
    const h = createHash('sha256')
    createReadStream(file)
      .on('data', (d) => h.update(d))
      .on('end', () => ok(h.digest('hex')))
      .on('error', fail)
  })
await Promise.all(made.map(async (m) => (m.sha = await sha(m.exe))))

// Which executables a profile may be of: those whose sections are the
// sizes its header gives for counters and names.
const shape = new Map()
for (const block of execFileSync(join(tools, 'llvm-size'), ['-A', ...made.map((m) => m.exe)], {
  encoding: 'utf8',
  maxBuffer: 1 << 24,
}).split(/\n(?=\S.* :\s*$)/m)) {
  const file = block.match(/^(\S.*?)\s*:\s*$/m)?.[1]
  const size = (s) => Number(block.match(new RegExp(`^_*${s}\\s+(\\d+)`, 'm'))?.[1])
  const k = `${size('llvm_prf_cnts') / 8}:${size('llvm_prf_names')}`
  shape.set(k, [...(shape.get(k) ?? []), ...made.filter((m) => m.exe === file)])
}
// The raw profile's header, as of version 10: counters, then names.
const header = (file) => {
  const b = Buffer.alloc(80)
  const fd = openSync(file, 'r')
  readSync(fd, b, 0, 80, 0)
  closeSync(fd)
  return (b.readBigUInt64LE(8) & 0xffffffffn) === 10n ? b : undefined
}
const counters = (file) => header(file)?.readBigUInt64LE(40)
const profiled = (file) => {
  const b = header(file)
  return b && shape.get(`${b.readBigUInt64LE(40)}:${b.readBigUInt64LE(72)}`)
}

// What else a run's outcome rests on: the harness and what it runs on,
// the PDS each net is, and this script, which sets the runs' environment.
const hashOf = (...parts) => createHash('sha256').update(parts.join('\0')).digest('hex')
const version = (cmd) => execFileSync(cmd, ['--version'], { encoding: 'utf8' }).trim()
const harnessed = hashOf(
  ...[
    'harness/cov.mjs',
    'harness/pds.mjs',
    'harness/tls-proxy.mjs',
    'harness/authorize.mjs',
    'harness/package-lock.json',
  ].map((f) => readFileSync(join(root, f), 'utf8')),
  version('node'),
  version('git'),
)
const netOf = {
  stock: '',
  spaces: `${process.env.ATPROTO_COMMIT ?? resolve(spaces)} ${process.env.ALPHA_BUILD ?? ''}`,
}
// A run's key, and the binaries besides its own it rests on: those of its
// own package always, as one may be stopped before it writes a profile,
// and those it was seen to run.
const keyOf = (t, net) => hashOf(t.sha, harnessed, net, netOf[net])
const depOf = (m) => `${m.test ? 'test' : 'bin'} ${m.name}`
const own = (t) =>
  Object.fromEntries(bins.filter((b) => b.pkg === t.pkg).map((b) => [depOf(b), b.sha]))
const now = Object.fromEntries(made.map((m) => [depOf(m), m.sha]))

// The report's own directory: this build's executables, linked, and the
// profiles of this build's runs, fresh or from the store.
const report = join(target, 'cov-report')
rmSync(report, { recursive: true, force: true })
mkdirSync(join(report, 'debug/deps'), { recursive: true })
for (const m of new Map(made.map((m) => [m.sha, m])).values()) {
  // Named as cargo names them, as the report finds them by name.
  const to = join(
    report,
    'debug/deps',
    `${basename(m.exe).replace(/-[0-9a-f]+$/, '')}-${m.sha.slice(0, 16)}`,
  )
  try {
    linkSync(m.exe, to)
  } catch {
    copyFileSync(m.exe, to)
  }
}
let profiles = 0
const profile = (file) => copyFileSync(file, join(report, `run-${profiles++}.profraw`))
// The XRPC answers the runs received, fresh or from the store, for
// harness/answers.mjs to hold to the lexicons.
const answered = join(report, 'answers')
mkdirSync(answered)
let answers = 0
const answer = (dir) => {
  if (!existsSync(dir)) return
  for (const f of readdirSync(dir))
    copyFileSync(join(dir, f), join(answered, `run-${answers++}.jsonl`))
}
// The scans of growing tables the index's queries made, as
// crates/knotless-serve/src/plans.rs finds them, for harness/plans.mjs.
const planned = join(report, 'plans')
mkdirSync(planned)
let plans = 0
const plan = (dir) => {
  if (!existsSync(dir)) return
  for (const f of readdirSync(dir))
    copyFileSync(join(dir, f), join(planned, `run-${plans++}.jsonl`))
}

/// A stored pass for this run, if every binary it rests on is as it was:
/// its profile goes to the report, and it is not run.
const stored = (r) => {
  if (!store) return false
  const dir = join(store, r.key)
  try {
    const kept = JSON.parse(readFileSync(join(dir, 'result.json'), 'utf8'))
    if (!Object.entries(kept.deps).every(([d, s]) => now[d] === s)) return false
    if (kept.profiled) profile(join(dir, 'run.profdata'))
    answer(join(dir, 'answers'))
    plan(join(dir, 'plans'))
    utimesSync(dir, new Date(), new Date())
    r.line = kept.line
    return true
  } catch {
    return false
  }
}

const timed = join(target, 'cov-times.json')
let times = {}
try {
  times = JSON.parse(readFileSync(timed, 'utf8'))
} catch {}
const timeKey = (t) => `${t.net} ${t.name}`
/// How long a run of `t` may go on, in ms (see the top).
const limitOf = (t) =>
  Number(process.env.COV_TIMEOUT) * 1000 || Math.max(600_000, 3 * (times[timeKey(t)] ?? 0))
const all = Object.keys(netOf).flatMap((net) =>
  tests.map((t) => ({ ...t, net, key: keyOf(t, net) })),
)
const reused = all.filter(stored)
// The longest first, as the last run timed them, so that the last to
// finish is not one started late: the integration tests before the rest
// where there is no time yet.
const guess = (t) => times[timeKey(t)] ?? (t.name.endsWith('(test)') ? 1e9 : 0)
const stockOnly = process.env.COV_STOCK?.split(',')
const wanted = (r) =>
  netsRun.includes(r.net) &&
  !(stockOnly && r.net === 'stock' && !stockOnly.includes(r.name.split(' ')[0]))
const runs = all.filter((r) => !reused.includes(r) && wanted(r)).sort((a, b) => guess(b) - guess(a))
console.log(
  `${reused.length} of ${all.length} runs passed before, as they are now, ${runs.length} to run`,
)
const left = all.filter((r) => !reused.includes(r) && !runs.includes(r))
if (left.length)
  console.log(
    `  not run, as COV_NETS is ${netsRun}${stockOnly ? ` and COV_STOCK ${stockOnly}` : ''}: ${left.map((r) => `${r.net} ${r.name}`).join(', ')}`,
  )
if (reused.length)
  console.log(`  passed before: ${reused.map((r) => `${r.net} ${r.name}`).join(', ')}`)

if (runs.length) {
  for (const n of Object.values(nets).flat()) n.file = await n.up
  console.log(`harnesses up in ${took()}`)
}

let dirs = 0
/// One binary against one harness, its output kept to show if it fails,
/// its profiles in a directory of their own.
const one = (t, args = []) =>
  new Promise((ok) => {
    const profiles = join(runsDir, String(dirs++))
    mkdirSync(profiles, { recursive: true })
    const answers = `${profiles}-answers`
    rmSync(answers, { recursive: true, force: true })
    const plans = `${profiles}-plans`
    rmSync(plans, { recursive: true, force: true })
    const began = Date.now()
    // The least busy harness of its net.
    const net = nets[t.net].reduce((a, b) => (b.busy < a.busy ? b : a))
    net.busy++
    const child = spawn(t.exe, args, {
      cwd: t.dir,
      env: {
        ...env,
        LLVM_PROFILE_FILE: join(profiles, '%p-%m.profraw'),
        KNOTLESS_TEST_NET: net.file,
        KNOTLESS_DEV_ENV: t.net === 'spaces' ? spaces : '',
        CARGO_MANIFEST_DIR: t.dir,
        KNOTLESS_ANSWERS: answers,
        KNOTLESS_PLANS: plans,
      },
      stdio: ['ignore', 'pipe', 'pipe'],
    })
    let out = ''
    child.stdout.on('data', (d) => (out += d))
    child.stderr.on('data', (d) => (out += d))
    const limit = limitOf(t)
    let timedOut = false
    const timer = setTimeout(() => {
      timedOut = true
      child.kill('SIGKILL')
    }, limit)
    // On close, not exit: the output is all read by then.
    child.on('close', (code) => {
      clearTimeout(timer)
      net.busy--
      if (timedOut) out += `\nkilled: still running after ${Math.round(limit / 1000)}s\n`
      ok({
        ...t,
        code: timedOut ? 'timeout' : code,
        timedOut,
        out,
        ms: Date.now() - began,
        profiles,
        answers,
        plans,
      })
    })
  })

/// Keeps a passing run: its profiles merged into one, and what it ran.
const keepRun = (r, line) => {
  const raws = readdirSync(r.profiles).map((f) => join(r.profiles, f))
  const deps = own(r)
  for (const f of raws) {
    // One with no counters counts nothing, and is of no binary of ours.
    if (counters(f) === 0n) continue
    const of = profiled(f)
    // A profile of no executable this build made: kept out of the store.
    if (!of)
      return console.log(
        `not stored: ${r.net} ${r.name} wrote ${basename(f)}, of no binary this build made`,
      )
    for (const m of of) if (m.exe !== r.exe) deps[depOf(m)] = m.sha
  }
  const tmp = join(store, `.tmp-${process.pid}-${r.key}`)
  rmSync(tmp, { recursive: true, force: true })
  mkdirSync(tmp, { recursive: true })
  if (raws.length)
    execFileSync(join(tools, 'llvm-profdata'), [
      'merge',
      '-sparse',
      ...raws,
      '-o',
      join(tmp, 'run.profdata'),
    ])
  if (existsSync(r.answers)) cpSync(r.answers, join(tmp, 'answers'), { recursive: true })
  if (existsSync(r.plans)) cpSync(r.plans, join(tmp, 'plans'), { recursive: true })
  writeFileSync(
    join(tmp, 'result.json'),
    JSON.stringify({ deps, line, profiled: raws.length > 0, at: new Date().toISOString() }),
  )
  rmSync(join(store, r.key), { recursive: true, force: true })
  renameSync(tmp, join(store, r.key))
}

// Tests failing now and then, while they are fixed: each by net, binary
// and test, with why and the last day it holds. Past that day it fails
// as any test does.
const today = new Date().toISOString().slice(0, 10)
const quarantine = JSON.parse(readFileSync(join(root, 'harness/quarantine.json'), 'utf8')).filter(
  (q) => {
    if (q.until >= today) return true
    console.log(`quarantine over since ${q.until}: ${q.net} ${q.binary} ${q.test}`)
    return false
  },
)
/// The tests `r` failed, if every one is quarantined; none otherwise.
const quarantined = (r) => {
  const names = [...r.out.matchAll(/^test (\S+) \.\.\. FAILED$/gm)].map((m) => m[1])
  const held = (n) => quarantine.some((q) => q.net === r.net && q.binary === r.name && q.test === n)
  return !r.timedOut && names.length && names.every(held) ? names : []
}

const failed = []
/// Runs each of `list`, `jobs` at a time, keeping those that pass.
const runAll = async (list) => {
  let next = 0
  await Promise.all(
    Array.from({ length: jobs }, async () => {
      while (next < list.length) {
        const first = await one(list[next++])
        // One that hung would hang again.
        const r = first.code === 0 || first.timedOut ? first : await one(first)
        const line = r.timedOut
          ? 'killed, still running at its limit'
          : (r.out.match(/^test result: .*$/m)?.[0] ?? `exited with ${r.code}`)
        const held = r.code === 0 ? [] : quarantined(r)
        console.log(
          `${r.code === 0 ? 'ok  ' : held.length ? 'HELD' : 'FAIL'} ${r.net.padEnd(6)} ${r.name}: ${line}`,
        )
        for (const n of held) console.log(`QUARANTINED: ${r.net} ${r.name} ${n}`)
        if (r.code !== 0 && !held.length) {
          failed.push(first, r)
          continue
        }
        times[timeKey(r)] = r.ms
        for (const f of readdirSync(r.profiles)) profile(join(r.profiles, f))
        answer(r.answers)
        plan(r.plans)
        if (held.length) {
          // Failed only where it is known to: counted, never stored. What
          // only a held test reaches is not reached when it fails, so each
          // is run alone, up to three times, for a run that passes.
          for (const n of held) {
            let alone
            for (let i = 0; i < 3 && alone?.code !== 0; i++) alone = await one(r, [n, '--exact'])
            console.log(`${alone.code === 0 ? 'passed' : 'failed'} alone: ${r.net} ${r.name} ${n}`)
            if (alone.code === 0)
              for (const f of readdirSync(alone.profiles)) profile(join(alone.profiles, f))
          }
          console.log(
            `---- its runs, which failed:\n${first.out}\n----\n${r.out}\n---- end of ${r.net} ${r.name}`,
          )
        } else if (r !== first) {
          // Passed only the second time: it says so, plainly, and is not stored.
          const names = [...first.out.matchAll(/^test (\S+) \.\.\. FAILED$/gm)].map((m) => m[1])
          for (const n of names.length ? names : ['(the binary itself)'])
            console.log(`FLAKY: ${r.net} ${r.name} ${n}`)
          console.log(
            `---- its first run, which failed:\n${first.out}\n---- end of ${r.net} ${r.name}`,
          )
        } else if (store) keepRun(r, line)
      }
    }),
  )
  writeFileSync(timed, JSON.stringify(times, null, 1))
  for (const r of failed) console.log(`\n---- ${r.net} ${r.name}\n${r.out}`)
  if (failed.length) process.exit(1)
}
await runAll(runs)
// The harnesses are kept while a run may yet be made again (see below).
if (!reused.length) stop()
console.log(`ran ${runs.length} in ${took()}`)

// The store keeps the runs used last, up to its bound.
if (store && existsSync(store)) {
  const entries = readdirSync(store)
    .filter((e) => !e.startsWith('.'))
    .map((e) => ({ e, at: statSync(join(store, e)).mtimeMs }))
    .sort((a, b) => b.at - a.at)
  for (const { e } of entries.slice(keep)) rmSync(join(store, e), { recursive: true, force: true })
  for (const e of readdirSync(store).filter(
    (e) => e.startsWith('.tmp-') && !e.startsWith(`.tmp-${process.pid}-`),
  )) {
    if (Date.now() - statSync(join(store, e)).mtimeMs > 3600e3)
      rmSync(join(store, e), { recursive: true, force: true })
  }
}

// The table goes to a file, and only its total and the files short of
// the gate are shown, with the lines they miss: a table of every file
// would push the misses out of what CI's log keeps.
const table = join(report, 'report.txt')
const gate = () => {
  const r = spawnSync(
    'rustup',
    [
      ...cargo,
      'llvm-cov',
      'report',
      '--output-path',
      table,
      '--ignore-filename-regex',
      ignore,
      '--fail-under-lines',
      '100',
      '--fail-under-functions',
      '100',
      '--show-missing-lines',
    ],
    {
      cwd: root,
      env: { ...env, CARGO_LLVM_COV_TARGET_DIR: report, CARGO_LLVM_COV_BUILD_DIR: report },
      stdio: 'inherit',
    },
  )
  const rows = existsSync(table) ? readFileSync(table, 'utf8').split('\n') : []
  const short = (l) =>
    l.split(/\s+/).length > 9 && (l.split(/\s+/)[5] !== '0' || l.split(/\s+/)[8] !== '0')
  const total = rows.filter((l) => l.startsWith('TOTAL'))
  console.log(
    [rows[0], ...rows.slice(1).filter((l) => !l.startsWith('TOTAL') && short(l)), ...total].join(
      '\n',
    ),
  )
  if (r.status !== 0)
    missed(
      rows
        .slice(1)
        .filter((l) => !l.startsWith('TOTAL') && short(l))
        .map((l) => l.split(/\s+/)[0]),
    )
  return r.status ?? 1
}

/// Says each line the gate counts as missed, by file and by the function
/// it is in. The table counts a function's lines once for all its
/// instances, a generic's or a macro's, at the instance that reaches the
/// most of them, and counts a line again for each function it is in: the
/// lines are read here the same way, so the list is the table's misses
/// and no more, though another instance of a generic reaches a line its
/// best one does not. Where instances differ in their lines, as when one
/// is built only for a crate's own tests, the table's misses are those of
/// the instance with the most lines, and those are named.
const missed = (short) => {
  const json = join(report, 'missed.json')
  spawnSync(
    'rustup',
    [
      ...cargo,
      'llvm-cov',
      'report',
      '--json',
      '--output-path',
      json,
      '--ignore-filename-regex',
      ignore,
    ],
    {
      cwd: root,
      env: { ...env, CARGO_LLVM_COV_TARGET_DIR: report, CARGO_LLVM_COV_BUILD_DIR: report },
      stdio: 'inherit',
    },
  )
  if (!existsSync(json)) return
  // A region: line and column it starts and ends at, its count, its
  // file, the file it expands, and its kind, 0 being code.
  const at = (f) => relative(root, f.filenames[0])
  const first = (f) => f.regions.find((r) => r[5] === 0)
  // The lines an instance misses: where each code region starts that no
  // reached region in the same file covers. A macro's call is an
  // expansion region, and one reached reaches its line.
  const misses = (f) => {
    const here = f.regions.filter((r) => r[5] === 0)
    const lines = here.filter((r) => r[7] === 0).map((r) => r[0])
    return [...new Set(lines.filter((l) => !here.some((r) => r[4] > 0 && r[0] <= l && l <= r[2])))]
  }
  // The lines an instance has: every line a code region of it spans, as
  // llvm-cov counts a line inside a region though nothing starts on it.
  const mapped = (f) => {
    const lines = new Set()
    for (const r of f.regions.filter((r) => r[5] === 0 && r[7] !== 2))
      for (let l = r[0]; l <= r[2]; l++) lines.add(l)
    return lines
  }
  // The instances of one function start at the same place.
  const groups = new Map()
  for (const f of JSON.parse(readFileSync(json, 'utf8')).data[0].functions) {
    const start = first(f)
    if (!start || !short.some((s) => at(f).endsWith(s))) continue
    const key = `${at(f)}:${start[0]}:${start[1]}`
    groups.set(key, [...(groups.get(key) ?? []), f])
  }
  const out = []
  for (const fs of groups.values()) {
    const f = fs[0]
    if (!fs.some((f) => f.count)) {
      out.push([at(f), first(f)[0], f.name, ', never called'])
      continue
    }
    const seen = fs.map((f) => {
      const miss = f.count ? misses(f) : []
      const lines = mapped(f)
      return { f, miss, lines, covered: f.count ? lines.size - miss.length : 0 }
    })
    const best = seen
      .filter((s) => s.f.count)
      .reduce((a, b) => (b.miss.length < a.miss.length ? b : a))
    for (const l of best.miss) out.push([at(best.f), l, best.f.name, ''])
    // The table takes the most lines any instance has, less the most any
    // reaches: an instance with lines the others lack, as one built only
    // for a crate's own tests (a `#[cfg(test)]` arm), counts the lines it
    // misses though another instance reaches them. Those are named too.
    const widest = seen.reduce((a, b) => (b.lines.size > a.lines.size ? b : a))
    const counted = widest.lines.size - Math.max(...seen.map((s) => s.covered))
    if (counted > best.miss.length) {
      const lacking = widest.f.count ? widest.miss : [...widest.lines]
      const extra = lacking.filter((l) => !best.miss.includes(l))
      for (const l of extra)
        out.push([at(widest.f), l, widest.f.name, ', in the instance with the most lines'])
    }
  }
  console.log('Uncovered:')
  const rows = out.map(([file, line, name, never]) => [file, line, name + never])
  rows.sort((a, b) => a[0].localeCompare(b[0]) || a[1] - b[1])
  for (const [file, line, name] of rows) console.log(`  ${file}:${line} in ${name}`)
}

let status = gate()
// A line some runs reach and others do not, as a race decides, can be
// one no stored run reached: the stored runs are made again, fresh, and
// the report read again, rather than fail a change that did not touch it.
if (status !== 0 && reused.length && !partial) {
  console.log(`short with ${reused.length} runs stored: running them again`)
  for (const n of Object.values(nets).flat()) n.file = await n.up
  await runAll(reused)
  status = gate()
}
if (status !== 0 && partial && left.length) {
  console.log(
    `short against ${netsRun} alone: making the ${left.length} runs left, against the rest`,
  )
  for (const r of left) bring(r.net)
  for (const n of Object.values(nets).flat()) n.file = await n.up
  await runAll(left)
  status = gate()
}
stop()
if (status !== 0) process.exit(status)
console.log(`coverage in ${took()}`)