chadtmiller.com/knotless

Switch repository

Loading…

// What moved: the pages as a base draws them against the pages as this
// checkout draws them, shot in the same browser in the same run, so no
// picture is kept between runs, in git or anywhere else.
//
//   node harness/shots/compare.mjs [playwright args]
//
// The base is KNOTLESS_RUN_BASE, which the runner gives a pull's run (the
// tip its head is merged with) and a batch's (the tip it is built on); in
// a push's run, the commit before; here, SHOTS_BASE, or where this
// checkout left main. Its web/ and lexicons/ are taken out of git into
// target/shots/base and built there, on this checkout's packages when its
// lockfile is the same, and its fixtures.json beside them. The harness
// that shoots both is this checkout's.
//
// Only pages whose HTML differs (digest.mjs) are shot: first as the base
// draws them, into target/shots/before, then as this checkout does,
// against those. A page that moved, or is new, is named in the log with
// its pictures in target/shots/results, and fails the run unless the
// change names it, on a line of its own:
//
//   Shots-moved: home, pull-open
//
// or `Shots-moved: all`. In a pull's run the line is read from the pull's
// body, from its record; otherwise from the commit messages since the
// base, which is where a batch's squashed pulls carry their bodies.
// SHOTS_MOVED names more; `just shots`, being for looking, passes `all`.
//
// With SHOTS_LIVE=1 this checkout's pages are drawn twice instead: from
// fixtures.json, then from a world through knotless-serve (setup.mjs), so
// a page that moved is one whose recording is out of date.
//
// Exits 0 when nothing moved unnamed, 1 when something did, 2 when the
// pages could not be shot at all.

import { execFileSync, spawnSync } from 'node:child_process'
import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { digest } from './digest.mjs'
import { cookies, fixtures, front, recording } from './fixtures.mjs'
import config from './playwright.config.mjs'
import { base as url, root } from './world.mjs'

const out = join(root, 'target/shots')
const say = (line) => console.log(`==> ${line}`)
const fail = (why) => {
  say(why)
  process.exit(2)
}
// A checkout another user made, as a container sees the runner's, is
// still git's to read.
const bytes = (...args) => execFileSync('git', ['-c', 'safe.directory=*', ...args], { cwd: root, maxBuffer: 1 << 28 })
const git = (...args) => bytes(...args).toString('utf8')
const run = (cmd, args, cwd, env = {}) =>
  spawnSync(cmd, args, { cwd, stdio: 'inherit', env: { ...process.env, ...env } }).status

/// The commit the pages are compared against.
function based() {
  if (process.env.KNOTLESS_RUN_BASE) return process.env.KNOTLESS_RUN_BASE
  // A pull is checked at its head merged with its base, the merge's first
  // parent; a push, against the commit before.
  if (process.env.KNOTLESS_RUN) return git('rev-parse', 'HEAD^1').trim()
  return process.env.SHOTS_BASE || git('merge-base', 'HEAD', 'main').trim()
}

/// `sha`'s pages, built in target/shots/base, which is kept for the next
/// run against the same commit.
function built(sha) {
  const dir = join(out, 'base')
  const stamp = join(dir, 'commit')
  if (existsSync(stamp) && readFileSync(stamp, 'utf8') === sha) return dir
  rmSync(dir, { recursive: true, force: true })
  mkdirSync(dir, { recursive: true })
  execFileSync('tar', ['-x', '-C', dir], { input: bytes('archive', '--format=tar', sha, 'web', 'lexicons') })
  writeFileSync(join(dir, 'fixtures.json'), git('show', `${sha}:harness/shots/fixtures.json`))
  const lock = (d) => readFileSync(join(d, 'web/package-lock.json'), 'utf8')
  const web = join(dir, 'web')
  if (lock(dir) === lock(root)) symlinkSync(join(root, 'web/node_modules'), join(web, 'node_modules'))
  else if (run('npm', ['ci', '--no-audit', '--no-fund'], web)) fail(`could not install ${sha}'s packages`)
  if (run('npm', ['run', 'build'], web)) fail(`could not build ${sha}'s pages`)
  writeFileSync(stamp, sha)
  return dir
}

/// Each page's digest, as `side` draws it from its recording.
async function digests(side) {
  const { views, answers } = recording(side.fixtures)
  const stand = await front({ recorded: answers, build: side.build })
  const sums = {}
  try {
    // Asked as the browser asks (views.spec.mjs): having said its zone, and
    // signed in where the view is.
    for (const [name, path] of Object.entries(views)) {
      sums[name] = digest(await (await fetch(url + path, { headers: { cookie: cookies(name) } })).text())
    }
  } finally {
    await stand.kill()
  }
  return sums
}

/// Playwright over the pages setup.mjs serves with `env`.
const shoot = (env, args) =>
  run('npx', ['playwright', 'test', '-c', 'shots/playwright.config.mjs', ...args], join(root, 'harness'), env)

/// The pages that moved, each with the projects it moved in and whether
/// it is new, from the report of the last run; and the tests that failed
/// for any other reason.
function moves() {
  const report = JSON.parse(readFileSync(join(out, 'report.json'), 'utf8'))
  const moved = new Map()
  const broken = []
  const walk = (suite) => {
    for (const s of suite.suites ?? []) walk(s)
    for (const spec of suite.specs ?? []) {
      for (const test of spec.tests) {
        if (test.status !== 'unexpected') continue
        const message = test.results.at(-1)?.error?.message ?? ''
        const added = /snapshot doesn't exist/.test(message)
        if (added || /are different|Expected an image \d+px/.test(message)) {
          const page = moved.get(spec.title) ?? { projects: [], added }
          page.projects.push(test.projectName)
          moved.set(spec.title, page)
        } else broken.push(`${spec.title} (${test.projectName}): ${message.split('\n').find((l) => l.trim())}`)
      }
    }
  }
  for (const suite of report.suites ?? []) walk(suite)
  return { moved, broken }
}

const NAMES = /^shots-moved:[ \t]*(.*)$/gim
const list = (text) => text.split(/[\s,]+/).filter(Boolean)
const names = (text) => [...text.matchAll(NAMES)].flatMap((m) => list(m[1]))

const json = async (at) => {
  const res = await fetch(at)
  if (!res.ok) throw new Error(`${at}: ${res.status}`)
  return res.json()
}

/// The body of the pull at `uri`, from its record on its author's PDS.
async function body(uri) {
  const [, did, collection, rkey] = uri.match(/^at:\/\/([^/]+)\/([^/]+)\/([^/]+)$/)
  const doc = did.startsWith('did:web:')
    ? await json(`https://${decodeURIComponent(did.slice(8))}/.well-known/did.json`)
    : await json(`${process.env.KNOTLESS_PLC_URL ?? 'https://plc.directory'}/${did}`)
  const pds = doc.service.find((s) => s.id === '#atproto_pds' || s.id === `${did}#atproto_pds`).serviceEndpoint
  const q = new URLSearchParams({ repo: did, collection, rkey })
  return (await json(`${pds}/xrpc/com.atproto.repo.getRecord?${q}`)).value.body ?? ''
}

/// The pages the change says it moves.
async function named(sha) {
  const said = list(process.env.SHOTS_MOVED ?? '')
  const pull = process.env.KNOTLESS_RUN_PULL
  if (pull) {
    try {
      said.push(...names(await body(pull)))
    } catch (e) {
      fail(`could not read the body of ${pull}: ${e.message}`)
    }
  } else said.push(...names(git('log', '--format=%B', `${sha}..HEAD`)))
  return new Set(said)
}

// The pages, not the cards, which are only looked at (`just shots-cards`).
const projects = config.projects.map((p) => p.name).filter((p) => p !== 'card')
const given = process.argv.slice(2)
const args = given.some((a) => a.startsWith('--project')) ? given : [...given, ...projects.map((p) => `--project=${p}`)]
const live = Boolean(process.env.SHOTS_LIVE)
mkdirSync(out, { recursive: true })
const here = { build: join(root, 'web/build'), fixtures }
let before = here
let sha
let unchanged = []
if (!live) {
  sha = based()
  say(`comparing the pages with ${sha}'s`)
  // What the pages are drawn from, against what is checked out now.
  const from = ['web', 'lexicons', 'harness/shots/fixtures.json']
  if (spawnSync('git', ['-c', 'safe.directory=*', 'diff', '--quiet', sha, '--', ...from], { cwd: root }).status === 0) {
    say(`nothing the pages are drawn from (${from.join(', ')}) changed: none moved`)
    process.exit(0)
  }
  const dir = built(sha)
  before = { build: join(dir, 'web/build'), fixtures: join(dir, 'fixtures.json') }
  const [then, now] = [await digests(before), await digests(here)]
  unchanged = Object.keys(now).filter((name) => then[name] === now[name])
  say(`${Object.keys(now).length - unchanged.length} of ${Object.keys(now).length} pages' HTML differs`)
}
writeFileSync(join(out, 'unchanged.json'), JSON.stringify(unchanged))
rmSync(join(out, 'before'), { recursive: true, force: true })
const env = { SHOTS_BUILD: before.build, SHOTS_FIXTURES: before.fixtures, SHOTS_LIVE: '' }
if (shoot(env, ['--update-snapshots=all', ...args])) fail(`the pages ${live ? 'as recorded' : `as ${sha} draws them`} could not be shot`)
shoot({ SHOTS_LIVE: live ? '1' : '' }, ['--update-snapshots=none', ...args])
const { moved, broken } = moves()
if (broken.length) fail(`the pages could not be shot:\n${broken.join('\n')}`)
if (!moved.size) {
  say('no page moved')
  process.exit(0)
}
const said = live ? new Set() : await named(sha)
const unnamed = [...moved.keys()].filter((name) => !said.has(name) && !said.has('all'))
for (const [name, { projects, added }] of moved) {
  const how = added ? 'is new' : `moved in ${projects.join(', ')}`
  say(`${name} ${how}${unnamed.includes(name) ? '' : ', as the change says'}`)
}
if (!unnamed.length) process.exit(0)
if (live) say('the recording is out of date: `just shots-record`')
else {
  say(`moved, but not named: ${unnamed.join(', ')}. Its pictures say how.`)
  say(`If they are meant to, name them on a line of the pull's body, \`Shots-moved: ${unnamed.join(', ')}\`, and run its checks again.`)
}
process.exit(1)