// What moved: the pages as a base draws them against the pages as this // checkout draws them, shot in the same browser in the same run, so no // picture is kept between runs, in git or anywhere else. // // node harness/shots/compare.mjs [playwright args] // // The base is KNOTLESS_RUN_BASE, which the runner gives a pull's run (the // tip its head is merged with) and a batch's (the tip it is built on); in // a push's run, the commit before; here, SHOTS_BASE, or where this // checkout left main. Its web/ and lexicons/ are taken out of git into // target/shots/base and built there, on this checkout's packages when its // lockfile is the same, and its fixtures.json beside them. The harness // that shoots both is this checkout's. // // Only pages whose HTML differs (digest.mjs) are shot: first as the base // draws them, into target/shots/before, then as this checkout does, // against those. A page that moved, or is new, is named in the log with // its pictures in target/shots/results, and fails the run unless the // change names it, on a line of its own: // // Shots-moved: home, pull-open // // or `Shots-moved: all`. In a pull's run the line is read from the pull's // body, from its record; otherwise from the commit messages since the // base, which is where a batch's squashed pulls carry their bodies. // SHOTS_MOVED names more; `just shots`, being for looking, passes `all`. // // With SHOTS_LIVE=1 this checkout's pages are drawn twice instead: from // fixtures.json, then from a world through knotless-serve (setup.mjs), so // a page that moved is one whose recording is out of date. // // Exits 0 when nothing moved unnamed, 1 when something did, 2 when the // pages could not be shot at all. import { execFileSync, spawnSync } from 'node:child_process' import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { digest } from './digest.mjs' import { cookies, fixtures, front, recording } from './fixtures.mjs' import config from './playwright.config.mjs' import { base as url, root } from './world.mjs' const out = join(root, 'target/shots') const say = (line) => console.log(`==> ${line}`) const fail = (why) => { say(why) process.exit(2) } // A checkout another user made, as a container sees the runner's, is // still git's to read. const bytes = (...args) => execFileSync('git', ['-c', 'safe.directory=*', ...args], { cwd: root, maxBuffer: 1 << 28 }) const git = (...args) => bytes(...args).toString('utf8') const run = (cmd, args, cwd, env = {}) => spawnSync(cmd, args, { cwd, stdio: 'inherit', env: { ...process.env, ...env } }).status /// The commit the pages are compared against. function based() { if (process.env.KNOTLESS_RUN_BASE) return process.env.KNOTLESS_RUN_BASE // A pull is checked at its head merged with its base, the merge's first // parent; a push, against the commit before. if (process.env.KNOTLESS_RUN) return git('rev-parse', 'HEAD^1').trim() return process.env.SHOTS_BASE || git('merge-base', 'HEAD', 'main').trim() } /// `sha`'s pages, built in target/shots/base, which is kept for the next /// run against the same commit. function built(sha) { const dir = join(out, 'base') const stamp = join(dir, 'commit') if (existsSync(stamp) && readFileSync(stamp, 'utf8') === sha) return dir rmSync(dir, { recursive: true, force: true }) mkdirSync(dir, { recursive: true }) execFileSync('tar', ['-x', '-C', dir], { input: bytes('archive', '--format=tar', sha, 'web', 'lexicons') }) writeFileSync(join(dir, 'fixtures.json'), git('show', `${sha}:harness/shots/fixtures.json`)) const lock = (d) => readFileSync(join(d, 'web/package-lock.json'), 'utf8') const web = join(dir, 'web') if (lock(dir) === lock(root)) symlinkSync(join(root, 'web/node_modules'), join(web, 'node_modules')) else if (run('npm', ['ci', '--no-audit', '--no-fund'], web)) fail(`could not install ${sha}'s packages`) if (run('npm', ['run', 'build'], web)) fail(`could not build ${sha}'s pages`) writeFileSync(stamp, sha) return dir } /// Each page's digest, as `side` draws it from its recording. async function digests(side) { const { views, answers } = recording(side.fixtures) const stand = await front({ recorded: answers, build: side.build }) const sums = {} try { // Asked as the browser asks (views.spec.mjs): having said its zone, and // signed in where the view is. for (const [name, path] of Object.entries(views)) { sums[name] = digest(await (await fetch(url + path, { headers: { cookie: cookies(name) } })).text()) } } finally { await stand.kill() } return sums } /// Playwright over the pages setup.mjs serves with `env`. const shoot = (env, args) => run('npx', ['playwright', 'test', '-c', 'shots/playwright.config.mjs', ...args], join(root, 'harness'), env) /// The pages that moved, each with the projects it moved in and whether /// it is new, from the report of the last run; and the tests that failed /// for any other reason. function moves() { const report = JSON.parse(readFileSync(join(out, 'report.json'), 'utf8')) const moved = new Map() const broken = [] const walk = (suite) => { for (const s of suite.suites ?? []) walk(s) for (const spec of suite.specs ?? []) { for (const test of spec.tests) { if (test.status !== 'unexpected') continue const message = test.results.at(-1)?.error?.message ?? '' const added = /snapshot doesn't exist/.test(message) if (added || /are different|Expected an image \d+px/.test(message)) { const page = moved.get(spec.title) ?? { projects: [], added } page.projects.push(test.projectName) moved.set(spec.title, page) } else broken.push(`${spec.title} (${test.projectName}): ${message.split('\n').find((l) => l.trim())}`) } } } for (const suite of report.suites ?? []) walk(suite) return { moved, broken } } const NAMES = /^shots-moved:[ \t]*(.*)$/gim const list = (text) => text.split(/[\s,]+/).filter(Boolean) const names = (text) => [...text.matchAll(NAMES)].flatMap((m) => list(m[1])) const json = async (at) => { const res = await fetch(at) if (!res.ok) throw new Error(`${at}: ${res.status}`) return res.json() } /// The body of the pull at `uri`, from its record on its author's PDS. async function body(uri) { const [, did, collection, rkey] = uri.match(/^at:\/\/([^/]+)\/([^/]+)\/([^/]+)$/) const doc = did.startsWith('did:web:') ? await json(`https://${decodeURIComponent(did.slice(8))}/.well-known/did.json`) : await json(`${process.env.KNOTLESS_PLC_URL ?? 'https://plc.directory'}/${did}`) const pds = doc.service.find((s) => s.id === '#atproto_pds' || s.id === `${did}#atproto_pds`).serviceEndpoint const q = new URLSearchParams({ repo: did, collection, rkey }) return (await json(`${pds}/xrpc/com.atproto.repo.getRecord?${q}`)).value.body ?? '' } /// The pages the change says it moves. async function named(sha) { const said = list(process.env.SHOTS_MOVED ?? '') const pull = process.env.KNOTLESS_RUN_PULL if (pull) { try { said.push(...names(await body(pull))) } catch (e) { fail(`could not read the body of ${pull}: ${e.message}`) } } else said.push(...names(git('log', '--format=%B', `${sha}..HEAD`))) return new Set(said) } // The pages, not the cards, which are only looked at (`just shots-cards`). const projects = config.projects.map((p) => p.name).filter((p) => p !== 'card') const given = process.argv.slice(2) const args = given.some((a) => a.startsWith('--project')) ? given : [...given, ...projects.map((p) => `--project=${p}`)] const live = Boolean(process.env.SHOTS_LIVE) mkdirSync(out, { recursive: true }) const here = { build: join(root, 'web/build'), fixtures } let before = here let sha let unchanged = [] if (!live) { sha = based() say(`comparing the pages with ${sha}'s`) // What the pages are drawn from, against what is checked out now. const from = ['web', 'lexicons', 'harness/shots/fixtures.json'] if (spawnSync('git', ['-c', 'safe.directory=*', 'diff', '--quiet', sha, '--', ...from], { cwd: root }).status === 0) { say(`nothing the pages are drawn from (${from.join(', ')}) changed: none moved`) process.exit(0) } const dir = built(sha) before = { build: join(dir, 'web/build'), fixtures: join(dir, 'fixtures.json') } const [then, now] = [await digests(before), await digests(here)] unchanged = Object.keys(now).filter((name) => then[name] === now[name]) say(`${Object.keys(now).length - unchanged.length} of ${Object.keys(now).length} pages' HTML differs`) } writeFileSync(join(out, 'unchanged.json'), JSON.stringify(unchanged)) rmSync(join(out, 'before'), { recursive: true, force: true }) const env = { SHOTS_BUILD: before.build, SHOTS_FIXTURES: before.fixtures, SHOTS_LIVE: '' } if (shoot(env, ['--update-snapshots=all', ...args])) fail(`the pages ${live ? 'as recorded' : `as ${sha} draws them`} could not be shot`) shoot({ SHOTS_LIVE: live ? '1' : '' }, ['--update-snapshots=none', ...args]) const { moved, broken } = moves() if (broken.length) fail(`the pages could not be shot:\n${broken.join('\n')}`) if (!moved.size) { say('no page moved') process.exit(0) } const said = live ? new Set() : await named(sha) const unnamed = [...moved.keys()].filter((name) => !said.has(name) && !said.has('all')) for (const [name, { projects, added }] of moved) { const how = added ? 'is new' : `moved in ${projects.join(', ')}` say(`${name} ${how}${unnamed.includes(name) ? '' : ', as the change says'}`) } if (!unnamed.length) process.exit(0) if (live) say('the recording is out of date: `just shots-record`') else { say(`moved, but not named: ${unnamed.join(', ')}. Its pictures say how.`) say(`If they are meant to, name them on a line of the pull's body, \`Shots-moved: ${unnamed.join(', ')}\`, and run its checks again.`) } process.exit(1)