// The harness's citations of the spec (spec/PDS.md, "How it is checked"): // a check names the section it holds the server to as // `(spec/.md, "")`, or several of one page as // `(spec/.md, "", "")`, as test262's `esid` names a section // of ECMAScript. A heading is cited as written, its backticks left out; a // paragraph's bold label, as `**When.**`, may be cited as a heading is. // Nothing under a page's "How it is checked", nor under a section that // opens with `*This section is informative.*`, is a section a check must // cite, though their headings may be cited. Every citation must name a // heading or label its page has; and every section of every page must be // cited by some check, but those UNCHECKED names, each with why. // // node harness/spec-coverage.mjs // // Prints each citation of no section, each section no check cites, and each // section UNCHECKED names that a check cites or that is gone, failing if // any; then how many sections of each page are cited. import { readdirSync, readFileSync } from 'node:fs' import { dirname, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const harness = dirname(fileURLToPath(import.meta.url)) const root = resolve(harness, '..') const spec = join(root, 'spec') /// The sections no check holds the server to yet, each with why. Every /// other section of every page must be cited; one listed here leaves the /// list once a check cites it. const UNCHECKED = { 'LIMITS.md': { getBlocks: "its check, on branch harness/getblocks, writes a repository of 100000 records in the --scale run, and the server outgrows its memory bound doing so (STORE.md, \"Scale\"): it lands once the server keeps within it, at://did:plc:bcgltzqazw5tb6k2g3ttenbj/space/dev.knotless.space/spec-pds/did:plc:bcgltzqazw5tb6k2g3ttenbj/dev.knotless.issue/3n24v3yorfeyq" }, } /// The section of every page that says what the harness does, which is no /// requirement of the server's. const CHECKED = 'How it is checked' /// The line a section opens with that says it requires nothing. const INFORMATIVE = '*This section is informative.*' /// Each page's headings below its title, outside code blocks, with their /// lines; the bold labels its paragraphs and list items start with; and the /// headings of its "How it is checked" and of its informative sections, with /// all below them, which a check need not cite. const plain = (text) => text.replaceAll('`', '').trim() const pages = new Map() const problems = [] for (const page of readdirSync(spec).filter((f) => f.endsWith('.md'))) { const headings = new Map() const labels = new Set() const exempt = new Set() const lines = readFileSync(join(spec, page), 'utf8').split('\n') let fenced = false let skipping = 0 lines.forEach((line, i) => { if (/^\s*```/.test(line)) fenced = !fenced if (fenced) return const h = line.match(/^(#{2,6})\s+(.+?)\s*$/) if (h) { const [level, heading] = [h[1].length, plain(h[2])] if (skipping && level > skipping) return void exempt.add(heading) const opening = lines.slice(i + 1).find((l) => l.trim() !== '')?.trim() skipping = heading === CHECKED || opening === INFORMATIVE ? level : 0 if (skipping) exempt.add(heading) else if (headings.has(heading)) problems.push(`spec/${page}:${i + 1}: "${heading}" is a heading twice, so no citation can tell which`) else headings.set(heading, i + 1) return } const l = !skipping && line.match(/^\s*(?:[-*]\s+|\d+\.\s+)?\*\*([^*]+?)\.?\*\*/) if (l) labels.add(plain(l[1])) }) pages.set(page, { headings, labels, exempt }) } /// Every citation in the harness's scripts: a heading quoted after its /// page, wrapped across comment lines or not. const cited = new Map([...pages.keys()].map((p) => [p, new Set()])) /// The harness's scripts, by path under it; `node_modules` is not entered, /// as walking the packages would take most of the run. const walk = (dir) => readdirSync(join(harness, dir), { withFileTypes: true }).flatMap((e) => { const path = dir ? `${dir}/${e.name}` : e.name if (e.isDirectory()) return e.name === 'node_modules' ? [] : walk(path) return e.name.endsWith('.mjs') ? [path] : [] }) const scripts = walk('') /// A citation's page, then its headings, each quoted, the comment it is in /// wrapped between them or not. const CITATION = /spec\/([\w-]+\.md)((?:,?(?:\s|\/\/+)+(?:and(?:\s|\/\/+)+)?"[^"]+")+)/g for (const file of scripts) { const src = readFileSync(join(harness, file), 'utf8') for (const m of src.matchAll(CITATION)) { const at = `harness/${file}:${src.slice(0, m.index).split('\n').length}` const page = pages.get(m[1]) for (const [, quoted] of m[2].matchAll(/"([^"]+)"/g)) { const heading = plain(quoted.replace(/\s*\n\s*(?:\/\/\/?|\*)?\s*/g, ' ').replaceAll("\\'", "'")) if (!page) problems.push(`${at}: spec/${m[1]} is no page`) else if (page.headings.has(heading)) cited.get(m[1]).add(heading) else if (!page.labels.has(heading) && !page.exempt.has(heading)) problems.push(`${at}: spec/${m[1]} has no section "${heading}"`) } } } for (const [page, { headings }] of pages) { const unchecked = UNCHECKED[page] ?? {} for (const [heading, line] of headings) if (!cited.get(page).has(heading) && !(heading in unchecked)) problems.push(`spec/${page}:${line}: no check cites "${heading}"`) for (const heading of Object.keys(unchecked)) { if (!headings.has(heading)) problems.push(`UNCHECKED names "${heading}" of spec/${page}, which has no such section`) else if (cited.get(page).has(heading)) problems.push(`spec/${page}: "${heading}" is cited now: take it out of UNCHECKED`) } } for (const page of Object.keys(UNCHECKED)) if (!pages.has(page)) problems.push(`UNCHECKED names spec/${page}, which is no page`) for (const p of problems) console.log(p) const lines = [...pages.keys()].sort().map((page) => { const n = Object.keys(UNCHECKED[page] ?? {}).length return ` ${page}: ${cited.get(page).size} of ${pages.get(page).headings.size}${n ? `, ${n} unchecked` : ''}` }) console.log(`sections cited by the harness (${relative(root, spec)}/):\n${lines.join('\n')}`) if (problems.length) process.exit(1) const unchecked = Object.values(UNCHECKED).reduce((n, s) => n + Object.keys(s).length, 0) console.log(`all citations name a section, and every section of every page is cited${unchecked ? ` but the ${unchecked} UNCHECKED names` : ''}`)