// The lexicons, as the Lexicon Style Guide and CLAUDE.md ("Lexicons") ask: // goat's lint, less the three warnings kept on purpose; what that lint does // not see; and that each one only grows from what `base` has, since a // schema already published is evolved, never changed, with records and // clients written to it out there. // // node harness/lexicons.mjs [base] base: a git revision; where this // branch left origin/main if that // exists, else none // // Prints each problem as `nsid#def.path: what`, failing if there is any. import { execFileSync } from 'node:child_process' import { readdirSync, readFileSync } from 'node:fs' import { dirname, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') const dir = join(root, 'lexicons') /// Every lexicon file under `dir`, by its path from the repository root. const files = (d) => readdirSync(d, { withFileTypes: true }).flatMap((e) => e.isDirectory() ? files(join(d, e.name)) : e.name.endsWith('.json') ? [join(d, e.name)] : [], ) const git = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }) // What is published is main's. A branch is judged from where it left // main, on what it changed: a feature branch behind main has not taken // back what main added since. A pull into main is checked merged with it, // so that is main itself. const base = process.argv[2] ?? (() => { try { return git('merge-base', 'HEAD', 'origin/main').trim() } catch { return null } })() // What was taken back on purpose before any record used it, each with why. // Once the base has none of it, an entry matches nothing and can go. // - `labellers` landed and was renamed `triagers` the same day: atproto's // labelers are moderation services, and the name read as one of those. const retracted = new Set(['dev.knotless.repo#main.record.labellers', 'dev.knotless.repo#labeller']) const problems = [] const say = (at, what) => retracted.has(at) || problems.push(`${at}: ${what}`) // goat (github.com/bluesky-social/goat), one finding a line. // `release`'s `cargo.index` is Cargo's own line, whose size is the crate's: // `web-sys`'s is 54 KB, and a limit once published cannot be raised. const kept = new Set([ 'dev.knotless.pack large-bytes', 'dev.knotless.release large-string', 'dev.knotless.listReports field-name-syntax', ]) let linted try { linted = execFileSync('goat', ['lex', 'lint', '--json', 'lexicons/'], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }) } catch (e) { if (e.code === 'ENOENT') { console.log( 'goat is not installed: brew install goat, or a release from github.com/bluesky-social/goat', ) process.exit(1) } linted = e.stdout } // goat 0.2.6 knows no space lexicon (`"type": "space"`, from the // permissioned-data alpha) and refuses the whole file, so those files are // checked below instead, against what the alpha's @atproto/lex-document // takes. Only that refusal is let go, and only for a file whose main is a // space: any other finding of it stands. const spaces = new Set( files(dir) .filter((f) => JSON.parse(readFileSync(f, 'utf8')).defs?.main?.type === 'space') .map((f) => relative(root, f)), ) const unknownSpace = (found) => found['lint-name'] === 'schema-json-parse' && found.message === 'unexpected schema type: space' && spaces.has(found['file-path']) for (const line of linted.split('\n').filter(Boolean)) { const found = JSON.parse(line) if (unknownSpace(found)) continue if (!kept.has(`${found.nsid} ${found['lint-name']}`)) say(found.nsid ?? found['file-path'], `${found['lint-name']}: ${found.message}`) } /// Walks every schema in `def`, calling `visit(schema, path)`. const walk = (schema, path, visit) => { if (!schema || typeof schema !== 'object') return visit(schema, path) if (schema.record) walk(schema.record, `${path}.record`, visit) for (const [k, v] of Object.entries(schema.properties ?? {})) walk(v, `${path}.${k}`, visit) if (schema.items) walk(schema.items, `${path}[]`, visit) for (const part of ['parameters', 'input', 'output', 'message']) { const p = schema[part] if (p) walk(p.schema ?? p, `${path}.${part}`, visit) } } // The style CLAUDE.md asks for, of every schema now. const now = new Map() for (const file of files(dir)) { const doc = JSON.parse(readFileSync(file, 'utf8')) now.set(relative(root, file), doc) // What people write: records, the shapes beside them, and what is sent // in. What the server answers it bounds itself, as it does a view, which // only an answer holds, wherever it is defined. const written = doc.defs?.main?.type === 'record' || doc.defs?.main?.type === undefined for (const [name, def] of Object.entries(doc.defs ?? {})) { const at = `${doc.id}#${name}` if (name === 'main' && !def.description) say(at, 'a main def needs a description') const view = /View(Basic|Detailed)?$/.test(name) walk(def, at, (s, path) => { const kept = (written && !view) || path.includes('.input') if (s.enum) say(path, 'knownValues, not enum: an enum can never grow') if ( kept && s.type === 'string' && !s.format && !s.enum && !s.knownValues && s.maxLength === undefined && s.const === undefined ) say(path, 'a string without a format needs a maxLength') if (s.type === 'union' && s.closed) say(path, 'unions stay open') if (s.type === 'string' && s.format === 'handle') say(path, 'accounts by DID, not handle') }) } } // A space lexicon, as the alpha's lexiconSpaceSchema takes it: a key, a // name of 1 to 64 characters for consent screens, and its collections, // each a record lexicon here. It is the file's only def, as goat, which // skips the file, lints none of it. const nsids = new Set( [...now.values()].filter((d) => d.defs?.main?.type === 'record').map((d) => d.id), ) for (const file of spaces) { const doc = now.get(file) const at = `${doc.id}#main` const def = doc.defs.main if (Object.keys(doc.defs).length > 1) say(doc.id, 'a space lexicon has its main def alone') if (!/^(any|nsid|tid|literal:.+)$/.test(def.key ?? '')) say(at, 'a space needs a key: any, nsid, tid or literal:') if (typeof def.name !== 'string' || def.name.length < 1 || def.name.length > 64) say(at, 'a space needs a name of 1 to 64 characters') if (!Array.isArray(def.collections)) say(at, 'a space lists its collections') for (const c of def.collections ?? []) if (!nsids.has(c)) say(`${at}.collections`, `${c} is not a record lexicon here`) } // What each schema in `base` was, against what it is now. const shape = (s) => JSON.stringify({ type: s.type, format: s.format, ref: s.ref, refs: s.refs?.slice().sort() }) const loosened = (before, after, path) => { if (!before || !after) return if (shape(before) !== shape(after) && !(before.type === 'union' && after.type === 'union')) { say(path, `was ${shape(before)}, now ${shape(after)}: a field is never retyped`) return } if (before.type === 'union') { for (const r of before.refs ?? []) if (!(after.refs ?? []).includes(r)) say(path, `the union lost ${r}`) } for (const k of ['maxLength', 'maxGraphemes', 'maxSize', 'maximum']) { if (after[k] !== undefined && (before[k] === undefined || after[k] < before[k])) say(path, `${k} went from ${before[k] ?? 'none'} to ${after[k]}: what was valid must stay so`) } for (const k of ['minLength', 'minGraphemes', 'minimum']) { if (after[k] !== undefined && (before[k] === undefined || after[k] > before[k])) say(path, `${k} went from ${before[k] ?? 'none'} to ${after[k]}: what was valid must stay so`) } if (before.enum) for (const v of before.enum) if (!(after.enum ?? [v]).includes(v)) say(path, `the enum lost ${v}`) if (before.const !== undefined && after.const !== before.const) say(path, 'a const never changes') const was = new Set(before.required ?? []) for (const r of after.required ?? []) if (!was.has(r)) say(path, `${r} is required now: only new optional fields`) for (const [k, v] of Object.entries(before.properties ?? {})) { if (!after.properties?.[k]) say(`${path}.${k}`, 'a field is never removed or renamed') else loosened(v, after.properties[k], `${path}.${k}`) } loosened(before.items, after.items, `${path}[]`) loosened(before.record, after.record, `${path}.record`) for (const part of ['parameters', 'input', 'output', 'message']) { const b = before[part]?.schema ?? before[part] const a = after[part]?.schema ?? after[part] if (b && !a) say(`${path}.${part}`, 'removed') else if (b?.type) loosened(b, a, `${path}.${part}`) } } // Schemas taken out on purpose, each named here as it goes, which a // change may remove: what is said of each is why. const RETIRED = new Set([ // Folded into dev.knotless.getPreferences and putPreferences, with few // accounts then to keep them for. 'dev.knotless.getNotificationPreferences', 'dev.knotless.putNotificationPreferences', // Renamed dev.knotless.orgAcceptance an hour after it went live, before // anyone had written one. 'dev.knotless.membership', ]) if (base) { const listed = git('ls-tree', '-r', '--name-only', base, '--', 'lexicons') .split('\n') .filter((f) => f.endsWith('.json')) for (const file of listed) { const before = JSON.parse(git('show', `${base}:${file}`)) const after = now.get(file) if (!after && RETIRED.has(before.id)) continue if (!after) { say(before.id, `removed since ${base}`) continue } for (const [name, def] of Object.entries(before.defs ?? {})) { const at = `${before.id}#${name}` if (!after.defs?.[name]) say(at, `removed since ${base}`) else loosened(def, after.defs[name], at) // A scope naming a space type alone is granted the collections its // lexicon lists, so taking one out takes writes from those logins. if (def.type === 'space') for (const c of def.collections ?? []) if (!(after.defs?.[name]?.collections ?? []).includes(c)) say(`${at}.collections`, `${c} was taken out`) } } } // Arrays hold objects, which can grow fields, rather than bare values, // which cannot: of arrays new since `base`, as those before it are kept. // A query's parameters are the exception: XRPC lets them hold only values. if (base) { const published = new Set() const listed = git('ls-tree', '-r', '--name-only', base, '--', 'lexicons') .split('\n') .filter((f) => f.endsWith('.json')) for (const file of listed) { const before = JSON.parse(git('show', `${base}:${file}`)) for (const [name, def] of Object.entries(before.defs ?? {})) walk(def, `${before.id}#${name}`, (_, path) => published.add(path)) } for (const doc of now.values()) { for (const [name, def] of Object.entries(doc.defs ?? {})) { walk(def, `${doc.id}#${name}`, (s, path) => { const param = path.includes('.parameters.') const bare = s.type === 'array' && !param && !['object', 'ref', 'union'].includes(s.items?.type) if (bare && !published.has(path)) say(path, 'an array holds objects, which can grow, not bare values') }) } } } for (const p of problems) console.log(p) if (problems.length) process.exit(1) console.log(`lexicons: as the guide asks${base ? `, and only grown since ${base}` : ''}`)