chadtmiller.com/monolith
An atproto PDS
0
0
// The site's pages, as Starlight reads them, in src/content/docs: the
// guides under docs/, the specification's pages (spec/*.md) and the
// compatibility check's and benchmark's, each copied with its first heading
// made its `title`; and two made from the specification itself, so they
// cannot drift from it: every method the server serves, from its lexicon,
// and every variable, from the spec's tables of them. The Markdown stays
// as it is in the repository, where it reads on its own.
//
// node content.mjs
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { dirname, join, relative } from 'node:path'
import { fileURLToPath } from 'node:url'
const here = dirname(fileURLToPath(import.meta.url))
export const root = join(here, '..')
const out = join(here, 'src/content/docs')
/** Where the site is served: the repository's path on its owner's public host. */
export const BASE = '/monolith'
export const REPO = 'https://knotless.dev/chadtmiller.com/monolith'
/** The site's two typefaces (src/monolith.css), which every page preloads. */
export const FONTS = [`${BASE}/fonts/space-grotesk-latin.woff2`, `${BASE}/fonts/jetbrains-mono-latin.woff2`]
const read = (path) => readFileSync(join(root, path), 'utf8')
const mds = (dir) =>
readdirSync(join(root, dir), { withFileTypes: true }).flatMap((e) =>
e.isDirectory()
? mds(`${dir}/${e.name}`)
: e.name.endsWith('.md')
? [`${dir}/${e.name}`]
: [],
)
/** A page's title: its first `# ` heading, as text. */
export const title = (text, path) => {
const heading = text.match(/^# (.+)$/m)
if (!heading) throw new Error(`${path}: no # heading to take its title from`)
return heading[1].replaceAll('`', '')
}
/** The specification's pages: spec/*.md, and none of the specs vendored beneath it. */
export const specPages = () =>
readdirSync(join(root, 'spec'))
.filter((f) => f.endsWith('.md'))
.sort()
/** Generated pages read their links as from spec/, where what they hold comes from. */
const GENERATED = { 'reference/methods': 'spec/.methods.md', 'reference/variables': 'spec/.variables.md' }
/**
* Every page of the site: its slug, and the file in the repository it is
* made from (or, for a generated one, where its links are read from).
*/
export function pages() {
const list = []
for (const source of mds('docs')) {
const slug = source.slice('docs/'.length).replace(/\.md$/, '').replace(/(^|\/)README$/, '')
// The site's root is the landing page (src/pages/index.astro); the guides' index is the overview.
list.push({ slug: slug || 'overview', source })
}
for (const file of specPages()) {
const slug = file === 'PDS.md' ? 'spec' : `spec/${file.replace(/\.md$/, '').toLowerCase()}`
list.push({ slug, source: `spec/${file}` })
}
list.push(
{ slug: 'reference/compat', source: 'compat/README.md' },
{ slug: 'reference/compat-results', source: 'compat/RESULTS.md' },
{ slug: 'reference/bench', source: 'bench/README.md' },
{ slug: 'reference/bench-results', source: 'bench/RESULTS.md' },
)
for (const [slug, source] of Object.entries(GENERATED)) list.push({ slug, source, generated: true })
return list
}
/** A slug's URL on the site. */
export const url = (slug) => `${BASE}/${slug ? `${slug}/` : ''}`
// ---- The methods, from their lexicons ----------------------------------
const lexiconFiles = (dir) =>
readdirSync(join(root, dir), { withFileTypes: true }).flatMap((e) =>
e.isDirectory()
? lexiconFiles(`${dir}/${e.name}`)
: e.name.endsWith('.json')
? [`${dir}/${e.name}`]
: [],
)
/** Every method the server serves: each lexicon in spec/lexicons whose main is a method (spec/PDS.md). */
function methods() {
return lexiconFiles('spec/lexicons')
.map((path) => ({ path, lexicon: JSON.parse(read(path)) }))
.filter(({ lexicon }) => ['query', 'procedure', 'subscription'].includes(lexicon.defs?.main?.type))
.sort((a, b) => a.lexicon.id.localeCompare(b.lexicon.id))
}
/**
* What a method is called (spec/COMMANDS.md): its HTTP verb, its MCP tool
* and its `pds` command, null where it has none.
*/
export function names(lexicon) {
const main = lexicon.defs.main
const [group, name] = lexicon.id.split('.').slice(-2)
const json = (e) => !e || e === JSON_TYPE
return {
verb: { query: 'GET', procedure: 'POST', subscription: 'WS' }[main.type],
tool:
main.type !== 'subscription' && json(main.input?.encoding) && json(main.output?.encoding)
? `${group}_${words(name, '_')}`
: null,
command: main.type === 'subscription' ? null : `pds ${group} ${words(name, '-')}`,
}
}
/** Which page of the spec says what each method does: spec/PDS.md's list of them, by page. */
function specifiedIn() {
const pds = read('spec/PDS.md')
const start = pds.indexOf('## What it serves')
const end = pds.indexOf('\n## ', start + 1)
const pageOf = new Map()
for (const item of pds.slice(start, end).split(/\n- /).slice(1).map((i) => i.split('\n\n')[0])) {
const page = [...item.matchAll(/\(`([A-Z-]+\.md)`\)/g)].at(-1)?.[1]
if (!page) continue
let namespace = ''
for (const [, code] of item.matchAll(/`([^`]+)`/g)) {
if (code.endsWith('.md') || code.includes(' ')) continue
if (code.includes('.')) {
namespace = code.slice(0, code.lastIndexOf('.'))
pageOf.set(code, page)
} else if (namespace) pageOf.set(`${namespace}.${code}`, page)
}
}
return pageOf
}
let methodIndex
/**
* The method an inline code names, on a page made from `source`: its NSID
* as written, or its name alone where one method has it, or, where several
* do (`getRecord`), the one `source` specifies; null for anything else.
*/
export function methodNamed(code, source) {
methodIndex ??= (() => {
const byName = new Map()
for (const { lexicon } of methods()) {
const name = lexicon.id.slice(lexicon.id.lastIndexOf('.') + 1)
byName.set(name, [...(byName.get(name) ?? []), lexicon.id])
}
return { byName, pageOf: specifiedIn() }
})()
if (methodIndex.pageOf.has(code)) return code
const ids = methodIndex.byName.get(code) ?? []
if (ids.length < 2) return ids[0] ?? null
const own = ids.filter((id) => `spec/${methodIndex.pageOf.get(id)}` === source)
return own.length === 1 ? own[0] : null
}
const JSON_TYPE = 'application/json'
const words = (name, sep) => name.replace(/[A-Z]/g, (c) => `${sep}${c.toLowerCase()}`)
const cell = (s) => String(s ?? '').replace(/\s*\n\s*/g, ' ').replaceAll('|', '\\|')
const short = (id, ref) => ref.replace(/^#/, '').replace(`${id}#`, '').replace(/#main$/, '')
/** A field's type as one reads it: `string (did)`, `integer, 1 to 100, default 50`, `ref strongRef`. */
function type(id, field) {
if (!field) return ''
const bits = []
switch (field.type) {
case 'string':
bits.push(field.format ? `string (${field.format})` : 'string')
if (field.maxLength) bits.push(`at most ${field.maxLength} bytes`)
if (field.knownValues) bits.push(field.knownValues.map((v) => `\`${v}\``).join(', '))
if (field.enum) bits.push(`one of ${field.enum.map((v) => `\`${v}\``).join(', ')}`)
break
case 'integer':
bits.push('integer')
if (field.minimum !== undefined && field.maximum !== undefined)
bits.push(`${field.minimum} to ${field.maximum}`)
break
case 'array':
bits.push(`array of ${type(id, field.items)}`)
if (field.maxLength) bits.push(`at most ${field.maxLength}`)
break
case 'ref':
bits.push(`\`${short(id, field.ref)}\``)
break
case 'union':
bits.push(`one of ${field.refs.map((r) => `\`${short(id, r)}\``).join(', ')}`)
break
default:
bits.push(field.type)
}
if (field.default !== undefined) bits.push(`default \`${JSON.stringify(field.default)}\``)
return bits.join(', ')
}
/** Markdown's inline code, and nothing else of it, as HTML. */
const inline = (s) =>
String(s ?? '')
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/`([^`]+)`/g, '<code>$1</code>')
/** An object's fields as a table under a label; a ref or union is a fact of its own, not a table. */
function fields(id, label, object) {
if (!object || object.type === 'ref' || object.type === 'union') return []
const props = Object.entries(object.properties ?? {})
if (!props.length) return []
const required = new Set(object.required ?? [])
return [
`<p class="method-label">${label}</p>`,
'',
'| Name | Type | Description |',
'| --- | --- | --- |',
...props.map(
([name, f]) =>
`| \`${name}\` | ${cell(type(id, f))}${required.has(name) ? '' : ', optional'} | ${cell(f.description)} |`,
),
'',
]
}
function methodsPage() {
const pageOf = specifiedIn()
const all = methods()
const unlisted = all.map((m) => m.lexicon.id).filter((id) => !pageOf.has(id))
if (unlisted.length)
throw new Error(`spec/PDS.md, "What it serves", lists no page for ${unlisted.join(', ')}`)
const lines = [
'---',
'title: Methods',
'---',
'',
`Every method the server serves: each lexicon in [\`spec/lexicons\`](${REPO}/tree/main/spec/lexicons) that defines one, as [\`PDS.md\`](${url('spec')}) says. Each is an XRPC method, an MCP tool and a \`pds\` command at once, named as [\`COMMANDS.md\`](${url('spec/commands')}) says; a method with no tool answers or takes something other than JSON. This page is made from the lexicons, and the page linked under each method says what it does.`,
'',
]
let namespace = ''
for (const { path, lexicon } of all) {
const id = lexicon.id
const main = lexicon.defs.main
const ns = id.slice(0, id.lastIndexOf('.'))
if (ns !== namespace) lines.push(`## \`${(namespace = ns)}\``, '')
const { verb, tool, command } = names(lexicon)
const page = pageOf.get(id)
// What a body is when it is not a table of fields: a ref, or bytes of a type.
const body = (io) =>
io?.schema?.type === 'ref' || io?.schema?.type === 'union'
? inline(type(id, io.schema))
: io?.encoding && io.encoding !== JSON_TYPE
? `<code>${inline(io.encoding)}</code>`
: null
const none = '<span class="method-none">none</span>'
const facts = [
['XRPC', `<span class="method method-${verb.toLowerCase()}">${verb}</span> <code>/xrpc/${id}</code>`],
['CLI', command ? `<code>${command}</code>` : none],
['MCP', tool ? `<code>${tool}</code>` : none],
['Input', body(main.input)],
['Output', body(main.output)],
['Errors', main.errors?.length ? main.errors.map((e) => `<code>${e.name}</code>`).join(' ') : null],
[
'Specified in',
`<a href="${url(`spec/${page.replace(/\.md$/, '').toLowerCase()}`)}">${inline(title(read(`spec/${page}`), `spec/${page}`))}</a> · <a href="${REPO}/blob/main/${path}">lexicon</a>`,
],
].filter(([, v]) => v)
lines.push(`### \`${id}\``, '')
if (main.description) lines.push(cell(main.description), '')
lines.push(
`<dl class="method-facts">${facts.map(([k, v]) => `<dt>${k}</dt><dd>${v}</dd>`).join('')}</dl>`,
'',
)
lines.push(...fields(id, 'Parameters', main.parameters))
lines.push(...fields(id, 'Input', main.input?.schema))
lines.push(...fields(id, 'Output', main.output?.schema))
}
return lines.join('\n')
}
// ---- The variables, from the spec's tables of them ----------------------
/** Each table of variables in a spec page: its header and rows, as written. */
const variableTables = (text) =>
[...text.matchAll(/^(\| *Variable *\|[^\n]*\n\|[ :|-]+\|\n)((?:\|[^\n]*\n?)+)/gm)].map((m) => ({
head: m[1],
rows: m[2].trimEnd().split('\n'),
}))
function variablesPage() {
const pages = specPages()
const lines = [
'---',
'title: Environment variables',
'---',
'',
`Every variable \`pds\` reads, as the specification's pages give them, page by page. This page is made from those pages' tables; each page says the rest of what its variables do.`,
'',
]
const tabled = new Set()
const order = ['PDS.md', ...pages.filter((p) => p !== 'PDS.md')]
for (const page of order) {
const text = read(`spec/${page}`)
const tables = variableTables(text)
if (!tables.length) continue
lines.push(`## [${title(text, `spec/${page}`)}](${page})`, '')
for (const { head, rows } of tables) {
for (const row of rows) for (const [, v] of row.split('|')[1].matchAll(/`(PDS_[A-Z0-9_]+)`/g)) tabled.add(v)
lines.push(head + rows.join('\n'), '')
}
}
// A variable the spec names but no table lists would be missing here.
const untabled = new Set()
for (const page of pages)
for (const [v] of read(`spec/${page}`).matchAll(/PDS_[A-Z0-9_]+/g)) if (!tabled.has(v)) untabled.add(`${v} (${page})`)
if (untabled.size)
throw new Error(`variables the spec names in no table of variables: ${[...untabled].join(', ')}`)
return lines.join('\n')
}
// ---- Writing them out ----------------------------------------------------
const target = (slug) => join(out, slug === '' ? 'index.md' : `${slug}.md`)
/** A slug that is another's directory is written as that directory's index. */
const targetOf = (slug, all) =>
all.some((p) => p.slug.startsWith(`${slug}/`)) ? join(out, slug, 'index.md') : target(slug)
/** The page a file Starlight read is, by its path under src/content/docs. */
export function pageAt(file) {
const rel = relative(out, file).replace(/\.md$/, '').replace(/(^|\/)index$/, '')
return pages().find((p) => p.slug === rel)
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
rmSync(out, { recursive: true, force: true })
const all = pages()
const generated = { 'reference/methods': methodsPage, 'reference/variables': variablesPage }
for (const { slug, source } of all) {
let text
if (generated[slug]) text = generated[slug]()
else {
if (!existsSync(join(root, source))) throw new Error(`${source}: missing`)
const body = read(source)
const heading = body.match(/^# .+\n+/m)
text = `---\ntitle: ${JSON.stringify(title(body, source))}\n---\n\n${body.slice(0, heading.index)}${body.slice(heading.index + heading[0].length)}`
}
const file = targetOf(slug, all)
mkdirSync(dirname(file), { recursive: true })
writeFileSync(file, text)
}
console.log(`${all.length} pages`)
}