type Json = Record; type Def = Json & { type: string; description?: string }; export type Lexicon = { id: string; file: string; description?: string; defs: Record }; export type RefPage = { out: string; title: string; body: string; aside: string }; export type Helpers = { esc: (s: string) => string; rel: (from: string, to: string) => string; hl: (code: string, lang: string) => string; }; const FAMILIES = [ "com.publicdomainrelay.temp", "com.fedfork.exec", "computer.socialweb", "software.release", "link.ident", "network.attested", "com.atproto", ]; const XRPC = new Set(["query", "procedure", "subscription"]); export const lexiconOut = (nsid: string) => `docs/lexicons/${nsid}.html`; const LEX_INDEX = "docs/lexicons/index.html"; const XRPC_INDEX = "docs/xrpc/index.html"; export async function loadLexicons(root: string): Promise> { const lexs = new Map(); const problems: string[] = []; async function walk(rel: string) { for await (const e of Deno.readDir(`${root}/${rel}`)) { const p = `${rel}/${e.name}`; if (e.isDirectory) await walk(p); else if (p.endsWith(".json")) { let j: Json; try { j = JSON.parse(await Deno.readTextFile(`${root}/${p}`)); } catch (err) { problems.push(`${p}: not JSON: ${err}`); continue; } const want = p.replace(/^lexicons\//, "").replace(/\.json$/, "").split("/").join("."); if (j.lexicon !== 1) problems.push(`${p}: lexicon is not 1`); if (j.id !== want) problems.push(`${p}: id ${j.id} is not ${want}`); if (!j.defs || typeof j.defs !== "object") problems.push(`${p}: no defs`); else lexs.set(want, { id: want, file: p, description: j.description as string | undefined, defs: j.defs as Record }); } } } await walk("lexicons"); for (const lex of lexs.values()) { for (const [name, def] of Object.entries(lex.defs)) { if (typeof def?.type !== "string") problems.push(`${lex.file}: #${name} has no type`); walkRefs(def, (ref) => { const [nsid, frag] = splitRef(lex.id, ref); const target = lexs.get(nsid); if (!target) problems.push(`${lex.file}: #${name} refers to ${ref}: no lexicon ${nsid}`); else if (!target.defs[frag]) problems.push(`${lex.file}: #${name} refers to ${ref}: ${nsid} has no #${frag}`); }); } } if (problems.length) throw new Error(`lexicons:\n${problems.join("\n")}`); return lexs; } function walkRefs(v: unknown, f: (ref: string) => void) { if (Array.isArray(v)) return v.forEach((x) => walkRefs(x, f)); if (!v || typeof v !== "object") return; const o = v as Json; if (o.type === "ref" && typeof o.ref === "string") f(o.ref); if (o.type === "union" && Array.isArray(o.refs)) (o.refs as string[]).forEach(f); for (const x of Object.values(o)) walkRefs(x, f); } function splitRef(from: string, ref: string): [string, string] { const [nsid, frag] = ref.split("#"); return [nsid || from, frag || "main"]; } function family(nsid: string): string { return FAMILIES.find((f) => nsid.startsWith(`${f}.`)) ?? nsid.split(".").slice(0, 2).join("."); } function group(nsid: string): string { return nsid.split(".").slice(0, -1).join("."); } function firstSentence(s = ""): string { const m = s.match(/^(.+?[.])(\s|$)/); return (m ? m[1] : s).trim(); } function mainType(lex: Lexicon): string { return lex.defs.main?.type ?? "defs"; } function sorted(lexs: Map): Lexicon[] { const rank = (n: string) => { const i = FAMILIES.indexOf(family(n)); return i < 0 ? FAMILIES.length : i; }; return [...lexs.values()].sort((a, b) => rank(a.id) - rank(b.id) || a.id.localeCompare(b.id)); } function groups(list: Lexicon[]): Map> { const fams = new Map>(); for (const lex of list) { const fam = family(lex.id); const g = group(lex.id); if (!fams.has(fam)) fams.set(fam, new Map()); const gs = fams.get(fam)!; if (!gs.has(g)) gs.set(g, []); gs.get(g)!.push(lex); } return fams; } export type UsedIn = Map; export function notInUse(lex: Lexicon): boolean { const d = (lex.defs.main?.description ?? lex.description ?? "").trim(); return /^(not in use|defined, not (written|served|called|used))\b/i.test(d); } export function lexiconKit(lexs: Map, h: Helpers) { const { esc, rel } = h; const list = sorted(lexs); const fams = groups(list); const badge = (t: string) => `${esc(t)}`; const nsidHtml = (nsid: string) => { const i = nsid.lastIndexOf("."); return `${esc(nsid.slice(0, i + 1))}${esc(nsid.slice(i + 1))}`; }; function aside(here: string, mode: "lexicons" | "xrpc"): string { const parts: string[] = [ ``, ]; for (const [fam, gs] of fams) { const famShown = [...gs.values()].flat().filter((l) => mode !== "xrpc" || XRPC.has(mainType(l))); if (!famShown.length) continue; parts.push(``); for (const [g, items] of gs) { const shown = mode === "xrpc" ? items.filter((l) => XRPC.has(mainType(l))) : items; if (!shown.length) continue; const label = g === fam ? fam.split(".").pop()! : g.slice(fam.length + 1); parts.push(`
    `); for (const l of shown) { const target = lexiconOut(l.id); const cur = target === here ? ' aria-current="page"' : ""; parts.push(`
  • ${esc(l.id.slice(g.length + 1))}
  • `); } parts.push("
"); } } return parts.join("\n"); } function refLink(here: string, from: string, ref: string): string { const [nsid, frag] = splitRef(from, ref); const href = rel(here, lexiconOut(nsid)) + (frag === "main" ? "" : `#${frag}`); const text = ref.startsWith("#") ? ref : ref; return `${esc(text)}`; } function typeHtml(here: string, from: string, f: Json): string { const t = String(f.type); if (t === "ref") return refLink(here, from, String(f.ref)); if (t === "union") { const refs = (f.refs as string[] ?? []).map((r) => refLink(here, from, r)).join(" | "); return `union${f.closed ? " (closed)" : ""} ${refs}`; } if (t === "array") return `array of ${typeHtml(here, from, f.items as Json)}`; const fmt = f.format ? ` ${esc(String(f.format))}` : ""; return `${esc(t)}${fmt}`; } function constraints(f: Json): string { const out: string[] = []; const show = (k: string, v: unknown) => out.push(`${esc(k)} ${esc(typeof v === "string" ? v : JSON.stringify(v))}`); for (const k of ["const", "default", "enum", "knownValues", "minimum", "maximum", "minLength", "maxLength", "minGraphemes", "maxGraphemes", "maxSize", "accept"]) { if (f[k] !== undefined) show(k, f[k]); } if (f.type === "array") { for (const k of ["minLength", "maxLength"]) if (f[k] !== undefined && !out.length) show(k, f[k]); const it = f.items as Json | undefined; if (it) for (const k of ["knownValues", "enum", "maxLength", "format"]) if (it[k] !== undefined) show(`items.${k}`, it[k]); } return out.join(" "); } function table(here: string, from: string, obj: Json, label: string, id: string): string { const props = (obj.properties ?? {}) as Record; const req = new Set((obj.required ?? []) as string[]); const nullable = new Set((obj.nullable ?? []) as string[]); const names = Object.keys(props); if (!names.length) return `

${esc(label)}: none.

`; const rows = names.map((n) => { const f = props[n]; const mark = req.has(n) ? ' *' : ""; const nul = nullable.has(n) ? ' nullable' : ""; const desc = f.description ? `
${esc(String(f.description))}
` : ""; const c = constraints(f); return `${esc(n)}${mark}${typeHtml(here, from, f)}${nul}${desc}${c ? `
${c}
` : ""}`; }).join("\n"); return `
${rows}
${esc(label)}typedescription
`; } function body(here: string, from: string, schema: Json | undefined, label: string, id: string): string { if (!schema) return ""; const enc = schema.encoding ? `

encoding ${esc(String(schema.encoding))}

` : ""; const s = schema.schema as Json | undefined; const desc = schema.description ? `

${esc(String(schema.description))}

` : ""; let inner = ""; if (s) { if (s.type === "object") inner = table(here, from, s, "field", id); else inner = `

${typeHtml(here, from, s)}

`; } return `

${esc(label)}

${enc}${desc}${inner}`; } function errors(def: Def): string { const errs = (def.errors ?? []) as Json[]; if (!errs.length) return ""; const rows = errs.map((e) => `${esc(String(e.name))}${esc(String(e.description ?? ""))}`).join("\n"); return `

errors

${rows}
errordescription
`; } function defSection(here: string, lex: Lexicon, name: string, def: Def): string { const parts: string[] = []; const isMain = name === "main"; if (!isMain) { parts.push(`

#${esc(name)} ${badge(def.type)}

`); if (def.description) parts.push(`

${esc(def.description)}

`); } const t = def.type; if (t === "record") { parts.push(`

record

`); const rec = def.record as Json; if (rec?.description) parts.push(`

${esc(String(rec.description))}

`); parts.push(table(here, lex.id, rec ?? {}, "field", "s-record")); } else if (t === "query" || t === "procedure" || t === "subscription") { const params = def.parameters as Json | undefined; if (params) parts.push(`

parameters

${table(here, lex.id, params, "parameter", "s-parameters")}`); parts.push(body(here, lex.id, def.input as Json | undefined, "input", "s-input")); parts.push(body(here, lex.id, def.output as Json | undefined, "output", "s-output")); parts.push(body(here, lex.id, def.message as Json | undefined, "message", "s-message")); parts.push(errors(def)); } else if (t === "object") { parts.push(table(here, lex.id, def, "field", isMain ? "s-fields" : name)); } else if (t === "array") { parts.push(`

${typeHtml(here, lex.id, def)}

`); } else { const c = constraints(def); parts.push(`

${typeHtml(here, lex.id, def)}${c ? ` ${c}` : ""}

`); } return parts.join("\n"); } const unused = `not in use`; function include(nsid: string, here: string): string { const lex = lexs.get(nsid); if (!lex) throw new Error(`${here}: include of ${nsid}: no such lexicon`); const main = lex.defs.main; const href = rel(here, lexiconOut(nsid)); const meta = [badge(mainType(lex))]; if (main?.key) meta.push(`key ${esc(String(main.key))}`); if (notInUse(lex)) meta.push(unused); const prefix = `inc-${nsid.replace(/\./g, "-")}-`; const inner = main ? defSection(here, lex, "main", main) .replace(/

([^<]*)<\/h3>/g, '

$1

') .replace(/ id="/g, ` id="${prefix}`) : ""; const others = Object.keys(lex.defs).filter((n) => n !== "main"); const more = others.length ? `

also: ${others.map((n) => `#${esc(n)}`).join(" ")}

` : ""; const desc = main?.description ?? lex.description; return `

${esc(nsid)} ${meta.join(" ")}

${desc ? `

${esc(desc)}

` : ""}${inner}${more}
\n`; } function usedList(here: string, nsid: string, usedIn: UsedIn): string { const pages = (usedIn.get(nsid) ?? []).filter((p) => p.out !== here); return pages.map((p) => `${esc(p.title)}`).join(", "); } function glossary(here: string, usedIn: UsedIn): string { const out: string[] = []; for (const [fam, gs] of fams) { for (const [g, items] of gs) { out.push(`

${esc(g)}

`); for (const l of items) { const d = firstSentence(l.defs.main?.description ?? l.description); out.push(``); } out.push(`
NSIDtypewhatdocs
${esc(l.id.slice(g.length + 1))}${badge(mainType(l))}${notInUse(l) ? ` ${unused}` : ""}${esc(d)}${usedList(here, l.id, usedIn)}
`); } void fam; } return out.join("\n"); } function pages(usedIn: UsedIn): RefPage[] { const pages: RefPage[] = []; for (const lex of list) { const here = lexiconOut(lex.id); const main = lex.defs.main; const t = mainType(lex); const meta: string[] = [badge(t)]; if (main?.key) meta.push(`key ${esc(String(main.key))}`); if (main?.type === "query") meta.push(`HTTP GET /xrpc/${esc(lex.id)}`); if (main?.type === "procedure") meta.push(`HTTP POST /xrpc/${esc(lex.id)}`); if (main?.type === "subscription") meta.push(`WebSocket /xrpc/${esc(lex.id)}`); if (notInUse(lex)) meta.push(unused); const used = usedList(here, lex.id, usedIn); const others = Object.keys(lex.defs).filter((n) => n !== "main"); const toc = others.length ? `

definitions: ${others.map((n) => `#${esc(n)}`).join(" ")}

` : ""; const desc = main?.description ?? lex.description; const sections = [main ? defSection(here, lex, "main", main) : "", ...others.map((n) => defSection(here, lex, n, lex.defs[n]))]; const raw = h.hl(JSON.stringify({ lexicon: 1, id: lex.id, ...(lex.description ? { description: lex.description } : {}), defs: lex.defs }, null, 2), "json"); const body = [ `

lexicon

`, `

${nsidHtml(lex.id)}

`, `

${meta.join(" ")}

`, desc ? `

${esc(desc)}

` : "", toc, ...sections, `

used in

${used || "no page of these docs names it yet."}

`, `
raw schema: ${esc(lex.file)}${raw}
`, ].join("\n"); pages.push({ out: here, title: lex.id, body, aside: aside(here, XRPC.has(t) ? "xrpc" : "lexicons") }); } const index: string[] = [ `

reference

`, `

Lexicons

`, `

Every record, object and XRPC method of the socialweb.computer market, generated from the repository's lexicons/ (${list.length} lexicons). XRPC methods only: xrpc.

`, ]; for (const [fam, gs] of fams) { const n = [...gs.values()].reduce((a, b) => a + b.length, 0); index.push(`

${esc(fam)}.* ${n}

`); for (const [g, items] of gs) { index.push(`

${esc(g)}

`); for (const l of items) { index.push(``); } index.push(`
${esc(l.id.slice(g.length + 1))}${badge(mainType(l))}${notInUse(l) ? ` ${unused}` : ""}${esc(firstSentence(l.defs.main?.description ?? l.description))}
`); } } pages.push({ out: LEX_INDEX, title: "Lexicons", body: index.join("\n"), aside: aside(LEX_INDEX, "lexicons") }); const methods = list.filter((l) => XRPC.has(mainType(l))); const xi: string[] = [ `

reference

`, `

XRPC

`, `

The ${methods.length} XRPC methods (queries, procedures, subscriptions) the market's services serve at /xrpc/<nsid>, generated from lexicons/. Records and objects: lexicons.

`, ]; for (const [, gs] of fams) { for (const [g, items] of gs) { const ms = items.filter((l) => XRPC.has(mainType(l))); if (!ms.length) continue; xi.push(`

${esc(g)} ${ms.length}

`); for (const l of ms) { xi.push(``); } xi.push(`
${esc(l.id.slice(g.length + 1))}${badge(mainType(l))}${esc(firstSentence(l.defs.main?.description))}
`); } } pages.push({ out: XRPC_INDEX, title: "XRPC", body: xi.join("\n"), aside: aside(XRPC_INDEX, "xrpc") }); return pages; } return { include, glossary, pages }; } export function nsidLookup(lexs: Map): (text: string) => string | undefined { const ids = [...lexs.keys()]; return (text: string) => { const t = text.trim(); if (!/^[a-zA-Z][\w.]*[a-zA-Z0-9]$/.test(t) || !t.includes(".")) return undefined; if (lexs.has(t)) return t; if (lexs.has(`com.publicdomainrelay.temp.${t}`)) return `com.publicdomainrelay.temp.${t}`; const hits = ids.filter((id) => id.endsWith(`.${t}`)); return hits.length === 1 ? hits[0] : undefined; }; }