0
0
type Json = Record<string, unknown>;
type Def = Json & { type: string; description?: string };
export type Lexicon = { id: string; file: string; description?: string; defs: Record<string, Def> };
export type RefPage = { out: string; title: string; body: string; aside: string };
export type Helpers = {
esc: (s: string) => string;
rel: (from: string, to: string) => string;
hl: (code: string, lang: string) => string;
};
const FAMILIES = [
"com.publicdomainrelay.temp",
"com.fedfork.exec",
"computer.socialweb",
"software.release",
"link.ident",
"network.attested",
"com.atproto",
];
const XRPC = new Set(["query", "procedure", "subscription"]);
export const lexiconOut = (nsid: string) => `docs/lexicons/${nsid}.html`;
const LEX_INDEX = "docs/lexicons/index.html";
const XRPC_INDEX = "docs/xrpc/index.html";
export async function loadLexicons(root: string): Promise<Map<string, Lexicon>> {
const lexs = new Map<string, Lexicon>();
const problems: string[] = [];
async function walk(rel: string) {
for await (const e of Deno.readDir(`${root}/${rel}`)) {
const p = `${rel}/${e.name}`;
if (e.isDirectory) await walk(p);
else if (p.endsWith(".json")) {
let j: Json;
try {
j = JSON.parse(await Deno.readTextFile(`${root}/${p}`));
} catch (err) {
problems.push(`${p}: not JSON: ${err}`);
continue;
}
const want = p.replace(/^lexicons\//, "").replace(/\.json$/, "").split("/").join(".");
if (j.lexicon !== 1) problems.push(`${p}: lexicon is not 1`);
if (j.id !== want) problems.push(`${p}: id ${j.id} is not ${want}`);
if (!j.defs || typeof j.defs !== "object") problems.push(`${p}: no defs`);
else lexs.set(want, { id: want, file: p, description: j.description as string | undefined, defs: j.defs as Record<string, Def> });
}
}
}
await walk("lexicons");
for (const lex of lexs.values()) {
for (const [name, def] of Object.entries(lex.defs)) {
if (typeof def?.type !== "string") problems.push(`${lex.file}: #${name} has no type`);
walkRefs(def, (ref) => {
const [nsid, frag] = splitRef(lex.id, ref);
const target = lexs.get(nsid);
if (!target) problems.push(`${lex.file}: #${name} refers to ${ref}: no lexicon ${nsid}`);
else if (!target.defs[frag]) problems.push(`${lex.file}: #${name} refers to ${ref}: ${nsid} has no #${frag}`);
});
}
}
if (problems.length) throw new Error(`lexicons:\n${problems.join("\n")}`);
return lexs;
}
function walkRefs(v: unknown, f: (ref: string) => void) {
if (Array.isArray(v)) return v.forEach((x) => walkRefs(x, f));
if (!v || typeof v !== "object") return;
const o = v as Json;
if (o.type === "ref" && typeof o.ref === "string") f(o.ref);
if (o.type === "union" && Array.isArray(o.refs)) (o.refs as string[]).forEach(f);
for (const x of Object.values(o)) walkRefs(x, f);
}
function splitRef(from: string, ref: string): [string, string] {
const [nsid, frag] = ref.split("#");
return [nsid || from, frag || "main"];
}
function family(nsid: string): string {
return FAMILIES.find((f) => nsid.startsWith(`${f}.`)) ?? nsid.split(".").slice(0, 2).join(".");
}
function group(nsid: string): string {
return nsid.split(".").slice(0, -1).join(".");
}
function firstSentence(s = ""): string {
const m = s.match(/^(.+?[.])(\s|$)/);
return (m ? m[1] : s).trim();
}
function mainType(lex: Lexicon): string {
return lex.defs.main?.type ?? "defs";
}
function sorted(lexs: Map<string, Lexicon>): Lexicon[] {
const rank = (n: string) => {
const i = FAMILIES.indexOf(family(n));
return i < 0 ? FAMILIES.length : i;
};
return [...lexs.values()].sort((a, b) => rank(a.id) - rank(b.id) || a.id.localeCompare(b.id));
}
function groups(list: Lexicon[]): Map<string, Map<string, Lexicon[]>> {
const fams = new Map<string, Map<string, Lexicon[]>>();
for (const lex of list) {
const fam = family(lex.id);
const g = group(lex.id);
if (!fams.has(fam)) fams.set(fam, new Map());
const gs = fams.get(fam)!;
if (!gs.has(g)) gs.set(g, []);
gs.get(g)!.push(lex);
}
return fams;
}
export type UsedIn = Map<string, { out: string; title: string }[]>;
export function notInUse(lex: Lexicon): boolean {
const d = (lex.defs.main?.description ?? lex.description ?? "").trim();
return /^(not in use|defined, not (written|served|called|used))\b/i.test(d);
}
export function lexiconKit(lexs: Map<string, Lexicon>, h: Helpers) {
const { esc, rel } = h;
const list = sorted(lexs);
const fams = groups(list);
const badge = (t: string) => `<span class="badge t-${esc(t)}">${esc(t)}</span>`;
const nsidHtml = (nsid: string) => {
const i = nsid.lastIndexOf(".");
return `<span class="ns">${esc(nsid.slice(0, i + 1))}</span><span class="nm">${esc(nsid.slice(i + 1))}</span>`;
};
function aside(here: string, mode: "lexicons" | "xrpc"): string {
const parts: string[] = [
`<p class="nav-top"><a href="${esc(rel(here, LEX_INDEX))}"${here === LEX_INDEX ? ' aria-current="page"' : ""}>lexicons</a>` +
` <a href="${esc(rel(here, XRPC_INDEX))}"${here === XRPC_INDEX ? ' aria-current="page"' : ""}>xrpc</a></p>`,
];
for (const [fam, gs] of fams) {
const famShown = [...gs.values()].flat().filter((l) => mode !== "xrpc" || XRPC.has(mainType(l)));
if (!famShown.length) continue;
parts.push(`<p class="nav-fam">${esc(fam)}.*</p>`);
for (const [g, items] of gs) {
const shown = mode === "xrpc" ? items.filter((l) => XRPC.has(mainType(l))) : items;
if (!shown.length) continue;
const label = g === fam ? fam.split(".").pop()! : g.slice(fam.length + 1);
parts.push(`<p class="nav-group"><span>${esc(label)}</span><span class="count">${shown.length}</span></p><ul>`);
for (const l of shown) {
const target = lexiconOut(l.id);
const cur = target === here ? ' aria-current="page"' : "";
parts.push(`<li><a href="${esc(rel(here, target))}"${cur}>${esc(l.id.slice(g.length + 1))}</a></li>`);
}
parts.push("</ul>");
}
}
return parts.join("\n");
}
function refLink(here: string, from: string, ref: string): string {
const [nsid, frag] = splitRef(from, ref);
const href = rel(here, lexiconOut(nsid)) + (frag === "main" ? "" : `#${frag}`);
const text = ref.startsWith("#") ? ref : ref;
return `<a class="ref" href="${esc(href)}"><code>${esc(text)}</code></a>`;
}
function typeHtml(here: string, from: string, f: Json): string {
const t = String(f.type);
if (t === "ref") return refLink(here, from, String(f.ref));
if (t === "union") {
const refs = (f.refs as string[] ?? []).map((r) => refLink(here, from, r)).join(" | ");
return `<span class="ty">union</span>${f.closed ? " (closed)" : ""} ${refs}`;
}
if (t === "array") return `<span class="ty">array</span> of ${typeHtml(here, from, f.items as Json)}`;
const fmt = f.format ? ` <span class="fmt">${esc(String(f.format))}</span>` : "";
return `<span class="ty">${esc(t)}</span>${fmt}`;
}
function constraints(f: Json): string {
const out: string[] = [];
const show = (k: string, v: unknown) => out.push(`<span class="c"><span class="ck">${esc(k)}</span> <code>${esc(typeof v === "string" ? v : JSON.stringify(v))}</code></span>`);
for (const k of ["const", "default", "enum", "knownValues", "minimum", "maximum", "minLength", "maxLength", "minGraphemes", "maxGraphemes", "maxSize", "accept"]) {
if (f[k] !== undefined) show(k, f[k]);
}
if (f.type === "array") {
for (const k of ["minLength", "maxLength"]) if (f[k] !== undefined && !out.length) show(k, f[k]);
const it = f.items as Json | undefined;
if (it) for (const k of ["knownValues", "enum", "maxLength", "format"]) if (it[k] !== undefined) show(`items.${k}`, it[k]);
}
return out.join(" ");
}
function table(here: string, from: string, obj: Json, label: string, id: string): string {
const props = (obj.properties ?? {}) as Record<string, Json>;
const req = new Set((obj.required ?? []) as string[]);
const nullable = new Set((obj.nullable ?? []) as string[]);
const names = Object.keys(props);
if (!names.length) return `<p class="empty">${esc(label)}: none.</p>`;
const rows = names.map((n) => {
const f = props[n];
const mark = req.has(n) ? ' <span class="req" title="required">*</span>' : "";
const nul = nullable.has(n) ? ' <span class="fmt">nullable</span>' : "";
const desc = f.description ? `<div class="desc">${esc(String(f.description))}</div>` : "";
const c = constraints(f);
return `<tr id="${esc(id)}-${esc(n)}"><td class="pn"><code>${esc(n)}</code>${mark}</td><td>${typeHtml(here, from, f)}${nul}</td><td>${desc}${c ? `<div class="cons">${c}</div>` : ""}</td></tr>`;
}).join("\n");
return `<div class="table"><table class="props"><thead><tr><th>${esc(label)}</th><th>type</th><th>description</th></tr></thead><tbody>${rows}</tbody></table></div>`;
}
function body(here: string, from: string, schema: Json | undefined, label: string, id: string): string {
if (!schema) return "";
const enc = schema.encoding ? `<p class="meta">encoding <code>${esc(String(schema.encoding))}</code></p>` : "";
const s = schema.schema as Json | undefined;
const desc = schema.description ? `<p>${esc(String(schema.description))}</p>` : "";
let inner = "";
if (s) {
if (s.type === "object") inner = table(here, from, s, "field", id);
else inner = `<p>${typeHtml(here, from, s)}</p>`;
}
return `<h3 id="${esc(id)}">${esc(label)}</h3>${enc}${desc}${inner}`;
}
function errors(def: Def): string {
const errs = (def.errors ?? []) as Json[];
if (!errs.length) return "";
const rows = errs.map((e) => `<tr><td class="pn"><code>${esc(String(e.name))}</code></td><td>${esc(String(e.description ?? ""))}</td></tr>`).join("\n");
return `<h3 id="s-errors">errors</h3><div class="table"><table class="props"><thead><tr><th>error</th><th>description</th></tr></thead><tbody>${rows}</tbody></table></div>`;
}
function defSection(here: string, lex: Lexicon, name: string, def: Def): string {
const parts: string[] = [];
const isMain = name === "main";
if (!isMain) {
parts.push(`<h2 id="${esc(name)}" class="def"><a class="anchor" href="#${esc(name)}"><code>#${esc(name)}</code></a> ${badge(def.type)}</h2>`);
if (def.description) parts.push(`<p>${esc(def.description)}</p>`);
}
const t = def.type;
if (t === "record") {
parts.push(`<h3 id="s-record">record</h3>`);
const rec = def.record as Json;
if (rec?.description) parts.push(`<p>${esc(String(rec.description))}</p>`);
parts.push(table(here, lex.id, rec ?? {}, "field", "s-record"));
} else if (t === "query" || t === "procedure" || t === "subscription") {
const params = def.parameters as Json | undefined;
if (params) parts.push(`<h3 id="s-parameters">parameters</h3>${table(here, lex.id, params, "parameter", "s-parameters")}`);
parts.push(body(here, lex.id, def.input as Json | undefined, "input", "s-input"));
parts.push(body(here, lex.id, def.output as Json | undefined, "output", "s-output"));
parts.push(body(here, lex.id, def.message as Json | undefined, "message", "s-message"));
parts.push(errors(def));
} else if (t === "object") {
parts.push(table(here, lex.id, def, "field", isMain ? "s-fields" : name));
} else if (t === "array") {
parts.push(`<p>${typeHtml(here, lex.id, def)}</p>`);
} else {
const c = constraints(def);
parts.push(`<p>${typeHtml(here, lex.id, def)}${c ? ` <span class="cons">${c}</span>` : ""}</p>`);
}
return parts.join("\n");
}
const unused = `<span class="badge t-unused" title="the lexicon's description says it is defined but not written or served today">not in use</span>`;
function include(nsid: string, here: string): string {
const lex = lexs.get(nsid);
if (!lex) throw new Error(`${here}: include of ${nsid}: no such lexicon`);
const main = lex.defs.main;
const href = rel(here, lexiconOut(nsid));
const meta = [badge(mainType(lex))];
if (main?.key) meta.push(`<span class="kv">key <code>${esc(String(main.key))}</code></span>`);
if (notInUse(lex)) meta.push(unused);
const prefix = `inc-${nsid.replace(/\./g, "-")}-`;
const inner = main
? defSection(here, lex, "main", main)
.replace(/<h3 id="[^"]+">([^<]*)<\/h3>/g, '<p class="lexinc-sub">$1</p>')
.replace(/ id="/g, ` id="${prefix}`)
: "";
const others = Object.keys(lex.defs).filter((n) => n !== "main");
const more = others.length ? `<p class="lexinc-more">also: ${others.map((n) => `<a href="${esc(href)}#${esc(n)}"><code>#${esc(n)}</code></a>`).join(" ")}</p>` : "";
const desc = main?.description ?? lex.description;
return `<div class="lexinc"><p class="lexinc-head"><a href="${esc(href)}"><code>${esc(nsid)}</code></a> ${meta.join(" ")}</p>${desc ? `<p class="lexinc-desc">${esc(desc)}</p>` : ""}${inner}${more}</div>\n`;
}
function usedList(here: string, nsid: string, usedIn: UsedIn): string {
const pages = (usedIn.get(nsid) ?? []).filter((p) => p.out !== here);
return pages.map((p) => `<a href="${esc(rel(here, p.out))}">${esc(p.title)}</a>`).join(", ");
}
function glossary(here: string, usedIn: UsedIn): string {
const out: string[] = [];
for (const [fam, gs] of fams) {
for (const [g, items] of gs) {
out.push(`<h3 id="${esc(g)}">${esc(g)}</h3><div class="table"><table class="props"><thead><tr><th>NSID</th><th>type</th><th>what</th><th>docs</th></tr></thead><tbody>`);
for (const l of items) {
const d = firstSentence(l.defs.main?.description ?? l.description);
out.push(`<tr><td class="pn"><a href="${esc(rel(here, lexiconOut(l.id)))}"><code>${esc(l.id.slice(g.length + 1))}</code></a></td><td>${badge(mainType(l))}${notInUse(l) ? ` ${unused}` : ""}</td><td>${esc(d)}</td><td>${usedList(here, l.id, usedIn)}</td></tr>`);
}
out.push(`</tbody></table></div>`);
}
void fam;
}
return out.join("\n");
}
function pages(usedIn: UsedIn): RefPage[] {
const pages: RefPage[] = [];
for (const lex of list) {
const here = lexiconOut(lex.id);
const main = lex.defs.main;
const t = mainType(lex);
const meta: string[] = [badge(t)];
if (main?.key) meta.push(`<span class="kv">key <code>${esc(String(main.key))}</code></span>`);
if (main?.type === "query") meta.push(`<span class="kv">HTTP <code>GET /xrpc/${esc(lex.id)}</code></span>`);
if (main?.type === "procedure") meta.push(`<span class="kv">HTTP <code>POST /xrpc/${esc(lex.id)}</code></span>`);
if (main?.type === "subscription") meta.push(`<span class="kv">WebSocket <code>/xrpc/${esc(lex.id)}</code></span>`);
if (notInUse(lex)) meta.push(unused);
const used = usedList(here, lex.id, usedIn);
const others = Object.keys(lex.defs).filter((n) => n !== "main");
const toc = others.length
? `<p class="toc">definitions: ${others.map((n) => `<a href="#${esc(n)}"><code>#${esc(n)}</code></a>`).join(" ")}</p>`
: "";
const desc = main?.description ?? lex.description;
const sections = [main ? defSection(here, lex, "main", main) : "", ...others.map((n) => defSection(here, lex, n, lex.defs[n]))];
const raw = h.hl(JSON.stringify({ lexicon: 1, id: lex.id, ...(lex.description ? { description: lex.description } : {}), defs: lex.defs }, null, 2), "json");
const body = [
`<p class="kicker">lexicon</p>`,
`<h1 class="nsid">${nsidHtml(lex.id)}</h1>`,
`<p class="meta">${meta.join(" ")}</p>`,
desc ? `<p class="lede">${esc(desc)}</p>` : "",
toc,
...sections,
`<h2 id="used-in" class="def">used in</h2><p>${used || "no page of these docs names it yet."}</p>`,
`<details id="raw" class="raw"><summary>raw schema: <code>${esc(lex.file)}</code></summary>${raw}</details>`,
].join("\n");
pages.push({ out: here, title: lex.id, body, aside: aside(here, XRPC.has(t) ? "xrpc" : "lexicons") });
}
const index: string[] = [
`<p class="kicker">reference</p>`,
`<h1 class="nsid"><span class="nm">Lexicons</span></h1>`,
`<p class="lede">Every record, object and XRPC method of the socialweb.computer market, generated from the repository's <code>lexicons/</code> (${list.length} lexicons). XRPC methods only: <a href="${esc(rel(LEX_INDEX, XRPC_INDEX))}">xrpc</a>.</p>`,
];
for (const [fam, gs] of fams) {
const n = [...gs.values()].reduce((a, b) => a + b.length, 0);
index.push(`<h2 id="${esc(fam)}">${esc(fam)}.* <span class="count">${n}</span></h2>`);
for (const [g, items] of gs) {
index.push(`<h3 id="${esc(g)}">${esc(g)}</h3><div class="table"><table class="props"><tbody>`);
for (const l of items) {
index.push(`<tr><td class="pn"><a href="${esc(rel(LEX_INDEX, lexiconOut(l.id)))}"><code>${esc(l.id.slice(g.length + 1))}</code></a></td><td>${badge(mainType(l))}${notInUse(l) ? ` ${unused}` : ""}</td><td>${esc(firstSentence(l.defs.main?.description ?? l.description))}</td></tr>`);
}
index.push(`</tbody></table></div>`);
}
}
pages.push({ out: LEX_INDEX, title: "Lexicons", body: index.join("\n"), aside: aside(LEX_INDEX, "lexicons") });
const methods = list.filter((l) => XRPC.has(mainType(l)));
const xi: string[] = [
`<p class="kicker">reference</p>`,
`<h1 class="nsid"><span class="nm">XRPC</span></h1>`,
`<p class="lede">The ${methods.length} XRPC methods (queries, procedures, subscriptions) the market's services serve at <code>/xrpc/<nsid></code>, generated from <code>lexicons/</code>. Records and objects: <a href="${esc(rel(XRPC_INDEX, LEX_INDEX))}">lexicons</a>.</p>`,
];
for (const [, gs] of fams) {
for (const [g, items] of gs) {
const ms = items.filter((l) => XRPC.has(mainType(l)));
if (!ms.length) continue;
xi.push(`<h2 id="${esc(g)}">${esc(g)} <span class="count">${ms.length}</span></h2><div class="table"><table class="props"><tbody>`);
for (const l of ms) {
xi.push(`<tr><td class="pn"><a href="${esc(rel(XRPC_INDEX, lexiconOut(l.id)))}"><code>${esc(l.id.slice(g.length + 1))}</code></a></td><td>${badge(mainType(l))}</td><td>${esc(firstSentence(l.defs.main?.description))}</td></tr>`);
}
xi.push(`</tbody></table></div>`);
}
}
pages.push({ out: XRPC_INDEX, title: "XRPC", body: xi.join("\n"), aside: aside(XRPC_INDEX, "xrpc") });
return pages;
}
return { include, glossary, pages };
}
export function nsidLookup(lexs: Map<string, Lexicon>): (text: string) => string | undefined {
const ids = [...lexs.keys()];
return (text: string) => {
const t = text.trim();
if (!/^[a-zA-Z][\w.]*[a-zA-Z0-9]$/.test(t) || !t.includes(".")) return undefined;
if (lexs.has(t)) return t;
if (lexs.has(`com.publicdomainrelay.temp.${t}`)) return `com.publicdomainrelay.temp.${t}`;
const hits = ids.filter((id) => id.endsWith(`.${t}`));
return hits.length === 1 ? hits[0] : undefined;
};
}