~ knotless Search/
↑↓ move↵ open⇧↵ all results

socialweb.computer/socialweb-computer-kcp

socialweb.computer: a compute market on atproto (kcp)

1 collaborator
0
0
388 lines · 18.8 KB Raw History
type Json = Record<string, unknown>;
type Def = Json & { type: string; description?: string };
export type Lexicon = { id: string; file: string; description?: string; defs: Record<string, Def> };
export type RefPage = { out: string; title: string; body: string; aside: string };
export type Helpers = {
  esc: (s: string) => string;
  rel: (from: string, to: string) => string;
  hl: (code: string, lang: string) => string;
};

const FAMILIES = [
  "com.publicdomainrelay.temp",
  "com.fedfork.exec",
  "computer.socialweb",
  "software.release",
  "link.ident",
  "network.attested",
  "com.atproto",
];
const XRPC = new Set(["query", "procedure", "subscription"]);

export const lexiconOut = (nsid: string) => `docs/lexicons/${nsid}.html`;
const LEX_INDEX = "docs/lexicons/index.html";
const XRPC_INDEX = "docs/xrpc/index.html";

export async function loadLexicons(root: string): Promise<Map<string, Lexicon>> {
  const lexs = new Map<string, Lexicon>();
  const problems: string[] = [];
  async function walk(rel: string) {
    for await (const e of Deno.readDir(`${root}/${rel}`)) {
      const p = `${rel}/${e.name}`;
      if (e.isDirectory) await walk(p);
      else if (p.endsWith(".json")) {
        let j: Json;
        try {
          j = JSON.parse(await Deno.readTextFile(`${root}/${p}`));
        } catch (err) {
          problems.push(`${p}: not JSON: ${err}`);
          continue;
        }
        const want = p.replace(/^lexicons\//, "").replace(/\.json$/, "").split("/").join(".");
        if (j.lexicon !== 1) problems.push(`${p}: lexicon is not 1`);
        if (j.id !== want) problems.push(`${p}: id ${j.id} is not ${want}`);
        if (!j.defs || typeof j.defs !== "object") problems.push(`${p}: no defs`);
        else lexs.set(want, { id: want, file: p, description: j.description as string | undefined, defs: j.defs as Record<string, Def> });
      }
    }
  }
  await walk("lexicons");
  for (const lex of lexs.values()) {
    for (const [name, def] of Object.entries(lex.defs)) {
      if (typeof def?.type !== "string") problems.push(`${lex.file}: #${name} has no type`);
      walkRefs(def, (ref) => {
        const [nsid, frag] = splitRef(lex.id, ref);
        const target = lexs.get(nsid);
        if (!target) problems.push(`${lex.file}: #${name} refers to ${ref}: no lexicon ${nsid}`);
        else if (!target.defs[frag]) problems.push(`${lex.file}: #${name} refers to ${ref}: ${nsid} has no #${frag}`);
      });
    }
  }
  if (problems.length) throw new Error(`lexicons:\n${problems.join("\n")}`);
  return lexs;
}

function walkRefs(v: unknown, f: (ref: string) => void) {
  if (Array.isArray(v)) return v.forEach((x) => walkRefs(x, f));
  if (!v || typeof v !== "object") return;
  const o = v as Json;
  if (o.type === "ref" && typeof o.ref === "string") f(o.ref);
  if (o.type === "union" && Array.isArray(o.refs)) (o.refs as string[]).forEach(f);
  for (const x of Object.values(o)) walkRefs(x, f);
}

function splitRef(from: string, ref: string): [string, string] {
  const [nsid, frag] = ref.split("#");
  return [nsid || from, frag || "main"];
}

function family(nsid: string): string {
  return FAMILIES.find((f) => nsid.startsWith(`${f}.`)) ?? nsid.split(".").slice(0, 2).join(".");
}

function group(nsid: string): string {
  return nsid.split(".").slice(0, -1).join(".");
}

function firstSentence(s = ""): string {
  const m = s.match(/^(.+?[.])(\s|$)/);
  return (m ? m[1] : s).trim();
}

function mainType(lex: Lexicon): string {
  return lex.defs.main?.type ?? "defs";
}

function sorted(lexs: Map<string, Lexicon>): Lexicon[] {
  const rank = (n: string) => {
    const i = FAMILIES.indexOf(family(n));
    return i < 0 ? FAMILIES.length : i;
  };
  return [...lexs.values()].sort((a, b) => rank(a.id) - rank(b.id) || a.id.localeCompare(b.id));
}

function groups(list: Lexicon[]): Map<string, Map<string, Lexicon[]>> {
  const fams = new Map<string, Map<string, Lexicon[]>>();
  for (const lex of list) {
    const fam = family(lex.id);
    const g = group(lex.id);
    if (!fams.has(fam)) fams.set(fam, new Map());
    const gs = fams.get(fam)!;
    if (!gs.has(g)) gs.set(g, []);
    gs.get(g)!.push(lex);
  }
  return fams;
}

export type UsedIn = Map<string, { out: string; title: string }[]>;

export function notInUse(lex: Lexicon): boolean {
  const d = (lex.defs.main?.description ?? lex.description ?? "").trim();
  return /^(not in use|defined, not (written|served|called|used))\b/i.test(d);
}

export function lexiconKit(lexs: Map<string, Lexicon>, h: Helpers) {
  const { esc, rel } = h;
  const list = sorted(lexs);
  const fams = groups(list);
  const badge = (t: string) => `<span class="badge t-${esc(t)}">${esc(t)}</span>`;
  const nsidHtml = (nsid: string) => {
    const i = nsid.lastIndexOf(".");
    return `<span class="ns">${esc(nsid.slice(0, i + 1))}</span><span class="nm">${esc(nsid.slice(i + 1))}</span>`;
  };

  function aside(here: string, mode: "lexicons" | "xrpc"): string {
    const parts: string[] = [
      `<p class="nav-top"><a href="${esc(rel(here, LEX_INDEX))}"${here === LEX_INDEX ? ' aria-current="page"' : ""}>lexicons</a>` +
      ` <a href="${esc(rel(here, XRPC_INDEX))}"${here === XRPC_INDEX ? ' aria-current="page"' : ""}>xrpc</a></p>`,
    ];
    for (const [fam, gs] of fams) {
      const famShown = [...gs.values()].flat().filter((l) => mode !== "xrpc" || XRPC.has(mainType(l)));
      if (!famShown.length) continue;
      parts.push(`<p class="nav-fam">${esc(fam)}.*</p>`);
      for (const [g, items] of gs) {
        const shown = mode === "xrpc" ? items.filter((l) => XRPC.has(mainType(l))) : items;
        if (!shown.length) continue;
        const label = g === fam ? fam.split(".").pop()! : g.slice(fam.length + 1);
        parts.push(`<p class="nav-group"><span>${esc(label)}</span><span class="count">${shown.length}</span></p><ul>`);
        for (const l of shown) {
          const target = lexiconOut(l.id);
          const cur = target === here ? ' aria-current="page"' : "";
          parts.push(`<li><a href="${esc(rel(here, target))}"${cur}>${esc(l.id.slice(g.length + 1))}</a></li>`);
        }
        parts.push("</ul>");
      }
    }
    return parts.join("\n");
  }

  function refLink(here: string, from: string, ref: string): string {
    const [nsid, frag] = splitRef(from, ref);
    const href = rel(here, lexiconOut(nsid)) + (frag === "main" ? "" : `#${frag}`);
    const text = ref.startsWith("#") ? ref : ref;
    return `<a class="ref" href="${esc(href)}"><code>${esc(text)}</code></a>`;
  }

  function typeHtml(here: string, from: string, f: Json): string {
    const t = String(f.type);
    if (t === "ref") return refLink(here, from, String(f.ref));
    if (t === "union") {
      const refs = (f.refs as string[] ?? []).map((r) => refLink(here, from, r)).join(" | ");
      return `<span class="ty">union</span>${f.closed ? " (closed)" : ""} ${refs}`;
    }
    if (t === "array") return `<span class="ty">array</span> of ${typeHtml(here, from, f.items as Json)}`;
    const fmt = f.format ? ` <span class="fmt">${esc(String(f.format))}</span>` : "";
    return `<span class="ty">${esc(t)}</span>${fmt}`;
  }

  function constraints(f: Json): string {
    const out: string[] = [];
    const show = (k: string, v: unknown) => out.push(`<span class="c"><span class="ck">${esc(k)}</span> <code>${esc(typeof v === "string" ? v : JSON.stringify(v))}</code></span>`);
    for (const k of ["const", "default", "enum", "knownValues", "minimum", "maximum", "minLength", "maxLength", "minGraphemes", "maxGraphemes", "maxSize", "accept"]) {
      if (f[k] !== undefined) show(k, f[k]);
    }
    if (f.type === "array") {
      for (const k of ["minLength", "maxLength"]) if (f[k] !== undefined && !out.length) show(k, f[k]);
      const it = f.items as Json | undefined;
      if (it) for (const k of ["knownValues", "enum", "maxLength", "format"]) if (it[k] !== undefined) show(`items.${k}`, it[k]);
    }
    return out.join(" ");
  }

  function table(here: string, from: string, obj: Json, label: string, id: string): string {
    const props = (obj.properties ?? {}) as Record<string, Json>;
    const req = new Set((obj.required ?? []) as string[]);
    const nullable = new Set((obj.nullable ?? []) as string[]);
    const names = Object.keys(props);
    if (!names.length) return `<p class="empty">${esc(label)}: none.</p>`;
    const rows = names.map((n) => {
      const f = props[n];
      const mark = req.has(n) ? ' <span class="req" title="required">*</span>' : "";
      const nul = nullable.has(n) ? ' <span class="fmt">nullable</span>' : "";
      const desc = f.description ? `<div class="desc">${esc(String(f.description))}</div>` : "";
      const c = constraints(f);
      return `<tr id="${esc(id)}-${esc(n)}"><td class="pn"><code>${esc(n)}</code>${mark}</td><td>${typeHtml(here, from, f)}${nul}</td><td>${desc}${c ? `<div class="cons">${c}</div>` : ""}</td></tr>`;
    }).join("\n");
    return `<div class="table"><table class="props"><thead><tr><th>${esc(label)}</th><th>type</th><th>description</th></tr></thead><tbody>${rows}</tbody></table></div>`;
  }

  function body(here: string, from: string, schema: Json | undefined, label: string, id: string): string {
    if (!schema) return "";
    const enc = schema.encoding ? `<p class="meta">encoding <code>${esc(String(schema.encoding))}</code></p>` : "";
    const s = schema.schema as Json | undefined;
    const desc = schema.description ? `<p>${esc(String(schema.description))}</p>` : "";
    let inner = "";
    if (s) {
      if (s.type === "object") inner = table(here, from, s, "field", id);
      else inner = `<p>${typeHtml(here, from, s)}</p>`;
    }
    return `<h3 id="${esc(id)}">${esc(label)}</h3>${enc}${desc}${inner}`;
  }

  function errors(def: Def): string {
    const errs = (def.errors ?? []) as Json[];
    if (!errs.length) return "";
    const rows = errs.map((e) => `<tr><td class="pn"><code>${esc(String(e.name))}</code></td><td>${esc(String(e.description ?? ""))}</td></tr>`).join("\n");
    return `<h3 id="s-errors">errors</h3><div class="table"><table class="props"><thead><tr><th>error</th><th>description</th></tr></thead><tbody>${rows}</tbody></table></div>`;
  }

  function defSection(here: string, lex: Lexicon, name: string, def: Def): string {
    const parts: string[] = [];
    const isMain = name === "main";
    if (!isMain) {
      parts.push(`<h2 id="${esc(name)}" class="def"><a class="anchor" href="#${esc(name)}"><code>#${esc(name)}</code></a> ${badge(def.type)}</h2>`);
      if (def.description) parts.push(`<p>${esc(def.description)}</p>`);
    }
    const t = def.type;
    if (t === "record") {
      parts.push(`<h3 id="s-record">record</h3>`);
      const rec = def.record as Json;
      if (rec?.description) parts.push(`<p>${esc(String(rec.description))}</p>`);
      parts.push(table(here, lex.id, rec ?? {}, "field", "s-record"));
    } else if (t === "query" || t === "procedure" || t === "subscription") {
      const params = def.parameters as Json | undefined;
      if (params) parts.push(`<h3 id="s-parameters">parameters</h3>${table(here, lex.id, params, "parameter", "s-parameters")}`);
      parts.push(body(here, lex.id, def.input as Json | undefined, "input", "s-input"));
      parts.push(body(here, lex.id, def.output as Json | undefined, "output", "s-output"));
      parts.push(body(here, lex.id, def.message as Json | undefined, "message", "s-message"));
      parts.push(errors(def));
    } else if (t === "object") {
      parts.push(table(here, lex.id, def, "field", isMain ? "s-fields" : name));
    } else if (t === "array") {
      parts.push(`<p>${typeHtml(here, lex.id, def)}</p>`);
    } else {
      const c = constraints(def);
      parts.push(`<p>${typeHtml(here, lex.id, def)}${c ? ` <span class="cons">${c}</span>` : ""}</p>`);
    }
    return parts.join("\n");
  }

  const unused = `<span class="badge t-unused" title="the lexicon's description says it is defined but not written or served today">not in use</span>`;

  function include(nsid: string, here: string): string {
    const lex = lexs.get(nsid);
    if (!lex) throw new Error(`${here}: include of ${nsid}: no such lexicon`);
    const main = lex.defs.main;
    const href = rel(here, lexiconOut(nsid));
    const meta = [badge(mainType(lex))];
    if (main?.key) meta.push(`<span class="kv">key <code>${esc(String(main.key))}</code></span>`);
    if (notInUse(lex)) meta.push(unused);
    const prefix = `inc-${nsid.replace(/\./g, "-")}-`;
    const inner = main
      ? defSection(here, lex, "main", main)
        .replace(/<h3 id="[^"]+">([^<]*)<\/h3>/g, '<p class="lexinc-sub">$1</p>')
        .replace(/ id="/g, ` id="${prefix}`)
      : "";
    const others = Object.keys(lex.defs).filter((n) => n !== "main");
    const more = others.length ? `<p class="lexinc-more">also: ${others.map((n) => `<a href="${esc(href)}#${esc(n)}"><code>#${esc(n)}</code></a>`).join(" ")}</p>` : "";
    const desc = main?.description ?? lex.description;
    return `<div class="lexinc"><p class="lexinc-head"><a href="${esc(href)}"><code>${esc(nsid)}</code></a> ${meta.join(" ")}</p>${desc ? `<p class="lexinc-desc">${esc(desc)}</p>` : ""}${inner}${more}</div>\n`;
  }

  function usedList(here: string, nsid: string, usedIn: UsedIn): string {
    const pages = (usedIn.get(nsid) ?? []).filter((p) => p.out !== here);
    return pages.map((p) => `<a href="${esc(rel(here, p.out))}">${esc(p.title)}</a>`).join(", ");
  }

  function glossary(here: string, usedIn: UsedIn): string {
    const out: string[] = [];
    for (const [fam, gs] of fams) {
      for (const [g, items] of gs) {
        out.push(`<h3 id="${esc(g)}">${esc(g)}</h3><div class="table"><table class="props"><thead><tr><th>NSID</th><th>type</th><th>what</th><th>docs</th></tr></thead><tbody>`);
        for (const l of items) {
          const d = firstSentence(l.defs.main?.description ?? l.description);
          out.push(`<tr><td class="pn"><a href="${esc(rel(here, lexiconOut(l.id)))}"><code>${esc(l.id.slice(g.length + 1))}</code></a></td><td>${badge(mainType(l))}${notInUse(l) ? ` ${unused}` : ""}</td><td>${esc(d)}</td><td>${usedList(here, l.id, usedIn)}</td></tr>`);
        }
        out.push(`</tbody></table></div>`);
      }
      void fam;
    }
    return out.join("\n");
  }

  function pages(usedIn: UsedIn): RefPage[] {
  const pages: RefPage[] = [];
  for (const lex of list) {
    const here = lexiconOut(lex.id);
    const main = lex.defs.main;
    const t = mainType(lex);
    const meta: string[] = [badge(t)];
    if (main?.key) meta.push(`<span class="kv">key <code>${esc(String(main.key))}</code></span>`);
    if (main?.type === "query") meta.push(`<span class="kv">HTTP <code>GET /xrpc/${esc(lex.id)}</code></span>`);
    if (main?.type === "procedure") meta.push(`<span class="kv">HTTP <code>POST /xrpc/${esc(lex.id)}</code></span>`);
    if (main?.type === "subscription") meta.push(`<span class="kv">WebSocket <code>/xrpc/${esc(lex.id)}</code></span>`);
    if (notInUse(lex)) meta.push(unused);
    const used = usedList(here, lex.id, usedIn);
    const others = Object.keys(lex.defs).filter((n) => n !== "main");
    const toc = others.length
      ? `<p class="toc">definitions: ${others.map((n) => `<a href="#${esc(n)}"><code>#${esc(n)}</code></a>`).join(" ")}</p>`
      : "";
    const desc = main?.description ?? lex.description;
    const sections = [main ? defSection(here, lex, "main", main) : "", ...others.map((n) => defSection(here, lex, n, lex.defs[n]))];
    const raw = h.hl(JSON.stringify({ lexicon: 1, id: lex.id, ...(lex.description ? { description: lex.description } : {}), defs: lex.defs }, null, 2), "json");
    const body = [
      `<p class="kicker">lexicon</p>`,
      `<h1 class="nsid">${nsidHtml(lex.id)}</h1>`,
      `<p class="meta">${meta.join(" ")}</p>`,
      desc ? `<p class="lede">${esc(desc)}</p>` : "",
      toc,
      ...sections,
      `<h2 id="used-in" class="def">used in</h2><p>${used || "no page of these docs names it yet."}</p>`,
      `<details id="raw" class="raw"><summary>raw schema: <code>${esc(lex.file)}</code></summary>${raw}</details>`,
    ].join("\n");
    pages.push({ out: here, title: lex.id, body, aside: aside(here, XRPC.has(t) ? "xrpc" : "lexicons") });
  }

  const index: string[] = [
    `<p class="kicker">reference</p>`,
    `<h1 class="nsid"><span class="nm">Lexicons</span></h1>`,
    `<p class="lede">Every record, object and XRPC method of the socialweb.computer market, generated from the repository's <code>lexicons/</code> (${list.length} lexicons). XRPC methods only: <a href="${esc(rel(LEX_INDEX, XRPC_INDEX))}">xrpc</a>.</p>`,
  ];
  for (const [fam, gs] of fams) {
    const n = [...gs.values()].reduce((a, b) => a + b.length, 0);
    index.push(`<h2 id="${esc(fam)}">${esc(fam)}.* <span class="count">${n}</span></h2>`);
    for (const [g, items] of gs) {
      index.push(`<h3 id="${esc(g)}">${esc(g)}</h3><div class="table"><table class="props"><tbody>`);
      for (const l of items) {
        index.push(`<tr><td class="pn"><a href="${esc(rel(LEX_INDEX, lexiconOut(l.id)))}"><code>${esc(l.id.slice(g.length + 1))}</code></a></td><td>${badge(mainType(l))}${notInUse(l) ? ` ${unused}` : ""}</td><td>${esc(firstSentence(l.defs.main?.description ?? l.description))}</td></tr>`);
      }
      index.push(`</tbody></table></div>`);
    }
  }
  pages.push({ out: LEX_INDEX, title: "Lexicons", body: index.join("\n"), aside: aside(LEX_INDEX, "lexicons") });

  const methods = list.filter((l) => XRPC.has(mainType(l)));
  const xi: string[] = [
    `<p class="kicker">reference</p>`,
    `<h1 class="nsid"><span class="nm">XRPC</span></h1>`,
    `<p class="lede">The ${methods.length} XRPC methods (queries, procedures, subscriptions) the market's services serve at <code>/xrpc/&lt;nsid&gt;</code>, generated from <code>lexicons/</code>. Records and objects: <a href="${esc(rel(XRPC_INDEX, LEX_INDEX))}">lexicons</a>.</p>`,
  ];
  for (const [, gs] of fams) {
    for (const [g, items] of gs) {
      const ms = items.filter((l) => XRPC.has(mainType(l)));
      if (!ms.length) continue;
      xi.push(`<h2 id="${esc(g)}">${esc(g)} <span class="count">${ms.length}</span></h2><div class="table"><table class="props"><tbody>`);
      for (const l of ms) {
        xi.push(`<tr><td class="pn"><a href="${esc(rel(XRPC_INDEX, lexiconOut(l.id)))}"><code>${esc(l.id.slice(g.length + 1))}</code></a></td><td>${badge(mainType(l))}</td><td>${esc(firstSentence(l.defs.main?.description))}</td></tr>`);
      }
      xi.push(`</tbody></table></div>`);
    }
  }
  pages.push({ out: XRPC_INDEX, title: "XRPC", body: xi.join("\n"), aside: aside(XRPC_INDEX, "xrpc") });
  return pages;
  }

  return { include, glossary, pages };
}

export function nsidLookup(lexs: Map<string, Lexicon>): (text: string) => string | undefined {
  const ids = [...lexs.keys()];
  return (text: string) => {
    const t = text.trim();
    if (!/^[a-zA-Z][\w.]*[a-zA-Z0-9]$/.test(t) || !t.includes(".")) return undefined;
    if (lexs.has(t)) return t;
    if (lexs.has(`com.publicdomainrelay.temp.${t}`)) return `com.publicdomainrelay.temp.${t}`;
    const hits = ids.filter((id) => id.endsWith(`.${t}`));
    return hits.length === 1 ? hits[0] : undefined;
  };
}