import { Marked } from "marked"; import puppeteer from "puppeteer-core"; import { createHighlighter } from "shiki"; import { lexiconKit, lexiconOut, loadLexicons, nsidLookup, type UsedIn } from "./lexref.ts"; const REPO = "https://knotless.dev/socialweb.computer/socialweb-computer-kcp"; const REF = "main"; const SITE = "socialweb.computer"; type Section = "root" | "docs" | "fancy"; type Page = { src: string; as: string; out: string; raw: boolean; section: Section }; const DOCS = [ "docs/README.md", "docs/quickstart-prod.md", "docs/quickstart-local.md", "docs/bidder.md", "docs/gateway.md", "docs/deploy.md", "docs/how-it-works.md", "docs/protocol/README.md", "docs/protocol/flow.md", "docs/protocol/compute-contract-flow-map.md", "docs/protocol/identity.md", "docs/protocol/market.md", "docs/protocol/policy.md", "docs/protocol/guest-events.md", "docs/protocol/release.md", "docs/protocol/glossary.md", "docs/protocol/lexicons-market.md", ]; const FANCY = [ ["index", "Compute Contracts"], ["how-it-works", "How it works"], ["get-started", "Get started"], ["request-compute", "Request compute"], ["marketplace", "Marketplace"], ["docs", "Spec"], ["trust", "Trust & policy"], ["security", "Security"], ["workload-identity", "Workload identity"], ["gateway", "Gateway"], ["about", "About"], ]; function htmlOf(md: string): string { return md.replace(/(^|\/)README\.md$/, "$1index.html").replace(/\.md$/, ".html"); } const PAGES: Page[] = [ { src: "site/index.md", as: "README.md", out: "index.html", raw: true, section: "root" }, ...DOCS.map((src): Page => ({ src, as: src, out: htmlOf(src), raw: false, section: "docs" })), ...FANCY.map(([name]): Page => ({ src: `site/fancy/${name}.md`, as: `site/fancy/${name}.md`, out: name === "index" ? "fancy/index.html" : `fancy/${name}.html`, raw: false, section: "fancy", })), ]; const OUT_OF = new Map(PAGES.map((p) => [p.as, p.out])); const root = new URL("..", import.meta.url).pathname.replace(/\/$/, ""); const out = `${root}/site/out`; const LEXICONS = await loadLexicons(root); for (const lex of LEXICONS.values()) OUT_OF.set(lex.file, lexiconOut(lex.id)); OUT_OF.set("lexicons", "docs/lexicons/index.html"); const findNsid = nsidLookup(LEXICONS); const KIT = lexiconKit(LEXICONS, { esc: escape, rel: relative, hl: highlight }); const LANGS: Record = { sh: "bash", bash: "bash", shell: "bash", console: "bash", zsh: "bash", json: "json", jsonc: "json", yaml: "yaml", yml: "yaml", go: "go", ts: "typescript", typescript: "typescript", js: "javascript", javascript: "javascript", toml: "toml", rust: "rust", rs: "rust", sql: "sql", diff: "diff", dockerfile: "dockerfile", ini: "ini", caddyfile: "nginx", }; const HIGHLIGHTER = await createHighlighter({ themes: ["tokyo-night"], langs: [...new Set(Object.values(LANGS))] }); const CODE_BG = "#070a10"; const readable = new Map(); function luminance(hex: string): number { const c = hex.slice(1, 7).match(/../g)!.map((x) => parseInt(x, 16) / 255).map((v) => v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4); return 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; } function readableOn(hex: string): string { const key = hex.toLowerCase(); if (readable.has(key)) return readable.get(key)!; let rgb = key.slice(1, 7).match(/../g)!.map((x) => parseInt(x, 16)); const bg = luminance(CODE_BG); const ratio = (c: number[]) => (luminance(`#${c.map((v) => v.toString(16).padStart(2, "0")).join("")}`) + 0.05) / (bg + 0.05); while (ratio(rgb) < 4.5) rgb = rgb.map((v) => Math.min(255, Math.round(v + (255 - v) * 0.08))); const fixed = `#${rgb.map((v) => v.toString(16).padStart(2, "0")).join("")}`; readable.set(key, fixed); return fixed; } const SITE_URL = "https://socialweb.computer/"; const GLOSSARY = "docs/protocol/glossary.md"; const used = new Map>(); function noteUse(nsid: string, page: Page) { if (page.src === GLOSSARY) return; if (!used.has(nsid)) used.set(nsid, new Set()); used.get(nsid)!.add(page.out); } function highlight(code: string, info: string): string { const word = (info ?? "").trim().split(/\s+/)[0].toLowerCase(); const lang = LANGS[word]; const cls = word ? ` class="language-${escape(word)}"` : ""; if (!lang) return `
${escape(code.replace(/\n$/, ""))}
\n`; const html = HIGHLIGHTER.codeToHtml(code.replace(/\n$/, ""), { lang, theme: "tokyo-night" }); return html .replace(/^]*>/, `
`)
    .replace(/color:(#[0-9a-fA-F]{6})/g, (_, c) => `color:${readableOn(c)}`) + "\n";
}

function dirname(p: string): string {
  const i = p.lastIndexOf("/");
  return i < 0 ? "" : p.slice(0, i);
}

function normalize(p: string): string {
  const parts: string[] = [];
  for (const s of p.split("/")) {
    if (s === "" || s === ".") continue;
    if (s === "..") parts.pop();
    else parts.push(s);
  }
  return parts.join("/");
}

function relative(fromFile: string, toFile: string): string {
  const from = dirname(fromFile).split("/").filter(Boolean);
  const to = toFile.split("/");
  let i = 0;
  while (i < from.length && i < to.length - 1 && from[i] === to[i]) i++;
  let rel = [...from.slice(i).map(() => ".."), ...to.slice(i)].join("/");
  if (rel === "index.html") rel = "./";
  else if (rel.endsWith("/index.html")) rel = rel.slice(0, -"index.html".length);
  return rel;
}

function isDir(path: string): boolean {
  try {
    return Deno.statSync(`${root}/${path}`).isDirectory;
  } catch {
    return false;
  }
}

function rewrite(page: Page, href: string): string {
  if (href.startsWith(SITE_URL)) {
    const u = new URL(href);
    let path = u.pathname.slice(1);
    if (path === "" || path.endsWith("/")) path += "index.html";
    return relative(page.out, path) + u.hash;
  }
  if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith("#")) return href;
  const hash = href.indexOf("#");
  const path = hash < 0 ? href : href.slice(0, hash);
  const frag = hash < 0 ? "" : href.slice(hash);
  const target = normalize(`${dirname(page.as)}/${path}`);
  const rendered = OUT_OF.get(target);
  if (rendered) return relative(page.out, rendered) + frag;
  const kind = path.endsWith("/") || isDir(target) ? "tree" : "blob";
  return `${REPO}/${kind}/${REF}/${target}${frag}`;
}

function escape(s: string): string {
  return s.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """);
}

function slug(text: string): string {
  return text.toLowerCase().trim().replace(/<[^>]*>/g, "").replace(/[^\p{L}\p{N}\s_-]/gu, "").replace(/\s/g, "-");
}

function section(md: string, id: string, from: string): string {
  const lines = md.split("\n");
  let fence = false;
  let start = -1;
  let level = 0;
  for (let i = 0; i < lines.length; i++) {
    if (/^(```|~~~)/.test(lines[i])) fence = !fence;
    const m = !fence && lines[i].match(/^(#{1,6})\s+(.*)$/);
    if (!m) continue;
    if (start >= 0 && m[1].length <= level) return lines.slice(start, i).join("\n").trim();
    if (start < 0 && slug(m[2].replace(/`/g, "")) === id) {
      start = i + 1;
      level = m[1].length;
    }
  }
  if (start < 0) throw new Error(`${from}: no section #${id}`);
  return lines.slice(start).join("\n").trim();
}

async function splice(md: string, page: Page): Promise<{ md: string; ids: string[] }> {
  const ids: string[] = [];
  const parts: string[] = [];
  let last = 0;
  for (const m of md.matchAll(/^$/gm)) {
    const file = normalize(`${dirname(page.as)}/${m[1]}`);
    const body = section(await Deno.readTextFile(`${root}/${file}`), m[2], page.src)
      .replace(/\]\(([^)\s]+)\)/g, (all, href: string) => {
        if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith("#")) return all;
        const target = normalize(`${dirname(file)}/${href}`);
        const from = dirname(page.as).split("/").filter(Boolean);
        return `](${[...from.map(() => ".."), target].join("/")})`;
      });
    ids.push(m[2]);
    parts.push(md.slice(last, m.index), `\n\n${body}`);
    last = m.index! + m[0].length;
  }
  parts.push(md.slice(last));
  return { md: parts.join(""), ids };
}

const CALLOUTS: Record = { NOTE: "Note", TIP: "Tip", IMPORTANT: "Important", WARNING: "Warning", CAUTION: "Caution" };

function semicolons(code: string): string {
  if (!/^\s*sequenceDiagram/.test(code)) return code;
  return code.split("\n").map((l) => {
    const m = l.match(/^(\s*(?:[^:]*?(?:->>|-->>|->|-->|-x|--x|-\)|--\))[^:]*|Note [^:]*)):(.*)$/);
    return m ? `${m[1]}:${m[2].replace(/;/g, "#59;")}` : l;
  }).join("\n");
}

type Rendered = { page: Page; title: string; body: string; aside?: string };
const diagrams: string[] = [];

async function render(page: Page): Promise {
  const spliced = await splice(await Deno.readTextFile(`${root}/${page.src}`), page);
  const seen = new Map(spliced.ids.map((id) => [id, 1]));
  let title = "";
  const marked = new Marked({ gfm: true });
  marked.use({
    walkTokens(token) {
      const quiet = (t: { tokens?: unknown[] }) => (t.tokens ?? []).forEach((c) => { (c as { noLink?: boolean }).noLink = true; });
      if (token.type === "link") {
        const m = token.href.match(/^https:\/\/socialweb\.computer\/docs\/lexicons\/([\w.]+)\.html/);
        if (m && LEXICONS.has(m[1])) noteUse(m[1], page);
        token.href = rewrite(page, token.href);
        quiet(token);
      }
      if (token.type === "heading") quiet(token);
      if (token.type === "codespan") {
        const nsid = findNsid(token.text.replace(/&/g, "&"));
        if (nsid) noteUse(nsid, page);
      }
    },
    renderer: {
      heading({ tokens, depth, text }) {
        const plain = text.replace(/`/g, "");
        if (depth === 1 && !title) title = plain;
        const base = slug(plain);
        const n = seen.get(base) ?? 0;
        seen.set(base, n + 1);
        const id = n ? `${base}-${n}` : base;
        return `${this.parser.parseInline(tokens)}\n`;
      },
      html({ text }) {
        if (page.raw || /^<(a|span) id="[\w-]+">$|^<\/(a|span)>$/.test(text)) return text;
        const m = text.match(/^\s*<(a|span) id="([\w-]+)"><\/\1>[ \t]*\n?/);
        if (!m) return escape(text);
        const rest = text.slice(m[0].length);
        return `<${m[1]} id="${m[2]}">\n${rest.trim() ? marked.parse(rest, { async: false }) : ""}`;
      },
      code({ text, lang }) {
        if ((lang ?? "").split(/\s/)[0] === "mermaid") {
          diagrams.push(semicolons(text));
          return `\n`;
        }
        return highlight(text, lang ?? "");
      },
      codespan(token) {
        const text = token.text;
        if (page.section === "root" || (token as { noLink?: boolean }).noLink) return false;
        const nsid = findNsid(text.replace(/&/g, "&"));
        if (!nsid) return false;
        return `${text}`;
      },
      paragraph({ tokens }) {
        const only = tokens.length === 1 && tokens[0].type === "link" ? tokens[0] as { href: string; text: string } : undefined;
        const inc = only?.text.match(/^lexicon: ([\w.]+)$/);
        if (inc) {
          const want = relative(page.out, lexiconOut(inc[1]));
          if (only!.href !== want) throw new Error(`${page.src}: include ${inc[1]} links ${only!.href}, not ${SITE_URL}${lexiconOut(inc[1])}`);
          noteUse(inc[1], page);
          return KIT.include(inc[1], page.out);
        }
        if (only?.text === "glossary: generated from lexicons/") return "\n";
        return false;
      },
      blockquote({ tokens }) {
        const body = this.parser.parse(tokens);
        const m = body.match(/^

\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*(<\/p>\s*)?/); if (!m) return `

\n${body}
\n`; const rest = m[2] ? body.slice(m[0].length) : `

${body.slice(m[0].length)}`; return `

${CALLOUTS[m[1]]}

\n${rest}
\n`; }, table(token) { const head = token.header.map((c) => `${this.parser.parseInline(c.tokens)}`).join(""); const rows = token.rows.map((r) => `${r.map((c) => `${this.parser.parseInline(c.tokens)}`).join("")}`).join("\n"); return `
${head}${rows}
\n`; }, }, }); const body = marked.parse(spliced.md, { async: false }) as string; return { page, title: title || page.src, body }; } async function renderDiagrams(sources: string[]): Promise { if (!sources.length) return []; const chrome = Deno.env.get("SITE_CHROME"); if (!chrome) throw new Error("SITE_CHROME is not set (site/build.sh sets it)"); const mermaid = await Deno.readTextFile(`${root}/site/node_modules/mermaid/dist/mermaid.min.js`); const profile = await Deno.makeTempDir({ prefix: "site-chrome-" }); const browser = await puppeteer.launch({ executablePath: chrome, headless: true, userDataDir: profile, args: ["--no-sandbox", "--disable-dev-shm-usage"], }); try { const tab = await browser.newPage(); await tab.setContent(""); await tab.addScriptTag({ content: mermaid }); return await tab.evaluate(async (codes: string[], config: unknown) => { const m = (globalThis as unknown as { mermaid: { initialize(c: unknown): void; render(id: string, code: string): Promise<{ svg: string }> } }).mermaid; m.initialize(config); const svgs: string[] = []; for (let i = 0; i < codes.length; i++) svgs.push((await m.render(`diagram-${i}`, codes[i])).svg); return svgs; }, sources, MERMAID); } finally { await browser.close(); await Deno.remove(profile, { recursive: true }).catch(() => {}); } } const CONSOLE = { bg: "#0b0e14", panel: "#11151f", surface: "#151a25", border: "#2a3242", line: "#4c566a", fg: "#d5dbe5", muted: "#7c869a", accent: "#7aa2f7", ok: "#9ece6a", warn: "#e0af68", }; const MERMAID = { startOnLoad: false, securityLevel: "strict", theme: "base", fontFamily: "'Courier New', 'Liberation Mono', Cousine, monospace", themeVariables: { darkMode: true, fontFamily: "'Courier New', 'Liberation Mono', Cousine, monospace", fontSize: "15px", background: CONSOLE.panel, primaryColor: CONSOLE.surface, primaryTextColor: CONSOLE.fg, primaryBorderColor: CONSOLE.accent, secondaryColor: CONSOLE.panel, tertiaryColor: CONSOLE.bg, lineColor: CONSOLE.muted, textColor: CONSOLE.fg, mainBkg: CONSOLE.surface, nodeBorder: CONSOLE.accent, nodeTextColor: CONSOLE.fg, clusterBkg: CONSOLE.panel, clusterBorder: CONSOLE.border, titleColor: CONSOLE.fg, edgeLabelBackground: CONSOLE.panel, actorBkg: CONSOLE.surface, actorBorder: CONSOLE.accent, actorTextColor: CONSOLE.fg, actorLineColor: CONSOLE.line, signalColor: CONSOLE.fg, signalTextColor: CONSOLE.fg, labelBoxBkgColor: CONSOLE.surface, labelBoxBorderColor: CONSOLE.border, labelTextColor: CONSOLE.fg, loopTextColor: CONSOLE.fg, noteBkgColor: "#1f2633", noteBorderColor: CONSOLE.warn, noteTextColor: CONSOLE.fg, activationBkgColor: "#1f2633", activationBorderColor: CONSOLE.accent, sequenceNumberColor: CONSOLE.bg, }, }; function naturalSize(svg: string): string { const vb = svg.match(/viewBox="[-\d.]+ [-\d.]+ ([\d.]+) ([\d.]+)"/); if (!vb) return svg; const [w, h] = [Math.ceil(Number(vb[1])), Math.ceil(Number(vb[2]))]; return svg .replace(/^]*?) style="max-width:[^"]*"/, "]*?) width="100%"/, ` { const c = [Number(r), Number(g), Number(b)]; if (Math.min(...c) < 200) return all; const avg = (c[0] + c[1] + c[2]) / 3; const base = [21, 26, 37]; const t = base.map((v, i) => Math.max(0, Math.min(255, Math.round(v + 1.6 * (c[i] - avg))))); return `fill="rgb(${t.join(", ")})"`; }); } async function git(...args: string[]): Promise { const r = await new Deno.Command("git", { args: ["-C", root, ...args], stdout: "piped", stderr: "inherit" }).output(); if (!r.success) throw new Error(`git ${args.join(" ")} failed`); return new TextDecoder().decode(r.stdout).trim(); } function shell(r: Rendered, commit: string, tagline: string): string { const here = r.page.out; const landing = r.page.section === "root"; const nav = [ ...(landing ? [] : [{ href: relative(here, "index.html"), text: "home" }]), { href: relative(here, "docs/index.html"), text: "docs" }, { href: relative(here, "docs/lexicons/index.html"), text: "lexicons" }, { href: relative(here, "fancy/index.html"), text: "compute contracts" }, ...(landing ? [] : [{ href: REPO, text: "source" }]), ]; const a = (l: { href: string; text: string }) => `${escape(l.text)}`; const sub = r.page.section !== "fancy" ? "" : `\n`; return ` ${escape(landing ? SITE : `${r.title} - ${SITE}`)}

${SITE}

${escape(tagline)}

${sub}${r.aside === undefined ? `
${r.body}
` : `
lexicons and xrpc ${r.aside}
${r.body}
`}

Built from commit ${escape(commit.slice(0, 12))}${commit.endsWith("-dirty") ? "-dirty" : ""}. Unlicense (public domain).

`; } function anchorsOk(pages: Map): string[] { const bad: string[] = []; const ids = new Map([...pages].map(([p, html]) => [p, new Set([...html.matchAll(/ id="([^"]+)"/g)].map((m) => m[1]))])); for (const [p, html] of pages) { for (const m of html.matchAll(/href="([^"]+)"/g)) { const u = new URL(m[1].replace(/&/g, "&"), `https://${SITE}/${p}`); if (u.host !== SITE) continue; let path = u.pathname.slice(1); if (path === "" || path.endsWith("/")) path += "index.html"; if (!ids.has(path) && path !== "style.css" && path !== "install.sh") bad.push(`${p}: ${m[1]}: no page`); else if (u.hash && !ids.get(path)?.has(decodeURIComponent(u.hash.slice(1)))) bad.push(`${p}: ${m[1]}: no anchor`); } } return bad; } const head = await git("rev-parse", "HEAD"); const dirty = (await git("status", "--porcelain", "--", "README.md", "scripts/install.sh", "lexicons", ...PAGES.map((p) => p.src), "site")) !== ""; const commit = dirty ? `${head}-dirty` : head; const readme = await Deno.readTextFile(`${root}/README.md`); const tagline = (readme.split(/\n\s*\n/).find((p) => p.trim() && !p.startsWith("#")) ?? "") .replace(/\s+/g, " ").replace(/`/g, "").trim().split(/(?<=\.) /)[0]; const rendered: Rendered[] = []; for (const page of PAGES) rendered.push(await render(page)); const titles = new Map(rendered.map((r) => [r.page.out, r.title])); const usedIn: UsedIn = new Map([...used].map(([nsid, outs]) => [nsid, [...outs].sort().map((o) => ({ out: o, title: titles.get(o) ?? o }))])); for (const r of rendered) r.body = r.body.replace("", () => KIT.glossary(r.page.out, usedIn)); for (const ref of KIT.pages(usedIn)) { const src = ref.out; rendered.push({ page: { src, as: src, out: ref.out, raw: false, section: "docs" }, title: ref.title, body: ref.body, aside: ref.aside }); } const svgs = await renderDiagrams(diagrams); const html = new Map(); for (const r of rendered) { r.body = r.body .replace(//g, (_, i) => `
diagram
${darkTint(svgs[Number(i)])}
`) .replace(//g, (f) => `
demo video
${f}
`); html.set(r.page.out, shell(r, commit, tagline)); } const bad = anchorsOk(html); if (bad.length) throw new Error(`dangling links:\n${bad.join("\n")}`); await Deno.remove(out, { recursive: true }).catch(() => {}); await Deno.mkdir(out, { recursive: true }); await Deno.copyFile(`${root}/site/style.css`, `${out}/style.css`); await Deno.copyFile(`${root}/scripts/install.sh`, `${out}/install.sh`); for (const [p, h] of html) { await Deno.mkdir(`${out}/${dirname(p)}`, { recursive: true }); await Deno.writeTextFile(`${out}/${p}`, h); } await Deno.writeTextFile(`${out}/COMMIT`, `${commit}\n`); console.log(`built ${out} from ${commit}: ${html.size} pages (${LEXICONS.size} lexicon pages), ${svgs.length} diagrams, install.sh`);