0
0
import { Marked } from "marked";
import puppeteer from "puppeteer-core";
import { createHighlighter } from "shiki";
import { lexiconKit, lexiconOut, loadLexicons, nsidLookup, type UsedIn } from "./lexref.ts";
const REPO = "https://knotless.dev/socialweb.computer/socialweb-computer-kcp";
const REF = "main";
const SITE = "socialweb.computer";
type Section = "root" | "docs" | "fancy";
type Page = { src: string; as: string; out: string; raw: boolean; section: Section };
const DOCS = [
"docs/README.md",
"docs/quickstart-prod.md",
"docs/quickstart-local.md",
"docs/bidder.md",
"docs/gateway.md",
"docs/deploy.md",
"docs/how-it-works.md",
"docs/protocol/README.md",
"docs/protocol/flow.md",
"docs/protocol/compute-contract-flow-map.md",
"docs/protocol/identity.md",
"docs/protocol/market.md",
"docs/protocol/policy.md",
"docs/protocol/guest-events.md",
"docs/protocol/release.md",
"docs/protocol/glossary.md",
"docs/protocol/lexicons-market.md",
];
const FANCY = [
["index", "Compute Contracts"],
["how-it-works", "How it works"],
["get-started", "Get started"],
["request-compute", "Request compute"],
["marketplace", "Marketplace"],
["docs", "Spec"],
["trust", "Trust & policy"],
["security", "Security"],
["workload-identity", "Workload identity"],
["gateway", "Gateway"],
["about", "About"],
];
function htmlOf(md: string): string {
return md.replace(/(^|\/)README\.md$/, "$1index.html").replace(/\.md$/, ".html");
}
const PAGES: Page[] = [
{ src: "site/index.md", as: "README.md", out: "index.html", raw: true, section: "root" },
...DOCS.map((src): Page => ({ src, as: src, out: htmlOf(src), raw: false, section: "docs" })),
...FANCY.map(([name]): Page => ({
src: `site/fancy/${name}.md`,
as: `site/fancy/${name}.md`,
out: name === "index" ? "fancy/index.html" : `fancy/${name}.html`,
raw: false,
section: "fancy",
})),
];
const OUT_OF = new Map(PAGES.map((p) => [p.as, p.out]));
const root = new URL("..", import.meta.url).pathname.replace(/\/$/, "");
const out = `${root}/site/out`;
const LEXICONS = await loadLexicons(root);
for (const lex of LEXICONS.values()) OUT_OF.set(lex.file, lexiconOut(lex.id));
OUT_OF.set("lexicons", "docs/lexicons/index.html");
const findNsid = nsidLookup(LEXICONS);
const KIT = lexiconKit(LEXICONS, { esc: escape, rel: relative, hl: highlight });
const LANGS: Record<string, string> = {
sh: "bash", bash: "bash", shell: "bash", console: "bash", zsh: "bash",
json: "json", jsonc: "json", yaml: "yaml", yml: "yaml", go: "go", ts: "typescript", typescript: "typescript",
js: "javascript", javascript: "javascript", toml: "toml", rust: "rust", rs: "rust", sql: "sql", diff: "diff",
dockerfile: "dockerfile", ini: "ini", caddyfile: "nginx",
};
const HIGHLIGHTER = await createHighlighter({ themes: ["tokyo-night"], langs: [...new Set(Object.values(LANGS))] });
const CODE_BG = "#070a10";
const readable = new Map<string, string>();
function luminance(hex: string): number {
const c = hex.slice(1, 7).match(/../g)!.map((x) => parseInt(x, 16) / 255).map((v) => v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4);
return 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2];
}
function readableOn(hex: string): string {
const key = hex.toLowerCase();
if (readable.has(key)) return readable.get(key)!;
let rgb = key.slice(1, 7).match(/../g)!.map((x) => parseInt(x, 16));
const bg = luminance(CODE_BG);
const ratio = (c: number[]) => (luminance(`#${c.map((v) => v.toString(16).padStart(2, "0")).join("")}`) + 0.05) / (bg + 0.05);
while (ratio(rgb) < 4.5) rgb = rgb.map((v) => Math.min(255, Math.round(v + (255 - v) * 0.08)));
const fixed = `#${rgb.map((v) => v.toString(16).padStart(2, "0")).join("")}`;
readable.set(key, fixed);
return fixed;
}
const SITE_URL = "https://socialweb.computer/";
const GLOSSARY = "docs/protocol/glossary.md";
const used = new Map<string, Set<string>>();
function noteUse(nsid: string, page: Page) {
if (page.src === GLOSSARY) return;
if (!used.has(nsid)) used.set(nsid, new Set());
used.get(nsid)!.add(page.out);
}
function highlight(code: string, info: string): string {
const word = (info ?? "").trim().split(/\s+/)[0].toLowerCase();
const lang = LANGS[word];
const cls = word ? ` class="language-${escape(word)}"` : "";
if (!lang) return `<pre class="code" tabindex="0"><code${cls}>${escape(code.replace(/\n$/, ""))}</code></pre>\n`;
const html = HIGHLIGHTER.codeToHtml(code.replace(/\n$/, ""), { lang, theme: "tokyo-night" });
return html
.replace(/^<pre[^>]*><code>/, `<pre class="code shiki" tabindex="0"><code${cls}>`)
.replace(/color:(#[0-9a-fA-F]{6})/g, (_, c) => `color:${readableOn(c)}`) + "\n";
}
function dirname(p: string): string {
const i = p.lastIndexOf("/");
return i < 0 ? "" : p.slice(0, i);
}
function normalize(p: string): string {
const parts: string[] = [];
for (const s of p.split("/")) {
if (s === "" || s === ".") continue;
if (s === "..") parts.pop();
else parts.push(s);
}
return parts.join("/");
}
function relative(fromFile: string, toFile: string): string {
const from = dirname(fromFile).split("/").filter(Boolean);
const to = toFile.split("/");
let i = 0;
while (i < from.length && i < to.length - 1 && from[i] === to[i]) i++;
let rel = [...from.slice(i).map(() => ".."), ...to.slice(i)].join("/");
if (rel === "index.html") rel = "./";
else if (rel.endsWith("/index.html")) rel = rel.slice(0, -"index.html".length);
return rel;
}
function isDir(path: string): boolean {
try {
return Deno.statSync(`${root}/${path}`).isDirectory;
} catch {
return false;
}
}
function rewrite(page: Page, href: string): string {
if (href.startsWith(SITE_URL)) {
const u = new URL(href);
let path = u.pathname.slice(1);
if (path === "" || path.endsWith("/")) path += "index.html";
return relative(page.out, path) + u.hash;
}
if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith("#")) return href;
const hash = href.indexOf("#");
const path = hash < 0 ? href : href.slice(0, hash);
const frag = hash < 0 ? "" : href.slice(hash);
const target = normalize(`${dirname(page.as)}/${path}`);
const rendered = OUT_OF.get(target);
if (rendered) return relative(page.out, rendered) + frag;
const kind = path.endsWith("/") || isDir(target) ? "tree" : "blob";
return `${REPO}/${kind}/${REF}/${target}${frag}`;
}
function escape(s: string): string {
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
}
function slug(text: string): string {
return text.toLowerCase().trim().replace(/<[^>]*>/g, "").replace(/[^\p{L}\p{N}\s_-]/gu, "").replace(/\s/g, "-");
}
function section(md: string, id: string, from: string): string {
const lines = md.split("\n");
let fence = false;
let start = -1;
let level = 0;
for (let i = 0; i < lines.length; i++) {
if (/^(```|~~~)/.test(lines[i])) fence = !fence;
const m = !fence && lines[i].match(/^(#{1,6})\s+(.*)$/);
if (!m) continue;
if (start >= 0 && m[1].length <= level) return lines.slice(start, i).join("\n").trim();
if (start < 0 && slug(m[2].replace(/`/g, "")) === id) {
start = i + 1;
level = m[1].length;
}
}
if (start < 0) throw new Error(`${from}: no section #${id}`);
return lines.slice(start).join("\n").trim();
}
async function splice(md: string, page: Page): Promise<{ md: string; ids: string[] }> {
const ids: string[] = [];
const parts: string[] = [];
let last = 0;
for (const m of md.matchAll(/^<!--\s*([\w./-]+\.md)#([\w-]+)\s*-->$/gm)) {
const file = normalize(`${dirname(page.as)}/${m[1]}`);
const body = section(await Deno.readTextFile(`${root}/${file}`), m[2], page.src)
.replace(/\]\(([^)\s]+)\)/g, (all, href: string) => {
if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith("#")) return all;
const target = normalize(`${dirname(file)}/${href}`);
const from = dirname(page.as).split("/").filter(Boolean);
return `](${[...from.map(() => ".."), target].join("/")})`;
});
ids.push(m[2]);
parts.push(md.slice(last, m.index), `<span id="${m[2]}"></span>\n\n${body}`);
last = m.index! + m[0].length;
}
parts.push(md.slice(last));
return { md: parts.join(""), ids };
}
const CALLOUTS: Record<string, string> = { NOTE: "Note", TIP: "Tip", IMPORTANT: "Important", WARNING: "Warning", CAUTION: "Caution" };
function semicolons(code: string): string {
if (!/^\s*sequenceDiagram/.test(code)) return code;
return code.split("\n").map((l) => {
const m = l.match(/^(\s*(?:[^:]*?(?:->>|-->>|->|-->|-x|--x|-\)|--\))[^:]*|Note [^:]*)):(.*)$/);
return m ? `${m[1]}:${m[2].replace(/;/g, "#59;")}` : l;
}).join("\n");
}
type Rendered = { page: Page; title: string; body: string; aside?: string };
const diagrams: string[] = [];
async function render(page: Page): Promise<Rendered> {
const spliced = await splice(await Deno.readTextFile(`${root}/${page.src}`), page);
const seen = new Map<string, number>(spliced.ids.map((id) => [id, 1]));
let title = "";
const marked = new Marked({ gfm: true });
marked.use({
walkTokens(token) {
const quiet = (t: { tokens?: unknown[] }) => (t.tokens ?? []).forEach((c) => { (c as { noLink?: boolean }).noLink = true; });
if (token.type === "link") {
const m = token.href.match(/^https:\/\/socialweb\.computer\/docs\/lexicons\/([\w.]+)\.html/);
if (m && LEXICONS.has(m[1])) noteUse(m[1], page);
token.href = rewrite(page, token.href);
quiet(token);
}
if (token.type === "heading") quiet(token);
if (token.type === "codespan") {
const nsid = findNsid(token.text.replace(/&/g, "&"));
if (nsid) noteUse(nsid, page);
}
},
renderer: {
heading({ tokens, depth, text }) {
const plain = text.replace(/`/g, "");
if (depth === 1 && !title) title = plain;
const base = slug(plain);
const n = seen.get(base) ?? 0;
seen.set(base, n + 1);
const id = n ? `${base}-${n}` : base;
return `<h${depth} id="${id}"><a class="anchor" href="#${id}">${this.parser.parseInline(tokens)}</a></h${depth}>\n`;
},
html({ text }) {
if (page.raw || /^<(a|span) id="[\w-]+">$|^<\/(a|span)>$/.test(text)) return text;
const m = text.match(/^\s*<(a|span) id="([\w-]+)"><\/\1>[ \t]*\n?/);
if (!m) return escape(text);
const rest = text.slice(m[0].length);
return `<${m[1]} id="${m[2]}"></${m[1]}>\n${rest.trim() ? marked.parse(rest, { async: false }) : ""}`;
},
code({ text, lang }) {
if ((lang ?? "").split(/\s/)[0] === "mermaid") {
diagrams.push(semicolons(text));
return `<!--diagram:${diagrams.length - 1}-->\n`;
}
return highlight(text, lang ?? "");
},
codespan(token) {
const text = token.text;
if (page.section === "root" || (token as { noLink?: boolean }).noLink) return false;
const nsid = findNsid(text.replace(/&/g, "&"));
if (!nsid) return false;
return `<a class="nsid-link" href="${escape(relative(page.out, lexiconOut(nsid)))}"><code>${text}</code></a>`;
},
paragraph({ tokens }) {
const only = tokens.length === 1 && tokens[0].type === "link" ? tokens[0] as { href: string; text: string } : undefined;
const inc = only?.text.match(/^lexicon: ([\w.]+)$/);
if (inc) {
const want = relative(page.out, lexiconOut(inc[1]));
if (only!.href !== want) throw new Error(`${page.src}: include ${inc[1]} links ${only!.href}, not ${SITE_URL}${lexiconOut(inc[1])}`);
noteUse(inc[1], page);
return KIT.include(inc[1], page.out);
}
if (only?.text === "glossary: generated from lexicons/") return "<!--glossary-->\n";
return false;
},
blockquote({ tokens }) {
const body = this.parser.parse(tokens);
const m = body.match(/^<p>\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*(<\/p>\s*)?/);
if (!m) return `<blockquote>\n${body}</blockquote>\n`;
const rest = m[2] ? body.slice(m[0].length) : `<p>${body.slice(m[0].length)}`;
return `<div class="callout ${m[1].toLowerCase()}"><p class="callout-title">${CALLOUTS[m[1]]}</p>\n${rest}</div>\n`;
},
table(token) {
const head = token.header.map((c) => `<th>${this.parser.parseInline(c.tokens)}</th>`).join("");
const rows = token.rows.map((r) => `<tr>${r.map((c) => `<td>${this.parser.parseInline(c.tokens)}</td>`).join("")}</tr>`).join("\n");
return `<div class="table"><table><thead><tr>${head}</tr></thead><tbody>${rows}</tbody></table></div>\n`;
},
},
});
const body = marked.parse(spliced.md, { async: false }) as string;
return { page, title: title || page.src, body };
}
async function renderDiagrams(sources: string[]): Promise<string[]> {
if (!sources.length) return [];
const chrome = Deno.env.get("SITE_CHROME");
if (!chrome) throw new Error("SITE_CHROME is not set (site/build.sh sets it)");
const mermaid = await Deno.readTextFile(`${root}/site/node_modules/mermaid/dist/mermaid.min.js`);
const profile = await Deno.makeTempDir({ prefix: "site-chrome-" });
const browser = await puppeteer.launch({
executablePath: chrome,
headless: true,
userDataDir: profile,
args: ["--no-sandbox", "--disable-dev-shm-usage"],
});
try {
const tab = await browser.newPage();
await tab.setContent("<!doctype html><html><body></body></html>");
await tab.addScriptTag({ content: mermaid });
return await tab.evaluate(async (codes: string[], config: unknown) => {
const m = (globalThis as unknown as { mermaid: { initialize(c: unknown): void; render(id: string, code: string): Promise<{ svg: string }> } }).mermaid;
m.initialize(config);
const svgs: string[] = [];
for (let i = 0; i < codes.length; i++) svgs.push((await m.render(`diagram-${i}`, codes[i])).svg);
return svgs;
}, sources, MERMAID);
} finally {
await browser.close();
await Deno.remove(profile, { recursive: true }).catch(() => {});
}
}
const CONSOLE = {
bg: "#0b0e14",
panel: "#11151f",
surface: "#151a25",
border: "#2a3242",
line: "#4c566a",
fg: "#d5dbe5",
muted: "#7c869a",
accent: "#7aa2f7",
ok: "#9ece6a",
warn: "#e0af68",
};
const MERMAID = {
startOnLoad: false,
securityLevel: "strict",
theme: "base",
fontFamily: "'Courier New', 'Liberation Mono', Cousine, monospace",
themeVariables: {
darkMode: true,
fontFamily: "'Courier New', 'Liberation Mono', Cousine, monospace",
fontSize: "15px",
background: CONSOLE.panel,
primaryColor: CONSOLE.surface,
primaryTextColor: CONSOLE.fg,
primaryBorderColor: CONSOLE.accent,
secondaryColor: CONSOLE.panel,
tertiaryColor: CONSOLE.bg,
lineColor: CONSOLE.muted,
textColor: CONSOLE.fg,
mainBkg: CONSOLE.surface,
nodeBorder: CONSOLE.accent,
nodeTextColor: CONSOLE.fg,
clusterBkg: CONSOLE.panel,
clusterBorder: CONSOLE.border,
titleColor: CONSOLE.fg,
edgeLabelBackground: CONSOLE.panel,
actorBkg: CONSOLE.surface,
actorBorder: CONSOLE.accent,
actorTextColor: CONSOLE.fg,
actorLineColor: CONSOLE.line,
signalColor: CONSOLE.fg,
signalTextColor: CONSOLE.fg,
labelBoxBkgColor: CONSOLE.surface,
labelBoxBorderColor: CONSOLE.border,
labelTextColor: CONSOLE.fg,
loopTextColor: CONSOLE.fg,
noteBkgColor: "#1f2633",
noteBorderColor: CONSOLE.warn,
noteTextColor: CONSOLE.fg,
activationBkgColor: "#1f2633",
activationBorderColor: CONSOLE.accent,
sequenceNumberColor: CONSOLE.bg,
},
};
function naturalSize(svg: string): string {
const vb = svg.match(/viewBox="[-\d.]+ [-\d.]+ ([\d.]+) ([\d.]+)"/);
if (!vb) return svg;
const [w, h] = [Math.ceil(Number(vb[1])), Math.ceil(Number(vb[2]))];
return svg
.replace(/^<svg([^>]*?) style="max-width:[^"]*"/, "<svg$1")
.replace(/^<svg([^>]*?) width="100%"/, `<svg$1 width="${w}" height="${h}" style="min-width:${Math.min(w, 560)}px"`);
}
function darkTint(svg: string): string {
return naturalSize(svg).replace(/fill="rgb\(\s*(\d+),\s*(\d+),\s*(\d+)\s*\)"/g, (all, r, g, b) => {
const c = [Number(r), Number(g), Number(b)];
if (Math.min(...c) < 200) return all;
const avg = (c[0] + c[1] + c[2]) / 3;
const base = [21, 26, 37];
const t = base.map((v, i) => Math.max(0, Math.min(255, Math.round(v + 1.6 * (c[i] - avg)))));
return `fill="rgb(${t.join(", ")})"`;
});
}
async function git(...args: string[]): Promise<string> {
const r = await new Deno.Command("git", { args: ["-C", root, ...args], stdout: "piped", stderr: "inherit" }).output();
if (!r.success) throw new Error(`git ${args.join(" ")} failed`);
return new TextDecoder().decode(r.stdout).trim();
}
function shell(r: Rendered, commit: string, tagline: string): string {
const here = r.page.out;
const landing = r.page.section === "root";
const nav = [
...(landing ? [] : [{ href: relative(here, "index.html"), text: "home" }]),
{ href: relative(here, "docs/index.html"), text: "docs" },
{ href: relative(here, "docs/lexicons/index.html"), text: "lexicons" },
{ href: relative(here, "fancy/index.html"), text: "compute contracts" },
...(landing ? [] : [{ href: REPO, text: "source" }]),
];
const a = (l: { href: string; text: string }) => `<a href="${escape(l.href)}">${escape(l.text)}</a>`;
const sub = r.page.section !== "fancy" ? "" : `<nav class="subnav" aria-label="Compute Contracts">${
FANCY.map(([name, label]) => {
const target = name === "index" ? "fancy/index.html" : `fancy/${name}.html`;
const cur = target === here ? ` aria-current="page"` : "";
return `<a href="${escape(relative(here, target))}"${cur}>${escape(label)}</a>`;
}).join("\n")
}</nav>\n`;
return `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="${escape(tagline)}">
<title>${escape(landing ? SITE : `${r.title} - ${SITE}`)}</title>
<link rel="icon" href="data:,">
<link rel="stylesheet" href="${relative(here, "style.css")}">
</head>
<body class="paper">
<header class="bar">
<p class="name"><a href="${relative(here, "index.html")}">${SITE}</a></p>
<p class="meta">${escape(tagline)}</p>
<nav>${nav.map(a).join("\n")}</nav>
</header>
${sub}${r.aside === undefined ? `<main>
${r.body}</main>` : `<main class="ref">
<details class="refnav-phone"><summary>lexicons and xrpc</summary>
${r.aside}
</details>
<div class="ref-grid">
<aside class="refnav" aria-label="Lexicon reference">
${r.aside}
</aside>
<div class="ref-body">
${r.body}
</div>
</div>
</main>`}
<footer>
<p>Built from commit <code>${escape(commit.slice(0, 12))}${commit.endsWith("-dirty") ? "-dirty" : ""}</code>. Unlicense (public domain).</p>
</footer>
</body>
</html>
`;
}
function anchorsOk(pages: Map<string, string>): string[] {
const bad: string[] = [];
const ids = new Map([...pages].map(([p, html]) => [p, new Set([...html.matchAll(/ id="([^"]+)"/g)].map((m) => m[1]))]));
for (const [p, html] of pages) {
for (const m of html.matchAll(/href="([^"]+)"/g)) {
const u = new URL(m[1].replace(/&/g, "&"), `https://${SITE}/${p}`);
if (u.host !== SITE) continue;
let path = u.pathname.slice(1);
if (path === "" || path.endsWith("/")) path += "index.html";
if (!ids.has(path) && path !== "style.css" && path !== "install.sh") bad.push(`${p}: ${m[1]}: no page`);
else if (u.hash && !ids.get(path)?.has(decodeURIComponent(u.hash.slice(1)))) bad.push(`${p}: ${m[1]}: no anchor`);
}
}
return bad;
}
const head = await git("rev-parse", "HEAD");
const dirty = (await git("status", "--porcelain", "--", "README.md", "scripts/install.sh", "lexicons", ...PAGES.map((p) => p.src), "site")) !== "";
const commit = dirty ? `${head}-dirty` : head;
const readme = await Deno.readTextFile(`${root}/README.md`);
const tagline = (readme.split(/\n\s*\n/).find((p) => p.trim() && !p.startsWith("#")) ?? "")
.replace(/\s+/g, " ").replace(/`/g, "").trim().split(/(?<=\.) /)[0];
const rendered: Rendered[] = [];
for (const page of PAGES) rendered.push(await render(page));
const titles = new Map(rendered.map((r) => [r.page.out, r.title]));
const usedIn: UsedIn = new Map([...used].map(([nsid, outs]) => [nsid, [...outs].sort().map((o) => ({ out: o, title: titles.get(o) ?? o }))]));
for (const r of rendered) r.body = r.body.replace("<!--glossary-->", () => KIT.glossary(r.page.out, usedIn));
for (const ref of KIT.pages(usedIn)) {
const src = ref.out;
rendered.push({ page: { src, as: src, out: ref.out, raw: false, section: "docs" }, title: ref.title, body: ref.body, aside: ref.aside });
}
const svgs = await renderDiagrams(diagrams);
const html = new Map<string, string>();
for (const r of rendered) {
r.body = r.body
.replace(/<!--diagram:(\d+)-->/g, (_, i) => `<figure class="panel diagram"><figcaption>diagram</figcaption>${darkTint(svgs[Number(i)])}</figure>`)
.replace(/<iframe[\s\S]*?<\/iframe>/g, (f) => `<figure class="panel video"><figcaption>demo video</figcaption>${f}</figure>`);
html.set(r.page.out, shell(r, commit, tagline));
}
const bad = anchorsOk(html);
if (bad.length) throw new Error(`dangling links:\n${bad.join("\n")}`);
await Deno.remove(out, { recursive: true }).catch(() => {});
await Deno.mkdir(out, { recursive: true });
await Deno.copyFile(`${root}/site/style.css`, `${out}/style.css`);
await Deno.copyFile(`${root}/scripts/install.sh`, `${out}/install.sh`);
for (const [p, h] of html) {
await Deno.mkdir(`${out}/${dirname(p)}`, { recursive: true });
await Deno.writeTextFile(`${out}/${p}`, h);
}
await Deno.writeTextFile(`${out}/COMMIT`, `${commit}\n`);
console.log(`built ${out} from ${commit}: ${html.size} pages (${LEXICONS.size} lexicon pages), ${svgs.length} diagrams, install.sh`);