render svfe repo pages for knot-static generations
Signed-off-by: dawn <90008@klbr.net>
edafa53e +870
123456789101112# svfe-pages
renders svfe's repo pages for a knot-static generation, so they work without js and hydrateinto the svfe SPA. it runs as knot-static's `[site_hook]`, reads only the knot's public xrpcsurface, and re-renders just the pages the generation's xrpc diff touches.
svfe-pages <hook.json> --server <svfe build-server> --out <dir> [--read <xrpc base>]
see `nginx-site.conf` for serving `<out>` in front of the SPA, and `budget.mjs` for what arepo may cost on disk.
node --test1234567891011121314151617181920212223// what a repo may cost on disk. pages past it aren't written and those urls get the SPA// shell, which renders them in the browser like any other routeexport const BUDGET_OPTIONS = { "max-pages": { type: "string", default: "1000" }, "max-repo-bytes": { type: "string", default: String(20 << 20) }, // the whole site: each repo gets at most an even share of it "max-total-bytes": { type: "string", default: String(2 * 2 ** 30) }, "max-blob-bytes": { type: "string", default: String(128 << 10) }, "max-commits": { type: "string", default: "60" }};
export const budgetFrom = (opts, repos) => ({ pages: Number(opts["max-pages"]), bytes: Math.min(Number(opts["max-repo-bytes"]), Math.floor(Number(opts["max-total-bytes"]) / Math.max(1, repos))), blobBytes: Number(opts["max-blob-bytes"]), commits: Number(opts["max-commits"])});
// which pages a repo keeps when it can't keep all: its front pages, then folders and files// nearest the root, then commits, newest first as the log lists themconst RANK = { "": 0, issues: 1, pulls: 1, branches: 1, tags: 1, languages: 1, commits: 2, tree: 3, blob: 4, commit: 5 };
export const rankOf = (view, depth) => (RANK[view] ?? 6) * 1000 + Math.min(depth, 999);123456789101112131415161718192021222324252627282930313233343536373839404142434445import { createHash } from "node:crypto";import { readFileSync } from "node:fs";import { mkdir, rename, writeFile } from "node:fs/promises";import path from "node:path";
// a bump means no saved page can be trustedexport const RENDERER_FORMAT = 2;
export const tokenOf = (did) => createHash("sha256").update(did).digest("hex").slice(0, 32);
// svfe's url for a repo: the owner's handle, or its did when it has noneexport const prefixOf = (repo) => `/${repo.owner_handle?.toLowerCase() ?? repo.owner_did}/${repo.name}`;
let written = 0;// concurrent writers of one file each rename their own tmp, the last rename winsexport const atomicJson = async (file, value) => { await mkdir(path.dirname(file), { recursive: true }); const tmp = `${file}.${process.pid}.${written++}.tmp`; await writeFile(tmp, JSON.stringify(value)); await rename(tmp, file);};
export const readJson = (file) => { try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; }};
// the knot answers a repo's manifest uri as the repo didconst MANIFEST = /^at:\/\/(did:[^/]+)\/org\.tangled\.repo\.manifest\/self$/;
// one spelling per response: the nsid and its params sorted by name, values decoded, empty// values left out. the knot's xrpc diff spells its keys the same wayexport const keyOf = (nsid, params) => { const pairs = Object.entries(params) .filter(([, value]) => value !== "" && value !== undefined && value !== null) .map(([name, value]) => [name, String(value).replace(MANIFEST, "$1")]) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); return `${nsid}?${pairs.map(([name, value]) => `${name}=${value}`).join("&")}`;};
export const keyOfUrl = (url) => keyOf(url.pathname.slice("/xrpc/".length), Object.fromEntries(url.searchParams));123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100import { readFile } from "node:fs/promises";import path from "node:path";import { keyOf, readJson } from "./common.mjs";
// the knot writes <generation>/hook.json and its diff against the generation before it.// a renderer that was stopped or skipped generations unions the diffs back to the// generation it last rendered; a missing link means it can't know, so it renders againconst chainTo = (hookPath, hook, since) => { const generations = path.dirname(path.dirname(hookPath)); const chain = []; for (let at = { path: hookPath, hook }; at.hook.generation !== since; ) { if (!at.hook.previous || !at.hook.paths?.diff) return null; chain.push(path.resolve(path.dirname(at.path), at.hook.paths.diff)); const previous = path.join(generations, at.hook.previous, "hook.json"); const hookThere = readJson(previous); if (!hookThere || hookThere.generation !== at.hook.previous) return null; at = { path: previous, hook: hookThere }; } return chain;};
const parse = async (files) => { const diffs = { entries: 0, exact: new Set(), defaults: new Set(), any: new Map(), resets: new Set() }; for (const file of files) { let text; try { text = await readFile(file, "utf8"); } catch { return null; } for (const line of text.split("\n")) { if (!line) continue; diffs.entries++; const entry = JSON.parse(line); if (entry.op === "reset") { diffs.resets.add(entry.params.repo); } else if (entry.default) { diffs.defaults.add(entry.nsid); } else if (entry.any) { const params = { ...entry.params }; for (const name of entry.any) delete params[name]; if (!diffs.any.has(entry.nsid)) diffs.any.set(entry.nsid, { names: entry.any, keys: new Set() }); diffs.any.get(entry.nsid).keys.add(keyOf(entry.nsid, params)); } else { // an added key is one a page may have read while the map default answered it diffs.exact.add(keyOf(entry.nsid, entry.params)); } } } return diffs;};
const cache = new Map();export const readDiff = async (hookPath, hook, since) => { const key = `${hookPath}\0${since}`; if (!cache.has(key)) { const chain = chainTo(hookPath, hook, since); cache.set(key, chain === null ? null : await parse(chain)); } return cache.get(key);};
const nsidOf = (key) => key.slice(0, key.indexOf("?"));
// a key spelled by keyOf is decoded, so split it by hand rather than as a query stringconst splitKey = (key) => { const query = key.slice(key.indexOf("?") + 1); const params = {}; if (query === "") return params; for (const pair of query.split("&")) { const at = pair.indexOf("="); params[pair.slice(0, at)] = pair.slice(at + 1); } return params;};
const touched = (diffs, key) => { if (diffs.exact.has(key)) return true; const nsid = nsidOf(key); if (diffs.defaults.has(nsid)) return true; const any = diffs.any.get(nsid); if (!any) return false; const params = splitKey(key); for (const name of any.names) delete params[name]; return any.keys.has(keyOf(nsid, params));};
// the routes whose reads changed: pages to render again, and pages the budget or a failed// read left out, which may stand nowexport const dirtyPages = (prior, diffs) => { if (diffs.resets.has(prior.did)) return "all"; const dirty = new Set(); for (const [route, page] of Object.entries(prior.pages)) { if (page.deps.some((key) => touched(diffs, key))) dirty.add(route); } for (const [route, deps] of Object.entries(prior.left ?? {})) { if (deps.some((key) => touched(diffs, key))) dirty.add(route); } return dirty;};123456789101112131415161718192021222324252627282930313233343536// the bits of a rendered page the renderer reads or rewrites
const FETCHED = /<script type="application\/json" data-sveltekit-fetched data-url="([^"]*)"[^>]*>[\s\S]*?<\/script>/g;
// kit inlines each read the page made in the order the reads finished, which follows// concurrency. sorted, the same reads give the same bytes, so an unchanged page is unchangedexport const canonicalHtml = (html) => { const blocks = [...html.matchAll(FETCHED)]; if (blocks.length < 2) return html; const sorted = blocks.map((m) => m[0]).sort(); let at = 0; let rebuilt = ""; blocks.forEach((m, i) => { rebuilt += html.slice(at, m.index) + sorted[i]; at = m.index + m[0].length; }); return rebuilt + html.slice(at);};
// same-repo links a page shows, as decoded pathnames routeOf acceptsexport const linksOf = (html, origin, routeOf) => { const found = new Set(); for (const [, raw] of html.matchAll(/href="([^"#?]+)/g)) { const href = raw.replaceAll("&", "&"); if (!href.startsWith("/")) continue; let pathname; try { pathname = decodeURI(new URL(href, origin).pathname); } catch { continue; } pathname = pathname.replace(/\/$/, "") || "/"; if (routeOf(pathname) !== null) found.add(pathname); } return found;};1234567891011121314151617181920# the renderer's pages in front of the svfe SPA. include inside the knot's server block,# after its /xrpc/ locations. <out> is svfe-pages --out, <build> svfe's client build/location / { root <out>/www; # pages are stored gzipped next to an empty stub, so try_files sees the name gzip_static always; gunzip on; # a file or folder page is <path>.html, the repo's front page <repo>/index.html try_files $uri.html $uri/index.html @spa;}location @spa { root <build>; gzip_static on; try_files $uri /index.html;}location /_app/ { root <build>; gzip_static on; expires max;}123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181#!/usr/bin/env node// renders svfe's repo pages for a knot-static generation. reads only the knot's public xrpc// surface and writes static html per repo, swapped in atomically per repo:// site/<repo did> -> store/<token>/<generation>, the pages of one repo// www/<owner>/<name> -> site/<repo did>, the urls svfe links to (owner = handle or did)// state/<token>.json what each page read, so a diff of the knot's reads names the// pages to render again// svfe-pages <hook.json> --server <svfe build-server> --out <dir> [--read <xrpc base>]import { fork } from "node:child_process";import { createHash } from "node:crypto";import { readFileSync } from "node:fs";import { rm } from "node:fs/promises";import { availableParallelism } from "node:os";import path from "node:path";import { parseArgs } from "node:util";import { budgetFrom, BUDGET_OPTIONS } from "./budget.mjs";import { atomicJson, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: { server: { type: "string" }, out: { type: "string" }, read: { type: "string" }, // pages in flight per repo process jobs: { type: "string", default: "6" }, // repo processes, and the syntax highlighting threads each one runs workers: { type: "string", default: String(Math.max(1, Math.floor(availableParallelism() / 2))) }, highlighters: { type: "string", default: "2" }, full: { type: "boolean", default: false }, ...BUDGET_OPTIONS }});
await coordinate();
async function coordinate() { const hookPath = path.resolve(positionals[0]); const hook = JSON.parse(readFileSync(hookPath, "utf8")); const out = path.resolve(opts.out); const stateDir = path.join(out, "state"); const statusPath = hook.paths?.status ?? path.join(out, "status.json"); const knotState = readJson(path.join(stateDir, "knot.json")); // pages load the client chunks of the build that rendered them, so a new svfe build // renders everything again const build = createHash("sha256").update(readFileSync(path.join(opts.server, "manifest-full.js"))).digest("hex").slice(0, 16); const loadState = (did) => readJson(path.join(stateDir, tokenOf(did) + ".json")); const budget = budgetFrom(opts, hook.repos.length); const t0 = performance.now();
const status = { generation: hook.generation, state: "rendering", started_at: new Date().toISOString(), repos: {} }; // one write at a time, so the file ends as the newest status let statusWrite = Promise.resolve(); const writeStatus = (extra = {}) => { const snapshot = JSON.parse(JSON.stringify({ ...status, ...extra })); statusWrite = statusWrite.catch(() => {}).then(() => atomicJson(statusPath, snapshot)); return statusWrite; };
const children = new Set(); // the knot stops a renderer it supersedes with SIGTERM to the process group process.on("SIGTERM", async () => { for (const child of children) child.kill("SIGTERM"); await writeStatus({ state: "superseded" }).catch(() => {}); process.exit(3); });
await writeStatus(); for (const did of hook.removed_repos ?? []) await removeRepo(out, did, loadState(did));
// a page is current if every read it made is unchanged since the generation it was // rendered at, so each repo walks the knot's diffs back to its own generation const full = opts.full || !knotState || knotState.knot_format !== hook.format || knotState.renderer_format !== RENDERER_FORMAT || knotState.build !== build; const plans = []; let diffEntries = 0; for (const repo of hook.repos) { const prior = loadState(repo.did); const fresh = full || !prior || repo.change === "added" || prior.prefix !== prefixOf(repo); if (fresh) { plans.push({ repo, work: "all", weight: prior ? Object.keys(prior.pages).length : Infinity }); continue; } if (prior.generation === hook.generation) continue; const diffs = await readDiff(hookPath, hook, prior.generation); if (diffs === null) { plans.push({ repo, work: "all", weight: Object.keys(prior.pages).length }); continue; } diffEntries = Math.max(diffEntries, diffs.entries); const dirty = dirtyPages(prior, diffs); if (dirty === "all" || dirty.size > 0 || repo.change !== "unchanged") { plans.push({ repo, work: dirty === "all" ? "all" : [...dirty], weight: dirty === "all" ? Object.keys(prior.pages).length : dirty.size }); } else { // nothing it read changed: the pages stand for this generation too await atomicJson(path.join(stateDir, tokenOf(repo.did) + ".json"), { ...prior, generation: hook.generation }); status.repos[repo.did] = { skipped: true }; } } // biggest first, so the last worker to finish isn't the one holding ghostty plans.sort((a, b) => b.weight - a.weight);
const workers = Math.max(1, Math.min(Number(opts.workers), plans.length)); const queue = [...plans]; const totals = { rendered: 0, kept: 0, dropped: 0, over_budget: 0, pages: 0, bytes: 0, reads: 0 }; const failures = []; await Promise.all( Array.from({ length: workers }, async () => { for (let plan; (plan = queue.shift()) !== undefined; ) { const t = performance.now(); const result = await runWorker(children, plan, { hookPath, out, budget }).catch((error) => ({ error: String(error) })); if (result.error) { failures.push(plan.repo.did); status.repos[plan.repo.did] = { error: result.error }; } else { status.repos[plan.repo.did] = { ...result, wall_ms: Math.round(performance.now() - t) }; for (const key of Object.keys(totals)) totals[key] += result[key] ?? 0; } await writeStatus(); } }) );
if (failures.length === 0) { await atomicJson(path.join(stateDir, "knot.json"), { knot_format: hook.format, renderer_format: RENDERER_FORMAT, build, generation: hook.generation }); } const summary = { state: failures.length === 0 ? "done" : "failed", finished_at: new Date().toISOString(), diff_entries: diffEntries, totals, total_ms: Math.round(performance.now() - t0) }; await writeStatus(summary); process.stdout.write(JSON.stringify({ ...summary, repos: status.repos }) + "\n", () => process.exit(failures.length === 0 ? 0 : 1));}
function runWorker(children, plan, { hookPath, out, budget }) { return new Promise((resolve, reject) => { const args = [hookPath, "--out", out, "--server", opts.server, "--jobs", opts.jobs, "--highlighters", opts.highlighters]; if (opts.read) args.push("--read", opts.read); const child = fork(new URL("./worker.mjs", import.meta.url).pathname, args, { stdio: ["ignore", "inherit", "inherit", "ipc"], // SVFE_PAGES_PROFILE=<dir> writes a cpu profile per worker there execArgv: process.env.SVFE_PAGES_PROFILE ? ["--cpu-prof", "--cpu-prof-dir", process.env.SVFE_PAGES_PROFILE] : [] }); children.add(child); let result = null; child.on("message", (message) => { if (message.ready) child.send({ repo: plan.repo, work: plan.work, budget }); else result = message; }); child.on("exit", (code, signal) => { children.delete(child); if (result) resolve(result); else reject(new Error(`worker for ${plan.repo.did} exited ${signal ?? code}`)); }); });}
async function removeRepo(out, did, prior) { await rm(path.join(out, "site", did), { force: true }); for (const alias of prior?.aliases ?? []) await rm(path.join(out, "www", alias), { force: true }); await rm(path.join(out, "store", tokenOf(did)), { recursive: true, force: true }); await rm(path.join(out, "state", tokenOf(did) + ".json"), { force: true });}123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100import assert from "node:assert/strict";import { mkdtemp, mkdir, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { test } from "node:test";import { keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";import { canonicalHtml, linksOf } from "./html.mjs";
test("keyOf sorts params, drops empty ones and matches a request url", () => { assert.equal(keyOf("org.tangled.git.getTree", { ref: "main", repo: "did:plc:a", path: "" }), "org.tangled.git.getTree?ref=main&repo=did:plc:a"); const url = new URL("https://k.test/xrpc/org.tangled.git.getBlob?repo=did%3Aplc%3Aa&ref=main&path=src%2Fa%20b.rs"); assert.equal(keyOfUrl(url), "org.tangled.git.getBlob?path=src/a b.rs&ref=main&repo=did:plc:a"); assert.equal(keyOf("org.tangled.feed.countStars", { subject: "at://did:plc:a/org.tangled.repo.manifest/self" }), "org.tangled.feed.countStars?subject=did:plc:a");});
test("prefixOf links by handle, or by owner did without one", () => { assert.equal(prefixOf({ owner_handle: "Alice.Test", owner_did: "did:plc:o", name: "Core" }), "/alice.test/Core"); assert.equal(prefixOf({ owner_handle: null, owner_did: "did:web:o.test", name: "x" }), "/did:web:o.test/x");});
const fetched = (url, body) => `<script type="application/json" data-sveltekit-fetched data-url="${url}">${body}</script>`;
test("canonicalHtml orders kit's inlined reads, whatever order they finished in", () => { const a = fetched("/xrpc/a", "1"); const b = fetched("/xrpc/b", "2"); assert.equal(canonicalHtml(`<p>x</p>${b}${a}<i></i>`), `<p>x</p>${a}${b}<i></i>`); assert.equal(canonicalHtml(`<p>x</p>${a}${b}`), `<p>x</p>${a}${b}`);});
test("linksOf keeps same-repo routes, decoded and without trailing slashes", () => { const html = '<a href="/o/r/tree/main/src/">x</a><a href="/o/r/blob/main/a%20b.rs#L1">y</a><a href="https://e.test/o/r">z</a><a href="/o/r/history/main/x">h</a>'; const found = linksOf(html, "https://k.test", (p) => (p.includes("history") ? null : p)); assert.deepEqual([...found], ["/o/r/tree/main/src", "/o/r/blob/main/a b.rs"]);});
const line = (entry) => JSON.stringify(entry) + "\n";
const chain = async (gens) => { const root = await mkdtemp(path.join(tmpdir(), "svfe-pages-")); let previous = null; let hookPath; for (const [generation, entries] of gens) { const dir = path.join(root, generation); await mkdir(dir); const hook = { generation, previous, paths: { diff: previous ? "xrpc-diff.jsonl" : null } }; if (previous) await writeFile(path.join(dir, "xrpc-diff.jsonl"), entries.map(line).join("")); hookPath = path.join(dir, "hook.json"); await writeFile(hookPath, JSON.stringify(hook)); previous = generation; } return { hookPath, hook: JSON.parse(await (await import("node:fs/promises")).readFile(hookPath, "utf8")) };};
const page = (...deps) => ({ deps, bytes: 1, links: [] });
test("a diff dirties the pages that read a changed key, unioned back to the page's generation", async () => { const tree = { op: "changed", nsid: "org.tangled.git.getTree", params: { repo: "did:plc:a", ref: "main", path: "src" }, old: "x", new: "y" }; const blob = { op: "removed", nsid: "org.tangled.git.getBlob", params: { repo: "did:plc:a", ref: "main", path: "b" }, old: "x", new: null }; const { hookPath, hook } = await chain([["g1", []], ["g2", [tree]], ["g3", [blob]]]); const prior = { did: "did:plc:a", pages: { "/o/r/tree/main/src": page(keyOf(tree.nsid, tree.params)), "/o/r/blob/main/b": page(keyOf(blob.nsid, blob.params)), "/o/r": page("org.tangled.git.getTree?ref=main&repo=did:plc:a") }, left: { "/o/r/blob/main/c": [keyOf(tree.nsid, tree.params)] } }; const fromG1 = await readDiff(hookPath, hook, "g1"); assert.deepEqual([...dirtyPages(prior, fromG1)].sort(), ["/o/r/blob/main/b", "/o/r/blob/main/c", "/o/r/tree/main/src"]); const fromG2 = await readDiff(hookPath, hook, "g2"); assert.deepEqual([...dirtyPages(prior, fromG2)], ["/o/r/blob/main/b"]); assert.equal(await readDiff(hookPath, hook, "gone"), null);});
test("default, any and reset lines", async () => { const { hookPath, hook } = await chain([ ["g1", []], [ "g2", [ { op: "changed", nsid: "org.tangled.repo.countForks", params: {}, default: true, old: "a", new: "b" }, { op: "changed", nsid: "org.tangled.search.query", params: { kind: "issue", repoDid: "did:plc:a" }, any: ["limit", "cursor"], old: "a", new: "b" }, { op: "reset", params: { repo: "did:plc:z" } } ] ] ]); const diffs = await readDiff(hookPath, hook, "g1"); const prior = { did: "did:plc:a", pages: { "/o/r": page("org.tangled.repo.countForks?subject=did:plc:a"), "/o/r/issues": page("org.tangled.search.query?cursor=3&kind=issue&limit=50&repoDid=did:plc:a"), "/o/r/pulls": page("org.tangled.search.query?kind=pull&repoDid=did:plc:a") } }; assert.deepEqual([...dirtyPages(prior, diffs)].sort(), ["/o/r", "/o/r/issues"]); assert.equal(dirtyPages({ ...prior, did: "did:plc:z" }, diffs), "all");});123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353// renders one repo's pages in its own process, so repos render in parallel across cores.// the coordinator sends {repo, work, budget} and gets the repo's stats backimport { AsyncLocalStorage } from "node:async_hooks";import { readFileSync } from "node:fs";import { link, mkdir, readdir, rename, rm, symlink, writeFile } from "node:fs/promises";import path from "node:path";import { pathToFileURL } from "node:url";import { parseArgs, promisify } from "node:util";import { gzip as gzipCallback } from "node:zlib";import { rankOf } from "./budget.mjs";import { atomicJson, keyOfUrl, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { canonicalHtml, linksOf } from "./html.mjs";
const gzip = promisify(gzipCallback);
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: { server: { type: "string" }, out: { type: "string" }, read: { type: "string" }, jobs: { type: "string", default: "6" }, highlighters: { type: "string", default: "2" } }});const hook = JSON.parse(readFileSync(positionals[0], "utf8"));const out = path.resolve(opts.out);const origin = hook.knot.base_url.replace(/\/$/, "");// where reads go: the knot's own nginx on loopback, or its public https originconst readBase = (opts.read ?? hook.knot.base_url).replace(/\/$/, "");const JOBS = Number(opts.jobs);
// --- reads -----------------------------------------------------------------------------
const deps = new AsyncLocalStorage();const reads = { count: 0, ms: 0 };// file sizes from the folder listings this run read, so a file over the budget never rendersconst sizes = new Map();const sizeKey = (ref, file) => `${ref}\0${file}`;
const learnSizes = (url, body) => { const ref = url.searchParams.get("ref") ?? ""; const dir = url.searchParams.get("path") ?? ""; try { for (const entry of JSON.parse(body).entries ?? []) { sizes.set(sizeKey(ref, dir ? `${dir}/${entry.name}` : entry.name), entry.size ?? 0); } } catch {}};
const answer = async (request) => { const url = new URL(request.url); if (url.origin !== origin || !url.pathname.startsWith("/xrpc/")) { // nothing off the knot at render time: the browser fetches it live after hydrating return new Response(null, { status: 404 }); } const t = performance.now(); const res = await fetch(readBase + url.pathname + url.search, { headers: { host: new URL(origin).host, "accept-encoding": "identity" } }); const body = await res.text(); reads.count++; reads.ms += performance.now() - t; deps.getStore()?.push({ key: keyOfUrl(url), status: res.status }); if (res.status === 200 && url.pathname === "/xrpc/org.tangled.git.getTree") learnSizes(url, body); const headers = new Headers(); if (res.headers.has("content-type")) headers.set("content-type", res.headers.get("content-type")); return new Response(body, { status: res.status, headers });};
globalThis.__knotStaticPrerender = { answer, renderWorkers: Number(opts.highlighters) };const { server } = await import(pathToFileURL(path.resolve(opts.server, "instance.js")).href);await server.init({ env: {}, read: () => { throw new Error("no server assets"); } });
const renderPage = async (route) => { const collected = []; const t = performance.now(); const res = await deps.run(collected, () => server.respond(new Request(origin + encodeURI(route)), { getClientAddress: () => "127.0.0.1" }) ); const html = await res.text(); return { status: res.status, html, reads: collected, ms: performance.now() - t };};
// a page is static only if every git read it made was answered: a ref, path or diff the// knot didn't project leaves it to the clientconst accepted = (page) => page.status === 200 && page.reads.every((r) => r.status === 200 || !r.key.startsWith("org.tangled.git."));
const pool = async (items, fn) => { const queue = [...items]; await Promise.all( Array.from({ length: JOBS }, async () => { for (let item; (item = queue.shift()) !== undefined; ) await fn(item); }) );};
// --- routes ------------------------------------------------------------------------------
// views a static page can stand for; the rest stay client-rendered. history is a mock// in svfe today, the issue and pull forms need a signed-in viewerconst STATIC_VIEWS = new Set(["", "tree", "blob", "commits", "commit", "branches", "tags", "issues", "pulls", "languages"]);const NOT_STATIC = /^\/(issues|pulls)\/new(\/|$)/;// files, folders and logs get pages at the default branch onlyconst BROWSED = new Set(["tree", "blob", "commits"]);
const routeOf = (prefix, pathname, repo) => { if (pathname !== prefix && !pathname.startsWith(prefix + "/")) return null; const rest = pathname.slice(prefix.length); if (NOT_STATIC.test(rest)) return null; const [, view = "", ref, ...file] = rest.split("/"); if (!STATIC_VIEWS.has(view)) return null; if (BROWSED.has(view) && ref && ref !== repo.default_branch) return null; if (view === "commits" && file.length > 0) return null; return { route: pathname, view, ref, file: file.join("/"), depth: rest.split("/").length };};
// a file is the file name plus ".html", a folder its own directory, so a file page costs// no directory. the repo's front page is the one index.htmlconst fileFor = (dir, prefix, route) => route === prefix ? path.join(dir, "index.html") : path.join(dir, route.slice(prefix.length + 1) + ".html");
// --- one repo ------------------------------------------------------------------------------
const storeOf = (did) => path.join(out, "store", tokenOf(did));
const activeGeneration = () => readJson(hook.paths?.active ?? "")?.generation ?? hook.generation;class Superseded extends Error {}const checkCurrent = () => { const active = activeGeneration(); if (active !== hook.generation) throw new Superseded(active);};
const seedsOf = (prefix) => [prefix, `${prefix}/issues`, `${prefix}/pulls`, `${prefix}/branches`, `${prefix}/tags`];
// pages nothing links to any more (a commit fallen off the log, a deleted file's page that// still answers through an old commit) go, so a repo's pages track what its front pages reachconst reachable = (prefix, pages, inBudget) => { const seen = new Set(seedsOf(prefix).filter((route) => pages[route])); const queue = [...seen]; for (let route; (route = queue.shift()) !== undefined; ) { for (const next of pages[route].links ?? []) { if (pages[next] && inBudget(next) && !seen.has(next)) { seen.add(next); queue.push(next); } } } return seen;};
const renderRepo = async (repo, work, budget) => { const prefix = prefixOf(repo); const stateFile = path.join(out, "state", tokenOf(repo.did) + ".json"); const saved = readJson(stateFile); const prior = work === "all" ? null : saved; const pages = prior ? structuredClone(prior.pages) : {}; // routes left out with the reads that failed them, retried when one of those changes const left = prior ? structuredClone(prior.left ?? {}) : {}; const dirty = new Set(prior ? work : []); const genDir = path.join(storeOf(repo.did), hook.generation); const tmpDir = genDir + ".tmp"; // a run that was superseded or killed leaves its .tmp behind for (const entry of await readdir(storeOf(repo.did)).catch(() => [])) if (entry.endsWith(".tmp")) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true }); await mkdir(tmpDir, { recursive: true }); // every page's stub is one empty inode: nginx's try_files needs the name, gzip_static // serves the .gz next to it const empty = path.join(tmpDir, ".empty"); await writeFile(empty, "");
const stats = { rendered: 0, kept: 0, dropped: 0, over_budget: 0, render_ms: 0 }; if (prior) { const priorDir = path.join(storeOf(repo.did), prior.generation); await pool(Object.keys(pages).filter((route) => !dirty.has(route)), async (route) => { const to = fileFor(tmpDir, prefix, route); await mkdir(path.dirname(to), { recursive: true }); await link(empty, to); // a page whose file went missing renders again await link(fileFor(priorDir, prefix, route) + ".gz", to + ".gz").catch(() => dirty.add(route)); }); }
let bytes = Object.values(pages).reduce((sum, page) => sum + page.bytes, 0); const overBudget = new Set(); // what the budget left out last time stays out until a full render const over = new Set(prior?.over ?? []); const seen = new Set([...Object.keys(pages), ...Object.keys(left), ...over]); let todo = prior ? [...dirty] : seedsOf(prefix); const rendered = new Set(); for (const route of todo) seen.add(route); const rankOfRoute = (route) => { const parsed = routeOf(prefix, route, repo); return parsed ? rankOf(parsed.view, parsed.depth) : 0; };
while (todo.length > 0) { checkCurrent(); const batch = todo.sort((a, b) => rankOfRoute(a) - rankOfRoute(b) || (a < b ? -1 : a > b ? 1 : 0)); todo = []; // the budget keeps the best ranked pages whatever order their renders finish in: // render the batch, then take its pages in rank order let room = budget.pages - Object.keys(pages).length; const wanted = batch.filter((route) => { if (route in pages) return true; if (room <= 0) { overBudget.add(route); return false; } room--; return true; }); const results = new Map(); await pool(wanted, async (route) => { const page = await renderPage(route); stats.render_ms += page.ms; const deps = [...new Set(page.reads.map((r) => r.key))].sort(); if (!accepted(page)) return void results.set(route, { deps }); const gz = await gzip(canonicalHtml(page.html), { level: 9, memLevel: 9 }); // commit pages link their parents and trees at their own commit: only the log // and the front page decide which commits get pages const links = route.startsWith(`${prefix}/commit/`) ? [] : [...linksOf(page.html, origin, (pathname) => routeOf(prefix, pathname, repo))]; results.set(route, { deps, gz, links }); }); const writes = []; for (const route of wanted) { const known = route in pages; const { deps, gz, links } = results.get(route); if (known) bytes -= pages[route].bytes; if (!gz || bytes + gz.length > budget.bytes) { if (known) { delete pages[route]; stats.dropped++; } if (gz) overBudget.add(route); else left[route] = deps; writes.push(route); continue; } bytes += gz.length; delete left[route]; const kept = links.filter((next) => keep(next)); pages[route] = { deps, bytes: gz.length, links: kept }; rendered.add(route); stats.rendered++; writes.push(route); for (const next of kept) { if (!seen.has(next)) { seen.add(next); todo.push(next); } } } await pool(writes, async (route) => { const file = fileFor(tmpDir, prefix, route); await rm(file, { force: true }); await rm(file + ".gz", { force: true }); const { gz } = results.get(route); if (!(route in pages)) return; await mkdir(path.dirname(file), { recursive: true }); await link(empty, file); await writeFile(file + ".gz", gz); }); }
// a link the budget rules out is never rendered, so its url stays the SPA's function keep(next) { const parsed = routeOf(prefix, next, repo); if (over.has(next)) return false; if (parsed.view === "blob" && (sizes.get(sizeKey(parsed.ref, parsed.file)) ?? 0) > budget.blobBytes) { overBudget.add(next); return false; } return true; }
// commit pages: the newest the front page and the log link, then branch and tag tips. // the rest go, and come back if a page links them again const commitOrder = [prefix, `${prefix}/commits/${repo.default_branch}`, `${prefix}/branches`, `${prefix}/tags`] .flatMap((route) => pages[route]?.links ?? []) .filter((route) => route.startsWith(`${prefix}/commit/`)); const commits = new Set([...new Set(commitOrder)].slice(0, budget.commits)); const inBudget = (route) => !route.startsWith(`${prefix}/commit/`) || commits.has(route);
const live = reachable(prefix, pages, inBudget); for (const route of Object.keys(pages)) { if (live.has(route)) continue; bytes -= pages[route].bytes; delete pages[route]; await rm(fileFor(tmpDir, prefix, route), { force: true }); await rm(fileFor(tmpDir, prefix, route) + ".gz", { force: true }); stats.dropped++; } stats.kept = Object.keys(pages).filter((route) => !rendered.has(route)).length; for (const route of overBudget) over.add(route); stats.over_budget = over.size; checkCurrent();
await publish(repo, saved, genDir, tmpDir); await atomicJson(stateFile, { did: repo.did, prefix, aliases: aliasesOf(repo), format: RENDERER_FORMAT, generation: hook.generation, pages, left, over: [...over].sort() }); // the state names the new generation now, so the old one can go for (const entry of await readdir(storeOf(repo.did))) { if (entry !== hook.generation) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true }); } return { pages: Object.keys(pages).length, bytes, reads: reads.count, read_ms: Math.round(reads.ms), ...stats, render_ms: Math.round(stats.render_ms) };};
// urls svfe serves the repo under: the handle form it links to, and the owner did form,// which hydrates and then redirects to the handle like the SPA doesconst aliasesOf = (repo) => [...new Set([prefixOf(repo).slice(1), `${repo.owner_did}/${repo.name}`])];
const swapLink = async (target, at) => { await mkdir(path.dirname(at), { recursive: true }); const tmp = `${at}.${process.pid}.tmp`; await rm(tmp, { force: true }); await symlink(path.relative(path.dirname(at), target), tmp); await rename(tmp, at);};
// the store dir by rename, then the site link by rename, so nginx sees the old or the new// pages of this repo and never half of eachconst publish = async (repo, saved, genDir, tmpDir) => { await rm(genDir, { recursive: true, force: true }); await rename(tmpDir, genDir); const site = path.join(out, "site", repo.did); await swapLink(genDir, site); const aliases = aliasesOf(repo); for (const alias of aliases) await swapLink(site, path.join(out, "www", alias)); for (const alias of saved?.aliases ?? []) { if (!aliases.includes(alias)) await rm(path.join(out, "www", alias), { force: true }); }};
process.on("message", async ({ repo, work, budget }) => { try { const result = await renderRepo(repo, work, budget); process.send(result, () => process.exit(0)); } catch (error) { const message = error instanceof Superseded ? "superseded" : String(error?.stack ?? error); process.send({ error: message }, () => process.exit(error instanceof Superseded ? 3 : 1)); }});process.send({ ready: true });