// renders one repo's pages in its own process, so repos render in parallel across cores.
// the coordinator sends {repo, work, budget} and gets the repo's stats back
import { AsyncLocalStorage } from "node:async_hooks";
import { linkSync, mkdirSync, readFileSync, statSync } from "node:fs";
import { link, mkdir, readdir, realpath, rename, rm, symlink, writeFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { parseArgs } from "node:util";
import { gzipSync } from "node:zlib";
import { rankOf } from "./budget.mjs";
import { atomicJson, fileOf, keyOfUrl, ownerFiles, PAGE_QUERY, prefixOf, readJson, RENDERER_FORMAT, tokenOf, unpublishOwner } from "./common.mjs";
import { canonicalHtml, inlinedOverCap, linksOf } from "./html.mjs";
import { COUNTS, latestPath, ROUTES, SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
const { values: opts, positionals } = parseArgs({
allowPositionals: true,
options: {
server: { type: "string" },
out: { type: "string" },
read: { type: "string" },
jobs: { type: "string", default: "6" },
highlighters: { type: "string", default: "0" }
}
});
const hook = JSON.parse(readFileSync(positionals[0], "utf8"));
const out = path.resolve(opts.out);
const origin = hook.knot.base_url.replace(/\/$/, "");
const originHost = new URL(origin).host;
// where reads go: the knot's own nginx on loopback, or its public https origin
const readBase = (opts.read ?? hook.knot.base_url).replace(/\/$/, "");
const JOBS = Number(opts.jobs);
// --- reads -----------------------------------------------------------------------------
const deps = new AsyncLocalStorage();
const reads = { count: 0, ms: 0 };
// file sizes from the folder listings this run read, so a file over the budget never renders
const sizes = new Map();
const files = new Set();
const readCache = new Map();
let readCacheChars = 0;
const MAX_CACHE_ENTRIES = 512;
const MAX_CACHE_BODY_CHARS = 262_144;
const MAX_CACHE_TOTAL_CHARS = 8 * 1024 * 1024;
const sizeKey = (ref, file) => `${ref}\0${file}`;
const learnSizes = (url, body) => {
const ref = url.searchParams.get("ref") ?? "";
const dir = url.searchParams.get("path") ?? "";
try {
for (const entry of JSON.parse(body).entries ?? []) {
const k = sizeKey(ref, dir ? `${dir}/${entry.name}` : entry.name);
sizes.set(k, entry.size ?? 0);
if (entry.mode && entry.mode !== "0040000" && entry.mode !== "040000") files.add(k);
}
} catch {}
};
const fetchXrpc = async (url, cacheKey) => {
const t = performance.now();
const res = await fetch(readBase + cacheKey, {
headers: { host: originHost, "accept-encoding": "identity" }
});
const body = await res.text();
reads.count++;
reads.ms += performance.now() - t;
if (res.status === 200 && url.pathname === "/xrpc/org.tangled.git.getTree") learnSizes(url, body);
const contentType = res.headers.get("content-type");
return { status: res.status, contentType, body };
};
// a read the page renders from: a change to it renders the page again
const answer = (request) => respond(request, true);
// a read only shared files carry, which the page doesn't depend on
const read = (request) => respond(request, false);
const respond = async (request, own) => {
const url = new URL(request.url);
if (url.origin !== origin || !url.pathname.startsWith("/xrpc/")) {
// nothing off the knot at render time: the browser fetches it live after hydrating
return new Response(null, { status: 404 });
}
const cacheKey = url.pathname + url.search;
let pending = readCache.get(cacheKey);
if (pending) {
readCache.delete(cacheKey);
readCache.set(cacheKey, pending);
} else {
pending = fetchXrpc(url, cacheKey).then((entry) => {
const len = entry.body.length;
if (len > MAX_CACHE_BODY_CHARS) {
readCache.delete(cacheKey);
return entry;
}
pending.chars = len;
readCacheChars += len;
while (readCache.size > MAX_CACHE_ENTRIES || readCacheChars > MAX_CACHE_TOTAL_CHARS) {
const oldest = readCache.keys().next().value;
if (oldest === undefined || oldest === cacheKey) break;
const oldEntry = readCache.get(oldest);
readCache.delete(oldest);
readCacheChars -= oldEntry?.chars ?? 0;
}
return entry;
});
readCache.set(cacheKey, pending);
}
const entry = await pending;
if (own) deps.getStore()?.reads.push({ key: keyOfUrl(url), status: entry.status });
const headers = new Headers();
if (entry.contentType) headers.set("content-type", entry.contentType);
return new Response(entry.body, { status: entry.status, headers });
};
const COUNT_READS = {
issues: ["org.tangled.repo.countIssues", { state: "open" }],
pulls: ["org.tangled.repo.countPulls", { status: "open" }]
};
// svfe's spelling of the counts, so the keys match the knot's diff lines for them. no value
// when one fails, and the browser asks the knot itself
const readCounts = async (did) => {
const value = {};
const keys = [];
for (const [name, [nsid, params]] of Object.entries(COUNT_READS)) {
const url = new URL(`/xrpc/${nsid}?${new URLSearchParams({ subject: did, ...params })}`, origin);
keys.push(keyOfUrl(url));
const entry = await fetchXrpc(url, url.pathname + url.search);
if (entry.status !== 200) return { value: null, keys };
value[name] = JSON.parse(entry.body).count ?? 0;
}
return { value, keys };
};
// this run's shared files of the repo, by hash
const shared = new Map();
const share = (value, name) => {
const { hash, body } = sharedOf(value);
if (!shared.has(hash)) shared.set(hash, body);
deps.getStore()?.shared.add(hash);
if (name && fitsFile(name) && name.split("/").every((part) => part && !part.startsWith("."))) deps.getStore()?.latest.set(name, hash);
return sharedPath(hash);
};
globalThis.__knotStaticPrerender = { answer, read, share, renderWorkers: Number(opts.highlighters) };
const { server } = await import(pathToFileURL(path.resolve(opts.server, "instance.js")).href);
await server.init({ env: {}, read: () => { throw new Error("no server assets"); } });
const renderPage = async (route) => {
const collected = { reads: [], shared: new Set(), latest: new Map() };
const t = performance.now();
const res = await deps.run(collected, () =>
server.respond(new Request(origin + encodeURI(route)), { getClientAddress: () => "127.0.0.1" })
);
const html = await res.text();
return {
status: res.status,
html,
reads: collected.reads,
shared: [...collected.shared].sort(),
latest: Object.fromEntries(collected.latest),
ms: performance.now() - t
};
};
// a page is static only if every git read of its repo was answered: a ref, path or diff the
// knot didn't project leaves it to the client. a fork hosted on another knot gets the same
// answer from this knot in the browser, so a pull from one doesn't hold its page back
const REPO_PARAM = /[?&]repo=([^&]*)/;
const accepted = (page, did) =>
page.status === 200 &&
page.reads.every((r) => r.status === 200 || !r.key.startsWith("org.tangled.git.") || REPO_PARAM.exec(r.key)?.[1] !== did);
// the repo's peak heap, reported with its stats, shows how near a worker runs to its cap
let heapPeak = 0;
setInterval(() => (heapPeak = Math.max(heapPeak, process.memoryUsage().heapUsed)), 50).unref();
const pool = async (items, fn) => {
const queue = [...items];
await Promise.all(
Array.from({ length: JOBS }, async () => {
for (let item; (item = queue.shift()) !== undefined; ) await fn(item);
})
);
};
// --- routes ------------------------------------------------------------------------------
// views a static page can stand for; the rest stay client-rendered. history is a mock
// in svfe today, the issue and pull forms need a signed-in viewer
const STATIC_VIEWS = new Set(["", "tree", "blob", "commits", "commit", "branches", "tags", "issues", "pulls", "languages"]);
const NOT_STATIC = /^\/(issues|pulls)\/new(\/|$)/;
// files, folders and logs get pages at the default branch only
const BROWSED = new Set(["tree", "blob", "commits"]);
// lists whose later pages get pages of their own
const PAGED = new Set(["commits", "branches", "tags"]);
// a path component a file system takes, with room for "~page-N.html.gz" (most cap a name at
// 255 bytes, git doesn't): a page whose name won't fit is left to the client
const NAME_BYTES = 230;
const fitsFile = (rest) => rest.split("/").every((name) => Buffer.byteLength(name) <= NAME_BYTES);
const routeOf = (prefix, route, repo) => {
const paged = PAGE_QUERY.exec(route);
const pathname = paged ? route.slice(0, paged.index) : route;
if (pathname !== prefix && !pathname.startsWith(prefix + "/")) return null;
const rest = pathname.slice(prefix.length);
if (NOT_STATIC.test(rest) || !fitsFile(rest)) return null;
const [, view = "", ref, ...file] = rest.split("/");
if (!STATIC_VIEWS.has(view)) return null;
if (BROWSED.has(view) && ref && ref !== repo.default_branch) return null;
if (view === "commits" && file.length > 0) return null;
const page = paged ? Number(paged[1]) : 1;
if (paged && (!PAGED.has(view) || page < 2)) return null;
return { route, view, ref, file: file.join("/"), depth: rest.split("/").length, page };
};
const fileFor = (dir, prefix, route) => path.join(dir, fileOf(prefix, route));
// every page of a folder links the folder's whole listing and most pages make the same reads,
// so a repo keeps one copy of each route and key: a folder of 1500 generated files otherwise
// holds 1500 copies of its listing, 30 MB of a worker's heap
const interned = new Map();
const intern = (value) => {
const known = interned.get(value);
if (known !== undefined) return known;
interned.set(value, value);
return value;
};
// --- one repo ------------------------------------------------------------------------------
const storeOf = (did) => path.join(out, "store", tokenOf(did));
const activeGeneration = () => readJson(hook.paths?.active ?? "")?.generation ?? hook.generation;
class Superseded extends Error {}
const checkCurrent = () => {
const active = activeGeneration();
if (active !== hook.generation) throw new Superseded(active);
};
const seedsOf = (prefix) => [prefix, `${prefix}/issues`, `${prefix}/pulls`, `${prefix}/branches`, `${prefix}/tags`];
// pages nothing links to any more (a commit fallen off the log, a deleted file's page that
// still answers through an old commit) go, so a repo's pages track what its front pages reach
const reachable = (prefix, pages, inBudget) => {
const seen = new Set(seedsOf(prefix).filter((route) => pages[route]));
const queue = [...seen];
for (let route; (route = queue.shift()) !== undefined; ) {
for (const next of pages[route].links ?? []) {
if (pages[next] && inBudget(next) && !seen.has(next)) {
seen.add(next);
queue.push(next);
}
}
}
return seen;
};
const renderRepo = async (repo, work, budget) => {
reads.count = 0;
reads.ms = 0;
heapPeak = 0;
sizes.clear();
files.clear();
readCache.clear();
readCacheChars = 0;
shared.clear();
interned.clear();
const prefix = prefixOf(repo);
const stateFile = path.join(out, "state", tokenOf(repo.did) + ".json");
const saved = readJson(stateFile);
const savedAliases = saved?.aliases ?? [];
const prior = work === "all" ? null : saved;
const pages = prior ? prior.pages : {};
for (const page of Object.values(pages)) {
page.deps = page.deps.map(intern);
page.links = page.links?.map((at) => intern(prior.routes[at]));
}
// routes left out with the reads that failed them, retried when one of those changes
const left = prior ? (prior.left ?? {}) : {};
const dirty = new Set(prior ? work : []);
const genDir = path.join(storeOf(repo.did), hook.generation);
const tmpDir = genDir + ".tmp";
// a run that was superseded or killed leaves its .tmp behind
for (const entry of await readdir(storeOf(repo.did)).catch(() => []))
if (entry.endsWith(".tmp")) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true });
await mkdir(tmpDir, { recursive: true });
// every page's stub is one empty inode: nginx's try_files needs the name, gzip_static
// serves the .gz next to it
const empty = path.join(tmpDir, ".empty");
await writeFile(empty, "");
// the shared files in tmpDir, by hash, with their gzipped size
const sharedDir = path.join(tmpDir, SHARED_DIR);
const sharedBytes = new Map();
// the hash each stable name points at, kept from the last run unless a page shares it anew
const latest = { ...(prior?.latest ?? {}) };
const priorDir = saved
? await realpath(path.join(out, "site", repo.did)).catch(() => path.join(storeOf(repo.did), saved.generation))
: null;
const madeDirs = new Set([tmpDir]);
const dirPromises = new Map();
const ensureDir = (dir) => {
if (madeDirs.has(dir)) return;
let p = dirPromises.get(dir);
if (!p) {
p = mkdir(dir, { recursive: true }).then(() => {
madeDirs.add(dir);
dirPromises.delete(dir);
});
dirPromises.set(dir, p);
}
return p;
};
const stats = { rendered: 0, kept: 0, dropped: 0, over_budget: 0, render_ms: 0, inlined_over_cap: 0 };
let inlinedMax = null;
if (prior) {
for (const route of Object.keys(pages)) {
if (dirty.has(route)) continue;
const to = fileFor(tmpDir, prefix, route);
const dir = path.dirname(to);
if (!madeDirs.has(dir)) {
mkdirSync(dir, { recursive: true });
madeDirs.add(dir);
}
try {
linkSync(empty, to);
linkSync(fileFor(priorDir, prefix, route) + ".gz", to + ".gz");
for (const hash of pages[route].shared ?? []) {
if (sharedBytes.has(hash)) continue;
const file = path.join(sharedDir, hash + ".json");
mkdirSync(sharedDir, { recursive: true });
linkSync(path.join(priorDir, SHARED_DIR, hash + ".json.gz"), file + ".gz");
linkSync(empty, file);
sharedBytes.set(hash, statSync(file + ".gz").size);
}
} catch {
// a page whose file went missing renders again
dirty.add(route);
}
}
}
// a shared file counts once, however many pages name it
let bytes = Object.values(pages).reduce((sum, page) => sum + page.bytes, 0);
for (const size of sharedBytes.values()) bytes += size;
const minViewBytes = new Map();
for (const [route, page] of Object.entries(pages)) {
const parsed = routeOf(prefix, route, repo);
if (!parsed) continue;
const prev = minViewBytes.get(parsed.view);
if (prev === undefined || page.bytes < prev) minViewBytes.set(parsed.view, page.bytes);
}
const overBudget = new Set();
// what the budget left out last time stays out until a full render
const over = new Set(prior?.over ?? []);
const seen = new Set([...Object.keys(pages), ...Object.keys(left), ...over]);
let todo = prior ? [...dirty] : seedsOf(prefix);
const rendered = new Set();
for (const route of todo) seen.add(route);
const rankOfRoute = (route) => {
const parsed = routeOf(prefix, route, repo);
if (!parsed) return 0;
return parsed.page > 1 ? rankOf("page", parsed.page) : rankOf(parsed.view, parsed.depth);
};
const commitSources = [prefix, `${prefix}/commits/${repo.default_branch}`, `${prefix}/branches`, `${prefix}/tags`];
const commitSourceSet = new Set(commitSources);
const commitsRoute = `${prefix}/commits/${repo.default_branch}`;
const currentAllowedCommits = () => {
const order = commitSources
.flatMap((route) => pages[route]?.links ?? [])
.filter((route) => route.startsWith(`${prefix}/commit/`));
return new Set([...new Set(order)].slice(0, budget.commits));
};
// a list's later pages wait until nothing else is left, so they only take the room the
// rest of the repo didn't
let later = [];
while (todo.length > 0 || later.length > 0) {
checkCurrent();
if (todo.length === 0) [todo, later] = [later, []];
const batch = todo.sort((a, b) => rankOfRoute(a) - rankOfRoute(b) || (a < b ? -1 : a > b ? 1 : 0));
todo = [];
const batchMinViewBytes = new Map(minViewBytes);
const minBytesFor = (route) => {
const parsed = routeOf(prefix, route, repo);
return ((parsed && batchMinViewBytes.get(parsed.view)) ?? 12_000) * 0.98;
};
// the budget keeps the best ranked pages whatever order their renders finish in:
// render the batch, then take its pages in rank order
let room = budget.pages - Object.keys(pages).length;
const hasKnownInBatch = batch.some((r) => r in pages);
const wanted = batch.filter((route) => {
if (route in pages) return true;
if (room <= 0 || (!hasKnownInBatch && budget.bytes - bytes < minBytesFor(route))) {
overBudget.add(route);
return false;
}
room--;
return true;
});
const results = new Map();
let at = 0;
let remainingKnown = wanted.filter((r) => r in pages).length;
// a page that outranks kept ones takes their room, worst first: the budget keeps the best
// ranked pages, also when a re-render of a kept page comes out bigger
const makeRoom = (route, need, ready) => {
const rank = rankOfRoute(route);
const pending = new Set(wanted.slice(at));
const worse = Object.keys(pages)
.filter((other) => !pending.has(other) && rankOfRoute(other) > rank)
.sort((a, b) => rankOfRoute(b) - rankOfRoute(a) || (a < b ? 1 : -1));
const freed = worse.reduce((sum, other) => sum + pages[other].bytes, 0);
if (bytes + need - freed > budget.bytes) return;
for (const other of worse) {
if (bytes + need <= budget.bytes) break;
bytes -= pages[other].bytes;
delete pages[other];
rendered.delete(other);
overBudget.add(other);
stats.dropped++;
ready.push({ route: other, gz: null });
}
};
const retire = () => {
const ready = [];
while (at < wanted.length && results.has(wanted[at])) {
const route = wanted[at++];
const res = results.get(route);
results.delete(route);
if (res.skippedOver) {
overBudget.add(route);
continue;
}
const known = route in pages;
const { deps, gz, links } = res;
const fresh = gz ? res.shared.filter((hash) => !sharedBytes.has(hash)) : [];
const sharedGz = fresh.map((hash) => [hash, gzipSync(shared.get(hash), { level: 6, memLevel: 9 })]);
const extra = sharedGz.reduce((sum, [, file]) => sum + file.length, 0);
if (known) {
bytes -= pages[route].bytes;
remainingKnown--;
}
if (gz) {
const parsed = routeOf(prefix, route, repo);
if (parsed) {
const prev = minViewBytes.get(parsed.view);
if (prev === undefined || gz.length < prev) minViewBytes.set(parsed.view, gz.length);
}
}
if (gz && bytes + gz.length + extra > budget.bytes) makeRoom(route, gz.length + extra, ready);
if (!gz || bytes + gz.length + extra > budget.bytes) {
if (known) {
delete pages[route];
stats.dropped++;
}
if (gz) overBudget.add(route);
else left[route] = deps;
ready.push({ route, gz: null });
continue;
}
bytes += gz.length + extra;
for (const [hash, file] of sharedGz) sharedBytes.set(hash, file.length);
Object.assign(latest, res.latest);
delete left[route];
const kept = links.filter((next) => keep(next));
pages[route] = { deps, bytes: gz.length, links: kept, ...(res.shared.length > 0 && { shared: res.shared }) };
rendered.add(route);
stats.rendered++;
ready.push({ route, gz, sharedGz });
for (const next of kept) {
if (next.startsWith(`${prefix}/commit/`)) continue;
if (!seen.has(next)) {
seen.add(next);
(PAGE_QUERY.test(next) ? later : todo).push(next);
}
}
}
return ready;
};
await pool(wanted, async (route) => {
if (!(route in pages) && remainingKnown === 0 && budget.bytes - bytes < minBytesFor(route)) {
results.set(route, { skippedOver: true });
} else {
let page = await renderPage(route);
stats.render_ms += page.ms;
const deps = [...new Set(page.reads.map((r) => r.key))].sort().map(intern);
if (!accepted(page, repo.did)) {
page = null;
results.set(route, { deps });
} else {
// commit pages link their parents and trees at their own commit: only the log
// and the front page decide which commits get pages
const rawLinks = route.startsWith(`${prefix}/commit/`)
? []
: [...linksOf(page.html, origin, (pathname) => routeOf(prefix, pathname, repo))];
const links = (
commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`))
).map(intern);
for (const inlined of inlinedOverCap(page.html)) {
stats.inlined_over_cap++;
if (!inlinedMax || inlined.bytes > inlinedMax.bytes) inlinedMax = { ...inlined, route };
}
const canonical = canonicalHtml(page.html);
const { shared, latest } = page;
page = null;
const gz = gzipSync(canonical, { level: 6, memLevel: 9 });
results.set(route, { deps, gz, links, shared, latest });
}
}
for (const item of retire()) {
const file = fileFor(tmpDir, prefix, item.route);
await rm(file, { force: true });
await rm(file + ".gz", { force: true });
if (!item.gz) continue;
await ensureDir(path.dirname(file));
await link(empty, file);
await writeFile(file + ".gz", item.gz);
item.gz = null;
for (const [hash, gz] of item.sharedGz) {
const sharedFile = path.join(sharedDir, hash + ".json");
await ensureDir(sharedDir);
await rm(sharedFile, { force: true });
await rm(sharedFile + ".gz", { force: true });
await link(empty, sharedFile);
await writeFile(sharedFile + ".gz", gz);
}
}
});
if (commitsRoute in pages || commitsRoute in left || !seen.has(commitsRoute)) {
for (const next of currentAllowedCommits()) {
if (!seen.has(next)) {
seen.add(next);
todo.push(next);
}
}
} else if (prefix in pages) {
for (const next of pages[prefix].links.filter((r) => r.startsWith(`${prefix}/commit/`)).slice(0, budget.commits)) {
if (!seen.has(next)) {
seen.add(next);
todo.push(next);
}
}
}
}
// a link the budget rules out is never rendered, so its url stays the SPA's
function keep(next) {
const parsed = routeOf(prefix, next, repo);
if (over.has(next)) return false;
if (parsed.view === "tree" && files.has(sizeKey(parsed.ref, parsed.file))) return false;
if (parsed.view === "blob" && (sizes.get(sizeKey(parsed.ref, parsed.file)) ?? 0) > budget.blobBytes) {
overBudget.add(next);
return false;
}
return true;
}
// commit pages: the newest the front page and the log link, then branch and tag tips.
// the rest go, and come back if a page links them again
const commits = currentAllowedCommits();
const inBudget = (route) => !route.startsWith(`${prefix}/commit/`) || commits.has(route);
const live = reachable(prefix, pages, inBudget);
for (const route of Object.keys(pages)) {
if (live.has(route)) continue;
bytes -= pages[route].bytes;
delete pages[route];
await rm(fileFor(tmpDir, prefix, route), { force: true });
await rm(fileFor(tmpDir, prefix, route) + ".gz", { force: true });
stats.dropped++;
}
// a name keeps its copy from the last run when no page shared it this time
for (const [name, hash] of Object.entries(latest)) {
if (sharedBytes.has(hash)) continue;
const file = path.join(sharedDir, hash + ".json");
try {
mkdirSync(sharedDir, { recursive: true });
linkSync(path.join(priorDir, SHARED_DIR, hash + ".json.gz"), file + ".gz");
linkSync(empty, file);
sharedBytes.set(hash, statSync(file + ".gz").size);
bytes += sharedBytes.get(hash);
} catch {
delete latest[name];
}
}
// a shared file goes with the last page or name that named it
const named = sharedNamed(pages, latest);
for (const [hash, size] of sharedBytes) {
if (named.has(hash)) continue;
bytes -= size;
await rm(path.join(sharedDir, hash + ".json"), { force: true });
await rm(path.join(sharedDir, hash + ".json.gz"), { force: true });
}
for (const [name, hash] of Object.entries(latest)) {
const file = path.join(tmpDir, latestPath(name));
await ensureDir(path.dirname(file));
await link(empty, file);
await link(path.join(sharedDir, hash + ".json.gz"), file + ".gz");
}
const writeLatest = async (name, value) => {
const file = path.join(tmpDir, latestPath(name));
const gz = gzipSync(JSON.stringify(value));
await ensureDir(path.dirname(file));
await link(empty, file);
await writeFile(file + ".gz", gz);
bytes += gz.length;
};
await writeLatest(ROUTES, { routes: Object.keys(pages).map((route) => route.slice(prefix.length)).sort() });
// read apart from the pages, so a ticket write rewrites this file and no page
const counts = await readCounts(repo.did);
if (counts.value) await writeLatest(COUNTS, counts.value);
stats.kept = Object.keys(pages).filter((route) => !rendered.has(route)).length;
for (const route of overBudget) over.add(route);
stats.over_budget = over.size;
checkCurrent();
await publish(repo, savedAliases, genDir, tmpDir);
// a page's links are places in one list of routes: a folder's pages all link the same
// siblings, and written out in full they made a big folder's state tens of MB
const routes = [];
const placeOf = new Map();
const placed = (route) => {
if (!placeOf.has(route)) placeOf.set(route, routes.push(route) - 1);
return placeOf.get(route);
};
const saving = {};
for (const [route, page] of Object.entries(pages)) saving[route] = { ...page, links: page.links?.map(placed) };
await atomicJson(stateFile, {
did: repo.did,
prefix,
aliases: aliasesOf(repo),
format: RENDERER_FORMAT,
generation: hook.generation,
pages: saving,
routes,
left,
over: [...over].sort(),
latest,
counts: counts.keys
});
// the state names the new generation now, so the old one can go
for (const entry of await readdir(storeOf(repo.did))) {
if (entry !== hook.generation) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true });
}
return {
pages: Object.keys(pages).length,
shared: named.size,
bytes,
reads: reads.count,
read_ms: Math.round(reads.ms),
...stats,
render_ms: Math.round(stats.render_ms),
...(inlinedMax && { inlined_max: inlinedMax }),
heap_peak_mb: Math.round(heapPeak / 1_048_576),
rss_mb: Math.round(process.memoryUsage().rss / 1_048_576),
// the worker's peak since it started, so the repo it jumps on is the one that spiked
max_rss_mb: Math.round(process.resourceUsage().maxRSS / 1024)
};
};
// a user's page: their profile and the repos this knot hosts for them, at the url of the
// handle. one svfe can't render (no such account) goes, and the SPA answers the url
const renderOwner = async (owner) => {
reads.count = 0;
reads.ms = 0;
heapPeak = 0;
readCache.clear();
readCacheChars = 0;
const page = await renderPage(owner.prefix);
const deps = [...new Set(page.reads.map((r) => r.key))].sort();
const stats = { reads: reads.count, read_ms: Math.round(reads.ms), render_ms: Math.round(page.ms), deps };
checkCurrent();
if (page.status !== 200) {
await unpublishOwner(out, owner.prefix);
return { pages: 0, rendered: 0, bytes: 0, status: page.status, ...stats };
}
const [file, routesFile] = ownerFiles(out, owner.prefix);
const gz = gzipSync(canonicalHtml(page.html), { level: 6, memLevel: 9 });
const routesGz = gzipSync(JSON.stringify({ routes: [""] }));
await publishFile(file, gz);
await publishFile(routesFile, routesGz);
return { pages: 1, rendered: 1, bytes: gz.length + routesGz.length, ...stats };
};
// the page by rename, then its stub, so nginx sees the old page or the new one
const publishFile = async (file, gz) => {
await mkdir(path.dirname(file), { recursive: true });
const tmp = `${file}.gz.${process.pid}.tmp`;
await writeFile(tmp, gz);
await rename(tmp, file + ".gz");
await writeFile(file, "", { flag: "a" });
};
// urls svfe serves the repo under: the handle form it links to, and the owner did form,
// which hydrates and then redirects to the handle like the SPA does
const aliasesOf = (repo) => [...new Set([prefixOf(repo).slice(1), `${repo.owner_did}/${repo.name}`])];
const swapLink = async (target, at) => {
await mkdir(path.dirname(at), { recursive: true });
const tmp = `${at}.${process.pid}.tmp`;
await rm(tmp, { force: true });
await symlink(path.relative(path.dirname(at), target), tmp);
await rename(tmp, at);
};
// the store dir by rename, then the site link by rename, so nginx sees the old or the new
// pages of this repo and never half of each
const publish = async (repo, savedAliases, genDir, tmpDir) => {
await rm(genDir, { recursive: true, force: true });
await rename(tmpDir, genDir);
const site = path.join(out, "site", repo.did);
await swapLink(genDir, site);
const aliases = aliasesOf(repo);
for (const alias of aliases) await swapLink(site, path.join(out, "www", alias));
for (const alias of savedAliases) {
if (!aliases.includes(alias)) await rm(path.join(out, "www", alias), { force: true });
}
};
process.on("message", async ({ repo, owner, work, budget }) => {
try {
const result = owner ? await renderOwner(owner) : await renderRepo(repo, work, budget);
process.send(result);
} catch (error) {
const message = error instanceof Superseded ? "superseded" : String(error?.stack ?? error);
process.send({ error: message }, () => process.exit(error instanceof Superseded ? 3 : 1));
}
});
process.on("disconnect", () => process.exit(0));
process.send({ ready: true });