store the sidebar every page of a repo shows once per content, outside the pages' reads
Signed-off-by: dawn <90008@klbr.net>
7 unmodified lines89101112131415167 unmodified lines
see `nginx-site.conf` for serving `<out>` in front of the SPA, and `budget.mjs` for what arepo may cost on disk.
what every page of a repo shows the same way, the sidebar, is stored once per content in`<repo>/_shared/<hash>.json` and isn't a page's own read, so a change to it re-rendersnothing.
node --test2 unmodified lines34567789101134 unmodified lines2 unmodified linesimport { mkdir, rename, writeFile } from "node:fs/promises";import path from "node:path";
// a bump means no saved page can be trustedexport const RENDERER_FORMAT = 2;export const RENDERER_FORMAT = 3;
export const tokenOf = (did) => createHash("sha256").update(did).digest("hex").slice(0, 32);
// svfe's url for a repo: the owner's handle, or its did when it has none34 unmodified lines6 unmodified lines78910111213141516171819202122236 unmodified lines6 unmodified lines gunzip on; # a file or folder page is <path>.html, the repo's front page <repo>/index.html try_files $uri.html $uri/index.html @spa;}# what all of a repo's pages share, named by content, so it never changes under a namelocation ~ ^/[^/]+/[^/]+/_shared/ { root <out>/www; gzip_static always; gunzip on; default_type application/json; expires max; try_files $uri =404;}location @spa { root <build>; gzip_static on; try_files $uri /index.html;6 unmodified lines4 unmodified lines567891011121384 unmodified lines98991001011021031041051061071081091101111121131141151161171181191204 unmodified linesimport { test } from "node:test";import { keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { maskShared, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
test("keyOf sorts params, drops empty ones and matches a request url", () => { assert.equal(keyOf("org.tangled.git.getTree", { ref: "main", repo: "did:plc:a", path: "" }), "org.tangled.git.getTree?ref=main&repo=did:plc:a"); const url = new URL("https://k.test/xrpc/org.tangled.git.getBlob?repo=did%3Aplc%3Aa&ref=main&path=src%2Fa%20b.rs");84 unmodified lines }; assert.deepEqual([...dirtyPages(prior, diffs)].sort(), ["/o/r", "/o/r/issues"]); assert.equal(dirtyPages({ ...prior, did: "did:plc:z" }, diffs), "all");});
test("a shared value is named by its content", () => { const a = sharedOf({ ref: "main", files: [] }); assert.equal(a.hash, sharedOf({ ref: "main", files: [] }).hash); assert.notEqual(a.hash, sharedOf({ ref: "dev", files: [] }).hash); assert.match(sharedPath(a.hash), /^_shared\/[0-9a-f]{16}\.json$/); assert.equal(JSON.parse(a.body).ref, "main");});
test("maskShared hides which copy of a shared file a page names", () => { const page = (hash) => `<div class="sb" data-sidebar-src="/o/r/${sharedPath(hash)}">`; assert.equal(maskShared(page("0123456789abcdef")), maskShared(page("fedcba9876543210"))); assert.notEqual(maskShared(page("0123456789abcdef")), maskShared('<div class="sb">'));});
test("sharedNamed collects what the kept pages name", () => { const pages = { "/o/r": { shared: ["a", "b"] }, "/o/r/tags": { shared: ["b"] }, "/o/r/pulls": {} }; assert.deepEqual([...sharedNamed(pages)].sort(), ["a", "b"]);});123456789101112131415161718192021import { createHash } from "node:crypto";
// what every page of a repo shows the same way (its sidebar) is stored once per content// under the repo's url, and a page names the files it reads back as it hydratesexport const SHARED_DIR = "_shared";
export const sharedOf = (value) => { const body = JSON.stringify(value); return { hash: createHash("sha256").update(body).digest("hex").slice(0, 16), body };};
export const sharedPath = (hash) => `${SHARED_DIR}/${hash}.json`;
const NAMED = /_shared\/[0-9a-f]{16}\.json/g;
// a page kept from an older render names an older copy of the same thing, so an incremental// run matches a fresh one only with the names maskedexport const maskShared = (html) => html.replace(NAMED, "_shared/-.json");
// the files a repo's pages name, which gc keepsexport const sharedNamed = (pages) => new Set(Object.values(pages).flatMap((page) => page.shared ?? []));1234456789101112131415161750 unmodified lines68697071717273747576777879808123 unmodified lines10510610710810310911011111211311411511611711811912012112212310912412512612712811412913013113213313412013513613713813969 unmodified lines20921021121221321421521621712 unmodified lines23023123223323423523623723823924022 unmodified lines26326426526626726826927027127227327427527627727827928028128228328428528628728857 unmodified lines3463473483493503513523533543553563 unmodified lines3603613623633323643653663673681 unmodified line37037137237334237437537637734537837938034838138238338438522 unmodified lines4084094104114124134143814154164174184193 unmodified lines42342442542642742842943043143243343443543643743837 unmodified lines47647747847948048148248348448548648748848949049114 unmodified lines50650750850951051151251351442 unmodified lines// renders one repo's pages in its own process, so repos render in parallel across cores.// the coordinator sends {repo, work, budget} and gets the repo's stats backimport { AsyncLocalStorage } from "node:async_hooks";import { linkSync, mkdirSync, readFileSync } from "node:fs";import { linkSync, mkdirSync, readFileSync, statSync } from "node:fs";import { link, mkdir, readdir, realpath, rename, rm, symlink, writeFile } from "node:fs/promises";import path from "node:path";import { pathToFileURL } from "node:url";import { parseArgs } from "node:util";import { gzipSync } from "node:zlib";import { rankOf } from "./budget.mjs";import { atomicJson, keyOfUrl, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: {50 unmodified lines const contentType = res.headers.get("content-type"); return { status: res.status, contentType, body };};
const answer = async (request) => {// a read the page renders from: a change to it renders the page againconst answer = (request) => respond(request, true);// a read only shared files carry, which the page doesn't depend onconst read = (request) => respond(request, false);
const respond = async (request, own) => { const url = new URL(request.url); if (url.origin !== origin || !url.pathname.startsWith("/xrpc/")) { // nothing off the knot at render time: the browser fetches it live after hydrating return new Response(null, { status: 404 });23 unmodified lines }); readCache.set(cacheKey, pending); } const entry = await pending; deps.getStore()?.push({ key: keyOfUrl(url), status: entry.status }); if (own) deps.getStore()?.reads.push({ key: keyOfUrl(url), status: entry.status }); const headers = new Headers(); if (entry.contentType) headers.set("content-type", entry.contentType); return new Response(entry.body, { status: entry.status, headers });};
// this run's shared files of the repo, by hashconst shared = new Map();const share = (value) => { const { hash, body } = sharedOf(value); if (!shared.has(hash)) shared.set(hash, body); deps.getStore()?.shared.add(hash); return sharedPath(hash);};
globalThis.__knotStaticPrerender = { answer, renderWorkers: Number(opts.highlighters) };globalThis.__knotStaticPrerender = { answer, read, share, renderWorkers: Number(opts.highlighters) };const { server } = await import(pathToFileURL(path.resolve(opts.server, "instance.js")).href);await server.init({ env: {}, read: () => { throw new Error("no server assets"); } });
const renderPage = async (route) => { const collected = []; const collected = { reads: [], shared: new Set() }; const t = performance.now(); const res = await deps.run(collected, () => server.respond(new Request(origin + encodeURI(route)), { getClientAddress: () => "127.0.0.1" }) ); const html = await res.text(); return { status: res.status, html, reads: collected, ms: performance.now() - t }; return { status: res.status, html, reads: collected.reads, shared: [...collected.shared].sort(), ms: performance.now() - t };};
// a page is static only if every git read it made was answered: a ref, path or diff the// knot didn't project leaves it to the client69 unmodified lines sizes.clear(); files.clear(); readCache.clear(); readCacheChars = 0; shared.clear();
const prefix = prefixOf(repo); const stateFile = path.join(out, "state", tokenOf(repo.did) + ".json"); const saved = readJson(stateFile);12 unmodified lines // every page's stub is one empty inode: nginx's try_files needs the name, gzip_static // serves the .gz next to it const empty = path.join(tmpDir, ".empty"); await writeFile(empty, ""); // the shared files in tmpDir, by hash, with their gzipped size const sharedDir = path.join(tmpDir, SHARED_DIR); const sharedBytes = new Map();
const madeDirs = new Set([tmpDir]); const dirPromises = new Map(); const ensureDir = (dir) => {22 unmodified lines } try { linkSync(empty, to); linkSync(fileFor(priorDir, prefix, route) + ".gz", to + ".gz"); for (const hash of pages[route].shared ?? []) { if (sharedBytes.has(hash)) continue; const file = path.join(sharedDir, hash + ".json"); mkdirSync(sharedDir, { recursive: true }); linkSync(path.join(priorDir, SHARED_DIR, hash + ".json.gz"), file + ".gz"); linkSync(empty, file); sharedBytes.set(hash, statSync(file + ".gz").size); } } catch { // a page whose file went missing renders again dirty.add(route); } } }
// a shared file counts once, however many pages name it let bytes = Object.values(pages).reduce((sum, page) => sum + page.bytes, 0); for (const size of sharedBytes.values()) bytes += size; const minViewBytes = new Map(); for (const [route, page] of Object.entries(pages)) { const parsed = routeOf(prefix, route, repo); if (!parsed) continue;57 unmodified lines continue; } const known = route in pages; const { deps, gz, links } = res; const fresh = gz ? res.shared.filter((hash) => !sharedBytes.has(hash)) : []; const sharedGz = fresh.map((hash) => [hash, gzipSync(shared.get(hash), { level: 6, memLevel: 9 })]); const extra = sharedGz.reduce((sum, [, file]) => sum + file.length, 0); if (known) { bytes -= pages[route].bytes; remainingKnown--; }3 unmodified lines const prev = minViewBytes.get(parsed.view); if (prev === undefined || gz.length < prev) minViewBytes.set(parsed.view, gz.length); } } if (!gz || bytes + gz.length > budget.bytes) { if (!gz || bytes + gz.length + extra > budget.bytes) { if (known) { delete pages[route]; stats.dropped++; }1 unmodified line else left[route] = deps; ready.push({ route, gz: null }); continue; } bytes += gz.length; bytes += gz.length + extra; for (const [hash, file] of sharedGz) sharedBytes.set(hash, file.length); delete left[route]; const kept = links.filter((next) => keep(next)); pages[route] = { deps, bytes: gz.length, links: kept }; pages[route] = { deps, bytes: gz.length, links: kept, ...(res.shared.length > 0 && { shared: res.shared }) }; rendered.add(route); stats.rendered++; ready.push({ route, gz }); ready.push({ route, gz, sharedGz }); for (const next of kept) { if (next.startsWith(`${prefix}/commit/`)) continue; if (!seen.has(next)) { seen.add(next);22 unmodified lines const links = commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`)); const canonical = canonicalHtml(page.html); const { shared } = page; page = null; const gz = gzipSync(canonical, { level: 6, memLevel: 9 }); results.set(route, { deps, gz, links }); results.set(route, { deps, gz, links, shared }); } } for (const item of retire()) { const file = fileFor(tmpDir, prefix, item.route);3 unmodified lines await ensureDir(path.dirname(file)); await link(empty, file); await writeFile(file + ".gz", item.gz); item.gz = null; for (const [hash, gz] of item.sharedGz) { const sharedFile = path.join(sharedDir, hash + ".json"); await ensureDir(sharedDir); await rm(sharedFile, { force: true }); await rm(sharedFile + ".gz", { force: true }); await link(empty, sharedFile); await writeFile(sharedFile + ".gz", gz); } } }); if (commitsRoute in pages || commitsRoute in left || !seen.has(commitsRoute)) { for (const next of currentAllowedCommits()) {37 unmodified lines await rm(fileFor(tmpDir, prefix, route), { force: true }); await rm(fileFor(tmpDir, prefix, route) + ".gz", { force: true }); stats.dropped++; } // a shared file goes with the last page that named it const named = sharedNamed(pages); for (const [hash, size] of sharedBytes) { if (named.has(hash)) continue; bytes -= size; await rm(path.join(sharedDir, hash + ".json"), { force: true }); await rm(path.join(sharedDir, hash + ".json.gz"), { force: true }); } stats.kept = Object.keys(pages).filter((route) => !rendered.has(route)).length; for (const route of overBudget) over.add(route); stats.over_budget = over.size; checkCurrent();14 unmodified lines if (entry !== hook.generation) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true }); } return { pages: Object.keys(pages).length, shared: named.size, bytes, reads: reads.count, read_ms: Math.round(reads.ms), ...stats,42 unmodified lines