keep each ref's newest sidebar under a stable name for pages left to the spa
Signed-off-by: dawn <90008@klbr.net>
14 unmodified lines151617181920212214 unmodified linesagainst a fresh one of the same generation with
node verify.mjs <incremental out> <fresh out>
the newest sidebar of each ref is also at `<repo>/_latest/sidebar/<ref>.json`, for a pagethe SPA renders without a prerendered copy to name.
node --test22 unmodified lines2324252627282930313233343536373839406 unmodified lines22 unmodified lines default_type application/json; expires max; try_files $uri =404;}# the newest copy under a stable name, for pages the renderer left to the SPA: revalidated# on every use, so a new render shows up at oncelocation ~ ^/[^/]+/[^/]+/_latest/ { root <out>/www; gzip_static always; gunzip on; default_type application/json; add_header Cache-Control "no-cache"; try_files $uri =404;}location @spa { root <build>; gzip_static on; try_files $uri /index.html;6 unmodified lines4 unmodified lines56789910111213138 unmodified linesExpand all1521531541551561571581591601611621631641651661678 unmodified lines4 unmodified linesimport { test } from "node:test";import { fileOf, keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { maskShared, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";import { latestPath, maskShared, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
test("keyOf sorts params, drops empty ones and matches a request url", () => { assert.equal(keyOf("org.tangled.git.getTree", { ref: "main", repo: "did:plc:a", path: "" }), "org.tangled.git.getTree?ref=main&repo=did:plc:a"); const url = new URL("https://k.test/xrpc/org.tangled.git.getBlob?repo=did%3Aplc%3Aa&ref=main&path=src%2Fa%20b.rs");138 unmodified linesExpand all
test("sharedNamed collects what the kept pages name", () => { const pages = { "/o/r": { shared: ["a", "b"] }, "/o/r/tags": { shared: ["b"] }, "/o/r/pulls": {} }; assert.deepEqual([...sharedNamed(pages)].sort(), ["a", "b"]); assert.deepEqual([...sharedNamed(pages, { "sidebar/main": "c" })].sort(), ["a", "b", "c"]);});
test("a stable name is a file under the repo, with a ref's slashes as folders", () => { assert.equal(latestPath("sidebar/main"), "_latest/sidebar/main.json"); assert.equal(latestPath("sidebar/feat/x"), "_latest/sidebar/feat/x.json");});
test("a later page of a list is stored next to page 1, under a name no ref can have", () => { assert.equal(fileOf("/o/r", "/o/r"), "index.html"); assert.equal(fileOf("/o/r", "/o/r/commits/main"), "commits/main.html"); assert.equal(fileOf("/o/r", "/o/r/commits/main?page=3"), "commits/main~page-3.html");8 unmodified lines9 unmodified lines10111213141516171819202122232425262127289 unmodified lines};
export const sharedPath = (hash) => `${SHARED_DIR}/${hash}.json`;
// the newest copy of a shared value under a name that stays put (the sidebar of a ref), for// a page the renderer didn't keep, which has no copy to nameexport const LATEST_DIR = "_latest";
export const latestPath = (name) => `${LATEST_DIR}/${name}.json`;
const NAMED = /_shared\/[0-9a-f]{16}\.json/g;
// a page kept from an older render names an older copy of the same thing, so an incremental// run matches a fresh one only with the names maskedexport const maskShared = (html) => html.replace(NAMED, "_shared/-.json");
// the files a repo's pages name, which gc keepsexport const sharedNamed = (pages) => new Set(Object.values(pages).flatMap((page) => page.shared ?? []));export const sharedNamed = (pages, latest = {}) => new Set([...Object.values(pages).flatMap((page) => page.shared ?? []), ...Object.values(latest)]);5 unmodified lines678910101112131415161718181920212223242425262728293031323334358 unmodified lines5 unmodified linesimport { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs";import path from "node:path";import { gunzipSync } from "node:zlib";import { fileOf, readJson } from "./common.mjs";import { maskShared, SHARED_DIR, sharedNamed } from "./shared.mjs";import { latestPath, maskShared, SHARED_DIR, sharedNamed } from "./shared.mjs";
const [incremental, fresh] = process.argv.slice(2);const html = (out, state, route) => { const dir = realpathSync(path.join(out, "site", state.did)); return maskShared(gunzipSync(readFileSync(path.join(dir, fileOf(state.prefix, route) + ".gz"))).toString());};
const result = { compared: 0, differ: [], only_incremental: 0, only_fresh: 0, missing_shared: [] };const result = { compared: 0, differ: [], only_incremental: 0, only_fresh: 0, missing_shared: [], latest_differ: [] };for (const name of readdirSync(path.join(incremental, "state"))) { const a = readJson(path.join(incremental, "state", name)); const b = readJson(path.join(fresh, "state", name)); if (!a?.pages || !b?.pages) continue; const dir = realpathSync(path.join(incremental, "site", a.did)); for (const hash of sharedNamed(a.pages)) { for (const hash of sharedNamed(a.pages, a.latest)) { if (!existsSync(path.join(dir, SHARED_DIR, hash + ".json.gz"))) result.missing_shared.push(`${a.prefix} ${hash}`); } for (const name of Object.keys(a.latest ?? {})) { if (!existsSync(path.join(dir, latestPath(name) + ".gz"))) result.missing_shared.push(`${a.prefix} ${name}`); // a name the fresh render points elsewhere is the newest copy going stale, not a broken page if (b.latest?.[name] !== a.latest[name]) result.latest_differ.push(`${a.prefix} ${name}`); } for (const route of Object.keys(a.pages)) { if (!(route in b.pages)) { result.only_incremental++; continue;8 unmodified lines8 unmodified lines910111213131415161795 unmodified lines113114115116117117118119120121122123124125126127128129129130131132133134135135136137138139140141142143144145146147105 unmodified linesExpand all25325425525625725825926026126226326426510 unmodified lines276277278279267280281282283117 unmodified linesExpand all40140240340440540640740840928 unmodified lines43843944044142944244344443244544644744844956 unmodified lines5065075085095105115125135145155165175185195205215225234974985245255265275285295305315325335345355365375385395405416 unmodified lines54854955055151955255355455555655750 unmodified lines8 unmodified linesimport { gzipSync } from "node:zlib";import { rankOf } from "./budget.mjs";import { atomicJson, fileOf, keyOfUrl, PAGE_QUERY, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";import { latestPath, SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: {95 unmodified lines};
// this run's shared files of the repo, by hashconst shared = new Map();const share = (value) => {const share = (value, name) => { const { hash, body } = sharedOf(value); if (!shared.has(hash)) shared.set(hash, body); deps.getStore()?.shared.add(hash); if (name && fitsFile(name) && name.split("/").every((part) => part && !part.startsWith("."))) deps.getStore()?.latest.set(name, hash); return sharedPath(hash);};
globalThis.__knotStaticPrerender = { answer, read, share, renderWorkers: Number(opts.highlighters) };const { server } = await import(pathToFileURL(path.resolve(opts.server, "instance.js")).href);await server.init({ env: {}, read: () => { throw new Error("no server assets"); } });
const renderPage = async (route) => { const collected = { reads: [], shared: new Set() }; const collected = { reads: [], shared: new Set(), latest: new Map() }; const t = performance.now(); const res = await deps.run(collected, () => server.respond(new Request(origin + encodeURI(route)), { getClientAddress: () => "127.0.0.1" }) ); const html = await res.text(); return { status: res.status, html, reads: collected.reads, shared: [...collected.shared].sort(), ms: performance.now() - t }; return { status: res.status, html, reads: collected.reads, shared: [...collected.shared].sort(), latest: Object.fromEntries(collected.latest), ms: performance.now() - t };};
// a page is static only if every git read of its repo was answered: a ref, path or diff the// knot didn't project leaves it to the client. a fork hosted on another knot gets the same105 unmodified linesExpand all await writeFile(empty, ""); // the shared files in tmpDir, by hash, with their gzipped size const sharedDir = path.join(tmpDir, SHARED_DIR); const sharedBytes = new Map(); // the hash each stable name points at, kept from the last run unless a page shares it anew const latest = { ...(prior?.latest ?? {}) }; const priorDir = saved ? await realpath(path.join(out, "site", repo.did)).catch(() => path.join(storeOf(repo.did), saved.generation)) : null;
const madeDirs = new Set([tmpDir]); const dirPromises = new Map(); const ensureDir = (dir) => {10 unmodified lines };
const stats = { rendered: 0, kept: 0, dropped: 0, over_budget: 0, render_ms: 0 }; if (prior) { const priorDir = await realpath(path.join(out, "site", repo.did)).catch(() => path.join(storeOf(repo.did), prior.generation)); for (const route of Object.keys(pages)) { if (dirty.has(route)) continue; const to = fileFor(tmpDir, prefix, route); const dir = path.dirname(to);117 unmodified linesExpand all continue; } bytes += gz.length + extra; for (const [hash, file] of sharedGz) sharedBytes.set(hash, file.length); Object.assign(latest, res.latest); delete left[route]; const kept = links.filter((next) => keep(next)); pages[route] = { deps, bytes: gz.length, links: kept, ...(res.shared.length > 0 && { shared: res.shared }) }; rendered.add(route);28 unmodified lines const links = commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`)); const canonical = canonicalHtml(page.html); const { shared } = page; const { shared, latest } = page; page = null; const gz = gzipSync(canonical, { level: 6, memLevel: 9 }); results.set(route, { deps, gz, links, shared }); results.set(route, { deps, gz, links, shared, latest }); } } for (const item of retire()) { const file = fileFor(tmpDir, prefix, item.route);56 unmodified lines await rm(fileFor(tmpDir, prefix, route), { force: true }); await rm(fileFor(tmpDir, prefix, route) + ".gz", { force: true }); stats.dropped++; } // a name keeps its copy from the last run when no page shared it this time for (const [name, hash] of Object.entries(latest)) { if (sharedBytes.has(hash)) continue; const file = path.join(sharedDir, hash + ".json"); try { mkdirSync(sharedDir, { recursive: true }); linkSync(path.join(priorDir, SHARED_DIR, hash + ".json.gz"), file + ".gz"); linkSync(empty, file); sharedBytes.set(hash, statSync(file + ".gz").size); bytes += sharedBytes.get(hash); } catch { delete latest[name]; } } // a shared file goes with the last page that named it const named = sharedNamed(pages); // a shared file goes with the last page or name that named it const named = sharedNamed(pages, latest); for (const [hash, size] of sharedBytes) { if (named.has(hash)) continue; bytes -= size; await rm(path.join(sharedDir, hash + ".json"), { force: true }); await rm(path.join(sharedDir, hash + ".json.gz"), { force: true }); } for (const [name, hash] of Object.entries(latest)) { const file = path.join(tmpDir, latestPath(name)); await ensureDir(path.dirname(file)); await link(empty, file); await link(path.join(sharedDir, hash + ".json.gz"), file + ".gz"); } stats.kept = Object.keys(pages).filter((route) => !rendered.has(route)).length; for (const route of overBudget) over.add(route); stats.over_budget = over.size; checkCurrent();6 unmodified lines format: RENDERER_FORMAT, generation: hook.generation, pages, left, over: [...over].sort() over: [...over].sort(), latest }); // the state names the new generation now, so the old one can go for (const entry of await readdir(storeOf(repo.did))) { if (entry !== hook.generation) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true });50 unmodified lines