keep the client chunks of every build a served page was rendered by, so a deploy never breaks a page before its re-render
Signed-off-by: dawn <90008@klbr.net>
2 unmodified lines3456778910111213141516171819202114 unmodified lines2 unmodified linesrenders svfe's repo pages for a knot-static generation, so they work without js and hydrateinto the svfe SPA. it runs as knot-static's `[site_hook]`, reads only the knot's public xrpcsurface, and re-renders just the pages the generation's xrpc diff touches.
svfe-pages <hook.json> --server <svfe build-server> --out <dir> [--read <xrpc base>] svfe-pages <hook.json> --server <svfe build-server> [--client <svfe build>] --out <dir> [--read <xrpc base>]
see `nginx-site.conf` for serving `<out>` in front of the SPA, and `budget.mjs` for what arepo may cost on disk.
a page loads the client chunks of the svfe build that rendered it, so `--client` (by defaultthe `build` next to `--server`) has its `_app/immutable` copied into `<out>/www` before anypage renders, and a build's files are removed once no served page was rendered by it. adeploy then never leaves a served page without its chunks, while the re-render runs.
what every page of a repo shows the same way, the sidebar, is stored once per content in`<repo>/_shared/<hash>.json` and isn't a page's own read, so a change to it re-rendersnothing. a page kept from an older render names an older copy, so check an incremental runagainst a fresh one of the same generation with14 unmodified lines1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556// a page loads the client chunks of the build that rendered it, under content-hashed names,// and a deploy replaces the build's own copy before the pages render again. so every build's// chunks are kept under www/_app/immutable, listed per build in state/builds/<build>.json,// until no page rendered by that build is servedimport { constants } from "node:fs";import { copyFile, mkdir, readdir, rename, rm, stat } from "node:fs/promises";import path from "node:path";import { atomicJson, readJson } from "./common.mjs";
const IMMUTABLE = "_app/immutable";
const filesUnder = async (dir, rel = "") => { const files = []; for (const entry of await readdir(path.join(dir, rel), { withFileTypes: true })) { const name = path.join(rel, entry.name); if (entry.isDirectory()) files.push(...(await filesUnder(dir, name))); else files.push(name); } return files;};
const buildsDir = (out) => path.join(out, "state", "builds");
// copies the files of a build its pages load that aren't kept yet, before any page rendersexport const keepClient = async (client, out, build) => { const from = path.join(client, IMMUTABLE); const to = path.join(out, "www", IMMUTABLE); const files = (await filesUnder(from)).sort(); let copied = 0; for (const file of files) { const target = path.join(to, file); if (await stat(target).then(() => true, () => false)) continue; await mkdir(path.dirname(target), { recursive: true }); await copyFile(path.join(from, file), target + ".tmp", constants.COPYFILE_FICLONE); await rename(target + ".tmp", target); copied++; } await atomicJson(path.join(buildsDir(out), build + ".json"), files); return copied;};
// removes the files of every build not in `served` that no served build sharesexport const collectClients = async (out, served) => { const builds = (await readdir(buildsDir(out)).catch(() => [])).filter((name) => name.endsWith(".json")).map((name) => name.slice(0, -5)); const kept = new Set(builds.filter((build) => served.has(build)).flatMap((build) => readJson(path.join(buildsDir(out), build + ".json")) ?? [])); let removed = 0; for (const build of builds.filter((build) => !served.has(build))) { for (const file of readJson(path.join(buildsDir(out), build + ".json")) ?? []) { if (kept.has(file)) continue; await rm(path.join(out, "www", IMMUTABLE, file), { force: true }); removed++; } await rm(path.join(buildsDir(out), build + ".json"), { force: true }); } return removed;};40 unmodified lines414243444546474849505152535455565758596061626340 unmodified lines gzip_static on; add_header Cache-Control "no-cache"; try_files $uri /index.html;}# version.json, which a page polls to learn of a new buildlocation /_app/ { root <build>; gzip_static on; add_header Cache-Control "no-cache";}# a page loads the chunks of the build that rendered it: the renderer keeps those of every# build a served page was rendered by, the newest build has its ownlocation ^~ /_app/immutable/ { root <out>/www; gzip_static on; expires max; try_files $uri @immutable;}location @immutable { root <build>; gzip_static on; expires max;}3 unmodified lines4567889101112131415161718192021222324252627282930313244 unmodified lines7778798081828384858611 unmodified lines989910010110210310410510698 unmodified lines2052062072082012092102112122132 unmodified lines2162172182192122202212222232242252262272282292303 unmodified lines23423523623723823924024124224324424524624724824925097 unmodified lines3 unmodified lines// site/<repo did> -> store/<token>/<generation>, the pages of one repo// www/<owner>/<name> -> site/<repo did>, the urls svfe links to (owner = handle or did)// state/<token>.json what each page read, so a diff of the knot's reads names the// pages to render again// www/_app/immutable the client chunks of every build a served page was rendered by// svfe-pages <hook.json> --server <svfe build-server> --out <dir> [--read <xrpc base>]// svfe-pages <hook.json> --server <svfe build-server> [--client <svfe build>] --out <dir>// [--read <xrpc base>]import { fork } from "node:child_process";import { createHash } from "node:crypto";import { readFileSync } from "node:fs";import { rm } from "node:fs/promises";import { availableParallelism } from "node:os";import path from "node:path";import { parseArgs } from "node:util";import { budgetFrom, BUDGET_OPTIONS } from "./budget.mjs";import { collectClients, keepClient } from "./client.mjs";import { atomicJson, ownerPrefixOf, prefixOf, readJson, RENDERER_FORMAT, tokenOf, unpublishOwner } from "./common.mjs";import { countsChanged, dirtyPages, readDiff, stale } from "./diff.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: { server: { type: "string" }, // the client build next to the server one, as svfe's package lays them out client: { type: "string" }, out: { type: "string" }, read: { type: "string" }, // pages in flight per repo process jobs: { type: "string", default: "6" },44 unmodified lines // work of a run that is superseded or killed counts for the next. an entry carries the // formats and build it was rendered with: one rendered by another build renders again const stamp = `${hook.format}/${RENDERER_FORMAT}/${build}`; const current = (entry) => (!opts.full && entry?.stamp === stamp ? entry : undefined); // the build whose pages an entry's url serves: a failed render leaves the pages before it const servedBy = (entry) => entry?.served ?? (entry?.failed ? undefined : entry?.stamp?.split("/").at(-1)); const nextRepos = {}; const nextOwners = {}; let knotWrite = Promise.resolve(); const saveKnot = () => {11 unmodified lines process.exit(3); });
await writeStatus(); const copied = await keepClient(opts.client ?? path.join(opts.server, "..", "build"), out, build); for (const did of hook.removed_repos ?? []) await removeRepo(out, did, loadState(did));
// a page is current if every read it made is unchanged since the generation it was // rendered at, so each repo walks the knot's diffs back to its own generation98 unmodified lines if (plan.owner) { const { did, prefix } = plan.owner; if (result.error) { failures.push(did); nextOwners[did] = { stamp, failed: true, prefix }; nextOwners[did] = { stamp, failed: true, prefix, served: servedBy(nextOwners[did]) }; status.owners[did] = { error: result.error }; } else { const { deps, ...stats } = result; nextOwners[did] = { stamp, generation: hook.generation, prefix, deps };2 unmodified lines } } else if (result.error) { failures.push(plan.repo.did); // its site link still names the pages of its last finished render nextRepos[plan.repo.did] = { stamp, failed: true, prefix: prefixOf(plan.repo), pages: nextRepos[plan.repo.did]?.pages }; nextRepos[plan.repo.did] = { stamp, failed: true, prefix: prefixOf(plan.repo), pages: nextRepos[plan.repo.did]?.pages, served: servedBy(nextRepos[plan.repo.did]) }; status.repos[plan.repo.did] = { error: result.error }; } else { nextRepos[plan.repo.did] = { stamp, generation: hook.generation, prefix: prefixOf(plan.repo), pages: result.pages }; status.repos[plan.repo.did] = { ...result, wall_ms: wallMs };3 unmodified lines }) ) );
await knotWrite; const served = new Set([build, ...[...Object.values(nextRepos), ...Object.values(nextOwners)].map(servedBy)]); const collected = await collectClients(out, served); const summary = { state: failures.length === 0 ? "done" : "partial", failed: failures, finished_at: new Date().toISOString(), diff_entries: diffEntries, client: { copied, collected }, totals, total_ms: Math.round(performance.now() - t0) }; await writeStatus(summary);97 unmodified lines122345678910202 unmodified linesExpand all213214215216217218219220221222223224225226227228229230231232233234import assert from "node:assert/strict";import { mkdtemp, mkdir, writeFile } from "node:fs/promises";import { access, mkdtemp, mkdir, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { test } from "node:test";import { collectClients, keepClient } from "./client.mjs";import { fileOf, keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { countsChanged, dirtyPages, readDiff } from "./diff.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { latestPath, maskShared, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";202 unmodified linesExpand all const paged = (route) => (/[?]page=/.test(route) ? (route.includes("/tags") ? route : null) : route); const html = '<a href="/o/r/tags?page=2">older</a><a href="/o/r/tags?page=1">newer</a><a href="/o/r/blob/main/a?view=raw">raw</a><a href="/o/r/issues?page=2">more</a>'; assert.deepEqual([...linksOf(html, "https://k.test", paged)].sort(), ["/o/r/blob/main/a", "/o/r/issues", "/o/r/tags", "/o/r/tags?page=2"]);});
test("a build's chunks stay until no served page was rendered by it", async () => { const dir = await mkdtemp(path.join(tmpdir(), "svfe-pages-client-")); const build = async (name, files) => { for (const file of files) { await mkdir(path.dirname(path.join(dir, name, "_app/immutable", file)), { recursive: true }); await writeFile(path.join(dir, name, "_app/immutable", file), name); } return path.join(dir, name); }; const out = path.join(dir, "out"); const kept = (file) => access(path.join(out, "www/_app/immutable", file)).then(() => true, () => false); assert.equal(await keepClient(await build("a", ["entry/app.A.js", "chunks/x.js"]), out, "a"), 2); assert.equal(await keepClient(await build("b", ["entry/app.B.js", "chunks/x.js"]), out, "b"), 1); assert.equal(await collectClients(out, new Set(["a", "b"])), 0); assert.equal(await collectClients(out, new Set(["b"])), 1); assert.deepEqual([await kept("entry/app.A.js"), await kept("entry/app.B.js"), await kept("chunks/x.js")], [false, true, true]);});