render each repo owner's user page, again when its profile, handle or repo list changes
Signed-off-by: dawn <90008@klbr.net>
20 unmodified lines21222324252627282920 unmodified lines`<repo>/_latest/routes.json`, so the SPA loads only those as documents. the repo's open ticketcounts, which every page's tabs show, are read apart from the pages into`<repo>/_latest/counts.json`, so a new ticket rewrites that file and no page.
each owner of a repo also gets a user page, `<owner>.html` beside the owner's repos, with`<owner>/_latest/routes.json` naming it, rendered again when a read it made changes (theprofile, the handle, the repo list) and moved when the handle changes.
node --test1233456789101111121213141516171819205 unmodified lines2627282930313233343536373839404142434445464728 unmodified linesimport { createHash } from "node:crypto";import { readFileSync } from "node:fs";import { mkdir, rename, writeFile } from "node:fs/promises";import { mkdir, rename, rm, writeFile } from "node:fs/promises";import path from "node:path";import { latestPath, ROUTES } from "./shared.mjs";
// a bump means no saved page can be trustedexport const RENDERER_FORMAT = 3;
export const tokenOf = (did) => createHash("sha256").update(did).digest("hex").slice(0, 32);
// svfe's url for a repo: the owner's handle, or its did when it has noneexport const prefixOf = (repo) => `/${repo.owner_handle?.toLowerCase() ?? repo.owner_did}/${repo.name}`;// svfe's url for a repo's owner: the handle, or the did when it has noneexport const ownerPrefixOf = (repo) => `/${repo.owner_handle?.toLowerCase() ?? repo.owner_did}`;
export const prefixOf = (repo) => `${ownerPrefixOf(repo)}/${repo.name}`;
// a later page of a list is a route of its own, "<list>?page=N", stored next to page 1 as// "<list>~page-N.html": no ref or path can hold a "~", so the name is never a real oneexport const PAGE_QUERY = /\?page=([1-9][0-9]*)$/;
5 unmodified lines const pathname = paged ? route.slice(0, paged.index) : route; return pathname.slice(prefix.length + 1) + (paged ? `~page-${paged[1]}` : "") + ".html";};
// a user's page, and the routes file that tells the SPA to load it as a documentexport const ownerFiles = (out, ownerPrefix) => { const at = path.join(out, "www", ownerPrefix.slice(1)); return [at + ".html", path.join(at, latestPath(ROUTES))];};
// the stub first, so nginx never serves a stub without its pageexport const unpublishOwner = async (out, ownerPrefix) => { for (const file of ownerFiles(out, ownerPrefix)) { await rm(file, { force: true }); await rm(file + ".gz", { force: true }); }};
let written = 0;// concurrent writers of one file each rename their own tmp, the last rename winsexport const atomicJson = async (file, value) => { await mkdir(path.dirname(file), { recursive: true });28 unmodified lines117 unmodified linesExpand all118119120121122123124125123126127128129130131132133134132135136137135138139140141117 unmodified linesExpand all } return false;};
// whether any of these reads changedexport const stale = (keys, diffs) => keys.some((key) => touched(diffs, key));
// whether the open ticket counts the repo publishes apart from its pages changedexport const countsChanged = (prior, diffs) => (prior.counts ?? []).some((key) => touched(diffs, key));export const countsChanged = (prior, diffs) => stale(prior.counts ?? [], diffs);
// the routes whose reads changed: pages to render again, and pages the budget or a failed// read left out, which may stand nowexport const dirtyPages = (prior, diffs) => { if (diffs.resets.has(prior.did)) return "all"; if (!diffs.global && diffs.repos && !diffs.repos.has(prior.did)) return new Set(); const dirty = new Set(); for (const [route, page] of Object.entries(prior.pages)) { if (page.deps.some((key) => touched(diffs, key))) dirty.add(route); if (stale(page.deps, diffs)) dirty.add(route); } for (const [route, deps] of Object.entries(prior.left ?? {})) { if (deps.some((key) => touched(diffs, key))) dirty.add(route); if (stale(deps, diffs)) dirty.add(route); } return dirty;};24 unmodified lines252627282930312930313233343514 unmodified lines24 unmodified lines default_type application/json; expires max; try_files $uri =404;}# the newest copy under a stable name, for pages the renderer left to the SPA: revalidated# on every use, so a new render shows up at oncelocation ~ ^/[^/]+/[^/]+/_latest/ {# the newest copy under a stable name, for pages the renderer left to the SPA, and a user's# routes file at <owner>/_latest/: revalidated on every use, so a new render shows up at oncelocation ~ ^/[^/]+(/[^/]+)?/_latest/ { root <out>/www; gzip_static always; gunzip on; default_type application/json;14 unmodified lines12 unmodified lines13141516171817181920212233 unmodified lines565758596060616263646573 unmodified lines13914014114214314414514614714814915015115215315415515615715815916016116216316416516616716814716917017117217315217417517615517717815717918018118218318418518618718816518919019119219319419519619719819920020120220311 unmodified lines2152162172181852192202212222232242 unmodified lines22722822923019623123223323423538 unmodified lines2742752762772432782792452802812822832842 unmodified lines28728828929025625729129229329429529612 unmodified lines3093103113122782793133143153163173189 unmodified lines12 unmodified linesimport { availableParallelism } from "node:os";import path from "node:path";import { parseArgs } from "node:util";import { budgetFrom, BUDGET_OPTIONS } from "./budget.mjs";import { atomicJson, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { countsChanged, dirtyPages, readDiff } from "./diff.mjs";import { atomicJson, ownerPrefixOf, prefixOf, readJson, RENDERER_FORMAT, tokenOf, unpublishOwner } from "./common.mjs";import { countsChanged, dirtyPages, readDiff, stale } from "./diff.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: {33 unmodified lines const status = { generation: hook.generation, state: "rendering", started_at: new Date().toISOString(), repos: {} repos: {}, owners: {} }; // one write at a time, so the file ends as the newest status let statusWrite = Promise.resolve(); const writeStatus = (extra = {}) => {73 unmodified lines nextRepos[repo.did] = { generation: hook.generation, prefix, pages: Object.keys(prior.pages).length }; status.repos[repo.did] = { skipped: true }; } }
// every owner of a repo here gets a user page, rendered again when a read it made changed const owners = new Map(hook.repos.map((repo) => [repo.owner_did, { did: repo.owner_did, prefix: ownerPrefixOf(repo) }])); const knownOwners = (!full && knotState?.owners) || {}; const nextOwners = {}; // one with no repo left here, or under a new handle, loses the page at the old url for (const [did, known] of Object.entries(knotState?.owners ?? {})) { if (owners.get(did)?.prefix !== known.prefix) await unpublishOwner(out, known.prefix); } for (const owner of owners.values()) { const known = knownOwners[owner.did]; if (known && !known.failed && known.prefix === owner.prefix) { const diffs = await readDiff(hookPath, hook, known.generation); if (diffs !== null && !stale(known.deps, diffs)) { nextOwners[owner.did] = { ...known, generation: hook.generation }; continue; } } plans.push({ owner, weight: 0 }); }
// biggest first, so the last worker to finish isn't the one holding ghostty plans.sort((a, b) => b.weight - a.weight);
const workers = Math.min(Number(opts.workers), plans.length); const queue = [...plans]; const familyOf = (name) => name.toLowerCase().replace(/(-fork|\d+)$/, ""); const familyOf = (plan) => (plan.repo?.name ?? plan.owner.prefix).toLowerCase().replace(/(-fork|\d+)$/, ""); const activeFamilies = new Set(); const takePlan = (lastFamily) => { if (queue.length === 0) return undefined; if (lastFamily) { const sameIdx = queue.findIndex((p) => familyOf(p.repo.name) === lastFamily); const sameIdx = queue.findIndex((p) => familyOf(p) === lastFamily); if (sameIdx !== -1) return queue.splice(sameIdx, 1)[0]; } const distinctIdx = queue.findIndex((p) => !activeFamilies.has(familyOf(p.repo.name))); const distinctIdx = queue.findIndex((p) => !activeFamilies.has(familyOf(p))); const plan = queue.splice(distinctIdx !== -1 ? distinctIdx : 0, 1)[0]; activeFamilies.add(familyOf(plan.repo.name)); activeFamilies.add(familyOf(plan)); return plan; }; const totals = { rendered: 0, kept: 0, dropped: 0, over_budget: 0, pages: 0, bytes: 0, reads: 0 }; const failures = []; await Promise.all( Array.from({ length: workers }, () => runWorkerLoop(children, queue, takePlan, activeFamilies, familyOf, { hookPath, out, budget }, async (plan, result, wallMs) => { if (plan.owner) { const { did, prefix } = plan.owner; if (result.error) { if (result.error) { failures.push(did); nextOwners[did] = { failed: true, prefix }; status.owners[did] = { error: result.error }; } else { const { deps, ...stats } = result; nextOwners[did] = { generation: hook.generation, prefix, deps }; status.owners[did] = { ...stats, wall_ms: wallMs }; for (const key of Object.keys(totals)) totals[key] += stats[key] ?? 0; } } else if (result.error) { failures.push(plan.repo.did); // its site link still names the pages of its last finished render nextRepos[plan.repo.did] = { failed: true, prefix: prefixOf(plan.repo), pages: knownRepos[plan.repo.did]?.pages }; status.repos[plan.repo.did] = { error: result.error };11 unmodified lines knot_format: hook.format, renderer_format: RENDERER_FORMAT, build, generation: hook.generation, repos: nextRepos repos: nextRepos, owners: nextOwners }); const summary = { state: failures.length === 0 ? "done" : "partial", failed: failures,2 unmodified lines totals, total_ms: Math.round(performance.now() - t0) }; await writeStatus(summary); process.stdout.write(JSON.stringify({ ...summary, repos: status.repos }) + "\n", () => process.exit(failures.length === 0 ? 0 : 1)); process.stdout.write(JSON.stringify({ ...summary, repos: status.repos, owners: status.owners }) + "\n", () => process.exit(failures.length === 0 ? 0 : 1));}
async function runWorkerLoop(children, queue, takePlan, activeFamilies, familyOf, { hookPath, out, budget }, onRepoDone) { const maxRepos = Math.max(1, Number(opts["worker-max-repos"]) || 12);38 unmodified lines if (child.connected) child.disconnect(); return; } current = plan; lastFamily = familyOf(plan.repo.name); lastFamily = familyOf(plan); tStart = performance.now(); child.send({ repo: plan.repo, work: plan.work, budget }); child.send({ repo: plan.repo, owner: plan.owner, work: plan.work, budget }); }; child.on("message", async (message) => { if (message.ready) { next();2 unmodified lines const finished = current; const wallMs = Math.round(performance.now() - tStart); current = null; served++; const fam = familyOf(finished.repo.name); const hasSameFamilyNext = queue.some((p) => familyOf(p.repo.name) === fam); const fam = familyOf(finished); const hasSameFamilyNext = queue.some((p) => familyOf(p) === fam); if (!hasSameFamilyNext) activeFamilies.delete(fam); const rssMb = message.rss_mb ?? 0; const { rss_mb: _rss, ...cleanMessage } = message; await onRepoDone(finished, cleanMessage, wallMs);12 unmodified lines const failed = current ?? (served === 0 ? takePlan(lastFamily) : null); if (failed) { const wallMs = tStart ? Math.round(performance.now() - tStart) : 0; current = null; activeFamilies.delete(familyOf(failed.repo.name)); await onRepoDone(failed, { error: `worker for ${failed.repo.did} exited ${signal ?? code}` }, wallMs); activeFamilies.delete(familyOf(failed)); await onRepoDone(failed, { error: `worker for ${(failed.repo ?? failed.owner).did} exited ${signal ?? code}` }, wallMs); } resolve(); }); });9 unmodified lines6 unmodified lines78910111112131415606 unmodified linesExpand all62262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166218 unmodified lines6816826836846526856866546876886896906914 unmodified lines6 unmodified linesimport { pathToFileURL } from "node:url";import { parseArgs } from "node:util";import { gzipSync } from "node:zlib";import { rankOf } from "./budget.mjs";import { atomicJson, fileOf, keyOfUrl, PAGE_QUERY, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { atomicJson, fileOf, keyOfUrl, ownerFiles, PAGE_QUERY, prefixOf, readJson, RENDERER_FORMAT, tokenOf, unpublishOwner } from "./common.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { COUNTS, latestPath, ROUTES, SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
const { values: opts, positionals } = parseArgs({606 unmodified linesExpand all rss_mb: Math.round(process.memoryUsage().rss / 1_048_576) };};
// a user's page: their profile and the repos this knot hosts for them, at the url of the// handle. one svfe can't render (no such account) goes, and the SPA answers the urlconst renderOwner = async (owner) => { reads.count = 0; reads.ms = 0; heapPeak = 0; readCache.clear(); readCacheChars = 0; const page = await renderPage(owner.prefix); const deps = [...new Set(page.reads.map((r) => r.key))].sort(); const stats = { reads: reads.count, read_ms: Math.round(reads.ms), render_ms: Math.round(page.ms), deps }; checkCurrent(); if (page.status !== 200) { await unpublishOwner(out, owner.prefix); return { pages: 0, rendered: 0, bytes: 0, status: page.status, ...stats }; } const [file, routesFile] = ownerFiles(out, owner.prefix); const gz = gzipSync(canonicalHtml(page.html), { level: 6, memLevel: 9 }); const routesGz = gzipSync(JSON.stringify({ routes: [""] })); await publishFile(file, gz); await publishFile(routesFile, routesGz); return { pages: 1, rendered: 1, bytes: gz.length + routesGz.length, ...stats };};
// the page by rename, then its stub, so nginx sees the old page or the new oneconst publishFile = async (file, gz) => { await mkdir(path.dirname(file), { recursive: true }); const tmp = `${file}.gz.${process.pid}.tmp`; await writeFile(tmp, gz); await rename(tmp, file + ".gz"); await writeFile(file, "", { flag: "a" });};
// urls svfe serves the repo under: the handle form it links to, and the owner did form,// which hydrates and then redirects to the handle like the SPA doesconst aliasesOf = (repo) => [...new Set([prefixOf(repo).slice(1), `${repo.owner_did}/${repo.name}`])];
18 unmodified lines if (!aliases.includes(alias)) await rm(path.join(out, "www", alias), { force: true }); }};
process.on("message", async ({ repo, work, budget }) => {process.on("message", async ({ repo, owner, work, budget }) => { try { const result = await renderRepo(repo, work, budget); const result = owner ? await renderOwner(owner) : await renderRepo(repo, work, budget); process.send(result); } catch (error) { const message = error instanceof Superseded ? "superseded" : String(error?.stack ?? error); process.send({ error: message }, () => process.exit(error instanceof Superseded ? 3 : 1));4 unmodified lines