publish a repo's open ticket counts apart from its pages, so a new ticket rewrites one file and no page
Signed-off-by: dawn <90008@klbr.net>
16 unmodified lines1718192021212223242516 unmodified lines node verify.mjs <incremental out> <fresh out>
the newest sidebar of each ref is also at `<repo>/_latest/sidebar/<ref>.json`, for a pagethe SPA renders without a prerendered copy to name, and the routes the repo has pages for at`<repo>/_latest/routes.json`, so the SPA loads only those as documents.`<repo>/_latest/routes.json`, so the SPA loads only those as documents. the repo's open ticketcounts, which every page's tabs show, are read apart from the pages into`<repo>/_latest/counts.json`, so a new ticket rewrites that file and no page.
node --test128 unmodified linesExpand all12913013113213313413513613713813910 unmodified lines128 unmodified linesExpand all for (const name of any.names) delete params[name]; return any.keys.has(keyOf(nsid, params));};
// whether the open ticket counts the repo publishes apart from its pages changedexport const countsChanged = (prior, diffs) => (prior.counts ?? []).some((key) => touched(diffs, key));
// the routes whose reads changed: pages to render again, and pages the budget or a failed// read left out, which may stand nowexport const dirtyPages = (prior, diffs) => { if (diffs.resets.has(prior.did)) return "all";10 unmodified lines13 unmodified lines14151617181819202122107 unmodified linesExpand all130131132133134134135136137138154 unmodified linesExpand all13 unmodified linesimport path from "node:path";import { parseArgs } from "node:util";import { budgetFrom, BUDGET_OPTIONS } from "./budget.mjs";import { atomicJson, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";import { countsChanged, dirtyPages, readDiff } from "./diff.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: {107 unmodified linesExpand all continue; } diffEntries = Math.max(diffEntries, diffs.entries); const dirty = dirtyPages(prior, diffs); if (dirty === "all" || dirty.size > 0 || repo.change !== "unchanged") { if (dirty === "all" || dirty.size > 0 || repo.change !== "unchanged" || countsChanged(prior, diffs)) { plans.push({ repo, work: dirty === "all" ? "all" : [...dirty], weight: dirty === "all" ? Object.keys(prior.pages).length : dirty.size }); } else { // nothing it read changed: the pages stand for this generation too nextRepos[repo.did] = { generation: hook.generation, prefix, pages: Object.keys(prior.pages).length };154 unmodified linesExpand all2 unmodified lines345677891011123 unmodified linesExpand all13513613713813914014114214314414514614714814915015115215315415515615715833 unmodified lines2 unmodified linesimport { tmpdir } from "node:os";import path from "node:path";import { test } from "node:test";import { fileOf, keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";import { countsChanged, dirtyPages, readDiff } from "./diff.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { latestPath, maskShared, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
test("keyOf sorts params, drops empty ones and matches a request url", () => {123 unmodified linesExpand all }; assert.deepEqual([...dirtyPages(prior, diffs)].sort(), ["/o/r/issues", "/o/r/issues/3", "/o/r/issues/5"]);});
test("a new ticket changes the repo's counts and no page that doesn't read them", async () => { const { hookPath, hook } = await chain([ ["g1", []], ["g2", [{ op: "changed", nsid: "org.tangled.repo.countIssues", params: { state: "open", subject: "did:plc:a" }, old: "1", new: "2" }]] ]); const diffs = await readDiff(hookPath, hook, "g1"); const prior = { did: "did:plc:a", pages: { "/o/r": page("org.tangled.git.getTree?ref=main&repo=did:plc:a") }, counts: ["org.tangled.repo.countIssues?state=open&subject=did:plc:a", "org.tangled.repo.countPulls?status=open&subject=did:plc:a"] }; assert.deepEqual([...dirtyPages(prior, diffs)], []); assert.equal(countsChanged(prior, diffs), true); assert.equal(countsChanged({ ...prior, counts: undefined }, diffs), false);});
test("a shared value is named by its content", () => { const a = sharedOf({ ref: "main", files: [] }); assert.equal(a.hash, sharedOf({ ref: "main", files: [] }).hash); assert.notEqual(a.hash, sharedOf({ ref: "dev", files: [] }).hash);33 unmodified lines18 unmodified lines192021222324252627286 unmodified lines18 unmodified lines
// the routes a repo has pages for, relative to its url: the SPA loads those as documents and// renders the rest itselfexport const ROUTES = "routes";// the repo's open ticket counts, which every page's tabs showexport const COUNTS = "counts";
const NAMED = /_shared\/[0-9a-f]{16}\.json/g;
// a page kept from an older render names an older copy of the same thing, so an incremental6 unmodified lines8 unmodified lines910111213131415161793 unmodified lines111112113114115116117118119120121122123124125126127128129130131132133134135136137138436 unmodified linesExpand all5755765775785595605615625635645795805815825835845855865875885895905915925935947 unmodified lines60260360460558060660760860961061151 unmodified lines8 unmodified linesimport { gzipSync } from "node:zlib";import { rankOf } from "./budget.mjs";import { atomicJson, fileOf, keyOfUrl, PAGE_QUERY, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { latestPath, ROUTES, SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";import { COUNTS, latestPath, ROUTES, SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
const { values: opts, positionals } = parseArgs({ allowPositionals: true, options: {93 unmodified lines if (entry.contentType) headers.set("content-type", entry.contentType); return new Response(entry.body, { status: entry.status, headers });};
const COUNT_READS = { issues: ["org.tangled.repo.countIssues", { state: "open" }], pulls: ["org.tangled.repo.countPulls", { status: "open" }]};
// svfe's spelling of the counts, so the keys match the knot's diff lines for them. no value// when one fails, and the browser asks the knot itselfconst readCounts = async (did) => { const value = {}; const keys = []; for (const [name, [nsid, params]] of Object.entries(COUNT_READS)) { const url = new URL(`/xrpc/${nsid}?${new URLSearchParams({ subject: did, ...params })}`, origin); keys.push(keyOfUrl(url)); const entry = await fetchXrpc(url, url.pathname + url.search); if (entry.status !== 200) return { value: null, keys }; value[name] = JSON.parse(entry.body).count ?? 0; } return { value, keys };};
// this run's shared files of the repo, by hashconst shared = new Map();const share = (value, name) => { const { hash, body } = sharedOf(value);436 unmodified linesExpand all await ensureDir(path.dirname(file)); await link(empty, file); await link(path.join(sharedDir, hash + ".json.gz"), file + ".gz"); } const routesFile = path.join(tmpDir, latestPath(ROUTES)); const routesGz = gzipSync(JSON.stringify({ routes: Object.keys(pages).map((route) => route.slice(prefix.length)).sort() })); await ensureDir(path.dirname(routesFile)); await link(empty, routesFile); await writeFile(routesFile + ".gz", routesGz); bytes += routesGz.length; const writeLatest = async (name, value) => { const file = path.join(tmpDir, latestPath(name)); const gz = gzipSync(JSON.stringify(value)); await ensureDir(path.dirname(file)); await link(empty, file); await writeFile(file + ".gz", gz); bytes += gz.length; }; await writeLatest(ROUTES, { routes: Object.keys(pages).map((route) => route.slice(prefix.length)).sort() }); // read apart from the pages, so a ticket write rewrites this file and no page const counts = await readCounts(repo.did); if (counts.value) await writeLatest(COUNTS, counts.value); stats.kept = Object.keys(pages).filter((route) => !rendered.has(route)).length; for (const route of overBudget) over.add(route); stats.over_budget = over.size; checkCurrent();7 unmodified lines generation: hook.generation, pages, left, over: [...over].sort(), latest latest, counts: counts.keys }); // the state names the new generation now, so the old one can go for (const entry of await readdir(storeOf(repo.did))) { if (entry !== hook.generation) await rm(path.join(storeOf(repo.did), entry), { recursive: true, force: true });51 unmodified lines