render the later pages of the commit log, branches and tags, after everything else
Signed-off-by: dawn <90008@klbr.net>
15 unmodified lines161718192021202122232415 unmodified lines commits: Number(opts["max-commits"])});
// which pages a repo keeps when it can't keep all: its front pages, then folders and files// nearest the root, then commits, newest first as the log lists themconst RANK = { "": 0, issues: 1, pulls: 1, branches: 1, tags: 1, languages: 1, commits: 2, tree: 3, blob: 4, commit: 5 };// nearest the root, then commits, newest first as the log lists them, then the later pages// of the lists, nearest firstconst RANK = { "": 0, issues: 1, pulls: 1, branches: 1, tags: 1, languages: 1, commits: 2, tree: 3, blob: 4, commit: 5, page: 6 };
export const rankOf = (view, depth) => (RANK[view] ?? 6) * 1000 + Math.min(depth, 999);9 unmodified lines10111213141516171819202122232425262728293028 unmodified lines9 unmodified lines
// svfe's url for a repo: the owner's handle, or its did when it has noneexport const prefixOf = (repo) => `/${repo.owner_handle?.toLowerCase() ?? repo.owner_did}/${repo.name}`;
// a later page of a list is a route of its own, "<list>?page=N", stored next to page 1 as// "<list>~page-N.html": no ref or path can hold a "~", so the name is never a real oneexport const PAGE_QUERY = /\?page=([1-9][0-9]*)$/;
// where a route's page lives under the repo's directory: a file is its name plus ".html", a// folder its own directory, so a file page costs no directory. the front page is index.htmlexport const fileOf = (prefix, route) => { if (route === prefix) return "index.html"; const paged = PAGE_QUERY.exec(route); const pathname = paged ? route.slice(0, paged.index) : route; return pathname.slice(prefix.length + 1) + (paged ? `~page-${paged[1]}` : "") + ".html";};
let written = 0;// concurrent writers of one file each rename their own tmp, the last rename winsexport const atomicJson = async (file, value) => { await mkdir(path.dirname(file), { recursive: true });28 unmodified lines18 unmodified lines192021222323242526262728293030313231323334353637384 unmodified lines434445464748454950515218 unmodified lines }); return rebuilt + tail.slice(at);};
const HREF = /href="([^"#?]+)/g;const HREF = /href="([^"#?]+)(\?[^"#]*)?/g;const PAGE = /(?:^\?|&(?:amp;)?)page=([0-9]+)(?:&|$)/;
// same-repo routes linked from rendered html, decoded the way routeOf and the filesystem// see them// see them. a link to a later page of a list keeps its "?page=N", other queries goexport const linksOf = (html, origin, routeOf) => { const found = new Set(); const seen = new Set(); for (const [, raw] of html.matchAll(HREF)) { if (seen.has(raw)) continue; seen.add(raw); for (const [, raw, query] of html.matchAll(HREF)) { const page = Number(PAGE.exec(query ?? "")?.[1] ?? 1); if (seen.has(raw + page)) continue; seen.add(raw + page); if (!raw.startsWith("/")) continue; const href = raw.includes("&") ? raw.replaceAll("&", "&") : raw; let pathname; try {4 unmodified lines } catch { continue; } if (pathname.length > 1 && pathname.endsWith("/")) pathname = pathname.replace(/\/+$/, "") || "/"; const route = page > 1 ? `${pathname}?page=${page}` : pathname; if (routeOf(route) !== null) found.add(route); if (routeOf(pathname) !== null) found.add(pathname); else if (page > 1 && routeOf(pathname) !== null) found.add(pathname); } return found;};1 unmodified line23456678910106 unmodified linesExpand all1171181191201211221231241251261271281291301311321331 unmodified lineimport { mkdtemp, mkdir, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { test } from "node:test";import { keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { fileOf, keyOf, keyOfUrl, prefixOf } from "./common.mjs";import { dirtyPages, readDiff } from "./diff.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { maskShared, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
106 unmodified linesExpand alltest("sharedNamed collects what the kept pages name", () => { const pages = { "/o/r": { shared: ["a", "b"] }, "/o/r/tags": { shared: ["b"] }, "/o/r/pulls": {} }; assert.deepEqual([...sharedNamed(pages)].sort(), ["a", "b"]);});
test("a later page of a list is stored next to page 1, under a name no ref can have", () => { assert.equal(fileOf("/o/r", "/o/r"), "index.html"); assert.equal(fileOf("/o/r", "/o/r/commits/main"), "commits/main.html"); assert.equal(fileOf("/o/r", "/o/r/commits/main?page=3"), "commits/main~page-3.html"); assert.equal(fileOf("/o/r", "/o/r/tags?page=2"), "tags~page-2.html");});
test("linksOf keeps a later page's ?page=N and drops other queries", () => { const paged = (route) => (/[?]page=/.test(route) ? (route.includes("/tags") ? route : null) : route); const html = '<a href="/o/r/tags?page=2">older</a><a href="/o/r/tags?page=1">newer</a><a href="/o/r/blob/main/a?view=raw">raw</a><a href="/o/r/issues?page=2">more</a>'; assert.deepEqual([...linksOf(html, "https://k.test", paged)].sort(), ["/o/r/blob/main/a", "/o/r/issues", "/o/r/tags", "/o/r/tags?page=2"]);});4 unmodified lines56789910111213141516151617181919 unmodified lines4 unmodified lines// node verify.mjs <incremental out> <fresh out>import { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs";import path from "node:path";import { gunzipSync } from "node:zlib";import { readJson } from "./common.mjs";import { fileOf, readJson } from "./common.mjs";import { maskShared, SHARED_DIR, sharedNamed } from "./shared.mjs";
const [incremental, fresh] = process.argv.slice(2);const html = (out, state, route) => { const dir = realpathSync(path.join(out, "site", state.did)); const file = route === state.prefix ? "index.html" : route.slice(state.prefix.length + 1) + ".html"; return maskShared(gunzipSync(readFileSync(path.join(dir, file + ".gz"))).toString()); return maskShared(gunzipSync(readFileSync(path.join(dir, fileOf(state.prefix, route) + ".gz"))).toString());};
const result = { compared: 0, differ: [], only_incremental: 0, only_fresh: 0, missing_shared: [] };for (const name of readdirSync(path.join(incremental, "state"))) {19 unmodified lines6 unmodified lines78910111112131415141 unmodified linesExpand all157158159160161162163161164165166167168169170171172173169174175176177178172173174175179180181182183117 unmodified linesExpand all3013023033043013053063073083093103 unmodified lines31431531631731831932031332132232332432532632763 unmodified lines391392393394386395396397398399166 unmodified linesExpand all6 unmodified linesimport { pathToFileURL } from "node:url";import { parseArgs } from "node:util";import { gzipSync } from "node:zlib";import { rankOf } from "./budget.mjs";import { atomicJson, keyOfUrl, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { atomicJson, fileOf, keyOfUrl, PAGE_QUERY, prefixOf, readJson, RENDERER_FORMAT, tokenOf } from "./common.mjs";import { canonicalHtml, linksOf } from "./html.mjs";import { SHARED_DIR, sharedNamed, sharedOf, sharedPath } from "./shared.mjs";
const { values: opts, positionals } = parseArgs({141 unmodified linesExpand allconst NOT_STATIC = /^\/(issues|pulls)\/new(\/|$)/;// files, folders and logs get pages at the default branch onlyconst BROWSED = new Set(["tree", "blob", "commits"]);
// lists whose later pages get pages of their ownconst PAGED = new Set(["commits", "branches", "tags"]);
const routeOf = (prefix, pathname, repo) => {const routeOf = (prefix, route, repo) => { const paged = PAGE_QUERY.exec(route); const pathname = paged ? route.slice(0, paged.index) : route; if (pathname !== prefix && !pathname.startsWith(prefix + "/")) return null; const rest = pathname.slice(prefix.length); if (NOT_STATIC.test(rest)) return null; const [, view = "", ref, ...file] = rest.split("/"); if (!STATIC_VIEWS.has(view)) return null; if (BROWSED.has(view) && ref && ref !== repo.default_branch) return null; if (view === "commits" && file.length > 0) return null; return { route: pathname, view, ref, file: file.join("/"), depth: rest.split("/").length }; const page = paged ? Number(paged[1]) : 1; if (paged && (!PAGED.has(view) || page < 2)) return null; return { route, view, ref, file: file.join("/"), depth: rest.split("/").length, page };};
// a file is the file name plus ".html", a folder its own directory, so a file page costs// no directory. the repo's front page is the one index.htmlconst fileFor = (dir, prefix, route) => route === prefix ? path.join(dir, "index.html") : path.join(dir, route.slice(prefix.length + 1) + ".html");const fileFor = (dir, prefix, route) => path.join(dir, fileOf(prefix, route));
// --- one repo ------------------------------------------------------------------------------
const storeOf = (did) => path.join(out, "store", tokenOf(did));117 unmodified linesExpand all const rendered = new Set(); for (const route of todo) seen.add(route); const rankOfRoute = (route) => { const parsed = routeOf(prefix, route, repo); return parsed ? rankOf(parsed.view, parsed.depth) : 0; if (!parsed) return 0; return parsed.page > 1 ? rankOf("page", parsed.page) : rankOf(parsed.view, parsed.depth); }; const commitSources = [prefix, `${prefix}/commits/${repo.default_branch}`, `${prefix}/branches`, `${prefix}/tags`]; const commitSourceSet = new Set(commitSources); const commitsRoute = `${prefix}/commits/${repo.default_branch}`;3 unmodified lines .filter((route) => route.startsWith(`${prefix}/commit/`)); return new Set([...new Set(order)].slice(0, budget.commits)); };
// a list's later pages wait until nothing else is left, so they only take the room the // rest of the repo didn't let later = []; while (todo.length > 0) { while (todo.length > 0 || later.length > 0) { checkCurrent(); if (todo.length === 0) [todo, later] = [later, []]; const batch = todo.sort((a, b) => rankOfRoute(a) - rankOfRoute(b) || (a < b ? -1 : a > b ? 1 : 0)); todo = []; const batchMinViewBytes = new Map(minViewBytes); const minBytesFor = (route) => {63 unmodified lines for (const next of kept) { if (next.startsWith(`${prefix}/commit/`)) continue; if (!seen.has(next)) { seen.add(next); todo.push(next); (PAGE_QUERY.test(next) ? later : todo).push(next); } } } return ready;166 unmodified linesExpand all