keep one copy per repo of the routes and read keys its pages share, so a folder of generated files fits a worker's heap
Signed-off-by: dawn <90008@klbr.net>
192 unmodified linesExpand all19319419519619719819920020120220320420520620720820921021129 unmodified lines241242243244245246247248249250251252253254255256257258259260179 unmodified linesExpand all4404414424434284444454464474481 unmodified line450451452453438439440454455456457458459460163 unmodified linesExpand all192 unmodified linesExpand all};
const fileFor = (dir, prefix, route) => path.join(dir, fileOf(prefix, route));
// every page of a folder links the folder's whole listing and most pages make the same reads,// so a repo keeps one copy of each route and key: a folder of 1500 generated files otherwise// holds 1500 copies of its listing, 30 MB of a worker's heapconst interned = new Map();const intern = (value) => { const known = interned.get(value); if (known !== undefined) return known; interned.set(value, value); return value;};
// --- one repo ------------------------------------------------------------------------------
const storeOf = (did) => path.join(out, "store", tokenOf(did));
29 unmodified lines files.clear(); readCache.clear(); readCacheChars = 0; shared.clear(); interned.clear();
const prefix = prefixOf(repo); const stateFile = path.join(out, "state", tokenOf(repo.did) + ".json"); const saved = readJson(stateFile); const savedAliases = saved?.aliases ?? []; const prior = work === "all" ? null : saved; const pages = prior ? prior.pages : {}; for (const page of Object.values(pages)) { page.deps = page.deps.map(intern); page.links = page.links?.map(intern); } // routes left out with the reads that failed them, retried when one of those changes const left = prior ? (prior.left ?? {}) : {}; const dirty = new Set(prior ? work : []); const genDir = path.join(storeOf(repo.did), hook.generation);179 unmodified linesExpand all results.set(route, { skippedOver: true }); } else { let page = await renderPage(route); stats.render_ms += page.ms; const deps = [...new Set(page.reads.map((r) => r.key))].sort(); const deps = [...new Set(page.reads.map((r) => r.key))].sort().map(intern); if (!accepted(page, repo.did)) { page = null; results.set(route, { deps }); } else {1 unmodified line // and the front page decide which commits get pages const rawLinks = route.startsWith(`${prefix}/commit/`) ? [] : [...linksOf(page.html, origin, (pathname) => routeOf(prefix, pathname, repo))]; const links = commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`)); const links = ( commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`)) ).map(intern); const canonical = canonicalHtml(page.html); const { shared, latest } = page; page = null; const gz = gzipSync(canonical, { level: 6, memLevel: 9 });163 unmodified linesExpand all