stop rendering a batch once the repo's byte budget is full
Signed-off-by: dawn <90008@klbr.net>
183 unmodified linesExpand all18418518618718818919019119219319419519619719817 unmodified lines21621721821922022122222322422522622722822923021823123223323423523623723823924024124224324423123224524624724824925025125225323425425525625725825926026126226326426526626720 unmodified lines288289290291263264265266267268292293294295296297298299300301270271272273274275276277278279280281302303304305306307308309310311312313314315316317318110 unmodified linesExpand all183 unmodified linesExpand all }); }
let bytes = Object.values(pages).reduce((sum, page) => sum + page.bytes, 0); const minViewBytes = new Map(); for (const [route, page] of Object.entries(pages)) { const parsed = routeOf(prefix, route, repo); if (!parsed) continue; const prev = minViewBytes.get(parsed.view); if (prev === undefined || page.bytes < prev) minViewBytes.set(parsed.view, page.bytes); } const overBudget = new Set(); // what the budget left out last time stays out until a full render const over = new Set(prior?.over ?? []); const seen = new Set([...Object.keys(pages), ...Object.keys(left), ...over]);17 unmodified lines while (todo.length > 0) { checkCurrent(); const batch = todo.sort((a, b) => rankOfRoute(a) - rankOfRoute(b) || (a < b ? -1 : a > b ? 1 : 0)); todo = []; const batchMinViewBytes = new Map(minViewBytes); const minBytesFor = (route) => { const parsed = routeOf(prefix, route, repo); return ((parsed && batchMinViewBytes.get(parsed.view)) ?? 12_000) * 0.98; }; // the budget keeps the best ranked pages whatever order their renders finish in: // render the batch, then take its pages in rank order let room = budget.pages - Object.keys(pages).length; const hasKnownInBatch = batch.some((r) => r in pages); const wanted = batch.filter((route) => { if (route in pages) return true; if (room <= 0) { if (room <= 0 || (!hasKnownInBatch && budget.bytes - bytes < minBytesFor(route))) { overBudget.add(route); return false; } room--; return true; }); const results = new Map(); let at = 0; let remainingKnown = wanted.filter((r) => r in pages).length; const retire = () => { const ready = []; while (at < wanted.length && results.has(wanted[at])) { const route = wanted[at++]; const known = route in pages; const { deps, gz, links } = results.get(route); const res = results.get(route); results.delete(route); if (res.skippedOver) { overBudget.add(route); continue; } const known = route in pages; const { deps, gz, links } = res; if (known) { if (known) bytes -= pages[route].bytes; bytes -= pages[route].bytes; remainingKnown--; } if (gz) { const parsed = routeOf(prefix, route, repo); if (parsed) { const prev = minViewBytes.get(parsed.view); if (prev === undefined || gz.length < prev) minViewBytes.set(parsed.view, gz.length); } } if (!gz || bytes + gz.length > budget.bytes) { if (known) { delete pages[route]; stats.dropped++;20 unmodified lines } return ready; }; await pool(wanted, async (route) => { let page = await renderPage(route); stats.render_ms += page.ms; const deps = [...new Set(page.reads.map((r) => r.key))].sort(); if (!accepted(page)) { page = null; results.set(route, { deps }); if (!(route in pages) && remainingKnown === 0 && budget.bytes - bytes < minBytesFor(route)) { results.set(route, { skippedOver: true }); } else { let page = await renderPage(route); stats.render_ms += page.ms; const deps = [...new Set(page.reads.map((r) => r.key))].sort(); if (!accepted(page)) { page = null; results.set(route, { deps }); } else { // commit pages link their parents and trees at their own commit: only the log // and the front page decide which commits get pages const rawLinks = route.startsWith(`${prefix}/commit/`) ? [] : [...linksOf(page.html, origin, (pathname) => routeOf(prefix, pathname, repo))]; const links = commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`)); const canonical = canonicalHtml(page.html); page = null; const gz = gzipSync(canonical, { level: 9, memLevel: 9 }); results.set(route, { deps, gz, links }); // commit pages link their parents and trees at their own commit: only the log // and the front page decide which commits get pages const rawLinks = route.startsWith(`${prefix}/commit/`) ? [] : [...linksOf(page.html, origin, (pathname) => routeOf(prefix, pathname, repo))]; const links = commitSourceSet.has(route) ? rawLinks : rawLinks.filter((next) => !next.startsWith(`${prefix}/commit/`)); const canonical = canonicalHtml(page.html); page = null; const gz = gzipSync(canonical, { level: 9, memLevel: 9 }); results.set(route, { deps, gz, links }); } } for (const item of retire()) { const file = fileFor(tmpDir, prefix, item.route); await rm(file, { force: true });110 unmodified linesExpand all