cache xrpc reads within a repo in a bounded lru
Signed-off-by: dawn <90008@klbr.net>
22 unmodified lines2324252627282930313 unmodified lines353637383940414243444546476 unmodified lines545556575253545556575859596060616263646565666768697071727374757677787980818283848586878889909192939495969798991001011021031046869105106107108109110355 unmodified linesExpand all22 unmodified lines});const hook = JSON.parse(readFileSync(positionals[0], "utf8"));const out = path.resolve(opts.out);const origin = hook.knot.base_url.replace(/\/$/, "");const originHost = new URL(origin).host;// where reads go: the knot's own nginx on loopback, or its public https originconst readBase = (opts.read ?? hook.knot.base_url).replace(/\/$/, "");const JOBS = Number(opts.jobs);
3 unmodified linesconst reads = { count: 0, ms: 0 };// file sizes from the folder listings this run read, so a file over the budget never rendersconst sizes = new Map();const files = new Set();const readCache = new Map();let readCacheChars = 0;const MAX_CACHE_ENTRIES = 512;const MAX_CACHE_BODY_CHARS = 262_144;const MAX_CACHE_TOTAL_CHARS = 8 * 1024 * 1024;const sizeKey = (ref, file) => `${ref}\0${file}`;
const learnSizes = (url, body) => { const ref = url.searchParams.get("ref") ?? "";6 unmodified lines } } catch {}};
const answer = async (request) => { const url = new URL(request.url); if (url.origin !== origin || !url.pathname.startsWith("/xrpc/")) { // nothing off the knot at render time: the browser fetches it live after hydrating return new Response(null, { status: 404 }); }const fetchXrpc = async (url, cacheKey) => { const t = performance.now(); const res = await fetch(readBase + url.pathname + url.search, { headers: { host: new URL(origin).host, "accept-encoding": "identity" } const res = await fetch(readBase + cacheKey, { headers: { host: originHost, "accept-encoding": "identity" } }); const body = await res.text(); reads.count++; reads.ms += performance.now() - t; deps.getStore()?.push({ key: keyOfUrl(url), status: res.status }); if (res.status === 200 && url.pathname === "/xrpc/org.tangled.git.getTree") learnSizes(url, body); const contentType = res.headers.get("content-type"); return { status: res.status, contentType, body };};
const answer = async (request) => { const url = new URL(request.url); if (url.origin !== origin || !url.pathname.startsWith("/xrpc/")) { // nothing off the knot at render time: the browser fetches it live after hydrating return new Response(null, { status: 404 }); } const cacheKey = url.pathname + url.search; let pending = readCache.get(cacheKey); if (pending) { readCache.delete(cacheKey); readCache.set(cacheKey, pending); } else { pending = fetchXrpc(url, cacheKey).then((entry) => { const len = entry.body.length; if (len > MAX_CACHE_BODY_CHARS) { readCache.delete(cacheKey); return entry; } pending.chars = len; readCacheChars += len; while (readCache.size > MAX_CACHE_ENTRIES || readCacheChars > MAX_CACHE_TOTAL_CHARS) { const oldest = readCache.keys().next().value; if (oldest === undefined || oldest === cacheKey) break; const oldEntry = readCache.get(oldest); readCache.delete(oldest); readCacheChars -= oldEntry?.chars ?? 0; } return entry; }); readCache.set(cacheKey, pending); } const entry = await pending; deps.getStore()?.push({ key: keyOfUrl(url), status: entry.status }); const headers = new Headers(); if (res.headers.has("content-type")) headers.set("content-type", res.headers.get("content-type")); return new Response(body, { status: res.status, headers }); if (entry.contentType) headers.set("content-type", entry.contentType); return new Response(entry.body, { status: entry.status, headers });};
globalThis.__knotStaticPrerender = { answer, renderWorkers: Number(opts.highlighters) };const { server } = await import(pathToFileURL(path.resolve(opts.server, "instance.js")).href);355 unmodified linesExpand all