web/repo: a rendered diff page carries the diff it built instead of the payload it built it from, so a big commit's page drops megabytes
Signed-off-by: dawn <90008@klbr.net>
2 unmodified lines345677891011121113131415161718470 unmodified linesExpand all4894904914924924934944954964974974984994984995005015025035045045055065055065075085095105115125135145155165175185195205215225235245255265275285295302 unmodified linesimport { gitTarget } from "#lib/api/gitclient.ts";import { cappedFetch, DiffTooLargeError, isDiffTooLarge, diffScript, fetchSides, firstDiff, isDiffTooLarge, loadDiff, loadDiffStat, patchesScript, readPatches, shippedPatches shippedDiff} from "#lib/api/repoDiff.ts";import { emptyTreeOid, type MirrorFileDiff } from "#lib/api/knotmirror.ts";import type { Fetch } from "#lib/fetch.ts";
470 unmodified linesExpand all [] ); });
it("hands the patches it shipped to the app's first load, in a script nothing closes early", async () => { it("hands the diff it shipped to the app's first load, in a script nothing closes early", async () => { const { deps } = harness([file("a.ts", 4), file("c.ts", 1)], { ...texts, "head:a.ts": "</script>\n" }); const loaded = await loadDiff(deps, diffSpec); const shipped = await readPatches(deps, loaded); const script = patchesScript(shipped) ?? ""; const shipped = await firstDiff(deps, diffSpec); const script = diffScript(shipped); const body = script.slice(script.indexOf(">") + 1, script.lastIndexOf("</script>")); const root = { querySelectorAll: () => [{ textContent: body }] } as unknown as ParentNode; const other = { ...diffSpec, version: { ...diffSpec.version, head: "other" } };
expect(body).not.toContain("<"); expect(shippedPatches(loaded, root).files.map((file) => file.patch)).toEqual( shipped.files.map((file) => file.patch) ); expect(shippedDiff(diffSpec, root)).toEqual(shipped); expect(shippedDiff(other, root)).toBeUndefined(); expect(shipped.files.map((file) => file.patch === undefined)).toEqual([false, false]); });
it("reads the diff payload from a clone, so the page carries the diff instead", async () => { const { deps } = harness([file("a.ts", 4)], texts); const read: string[] = []; const watched: Fetch = async (input, init) => new Proxy(await deps.ctx.fetch(input, init), { get: (response, key) => { read.push(String(key)); const value: unknown = Reflect.get(response, key, response); return typeof value === "function" ? value.bind(response) : value; } });
const diff = await firstDiff({ ...deps, ctx: { ...deps.ctx, fetch: watched } }, diffSpec);
expect(diff.files).toHaveLength(1); expect(read).toContain("clone"); expect(read.filter((key) => ["body", "text", "json", "arrayBuffer"].includes(key))).toEqual( [] ); });});209 unmodified linesExpand all21021121221321421421521621721821621721822022122222322422522622722822921922022122222322423223323423523623723823924024124224324424524622522622722822923023123223323423523623724925023823924024124224324425625725825924524624724824925025125225325425525645 unmodified lines209 unmodified linesExpand all// reads the rest as it scrolls, as beforeexport const SSR_PATCH_BYTES = 64 * 1024;export const SSR_SIDES_READ_BYTES = 512 * 1024;
const PATCHES_ATTRIBUTE = "data-diff-patches";const DIFF_ATTRIBUTE = "data-diff";
// a patch is named by the sides it was built from, so one never lands on another diff's fileconst sidesKey = (sides: { lhs: DiffSrc; rhs: DiffSrc }): string => `${sides.lhs.oid}:${sides.rhs.oid}`;// a diff is named by the revisions it compares, so one never stands in for anotherconst diffKey = (contents: ContentsSource): string => `${contents.oldRef}..${contents.newRef}`;
/** the script a page carries its diff's patches in, for the app to take up as it starts */export const patchesScript = (diff: RepoDiff): string | null => { const patches = Object.fromEntries( diff.files.flatMap((file) => file.patch && file.sides ? [[sidesKey(file.sides), file.patch]] : [] ) ); if (Object.keys(patches).length === 0) return null; const json = JSON.stringify(patches).replaceAll("<", "\\u003c"); return `<script type="application/json" ${PATCHES_ATTRIBUTE}>${json}</script>`;/** the script a page carries its diff in, for the app to take up as it starts */export const diffScript = (diff: RepoDiff): string => { const json = JSON.stringify({ key: diffKey(diff.contents), diff }).replaceAll("<", "\\u003c"); return `<script type="application/json" ${DIFF_ATTRIBUTE}>${json}</script>`;};
/** the patches the page came with, which the app's first load draws without reading */export const shippedPatches = (diff: RepoDiff, root: ParentNode = document): RepoDiff => { const patches: Record<string, HunkPatch> = Object.assign( {}, ...[...root.querySelectorAll(`script[${PATCHES_ATTRIBUTE}]`)].map((script) => JSON.parse(script.textContent ?? "{}") ) ); return { ...diff, files: diff.files.map((file) => { const patch = file.sides && patches[sidesKey(file.sides)]; return patch ? { ...file, patch } : file; }) };/** the diff the page came with, which the app's first load draws without reading */export const shippedDiff = ( spec: RepoDiffSpec, root: ParentNode = document): RepoDiff | undefined => { const key = diffKey({ oldRef: spec.version.base, newRef: spec.version.head }); for (const script of root.querySelectorAll(`script[${DIFF_ATTRIBUTE}]`)) { const shipped = JSON.parse(script.textContent ?? "") as { key: string; diff: RepoDiff }; if (shipped.key === key) return shipped.diff; } return undefined;};
// kit inlines into the page every response its fetch reads, but not a clone's, so the sides// a patch is built from stay out of it// kit inlines into the page every response its fetch reads, but not a clone's, so the diff// payload and the sides a patch is built from stay out of itconst unshipped = (fetch: Fetch): Fetch => async (input, init) => (await fetch(input, init)).clone();
/** a first load's diff with the patches it ships: read on the server, taken from the page in the browser */export const withPatches = (deps: RepoDiffDeps, diff: RepoDiff): Promise<RepoDiff> => browser ? Promise.resolve(shippedPatches(diff)) : readPatches(deps, diff);/** a first load's diff with the patches it ships: read on the server, where the page carries what it built from the payload rather than the payload, and taken from the page in the browser */export const firstDiff = async (deps: RepoDiffDeps, spec: RepoDiffSpec): Promise<RepoDiff> => { if (browser) return shippedDiff(spec) ?? loadDiff(deps, spec); const reading = { ...deps, ctx: { ...deps.ctx, fetch: unshipped(deps.ctx.fetch) } }; return readPatches(deps, await loadDiff(reading, spec));};
export const readPatches = async ( deps: RepoDiffDeps, diff: RepoDiff,45 unmodified lines123345671 unmodified line910111213131415161718108 unmodified linesExpand all1271281291301301311321331341351361 unmodified line13813914014114014114214214314414514614714819 unmodified lines<script lang="ts"> import type { SelectedLineRange } from "@pierre/diffs"; import { onDestroy } from "svelte"; import { onDestroy, untrack } from "svelte"; import { goto } from "$app/navigation"; import { page } from "$app/state"; import { SvelteMap } from "svelte/reactivity"; import DiffCodeView, {1 unmodified line type CodeViewFile, type DiffStock } from "#lib/components/repo/DiffCodeView.svelte"; import { fileAnchor, jumpToAnchoredFile } from "#lib/components/repo/fileAnchor.svelte.ts"; import { patchesScript, type RepoDiff, type RepoDiffDeps } from "#lib/api/repoDiff.ts"; import { diffScript, type RepoDiff, type RepoDiffDeps } from "#lib/api/repoDiff.ts"; import { servesRenderedPage } from "#lib/ssr/firstLoad.ts"; import type { DiffStyle } from "#lib/components/repo/pierre.ts"; import type { DiffEntry } from "#lib/components/repo/fileDiff.ts"; import { createDiffQueue } from "#lib/components/repo/diffQueue.ts"; import { diffRows, findDiffMatches, type DiffMatch } from "#lib/components/lines/diff.ts";108 unmodified linesExpand all return found; };
export const resync = () => view?.resync(); const shippedPatches = $derived(patchesScript(diff)); // a first load's diff stays in its page for the app to take up; later ones never ship const shipped = servesRenderedPage() ? untrack(() => diffScript(diff)) : null; jumpToAnchoredFile(jumpToFile);</script>
<!-- one scroller for the whole diff: the page itself never grows with the file1 unmodified line a diff of this size --><!-- fills whatever the sticky toolbar leaves of the column, so the scroller always reaches the bottom of the card --><div class="flex min-h-0 shrink grow basis-auto flex-col"> {#if shippedPatches} <!-- eslint-disable-next-line svelte/no-at-html-tags -- json of our own patches, its < escaped --> {@html shippedPatches} {#if shipped} <!-- eslint-disable-next-line svelte/no-at-html-tags -- json of our own diff, its < escaped --> {@html shipped} {/if} <DiffCodeView bind:this={view} files={viewFiles}19 unmodified lines10 unmodified lines11121314151617171819202141 unmodified lines636465666768676869707169 unmodified lines10 unmodified linesimport { servesRenderedPage } from "#lib/ssr/firstLoad.ts";import { errorMessage, live, settle, stream, toHttpError } from "#lib/api/load.ts";import { cappedFetch, firstDiff, isDiffTooLarge, loadDiff, withPatches, type RepoDiffDeps, type RepoDiffSpec} from "#lib/api/repoDiff.ts";
41 unmodified lines let diff; let diffSpec: RepoDiffSpec | undefined; let diffError: string | undefined; try { diff = await loadDiff(diffDeps, spec); if (renderedPage) diff = await withPatches(diffDeps, diff); diff = await (renderedPage ? firstDiff : loadDiff)(diffDeps, spec); } catch (cause) { if (isDiffTooLarge(cause)) diffSpec = spec; else diffError = errorMessage(cause, "could not load commit diff from the mirror"); }69 unmodified lines2 unmodified lines34567789101149 unmodified lines61626364656667656667686970717214 unmodified lines2 unmodified linesimport { gitTarget } from "#lib/api/gitclient.ts";import { listLog } from "#lib/api/knotmirror.ts";import { early, orNull, settle, stream } from "#lib/api/load.ts";import { isRedirect, parseDiffRoute } from "#lib/api/pullRoute.ts";import { cappedFetch, loadDiff, withPatches } from "#lib/api/repoDiff.ts";import { cappedFetch, firstDiff, loadDiff } from "#lib/api/repoDiff.ts";import { withAuthorHandles } from "#lib/api/repoIndex.ts";import { countCommentsByCommit } from "#lib/components/comment/comments.ts";import { didFromUri } from "#lib/api/uri.ts";import { servesRenderedPage } from "#lib/ssr/firstLoad.ts";49 unmodified lines }; const diff = version && commits !== null ? await orNull( loadDiff(diffDeps, { repo: repo.repoDid as Did, version }).then((loaded) => renderedPage ? withPatches(diffDeps, loaded) : loaded ) (renderedPage ? firstDiff : loadDiff)(diffDeps, { repo: repo.repoDid as Did, version }) ) : null;
return {14 unmodified lines