web/repo: a knot's file page names its folder's listing in a file shared by the folder's pages, instead of carrying the payload it was read from
Signed-off-by: dawn <90008@klbr.net>
1234567export type Fetch = (input: URL | RequestInfo, init?: RequestInit) => Promise<Response>;
// kit inlines into the page every response its fetch reads, but not a clone'sexport const unshipped = (fetch: Fetch): Fetch => async (input, init) => (await fetch(input, init)).clone();15 unmodified lines16171819202021222324209 unmodified linesExpand all2342352362372382392402412422432442382392402417 unmodified lines24925025125225325425525625738 unmodified lines15 unmodified linesimport type { BobbinContext } from "#lib/api/client.ts";import type { GitTarget } from "#lib/api/gitclient.ts";import type { DiffStat } from "#lib/api/diff.ts";import type { Revspec } from "#lib/api/pullRoute.ts";import type { Fetch } from "#lib/fetch.ts";import { unshipped, type Fetch } from "#lib/fetch.ts";
type DiffSrc = ShTangledGitDefs.DiffSrc;
// pierre renders roughly one row per line pair, plus a little chrome per hunk. only209 unmodified linesExpand all } return undefined;};
// kit inlines into the page every response its fetch reads, but not a clone's, so the diff// payload and the sides a patch is built from stay out of itconst unshipped = (fetch: Fetch): Fetch => async (input, init) => (await fetch(input, init)).clone();
/** a first load's diff with the patches it ships: read on the server, where the page carries what it built from the payload rather than the payload, and taken from the page in the browser */export const firstDiff = async (deps: RepoDiffDeps, spec: RepoDiffSpec): Promise<RepoDiff> => {7 unmodified lines diff: RepoDiff, maxBytes: number = SSR_PATCH_BYTES, maxReadBytes: number = SSR_SIDES_READ_BYTES): Promise<RepoDiff> => { // the sides stay out of the page too const ctx = { ...deps.git.ctx, fetch: unshipped(deps.git.ctx.fetch) }; const reading = { ...deps, git: { ...deps.git, ctx } }; let unread = maxReadBytes; const reads = diff.files.map((file) => {38 unmodified lines121234567891011121314151644 unmodified lines616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132import { describe, expect, it } from "vitest";import { chainEnd, findCompact, rootedChain, treeRoot } from "#lib/api/repoSidebar.ts";import { afterEach, describe, expect, it, vi } from "vitest";import { chainEnd, findCompact, LISTING_INLINE_BYTES, listingsScript, pageListings, rootedChain, treeRoot} from "#lib/api/repoSidebar.ts";import type { TreeEntrySummary } from "#lib/components/repo/types.ts";import type { Fetch } from "#lib/fetch.ts";
const dir = (name: string): TreeEntrySummary => ({ name, kind: "directory", size: 0 });const file = (name: string): TreeEntrySummary => ({ name, kind: "file", size: 1 });
44 unmodified lines it("stops reading at its limit", async () => { expect(await findCompact(list, ["src", "docs"], 2)).toEqual({ src: "main" }); });});
describe("pageListings", () => { afterEach(() => { vi.unstubAllGlobals(); });
const entries = (count: number, length: number) => Array.from({ length: count }, (_, index) => ({ name: `${String(index).padStart(length, "x")}.json`, mode: "0100644", size: 1 })); const folders: Record<string, ReturnType<typeof entries>> = { big: entries(1000, 40), "big/small": entries(2, 4) }; const parent = { publicConfig: { bobbinUrl: "https://bobbin.test" }, repo: Promise.resolve({ uri: "at://did:plc:alice/sh.tangled.repo/core", repoDid: "did:plc:repo", git: { defaultBranch: "main" } }) } as never;
it("on a knot, names a big folder's shared file and carries a small one, and inlines neither", async () => { const share = vi.fn(() => "_shared/listing.json"); vi.stubGlobal("__knotStaticPrerender", { share }); const read: string[] = []; const fetch: Fetch = async (input) => { const path = new URL(String(input)).searchParams.get("path") ?? ""; const response = new Response(JSON.stringify({ entries: folders[path] }), { headers: { "content-type": "application/json" } }); return new Proxy(response, { get: (target, key) => { read.push(String(key)); const value: unknown = Reflect.get(target, key, target); return typeof value === "function" ? value.bind(target) : value; } }); };
const { listings, sources } = await pageListings( parent, "main", ["big", "big/small"], fetch, "/alice/core" );
expect(JSON.stringify(listings.big).length).toBeGreaterThan(LISTING_INLINE_BYTES); expect(sources.big).toBe("/alice/core/_shared/listing.json"); expect(share).toHaveBeenCalledExactlyOnceWith(listings.big); expect(sources["big/small"]).toEqual(listings["big/small"]); expect(read.filter((key) => ["body", "text", "json", "arrayBuffer"].includes(key))).toEqual( [] ); });
it("ships its sources in a script nothing closes early", () => { expect(listingsScript({})).toBeNull(); const script = listingsScript({ a: [{ name: "</script>", kind: "file", size: 1 }] }) ?? ""; expect( script.slice(script.indexOf(">") + 1, script.lastIndexOf("</script>")) ).not.toContain("<"); });});1234512 unmodified lines18192021212223242526272893 unmodified lines12212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819963 unmodified linesimport { browser } from "$app/env";import { branches as gitBranches, gitTarget, hasGit,12 unmodified lines} from "#lib/api/repo.ts";import { attempt, REF_LIMIT, toLanguageSlices } from "#lib/api/repoIndex.ts";import type { RepoParent } from "#lib/api/repoLoad.ts";import type { LanguageSlice } from "#lib/components/repo/types.ts";import type { Fetch } from "#lib/fetch.ts";import { unshipped, type Fetch } from "#lib/fetch.ts";import { prerender } from "#lib/prerender.ts";import { servesRenderedPage } from "#lib/ssr/firstLoad.ts";
// the refs the sidebar draws. it keeps every ref up to REF_LIMIT (the most static knots// answer in a page), so a page that names a commit's refs reads them from itexport const SIDEBAR_BRANCHES = 6;93 unmodified lines ); return Object.fromEntries(listed.filter((entry) => entry !== null));};
// a page carries a folder's listing up to this size; past it the listing is a file the knot// stores once per repo, which every page of the folder names, as they all draw the same listexport const LISTING_INLINE_BYTES = 32 * 1024;
const LISTINGS_ATTRIBUTE = "data-folder-listings";
// each folder's entries, or the url of the shared file holding themexport type ListingSources = Record<string, TreeEntrySummary[] | string>;
export interface PageListings { listings: FolderListings; sources: ListingSources;}
const shippedSources = (root: ParentNode = document): ListingSources | null => { const script = root.querySelector(`script[${LISTINGS_ATTRIBUTE}]`); return script ? (JSON.parse(script.textContent ?? "") as ListingSources) : null;};
/** the folder listings a page draws. a knot's render reads them without kit inlining the payload and ships them trimmed, in the page or as shared files; the browser's first load takes them from there */export const pageListings = async ( parent: RepoParent, ref: string, folders: string[], fetch: Fetch, base: string): Promise<PageListings> => { const knot = prerender(); if (knot) { const listings = await loadFolderListings(parent, ref, folders, unshipped(fetch)); const sources: ListingSources = {}; for (const [folder, entries] of Object.entries(listings)) { sources[folder] = JSON.stringify(entries).length > LISTING_INLINE_BYTES ? `${base}/${knot.share(entries)}` : entries; } return { listings, sources }; } const shipped = browser && servesRenderedPage() ? shippedSources() : null; if (shipped) { try { const listed = await Promise.all( Object.entries(shipped).map(async ([folder, source]) => { if (typeof source !== "string") return [folder, source] as const; const response = await globalThis.fetch(source); if (!response.ok) throw new Error(`${source}: ${response.status}`); return [folder, (await response.json()) as TreeEntrySummary[]] as const; }) ); return { listings: Object.fromEntries(listed), sources: shipped }; } catch { // a page outlived its shared file: list the folders again, under the same script return { listings: await loadFolderListings(parent, ref, folders, fetch), sources: shipped }; } } return { listings: await loadFolderListings(parent, ref, folders, fetch), sources: {} };};
/** the script a page carries its listings, or their urls, in */export const listingsScript = (sources: ListingSources): string | null => Object.keys(sources).length === 0 ? null : `<script type="application/json" ${LISTINGS_ATTRIBUTE}>${JSON.stringify(sources).replaceAll("<", "\\u003c")}</script>`;
export const loadCompact = async ( parent: RepoParent, ref: string, files: TreeEntrySummary[],63 unmodified lines5 unmodified lines678910111213141516171819202115 unmodified lines373839404142434445464748495051107 unmodified linesExpand all159160161162163164165166167168169170171172173174175176177136 unmodified linesExpand all5 unmodified lines import { sortTreeEntries, toTreeEntrySummary } from "#lib/api/repo.ts"; import { findCompact, folderChain, listingsScript, SIDEBAR_BRANCHES, SIDEBAR_TAGS, TREE_DEPTH, treeRoot, type Compact, type FolderListings, type ListingSources, type RepoSidebar } from "#lib/api/repoSidebar.ts"; import type { TreeEntrySummary } from "#lib/components/repo/types.ts"; import LanguageWaffle from "#lib/components/repo/sidebar/LanguageWaffle.svelte";15 unmodified lines
let { repo, sidebar, currentPath }: Props = $props();
const listings = $derived((await page.data.listings) as FolderListings | undefined); const listingSources = $derived( ((await page.data.listingSources) as ListingSources | undefined) ?? {} ); const sharedListings = $derived( Object.values(listingSources).filter((source) => typeof source === "string") ); const shippedListings = $derived(listingsScript(listingSources));
// a file page starts from the folder it sits in const folder = $derived( page.route.id?.includes("/blob/")107 unmodified linesExpand all<svelte:head> {#if sidebar?.src} <link rel="preload" href={sidebar.src} as="fetch" crossorigin="anonymous" /> {/if} {#each sharedListings as src (src)} <link rel="preload" href={src} as="fetch" crossorigin="anonymous" /> {/each}</svelte:head>
{#if sidebar} <div class="sb" data-sidebar-src={sidebar.src}> {#if shippedListings} <!-- eslint-disable-next-line svelte/no-at-html-tags -- json of our own listings, its < escaped --> {@html shippedListings} {/if} <Window class="sb-files" bodyClass="sb-tree"> {#snippet header()} <header class="bar"> <div class="y-rb">136 unmodified linesExpand all1234456786 unmodified lines1516171819202122232425262728293031322324252627282930313233343536import { error } from "@sveltejs/kit";import { loadRepoBlob } from "#lib/api/blobLoad.ts";import { early, settle, stream } from "#lib/api/load.ts";import { folderChain, loadFolderListings, rootedChain, treeRoot } from "#lib/api/repoSidebar.ts";import { folderChain, pageListings, rootedChain, treeRoot } from "#lib/api/repoSidebar.ts";import type { PageLoad } from "./$types";
// the ref is one encoded segment, which keeps `feature/x` wholeexport const load: PageLoad = async (event) => {6 unmodified lines loadRepoBlob(event, parent, event.params.ref, event.params.path, event.fetch) ); await early(blob);
const listed = Promise.resolve(parent.sidebar).then((sidebar) => pageListings( parent, event.params.ref, rootedChain(folder, treeRoot(folder, sidebar?.compact)), event.fetch, `/${event.params.handle}/${event.params.repo}` ) );
return settle({ ref: event.params.ref, path: event.params.path, blob, listings: stream( Promise.resolve(parent.sidebar).then((sidebar) => loadFolderListings( parent, event.params.ref, rootedChain(folder, treeRoot(folder, sidebar?.compact)), event.fetch ) ) ) listings: stream(listed.then(({ listings }) => listings)), listingSources: stream(listed.then(({ sources }) => sources)) });};123345674 unmodified lines121314151616171819202122232425202627282930312 unmodified linesimport { settle, stream } from "#lib/api/load.ts";import { loadRepoTree } from "#lib/api/repoLoad.ts";import { loadFolderListings, rootedChain, treeRoot } from "#lib/api/repoSidebar.ts";import { pageListings, rootedChain, treeRoot } from "#lib/api/repoSidebar.ts";import type { PageLoad } from "./$types";
// the rest param is already decoded, decoding it again breaks a name with a %export const load: PageLoad = async (event) => {4 unmodified lines const { files } = await tree.tree; // path's own listing is the tree read's files, and the chain leaves it out const ancestors = Promise.resolve(parent.sidebar).then((sidebar) => { const chain = rootedChain(path, treeRoot(path, sidebar?.compact)).slice(0, -1); return loadFolderListings(parent, ref, chain, event.fetch); return pageListings( parent, ref, chain, event.fetch, `/${event.params.handle}/${event.params.repo}` ); }); return settle({ ...tree, listings: stream(ancestors.then((listed) => ({ ...listed, [path]: files }))), listings: stream(ancestors.then(({ listings }) => ({ ...listings, [path]: files }))), listingSources: stream(ancestors.then(({ sources }) => sources)), meta: { title: `${path} · ${event.params.repo} · Tangled`, description: `Viewing ${path} in ${event.params.repo} on Tangled.` }2 unmodified lines