save each repo's and owner's render as it finishes, so a superseded or killed run's work counts for the next
Signed-off-by: dawn <90008@klbr.net>
66 unmodified lines67686970717273747576777879808182838485728687888975909192939495969798848586878889909199100101102961031041051061071081091101021111121131141151161171181191111201211221231241251261181271281291301311231321331251341351361371385 unmodified lines1441451461471391481491501511521531541461471551561571581591601531611621631641652 unmodified lines16816917017117217317417517617719 unmodified lines197198199200191201202203204195205206207208209210211202212213214205215216217218209219220221222223214215216217218219220221224225226227102 unmodified linesExpand all66 unmodified lines statusWrite = statusWrite.catch(() => {}).then(() => atomicJson(statusPath, snapshot)); return statusWrite; };
// what each repo's and owner's pages were rendered at, saved as each one finishes, so the // work of a run that is superseded or killed counts for the next. an entry carries the // formats and build it was rendered with: one rendered by another build renders again const stamp = `${hook.format}/${RENDERER_FORMAT}/${build}`; const current = (entry) => (!opts.full && entry?.stamp === stamp ? entry : undefined); const nextRepos = {}; const nextOwners = {}; let knotWrite = Promise.resolve(); const saveKnot = () => { const snapshot = JSON.parse(JSON.stringify({ generation: hook.generation, repos: nextRepos, owners: nextOwners })); knotWrite = knotWrite.catch(() => {}).then(() => atomicJson(path.join(stateDir, "knot.json"), snapshot)); return knotWrite; };
const children = new Set(); // the knot stops a renderer it supersedes with SIGTERM to the process group // the knot stops a renderer it supersedes with SIGTERM to the process group: the repo in // flight is dropped (its next render clears the half-written copy), the finished ones stay process.on("SIGTERM", async () => { for (const child of children) child.kill("SIGTERM"); await writeStatus({ state: "superseded" }).catch(() => {}); await Promise.all([knotWrite, writeStatus({ state: "superseded" })]).catch(() => {}); process.exit(3); });
await writeStatus(); for (const did of hook.removed_repos ?? []) await removeRepo(out, did, loadState(did));
// a page is current if every read it made is unchanged since the generation it was // rendered at, so each repo walks the knot's diffs back to its own generation const full = opts.full || !knotState || knotState.knot_format !== hook.format || knotState.renderer_format !== RENDERER_FORMAT || knotState.build !== build; const knownRepos = (!full && knotState?.repos) || {}; const nextRepos = {}; const plans = []; let diffEntries = 0; for (const repo of hook.repos) { const prefix = prefixOf(repo); const summary = knownRepos[repo.did]; // until it renders again, a repo keeps the entry it had, whatever stamp it carries if (knotState?.repos?.[repo.did]) nextRepos[repo.did] = knotState.repos[repo.did]; const summary = current(knotState?.repos?.[repo.did]); // a repo whose render failed last time keeps pages of an older run, maybe of another build if (summary?.failed) { plans.push({ repo, work: "all", weight: summary.pages ?? Infinity }); continue; } if (!full && summary && repo.change === "unchanged" && summary.prefix === prefix) { if (summary && repo.change === "unchanged" && summary.prefix === prefix) { if (summary.generation === hook.generation) { nextRepos[repo.did] = summary; continue; } const diffs = await readDiff(hookPath, hook, summary.generation); if (diffs !== null) { diffEntries = Math.max(diffEntries, diffs.entries); if (!diffs.global && !diffs.repos.has(repo.did)) { nextRepos[repo.did] = { generation: hook.generation, prefix, pages: summary.pages }; nextRepos[repo.did] = { stamp, generation: hook.generation, prefix, pages: summary.pages }; status.repos[repo.did] = { skipped: true }; continue; } } } const prior = loadState(repo.did); const fresh = full || !prior || repo.change === "added" || prior.prefix !== prefix; const fresh = !summary || !prior || repo.change === "added" || prior.prefix !== prefix; if (fresh) { plans.push({ repo, work: "all", weight: prior ? Object.keys(prior.pages).length : (summary?.pages ?? Infinity) }); continue; } const since = summary?.generation ?? prior.generation; const since = summary.generation; if (since === hook.generation) { nextRepos[repo.did] = { generation: hook.generation, prefix, pages: Object.keys(prior.pages).length }; nextRepos[repo.did] = { stamp, generation: hook.generation, prefix, pages: Object.keys(prior.pages).length }; continue; } const diffs = await readDiff(hookPath, hook, since); if (diffs === null) {5 unmodified lines if (dirty === "all" || dirty.size > 0 || repo.change !== "unchanged" || countsChanged(prior, diffs)) { plans.push({ repo, work: dirty === "all" ? "all" : [...dirty], weight: dirty === "all" ? Object.keys(prior.pages).length : dirty.size }); } else { // nothing it read changed: the pages stand for this generation too nextRepos[repo.did] = { generation: hook.generation, prefix, pages: Object.keys(prior.pages).length }; nextRepos[repo.did] = { stamp, generation: hook.generation, prefix, pages: Object.keys(prior.pages).length }; status.repos[repo.did] = { skipped: true }; } }
// every owner of a repo here gets a user page, rendered again when a read it made changed const owners = new Map(hook.repos.map((repo) => [repo.owner_did, { did: repo.owner_did, prefix: ownerPrefixOf(repo) }])); const knownOwners = (!full && knotState?.owners) || {}; const nextOwners = {}; // one with no repo left here, or under a new handle, loses the page at the old url for (const [did, known] of Object.entries(knotState?.owners ?? {})) { if (owners.get(did)?.prefix !== known.prefix) await unpublishOwner(out, known.prefix); else nextOwners[did] = known; } for (const owner of owners.values()) { const known = knownOwners[owner.did]; const known = current(knotState?.owners?.[owner.did]); if (known && !known.failed && known.prefix === owner.prefix) { const diffs = await readDiff(hookPath, hook, known.generation); if (diffs !== null && !stale(known.deps, diffs)) { nextOwners[owner.did] = { ...known, generation: hook.generation };2 unmodified lines } plans.push({ owner, weight: 0 }); }
await saveKnot();
// biggest first, so the last worker to finish isn't the one holding ghostty plans.sort((a, b) => b.weight - a.weight);
const workers = Math.min(Number(opts.workers), plans.length);19 unmodified lines if (plan.owner) { const { did, prefix } = plan.owner; if (result.error) { failures.push(did); nextOwners[did] = { failed: true, prefix }; nextOwners[did] = { stamp, failed: true, prefix }; status.owners[did] = { error: result.error }; } else { const { deps, ...stats } = result; nextOwners[did] = { generation: hook.generation, prefix, deps }; nextOwners[did] = { stamp, generation: hook.generation, prefix, deps }; status.owners[did] = { ...stats, wall_ms: wallMs }; for (const key of Object.keys(totals)) totals[key] += stats[key] ?? 0; } } else if (result.error) { failures.push(plan.repo.did); // its site link still names the pages of its last finished render nextRepos[plan.repo.did] = { failed: true, prefix: prefixOf(plan.repo), pages: knownRepos[plan.repo.did]?.pages }; nextRepos[plan.repo.did] = { stamp, failed: true, prefix: prefixOf(plan.repo), pages: nextRepos[plan.repo.did]?.pages }; status.repos[plan.repo.did] = { error: result.error }; } else { nextRepos[plan.repo.did] = { generation: hook.generation, prefix: prefixOf(plan.repo), pages: result.pages }; nextRepos[plan.repo.did] = { stamp, generation: hook.generation, prefix: prefixOf(plan.repo), pages: result.pages }; status.repos[plan.repo.did] = { ...result, wall_ms: wallMs }; for (const key of Object.keys(totals)) totals[key] += result[key] ?? 0; } await writeStatus(); await Promise.all([saveKnot(), writeStatus()]); }) ) );
await atomicJson(path.join(stateDir, "knot.json"), { knot_format: hook.format, renderer_format: RENDERER_FORMAT, build, generation: hook.generation, repos: nextRepos, owners: nextOwners }); const summary = { state: failures.length === 0 ? "done" : "partial", failed: failures, finished_at: new Date().toISOString(),102 unmodified linesExpand all