#!/usr/bin/env node
import { createReadStream } from "node:fs";
import { promises as fs } from "node:fs";
import { homedir } from "node:os";
import path from "node:path";
import { createInterface } from "node:readline";
const DEFAULT_SESSIONS_DIR = path.join(homedir(), ".pi/agent/sessions");
const DEFAULT_ACTIVE_EDIT_EXTENSION_PATH = path.join(homedir(), ".pi/agent/extensions/edit.ts");
const DEFAULT_TOP = 20;
function parseArgs(argv) {
const options = {
sessionsDir: DEFAULT_SESSIONS_DIR,
json: false,
includeRecords: false,
failedOnly: false,
modelFilter: undefined,
extFilter: undefined,
top: DEFAULT_TOP,
help: false,
allSessions: false,
since: undefined,
autoSincePath: DEFAULT_ACTIVE_EDIT_EXTENSION_PATH,
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (arg === "--help" || arg === "-h") {
options.help = true;
} else if (arg === "--json") {
options.json = true;
} else if (arg === "--include-records") {
options.includeRecords = true;
} else if (arg === "--failed-only") {
options.failedOnly = true;
} else if (arg === "--model") {
options.modelFilter = argv[++i];
} else if (arg === "--ext") {
options.extFilter = argv[++i]?.toLowerCase();
} else if (arg === "--top") {
const value = Number.parseInt(argv[++i] ?? "", 10);
if (!Number.isFinite(value) || value <= 0) {
throw new Error("--top must be a positive integer");
}
options.top = value;
} else if (arg === "--sessions-dir") {
options.sessionsDir = argv[++i];
} else if (arg === "--all-sessions") {
options.allSessions = true;
} else if (arg === "--since") {
options.since = argv[++i];
} else if (arg === "--auto-since-path") {
options.autoSincePath = argv[++i];
} else {
throw new Error(`Unknown argument: ${arg}`);
}
}
return options;
}
function printHelp() {
console.log(`Usage: node scripts/edit-tool-stats.mjs [options]
Options:
--sessions-dir <path> Sessions directory (default: ~/.pi/agent/sessions)
--model <substring> Filter provider/model by substring
--ext <extension> Filter by file extension, e.g. .ts
--failed-only Include only failed edit calls
--top <n> Number of examples to show (default: ${DEFAULT_TOP})
--since <iso> Only scan session files created at or after this ISO time
--all-sessions Disable the automatic since filter
--auto-since-path <p> Use birth time of this file for the automatic since filter
--json Print JSON summary instead of human report
--include-records Include raw records in JSON output
-h, --help Show this help
`);
}
function parseSessionFileTimestamp(sessionFile) {
const base = path.basename(sessionFile);
const rawTimestamp = base.split("_")[0];
if (!rawTimestamp) return null;
const isoTimestamp = rawTimestamp.replace(/T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "T$1:$2:$3.$4Z");
const ms = Date.parse(isoTimestamp);
if (!Number.isFinite(ms)) return null;
return ms;
}
function formatIso(ms) {
return new Date(ms).toISOString();
}
async function resolveAutoSinceMs(options) {
if (options.allSessions) return null;
if (options.since) {
const ms = Date.parse(options.since);
if (!Number.isFinite(ms)) {
throw new Error(`Invalid --since value: ${options.since}`);
}
return { ms, source: `--since ${options.since}` };
}
if (!options.autoSincePath) return null;
try {
const stats = await fs.stat(options.autoSincePath);
const birthtimeMs = Number.isFinite(stats.birthtimeMs) && stats.birthtimeMs > 0 ? stats.birthtimeMs : stats.mtimeMs;
if (!Number.isFinite(birthtimeMs) || birthtimeMs <= 0) return null;
return { ms: birthtimeMs, source: `birth time of ${options.autoSincePath}` };
} catch {
return null;
}
}
async function* walkJsonlFiles(dir) {
const entries = await fs.readdir(dir, { withFileTypes: true });
entries.sort((a, b) => a.name.localeCompare(b.name));
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
yield* walkJsonlFiles(fullPath);
} else if (entry.isFile() && entry.name.endsWith(".jsonl")) {
yield fullPath;
}
}
}
function getPathExtension(filePath) {
if (typeof filePath !== "string" || filePath.length === 0) return "[unknown]";
const ext = path.extname(filePath).toLowerCase();
if (ext) return ext;
const base = path.basename(filePath);
if (base.startsWith(".") && !base.slice(1).includes(".")) return base.toLowerCase();
return "[no_ext]";
}
function utf8Bytes(value) {
return Buffer.byteLength(value ?? "", "utf8");
}
function longestCommonPrefixLength(a, b) {
const max = Math.min(a.length, b.length);
let index = 0;
while (index < max && a[index] === b[index]) index++;
return index;
}
function longestCommonSuffixLength(a, b) {
const max = Math.min(a.length, b.length);
let index = 0;
while (index < max && a[a.length - 1 - index] === b[b.length - 1 - index]) index++;
return index;
}
function analyzeReplacement(oldText, newText) {
const prefixChars = longestCommonPrefixLength(oldText, newText);
const oldRemainder = oldText.slice(prefixChars);
const newRemainder = newText.slice(prefixChars);
const suffixChars = longestCommonSuffixLength(oldRemainder, newRemainder);
const oldCore = suffixChars > 0 ? oldRemainder.slice(0, -suffixChars) : oldRemainder;
const newCore = suffixChars > 0 ? newRemainder.slice(0, -suffixChars) : newRemainder;
const prefix = oldText.slice(0, prefixChars);
const suffix = suffixChars > 0 ? oldRemainder.slice(-suffixChars) : "";
const oldBytes = utf8Bytes(oldText);
const newBytes = utf8Bytes(newText);
const sharedPrefixBytes = utf8Bytes(prefix);
const sharedSuffixBytes = utf8Bytes(suffix);
const sharedContextBytes = sharedPrefixBytes + sharedSuffixBytes;
const coreOldBytes = utf8Bytes(oldCore);
const coreNewBytes = utf8Bytes(newCore);
const coreBytes = coreOldBytes + coreNewBytes;
const totalEditBytes = oldBytes + newBytes;
const wrapperPayloadBytes = totalEditBytes - coreBytes;
const inflationRatio = coreBytes === 0 ? null : totalEditBytes / coreBytes;
return {
oldBytes,
newBytes,
totalEditBytes,
sharedPrefixBytes,
sharedSuffixBytes,
sharedContextBytes,
coreOldBytes,
coreNewBytes,
coreBytes,
wrapperPayloadBytes,
inflationRatio,
noCoreChange: coreBytes === 0,
};
}
function median(numbers) {
return quantile(numbers, 0.5);
}
function quantile(numbers, q) {
const finite = numbers.filter((value) => Number.isFinite(value)).sort((a, b) => a - b);
if (finite.length === 0) return null;
if (finite.length === 1) return finite[0];
const position = (finite.length - 1) * q;
const lower = Math.floor(position);
const upper = Math.ceil(position);
if (lower === upper) return finite[lower];
const weight = position - lower;
return finite[lower] * (1 - weight) + finite[upper] * weight;
}
function formatInt(value) {
return new Intl.NumberFormat("en-US").format(value);
}
function formatPercent(part, total) {
if (total === 0) return "n/a";
return `${((part / total) * 100).toFixed(1)}%`;
}
function formatRatio(value) {
if (value === null) return "no-core-change";
if (!Number.isFinite(value)) return "∞";
if (value >= 100) return `${value.toFixed(0)}x`;
if (value >= 10) return `${value.toFixed(1)}x`;
return `${value.toFixed(2)}x`;
}
function formatBytes(value) {
if (value < 1024) return `${value}B`;
if (value < 1024 * 1024) return `${(value / 1024).toFixed(value >= 10 * 1024 ? 0 : 1)}KB`;
return `${(value / (1024 * 1024)).toFixed(1)}MB`;
}
function extractTextContent(content) {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
return content
.filter((block) => block?.type === "text" && typeof block.text === "string")
.map((block) => block.text)
.join("\n");
}
function classifyErrorKind(text, isError, matchedResult) {
if (!matchedResult) return "missing_result";
if (!isError) return null;
const normalized = text.toLowerCase();
if (normalized.includes("file not found")) return "file_not_found";
if (normalized.includes("could not find the exact text")) return "not_found_exact_text";
if (normalized.includes("found multiple occurrences") || /^found \d+ occurrences/m.test(normalized)) {
return "multiple_occurrences";
}
if (normalized.includes("no changes made")) return "no_changes_made";
if (normalized.includes("input is invalid")) return "invalid_input";
if (normalized.includes("must not overlap")) return "overlapping_edits";
if (normalized.includes("aborted")) return "aborted";
return "other";
}
function getArgStyle(args) {
const hasEdits = Array.isArray(args?.edits);
const hasOldText = typeof args?.oldText === "string" || typeof args?.newText === "string";
const hasOldString = typeof args?.old_string === "string" || typeof args?.new_string === "string";
if (hasEdits) return "edits";
if (hasOldText && hasOldString) return "mixed";
if (hasOldText) return "oldText/newText";
if (hasOldString) return "old_string/new_string";
return "unknown";
}
function analyzeToolArguments(args) {
const normalizedArgs = args && typeof args === "object" ? args : {};
const filePath = typeof normalizedArgs.path === "string" ? normalizedArgs.path : "";
const extension = getPathExtension(filePath);
const argStyle = getArgStyle(normalizedArgs);
if (Array.isArray(normalizedArgs.edits)) {
const perEdit = normalizedArgs.edits.map((edit) =>
analyzeReplacement(typeof edit?.oldText === "string" ? edit.oldText : "", typeof edit?.newText === "string" ? edit.newText : "")
);
const inflations = perEdit.map((edit) => edit.inflationRatio).filter((value) => value !== null);
const totals = perEdit.reduce(
(acc, edit) => ({
oldBytes: acc.oldBytes + edit.oldBytes,
newBytes: acc.newBytes + edit.newBytes,
totalEditBytes: acc.totalEditBytes + edit.totalEditBytes,
sharedPrefixBytes: acc.sharedPrefixBytes + edit.sharedPrefixBytes,
sharedSuffixBytes: acc.sharedSuffixBytes + edit.sharedSuffixBytes,
sharedContextBytes: acc.sharedContextBytes + edit.sharedContextBytes,
coreOldBytes: acc.coreOldBytes + edit.coreOldBytes,
coreNewBytes: acc.coreNewBytes + edit.coreNewBytes,
coreBytes: acc.coreBytes + edit.coreBytes,
wrapperPayloadBytes: acc.wrapperPayloadBytes + edit.wrapperPayloadBytes,
noCoreChangeCount: acc.noCoreChangeCount + (edit.noCoreChange ? 1 : 0),
}),
{
oldBytes: 0,
newBytes: 0,
totalEditBytes: 0,
sharedPrefixBytes: 0,
sharedSuffixBytes: 0,
sharedContextBytes: 0,
coreOldBytes: 0,
coreNewBytes: 0,
coreBytes: 0,
wrapperPayloadBytes: 0,
noCoreChangeCount: 0,
}
);
return {
path: filePath,
extension,
mode: "multi",
argStyle,
editsCount: normalizedArgs.edits.length,
...totals,
inflationRatio: totals.coreBytes === 0 ? null : totals.totalEditBytes / totals.coreBytes,
medianEditInflationRatio: median(inflations),
maxEditInflationRatio: inflations.length > 0 ? Math.max(...inflations) : null,
perEdit,
};
}
const oldText = typeof normalizedArgs.oldText === "string" ? normalizedArgs.oldText : typeof normalizedArgs.old_string === "string" ? normalizedArgs.old_string : "";
const newText = typeof normalizedArgs.newText === "string" ? normalizedArgs.newText : typeof normalizedArgs.new_string === "string" ? normalizedArgs.new_string : "";
const replacement = analyzeReplacement(oldText, newText);
return {
path: filePath,
extension,
mode: "single",
argStyle,
editsCount: 1,
...replacement,
medianEditInflationRatio: replacement.inflationRatio,
maxEditInflationRatio: replacement.inflationRatio,
perEdit: [replacement],
};
}
function groupCounts(records, keyFn) {
const counts = new Map();
for (const record of records) {
const key = keyFn(record);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
return counts;
}
function collectInflations(records) {
return records.map((record) => record.inflationRatio).filter((value) => value !== null);
}
function summarizeGroups(records, keyFn) {
const groups = new Map();
for (const record of records) {
const key = keyFn(record);
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(record);
}
return [...groups.entries()]
.map(([key, group]) => {
const resolved = group.filter((record) => record.success !== null);
const success = resolved.filter((record) => record.success).length;
const failed = resolved.filter((record) => record.success === false).length;
const multi = group.filter((record) => record.mode === "multi").length;
const inflations = collectInflations(group);
return {
key,
calls: group.length,
resolved: resolved.length,
success,
failed,
unresolved: group.length - resolved.length,
multi,
multiRate: group.length === 0 ? null : multi / group.length,
successRate: resolved.length === 0 ? null : success / resolved.length,
medianInflation: quantile(inflations, 0.5),
p95Inflation: quantile(inflations, 0.95),
};
})
.sort((a, b) => b.calls - a.calls || a.key.localeCompare(b.key));
}
function buildSameFileClusterStats(records) {
const groups = new Map();
for (const record of records) {
const key = `${record.sessionFile}::${record.assistantEntryId}::${record.path}`;
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(record);
}
const clusters = [...groups.values()].filter((group) => group.length >= 2);
const assistantMessagesWithCluster = new Set(clusters.map((group) => `${group[0].sessionFile}::${group[0].assistantEntryId}`));
const assistantMessagesWithMultiEdit = new Set(
records
.filter((record) => record.mode === "multi" && record.editsCount > 1)
.map((record) => `${record.sessionFile}::${record.assistantEntryId}`)
);
const callsInsideClusters = clusters.reduce((sum, group) => sum + group.length, 0);
return {
clustersCount: clusters.length,
assistantMessagesWithCluster: assistantMessagesWithCluster.size,
assistantMessagesWithMultiEdit: assistantMessagesWithMultiEdit.size,
callsInsideClusters,
averageCallsPerCluster: clusters.length === 0 ? null : callsInsideClusters / clusters.length,
ratioClusterAssistantMessagesToMultiEditAssistantMessages:
assistantMessagesWithMultiEdit.size === 0 ? null : assistantMessagesWithCluster.size / assistantMessagesWithMultiEdit.size,
};
}
function buildInflationBuckets(records) {
const buckets = [
{ key: "no_core_change", label: "no-core-change", test: (record) => record.inflationRatio === null },
{ key: "lt4", label: "<4x", test: (record) => record.inflationRatio !== null && record.inflationRatio < 4 },
{ key: "4to10", label: "4-10x", test: (record) => record.inflationRatio !== null && record.inflationRatio >= 4 && record.inflationRatio < 10 },
{ key: "10to25", label: "10-25x", test: (record) => record.inflationRatio !== null && record.inflationRatio >= 10 && record.inflationRatio < 25 },
{ key: "gte25", label: "25x+", test: (record) => record.inflationRatio !== null && record.inflationRatio >= 25 },
];
return buckets.map((bucket) => {
const bucketRecords = records.filter(bucket.test);
const resolved = bucketRecords.filter((record) => record.success !== null);
const failed = resolved.filter((record) => record.success === false).length;
return {
key: bucket.key,
label: bucket.label,
count: bucketRecords.length,
resolved: resolved.length,
failed,
failureRate: resolved.length === 0 ? null : failed / resolved.length,
};
});
}
function buildHugeReplacementStats(records) {
const thresholds = [1024, 4096, 16384, 65536];
return thresholds.map((threshold) => ({
threshold,
count: records.filter((record) => record.totalEditBytes > threshold).length,
}));
}
function buildWorstExamples(records, top) {
const scored = [...records].sort((a, b) => {
const aScore = a.inflationRatio === null ? Number.POSITIVE_INFINITY : a.inflationRatio;
const bScore = b.inflationRatio === null ? Number.POSITIVE_INFINITY : b.inflationRatio;
if (aScore !== bScore) return bScore - aScore;
if (a.totalEditBytes !== b.totalEditBytes) return b.totalEditBytes - a.totalEditBytes;
return a.path.localeCompare(b.path);
});
return scored.slice(0, top).map((record) => ({
providerModel: record.providerModel,
extension: record.extension,
path: record.path,
inflationRatio: record.inflationRatio,
totalEditBytes: record.totalEditBytes,
coreBytes: record.coreBytes,
mode: record.mode,
editsCount: record.editsCount,
failed: record.success === false,
errorKind: record.errorKind,
sessionFile: record.sessionFile,
}));
}
function buildSummary(records, meta, options) {
const uniqueAssistantMessages = new Set(records.map((record) => `${record.sessionFile}::${record.assistantEntryId}`)).size;
const resolved = records.filter((record) => record.success !== null);
const success = resolved.filter((record) => record.success).length;
const failed = resolved.filter((record) => record.success === false).length;
const unresolved = records.length - resolved.length;
const single = records.filter((record) => record.mode === "single").length;
const multi = records.filter((record) => record.mode === "multi").length;
const modeStats = ["single", "multi"].map((mode) => {
const modeRecords = records.filter((record) => record.mode === mode);
const modeResolved = modeRecords.filter((record) => record.success !== null);
const modeSuccess = modeResolved.filter((record) => record.success).length;
const modeFailed = modeResolved.filter((record) => record.success === false).length;
return {
mode,
calls: modeRecords.length,
resolved: modeResolved.length,
success: modeSuccess,
failed: modeFailed,
unresolved: modeRecords.length - modeResolved.length,
successRate: modeResolved.length === 0 ? null : modeSuccess / modeResolved.length,
failureRate: modeResolved.length === 0 ? null : modeFailed / modeResolved.length,
};
});
const multiEditLengthBuckets = [
{ key: "1", label: "edits.length === 1", test: (record) => record.mode === "multi" && record.editsCount === 1 },
{ key: "2", label: "edits.length === 2", test: (record) => record.mode === "multi" && record.editsCount === 2 },
{ key: "3plus", label: "edits.length >= 3", test: (record) => record.mode === "multi" && record.editsCount >= 3 },
].map((bucket) => {
const bucketRecords = records.filter(bucket.test);
const bucketResolved = bucketRecords.filter((record) => record.success !== null);
const bucketSuccess = bucketResolved.filter((record) => record.success).length;
const bucketFailed = bucketResolved.filter((record) => record.success === false).length;
return {
key: bucket.key,
label: bucket.label,
calls: bucketRecords.length,
resolved: bucketResolved.length,
success: bucketSuccess,
failed: bucketFailed,
unresolved: bucketRecords.length - bucketResolved.length,
successRate: bucketResolved.length === 0 ? null : bucketSuccess / bucketResolved.length,
failureRate: bucketResolved.length === 0 ? null : bucketFailed / bucketResolved.length,
};
});
const argStyles = [...groupCounts(records, (record) => record.argStyle).entries()]
.map(([style, count]) => ({ style, count }))
.sort((a, b) => b.count - a.count || a.style.localeCompare(b.style));
const providerStats = summarizeGroups(records, (record) => record.providerModel);
const extensionStats = summarizeGroups(records, (record) => record.extension);
const inflations = collectInflations(records);
const noCoreChange = records.filter((record) => record.inflationRatio === null).length;
const pathologicalThresholds = [10, 25, 100].map((threshold) => ({
threshold,
count: records.filter((record) => record.inflationRatio !== null && record.inflationRatio >= threshold).length,
}));
const failureKinds = [...groupCounts(records.filter((record) => record.success === false), (record) => record.errorKind ?? "other").entries()]
.map(([kind, count]) => ({ kind, count }))
.sort((a, b) => b.count - a.count || a.kind.localeCompare(b.kind));
return {
filters: {
model: options.modelFilter ?? null,
extension: options.extFilter ?? null,
failedOnly: options.failedOnly,
},
scan: {
sessionsDir: meta.sessionsDir,
sessionFilesScanned: meta.sessionFilesScanned,
sessionFilesIncluded: meta.sessionFilesIncluded,
sessionFilesSkippedOlderThanSince: meta.sessionFilesSkippedOlderThanSince,
sessionFilesWithEditCalls: meta.sessionFilesWithEditCalls,
since: meta.since ? { ms: meta.since.ms, iso: formatIso(meta.since.ms), source: meta.since.source } : null,
malformedLines: meta.malformedLines,
unmatchedToolResults: meta.unmatchedToolResults,
},
counts: {
assistantMessagesWithEditCalls: uniqueAssistantMessages,
totalEditCalls: records.length,
resolvedEditCalls: resolved.length,
success,
failed,
unresolved,
single,
multi,
noCoreChange,
},
modeStats,
multiEditLengthBuckets,
argStyles,
providerStats,
extensionStats,
inflation: {
median: quantile(inflations, 0.5),
p90: quantile(inflations, 0.9),
p95: quantile(inflations, 0.95),
p99: quantile(inflations, 0.99),
pathologicalThresholds,
hugeReplacements: buildHugeReplacementStats(records),
failureByBucket: buildInflationBuckets(records),
},
sameFileClusters: buildSameFileClusterStats(records),
failureKinds,
worstExamples: buildWorstExamples(records, options.top),
};
}
function printGroupTable(title, groups, formatter) {
if (groups.length === 0) return;
console.log(`\n${title}`);
for (const group of groups) {
console.log(formatter(group));
}
}
function printHumanReport(summary) {
const { scan, counts, modeStats, multiEditLengthBuckets, argStyles, providerStats, extensionStats, inflation, sameFileClusters, failureKinds, worstExamples, filters } = summary;
console.log(`Scanned ${formatInt(scan.sessionFilesIncluded)} session files in ${scan.sessionsDir}`);
if (scan.since) {
console.log(`Session filter: files created at or after ${scan.since.iso} (${scan.since.source})`);
console.log(`Skipped older session files: ${formatInt(scan.sessionFilesSkippedOlderThanSince)} of ${formatInt(scan.sessionFilesScanned)}`);
}
console.log(`Found ${formatInt(counts.totalEditCalls)} edit tool calls in ${formatInt(counts.assistantMessagesWithEditCalls)} assistant messages`);
if (filters.model || filters.extension || filters.failedOnly) {
const filterParts = [];
if (filters.model) filterParts.push(`model contains \"${filters.model}\"`);
if (filters.extension) filterParts.push(`extension = ${filters.extension}`);
if (filters.failedOnly) filterParts.push("failed only");
console.log(`Filters: ${filterParts.join(", ")}`);
}
console.log("\nSuccess rate");
console.log(` success: ${formatInt(counts.success)} ${formatPercent(counts.success, counts.resolvedEditCalls)}`);
console.log(` failed: ${formatInt(counts.failed)} ${formatPercent(counts.failed, counts.resolvedEditCalls)}`);
console.log(` unresolved: ${formatInt(counts.unresolved)}`);
console.log("\nMode usage");
console.log(` single replacement: ${formatInt(counts.single)} ${formatPercent(counts.single, counts.totalEditCalls)}`);
console.log(` multi-edit (edits): ${formatInt(counts.multi)} ${formatPercent(counts.multi, counts.totalEditCalls)}`);
console.log("\nFailures by edit type");
for (const mode of modeStats) {
console.log(
` ${mode.mode.padEnd(6)} calls=${formatInt(mode.calls).padStart(4)} success=${mode.successRate === null ? "n/a" : formatPercent(mode.success, mode.resolved).padStart(6)} failed=${mode.failureRate === null ? "n/a" : formatPercent(mode.failed, mode.resolved).padStart(6)} unresolved=${formatInt(mode.unresolved)}`
);
}
console.log("\nMulti-edit bucket split");
for (const bucket of multiEditLengthBuckets) {
console.log(
` ${bucket.label.padEnd(20)} ${formatInt(bucket.calls).padStart(4)} calls success=${bucket.successRate === null ? "n/a" : formatPercent(bucket.success, bucket.resolved).padStart(6)} failed=${bucket.failureRate === null ? "n/a" : formatPercent(bucket.failed, bucket.resolved).padStart(6)}`
);
}
console.log("\nArgument style");
for (const entry of argStyles) {
console.log(` ${entry.style.padEnd(22)} ${formatInt(entry.count).padStart(8)} ${formatPercent(entry.count, counts.totalEditCalls)}`);
}
printGroupTable("By provider/model", providerStats, (group) => {
return [
` ${group.key}`,
` calls: ${formatInt(group.calls)}`,
` success: ${group.successRate === null ? "n/a" : formatPercent(group.success, group.resolved)}`,
` multi-edit: ${group.multiRate === null ? "n/a" : formatPercent(group.multi, group.calls)}`,
` median inflation: ${formatRatio(group.medianInflation)}`,
` p95 inflation: ${formatRatio(group.p95Inflation)}`,
].join("\n");
});
printGroupTable("By file extension", extensionStats, (group) => {
return ` ${group.key.padEnd(10)} calls=${formatInt(group.calls).padStart(6)} success=${group.successRate === null ? "n/a" : formatPercent(group.success, group.resolved).padStart(6)} medianInflation=${formatRatio(group.medianInflation)}`;
});
console.log("\nContext inflation");
console.log(` median inflation: ${formatRatio(inflation.median)}`);
console.log(` p90 inflation: ${formatRatio(inflation.p90)}`);
console.log(` p95 inflation: ${formatRatio(inflation.p95)}`);
console.log(` p99 inflation: ${formatRatio(inflation.p99)}`);
console.log(` no-core-change: ${formatInt(counts.noCoreChange)}`);
console.log("\nHuge replacements");
for (const entry of inflation.hugeReplacements) {
console.log(` >${formatBytes(entry.threshold).padEnd(6)} ${formatInt(entry.count).padStart(8)}`);
}
console.log("\nPathological wrappers");
for (const entry of inflation.pathologicalThresholds) {
console.log(` inflation >= ${String(entry.threshold).padEnd(3)}x ${formatInt(entry.count).padStart(8)}`);
}
console.log("\nSame-file multi-call behavior");
console.log(` assistant msgs with 2+ edit calls to same file: ${formatInt(sameFileClusters.assistantMessagesWithCluster)}`);
console.log(` total same-file clusters: ${formatInt(sameFileClusters.clustersCount)}`);
console.log(` calls inside those clusters: ${formatInt(sameFileClusters.callsInsideClusters)}`);
console.log(` average calls per cluster: ${sameFileClusters.averageCallsPerCluster === null ? "n/a" : sameFileClusters.averageCallsPerCluster.toFixed(2)}`);
console.log(` assistant msgs using one multi-edit call: ${formatInt(sameFileClusters.assistantMessagesWithMultiEdit)}`);
console.log(` ratio multi-call / multi-edit assistant msgs: ${sameFileClusters.ratioClusterAssistantMessagesToMultiEditAssistantMessages === null ? "n/a" : sameFileClusters.ratioClusterAssistantMessagesToMultiEditAssistantMessages.toFixed(2)}`);
console.log("\nFailures by kind");
if (failureKinds.length === 0) {
console.log(" none");
} else {
for (const failure of failureKinds) {
console.log(` ${failure.kind.padEnd(22)} ${formatInt(failure.count).padStart(8)}`);
}
}
console.log("\nFailure rate by inflation bucket");
for (const bucket of inflation.failureByBucket) {
console.log(` ${bucket.label.padEnd(14)} ${bucket.failureRate === null ? "n/a" : formatPercent(bucket.failed, bucket.resolved).padStart(6)} (${formatInt(bucket.count)} calls)`);
}
console.log(`\nWorst ${formatInt(worstExamples.length)} examples`);
for (let i = 0; i < worstExamples.length; i++) {
const example = worstExamples[i];
console.log(
` ${i + 1}. ${example.providerModel} ${example.extension} inflation=${formatRatio(example.inflationRatio)} failed=${example.failed ? example.errorKind : "false"}`
);
console.log(` path: ${example.path}`);
console.log(` totalEditBytes=${formatBytes(example.totalEditBytes)} coreBytes=${formatBytes(example.coreBytes)} mode=${example.mode} edits=${example.editsCount}`);
}
if (scan.malformedLines > 0 || scan.unmatchedToolResults > 0) {
console.log("\nParser notes");
if (scan.malformedLines > 0) console.log(` malformed lines skipped: ${formatInt(scan.malformedLines)}`);
if (scan.unmatchedToolResults > 0) console.log(` unmatched edit tool results: ${formatInt(scan.unmatchedToolResults)}`);
}
}
async function scanSessions(sessionsDir, since) {
const records = [];
const meta = {
sessionsDir,
sessionFilesScanned: 0,
sessionFilesIncluded: 0,
sessionFilesSkippedOlderThanSince: 0,
sessionFilesWithEditCalls: 0,
since,
malformedLines: 0,
unmatchedToolResults: 0,
};
for await (const sessionFile of walkJsonlFiles(sessionsDir)) {
meta.sessionFilesScanned++;
const sessionTimestampMs = parseSessionFileTimestamp(sessionFile);
if (since && sessionTimestampMs !== null && sessionTimestampMs < since.ms) {
meta.sessionFilesSkippedOlderThanSince++;
continue;
}
meta.sessionFilesIncluded++;
const pending = new Map();
let fileHadEditCall = false;
const input = createReadStream(sessionFile, { encoding: "utf8" });
const rl = createInterface({ input, crlfDelay: Infinity });
for await (const line of rl) {
if (!line.trim()) continue;
let entry;
try {
entry = JSON.parse(line);
} catch {
meta.malformedLines++;
continue;
}
if (entry?.type !== "message" || !entry.message) continue;
const message = entry.message;
if (message.role === "assistant" && Array.isArray(message.content)) {
for (const block of message.content) {
if (block?.type !== "toolCall" || block.name !== "edit") continue;
fileHadEditCall = true;
const analysis = analyzeToolArguments(block.arguments);
const record = {
sessionFile,
assistantEntryId: entry.id,
toolCallId: typeof block.id === "string" ? block.id : "",
timestamp: entry.timestamp,
api: typeof message.api === "string" ? message.api : null,
provider: typeof message.provider === "string" ? message.provider : "[unknown]",
model: typeof message.model === "string" ? message.model : "[unknown]",
providerModel: `${typeof message.provider === "string" ? message.provider : "[unknown]"}/${typeof message.model === "string" ? message.model : "[unknown]"}`,
success: null,
errorKind: null,
errorText: "",
resultSummary: "",
matchedResult: false,
...analysis,
};
records.push(record);
if (record.toolCallId) pending.set(record.toolCallId, record);
}
}
if (message.role === "toolResult" && message.toolName === "edit") {
const toolCallId = typeof message.toolCallId === "string" ? message.toolCallId : "";
const record = pending.get(toolCallId);
if (!record) {
meta.unmatchedToolResults++;
continue;
}
const text = extractTextContent(message.content);
record.matchedResult = true;
record.success = message.isError === true ? false : true;
record.resultSummary = text;
record.errorText = message.isError === true ? text : "";
record.errorKind = classifyErrorKind(text, message.isError === true, true);
pending.delete(toolCallId);
}
}
for (const record of pending.values()) {
record.matchedResult = false;
record.success = null;
record.errorKind = classifyErrorKind("", false, false);
}
if (fileHadEditCall) meta.sessionFilesWithEditCalls++;
}
return { records, meta };
}
function applyFilters(records, options) {
return records.filter((record) => {
if (options.modelFilter && !record.providerModel.toLowerCase().includes(options.modelFilter.toLowerCase())) {
return false;
}
if (options.extFilter && record.extension !== options.extFilter) {
return false;
}
if (options.failedOnly && record.success !== false) {
return false;
}
return true;
});
}
async function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const sessionsDir = path.resolve(options.sessionsDir);
await fs.access(sessionsDir);
const since = await resolveAutoSinceMs(options);
const { records, meta } = await scanSessions(sessionsDir, since);
const filteredRecords = applyFilters(records, options);
const summary = buildSummary(filteredRecords, meta, options);
if (options.json) {
const output = options.includeRecords ? { summary, records: filteredRecords } : { summary };
console.log(JSON.stringify(output, null, 2));
return;
}
printHumanReport(summary);
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});