import { describe, test, expect } from "bun:test";
import { buildRows, groupTurns, computeInputOutcomes } from "./turns";
import type { TimelineItem } from "./streaming-v2";
import type { InputOutcome, WorkItemDiagnostic } from "./protocol-v2";
// Fixtures for the outcomes of a turn: each builder produces the record the runtime would send.
const input = (seq: number, id?: string): TimelineItem => ({
lane: "input",
id: id ?? `in-${seq}`,
seq,
created_ms: seq * 10,
data: { source: "operator", text: `turn ${seq}` },
} as unknown as TimelineItem);
const speech = (seq: number, status: string, reason?: string): TimelineItem => ({
lane: "speech",
id: `sp-${seq}`,
seq,
created_ms: seq * 10,
data: { text: "words", receipt: { status, reason } },
} as unknown as TimelineItem);
const cell = (
seq: number,
{ status, exit_code, stderr }: { status: string; exit_code: number; stderr?: string }
): TimelineItem => ({
lane: "cell",
id: `cell-${seq}`,
seq,
created_ms: seq * 10,
data: {
type: "completed",
execution_id: `exec-${seq}`,
status,
exit_code,
outputs: stderr === undefined ? [{ stream: "stdout", text: "ok\n" }] : [{ stream: "stderr", text: stderr }],
},
} as unknown as TimelineItem);
const lifecycle = (seq: number, kind: string, data?: Record<string, unknown>): TimelineItem => ({
lane: "lifecycle",
id: `life-${seq}`,
seq,
created_ms: seq * 10,
data: { kind, details: kind, reason: data?.reason, data },
} as unknown as TimelineItem);
const workItem = (id: string, status: WorkItemDiagnostic["status"], title = "repair"): WorkItemDiagnostic => ({
id,
title,
purpose: "repair",
status,
origin: "human",
created_ms: 1,
});
/**
* The implementation this replaced, kept here as the reference: evidence gathered by scanning the
* transcript once per input. The rewrite has to answer exactly what this answers.
*/
function referenceOutcomes(items: TimelineItem[], work: WorkItemDiagnostic[]): Map<string, InputOutcome> {
const outcomes = new Map<string, InputOutcome>();
const exactBySeq = new Map<number, InputOutcome>();
for (const item of items) {
if (item.lane !== "lifecycle") continue;
const kind = (item.data as { kind?: string }).kind;
if (kind !== "turn.outcome" && kind !== "turn.failed") continue;
const data = (item.data as { data?: Record<string, unknown> }).data ?? {};
const ids = Array.isArray(data.input_event_ids) ? (data.input_event_ids as number[]) : [];
const status = String(data.status ?? (kind === "turn.failed" ? "failed" : ""));
const reason = String(data.error ?? data.reason ?? (item.data as { reason?: string }).reason ?? "");
const outcome: InputOutcome =
kind === "turn.failed" || status === "failed"
? { status: "failed", reason: reason || "the turn failed" }
: status === "held"
? { status: "held", reason: reason || "held by delivery review" }
: status === "replied"
? { status: "replied" }
: { status: "no_reply", reason: "the turn completed without speech" };
for (const id of ids) exactBySeq.set(id, outcome);
}
const inputs = items.filter((it): it is Extract<TimelineItem, { lane: "input" }> => it.lane === "input");
inputs.forEach((inputItem, index) => {
if (inputItem.seq !== undefined && exactBySeq.has(inputItem.seq)) {
outcomes.set(inputItem.id, exactBySeq.get(inputItem.seq)!);
return;
}
const start = items.findIndex((it) => it.id === inputItem.id);
const nextInput = inputs[index + 1];
const end = nextInput ? items.findIndex((it) => it.id === nextInput.id) : items.length;
const turnItems = items.slice(start + 1, end);
const delivered = turnItems.find((it) => it.lane === "speech" && it.data.receipt?.status === "confirmed");
if (delivered) { outcomes.set(inputItem.id, { status: "replied" }); return; }
const held = turnItems.find((it) => it.lane === "speech" && it.data.receipt?.status === "held");
if (held && held.lane === "speech") {
outcomes.set(inputItem.id, { status: "held", reason: held.data.receipt.reason || "held by delivery review" });
return;
}
const failedCell = turnItems.find((it) => it.lane === "cell" && (it.data.status === "error" || (it.data.exit_code ?? 0) !== 0));
const failedLifecycle = turnItems.find((it) => it.lane === "lifecycle" && String((it.data as { kind?: string }).kind) === "failed");
const failedWork = work?.find((w) => w.status === "failed" && inputItem.seq !== undefined && w.id.includes(`input-${inputItem.seq}`));
if (failedCell || failedLifecycle || failedWork) {
const stderr = failedCell && failedCell.lane === "cell"
? failedCell.data.outputs.find((o) => o.stream === "stderr")?.text
: undefined;
const reason = failedCell && failedCell.lane === "cell"
? stderr?.split("\n").filter(Boolean).slice(-1)[0] || `the cell exited ${failedCell.data.exit_code ?? "non-zero"}`
: failedWork?.title || "the turn failed";
outcomes.set(inputItem.id, { status: "failed", reason });
return;
}
if (index < inputs.length - 1) {
outcomes.set(inputItem.id, { status: "no_reply", reason: "the turn completed without speech" });
return;
}
outcomes.set(inputItem.id, { status: "running" });
});
return outcomes;
}
/** Deterministic corpora: a fixed seed, so a failure is reproducible rather than a flake. */
function seed(seedValue: number) {
let x = seedValue >>> 0;
return () => {
x = (x * 1664525 + 1013904223) >>> 0;
return x / 4294967296;
};
}
function transcript(turns: number, seedValue = 7): { items: TimelineItem[]; work: WorkItemDiagnostic[] } {
const next = seed(seedValue);
const items: TimelineItem[] = [];
const work: WorkItemDiagnostic[] = [];
let seq = 0;
// records before any input: a session that started itself
items.push(lifecycle(++seq, "session.activated"));
items.push(speech(++seq, "confirmed"));
for (let t = 1; t <= turns; t++) {
const inputSeq = ++seq;
items.push(input(inputSeq));
const shape = Math.floor(next() * 8);
if (shape === 0) items.push(speech(++seq, "confirmed"));
else if (shape === 1) items.push(speech(++seq, "held", next() < 0.5 ? "awaiting delivery review" : undefined));
else if (shape === 2) items.push(cell(++seq, { status: "error", exit_code: 101, stderr: "assertion failed\nleft == right" }));
else if (shape === 3) items.push(cell(++seq, { status: "error", exit_code: 0, stderr: "" }));
else if (shape === 4) items.push(cell(++seq, { status: "ok", exit_code: 0 }));
else if (shape === 5) items.push(lifecycle(++seq, "failed"));
else if (shape === 6) {
items.push(speech(++seq, "held", "held at the boundary"));
items.push(speech(++seq, "confirmed"));
} else {
items.push(lifecycle(++seq, "turn.outcome", { status: "held", reason: "delivery review refused" }));
items.push(cell(++seq, { status: "ok", exit_code: 0 }));
}
if (next() < 0.3) work.push(workItem(`input-${inputSeq}`, "failed", `repair for turn ${t}`));
if (next() < 0.2) work.push(workItem(`input-${inputSeq}-extra`, "pending"));
}
// a second, later answer for an input that already had one
const targets = items.filter((it) => it.lane === "input").slice(0, 3);
for (const target of targets) {
if (target.seq === undefined) continue;
items.push(lifecycle(++seq, "turn.outcome", { status: "replied", input_event_ids: [target.seq] }));
}
return { items, work };
}
describe("computeInputOutcomes", () => {
test("answers what the scan-per-input implementation answered", () => {
for (const size of [1, 2, 5, 40, 200]) {
const { items, work } = transcript(size, size * 13 + 1);
const mine = computeInputOutcomes(items, work);
const reference = referenceOutcomes(items, work);
expect([...mine.entries()]).toEqual([...reference.entries()]);
}
});
test("names the outcome of a turn by the evidence in it", () => {
const items = [
input(1), speech(2, "confirmed"), // replied
input(3), cell(4, { status: "error", exit_code: 101, stderr: "one\ntwo\nboom" }), // failed
input(5), speech(6, "held", "awaiting delivery review"), // held
input(7), cell(8, { status: "ok", exit_code: 0 }), // no reply
input(9), cell(10, { status: "ok", exit_code: 0 }), // running (last, nothing said yet)
];
const outcomes = computeInputOutcomes(items, []);
expect(outcomes.get("in-1")).toEqual({ status: "replied" });
expect(outcomes.get("in-3")).toEqual({ status: "failed", reason: "boom" });
expect(outcomes.get("in-5")).toEqual({ status: "held", reason: "awaiting delivery review" });
expect(outcomes.get("in-7")).toEqual({ status: "no_reply", reason: "the turn completed without speech" });
expect(outcomes.get("in-9")).toEqual({ status: "running" });
expect(outcomes.size).toBe(5);
});
test("an explicit outcome for an input outranks the evidence that followed it", () => {
const items = [
input(1), cell(2, { status: "error", exit_code: 101, stderr: "boom" }),
speech(3, "confirmed"),
// the runtime names input 1's own event id: that is the answer, not the failing cell beside it
lifecycle(4, "turn.outcome", { status: "replied", input_event_ids: [1] }),
];
expect(computeInputOutcomes(items, []).get("in-1")).toEqual({ status: "replied" });
});
test("a failed work item tied to the input's sequence is a failure, with its title as the reason", () => {
const items = [input(1), cell(2, { status: "ok", exit_code: 0 }), input(3)];
const work = [workItem("input-1", "failed", "repair the scheduler")];
expect(computeInputOutcomes(items, work).get("in-1")).toEqual({ status: "failed", reason: "repair the scheduler" });
});
test("reads the transcript a fixed number of times per record, not once per record per input", () => {
const countReads = (turns: number) => {
const { items, work } = transcript(turns, 99);
let reads = 0;
const counted = new Proxy(items, {
get(target, prop, receiver) {
if (typeof prop === "string" && /^\d+$/.test(prop)) reads += 1;
return Reflect.get(target, prop, receiver);
},
});
computeInputOutcomes(counted as unknown as TimelineItem[], work);
return { reads, items: items.length };
};
const small = countReads(100);
const large = countReads(200);
// One pass reads each record for what it is, at every length: 2 reads per record measured at 50, 100,
// 200 and 400 turns. The scan-per-input version doubled its reads per record every time the
// transcript doubled (52, 102, 199, 398 at those same sizes) — 199x the reads at the largest.
expect(large.reads / large.items).toBeLessThan(small.reads / small.items * 1.6);
});
});
describe("turn coordinates and the window's own edge", () => {
test("a turn is numbered by its place in the session, not by its rank in the loaded window", () => {
// The same three turns, loaded as a window that starts at the third: their numbers are the same
// whether the console holds all of them or the last one. A count from one would say the session
// began at the window's edge, which is exactly the claim a window cannot make.
const items = [input(100), speech(101, "confirmed"), input(120), speech(121, "confirmed"), input(140), cell(141, { status: "ok", exit_code: 0 })];
const all = buildRows(groupTurns(items.filter((i) => i.lane === "speech" || i.lane === "input" || i.lane === "cell")));
const heads = all.filter((row) => row.kind === "head").map((row: any) => row.no);
expect(heads).toEqual([100, 120, 140]);
const windowed = buildRows(groupTurns(items.slice(4)));
expect(windowed.filter((row) => row.kind === "head").map((row: any) => row.no)).toEqual([140]);
});
test("a turn the window cut in half claims nothing about the turn", () => {
// These records are the middle of a turn: the input that opened it is above the loaded window, so
// this summary would be a count of what happens to be loaded, in the shape of a count of the turn.
const orphanTail = [cell(300, { status: "error", exit_code: 1 }), speech(301, "confirmed")];
const cut = buildRows(groupTurns(orphanTail), {}, true);
const head = cut.find((row) => row.kind === "head") as any;
expect(head.partial).toBe(true);
// The same records with nothing above them are a turn of a session that started without an input,
// and there is no cut to report: it is numbered and summarised like any other.
const whole = buildRows(groupTurns(orphanTail), {}, false);
expect((whole.find((row) => row.kind === "head") as any).partial).toBe(false);
});
test("the window's own edge is the first row, and it is not a turn", () => {
const items = [input(100), speech(101, "confirmed")];
const rows = buildRows(groupTurns(items), {}, true);
expect(rows[0].kind).toBe("history");
expect(rows[0].at).toBe(0);
// A transcript with nothing behind it has no edge to draw.
expect(buildRows(groupTurns(items), {}, false)[0].kind).toBe("head");
});
});