fix(coding-agent): survive daemon restarts and retry re-attachment with backoff
1- Fixed attached terminal windows dying after daemon restarts by retrying re-attachment with bounded backoff and accepting prompts once restored.46 unmodified lines47484950515152535455114 unmodified linesExpand all46 unmodified lines```text{ generation, sequence }```
The adapter rejects duplicate or retired-generation events. After a transient socket loss it reconnects with the same client identity and last cursor, reattaches, and emits a resynchronized snapshot. If incremental replay is unavailable, the session snapshot is the source of truth.The adapter rejects duplicate or retired-generation events. After a transient socket loss it reconnects with the same client identity and last cursor, reattaches, and emits a resynchronized snapshot. If incremental replay is unavailable, the session snapshot is the source of truth. During daemon restarts or updates, attached windows remain open and retry re-attachment with bounded exponential backoff while workers start, accepting submitted prompts once the restored session is ready.
Key files:
- `src/modes/agent-connection/daemon-agent-connection.ts`114 unmodified linesExpand all1217 unmodified linesExpand all12181219122012211222122312241225122612271228122212291230123112321233174 unmodified linesExpand all1408140914101411140514121413141414151416141714181419142014211422387 unmodified linesExpand all1217 unmodified linesExpand all let resumed = 0; const failures: DaemonUpdateRestartFailure[] = []; try { await client.connect(10000); const orderedSessions = restartOriginActiveSessionId ? [...manifest.sessions].sort((a, b) => { if (a.activeSessionId === restartOriginActiveSessionId) return -1; if (b.activeSessionId === restartOriginActiveSessionId) return 1; return 0; }) : manifest.sessions; for (const session of manifest.sessions) { for (const session of orderedSessions) { try { const result = await restoreDaemonUpdateRestartSession( client, session,174 unmodified linesExpand all failed: 0, }, }); await shutdownAdmission.assertOrRenew(); const stopped = await shutdownConnectedDaemonAndWait(connectedClient, options.socketPath, 10000, hello); const stopped = await shutdownConnectedDaemonAndWait( connectedClient, options.socketPath, 10000, hello, "update", ); connectedClient = undefined; if (!stopped) { const remainingDaemon = await probeRunningDaemonSessions(options.socketPath); if (remainingDaemon.reachable) {387 unmodified linesExpand all17 unmodified lines18192021222324252618 unmodified lines45464748495051525354 unmodified lines1081091101111101121131141151161818 unmodified linesExpand all19351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162110 unmodified linesExpand all17 unmodified lines type DaemonHello, DaemonSocketClosedError, type DaemonTransportClient,} from "../src/modes/daemon/daemon-client.js";import { DaemonSessionStartingError } from "../src/modes/daemon/daemon-errors.js";import { DAEMON_PROTOCOL_INFO, DAEMON_SCHEMA_REVISION, type DaemonAttachResult,18 unmodified lines reconnectCount = 0; resetTransportCount = 0; reconnectError: Error | undefined; attachFailures = 0; attachError: Error | undefined; connectionStateGate: Promise<void> | undefined; connectionStateFactory: ((activeSessionId: string) => AgentConnectionState) | undefined; rlmChildren: AgentConnectionRlmChildAgentSnapshot[] = []; rlmChildrenEventSequence = 12;54 unmodified lines }; case "attach": { if (this.attachFailures > 0) { this.attachFailures--; throw new Error("attach failed"); throw this.attachError ?? new Error("attach failed"); } if (command.activeSessionId === "active-restored" && this.restoredAttachGate) { await this.restoredAttachGate; this.restoredAttachCompleted++;1818 unmodified linesExpand all const closedError = closedEvents[0]?.type === "closed" ? closedEvents[0].error : undefined; expect(closedError).toContain("The Prime Agent daemon shut down while this window was attached."); });
it("re-attaches and accepts prompt when connection drops with recoverDaemon", async () => { const fakeClient = new FakeDaemonClient(); const connection = new DaemonAgentConnection(asDaemonClient(fakeClient), "active-original", { recoverDaemon: async () => undefined, reconnectTimeoutMs: 5000, }); const events: AgentConnectionEvent[] = []; connection.subscribe((event) => { events.push(event); }); await connection.attach();
fakeClient.connected = false; fakeClient.emitClose(new DaemonSocketClosedError("/tmp/prime-agent.sock", "shutdown"));
await connection.prompt("hello after restart");
expect(events.some((e) => e.type === "connection_status" && e.status === "reconnecting")).toBe(true); expect(events.some((e) => e.type === "session_resynced")).toBe(true); expect(events.some((e) => e.type === "connection_status" && e.status === "connected")).toBe(true); expect( fakeClient.requests.some( (r) => r.type === "prompt" && "message" in r && (r as { message: unknown }).message === "hello after restart", ), ).toBe(true); });
it("retries and succeeds when attach is refused with Session worker is starting", async () => { const fakeClient = new FakeDaemonClient(); const connection = new DaemonAgentConnection(asDaemonClient(fakeClient), "active-original", { recoverDaemon: async () => undefined, reconnectTimeoutMs: 5000, }); const resynced = new Promise<AgentConnectionEvent>((resolve) => { connection.subscribe((event) => { if (event.type === "session_resynced") resolve(event); }); }); await connection.attach();
fakeClient.attachFailures = 1; fakeClient.attachError = new DaemonSessionStartingError(undefined, "Session worker is starting"); fakeClient.connected = false; fakeClient.emitClose(new Error("socket closed"));
const event = await resynced; expect(event.type).toBe("session_resynced"); expect(fakeClient.resetTransportCount).toBe(0); });
it("emits a clear give-up error with transcript details after reconnect timeout expires", async () => { const fakeClient = new FakeDaemonClient(); fakeClient.reconnectError = new Error("connection refused"); const connection = new DaemonAgentConnection(asDaemonClient(fakeClient), "active-original", { recoverDaemon: async () => undefined, reconnectTimeoutMs: 50, }); const closed = new Promise<string>((resolve) => { connection.subscribe((event) => { if (event.type === "closed") resolve(event.error ?? ""); }); }); await connection.attach();
fakeClient.connected = false; fakeClient.emitClose(new Error("socket died"));
const error = await closed; expect(error).toContain("Daemon reconnection failed"); expect(error).toContain("restart Prime Agent or reopen the session from Agents View"); expect(error).toContain("Session ID: session-current."); });
it("does not emit a restored session after disposal begins", async () => { const fakeClient = new FakeDaemonClient(); fakeClient.updateRestartSessions = [ {2110 unmodified linesExpand all9 unmodified lines1011121314141516171819202122231 unmodified linesExpand all2542552562572582592602612622582632642652662671 unmodified line269270271272268273274275276277278279280281273282283284285286289 unmodified linesExpand all9 unmodified linesimport { appendRotatingLog, expandTildePath, getClientErrorLogPath, getDaemonLogPath, VERSION } from "../config.js";import { ORPHAN_PROCESS_JOURNAL_ENV } from "../core/orphan-process-journal.js";import { getProcessStartId, SESSION_LEASE_OWNER_ID_ENV, SESSION_LEASES_ENABLED_ENV } from "../core/session-lease.js";import { DaemonClient, type DaemonHello } from "../modes/daemon/daemon-client.js";import { DAEMON_PROTOCOL_VERSION, DAEMON_SCHEMA_ID } from "../modes/daemon/daemon-protocol.js";import { DAEMON_PROTOCOL_VERSION, DAEMON_SCHEMA_ID, type DaemonClosingReason,} from "../modes/daemon/daemon-protocol.js";import { getDaemonRuntimeIdentity } from "../modes/daemon/daemon-runtime-identity.js";import { isSessionSummaryBusy, type SessionSummary } from "../modes/daemon/daemon-session-list.js";import { defaultDaemonSocketPath, normalizeSocketPath } from "../modes/daemon/daemon-socket.js";import {231 unmodified linesExpand all client: DaemonClient, socketPath: string, timeoutMs = 5000, hello: DaemonHello | undefined = client.hello, reason?: DaemonClosingReason,): Promise<boolean> { let shutdownAccepted = false; const expectedIdentity = processIdentityFromDaemonHello(hello); try { const response = await client.request({ type: "shutdown" }).catch(() => undefined); const response = await client.request({ type: "shutdown", ...(reason ? { reason } : {}) }).catch(() => undefined); shutdownAccepted = response?.success === true; } catch { // A connect failure isn't treated as "gone"; waitForDaemonGone is the source of truth. } finally {1 unmodified line } return waitForDaemonGone(socketPath, timeoutMs, shutdownAccepted, expectedIdentity);}
export async function shutdownDaemonAndWait(socketPath: string, timeoutMs = 5000): Promise<boolean> {export async function shutdownDaemonAndWait( socketPath: string, timeoutMs = 5000, reason?: DaemonClosingReason,): Promise<boolean> { const client = new DaemonClient(socketPath); try { await client.connect(1000); const hello = await client.waitForHello(2000).catch(() => undefined); return shutdownConnectedDaemonAndWait(client, socketPath, timeoutMs, hello); return shutdownConnectedDaemonAndWait(client, socketPath, timeoutMs, hello, reason); } catch { client.close(); return waitForDaemonGone(socketPath, timeoutMs); }289 unmodified linesExpand all