From ff230df47c50f43993afb2a5e7817bbc92f08d74 Mon Sep 17 00:00:00 2001 From: Youngbin Kim <64558592+youngbinkim0@users.noreply.github.com> Date: Thu, 12 Feb 2026 13:10:47 -0500 Subject: [PATCH] fix(runtime-fallback): harden fallback progression and success detection --- src/hooks/runtime-fallback/index.test.ts | 1059 +++++++++++++++++++++- src/hooks/runtime-fallback/index.ts | 373 +++++++- 2 files changed, 1426 insertions(+), 6 deletions(-) diff --git a/src/hooks/runtime-fallback/index.test.ts b/src/hooks/runtime-fallback/index.test.ts index 0d277d2b..38e57d51 100644 --- a/src/hooks/runtime-fallback/index.test.ts +++ b/src/hooks/runtime-fallback/index.test.ts @@ -27,6 +27,7 @@ describe("runtime-fallback", () => { session?: { messages?: (args: unknown) => Promise promptAsync?: (args: unknown) => Promise + abort?: (args: unknown) => Promise } }) { return { @@ -43,6 +44,7 @@ describe("runtime-fallback", () => { session: { messages: overrides?.session?.messages ?? (async () => ({ data: [] })), promptAsync: overrides?.session?.promptAsync ?? (async () => ({})), + abort: overrides?.session?.abort ?? (async () => ({})), }, }, directory: "/test/dir", @@ -160,6 +162,77 @@ describe("runtime-fallback", () => { expect(skipLog).toBeDefined() }) + test("should log missing API key errors with classification details", async () => { + const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig() }) + const sessionID = "test-session-missing-api-key" + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "AI_LoadAPIKeyError", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }) + + const sessionErrorLog = logCalls.find((c) => c.msg.includes("session.error received")) + expect(sessionErrorLog).toBeDefined() + expect(sessionErrorLog?.data).toMatchObject({ + sessionID, + errorName: "AI_LoadAPIKeyError", + errorType: "missing_api_key", + }) + + const skipLog = logCalls.find((c) => c.msg.includes("Error not retryable")) + expect(skipLog).toBeUndefined() + }) + + test("should trigger fallback for missing API key errors when fallback models are configured", async () => { + const hook = createRuntimeFallbackHook(createMockPluginInput(), { + config: createMockConfig({ notify_on_fallback: false }), + pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]), + }) + const sessionID = "test-session-missing-api-key-fallback" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "AI_LoadAPIKeyError", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }) + + const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback")) + expect(fallbackLog).toBeDefined() + expect(fallbackLog?.data).toMatchObject({ from: "google/gemini-2.5-pro", to: "openai/gpt-5.2" }) + }) + test("should detect retryable error from message pattern 'rate limit'", async () => { const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig() }) const sessionID = "test-session-pattern" @@ -182,6 +255,100 @@ describe("runtime-fallback", () => { expect(errorLog).toBeDefined() }) + test("should continue fallback chain when fallback model is not found", async () => { + const hook = createRuntimeFallbackHook(createMockPluginInput(), { + config: createMockConfig({ notify_on_fallback: false }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "anthropic/claude-opus-4.6", + "openai/gpt-5.2", + ]), + }) + const sessionID = "test-session-model-not-found" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { name: "UnknownError", data: { message: "Model not found: anthropic/claude-opus-4.6." } }, + }, + }, + }) + + const fallbackLogs = logCalls.filter((c) => c.msg.includes("Preparing fallback")) + expect(fallbackLogs.length).toBeGreaterThanOrEqual(2) + expect(fallbackLogs[1]?.data).toMatchObject({ from: "anthropic/claude-opus-4.6", to: "openai/gpt-5.2" }) + + const nonRetryLog = logCalls.find( + (c) => c.msg.includes("Error not retryable") && (c.data as { sessionID?: string } | undefined)?.sessionID === sessionID + ) + expect(nonRetryLog).toBeUndefined() + }) + + test("should trigger fallback on Copilot auto-retry signal in message.updated", async () => { + const hook = createRuntimeFallbackHook(createMockPluginInput(), { + config: createMockConfig({ notify_on_fallback: false }), + pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]), + }) + + const sessionID = "test-session-copilot-auto-retry" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "github-copilot/claude-opus-4.6" } }, + }, + }) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + model: "github-copilot/claude-opus-4.6", + status: + "Too Many Requests: quota exceeded [retrying in ~2 weeks attempt #1]", + }, + }, + }, + }) + + const signalLog = logCalls.find((c) => c.msg.includes("Detected Copilot auto-retry signal")) + expect(signalLog).toBeDefined() + + const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback")) + expect(fallbackLog).toBeDefined() + expect(fallbackLog?.data).toMatchObject({ from: "github-copilot/claude-opus-4.6", to: "openai/gpt-5.2" }) + }) + test("should log when no fallback models configured", async () => { const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig(), @@ -410,6 +577,893 @@ describe("runtime-fallback", () => { const errorLog = logCalls.find((c) => c.msg.includes("message.updated with assistant error")) expect(errorLog).toBeUndefined() }) + + test("should trigger fallback when message.updated has missing API key error without model", async () => { + const hook = createRuntimeFallbackHook(createMockPluginInput(), { + config: createMockConfig({ notify_on_fallback: false }), + pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]), + }) + const sessionID = "test-message-updated-missing-model" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + error: { + name: "AI_LoadAPIKeyError", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback")) + expect(fallbackLog).toBeDefined() + expect(fallbackLog?.data).toMatchObject({ from: "google/gemini-2.5-pro", to: "openai/gpt-5.2" }) + }) + + test("should not advance fallback state from message.updated while retry is already in flight", async () => { + const pending = new Promise(() => {}) + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "hello" }] }], + }), + promptAsync: async () => pending, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "anthropic/claude-opus-4-6", + "openai/gpt-5.2", + ]), + } + ) + + const sessionID = "test-message-updated-inflight-race" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + const sessionErrorPromise = hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 0)) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + model: "github-copilot/claude-opus-4.6", + }, + }, + }, + }) + + const fallbackLogs = logCalls.filter((c) => c.msg.includes("Preparing fallback")) + expect(fallbackLogs).toHaveLength(1) + + void sessionErrorPromise + }) + + test("should force advance fallback from message.updated when Copilot auto-retry signal appears during in-flight retry", async () => { + const retriedModels: string[] = [] + const pending = new Promise(() => {}) + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "hello" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + + if (retriedModels.length === 1) { + await pending + } + + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "anthropic/claude-opus-4-6", + "openai/gpt-5.2", + ]), + } + ) + + const sessionID = "test-message-updated-inflight-retry-signal" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + const sessionErrorPromise = hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 0)) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + model: "github-copilot/claude-opus-4.6", + status: + "Too Many Requests: quota exceeded [retrying in ~2 weeks attempt #1]", + }, + }, + }, + }) + + expect(retriedModels.length).toBeGreaterThanOrEqual(2) + expect(retriedModels[0]).toBe("github-copilot/claude-opus-4.6") + expect(retriedModels[1]).toBe("anthropic/claude-opus-4-6") + + void sessionErrorPromise + }) + + test("should advance fallback after session timeout when Copilot retry emits no retryable events", async () => { + const retriedModels: string[] = [] + const abortCalls: Array<{ path?: { id?: string } }> = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "hello" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + abort: async (args: unknown) => { + abortCalls.push(args as { path?: { id?: string } }) + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "anthropic/claude-opus-4-6", + "openai/gpt-5.2", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-timeout-watchdog" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(retriedModels).toContain("github-copilot/claude-opus-4.6") + expect(retriedModels).toContain("anthropic/claude-opus-4-6") + expect(abortCalls.some((call) => call.path?.id === sessionID)).toBe(true) + + const timeoutLog = logCalls.find((c) => c.msg.includes("Session fallback timeout reached")) + expect(timeoutLog).toBeDefined() + }) + + test("should keep session timeout active after chat.message model override", async () => { + const retriedModels: string[] = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "hello" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "anthropic/claude-opus-4-6", + "openai/gpt-5.2", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-timeout-after-chat-message" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + const output: { message: { model?: { providerID: string; modelID: string } }; parts: Array<{ type: string; text?: string }> } = { + message: {}, + parts: [], + } + await hook["chat.message"]?.( + { + sessionID, + model: { providerID: "github-copilot", modelID: "claude-opus-4.6" }, + }, + output + ) + + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(retriedModels).toContain("github-copilot/claude-opus-4.6") + expect(retriedModels).toContain("anthropic/claude-opus-4-6") + }) + + test("should abort in-flight fallback request before advancing on timeout", async () => { + const retriedModels: string[] = [] + const abortCalls: Array<{ path?: { id?: string } }> = [] + const never = new Promise(() => {}) + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "hello" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + + if (retriedModels.length === 1) { + await never + } + + return {} + }, + abort: async (args: unknown) => { + abortCalls.push(args as { path?: { id?: string } }) + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "anthropic/claude-opus-4-6", + "openai/gpt-5.2", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-timeout-abort-inflight" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + const sessionErrorPromise = hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(abortCalls.some((call) => call.path?.id === sessionID)).toBe(true) + expect(retriedModels).toContain("github-copilot/claude-opus-4.6") + expect(retriedModels).toContain("anthropic/claude-opus-4-6") + + void sessionErrorPromise + }) + + test("should not advance fallback after session.stop cancels timeout-driven retry", async () => { + const retriedModels: string[] = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "hello" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "anthropic/claude-opus-4-6", + "openai/gpt-5.2", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-stop-cancels-timeout-fallback" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + expect(retriedModels).toContain("github-copilot/claude-opus-4.6") + + await hook.event({ + event: { + type: "session.stop", + properties: { sessionID }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(retriedModels).toHaveLength(1) + }) + + test("should not trigger second fallback after successful assistant reply", async () => { + const retriedModels: string[] = [] + const mockMessages = [ + { info: { role: "user" }, parts: [{ type: "text", text: "test" }] }, + ] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: mockMessages, + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "openai/gpt-5.3-codex", + "anthropic/claude-opus-4-6", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-success-clears-timeout" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + expect(retriedModels).toEqual(["github-copilot/claude-opus-4.6"]) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + model: "openai/gpt-5.3-codex", + }, + }, + }, + }) + + mockMessages.push({ + info: { role: "assistant" }, + parts: [{ type: "text", text: "Got it - I'm here." }], + }) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + model: "openai/gpt-5.3-codex", + message: "Got it - I'm here.", + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(retriedModels).toEqual(["github-copilot/claude-opus-4.6"]) + }) + + test("should not clear fallback timeout on assistant non-error update with Copilot retry signal", async () => { + const retriedModels: string[] = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "test" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "openai/gpt-5.3-codex", + "anthropic/claude-opus-4-6", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-copilot-retry-signal-no-error" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + expect(retriedModels).toEqual(["github-copilot/claude-opus-4.6"]) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + status: "Too Many Requests: quota exceeded [retrying in ~2 weeks attempt #1]", + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 60)) + + expect(retriedModels).toContain("openai/gpt-5.3-codex") + }) + + test("should not clear fallback timeout on assistant non-error update without user-visible content", async () => { + const retriedModels: string[] = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "test" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "openai/gpt-5.3-codex", + "anthropic/claude-opus-4-6", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-no-content-non-error-update" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + expect(retriedModels).toEqual(["github-copilot/claude-opus-4.6"]) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + model: "github-copilot/claude-opus-4.6", + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 60)) + + expect(retriedModels).toContain("openai/gpt-5.3-codex") + }) + + test("should not clear fallback timeout from info.message alone without persisted assistant text", async () => { + const retriedModels: string[] = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "test" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "openai/gpt-5.3-codex", + "anthropic/claude-opus-4-6", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-info-message-without-persisted-text" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + expect(retriedModels).toEqual(["github-copilot/claude-opus-4.6"]) + + await hook.event({ + event: { + type: "message.updated", + properties: { + info: { + sessionID, + role: "assistant", + message: "Thinking: retrying provider request...", + }, + }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 60)) + + expect(retriedModels).toContain("openai/gpt-5.3-codex") + }) + + test("should keep timeout armed when session.idle fires before fallback result", async () => { + const retriedModels: string[] = [] + + const hook = createRuntimeFallbackHook( + createMockPluginInput({ + session: { + messages: async () => ({ + data: [{ info: { role: "user" }, parts: [{ type: "text", text: "test" }] }], + }), + promptAsync: async (args: unknown) => { + const model = (args as { body?: { model?: { providerID?: string; modelID?: string } } })?.body?.model + if (model?.providerID && model?.modelID) { + retriedModels.push(`${model.providerID}/${model.modelID}`) + } + return {} + }, + }, + }), + { + config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 30 }), + pluginConfig: createMockPluginConfigWithCategoryFallback([ + "github-copilot/claude-opus-4.6", + "openai/gpt-5.3-codex", + "anthropic/claude-opus-4-6", + ]), + session_timeout_ms: 20, + } + ) + + const sessionID = "test-session-idle-before-fallback-result" + SessionCategoryRegistry.register(sessionID, "test") + + await hook.event({ + event: { + type: "session.created", + properties: { info: { id: sessionID, model: "google/gemini-2.5-pro" } }, + }, + }) + + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID, + error: { + name: "ProviderAuthError", + data: { + providerID: "google", + message: + "Google Generative AI API key is missing. Pass it using the 'apiKey' parameter or the GOOGLE_GENERATIVE_AI_API_KEY environment variable.", + }, + }, + }, + }, + }) + + expect(retriedModels).toEqual(["github-copilot/claude-opus-4.6"]) + + await hook.event({ + event: { + type: "session.idle", + properties: { sessionID }, + }, + }) + + await new Promise((resolve) => setTimeout(resolve, 60)) + + expect(retriedModels).toContain("openai/gpt-5.3-codex") + }) }) describe("edge cases", () => { @@ -497,7 +1551,10 @@ describe("runtime-fallback", () => { }, }) - const output = { message: {}, parts: [] } + const output: { message: { model?: { providerID: string; modelID: string } }; parts: Array<{ type: string; text?: string }> } = { + message: {}, + parts: [], + } await hook["chat.message"]?.( { sessionID }, output diff --git a/src/hooks/runtime-fallback/index.ts b/src/hooks/runtime-fallback/index.ts index 202b917d..29fac0e2 100644 --- a/src/hooks/runtime-fallback/index.ts +++ b/src/hooks/runtime-fallback/index.ts @@ -65,9 +65,91 @@ function extractStatusCode(error: unknown): number | undefined { return undefined } +function extractErrorName(error: unknown): string | undefined { + if (!error || typeof error !== "object") return undefined + + const errorObj = error as Record + const directName = errorObj.name + if (typeof directName === "string" && directName.length > 0) { + return directName + } + + const nestedError = errorObj.error as Record | undefined + const nestedName = nestedError?.name + if (typeof nestedName === "string" && nestedName.length > 0) { + return nestedName + } + + const dataError = (errorObj.data as Record | undefined)?.error as Record | undefined + const dataErrorName = dataError?.name + if (typeof dataErrorName === "string" && dataErrorName.length > 0) { + return dataErrorName + } + + return undefined +} + +function classifyErrorType(error: unknown): string | undefined { + const message = getErrorMessage(error) + const errorName = extractErrorName(error)?.toLowerCase() + + if ( + errorName?.includes("loadapi") || + (/api.?key.?is.?missing/i.test(message) && /environment variable/i.test(message)) + ) { + return "missing_api_key" + } + + if (/api.?key/i.test(message) && /must be a string/i.test(message)) { + return "invalid_api_key" + } + + if (errorName?.includes("unknownerror") && /model\s+not\s+found/i.test(message)) { + return "model_not_found" + } + + return undefined +} + +function extractCopilotAutoRetrySignal(info: Record | undefined): string | undefined { + if (!info) return undefined + + const candidates: string[] = [] + + const directStatus = info.status + if (typeof directStatus === "string") candidates.push(directStatus) + + const summary = info.summary + if (typeof summary === "string") candidates.push(summary) + + const message = info.message + if (typeof message === "string") candidates.push(message) + + const details = info.details + if (typeof details === "string") candidates.push(details) + + const combined = candidates.join("\n") + if (!combined) return undefined + + if (/too.?many.?requests/i.test(combined) && /quota.?exceeded/i.test(combined) && /retrying\s+in/i.test(combined)) { + return combined + } + + return undefined +} + function isRetryableError(error: unknown, retryOnErrors: number[]): boolean { const statusCode = extractStatusCode(error) const message = getErrorMessage(error) + const errorType = classifyErrorType(error) + + if (errorType === "missing_api_key") { + return true + } + + if (errorType === "model_not_found") { + return true + } if (statusCode && retryOnErrors.includes(statusCode)) { return true @@ -265,14 +347,78 @@ export function createRuntimeFallbackHook( retry_on_errors: options?.config?.retry_on_errors ?? DEFAULT_CONFIG.retry_on_errors, max_fallback_attempts: options?.config?.max_fallback_attempts ?? DEFAULT_CONFIG.max_fallback_attempts, cooldown_seconds: options?.config?.cooldown_seconds ?? DEFAULT_CONFIG.cooldown_seconds, + timeout_seconds: options?.config?.timeout_seconds ?? DEFAULT_CONFIG.timeout_seconds, notify_on_fallback: options?.config?.notify_on_fallback ?? DEFAULT_CONFIG.notify_on_fallback, } const sessionStates = new Map() const sessionLastAccess = new Map() const sessionRetryInFlight = new Set() + const sessionAwaitingFallbackResult = new Set() + const sessionFallbackTimeouts = new Map>() const SESSION_TTL_MS = 30 * 60 * 1000 // 30 minutes TTL for stale sessions + const abortSessionRequest = async (sessionID: string, source: string): Promise => { + try { + await ctx.client.session.abort({ path: { id: sessionID } }) + log(`[${HOOK_NAME}] Aborted in-flight session request (${source})`, { sessionID }) + } catch (error) { + log(`[${HOOK_NAME}] Failed to abort in-flight session request (${source})`, { + sessionID, + error: String(error), + }) + } + } + + const clearSessionFallbackTimeout = (sessionID: string) => { + const timer = sessionFallbackTimeouts.get(sessionID) + if (timer) { + clearTimeout(timer) + sessionFallbackTimeouts.delete(sessionID) + } + } + + const scheduleSessionFallbackTimeout = (sessionID: string, resolvedAgent?: string) => { + clearSessionFallbackTimeout(sessionID) + + const timeoutMs = options?.session_timeout_ms ?? config.timeout_seconds * 1000 + if (timeoutMs <= 0) return + + const timer = setTimeout(async () => { + sessionFallbackTimeouts.delete(sessionID) + + const state = sessionStates.get(sessionID) + if (!state) return + + if (sessionRetryInFlight.has(sessionID)) { + log(`[${HOOK_NAME}] Overriding in-flight retry due to session timeout`, { sessionID }) + } + + await abortSessionRequest(sessionID, "session.timeout") + sessionRetryInFlight.delete(sessionID) + + if (state.pendingFallbackModel) { + state.pendingFallbackModel = undefined + } + + const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig) + if (fallbackModels.length === 0) return + + log(`[${HOOK_NAME}] Session fallback timeout reached`, { + sessionID, + timeoutSeconds: config.timeout_seconds, + currentModel: state.currentModel, + }) + + const result = prepareFallback(sessionID, state, fallbackModels, config) + if (result.success && result.newModel) { + await autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.timeout") + } + }, timeoutMs) + + sessionFallbackTimeouts.set(sessionID, timer) + } + // Periodic cleanup of stale session states to prevent memory leaks const cleanupStaleSessions = () => { const now = Date.now() @@ -282,6 +428,8 @@ export function createRuntimeFallbackHook( sessionStates.delete(sessionID) sessionLastAccess.delete(sessionID) sessionRetryInFlight.delete(sessionID) + sessionAwaitingFallbackResult.delete(sessionID) + clearSessionFallbackTimeout(sessionID) SessionCategoryRegistry.remove(sessionID) cleanedCount++ } @@ -354,6 +502,9 @@ export function createRuntimeFallbackHook( if (retryParts.length > 0) { const retryAgent = resolvedAgent ?? getSessionAgent(sessionID) + sessionAwaitingFallbackResult.add(sessionID) + scheduleSessionFallbackTimeout(sessionID, retryAgent) + await ctx.client.session.promptAsync({ path: { id: sessionID }, body: { @@ -370,6 +521,10 @@ export function createRuntimeFallbackHook( } catch (retryError) { log(`[${HOOK_NAME}] Auto-retry failed (${source})`, { sessionID, error: String(retryError) }) } finally { + const state = sessionStates.get(sessionID) + if (state?.pendingFallbackModel === newModel) { + state.pendingFallbackModel = undefined + } sessionRetryInFlight.delete(sessionID) } } @@ -404,6 +559,47 @@ export function createRuntimeFallbackHook( return undefined } + const hasVisibleAssistantResponse = async ( + sessionID: string, + _info: Record | undefined, + ): Promise => { + try { + const messagesResp = await ctx.client.session.messages({ + path: { id: sessionID }, + query: { directory: ctx.directory }, + }) + + const msgs = (messagesResp as { + data?: Array<{ + info?: Record + parts?: Array<{ type?: string; text?: string }> + }> + }).data + + if (!msgs || msgs.length === 0) return false + + const lastAssistant = [...msgs].reverse().find((m) => m.info?.role === "assistant") + if (!lastAssistant) return false + if (lastAssistant.info?.error) return false + + const parts = lastAssistant.parts ?? + (lastAssistant.info?.parts as Array<{ type?: string; text?: string }> | undefined) + + const textFromParts = (parts ?? []) + .filter((p) => p.type === "text" && typeof p.text === "string") + .map((p) => p.text!.trim()) + .filter((text) => text.length > 0) + .join("\n") + + if (!textFromParts) return false + if (extractCopilotAutoRetrySignal({ message: textFromParts })) return false + + return true + } catch { + return false + } + } + const eventHandler = async ({ event }: { event: { type: string; properties?: unknown } }) => { if (!config.enabled) return @@ -431,11 +627,59 @@ export function createRuntimeFallbackHook( sessionStates.delete(sessionID) sessionLastAccess.delete(sessionID) sessionRetryInFlight.delete(sessionID) + sessionAwaitingFallbackResult.delete(sessionID) + clearSessionFallbackTimeout(sessionID) SessionCategoryRegistry.remove(sessionID) } return } + if (event.type === "session.stop") { + const sessionID = props?.sessionID as string | undefined + if (!sessionID) return + + clearSessionFallbackTimeout(sessionID) + + if (sessionRetryInFlight.has(sessionID)) { + await abortSessionRequest(sessionID, "session.stop") + } + + sessionRetryInFlight.delete(sessionID) + sessionAwaitingFallbackResult.delete(sessionID) + + const state = sessionStates.get(sessionID) + if (state?.pendingFallbackModel) { + state.pendingFallbackModel = undefined + } + + log(`[${HOOK_NAME}] Cleared fallback retry state on session.stop`, { sessionID }) + return + } + + if (event.type === "session.idle") { + const sessionID = props?.sessionID as string | undefined + if (!sessionID) return + + if (sessionAwaitingFallbackResult.has(sessionID)) { + log(`[${HOOK_NAME}] session.idle while awaiting fallback result; keeping timeout armed`, { sessionID }) + return + } + + const hadTimeout = sessionFallbackTimeouts.has(sessionID) + clearSessionFallbackTimeout(sessionID) + sessionRetryInFlight.delete(sessionID) + + const state = sessionStates.get(sessionID) + if (state?.pendingFallbackModel) { + state.pendingFallbackModel = undefined + } + + if (hadTimeout) { + log(`[${HOOK_NAME}] Cleared fallback timeout after session completion`, { sessionID }) + } + return + } + if (event.type === "session.error") { const sessionID = props?.sessionID as string | undefined const error = props?.error @@ -447,16 +691,27 @@ export function createRuntimeFallbackHook( } const resolvedAgent = await resolveAgentForSessionFromContext(sessionID, agent) + sessionAwaitingFallbackResult.delete(sessionID) + + clearSessionFallbackTimeout(sessionID) log(`[${HOOK_NAME}] session.error received`, { sessionID, agent, resolvedAgent, statusCode: extractStatusCode(error), + errorName: extractErrorName(error), + errorType: classifyErrorType(error), }) if (!isRetryableError(error, config.retry_on_errors)) { - log(`[${HOOK_NAME}] Error not retryable, skipping fallback`, { sessionID }) + log(`[${HOOK_NAME}] Error not retryable, skipping fallback`, { + sessionID, + retryable: false, + statusCode: extractStatusCode(error), + errorName: extractErrorName(error), + errorType: classifyErrorType(error), + }) return } @@ -524,14 +779,74 @@ export function createRuntimeFallbackHook( if (event.type === "message.updated") { const info = props?.info as Record | undefined const sessionID = info?.sessionID as string | undefined - const error = info?.error + const retrySignal = extractCopilotAutoRetrySignal(info) + const error = info?.error ?? (retrySignal ? { name: "ProviderRateLimitError", message: retrySignal } : undefined) const role = info?.role as string | undefined const model = info?.model as string | undefined - if (sessionID && role === "assistant" && error && model) { - log(`[${HOOK_NAME}] message.updated with assistant error`, { sessionID, model }) + if (sessionID && role === "assistant" && !error) { + if (!sessionAwaitingFallbackResult.has(sessionID)) { + return + } + + const hasVisibleResponse = await hasVisibleAssistantResponse(sessionID, info) + if (!hasVisibleResponse) { + log(`[${HOOK_NAME}] Assistant update observed without visible final response; keeping fallback timeout`, { + sessionID, + model, + }) + return + } + + sessionAwaitingFallbackResult.delete(sessionID) + clearSessionFallbackTimeout(sessionID) + const state = sessionStates.get(sessionID) + if (state?.pendingFallbackModel) { + state.pendingFallbackModel = undefined + } + log(`[${HOOK_NAME}] Assistant response observed; cleared fallback timeout`, { sessionID, model }) + return + } + + if (sessionID && role === "assistant" && error) { + sessionAwaitingFallbackResult.delete(sessionID) + if (sessionRetryInFlight.has(sessionID) && !retrySignal) { + log(`[${HOOK_NAME}] message.updated fallback skipped (retry in flight)`, { sessionID }) + return + } + + if (retrySignal && sessionRetryInFlight.has(sessionID)) { + log(`[${HOOK_NAME}] Overriding in-flight retry due to Copilot auto-retry signal`, { + sessionID, + model, + }) + await abortSessionRequest(sessionID, "message.updated.retry-signal") + sessionRetryInFlight.delete(sessionID) + } + + if (retrySignal) { + log(`[${HOOK_NAME}] Detected Copilot auto-retry signal`, { sessionID, model }) + } + + if (!retrySignal) { + clearSessionFallbackTimeout(sessionID) + } + + log(`[${HOOK_NAME}] message.updated with assistant error`, { + sessionID, + model, + statusCode: extractStatusCode(error), + errorName: extractErrorName(error), + errorType: classifyErrorType(error), + }) if (!isRetryableError(error, config.retry_on_errors)) { + log(`[${HOOK_NAME}] message.updated error not retryable, skipping fallback`, { + sessionID, + statusCode: extractStatusCode(error), + errorName: extractErrorName(error), + errorType: classifyErrorType(error), + }) return } @@ -545,11 +860,53 @@ export function createRuntimeFallbackHook( } if (!state) { - state = createFallbackState(model) + let initialModel = model + if (!initialModel) { + const detectedAgent = resolvedAgent + const agentConfig = detectedAgent + ? pluginConfig?.agents?.[detectedAgent as keyof typeof pluginConfig.agents] + : undefined + const agentModel = agentConfig?.model as string | undefined + if (agentModel) { + log(`[${HOOK_NAME}] Derived model from agent config for message.updated`, { + sessionID, + agent: detectedAgent, + model: agentModel, + }) + initialModel = agentModel + } + } + + if (!initialModel) { + log(`[${HOOK_NAME}] message.updated missing model info, cannot fallback`, { + sessionID, + errorName: extractErrorName(error), + errorType: classifyErrorType(error), + }) + return + } + + state = createFallbackState(initialModel) sessionStates.set(sessionID, state) sessionLastAccess.set(sessionID, Date.now()) } else { sessionLastAccess.set(sessionID, Date.now()) + + if (state.pendingFallbackModel) { + if (retrySignal) { + log(`[${HOOK_NAME}] Clearing pending fallback due to Copilot auto-retry signal`, { + sessionID, + pendingFallbackModel: state.pendingFallbackModel, + }) + state.pendingFallbackModel = undefined + } else { + log(`[${HOOK_NAME}] message.updated fallback skipped (pending fallback in progress)`, { + sessionID, + pendingFallbackModel: state.pendingFallbackModel, + }) + return + } + } } const result = prepareFallback(sessionID, state, fallbackModels, config) @@ -591,6 +948,12 @@ export function createRuntimeFallbackHook( : undefined if (requestedModel && requestedModel !== state.currentModel) { + if (state.pendingFallbackModel && state.pendingFallbackModel === requestedModel) { + state.pendingFallbackModel = undefined + sessionLastAccess.set(sessionID, Date.now()) + return + } + log(`[${HOOK_NAME}] Detected manual model change, resetting fallback state`, { sessionID, from: state.currentModel,