fix(runtime-fallback): per-model cooldown and stricter retry patterns
This commit is contained in:
parent
0ef17aa6c9
commit
d947743932
@ -29,9 +29,9 @@ export const RETRYABLE_ERROR_PATTERNS = [
|
|||||||
/overloaded/i,
|
/overloaded/i,
|
||||||
/temporarily.?unavailable/i,
|
/temporarily.?unavailable/i,
|
||||||
/try.?again/i,
|
/try.?again/i,
|
||||||
/429/,
|
/\b429\b/,
|
||||||
/503/,
|
/\b503\b/,
|
||||||
/529/,
|
/\b529\b/,
|
||||||
]
|
]
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@ -1,7 +1,8 @@
|
|||||||
import { describe, expect, test, beforeEach, afterEach, spyOn } from "bun:test"
|
import { describe, expect, test, beforeEach, afterEach, spyOn } from "bun:test"
|
||||||
import { createRuntimeFallbackHook, type RuntimeFallbackHook } from "./index"
|
import { createRuntimeFallbackHook, type RuntimeFallbackHook } from "./index"
|
||||||
import type { RuntimeFallbackConfig } from "../../config"
|
import type { RuntimeFallbackConfig, OhMyOpenCodeConfig } from "../../config"
|
||||||
import * as sharedModule from "../../shared"
|
import * as sharedModule from "../../shared"
|
||||||
|
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||||
|
|
||||||
describe("runtime-fallback", () => {
|
describe("runtime-fallback", () => {
|
||||||
let logCalls: Array<{ msg: string; data?: unknown }>
|
let logCalls: Array<{ msg: string; data?: unknown }>
|
||||||
@ -11,12 +12,14 @@ describe("runtime-fallback", () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
logCalls = []
|
logCalls = []
|
||||||
toastCalls = []
|
toastCalls = []
|
||||||
|
SessionCategoryRegistry.clear()
|
||||||
logSpy = spyOn(sharedModule, "log").mockImplementation((msg: string, data?: unknown) => {
|
logSpy = spyOn(sharedModule, "log").mockImplementation((msg: string, data?: unknown) => {
|
||||||
logCalls.push({ msg, data })
|
logCalls.push({ msg, data })
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
SessionCategoryRegistry.clear()
|
||||||
logSpy?.mockRestore()
|
logSpy?.mockRestore()
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -48,6 +51,16 @@ describe("runtime-fallback", () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createMockPluginConfigWithCategoryFallback(fallbackModels: string[]): OhMyOpenCodeConfig {
|
||||||
|
return {
|
||||||
|
categories: {
|
||||||
|
test: {
|
||||||
|
fallback_models: fallbackModels,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
describe("session.error handling", () => {
|
describe("session.error handling", () => {
|
||||||
test("should detect retryable error with status code 429", async () => {
|
test("should detect retryable error with status code 429", async () => {
|
||||||
const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig() })
|
const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig() })
|
||||||
@ -448,11 +461,15 @@ describe("runtime-fallback", () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
describe("model switching via chat.message", () => {
|
describe("model switching via chat.message", () => {
|
||||||
test("should set pending fallback model after error", async () => {
|
test("should apply fallback model on next chat.message after error", async () => {
|
||||||
const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig() })
|
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
|
||||||
|
config: createMockConfig({ notify_on_fallback: false }),
|
||||||
|
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2", "google/gemini-3-pro"]),
|
||||||
|
})
|
||||||
const sessionID = "test-session-switch"
|
const sessionID = "test-session-switch"
|
||||||
|
SessionCategoryRegistry.register(sessionID, "test")
|
||||||
|
|
||||||
//#given - session with fallback models configured
|
//#given
|
||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
type: "session.created",
|
type: "session.created",
|
||||||
@ -460,25 +477,30 @@ describe("runtime-fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
//#when - retryable error occurs
|
//#when
|
||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
type: "session.error",
|
type: "session.error",
|
||||||
properties: {
|
properties: { sessionID, error: { statusCode: 429, message: "Rate limit" } },
|
||||||
sessionID,
|
|
||||||
error: { statusCode: 429, message: "Rate limit" },
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
//#then - fallback preparation should be logged
|
const output = { message: {}, parts: [] }
|
||||||
const fallbackPrepLog = logCalls.find((c) => c.msg.includes("Preparing fallback") || c.msg.includes("fallback"))
|
await hook["chat.message"]?.(
|
||||||
expect(fallbackPrepLog !== undefined || logCalls.some(c => c.msg.includes("No fallback"))).toBe(true)
|
{ sessionID, model: { providerID: "anthropic", modelID: "claude-opus-4-5" } },
|
||||||
|
output
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(output.message.model).toEqual({ providerID: "openai", modelID: "gpt-5.2" })
|
||||||
})
|
})
|
||||||
|
|
||||||
test("should notify when fallback occurs", async () => {
|
test("should notify when fallback occurs", async () => {
|
||||||
const hook = createRuntimeFallbackHook(createMockPluginInput(), { config: createMockConfig() })
|
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
|
||||||
|
config: createMockConfig({ notify_on_fallback: true }),
|
||||||
|
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
|
||||||
|
})
|
||||||
const sessionID = "test-session-notify"
|
const sessionID = "test-session-notify"
|
||||||
|
SessionCategoryRegistry.register(sessionID, "test")
|
||||||
|
|
||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
@ -490,13 +512,12 @@ describe("runtime-fallback", () => {
|
|||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
type: "session.error",
|
type: "session.error",
|
||||||
properties: { sessionID, error: { statusCode: 429 }, agent: "sisyphus" },
|
properties: { sessionID, error: { statusCode: 429 } },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
//#then - should show notification toast or prepare fallback
|
expect(toastCalls.length).toBe(1)
|
||||||
const notifyLog = logCalls.find((c) => c.msg.includes("Preparing fallback") || c.msg.includes("No fallback models"))
|
expect(toastCalls[0]?.message.includes("gpt-5.2")).toBe(true)
|
||||||
expect(notifyLog).toBeDefined()
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -553,9 +574,14 @@ describe("runtime-fallback", () => {
|
|||||||
describe("cooldown mechanism", () => {
|
describe("cooldown mechanism", () => {
|
||||||
test("should respect cooldown period before retrying failed model", async () => {
|
test("should respect cooldown period before retrying failed model", async () => {
|
||||||
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
|
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
|
||||||
config: createMockConfig({ cooldown_seconds: 1 }),
|
config: createMockConfig({ cooldown_seconds: 60, notify_on_fallback: false }),
|
||||||
|
pluginConfig: createMockPluginConfigWithCategoryFallback([
|
||||||
|
"openai/gpt-5.2",
|
||||||
|
"anthropic/claude-opus-4-5",
|
||||||
|
]),
|
||||||
})
|
})
|
||||||
const sessionID = "test-session-cooldown"
|
const sessionID = "test-session-cooldown"
|
||||||
|
SessionCategoryRegistry.register(sessionID, "test")
|
||||||
|
|
||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
@ -564,7 +590,7 @@ describe("runtime-fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
//#when - first error occurs
|
//#when - first error occurs, switches to openai
|
||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
type: "session.error",
|
type: "session.error",
|
||||||
@ -572,11 +598,7 @@ describe("runtime-fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
const firstFallback = logCalls.find((c) => c.msg.includes("Preparing fallback") || c.msg.includes("No fallback models"))
|
//#when - second error occurs immediately; tries to switch back to original model but should be in cooldown
|
||||||
expect(firstFallback).toBeDefined()
|
|
||||||
|
|
||||||
//#when - second error occurs immediately (within cooldown)
|
|
||||||
logCalls = []
|
|
||||||
await hook.event({
|
await hook.event({
|
||||||
event: {
|
event: {
|
||||||
type: "session.error",
|
type: "session.error",
|
||||||
@ -584,11 +606,8 @@ describe("runtime-fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
//#then - should skip due to cooldown (no new logs or cooldown message)
|
const cooldownSkipLog = logCalls.find((c) => c.msg.includes("Skipping fallback model in cooldown"))
|
||||||
const hasCooldownSkip = logCalls.some((c) =>
|
expect(cooldownSkipLog).toBeDefined()
|
||||||
c.msg.includes("cooldown") || c.msg.includes("Skipping")
|
|
||||||
)
|
|
||||||
expect(hasCooldownSkip || logCalls.length <= 2).toBe(true)
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
import type { PluginInput } from "@opencode-ai/plugin"
|
import type { PluginInput } from "@opencode-ai/plugin"
|
||||||
import type { RuntimeFallbackConfig, OhMyOpenCodeConfig } from "../../config"
|
import type { RuntimeFallbackConfig, OhMyOpenCodeConfig } from "../../config"
|
||||||
import type { FallbackState, FallbackResult, RuntimeFallbackHook } from "./types"
|
import type { FallbackState, FallbackResult, RuntimeFallbackHook, RuntimeFallbackOptions } from "./types"
|
||||||
import { DEFAULT_CONFIG, RETRYABLE_ERROR_PATTERNS, HOOK_NAME } from "./constants"
|
import { DEFAULT_CONFIG, RETRYABLE_ERROR_PATTERNS, HOOK_NAME } from "./constants"
|
||||||
import { log } from "../../shared/logger"
|
import { log } from "../../shared/logger"
|
||||||
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||||
@ -10,8 +10,7 @@ function createFallbackState(originalModel: string): FallbackState {
|
|||||||
originalModel,
|
originalModel,
|
||||||
currentModel: originalModel,
|
currentModel: originalModel,
|
||||||
fallbackIndex: -1,
|
fallbackIndex: -1,
|
||||||
lastFallbackTime: 0,
|
failedModels: new Map<string, number>(),
|
||||||
failedModels: new Set<string>(),
|
|
||||||
attemptCount: 0,
|
attemptCount: 0,
|
||||||
pendingFallbackModel: undefined,
|
pendingFallbackModel: undefined,
|
||||||
}
|
}
|
||||||
@ -132,12 +131,10 @@ function getFallbackModelsForSession(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function isModelInCooldown(model: string, state: FallbackState, cooldownSeconds: number): boolean {
|
function isModelInCooldown(model: string, state: FallbackState, cooldownSeconds: number): boolean {
|
||||||
if (!state.failedModels.has(model)) return false
|
const failedAt = state.failedModels.get(model)
|
||||||
|
if (failedAt === undefined) return false
|
||||||
const cooldownMs = cooldownSeconds * 1000
|
const cooldownMs = cooldownSeconds * 1000
|
||||||
const timeSinceLastFallback = Date.now() - state.lastFallbackTime
|
return Date.now() - failedAt < cooldownMs
|
||||||
|
|
||||||
return timeSinceLastFallback < cooldownMs
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function findNextAvailableFallback(
|
function findNextAvailableFallback(
|
||||||
@ -180,9 +177,11 @@ function prepareFallback(
|
|||||||
attempt: state.attemptCount + 1,
|
attempt: state.attemptCount + 1,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const failedModel = state.currentModel
|
||||||
|
const now = Date.now()
|
||||||
|
|
||||||
state.fallbackIndex = fallbackModels.indexOf(nextModel)
|
state.fallbackIndex = fallbackModels.indexOf(nextModel)
|
||||||
state.failedModels.add(state.currentModel)
|
state.failedModels.set(failedModel, now)
|
||||||
state.lastFallbackTime = Date.now()
|
|
||||||
state.attemptCount++
|
state.attemptCount++
|
||||||
state.currentModel = nextModel
|
state.currentModel = nextModel
|
||||||
state.pendingFallbackModel = nextModel
|
state.pendingFallbackModel = nextModel
|
||||||
@ -194,7 +193,7 @@ export type { RuntimeFallbackHook, RuntimeFallbackOptions } from "./types"
|
|||||||
|
|
||||||
export function createRuntimeFallbackHook(
|
export function createRuntimeFallbackHook(
|
||||||
ctx: PluginInput,
|
ctx: PluginInput,
|
||||||
options?: { config?: RuntimeFallbackConfig }
|
options?: RuntimeFallbackOptions
|
||||||
): RuntimeFallbackHook {
|
): RuntimeFallbackHook {
|
||||||
const config: Required<RuntimeFallbackConfig> = {
|
const config: Required<RuntimeFallbackConfig> = {
|
||||||
enabled: options?.config?.enabled ?? DEFAULT_CONFIG.enabled,
|
enabled: options?.config?.enabled ?? DEFAULT_CONFIG.enabled,
|
||||||
@ -207,11 +206,15 @@ export function createRuntimeFallbackHook(
|
|||||||
const sessionStates = new Map<string, FallbackState>()
|
const sessionStates = new Map<string, FallbackState>()
|
||||||
|
|
||||||
let pluginConfig: OhMyOpenCodeConfig | undefined
|
let pluginConfig: OhMyOpenCodeConfig | undefined
|
||||||
try {
|
if (options?.pluginConfig) {
|
||||||
const { loadPluginConfig } = require("../../plugin-config")
|
pluginConfig = options.pluginConfig
|
||||||
pluginConfig = loadPluginConfig(ctx.directory, ctx)
|
} else {
|
||||||
} catch {
|
try {
|
||||||
log(`[${HOOK_NAME}] Plugin config not available`)
|
const { loadPluginConfig } = require("../../plugin-config")
|
||||||
|
pluginConfig = loadPluginConfig(ctx.directory, ctx)
|
||||||
|
} catch {
|
||||||
|
log(`[${HOOK_NAME}] Plugin config not available`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const eventHandler = async ({ event }: { event: { type: string; properties?: unknown } }) => {
|
const eventHandler = async ({ event }: { event: { type: string; properties?: unknown } }) => {
|
||||||
@ -238,6 +241,7 @@ export function createRuntimeFallbackHook(
|
|||||||
if (sessionID) {
|
if (sessionID) {
|
||||||
log(`[${HOOK_NAME}] Cleaning up session state`, { sessionID })
|
log(`[${HOOK_NAME}] Cleaning up session state`, { sessionID })
|
||||||
sessionStates.delete(sessionID)
|
sessionStates.delete(sessionID)
|
||||||
|
SessionCategoryRegistry.remove(sessionID)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,7 +4,7 @@
|
|||||||
* Types for managing runtime model fallback when API errors occur.
|
* Types for managing runtime model fallback when API errors occur.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { RuntimeFallbackConfig } from "../../config"
|
import type { RuntimeFallbackConfig, OhMyOpenCodeConfig } from "../../config"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tracks the state of fallback attempts for a session
|
* Tracks the state of fallback attempts for a session
|
||||||
@ -13,8 +13,7 @@ export interface FallbackState {
|
|||||||
originalModel: string
|
originalModel: string
|
||||||
currentModel: string
|
currentModel: string
|
||||||
fallbackIndex: number
|
fallbackIndex: number
|
||||||
lastFallbackTime: number
|
failedModels: Map<string, number>
|
||||||
failedModels: Set<string>
|
|
||||||
attemptCount: number
|
attemptCount: number
|
||||||
pendingFallbackModel?: string
|
pendingFallbackModel?: string
|
||||||
}
|
}
|
||||||
@ -57,6 +56,8 @@ export interface FallbackResult {
|
|||||||
export interface RuntimeFallbackOptions {
|
export interface RuntimeFallbackOptions {
|
||||||
/** Runtime fallback configuration */
|
/** Runtime fallback configuration */
|
||||||
config?: RuntimeFallbackConfig
|
config?: RuntimeFallbackConfig
|
||||||
|
/** Optional plugin config override (primarily for testing) */
|
||||||
|
pluginConfig?: OhMyOpenCodeConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RuntimeFallbackHook {
|
export interface RuntimeFallbackHook {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user