refactor(runtime-fallback): decompose index.ts into focused modules
Split 1021-line index.ts into 10 focused modules per project conventions. New structure: - error-classifier.ts: error analysis with dynamic status code extraction - agent-resolver.ts: agent detection utilities - fallback-state.ts: state management and cooldown logic - fallback-models.ts: model resolution from config - auto-retry.ts: retry helpers with mutual recursion support - event-handler.ts: session lifecycle events - message-update-handler.ts: message.updated event handling - chat-message-handler.ts: chat message interception - hook.ts: main factory with proper cleanup - types.ts: updated with HookDeps interface - index.ts: 2-line barrel re-export Embedded fixes: - Fix setInterval leak with .unref() - Replace require() with ESM import - Add log warning on invalid model format - Update sessionLastAccess on normal traffic - Make extractStatusCode dynamic from config - Remove unused SessionErrorInfo type All 61 tests pass without modification. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
parent
22dda6178a
commit
b6456faea8
54
src/hooks/runtime-fallback/agent-resolver.ts
Normal file
54
src/hooks/runtime-fallback/agent-resolver.ts
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
import { getSessionAgent } from "../../features/claude-code-session-state"
|
||||||
|
|
||||||
|
export const AGENT_NAMES = [
|
||||||
|
"sisyphus",
|
||||||
|
"oracle",
|
||||||
|
"librarian",
|
||||||
|
"explore",
|
||||||
|
"prometheus",
|
||||||
|
"atlas",
|
||||||
|
"metis",
|
||||||
|
"momus",
|
||||||
|
"hephaestus",
|
||||||
|
"sisyphus-junior",
|
||||||
|
"build",
|
||||||
|
"plan",
|
||||||
|
"multimodal-looker",
|
||||||
|
]
|
||||||
|
|
||||||
|
export const agentPattern = new RegExp(
|
||||||
|
`\\b(${AGENT_NAMES
|
||||||
|
.sort((a, b) => b.length - a.length)
|
||||||
|
.map((a) => a.replace(/-/g, "\\-"))
|
||||||
|
.join("|")})\\b`,
|
||||||
|
"i",
|
||||||
|
)
|
||||||
|
|
||||||
|
export function detectAgentFromSession(sessionID: string): string | undefined {
|
||||||
|
const match = sessionID.match(agentPattern)
|
||||||
|
if (match) {
|
||||||
|
return match[1].toLowerCase()
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAgentName(agent: string | undefined): string | undefined {
|
||||||
|
if (!agent) return undefined
|
||||||
|
const normalized = agent.toLowerCase().trim()
|
||||||
|
if (AGENT_NAMES.includes(normalized)) {
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
const match = normalized.match(agentPattern)
|
||||||
|
if (match) {
|
||||||
|
return match[1].toLowerCase()
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveAgentForSession(sessionID: string, eventAgent?: string): string | undefined {
|
||||||
|
return (
|
||||||
|
normalizeAgentName(eventAgent) ??
|
||||||
|
normalizeAgentName(getSessionAgent(sessionID)) ??
|
||||||
|
detectAgentFromSession(sessionID)
|
||||||
|
)
|
||||||
|
}
|
||||||
213
src/hooks/runtime-fallback/auto-retry.ts
Normal file
213
src/hooks/runtime-fallback/auto-retry.ts
Normal file
@ -0,0 +1,213 @@
|
|||||||
|
import type { HookDeps } from "./types"
|
||||||
|
import { HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { normalizeAgentName, resolveAgentForSession } from "./agent-resolver"
|
||||||
|
import { getSessionAgent } from "../../features/claude-code-session-state"
|
||||||
|
import { getFallbackModelsForSession } from "./fallback-models"
|
||||||
|
import { prepareFallback } from "./fallback-state"
|
||||||
|
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||||
|
|
||||||
|
const SESSION_TTL_MS = 30 * 60 * 1000
|
||||||
|
|
||||||
|
export function createAutoRetryHelpers(deps: HookDeps) {
|
||||||
|
const { ctx, config, options, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts, pluginConfig } = deps
|
||||||
|
|
||||||
|
const abortSessionRequest = async (sessionID: string, source: string): Promise<void> => {
|
||||||
|
try {
|
||||||
|
await ctx.client.session.abort({ path: { id: sessionID } })
|
||||||
|
log(`[${HOOK_NAME}] Aborted in-flight session request (${source})`, { sessionID })
|
||||||
|
} catch (error) {
|
||||||
|
log(`[${HOOK_NAME}] Failed to abort in-flight session request (${source})`, {
|
||||||
|
sessionID,
|
||||||
|
error: String(error),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const clearSessionFallbackTimeout = (sessionID: string) => {
|
||||||
|
const timer = sessionFallbackTimeouts.get(sessionID)
|
||||||
|
if (timer) {
|
||||||
|
clearTimeout(timer)
|
||||||
|
sessionFallbackTimeouts.delete(sessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const scheduleSessionFallbackTimeout = (sessionID: string, resolvedAgent?: string) => {
|
||||||
|
clearSessionFallbackTimeout(sessionID)
|
||||||
|
|
||||||
|
const timeoutMs = options?.session_timeout_ms ?? config.timeout_seconds * 1000
|
||||||
|
if (timeoutMs <= 0) return
|
||||||
|
|
||||||
|
const timer = setTimeout(async () => {
|
||||||
|
sessionFallbackTimeouts.delete(sessionID)
|
||||||
|
|
||||||
|
const state = sessionStates.get(sessionID)
|
||||||
|
if (!state) return
|
||||||
|
|
||||||
|
if (sessionRetryInFlight.has(sessionID)) {
|
||||||
|
log(`[${HOOK_NAME}] Overriding in-flight retry due to session timeout`, { sessionID })
|
||||||
|
}
|
||||||
|
|
||||||
|
await abortSessionRequest(sessionID, "session.timeout")
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
|
||||||
|
if (state.pendingFallbackModel) {
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
|
||||||
|
if (fallbackModels.length === 0) return
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] Session fallback timeout reached`, {
|
||||||
|
sessionID,
|
||||||
|
timeoutSeconds: config.timeout_seconds,
|
||||||
|
currentModel: state.currentModel,
|
||||||
|
})
|
||||||
|
|
||||||
|
const result = prepareFallback(sessionID, state, fallbackModels, config)
|
||||||
|
if (result.success && result.newModel) {
|
||||||
|
await autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.timeout")
|
||||||
|
}
|
||||||
|
}, timeoutMs)
|
||||||
|
|
||||||
|
sessionFallbackTimeouts.set(sessionID, timer)
|
||||||
|
}
|
||||||
|
|
||||||
|
const autoRetryWithFallback = async (
|
||||||
|
sessionID: string,
|
||||||
|
newModel: string,
|
||||||
|
resolvedAgent: string | undefined,
|
||||||
|
source: string,
|
||||||
|
): Promise<void> => {
|
||||||
|
if (sessionRetryInFlight.has(sessionID)) {
|
||||||
|
log(`[${HOOK_NAME}] Retry already in flight, skipping (${source})`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelParts = newModel.split("/")
|
||||||
|
if (modelParts.length < 2) {
|
||||||
|
log(`[${HOOK_NAME}] Invalid model format (missing provider prefix): ${newModel}`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const fallbackModelObj = {
|
||||||
|
providerID: modelParts[0],
|
||||||
|
modelID: modelParts.slice(1).join("/"),
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionRetryInFlight.add(sessionID)
|
||||||
|
try {
|
||||||
|
const messagesResp = await ctx.client.session.messages({
|
||||||
|
path: { id: sessionID },
|
||||||
|
query: { directory: ctx.directory },
|
||||||
|
})
|
||||||
|
const msgs = (messagesResp as {
|
||||||
|
data?: Array<{
|
||||||
|
info?: Record<string, unknown>
|
||||||
|
parts?: Array<{ type?: string; text?: string }>
|
||||||
|
}>
|
||||||
|
}).data
|
||||||
|
const lastUserMsg = msgs?.filter((m) => m.info?.role === "user").pop()
|
||||||
|
const lastUserPartsRaw =
|
||||||
|
lastUserMsg?.parts ??
|
||||||
|
(lastUserMsg?.info?.parts as Array<{ type?: string; text?: string }> | undefined)
|
||||||
|
|
||||||
|
if (lastUserPartsRaw && lastUserPartsRaw.length > 0) {
|
||||||
|
log(`[${HOOK_NAME}] Auto-retrying with fallback model (${source})`, {
|
||||||
|
sessionID,
|
||||||
|
model: newModel,
|
||||||
|
})
|
||||||
|
|
||||||
|
const retryParts = lastUserPartsRaw
|
||||||
|
.filter((p) => p.type === "text" && typeof p.text === "string" && p.text.length > 0)
|
||||||
|
.map((p) => ({ type: "text" as const, text: p.text! }))
|
||||||
|
|
||||||
|
if (retryParts.length > 0) {
|
||||||
|
const retryAgent = resolvedAgent ?? getSessionAgent(sessionID)
|
||||||
|
sessionAwaitingFallbackResult.add(sessionID)
|
||||||
|
scheduleSessionFallbackTimeout(sessionID, retryAgent)
|
||||||
|
|
||||||
|
await ctx.client.session.promptAsync({
|
||||||
|
path: { id: sessionID },
|
||||||
|
body: {
|
||||||
|
...(retryAgent ? { agent: retryAgent } : {}),
|
||||||
|
model: fallbackModelObj,
|
||||||
|
parts: retryParts,
|
||||||
|
},
|
||||||
|
query: { directory: ctx.directory },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log(`[${HOOK_NAME}] No user message found for auto-retry (${source})`, { sessionID })
|
||||||
|
}
|
||||||
|
} catch (retryError) {
|
||||||
|
log(`[${HOOK_NAME}] Auto-retry failed (${source})`, { sessionID, error: String(retryError) })
|
||||||
|
} finally {
|
||||||
|
const state = sessionStates.get(sessionID)
|
||||||
|
if (state?.pendingFallbackModel === newModel) {
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
}
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const resolveAgentForSessionFromContext = async (
|
||||||
|
sessionID: string,
|
||||||
|
eventAgent?: string,
|
||||||
|
): Promise<string | undefined> => {
|
||||||
|
const resolved = resolveAgentForSession(sessionID, eventAgent)
|
||||||
|
if (resolved) return resolved
|
||||||
|
|
||||||
|
try {
|
||||||
|
const messagesResp = await ctx.client.session.messages({
|
||||||
|
path: { id: sessionID },
|
||||||
|
query: { directory: ctx.directory },
|
||||||
|
})
|
||||||
|
const msgs = (messagesResp as { data?: Array<{ info?: Record<string, unknown> }> }).data
|
||||||
|
if (!msgs || msgs.length === 0) return undefined
|
||||||
|
|
||||||
|
for (let i = msgs.length - 1; i >= 0; i--) {
|
||||||
|
const info = msgs[i]?.info
|
||||||
|
const infoAgent = typeof info?.agent === "string" ? info.agent : undefined
|
||||||
|
const normalized = normalizeAgentName(infoAgent)
|
||||||
|
if (normalized) {
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
const cleanupStaleSessions = () => {
|
||||||
|
const now = Date.now()
|
||||||
|
let cleanedCount = 0
|
||||||
|
for (const [sessionID, lastAccess] of sessionLastAccess.entries()) {
|
||||||
|
if (now - lastAccess > SESSION_TTL_MS) {
|
||||||
|
sessionStates.delete(sessionID)
|
||||||
|
sessionLastAccess.delete(sessionID)
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
sessionAwaitingFallbackResult.delete(sessionID)
|
||||||
|
clearSessionFallbackTimeout(sessionID)
|
||||||
|
SessionCategoryRegistry.remove(sessionID)
|
||||||
|
cleanedCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cleanedCount > 0) {
|
||||||
|
log(`[${HOOK_NAME}] Cleaned up ${cleanedCount} stale session states`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
abortSessionRequest,
|
||||||
|
clearSessionFallbackTimeout,
|
||||||
|
scheduleSessionFallbackTimeout,
|
||||||
|
autoRetryWithFallback,
|
||||||
|
resolveAgentForSessionFromContext,
|
||||||
|
cleanupStaleSessions,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AutoRetryHelpers = ReturnType<typeof createAutoRetryHelpers>
|
||||||
62
src/hooks/runtime-fallback/chat-message-handler.ts
Normal file
62
src/hooks/runtime-fallback/chat-message-handler.ts
Normal file
@ -0,0 +1,62 @@
|
|||||||
|
import type { HookDeps } from "./types"
|
||||||
|
import { HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { createFallbackState } from "./fallback-state"
|
||||||
|
|
||||||
|
export function createChatMessageHandler(deps: HookDeps) {
|
||||||
|
const { config, sessionStates, sessionLastAccess } = deps
|
||||||
|
|
||||||
|
return async (
|
||||||
|
input: { sessionID: string; agent?: string; model?: { providerID: string; modelID: string } },
|
||||||
|
output: { message: { model?: { providerID: string; modelID: string } }; parts?: Array<{ type: string; text?: string }> }
|
||||||
|
) => {
|
||||||
|
if (!config.enabled) return
|
||||||
|
|
||||||
|
const { sessionID } = input
|
||||||
|
let state = sessionStates.get(sessionID)
|
||||||
|
|
||||||
|
if (!state) return
|
||||||
|
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
|
||||||
|
const requestedModel = input.model
|
||||||
|
? `${input.model.providerID}/${input.model.modelID}`
|
||||||
|
: undefined
|
||||||
|
|
||||||
|
if (requestedModel && requestedModel !== state.currentModel) {
|
||||||
|
if (state.pendingFallbackModel && state.pendingFallbackModel === requestedModel) {
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] Detected manual model change, resetting fallback state`, {
|
||||||
|
sessionID,
|
||||||
|
from: state.currentModel,
|
||||||
|
to: requestedModel,
|
||||||
|
})
|
||||||
|
state = createFallbackState(requestedModel)
|
||||||
|
sessionStates.set(sessionID, state)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.currentModel === state.originalModel) return
|
||||||
|
|
||||||
|
const activeModel = state.currentModel
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] Applying fallback model override`, {
|
||||||
|
sessionID,
|
||||||
|
from: input.model,
|
||||||
|
to: activeModel,
|
||||||
|
})
|
||||||
|
|
||||||
|
if (output.message && activeModel) {
|
||||||
|
const parts = activeModel.split("/")
|
||||||
|
if (parts.length >= 2) {
|
||||||
|
output.message.model = {
|
||||||
|
providerID: parts[0],
|
||||||
|
modelID: parts.slice(1).join("/"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
154
src/hooks/runtime-fallback/error-classifier.ts
Normal file
154
src/hooks/runtime-fallback/error-classifier.ts
Normal file
@ -0,0 +1,154 @@
|
|||||||
|
import { DEFAULT_CONFIG, RETRYABLE_ERROR_PATTERNS } from "./constants"
|
||||||
|
|
||||||
|
export function getErrorMessage(error: unknown): string {
|
||||||
|
if (!error) return ""
|
||||||
|
if (typeof error === "string") return error.toLowerCase()
|
||||||
|
|
||||||
|
const errorObj = error as Record<string, unknown>
|
||||||
|
const paths = [
|
||||||
|
errorObj.data,
|
||||||
|
errorObj.error,
|
||||||
|
errorObj,
|
||||||
|
(errorObj.data as Record<string, unknown>)?.error,
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const obj of paths) {
|
||||||
|
if (obj && typeof obj === "object") {
|
||||||
|
const msg = (obj as Record<string, unknown>).message
|
||||||
|
if (typeof msg === "string" && msg.length > 0) {
|
||||||
|
return msg.toLowerCase()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return JSON.stringify(error).toLowerCase()
|
||||||
|
} catch {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractStatusCode(error: unknown, retryOnErrors?: number[]): number | undefined {
|
||||||
|
if (!error) return undefined
|
||||||
|
|
||||||
|
const errorObj = error as Record<string, unknown>
|
||||||
|
|
||||||
|
const statusCode = errorObj.statusCode ?? errorObj.status ?? (errorObj.data as Record<string, unknown>)?.statusCode
|
||||||
|
if (typeof statusCode === "number") {
|
||||||
|
return statusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
const codes = retryOnErrors ?? DEFAULT_CONFIG.retry_on_errors
|
||||||
|
const pattern = new RegExp(`\\b(${codes.join("|")})\\b`)
|
||||||
|
const message = getErrorMessage(error)
|
||||||
|
const statusMatch = message.match(pattern)
|
||||||
|
if (statusMatch) {
|
||||||
|
return parseInt(statusMatch[1], 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractErrorName(error: unknown): string | undefined {
|
||||||
|
if (!error || typeof error !== "object") return undefined
|
||||||
|
|
||||||
|
const errorObj = error as Record<string, unknown>
|
||||||
|
const directName = errorObj.name
|
||||||
|
if (typeof directName === "string" && directName.length > 0) {
|
||||||
|
return directName
|
||||||
|
}
|
||||||
|
|
||||||
|
const nestedError = errorObj.error as Record<string, unknown> | undefined
|
||||||
|
const nestedName = nestedError?.name
|
||||||
|
if (typeof nestedName === "string" && nestedName.length > 0) {
|
||||||
|
return nestedName
|
||||||
|
}
|
||||||
|
|
||||||
|
const dataError = (errorObj.data as Record<string, unknown> | undefined)?.error as Record<string, unknown> | undefined
|
||||||
|
const dataErrorName = dataError?.name
|
||||||
|
if (typeof dataErrorName === "string" && dataErrorName.length > 0) {
|
||||||
|
return dataErrorName
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function classifyErrorType(error: unknown): string | undefined {
|
||||||
|
const message = getErrorMessage(error)
|
||||||
|
const errorName = extractErrorName(error)?.toLowerCase()
|
||||||
|
|
||||||
|
if (
|
||||||
|
errorName?.includes("loadapi") ||
|
||||||
|
(/api.?key.?is.?missing/i.test(message) && /environment variable/i.test(message))
|
||||||
|
) {
|
||||||
|
return "missing_api_key"
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/api.?key/i.test(message) && /must be a string/i.test(message)) {
|
||||||
|
return "invalid_api_key"
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errorName?.includes("unknownerror") && /model\s+not\s+found/i.test(message)) {
|
||||||
|
return "model_not_found"
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AutoRetrySignal {
|
||||||
|
signal: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||||
|
(combined) => /retrying\s+in/i.test(combined),
|
||||||
|
(combined) =>
|
||||||
|
/(?:too\s+many\s+requests|quota\s*exceeded|usage\s+limit|rate\s+limit|limit\s+reached)/i.test(combined),
|
||||||
|
]
|
||||||
|
|
||||||
|
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||||
|
if (!info) return undefined
|
||||||
|
|
||||||
|
const candidates: string[] = []
|
||||||
|
|
||||||
|
const directStatus = info.status
|
||||||
|
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||||
|
|
||||||
|
const summary = info.summary
|
||||||
|
if (typeof summary === "string") candidates.push(summary)
|
||||||
|
|
||||||
|
const message = info.message
|
||||||
|
if (typeof message === "string") candidates.push(message)
|
||||||
|
|
||||||
|
const details = info.details
|
||||||
|
if (typeof details === "string") candidates.push(details)
|
||||||
|
|
||||||
|
const combined = candidates.join("\n")
|
||||||
|
if (!combined) return undefined
|
||||||
|
|
||||||
|
const isAutoRetry = AUTO_RETRY_PATTERNS.every((test) => test(combined))
|
||||||
|
if (isAutoRetry) {
|
||||||
|
return { signal: combined }
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRetryableError(error: unknown, retryOnErrors: number[]): boolean {
|
||||||
|
const statusCode = extractStatusCode(error, retryOnErrors)
|
||||||
|
const message = getErrorMessage(error)
|
||||||
|
const errorType = classifyErrorType(error)
|
||||||
|
|
||||||
|
if (errorType === "missing_api_key") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errorType === "model_not_found") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if (statusCode && retryOnErrors.includes(statusCode)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return RETRYABLE_ERROR_PATTERNS.some((pattern) => pattern.test(message))
|
||||||
|
}
|
||||||
187
src/hooks/runtime-fallback/event-handler.ts
Normal file
187
src/hooks/runtime-fallback/event-handler.ts
Normal file
@ -0,0 +1,187 @@
|
|||||||
|
import type { HookDeps } from "./types"
|
||||||
|
import type { AutoRetryHelpers } from "./auto-retry"
|
||||||
|
import { HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError } from "./error-classifier"
|
||||||
|
import { createFallbackState, prepareFallback } from "./fallback-state"
|
||||||
|
import { getFallbackModelsForSession } from "./fallback-models"
|
||||||
|
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||||
|
|
||||||
|
export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
|
||||||
|
const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
|
||||||
|
|
||||||
|
const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
|
||||||
|
const sessionInfo = props?.info as { id?: string; model?: string } | undefined
|
||||||
|
const sessionID = sessionInfo?.id
|
||||||
|
const model = sessionInfo?.model
|
||||||
|
|
||||||
|
if (sessionID && model) {
|
||||||
|
log(`[${HOOK_NAME}] Session created with model`, { sessionID, model })
|
||||||
|
sessionStates.set(sessionID, createFallbackState(model))
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSessionDeleted = (props: Record<string, unknown> | undefined) => {
|
||||||
|
const sessionInfo = props?.info as { id?: string } | undefined
|
||||||
|
const sessionID = sessionInfo?.id
|
||||||
|
|
||||||
|
if (sessionID) {
|
||||||
|
log(`[${HOOK_NAME}] Cleaning up session state`, { sessionID })
|
||||||
|
sessionStates.delete(sessionID)
|
||||||
|
sessionLastAccess.delete(sessionID)
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
sessionAwaitingFallbackResult.delete(sessionID)
|
||||||
|
helpers.clearSessionFallbackTimeout(sessionID)
|
||||||
|
SessionCategoryRegistry.remove(sessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSessionStop = async (props: Record<string, unknown> | undefined) => {
|
||||||
|
const sessionID = props?.sessionID as string | undefined
|
||||||
|
if (!sessionID) return
|
||||||
|
|
||||||
|
helpers.clearSessionFallbackTimeout(sessionID)
|
||||||
|
|
||||||
|
if (sessionRetryInFlight.has(sessionID)) {
|
||||||
|
await helpers.abortSessionRequest(sessionID, "session.stop")
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
sessionAwaitingFallbackResult.delete(sessionID)
|
||||||
|
|
||||||
|
const state = sessionStates.get(sessionID)
|
||||||
|
if (state?.pendingFallbackModel) {
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] Cleared fallback retry state on session.stop`, { sessionID })
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSessionIdle = (props: Record<string, unknown> | undefined) => {
|
||||||
|
const sessionID = props?.sessionID as string | undefined
|
||||||
|
if (!sessionID) return
|
||||||
|
|
||||||
|
if (sessionAwaitingFallbackResult.has(sessionID)) {
|
||||||
|
log(`[${HOOK_NAME}] session.idle while awaiting fallback result; keeping timeout armed`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const hadTimeout = sessionFallbackTimeouts.has(sessionID)
|
||||||
|
helpers.clearSessionFallbackTimeout(sessionID)
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
|
||||||
|
const state = sessionStates.get(sessionID)
|
||||||
|
if (state?.pendingFallbackModel) {
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hadTimeout) {
|
||||||
|
log(`[${HOOK_NAME}] Cleared fallback timeout after session completion`, { sessionID })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSessionError = async (props: Record<string, unknown> | undefined) => {
|
||||||
|
const sessionID = props?.sessionID as string | undefined
|
||||||
|
const error = props?.error
|
||||||
|
const agent = props?.agent as string | undefined
|
||||||
|
|
||||||
|
if (!sessionID) {
|
||||||
|
log(`[${HOOK_NAME}] session.error without sessionID, skipping`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
|
||||||
|
sessionAwaitingFallbackResult.delete(sessionID)
|
||||||
|
helpers.clearSessionFallbackTimeout(sessionID)
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] session.error received`, {
|
||||||
|
sessionID,
|
||||||
|
agent,
|
||||||
|
resolvedAgent,
|
||||||
|
statusCode: extractStatusCode(error, config.retry_on_errors),
|
||||||
|
errorName: extractErrorName(error),
|
||||||
|
errorType: classifyErrorType(error),
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!isRetryableError(error, config.retry_on_errors)) {
|
||||||
|
log(`[${HOOK_NAME}] Error not retryable, skipping fallback`, {
|
||||||
|
sessionID,
|
||||||
|
retryable: false,
|
||||||
|
statusCode: extractStatusCode(error, config.retry_on_errors),
|
||||||
|
errorName: extractErrorName(error),
|
||||||
|
errorType: classifyErrorType(error),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let state = sessionStates.get(sessionID)
|
||||||
|
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
|
||||||
|
|
||||||
|
if (fallbackModels.length === 0) {
|
||||||
|
log(`[${HOOK_NAME}] No fallback models configured`, { sessionID, agent })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!state) {
|
||||||
|
const currentModel = props?.model as string | undefined
|
||||||
|
if (currentModel) {
|
||||||
|
state = createFallbackState(currentModel)
|
||||||
|
sessionStates.set(sessionID, state)
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
} else {
|
||||||
|
const detectedAgent = resolvedAgent
|
||||||
|
const agentConfig = detectedAgent
|
||||||
|
? pluginConfig?.agents?.[detectedAgent as keyof typeof pluginConfig.agents]
|
||||||
|
: undefined
|
||||||
|
const agentModel = agentConfig?.model as string | undefined
|
||||||
|
if (agentModel) {
|
||||||
|
log(`[${HOOK_NAME}] Derived model from agent config`, { sessionID, agent: detectedAgent, model: agentModel })
|
||||||
|
state = createFallbackState(agentModel)
|
||||||
|
sessionStates.set(sessionID, state)
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
} else {
|
||||||
|
log(`[${HOOK_NAME}] No model info available, cannot fallback`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = prepareFallback(sessionID, state, fallbackModels, config)
|
||||||
|
|
||||||
|
if (result.success && config.notify_on_fallback) {
|
||||||
|
await deps.ctx.client.tui
|
||||||
|
.showToast({
|
||||||
|
body: {
|
||||||
|
title: "Model Fallback",
|
||||||
|
message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
|
||||||
|
variant: "warning",
|
||||||
|
duration: 5000,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.catch(() => {})
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.success && result.newModel) {
|
||||||
|
await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.error")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.success) {
|
||||||
|
log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return async ({ event }: { event: { type: string; properties?: unknown } }) => {
|
||||||
|
if (!config.enabled) return
|
||||||
|
|
||||||
|
const props = event.properties as Record<string, unknown> | undefined
|
||||||
|
|
||||||
|
if (event.type === "session.created") { handleSessionCreated(props); return }
|
||||||
|
if (event.type === "session.deleted") { handleSessionDeleted(props); return }
|
||||||
|
if (event.type === "session.stop") { await handleSessionStop(props); return }
|
||||||
|
if (event.type === "session.idle") { handleSessionIdle(props); return }
|
||||||
|
if (event.type === "session.error") { await handleSessionError(props); return }
|
||||||
|
}
|
||||||
|
}
|
||||||
69
src/hooks/runtime-fallback/fallback-models.ts
Normal file
69
src/hooks/runtime-fallback/fallback-models.ts
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
import type { OhMyOpenCodeConfig } from "../../config"
|
||||||
|
import { AGENT_NAMES, agentPattern } from "./agent-resolver"
|
||||||
|
import { HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||||
|
import { normalizeFallbackModels } from "../../shared/model-resolver"
|
||||||
|
|
||||||
|
export function getFallbackModelsForSession(
|
||||||
|
sessionID: string,
|
||||||
|
agent: string | undefined,
|
||||||
|
pluginConfig: OhMyOpenCodeConfig | undefined
|
||||||
|
): string[] {
|
||||||
|
if (!pluginConfig) return []
|
||||||
|
|
||||||
|
const sessionCategory = SessionCategoryRegistry.get(sessionID)
|
||||||
|
if (sessionCategory && pluginConfig.categories?.[sessionCategory]) {
|
||||||
|
const categoryConfig = pluginConfig.categories[sessionCategory]
|
||||||
|
if (categoryConfig?.fallback_models) {
|
||||||
|
return normalizeFallbackModels(categoryConfig.fallback_models) ?? []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const tryGetFallbackFromAgent = (agentName: string): string[] | undefined => {
|
||||||
|
const agentConfig = pluginConfig.agents?.[agentName as keyof typeof pluginConfig.agents]
|
||||||
|
if (!agentConfig) return undefined
|
||||||
|
|
||||||
|
if (agentConfig?.fallback_models) {
|
||||||
|
return normalizeFallbackModels(agentConfig.fallback_models)
|
||||||
|
}
|
||||||
|
|
||||||
|
const agentCategory = agentConfig?.category
|
||||||
|
if (agentCategory && pluginConfig.categories?.[agentCategory]) {
|
||||||
|
const categoryConfig = pluginConfig.categories[agentCategory]
|
||||||
|
if (categoryConfig?.fallback_models) {
|
||||||
|
return normalizeFallbackModels(categoryConfig.fallback_models)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
if (agent) {
|
||||||
|
const result = tryGetFallbackFromAgent(agent)
|
||||||
|
if (result) return result
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionAgentMatch = sessionID.match(agentPattern)
|
||||||
|
if (sessionAgentMatch) {
|
||||||
|
const detectedAgent = sessionAgentMatch[1].toLowerCase()
|
||||||
|
const result = tryGetFallbackFromAgent(detectedAgent)
|
||||||
|
if (result) return result
|
||||||
|
}
|
||||||
|
|
||||||
|
const sisyphusFallback = tryGetFallbackFromAgent("sisyphus")
|
||||||
|
if (sisyphusFallback) {
|
||||||
|
log(`[${HOOK_NAME}] Using sisyphus fallback models (no agent detected)`, { sessionID })
|
||||||
|
return sisyphusFallback
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const agentName of AGENT_NAMES) {
|
||||||
|
const result = tryGetFallbackFromAgent(agentName)
|
||||||
|
if (result) {
|
||||||
|
log(`[${HOOK_NAME}] Using ${agentName} fallback models (no agent detected)`, { sessionID })
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return []
|
||||||
|
}
|
||||||
74
src/hooks/runtime-fallback/fallback-state.ts
Normal file
74
src/hooks/runtime-fallback/fallback-state.ts
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
import type { FallbackState, FallbackResult } from "./types"
|
||||||
|
import { HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import type { RuntimeFallbackConfig } from "../../config"
|
||||||
|
|
||||||
|
export function createFallbackState(originalModel: string): FallbackState {
|
||||||
|
return {
|
||||||
|
originalModel,
|
||||||
|
currentModel: originalModel,
|
||||||
|
fallbackIndex: -1,
|
||||||
|
failedModels: new Map<string, number>(),
|
||||||
|
attemptCount: 0,
|
||||||
|
pendingFallbackModel: undefined,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isModelInCooldown(model: string, state: FallbackState, cooldownSeconds: number): boolean {
|
||||||
|
const failedAt = state.failedModels.get(model)
|
||||||
|
if (failedAt === undefined) return false
|
||||||
|
const cooldownMs = cooldownSeconds * 1000
|
||||||
|
return Date.now() - failedAt < cooldownMs
|
||||||
|
}
|
||||||
|
|
||||||
|
export function findNextAvailableFallback(
|
||||||
|
state: FallbackState,
|
||||||
|
fallbackModels: string[],
|
||||||
|
cooldownSeconds: number
|
||||||
|
): string | undefined {
|
||||||
|
for (let i = state.fallbackIndex + 1; i < fallbackModels.length; i++) {
|
||||||
|
const candidate = fallbackModels[i]
|
||||||
|
if (!isModelInCooldown(candidate, state, cooldownSeconds)) {
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
log(`[${HOOK_NAME}] Skipping fallback model in cooldown`, { model: candidate, index: i })
|
||||||
|
}
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
export function prepareFallback(
|
||||||
|
sessionID: string,
|
||||||
|
state: FallbackState,
|
||||||
|
fallbackModels: string[],
|
||||||
|
config: Required<RuntimeFallbackConfig>
|
||||||
|
): FallbackResult {
|
||||||
|
if (state.attemptCount >= config.max_fallback_attempts) {
|
||||||
|
log(`[${HOOK_NAME}] Max fallback attempts reached`, { sessionID, attempts: state.attemptCount })
|
||||||
|
return { success: false, error: "Max fallback attempts reached", maxAttemptsReached: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextModel = findNextAvailableFallback(state, fallbackModels, config.cooldown_seconds)
|
||||||
|
|
||||||
|
if (!nextModel) {
|
||||||
|
log(`[${HOOK_NAME}] No available fallback models`, { sessionID })
|
||||||
|
return { success: false, error: "No available fallback models (all in cooldown or exhausted)" }
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] Preparing fallback`, {
|
||||||
|
sessionID,
|
||||||
|
from: state.currentModel,
|
||||||
|
to: nextModel,
|
||||||
|
attempt: state.attemptCount + 1,
|
||||||
|
})
|
||||||
|
|
||||||
|
const failedModel = state.currentModel
|
||||||
|
const now = Date.now()
|
||||||
|
|
||||||
|
state.fallbackIndex = fallbackModels.indexOf(nextModel)
|
||||||
|
state.failedModels.set(failedModel, now)
|
||||||
|
state.attemptCount++
|
||||||
|
state.currentModel = nextModel
|
||||||
|
state.pendingFallbackModel = nextModel
|
||||||
|
|
||||||
|
return { success: true, newModel: nextModel }
|
||||||
|
}
|
||||||
67
src/hooks/runtime-fallback/hook.ts
Normal file
67
src/hooks/runtime-fallback/hook.ts
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
import type { PluginInput } from "@opencode-ai/plugin"
|
||||||
|
import type { HookDeps, RuntimeFallbackHook, RuntimeFallbackOptions } from "./types"
|
||||||
|
import { DEFAULT_CONFIG, HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { loadPluginConfig } from "../../plugin-config"
|
||||||
|
import { createAutoRetryHelpers } from "./auto-retry"
|
||||||
|
import { createEventHandler } from "./event-handler"
|
||||||
|
import { createMessageUpdateHandler } from "./message-update-handler"
|
||||||
|
import { createChatMessageHandler } from "./chat-message-handler"
|
||||||
|
|
||||||
|
export function createRuntimeFallbackHook(
|
||||||
|
ctx: PluginInput,
|
||||||
|
options?: RuntimeFallbackOptions
|
||||||
|
): RuntimeFallbackHook {
|
||||||
|
const config = {
|
||||||
|
enabled: options?.config?.enabled ?? DEFAULT_CONFIG.enabled,
|
||||||
|
retry_on_errors: options?.config?.retry_on_errors ?? DEFAULT_CONFIG.retry_on_errors,
|
||||||
|
max_fallback_attempts: options?.config?.max_fallback_attempts ?? DEFAULT_CONFIG.max_fallback_attempts,
|
||||||
|
cooldown_seconds: options?.config?.cooldown_seconds ?? DEFAULT_CONFIG.cooldown_seconds,
|
||||||
|
timeout_seconds: options?.config?.timeout_seconds ?? DEFAULT_CONFIG.timeout_seconds,
|
||||||
|
notify_on_fallback: options?.config?.notify_on_fallback ?? DEFAULT_CONFIG.notify_on_fallback,
|
||||||
|
}
|
||||||
|
|
||||||
|
let pluginConfig = options?.pluginConfig
|
||||||
|
if (!pluginConfig) {
|
||||||
|
try {
|
||||||
|
pluginConfig = loadPluginConfig(ctx.directory, ctx)
|
||||||
|
} catch {
|
||||||
|
log(`[${HOOK_NAME}] Plugin config not available`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const deps: HookDeps = {
|
||||||
|
ctx,
|
||||||
|
config,
|
||||||
|
options,
|
||||||
|
pluginConfig,
|
||||||
|
sessionStates: new Map(),
|
||||||
|
sessionLastAccess: new Map(),
|
||||||
|
sessionRetryInFlight: new Set(),
|
||||||
|
sessionAwaitingFallbackResult: new Set(),
|
||||||
|
sessionFallbackTimeouts: new Map(),
|
||||||
|
}
|
||||||
|
|
||||||
|
const helpers = createAutoRetryHelpers(deps)
|
||||||
|
const baseEventHandler = createEventHandler(deps, helpers)
|
||||||
|
const messageUpdateHandler = createMessageUpdateHandler(deps, helpers)
|
||||||
|
const chatMessageHandler = createChatMessageHandler(deps)
|
||||||
|
|
||||||
|
const cleanupInterval = setInterval(helpers.cleanupStaleSessions, 5 * 60 * 1000)
|
||||||
|
cleanupInterval.unref()
|
||||||
|
|
||||||
|
const eventHandler = async ({ event }: { event: { type: string; properties?: unknown } }) => {
|
||||||
|
if (event.type === "message.updated") {
|
||||||
|
if (!config.enabled) return
|
||||||
|
const props = event.properties as Record<string, unknown> | undefined
|
||||||
|
await messageUpdateHandler(props)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await baseEventHandler({ event })
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
event: eventHandler,
|
||||||
|
"chat.message": chatMessageHandler,
|
||||||
|
} as RuntimeFallbackHook
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
212
src/hooks/runtime-fallback/message-update-handler.ts
Normal file
212
src/hooks/runtime-fallback/message-update-handler.ts
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
import type { HookDeps } from "./types"
|
||||||
|
import type { AutoRetryHelpers } from "./auto-retry"
|
||||||
|
import { HOOK_NAME } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError, extractAutoRetrySignal } from "./error-classifier"
|
||||||
|
import { createFallbackState, prepareFallback } from "./fallback-state"
|
||||||
|
import { getFallbackModelsForSession } from "./fallback-models"
|
||||||
|
|
||||||
|
export function hasVisibleAssistantResponse(extractAutoRetrySignalFn: typeof extractAutoRetrySignal) {
|
||||||
|
return async (
|
||||||
|
ctx: HookDeps["ctx"],
|
||||||
|
sessionID: string,
|
||||||
|
_info: Record<string, unknown> | undefined,
|
||||||
|
): Promise<boolean> => {
|
||||||
|
try {
|
||||||
|
const messagesResp = await ctx.client.session.messages({
|
||||||
|
path: { id: sessionID },
|
||||||
|
query: { directory: ctx.directory },
|
||||||
|
})
|
||||||
|
|
||||||
|
const msgs = (messagesResp as {
|
||||||
|
data?: Array<{
|
||||||
|
info?: Record<string, unknown>
|
||||||
|
parts?: Array<{ type?: string; text?: string }>
|
||||||
|
}>
|
||||||
|
}).data
|
||||||
|
|
||||||
|
if (!msgs || msgs.length === 0) return false
|
||||||
|
|
||||||
|
const lastAssistant = [...msgs].reverse().find((m) => m.info?.role === "assistant")
|
||||||
|
if (!lastAssistant) return false
|
||||||
|
if (lastAssistant.info?.error) return false
|
||||||
|
|
||||||
|
const parts = lastAssistant.parts ??
|
||||||
|
(lastAssistant.info?.parts as Array<{ type?: string; text?: string }> | undefined)
|
||||||
|
|
||||||
|
const textFromParts = (parts ?? [])
|
||||||
|
.filter((p) => p.type === "text" && typeof p.text === "string")
|
||||||
|
.map((p) => p.text!.trim())
|
||||||
|
.filter((text) => text.length > 0)
|
||||||
|
.join("\n")
|
||||||
|
|
||||||
|
if (!textFromParts) return false
|
||||||
|
if (extractAutoRetrySignalFn({ message: textFromParts })) return false
|
||||||
|
|
||||||
|
return true
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createMessageUpdateHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
|
||||||
|
const { ctx, config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult } = deps
|
||||||
|
const checkVisibleResponse = hasVisibleAssistantResponse(extractAutoRetrySignal)
|
||||||
|
|
||||||
|
return async (props: Record<string, unknown> | undefined) => {
|
||||||
|
const info = props?.info as Record<string, unknown> | undefined
|
||||||
|
const sessionID = info?.sessionID as string | undefined
|
||||||
|
const retrySignalResult = extractAutoRetrySignal(info)
|
||||||
|
const retrySignal = retrySignalResult?.signal
|
||||||
|
const timeoutEnabled = config.timeout_seconds > 0
|
||||||
|
const error = info?.error ?? (retrySignal && timeoutEnabled ? { name: "ProviderRateLimitError", message: retrySignal } : undefined)
|
||||||
|
const role = info?.role as string | undefined
|
||||||
|
const model = info?.model as string | undefined
|
||||||
|
|
||||||
|
if (sessionID && role === "assistant" && !error) {
|
||||||
|
if (!sessionAwaitingFallbackResult.has(sessionID)) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasVisible = await checkVisibleResponse(ctx, sessionID, info)
|
||||||
|
if (!hasVisible) {
|
||||||
|
log(`[${HOOK_NAME}] Assistant update observed without visible final response; keeping fallback timeout`, {
|
||||||
|
sessionID,
|
||||||
|
model,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionAwaitingFallbackResult.delete(sessionID)
|
||||||
|
helpers.clearSessionFallbackTimeout(sessionID)
|
||||||
|
const state = sessionStates.get(sessionID)
|
||||||
|
if (state?.pendingFallbackModel) {
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
}
|
||||||
|
log(`[${HOOK_NAME}] Assistant response observed; cleared fallback timeout`, { sessionID, model })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sessionID && role === "assistant" && error) {
|
||||||
|
sessionAwaitingFallbackResult.delete(sessionID)
|
||||||
|
if (sessionRetryInFlight.has(sessionID) && !retrySignal) {
|
||||||
|
log(`[${HOOK_NAME}] message.updated fallback skipped (retry in flight)`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (retrySignal && sessionRetryInFlight.has(sessionID) && timeoutEnabled) {
|
||||||
|
log(`[${HOOK_NAME}] Overriding in-flight retry due to provider auto-retry signal`, {
|
||||||
|
sessionID,
|
||||||
|
model,
|
||||||
|
})
|
||||||
|
await helpers.abortSessionRequest(sessionID, "message.updated.retry-signal")
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (retrySignal && timeoutEnabled) {
|
||||||
|
log(`[${HOOK_NAME}] Detected provider auto-retry signal`, { sessionID, model })
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!retrySignal) {
|
||||||
|
helpers.clearSessionFallbackTimeout(sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] message.updated with assistant error`, {
|
||||||
|
sessionID,
|
||||||
|
model,
|
||||||
|
statusCode: extractStatusCode(error, config.retry_on_errors),
|
||||||
|
errorName: extractErrorName(error),
|
||||||
|
errorType: classifyErrorType(error),
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!isRetryableError(error, config.retry_on_errors)) {
|
||||||
|
log(`[${HOOK_NAME}] message.updated error not retryable, skipping fallback`, {
|
||||||
|
sessionID,
|
||||||
|
statusCode: extractStatusCode(error, config.retry_on_errors),
|
||||||
|
errorName: extractErrorName(error),
|
||||||
|
errorType: classifyErrorType(error),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let state = sessionStates.get(sessionID)
|
||||||
|
const agent = info?.agent as string | undefined
|
||||||
|
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
|
||||||
|
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
|
||||||
|
|
||||||
|
if (fallbackModels.length === 0) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!state) {
|
||||||
|
let initialModel = model
|
||||||
|
if (!initialModel) {
|
||||||
|
const detectedAgent = resolvedAgent
|
||||||
|
const agentConfig = detectedAgent
|
||||||
|
? pluginConfig?.agents?.[detectedAgent as keyof typeof pluginConfig.agents]
|
||||||
|
: undefined
|
||||||
|
const agentModel = agentConfig?.model as string | undefined
|
||||||
|
if (agentModel) {
|
||||||
|
log(`[${HOOK_NAME}] Derived model from agent config for message.updated`, {
|
||||||
|
sessionID,
|
||||||
|
agent: detectedAgent,
|
||||||
|
model: agentModel,
|
||||||
|
})
|
||||||
|
initialModel = agentModel
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!initialModel) {
|
||||||
|
log(`[${HOOK_NAME}] message.updated missing model info, cannot fallback`, {
|
||||||
|
sessionID,
|
||||||
|
errorName: extractErrorName(error),
|
||||||
|
errorType: classifyErrorType(error),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
state = createFallbackState(initialModel)
|
||||||
|
sessionStates.set(sessionID, state)
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
} else {
|
||||||
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
|
||||||
|
if (state.pendingFallbackModel) {
|
||||||
|
if (retrySignal && timeoutEnabled) {
|
||||||
|
log(`[${HOOK_NAME}] Clearing pending fallback due to provider auto-retry signal`, {
|
||||||
|
sessionID,
|
||||||
|
pendingFallbackModel: state.pendingFallbackModel,
|
||||||
|
})
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
} else {
|
||||||
|
log(`[${HOOK_NAME}] message.updated fallback skipped (pending fallback in progress)`, {
|
||||||
|
sessionID,
|
||||||
|
pendingFallbackModel: state.pendingFallbackModel,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = prepareFallback(sessionID, state, fallbackModels, config)
|
||||||
|
|
||||||
|
if (result.success && config.notify_on_fallback) {
|
||||||
|
await deps.ctx.client.tui
|
||||||
|
.showToast({
|
||||||
|
body: {
|
||||||
|
title: "Model Fallback",
|
||||||
|
message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
|
||||||
|
variant: "warning",
|
||||||
|
duration: 5000,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.catch(() => {})
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.success && result.newModel) {
|
||||||
|
await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "message.updated")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,14 +1,6 @@
|
|||||||
/**
|
import type { PluginInput } from "@opencode-ai/plugin"
|
||||||
* Runtime Fallback Hook - Type Definitions
|
|
||||||
*
|
|
||||||
* Types for managing runtime model fallback when API errors occur.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type { RuntimeFallbackConfig, OhMyOpenCodeConfig } from "../../config"
|
import type { RuntimeFallbackConfig, OhMyOpenCodeConfig } from "../../config"
|
||||||
|
|
||||||
/**
|
|
||||||
* Tracks the state of fallback attempts for a session
|
|
||||||
*/
|
|
||||||
export interface FallbackState {
|
export interface FallbackState {
|
||||||
originalModel: string
|
originalModel: string
|
||||||
currentModel: string
|
currentModel: string
|
||||||
@ -18,47 +10,16 @@ export interface FallbackState {
|
|||||||
pendingFallbackModel?: string
|
pendingFallbackModel?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Error information extracted from session.error event
|
|
||||||
*/
|
|
||||||
export interface SessionErrorInfo {
|
|
||||||
/** Session ID that encountered the error */
|
|
||||||
sessionID: string
|
|
||||||
/** The error object */
|
|
||||||
error: unknown
|
|
||||||
/** Error message (extracted) */
|
|
||||||
message: string
|
|
||||||
/** HTTP status code if available */
|
|
||||||
statusCode?: number
|
|
||||||
/** Current model when error occurred */
|
|
||||||
currentModel?: string
|
|
||||||
/** Agent name if available */
|
|
||||||
agent?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Result of a fallback attempt
|
|
||||||
*/
|
|
||||||
export interface FallbackResult {
|
export interface FallbackResult {
|
||||||
/** Whether the fallback was successful */
|
|
||||||
success: boolean
|
success: boolean
|
||||||
/** The model switched to (if successful) */
|
|
||||||
newModel?: string
|
newModel?: string
|
||||||
/** Error message (if failed) */
|
|
||||||
error?: string
|
error?: string
|
||||||
/** Whether max attempts were reached */
|
|
||||||
maxAttemptsReached?: boolean
|
maxAttemptsReached?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Options for creating the runtime fallback hook
|
|
||||||
*/
|
|
||||||
export interface RuntimeFallbackOptions {
|
export interface RuntimeFallbackOptions {
|
||||||
/** Runtime fallback configuration */
|
|
||||||
config?: RuntimeFallbackConfig
|
config?: RuntimeFallbackConfig
|
||||||
/** Optional plugin config override (primarily for testing) */
|
|
||||||
pluginConfig?: OhMyOpenCodeConfig
|
pluginConfig?: OhMyOpenCodeConfig
|
||||||
/** Optional session-level timeout override in milliseconds (primarily for testing) */
|
|
||||||
session_timeout_ms?: number
|
session_timeout_ms?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -66,3 +27,15 @@ export interface RuntimeFallbackHook {
|
|||||||
event: (input: { event: { type: string; properties?: unknown } }) => Promise<void>
|
event: (input: { event: { type: string; properties?: unknown } }) => Promise<void>
|
||||||
"chat.message"?: (input: { sessionID: string; agent?: string; model?: { providerID: string; modelID: string } }, output: { message: { model?: { providerID: string; modelID: string } }; parts?: Array<{ type: string; text?: string }> }) => Promise<void>
|
"chat.message"?: (input: { sessionID: string; agent?: string; model?: { providerID: string; modelID: string } }, output: { message: { model?: { providerID: string; modelID: string } }; parts?: Array<{ type: string; text?: string }> }) => Promise<void>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HookDeps {
|
||||||
|
ctx: PluginInput
|
||||||
|
config: Required<RuntimeFallbackConfig>
|
||||||
|
options: RuntimeFallbackOptions | undefined
|
||||||
|
pluginConfig: OhMyOpenCodeConfig | undefined
|
||||||
|
sessionStates: Map<string, FallbackState>
|
||||||
|
sessionLastAccess: Map<string, number>
|
||||||
|
sessionRetryInFlight: Set<string>
|
||||||
|
sessionAwaitingFallbackResult: Set<string>
|
||||||
|
sessionFallbackTimeouts: Map<string, ReturnType<typeof setTimeout>>
|
||||||
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user