From 76211a31857398b805dc850b2a52b0c9bf495bd5 Mon Sep 17 00:00:00 2001 From: justsisyphus Date: Thu, 22 Jan 2026 22:44:11 +0900 Subject: [PATCH] feat(cli): add doctor check for model resolution Add new 'model-resolution' check to diagnose model fallback chain health. Validates that configured agents and categories can resolve to available models, surfacing misconfiguration early. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-opencode) Co-authored-by: Sisyphus --- src/cli/doctor/checks/index.ts | 3 + .../doctor/checks/model-resolution.test.ts | 139 ++++++++++++ src/cli/doctor/checks/model-resolution.ts | 210 ++++++++++++++++++ src/cli/doctor/constants.ts | 2 + 4 files changed, 354 insertions(+) create mode 100644 src/cli/doctor/checks/model-resolution.test.ts create mode 100644 src/cli/doctor/checks/model-resolution.ts diff --git a/src/cli/doctor/checks/index.ts b/src/cli/doctor/checks/index.ts index af82d3c1..d8d4b7e7 100644 --- a/src/cli/doctor/checks/index.ts +++ b/src/cli/doctor/checks/index.ts @@ -2,6 +2,7 @@ import type { CheckDefinition } from "../types" import { getOpenCodeCheckDefinition } from "./opencode" import { getPluginCheckDefinition } from "./plugin" import { getConfigCheckDefinition } from "./config" +import { getModelResolutionCheckDefinition } from "./model-resolution" import { getAuthCheckDefinitions } from "./auth" import { getDependencyCheckDefinitions } from "./dependencies" import { getGhCliCheckDefinition } from "./gh" @@ -12,6 +13,7 @@ import { getVersionCheckDefinition } from "./version" export * from "./opencode" export * from "./plugin" export * from "./config" +export * from "./model-resolution" export * from "./auth" export * from "./dependencies" export * from "./gh" @@ -24,6 +26,7 @@ export function getAllCheckDefinitions(): CheckDefinition[] { getOpenCodeCheckDefinition(), getPluginCheckDefinition(), getConfigCheckDefinition(), + getModelResolutionCheckDefinition(), ...getAuthCheckDefinitions(), ...getDependencyCheckDefinitions(), getGhCliCheckDefinition(), diff --git a/src/cli/doctor/checks/model-resolution.test.ts b/src/cli/doctor/checks/model-resolution.test.ts new file mode 100644 index 00000000..1121dc32 --- /dev/null +++ b/src/cli/doctor/checks/model-resolution.test.ts @@ -0,0 +1,139 @@ +import { describe, it, expect, beforeEach, afterEach, spyOn, mock } from "bun:test" + +describe("model-resolution check", () => { + describe("getModelResolutionInfo", () => { + // #given: Model requirements are defined in model-requirements.ts + // #when: Getting model resolution info + // #then: Returns info for all agents and categories with their provider chains + + it("returns agent requirements with provider chains", async () => { + const { getModelResolutionInfo } = await import("./model-resolution") + + const info = getModelResolutionInfo() + + // #then: Should have agent entries + const sisyphus = info.agents.find((a) => a.name === "Sisyphus") + expect(sisyphus).toBeDefined() + expect(sisyphus!.requirement.fallbackChain[0]?.model).toBe("claude-opus-4-5") + expect(sisyphus!.requirement.fallbackChain[0]?.providers).toContain("anthropic") + expect(sisyphus!.requirement.fallbackChain[0]?.providers).toContain("github-copilot") + }) + + it("returns category requirements with provider chains", async () => { + const { getModelResolutionInfo } = await import("./model-resolution") + + const info = getModelResolutionInfo() + + // #then: Should have category entries + const visual = info.categories.find((c) => c.name === "visual-engineering") + expect(visual).toBeDefined() + expect(visual!.requirement.fallbackChain[0]?.model).toBe("gemini-3-pro-preview") + expect(visual!.requirement.fallbackChain[0]?.providers).toContain("google") + }) + }) + + describe("getModelResolutionInfoWithOverrides", () => { + // #given: User has overrides in oh-my-opencode.json + // #when: Getting resolution info with config + // #then: Shows user override in Step 1 position + + it("shows user override for agent when configured", async () => { + const { getModelResolutionInfoWithOverrides } = await import("./model-resolution") + + // #given: User has override for oracle agent + const mockConfig = { + agents: { + oracle: { model: "anthropic/claude-opus-4-5" }, + }, + } + + const info = getModelResolutionInfoWithOverrides(mockConfig) + + // #then: Oracle should show the override + const oracle = info.agents.find((a) => a.name === "oracle") + expect(oracle).toBeDefined() + expect(oracle!.userOverride).toBe("anthropic/claude-opus-4-5") + expect(oracle!.effectiveResolution).toBe("User override: anthropic/claude-opus-4-5") + }) + + it("shows user override for category when configured", async () => { + const { getModelResolutionInfoWithOverrides } = await import("./model-resolution") + + // #given: User has override for visual-engineering category + const mockConfig = { + categories: { + "visual-engineering": { model: "openai/gpt-5.2" }, + }, + } + + const info = getModelResolutionInfoWithOverrides(mockConfig) + + // #then: visual-engineering should show the override + const visual = info.categories.find((c) => c.name === "visual-engineering") + expect(visual).toBeDefined() + expect(visual!.userOverride).toBe("openai/gpt-5.2") + expect(visual!.effectiveResolution).toBe("User override: openai/gpt-5.2") + }) + + it("shows provider fallback when no override exists", async () => { + const { getModelResolutionInfoWithOverrides } = await import("./model-resolution") + + // #given: No overrides configured + const mockConfig = {} + + const info = getModelResolutionInfoWithOverrides(mockConfig) + + // #then: Should show provider fallback chain + const sisyphus = info.agents.find((a) => a.name === "Sisyphus") + expect(sisyphus).toBeDefined() + expect(sisyphus!.userOverride).toBeUndefined() + expect(sisyphus!.effectiveResolution).toContain("Provider fallback:") + expect(sisyphus!.effectiveResolution).toContain("anthropic") + }) + }) + + describe("checkModelResolution", () => { + // #given: Doctor check is executed + // #when: Running the model resolution check + // #then: Returns pass with details showing resolution flow + + it("returns pass status with agent and category counts", async () => { + const { checkModelResolution } = await import("./model-resolution") + + const result = await checkModelResolution() + + // #then: Should pass and show counts + expect(result.status).toBe("pass") + expect(result.message).toMatch(/\d+ agents?, \d+ categories?/) + }) + + it("includes resolution details in verbose mode details array", async () => { + const { checkModelResolution } = await import("./model-resolution") + + const result = await checkModelResolution() + + // #then: Details should contain agent/category resolution info + expect(result.details).toBeDefined() + expect(result.details!.length).toBeGreaterThan(0) + // Should have Current Models header and sections + expect(result.details!.some((d) => d.includes("Current Models"))).toBe(true) + expect(result.details!.some((d) => d.includes("Agents:"))).toBe(true) + expect(result.details!.some((d) => d.includes("Categories:"))).toBe(true) + // Should have legend + expect(result.details!.some((d) => d.includes("user override"))).toBe(true) + }) + }) + + describe("getModelResolutionCheckDefinition", () => { + it("returns valid check definition", async () => { + const { getModelResolutionCheckDefinition } = await import("./model-resolution") + + const def = getModelResolutionCheckDefinition() + + expect(def.id).toBe("model-resolution") + expect(def.name).toBe("Model Resolution") + expect(def.category).toBe("configuration") + expect(typeof def.check).toBe("function") + }) + }) +}) diff --git a/src/cli/doctor/checks/model-resolution.ts b/src/cli/doctor/checks/model-resolution.ts new file mode 100644 index 00000000..8bf85a8c --- /dev/null +++ b/src/cli/doctor/checks/model-resolution.ts @@ -0,0 +1,210 @@ +import { readFileSync } from "node:fs" +import type { CheckResult, CheckDefinition } from "../types" +import { CHECK_IDS, CHECK_NAMES } from "../constants" +import { parseJsonc, detectConfigFile } from "../../../shared" +import { + AGENT_MODEL_REQUIREMENTS, + CATEGORY_MODEL_REQUIREMENTS, + type ModelRequirement, +} from "../../../shared/model-requirements" +import { homedir } from "node:os" +import { join } from "node:path" + +const PACKAGE_NAME = "oh-my-opencode" +const USER_CONFIG_DIR = join(homedir(), ".config", "opencode") +const USER_CONFIG_BASE = join(USER_CONFIG_DIR, PACKAGE_NAME) +const PROJECT_CONFIG_BASE = join(process.cwd(), ".opencode", PACKAGE_NAME) + +export interface AgentResolutionInfo { + name: string + requirement: ModelRequirement + userOverride?: string + effectiveModel: string + effectiveResolution: string +} + +export interface CategoryResolutionInfo { + name: string + requirement: ModelRequirement + userOverride?: string + effectiveModel: string + effectiveResolution: string +} + +export interface ModelResolutionInfo { + agents: AgentResolutionInfo[] + categories: CategoryResolutionInfo[] +} + +interface OmoConfig { + agents?: Record + categories?: Record +} + +function loadConfig(): OmoConfig | null { + const projectDetected = detectConfigFile(PROJECT_CONFIG_BASE) + if (projectDetected.format !== "none") { + try { + const content = readFileSync(projectDetected.path, "utf-8") + return parseJsonc(content) + } catch { + return null + } + } + + const userDetected = detectConfigFile(USER_CONFIG_BASE) + if (userDetected.format !== "none") { + try { + const content = readFileSync(userDetected.path, "utf-8") + return parseJsonc(content) + } catch { + return null + } + } + + return null +} + +function formatProviderChain(providers: string[]): string { + return providers.join(" → ") +} + +function getEffectiveModel(requirement: ModelRequirement, userOverride?: string): string { + if (userOverride) { + return userOverride + } + const firstEntry = requirement.fallbackChain[0] + if (!firstEntry) { + return "unknown" + } + return `${firstEntry.providers[0]}/${firstEntry.model}` +} + +function buildEffectiveResolution( + requirement: ModelRequirement, + userOverride?: string, +): string { + if (userOverride) { + return `User override: ${userOverride}` + } + const firstEntry = requirement.fallbackChain[0] + if (!firstEntry) { + return "No fallback chain defined" + } + return `Provider fallback: ${formatProviderChain(firstEntry.providers)} → ${firstEntry.model}` +} + +export function getModelResolutionInfo(): ModelResolutionInfo { + const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map( + ([name, requirement]) => ({ + name, + requirement, + effectiveModel: getEffectiveModel(requirement), + effectiveResolution: buildEffectiveResolution(requirement), + }), + ) + + const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map( + ([name, requirement]) => ({ + name, + requirement, + effectiveModel: getEffectiveModel(requirement), + effectiveResolution: buildEffectiveResolution(requirement), + }), + ) + + return { agents, categories } +} + +export function getModelResolutionInfoWithOverrides(config: OmoConfig): ModelResolutionInfo { + const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map( + ([name, requirement]) => { + const userOverride = config.agents?.[name]?.model + return { + name, + requirement, + userOverride, + effectiveModel: getEffectiveModel(requirement, userOverride), + effectiveResolution: buildEffectiveResolution(requirement, userOverride), + } + }, + ) + + const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map( + ([name, requirement]) => { + const userOverride = config.categories?.[name]?.model + return { + name, + requirement, + userOverride, + effectiveModel: getEffectiveModel(requirement, userOverride), + effectiveResolution: buildEffectiveResolution(requirement, userOverride), + } + }, + ) + + return { agents, categories } +} + +function formatModelWithVariant(model: string, variant?: string): string { + return variant ? `${model} (${variant})` : model +} + +function getEffectiveVariant(requirement: ModelRequirement): string | undefined { + const firstEntry = requirement.fallbackChain[0] + return firstEntry?.variant ?? requirement.variant +} + +function buildDetailsArray(info: ModelResolutionInfo): string[] { + const details: string[] = [] + + details.push("═══ Current Models ═══") + details.push("") + details.push("Agents:") + for (const agent of info.agents) { + const marker = agent.userOverride ? "●" : "○" + const display = formatModelWithVariant(agent.effectiveModel, getEffectiveVariant(agent.requirement)) + details.push(` ${marker} ${agent.name}: ${display}`) + } + details.push("") + details.push("Categories:") + for (const category of info.categories) { + const marker = category.userOverride ? "●" : "○" + const display = formatModelWithVariant(category.effectiveModel, getEffectiveVariant(category.requirement)) + details.push(` ${marker} ${category.name}: ${display}`) + } + details.push("") + details.push("● = user override, ○ = provider fallback") + + return details +} + +export async function checkModelResolution(): Promise { + const config = loadConfig() ?? {} + const info = getModelResolutionInfoWithOverrides(config) + + const agentCount = info.agents.length + const categoryCount = info.categories.length + const agentOverrides = info.agents.filter((a) => a.userOverride).length + const categoryOverrides = info.categories.filter((c) => c.userOverride).length + const totalOverrides = agentOverrides + categoryOverrides + + const overrideNote = totalOverrides > 0 ? ` (${totalOverrides} override${totalOverrides > 1 ? "s" : ""})` : "" + + return { + name: CHECK_NAMES[CHECK_IDS.MODEL_RESOLUTION], + status: "pass", + message: `${agentCount} agents, ${categoryCount} categories${overrideNote}`, + details: buildDetailsArray(info), + } +} + +export function getModelResolutionCheckDefinition(): CheckDefinition { + return { + id: CHECK_IDS.MODEL_RESOLUTION, + name: CHECK_NAMES[CHECK_IDS.MODEL_RESOLUTION], + category: "configuration", + check: checkModelResolution, + critical: false, + } +} diff --git a/src/cli/doctor/constants.ts b/src/cli/doctor/constants.ts index 3b9a2851..26dbcc01 100644 --- a/src/cli/doctor/constants.ts +++ b/src/cli/doctor/constants.ts @@ -21,6 +21,7 @@ export const CHECK_IDS = { OPENCODE_INSTALLATION: "opencode-installation", PLUGIN_REGISTRATION: "plugin-registration", CONFIG_VALIDATION: "config-validation", + MODEL_RESOLUTION: "model-resolution", AUTH_ANTHROPIC: "auth-anthropic", AUTH_OPENAI: "auth-openai", AUTH_GOOGLE: "auth-google", @@ -38,6 +39,7 @@ export const CHECK_NAMES: Record = { [CHECK_IDS.OPENCODE_INSTALLATION]: "OpenCode Installation", [CHECK_IDS.PLUGIN_REGISTRATION]: "Plugin Registration", [CHECK_IDS.CONFIG_VALIDATION]: "Configuration Validity", + [CHECK_IDS.MODEL_RESOLUTION]: "Model Resolution", [CHECK_IDS.AUTH_ANTHROPIC]: "Anthropic (Claude) Auth", [CHECK_IDS.AUTH_OPENAI]: "OpenAI (ChatGPT) Auth", [CHECK_IDS.AUTH_GOOGLE]: "Google (Gemini) Auth",