diff --git a/src/tools/look-at/image-converter.ts b/src/tools/look-at/image-converter.ts index af9aef2e..6718bd0f 100644 --- a/src/tools/look-at/image-converter.ts +++ b/src/tools/look-at/image-converter.ts @@ -1,5 +1,5 @@ import { execSync } from "node:child_process" -import { existsSync, mkdtempSync, unlinkSync } from "node:fs" +import { existsSync, mkdtempSync, unlinkSync, writeFileSync, readFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { log } from "../../shared" @@ -112,3 +112,38 @@ export function cleanupConvertedImage(filePath: string): void { log(`[image-converter] Failed to cleanup ${filePath}: ${error}`) } } + +export function convertBase64ImageToJpeg( + base64Data: string, + mimeType: string +): { base64: string; tempFiles: string[] } { + const tempDir = mkdtempSync(join(tmpdir(), "opencode-b64-")) + const inputExt = mimeType.split("/")[1] || "bin" + const inputPath = join(tempDir, `input.${inputExt}`) + const tempFiles: string[] = [inputPath] + + try { + const cleanBase64 = base64Data.replace(/^data:[^;]+;base64,/, "") + const buffer = Buffer.from(cleanBase64, "base64") + writeFileSync(inputPath, buffer) + + log(`[image-converter] Converting Base64 ${mimeType} to JPEG`) + + const outputPath = convertImageToJpeg(inputPath, mimeType) + tempFiles.push(outputPath) + + const convertedBuffer = readFileSync(outputPath) + const convertedBase64 = convertedBuffer.toString("base64") + + log(`[image-converter] Base64 conversion successful`) + + return { base64: convertedBase64, tempFiles } + } catch (error) { + tempFiles.forEach(file => { + try { + if (existsSync(file)) unlinkSync(file) + } catch {} + }) + throw error + } +} diff --git a/src/tools/look-at/tools.ts b/src/tools/look-at/tools.ts index 9980ed1b..b7643860 100644 --- a/src/tools/look-at/tools.ts +++ b/src/tools/look-at/tools.ts @@ -16,6 +16,7 @@ import { resolveMultimodalLookerAgentMetadata } from "./multimodal-agent-metadat import { needsConversion, convertImageToJpeg, + convertBase64ImageToJpeg, cleanupConvertedImage, } from "./image-converter" @@ -47,17 +48,36 @@ export function createLookAt(ctx: PluginInput): ToolDefinition { let mimeType: string let filePart: { type: "file"; mime: string; url: string; filename: string } let tempFilePath: string | null = null + let tempFilesToCleanup: string[] = [] try { if (imageData) { - mimeType = inferMimeTypeFromBase64(imageData) - filePart = { - type: "file", - mime: mimeType, - url: `data:${mimeType};base64,${extractBase64Data(imageData)}`, - filename: `clipboard-image.${mimeType.split("/")[1] || "png"}`, - } - } else if (filePath) { + mimeType = inferMimeTypeFromBase64(imageData) + + let finalBase64Data = extractBase64Data(imageData) + let finalMimeType = mimeType + + if (needsConversion(mimeType)) { + log(`[look_at] Detected unsupported Base64 format: ${mimeType}, converting to JPEG...`) + try { + const { base64, tempFiles } = convertBase64ImageToJpeg(imageData, mimeType) + finalBase64Data = base64 + finalMimeType = "image/jpeg" + tempFilesToCleanup = tempFiles + log(`[look_at] Base64 conversion successful`) + } catch (conversionError) { + log(`[look_at] Base64 conversion failed: ${conversionError}`) + return `Error: Failed to convert Base64 image format. ${conversionError}` + } + } + + filePart = { + type: "file", + mime: finalMimeType, + url: `data:${finalMimeType};base64,${finalBase64Data}`, + filename: `clipboard-image.${finalMimeType.split("/")[1] || "png"}`, + } + } else if (filePath) { mimeType = inferMimeTypeFromFilePath(filePath) let actualFilePath = filePath @@ -177,6 +197,7 @@ Original error: ${createResult.error}` if (tempFilePath) { cleanupConvertedImage(tempFilePath) } + tempFilesToCleanup.forEach(file => cleanupConvertedImage(file)) } }, })