From ae19ff60cfb40867104918d60ff9fae4c138b7d9 Mon Sep 17 00:00:00 2001 From: XIN PENG Date: Mon, 16 Feb 2026 10:44:54 -0800 Subject: [PATCH] feat: Add automatic image format conversion for HEIC/RAW/PSD files Adds automatic conversion of unsupported image formats (HEIC, HEIF, RAW, PSD) to JPEG before sending to multimodal-looker agent. Changes: - Add image-converter.ts module with format detection and conversion - Modify look_at tool to auto-convert unsupported formats - Extend mime-type-inference.ts to support 15+ additional formats - Use sips (macOS) and ImageMagick (Linux/Windows) for conversion - Add proper cleanup of temporary files Fixes #722 Testing: - All existing tests pass (29/29) - TypeScript type checking passes - Verified HEIC to JPEG conversion on macOS --- src/tools/look-at/image-converter.ts | 114 +++++++++++++++++++++++ src/tools/look-at/mime-type-inference.ts | 17 ++++ src/tools/look-at/tools.ts | 37 +++++++- 3 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 src/tools/look-at/image-converter.ts diff --git a/src/tools/look-at/image-converter.ts b/src/tools/look-at/image-converter.ts new file mode 100644 index 00000000..af9aef2e --- /dev/null +++ b/src/tools/look-at/image-converter.ts @@ -0,0 +1,114 @@ +import { execSync } from "node:child_process" +import { existsSync, mkdtempSync, unlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { log } from "../../shared" + +const SUPPORTED_FORMATS = new Set([ + "image/jpeg", + "image/png", + "image/webp", + "image/gif", + "image/bmp", + "image/tiff", +]) + +const UNSUPPORTED_FORMATS = new Set([ + "image/heic", + "image/heif", + "image/x-canon-cr2", + "image/x-canon-crw", + "image/x-nikon-nef", + "image/x-nikon-nrw", + "image/x-sony-arw", + "image/x-sony-sr2", + "image/x-sony-srf", + "image/x-pentax-pef", + "image/x-olympus-orf", + "image/x-panasonic-raw", + "image/x-fuji-raf", + "image/x-adobe-dng", + "image/vnd.adobe.photoshop", + "image/x-photoshop", +]) + +export function needsConversion(mimeType: string): boolean { + if (SUPPORTED_FORMATS.has(mimeType)) { + return false + } + + if (UNSUPPORTED_FORMATS.has(mimeType)) { + return true + } + + return mimeType.startsWith("image/") +} + +export function convertImageToJpeg(inputPath: string, mimeType: string): string { + if (!existsSync(inputPath)) { + throw new Error(`File not found: ${inputPath}`) + } + + const tempDir = mkdtempSync(join(tmpdir(), "opencode-img-")) + const outputPath = join(tempDir, "converted.jpg") + + log(`[image-converter] Converting ${mimeType} to JPEG: ${inputPath}`) + + try { + if (process.platform === "darwin") { + try { + execSync(`sips -s format jpeg "${inputPath}" --out "${outputPath}"`, { + stdio: "pipe", + encoding: "utf-8", + }) + + if (existsSync(outputPath)) { + log(`[image-converter] Converted using sips: ${outputPath}`) + return outputPath + } + } catch (sipsError) { + log(`[image-converter] sips failed: ${sipsError}`) + } + } + + try { + execSync(`convert "${inputPath}" "${outputPath}"`, { + stdio: "pipe", + encoding: "utf-8", + }) + + if (existsSync(outputPath)) { + log(`[image-converter] Converted using ImageMagick: ${outputPath}`) + return outputPath + } + } catch (convertError) { + log(`[image-converter] ImageMagick convert failed: ${convertError}`) + } + + throw new Error( + `No image conversion tool available. Please install ImageMagick:\n` + + ` macOS: brew install imagemagick\n` + + ` Ubuntu/Debian: sudo apt install imagemagick\n` + + ` RHEL/CentOS: sudo yum install ImageMagick` + ) + } catch (error) { + try { + if (existsSync(outputPath)) { + unlinkSync(outputPath) + } + } catch {} + + throw error + } +} + +export function cleanupConvertedImage(filePath: string): void { + try { + if (existsSync(filePath)) { + unlinkSync(filePath) + log(`[image-converter] Cleaned up temporary file: ${filePath}`) + } + } catch (error) { + log(`[image-converter] Failed to cleanup ${filePath}: ${error}`) + } +} diff --git a/src/tools/look-at/mime-type-inference.ts b/src/tools/look-at/mime-type-inference.ts index 18954c46..3a7c0010 100644 --- a/src/tools/look-at/mime-type-inference.ts +++ b/src/tools/look-at/mime-type-inference.ts @@ -29,8 +29,25 @@ export function inferMimeTypeFromFilePath(filePath: string): string { ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", + ".gif": "image/gif", + ".bmp": "image/bmp", + ".tiff": "image/tiff", + ".tif": "image/tiff", ".heic": "image/heic", ".heif": "image/heif", + ".cr2": "image/x-canon-cr2", + ".crw": "image/x-canon-crw", + ".nef": "image/x-nikon-nef", + ".nrw": "image/x-nikon-nrw", + ".arw": "image/x-sony-arw", + ".sr2": "image/x-sony-sr2", + ".srf": "image/x-sony-srf", + ".pef": "image/x-pentax-pef", + ".orf": "image/x-olympus-orf", + ".raw": "image/x-panasonic-raw", + ".raf": "image/x-fuji-raf", + ".dng": "image/x-adobe-dng", + ".psd": "image/vnd.adobe.photoshop", ".mp4": "video/mp4", ".mpeg": "video/mpeg", ".mpg": "video/mpeg", diff --git a/src/tools/look-at/tools.ts b/src/tools/look-at/tools.ts index 0d5c1c0b..9980ed1b 100644 --- a/src/tools/look-at/tools.ts +++ b/src/tools/look-at/tools.ts @@ -13,6 +13,11 @@ import { inferMimeTypeFromFilePath, } from "./mime-type-inference" import { resolveMultimodalLookerAgentMetadata } from "./multimodal-agent-metadata" +import { + needsConversion, + convertImageToJpeg, + cleanupConvertedImage, +} from "./image-converter" export { normalizeArgs, validateArgs } from "./look-at-arguments" @@ -41,8 +46,10 @@ export function createLookAt(ctx: PluginInput): ToolDefinition { let mimeType: string let filePart: { type: "file"; mime: string; url: string; filename: string } + let tempFilePath: string | null = null - if (imageData) { + try { + if (imageData) { mimeType = inferMimeTypeFromBase64(imageData) filePart = { type: "file", @@ -52,11 +59,26 @@ export function createLookAt(ctx: PluginInput): ToolDefinition { } } else if (filePath) { mimeType = inferMimeTypeFromFilePath(filePath) + + let actualFilePath = filePath + if (needsConversion(mimeType)) { + log(`[look_at] Detected unsupported format: ${mimeType}, converting to JPEG...`) + try { + tempFilePath = convertImageToJpeg(filePath, mimeType) + actualFilePath = tempFilePath + mimeType = "image/jpeg" + log(`[look_at] Conversion successful: ${tempFilePath}`) + } catch (conversionError) { + log(`[look_at] Conversion failed: ${conversionError}`) + return `Error: Failed to convert image format. ${conversionError}` + } + } + filePart = { type: "file", mime: mimeType, - url: pathToFileURL(filePath).href, - filename: basename(filePath), + url: pathToFileURL(actualFilePath).href, + filename: basename(actualFilePath), } } else { return "Error: Must provide either 'file_path' or 'image_data'." @@ -149,8 +171,13 @@ Original error: ${createResult.error}` return "Error: No response from multimodal-looker agent" } - log(`[look_at] Got response, length: ${responseText.length}`) - return responseText + log(`[look_at] Got response, length: ${responseText.length}`) + return responseText + } finally { + if (tempFilePath) { + cleanupConvertedImage(tempFilePath) + } + } }, }) }