fix: Add Base64 image format conversion support
Extends conversion logic to handle Base64-encoded images (e.g., from clipboard). Previously, unsupported formats like HEIC/RAW/PSD in Base64 form bypassed the conversion check and caused failures at multimodal-looker agent. Changes: - Add convertBase64ImageToJpeg() function in image-converter.ts - Save Base64 data to temp file, convert, read back as Base64 - Update tools.ts to check and convert Base64 images when needed - Ensure proper cleanup of all temporary files Testing: - All tests pass (29/29) - Verified with 1.7MB HEIC file converted from Base64 - Type checking passes
This commit is contained in:
parent
ae19ff60cf
commit
116ca090e0
@ -1,5 +1,5 @@
|
|||||||
import { execSync } from "node:child_process"
|
import { execSync } from "node:child_process"
|
||||||
import { existsSync, mkdtempSync, unlinkSync } from "node:fs"
|
import { existsSync, mkdtempSync, unlinkSync, writeFileSync, readFileSync } from "node:fs"
|
||||||
import { tmpdir } from "node:os"
|
import { tmpdir } from "node:os"
|
||||||
import { join } from "node:path"
|
import { join } from "node:path"
|
||||||
import { log } from "../../shared"
|
import { log } from "../../shared"
|
||||||
@ -112,3 +112,38 @@ export function cleanupConvertedImage(filePath: string): void {
|
|||||||
log(`[image-converter] Failed to cleanup ${filePath}: ${error}`)
|
log(`[image-converter] Failed to cleanup ${filePath}: ${error}`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function convertBase64ImageToJpeg(
|
||||||
|
base64Data: string,
|
||||||
|
mimeType: string
|
||||||
|
): { base64: string; tempFiles: string[] } {
|
||||||
|
const tempDir = mkdtempSync(join(tmpdir(), "opencode-b64-"))
|
||||||
|
const inputExt = mimeType.split("/")[1] || "bin"
|
||||||
|
const inputPath = join(tempDir, `input.${inputExt}`)
|
||||||
|
const tempFiles: string[] = [inputPath]
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cleanBase64 = base64Data.replace(/^data:[^;]+;base64,/, "")
|
||||||
|
const buffer = Buffer.from(cleanBase64, "base64")
|
||||||
|
writeFileSync(inputPath, buffer)
|
||||||
|
|
||||||
|
log(`[image-converter] Converting Base64 ${mimeType} to JPEG`)
|
||||||
|
|
||||||
|
const outputPath = convertImageToJpeg(inputPath, mimeType)
|
||||||
|
tempFiles.push(outputPath)
|
||||||
|
|
||||||
|
const convertedBuffer = readFileSync(outputPath)
|
||||||
|
const convertedBase64 = convertedBuffer.toString("base64")
|
||||||
|
|
||||||
|
log(`[image-converter] Base64 conversion successful`)
|
||||||
|
|
||||||
|
return { base64: convertedBase64, tempFiles }
|
||||||
|
} catch (error) {
|
||||||
|
tempFiles.forEach(file => {
|
||||||
|
try {
|
||||||
|
if (existsSync(file)) unlinkSync(file)
|
||||||
|
} catch {}
|
||||||
|
})
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import { resolveMultimodalLookerAgentMetadata } from "./multimodal-agent-metadat
|
|||||||
import {
|
import {
|
||||||
needsConversion,
|
needsConversion,
|
||||||
convertImageToJpeg,
|
convertImageToJpeg,
|
||||||
|
convertBase64ImageToJpeg,
|
||||||
cleanupConvertedImage,
|
cleanupConvertedImage,
|
||||||
} from "./image-converter"
|
} from "./image-converter"
|
||||||
|
|
||||||
@ -47,17 +48,36 @@ export function createLookAt(ctx: PluginInput): ToolDefinition {
|
|||||||
let mimeType: string
|
let mimeType: string
|
||||||
let filePart: { type: "file"; mime: string; url: string; filename: string }
|
let filePart: { type: "file"; mime: string; url: string; filename: string }
|
||||||
let tempFilePath: string | null = null
|
let tempFilePath: string | null = null
|
||||||
|
let tempFilesToCleanup: string[] = []
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (imageData) {
|
if (imageData) {
|
||||||
mimeType = inferMimeTypeFromBase64(imageData)
|
mimeType = inferMimeTypeFromBase64(imageData)
|
||||||
filePart = {
|
|
||||||
type: "file",
|
let finalBase64Data = extractBase64Data(imageData)
|
||||||
mime: mimeType,
|
let finalMimeType = mimeType
|
||||||
url: `data:${mimeType};base64,${extractBase64Data(imageData)}`,
|
|
||||||
filename: `clipboard-image.${mimeType.split("/")[1] || "png"}`,
|
if (needsConversion(mimeType)) {
|
||||||
}
|
log(`[look_at] Detected unsupported Base64 format: ${mimeType}, converting to JPEG...`)
|
||||||
} else if (filePath) {
|
try {
|
||||||
|
const { base64, tempFiles } = convertBase64ImageToJpeg(imageData, mimeType)
|
||||||
|
finalBase64Data = base64
|
||||||
|
finalMimeType = "image/jpeg"
|
||||||
|
tempFilesToCleanup = tempFiles
|
||||||
|
log(`[look_at] Base64 conversion successful`)
|
||||||
|
} catch (conversionError) {
|
||||||
|
log(`[look_at] Base64 conversion failed: ${conversionError}`)
|
||||||
|
return `Error: Failed to convert Base64 image format. ${conversionError}`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
filePart = {
|
||||||
|
type: "file",
|
||||||
|
mime: finalMimeType,
|
||||||
|
url: `data:${finalMimeType};base64,${finalBase64Data}`,
|
||||||
|
filename: `clipboard-image.${finalMimeType.split("/")[1] || "png"}`,
|
||||||
|
}
|
||||||
|
} else if (filePath) {
|
||||||
mimeType = inferMimeTypeFromFilePath(filePath)
|
mimeType = inferMimeTypeFromFilePath(filePath)
|
||||||
|
|
||||||
let actualFilePath = filePath
|
let actualFilePath = filePath
|
||||||
@ -177,6 +197,7 @@ Original error: ${createResult.error}`
|
|||||||
if (tempFilePath) {
|
if (tempFilePath) {
|
||||||
cleanupConvertedImage(tempFilePath)
|
cleanupConvertedImage(tempFilePath)
|
||||||
}
|
}
|
||||||
|
tempFilesToCleanup.forEach(file => cleanupConvertedImage(file))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user