feat: auto-compress oversized images before API submission

packages/opencode/package.json

@@ -155,6 +155,7 @@
     "partial-json": "0.1.7",
     "remeda": "catalog:",
     "semver": "^7.6.3",
+    "sharp": "0.34.5",
     "solid-js": "catalog:",
     "strip-ansi": "7.1.2",
     "tree-sitter-bash": "0.25.0",

packages/opencode/src/provider/transform.ts

@@ -6,6 +6,7 @@ import type { Provider } from "./provider"
 import type { ModelsDev } from "./models"
 import { iife } from "@/util/iife"
 import { Flag } from "@/flag/flag"
+import { Image } from "@/util/image"
 
 type Modality = NonNullable<ModelsDev.Model["modalities"]>["input"][number]
 
@@ -275,8 +276,87 @@ export namespace ProviderTransform {
     })
   }
 
-  export function message(msgs: ModelMessage[], model: Provider.Model, options: Record<string, unknown>) {
+
+  async function compressImages(msgs: ModelMessage[]): Promise<ModelMessage[]> {
+    const imageError = {
+      type: "text" as const,
+      text: "ERROR: Image could not be processed (file may be corrupted, too large, or in an unsupported format). Inform the user.",
+    }
+
+    const results: ModelMessage[] = []
+    for (const msg of msgs) {
+      if (!Array.isArray(msg.content)) {
+        results.push(msg)
+        continue
+      }
+
+      const parts = await Promise.all(
+        msg.content.map(async (part) => {
+          if (part.type === "image") {
+            const imageStr = part.image.toString()
+            const match = imageStr.match(/^data:([^;]+);base64,(.+)$/)
+            if (match && match[2] && Image.needsCompression(match[2])) {
+              const compressed = await Image.compress({
+                data: match[2],
+                mime: match[1],
+                allowFormatChange: true,
+              }).catch(() => undefined)
+              if (compressed)
+                return {
+                  ...part,
+                  image: new URL(`data:${compressed.mime};base64,${compressed.data}`),
+                }
+              return imageError
+            }
+          }
+          if (part.type === "file" && part.mediaType.startsWith("image/")) {
+            const raw = part.data
+            if (raw instanceof Uint8Array || raw instanceof ArrayBuffer || Buffer.isBuffer(raw)) {
+              const buf = Buffer.isBuffer(raw) ? raw : Buffer.from(raw as ArrayBuffer)
+              const data = buf.toString("base64")
+              if (Image.needsCompression(data)) {
+                const compressed = await Image.compress({
+                  data,
+                  mime: part.mediaType,
+                  allowFormatChange: true,
+                }).catch(() => undefined)
+                if (compressed)
+                  return {
+                    ...part,
+                    mediaType: compressed.mime,
+                    data: Buffer.from(compressed.data, "base64"),
+                  }
+                return imageError
+              }
+            } else if (typeof raw === "string") {
+              if (Image.needsCompression(raw)) {
+                const compressed = await Image.compress({
+                  data: raw,
+                  mime: part.mediaType,
+                  allowFormatChange: true,
+                }).catch(() => undefined)
+                if (compressed)
+                  return {
+                    ...part,
+                    mediaType: compressed.mime,
+                    data: Buffer.from(compressed.data, "base64"),
+                  }
+                return imageError
+              }
+            }
+          }
+          return part
+        }),
+      )
+
+      results.push({ ...msg, content: parts } as ModelMessage)
+    }
+    return results
+  }
+
+  export async function message(msgs: ModelMessage[], model: Provider.Model, options: Record<string, unknown>) {
     msgs = unsupportedParts(msgs, model)
+    msgs = await compressImages(msgs)
     msgs = normalizeMessages(msgs, model, options)
     if (
       (model.providerID === "anthropic" ||

packages/opencode/src/session/llm.ts

@@ -317,7 +317,7 @@ export namespace LLM {
             async transformParams(args) {
               if (args.type === "stream") {
                 // @ts-expect-error
-                args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, options)
+                args.params.prompt = await ProviderTransform.message(args.params.prompt, input.model, options)
               }
               return args.params
             },

packages/opencode/src/tool/read.ts

@@ -12,6 +12,7 @@ import DESCRIPTION from "./read.txt"
 import { Instance } from "../project/instance"
 import { assertExternalDirectoryEffect } from "./external-directory"
 import { Instruction } from "../session/instruction"
+import { Image } from "../util/image"
 
 const DEFAULT_READ_LIMIT = 2000
 const MAX_LINE_LENGTH = 2000
@@ -153,6 +154,22 @@ export const ReadTool = Tool.defineEffect(
       const isImage = mime.startsWith("image/") && mime !== "image/svg+xml" && mime !== "image/vnd.fastbidsheet"
       const isPdf = mime === "application/pdf"
       if (isImage || isPdf) {
+        const raw = Buffer.from(yield* fs.readFile(filepath))
+        let finalMime = mime
+        let base64 = raw.toString("base64")
+
+        if (isImage) {
+          const validated = yield* Effect.tryPromise({
+            try: () => Image.optimizeForUpload({ data: base64, mime }),
+            catch: () =>
+              new Error(
+                `Cannot read image: ${filepath} (file may be corrupted or in an unsupported format)`,
+              ),
+          })
+          finalMime = validated.mime
+          base64 = validated.data
+        }
+
         const msg = `${isImage ? "Image" : "PDF"} read successfully`
         return {
           title,
@@ -165,8 +182,8 @@ export const ReadTool = Tool.defineEffect(
           attachments: [
             {
               type: "file" as const,
-              mime,
-              url: `data:${mime};base64,${Buffer.from(yield* fs.readFile(filepath)).toString("base64")}`,
+              mime: finalMime,
+              url: `data:${finalMime};base64,${base64}`,
             },
           ],
         }

packages/opencode/src/util/image.ts

@@ -0,0 +1,229 @@
+import sharp from 'sharp'
+
+// Anthropic limit is 5MB base64; base64 inflates ~4/3x, so cap raw bytes at 3.75MB
+const DEFAULT_MAX_BYTES = 3.75 * 1024 * 1024
+const DEFAULT_QUALITY = 85
+const MAX_DIMENSION = 2048
+
+export namespace Image {
+  export interface CompressOptions {
+    data: string
+    mime: string
+    maxBytes?: number
+    quality?: number
+    allowFormatChange?: boolean
+  }
+
+  export interface CompressResult {
+    data: string
+    mime: string
+    compressed: boolean
+    originalSize: number
+    finalSize: number
+  }
+
+  export interface ImageInfo {
+    width: number
+    height: number
+    format: string
+    hasAlpha: boolean
+  }
+
+  export interface ResizeOptions {
+    data: string
+    maxWidth: number
+    maxHeight: number
+  }
+
+  export interface ResizeResult {
+    data: string
+    width: number
+    height: number
+  }
+
+  export interface OptimizeOptions {
+    data: string
+    mime: string
+    targetBytes?: number
+  }
+
+  export function needsCompression(base64: string, thresholdBytes = DEFAULT_MAX_BYTES): boolean {
+    const sizeBytes = Math.ceil((base64.length * 3) / 4)
+    return sizeBytes > thresholdBytes
+  }
+
+  export async function getInfo(base64: string): Promise<ImageInfo> {
+    const buffer = Buffer.from(base64, 'base64')
+    const metadata = await sharp(buffer).metadata()
+
+    return {
+      width: metadata.width ?? 0,
+      height: metadata.height ?? 0,
+      format: metadata.format ?? 'unknown',
+      hasAlpha: metadata.hasAlpha ?? false
+    }
+  }
+
+  export async function resize(options: ResizeOptions): Promise<ResizeResult> {
+    const buffer = Buffer.from(options.data, 'base64')
+    const metadata = await sharp(buffer).metadata()
+
+    const currentWidth = metadata.width ?? 0
+    const currentHeight = metadata.height ?? 0
+
+    if (currentWidth <= options.maxWidth && currentHeight <= options.maxHeight) {
+      return {
+        data: options.data,
+        width: currentWidth,
+        height: currentHeight
+      }
+    }
+
+    const resized = await sharp(buffer)
+      .resize(options.maxWidth, options.maxHeight, {
+        fit: 'inside',
+        withoutEnlargement: true
+      })
+      .toBuffer()
+
+    const newMetadata = await sharp(resized).metadata()
+
+    return {
+      data: resized.toString('base64'),
+      width: newMetadata.width ?? 0,
+      height: newMetadata.height ?? 0
+    }
+  }
+
+  export function estimateCompressedSize(originalSize: number, format: 'jpeg' | 'webp' | 'png', quality: number): number {
+    const qualityFactor = quality / 100
+    switch (format) {
+      case 'jpeg':
+        return Math.ceil(originalSize * qualityFactor * 0.3)
+      case 'webp':
+        return Math.ceil(originalSize * qualityFactor * 0.25)
+      case 'png':
+        return Math.ceil(originalSize * 0.7)
+    }
+  }
+
+  export async function compress(options: CompressOptions): Promise<CompressResult> {
+    if (!options.data) {
+      throw new Error('Image data is required')
+    }
+
+    const buffer = Buffer.from(options.data, 'base64')
+    const originalSize = buffer.length
+    const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
+    const quality = options.quality ?? DEFAULT_QUALITY
+
+    if (originalSize <= maxBytes) {
+      return {
+        data: options.data,
+        mime: options.mime,
+        compressed: false,
+        originalSize,
+        finalSize: originalSize
+      }
+    }
+
+    const metadata = await sharp(buffer).metadata()
+    const hasAlpha = metadata.hasAlpha ?? false
+    const currentWidth = metadata.width ?? 0
+    const currentHeight = metadata.height ?? 0
+
+    let pipeline = sharp(buffer)
+
+    if (currentWidth > MAX_DIMENSION || currentHeight > MAX_DIMENSION) {
+      pipeline = pipeline.resize(MAX_DIMENSION, MAX_DIMENSION, {
+        fit: 'inside',
+        withoutEnlargement: true
+      })
+    }
+
+    let outputFormat: 'jpeg' | 'webp' | 'png' = 'jpeg'
+    let outputMime = 'image/jpeg'
+
+    if (hasAlpha) {
+      outputFormat = 'webp'
+      outputMime = 'image/webp'
+    } else if (options.allowFormatChange !== false) {
+      outputFormat = 'jpeg'
+      outputMime = 'image/jpeg'
+    } else {
+      outputFormat = 'webp'
+      outputMime = 'image/webp'
+    }
+
+    let currentQuality = quality
+    let result: Buffer
+    let currentMaxDimension = MAX_DIMENSION
+
+    // Phase 1: Try quality reduction at current dimensions
+    while (currentQuality >= 20) {
+      if (outputFormat === 'jpeg') {
+        result = await pipeline.clone().jpeg({ quality: currentQuality, mozjpeg: true }).toBuffer()
+      } else if (outputFormat === 'webp') {
+        result = await pipeline.clone().webp({ quality: currentQuality }).toBuffer()
+      } else {
+        result = await pipeline.clone().png({ compressionLevel: 9 }).toBuffer()
+      }
+
+      if (result.length <= maxBytes) {
+        return {
+          data: result.toString('base64'),
+          mime: outputMime,
+          compressed: true,
+          originalSize,
+          finalSize: result.length
+        }
+      }
+
+      currentQuality -= 10
+    }
+
+    // Phase 2: Progressive dimension reduction with low quality
+    while (currentMaxDimension >= 512) {
+      pipeline = sharp(buffer).resize(currentMaxDimension, currentMaxDimension, {
+        fit: 'inside',
+        withoutEnlargement: true
+      })
+
+      result = await pipeline.clone().webp({ quality: 30 }).toBuffer()
+
+      if (result.length <= maxBytes) {
+        return {
+          data: result.toString('base64'),
+          mime: 'image/webp',
+          compressed: true,
+          originalSize,
+          finalSize: result.length
+        }
+      }
+
+      currentMaxDimension = Math.floor(currentMaxDimension * 0.75)
+    }
+
+    // Phase 3: Final fallback — small dimensions, lowest quality
+    result = await sharp(buffer).resize(512, 512, { fit: 'inside', withoutEnlargement: true }).webp({ quality: 20 }).toBuffer()
+
+    return {
+      data: result.toString('base64'),
+      mime: 'image/webp',
+      compressed: true,
+      originalSize,
+      finalSize: result.length
+    }
+  }
+
+  export async function optimizeForUpload(options: OptimizeOptions): Promise<CompressResult> {
+    const targetBytes = options.targetBytes ?? DEFAULT_MAX_BYTES
+
+    return compress({
+      data: options.data,
+      mime: options.mime,
+      maxBytes: targetBytes,
+      allowFormatChange: true
+    })
+  }
+}