Add media attachments to read tool

soaringk · soaringk · commit 98f37ced7d61 · 2026-04-13T15:01:20.000+08:00
diff --git a/packages/opencode/src/flag/flag.ts b/packages/opencode/src/flag/flag.ts
@@ -78,6 +78,7 @@ export namespace Flag {
   export const OPENCODE_EXPERIMENTAL_MARKDOWN = !falsy("OPENCODE_EXPERIMENTAL_MARKDOWN")
   export const OPENCODE_MODELS_URL = process.env["OPENCODE_MODELS_URL"]
   export const OPENCODE_MODELS_PATH = process.env["OPENCODE_MODELS_PATH"]
+  export const OPENCODE_READ_MAX_ATTACHMENT_BYTES = number("OPENCODE_READ_MAX_ATTACHMENT_BYTES")
   export const OPENCODE_DISABLE_EMBEDDED_WEB_UI = truthy("OPENCODE_DISABLE_EMBEDDED_WEB_UI")
   export const OPENCODE_DB = process.env["OPENCODE_DB"]
   export const OPENCODE_DISABLE_CHANNEL_DB = truthy("OPENCODE_DISABLE_CHANNEL_DB")
diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts
@@ -320,7 +320,7 @@ When constructing the summary, try to stick to this template:
             })
             const text =
               (input.overflow
-                ? "The previous request exceeded the provider's size limit due to large media attachments. The conversation was compacted and media files were removed from context. If the user was asking about attached images or files, explain that the attachments were too large to process and suggest they try again with smaller or fewer files.\n\n"
+                ? "The previous request exceeded the provider's size limit due to large media attachments. The conversation was compacted and media files were removed from context. If the user was asking about attached media or files, explain that the attachments were too large to process and suggest they try again with smaller or fewer files.\n\n"
                 : "") +
               "Continue if you have next steps, or stop and ask for clarification if you are unsure how to proceed."
             yield* session.updatePart({
diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts
@@ -25,10 +25,15 @@ interface FetchDecompressionError extends Error {
 }
 
 export namespace MessageV2 {
-  export const SYNTHETIC_ATTACHMENT_PROMPT = "Attached image(s) from tool result:"
+  export const SYNTHETIC_ATTACHMENT_PROMPT = "Attached media file(s) from tool result:"
 
   export function isMedia(mime: string) {
-    return mime.startsWith("image/") || mime === "application/pdf"
+    return (
+      mime.startsWith("image/") ||
+      mime.startsWith("video/") ||
+      mime.startsWith("audio/") ||
+      mime === "application/pdf"
+    )
   }
 
   export const OutputLengthError = NamedError.create("MessageOutputLengthError", z.object({}))
@@ -596,8 +601,7 @@ export namespace MessageV2 {
     // to extract media and inject as user messages. Other SDKs (anthropic, google,
     // bedrock) handle type: "content" with media parts natively.
     //
-    // Only apply this workaround if the model actually supports image input -
-    // otherwise there's no point extracting images.
+    // Apply this workaround only for adapters that cannot carry media in tool results.
     const supportsMediaInToolResults = (() => {
       if (model.api.npm === "@ai-sdk/anthropic") return true
       if (model.api.npm === "@ai-sdk/openai") return true
@@ -728,7 +732,7 @@ export namespace MessageV2 {
               const attachments = part.state.time.compacted || options?.stripMedia ? [] : (part.state.attachments ?? [])
 
               // For providers that don't support media in tool results, extract media files
-              // (images, PDFs) to be sent as a separate user message
+              // to be sent as a separate user message.
               const mediaAttachments = attachments.filter((a) => isMedia(a.mime))
               const nonMediaAttachments = attachments.filter((a) => !isMedia(a.mime))
               if (!supportsMediaInToolResults && mediaAttachments.length > 0) {
@@ -802,7 +806,7 @@ export namespace MessageV2 {
         if (assistantMessage.parts.length > 0) {
           result.push(assistantMessage)
           // Inject pending media as a user message for providers that don't support
-          // media (images, PDFs) in tool results
+          // media in tool results.
           if (media.length > 0) {
             result.push({
               id: MessageID.ascending(),
diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts
@@ -12,12 +12,14 @@ import DESCRIPTION from "./read.txt"
 import { Instance } from "../project/instance"
 import { assertExternalDirectoryEffect } from "./external-directory"
 import { Instruction } from "../session/instruction"
+import { Flag } from "../flag/flag"
 
 const DEFAULT_READ_LIMIT = 2000
 const MAX_LINE_LENGTH = 2000
 const MAX_LINE_SUFFIX = `... (line truncated to ${MAX_LINE_LENGTH} chars)`
 const MAX_BYTES = 50 * 1024
 const MAX_BYTES_LABEL = `${MAX_BYTES / 1024} KB`
+const MAX_ATTACHMENT_BYTES = 256 * 1024 * 1024
 
 const parameters = z.object({
   filePath: z.string().describe("The absolute path to the file or directory to read"),
@@ -146,10 +148,19 @@ export const ReadTool = Tool.define(
       const loaded = yield* instruction.resolve(ctx.messages, filepath, ctx.messageID)
 
       const mime = AppFileSystem.mimeType(filepath)
-      const isImage = mime.startsWith("image/") && mime !== "image/svg+xml" && mime !== "image/vnd.fastbidsheet"
-      const isPdf = mime === "application/pdf"
-      if (isImage || isPdf) {
-        const msg = `${isImage ? "Image" : "PDF"} read successfully`
+      const attach = attachable(mime)
+      if (attach) {
+        const bytes = Number(stat.size)
+        const max = Flag.OPENCODE_READ_MAX_ATTACHMENT_BYTES ?? MAX_ATTACHMENT_BYTES
+        if (bytes > max) {
+          return yield* Effect.fail(
+            new Error(
+              `Cannot attach ${attach} file larger than ${format(max)}: ${filepath} (${format(bytes)})`,
+            ),
+          )
+        }
+
+        const msg = `${attach[0]!.toUpperCase()}${attach.slice(1)} read successfully`
         return {
           title,
           output: msg,
@@ -265,6 +276,21 @@ async function lines(filepath: string, opts: { limit: number; offset: number })
   return { raw, count, cut, more, offset: opts.offset }
 }
 
+function attachable(mime: string): "image" | "PDF" | "video" | "audio" | undefined {
+  if (mime.startsWith("image/") && mime !== "image/svg+xml" && mime !== "image/vnd.fastbidsheet") return "image"
+  if (mime === "application/pdf") return "PDF"
+  if (mime.startsWith("video/")) return "video"
+  if (mime.startsWith("audio/")) return "audio"
+  return undefined
+}
+
+function format(bytes: number) {
+  if (bytes < 1024) return `${bytes} B`
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`
+}
+
 async function isBinaryFile(filepath: string, fileSize: number): Promise<boolean> {
   const ext = path.extname(filepath).toLowerCase()
   // binary check for common non-text extensions
diff --git a/packages/opencode/src/tool/read.txt b/packages/opencode/src/tool/read.txt
@@ -11,4 +11,5 @@ Usage:
 - Any line longer than 2000 characters is truncated.
 - Call this tool in parallel when you know there are multiple files you want to read.
 - Avoid tiny repeated slices (30 line chunks). If you need more context, read a larger window.
-- This tool can read image files and PDFs and return them as file attachments.
+- This tool can read images, PDFs, audio, and video files and return them as file attachments when the file type and size are supported.
+- Oversized binary attachments fail explicitly instead of being returned as text or truncated base64. The default attachment limit is 256 MB and can be changed with OPENCODE_READ_MAX_ATTACHMENT_BYTES.
diff --git a/packages/opencode/test/session/compaction.test.ts b/packages/opencode/test/session/compaction.test.ts
@@ -678,6 +678,15 @@ describe("session.compaction.process", () => {
           filename: "cat.png",
           url: "https://example.com/cat.png",
         })
+        await Session.updatePart({
+          id: PartID.ascending(),
+          messageID: replay.id,
+          sessionID: session.id,
+          type: "file",
+          mime: "video/mp4",
+          filename: "clip.mp4",
+          url: "https://example.com/clip.mp4",
+        })
         const msg = await user(session.id, "current")
         const rt = runtime("continue", Plugin.defaultLayer, wide())
         try {
@@ -702,6 +711,9 @@ describe("session.compaction.process", () => {
           expect(
             last?.parts.some((part) => part.type === "text" && part.text.includes("Attached image/png: cat.png")),
           ).toBe(true)
+          expect(
+            last?.parts.some((part) => part.type === "text" && part.text.includes("Attached video/mp4: clip.mp4")),
+          ).toBe(true)
         } finally {
           await rt.dispose()
         }
diff --git a/packages/opencode/test/session/message-v2.test.ts b/packages/opencode/test/session/message-v2.test.ts
@@ -107,6 +107,16 @@ function basePart(messageID: string, id: string) {
   }
 }
 
+describe("session.message-v2.isMedia", () => {
+  test("classifies images, PDFs, video, and audio as media", () => {
+    expect(MessageV2.isMedia("image/png")).toBe(true)
+    expect(MessageV2.isMedia("application/pdf")).toBe(true)
+    expect(MessageV2.isMedia("video/mp4")).toBe(true)
+    expect(MessageV2.isMedia("audio/mpeg")).toBe(true)
+    expect(MessageV2.isMedia("text/plain")).toBe(false)
+  })
+})
+
 describe("session.message-v2.toModelMessage", () => {
   test("filters out messages with no parts", async () => {
     const input: MessageV2.WithParts[] = [
@@ -359,6 +369,101 @@ describe("session.message-v2.toModelMessage", () => {
     ])
   })
 
+  test("injects media tool attachments for providers without media tool results", async () => {
+    const userID = "m-user"
+    const assistantID = "m-assistant"
+    const next = {
+      ...model,
+      api: {
+        ...model.api,
+        npm: "@ai-sdk/openai-compatible",
+      },
+    } as Provider.Model
+
+    const input: MessageV2.WithParts[] = [
+      {
+        info: userInfo(userID),
+        parts: [
+          {
+            ...basePart(userID, "u1"),
+            type: "text",
+            text: "read video",
+          },
+        ] as MessageV2.Part[],
+      },
+      {
+        info: assistantInfo(assistantID, userID),
+        parts: [
+          {
+            ...basePart(assistantID, "a1"),
+            type: "tool",
+            callID: "call-1",
+            tool: "read",
+            state: {
+              status: "completed",
+              input: { filePath: "/tmp/clip.mp4" },
+              output: "Video read successfully",
+              title: "clip.mp4",
+              metadata: {},
+              time: { start: 0, end: 1 },
+              attachments: [
+                {
+                  ...basePart(assistantID, "file-1"),
+                  type: "file",
+                  mime: "video/mp4",
+                  filename: "clip.mp4",
+                  url: "data:video/mp4;base64,Zm9v",
+                },
+              ],
+            },
+          },
+        ] as MessageV2.Part[],
+      },
+    ]
+
+    expect(await MessageV2.toModelMessages(input, next)).toStrictEqual([
+      {
+        role: "user",
+        content: [{ type: "text", text: "read video" }],
+      },
+      {
+        role: "assistant",
+        content: [
+          {
+            type: "tool-call",
+            toolCallId: "call-1",
+            toolName: "read",
+            input: { filePath: "/tmp/clip.mp4" },
+            providerExecuted: undefined,
+          },
+        ],
+      },
+      {
+        role: "tool",
+        content: [
+          {
+            type: "tool-result",
+            toolCallId: "call-1",
+            toolName: "read",
+            output: { type: "text", value: "Video read successfully" },
+          },
+        ],
+      },
+      {
+        role: "user",
+        content: [
+          { type: "text", text: MessageV2.SYNTHETIC_ATTACHMENT_PROMPT },
+          {
+            type: "file",
+            data: "data:video/mp4;base64,Zm9v",
+            filename: undefined,
+            mediaType: "video/mp4",
+          },
+        ],
+      },
+    ])
+  })
+
   test("omits provider metadata when assistant model differs", async () => {
     const userID = "m-user"
     const assistantID = "m-assistant"
diff --git a/packages/opencode/test/tool/read.test.ts b/packages/opencode/test/tool/read.test.ts
@@ -1,10 +1,12 @@
 import { afterEach, describe, expect } from "bun:test"
 import { Cause, Effect, Exit, Layer } from "effect"
+import { truncate as resize } from "fs/promises"
 import path from "path"
 import { Agent } from "../../src/agent/agent"
 import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner"
 import { AppFileSystem } from "../../src/filesystem"
 import { FileTime } from "../../src/file/time"
+import { Flag } from "../../src/flag/flag"
 import { LSP } from "../../src/lsp"
 import { Permission } from "../../src/permission"
 import { Instance } from "../../src/project/instance"
@@ -409,6 +411,69 @@ describe("tool.read truncation", () => {
     }),
   )
 
+  it.live("attaches PDFs, video, and audio as file attachments", () =>
+    Effect.gen(function* () {
+      const dir = yield* tmpdirScoped()
+      const cases = [
+        { file: "doc.pdf", mime: "application/pdf", msg: "PDF read successfully" },
+        { file: "clip.mp4", mime: "video/mp4", msg: "Video read successfully" },
+        { file: "clip.webm", mime: "video/webm", msg: "Video read successfully" },
+        { file: "sound.mp3", mime: "audio/mpeg", msg: "Audio read successfully" },
+      ]
+
+      yield* Effect.forEach(
+        cases,
+        (item) =>
+          Effect.gen(function* () {
+            yield* put(path.join(dir, item.file), "media")
+            const result = yield* exec(dir, { filePath: path.join(dir, item.file) })
+
+            expect(result.output).toBe(item.msg)
+            expect(result.metadata.preview).toBe(item.msg)
+            expect(result.metadata.truncated).toBe(false)
+            expect(result.attachments).toBeDefined()
+            expect(result.attachments?.[0].type).toBe("file")
+            expect(result.attachments?.[0].mime).toBe(item.mime)
+            expect(result.attachments?.[0].url).toStartWith(`data:${item.mime};base64,`)
+          }),
+        { concurrency: "unbounded" },
+      )
+    }),
+  )
+
+  it.live("rejects oversized media attachments before reading content", () =>
+    Effect.gen(function* () {
+      const dir = yield* tmpdirScoped()
+      const file = path.join(dir, "large.mp4")
+      yield* put(file, "")
+      yield* Effect.promise(() => resize(file, 256 * 1024 * 1024 + 1))
+
+      const err = yield* fail(dir, { filePath: file })
+      expect(err.message).toContain("Cannot attach video file larger than 256.0 MB")
+      expect(err.message).toContain(file)
+    }),
+  )
+
+  it.live("uses OPENCODE_READ_MAX_ATTACHMENT_BYTES for media attachment limit", () =>
+    Effect.gen(function* () {
+      const prev = Flag.OPENCODE_READ_MAX_ATTACHMENT_BYTES
+      try {
+        // @ts-expect-error tests can override static env flags
+        Flag.OPENCODE_READ_MAX_ATTACHMENT_BYTES = 4
+        const dir = yield* tmpdirScoped()
+        const file = path.join(dir, "small.mp4")
+        yield* put(file, "media")
+
+        const err = yield* fail(dir, { filePath: file })
+        expect(err.message).toContain("Cannot attach video file larger than 4 B")
+        expect(err.message).toContain("(5 B)")
+      } finally {
+        // @ts-expect-error tests can override static env flags
+        Flag.OPENCODE_READ_MAX_ATTACHMENT_BYTES = prev
+      }
+    }),
+  )
+
   it.live(".fbs files (FlatBuffers schema) are read as text, not images", () =>
     Effect.gen(function* () {
       const dir = yield* tmpdirScoped()
