feat(core): preserve multimodal content blocks in provider responses

packages/core/src/evaluation/providers/claude-cli.ts

@@ -5,7 +5,7 @@ import type { WriteStream } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 
-import type { Content } from '../content.js';
+import { extractTextContent, toContentArray } from './claude-content.js';
 import { recordClaudeLogEntry } from './claude-log-tracker.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
 import type { ClaudeResolvedConfig } from './targets.js';
@@ -479,72 +479,6 @@ function summarizeEvent(event: Record<string, unknown>): string | undefined {
   }
 }
 
-/**
- * Convert Claude's content array to Content[] preserving non-text blocks.
- * Returns undefined if content is a plain string or has only text blocks
- * (no benefit over the simpler string representation).
- */
-function toContentArray(content: unknown): Content[] | undefined {
-  if (!Array.isArray(content)) return undefined;
-
-  let hasNonText = false;
-  const blocks: Content[] = [];
-
-  for (const part of content) {
-    if (!part || typeof part !== 'object') continue;
-    const p = part as Record<string, unknown>;
-
-    if (p.type === 'text' && typeof p.text === 'string') {
-      blocks.push({ type: 'text', text: p.text });
-    } else if (p.type === 'image' && typeof p.source === 'object' && p.source !== null) {
-      const src = p.source as Record<string, unknown>;
-      const mediaType =
-        typeof p.media_type === 'string'
-          ? p.media_type
-          : typeof src.media_type === 'string'
-            ? src.media_type
-            : 'application/octet-stream';
-      const data =
-        typeof src.data === 'string'
-          ? `data:${mediaType};base64,${src.data}`
-          : typeof p.url === 'string'
-            ? (p.url as string)
-            : '';
-      blocks.push({ type: 'image', media_type: mediaType, source: data });
-      hasNonText = true;
-    } else if (p.type === 'tool_use') {
-      // tool_use blocks are handled separately as ToolCall — skip
-    } else if (p.type === 'tool_result') {
-      // tool_result blocks are not user content — skip
-    }
-  }
-
-  return hasNonText && blocks.length > 0 ? blocks : undefined;
-}
-
-/**
- * Extract text content from Claude's content array format.
- */
-function extractTextContent(content: unknown): string | undefined {
-  if (typeof content === 'string') {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return undefined;
-  }
-  const textParts: string[] = [];
-  for (const part of content) {
-    if (!part || typeof part !== 'object') {
-      continue;
-    }
-    const p = part as Record<string, unknown>;
-    if (p.type === 'text' && typeof p.text === 'string') {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join('\n') : undefined;
-}
-
 /**
  * Extract tool calls from Claude's content array format.
  */

packages/core/src/evaluation/providers/claude-content.ts

@@ -0,0 +1,96 @@
+/**
+ * Shared content-mapping utilities for Claude-based providers.
+ *
+ * Converts Claude's raw content array format (Anthropic API) into the AgentV
+ * Content[] union so that non-text blocks (images) flow through the pipeline
+ * without lossy flattening.
+ *
+ * Used by: claude-cli, claude-sdk, claude (legacy).
+ *
+ * ## Claude content format
+ *
+ * Claude responses use:
+ * ```json
+ * { "content": [
+ *     { "type": "text", "text": "..." },
+ *     { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "..." } },
+ *     { "type": "tool_use", "name": "...", "input": {...}, "id": "..." }
+ * ]}
+ * ```
+ *
+ * `toContentArray` maps text and image blocks to `Content[]`.
+ * `tool_use` and `tool_result` blocks are handled separately as `ToolCall`.
+ */
+
+import type { Content } from '../content.js';
+
+/**
+ * Convert Claude's raw content array to `Content[]`, preserving non-text blocks.
+ *
+ * Returns `undefined` when the content is a plain string or contains only text
+ * blocks — callers should fall back to the text-only string representation in
+ * that case (no benefit from wrapping plain text in `Content[]`).
+ */
+export function toContentArray(content: unknown): Content[] | undefined {
+  if (!Array.isArray(content)) return undefined;
+
+  let hasNonText = false;
+  const blocks: Content[] = [];
+
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue;
+    const p = part as Record<string, unknown>;
+
+    if (p.type === 'text' && typeof p.text === 'string') {
+      blocks.push({ type: 'text', text: p.text });
+    } else if (p.type === 'image' && typeof p.source === 'object' && p.source !== null) {
+      const src = p.source as Record<string, unknown>;
+      const mediaType =
+        typeof p.media_type === 'string'
+          ? p.media_type
+          : typeof src.media_type === 'string'
+            ? src.media_type
+            : 'application/octet-stream';
+      const data =
+        typeof src.data === 'string' && src.data.length > 0
+          ? `data:${mediaType};base64,${src.data}`
+          : typeof p.url === 'string' && (p.url as string).length > 0
+            ? (p.url as string)
+            : '';
+      if (data) {
+        blocks.push({ type: 'image', media_type: mediaType, source: data });
+        hasNonText = true;
+      }
+    } else if (p.type === 'tool_use') {
+      // tool_use blocks are handled separately as ToolCall — skip
+    } else if (p.type === 'tool_result') {
+      // tool_result blocks are not user content — skip
+    }
+  }
+
+  return hasNonText && blocks.length > 0 ? blocks : undefined;
+}
+
+/**
+ * Extract text content from Claude's content array format.
+ * Returns joined text from all `type: 'text'` blocks (newline-separated).
+ */
+export function extractTextContent(content: unknown): string | undefined {
+  if (typeof content === 'string') {
+    return content;
+  }
+  if (!Array.isArray(content)) {
+    return undefined;
+  }
+  const textParts: string[] = [];
+  for (const part of content) {
+    if (!part || typeof part !== 'object') {
+      continue;
+    }
+    const p = part as Record<string, unknown>;
+    if (p.type === 'text' && typeof p.text === 'string') {
+      textParts.push(p.text);
+    }
+  }
+  return textParts.length > 0 ? textParts.join('\n') : undefined;
+}

packages/core/src/evaluation/providers/claude-sdk.ts

@@ -4,6 +4,7 @@ import type { WriteStream } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 
+import { extractTextContent, toContentArray } from './claude-content.js';
 import { recordClaudeLogEntry } from './claude-log-tracker.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
 import type { ClaudeResolvedConfig } from './targets.js';
@@ -139,12 +140,13 @@ export class ClaudeSdkProvider implements Provider {
             if (betaMessage && typeof betaMessage === 'object') {
               const msg = betaMessage as Record<string, unknown>;
               const content = msg.content;
+              const structuredContent = toContentArray(content);
               const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls(content);
 
               const outputMsg: Message = {
                 role: 'assistant',
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
               };
               output.push(outputMsg);
@@ -280,30 +282,6 @@ export class ClaudeSdkProvider implements Provider {
   }
 }
 
-/**
- * Extract text content from Claude's content array format.
- * Claude uses: content: [{ type: "text", text: "..." }, ...]
- */
-function extractTextContent(content: unknown): string | undefined {
-  if (typeof content === 'string') {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return undefined;
-  }
-  const textParts: string[] = [];
-  for (const part of content) {
-    if (!part || typeof part !== 'object') {
-      continue;
-    }
-    const p = part as Record<string, unknown>;
-    if (p.type === 'text' && typeof p.text === 'string') {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join('\n') : undefined;
-}
-
 /**
  * Extract tool calls from Claude's content array format.
  * Claude uses: content: [{ type: "tool_use", name: "...", input: {...}, id: "..." }, ...]

packages/core/src/evaluation/providers/claude.ts

@@ -4,6 +4,7 @@ import type { WriteStream } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 
+import { extractTextContent, toContentArray } from './claude-content.js';
 import { recordClaudeLogEntry } from './claude-log-tracker.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
 import type { ClaudeResolvedConfig } from './targets.js';
@@ -139,12 +140,13 @@ export class ClaudeProvider implements Provider {
             if (betaMessage && typeof betaMessage === 'object') {
               const msg = betaMessage as Record<string, unknown>;
               const content = msg.content;
+              const structuredContent = toContentArray(content);
               const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls(content);
 
               const outputMsg: Message = {
                 role: 'assistant',
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
               };
               output.push(outputMsg);
@@ -278,30 +280,6 @@ export class ClaudeProvider implements Provider {
   }
 }
 
-/**
- * Extract text content from Claude's content array format.
- * Claude uses: content: [{ type: "text", text: "..." }, ...]
- */
-function extractTextContent(content: unknown): string | undefined {
-  if (typeof content === 'string') {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return undefined;
-  }
-  const textParts: string[] = [];
-  for (const part of content) {
-    if (!part || typeof part !== 'object') {
-      continue;
-    }
-    const p = part as Record<string, unknown>;
-    if (p.type === 'text' && typeof p.text === 'string') {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join('\n') : undefined;
-}
-
 /**
  * Extract tool calls from Claude's content array format.
  * Claude uses: content: [{ type: "tool_use", name: "...", input: {...}, id: "..." }, ...]

packages/core/src/evaluation/providers/pi-coding-agent.ts

@@ -18,7 +18,7 @@ import { createInterface } from 'node:readline';
 import { fileURLToPath } from 'node:url';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
+import { extractPiTextContent, toFiniteNumber, toPiContentArray } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCodingAgentResolvedConfig } from './targets.js';
 import type {
@@ -564,7 +564,8 @@ function convertAgentMessage(
 
   const msg = message as Record<string, unknown>;
   const role = typeof msg.role === 'string' ? msg.role : 'unknown';
-  const content = extractPiTextContent(msg.content);
+  const structuredContent = toPiContentArray(msg.content);
+  const content = structuredContent ?? extractPiTextContent(msg.content);
   const toolCalls = extractToolCalls(msg.content, toolTrackers, completedToolResults);
   const startTimeVal =
     typeof msg.timestamp === 'number'

packages/core/src/evaluation/providers/pi-utils.ts

@@ -5,6 +5,8 @@
  * and safe numeric conversions.
  */
 
+import type { Content } from '../content.js';
+
 /**
  * Extract text content from Pi's content array format.
  * Pi uses: content: [{ type: "text", text: "..." }, ...]
@@ -32,6 +34,50 @@ export function extractPiTextContent(content: unknown): string | undefined {
   return textParts.length > 0 ? textParts.join('\n') : undefined;
 }
 
+/**
+ * Convert Pi's content array to `Content[]`, preserving non-text blocks.
+ *
+ * Returns `undefined` when content is a plain string or contains only text
+ * blocks — callers should fall back to the text-only string representation.
+ */
+export function toPiContentArray(content: unknown): Content[] | undefined {
+  if (!Array.isArray(content)) return undefined;
+
+  let hasNonText = false;
+  const blocks: Content[] = [];
+
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue;
+    const p = part as Record<string, unknown>;
+
+    if (p.type === 'text' && typeof p.text === 'string') {
+      blocks.push({ type: 'text', text: p.text });
+    } else if (p.type === 'image') {
+      const mediaType =
+        typeof p.media_type === 'string' ? p.media_type : 'application/octet-stream';
+
+      let source = '';
+      if (typeof p.source === 'object' && p.source !== null) {
+        const src = p.source as Record<string, unknown>;
+        const srcMediaType = typeof src.media_type === 'string' ? src.media_type : mediaType;
+        source = typeof src.data === 'string' ? `data:${srcMediaType};base64,${src.data}` : '';
+      }
+      if (!source && typeof p.url === 'string') {
+        source = p.url;
+      }
+
+      if (source) {
+        blocks.push({ type: 'image', media_type: mediaType, source });
+        hasNonText = true;
+      }
+    } else if (p.type === 'tool_use' || p.type === 'tool_result') {
+      // Handled separately — skip
+    }
+  }
+
+  return hasNonText && blocks.length > 0 ? blocks : undefined;
+}
+
 /**
  * Safely convert an unknown value to a finite number, or undefined.
  */