From 9a9098848e75c1fe9210df00d716a03c02099f94 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Sun, 29 Mar 2026 02:34:08 +0000
Subject: [PATCH 1/2] feat(core): preserve multimodal content blocks in
 provider responses

Update Claude and Pi providers to preserve non-text content blocks
(images) in Message.content instead of discarding them via
extractTextContent(). This enables multimodal content to flow from
provider response through to evaluators.

Changes:
- Create shared claude-content.ts with toContentArray() and
  extractTextContent() used by all 3 Claude providers
- Update claude-cli, claude-sdk, claude providers to use
  structuredContent ?? textContent pattern
- Add toPiContentArray() to pi-utils.ts for Pi provider
- Update pi-coding-agent convertAgentMessage() to preserve
  structured content
- Add 23 unit tests covering content preservation, backward
  compat, and end-to-end multimodal flow

Text-only responses still produce plain strings (no unnecessary
wrapping). extractTextContent() remains available for backward
compatibility.

Closes #818

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../src/evaluation/providers/claude-cli.ts    |  68 +----
 .../evaluation/providers/claude-content.ts    |  94 ++++++
 .../src/evaluation/providers/claude-sdk.ts    |  28 +-
 .../core/src/evaluation/providers/claude.ts   |  28 +-
 .../evaluation/providers/pi-coding-agent.ts   |   5 +-
 .../core/src/evaluation/providers/pi-utils.ts |  50 +++
 .../providers/content-preserve.test.ts        | 288 ++++++++++++++++++
 7 files changed, 442 insertions(+), 119 deletions(-)
 create mode 100644 packages/core/src/evaluation/providers/claude-content.ts
 create mode 100644 packages/core/test/evaluation/providers/content-preserve.test.ts

diff --git a/packages/core/src/evaluation/providers/claude-cli.ts b/packages/core/src/evaluation/providers/claude-cli.ts
index d400c2069..1699810dd 100644
--- a/packages/core/src/evaluation/providers/claude-cli.ts
+++ b/packages/core/src/evaluation/providers/claude-cli.ts
@@ -5,7 +5,7 @@ import type { WriteStream } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 
-import type { Content } from '../content.js';
+import { extractTextContent, toContentArray } from './claude-content.js';
 import { recordClaudeLogEntry } from './claude-log-tracker.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
 import type { ClaudeResolvedConfig } from './targets.js';
@@ -479,72 +479,6 @@ function summarizeEvent(event: Record<string, unknown>): string | undefined {
   }
 }
 
-/**
- * Convert Claude's content array to Content[] preserving non-text blocks.
- * Returns undefined if content is a plain string or has only text blocks
- * (no benefit over the simpler string representation).
- */
-function toContentArray(content: unknown): Content[] | undefined {
-  if (!Array.isArray(content)) return undefined;
-
-  let hasNonText = false;
-  const blocks: Content[] = [];
-
-  for (const part of content) {
-    if (!part || typeof part !== 'object') continue;
-    const p = part as Record<string, unknown>;
-
-    if (p.type === 'text' && typeof p.text === 'string') {
-      blocks.push({ type: 'text', text: p.text });
-    } else if (p.type === 'image' && typeof p.source === 'object' && p.source !== null) {
-      const src = p.source as Record<string, unknown>;
-      const mediaType =
-        typeof p.media_type === 'string'
-          ? p.media_type
-          : typeof src.media_type === 'string'
-            ? src.media_type
-            : 'application/octet-stream';
-      const data =
-        typeof src.data === 'string'
-          ? `data:${mediaType};base64,${src.data}`
-          : typeof p.url === 'string'
-            ? (p.url as string)
-            : '';
-      blocks.push({ type: 'image', media_type: mediaType, source: data });
-      hasNonText = true;
-    } else if (p.type === 'tool_use') {
-      // tool_use blocks are handled separately as ToolCall — skip
-    } else if (p.type === 'tool_result') {
-      // tool_result blocks are not user content — skip
-    }
-  }
-
-  return hasNonText && blocks.length > 0 ? blocks : undefined;
-}
-
-/**
- * Extract text content from Claude's content array format.
- */
-function extractTextContent(content: unknown): string | undefined {
-  if (typeof content === 'string') {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return undefined;
-  }
-  const textParts: string[] = [];
-  for (const part of content) {
-    if (!part || typeof part !== 'object') {
-      continue;
-    }
-    const p = part as Record<string, unknown>;
-    if (p.type === 'text' && typeof p.text === 'string') {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join('\n') : undefined;
-}
-
 /**
  * Extract tool calls from Claude's content array format.
  */
diff --git a/packages/core/src/evaluation/providers/claude-content.ts b/packages/core/src/evaluation/providers/claude-content.ts
new file mode 100644
index 000000000..889029fc9
--- /dev/null
+++ b/packages/core/src/evaluation/providers/claude-content.ts
@@ -0,0 +1,94 @@
+/**
+ * Shared content-mapping utilities for Claude-based providers.
+ *
+ * Converts Claude's raw content array format (Anthropic API) into the AgentV
+ * Content[] union so that non-text blocks (images) flow through the pipeline
+ * without lossy flattening.
+ *
+ * Used by: claude-cli, claude-sdk, claude (legacy).
+ *
+ * ## Claude content format
+ *
+ * Claude responses use:
+ * ```json
+ * { "content": [
+ *     { "type": "text", "text": "..." },
+ *     { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "..." } },
+ *     { "type": "tool_use", "name": "...", "input": {...}, "id": "..." }
+ * ]}
+ * ```
+ *
+ * `toContentArray` maps text and image blocks to `Content[]`.
+ * `tool_use` and `tool_result` blocks are handled separately as `ToolCall`.
+ */
+
+import type { Content } from '../content.js';
+
+/**
+ * Convert Claude's raw content array to `Content[]`, preserving non-text blocks.
+ *
+ * Returns `undefined` when the content is a plain string or contains only text
+ * blocks — callers should fall back to the text-only string representation in
+ * that case (no benefit from wrapping plain text in `Content[]`).
+ */
+export function toContentArray(content: unknown): Content[] | undefined {
+  if (!Array.isArray(content)) return undefined;
+
+  let hasNonText = false;
+  const blocks: Content[] = [];
+
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue;
+    const p = part as Record<string, unknown>;
+
+    if (p.type === 'text' && typeof p.text === 'string') {
+      blocks.push({ type: 'text', text: p.text });
+    } else if (p.type === 'image' && typeof p.source === 'object' && p.source !== null) {
+      const src = p.source as Record<string, unknown>;
+      const mediaType =
+        typeof p.media_type === 'string'
+          ? p.media_type
+          : typeof src.media_type === 'string'
+            ? src.media_type
+            : 'application/octet-stream';
+      const data =
+        typeof src.data === 'string'
+          ? `data:${mediaType};base64,${src.data}`
+          : typeof p.url === 'string'
+            ? (p.url as string)
+            : '';
+      blocks.push({ type: 'image', media_type: mediaType, source: data });
+      hasNonText = true;
+    } else if (p.type === 'tool_use') {
+      // tool_use blocks are handled separately as ToolCall — skip
+    } else if (p.type === 'tool_result') {
+      // tool_result blocks are not user content — skip
+    }
+  }
+
+  return hasNonText && blocks.length > 0 ? blocks : undefined;
+}
+
+/**
+ * Extract text content from Claude's content array format.
+ * Returns joined text from all `type: 'text'` blocks (newline-separated).
+ */
+export function extractTextContent(content: unknown): string | undefined {
+  if (typeof content === 'string') {
+    return content;
+  }
+  if (!Array.isArray(content)) {
+    return undefined;
+  }
+  const textParts: string[] = [];
+  for (const part of content) {
+    if (!part || typeof part !== 'object') {
+      continue;
+    }
+    const p = part as Record<string, unknown>;
+    if (p.type === 'text' && typeof p.text === 'string') {
+      textParts.push(p.text);
+    }
+  }
+  return textParts.length > 0 ? textParts.join('\n') : undefined;
+}
diff --git a/packages/core/src/evaluation/providers/claude-sdk.ts b/packages/core/src/evaluation/providers/claude-sdk.ts
index aab8cc16b..6e8985fa4 100644
--- a/packages/core/src/evaluation/providers/claude-sdk.ts
+++ b/packages/core/src/evaluation/providers/claude-sdk.ts
@@ -4,6 +4,7 @@ import type { WriteStream } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 
+import { extractTextContent, toContentArray } from './claude-content.js';
 import { recordClaudeLogEntry } from './claude-log-tracker.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
 import type { ClaudeResolvedConfig } from './targets.js';
@@ -139,12 +140,13 @@ export class ClaudeSdkProvider implements Provider {
             if (betaMessage && typeof betaMessage === 'object') {
               const msg = betaMessage as Record<string, unknown>;
               const content = msg.content;
+              const structuredContent = toContentArray(content);
               const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls(content);
 
               const outputMsg: Message = {
                 role: 'assistant',
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
               };
               output.push(outputMsg);
@@ -280,30 +282,6 @@ export class ClaudeSdkProvider implements Provider {
   }
 }
 
-/**
- * Extract text content from Claude's content array format.
- * Claude uses: content: [{ type: "text", text: "..." }, ...]
- */
-function extractTextContent(content: unknown): string | undefined {
-  if (typeof content === 'string') {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return undefined;
-  }
-  const textParts: string[] = [];
-  for (const part of content) {
-    if (!part || typeof part !== 'object') {
-      continue;
-    }
-    const p = part as Record<string, unknown>;
-    if (p.type === 'text' && typeof p.text === 'string') {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join('\n') : undefined;
-}
-
 /**
  * Extract tool calls from Claude's content array format.
  * Claude uses: content: [{ type: "tool_use", name: "...", input: {...}, id: "..." }, ...]
diff --git a/packages/core/src/evaluation/providers/claude.ts b/packages/core/src/evaluation/providers/claude.ts
index 62382a604..2ac222e4f 100644
--- a/packages/core/src/evaluation/providers/claude.ts
+++ b/packages/core/src/evaluation/providers/claude.ts
@@ -4,6 +4,7 @@ import type { WriteStream } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
 
+import { extractTextContent, toContentArray } from './claude-content.js';
 import { recordClaudeLogEntry } from './claude-log-tracker.js';
 import { buildPromptDocument, normalizeInputFiles } from './preread.js';
 import type { ClaudeResolvedConfig } from './targets.js';
@@ -139,12 +140,13 @@ export class ClaudeProvider implements Provider {
             if (betaMessage && typeof betaMessage === 'object') {
               const msg = betaMessage as Record<string, unknown>;
               const content = msg.content;
+              const structuredContent = toContentArray(content);
               const textContent = extractTextContent(content);
               const toolCalls = extractToolCalls(content);
 
               const outputMsg: Message = {
                 role: 'assistant',
-                content: textContent,
+                content: structuredContent ?? textContent,
                 toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
               };
               output.push(outputMsg);
@@ -278,30 +280,6 @@ export class ClaudeProvider implements Provider {
   }
 }
 
-/**
- * Extract text content from Claude's content array format.
- * Claude uses: content: [{ type: "text", text: "..." }, ...]
- */
-function extractTextContent(content: unknown): string | undefined {
-  if (typeof content === 'string') {
-    return content;
-  }
-  if (!Array.isArray(content)) {
-    return undefined;
-  }
-  const textParts: string[] = [];
-  for (const part of content) {
-    if (!part || typeof part !== 'object') {
-      continue;
-    }
-    const p = part as Record<string, unknown>;
-    if (p.type === 'text' && typeof p.text === 'string') {
-      textParts.push(p.text);
-    }
-  }
-  return textParts.length > 0 ? textParts.join('\n') : undefined;
-}
-
 /**
  * Extract tool calls from Claude's content array format.
  * Claude uses: content: [{ type: "tool_use", name: "...", input: {...}, id: "..." }, ...]
diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts
index 1c92f92cd..3e4691bd0 100644
--- a/packages/core/src/evaluation/providers/pi-coding-agent.ts
+++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts
@@ -18,7 +18,7 @@ import { createInterface } from 'node:readline';
 import { fileURLToPath } from 'node:url';
 
 import { recordPiLogEntry } from './pi-log-tracker.js';
-import { extractPiTextContent, toFiniteNumber } from './pi-utils.js';
+import { extractPiTextContent, toFiniteNumber, toPiContentArray } from './pi-utils.js';
 import { normalizeInputFiles } from './preread.js';
 import type { PiCodingAgentResolvedConfig } from './targets.js';
 import type {
@@ -564,7 +564,8 @@ function convertAgentMessage(
 
   const msg = message as Record<string, unknown>;
   const role = typeof msg.role === 'string' ? msg.role : 'unknown';
-  const content = extractPiTextContent(msg.content);
+  const structuredContent = toPiContentArray(msg.content);
+  const content = structuredContent ?? extractPiTextContent(msg.content);
   const toolCalls = extractToolCalls(msg.content, toolTrackers, completedToolResults);
   const startTimeVal =
     typeof msg.timestamp === 'number'
diff --git a/packages/core/src/evaluation/providers/pi-utils.ts b/packages/core/src/evaluation/providers/pi-utils.ts
index 058720870..3ea78d3d8 100644
--- a/packages/core/src/evaluation/providers/pi-utils.ts
+++ b/packages/core/src/evaluation/providers/pi-utils.ts
@@ -5,6 +5,8 @@
  * and safe numeric conversions.
  */
 
+import type { Content } from '../content.js';
+
 /**
  * Extract text content from Pi's content array format.
  * Pi uses: content: [{ type: "text", text: "..." }, ...]
@@ -32,6 +34,54 @@ export function extractPiTextContent(content: unknown): string | undefined {
   return textParts.length > 0 ? textParts.join('\n') : undefined;
 }
 
+/**
+ * Convert Pi's content array to `Content[]`, preserving non-text blocks.
+ *
+ * Returns `undefined` when content is a plain string or contains only text
+ * blocks — callers should fall back to the text-only string representation.
+ */
+export function toPiContentArray(content: unknown): Content[] | undefined {
+  if (!Array.isArray(content)) return undefined;
+
+  let hasNonText = false;
+  const blocks: Content[] = [];
+
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue;
+    const p = part as Record<string, unknown>;
+
+    if (p.type === 'text' && typeof p.text === 'string') {
+      blocks.push({ type: 'text', text: p.text });
+    } else if (p.type === 'image') {
+      const mediaType =
+        typeof p.media_type === 'string' ? p.media_type : 'application/octet-stream';
+
+      let source = '';
+      if (typeof p.source === 'object' && p.source !== null) {
+        const src = p.source as Record<string, unknown>;
+        const srcMediaType =
+          typeof src.media_type === 'string' ? src.media_type : mediaType;
+        source =
+          typeof src.data === 'string'
+            ? `data:${srcMediaType};base64,${src.data}`
+            : '';
+      }
+      if (!source && typeof p.url === 'string') {
+        source = p.url;
+      }
+
+      if (source) {
+        blocks.push({ type: 'image', media_type: mediaType, source });
+        hasNonText = true;
+      }
+    } else if (p.type === 'tool_use' || p.type === 'tool_result') {
+      // Handled separately — skip
+    }
+  }
+
+  return hasNonText && blocks.length > 0 ? blocks : undefined;
+}
+
 /**
  * Safely convert an unknown value to a finite number, or undefined.
  */
diff --git a/packages/core/test/evaluation/providers/content-preserve.test.ts b/packages/core/test/evaluation/providers/content-preserve.test.ts
new file mode 100644
index 000000000..626ed0e35
--- /dev/null
+++ b/packages/core/test/evaluation/providers/content-preserve.test.ts
@@ -0,0 +1,288 @@
+import { describe, expect, it } from 'vitest';
+
+import { getTextContent } from '../../../src/evaluation/content.js';
+import {
+  extractTextContent,
+  toContentArray,
+} from '../../../src/evaluation/providers/claude-content.js';
+import {
+  extractPiTextContent,
+  toPiContentArray,
+} from '../../../src/evaluation/providers/pi-utils.js';
+import type { Content } from '../../../src/evaluation/content.js';
+import type { Message } from '../../../src/evaluation/providers/types.js';
+
+// ---------------------------------------------------------------------------
+// toContentArray (Claude)
+// ---------------------------------------------------------------------------
+describe('toContentArray', () => {
+  it('returns undefined for non-array input', () => {
+    expect(toContentArray('plain string')).toBeUndefined();
+    expect(toContentArray(42)).toBeUndefined();
+    expect(toContentArray(null)).toBeUndefined();
+    expect(toContentArray(undefined)).toBeUndefined();
+  });
+
+  it('returns undefined when content has only text blocks', () => {
+    const content = [
+      { type: 'text', text: 'hello' },
+      { type: 'text', text: 'world' },
+    ];
+    expect(toContentArray(content)).toBeUndefined();
+  });
+
+  it('preserves image + text with base64 data', () => {
+    const content = [
+      { type: 'text', text: 'Here is an image:' },
+      {
+        type: 'image',
+        source: { type: 'base64', media_type: 'image/png', data: 'abc123' },
+      },
+    ];
+    const result = toContentArray(content);
+    expect(result).toBeDefined();
+    expect(result).toHaveLength(2);
+    expect(result![0]).toEqual({ type: 'text', text: 'Here is an image:' });
+    expect(result![1]).toEqual({
+      type: 'image',
+      media_type: 'image/png',
+      source: 'data:image/png;base64,abc123',
+    });
+  });
+
+  it('handles url images', () => {
+    const content = [
+      {
+        type: 'image',
+        url: 'https://example.com/img.png',
+        source: { type: 'url' },
+        media_type: 'image/png',
+      },
+    ];
+    const result = toContentArray(content);
+    expect(result).toBeDefined();
+    expect(result![0]).toEqual({
+      type: 'image',
+      media_type: 'image/png',
+      source: 'https://example.com/img.png',
+    });
+  });
+
+  it('skips tool_use and tool_result blocks', () => {
+    const content = [
+      { type: 'text', text: 'hi' },
+      { type: 'tool_use', name: 'bash', input: { cmd: 'ls' }, id: 't1' },
+      { type: 'tool_result', tool_use_id: 't1', content: 'ok' },
+      {
+        type: 'image',
+        source: { data: 'AAAA', media_type: 'image/jpeg' },
+      },
+    ];
+    const result = toContentArray(content);
+    expect(result).toBeDefined();
+    expect(result).toHaveLength(2);
+    expect(result![0]).toEqual({ type: 'text', text: 'hi' });
+    expect(result![1].type).toBe('image');
+  });
+
+  it('handles invalid parts gracefully', () => {
+    const content = [null, undefined, 42, 'string', { type: 'text', text: 'ok' }];
+    // only text → undefined (no non-text blocks)
+    expect(toContentArray(content)).toBeUndefined();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// extractTextContent (Claude)
+// ---------------------------------------------------------------------------
+describe('extractTextContent', () => {
+  it('passes through a plain string', () => {
+    expect(extractTextContent('hello')).toBe('hello');
+  });
+
+  it('returns undefined for non-array non-string', () => {
+    expect(extractTextContent(42)).toBeUndefined();
+    expect(extractTextContent(null)).toBeUndefined();
+    expect(extractTextContent(undefined)).toBeUndefined();
+    expect(extractTextContent({})).toBeUndefined();
+  });
+
+  it('extracts text from content array', () => {
+    const content = [
+      { type: 'text', text: 'hello' },
+      { type: 'text', text: 'world' },
+    ];
+    expect(extractTextContent(content)).toBe('hello\nworld');
+  });
+
+  it('skips non-text blocks', () => {
+    const content = [
+      { type: 'text', text: 'hello' },
+      { type: 'image', source: { data: 'abc' } },
+      { type: 'tool_use', name: 'bash' },
+    ];
+    expect(extractTextContent(content)).toBe('hello');
+  });
+
+  it('returns undefined for empty array', () => {
+    expect(extractTextContent([])).toBeUndefined();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// toPiContentArray
+// ---------------------------------------------------------------------------
+describe('toPiContentArray', () => {
+  it('returns undefined for non-array input', () => {
+    expect(toPiContentArray('plain string')).toBeUndefined();
+    expect(toPiContentArray(42)).toBeUndefined();
+    expect(toPiContentArray(null)).toBeUndefined();
+  });
+
+  it('returns undefined when content has only text blocks', () => {
+    const content = [
+      { type: 'text', text: 'hello' },
+      { type: 'text', text: 'world' },
+    ];
+    expect(toPiContentArray(content)).toBeUndefined();
+  });
+
+  it('preserves image + text with base64 source', () => {
+    const content = [
+      { type: 'text', text: 'Here is an image:' },
+      {
+        type: 'image',
+        media_type: 'image/png',
+        source: { data: 'abc123', media_type: 'image/png' },
+      },
+    ];
+    const result = toPiContentArray(content);
+    expect(result).toBeDefined();
+    expect(result).toHaveLength(2);
+    expect(result![0]).toEqual({ type: 'text', text: 'Here is an image:' });
+    expect(result![1]).toEqual({
+      type: 'image',
+      media_type: 'image/png',
+      source: 'data:image/png;base64,abc123',
+    });
+  });
+
+  it('handles url images', () => {
+    const content = [
+      {
+        type: 'image',
+        url: 'https://example.com/img.png',
+        media_type: 'image/png',
+      },
+    ];
+    const result = toPiContentArray(content);
+    expect(result).toBeDefined();
+    expect(result![0]).toEqual({
+      type: 'image',
+      media_type: 'image/png',
+      source: 'https://example.com/img.png',
+    });
+  });
+
+  it('skips tool_use and tool_result blocks', () => {
+    const content = [
+      { type: 'text', text: 'hi' },
+      { type: 'tool_use', name: 'bash' },
+      { type: 'tool_result', content: 'ok' },
+      {
+        type: 'image',
+        media_type: 'image/jpeg',
+        source: { data: 'AAAA', media_type: 'image/jpeg' },
+      },
+    ];
+    const result = toPiContentArray(content);
+    expect(result).toBeDefined();
+    expect(result).toHaveLength(2);
+    expect(result![0]).toEqual({ type: 'text', text: 'hi' });
+    expect(result![1].type).toBe('image');
+  });
+});
+
+// ---------------------------------------------------------------------------
+// extractPiTextContent (backward compat)
+// ---------------------------------------------------------------------------
+describe('extractPiTextContent', () => {
+  it('passes through a plain string', () => {
+    expect(extractPiTextContent('hello')).toBe('hello');
+  });
+
+  it('extracts text from content array', () => {
+    const content = [
+      { type: 'text', text: 'hello' },
+      { type: 'text', text: 'world' },
+    ];
+    expect(extractPiTextContent(content)).toBe('hello\nworld');
+  });
+
+  it('returns undefined for non-array non-string', () => {
+    expect(extractPiTextContent(42)).toBeUndefined();
+    expect(extractPiTextContent(null)).toBeUndefined();
+  });
+
+  it('returns undefined for empty array', () => {
+    expect(extractPiTextContent([])).toBeUndefined();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// End-to-end: Content[] interop
+// ---------------------------------------------------------------------------
+describe('End-to-end content preservation', () => {
+  it('Content[] is compatible with getTextContent', () => {
+    const blocks: Content[] = [
+      { type: 'text', text: 'hello' },
+      { type: 'image', media_type: 'image/png', source: 'data:image/png;base64,abc' },
+      { type: 'text', text: 'world' },
+    ];
+    expect(getTextContent(blocks)).toBe('hello\nworld');
+  });
+
+  it('image block survives into Message.content', () => {
+    const rawClaudeContent = [
+      { type: 'text', text: 'Look at this:' },
+      {
+        type: 'image',
+        source: { type: 'base64', media_type: 'image/png', data: 'DEADBEEF' },
+      },
+    ];
+
+    const structuredContent = toContentArray(rawClaudeContent);
+    const textContent = extractTextContent(rawClaudeContent);
+
+    const msg: Message = {
+      role: 'assistant',
+      content: structuredContent ?? textContent,
+    };
+
+    // content should be Content[] (not flattened to string)
+    expect(Array.isArray(msg.content)).toBe(true);
+    const blocks = msg.content as Content[];
+    expect(blocks).toHaveLength(2);
+    expect(blocks[0]).toEqual({ type: 'text', text: 'Look at this:' });
+    expect(blocks[1].type).toBe('image');
+    expect((blocks[1] as { source: string }).source).toContain('base64,DEADBEEF');
+  });
+
+  it('text-only content falls back to string', () => {
+    const rawClaudeContent = [
+      { type: 'text', text: 'Just text' },
+    ];
+
+    const structuredContent = toContentArray(rawClaudeContent);
+    const textContent = extractTextContent(rawClaudeContent);
+
+    const msg: Message = {
+      role: 'assistant',
+      content: structuredContent ?? textContent,
+    };
+
+    // text-only → toContentArray returns undefined → falls back to string
+    expect(typeof msg.content).toBe('string');
+    expect(msg.content).toBe('Just text');
+  });
+});

From 989b04b2cf30b840a8093f68f81e92e66afd0d65 Mon Sep 17 00:00:00 2001
From: Christopher Tso <christso@gmail.com>
Date: Sun, 29 Mar 2026 03:29:04 +0000
Subject: [PATCH 2/2] fix: guard against empty image source in toContentArray()

Skip ContentImage blocks when Claude's image block has neither valid
source.data nor a non-empty url, preventing invalid empty-source blocks
in the structured content output.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../evaluation/providers/claude-content.ts    | 10 ++--
 .../core/src/evaluation/providers/pi-utils.ts |  8 +--
 .../providers/claude-content.test.ts          | 59 +++++++++++++++++++
 .../providers/content-preserve.test.ts        | 26 ++++----
 4 files changed, 79 insertions(+), 24 deletions(-)
 create mode 100644 packages/core/test/evaluation/providers/claude-content.test.ts

diff --git a/packages/core/src/evaluation/providers/claude-content.ts b/packages/core/src/evaluation/providers/claude-content.ts
index 889029fc9..1f61d24fe 100644
--- a/packages/core/src/evaluation/providers/claude-content.ts
+++ b/packages/core/src/evaluation/providers/claude-content.ts
@@ -52,13 +52,15 @@ export function toContentArray(content: unknown): Content[] | undefined {
             ? src.media_type
             : 'application/octet-stream';
       const data =
-        typeof src.data === 'string'
+        typeof src.data === 'string' && src.data.length > 0
           ? `data:${mediaType};base64,${src.data}`
-          : typeof p.url === 'string'
+          : typeof p.url === 'string' && (p.url as string).length > 0
             ? (p.url as string)
             : '';
-      blocks.push({ type: 'image', media_type: mediaType, source: data });
-      hasNonText = true;
+      if (data) {
+        blocks.push({ type: 'image', media_type: mediaType, source: data });
+        hasNonText = true;
+      }
     } else if (p.type === 'tool_use') {
       // tool_use blocks are handled separately as ToolCall — skip
     } else if (p.type === 'tool_result') {
diff --git a/packages/core/src/evaluation/providers/pi-utils.ts b/packages/core/src/evaluation/providers/pi-utils.ts
index 3ea78d3d8..da32918ce 100644
--- a/packages/core/src/evaluation/providers/pi-utils.ts
+++ b/packages/core/src/evaluation/providers/pi-utils.ts
@@ -59,12 +59,8 @@ export function toPiContentArray(content: unknown): Content[] | undefined {
       let source = '';
       if (typeof p.source === 'object' && p.source !== null) {
         const src = p.source as Record<string, unknown>;
-        const srcMediaType =
-          typeof src.media_type === 'string' ? src.media_type : mediaType;
-        source =
-          typeof src.data === 'string'
-            ? `data:${srcMediaType};base64,${src.data}`
-            : '';
+        const srcMediaType = typeof src.media_type === 'string' ? src.media_type : mediaType;
+        source = typeof src.data === 'string' ? `data:${srcMediaType};base64,${src.data}` : '';
       }
       if (!source && typeof p.url === 'string') {
         source = p.url;
diff --git a/packages/core/test/evaluation/providers/claude-content.test.ts b/packages/core/test/evaluation/providers/claude-content.test.ts
new file mode 100644
index 000000000..d04116f16
--- /dev/null
+++ b/packages/core/test/evaluation/providers/claude-content.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, it } from 'vitest';
+
+import { toContentArray } from '../../../src/evaluation/providers/claude-content.js';
+
+describe('toContentArray – empty image source guard', () => {
+  it('skips image block when source.data is missing and url is absent', () => {
+    const input = [
+      { type: 'image', source: { media_type: 'image/png' } },
+      { type: 'text', text: 'hello' },
+    ];
+    const result = toContentArray(input);
+    // No valid image → no non-text content → returns undefined
+    expect(result).toBeUndefined();
+  });
+
+  it('skips image block when source.data is empty string', () => {
+    const input = [
+      { type: 'image', source: { data: '', media_type: 'image/png' } },
+      { type: 'text', text: 'hello' },
+    ];
+    const result = toContentArray(input);
+    expect(result).toBeUndefined();
+  });
+
+  it('skips image block when source.data is missing and url is empty string', () => {
+    const input = [
+      { type: 'image', source: { media_type: 'image/jpeg' }, url: '' },
+      { type: 'text', text: 'hello' },
+    ];
+    const result = toContentArray(input);
+    expect(result).toBeUndefined();
+  });
+
+  it('includes image block when source.data has valid base64', () => {
+    const input = [
+      { type: 'image', source: { data: 'abc123', media_type: 'image/png' } },
+      { type: 'text', text: 'hello' },
+    ];
+    const result = toContentArray(input);
+    expect(result).toBeDefined();
+    expect(result).toEqual([
+      { type: 'image', media_type: 'image/png', source: 'data:image/png;base64,abc123' },
+      { type: 'text', text: 'hello' },
+    ]);
+  });
+
+  it('includes image block when url is valid', () => {
+    const input = [
+      { type: 'image', source: { media_type: 'image/png' }, url: 'https://example.com/img.png' },
+      { type: 'text', text: 'caption' },
+    ];
+    const result = toContentArray(input);
+    expect(result).toBeDefined();
+    expect(result).toEqual([
+      { type: 'image', media_type: 'image/png', source: 'https://example.com/img.png' },
+      { type: 'text', text: 'caption' },
+    ]);
+  });
+});
diff --git a/packages/core/test/evaluation/providers/content-preserve.test.ts b/packages/core/test/evaluation/providers/content-preserve.test.ts
index 626ed0e35..fd700adaa 100644
--- a/packages/core/test/evaluation/providers/content-preserve.test.ts
+++ b/packages/core/test/evaluation/providers/content-preserve.test.ts
@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'vitest';
 
 import { getTextContent } from '../../../src/evaluation/content.js';
+import type { Content } from '../../../src/evaluation/content.js';
 import {
   extractTextContent,
   toContentArray,
@@ -9,7 +10,6 @@ import {
   extractPiTextContent,
   toPiContentArray,
 } from '../../../src/evaluation/providers/pi-utils.js';
-import type { Content } from '../../../src/evaluation/content.js';
 import type { Message } from '../../../src/evaluation/providers/types.js';
 
 // ---------------------------------------------------------------------------
@@ -42,8 +42,8 @@ describe('toContentArray', () => {
     const result = toContentArray(content);
     expect(result).toBeDefined();
     expect(result).toHaveLength(2);
-    expect(result![0]).toEqual({ type: 'text', text: 'Here is an image:' });
-    expect(result![1]).toEqual({
+    expect(result?.[0]).toEqual({ type: 'text', text: 'Here is an image:' });
+    expect(result?.[1]).toEqual({
       type: 'image',
       media_type: 'image/png',
       source: 'data:image/png;base64,abc123',
@@ -61,7 +61,7 @@ describe('toContentArray', () => {
     ];
     const result = toContentArray(content);
     expect(result).toBeDefined();
-    expect(result![0]).toEqual({
+    expect(result?.[0]).toEqual({
       type: 'image',
       media_type: 'image/png',
       source: 'https://example.com/img.png',
@@ -81,8 +81,8 @@ describe('toContentArray', () => {
     const result = toContentArray(content);
     expect(result).toBeDefined();
     expect(result).toHaveLength(2);
-    expect(result![0]).toEqual({ type: 'text', text: 'hi' });
-    expect(result![1].type).toBe('image');
+    expect(result?.[0]).toEqual({ type: 'text', text: 'hi' });
+    expect(result?.[1].type).toBe('image');
   });
 
   it('handles invalid parts gracefully', () => {
@@ -159,8 +159,8 @@ describe('toPiContentArray', () => {
     const result = toPiContentArray(content);
     expect(result).toBeDefined();
     expect(result).toHaveLength(2);
-    expect(result![0]).toEqual({ type: 'text', text: 'Here is an image:' });
-    expect(result![1]).toEqual({
+    expect(result?.[0]).toEqual({ type: 'text', text: 'Here is an image:' });
+    expect(result?.[1]).toEqual({
       type: 'image',
       media_type: 'image/png',
       source: 'data:image/png;base64,abc123',
@@ -177,7 +177,7 @@ describe('toPiContentArray', () => {
     ];
     const result = toPiContentArray(content);
     expect(result).toBeDefined();
-    expect(result![0]).toEqual({
+    expect(result?.[0]).toEqual({
       type: 'image',
       media_type: 'image/png',
       source: 'https://example.com/img.png',
@@ -198,8 +198,8 @@ describe('toPiContentArray', () => {
     const result = toPiContentArray(content);
     expect(result).toBeDefined();
     expect(result).toHaveLength(2);
-    expect(result![0]).toEqual({ type: 'text', text: 'hi' });
-    expect(result![1].type).toBe('image');
+    expect(result?.[0]).toEqual({ type: 'text', text: 'hi' });
+    expect(result?.[1].type).toBe('image');
   });
 });
 
@@ -269,9 +269,7 @@ describe('End-to-end content preservation', () => {
   });
 
   it('text-only content falls back to string', () => {
-    const rawClaudeContent = [
-      { type: 'text', text: 'Just text' },
-    ];
+    const rawClaudeContent = [{ type: 'text', text: 'Just text' }];
 
     const structuredContent = toContentArray(rawClaudeContent);
     const textContent = extractTextContent(rawClaudeContent);