Skip to content

Commit 39f626c

Browse files
christsoCopilot
christso
and
Copilot
authored
feat(schema): add type: 'image' to EVAL.yaml ContentItemSchema (#831)
* feat(core): Content union type for multimodal content model Introduce a discriminated union Content type (ContentText | ContentImage | ContentFile) that enables multimodal content to flow through the pipeline without lossy flattening. Changes: - Add packages/core/src/evaluation/content.ts with Content union type, type guards (isContent, isContentArray), and getTextContent() accessor - Update Message.content from 'unknown' to 'string | Content[]' - Update extractLastAssistantContent() to handle Content[] via getTextContent() - Update claude-cli provider to preserve non-text content blocks (images) instead of dropping them during extraction - Update cli provider to handle Content[] from external processes - Export all content types from @agentv/core public API - Add 25 unit tests covering type guards, accessors, backward compat, and extractLastAssistantContent with Content[] Closes #817 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * feat(schema): add type: 'image' to EVAL.yaml ContentItemSchema - Add 'image' to ContentItemSchema type enum in eval-file.schema.ts - Add image file processing in message-processor.ts (base64 encoding, media type detection) - Support both processMessages and processExpectedMessages - Extension-based media type mapping: png, jpg, jpeg, gif, webp, svg, bmp - Image content produces ContentImage blocks with data URI source - Clear error messages for missing files and unsupported extensions - Regenerated eval-schema.json - Unit tests for media type detection, image processing, and error cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 700e282 commit 39f626c

4 files changed

Lines changed: 329 additions & 6 deletions

File tree

‎packages/core/src/evaluation/loaders/message-processor.ts‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,29 @@ import type { JsonObject, TestMessage } from '../types.js';
77
import { isJsonObject } from '../types.js';
88
import { resolveFileReference } from './file-resolver.js';
99

10+
/**
11+
* Maps image file extensions to MIME types.
12+
* To add a new image format: add the extension (with leading dot) and its MIME type.
13+
*/
14+
const IMAGE_MEDIA_TYPES: Record<string, string> = {
15+
'.png': 'image/png',
16+
'.jpg': 'image/jpeg',
17+
'.jpeg': 'image/jpeg',
18+
'.gif': 'image/gif',
19+
'.webp': 'image/webp',
20+
'.svg': 'image/svg+xml',
21+
'.bmp': 'image/bmp',
22+
};
23+
24+
/**
25+
* Detect image MIME type from file extension.
26+
* Returns undefined for unsupported extensions.
27+
*/
28+
export function detectImageMediaType(filePath: string): string | undefined {
29+
const ext = path.extname(filePath).toLowerCase();
30+
return IMAGE_MEDIA_TYPES[ext];
31+
}
32+
1033
const ANSI_YELLOW = '\u001b[33m';
1134
const ANSI_RESET = '\u001b[0m';
1235

@@ -103,6 +126,56 @@ export async function processMessages(options: ProcessMessagesOptions): Promise<
103126
continue;
104127
}
105128

129+
if (segmentType === 'image') {
130+
const rawValue = asString(rawSegment.value);
131+
if (!rawValue) {
132+
continue;
133+
}
134+
135+
const { displayPath, resolvedPath, attempted } = await resolveFileReference(
136+
rawValue,
137+
searchRoots,
138+
);
139+
140+
if (!resolvedPath) {
141+
const attempts = attempted.length
142+
? [' Tried:', ...attempted.map((candidate) => ` ${candidate}`)]
143+
: undefined;
144+
const context = messageType === 'input' ? '' : ' in expected_output';
145+
logWarning(`Image file not found${context}: ${displayPath}`, attempts);
146+
continue;
147+
}
148+
149+
const mediaType = detectImageMediaType(resolvedPath);
150+
if (!mediaType) {
151+
logWarning(
152+
`Unsupported image extension for ${displayPath}. Supported: ${Object.keys(IMAGE_MEDIA_TYPES).join(', ')}`,
153+
);
154+
continue;
155+
}
156+
157+
try {
158+
const imageBuffer = await readFile(resolvedPath);
159+
const base64 = imageBuffer.toString('base64');
160+
161+
processedContent.push({
162+
type: 'image',
163+
media_type: mediaType,
164+
source: `data:${mediaType};base64,${base64}`,
165+
});
166+
167+
if (verbose) {
168+
const label = messageType === 'input' ? '[Image]' : '[Expected Output Image]';
169+
console.log(` ${label} Found: ${displayPath}`);
170+
console.log(` Resolved to: ${resolvedPath} (${mediaType})`);
171+
}
172+
} catch (error) {
173+
const context = messageType === 'input' ? '' : ' expected output';
174+
logWarning(`Could not read${context} image ${resolvedPath}: ${(error as Error).message}`);
175+
}
176+
continue;
177+
}
178+
106179
const clonedSegment = cloneJsonObject(rawSegment);
107180
processedContent.push(clonedSegment);
108181
const inlineValue = clonedSegment.value;
@@ -306,6 +379,54 @@ export async function processExpectedMessages(
306379
continue;
307380
}
308381

382+
if (segmentType === 'image') {
383+
const rawValue = asString(rawSegment.value);
384+
if (!rawValue) {
385+
continue;
386+
}
387+
388+
const { displayPath, resolvedPath, attempted } = await resolveFileReference(
389+
rawValue,
390+
searchRoots,
391+
);
392+
393+
if (!resolvedPath) {
394+
const attempts = attempted.length
395+
? [' Tried:', ...attempted.map((candidate) => ` ${candidate}`)]
396+
: undefined;
397+
logWarning(`Image file not found in expected_output: ${displayPath}`, attempts);
398+
continue;
399+
}
400+
401+
const mediaType = detectImageMediaType(resolvedPath);
402+
if (!mediaType) {
403+
logWarning(
404+
`Unsupported image extension for ${displayPath}. Supported: ${Object.keys(IMAGE_MEDIA_TYPES).join(', ')}`,
405+
);
406+
continue;
407+
}
408+
409+
try {
410+
const imageBuffer = await readFile(resolvedPath);
411+
const base64 = imageBuffer.toString('base64');
412+
processedContent.push({
413+
type: 'image',
414+
media_type: mediaType,
415+
source: `data:${mediaType};base64,${base64}`,
416+
});
417+
418+
if (verbose) {
419+
console.log(` [Expected Output Image] Found: ${displayPath}`);
420+
console.log(` Resolved to: ${resolvedPath} (${mediaType})`);
421+
}
422+
} catch (error) {
423+
logWarning(
424+
`Could not read expected output image ${resolvedPath}: ${(error as Error).message}`,
425+
);
426+
}
427+
continue;
428+
}
429+
309430
processedContent.push(cloneJsonObject(rawSegment));
310431
}
311432
segment.content = processedContent;

‎packages/core/src/evaluation/validation/eval-file.schema.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import { z } from 'zod';
1414

1515
/** Message content: string or structured array */
1616
const ContentItemSchema = z.object({
17-
type: z.enum(['text', 'file']),
17+
type: z.enum(['text', 'file', 'image']),
1818
value: z.string(),
1919
});
2020

‎packages/core/test/evaluation/loaders/message-processor.test.ts‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
import { describe, expect, it } from 'bun:test';
2+
import { mkdir, rm, writeFile } from 'node:fs/promises';
3+
import path from 'node:path';
4+
5+
import {
6+
detectImageMediaType,
7+
processMessages,
8+
} from '../../../src/evaluation/loaders/message-processor.js';
9+
import type { TestMessage } from '../../../src/evaluation/types.js';
10+
11+
// Minimal 1x1 red PNG (68 bytes)
12+
const TINY_PNG_BASE64 =
13+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
14+
15+
const FIXTURE_DIR = path.join(import.meta.dirname, '__fixtures__');
16+
const PNG_PATH = path.join(FIXTURE_DIR, 'test-image.png');
17+
const JPG_PATH = path.join(FIXTURE_DIR, 'test-image.jpg');
18+
const TXT_PATH = path.join(FIXTURE_DIR, 'test-file.txt');
19+
20+
// Setup & teardown
21+
async function setupFixtures() {
22+
await mkdir(FIXTURE_DIR, { recursive: true });
23+
await writeFile(PNG_PATH, Buffer.from(TINY_PNG_BASE64, 'base64'));
24+
await writeFile(JPG_PATH, Buffer.from(TINY_PNG_BASE64, 'base64'));
25+
await writeFile(TXT_PATH, 'hello world');
26+
}
27+
28+
async function cleanupFixtures() {
29+
await rm(FIXTURE_DIR, { recursive: true, force: true });
30+
}
31+
32+
// ---------------------------------------------------------------------------
33+
// detectImageMediaType
34+
// ---------------------------------------------------------------------------
35+
36+
describe('detectImageMediaType', () => {
37+
it('detects PNG', () => {
38+
expect(detectImageMediaType('photo.png')).toBe('image/png');
39+
});
40+
41+
it('detects JPG', () => {
42+
expect(detectImageMediaType('photo.jpg')).toBe('image/jpeg');
43+
});
44+
45+
it('detects JPEG', () => {
46+
expect(detectImageMediaType('photo.jpeg')).toBe('image/jpeg');
47+
});
48+
49+
it('detects GIF', () => {
50+
expect(detectImageMediaType('anim.gif')).toBe('image/gif');
51+
});
52+
53+
it('detects WebP', () => {
54+
expect(detectImageMediaType('modern.webp')).toBe('image/webp');
55+
});
56+
57+
it('detects SVG', () => {
58+
expect(detectImageMediaType('icon.svg')).toBe('image/svg+xml');
59+
});
60+
61+
it('detects BMP', () => {
62+
expect(detectImageMediaType('old.bmp')).toBe('image/bmp');
63+
});
64+
65+
it('is case-insensitive', () => {
66+
expect(detectImageMediaType('PHOTO.PNG')).toBe('image/png');
67+
expect(detectImageMediaType('Photo.JPG')).toBe('image/jpeg');
68+
});
69+
70+
it('returns undefined for unsupported extensions', () => {
71+
expect(detectImageMediaType('file.txt')).toBeUndefined();
72+
expect(detectImageMediaType('file.pdf')).toBeUndefined();
73+
expect(detectImageMediaType('file')).toBeUndefined();
74+
});
75+
});
76+
77+
// ---------------------------------------------------------------------------
78+
// processMessages – type: 'image'
79+
// ---------------------------------------------------------------------------
80+
81+
describe('processMessages – image content', () => {
82+
it('reads a PNG file and produces a ContentImage with base64 data URI', async () => {
83+
await setupFixtures();
84+
try {
85+
const messages: TestMessage[] = [
86+
{
87+
role: 'user',
88+
content: [{ type: 'image', value: './test-image.png' }],
89+
},
90+
];
91+
92+
const result = await processMessages({
93+
messages,
94+
searchRoots: [FIXTURE_DIR],
95+
repoRootPath: FIXTURE_DIR,
96+
messageType: 'input',
97+
verbose: false,
98+
});
99+
100+
expect(result).toHaveLength(1);
101+
const content = result[0].content;
102+
expect(Array.isArray(content)).toBe(true);
103+
const items = content as Record<string, unknown>[];
104+
expect(items).toHaveLength(1);
105+
expect(items[0].type).toBe('image');
106+
expect(items[0].media_type).toBe('image/png');
107+
expect(typeof items[0].source).toBe('string');
108+
expect((items[0].source as string).startsWith('data:image/png;base64,')).toBe(true);
109+
} finally {
110+
await cleanupFixtures();
111+
}
112+
});
113+
114+
it('reads a JPG file and detects correct media type', async () => {
115+
await setupFixtures();
116+
try {
117+
const messages: TestMessage[] = [
118+
{
119+
role: 'user',
120+
content: [{ type: 'image', value: './test-image.jpg' }],
121+
},
122+
];
123+
124+
const result = await processMessages({
125+
messages,
126+
searchRoots: [FIXTURE_DIR],
127+
repoRootPath: FIXTURE_DIR,
128+
messageType: 'input',
129+
verbose: false,
130+
});
131+
132+
const items = result[0].content as Record<string, unknown>[];
133+
expect(items[0].media_type).toBe('image/jpeg');
134+
expect((items[0].source as string).startsWith('data:image/jpeg;base64,')).toBe(true);
135+
} finally {
136+
await cleanupFixtures();
137+
}
138+
});
139+
140+
it('warns and skips when image file does not exist', async () => {
141+
await setupFixtures();
142+
try {
143+
const messages: TestMessage[] = [
144+
{
145+
role: 'user',
146+
content: [{ type: 'image', value: './nonexistent.png' }],
147+
},
148+
];
149+
150+
const result = await processMessages({
151+
messages,
152+
searchRoots: [FIXTURE_DIR],
153+
repoRootPath: FIXTURE_DIR,
154+
messageType: 'input',
155+
verbose: false,
156+
});
157+
158+
const content = result[0].content as Record<string, unknown>[];
159+
expect(content).toHaveLength(0);
160+
} finally {
161+
await cleanupFixtures();
162+
}
163+
});
164+
165+
it('preserves existing type: text and type: file behavior', async () => {
166+
await setupFixtures();
167+
try {
168+
const messages: TestMessage[] = [
169+
{
170+
role: 'user',
171+
content: [
172+
{ type: 'text', value: 'describe this' },
173+
{ type: 'file', value: './test-file.txt' },
174+
{ type: 'image', value: './test-image.png' },
175+
],
176+
},
177+
];
178+
179+
const result = await processMessages({
180+
messages,
181+
searchRoots: [FIXTURE_DIR],
182+
repoRootPath: FIXTURE_DIR,
183+
messageType: 'input',
184+
verbose: false,
185+
});
186+
187+
const items = result[0].content as Record<string, unknown>[];
188+
expect(items).toHaveLength(3);
189+
// text preserved
190+
expect(items[0].type).toBe('text');
191+
expect(items[0].value).toBe('describe this');
192+
// file preserved with resolved content
193+
expect(items[1].type).toBe('file');
194+
expect(items[1].text).toBe('hello world');
195+
// image has base64
196+
expect(items[2].type).toBe('image');
197+
expect(items[2].media_type).toBe('image/png');
198+
} finally {
199+
await cleanupFixtures();
200+
}
201+
});
202+
});

‎plugins/agentv-dev/skills/agentv-eval-writer/references/eval-schema.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
"properties": {
6868
"type": {
6969
"type": "string",
70-
"enum": ["text", "file"]
70+
"enum": ["text", "file", "image"]
7171
},
7272
"value": {
7373
"type": "string"
@@ -135,7 +135,7 @@
135135
"properties": {
136136
"type": {
137137
"type": "string",
138-
"enum": ["text", "file"]
138+
"enum": ["text", "file", "image"]
139139
},
140140
"value": {
141141
"type": "string"
@@ -190,7 +190,7 @@
190190
"properties": {
191191
"type": {
192192
"type": "string",
193-
"enum": ["text", "file"]
193+
"enum": ["text", "file", "image"]
194194
},
195195
"value": {
196196
"type": "string"
@@ -6446,7 +6446,7 @@
64466446
"properties": {
64476447
"type": {
64486448
"type": "string",
6449-
"enum": ["text", "file"]
6449+
"enum": ["text", "file", "image"]
64506450
},
64516451
"value": {
64526452
"type": "string"
@@ -6501,7 +6501,7 @@
65016501
"properties": {
65026502
"type": {
65036503
"type": "string",
6504-
"enum": ["text", "file"]
6504+
"enum": ["text", "file", "image"]
65056505
},
65066506
"value": {
65076507
"type": "string"

0 commit comments

Comments
 (0)