feat(integrations): add transformation functions for OpenAI Agents content and update message handling

Author: constantinius

sentry_sdk/integrations/openai_agents/spans/invoke_agent.py

@@ -3,14 +3,19 @@
     get_start_span_function,
     set_data_normalized,
     normalize_message_roles,
+    normalize_message_role,
     truncate_and_annotate_messages,
 )
 from sentry_sdk.consts import OP, SPANDATA
 from sentry_sdk.scope import should_send_default_pii
 from sentry_sdk.utils import safe_serialize
 
 from ..consts import SPAN_ORIGIN
-from ..utils import _set_agent_data, _set_usage_data
+from ..utils import (
+    _set_agent_data,
+    _set_usage_data,
+    _transform_openai_agents_message_content,
+)
 
 from typing import TYPE_CHECKING
 
@@ -49,17 +54,40 @@ def invoke_agent_span(
 
         original_input = kwargs.get("original_input")
         if original_input is not None:
-            message = (
-                original_input
-                if isinstance(original_input, str)
-                else safe_serialize(original_input)
-            )
-            messages.append(
-                {
-                    "content": [{"text": message, "type": "text"}],
-                    "role": "user",
-                }
-            )
+            if isinstance(original_input, str):
+                # String input: wrap in text block
+                messages.append(
+                    {
+                        "content": [{"text": original_input, "type": "text"}],
+                        "role": "user",
+                    }
+                )
+            elif isinstance(original_input, list) and len(original_input) > 0:
+                # Check if list contains message objects (with type="message")
+                # or content parts (input_text, input_image, etc.)
+                first_item = original_input[0]
+                if isinstance(first_item, dict) and first_item.get("type") == "message":
+                    # List of message objects - process each individually
+                    for msg in original_input:
+                        if isinstance(msg, dict) and msg.get("type") == "message":
+                            role = normalize_message_role(msg.get("role", "user"))
+                            content = msg.get("content")
+                            transformed = _transform_openai_agents_message_content(
+                                content
+                            )
+                            if isinstance(transformed, str):
+                                transformed = [{"text": transformed, "type": "text"}]
+                            elif not isinstance(transformed, list):
+                                transformed = [
+                                    {"text": str(transformed), "type": "text"}
+                                ]
+                            messages.append({"content": transformed, "role": role})
+                else:
+                    # List of content parts - transform and wrap as user message
+                    content = _transform_openai_agents_message_content(original_input)
+                    if not isinstance(content, list):
+                        content = [{"text": str(content), "type": "text"}]
+                    messages.append({"content": content, "role": "user"})
 
         if len(messages) > 0:
             normalized_messages = normalize_message_roles(messages)

sentry_sdk/integrations/openai_agents/utils.py

@@ -27,6 +27,126 @@
     raise DidNotEnable("OpenAI Agents not installed")
 
 
+def _transform_openai_agents_content_part(
+    content_part: "dict[str, Any]",
+) -> "dict[str, Any]":
+    """
+    Transform an OpenAI Agents content part to Sentry-compatible format.
+
+    Handles multimodal content (images, audio, files) by converting them
+    to the standardized format:
+    - base64 encoded data -> type: "blob"
+    - URL references -> type: "uri"
+    - file_id references -> type: "file"
+    """
+    if not isinstance(content_part, dict):
+        return content_part
+
+    part_type = content_part.get("type")
+
+    # Handle input_text (OpenAI Agents SDK text format) -> normalize to standard text format
+    if part_type == "input_text":
+        return {
+            "type": "text",
+            "text": content_part.get("text", ""),
+        }
+
+    # Handle image_url (OpenAI vision format) and input_image (OpenAI Agents SDK format)
+    if part_type in ("image_url", "input_image"):
+        # Get URL from either format
+        if part_type == "image_url":
+            image_url = content_part.get("image_url", {})
+            url = (
+                image_url.get("url", "")
+                if isinstance(image_url, dict)
+                else str(image_url)
+            )
+        else:
+            # input_image format has image_url directly
+            url = content_part.get("image_url", "")
+
+        if url.startswith("data:"):
+            # Parse data URI: data:image/jpeg;base64,/9j/4AAQ...
+            try:
+                header, content = url.split(",", 1)
+                mime_type = header.split(":")[1].split(";")[0] if ":" in header else ""
+                return {
+                    "type": "blob",
+                    "modality": "image",
+                    "mime_type": mime_type,
+                    "content": content,
+                }
+            except (ValueError, IndexError):
+                # If parsing fails, return as URI
+                return {
+                    "type": "uri",
+                    "modality": "image",
+                    "mime_type": "",
+                    "uri": url,
+                }
+        else:
+            return {
+                "type": "uri",
+                "modality": "image",
+                "mime_type": "",
+                "uri": url,
+            }
+
+    # Handle input_audio (OpenAI audio input format)
+    if part_type == "input_audio":
+        input_audio = content_part.get("input_audio", {})
+        audio_format = input_audio.get("format", "")
+        mime_type = f"audio/{audio_format}" if audio_format else ""
+        return {
+            "type": "blob",
+            "modality": "audio",
+            "mime_type": mime_type,
+            "content": input_audio.get("data", ""),
+        }
+
+    # Handle image_file (Assistants API file-based images)
+    if part_type == "image_file":
+        image_file = content_part.get("image_file", {})
+        return {
+            "type": "file",
+            "modality": "image",
+            "mime_type": "",
+            "file_id": image_file.get("file_id", ""),
+        }
+
+    # Handle file (document attachments)
+    if part_type == "file":
+        file_data = content_part.get("file", {})
+        return {
+            "type": "file",
+            "modality": "document",
+            "mime_type": "",
+            "file_id": file_data.get("file_id", ""),
+        }
+
+    return content_part
+
+
+def _transform_openai_agents_message_content(content: "Any") -> "Any":
+    """
+    Transform OpenAI Agents message content, handling both string content and
+    list of content parts.
+    """
+    if isinstance(content, str):
+        return content
+
+    if isinstance(content, (list, tuple)):
+        transformed = []
+        for item in content:
+            if isinstance(item, dict):
+                transformed.append(_transform_openai_agents_content_part(item))
+            else:
+                transformed.append(item)
+        return transformed
+
+    return content
+
+
 def _capture_exception(exc: "Any") -> None:
     set_span_errored()
 
@@ -128,13 +248,15 @@ def _set_input_data(
         if "role" in message:
             normalized_role = normalize_message_role(message.get("role"))
             content = message.get("content")
+            # Transform content to handle multimodal data (images, audio, files)
+            transformed_content = _transform_openai_agents_message_content(content)
             request_messages.append(
                 {
                     "role": normalized_role,
                     "content": (
-                        [{"type": "text", "text": content}]
-                        if isinstance(content, str)
-                        else content
+                        [{"type": "text", "text": transformed_content}]
+                        if isinstance(transformed_content, str)
+                        else transformed_content
                     ),
                 }
             )

tests/integrations/openai_agents/test_openai_agents.py

@@ -9,7 +9,12 @@
 from sentry_sdk import start_span
 from sentry_sdk.consts import SPANDATA
 from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration
-from sentry_sdk.integrations.openai_agents.utils import _set_input_data, safe_serialize
+from sentry_sdk.integrations.openai_agents.utils import (
+    _set_input_data,
+    safe_serialize,
+    _transform_openai_agents_content_part,
+    _transform_openai_agents_message_content,
+)
 from sentry_sdk.utils import parse_version
 
 import agents
@@ -1998,3 +2003,90 @@ def test_openai_agents_message_truncation(sentry_init, capture_events):
         assert len(parsed_messages) == 2
         assert "small message 4" in str(parsed_messages[0])
         assert "small message 5" in str(parsed_messages[1])
+
+
+def test_transform_image_url_to_blob():
+    """Test that OpenAI image_url with data URI is converted to blob format."""
+    content_part = {
+        "type": "image_url",
+        "image_url": {
+            "url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD",
+            "detail": "high",
+        },
+    }
+    result = _transform_openai_agents_content_part(content_part)
+    assert result == {
+        "type": "blob",
+        "modality": "image",
+        "mime_type": "image/jpeg",
+        "content": "/9j/4AAQSkZJRgABAQAAAQABAAD",
+    }
+
+
+def test_transform_image_url_to_uri():
+    """Test that OpenAI image_url with HTTP URL is converted to uri format."""
+    content_part = {
+        "type": "image_url",
+        "image_url": {
+            "url": "https://example.com/image.jpg",
+            "detail": "low",
+        },
+    }
+    result = _transform_openai_agents_content_part(content_part)
+    assert result == {
+        "type": "uri",
+        "modality": "image",
+        "mime_type": "",
+        "uri": "https://example.com/image.jpg",
+    }
+
+
+def test_transform_message_content_with_image():
+    """Test that message content with image is properly transformed."""
+    content = [
+        {"type": "text", "text": "What is in this image?"},
+        {
+            "type": "image_url",
+            "image_url": {
+                "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==",
+            },
+        },
+    ]
+    result = _transform_openai_agents_message_content(content)
+    assert len(result) == 2
+    assert result[0] == {"type": "text", "text": "What is in this image?"}
+    assert result[1] == {
+        "type": "blob",
+        "modality": "image",
+        "mime_type": "image/png",
+        "content": "iVBORw0KGgoAAAANSUhEUg==",
+    }
+
+
+def test_transform_input_image_to_blob():
+    """Test that OpenAI Agents SDK input_image format is converted to blob format."""
+    # OpenAI Agents SDK uses input_image type with image_url as a direct string
+    content_part = {
+        "type": "input_image",
+        "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==",
+    }
+    result = _transform_openai_agents_content_part(content_part)
+    assert result == {
+        "type": "blob",
+        "modality": "image",
+        "mime_type": "image/png",
+        "content": "iVBORw0KGgoAAAANSUhEUg==",
+    }
+
+
+def test_transform_input_text_to_text():
+    """Test that OpenAI Agents SDK input_text format is normalized to text format."""
+    content_part = {
+        "type": "input_text",
+        "text": "Hello, world!",
+    }
+    result = _transform_openai_agents_content_part(content_part)
+    assert result == {
+        "type": "text",
+        "text": "Hello, world!",
+    }