From 4f25a56b1cb514a493c4eb6a278b2ff66cefe8e3 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:40:02 +0100 Subject: [PATCH 1/9] feat(openai): Set system instruction attribute --- sentry_sdk/ai/_openai_completions_api.py | 20 +++++++++++ sentry_sdk/consts.py | 6 ++++ sentry_sdk/integrations/openai.py | 42 +++++++++++++++++++++++- 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 sentry_sdk/ai/_openai_completions_api.py diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py new file mode 100644 index 0000000000..6697f285c6 --- /dev/null +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -0,0 +1,20 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ) + from typing import Iterable, Union + + +def _get_system_instructions( + messages: "Iterable[Union[ChatCompletionMessageParam, str]]", +) -> "list[ChatCompletionSystemMessageParam]": + system_messages = [] + + for message in messages: + if isinstance(message, dict) and message.get("role") == "system": + system_messages.append(message) + + return system_messages diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 93fca6ba3e..4b61a317fb 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -542,6 +542,12 @@ class SPANDATA: Example: 2048 """ + GEN_AI_SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions" + """ + The system instructions passed to the model. + Example: [{"type": "text", "text": "You are a helpful assistant."},{"type": "text", "text": "Be concise and clear."}] + """ + GEN_AI_REQUEST_MESSAGES = "gen_ai.request.messages" """ The messages passed to the model. The "content" can be a string or an array of objects. diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 553d93d195..6a269c40c7 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -9,6 +9,9 @@ normalize_message_roles, truncate_and_annotate_messages, ) +from sentry_sdk.ai._openai_completions_api import ( + _get_system_instructions as _get_system_instructions_completions, +) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -35,7 +38,7 @@ ) from sentry_sdk.tracing import Span - from openai.types.responses import ResponseInputParam + from openai.types.responses import ResponseInputParam, ResponseInputItemParam try: try: @@ -193,6 +196,25 @@ def _calculate_token_usage( ) +def _get_system_instructions_responses( + input_items: "Union[ResponseInputParam, list[str]]", +) -> "list[ResponseInputItemParam]": + if isinstance(input_items, str): + return [] + + system_messages = [] + + for item in input_items: + if ( + isinstance(item, dict) + and item.get("type") == "message" + and item.get("role") == "system" + ): + system_messages.append(item) + + return system_messages + + def _get_input_messages( kwargs: "dict[str, Any]", ) -> "Optional[Iterable[Any] | list[str]]": @@ -245,6 +267,15 @@ def _set_responses_api_input_data( ): messages: "Optional[ResponseInputParam | list[str]]" = _get_input_messages(kwargs) # type: ignore + if messages is not None: + system_instructions = _get_system_instructions_responses(messages) + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + system_instructions, + unpack=False, + ) + if ( messages is not None and len(messages) > 0 @@ -272,6 +303,15 @@ def _set_completions_api_input_data( _get_input_messages(kwargs) ) + if messages is not None: + system_instructions = _get_system_instructions_completions(messages) + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + system_instructions, + unpack=False, + ) + if ( messages is not None and len(messages) > 0 # type: ignore From 9f5d8015f5ab0e222c784f1bee3ea05ef26b9f7e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 19:06:03 +0100 Subject: [PATCH 2/9] . --- sentry_sdk/ai/_openai_completions_api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 6697f285c6..8309c76c31 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -5,12 +5,14 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ) - from typing import Iterable, Union + from typing import Iterable, Union, TypeVar + + T = TypeVar("T") def _get_system_instructions( - messages: "Iterable[Union[ChatCompletionMessageParam, str]]", -) -> "list[ChatCompletionSystemMessageParam]": + messages: "Iterable[Union[T, str]]", +) -> "list[T]": system_messages = [] for message in messages: From d66ffe12aee0f979d22140b3e4c05d4782127e37 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 19:20:08 +0100 Subject: [PATCH 3/9] use specific openai types --- sentry_sdk/ai/_openai_completions_api.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 8309c76c31..6697f285c6 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -5,14 +5,12 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ) - from typing import Iterable, Union, TypeVar - - T = TypeVar("T") + from typing import Iterable, Union def _get_system_instructions( - messages: "Iterable[Union[T, str]]", -) -> "list[T]": + messages: "Iterable[Union[ChatCompletionMessageParam, str]]", +) -> "list[ChatCompletionSystemMessageParam]": system_messages = [] for message in messages: From ac3ce00e26e6e86b5cec7ecb5170fad729f9a31d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 10:43:32 +0100 Subject: [PATCH 4/9] wip --- sentry_sdk/integrations/openai.py | 107 +++++++++++++++++------ tests/integrations/openai/test_openai.py | 73 +++++++++++++++- 2 files changed, 147 insertions(+), 33 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index b747596a61..cace34ebf1 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -11,6 +11,7 @@ ) from sentry_sdk.ai._openai_completions_api import ( _get_system_instructions as _get_system_instructions_completions, + _is_system_instruction as _is_system_instruction_completions, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -196,23 +197,27 @@ def _calculate_token_usage( ) +def _is_system_instruction_responses(message: "ResponseInputItemParam"): + return ( + isinstance(message, dict) + and message.get("type") == "message" + and message.get("role") == "system" + ) + + def _get_system_instructions_responses( - input_items: "Union[ResponseInputParam, list[str]]", + messages: "Union[str, ResponseInputParam]", ) -> "list[ResponseInputItemParam]": - if isinstance(input_items, str): + if isinstance(messages, str): return [] - system_messages = [] + system_instructions = [] - for item in input_items: - if ( - isinstance(item, dict) - and item.get("type") == "message" - and item.get("role") == "system" - ): - system_messages.append(item) + for message in messages: + if _is_system_instruction_responses(message): + system_instructions.append(message) - return system_messages + return system_instructions def _get_input_messages( @@ -265,12 +270,19 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Optional[Union[ResponseInputParam, list[str]]]" = _get_input_messages( - kwargs - ) + messages: "Optional[Union[str, ResponseInputParam]]" = kwargs.get("input") + + if messages is None: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return - if messages is not None: - system_instructions = _get_system_instructions_responses(messages) + system_instructions = _get_system_instructions_responses(messages) + if ( + len(system_instructions) > 0 + and should_send_default_pii() + and integration.include_prompts + ): set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, @@ -279,12 +291,11 @@ def _set_responses_api_input_data( ) if ( - messages is not None - and len(messages) > 0 + isinstance(messages, str) and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) # type: ignore + normalized_messages = normalize_message_roles([messages]) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: @@ -292,6 +303,23 @@ def _set_responses_api_input_data( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) + elif should_send_default_pii() and integration.include_prompts: + non_system_messages = [ + message + for message in messages + if not _is_system_instruction_responses(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(messages) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope + ) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") _commmon_set_input_data(span, kwargs) @@ -301,12 +329,21 @@ def _set_completions_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]" = ( - _get_input_messages(kwargs) + messages: "Optional[Union[str, Iterable[ChatCompletionMessageParam]]]" = kwargs.get( + "messages" ) - if messages is not None: - system_instructions = _get_system_instructions_completions(messages) + if messages is None: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + return + + system_instructions = _get_system_instructions_completions(messages) + if ( + len(system_instructions) > 0 + and should_send_default_pii() + and integration.include_prompts + ): set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, @@ -315,21 +352,33 @@ def _set_completions_api_input_data( ) if ( - messages is not None - and len(messages) > 0 # type: ignore + isinstance(messages, str) and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) # type: ignore + normalized_messages = normalize_message_roles([messages]) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False ) - - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") - _commmon_set_input_data(span, kwargs) + elif should_send_default_pii() and integration.include_prompts: + non_system_messages = [ + message + for message in messages + if not _is_system_instruction_completions(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope + ) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) def _set_embeddings_input_data( diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 814289c887..c09ca61cb3 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -147,7 +147,11 @@ def test_nonstreaming_chat_completion( with start_transaction(name="openai tx"): response = ( client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) .choices[0] .message.content @@ -160,9 +164,17 @@ def test_nonstreaming_chat_completion( assert span["op"] == "gen_ai.chat" if send_default_pii and include_prompts: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "content": "You are a helpful assistant.", + "role": "system", + } + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -191,7 +203,11 @@ async def test_nonstreaming_chat_completion_async( with start_transaction(name="openai tx"): response = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response = response.choices[0].message.content @@ -202,9 +218,17 @@ async def test_nonstreaming_chat_completion_async( assert span["op"] == "gen_ai.chat" if send_default_pii and include_prompts: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "content": "You are a helpful assistant.", + "role": "system", + } + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -283,7 +307,11 @@ def test_streaming_chat_completion( client.chat.completions._post = mock.Mock(return_value=returned_stream) with start_transaction(name="openai tx"): response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "".join( map(lambda x: x.choices[0].delta.content, response_stream) @@ -298,6 +326,7 @@ def test_streaming_chat_completion( assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -377,7 +406,11 @@ async def test_streaming_chat_completion_async( client.chat.completions._post = AsyncMock(return_value=returned_stream) with start_transaction(name="openai tx"): response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "" @@ -394,6 +427,7 @@ async def test_streaming_chat_completion_async( assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] @@ -1427,6 +1461,37 @@ async def test_streaming_responses_api_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 +@pytest.mark.skipif( + OPENAI_VERSION <= (1, 1, 0), + reason="OpenAI versions <=1.1.0 do not support the tools parameter.", +) +@pytest.mark.parametrize( + "tools", + [[], None, NOT_GIVEN, omit], +) +def test_chat_completion_with_system_instruction(sentry_init, capture_events, tools): + sentry_init( + integrations=[OpenAIIntegration()], + traces_sample_rate=1.0, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + tools=tools, + ) + + (event,) = events + span = event["spans"][0] + + assert "gen_ai.request.available_tools" not in span["data"] + + @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the tools parameter.", From ef9fe6fee5c70cf2091effcbd190def07fbcc985 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 10:44:10 +0100 Subject: [PATCH 5/9] . --- sentry_sdk/ai/_openai_completions_api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 6697f285c6..d1c1c7d23b 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -5,16 +5,20 @@ ChatCompletionMessageParam, ChatCompletionSystemMessageParam, ) - from typing import Iterable, Union + from typing import Iterable + + +def _is_system_instruction(message: "ChatCompletionMessageParam"): + return isinstance(message, dict) and message.get("role") == "system" def _get_system_instructions( - messages: "Iterable[Union[ChatCompletionMessageParam, str]]", + messages: "Iterable[ChatCompletionMessageParam]", ) -> "list[ChatCompletionSystemMessageParam]": - system_messages = [] + system_instructions = [] for message in messages: - if isinstance(message, dict) and message.get("role") == "system": - system_messages.append(message) + if _is_system_instruction(message): + system_instructions.append(message) - return system_messages + return system_instructions From ce84a29356a29c4313b83bf4717e0dbbb64c9b32 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:13:40 +0100 Subject: [PATCH 6/9] . --- sentry_sdk/_types.py | 4 + sentry_sdk/ai/_openai_completions_api.py | 26 +++ sentry_sdk/integrations/openai.py | 22 +- tests/integrations/openai/test_openai.py | 262 +++++++++++++++++++---- 4 files changed, 267 insertions(+), 47 deletions(-) diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index 7043bbc2ee..ecb8abcd10 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -359,3 +359,7 @@ class SDKInfo(TypedDict): ) HttpStatusCodeRange = Union[int, Container[int]] + + class TextPart(TypedDict): + type: Literal["text"] + content: str diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index d1c1c7d23b..3bb4c82448 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -7,6 +7,8 @@ ) from typing import Iterable + from sentry_sdk._types import TextPart + def _is_system_instruction(message: "ChatCompletionMessageParam"): return isinstance(message, dict) and message.get("role") == "system" @@ -22,3 +24,27 @@ def _get_system_instructions( system_instructions.append(message) return system_instructions + + +def _transform_system_instructions( + system_instructions: "list[ChatCompletionSystemMessageParam]", +) -> "list[TextPart]": + instruction_text_parts = [] + + for instruction in system_instructions: + if not isinstance(instruction, dict): + continue + + content = instruction.get("content") + + if isinstance(content, str): + instruction_text_parts.append({"type": "text", "content": content}) + + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text = part.get("text", "") + if text: + instruction_text_parts.append({"type": "text", "content": text}) + + return instruction_text_parts diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index cace34ebf1..1e587b3a3e 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -12,6 +12,7 @@ from sentry_sdk.ai._openai_completions_api import ( _get_system_instructions as _get_system_instructions_completions, _is_system_instruction as _is_system_instruction_completions, + _transform_system_instructions, ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -40,6 +41,7 @@ from sentry_sdk.tracing import Span from openai.types.responses import ResponseInputParam, ResponseInputItemParam + from openai import Omit try: try: @@ -277,16 +279,28 @@ def _set_responses_api_input_data( _commmon_set_input_data(span, kwargs) return + explicit_instructions: "Union[Optional[str], Omit]" = kwargs.get("instructions") system_instructions = _get_system_instructions_responses(messages) if ( - len(system_instructions) > 0 + (_is_given(explicit_instructions) or len(system_instructions) > 0) and should_send_default_pii() and integration.include_prompts ): + # Deliberate use of function accepting completions API type because + # of shared structure FOR THIS PURPOSE ONLY. + instructions_text_parts = _transform_system_instructions(system_instructions) # type: ignore + if _is_given(explicit_instructions): + instructions_text_parts.append( + { + "type": "text", + "content": explicit_instructions, + } + ) + set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - system_instructions, + instructions_text_parts, unpack=False, ) @@ -310,7 +324,7 @@ def _set_responses_api_input_data( if not _is_system_instruction_responses(message) ] if len(non_system_messages) > 0: - normalized_messages = normalize_message_roles(messages) # type: ignore + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages( normalized_messages, span, scope @@ -347,7 +361,7 @@ def _set_completions_api_input_data( set_data_normalized( span, SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - system_instructions, + _transform_system_instructions(system_instructions), unpack=False, ) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index c09ca61cb3..a9bfa40fef 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -166,8 +166,8 @@ def test_nonstreaming_chat_completion( if send_default_pii and include_prompts: assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ { + "type": "text", "content": "You are a helpful assistant.", - "role": "system", } ] @@ -220,8 +220,8 @@ async def test_nonstreaming_chat_completion_async( if send_default_pii and include_prompts: assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ { + "type": "text", "content": "You are a helpful assistant.", - "role": "system", } ] @@ -251,8 +251,38 @@ def tiktoken_encoding_if_installed(): "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], ) +@pytest.mark.parametrize( + "input", + [ + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) def test_streaming_chat_completion( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts, input, request ): sentry_init( integrations=[ @@ -308,10 +338,7 @@ def test_streaming_chat_completion( with start_transaction(name="openai tx"): response_stream = client.chat.completions.create( model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], + messages=input, ) response_string = "".join( map(lambda x: x.choices[0].delta.content, response_stream) @@ -322,7 +349,27 @@ def test_streaming_chat_completion( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" + param_id = request.node.callspec.id if send_default_pii and include_prompts: + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: @@ -333,9 +380,14 @@ def test_streaming_chat_completion( try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -346,8 +398,38 @@ def test_streaming_chat_completion( "send_default_pii, include_prompts", [(True, True), (True, False), (False, True), (False, False)], ) +@pytest.mark.parametrize( + "input", + [ + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) async def test_streaming_chat_completion_async( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts, input, request ): sentry_init( integrations=[ @@ -407,10 +489,7 @@ async def test_streaming_chat_completion_async( with start_transaction(name="openai tx"): response_stream = await client.chat.completions.create( model="some-model", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "hello"}, - ], + messages=input, ) response_string = "" @@ -423,7 +502,27 @@ async def test_streaming_chat_completion_async( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" + param_id = request.node.callspec.id if send_default_pii and include_prompts: + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] else: @@ -434,9 +533,15 @@ async def test_streaming_chat_completion_async( try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import - assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 + except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -1068,12 +1173,46 @@ def test_ai_client_span_responses_api_no_pii(sentry_init, capture_events): "thread.name": mock.ANY, } + assert "gen_ai.system_instructions" not in spans[0]["data"] assert "gen_ai.request.messages" not in spans[0]["data"] assert "gen_ai.response.text" not in spans[0]["data"] +@pytest.mark.parametrize( + "input", + [ + pytest.param( + "How do I check if a Python object is an instance of a class?", id="string" + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": "You are a helpful assistant.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "type": "message", + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"type": "message", "role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") -def test_ai_client_span_responses_api(sentry_init, capture_events): +def test_ai_client_span_responses_api(sentry_init, capture_events, input, request): sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, @@ -1088,7 +1227,7 @@ def test_ai_client_span_responses_api(sentry_init, capture_events): client.responses.create( model="gpt-4o", instructions="You are a coding assistant that talks like a pirate.", - input="How do I check if a Python object is an instance of a class?", + input=input, ) (transaction,) = events @@ -1097,21 +1236,59 @@ def test_ai_client_span_responses_api(sentry_init, capture_events): assert len(spans) == 1 assert spans[0]["op"] == "gen_ai.responses" assert spans[0]["origin"] == "auto.ai.openai" - assert spans[0]["data"] == { - "gen_ai.operation.name": "responses", - "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', - "gen_ai.request.model": "gpt-4o", - "gen_ai.system": "openai", - "gen_ai.response.model": "response-model-id", - "gen_ai.usage.input_tokens": 20, - "gen_ai.usage.input_tokens.cached": 5, - "gen_ai.usage.output_tokens": 10, - "gen_ai.usage.output_tokens.reasoning": 8, - "gen_ai.usage.total_tokens": 30, - "gen_ai.response.text": "the model response", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + + param_id = request.node.callspec.id + if param_id == "string": + assert spans[0]["data"] == { + "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '["How do I check if a Python object is an instance of a class?"]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + elif param_id == "blocks": + assert spans[0]["data"] == { + "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '[{"type": "message", "role": "user", "content": "hello"}]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a helpful assistant."}, {"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + else: + assert spans[0]["data"] == { + "gen_ai.operation.name": "responses", + "gen_ai.request.messages": '[{"type": "message", "role": "user", "content": "hello"}]', + "gen_ai.request.model": "gpt-4o", + "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a helpful assistant."}, {"type": "text", "content": "Be concise and clear."}, {"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', + "gen_ai.response.model": "response-model-id", + "gen_ai.usage.input_tokens": 20, + "gen_ai.usage.input_tokens.cached": 5, + "gen_ai.usage.output_tokens": 10, + "gen_ai.usage.output_tokens.reasoning": 8, + "gen_ai.usage.total_tokens": 30, + "gen_ai.response.text": "the model response", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") @@ -1183,6 +1360,7 @@ async def test_ai_client_span_responses_async_api(sentry_init, capture_events): "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, @@ -1230,6 +1408,7 @@ async def test_ai_client_span_streaming_responses_async_api( "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": True, "gen_ai.system": "openai", + "gen_ai.system_instructions": '[{"type": "text", "content": "You are a coding assistant that talks like a pirate."}]', "gen_ai.usage.input_tokens": 20, "gen_ai.usage.input_tokens.cached": 5, "gen_ai.usage.output_tokens": 10, @@ -1537,7 +1716,6 @@ def test_openai_message_role_mapping(sentry_init, capture_events): client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) # Test messages with mixed roles including "ai" that should be mapped to "assistant" test_messages = [ - {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}, {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" @@ -1557,17 +1735,15 @@ def test_openai_message_role_mapping(sentry_init, capture_events): stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) # Verify that "ai" role was mapped to "assistant" - assert len(stored_messages) == 4 - assert stored_messages[0]["role"] == "system" - assert stored_messages[1]["role"] == "user" + assert len(stored_messages) == 3 assert ( - stored_messages[2]["role"] == "assistant" + stored_messages[1]["role"] == "assistant" ) # "ai" should be mapped to "assistant" - assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + assert stored_messages[2]["role"] == "assistant" # should stay "assistant" # Verify content is preserved - assert stored_messages[2]["content"] == "Hi there!" - assert stored_messages[3]["content"] == "How can I help?" + assert stored_messages[1]["content"] == "Hi there!" + assert stored_messages[2]["content"] == "How can I help?" # Verify no "ai" roles remain roles = [msg["role"] for msg in stored_messages] From dee993063e181fef4359036e3f3ea3bf73740fca Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:14:42 +0100 Subject: [PATCH 7/9] remove test --- tests/integrations/openai/test_openai.py | 31 ------------------------ 1 file changed, 31 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index a9bfa40fef..49fc7baf1b 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -1640,37 +1640,6 @@ async def test_streaming_responses_api_async( assert span["data"]["gen_ai.usage.total_tokens"] == 30 -@pytest.mark.skipif( - OPENAI_VERSION <= (1, 1, 0), - reason="OpenAI versions <=1.1.0 do not support the tools parameter.", -) -@pytest.mark.parametrize( - "tools", - [[], None, NOT_GIVEN, omit], -) -def test_chat_completion_with_system_instruction(sentry_init, capture_events, tools): - sentry_init( - integrations=[OpenAIIntegration()], - traces_sample_rate=1.0, - ) - events = capture_events() - - client = OpenAI(api_key="z") - client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=[{"role": "system", "content": "hello"}], - tools=tools, - ) - - (event,) = events - span = event["spans"][0] - - assert "gen_ai.request.available_tools" not in span["data"] - - @pytest.mark.skipif( OPENAI_VERSION <= (1, 1, 0), reason="OpenAI versions <=1.1.0 do not support the tools parameter.", From cb00ab3acd79431612b01f4edd1465b096ec8d75 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:18:52 +0100 Subject: [PATCH 8/9] . --- sentry_sdk/ai/_openai_completions_api.py | 4 ++-- sentry_sdk/integrations/openai.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/ai/_openai_completions_api.py b/sentry_sdk/ai/_openai_completions_api.py index 3bb4c82448..c77fdb82dc 100644 --- a/sentry_sdk/ai/_openai_completions_api.py +++ b/sentry_sdk/ai/_openai_completions_api.py @@ -10,7 +10,7 @@ from sentry_sdk._types import TextPart -def _is_system_instruction(message: "ChatCompletionMessageParam"): +def _is_system_instruction(message: "ChatCompletionMessageParam") -> bool: return isinstance(message, dict) and message.get("role") == "system" @@ -29,7 +29,7 @@ def _get_system_instructions( def _transform_system_instructions( system_instructions: "list[ChatCompletionSystemMessageParam]", ) -> "list[TextPart]": - instruction_text_parts = [] + instruction_text_parts: "list[TextPart]" = [] for instruction in system_instructions: if not isinstance(instruction, dict): diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 1e587b3a3e..538db1a603 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -199,7 +199,7 @@ def _calculate_token_usage( ) -def _is_system_instruction_responses(message: "ResponseInputItemParam"): +def _is_system_instruction_responses(message: "ResponseInputItemParam") -> bool: return ( isinstance(message, dict) and message.get("type") == "message" @@ -288,12 +288,12 @@ def _set_responses_api_input_data( ): # Deliberate use of function accepting completions API type because # of shared structure FOR THIS PURPOSE ONLY. - instructions_text_parts = _transform_system_instructions(system_instructions) # type: ignore + instructions_text_parts = _transform_system_instructions(system_instructions) if _is_given(explicit_instructions): instructions_text_parts.append( { "type": "text", - "content": explicit_instructions, + "content": explicit_instructions, # type: ignore } ) From 26b932bad88285d50a59c06a436d47ab2cb3d5c8 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 22 Jan 2026 13:40:58 +0100 Subject: [PATCH 9/9] . --- tests/integrations/openai/test_openai.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 49fc7baf1b..33d476bb53 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -257,25 +257,23 @@ def tiktoken_encoding_if_installed(): pytest.param( [ { - "type": "message", "role": "system", "content": "You are a helpful assistant.", }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="blocks", ), pytest.param( [ { - "type": "message", "role": "system", "content": [ {"type": "text", "text": "You are a helpful assistant."}, {"type": "text", "text": "Be concise and clear."}, ], }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="parts", ), @@ -404,25 +402,23 @@ def test_streaming_chat_completion( pytest.param( [ { - "type": "message", "role": "system", "content": "You are a helpful assistant.", }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="blocks", ), pytest.param( [ { - "type": "message", "role": "system", "content": [ {"type": "text", "text": "You are a helpful assistant."}, {"type": "text", "text": "Be concise and clear."}, ], }, - {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, ], id="parts", ),