From a03d7ce063dad3a591f8588c948ee61b659f4386 Mon Sep 17 00:00:00 2001 From: claude89757 <138977524+claude89757@users.noreply.github.com> Date: Mon, 12 Jan 2026 00:15:06 +0800 Subject: [PATCH 1/3] Fix: Add system_instructions to ChatClient LLM span tracing - Add system_instructions parameter to _capture_messages() calls in _trace_get_response() and _trace_get_streaming_response() - Extract instructions from chat_options in kwargs - Add unit tests to verify system_instructions are captured correctly When using ChatClient with ChatOptions.instructions, the OpenTelemetry LLM span was missing system messages in gen_ai.input.messages and the gen_ai.system_instructions attribute was not being set. This fix aligns the ChatClient-level tracing with the Agent-level tracing which already correctly passes system_instructions. Fixes #3163 Co-Authored-By: Claude Opus 4.5 --- .../core/agent_framework/observability.py | 10 ++- .../core/tests/core/test_observability.py | 79 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/observability.py b/python/packages/core/agent_framework/observability.py index 26c261038b..131445de21 100644 --- a/python/packages/core/agent_framework/observability.py +++ b/python/packages/core/agent_framework/observability.py @@ -1097,7 +1097,13 @@ async def trace_get_response( ) with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span: if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages: - _capture_messages(span=span, provider_name=provider_name, messages=messages) + chat_options = kwargs.get("chat_options") + _capture_messages( + span=span, + provider_name=provider_name, + messages=messages, + system_instructions=getattr(chat_options, "instructions", None) if chat_options else None, + ) start_time_stamp = perf_counter() end_time_stamp: float | None = None try: @@ -1186,10 +1192,12 @@ async def trace_get_streaming_response( all_updates: list["ChatResponseUpdate"] = [] with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span: if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages: + chat_options = kwargs.get("chat_options") _capture_messages( span=span, provider_name=provider_name, messages=messages, + system_instructions=getattr(chat_options, "instructions", None) if chat_options else None, ) start_time_stamp = perf_counter() end_time_stamp: float | None = None diff --git a/python/packages/core/tests/core/test_observability.py b/python/packages/core/tests/core/test_observability.py index 8528295406..00b9a4e4b2 100644 --- a/python/packages/core/tests/core/test_observability.py +++ b/python/packages/core/tests/core/test_observability.py @@ -280,6 +280,85 @@ async def test_chat_client_streaming_observability( assert span.attributes[OtelAttr.OUTPUT_MESSAGES] is not None +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_chat_client_observability_with_instructions( + mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data +): + """Test that system_instructions from chat_options are captured in LLM span.""" + import json + + client = use_instrumentation(mock_chat_client)() + + messages = [ChatMessage(role=Role.USER, text="Test message")] + chat_options = ChatOptions(model_id="Test", instructions="You are a helpful assistant.") + span_exporter.clear() + response = await client.get_response(messages=messages, chat_options=chat_options) + + assert response is not None + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + # Verify system_instructions attribute is set + assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes + system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS]) + assert len(system_instructions) == 1 + assert system_instructions[0]["content"] == "You are a helpful assistant." + + # Verify input_messages contains system message + input_messages = json.loads(span.attributes[OtelAttr.INPUT_MESSAGES]) + assert any(msg.get("role") == "system" for msg in input_messages) + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_chat_client_streaming_observability_with_instructions( + mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data +): + """Test streaming telemetry captures system_instructions from chat_options.""" + import json + + client = use_instrumentation(mock_chat_client)() + messages = [ChatMessage(role=Role.USER, text="Test")] + chat_options = ChatOptions(model_id="Test", instructions="You are a helpful assistant.") + span_exporter.clear() + + updates = [] + async for update in client.get_streaming_response(messages=messages, chat_options=chat_options): + updates.append(update) + + assert len(updates) == 2 + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + # Verify system_instructions attribute is set + assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes + system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS]) + assert len(system_instructions) == 1 + assert system_instructions[0]["content"] == "You are a helpful assistant." + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_chat_client_observability_without_instructions( + mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data +): + """Test that system_instructions attribute is not set when instructions are not provided.""" + client = use_instrumentation(mock_chat_client)() + + messages = [ChatMessage(role=Role.USER, text="Test message")] + chat_options = ChatOptions(model_id="Test") # No instructions + span_exporter.clear() + response = await client.get_response(messages=messages, chat_options=chat_options) + + assert response is not None + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + # Verify system_instructions attribute is NOT set + assert OtelAttr.SYSTEM_INSTRUCTIONS not in span.attributes + + async def test_chat_client_without_model_id_observability(mock_chat_client, span_exporter: InMemorySpanExporter): """Test telemetry shouldn't fail when the model_id is not provided for unknown reason.""" client = use_instrumentation(mock_chat_client)() From 0ba22a5b736627443c056e8f2e4e6bbcb2bef3a3 Mon Sep 17 00:00:00 2001 From: claude89757 <138977524+claude89757@users.noreply.github.com> Date: Mon, 12 Jan 2026 00:22:21 +0800 Subject: [PATCH 2/3] Add edge case tests for system_instructions - Add test for empty string instructions (should not set attribute) - Add test for list-type instructions (verify multiple items captured) Co-Authored-By: Claude Opus 4.5 --- .../core/tests/core/test_observability.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/python/packages/core/tests/core/test_observability.py b/python/packages/core/tests/core/test_observability.py index 00b9a4e4b2..9fe059012d 100644 --- a/python/packages/core/tests/core/test_observability.py +++ b/python/packages/core/tests/core/test_observability.py @@ -359,6 +359,54 @@ async def test_chat_client_observability_without_instructions( assert OtelAttr.SYSTEM_INSTRUCTIONS not in span.attributes +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_chat_client_observability_with_empty_instructions( + mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data +): + """Test that system_instructions attribute is not set when instructions is an empty string.""" + client = use_instrumentation(mock_chat_client)() + + messages = [ChatMessage(role=Role.USER, text="Test message")] + chat_options = ChatOptions(model_id="Test", instructions="") # Empty string + span_exporter.clear() + response = await client.get_response(messages=messages, chat_options=chat_options) + + assert response is not None + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + # Empty string should not set system_instructions + assert OtelAttr.SYSTEM_INSTRUCTIONS not in span.attributes + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_chat_client_observability_with_list_instructions( + mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data +): + """Test that list-type instructions are correctly captured.""" + import json + + client = use_instrumentation(mock_chat_client)() + + messages = [ChatMessage(role=Role.USER, text="Test message")] + chat_options = ChatOptions(model_id="Test", instructions=["Instruction 1", "Instruction 2"]) + span_exporter.clear() + response = await client.get_response(messages=messages, chat_options=chat_options) + + assert response is not None + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + + # Verify system_instructions attribute contains both instructions + assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes + system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS]) + assert len(system_instructions) == 2 + assert system_instructions[0]["content"] == "Instruction 1" + assert system_instructions[1]["content"] == "Instruction 2" + + async def test_chat_client_without_model_id_observability(mock_chat_client, span_exporter: InMemorySpanExporter): """Test telemetry shouldn't fail when the model_id is not provided for unknown reason.""" client = use_instrumentation(mock_chat_client)() From e544a61eefb392548ddc8ef665b98cd1913be86a Mon Sep 17 00:00:00 2001 From: claude89757 <138977524+claude89757@users.noreply.github.com> Date: Tue, 20 Jan 2026 21:44:26 +0800 Subject: [PATCH 3/3] Simplify: use options.get('instructions') directly instead of kwargs.get('chat_options') Addresses reviewer feedback: - Removed unnecessary chat_options variable from kwargs - Directly access instructions from the options parameter - Updated tests to use dict syntax for options (TypedDict convention) --- .../core/agent_framework/observability.py | 6 ++--- .../core/tests/core/test_observability.py | 24 +++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/python/packages/core/agent_framework/observability.py b/python/packages/core/agent_framework/observability.py index 6cd092f08b..20b9843468 100644 --- a/python/packages/core/agent_framework/observability.py +++ b/python/packages/core/agent_framework/observability.py @@ -1096,12 +1096,11 @@ async def trace_get_response( ) with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span: if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages: - chat_options = kwargs.get("chat_options") _capture_messages( span=span, provider_name=provider_name, messages=messages, - system_instructions=getattr(chat_options, "instructions", None) if chat_options else None, + system_instructions=options.get("instructions"), ) start_time_stamp = perf_counter() end_time_stamp: float | None = None @@ -1191,12 +1190,11 @@ async def trace_get_streaming_response( all_updates: list["ChatResponseUpdate"] = [] with _get_span(attributes=attributes, span_name_attribute=SpanAttributes.LLM_REQUEST_MODEL) as span: if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages: - chat_options = kwargs.get("chat_options") _capture_messages( span=span, provider_name=provider_name, messages=messages, - system_instructions=getattr(chat_options, "instructions", None) if chat_options else None, + system_instructions=options.get("instructions"), ) start_time_stamp = perf_counter() end_time_stamp: float | None = None diff --git a/python/packages/core/tests/core/test_observability.py b/python/packages/core/tests/core/test_observability.py index 986549d651..88245cfa52 100644 --- a/python/packages/core/tests/core/test_observability.py +++ b/python/packages/core/tests/core/test_observability.py @@ -283,15 +283,15 @@ async def test_chat_client_streaming_observability( async def test_chat_client_observability_with_instructions( mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data ): - """Test that system_instructions from chat_options are captured in LLM span.""" + """Test that system_instructions from options are captured in LLM span.""" import json client = use_instrumentation(mock_chat_client)() messages = [ChatMessage(role=Role.USER, text="Test message")] - chat_options = ChatOptions(model_id="Test", instructions="You are a helpful assistant.") + options = {"model_id": "Test", "instructions": "You are a helpful assistant."} span_exporter.clear() - response = await client.get_response(messages=messages, chat_options=chat_options) + response = await client.get_response(messages=messages, options=options) assert response is not None spans = span_exporter.get_finished_spans() @@ -313,16 +313,16 @@ async def test_chat_client_observability_with_instructions( async def test_chat_client_streaming_observability_with_instructions( mock_chat_client, span_exporter: InMemorySpanExporter, enable_sensitive_data ): - """Test streaming telemetry captures system_instructions from chat_options.""" + """Test streaming telemetry captures system_instructions from options.""" import json client = use_instrumentation(mock_chat_client)() messages = [ChatMessage(role=Role.USER, text="Test")] - chat_options = ChatOptions(model_id="Test", instructions="You are a helpful assistant.") + options = {"model_id": "Test", "instructions": "You are a helpful assistant."} span_exporter.clear() updates = [] - async for update in client.get_streaming_response(messages=messages, chat_options=chat_options): + async for update in client.get_streaming_response(messages=messages, options=options): updates.append(update) assert len(updates) == 2 @@ -345,9 +345,9 @@ async def test_chat_client_observability_without_instructions( client = use_instrumentation(mock_chat_client)() messages = [ChatMessage(role=Role.USER, text="Test message")] - chat_options = ChatOptions(model_id="Test") # No instructions + options = {"model_id": "Test"} # No instructions span_exporter.clear() - response = await client.get_response(messages=messages, chat_options=chat_options) + response = await client.get_response(messages=messages, options=options) assert response is not None spans = span_exporter.get_finished_spans() @@ -366,9 +366,9 @@ async def test_chat_client_observability_with_empty_instructions( client = use_instrumentation(mock_chat_client)() messages = [ChatMessage(role=Role.USER, text="Test message")] - chat_options = ChatOptions(model_id="Test", instructions="") # Empty string + options = {"model_id": "Test", "instructions": ""} # Empty string span_exporter.clear() - response = await client.get_response(messages=messages, chat_options=chat_options) + response = await client.get_response(messages=messages, options=options) assert response is not None spans = span_exporter.get_finished_spans() @@ -389,9 +389,9 @@ async def test_chat_client_observability_with_list_instructions( client = use_instrumentation(mock_chat_client)() messages = [ChatMessage(role=Role.USER, text="Test message")] - chat_options = ChatOptions(model_id="Test", instructions=["Instruction 1", "Instruction 2"]) + options = {"model_id": "Test", "instructions": ["Instruction 1", "Instruction 2"]} span_exporter.clear() - response = await client.get_response(messages=messages, chat_options=chat_options) + response = await client.get_response(messages=messages, options=options) assert response is not None spans = span_exporter.get_finished_spans()