Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@
from anthropic.types.beta.beta_bash_code_execution_tool_result_error import (
BetaBashCodeExecutionToolResultError,
)
from anthropic.types.beta.beta_code_execution_result_block import BetaCodeExecutionResultBlock
from anthropic.types.beta.beta_code_execution_tool_result_error import (
BetaCodeExecutionToolResultError,
)
from anthropic.types.beta.beta_encrypted_code_execution_result_block import BetaEncryptedCodeExecutionResultBlock
from pydantic import BaseModel

if sys.version_info >= (3, 11):
Expand Down Expand Up @@ -934,13 +936,26 @@ def _parse_contents_from_anthropic(
)
)
else:
if content_block.content.stdout:
if (
isinstance(content_block.content, BetaCodeExecutionResultBlock)
and content_block.content.stdout
):
code_outputs.append(
Content.from_text(
text=content_block.content.stdout,
raw_representation=content_block.content,
)
)
if (
isinstance(content_block.content, BetaEncryptedCodeExecutionResultBlock)
and content_block.content.encrypted_stdout
):
code_outputs.append(
Content.from_text(
text=content_block.content.encrypted_stdout,
raw_representation=content_block.content,
)
)
if content_block.content.stderr:
code_outputs.append(
Content.from_error(
Expand Down
4 changes: 2 additions & 2 deletions python/packages/core/agent_framework/observability.py
Original file line number Diff line number Diff line change
Expand Up @@ -1366,7 +1366,7 @@ def run(
span=span,
provider_name=provider_name,
messages=messages,
system_instructions=_get_instructions_from_options(options),
system_instructions=_get_instructions_from_options(merged_options),
)

span_state = {"closed": False}
Expand Down Expand Up @@ -1423,7 +1423,7 @@ async def _run() -> AgentResponse:
span=span,
provider_name=provider_name,
messages=messages,
system_instructions=_get_instructions_from_options(options),
system_instructions=_get_instructions_from_options(merged_options),
)
start_time_stamp = perf_counter()
try:
Expand Down
160 changes: 160 additions & 0 deletions python/packages/core/tests/core/test_observability.py
Original file line number Diff line number Diff line change
Expand Up @@ -2437,3 +2437,163 @@ def greet_with_model(greeting: Greeting) -> str:
# Verify JSON is valid and contains the text
tool_arguments = json.loads(tool_arguments_json)
assert tool_arguments["greeting"]["message"] == japanese_text


# region Test merged options for instructions


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
async def test_agent_instructions_from_default_options(
mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
):
"""Test that instructions from default_options are captured in agent telemetry."""
import json

agent = mock_chat_agent()
agent.default_options = {"model_id": "TestModel", "instructions": "Default system instructions."}

messages = [Message(role="user", text="Test message")]
span_exporter.clear()
response = await agent.run(messages)

assert response is not None
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]

# Instructions from default_options should be captured
assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
assert len(system_instructions) == 1
assert system_instructions[0]["content"] == "Default system instructions."


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
async def test_agent_instructions_from_options_override(
mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
):
"""Test that instructions from options are captured when no default_options instructions exist."""
import json

agent = mock_chat_agent()
agent.default_options = {"model_id": "TestModel"} # No default instructions

messages = [Message(role="user", text="Test message")]
span_exporter.clear()
response = await agent.run(messages, options={"instructions": "Override instructions."})

assert response is not None
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]

assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
assert len(system_instructions) == 1
assert system_instructions[0]["content"] == "Override instructions."


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
async def test_agent_instructions_merged_from_default_and_options(
mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
):
"""Test that instructions from both default_options and options are merged (concatenated)."""
import json

agent = mock_chat_agent()
agent.default_options = {"model_id": "TestModel", "instructions": "Default instructions."}

messages = [Message(role="user", text="Test message")]
span_exporter.clear()
response = await agent.run(messages, options={"instructions": "Additional instructions."})

assert response is not None
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]

# Merged instructions should contain both default and override, concatenated with newline
assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
assert len(system_instructions) == 1
assert "Default instructions." in system_instructions[0]["content"]
assert "Additional instructions." in system_instructions[0]["content"]


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
async def test_agent_streaming_instructions_from_default_options(
mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
):
"""Test that streaming agent telemetry captures instructions from default_options."""
import json

agent = mock_chat_agent()
agent.default_options = {"model_id": "TestModel", "instructions": "Default streaming instructions."}

messages = [Message(role="user", text="Test message")]
span_exporter.clear()
updates = []
stream = agent.run(messages, stream=True)
async for update in stream:
updates.append(update)
await stream.get_final_response()

assert len(updates) == 2
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]

assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
assert len(system_instructions) == 1
assert system_instructions[0]["content"] == "Default streaming instructions."


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
async def test_agent_streaming_instructions_merged_from_default_and_options(
mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
):
"""Test that streaming agent telemetry captures merged instructions from default_options and options."""
import json

agent = mock_chat_agent()
agent.default_options = {"model_id": "TestModel", "instructions": "Default instructions."}

messages = [Message(role="user", text="Test message")]
span_exporter.clear()
updates = []
stream = agent.run(messages, stream=True, options={"instructions": "Stream override."})
async for update in stream:
updates.append(update)
await stream.get_final_response()

assert len(updates) == 2
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]

assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
assert len(system_instructions) == 1
assert "Default instructions." in system_instructions[0]["content"]
assert "Stream override." in system_instructions[0]["content"]


@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
async def test_agent_no_instructions_in_default_or_options(
mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
):
"""Test that system_instructions is not set when neither default_options nor options have instructions."""
agent = mock_chat_agent()
agent.default_options = {"model_id": "TestModel"} # No instructions

messages = [Message(role="user", text="Test message")]
span_exporter.clear()
response = await agent.run(messages)

assert response is not None
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]

assert OtelAttr.SYSTEM_INSTRUCTIONS not in span.attributes
7 changes: 5 additions & 2 deletions python/samples/02-agents/observability/agent_observability.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
"""


# NOTE: approval_mode="never_require" is for sample brevity. Use "always_require" in production; see samples/02-agents/tools/function_tool_with_approval.py and samples/02-agents/tools/function_tool_with_approval_and_sessions.py.
# NOTE: approval_mode="never_require" is for sample brevity. Use "always_require" in production;
# See:
# samples/02-agents/tools/function_tool_with_approval.py
# samples/02-agents/tools/function_tool_with_approval_and_sessions.py.
@tool(approval_mode="never_require")
async def get_weather(
location: Annotated[str, Field(description="The location to get the weather for.")],
Expand All @@ -32,7 +35,7 @@ async def main():
# calling `configure_otel_providers` will *enable* tracing and create the necessary tracing, logging
# and metrics providers based on environment variables.
# See the .env.example file for the available configuration options.
configure_otel_providers()
configure_otel_providers(enable_sensitive_data=True)

questions = ["What's the weather in Amsterdam?", "and in Paris, and which is better?", "Why is the sky blue?"]

Expand Down
Loading