test(langchain): Add basic agent test with Responses call

tests/conftest.py

@@ -48,6 +48,12 @@
 from sentry_sdk.transport import Transport
 from sentry_sdk.utils import reraise
 
+try:
+    import openai
+except ImportError:
+    openai = None
+
+
 from tests import _warning_recorder, _warning_recorder_mgr
 
 from typing import TYPE_CHECKING
@@ -1033,10 +1039,11 @@ def inner(events, include_event_type=True):
 
 @pytest.fixture
 def get_model_response():
-    def inner(response_content, serialize_pydantic=False):
+    def inner(response_content, serialize_pydantic=False, request_headers={}):
         model_request = HttpxRequest(
             "POST",
             "/responses",
+            headers=request_headers,
         )
 
         if serialize_pydantic:
@@ -1053,6 +1060,45 @@ def inner(response_content, serialize_pydantic=False):
     return inner
 
 
+@pytest.fixture
+def nonstreaming_responses_model_response():
+    return openai.types.responses.Response(
+        id="resp_123",
+        output=[
+            openai.types.responses.ResponseOutputMessage(
+                id="msg_123",
+                type="message",
+                status="completed",
+                content=[
+                    openai.types.responses.ResponseOutputText(
+                        text="Hello, how can I help you?",
+                        type="output_text",
+                        annotations=[],
+                    )
+                ],
+                role="assistant",
+            )
+        ],
+        parallel_tool_calls=False,
+        tool_choice="none",
+        tools=[],
+        created_at=10000000,
+        model="gpt-4",
+        object="response",
+        usage=openai.types.responses.ResponseUsage(
+            input_tokens=10,
+            input_tokens_details=openai.types.responses.response_usage.InputTokensDetails(
+                cached_tokens=0,
+            ),
+            output_tokens=20,
+            output_tokens_details=openai.types.responses.response_usage.OutputTokensDetails(
+                reasoning_tokens=5,
+            ),
+            total_tokens=30,
+        ),
+    )
+
+
 class MockServerRequestHandler(BaseHTTPRequestHandler):
     def do_GET(self):  # noqa: N802
         # Process an HTTP GET request and return a response.

tests/integrations/langchain/test_langchain.py

@@ -22,6 +22,7 @@
 
 import sentry_sdk
 from sentry_sdk import start_transaction
+from sentry_sdk.utils import package_version
 from sentry_sdk.integrations.langchain import (
     LangchainIntegration,
     SentryLangchainCallback,
@@ -32,13 +33,14 @@
 try:
     # langchain v1+
     from langchain.tools import tool
+    from langchain.agents import create_agent
     from langchain_classic.agents import AgentExecutor, create_openai_tools_agent  # type: ignore[import-not-found]
 except ImportError:
     # langchain <v1
     from langchain.agents import tool, AgentExecutor, create_openai_tools_agent
 
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-
+from langchain_core.messages import HumanMessage, SystemMessage
 
 from openai.types.chat.chat_completion_chunk import (
     ChatCompletionChunk,
@@ -54,6 +56,8 @@
     PromptTokensDetails,
 )
 
+LANGCHAIN_VERSION = package_version("langchain")
+
 
 @tool
 def get_word_length(word: str) -> int:
@@ -81,6 +85,132 @@ def _llm_type(self) -> str:
         return llm_type
 
 
+@pytest.mark.skipif(
+    LANGCHAIN_VERSION < (1,),
+    reason="LangChain 1.0+ required (ONE AGENT refactor)",
+)
+@pytest.mark.parametrize(
+    "send_default_pii, include_prompts",
+    [
+        (True, True),
+        (True, False),
+        (False, True),
+        (False, False),
+    ],
+)
+@pytest.mark.parametrize(
+    "system_instructions_content",
+    [
+        "You are very powerful assistant, but don't know current events",
+        [
+            {"type": "text", "text": "You are a helpful assistant."},
+            {"type": "text", "text": "Be concise and clear."},
+        ],
+    ],
+    ids=["string", "blocks"],
+)
+def test_langchain_create_agent(
+    sentry_init,
+    capture_events,
+    send_default_pii,
+    include_prompts,
+    system_instructions_content,
+    request,
+    get_model_response,
+    nonstreaming_responses_model_response,
+):
+    sentry_init(
+        integrations=[
+            LangchainIntegration(
+                include_prompts=include_prompts,
+            )
+        ],
+        traces_sample_rate=1.0,
+        send_default_pii=send_default_pii,
+    )
+    events = capture_events()
+
+    model_repsonse = get_model_response(
+        nonstreaming_responses_model_response,
+        serialize_pydantic=True,
+        request_headers={
+            "X-Stainless-Raw-Response": "True",
+        },
+    )
+
+    llm = ChatOpenAI(
+        model_name="gpt-3.5-turbo",
+        temperature=0,
+        openai_api_key="badkey",
+        use_responses_api=True,
+    )
+    agent = create_agent(
+        model=llm,
+        tools=[get_word_length],
+        system_prompt=SystemMessage(content=system_instructions_content),
+        name="word_length_agent",
+    )
+
+    with patch.object(
+        llm.client._client._client,
+        "send",
+        return_value=model_repsonse,
+    ) as _:
+        with start_transaction():
+            agent.invoke(
+                {
+                    "messages": [
+                        HumanMessage(content="How many letters in the word eudca"),
+                    ],
+                },
+            )
+
+    tx = events[0]
+    assert tx["type"] == "transaction"
+    assert tx["contexts"]["trace"]["origin"] == "manual"
+
+    chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")
+    assert len(chat_spans) == 1
+    assert chat_spans[0]["origin"] == "auto.ai.langchain"
+
+    # Token usage is only available in newer versions of langchain (v0.2+)
+    # where usage_metadata is supported on AIMessageChunk
+    if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]:
+        assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10
+        assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20
+        assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30
+
+    if send_default_pii and include_prompts:
+        assert (
+            chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT]
+            == "Hello, how can I help you?"
+        )
+
+        param_id = request.node.callspec.id
+        if "string" in param_id:
+            assert [
+                {
+                    "type": "text",
+                    "content": "You are very powerful assistant, but don't know current events",
+                }
+            ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
+        else:
+            assert [
+                {
+                    "type": "text",
+                    "content": "You are a helpful assistant.",
+                },
+                {
+                    "type": "text",
+                    "content": "Be concise and clear.",
+                },
+            ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
+    else:
+        assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {})
+        assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {})
+        assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {})
+
+
 @pytest.mark.parametrize(
     "send_default_pii, include_prompts",
     [
@@ -102,7 +232,7 @@ def _llm_type(self) -> str:
     ],
     ids=["string", "list", "blocks"],
 )
-def test_langchain_agent(
+def test_langchain_openai_tools_agent(
     sentry_init,
     capture_events,
     send_default_pii,