From ed4d833a61c49515aa32909c582ea48f8d984dd6 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 26 Aug 2025 15:00:34 +0200 Subject: [PATCH 01/38] feat: add langgraph integration --- sentry_sdk/integrations/langgraph.py | 355 +++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 sentry_sdk/integrations/langgraph.py diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py new file mode 100644 index 0000000000..2b4cbc8060 --- /dev/null +++ b/sentry_sdk/integrations/langgraph.py @@ -0,0 +1,355 @@ +from functools import wraps +from typing import Any, AsyncIterator, Callable, Iterator, List, Optional + +import sentry_sdk +from sentry_sdk.ai.monitoring import set_ai_pipeline_name +from sentry_sdk.ai.utils import set_data_normalized +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations import DidNotEnable, Integration +from sentry_sdk.scope import should_send_default_pii + + +try: + from langgraph.graph import StateGraph + from langgraph.pregel import Pregel +except ImportError: + raise DidNotEnable("langgraph not installed") + + +class LanggraphIntegration(Integration): + identifier = "langgraph" + origin = f"auto.ai.{identifier}" + + def __init__(self, include_prompts=True): + # type: (LanggraphIntegration, bool) -> None + self.include_prompts = include_prompts + + @staticmethod + def setup_once(): + # type: () -> None + # Wrap StateGraph methods - these get called when the graph is compiled + StateGraph.compile = _wrap_state_graph_compile(StateGraph.compile) + + # Wrap Pregel methods - these are the actual execution methods on compiled graphs + if hasattr(Pregel, "invoke"): + Pregel.invoke = _wrap_pregel_invoke(Pregel.invoke) + if hasattr(Pregel, "ainvoke"): + Pregel.ainvoke = _wrap_pregel_ainvoke(Pregel.ainvoke) + if hasattr(Pregel, "stream"): + Pregel.stream = _wrap_pregel_stream(Pregel.stream) + if hasattr(Pregel, "astream"): + Pregel.astream = _wrap_pregel_astream(Pregel.astream) + + +def _get_graph_name(graph_obj): + # type: (Any) -> Optional[str] + """Extract graph name from various possible attributes.""" + # Try to get name from different possible attributes + for attr in ["name", "graph_name", "__name__", "_name"]: + if hasattr(graph_obj, attr): + name = getattr(graph_obj, attr) + if name and isinstance(name, str): + return name + return None + + +def _get_graph_metadata(graph_obj): + # type: (Any) -> tuple[Optional[str], Optional[List[str]]] + """Extract graph name and node names if available.""" + graph_name = _get_graph_name(graph_obj) + + # Try to get node names from the graph + node_names = None + if hasattr(graph_obj, "nodes"): + try: + nodes = graph_obj.nodes + if isinstance(nodes, dict): + node_names = list(nodes.keys()) + elif hasattr(nodes, "__iter__"): + node_names = list(nodes) + except Exception: + pass + elif hasattr(graph_obj, "graph") and hasattr(graph_obj.graph, "nodes"): + try: + nodes = graph_obj.graph.nodes + if isinstance(nodes, dict): + node_names = list(nodes.keys()) + elif hasattr(nodes, "__iter__"): + node_names = list(nodes) + except Exception: + pass + + return graph_name, node_names + + +def _wrap_state_graph_compile(f): + # type: (Callable[..., Any]) -> Callable[..., Any] + """Wrap StateGraph.compile to add instrumentation to the resulting compiled graph.""" + + @wraps(f) + def new_compile(self, *args, **kwargs): + # type: (Any, Any, Any) -> Any + # Compile the graph normally + compiled_graph = f(self, *args, **kwargs) + + # Store metadata on the compiled graph for later use + if hasattr(self, "__dict__"): + compiled_graph._sentry_source_graph = self + + return compiled_graph + + return new_compile + + +def _wrap_pregel_invoke(f): + # type: (Callable[..., Any]) -> Callable[..., Any] + + @wraps(f) + def new_invoke(self, *args, **kwargs): + # type: (Any, Any, Any) -> Any + integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) + if integration is None: + return f(self, *args, **kwargs) + + graph_name, node_names = _get_graph_metadata(self) + + with sentry_sdk.start_span( + op=OP.GEN_AI_PIPELINE, + name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + origin=LanggraphIntegration.origin, + ) as span: + # Set pipeline metadata + if graph_name: + set_ai_pipeline_name(graph_name) + span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke") + span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) + + # Capture input state if PII is allowed + if ( + len(args) > 0 + and should_send_default_pii() + and integration.include_prompts + ): + set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) + + # Execute the graph + try: + result = f(self, *args, **kwargs) + + # Capture output state if PII is allowed + if should_send_default_pii() and integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) + + return result + + except Exception: + span.set_status("internal_error") + raise + finally: + if graph_name: + set_ai_pipeline_name(None) + + return new_invoke + + +def _wrap_pregel_ainvoke(f): + # type: (Callable[..., Any]) -> Callable[..., Any] + + @wraps(f) + async def new_ainvoke(self, *args, **kwargs): + # type: (Any, Any, Any) -> Any + integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) + if integration is None: + return await f(self, *args, **kwargs) + + graph_name, node_names = _get_graph_metadata(self) + + with sentry_sdk.start_span( + op=OP.GEN_AI_PIPELINE, + name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + origin=LanggraphIntegration.origin, + ) as span: + # Set pipeline metadata + if graph_name: + set_ai_pipeline_name(graph_name) + span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "ainvoke") + span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) + + # Capture input state if PII is allowed + if ( + len(args) > 0 + and should_send_default_pii() + and integration.include_prompts + ): + set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) + + # Execute the graph + try: + result = await f(self, *args, **kwargs) + + # Capture output state if PII is allowed + if should_send_default_pii() and integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) + + return result + + except Exception: + span.set_status("internal_error") + raise + finally: + if graph_name: + set_ai_pipeline_name(None) + + new_ainvoke.__wrapped__ = True + return new_ainvoke + + +def _wrap_pregel_stream(f): + # type: (Callable[..., Any]) -> Callable[..., Any] + + @wraps(f) + def new_stream(self, *args, **kwargs): + # type: (Any, Any, Any) -> Any + integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) + if integration is None: + return f(self, *args, **kwargs) + + graph_name, node_names = _get_graph_metadata(self) + + span = sentry_sdk.start_span( + op=OP.GEN_AI_PIPELINE, + name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + origin=LanggraphIntegration.origin, + ) + span.__enter__() + + # Set pipeline metadata + if graph_name: + set_ai_pipeline_name(graph_name) + span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "stream") + span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + + # Capture input state if PII is allowed + if len(args) > 0 and should_send_default_pii() and integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) + + # Execute the graph + try: + result = f(self, *args, **kwargs) + except Exception: + span.set_status("internal_error") + if graph_name: + set_ai_pipeline_name(None) + span.__exit__(None, None, None) + raise + + old_iterator = result + + def new_iterator(): + # type: () -> Iterator[Any] + final_output = None + try: + for chunk in old_iterator: + final_output = chunk # Keep track of the last chunk + yield chunk + except Exception: + span.set_status("internal_error") + raise + finally: + # Capture final output if available and PII is allowed + if ( + final_output is not None + and should_send_default_pii() + and integration.include_prompts + ): + set_data_normalized( + span, SPANDATA.GEN_AI_RESPONSE_TEXT, final_output + ) + + if graph_name: + set_ai_pipeline_name(None) + span.__exit__(None, None, None) + + return new_iterator() + + return new_stream + + +def _wrap_pregel_astream(f): + # type: (Callable[..., Any]) -> Callable[..., Any] + + @wraps(f) + def new_astream(self, *args, **kwargs): + # type: (Any, Any, Any) -> Any + integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) + if integration is None: + return f(self, *args, **kwargs) + + graph_name, node_names = _get_graph_metadata(self) + + span = sentry_sdk.start_span( + op=OP.GEN_AI_PIPELINE, + name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + origin=LanggraphIntegration.origin, + ) + span.__enter__() + + # Set pipeline metadata + if graph_name: + set_ai_pipeline_name(graph_name) + span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "astream") + span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) + + # Capture input state if PII is allowed + if len(args) > 0 and should_send_default_pii() and integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) + + # Execute the graph + try: + result = f(self, *args, **kwargs) + except Exception: + span.set_status("internal_error") + if graph_name: + set_ai_pipeline_name(None) + span.__exit__(None, None, None) + raise + + old_async_iterator = result + + async def new_async_iterator(): + # type: () -> AsyncIterator[Any] + final_output = None + try: + async for chunk in old_async_iterator: + final_output = chunk # Keep track of the last chunk + yield chunk + except Exception: + span.set_status("internal_error") + raise + finally: + # Capture final output if available and PII is allowed + if ( + final_output is not None + and should_send_default_pii() + and integration.include_prompts + ): + set_data_normalized( + span, SPANDATA.GEN_AI_RESPONSE_TEXT, final_output + ) + + if graph_name: + set_ai_pipeline_name(None) + span.__exit__(None, None, None) + + return new_async_iterator() + + new_astream.__wrapped__ = True + return new_astream From 4b3b6649ec26ee39821c28bb4874ecfc66a55402 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 26 Aug 2025 15:14:03 +0200 Subject: [PATCH 02/38] use the correct span ops / data names --- sentry_sdk/integrations/langgraph.py | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 2b4cbc8060..a4a15be7f1 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -114,16 +114,17 @@ def new_invoke(self, *args, **kwargs): graph_name, node_names = _get_graph_metadata(self) with sentry_sdk.start_span( - op=OP.GEN_AI_PIPELINE, - name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + op=OP.GEN_AI_INVOKE_AGENT, + name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", origin=LanggraphIntegration.origin, ) as span: - # Set pipeline metadata + # Set agent metadata if graph_name: set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke") + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) # Capture input state if PII is allowed @@ -167,16 +168,17 @@ async def new_ainvoke(self, *args, **kwargs): graph_name, node_names = _get_graph_metadata(self) with sentry_sdk.start_span( - op=OP.GEN_AI_PIPELINE, - name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + op=OP.GEN_AI_INVOKE_AGENT, + name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", origin=LanggraphIntegration.origin, ) as span: - # Set pipeline metadata + # Set agent metadata if graph_name: set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "ainvoke") + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) # Capture input state if PII is allowed @@ -221,18 +223,19 @@ def new_stream(self, *args, **kwargs): graph_name, node_names = _get_graph_metadata(self) span = sentry_sdk.start_span( - op=OP.GEN_AI_PIPELINE, - name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + op=OP.GEN_AI_INVOKE_AGENT, + name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", origin=LanggraphIntegration.origin, ) span.__enter__() - # Set pipeline metadata + # Set agent metadata if graph_name: set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "stream") + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) # Capture input state if PII is allowed @@ -294,18 +297,19 @@ def new_astream(self, *args, **kwargs): graph_name, node_names = _get_graph_metadata(self) span = sentry_sdk.start_span( - op=OP.GEN_AI_PIPELINE, - name=f"langgraph {graph_name}".strip() if graph_name else "langgraph", + op=OP.GEN_AI_INVOKE_AGENT, + name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", origin=LanggraphIntegration.origin, ) span.__enter__() - # Set pipeline metadata + # Set agent metadata if graph_name: set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) + span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "astream") + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) # Capture input state if PII is allowed From e5d25204357b06379f30a6004306040fd4ed0238 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 28 Aug 2025 11:48:02 +0200 Subject: [PATCH 03/38] add create agent and make instrumentation more semantically meaningful --- sentry_sdk/consts.py | 1 + sentry_sdk/integrations/langgraph.py | 67 ++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index d7a0603a10..dcedd26646 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -792,6 +792,7 @@ class OP: FUNCTION_AWS = "function.aws" FUNCTION_GCP = "function.gcp" GEN_AI_CHAT = "gen_ai.chat" + GEN_AI_CREATE_AGENT = "gen_ai.create_agent" GEN_AI_EMBEDDINGS = "gen_ai.embeddings" GEN_AI_EXECUTE_TOOL = "gen_ai.execute_tool" GEN_AI_HANDOFF = "gen_ai.handoff" diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index a4a15be7f1..a011787062 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -35,15 +35,28 @@ def setup_once(): Pregel.invoke = _wrap_pregel_invoke(Pregel.invoke) if hasattr(Pregel, "ainvoke"): Pregel.ainvoke = _wrap_pregel_ainvoke(Pregel.ainvoke) - if hasattr(Pregel, "stream"): - Pregel.stream = _wrap_pregel_stream(Pregel.stream) - if hasattr(Pregel, "astream"): - Pregel.astream = _wrap_pregel_astream(Pregel.astream) + # if hasattr(Pregel, "stream"): + # Pregel.stream = _wrap_pregel_stream(Pregel.stream) + # if hasattr(Pregel, "astream"): + # Pregel.astream = _wrap_pregel_astream(Pregel.astream) + + # Wrap prebuilt agent creation functions + import langgraph.prebuilt as prebuilt + + if hasattr(prebuilt, "create_react_agent"): + prebuilt.create_react_agent = _wrap_create_react_agent( + prebuilt.create_react_agent + ) + print("Wrapped create_react_agent") def _get_graph_name(graph_obj): # type: (Any) -> Optional[str] """Extract graph name from various possible attributes.""" + # Check for Sentry-specific agent name first + if hasattr(graph_obj, "_sentry_agent_name"): + return graph_obj._sentry_agent_name + # Try to get name from different possible attributes for attr in ["name", "graph_name", "__name__", "_name"]: if hasattr(graph_obj, attr): @@ -82,6 +95,28 @@ def _get_graph_metadata(graph_obj): return graph_name, node_names +def _wrap_create_react_agent(f): + # type: (Callable[..., Any]) -> Callable[..., Any] + """Wrap create_react_agent to create a create_agent span.""" + + @wraps(f) + def new_create_react_agent(*args, **kwargs): + # type: (Any, Any) -> Any + integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) + if integration is None: + return f(*args, **kwargs) + with sentry_sdk.start_span( + op=OP.GEN_AI_CREATE_AGENT, + name="create_agent", + origin=LanggraphIntegration.origin, + ) as span: + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") + span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) + return f(*args, **kwargs) + + return new_create_react_agent + + def _wrap_state_graph_compile(f): # type: (Callable[..., Any]) -> Callable[..., Any] """Wrap StateGraph.compile to add instrumentation to the resulting compiled graph.""" @@ -115,7 +150,11 @@ def new_invoke(self, *args, **kwargs): with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", + name=( + f"invoke_agent invoke {graph_name}".strip() + if graph_name + else "invoke_agent" + ), origin=LanggraphIntegration.origin, ) as span: # Set agent metadata @@ -169,7 +208,11 @@ async def new_ainvoke(self, *args, **kwargs): with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", + name=( + f"invoke_agent ainvoke {graph_name}".strip() + if graph_name + else "invoke_agent" + ), origin=LanggraphIntegration.origin, ) as span: # Set agent metadata @@ -224,7 +267,11 @@ def new_stream(self, *args, **kwargs): span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", + name=( + f"invoke_agent stream {graph_name}".strip() + if graph_name + else "invoke_agent" + ), origin=LanggraphIntegration.origin, ) span.__enter__() @@ -298,7 +345,11 @@ def new_astream(self, *args, **kwargs): span = sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent", + name=( + f"invoke_agent astream {graph_name}".strip() + if graph_name + else "invoke_agent" + ), origin=LanggraphIntegration.origin, ) span.__enter__() From b2ec8c29830191e529eb6889ff4559e91660e4fb Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 28 Aug 2025 13:59:48 +0200 Subject: [PATCH 04/38] attach tools to agent creation --- sentry_sdk/integrations/langgraph.py | 61 +++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index a011787062..4c3b10a574 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -111,7 +111,6 @@ def new_create_react_agent(*args, **kwargs): origin=LanggraphIntegration.origin, ) as span: span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") - span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) return f(*args, **kwargs) return new_create_react_agent @@ -119,19 +118,61 @@ def new_create_react_agent(*args, **kwargs): def _wrap_state_graph_compile(f): # type: (Callable[..., Any]) -> Callable[..., Any] - """Wrap StateGraph.compile to add instrumentation to the resulting compiled graph.""" - @wraps(f) def new_compile(self, *args, **kwargs): - # type: (Any, Any, Any) -> Any - # Compile the graph normally - compiled_graph = f(self, *args, **kwargs) + integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) + + # import ipdb; ipdb.set_trace() + def get_tools(graph): + tools = [] + edges = getattr(graph, "edges", None) + if edges: + if isinstance(edges, dict): + if "tools" in edges: + tools.append("tools") + elif hasattr(edges, "__iter__"): + for edge in edges: + if getattr(edge, "name", None) == "tools" or edge == "tools": + tools.append("tools") + cond_edges = getattr(graph, "conditional_edges", None) + if cond_edges: + if isinstance(cond_edges, dict): + if "tools" in cond_edges: + tools.append("tools") + elif hasattr(cond_edges, "__iter__"): + for edge in cond_edges: + if getattr(edge, "name", None) == "tools" or edge == "tools": + tools.append("tools") + return tools - # Store metadata on the compiled graph for later use - if hasattr(self, "__dict__"): - compiled_graph._sentry_source_graph = self + compiled_graph = f(self, *args, **kwargs) + if integration is None: + return compiled_graph - return compiled_graph + with sentry_sdk.start_span( + op=OP.GEN_AI_CREATE_AGENT, + name="create_agent", + origin=LanggraphIntegration.origin, + ) as span: + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") + span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) + # import ipdb; ipdb.set_trace() + tools = None + graph = getattr(compiled_graph, "get_graph", None) + if callable(graph): + graph_obj = graph() + nodes = getattr(graph_obj, "nodes", None) + if nodes and isinstance(nodes, dict): + tools_node = nodes.get("tools") + if tools_node: + data = getattr(tools_node, "data", None) + if data and hasattr(data, "tools_by_name"): + tools = list(data.tools_by_name.keys()) + span.set_data(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools) + compiled_graph = f(self, *args, **kwargs) + if hasattr(self, "__dict__"): + compiled_graph._sentry_source_graph = self + return compiled_graph return new_compile From e65168deef9b8a0e26cad9cc7404e8c34482a8df Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 28 Aug 2025 14:00:23 +0200 Subject: [PATCH 05/38] remove unused code --- sentry_sdk/integrations/langgraph.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 4c3b10a574..8188feac0b 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -122,29 +122,6 @@ def _wrap_state_graph_compile(f): def new_compile(self, *args, **kwargs): integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) - # import ipdb; ipdb.set_trace() - def get_tools(graph): - tools = [] - edges = getattr(graph, "edges", None) - if edges: - if isinstance(edges, dict): - if "tools" in edges: - tools.append("tools") - elif hasattr(edges, "__iter__"): - for edge in edges: - if getattr(edge, "name", None) == "tools" or edge == "tools": - tools.append("tools") - cond_edges = getattr(graph, "conditional_edges", None) - if cond_edges: - if isinstance(cond_edges, dict): - if "tools" in cond_edges: - tools.append("tools") - elif hasattr(cond_edges, "__iter__"): - for edge in cond_edges: - if getattr(edge, "name", None) == "tools" or edge == "tools": - tools.append("tools") - return tools - compiled_graph = f(self, *args, **kwargs) if integration is None: return compiled_graph @@ -156,7 +133,6 @@ def get_tools(graph): ) as span: span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) - # import ipdb; ipdb.set_trace() tools = None graph = getattr(compiled_graph, "get_graph", None) if callable(graph): From e9e53c3b885ea671f24a8d8bfe136133df1212ad Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 28 Aug 2025 14:36:49 +0200 Subject: [PATCH 06/38] add some more data --- sentry_sdk/integrations/langgraph.py | 34 ++++------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 8188feac0b..6af4e83381 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -40,15 +40,6 @@ def setup_once(): # if hasattr(Pregel, "astream"): # Pregel.astream = _wrap_pregel_astream(Pregel.astream) - # Wrap prebuilt agent creation functions - import langgraph.prebuilt as prebuilt - - if hasattr(prebuilt, "create_react_agent"): - prebuilt.create_react_agent = _wrap_create_react_agent( - prebuilt.create_react_agent - ) - print("Wrapped create_react_agent") - def _get_graph_name(graph_obj): # type: (Any) -> Optional[str] @@ -95,27 +86,6 @@ def _get_graph_metadata(graph_obj): return graph_name, node_names -def _wrap_create_react_agent(f): - # type: (Callable[..., Any]) -> Callable[..., Any] - """Wrap create_react_agent to create a create_agent span.""" - - @wraps(f) - def new_create_react_agent(*args, **kwargs): - # type: (Any, Any) -> Any - integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) - if integration is None: - return f(*args, **kwargs) - with sentry_sdk.start_span( - op=OP.GEN_AI_CREATE_AGENT, - name="create_agent", - origin=LanggraphIntegration.origin, - ) as span: - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") - return f(*args, **kwargs) - - return new_create_react_agent - - def _wrap_state_graph_compile(f): # type: (Callable[..., Any]) -> Callable[..., Any] @wraps(f) @@ -131,7 +101,11 @@ def new_compile(self, *args, **kwargs): name="create_agent", origin=LanggraphIntegration.origin, ) as span: + # import ipdb; ipdb.set_trace() span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") + span.set_data( + SPANDATA.GEN_AI_AGENT_NAME, getattr(compiled_graph, "name", None) + ) span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) tools = None graph = getattr(compiled_graph, "get_graph", None) From aec2e26fc84bfd7a92fdb6252669aea27f1633fe Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 28 Aug 2025 15:18:13 +0200 Subject: [PATCH 07/38] cleanup --- sentry_sdk/integrations/langgraph.py | 163 +-------------------------- 1 file changed, 1 insertion(+), 162 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 6af4e83381..c5b44f7195 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -1,5 +1,5 @@ from functools import wraps -from typing import Any, AsyncIterator, Callable, Iterator, List, Optional +from typing import Any, Callable, List, Optional import sentry_sdk from sentry_sdk.ai.monitoring import set_ai_pipeline_name @@ -35,10 +35,6 @@ def setup_once(): Pregel.invoke = _wrap_pregel_invoke(Pregel.invoke) if hasattr(Pregel, "ainvoke"): Pregel.ainvoke = _wrap_pregel_ainvoke(Pregel.ainvoke) - # if hasattr(Pregel, "stream"): - # Pregel.stream = _wrap_pregel_stream(Pregel.stream) - # if hasattr(Pregel, "astream"): - # Pregel.astream = _wrap_pregel_astream(Pregel.astream) def _get_graph_name(graph_obj): @@ -242,160 +238,3 @@ async def new_ainvoke(self, *args, **kwargs): new_ainvoke.__wrapped__ = True return new_ainvoke - - -def _wrap_pregel_stream(f): - # type: (Callable[..., Any]) -> Callable[..., Any] - - @wraps(f) - def new_stream(self, *args, **kwargs): - # type: (Any, Any, Any) -> Any - integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) - if integration is None: - return f(self, *args, **kwargs) - - graph_name, node_names = _get_graph_metadata(self) - - span = sentry_sdk.start_span( - op=OP.GEN_AI_INVOKE_AGENT, - name=( - f"invoke_agent stream {graph_name}".strip() - if graph_name - else "invoke_agent" - ), - origin=LanggraphIntegration.origin, - ) - span.__enter__() - - # Set agent metadata - if graph_name: - set_ai_pipeline_name(graph_name) - span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) - span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) - - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) - - # Capture input state if PII is allowed - if len(args) > 0 and should_send_default_pii() and integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) - - # Execute the graph - try: - result = f(self, *args, **kwargs) - except Exception: - span.set_status("internal_error") - if graph_name: - set_ai_pipeline_name(None) - span.__exit__(None, None, None) - raise - - old_iterator = result - - def new_iterator(): - # type: () -> Iterator[Any] - final_output = None - try: - for chunk in old_iterator: - final_output = chunk # Keep track of the last chunk - yield chunk - except Exception: - span.set_status("internal_error") - raise - finally: - # Capture final output if available and PII is allowed - if ( - final_output is not None - and should_send_default_pii() - and integration.include_prompts - ): - set_data_normalized( - span, SPANDATA.GEN_AI_RESPONSE_TEXT, final_output - ) - - if graph_name: - set_ai_pipeline_name(None) - span.__exit__(None, None, None) - - return new_iterator() - - return new_stream - - -def _wrap_pregel_astream(f): - # type: (Callable[..., Any]) -> Callable[..., Any] - - @wraps(f) - def new_astream(self, *args, **kwargs): - # type: (Any, Any, Any) -> Any - integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) - if integration is None: - return f(self, *args, **kwargs) - - graph_name, node_names = _get_graph_metadata(self) - - span = sentry_sdk.start_span( - op=OP.GEN_AI_INVOKE_AGENT, - name=( - f"invoke_agent astream {graph_name}".strip() - if graph_name - else "invoke_agent" - ), - origin=LanggraphIntegration.origin, - ) - span.__enter__() - - # Set agent metadata - if graph_name: - set_ai_pipeline_name(graph_name) - span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) - span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) - - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, True) - - # Capture input state if PII is allowed - if len(args) > 0 and should_send_default_pii() and integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) - - # Execute the graph - try: - result = f(self, *args, **kwargs) - except Exception: - span.set_status("internal_error") - if graph_name: - set_ai_pipeline_name(None) - span.__exit__(None, None, None) - raise - - old_async_iterator = result - - async def new_async_iterator(): - # type: () -> AsyncIterator[Any] - final_output = None - try: - async for chunk in old_async_iterator: - final_output = chunk # Keep track of the last chunk - yield chunk - except Exception: - span.set_status("internal_error") - raise - finally: - # Capture final output if available and PII is allowed - if ( - final_output is not None - and should_send_default_pii() - and integration.include_prompts - ): - set_data_normalized( - span, SPANDATA.GEN_AI_RESPONSE_TEXT, final_output - ) - - if graph_name: - set_ai_pipeline_name(None) - span.__exit__(None, None, None) - - return new_async_iterator() - - new_astream.__wrapped__ = True - return new_astream From ba7e4a020a2070442fa7e3950ac4509280e3ba63 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Fri, 29 Aug 2025 09:38:02 +0200 Subject: [PATCH 08/38] handle messages better --- sentry_sdk/integrations/langgraph.py | 80 ++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index c5b44f7195..900a273e94 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -7,6 +7,7 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii +from sentry_sdk.utils import safe_serialize try: @@ -82,6 +83,66 @@ def _get_graph_metadata(graph_obj): return graph_name, node_names +def _parse_langgraph_messages(state): + # type: (Any) -> Optional[List[Any]] + role_map = { + "human": "user", + "ai": "assistant", + } + if not state: + return None + + messages = None + + if isinstance(state, dict): + messages = state.get("messages") + elif hasattr(state, "messages"): + messages = state.messages + elif hasattr(state, "get") and callable(state.get): + try: + messages = state.get("messages") + except Exception: + pass + + if not messages or not isinstance(messages, (list, tuple)): + return None + + normalized_messages = [] + for message in messages: + try: + if isinstance(message, dict): + role = message.get("role") or message.get("type") + if role in role_map: + message = dict(message) + message["role"] = role_map[role] + normalized_messages.append(message) + elif hasattr(message, "type") and hasattr(message, "content"): + role = getattr(message, "type", None) + mapped_role = role_map.get(role, role) + parsed = {"role": mapped_role, "content": message.content} + if hasattr(message, "additional_kwargs"): + parsed.update(message.additional_kwargs) + normalized_messages.append(parsed) + elif hasattr(message, "role") and hasattr(message, "content"): + role = getattr(message, "role", None) + mapped_role = role_map.get(role, role) + parsed = {"role": mapped_role, "content": message.content} + for attr in ["name", "tool_calls", "function_call"]: + if hasattr(message, attr): + value = getattr(message, attr) + if value is not None: + parsed[attr] = value + normalized_messages.append(parsed) + elif hasattr(message, "__dict__"): + normalized_messages.append(vars(message)) + else: + normalized_messages.append({"content": str(message)}) + except Exception: + continue + + return normalized_messages if normalized_messages else None + + def _wrap_state_graph_compile(f): # type: (Callable[..., Any]) -> Callable[..., Any] @wraps(f) @@ -153,13 +214,19 @@ def new_invoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) - # Capture input state if PII is allowed + # Capture input messages if PII is allowed if ( len(args) > 0 and should_send_default_pii() and integration.include_prompts ): - set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) + # import ipdb; ipdb.set_trace() + parsed_messages = _parse_langgraph_messages(args[0]) + if parsed_messages: + span.set_data( + SPANDATA.GEN_AI_REQUEST_MESSAGES, + safe_serialize(parsed_messages), + ) # Execute the graph try: @@ -211,13 +278,18 @@ async def new_ainvoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) - # Capture input state if PII is allowed + # Capture input messages if PII is allowed if ( len(args) > 0 and should_send_default_pii() and integration.include_prompts ): - set_data_normalized(span, SPANDATA.GEN_AI_REQUEST_MESSAGES, args[0]) + parsed_messages = _parse_langgraph_messages(args[0]) + if parsed_messages: + span.set_data( + SPANDATA.GEN_AI_REQUEST_MESSAGES, + safe_serialize(parsed_messages), + ) # Execute the graph try: From c7e326d616f3a5782488af953a6a9c80bd3a2c67 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Mon, 1 Sep 2025 15:16:35 +0200 Subject: [PATCH 09/38] simplify --- sentry_sdk/integrations/langgraph.py | 89 ++++------------------------ 1 file changed, 13 insertions(+), 76 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 900a273e94..26b537de7d 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -28,10 +28,14 @@ def __init__(self, include_prompts=True): @staticmethod def setup_once(): # type: () -> None - # Wrap StateGraph methods - these get called when the graph is compiled + # LangGraph lets users create agents using a StateGraph or the Functional API. + # StateGraphs are then compiled to a CompiledStateGraph. Both CompiledStateGraph and + # the functional API execute on a Pregel instance. Pregel is the runtime for the graph + # and the invocation happens on Pregel, so patching the invoke methods takes care of both. + # The streaming methods are not patched, because due to some internal reasons, LangGraph + # will automatically patch the streaming methods to run through invoke, and by doing this + # we prevent duplicate spans for invocations. StateGraph.compile = _wrap_state_graph_compile(StateGraph.compile) - - # Wrap Pregel methods - these are the actual execution methods on compiled graphs if hasattr(Pregel, "invoke"): Pregel.invoke = _wrap_pregel_invoke(Pregel.invoke) if hasattr(Pregel, "ainvoke"): @@ -40,12 +44,6 @@ def setup_once(): def _get_graph_name(graph_obj): # type: (Any) -> Optional[str] - """Extract graph name from various possible attributes.""" - # Check for Sentry-specific agent name first - if hasattr(graph_obj, "_sentry_agent_name"): - return graph_obj._sentry_agent_name - - # Try to get name from different possible attributes for attr in ["name", "graph_name", "__name__", "_name"]: if hasattr(graph_obj, attr): name = getattr(graph_obj, attr) @@ -54,41 +52,8 @@ def _get_graph_name(graph_obj): return None -def _get_graph_metadata(graph_obj): - # type: (Any) -> tuple[Optional[str], Optional[List[str]]] - """Extract graph name and node names if available.""" - graph_name = _get_graph_name(graph_obj) - - # Try to get node names from the graph - node_names = None - if hasattr(graph_obj, "nodes"): - try: - nodes = graph_obj.nodes - if isinstance(nodes, dict): - node_names = list(nodes.keys()) - elif hasattr(nodes, "__iter__"): - node_names = list(nodes) - except Exception: - pass - elif hasattr(graph_obj, "graph") and hasattr(graph_obj.graph, "nodes"): - try: - nodes = graph_obj.graph.nodes - if isinstance(nodes, dict): - node_names = list(nodes.keys()) - elif hasattr(nodes, "__iter__"): - node_names = list(nodes) - except Exception: - pass - - return graph_name, node_names - - def _parse_langgraph_messages(state): # type: (Any) -> Optional[List[Any]] - role_map = { - "human": "user", - "ai": "assistant", - } if not state: return None @@ -110,33 +75,15 @@ def _parse_langgraph_messages(state): normalized_messages = [] for message in messages: try: - if isinstance(message, dict): - role = message.get("role") or message.get("type") - if role in role_map: - message = dict(message) - message["role"] = role_map[role] - normalized_messages.append(message) - elif hasattr(message, "type") and hasattr(message, "content"): - role = getattr(message, "type", None) - mapped_role = role_map.get(role, role) - parsed = {"role": mapped_role, "content": message.content} - if hasattr(message, "additional_kwargs"): - parsed.update(message.additional_kwargs) - normalized_messages.append(parsed) - elif hasattr(message, "role") and hasattr(message, "content"): - role = getattr(message, "role", None) - mapped_role = role_map.get(role, role) - parsed = {"role": mapped_role, "content": message.content} + if hasattr(message, "content"): + parsed = {"content": message.content} for attr in ["name", "tool_calls", "function_call"]: if hasattr(message, attr): value = getattr(message, attr) if value is not None: parsed[attr] = value normalized_messages.append(parsed) - elif hasattr(message, "__dict__"): - normalized_messages.append(vars(message)) - else: - normalized_messages.append({"content": str(message)}) + except Exception: continue @@ -158,7 +105,6 @@ def new_compile(self, *args, **kwargs): name="create_agent", origin=LanggraphIntegration.origin, ) as span: - # import ipdb; ipdb.set_trace() span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") span.set_data( SPANDATA.GEN_AI_AGENT_NAME, getattr(compiled_graph, "name", None) @@ -176,9 +122,6 @@ def new_compile(self, *args, **kwargs): if data and hasattr(data, "tools_by_name"): tools = list(data.tools_by_name.keys()) span.set_data(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools) - compiled_graph = f(self, *args, **kwargs) - if hasattr(self, "__dict__"): - compiled_graph._sentry_source_graph = self return compiled_graph return new_compile @@ -194,7 +137,7 @@ def new_invoke(self, *args, **kwargs): if integration is None: return f(self, *args, **kwargs) - graph_name, node_names = _get_graph_metadata(self) + graph_name = _get_graph_name(self) with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, @@ -258,18 +201,13 @@ async def new_ainvoke(self, *args, **kwargs): if integration is None: return await f(self, *args, **kwargs) - graph_name, node_names = _get_graph_metadata(self) + graph_name = _get_graph_name(self) with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=( - f"invoke_agent ainvoke {graph_name}".strip() - if graph_name - else "invoke_agent" - ), + name="invoke_agent", origin=LanggraphIntegration.origin, ) as span: - # Set agent metadata if graph_name: set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) @@ -278,7 +216,6 @@ async def new_ainvoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) - # Capture input messages if PII is allowed if ( len(args) > 0 and should_send_default_pii() From affa2f966ac3ff9602e72a86db06cd6ffed23f7e Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Mon, 1 Sep 2025 15:47:13 +0200 Subject: [PATCH 10/38] simplify & add tests --- sentry_sdk/integrations/langgraph.py | 45 +- tests/integrations/langgraph/__init__.py | 1 + .../integrations/langgraph/test_langgraph.py | 716 ++++++++++++++++++ 3 files changed, 722 insertions(+), 40 deletions(-) create mode 100644 tests/integrations/langgraph/__init__.py create mode 100644 tests/integrations/langgraph/test_langgraph.py diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 26b537de7d..f2cb57d578 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -2,7 +2,6 @@ from typing import Any, Callable, List, Optional import sentry_sdk -from sentry_sdk.ai.monitoring import set_ai_pipeline_name from sentry_sdk.ai.utils import set_data_normalized from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -141,29 +140,21 @@ def new_invoke(self, *args, **kwargs): with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=( - f"invoke_agent invoke {graph_name}".strip() - if graph_name - else "invoke_agent" - ), + name="invoke_agent", origin=LanggraphIntegration.origin, ) as span: - # Set agent metadata if graph_name: - set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) - # Capture input messages if PII is allowed if ( len(args) > 0 and should_send_default_pii() and integration.include_prompts ): - # import ipdb; ipdb.set_trace() parsed_messages = _parse_langgraph_messages(args[0]) if parsed_messages: span.set_data( @@ -171,22 +162,10 @@ def new_invoke(self, *args, **kwargs): safe_serialize(parsed_messages), ) - # Execute the graph - try: - result = f(self, *args, **kwargs) - - # Capture output state if PII is allowed - if should_send_default_pii() and integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) - - return result - - except Exception: - span.set_status("internal_error") - raise - finally: - if graph_name: - set_ai_pipeline_name(None) + result = f(self, *args, **kwargs) + if should_send_default_pii() and integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) + return result return new_invoke @@ -209,7 +188,6 @@ async def new_ainvoke(self, *args, **kwargs): origin=LanggraphIntegration.origin, ) as span: if graph_name: - set_ai_pipeline_name(graph_name) span.set_data(SPANDATA.GEN_AI_PIPELINE_NAME, graph_name) span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) @@ -227,23 +205,10 @@ async def new_ainvoke(self, *args, **kwargs): SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(parsed_messages), ) - - # Execute the graph - try: result = await f(self, *args, **kwargs) - - # Capture output state if PII is allowed if should_send_default_pii() and integration.include_prompts: set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) - return result - except Exception: - span.set_status("internal_error") - raise - finally: - if graph_name: - set_ai_pipeline_name(None) - new_ainvoke.__wrapped__ = True return new_ainvoke diff --git a/tests/integrations/langgraph/__init__.py b/tests/integrations/langgraph/__init__.py new file mode 100644 index 0000000000..cf697bfaf2 --- /dev/null +++ b/tests/integrations/langgraph/__init__.py @@ -0,0 +1 @@ +# Langgraph integration tests diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py new file mode 100644 index 0000000000..549a6c5520 --- /dev/null +++ b/tests/integrations/langgraph/test_langgraph.py @@ -0,0 +1,716 @@ +import asyncio +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from sentry_sdk import start_transaction +from sentry_sdk.consts import SPANDATA, OP + + +# Mock langgraph modules before importing the integration +def mock_langgraph_imports(): + """Mock langgraph modules to prevent import errors.""" + mock_state_graph = MagicMock() + mock_pregel = MagicMock() + + langgraph_graph_mock = MagicMock() + langgraph_graph_mock.StateGraph = mock_state_graph + + langgraph_pregel_mock = MagicMock() + langgraph_pregel_mock.Pregel = mock_pregel + + sys.modules["langgraph"] = MagicMock() + sys.modules["langgraph.graph"] = langgraph_graph_mock + sys.modules["langgraph.pregel"] = langgraph_pregel_mock + + return mock_state_graph, mock_pregel + + +# Mock the imports +mock_state_graph, mock_pregel = mock_langgraph_imports() + +# Now we can safely import the integration +from sentry_sdk.integrations.langgraph import ( + LanggraphIntegration, + _parse_langgraph_messages, + _get_graph_name, + _wrap_state_graph_compile, + _wrap_pregel_invoke, + _wrap_pregel_ainvoke, +) + + +# Mock LangGraph dependencies +class MockStateGraph: + def __init__(self, schema=None): + self.name = "test_graph" + self.schema = schema + self._compiled_graph = None + + def compile(self, *args, **kwargs): + compiled = MockCompiledGraph(self.name) + compiled.graph = self + return compiled + + +class MockCompiledGraph: + def __init__(self, name="test_graph"): + self.name = name + self._graph = None + + def get_graph(self): + return MockGraphRepresentation() + + def invoke(self, state, config=None): + # Simulate graph execution + return {"messages": [MockMessage("Response from graph")]} + + async def ainvoke(self, state, config=None): + # Simulate async graph execution + return {"messages": [MockMessage("Async response from graph")]} + + +class MockGraphRepresentation: + def __init__(self): + self.nodes = {"tools": MockToolsNode()} + + +class MockToolsNode: + def __init__(self): + self.data = MockToolsData() + + +class MockToolsData: + def __init__(self): + self.tools_by_name = { + "search_tool": MockTool("search_tool"), + "calculator": MockTool("calculator"), + } + + +class MockTool: + def __init__(self, name): + self.name = name + + +class MockMessage: + def __init__(self, content, name=None, tool_calls=None, function_call=None): + self.content = content + self.name = name + self.tool_calls = tool_calls + self.function_call = function_call + + +class MockPregelInstance: + def __init__(self, name="test_pregel"): + self.name = name + self.graph_name = name + + def invoke(self, state, config=None): + return {"messages": [MockMessage("Pregel response")]} + + async def ainvoke(self, state, config=None): + return {"messages": [MockMessage("Async Pregel response")]} + + +def test_langgraph_integration_init(): + """Test LanggraphIntegration initialization with different parameters.""" + # Test default initialization + integration = LanggraphIntegration() + assert integration.include_prompts is True + assert integration.identifier == "langgraph" + assert integration.origin == "auto.ai.langgraph" + + # Test with include_prompts=False + integration = LanggraphIntegration(include_prompts=False) + assert integration.include_prompts is False + + +@pytest.mark.parametrize( + "send_default_pii, include_prompts", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], +) +def test_state_graph_compile( + sentry_init, capture_events, send_default_pii, include_prompts +): + """Test StateGraph.compile() wrapper creates proper create_agent span.""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=include_prompts)], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + ) + events = capture_events() + + # Create a mock StateGraph + graph = MockStateGraph() + + # Mock the original compile method + def original_compile(self, *args, **kwargs): + return MockCompiledGraph(self.name) + + with patch("sentry_sdk.integrations.langgraph.StateGraph"): + # Test the compile wrapper + with start_transaction(): + + wrapped_compile = _wrap_state_graph_compile(original_compile) + compiled_graph = wrapped_compile( + graph, model="test-model", checkpointer=None + ) + + # Verify the compiled graph is returned + assert compiled_graph is not None + assert compiled_graph.name == "test_graph" + + # Check events + tx = events[0] + assert tx["type"] == "transaction" + + # Find the create_agent span + agent_spans = [span for span in tx["spans"] if span["op"] == OP.GEN_AI_CREATE_AGENT] + assert len(agent_spans) == 1 + + agent_span = agent_spans[0] + assert agent_span["description"] == "create_agent" + assert agent_span["origin"] == "auto.ai.langgraph" + assert agent_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "create_agent" + assert agent_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "test_graph" + assert agent_span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "test-model" + assert SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in agent_span["data"] + + +@pytest.mark.parametrize( + "send_default_pii, include_prompts", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], +) +def test_pregel_invoke(sentry_init, capture_events, send_default_pii, include_prompts): + """Test Pregel.invoke() wrapper creates proper invoke_agent span.""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=include_prompts)], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + ) + events = capture_events() + + # Create test state with messages + test_state = { + "messages": [ + MockMessage("Hello, can you help me?", name="user"), + MockMessage("Of course! How can I assist you?", name="assistant"), + ] + } + + # Create mock Pregel instance + pregel = MockPregelInstance("test_graph") + + # Mock the original invoke method + def original_invoke(self, *args, **kwargs): + return {"messages": [MockMessage("Response")]} + + with start_transaction(): + + wrapped_invoke = _wrap_pregel_invoke(original_invoke) + result = wrapped_invoke(pregel, test_state) + + # Verify result + assert result is not None + + # Check events + tx = events[0] + assert tx["type"] == "transaction" + + # Find the invoke_agent span + invoke_spans = [ + span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) == 1 + + invoke_span = invoke_spans[0] + assert "invoke_agent invoke test_graph" in invoke_span["description"] + assert invoke_span["origin"] == "auto.ai.langgraph" + assert invoke_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent" + assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "test_graph" + assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "test_graph" + assert invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False + + # Check PII handling + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT in invoke_span["data"] + # Verify message content is captured + request_messages = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + # The messages might be serialized as a string, so parse if needed + if isinstance(request_messages, str): + import json + + request_messages = json.loads(request_messages) + assert len(request_messages) == 2 + assert request_messages[0]["content"] == "Hello, can you help me?" + assert request_messages[1]["content"] == "Of course! How can I assist you?" + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in invoke_span.get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in invoke_span.get("data", {}) + + +@pytest.mark.parametrize( + "send_default_pii, include_prompts", + [ + (True, True), + (True, False), + (False, True), + (False, False), + ], +) +def test_pregel_ainvoke(sentry_init, capture_events, send_default_pii, include_prompts): + """Test Pregel.ainvoke() async wrapper creates proper invoke_agent span.""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=include_prompts)], + traces_sample_rate=1.0, + send_default_pii=send_default_pii, + ) + events = capture_events() + + # Create test state with messages + test_state = {"messages": [MockMessage("What's the weather like?", name="user")]} + + # Create mock Pregel instance + pregel = MockPregelInstance("async_graph") + + # Mock the original ainvoke method + async def original_ainvoke(self, *args, **kwargs): + return {"messages": [MockMessage("It's sunny today!")]} + + async def run_test(): + with start_transaction(): + + wrapped_ainvoke = _wrap_pregel_ainvoke(original_ainvoke) + result = await wrapped_ainvoke(pregel, test_state) + return result + + # Run the async test + result = asyncio.run(run_test()) + assert result is not None + + # Check events + tx = events[0] + assert tx["type"] == "transaction" + + # Find the invoke_agent span + invoke_spans = [ + span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) == 1 + + invoke_span = invoke_spans[0] + assert invoke_span["description"] == "invoke_agent" + assert invoke_span["origin"] == "auto.ai.langgraph" + assert invoke_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent" + assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "async_graph" + assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "async_graph" + + # Check PII handling + if send_default_pii and include_prompts: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT in invoke_span["data"] + else: + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in invoke_span.get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in invoke_span.get("data", {}) + + +def test_pregel_invoke_error(sentry_init, capture_events): + """Test error handling during graph execution.""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + # Create test state + test_state = {"messages": [MockMessage("This will fail")]} + + # Create mock Pregel instance + pregel = MockPregelInstance("error_graph") + + # Mock the original invoke method to raise an exception + def original_invoke(self, *args, **kwargs): + raise Exception("Graph execution failed") + + with start_transaction(), pytest.raises(Exception, match="Graph execution failed"): + + wrapped_invoke = _wrap_pregel_invoke(original_invoke) + wrapped_invoke(pregel, test_state) + + # Check that error was captured + tx = events[0] + invoke_spans = [ + span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) == 1 + + invoke_span = invoke_spans[0] + # Check if error status is recorded (status is stored in tags) + assert invoke_span.get("tags", {}).get("status") == "internal_error" + + +def test_pregel_ainvoke_error(sentry_init, capture_events): + """Test error handling during async graph execution.""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + # Create test state + test_state = {"messages": [MockMessage("This will fail async")]} + + # Create mock Pregel instance + pregel = MockPregelInstance("async_error_graph") + + # Mock the original ainvoke method to raise an exception + async def original_ainvoke(self, *args, **kwargs): + raise Exception("Async graph execution failed") + + async def run_error_test(): + with start_transaction(), pytest.raises( + Exception, match="Async graph execution failed" + ): + + wrapped_ainvoke = _wrap_pregel_ainvoke(original_ainvoke) + await wrapped_ainvoke(pregel, test_state) + + # Run the async error test + asyncio.run(run_error_test()) + + # Check that error was captured + tx = events[0] + invoke_spans = [ + span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) == 1 + + invoke_span = invoke_spans[0] + # Check if error status is recorded (status is stored in tags) + assert invoke_span.get("tags", {}).get("status") == "internal_error" + + +def test_parse_langgraph_messages_dict_state(): + """Test _parse_langgraph_messages with dict state containing messages.""" + messages = [ + MockMessage("Hello", name="user"), + MockMessage("Hi there", name="assistant", tool_calls=[{"name": "search"}]), + MockMessage("Search result", function_call={"name": "search", "args": {}}), + ] + + state = {"messages": messages} + + result = _parse_langgraph_messages(state) + + assert result is not None + assert len(result) == 3 + + assert result[0]["content"] == "Hello" + assert result[0]["name"] == "user" + + assert result[1]["content"] == "Hi there" + assert result[1]["name"] == "assistant" + assert result[1]["tool_calls"] == [{"name": "search"}] + + assert result[2]["content"] == "Search result" + assert result[2]["function_call"] == {"name": "search", "args": {}} + + +def test_parse_langgraph_messages_object_state(): + """Test _parse_langgraph_messages with object state having messages attribute.""" + + class StateWithMessages: + def __init__(self): + self.messages = [ + MockMessage("Test message", name="user"), + MockMessage("Response", name="assistant"), + ] + + state = StateWithMessages() + + result = _parse_langgraph_messages(state) + + assert result is not None + assert len(result) == 2 + assert result[0]["content"] == "Test message" + assert result[1]["content"] == "Response" + + +def test_parse_langgraph_messages_callable_get(): + """Test _parse_langgraph_messages with state that has callable get method.""" + + class StateWithGet: + def __init__(self): + self._data = {"messages": [MockMessage("From get method", name="system")]} + + def get(self, key): + return self._data.get(key) + + state = StateWithGet() + + result = _parse_langgraph_messages(state) + + assert result is not None + assert len(result) == 1 + assert result[0]["content"] == "From get method" + assert result[0]["name"] == "system" + + +def test_parse_langgraph_messages_invalid_cases(): + """Test _parse_langgraph_messages with various invalid inputs.""" + # None state + assert _parse_langgraph_messages(None) is None + + # Empty state + assert _parse_langgraph_messages({}) is None + + # State without messages + assert _parse_langgraph_messages({"other": "data"}) is None + + # State with non-list messages + assert _parse_langgraph_messages({"messages": "not a list"}) is None + + # State with empty messages list + assert _parse_langgraph_messages({"messages": []}) is None + + # Messages that don't have content attribute + class BadMessage: + def __init__(self): + self.text = "I don't have content" + + assert _parse_langgraph_messages({"messages": [BadMessage()]}) is None + + +def test_parse_langgraph_messages_exception_handling(): + """Test _parse_langgraph_messages handles exceptions gracefully.""" + + class ProblematicState: + def get(self, key): + raise Exception("Something went wrong") + + state = ProblematicState() + + result = _parse_langgraph_messages(state) + assert result is None + + +def test_get_graph_name(): + """Test _get_graph_name function with different graph objects.""" + + # Test with name attribute + class GraphWithName: + name = "graph_with_name" + + assert _get_graph_name(GraphWithName()) == "graph_with_name" + + # Test with graph_name attribute + class GraphWithGraphName: + graph_name = "graph_with_graph_name" + + assert _get_graph_name(GraphWithGraphName()) == "graph_with_graph_name" + + # Test with __name__ attribute + class GraphWithDunderName: + __name__ = "graph_with_dunder_name" + + assert _get_graph_name(GraphWithDunderName()) == "graph_with_dunder_name" + + # Test with _name attribute + class GraphWithPrivateName: + _name = "graph_with_private_name" + + assert _get_graph_name(GraphWithPrivateName()) == "graph_with_private_name" + + # Test with no name attributes + class GraphWithoutName: + pass + + assert _get_graph_name(GraphWithoutName()) is None + + # Test with empty/None name + class GraphWithEmptyName: + name = None + + assert _get_graph_name(GraphWithEmptyName()) is None + + class GraphWithEmptyStringName: + name = "" + + assert _get_graph_name(GraphWithEmptyStringName()) is None + + +def test_span_origin(sentry_init, capture_events): + """Test that span origins are correctly set.""" + sentry_init( + integrations=[LanggraphIntegration()], + traces_sample_rate=1.0, + ) + events = capture_events() + + # Test state graph compile + graph = MockStateGraph() + + def original_compile(self, *args, **kwargs): + return MockCompiledGraph(self.name) + + with start_transaction(): + from sentry_sdk.integrations.langgraph import _wrap_state_graph_compile + + wrapped_compile = _wrap_state_graph_compile(original_compile) + wrapped_compile(graph) + + tx = events[0] + assert tx["contexts"]["trace"]["origin"] == "manual" + + # Check all spans have the correct origin + for span in tx["spans"]: + assert span["origin"] == "auto.ai.langgraph" + + +def test_no_spans_without_integration(sentry_init, capture_events): + """Test that no spans are created when integration is not enabled.""" + # Initialize without LanggraphIntegration + sentry_init( + integrations=[], + traces_sample_rate=1.0, + ) + events = capture_events() + + graph = MockStateGraph() + pregel = MockPregelInstance() + + with start_transaction(): + # These should not create spans + + def original_compile(self, *args, **kwargs): + return MockCompiledGraph(self.name) + + wrapped_compile = _wrap_state_graph_compile(original_compile) + wrapped_compile(graph) + + def original_invoke(self, *args, **kwargs): + return {"result": "test"} + + wrapped_invoke = _wrap_pregel_invoke(original_invoke) + wrapped_invoke(pregel, {"messages": []}) + + tx = events[0] + + # Should only have the manual transaction, no AI spans + ai_spans = [ + span + for span in tx["spans"] + if span["op"] in [OP.GEN_AI_CREATE_AGENT, OP.GEN_AI_INVOKE_AGENT] + ] + assert len(ai_spans) == 0 + + +@pytest.mark.parametrize("graph_name", ["my_graph", None, ""]) +def test_pregel_invoke_with_different_graph_names( + sentry_init, capture_events, graph_name +): + """Test Pregel.invoke() with different graph name scenarios.""" + sentry_init( + integrations=[LanggraphIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + # Create mock Pregel instance with specific name + pregel = MockPregelInstance(graph_name) if graph_name else MockPregelInstance() + if not graph_name: + # Remove name attributes to test None case + delattr(pregel, "name") + delattr(pregel, "graph_name") + + def original_invoke(self, *args, **kwargs): + return {"result": "test"} + + with start_transaction(): + + wrapped_invoke = _wrap_pregel_invoke(original_invoke) + wrapped_invoke(pregel, {"messages": []}) + + tx = events[0] + invoke_spans = [ + span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) == 1 + + invoke_span = invoke_spans[0] + + if graph_name and graph_name.strip(): + assert f"invoke_agent invoke {graph_name}" in invoke_span["description"] + assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == graph_name + assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == graph_name + else: + assert invoke_span["description"] == "invoke_agent" + assert SPANDATA.GEN_AI_PIPELINE_NAME not in invoke_span.get("data", {}) + assert SPANDATA.GEN_AI_AGENT_NAME not in invoke_span.get("data", {}) + + +def test_complex_message_parsing(): + """Test message parsing with complex message structures.""" + # Create messages with various attributes + messages = [ + MockMessage(content="User query", name="user"), + MockMessage( + content="Assistant response with tools", + name="assistant", + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + }, + { + "id": "call_2", + "type": "function", + "function": {"name": "calculate", "arguments": '{"x": 5}'}, + }, + ], + ), + MockMessage( + content="Function call response", + name="function", + function_call={"name": "search", "arguments": '{"query": "test"}'}, + ), + ] + + state = {"messages": messages} + result = _parse_langgraph_messages(state) + + assert result is not None + assert len(result) == 3 + + # Check first message + assert result[0]["content"] == "User query" + assert result[0]["name"] == "user" + assert "tool_calls" not in result[0] + assert "function_call" not in result[0] + + # Check second message with tool calls + assert result[1]["content"] == "Assistant response with tools" + assert result[1]["name"] == "assistant" + assert len(result[1]["tool_calls"]) == 2 + + # Check third message with function call + assert result[2]["content"] == "Function call response" + assert result[2]["name"] == "function" + assert result[2]["function_call"]["name"] == "search" From 0de085a19e8f8fcc8c70353ddfd4cabc7aba58e6 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Mon, 1 Sep 2025 15:47:42 +0200 Subject: [PATCH 11/38] remove comment --- tests/integrations/langgraph/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/langgraph/__init__.py b/tests/integrations/langgraph/__init__.py index cf697bfaf2..e69de29bb2 100644 --- a/tests/integrations/langgraph/__init__.py +++ b/tests/integrations/langgraph/__init__.py @@ -1 +0,0 @@ -# Langgraph integration tests From 71af1cc31c27c0c5c5b2bf01db951fff1dd01ca0 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 2 Sep 2025 09:23:28 +0200 Subject: [PATCH 12/38] Added langgraph to test matrix --- .github/workflows/test-integrations-ai.yml | 89 +++++++++++++++++++ scripts/populate_tox/config.py | 2 + scripts/populate_tox/tox.jinja | 2 + .../split_tox_gh_actions.py | 2 + sentry_sdk/integrations/__init__.py | 3 +- tox.ini | 9 ++ 6 files changed, 106 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml index 72a4253744..3ec0ed7187 100644 --- a/.github/workflows/test-integrations-ai.yml +++ b/.github/workflows/test-integrations-ai.yml @@ -22,6 +22,89 @@ env: CACHED_BUILD_PATHS: | ${{ github.workspace }}/dist-serverless jobs: + test-ai-latest: + name: AI (latest) + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.9","3.11","3.12"] + # python3.6 reached EOL and is no longer being supported on + # new versions of hosted runners on Github Actions + # ubuntu-20.04 is the last version that supported python3.6 + # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 + os: [ubuntu-22.04] + # Use Docker container only for Python 3.6 + container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }} + steps: + - uses: actions/checkout@v5.0.0 + - uses: actions/setup-python@v5 + if: ${{ matrix.python-version != '3.6' }} + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Setup Test Env + run: | + pip install "coverage[toml]" tox + - name: Erase coverage + run: | + coverage erase + - name: Test anthropic latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-anthropic-latest" + - name: Test cohere latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-cohere-latest" + - name: Test langchain latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-latest" + - name: Test langgraph latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph-latest" + - name: Test openai latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest" + - name: Test openai_agents latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents-latest" + - name: Test huggingface_hub latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors + - name: Generate coverage XML + if: ${{ !cancelled() && matrix.python-version != '3.6' }} + run: | + coverage combine .coverage-sentry-* + coverage xml + - name: Upload coverage to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5.5.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + # make sure no plugins alter our coverage reports + plugins: noop + verbose: true + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: .junitxml + verbose: true test-ai-pinned: name: AI (pinned) timeout-minutes: 30 @@ -63,6 +146,12 @@ jobs: set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain-base" - name: Test langchain-notiktoken pinned + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain" + - name: Test langgraph pinned + run: | + set -x # print commands that are executed + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langgraph" + - name: Test openai pinned run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain-notiktoken" diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 69f7b02e21..a3d9d226ad 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -145,6 +145,8 @@ ">=0.3": ["langchain-community"], }, "include": "<1.0", + "langgraph": { + "package": "langgraph", }, "launchdarkly": { "package": "launchdarkly-server-sdk", diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index 42c570b111..d172f6c3f3 100755 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -358,6 +358,8 @@ setenv = huggingface_hub: TESTPATH=tests/integrations/huggingface_hub langchain-base: TESTPATH=tests/integrations/langchain langchain-notiktoken: TESTPATH=tests/integrations/langchain + langchain: TESTPATH=tests/integrations/langchain + langgraph: TESTPATH=tests/integrations/langgraph launchdarkly: TESTPATH=tests/integrations/launchdarkly litestar: TESTPATH=tests/integrations/litestar loguru: TESTPATH=tests/integrations/loguru diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index 1c3435f43b..bb98266ec6 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -65,6 +65,8 @@ "langchain-notiktoken", "openai-base", "openai-notiktoken", + "langchain", + "langgraph", "openai_agents", "huggingface_hub", ], diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index 6f0109aced..683c8645fd 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -141,7 +141,8 @@ def iter_default_integrations(with_auto_enabling_integrations): "graphene": (3, 3), "grpc": (1, 32, 0), # grpcio "huggingface_hub": (0, 22), - "langchain": (0, 1, 0), + "langchain": (0, 0, 210), + "langgraph": (0, 6, 6), "launchdarkly": (9, 8, 0), "loguru": (0, 7, 0), "openai": (1, 0, 0), diff --git a/tox.ini b/tox.ini index c45c72bf85..6ee9163080 100644 --- a/tox.ini +++ b/tox.ini @@ -132,6 +132,7 @@ envlist = {py3.9,py3.11,py3.12}-cohere-v5.13.12 {py3.9,py3.11,py3.12}-cohere-v5.17.0 + {py3.9,py3.11,py3.12}-langchain-base-v0.1.20 {py3.9,py3.11,py3.12}-langchain-base-v0.2.17 {py3.9,py3.12,py3.13}-langchain-base-v0.3.27 @@ -149,6 +150,9 @@ envlist = {py3.8,py3.11,py3.12}-openai-notiktoken-v1.35.15 {py3.8,py3.11,py3.12}-openai-notiktoken-v1.69.0 {py3.8,py3.12,py3.13}-openai-notiktoken-v1.103.0 + {py3.9,py3.12,py3.13}-langgraph-v0.6.6 + {py3.10,py3.12,py3.13}-langgraph-v1.0.0a1 + {py3.10,py3.11,py3.12}-openai_agents-v0.0.19 {py3.10,py3.12,py3.13}-openai_agents-v0.1.0 @@ -536,6 +540,9 @@ deps = openai-notiktoken-v1.0.1: httpx<0.28 openai-notiktoken-v1.35.15: httpx<0.28 + langgraph-v0.6.6: langgraph==0.6.6 + langgraph-v1.0.0a1: langgraph==1.0.0a1 + openai_agents-v0.0.19: openai-agents==0.0.19 openai_agents-v0.1.0: openai-agents==0.1.0 openai_agents-v0.2.10: openai-agents==0.2.10 @@ -841,6 +848,8 @@ setenv = huggingface_hub: TESTPATH=tests/integrations/huggingface_hub langchain-base: TESTPATH=tests/integrations/langchain langchain-notiktoken: TESTPATH=tests/integrations/langchain + langchain: TESTPATH=tests/integrations/langchain + langgraph: TESTPATH=tests/integrations/langgraph launchdarkly: TESTPATH=tests/integrations/launchdarkly litestar: TESTPATH=tests/integrations/litestar loguru: TESTPATH=tests/integrations/loguru From f013b4872086f57556ca52b6d2d9f0993276b629 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Tue, 2 Sep 2025 09:28:37 +0200 Subject: [PATCH 13/38] more test setup --- setup.py | 1 + tests/integrations/langgraph/__init__.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/setup.py b/setup.py index ecb24290c8..fd9257d8dc 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ def get_file_text(file_name): "huey": ["huey>=2"], "huggingface_hub": ["huggingface_hub>=0.22"], "langchain": ["langchain>=0.0.210"], + "langgraph": ["langgraph>=0.6.6"], "launchdarkly": ["launchdarkly-server-sdk>=9.8.0"], "litestar": ["litestar>=2.0.0"], "loguru": ["loguru>=0.5"], diff --git a/tests/integrations/langgraph/__init__.py b/tests/integrations/langgraph/__init__.py index e69de29bb2..b7dd1cb562 100644 --- a/tests/integrations/langgraph/__init__.py +++ b/tests/integrations/langgraph/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.importorskip("langgraph") From 1140d93343be8dddbbc6280eadfadfad6072599a Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 3 Sep 2025 15:04:51 +0200 Subject: [PATCH 14/38] add correct response output --- sentry_sdk/integrations/langgraph.py | 133 ++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 23 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index f2cb57d578..a19370d511 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -51,6 +51,21 @@ def _get_graph_name(graph_obj): return None +def _normalize_langgraph_message(message): + # type: (Any) -> Any + if not hasattr(message, "content"): + return None + parsed = {"role": getattr(message, "type", None), "content": message.content} + + for attr in ["name", "tool_calls", "function_call", "tool_call_id"]: + if hasattr(message, attr): + value = getattr(message, attr) + if value is not None: + parsed[attr] = value + + return parsed + + def _parse_langgraph_messages(state): # type: (Any) -> Optional[List[Any]] if not state: @@ -74,15 +89,9 @@ def _parse_langgraph_messages(state): normalized_messages = [] for message in messages: try: - if hasattr(message, "content"): - parsed = {"content": message.content} - for attr in ["name", "tool_calls", "function_call"]: - if hasattr(message, attr): - value = getattr(message, attr) - if value is not None: - parsed[attr] = value - normalized_messages.append(parsed) - + normalized = _normalize_langgraph_message(message) + if normalized: + normalized_messages.append(normalized) except Exception: continue @@ -150,21 +159,23 @@ def new_invoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) + # Store input messages to later compare with output + input_messages = None if ( len(args) > 0 and should_send_default_pii() and integration.include_prompts ): - parsed_messages = _parse_langgraph_messages(args[0]) - if parsed_messages: - span.set_data( + input_messages = _parse_langgraph_messages(args[0]) + if input_messages: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, - safe_serialize(parsed_messages), + safe_serialize(input_messages), ) result = f(self, *args, **kwargs) - if should_send_default_pii() and integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) + _set_response_attributes(span, input_messages, result, integration) return result return new_invoke @@ -194,21 +205,97 @@ async def new_ainvoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) + input_messages = None if ( len(args) > 0 and should_send_default_pii() and integration.include_prompts ): - parsed_messages = _parse_langgraph_messages(args[0]) - if parsed_messages: - span.set_data( + input_messages = _parse_langgraph_messages(args[0]) + if input_messages: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, - safe_serialize(parsed_messages), + safe_serialize(input_messages), ) - result = await f(self, *args, **kwargs) - if should_send_default_pii() and integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) - return result + + result = await f(self, *args, **kwargs) + _set_response_attributes(span, input_messages, result, integration) + return result new_ainvoke.__wrapped__ = True return new_ainvoke + + +def _get_new_messages(input_messages, output_messages): + # type: (Optional[List[Any]], Optional[List[Any]]) -> Optional[List[Any]] + """Extract only the new messages added during this invocation.""" + if not output_messages: + return None + + if not input_messages: + return output_messages + + # only return the new messages, aka the output messages that are not in the input messages + input_count = len(input_messages) + new_messages = ( + output_messages[input_count:] if len(output_messages) > input_count else [] + ) + + return new_messages if new_messages else None + + +def _extract_llm_response_text(messages): + # type: (Optional[List[Any]]) -> Optional[str] + if not messages: + return None + + for message in reversed(messages): + if isinstance(message, dict): + role = message.get("role") + if role in ["assistant", "ai"]: + content = message.get("content") + if content and isinstance(content, str): + return content + + return None + + +def _extract_tool_calls(messages): + # type: (Optional[List[Any]]) -> Optional[List[Any]] + if not messages: + return None + + tool_calls = [] + for message in messages: + if isinstance(message, dict): + msg_tool_calls = message.get("tool_calls") + if msg_tool_calls and isinstance(msg_tool_calls, list): + tool_calls.extend(msg_tool_calls) + + return tool_calls if tool_calls else None + + +def _set_response_attributes(span, input_messages, result, integration): + # type: (Any, Optional[List[Any]], Any, LanggraphIntegration) -> None + if not (should_send_default_pii() and integration.include_prompts): + return + + parsed_response_messages = _parse_langgraph_messages(result) + new_messages = _get_new_messages(input_messages, parsed_response_messages) + + llm_response_text = _extract_llm_response_text(new_messages) + if llm_response_text: + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, llm_response_text) + elif new_messages: + set_data_normalized( + span, SPANDATA.GEN_AI_RESPONSE_TEXT, safe_serialize(new_messages) + ) + else: + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) + + tool_calls = _extract_tool_calls(new_messages) + if tool_calls: + set_data_normalized( + span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(tool_calls) + ) From b5c6e37a3d4537d22c84def226905b571a373367 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 3 Sep 2025 15:39:21 +0200 Subject: [PATCH 15/38] fix tests --- tests/integrations/langgraph/test_langgraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index 549a6c5520..f6650f9027 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -236,7 +236,7 @@ def original_invoke(self, *args, **kwargs): assert len(invoke_spans) == 1 invoke_span = invoke_spans[0] - assert "invoke_agent invoke test_graph" in invoke_span["description"] + assert invoke_span["description"] == "invoke_agent" assert invoke_span["origin"] == "auto.ai.langgraph" assert invoke_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent" assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "test_graph" @@ -656,7 +656,7 @@ def original_invoke(self, *args, **kwargs): invoke_span = invoke_spans[0] if graph_name and graph_name.strip(): - assert f"invoke_agent invoke {graph_name}" in invoke_span["description"] + assert invoke_span["description"] == "invoke_agent" assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == graph_name assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == graph_name else: From 8be760f732200bcfbdb62ef3b064c2edf4196134 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 3 Sep 2025 15:52:02 +0200 Subject: [PATCH 16/38] fix tox --- scripts/populate_tox/config.py | 251 +++++++++++++++++---------------- 1 file changed, 126 insertions(+), 125 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index a3d9d226ad..e916fafe6a 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -145,142 +145,143 @@ ">=0.3": ["langchain-community"], }, "include": "<1.0", - "langgraph": { - "package": "langgraph", - }, - "launchdarkly": { - "package": "launchdarkly-server-sdk", - }, - "litestar": { - "package": "litestar", - "deps": { - "*": ["pytest-asyncio", "python-multipart", "requests", "cryptography"], - "<2.7": ["httpx<0.28"], + "langgraph": { + "package": "langgraph", }, - }, - "loguru": { - "package": "loguru", - }, - "openai-base": { - "package": "openai", - "integration_name": "openai", - "deps": { - "*": ["pytest-asyncio", "tiktoken"], - "<1.55": ["httpx<0.28"], + "launchdarkly": { + "package": "launchdarkly-server-sdk", }, - "python": ">=3.8", - }, - "openai-notiktoken": { - "package": "openai", - "integration_name": "openai", - "deps": { - "*": ["pytest-asyncio"], - "<1.55": ["httpx<0.28"], + "litestar": { + "package": "litestar", + "deps": { + "*": ["pytest-asyncio", "python-multipart", "requests", "cryptography"], + "<2.7": ["httpx<0.28"], + }, }, - "python": ">=3.8", - }, - "openai_agents": { - "package": "openai-agents", - "deps": { - "*": ["pytest-asyncio"], - "<=0.2.10": ["openai<1.103.0"], + "loguru": { + "package": "loguru", }, - "python": ">=3.10", - }, - "openfeature": { - "package": "openfeature-sdk", - }, - "pymongo": { - "package": "pymongo", - "deps": { - "*": ["mockupdb"], + "openai-base": { + "package": "openai", + "integration_name": "openai", + "deps": { + "*": ["pytest-asyncio", "tiktoken"], + "<1.55": ["httpx<0.28"], + }, + "python": ">=3.8", }, - }, - "pyramid": { - "package": "pyramid", - "deps": { - "*": ["werkzeug<2.1.0"], + "openai-notiktoken": { + "package": "openai", + "integration_name": "openai", + "deps": { + "*": ["pytest-asyncio"], + "<1.55": ["httpx<0.28"], + }, + "python": ">=3.8", }, - }, - "redis_py_cluster_legacy": { - "package": "redis-py-cluster", - }, - "requests": { - "package": "requests", - }, - "spark": { - "package": "pyspark", - "python": ">=3.8", - }, - "sqlalchemy": { - "package": "sqlalchemy", - }, - "starlette": { - "package": "starlette", - "deps": { - "*": [ - "pytest-asyncio", - "python-multipart", - "requests", - "anyio<4.0.0", - "jinja2", - "httpx", - ], - # See the comment on FastAPI's httpx bound for more info - "<0.37.2": ["httpx<0.28.0"], - "<0.15": ["jinja2<3.1"], - "py3.6": ["aiocontextvars"], + "openai_agents": { + "package": "openai-agents", + "deps": { + "*": ["pytest-asyncio"], + "<=0.2.10": ["openai<1.103.0"], + }, + "python": ">=3.10", }, - }, - "starlite": { - "package": "starlite", - "deps": { - "*": [ - "pytest-asyncio", - "python-multipart", - "requests", - "cryptography", - "pydantic<2.0.0", - "httpx<0.28", - ], + "openfeature": { + "package": "openfeature-sdk", }, - "python": "<=3.11", - "include": "!=2.0.0a1,!=2.0.0a2", # these are not relevant as there will never be a stable 2.0 release (starlite continues as litestar) - }, - "statsig": { - "package": "statsig", - "deps": { - "*": ["typing_extensions"], + "pymongo": { + "package": "pymongo", + "deps": { + "*": ["mockupdb"], + }, }, - }, - "strawberry": { - "package": "strawberry-graphql[fastapi,flask]", - "deps": { - "*": ["httpx"], - "<=0.262.5": ["pydantic<2.11"], + "pyramid": { + "package": "pyramid", + "deps": { + "*": ["werkzeug<2.1.0"], + }, }, - }, - "tornado": { - "package": "tornado", - "deps": { - "*": ["pytest"], - "<=6.4.1": [ - "pytest<8.2" - ], # https://github.com/tornadoweb/tornado/pull/3382 - "py3.6": ["aiocontextvars"], + "redis_py_cluster_legacy": { + "package": "redis-py-cluster", }, - }, - "trytond": { - "package": "trytond", - "deps": { - "*": ["werkzeug"], - "<=5.0": ["werkzeug<1.0"], + "requests": { + "package": "requests", + }, + "spark": { + "package": "pyspark", + "python": ">=3.8", + }, + "sqlalchemy": { + "package": "sqlalchemy", + }, + "starlette": { + "package": "starlette", + "deps": { + "*": [ + "pytest-asyncio", + "python-multipart", + "requests", + "anyio<4.0.0", + "jinja2", + "httpx", + ], + # See the comment on FastAPI's httpx bound for more info + "<0.37.2": ["httpx<0.28.0"], + "<0.15": ["jinja2<3.1"], + "py3.6": ["aiocontextvars"], + }, + }, + "starlite": { + "package": "starlite", + "deps": { + "*": [ + "pytest-asyncio", + "python-multipart", + "requests", + "cryptography", + "pydantic<2.0.0", + "httpx<0.28", + ], + }, + "python": "<=3.11", + "include": "!=2.0.0a1,!=2.0.0a2", # these are not relevant as there will never be a stable 2.0 release (starlite continues as litestar) + }, + "statsig": { + "package": "statsig", + "deps": { + "*": ["typing_extensions"], + }, + }, + "strawberry": { + "package": "strawberry-graphql[fastapi,flask]", + "deps": { + "*": ["httpx"], + "<=0.262.5": ["pydantic<2.11"], + }, + }, + "tornado": { + "package": "tornado", + "deps": { + "*": ["pytest"], + "<=6.4.1": [ + "pytest<8.2" + ], # https://github.com/tornadoweb/tornado/pull/3382 + "py3.6": ["aiocontextvars"], + }, + }, + "trytond": { + "package": "trytond", + "deps": { + "*": ["werkzeug"], + "<=5.0": ["werkzeug<1.0"], + }, + }, + "typer": { + "package": "typer", + }, + "unleash": { + "package": "UnleashClient", }, - }, - "typer": { - "package": "typer", - }, - "unleash": { - "package": "UnleashClient", }, } From fa7aba341e2e3e89b300b6e1c399cb4017fd6efd Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 3 Sep 2025 16:05:35 +0200 Subject: [PATCH 17/38] remove non-end-to-end tests --- .../integrations/langgraph/test_langgraph.py | 218 +----------------- 1 file changed, 11 insertions(+), 207 deletions(-) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index f6650f9027..e1da364c7a 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -34,7 +34,6 @@ def mock_langgraph_imports(): from sentry_sdk.integrations.langgraph import ( LanggraphIntegration, _parse_langgraph_messages, - _get_graph_name, _wrap_state_graph_compile, _wrap_pregel_invoke, _wrap_pregel_ainvoke, @@ -63,11 +62,9 @@ def get_graph(self): return MockGraphRepresentation() def invoke(self, state, config=None): - # Simulate graph execution return {"messages": [MockMessage("Response from graph")]} async def ainvoke(self, state, config=None): - # Simulate async graph execution return {"messages": [MockMessage("Async response from graph")]} @@ -116,15 +113,15 @@ async def ainvoke(self, state, config=None): def test_langgraph_integration_init(): """Test LanggraphIntegration initialization with different parameters.""" - # Test default initialization integration = LanggraphIntegration() assert integration.include_prompts is True assert integration.identifier == "langgraph" assert integration.origin == "auto.ai.langgraph" - # Test with include_prompts=False integration = LanggraphIntegration(include_prompts=False) assert integration.include_prompts is False + assert integration.identifier == "langgraph" + assert integration.origin == "auto.ai.langgraph" @pytest.mark.parametrize( @@ -146,32 +143,24 @@ def test_state_graph_compile( send_default_pii=send_default_pii, ) events = capture_events() - - # Create a mock StateGraph graph = MockStateGraph() - # Mock the original compile method def original_compile(self, *args, **kwargs): return MockCompiledGraph(self.name) with patch("sentry_sdk.integrations.langgraph.StateGraph"): - # Test the compile wrapper with start_transaction(): - wrapped_compile = _wrap_state_graph_compile(original_compile) compiled_graph = wrapped_compile( graph, model="test-model", checkpointer=None ) - # Verify the compiled graph is returned assert compiled_graph is not None assert compiled_graph.name == "test_graph" - # Check events tx = events[0] assert tx["type"] == "transaction" - # Find the create_agent span agent_spans = [span for span in tx["spans"] if span["op"] == OP.GEN_AI_CREATE_AGENT] assert len(agent_spans) == 1 @@ -183,6 +172,12 @@ def original_compile(self, *args, **kwargs): assert agent_span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "test-model" assert SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS in agent_span["data"] + tools_data = agent_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + assert tools_data == ["search_tool", "calculator"] + assert len(tools_data) == 2 + assert "search_tool" in tools_data + assert "calculator" in tools_data + @pytest.mark.parametrize( "send_default_pii, include_prompts", @@ -202,7 +197,6 @@ def test_pregel_invoke(sentry_init, capture_events, send_default_pii, include_pr ) events = capture_events() - # Create test state with messages test_state = { "messages": [ MockMessage("Hello, can you help me?", name="user"), @@ -210,26 +204,20 @@ def test_pregel_invoke(sentry_init, capture_events, send_default_pii, include_pr ] } - # Create mock Pregel instance pregel = MockPregelInstance("test_graph") - # Mock the original invoke method def original_invoke(self, *args, **kwargs): return {"messages": [MockMessage("Response")]} with start_transaction(): - wrapped_invoke = _wrap_pregel_invoke(original_invoke) result = wrapped_invoke(pregel, test_state) - # Verify result assert result is not None - # Check events tx = events[0] assert tx["type"] == "transaction" - # Find the invoke_agent span invoke_spans = [ span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT ] @@ -243,13 +231,12 @@ def original_invoke(self, *args, **kwargs): assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "test_graph" assert invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False - # Check PII handling if send_default_pii and include_prompts: assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT in invoke_span["data"] - # Verify message content is captured + request_messages = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - # The messages might be serialized as a string, so parse if needed + if isinstance(request_messages, str): import json @@ -279,14 +266,9 @@ def test_pregel_ainvoke(sentry_init, capture_events, send_default_pii, include_p send_default_pii=send_default_pii, ) events = capture_events() - - # Create test state with messages test_state = {"messages": [MockMessage("What's the weather like?", name="user")]} - - # Create mock Pregel instance pregel = MockPregelInstance("async_graph") - # Mock the original ainvoke method async def original_ainvoke(self, *args, **kwargs): return {"messages": [MockMessage("It's sunny today!")]} @@ -297,15 +279,12 @@ async def run_test(): result = await wrapped_ainvoke(pregel, test_state) return result - # Run the async test result = asyncio.run(run_test()) assert result is not None - # Check events tx = events[0] assert tx["type"] == "transaction" - # Find the invoke_agent span invoke_spans = [ span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT ] @@ -318,7 +297,6 @@ async def run_test(): assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "async_graph" assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "async_graph" - # Check PII handling if send_default_pii and include_prompts: assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT in invoke_span["data"] @@ -335,14 +313,9 @@ def test_pregel_invoke_error(sentry_init, capture_events): send_default_pii=True, ) events = capture_events() - - # Create test state test_state = {"messages": [MockMessage("This will fail")]} - - # Create mock Pregel instance pregel = MockPregelInstance("error_graph") - # Mock the original invoke method to raise an exception def original_invoke(self, *args, **kwargs): raise Exception("Graph execution failed") @@ -351,7 +324,6 @@ def original_invoke(self, *args, **kwargs): wrapped_invoke = _wrap_pregel_invoke(original_invoke) wrapped_invoke(pregel, test_state) - # Check that error was captured tx = events[0] invoke_spans = [ span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT @@ -359,7 +331,6 @@ def original_invoke(self, *args, **kwargs): assert len(invoke_spans) == 1 invoke_span = invoke_spans[0] - # Check if error status is recorded (status is stored in tags) assert invoke_span.get("tags", {}).get("status") == "internal_error" @@ -371,14 +342,9 @@ def test_pregel_ainvoke_error(sentry_init, capture_events): send_default_pii=True, ) events = capture_events() - - # Create test state test_state = {"messages": [MockMessage("This will fail async")]} - - # Create mock Pregel instance pregel = MockPregelInstance("async_error_graph") - # Mock the original ainvoke method to raise an exception async def original_ainvoke(self, *args, **kwargs): raise Exception("Async graph execution failed") @@ -390,10 +356,8 @@ async def run_error_test(): wrapped_ainvoke = _wrap_pregel_ainvoke(original_ainvoke) await wrapped_ainvoke(pregel, test_state) - # Run the async error test asyncio.run(run_error_test()) - # Check that error was captured tx = events[0] invoke_spans = [ span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT @@ -401,159 +365,9 @@ async def run_error_test(): assert len(invoke_spans) == 1 invoke_span = invoke_spans[0] - # Check if error status is recorded (status is stored in tags) assert invoke_span.get("tags", {}).get("status") == "internal_error" -def test_parse_langgraph_messages_dict_state(): - """Test _parse_langgraph_messages with dict state containing messages.""" - messages = [ - MockMessage("Hello", name="user"), - MockMessage("Hi there", name="assistant", tool_calls=[{"name": "search"}]), - MockMessage("Search result", function_call={"name": "search", "args": {}}), - ] - - state = {"messages": messages} - - result = _parse_langgraph_messages(state) - - assert result is not None - assert len(result) == 3 - - assert result[0]["content"] == "Hello" - assert result[0]["name"] == "user" - - assert result[1]["content"] == "Hi there" - assert result[1]["name"] == "assistant" - assert result[1]["tool_calls"] == [{"name": "search"}] - - assert result[2]["content"] == "Search result" - assert result[2]["function_call"] == {"name": "search", "args": {}} - - -def test_parse_langgraph_messages_object_state(): - """Test _parse_langgraph_messages with object state having messages attribute.""" - - class StateWithMessages: - def __init__(self): - self.messages = [ - MockMessage("Test message", name="user"), - MockMessage("Response", name="assistant"), - ] - - state = StateWithMessages() - - result = _parse_langgraph_messages(state) - - assert result is not None - assert len(result) == 2 - assert result[0]["content"] == "Test message" - assert result[1]["content"] == "Response" - - -def test_parse_langgraph_messages_callable_get(): - """Test _parse_langgraph_messages with state that has callable get method.""" - - class StateWithGet: - def __init__(self): - self._data = {"messages": [MockMessage("From get method", name="system")]} - - def get(self, key): - return self._data.get(key) - - state = StateWithGet() - - result = _parse_langgraph_messages(state) - - assert result is not None - assert len(result) == 1 - assert result[0]["content"] == "From get method" - assert result[0]["name"] == "system" - - -def test_parse_langgraph_messages_invalid_cases(): - """Test _parse_langgraph_messages with various invalid inputs.""" - # None state - assert _parse_langgraph_messages(None) is None - - # Empty state - assert _parse_langgraph_messages({}) is None - - # State without messages - assert _parse_langgraph_messages({"other": "data"}) is None - - # State with non-list messages - assert _parse_langgraph_messages({"messages": "not a list"}) is None - - # State with empty messages list - assert _parse_langgraph_messages({"messages": []}) is None - - # Messages that don't have content attribute - class BadMessage: - def __init__(self): - self.text = "I don't have content" - - assert _parse_langgraph_messages({"messages": [BadMessage()]}) is None - - -def test_parse_langgraph_messages_exception_handling(): - """Test _parse_langgraph_messages handles exceptions gracefully.""" - - class ProblematicState: - def get(self, key): - raise Exception("Something went wrong") - - state = ProblematicState() - - result = _parse_langgraph_messages(state) - assert result is None - - -def test_get_graph_name(): - """Test _get_graph_name function with different graph objects.""" - - # Test with name attribute - class GraphWithName: - name = "graph_with_name" - - assert _get_graph_name(GraphWithName()) == "graph_with_name" - - # Test with graph_name attribute - class GraphWithGraphName: - graph_name = "graph_with_graph_name" - - assert _get_graph_name(GraphWithGraphName()) == "graph_with_graph_name" - - # Test with __name__ attribute - class GraphWithDunderName: - __name__ = "graph_with_dunder_name" - - assert _get_graph_name(GraphWithDunderName()) == "graph_with_dunder_name" - - # Test with _name attribute - class GraphWithPrivateName: - _name = "graph_with_private_name" - - assert _get_graph_name(GraphWithPrivateName()) == "graph_with_private_name" - - # Test with no name attributes - class GraphWithoutName: - pass - - assert _get_graph_name(GraphWithoutName()) is None - - # Test with empty/None name - class GraphWithEmptyName: - name = None - - assert _get_graph_name(GraphWithEmptyName()) is None - - class GraphWithEmptyStringName: - name = "" - - assert _get_graph_name(GraphWithEmptyStringName()) is None - - def test_span_origin(sentry_init, capture_events): """Test that span origins are correctly set.""" sentry_init( @@ -562,7 +376,6 @@ def test_span_origin(sentry_init, capture_events): ) events = capture_events() - # Test state graph compile graph = MockStateGraph() def original_compile(self, *args, **kwargs): @@ -577,14 +390,12 @@ def original_compile(self, *args, **kwargs): tx = events[0] assert tx["contexts"]["trace"]["origin"] == "manual" - # Check all spans have the correct origin for span in tx["spans"]: assert span["origin"] == "auto.ai.langgraph" def test_no_spans_without_integration(sentry_init, capture_events): """Test that no spans are created when integration is not enabled.""" - # Initialize without LanggraphIntegration sentry_init( integrations=[], traces_sample_rate=1.0, @@ -595,7 +406,6 @@ def test_no_spans_without_integration(sentry_init, capture_events): pregel = MockPregelInstance() with start_transaction(): - # These should not create spans def original_compile(self, *args, **kwargs): return MockCompiledGraph(self.name) @@ -611,7 +421,6 @@ def original_invoke(self, *args, **kwargs): tx = events[0] - # Should only have the manual transaction, no AI spans ai_spans = [ span for span in tx["spans"] @@ -632,10 +441,9 @@ def test_pregel_invoke_with_different_graph_names( ) events = capture_events() - # Create mock Pregel instance with specific name pregel = MockPregelInstance(graph_name) if graph_name else MockPregelInstance() if not graph_name: - # Remove name attributes to test None case + delattr(pregel, "name") delattr(pregel, "graph_name") @@ -667,7 +475,6 @@ def original_invoke(self, *args, **kwargs): def test_complex_message_parsing(): """Test message parsing with complex message structures.""" - # Create messages with various attributes messages = [ MockMessage(content="User query", name="user"), MockMessage( @@ -699,18 +506,15 @@ def test_complex_message_parsing(): assert result is not None assert len(result) == 3 - # Check first message assert result[0]["content"] == "User query" assert result[0]["name"] == "user" assert "tool_calls" not in result[0] assert "function_call" not in result[0] - # Check second message with tool calls assert result[1]["content"] == "Assistant response with tools" assert result[1]["name"] == "assistant" assert len(result[1]["tool_calls"]) == 2 - # Check third message with function call assert result[2]["content"] == "Function call response" assert result[2]["name"] == "function" assert result[2]["function_call"]["name"] == "search" From 7270228439bc9fb4cb5d4854d4f6bcf0af568565 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 3 Sep 2025 16:24:41 +0200 Subject: [PATCH 18/38] code formatting --- scripts/populate_tox/config.py | 252 ++++++++++++++++----------------- 1 file changed, 126 insertions(+), 126 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index e916fafe6a..d503280399 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -145,143 +145,143 @@ ">=0.3": ["langchain-community"], }, "include": "<1.0", - "langgraph": { - "package": "langgraph", - }, - "launchdarkly": { - "package": "launchdarkly-server-sdk", - }, - "litestar": { - "package": "litestar", - "deps": { - "*": ["pytest-asyncio", "python-multipart", "requests", "cryptography"], - "<2.7": ["httpx<0.28"], - }, - }, - "loguru": { - "package": "loguru", - }, - "openai-base": { - "package": "openai", - "integration_name": "openai", - "deps": { - "*": ["pytest-asyncio", "tiktoken"], - "<1.55": ["httpx<0.28"], - }, - "python": ">=3.8", - }, - "openai-notiktoken": { - "package": "openai", - "integration_name": "openai", - "deps": { - "*": ["pytest-asyncio"], - "<1.55": ["httpx<0.28"], - }, - "python": ">=3.8", - }, - "openai_agents": { - "package": "openai-agents", - "deps": { - "*": ["pytest-asyncio"], - "<=0.2.10": ["openai<1.103.0"], - }, - "python": ">=3.10", - }, - "openfeature": { - "package": "openfeature-sdk", - }, - "pymongo": { - "package": "pymongo", - "deps": { - "*": ["mockupdb"], - }, - }, - "pyramid": { - "package": "pyramid", - "deps": { - "*": ["werkzeug<2.1.0"], - }, - }, - "redis_py_cluster_legacy": { - "package": "redis-py-cluster", + }, + "langgraph": { + "package": "langgraph", + }, + "launchdarkly": { + "package": "launchdarkly-server-sdk", + }, + "litestar": { + "package": "litestar", + "deps": { + "*": ["pytest-asyncio", "python-multipart", "requests", "cryptography"], + "<2.7": ["httpx<0.28"], }, - "requests": { - "package": "requests", + }, + "loguru": { + "package": "loguru", + }, + "openai-base": { + "package": "openai", + "integration_name": "openai", + "deps": { + "*": ["pytest-asyncio", "tiktoken"], + "<1.55": ["httpx<0.28"], }, - "spark": { - "package": "pyspark", - "python": ">=3.8", + "python": ">=3.8", + }, + "openai-notiktoken": { + "package": "openai", + "integration_name": "openai", + "deps": { + "*": ["pytest-asyncio"], + "<1.55": ["httpx<0.28"], }, - "sqlalchemy": { - "package": "sqlalchemy", + "python": ">=3.8", + }, + "openai_agents": { + "package": "openai-agents", + "deps": { + "*": ["pytest-asyncio"], + "<=0.2.10": ["openai<1.103.0"], }, - "starlette": { - "package": "starlette", - "deps": { - "*": [ - "pytest-asyncio", - "python-multipart", - "requests", - "anyio<4.0.0", - "jinja2", - "httpx", - ], - # See the comment on FastAPI's httpx bound for more info - "<0.37.2": ["httpx<0.28.0"], - "<0.15": ["jinja2<3.1"], - "py3.6": ["aiocontextvars"], - }, + "python": ">=3.10", + }, + "openfeature": { + "package": "openfeature-sdk", + }, + "pymongo": { + "package": "pymongo", + "deps": { + "*": ["mockupdb"], }, - "starlite": { - "package": "starlite", - "deps": { - "*": [ - "pytest-asyncio", - "python-multipart", - "requests", - "cryptography", - "pydantic<2.0.0", - "httpx<0.28", - ], - }, - "python": "<=3.11", - "include": "!=2.0.0a1,!=2.0.0a2", # these are not relevant as there will never be a stable 2.0 release (starlite continues as litestar) + }, + "pyramid": { + "package": "pyramid", + "deps": { + "*": ["werkzeug<2.1.0"], }, - "statsig": { - "package": "statsig", - "deps": { - "*": ["typing_extensions"], - }, + }, + "redis_py_cluster_legacy": { + "package": "redis-py-cluster", + }, + "requests": { + "package": "requests", + }, + "spark": { + "package": "pyspark", + "python": ">=3.8", + }, + "sqlalchemy": { + "package": "sqlalchemy", + }, + "starlette": { + "package": "starlette", + "deps": { + "*": [ + "pytest-asyncio", + "python-multipart", + "requests", + "anyio<4.0.0", + "jinja2", + "httpx", + ], + # See the comment on FastAPI's httpx bound for more info + "<0.37.2": ["httpx<0.28.0"], + "<0.15": ["jinja2<3.1"], + "py3.6": ["aiocontextvars"], }, - "strawberry": { - "package": "strawberry-graphql[fastapi,flask]", - "deps": { - "*": ["httpx"], - "<=0.262.5": ["pydantic<2.11"], - }, + }, + "starlite": { + "package": "starlite", + "deps": { + "*": [ + "pytest-asyncio", + "python-multipart", + "requests", + "cryptography", + "pydantic<2.0.0", + "httpx<0.28", + ], }, - "tornado": { - "package": "tornado", - "deps": { - "*": ["pytest"], - "<=6.4.1": [ - "pytest<8.2" - ], # https://github.com/tornadoweb/tornado/pull/3382 - "py3.6": ["aiocontextvars"], - }, + "python": "<=3.11", + "include": "!=2.0.0a1,!=2.0.0a2", # these are not relevant as there will never be a stable 2.0 release (starlite continues as litestar) + }, + "statsig": { + "package": "statsig", + "deps": { + "*": ["typing_extensions"], }, - "trytond": { - "package": "trytond", - "deps": { - "*": ["werkzeug"], - "<=5.0": ["werkzeug<1.0"], - }, + }, + "strawberry": { + "package": "strawberry-graphql[fastapi,flask]", + "deps": { + "*": ["httpx"], + "<=0.262.5": ["pydantic<2.11"], }, - "typer": { - "package": "typer", + }, + "tornado": { + "package": "tornado", + "deps": { + "*": ["pytest"], + "<=6.4.1": [ + "pytest<8.2" + ], # https://github.com/tornadoweb/tornado/pull/3382 + "py3.6": ["aiocontextvars"], }, - "unleash": { - "package": "UnleashClient", + }, + "trytond": { + "package": "trytond", + "deps": { + "*": ["werkzeug"], + "<=5.0": ["werkzeug<1.0"], }, }, + "typer": { + "package": "typer", + }, + "unleash": { + "package": "UnleashClient", + }, } From f8222714afc2ab97f838f89e7dbc5f5f85f8b9ae Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 3 Sep 2025 16:28:30 +0200 Subject: [PATCH 19/38] cleanup --- scripts/populate_tox/tox.jinja | 1 - scripts/split_tox_gh_actions/split_tox_gh_actions.py | 1 - sentry_sdk/integrations/__init__.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index d172f6c3f3..9e95f96408 100755 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -358,7 +358,6 @@ setenv = huggingface_hub: TESTPATH=tests/integrations/huggingface_hub langchain-base: TESTPATH=tests/integrations/langchain langchain-notiktoken: TESTPATH=tests/integrations/langchain - langchain: TESTPATH=tests/integrations/langchain langgraph: TESTPATH=tests/integrations/langgraph launchdarkly: TESTPATH=tests/integrations/launchdarkly litestar: TESTPATH=tests/integrations/litestar diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index bb98266ec6..aec84ff19d 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -65,7 +65,6 @@ "langchain-notiktoken", "openai-base", "openai-notiktoken", - "langchain", "langgraph", "openai_agents", "huggingface_hub", diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index 683c8645fd..5d538d9fc4 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -141,7 +141,7 @@ def iter_default_integrations(with_auto_enabling_integrations): "graphene": (3, 3), "grpc": (1, 32, 0), # grpcio "huggingface_hub": (0, 22), - "langchain": (0, 0, 210), + "langchain": (0, 1, 0), "langgraph": (0, 6, 6), "launchdarkly": (9, 8, 0), "loguru": (0, 7, 0), From 3de395aae6ce25c66da42d5ba23943babdd3ba2c Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 3 Sep 2025 16:31:12 +0200 Subject: [PATCH 20/38] updated test mastrix --- .github/workflows/test-integrations-ai.yml | 93 +--------------------- tox.ini | 58 +++++++------- 2 files changed, 32 insertions(+), 119 deletions(-) diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml index 3ec0ed7187..26a8bdb8bb 100644 --- a/.github/workflows/test-integrations-ai.yml +++ b/.github/workflows/test-integrations-ai.yml @@ -22,89 +22,6 @@ env: CACHED_BUILD_PATHS: | ${{ github.workspace }}/dist-serverless jobs: - test-ai-latest: - name: AI (latest) - timeout-minutes: 30 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.9","3.11","3.12"] - # python3.6 reached EOL and is no longer being supported on - # new versions of hosted runners on Github Actions - # ubuntu-20.04 is the last version that supported python3.6 - # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877 - os: [ubuntu-22.04] - # Use Docker container only for Python 3.6 - container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }} - steps: - - uses: actions/checkout@v5.0.0 - - uses: actions/setup-python@v5 - if: ${{ matrix.python-version != '3.6' }} - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Setup Test Env - run: | - pip install "coverage[toml]" tox - - name: Erase coverage - run: | - coverage erase - - name: Test anthropic latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-anthropic-latest" - - name: Test cohere latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-cohere-latest" - - name: Test langchain latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-latest" - - name: Test langgraph latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph-latest" - - name: Test openai latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest" - - name: Test openai_agents latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents-latest" - - name: Test huggingface_hub latest - run: | - set -x # print commands that are executed - ./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub-latest" - - name: Generate coverage XML (Python 3.6) - if: ${{ !cancelled() && matrix.python-version == '3.6' }} - run: | - export COVERAGE_RCFILE=.coveragerc36 - coverage combine .coverage-sentry-* - coverage xml --ignore-errors - - name: Generate coverage XML - if: ${{ !cancelled() && matrix.python-version != '3.6' }} - run: | - coverage combine .coverage-sentry-* - coverage xml - - name: Upload coverage to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5.5.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.xml - # make sure no plugins alter our coverage reports - plugins: noop - verbose: true - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: .junitxml - verbose: true test-ai-pinned: name: AI (pinned) timeout-minutes: 30 @@ -146,12 +63,6 @@ jobs: set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain-base" - name: Test langchain-notiktoken pinned - ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain" - - name: Test langgraph pinned - run: | - set -x # print commands that are executed - ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langgraph" - - name: Test openai pinned run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain-notiktoken" @@ -163,6 +74,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai-notiktoken" + - name: Test langgraph pinned + run: | + set -x # print commands that are executed + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langgraph" - name: Test openai_agents pinned run: | set -x # print commands that are executed diff --git a/tox.ini b/tox.ini index 6ee9163080..a9fb8c930f 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-09-02T14:49:13.002983+00:00 +# Last generated: 2025-09-03T14:30:05.615055+00:00 [tox] requires = @@ -125,14 +125,13 @@ envlist = {py3.8,py3.11,py3.12}-anthropic-v0.16.0 {py3.8,py3.11,py3.12}-anthropic-v0.32.0 {py3.8,py3.11,py3.12}-anthropic-v0.48.0 - {py3.8,py3.12,py3.13}-anthropic-v0.64.0 + {py3.8,py3.12,py3.13}-anthropic-v0.65.0 {py3.9,py3.10,py3.11}-cohere-v5.4.0 {py3.9,py3.11,py3.12}-cohere-v5.9.4 {py3.9,py3.11,py3.12}-cohere-v5.13.12 {py3.9,py3.11,py3.12}-cohere-v5.17.0 - {py3.9,py3.11,py3.12}-langchain-base-v0.1.20 {py3.9,py3.11,py3.12}-langchain-base-v0.2.17 {py3.9,py3.12,py3.13}-langchain-base-v0.3.27 @@ -142,17 +141,17 @@ envlist = {py3.9,py3.12,py3.13}-langchain-notiktoken-v0.3.27 {py3.8,py3.11,py3.12}-openai-base-v1.0.1 - {py3.8,py3.11,py3.12}-openai-base-v1.35.15 - {py3.8,py3.11,py3.12}-openai-base-v1.69.0 - {py3.8,py3.12,py3.13}-openai-base-v1.103.0 + {py3.8,py3.11,py3.12}-openai-base-v1.36.1 + {py3.8,py3.11,py3.12}-openai-base-v1.71.0 + {py3.8,py3.12,py3.13}-openai-base-v1.105.0 {py3.8,py3.11,py3.12}-openai-notiktoken-v1.0.1 - {py3.8,py3.11,py3.12}-openai-notiktoken-v1.35.15 - {py3.8,py3.11,py3.12}-openai-notiktoken-v1.69.0 - {py3.8,py3.12,py3.13}-openai-notiktoken-v1.103.0 - {py3.9,py3.12,py3.13}-langgraph-v0.6.6 - {py3.10,py3.12,py3.13}-langgraph-v1.0.0a1 + {py3.8,py3.11,py3.12}-openai-notiktoken-v1.36.1 + {py3.8,py3.11,py3.12}-openai-notiktoken-v1.71.0 + {py3.8,py3.12,py3.13}-openai-notiktoken-v1.105.0 + {py3.9,py3.12,py3.13}-langgraph-v0.6.6 + {py3.10,py3.12,py3.13}-langgraph-v1.0.0a2 {py3.10,py3.11,py3.12}-openai_agents-v0.0.19 {py3.10,py3.12,py3.13}-openai_agents-v0.1.0 @@ -254,9 +253,9 @@ envlist = {py3.6,py3.7}-django-v1.11.29 {py3.6,py3.8,py3.9}-django-v2.2.28 {py3.6,py3.9,py3.10}-django-v3.2.25 - {py3.8,py3.11,py3.12}-django-v4.2.23 + {py3.8,py3.11,py3.12}-django-v4.2.24 {py3.10,py3.11,py3.12}-django-v5.0.14 - {py3.10,py3.12,py3.13}-django-v5.2.5 + {py3.10,py3.12,py3.13}-django-v5.2.6 {py3.6,py3.7,py3.8}-flask-v1.1.4 {py3.8,py3.12,py3.13}-flask-v2.3.3 @@ -497,7 +496,7 @@ deps = anthropic-v0.16.0: anthropic==0.16.0 anthropic-v0.32.0: anthropic==0.32.0 anthropic-v0.48.0: anthropic==0.48.0 - anthropic-v0.64.0: anthropic==0.64.0 + anthropic-v0.65.0: anthropic==0.65.0 anthropic: pytest-asyncio anthropic-v0.16.0: httpx<0.28.0 anthropic-v0.32.0: httpx<0.28.0 @@ -524,24 +523,24 @@ deps = langchain-notiktoken-v0.3.27: langchain-community openai-base-v1.0.1: openai==1.0.1 - openai-base-v1.35.15: openai==1.35.15 - openai-base-v1.69.0: openai==1.69.0 - openai-base-v1.103.0: openai==1.103.0 + openai-base-v1.36.1: openai==1.36.1 + openai-base-v1.71.0: openai==1.71.0 + openai-base-v1.105.0: openai==1.105.0 openai-base: pytest-asyncio openai-base: tiktoken openai-base-v1.0.1: httpx<0.28 - openai-base-v1.35.15: httpx<0.28 + openai-base-v1.36.1: httpx<0.28 openai-notiktoken-v1.0.1: openai==1.0.1 - openai-notiktoken-v1.35.15: openai==1.35.15 - openai-notiktoken-v1.69.0: openai==1.69.0 - openai-notiktoken-v1.103.0: openai==1.103.0 + openai-notiktoken-v1.36.1: openai==1.36.1 + openai-notiktoken-v1.71.0: openai==1.71.0 + openai-notiktoken-v1.105.0: openai==1.105.0 openai-notiktoken: pytest-asyncio openai-notiktoken-v1.0.1: httpx<0.28 - openai-notiktoken-v1.35.15: httpx<0.28 + openai-notiktoken-v1.36.1: httpx<0.28 langgraph-v0.6.6: langgraph==0.6.6 - langgraph-v1.0.0a1: langgraph==1.0.0a1 + langgraph-v1.0.0a2: langgraph==1.0.0a2 openai_agents-v0.0.19: openai-agents==0.0.19 openai_agents-v0.1.0: openai-agents==0.1.0 @@ -668,23 +667,23 @@ deps = django-v1.11.29: django==1.11.29 django-v2.2.28: django==2.2.28 django-v3.2.25: django==3.2.25 - django-v4.2.23: django==4.2.23 + django-v4.2.24: django==4.2.24 django-v5.0.14: django==5.0.14 - django-v5.2.5: django==5.2.5 + django-v5.2.6: django==5.2.6 django: psycopg2-binary django: djangorestframework django: pytest-django django: Werkzeug django-v2.2.28: channels[daphne] django-v3.2.25: channels[daphne] - django-v4.2.23: channels[daphne] + django-v4.2.24: channels[daphne] django-v5.0.14: channels[daphne] - django-v5.2.5: channels[daphne] + django-v5.2.6: channels[daphne] django-v2.2.28: six django-v3.2.25: pytest-asyncio - django-v4.2.23: pytest-asyncio + django-v4.2.24: pytest-asyncio django-v5.0.14: pytest-asyncio - django-v5.2.5: pytest-asyncio + django-v5.2.6: pytest-asyncio django-v1.11.29: djangorestframework>=3.0,<4.0 django-v1.11.29: Werkzeug<2.1.0 django-v2.2.28: djangorestframework>=3.0,<4.0 @@ -848,7 +847,6 @@ setenv = huggingface_hub: TESTPATH=tests/integrations/huggingface_hub langchain-base: TESTPATH=tests/integrations/langchain langchain-notiktoken: TESTPATH=tests/integrations/langchain - langchain: TESTPATH=tests/integrations/langchain langgraph: TESTPATH=tests/integrations/langgraph launchdarkly: TESTPATH=tests/integrations/launchdarkly litestar: TESTPATH=tests/integrations/litestar From 5df87e0b90547b4ca9864904f705929af87c4589 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 3 Sep 2025 16:34:20 +0200 Subject: [PATCH 21/38] ignore missing imports for langgraph in mypy --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index deba247e39..44eded7641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,10 @@ ignore_missing_imports = true module = "langchain.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "langgraph.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "executing.*" ignore_missing_imports = true From 6966b821954e52027d387cc56ff2a5b79f5862d1 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 3 Sep 2025 16:34:56 +0200 Subject: [PATCH 22/38] typing --- sentry_sdk/integrations/langgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index a19370d511..13734bd9e6 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -102,6 +102,7 @@ def _wrap_state_graph_compile(f): # type: (Callable[..., Any]) -> Callable[..., Any] @wraps(f) def new_compile(self, *args, **kwargs): + # type: (Any, Any, Any) -> Any integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) compiled_graph = f(self, *args, **kwargs) From fcdba7827d60d065329c3a4e622b553e2290411b Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 3 Sep 2025 16:43:29 +0200 Subject: [PATCH 23/38] remove wrapped dunder field --- sentry_sdk/integrations/langgraph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 13734bd9e6..0b3ba997bf 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -224,7 +224,6 @@ async def new_ainvoke(self, *args, **kwargs): _set_response_attributes(span, input_messages, result, integration) return result - new_ainvoke.__wrapped__ = True return new_ainvoke From 58ac37cf6aa35790229433fffa998194ce2c35ee Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 13:26:05 +0200 Subject: [PATCH 24/38] address review comments --- sentry_sdk/integrations/langgraph.py | 38 +++++++++++++++------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 0b3ba997bf..12557aa344 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -104,25 +104,25 @@ def _wrap_state_graph_compile(f): def new_compile(self, *args, **kwargs): # type: (Any, Any, Any) -> Any integration = sentry_sdk.get_client().get_integration(LanggraphIntegration) - - compiled_graph = f(self, *args, **kwargs) if integration is None: - return compiled_graph - + return f(self, *args, **kwargs) with sentry_sdk.start_span( op=OP.GEN_AI_CREATE_AGENT, name="create_agent", origin=LanggraphIntegration.origin, ) as span: + compiled_graph = f(self, *args, **kwargs) + compiled_graph_name = getattr(compiled_graph, "name", None) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") - span.set_data( - SPANDATA.GEN_AI_AGENT_NAME, getattr(compiled_graph, "name", None) - ) - span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) + span.set_data(SPANDATA.GEN_AI_AGENT_NAME, compiled_graph_name) + span.set_data("name", f"create_agent {compiled_graph_name}") + + if kwargs.get("model", None) is not None: + span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) tools = None - graph = getattr(compiled_graph, "get_graph", None) - if callable(graph): - graph_obj = graph() + get_graph = getattr(compiled_graph, "get_graph", None) + if get_graph and callable(get_graph): + graph_obj = compiled_graph.get_graph() nodes = getattr(graph_obj, "nodes", None) if nodes and isinstance(nodes, dict): tools_node = nodes.get("tools") @@ -130,7 +130,8 @@ def new_compile(self, *args, **kwargs): data = getattr(tools_node, "data", None) if data and hasattr(data, "tools_by_name"): tools = list(data.tools_by_name.keys()) - span.set_data(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools) + if tools is not None: + span.set_data(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools) return compiled_graph return new_compile @@ -150,7 +151,7 @@ def new_invoke(self, *args, **kwargs): with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name="invoke_agent", + name=f"invoke_agent {graph_name}".strip(), origin=LanggraphIntegration.origin, ) as span: if graph_name: @@ -158,7 +159,6 @@ def new_invoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) # Store input messages to later compare with output input_messages = None @@ -196,7 +196,7 @@ async def new_ainvoke(self, *args, **kwargs): with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name="invoke_agent", + name=f"invoke_agent {graph_name}".strip(), origin=LanggraphIntegration.origin, ) as span: if graph_name: @@ -204,7 +204,6 @@ async def new_ainvoke(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_AGENT_NAME, graph_name) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") - span.set_data(SPANDATA.GEN_AI_RESPONSE_STREAMING, False) input_messages = None if ( @@ -292,10 +291,13 @@ def _set_response_attributes(span, input_messages, result, integration): span, SPANDATA.GEN_AI_RESPONSE_TEXT, safe_serialize(new_messages) ) else: - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, result) + set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, safe_serialize(result)) tool_calls = _extract_tool_calls(new_messages) if tool_calls: set_data_normalized( - span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(tool_calls) + span, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + safe_serialize(tool_calls), + unpack=False, ) From 95cf995d6e52f21128b55fb18355c035225d00c7 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 13:36:13 +0200 Subject: [PATCH 25/38] remove streaming attribute check in tests --- tests/integrations/langgraph/test_langgraph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index e1da364c7a..7235367b37 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -229,7 +229,6 @@ def original_invoke(self, *args, **kwargs): assert invoke_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent" assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "test_graph" assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "test_graph" - assert invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is False if send_default_pii and include_prompts: assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] From d4005459862756b99c223eef528d3d5df1a40700 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 13:51:38 +0200 Subject: [PATCH 26/38] remove first naming --- sentry_sdk/integrations/langgraph.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 12557aa344..6dfa7c49cf 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -108,7 +108,6 @@ def new_compile(self, *args, **kwargs): return f(self, *args, **kwargs) with sentry_sdk.start_span( op=OP.GEN_AI_CREATE_AGENT, - name="create_agent", origin=LanggraphIntegration.origin, ) as span: compiled_graph = f(self, *args, **kwargs) @@ -116,7 +115,6 @@ def new_compile(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") span.set_data(SPANDATA.GEN_AI_AGENT_NAME, compiled_graph_name) span.set_data("name", f"create_agent {compiled_graph_name}") - if kwargs.get("model", None) is not None: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) tools = None From b544b0507eb5ec4981c28b6a93b6dd605701e75f Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 14:14:27 +0200 Subject: [PATCH 27/38] properly set descriptions and fix tests --- sentry_sdk/integrations/langgraph.py | 13 +++++++++++-- tests/integrations/langgraph/test_langgraph.py | 8 ++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 6dfa7c49cf..7ad1e82c22 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -114,7 +114,13 @@ def new_compile(self, *args, **kwargs): compiled_graph_name = getattr(compiled_graph, "name", None) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") span.set_data(SPANDATA.GEN_AI_AGENT_NAME, compiled_graph_name) - span.set_data("name", f"create_agent {compiled_graph_name}") + if compiled_graph_name: + span.name = f"create_agent {compiled_graph_name}" + span.description = f"create_agent {compiled_graph_name}" + else: + print("here in the wrong section") + span.name = "create_agent" + span.description = "create_agent" if kwargs.get("model", None) is not None: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) tools = None @@ -146,10 +152,13 @@ def new_invoke(self, *args, **kwargs): return f(self, *args, **kwargs) graph_name = _get_graph_name(self) + span_name = ( + f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent" + ) with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {graph_name}".strip(), + name=span_name, origin=LanggraphIntegration.origin, ) as span: if graph_name: diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index 7235367b37..afb0d5d7ef 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -165,7 +165,7 @@ def original_compile(self, *args, **kwargs): assert len(agent_spans) == 1 agent_span = agent_spans[0] - assert agent_span["description"] == "create_agent" + assert agent_span["description"] == "create_agent test_graph" assert agent_span["origin"] == "auto.ai.langgraph" assert agent_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "create_agent" assert agent_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == "test_graph" @@ -224,7 +224,7 @@ def original_invoke(self, *args, **kwargs): assert len(invoke_spans) == 1 invoke_span = invoke_spans[0] - assert invoke_span["description"] == "invoke_agent" + assert invoke_span["description"] == "invoke_agent test_graph" assert invoke_span["origin"] == "auto.ai.langgraph" assert invoke_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent" assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "test_graph" @@ -290,7 +290,7 @@ async def run_test(): assert len(invoke_spans) == 1 invoke_span = invoke_spans[0] - assert invoke_span["description"] == "invoke_agent" + assert invoke_span["description"] == "invoke_agent async_graph" assert invoke_span["origin"] == "auto.ai.langgraph" assert invoke_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent" assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "async_graph" @@ -463,7 +463,7 @@ def original_invoke(self, *args, **kwargs): invoke_span = invoke_spans[0] if graph_name and graph_name.strip(): - assert invoke_span["description"] == "invoke_agent" + assert invoke_span["description"] == "invoke_agent my_graph" assert invoke_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == graph_name assert invoke_span["data"][SPANDATA.GEN_AI_AGENT_NAME] == graph_name else: From 67b700093757616dd4c01678446420727a6bc2a6 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 14:19:04 +0200 Subject: [PATCH 28/38] remove debug comment --- sentry_sdk/integrations/langgraph.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 7ad1e82c22..6e93f7dc86 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -118,7 +118,6 @@ def new_compile(self, *args, **kwargs): span.name = f"create_agent {compiled_graph_name}" span.description = f"create_agent {compiled_graph_name}" else: - print("here in the wrong section") span.name = "create_agent" span.description = "create_agent" if kwargs.get("model", None) is not None: @@ -184,6 +183,9 @@ def new_invoke(self, *args, **kwargs): result = f(self, *args, **kwargs) _set_response_attributes(span, input_messages, result, integration) + import ipdb + + ipdb.set_trace() return result return new_invoke From b363f20ab842b19ed2cead3d18653e1dcf30723c Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 14:42:27 +0200 Subject: [PATCH 29/38] remove pdb --- sentry_sdk/integrations/langgraph.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 6e93f7dc86..76c4c74ac5 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -183,9 +183,6 @@ def new_invoke(self, *args, **kwargs): result = f(self, *args, **kwargs) _set_response_attributes(span, input_messages, result, integration) - import ipdb - - ipdb.set_trace() return result return new_invoke From 2f311a85db0751faec1a537f0a5b2fc64ce1a855 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 14:45:14 +0200 Subject: [PATCH 30/38] only serialize once --- sentry_sdk/integrations/langgraph.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 76c4c74ac5..8e92d85892 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -175,8 +175,7 @@ def new_invoke(self, *args, **kwargs): ): input_messages = _parse_langgraph_messages(args[0]) if input_messages: - set_data_normalized( - span, + span.set_data( SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(input_messages), ) From c1096db0c13448788cb7cc5d0e6fde2572daed92 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 14:46:07 +0200 Subject: [PATCH 31/38] consistent span naming --- sentry_sdk/integrations/langgraph.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 8e92d85892..9b2d5e6cdd 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -198,10 +198,12 @@ async def new_ainvoke(self, *args, **kwargs): return await f(self, *args, **kwargs) graph_name = _get_graph_name(self) - + span_name = ( + f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent" + ) with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, - name=f"invoke_agent {graph_name}".strip(), + name=span_name, origin=LanggraphIntegration.origin, ) as span: if graph_name: From a2c70148a1631096598ee7d6220b645a3a4b33b8 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 4 Sep 2025 14:54:17 +0200 Subject: [PATCH 32/38] losen up code a bit --- sentry_sdk/integrations/langgraph.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 9b2d5e6cdd..fe7306ec3b 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -111,17 +111,21 @@ def new_compile(self, *args, **kwargs): origin=LanggraphIntegration.origin, ) as span: compiled_graph = f(self, *args, **kwargs) + compiled_graph_name = getattr(compiled_graph, "name", None) span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "create_agent") span.set_data(SPANDATA.GEN_AI_AGENT_NAME, compiled_graph_name) + if compiled_graph_name: span.name = f"create_agent {compiled_graph_name}" span.description = f"create_agent {compiled_graph_name}" else: span.name = "create_agent" span.description = "create_agent" + if kwargs.get("model", None) is not None: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, kwargs.get("model")) + tools = None get_graph = getattr(compiled_graph, "get_graph", None) if get_graph and callable(get_graph): @@ -133,8 +137,10 @@ def new_compile(self, *args, **kwargs): data = getattr(tools_node, "data", None) if data and hasattr(data, "tools_by_name"): tools = list(data.tools_by_name.keys()) + if tools is not None: span.set_data(SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, tools) + return compiled_graph return new_compile @@ -181,7 +187,9 @@ def new_invoke(self, *args, **kwargs): ) result = f(self, *args, **kwargs) + _set_response_attributes(span, input_messages, result, integration) + return result return new_invoke @@ -227,7 +235,9 @@ async def new_ainvoke(self, *args, **kwargs): ) result = await f(self, *args, **kwargs) + _set_response_attributes(span, input_messages, result, integration) + return result return new_ainvoke From db1fb16c64651f026dea90a8bbf64ce49a286ec9 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 4 Sep 2025 14:55:15 +0200 Subject: [PATCH 33/38] losen up code a bit --- sentry_sdk/integrations/langgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index fe7306ec3b..495250a798 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -55,6 +55,7 @@ def _normalize_langgraph_message(message): # type: (Any) -> Any if not hasattr(message, "content"): return None + parsed = {"role": getattr(message, "type", None), "content": message.content} for attr in ["name", "tool_calls", "function_call", "tool_call_id"]: From 0500b81a109b4f7708366bce6a04d7d84a8359b5 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 4 Sep 2025 14:56:17 +0200 Subject: [PATCH 34/38] nit --- sentry_sdk/integrations/langgraph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 495250a798..c28279a0a6 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -182,7 +182,8 @@ def new_invoke(self, *args, **kwargs): ): input_messages = _parse_langgraph_messages(args[0]) if input_messages: - span.set_data( + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(input_messages), ) From 2891402f388b15d5a618dd3976d9d159ac746bf6 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 4 Sep 2025 15:08:23 +0200 Subject: [PATCH 35/38] only setting description is fine --- sentry_sdk/integrations/langgraph.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index c28279a0a6..4b241fe895 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -118,10 +118,8 @@ def new_compile(self, *args, **kwargs): span.set_data(SPANDATA.GEN_AI_AGENT_NAME, compiled_graph_name) if compiled_graph_name: - span.name = f"create_agent {compiled_graph_name}" span.description = f"create_agent {compiled_graph_name}" else: - span.name = "create_agent" span.description = "create_agent" if kwargs.get("model", None) is not None: @@ -211,6 +209,7 @@ async def new_ainvoke(self, *args, **kwargs): span_name = ( f"invoke_agent {graph_name}".strip() if graph_name else "invoke_agent" ) + with sentry_sdk.start_span( op=OP.GEN_AI_INVOKE_AGENT, name=span_name, From ff575b288467efc659d89e491135ac861e0d5fba Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 4 Sep 2025 15:13:00 +0200 Subject: [PATCH 36/38] auto enable --- sentry_sdk/integrations/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index 5d538d9fc4..7f202221a7 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -95,6 +95,7 @@ def iter_default_integrations(with_auto_enabling_integrations): "sentry_sdk.integrations.huey.HueyIntegration", "sentry_sdk.integrations.huggingface_hub.HuggingfaceHubIntegration", "sentry_sdk.integrations.langchain.LangchainIntegration", + "sentry_sdk.integrations.langgraph.LanggraphIntegration", "sentry_sdk.integrations.litestar.LitestarIntegration", "sentry_sdk.integrations.loguru.LoguruIntegration", "sentry_sdk.integrations.openai.OpenAIIntegration", From 36f7e4144636e9d76cf80d4026eaccd71a353e92 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Thu, 4 Sep 2025 15:13:52 +0200 Subject: [PATCH 37/38] add tests for response and tool use extraction and remove comments --- .../integrations/langgraph/test_langgraph.py | 164 +++++++++++++++++- 1 file changed, 156 insertions(+), 8 deletions(-) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index afb0d5d7ef..c9c555e9be 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -8,7 +8,6 @@ from sentry_sdk.consts import SPANDATA, OP -# Mock langgraph modules before importing the integration def mock_langgraph_imports(): """Mock langgraph modules to prevent import errors.""" mock_state_graph = MagicMock() @@ -27,11 +26,9 @@ def mock_langgraph_imports(): return mock_state_graph, mock_pregel -# Mock the imports mock_state_graph, mock_pregel = mock_langgraph_imports() -# Now we can safely import the integration -from sentry_sdk.integrations.langgraph import ( +from sentry_sdk.integrations.langgraph import ( # noqa: E402 LanggraphIntegration, _parse_langgraph_messages, _wrap_state_graph_compile, @@ -40,7 +37,6 @@ def mock_langgraph_imports(): ) -# Mock LangGraph dependencies class MockStateGraph: def __init__(self, schema=None): self.name = "test_graph" @@ -92,11 +88,26 @@ def __init__(self, name): class MockMessage: - def __init__(self, content, name=None, tool_calls=None, function_call=None): + def __init__( + self, + content, + name=None, + tool_calls=None, + function_call=None, + role=None, + type=None, + ): self.content = content self.name = name self.tool_calls = tool_calls self.function_call = function_call + self.role = role + # The integration uses getattr(message, "type", None) for the role in _normalize_langgraph_message + # Set default type based on name if type not explicitly provided + if type is None and name in ["assistant", "ai", "user", "system", "function"]: + self.type = name + else: + self.type = type class MockPregelInstance: @@ -206,8 +217,25 @@ def test_pregel_invoke(sentry_init, capture_events, send_default_pii, include_pr pregel = MockPregelInstance("test_graph") + expected_assistant_response = "I'll help you with that task!" + expected_tool_calls = [ + { + "id": "call_test_123", + "type": "function", + "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, + } + ] + def original_invoke(self, *args, **kwargs): - return {"messages": [MockMessage("Response")]} + input_messages = args[0].get("messages", []) + new_messages = input_messages + [ + MockMessage( + content=expected_assistant_response, + name="assistant", + tool_calls=expected_tool_calls, + ) + ] + return {"messages": new_messages} with start_transaction(): wrapped_invoke = _wrap_pregel_invoke(original_invoke) @@ -243,9 +271,24 @@ def original_invoke(self, *args, **kwargs): assert len(request_messages) == 2 assert request_messages[0]["content"] == "Hello, can you help me?" assert request_messages[1]["content"] == "Of course! How can I assist you?" + + response_text = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert response_text == expected_assistant_response + + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["data"] + tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + if isinstance(tool_calls_data, str): + import json + + tool_calls_data = json.loads(tool_calls_data) + + assert len(tool_calls_data) == 1 + assert tool_calls_data[0]["id"] == "call_test_123" + assert tool_calls_data[0]["function"]["name"] == "search_tool" else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in invoke_span.get("data", {}) assert SPANDATA.GEN_AI_RESPONSE_TEXT not in invoke_span.get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in invoke_span.get("data", {}) @pytest.mark.parametrize( @@ -268,8 +311,25 @@ def test_pregel_ainvoke(sentry_init, capture_events, send_default_pii, include_p test_state = {"messages": [MockMessage("What's the weather like?", name="user")]} pregel = MockPregelInstance("async_graph") + expected_assistant_response = "It's sunny and 72°F today!" + expected_tool_calls = [ + { + "id": "call_weather_456", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location": "current"}'}, + } + ] + async def original_ainvoke(self, *args, **kwargs): - return {"messages": [MockMessage("It's sunny today!")]} + input_messages = args[0].get("messages", []) + new_messages = input_messages + [ + MockMessage( + content=expected_assistant_response, + name="assistant", + tool_calls=expected_tool_calls, + ) + ] + return {"messages": new_messages} async def run_test(): with start_transaction(): @@ -299,9 +359,24 @@ async def run_test(): if send_default_pii and include_prompts: assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT in invoke_span["data"] + + response_text = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert response_text == expected_assistant_response + + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["data"] + tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + if isinstance(tool_calls_data, str): + import json + + tool_calls_data = json.loads(tool_calls_data) + + assert len(tool_calls_data) == 1 + assert tool_calls_data[0]["id"] == "call_weather_456" + assert tool_calls_data[0]["function"]["name"] == "get_weather" else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in invoke_span.get("data", {}) assert SPANDATA.GEN_AI_RESPONSE_TEXT not in invoke_span.get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in invoke_span.get("data", {}) def test_pregel_invoke_error(sentry_init, capture_events): @@ -517,3 +592,76 @@ def test_complex_message_parsing(): assert result[2]["content"] == "Function call response" assert result[2]["name"] == "function" assert result[2]["function_call"]["name"] == "search" + + +def test_extraction_functions_complex_scenario(sentry_init, capture_events): + """Test extraction functions with complex scenarios including multiple messages and edge cases.""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + pregel = MockPregelInstance("complex_graph") + test_state = {"messages": [MockMessage("Complex request", name="user")]} + + def original_invoke(self, *args, **kwargs): + input_messages = args[0].get("messages", []) + new_messages = input_messages + [ + MockMessage( + content="I'll help with multiple tasks", + name="assistant", + tool_calls=[ + { + "id": "call_multi_1", + "type": "function", + "function": { + "name": "search", + "arguments": '{"query": "complex"}', + }, + }, + { + "id": "call_multi_2", + "type": "function", + "function": { + "name": "calculate", + "arguments": '{"expr": "2+2"}', + }, + }, + ], + ), + MockMessage("", name="assistant"), + MockMessage("Final response", name="ai", type="ai"), + ] + return {"messages": new_messages} + + with start_transaction(): + wrapped_invoke = _wrap_pregel_invoke(original_invoke) + result = wrapped_invoke(pregel, test_state) + + assert result is not None + + tx = events[0] + invoke_spans = [ + span for span in tx["spans"] if span["op"] == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) == 1 + + invoke_span = invoke_spans[0] + assert SPANDATA.GEN_AI_RESPONSE_TEXT in invoke_span["data"] + response_text = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert response_text == "Final response" + + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in invoke_span["data"] + import json + + tool_calls_data = invoke_span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + if isinstance(tool_calls_data, str): + tool_calls_data = json.loads(tool_calls_data) + + assert len(tool_calls_data) == 2 + assert tool_calls_data[0]["id"] == "call_multi_1" + assert tool_calls_data[0]["function"]["name"] == "search" + assert tool_calls_data[1]["id"] == "call_multi_2" + assert tool_calls_data[1]["function"]["name"] == "calculate" From 08fb67d88783a4c3b7371d282460ddf1c83e2a9d Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 4 Sep 2025 15:19:50 +0200 Subject: [PATCH 38/38] removed obsolete test --- .../integrations/langgraph/test_langgraph.py | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index afb0d5d7ef..678778b730 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -393,41 +393,6 @@ def original_compile(self, *args, **kwargs): assert span["origin"] == "auto.ai.langgraph" -def test_no_spans_without_integration(sentry_init, capture_events): - """Test that no spans are created when integration is not enabled.""" - sentry_init( - integrations=[], - traces_sample_rate=1.0, - ) - events = capture_events() - - graph = MockStateGraph() - pregel = MockPregelInstance() - - with start_transaction(): - - def original_compile(self, *args, **kwargs): - return MockCompiledGraph(self.name) - - wrapped_compile = _wrap_state_graph_compile(original_compile) - wrapped_compile(graph) - - def original_invoke(self, *args, **kwargs): - return {"result": "test"} - - wrapped_invoke = _wrap_pregel_invoke(original_invoke) - wrapped_invoke(pregel, {"messages": []}) - - tx = events[0] - - ai_spans = [ - span - for span in tx["spans"] - if span["op"] in [OP.GEN_AI_CREATE_AGENT, OP.GEN_AI_INVOKE_AGENT] - ] - assert len(ai_spans) == 0 - - @pytest.mark.parametrize("graph_name", ["my_graph", None, ""]) def test_pregel_invoke_with_different_graph_names( sentry_init, capture_events, graph_name