From dd166a8fea49118f14c579086718f443a4b980ad Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 13 Aug 2025 14:21:06 +0200 Subject: [PATCH 1/6] updated tool span --- sentry_sdk/integrations/langchain.py | 43 ++++++++++++++++++---------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 0b377169d0..36e07dbe95 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -257,6 +257,7 @@ def on_llm_start( ): # type: (SentryLangchainCallback, Dict[str, Any], List[str], UUID, Optional[List[str]], Optional[UUID], Optional[Dict[str, Any]], Any) -> Any """Run when LLM starts running.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -286,6 +287,7 @@ def on_llm_start( def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): # type: (SentryLangchainCallback, Dict[str, Any], List[List[BaseMessage]], UUID, Any) -> Any """Run when Chat Model starts running.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -327,6 +329,7 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): def on_chat_model_end(self, response, *, run_id, **kwargs): # type: (SentryLangchainCallback, LLMResult, UUID, Any) -> Any """Run when Chat Model ends running.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -395,6 +398,7 @@ def on_llm_new_token(self, token, *, run_id, **kwargs): def on_llm_end(self, response, *, run_id, **kwargs): # type: (SentryLangchainCallback, LLMResult, UUID, Any) -> Any """Run when LLM ends running.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -450,23 +454,27 @@ def on_llm_end(self, response, *, run_id, **kwargs): def on_llm_error(self, error, *, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when LLM errors.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): self._handle_error(run_id, error) def on_chat_model_error(self, error, *, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when Chat Model errors.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): self._handle_error(run_id, error) def on_chain_start(self, serialized, inputs, *, run_id, **kwargs): # type: (SentryLangchainCallback, Dict[str, Any], Dict[str, Any], UUID, Any) -> Any """Run when chain starts running.""" + # import ipdb; ipdb.set_trace() pass def on_chain_end(self, outputs, *, run_id, **kwargs): # type: (SentryLangchainCallback, Dict[str, Any], UUID, Any) -> Any """Run when chain ends running.""" + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id or run_id not in self.span_map: return @@ -479,10 +487,12 @@ def on_chain_end(self, outputs, *, run_id, **kwargs): def on_chain_error(self, error, *, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when chain errors.""" + # import ipdb; ipdb.set_trace() self._handle_error(run_id, error) def on_agent_action(self, action, *, run_id, **kwargs): # type: (SentryLangchainCallback, AgentAction, UUID, Any) -> Any + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -502,6 +512,7 @@ def on_agent_action(self, action, *, run_id, **kwargs): def on_agent_finish(self, finish, *, run_id, **kwargs): # type: (SentryLangchainCallback, AgentFinish, UUID, Any) -> Any + # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -523,28 +534,31 @@ def on_tool_start(self, serialized, input_str, *, run_id, **kwargs): with capture_internal_exceptions(): if not run_id: return + + tool_name = serialized.get("name") or kwargs.get("name") + watched_span = self._create_span( run_id, kwargs.get("parent_run_id"), op=OP.GEN_AI_EXECUTE_TOOL, - name=serialized.get("name") or kwargs.get("name") or "AI tool usage", + name=f"execute_tool {tool_name}", origin=LangchainIntegration.origin, ) - watched_span.span.set_data( - SPANDATA.GEN_AI_TOOL_NAME, serialized.get("name") - ) + span = watched_span.span + + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") + span.set_data(SPANDATA.GEN_AI_TOOL_NAME, tool_name) + + tool_description = serialized.get("description") + if tool_description is not None: + span.set_data(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool_description) + if should_send_default_pii() and self.include_prompts: set_data_normalized( - watched_span.span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, + span, + SPANDATA.GEN_AI_TOOL_INPUT, kwargs.get("inputs", [input_str]), ) - if kwargs.get("metadata"): - set_data_normalized( - watched_span.span, - SPANDATA.GEN_AI_REQUEST_METADATA, - kwargs.get("metadata"), - ) def on_tool_end(self, output, *, run_id, **kwargs): # type: (SentryLangchainCallback, str, UUID, Any) -> Any @@ -557,14 +571,13 @@ def on_tool_end(self, output, *, run_id, **kwargs): if not span_data: return if should_send_default_pii() and self.include_prompts: - set_data_normalized( - span_data.span, SPANDATA.GEN_AI_RESPONSE_TEXT, output - ) + set_data_normalized(span_data.span, SPANDATA.GEN_AI_TOOL_OUTPUT, output) self._exit_span(span_data, run_id) def on_tool_error(self, error, *args, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when tool errors.""" + # import ipdb; ipdb.set_trace() # TODO(shellmayr): how to correctly set the status when the toolfails if run_id and run_id in self.span_map: span_data = self.span_map[run_id] From adaabbf7e9b310486d44ab0a445164ae66ed4988 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 13 Aug 2025 14:23:11 +0200 Subject: [PATCH 2/6] . --- sentry_sdk/integrations/langchain.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 36e07dbe95..b26d752bf3 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -257,7 +257,6 @@ def on_llm_start( ): # type: (SentryLangchainCallback, Dict[str, Any], List[str], UUID, Optional[List[str]], Optional[UUID], Optional[Dict[str, Any]], Any) -> Any """Run when LLM starts running.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -287,7 +286,6 @@ def on_llm_start( def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): # type: (SentryLangchainCallback, Dict[str, Any], List[List[BaseMessage]], UUID, Any) -> Any """Run when Chat Model starts running.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -329,7 +327,6 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): def on_chat_model_end(self, response, *, run_id, **kwargs): # type: (SentryLangchainCallback, LLMResult, UUID, Any) -> Any """Run when Chat Model ends running.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -398,7 +395,6 @@ def on_llm_new_token(self, token, *, run_id, **kwargs): def on_llm_end(self, response, *, run_id, **kwargs): # type: (SentryLangchainCallback, LLMResult, UUID, Any) -> Any """Run when LLM ends running.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -454,27 +450,23 @@ def on_llm_end(self, response, *, run_id, **kwargs): def on_llm_error(self, error, *, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when LLM errors.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): self._handle_error(run_id, error) def on_chat_model_error(self, error, *, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when Chat Model errors.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): self._handle_error(run_id, error) def on_chain_start(self, serialized, inputs, *, run_id, **kwargs): # type: (SentryLangchainCallback, Dict[str, Any], Dict[str, Any], UUID, Any) -> Any """Run when chain starts running.""" - # import ipdb; ipdb.set_trace() pass def on_chain_end(self, outputs, *, run_id, **kwargs): # type: (SentryLangchainCallback, Dict[str, Any], UUID, Any) -> Any """Run when chain ends running.""" - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id or run_id not in self.span_map: return @@ -487,12 +479,10 @@ def on_chain_end(self, outputs, *, run_id, **kwargs): def on_chain_error(self, error, *, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when chain errors.""" - # import ipdb; ipdb.set_trace() self._handle_error(run_id, error) def on_agent_action(self, action, *, run_id, **kwargs): # type: (SentryLangchainCallback, AgentAction, UUID, Any) -> Any - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -512,7 +502,6 @@ def on_agent_action(self, action, *, run_id, **kwargs): def on_agent_finish(self, finish, *, run_id, **kwargs): # type: (SentryLangchainCallback, AgentFinish, UUID, Any) -> Any - # import ipdb; ipdb.set_trace() with capture_internal_exceptions(): if not run_id: return @@ -577,12 +566,12 @@ def on_tool_end(self, output, *, run_id, **kwargs): def on_tool_error(self, error, *args, run_id, **kwargs): # type: (SentryLangchainCallback, Union[Exception, KeyboardInterrupt], UUID, Any) -> Any """Run when tool errors.""" - # import ipdb; ipdb.set_trace() - # TODO(shellmayr): how to correctly set the status when the toolfails + # TODO(shellmayr): how to correctly set the status when the tool fails? if run_id and run_id in self.span_map: span_data = self.span_map[run_id] if span_data: span_data.span.set_status("unknown") + self._handle_error(run_id, error) From 3795d638788529c1d95a1b5386a27da7528a0a3f Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 13 Aug 2025 14:31:32 +0200 Subject: [PATCH 3/6] tool name --- sentry_sdk/integrations/langchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index b26d752bf3..6b72096d2e 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -524,13 +524,13 @@ def on_tool_start(self, serialized, input_str, *, run_id, **kwargs): if not run_id: return - tool_name = serialized.get("name") or kwargs.get("name") + tool_name = serialized.get("name") or kwargs.get("name") or "" watched_span = self._create_span( run_id, kwargs.get("parent_run_id"), op=OP.GEN_AI_EXECUTE_TOOL, - name=f"execute_tool {tool_name}", + name=f"execute_tool {tool_name}".strip(), origin=LangchainIntegration.origin, ) span = watched_span.span From eb952933b90ea4e5e2d89c6a0a459450aa2e8723 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 13 Aug 2025 15:12:40 +0200 Subject: [PATCH 4/6] chat span --- sentry_sdk/integrations/langchain.py | 135 +++++++++++++-------------- 1 file changed, 65 insertions(+), 70 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 6b72096d2e..a85623c703 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -32,27 +32,20 @@ DATA_FIELDS = { - "temperature": SPANDATA.GEN_AI_REQUEST_TEMPERATURE, - "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, - "top_k": SPANDATA.GEN_AI_REQUEST_TOP_K, + "frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, "function_call": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - "tool_calls": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - "tools": SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, - "response_format": SPANDATA.GEN_AI_RESPONSE_FORMAT, "logit_bias": SPANDATA.GEN_AI_REQUEST_LOGIT_BIAS, + "max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, + "presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, + "response_format": SPANDATA.GEN_AI_RESPONSE_FORMAT, "tags": SPANDATA.GEN_AI_REQUEST_TAGS, + "temperature": SPANDATA.GEN_AI_REQUEST_TEMPERATURE, + "tool_calls": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + "tools": SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + "top_k": SPANDATA.GEN_AI_REQUEST_TOP_K, + "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, } -# TODO(shellmayr): is this still the case? -# To avoid double collecting tokens, we do *not* measure -# token counts for models for which we have an explicit integration -NO_COLLECT_TOKEN_MODELS = [ - # "openai-chat", - # "anthropic-chat", - "cohere-chat", - "huggingface_endpoint", -] - class LangchainIntegration(Integration): identifier = "langchain" @@ -74,7 +67,6 @@ def setup_once(): class WatchedSpan: span = None # type: Span - no_collect_tokens = False # type: bool children = [] # type: List[WatchedSpan] is_pipeline = False # type: bool @@ -291,25 +283,34 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): return all_params = kwargs.get("invocation_params", {}) all_params.update(serialized.get("kwargs", {})) + + model = ( + all_params.get("model") + or all_params.get("model_name") + or all_params.get("model_id") + or "" + ) + watched_span = self._create_span( run_id, kwargs.get("parent_run_id"), op=OP.GEN_AI_CHAT, - name=kwargs.get("name") or "Langchain Chat Model", + name=f"chat {model}".strip(), origin=LangchainIntegration.origin, ) span = watched_span.span - model = all_params.get( - "model", all_params.get("model_name", all_params.get("model_id")) - ) - watched_span.no_collect_tokens = any( - x in all_params.get("_type", "") for x in NO_COLLECT_TOKEN_MODELS - ) - if not model and "anthropic" in all_params.get("_type"): - model = "claude-2" + span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "chat") if model: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, model) + + import ipdb + + ipdb.set_trace() + for key, attribute in DATA_FIELDS.items(): + if key in all_params: + set_data_normalized(span, attribute, all_params[key]) + if should_send_default_pii() and self.include_prompts: set_data_normalized( span, @@ -319,10 +320,6 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): for list_ in messages ], ) - for k, v in DATA_FIELDS.items(): - if k in all_params: - set_data_normalized(span, v, all_params[k]) - # no manual token counting def on_chat_model_end(self, response, *, run_id, **kwargs): # type: (SentryLangchainCallback, LLMResult, UUID, Any) -> Any @@ -361,27 +358,26 @@ def on_chat_model_end(self, response, *, run_id, **kwargs): [[x.text for x in list_] for list_ in response.generations], ) - if not span_data.no_collect_tokens: - if token_usage: - input_tokens, output_tokens, total_tokens = ( - self._extract_token_usage(token_usage) - ) - else: - input_tokens, output_tokens, total_tokens = ( - self._extract_token_usage_from_generations(response.generations) - ) + if token_usage: + input_tokens, output_tokens, total_tokens = self._extract_token_usage( + token_usage + ) + else: + input_tokens, output_tokens, total_tokens = ( + self._extract_token_usage_from_generations(response.generations) + ) - if ( - input_tokens is not None - or output_tokens is not None - or total_tokens is not None - ): - record_token_usage( - span_data.span, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - ) + if ( + input_tokens is not None + or output_tokens is not None + or total_tokens is not None + ): + record_token_usage( + span_data.span, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) self._exit_span(span_data, run_id) @@ -423,27 +419,26 @@ def on_llm_end(self, response, *, run_id, **kwargs): [[x.text for x in list_] for list_ in response.generations], ) - if not span_data.no_collect_tokens: - if token_usage: - input_tokens, output_tokens, total_tokens = ( - self._extract_token_usage(token_usage) - ) - else: - input_tokens, output_tokens, total_tokens = ( - self._extract_token_usage_from_generations(response.generations) - ) + if token_usage: + input_tokens, output_tokens, total_tokens = self._extract_token_usage( + token_usage + ) + else: + input_tokens, output_tokens, total_tokens = ( + self._extract_token_usage_from_generations(response.generations) + ) - if ( - input_tokens is not None - or output_tokens is not None - or total_tokens is not None - ): - record_token_usage( - span_data.span, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - ) + if ( + input_tokens is not None + or output_tokens is not None + or total_tokens is not None + ): + record_token_usage( + span_data.span, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) self._exit_span(span_data, run_id) From 9b09474e7771e7dd3526bb49588416d78943a565 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 13 Aug 2025 15:18:18 +0200 Subject: [PATCH 5/6] fix available tools --- sentry_sdk/integrations/langchain.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index a85623c703..2a2f407c12 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -304,12 +304,9 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): if model: span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, model) - import ipdb - - ipdb.set_trace() for key, attribute in DATA_FIELDS.items(): if key in all_params: - set_data_normalized(span, attribute, all_params[key]) + set_data_normalized(span, attribute, all_params[key], unpack=False) if should_send_default_pii() and self.include_prompts: set_data_normalized( From 94fd28fc205339f009dd62bed5b53ed18afdb160 Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Wed, 13 Aug 2025 15:24:39 +0200 Subject: [PATCH 6/6] cleanup --- sentry_sdk/integrations/langchain.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 2a2f407c12..0a23a61dd4 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -256,8 +256,8 @@ def on_llm_start( all_params.update(serialized.get("kwargs", {})) watched_span = self._create_span( - run_id, - kwargs.get("parent_run_id"), + run_id=run_id, + parent_id=kwargs.get("parent_run_id"), op=OP.GEN_AI_PIPELINE, name=kwargs.get("name") or "Langchain LLM call", origin=LangchainIntegration.origin, @@ -292,8 +292,8 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): ) watched_span = self._create_span( - run_id, - kwargs.get("parent_run_id"), + run_id=run_id, + parent_id=kwargs.get("parent_run_id"), op=OP.GEN_AI_CHAT, name=f"chat {model}".strip(), origin=LangchainIntegration.origin, @@ -324,7 +324,6 @@ def on_chat_model_end(self, response, *, run_id, **kwargs): with capture_internal_exceptions(): if not run_id: return - token_usage = None # Try multiple paths to extract token usage, prioritizing streaming-aware approaches @@ -479,8 +478,8 @@ def on_agent_action(self, action, *, run_id, **kwargs): if not run_id: return watched_span = self._create_span( - run_id, - kwargs.get("parent_run_id"), + run_id=run_id, + parent_id=kwargs.get("parent_run_id"), op=OP.GEN_AI_INVOKE_AGENT, name=action.tool or "AI tool usage", origin=LangchainIntegration.origin, @@ -519,8 +518,8 @@ def on_tool_start(self, serialized, input_str, *, run_id, **kwargs): tool_name = serialized.get("name") or kwargs.get("name") or "" watched_span = self._create_span( - run_id, - kwargs.get("parent_run_id"), + run_id=run_id, + parent_id=kwargs.get("parent_run_id"), op=OP.GEN_AI_EXECUTE_TOOL, name=f"execute_tool {tool_name}".strip(), origin=LangchainIntegration.origin,