From e0703eb0bd54a59f6e1d2dff9e04f1d9ca6ff152 Mon Sep 17 00:00:00 2001 From: PR Bot Date: Sat, 21 Mar 2026 17:16:42 +0800 Subject: [PATCH] feat: detect OpenAI-compatible providers (MiniMax, Groq, etc.) in telemetry spans When users use the OpenAI SDK with a custom base_url pointing to an OpenAI-compatible provider like MiniMax, Groq, DeepSeek, or others, the gen_ai.system span attribute was always set to "OpenAI". This made it impossible to distinguish MiniMax or Groq calls from actual OpenAI calls in observability dashboards. This commit adds automatic provider detection by inspecting the OpenAI client's base_url at instrumentation time. The detection is applied to chat completions and Responses API wrappers (both sync and async), covering the most commonly used endpoints. Supported providers: MiniMax, Groq, Together AI, Fireworks AI, DeepSeek, Mistral AI, Perplexity AI, Google AI, xAI, SambaNova, Cerebras. --- .../providers/openai/provider_detection.py | 85 +++++++++ .../providers/openai/stream_wrapper.py | 23 +++ .../openai_core/test_provider_detection.py | 162 +++++++++++++++++ .../test_provider_detection_integration.py | 172 ++++++++++++++++++ 4 files changed, 442 insertions(+) create mode 100644 agentops/instrumentation/providers/openai/provider_detection.py create mode 100644 tests/unit/instrumentation/openai_core/test_provider_detection.py create mode 100644 tests/unit/instrumentation/openai_core/test_provider_detection_integration.py diff --git a/agentops/instrumentation/providers/openai/provider_detection.py b/agentops/instrumentation/providers/openai/provider_detection.py new file mode 100644 index 000000000..0b3e2c1f2 --- /dev/null +++ b/agentops/instrumentation/providers/openai/provider_detection.py @@ -0,0 +1,85 @@ +"""OpenAI-compatible provider detection for AgentOps instrumentation. + +When users use the OpenAI SDK with a custom base_url pointing to an +OpenAI-compatible provider (e.g., MiniMax, Groq, Together AI), this module +detects the actual provider from the client's base_url so that telemetry +spans are attributed to the correct system. +""" + +import logging +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# Mapping of base_url host patterns to provider names. +# Each entry maps a substring found in the base_url host to the provider name +# used in the gen_ai.system span attribute. +_PROVIDER_HOST_MAP = { + "api.minimax.io": "MiniMax", + "api.minimax.chat": "MiniMax", + "api.groq.com": "Groq", + "api.together.xyz": "Together AI", + "api.together.ai": "Together AI", + "api.fireworks.ai": "Fireworks AI", + "api.deepseek.com": "DeepSeek", + "api.mistral.ai": "Mistral AI", + "api.perplexity.ai": "Perplexity AI", + "generativelanguage.googleapis.com": "Google AI", + "api.x.ai": "xAI", + "api.sambanova.ai": "SambaNova", + "api.cerebras.ai": "Cerebras", +} + +_DEFAULT_PROVIDER = "OpenAI" + + +def detect_provider_from_instance(instance: Any) -> str: + """Detect the LLM provider from an OpenAI SDK resource instance. + + Inspects the client's base_url to determine if the OpenAI SDK is being + used with an OpenAI-compatible provider (e.g., MiniMax, Groq). + + Args: + instance: The OpenAI SDK resource instance (e.g., Completions, + AsyncCompletions). Expected to have ``_client.base_url``. + + Returns: + The detected provider name (e.g., "MiniMax", "OpenAI"). + """ + base_url = _extract_base_url(instance) + if not base_url: + return _DEFAULT_PROVIDER + + return _match_provider(base_url) + + +def _extract_base_url(instance: Any) -> Optional[str]: + """Extract the base_url string from an OpenAI SDK resource instance.""" + try: + client = getattr(instance, "_client", None) + if client is None: + return None + base_url = getattr(client, "base_url", None) + if base_url is None: + return None + # base_url may be a URL object or a string + return str(base_url) + except Exception: + logger.debug("[PROVIDER DETECTION] Failed to extract base_url from instance") + return None + + +def _match_provider(base_url: str) -> str: + """Match a base_url string against known provider hosts. + + Args: + base_url: The base URL string (e.g., "https://api.minimax.io/v1/"). + + Returns: + The matched provider name, or "OpenAI" if no match is found. + """ + base_url_lower = base_url.lower() + for host_pattern, provider_name in _PROVIDER_HOST_MAP.items(): + if host_pattern in base_url_lower: + return provider_name + return _DEFAULT_PROVIDER diff --git a/agentops/instrumentation/providers/openai/stream_wrapper.py b/agentops/instrumentation/providers/openai/stream_wrapper.py index 4d23c5b35..035dec170 100644 --- a/agentops/instrumentation/providers/openai/stream_wrapper.py +++ b/agentops/instrumentation/providers/openai/stream_wrapper.py @@ -16,6 +16,7 @@ from agentops.instrumentation.common.wrappers import _with_tracer_wrapper from agentops.instrumentation.providers.openai.utils import is_metrics_enabled from agentops.instrumentation.providers.openai.wrappers.chat import handle_chat_attributes, _create_tool_span +from agentops.instrumentation.providers.openai.provider_detection import detect_provider_from_instance from agentops.semconv import SpanAttributes, LLMRequestTypeValues, MessageAttributes @@ -477,6 +478,11 @@ def chat_completion_stream_wrapper(tracer, wrapped, instance, args, kwargs): # Extract and set request attributes request_attributes = handle_chat_attributes(kwargs=kwargs) + # Detect actual provider from client base_url (e.g., MiniMax, Groq) + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + for key, value in request_attributes.items(): span.set_attribute(key, value) @@ -546,6 +552,11 @@ async def async_chat_completion_stream_wrapper(tracer, wrapped, instance, args, # Extract and set request attributes request_attributes = handle_chat_attributes(kwargs=kwargs) + # Detect actual provider from client base_url (e.g., MiniMax, Groq) + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + for key, value in request_attributes.items(): span.set_attribute(key, value) @@ -852,6 +863,12 @@ def responses_stream_wrapper(tracer, wrapped, instance, args, kwargs): from agentops.instrumentation.providers.openai.wrappers.responses import handle_responses_attributes request_attributes = handle_responses_attributes(kwargs=kwargs) + + # Detect actual provider from client base_url (e.g., MiniMax, Groq) + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + for key, value in request_attributes.items(): span.set_attribute(key, value) @@ -909,6 +926,12 @@ async def async_responses_stream_wrapper(tracer, wrapped, instance, args, kwargs from agentops.instrumentation.providers.openai.wrappers.responses import handle_responses_attributes request_attributes = handle_responses_attributes(kwargs=kwargs) + + # Detect actual provider from client base_url (e.g., MiniMax, Groq) + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + for key, value in request_attributes.items(): span.set_attribute(key, value) diff --git a/tests/unit/instrumentation/openai_core/test_provider_detection.py b/tests/unit/instrumentation/openai_core/test_provider_detection.py new file mode 100644 index 000000000..86ee40537 --- /dev/null +++ b/tests/unit/instrumentation/openai_core/test_provider_detection.py @@ -0,0 +1,162 @@ +"""Tests for OpenAI-compatible provider detection. + +Verifies that the provider detection utility correctly identifies +LLM providers from the OpenAI SDK client's base_url. +""" + +import pytest + +from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + _extract_base_url, + _match_provider, + _PROVIDER_HOST_MAP, +) + + +class MockClient: + """Mock OpenAI client with configurable base_url.""" + + def __init__(self, base_url=None): + self.base_url = base_url + + +class MockResource: + """Mock OpenAI SDK resource (e.g., Completions) with a _client attribute.""" + + def __init__(self, client=None): + self._client = client + + +class TestMatchProvider: + """Tests for _match_provider function.""" + + def test_minimax_io(self): + assert _match_provider("https://api.minimax.io/v1/") == "MiniMax" + + def test_minimax_chat(self): + assert _match_provider("https://api.minimax.chat/v1") == "MiniMax" + + def test_groq(self): + assert _match_provider("https://api.groq.com/openai/v1") == "Groq" + + def test_together_xyz(self): + assert _match_provider("https://api.together.xyz/v1") == "Together AI" + + def test_together_ai(self): + assert _match_provider("https://api.together.ai/v1") == "Together AI" + + def test_fireworks(self): + assert _match_provider("https://api.fireworks.ai/inference/v1") == "Fireworks AI" + + def test_deepseek(self): + assert _match_provider("https://api.deepseek.com/v1") == "DeepSeek" + + def test_mistral(self): + assert _match_provider("https://api.mistral.ai/v1") == "Mistral AI" + + def test_perplexity(self): + assert _match_provider("https://api.perplexity.ai/") == "Perplexity AI" + + def test_xai(self): + assert _match_provider("https://api.x.ai/v1") == "xAI" + + def test_sambanova(self): + assert _match_provider("https://api.sambanova.ai/v1") == "SambaNova" + + def test_cerebras(self): + assert _match_provider("https://api.cerebras.ai/v1") == "Cerebras" + + def test_openai_default(self): + assert _match_provider("https://api.openai.com/v1") == "OpenAI" + + def test_unknown_url(self): + assert _match_provider("https://my-custom-llm.example.com/v1") == "OpenAI" + + def test_case_insensitive(self): + assert _match_provider("https://API.MINIMAX.IO/v1") == "MiniMax" + + def test_empty_url(self): + assert _match_provider("") == "OpenAI" + + +class TestExtractBaseUrl: + """Tests for _extract_base_url function.""" + + def test_with_string_base_url(self): + client = MockClient(base_url="https://api.minimax.io/v1/") + resource = MockResource(client=client) + assert _extract_base_url(resource) == "https://api.minimax.io/v1/" + + def test_with_url_object(self): + """Test with URL-like object that has __str__.""" + + class URLObject: + def __str__(self): + return "https://api.minimax.io/v1/" + + client = MockClient(base_url=URLObject()) + resource = MockResource(client=client) + assert _extract_base_url(resource) == "https://api.minimax.io/v1/" + + def test_no_client(self): + resource = MockResource(client=None) + assert _extract_base_url(resource) is None + + def test_no_base_url(self): + client = MockClient(base_url=None) + resource = MockResource(client=client) + assert _extract_base_url(resource) is None + + def test_no_client_attribute(self): + """Test with an object that has no _client attribute.""" + + class NoClient: + pass + + assert _extract_base_url(NoClient()) is None + + +class TestDetectProviderFromInstance: + """Tests for detect_provider_from_instance function.""" + + def test_minimax_provider(self): + client = MockClient(base_url="https://api.minimax.io/v1/") + resource = MockResource(client=client) + assert detect_provider_from_instance(resource) == "MiniMax" + + def test_groq_provider(self): + client = MockClient(base_url="https://api.groq.com/openai/v1") + resource = MockResource(client=client) + assert detect_provider_from_instance(resource) == "Groq" + + def test_openai_provider(self): + client = MockClient(base_url="https://api.openai.com/v1") + resource = MockResource(client=client) + assert detect_provider_from_instance(resource) == "OpenAI" + + def test_none_instance(self): + assert detect_provider_from_instance(None) == "OpenAI" + + def test_no_client_attribute(self): + assert detect_provider_from_instance(object()) == "OpenAI" + + def test_no_base_url(self): + client = MockClient(base_url=None) + resource = MockResource(client=client) + assert detect_provider_from_instance(resource) == "OpenAI" + + def test_deepseek_provider(self): + client = MockClient(base_url="https://api.deepseek.com/v1") + resource = MockResource(client=client) + assert detect_provider_from_instance(resource) == "DeepSeek" + + def test_all_registered_providers(self): + """Verify all providers in the host map are detectable.""" + for host, expected_name in _PROVIDER_HOST_MAP.items(): + client = MockClient(base_url=f"https://{host}/v1") + resource = MockResource(client=client) + result = detect_provider_from_instance(resource) + assert result == expected_name, ( + f"Expected '{expected_name}' for host '{host}', got '{result}'" + ) diff --git a/tests/unit/instrumentation/openai_core/test_provider_detection_integration.py b/tests/unit/instrumentation/openai_core/test_provider_detection_integration.py new file mode 100644 index 000000000..8d09645e2 --- /dev/null +++ b/tests/unit/instrumentation/openai_core/test_provider_detection_integration.py @@ -0,0 +1,172 @@ +"""Integration tests for provider detection in OpenAI stream wrappers. + +Verifies that the chat completion stream wrapper correctly detects +OpenAI-compatible providers (e.g., MiniMax) and sets the gen_ai.system +span attribute accordingly. +""" + +import pytest +from unittest.mock import MagicMock, patch, PropertyMock + +from agentops.semconv import SpanAttributes + + +class MockSpan: + """Mock OpenTelemetry span that records set_attribute calls.""" + + def __init__(self): + self._attributes = {} + + def set_attribute(self, key, value): + self._attributes[key] = value + + def set_status(self, status): + pass + + def end(self): + pass + + def record_exception(self, exception): + pass + + def add_event(self, name, attributes=None): + pass + + +class MockClient: + """Mock OpenAI client.""" + + def __init__(self, base_url): + self.base_url = base_url + + +class MockCompletionsInstance: + """Mock Completions resource instance.""" + + def __init__(self, base_url="https://api.openai.com/v1"): + self._client = MockClient(base_url) + + +class MockTracer: + """Mock OpenTelemetry tracer.""" + + def __init__(self): + self.spans = [] + + def start_span(self, name, kind=None, attributes=None): + span = MockSpan() + if attributes: + for k, v in attributes.items(): + span.set_attribute(k, v) + self.spans.append(span) + return span + + +class TestProviderDetectionInWrapper: + """Test that provider detection works in the stream wrapper context.""" + + def test_detect_minimax_sets_llm_system(self): + """When instance points to MiniMax, LLM_SYSTEM should be 'MiniMax'.""" + from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + ) + + instance = MockCompletionsInstance("https://api.minimax.io/v1/") + provider = detect_provider_from_instance(instance) + assert provider == "MiniMax" + + def test_detect_openai_keeps_default(self): + """When instance points to OpenAI, LLM_SYSTEM should be 'OpenAI'.""" + from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + ) + + instance = MockCompletionsInstance("https://api.openai.com/v1") + provider = detect_provider_from_instance(instance) + assert provider == "OpenAI" + + def test_request_attributes_overridden_for_minimax(self): + """Simulate the wrapper flow: handle_chat_attributes returns OpenAI, + then provider detection overrides to MiniMax.""" + from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + ) + + # Simulate handle_chat_attributes output + request_attributes = { + SpanAttributes.LLM_REQUEST_TYPE: "chat", + SpanAttributes.LLM_SYSTEM: "OpenAI", + SpanAttributes.LLM_REQUEST_MODEL: "MiniMax-M2.7", + } + + # Detect provider from MiniMax instance + instance = MockCompletionsInstance("https://api.minimax.io/v1/") + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + + # Verify the override + assert request_attributes[SpanAttributes.LLM_SYSTEM] == "MiniMax" + + # Apply to span + span = MockSpan() + for key, value in request_attributes.items(): + span.set_attribute(key, value) + + assert span._attributes[SpanAttributes.LLM_SYSTEM] == "MiniMax" + assert span._attributes[SpanAttributes.LLM_REQUEST_MODEL] == "MiniMax-M2.7" + + def test_openai_not_overridden(self): + """When using standard OpenAI, LLM_SYSTEM should remain 'OpenAI'.""" + from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + ) + + request_attributes = { + SpanAttributes.LLM_SYSTEM: "OpenAI", + SpanAttributes.LLM_REQUEST_MODEL: "gpt-4o", + } + + instance = MockCompletionsInstance("https://api.openai.com/v1") + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + + # Should remain OpenAI + assert request_attributes[SpanAttributes.LLM_SYSTEM] == "OpenAI" + + def test_groq_detection_in_wrapper_flow(self): + """Verify Groq is correctly detected in the wrapper flow.""" + from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + ) + + request_attributes = { + SpanAttributes.LLM_SYSTEM: "OpenAI", + SpanAttributes.LLM_REQUEST_MODEL: "llama-3.3-70b-versatile", + } + + instance = MockCompletionsInstance("https://api.groq.com/openai/v1") + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + + assert request_attributes[SpanAttributes.LLM_SYSTEM] == "Groq" + + def test_deepseek_detection_in_wrapper_flow(self): + """Verify DeepSeek is correctly detected in the wrapper flow.""" + from agentops.instrumentation.providers.openai.provider_detection import ( + detect_provider_from_instance, + ) + + request_attributes = { + SpanAttributes.LLM_SYSTEM: "OpenAI", + SpanAttributes.LLM_REQUEST_MODEL: "deepseek-chat", + } + + instance = MockCompletionsInstance("https://api.deepseek.com/v1") + provider = detect_provider_from_instance(instance) + if provider != "OpenAI": + request_attributes[SpanAttributes.LLM_SYSTEM] = provider + + assert request_attributes[SpanAttributes.LLM_SYSTEM] == "DeepSeek"