Skip to content

TEST DON'T MERGE #18961

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
RulaKhaled wants to merge 8 commits into from
Closed

TEST DON'T MERGE #18961

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8821f06
ref(tests): use constants in ai integration tests and add missing one…
nicohrubec Jan 23, 2026
1156b0a
fix vercel ai op name, and fix tests
RulaKhaled Jan 22, 2026
47cb2ce
fix vercel ai op name, and fix tests
RulaKhaled Jan 22, 2026
fafe00b
fix cloudflare tests
RulaKhaled Jan 22, 2026
c3ec4d5
fix tests
RulaKhaled Jan 22, 2026
cebf9a6
refactor to use chat for llms and chat models
RulaKhaled Jan 23, 2026
7111057
update with linting
RulaKhaled Jan 23, 2026
2c9ea07
fix cloudflare and browser tests
RulaKhaled Jan 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions dev-packages/browser-integration-tests/suites/tracing/ai-providers/anthropic/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ sentryTest('manual Anthropic instrumentation sends gen_ai transactions', async (
const eventData = envelopeRequestParser(req);

// Verify it's a gen_ai transaction
expect(eventData.transaction).toBe('messages claude-3-haiku-20240307');
expect(eventData.contexts?.trace?.op).toBe('gen_ai.messages');
expect(eventData.transaction).toBe('chat claude-3-haiku-20240307');
expect(eventData.contexts?.trace?.op).toBe('gen_ai.chat');
expect(eventData.contexts?.trace?.origin).toBe('auto.ai.anthropic');
expect(eventData.contexts?.trace?.data).toMatchObject({
'gen_ai.operation.name': 'messages',
'gen_ai.operation.name': 'chat',
'gen_ai.system': 'anthropic',
'gen_ai.request.model': 'claude-3-haiku-20240307',
'gen_ai.request.temperature': 0.7,
Expand Down
24 changes: 12 additions & 12 deletions dev-packages/cloudflare-integration-tests/suites/tracing/anthropic-ai/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,19 @@ it('traces a basic message creation request', async ({ signal }) => {
expect.arrayContaining([
expect.objectContaining({
data: expect.objectContaining({
'gen_ai.operation.name': 'messages',
'sentry.op': 'gen_ai.messages',
'sentry.origin': 'auto.ai.anthropic',
'gen_ai.system': 'anthropic',
'gen_ai.request.model': 'claude-3-haiku-20240307',
'gen_ai.request.temperature': 0.7,
'gen_ai.response.model': 'claude-3-haiku-20240307',
'gen_ai.response.id': 'msg_mock123',
'gen_ai.usage.input_tokens': 10,
'gen_ai.usage.output_tokens': 15,
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic',
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307',
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7,
[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307',
[GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'msg_mock123',
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10,
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15,
}),
description: 'messages claude-3-haiku-20240307',
op: 'gen_ai.messages',
description: 'chat claude-3-haiku-20240307',
op: 'gen_ai.chat',
origin: 'auto.ai.anthropic',
}),
]),
Expand Down
54 changes: 27 additions & 27 deletions dev-packages/cloudflare-integration-tests/suites/tracing/google-genai/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ it('traces Google GenAI chat creation and message sending', async () => {
// First span - chats.create
expect.objectContaining({
data: expect.objectContaining({
'gen_ai.operation.name': 'chat',
'sentry.op': 'gen_ai.chat',
'sentry.origin': 'auto.ai.google_genai',
'gen_ai.system': 'google_genai',
'gen_ai.request.model': 'gemini-1.5-pro',
'gen_ai.request.temperature': 0.8,
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai',
[GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai',
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro',
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.8,
'gen_ai.request.top_p': 0.9,
'gen_ai.request.max_tokens': 150,
[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 150,
}),
description: 'chat gemini-1.5-pro create',
op: 'gen_ai.chat',
Expand All @@ -34,14 +34,14 @@ it('traces Google GenAI chat creation and message sending', async () => {
// Second span - chat.sendMessage
expect.objectContaining({
data: expect.objectContaining({
'gen_ai.operation.name': 'chat',
'sentry.op': 'gen_ai.chat',
'sentry.origin': 'auto.ai.google_genai',
'gen_ai.system': 'google_genai',
'gen_ai.request.model': 'gemini-1.5-pro',
'gen_ai.usage.input_tokens': 8,
'gen_ai.usage.output_tokens': 12,
'gen_ai.usage.total_tokens': 20,
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai',
[GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai',
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-pro',
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8,
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12,
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20,
}),
description: 'chat gemini-1.5-pro',
op: 'gen_ai.chat',
Expand All @@ -50,20 +50,20 @@ it('traces Google GenAI chat creation and message sending', async () => {
// Third span - models.generateContent
expect.objectContaining({
data: expect.objectContaining({
'gen_ai.operation.name': 'models',
'sentry.op': 'gen_ai.models',
'sentry.origin': 'auto.ai.google_genai',
'gen_ai.system': 'google_genai',
'gen_ai.request.model': 'gemini-1.5-flash',
'gen_ai.request.temperature': 0.7,
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai',
[GEN_AI_SYSTEM_ATTRIBUTE]: 'google_genai',
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gemini-1.5-flash',
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7,
'gen_ai.request.top_p': 0.9,
'gen_ai.request.max_tokens': 100,
'gen_ai.usage.input_tokens': 8,
'gen_ai.usage.output_tokens': 12,
'gen_ai.usage.total_tokens': 20,
[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100,
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 8,
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12,
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 20,
}),
description: 'models gemini-1.5-flash',
op: 'gen_ai.models',
description: 'generate_content gemini-1.5-flash',
op: 'gen_ai.generate_content',
origin: 'auto.ai.google_genai',
}),
]),
Expand Down
42 changes: 27 additions & 15 deletions dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,16 @@
import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core';
import { expect, it } from 'vitest';
import {
GEN_AI_OPERATION_NAME_ATTRIBUTE,
GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE,
GEN_AI_REQUEST_MODEL_ATTRIBUTE,
GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE,
GEN_AI_SYSTEM_ATTRIBUTE,
GEN_AI_TOOL_NAME_ATTRIBUTE,
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
} from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
import { createRunner } from '../../../runner';

// These tests are not exhaustive because the instrumentation is
Expand All @@ -18,16 +30,16 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal }
// Chat model span
expect.objectContaining({
data: expect.objectContaining({
'gen_ai.operation.name': 'chat',
'sentry.op': 'gen_ai.chat',
'sentry.origin': 'auto.ai.langchain',
'gen_ai.system': 'anthropic',
'gen_ai.request.model': 'claude-3-5-sonnet-20241022',
'gen_ai.request.temperature': 0.7,
'gen_ai.request.max_tokens': 100,
'gen_ai.usage.input_tokens': 10,
'gen_ai.usage.output_tokens': 15,
'gen_ai.usage.total_tokens': 25,
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022',
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7,
[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE]: 100,
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10,
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15,
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25,
}),
description: 'chat claude-3-5-sonnet-20241022',
op: 'gen_ai.chat',
Expand All @@ -36,8 +48,8 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal }
// Chain span
expect.objectContaining({
data: expect.objectContaining({
'sentry.origin': 'auto.ai.langchain',
'sentry.op': 'gen_ai.invoke_agent',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent',
'langchain.chain.name': 'my_test_chain',
}),
description: 'chain my_test_chain',
Expand All @@ -47,9 +59,9 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal }
// Tool span
expect.objectContaining({
data: expect.objectContaining({
'sentry.origin': 'auto.ai.langchain',
'sentry.op': 'gen_ai.execute_tool',
'gen_ai.tool.name': 'search_tool',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool',
[GEN_AI_TOOL_NAME_ATTRIBUTE]: 'search_tool',
}),
description: 'execute_tool search_tool',
op: 'gen_ai.execute_tool',
Expand Down
44 changes: 28 additions & 16 deletions dev-packages/cloudflare-integration-tests/suites/tracing/langgraph/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,16 @@
import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core';
import { expect, it } from 'vitest';
import {
GEN_AI_AGENT_NAME_ATTRIBUTE,
GEN_AI_OPERATION_NAME_ATTRIBUTE,
GEN_AI_PIPELINE_NAME_ATTRIBUTE,
GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE,
GEN_AI_REQUEST_MESSAGES_ATTRIBUTE,
GEN_AI_RESPONSE_MODEL_ATTRIBUTE,
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
} from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
import { createRunner } from '../../../runner';

// These tests are not exhaustive because the instrumentation is
Expand All @@ -18,10 +30,10 @@ it('traces langgraph compile and invoke operations', async ({ signal }) => {
const createAgentSpan = transactionEvent.spans.find((span: any) => span.op === 'gen_ai.create_agent');
expect(createAgentSpan).toMatchObject({
data: {
'gen_ai.operation.name': 'create_agent',
'sentry.op': 'gen_ai.create_agent',
'sentry.origin': 'auto.ai.langgraph',
'gen_ai.agent.name': 'weather_assistant',
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'create_agent',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.create_agent',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph',
[GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant',
},
description: 'create_agent weather_assistant',
op: 'gen_ai.create_agent',
Expand All @@ -32,25 +44,25 @@ it('traces langgraph compile and invoke operations', async ({ signal }) => {
const invokeAgentSpan = transactionEvent.spans.find((span: any) => span.op === 'gen_ai.invoke_agent');
expect(invokeAgentSpan).toMatchObject({
data: expect.objectContaining({
'gen_ai.operation.name': 'invoke_agent',
'sentry.op': 'gen_ai.invoke_agent',
'sentry.origin': 'auto.ai.langgraph',
'gen_ai.agent.name': 'weather_assistant',
'gen_ai.pipeline.name': 'weather_assistant',
'gen_ai.request.messages': '[{"role":"user","content":"What is the weather in SF?"}]',
'gen_ai.response.model': 'mock-model',
'gen_ai.usage.input_tokens': 20,
'gen_ai.usage.output_tokens': 10,
'gen_ai.usage.total_tokens': 30,
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langgraph',
[GEN_AI_AGENT_NAME_ATTRIBUTE]: 'weather_assistant',
[GEN_AI_PIPELINE_NAME_ATTRIBUTE]: 'weather_assistant',
[GEN_AI_REQUEST_MESSAGES_ATTRIBUTE]: '[{"role":"user","content":"What is the weather in SF?"}]',
[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'mock-model',
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 20,
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10,
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 30,
}),
description: 'invoke_agent weather_assistant',
op: 'gen_ai.invoke_agent',
origin: 'auto.ai.langgraph',
});

// Verify tools are captured
if (invokeAgentSpan.data['gen_ai.request.available_tools']) {
expect(invokeAgentSpan.data['gen_ai.request.available_tools']).toMatch(/get_weather/);
if (invokeAgentSpan.data[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]) {
expect(invokeAgentSpan.data[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toMatch(/get_weather/);
}
})
.start(signal);
Expand Down
37 changes: 25 additions & 12 deletions dev-packages/cloudflare-integration-tests/suites/tracing/openai/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@
import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core';
import { expect, it } from 'vitest';
import {
GEN_AI_OPERATION_NAME_ATTRIBUTE,
GEN_AI_REQUEST_MODEL_ATTRIBUTE,
GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE,
GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE,
GEN_AI_RESPONSE_ID_ATTRIBUTE,
GEN_AI_RESPONSE_MODEL_ATTRIBUTE,
GEN_AI_SYSTEM_ATTRIBUTE,
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
} from '../../../../../packages/core/src/tracing/ai/gen-ai-attributes';
import { createRunner } from '../../../runner';

// These tests are not exhaustive because the instrumentation is
Expand All @@ -17,18 +30,18 @@ it('traces a basic chat completion request', async ({ signal }) => {
expect.arrayContaining([
expect.objectContaining({
data: expect.objectContaining({
'gen_ai.operation.name': 'chat',
'sentry.op': 'gen_ai.chat',
'sentry.origin': 'auto.ai.openai',
'gen_ai.system': 'openai',
'gen_ai.request.model': 'gpt-3.5-turbo',
'gen_ai.request.temperature': 0.7,
'gen_ai.response.model': 'gpt-3.5-turbo',
'gen_ai.response.id': 'chatcmpl-mock123',
'gen_ai.usage.input_tokens': 10,
'gen_ai.usage.output_tokens': 15,
'gen_ai.usage.total_tokens': 25,
'gen_ai.response.finish_reasons': '["stop"]',
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat',
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat',
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.openai',
[GEN_AI_SYSTEM_ATTRIBUTE]: 'openai',
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo',
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7,
[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gpt-3.5-turbo',
[GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'chatcmpl-mock123',
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10,
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 15,
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 25,
[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: '["stop"]',
}),
description: 'chat gpt-3.5-turbo',
op: 'gen_ai.chat',
Expand Down
Loading