diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs index 73891ad30b6f..7df934404ff9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs @@ -49,10 +49,15 @@ async function run() { const client = instrumentAnthropicAiClient(mockClient); // Send the image showing the number 3 + // Put the image in the last message so it doesn't get dropped await client.messages.create({ model: 'claude-3-haiku-20240307', max_tokens: 1024, messages: [ + { + role: 'user', + content: 'what number is this?', + }, { role: 'user', content: [ @@ -66,10 +71,6 @@ async function run() { }, ], }, - { - role: 'user', - content: 'what number is this?', - }, ], temperature: 0.7, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs index 21821cdc5aae..49cee7e3067d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs @@ -48,12 +48,11 @@ async function run() { const client = instrumentAnthropicAiClient(mockClient); - // Create 3 large messages where: - // - First 2 messages are very large (will be dropped) - // - Last message is large but will be truncated to fit within the 20KB limit + // Test 1: Given an array of messages only the last message should be kept + // The last message should be truncated to fit within the 20KB limit const largeContent1 = 'A'.repeat(15000); // ~15KB const largeContent2 = 'B'.repeat(15000); // ~15KB - const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + const largeContent3 = 'C'.repeat(25000) + 'D'.repeat(25000); // ~50KB (will be truncated, only C's remain) await client.messages.create({ model: 'claude-3-haiku-20240307', @@ -65,6 +64,20 @@ async function run() { ], temperature: 0.7, }); + + // Test 2: Given an array of messages only the last message should be kept + // The last message is small, so it should be kept intact + const smallContent = 'This is a small message that fits within the limit'; + await client.messages.create({ + model: 'claude-3-haiku-20240307', + max_tokens: 100, + messages: [ + { role: 'user', content: largeContent1 }, + { role: 'assistant', content: largeContent2 }, + { role: 'user', content: smallContent }, + ], + temperature: 0.7, + }); }); } diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index f62975dafb71..a70e51858113 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -86,8 +86,7 @@ describe('Anthropic integration', () => { data: expect.objectContaining({ 'gen_ai.operation.name': 'messages', 'gen_ai.request.max_tokens': 100, - 'gen_ai.request.messages': - '[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is the capital of France?"}]', + 'gen_ai.request.messages': '[{"role":"user","content":"What is the capital of France?"}]', 'gen_ai.request.model': 'claude-3-haiku-20240307', 'gen_ai.request.temperature': 0.7, 'gen_ai.response.id': 'msg_mock123', @@ -638,6 +637,7 @@ describe('Anthropic integration', () => { transaction: { transaction: 'main', spans: expect.arrayContaining([ + // First call: Last message is large and gets truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'messages', @@ -653,6 +653,24 @@ describe('Anthropic integration', () => { origin: 'auto.ai.anthropic', status: 'ok', }), + // Second call: Last message is small and kept without truncation + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'messages', + 'sentry.op': 'gen_ai.messages', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + // Small message should be kept intact + 'gen_ai.request.messages': JSON.stringify([ + { role: 'user', content: 'This is a small message that fits within the limit' }, + ]), + }), + description: 'messages claude-3-haiku-20240307', + op: 'gen_ai.messages', + origin: 'auto.ai.anthropic', + status: 'ok', + }), ]), }, }) @@ -677,6 +695,7 @@ describe('Anthropic integration', () => { 'sentry.origin': 'auto.ai.anthropic', 'gen_ai.system': 'anthropic', 'gen_ai.request.model': 'claude-3-haiku-20240307', + // Only the last message (with filtered media) should be kept 'gen_ai.request.messages': JSON.stringify([ { role: 'user', @@ -691,10 +710,6 @@ describe('Anthropic integration', () => { }, ], }, - { - role: 'user', - content: 'what number is this?', - }, ]), }), description: 'messages claude-3-haiku-20240307', diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-message-truncation.mjs index bb24b6835db2..595728e06531 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-message-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-message-truncation.mjs @@ -43,12 +43,11 @@ async function run() { const client = instrumentGoogleGenAIClient(mockClient); - // Create 3 large messages where: - // - First 2 messages are very large (will be dropped) - // - Last message is large but will be truncated to fit within the 20KB limit + // Test 1: Given an array of messages only the last message should be kept + // The last message should be truncated to fit within the 20KB limit const largeContent1 = 'A'.repeat(15000); // ~15KB const largeContent2 = 'B'.repeat(15000); // ~15KB - const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + const largeContent3 = 'C'.repeat(25000) + 'D'.repeat(25000); // ~50KB (will be truncated, only C's remain) await client.models.generateContent({ model: 'gemini-1.5-flash', @@ -63,6 +62,23 @@ async function run() { { role: 'user', parts: [{ text: largeContent3 }] }, ], }); + + // Test 2: Given an array of messages only the last message should be kept + // The last message is small, so it should be kept intact + const smallContent = 'This is a small message that fits within the limit'; + await client.models.generateContent({ + model: 'gemini-1.5-flash', + config: { + temperature: 0.7, + topP: 0.9, + maxOutputTokens: 100, + }, + contents: [ + { role: 'user', parts: [{ text: largeContent1 }] }, + { role: 'model', parts: [{ text: largeContent2 }] }, + { role: 'user', parts: [{ text: smallContent }] }, + ], + }); }); } diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts index 486b71dfedc7..d6ff72cde6d8 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts @@ -94,9 +94,7 @@ describe('Google GenAI integration', () => { 'gen_ai.request.temperature': 0.8, 'gen_ai.request.top_p': 0.9, 'gen_ai.request.max_tokens': 150, - 'gen_ai.request.messages': expect.stringMatching( - /\[\{"role":"system","content":"You are a friendly robot who likes to be funny."\},/, - ), // Should include history when recordInputs: true + 'gen_ai.request.messages': '[{"role":"user","parts":[{"text":"Hello, how are you?"}]}]', }), description: 'chat gemini-1.5-pro create', op: 'gen_ai.chat', @@ -504,6 +502,7 @@ describe('Google GenAI integration', () => { transaction: { transaction: 'main', spans: expect.arrayContaining([ + // First call: Last message is large and gets truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'models', @@ -521,6 +520,27 @@ describe('Google GenAI integration', () => { origin: 'auto.ai.google_genai', status: 'ok', }), + // Second call: Last message is small and kept without truncation + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'models', + 'sentry.op': 'gen_ai.models', + 'sentry.origin': 'auto.ai.google_genai', + 'gen_ai.system': 'google_genai', + 'gen_ai.request.model': 'gemini-1.5-flash', + // Small message should be kept intact + 'gen_ai.request.messages': JSON.stringify([ + { + role: 'user', + parts: [{ text: 'This is a small message that fits within the limit' }], + }, + ]), + }), + description: 'models gemini-1.5-flash', + op: 'gen_ai.models', + origin: 'auto.ai.google_genai', + status: 'ok', + }), ]), }, }) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs index 6dafe8572cec..9e5e59f264ca 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs @@ -51,17 +51,27 @@ async function run() { const largeContent1 = 'A'.repeat(15000); // ~15KB const largeContent2 = 'B'.repeat(15000); // ~15KB - const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + const largeContent3 = 'C'.repeat(25000) + 'D'.repeat(25000); // ~50KB (will be truncated, only C's remain) - // Create one very large string that gets truncated to only include Cs - await model.invoke(largeContent3 + largeContent2); + // Test 1: Create one very large string that gets truncated to only include Cs + await model.invoke(largeContent3); - // Create an array of messages that gets truncated to only include the last message (result should again contain only Cs) + // Test 2: Create an array of messages that gets truncated to only include the last message + // The last message should be truncated to fit within the 20KB limit (result should again contain only Cs) await model.invoke([ { role: 'system', content: largeContent1 }, { role: 'user', content: largeContent2 }, { role: 'user', content: largeContent3 }, ]); + + // Test 3: Given an array of messages only the last message should be kept + // The last message is small, so it should be kept intact + const smallContent = 'This is a small message that fits within the limit'; + await model.invoke([ + { role: 'system', content: largeContent1 }, + { role: 'user', content: largeContent2 }, + { role: 'user', content: smallContent }, + ]); }); await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index e75e0ec7f5da..8d8f1d542f70 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -198,6 +198,7 @@ describe('LangChain integration', () => { const EXPECTED_TRANSACTION_MESSAGE_TRUNCATION = { transaction: 'main', spans: expect.arrayContaining([ + // First call: String input truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'chat', @@ -213,6 +214,7 @@ describe('LangChain integration', () => { origin: 'auto.ai.langchain', status: 'ok', }), + // Second call: Array input, last message truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'chat', @@ -228,6 +230,24 @@ describe('LangChain integration', () => { origin: 'auto.ai.langchain', status: 'ok', }), + // Third call: Last message is small and kept without truncation + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + // Small message should be kept intact + 'gen_ai.request.messages': JSON.stringify([ + { role: 'user', content: 'This is a small message that fits within the limit' }, + ]), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), ]), }; diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs index 6dafe8572cec..9e5e59f264ca 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs @@ -51,17 +51,27 @@ async function run() { const largeContent1 = 'A'.repeat(15000); // ~15KB const largeContent2 = 'B'.repeat(15000); // ~15KB - const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + const largeContent3 = 'C'.repeat(25000) + 'D'.repeat(25000); // ~50KB (will be truncated, only C's remain) - // Create one very large string that gets truncated to only include Cs - await model.invoke(largeContent3 + largeContent2); + // Test 1: Create one very large string that gets truncated to only include Cs + await model.invoke(largeContent3); - // Create an array of messages that gets truncated to only include the last message (result should again contain only Cs) + // Test 2: Create an array of messages that gets truncated to only include the last message + // The last message should be truncated to fit within the 20KB limit (result should again contain only Cs) await model.invoke([ { role: 'system', content: largeContent1 }, { role: 'user', content: largeContent2 }, { role: 'user', content: largeContent3 }, ]); + + // Test 3: Given an array of messages only the last message should be kept + // The last message is small, so it should be kept intact + const smallContent = 'This is a small message that fits within the limit'; + await model.invoke([ + { role: 'system', content: largeContent1 }, + { role: 'user', content: largeContent2 }, + { role: 'user', content: smallContent }, + ]); }); await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 3e6b147d4e0d..b05a70acdeb4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -241,6 +241,7 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { const EXPECTED_TRANSACTION_MESSAGE_TRUNCATION = { transaction: 'main', spans: expect.arrayContaining([ + // First call: String input truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'chat', @@ -256,6 +257,7 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { origin: 'auto.ai.langchain', status: 'ok', }), + // Second call: Array input, last message truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'chat', @@ -271,6 +273,24 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => { origin: 'auto.ai.langchain', status: 'ok', }), + // Third call: Last message is small and kept without truncation + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + // Small message should be kept intact + 'gen_ai.request.messages': JSON.stringify([ + { role: 'user', content: 'This is a small message that fits within the limit' }, + ]), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), ]), }; diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-embeddings.mjs index f6cbe1160bf5..42c6a94c5199 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/scenario-embeddings.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-embeddings.mjs @@ -67,6 +67,12 @@ async function run() { } catch { // Error is expected and handled } + + // Third test: embeddings API with multiple inputs + await client.embeddings.create({ + input: ['First input text', 'Second input text', 'Third input text'], + model: 'text-embedding-3-small', + }); }); server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 4d41b34b8c31..bf64d2b92b72 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -160,8 +160,7 @@ describe('OpenAI integration', () => { 'gen_ai.request.model': 'gpt-3.5-turbo', 'gen_ai.request.temperature': 0.7, 'gen_ai.request.messages.original_length': 2, - 'gen_ai.request.messages': - '[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is the capital of France?"}]', + 'gen_ai.request.messages': '[{"role":"user","content":"What is the capital of France?"}]', 'gen_ai.response.model': 'gpt-3.5-turbo', 'gen_ai.response.id': 'chatcmpl-mock123', 'gen_ai.response.finish_reasons': '["stop"]', @@ -234,8 +233,7 @@ describe('OpenAI integration', () => { 'gen_ai.request.temperature': 0.8, 'gen_ai.request.stream': true, 'gen_ai.request.messages.original_length': 2, - 'gen_ai.request.messages': - '[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Tell me about streaming"}]', + 'gen_ai.request.messages': '[{"role":"user","content":"Tell me about streaming"}]', 'gen_ai.response.text': 'Hello from OpenAI streaming!', 'gen_ai.response.finish_reasons': '["stop"]', 'gen_ai.response.id': 'chatcmpl-stream-123', @@ -409,7 +407,7 @@ describe('OpenAI integration', () => { 'gen_ai.request.model': 'text-embedding-3-small', 'gen_ai.request.encoding_format': 'float', 'gen_ai.request.dimensions': 1536, - 'gen_ai.request.messages': 'Embedding test!', + 'gen_ai.embeddings.input': 'Embedding test!', 'gen_ai.response.model': 'text-embedding-3-small', 'gen_ai.usage.input_tokens': 10, 'gen_ai.usage.total_tokens': 10, @@ -429,13 +427,33 @@ describe('OpenAI integration', () => { 'sentry.origin': 'auto.ai.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'error-model', - 'gen_ai.request.messages': 'Error embedding test!', + 'gen_ai.embeddings.input': 'Error embedding test!', }, description: 'embeddings error-model', op: 'gen_ai.embeddings', origin: 'auto.ai.openai', status: 'internal_error', }), + // Third span - embeddings API with multiple inputs (this does not get truncated) + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'embeddings', + 'sentry.op': 'gen_ai.embeddings', + 'sentry.origin': 'auto.ai.openai', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'text-embedding-3-small', + 'gen_ai.embeddings.input': '["First input text","Second input text","Third input text"]', + 'gen_ai.response.model': 'text-embedding-3-small', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.total_tokens': 10, + 'openai.response.model': 'text-embedding-3-small', + 'openai.usage.prompt_tokens': 10, + }, + description: 'embeddings text-embedding-3-small', + op: 'gen_ai.embeddings', + origin: 'auto.ai.openai', + status: 'ok', + }), ]), }; createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { @@ -564,6 +582,7 @@ describe('OpenAI integration', () => { transaction: { transaction: 'main', spans: expect.arrayContaining([ + // First call: Last message is large and gets truncated (only C's remain, D's are cropped) expect.objectContaining({ data: expect.objectContaining({ 'gen_ai.operation.name': 'chat', @@ -579,6 +598,24 @@ describe('OpenAI integration', () => { origin: 'auto.ai.openai', status: 'ok', }), + // Second call: Last message is small and kept without truncation + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.openai', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-3.5-turbo', + // Small message should be kept intact + 'gen_ai.request.messages': JSON.stringify([ + { role: 'user', content: 'This is a small message that fits within the limit' }, + ]), + }), + description: 'chat gpt-3.5-turbo', + op: 'gen_ai.chat', + origin: 'auto.ai.openai', + status: 'ok', + }), ]), }, }) @@ -624,32 +661,6 @@ describe('OpenAI integration', () => { }, ); - createEsmAndCjsTests( - __dirname, - 'truncation/scenario-message-truncation-embeddings.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('truncates messages when they exceed byte limit - keeps only last message and crops it', async () => { - await createRunner() - .ignore('event') - .expect({ - transaction: { - transaction: 'main', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - 'gen_ai.operation.name': 'embeddings', - }), - }), - ]), - }, - }) - .start() - .completed(); - }); - }, - ); - // Test for conversation ID support (Conversations API and previous_response_id) const EXPECTED_TRANSACTION_CONVERSATION = { transaction: 'conversation-test', diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-completions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-completions.mjs index 96684ed9ec4f..7b0cdd730aa3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-completions.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-completions.mjs @@ -47,12 +47,11 @@ async function run() { const client = instrumentOpenAiClient(mockClient); - // Create 3 large messages where: - // - First 2 messages are very large (will be dropped) - // - Last message is large but will be truncated to fit within the 20KB limit + // Test 1: Given an array of messages only the last message should be kept + // The last message should be truncated to fit within the 20KB limit const largeContent1 = 'A'.repeat(15000); // ~15KB const largeContent2 = 'B'.repeat(15000); // ~15KB - const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + const largeContent3 = 'C'.repeat(25000) + 'D'.repeat(25000); // ~50KB (will be truncated, only C's remain) await client.chat.completions.create({ model: 'gpt-3.5-turbo', @@ -63,6 +62,19 @@ async function run() { ], temperature: 0.7, }); + + // Test 2: Given an array of messages only the last message should be kept + // The last message is small, so it should be kept intact + const smallContent = 'This is a small message that fits within the limit'; + await client.chat.completions.create({ + model: 'gpt-3.5-turbo', + messages: [ + { role: 'system', content: largeContent1 }, + { role: 'user', content: largeContent2 }, + { role: 'user', content: smallContent }, + ], + temperature: 0.7, + }); }); } diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-embeddings.mjs deleted file mode 100644 index b2e5cf3206fe..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/truncation/scenario-message-truncation-embeddings.mjs +++ /dev/null @@ -1,66 +0,0 @@ -import { instrumentOpenAiClient } from '@sentry/core'; -import * as Sentry from '@sentry/node'; - -class MockOpenAI { - constructor(config) { - this.apiKey = config.apiKey; - - this.embeddings = { - create: async params => { - await new Promise(resolve => setTimeout(resolve, 10)); - - return { - object: 'list', - data: [ - { - object: 'embedding', - embedding: [0.1, 0.2, 0.3], - index: 0, - }, - ], - model: params.model, - usage: { - prompt_tokens: 10, - total_tokens: 10, - }, - }; - }, - }; - } -} - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockOpenAI({ - apiKey: 'mock-api-key', - }); - - const client = instrumentOpenAiClient(mockClient); - - // Create 1 large input that gets truncated to fit within the 20KB limit - const largeContent = 'A'.repeat(25000) + 'B'.repeat(25000); // ~50KB gets truncated to include only As - - await client.embeddings.create({ - input: largeContent, - model: 'text-embedding-3-small', - dimensions: 1536, - encoding_format: 'float', - }); - - // Create 3 large inputs where: - // - First 2 inputs are very large (will be dropped) - // - Last input is large but will be truncated to fit within the 20KB limit - const largeContent1 = 'A'.repeat(15000); // ~15KB - const largeContent2 = 'B'.repeat(15000); // ~15KB - const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) - - await client.embeddings.create({ - input: [largeContent1, largeContent2, largeContent3], - model: 'text-embedding-3-small', - dimensions: 1536, - encoding_format: 'float', - }); - }); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/scenario-embeddings.mjs index f6cbe1160bf5..42c6a94c5199 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/scenario-embeddings.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/scenario-embeddings.mjs @@ -67,6 +67,12 @@ async function run() { } catch { // Error is expected and handled } + + // Third test: embeddings API with multiple inputs + await client.embeddings.create({ + input: ['First input text', 'Second input text', 'Third input text'], + model: 'text-embedding-3-small', + }); }); server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 3784fb7e4631..9b4120b143e4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -160,8 +160,7 @@ describe('OpenAI integration (V6)', () => { 'gen_ai.request.model': 'gpt-3.5-turbo', 'gen_ai.request.temperature': 0.7, 'gen_ai.request.messages.original_length': 2, - 'gen_ai.request.messages': - '[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is the capital of France?"}]', + 'gen_ai.request.messages': '[{"role":"user","content":"What is the capital of France?"}]', 'gen_ai.response.model': 'gpt-3.5-turbo', 'gen_ai.response.id': 'chatcmpl-mock123', 'gen_ai.response.finish_reasons': '["stop"]', @@ -234,8 +233,7 @@ describe('OpenAI integration (V6)', () => { 'gen_ai.request.temperature': 0.8, 'gen_ai.request.stream': true, 'gen_ai.request.messages.original_length': 2, - 'gen_ai.request.messages': - '[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Tell me about streaming"}]', + 'gen_ai.request.messages': '[{"role":"user","content":"Tell me about streaming"}]', 'gen_ai.response.text': 'Hello from OpenAI streaming!', 'gen_ai.response.finish_reasons': '["stop"]', 'gen_ai.response.id': 'chatcmpl-stream-123', @@ -381,7 +379,7 @@ describe('OpenAI integration (V6)', () => { 'gen_ai.request.model': 'text-embedding-3-small', 'gen_ai.request.encoding_format': 'float', 'gen_ai.request.dimensions': 1536, - 'gen_ai.request.messages': 'Embedding test!', + 'gen_ai.embeddings.input': 'Embedding test!', 'gen_ai.response.model': 'text-embedding-3-small', 'gen_ai.usage.input_tokens': 10, 'gen_ai.usage.total_tokens': 10, @@ -401,13 +399,33 @@ describe('OpenAI integration (V6)', () => { 'sentry.origin': 'auto.ai.openai', 'gen_ai.system': 'openai', 'gen_ai.request.model': 'error-model', - 'gen_ai.request.messages': 'Error embedding test!', + 'gen_ai.embeddings.input': 'Error embedding test!', }, description: 'embeddings error-model', op: 'gen_ai.embeddings', origin: 'auto.ai.openai', status: 'internal_error', }), + // Third span - embeddings API with multiple inputs (this does not get truncated) + expect.objectContaining({ + data: { + 'gen_ai.operation.name': 'embeddings', + 'sentry.op': 'gen_ai.embeddings', + 'sentry.origin': 'auto.ai.openai', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'text-embedding-3-small', + 'gen_ai.embeddings.input': '["First input text","Second input text","Third input text"]', + 'gen_ai.response.model': 'text-embedding-3-small', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.total_tokens': 10, + 'openai.response.model': 'text-embedding-3-small', + 'openai.usage.prompt_tokens': 10, + }, + description: 'embeddings text-embedding-3-small', + op: 'gen_ai.embeddings', + origin: 'auto.ai.openai', + status: 'ok', + }), ]), }; diff --git a/packages/core/src/tracing/ai/gen-ai-attributes.ts b/packages/core/src/tracing/ai/gen-ai-attributes.ts index 7959ee05bcdf..4fa7274d7281 100644 --- a/packages/core/src/tracing/ai/gen-ai-attributes.ts +++ b/packages/core/src/tracing/ai/gen-ai-attributes.ts @@ -211,6 +211,12 @@ export const GEN_AI_GENERATE_OBJECT_DO_GENERATE_OPERATION_ATTRIBUTE = 'gen_ai.ge */ export const GEN_AI_STREAM_OBJECT_DO_STREAM_OPERATION_ATTRIBUTE = 'gen_ai.stream_object'; +/** + * The embeddings input + * Only recorded when recordInputs is enabled + */ +export const GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE = 'gen_ai.embeddings.input'; + /** * The span operation name for embedding */ diff --git a/packages/core/src/tracing/ai/messageTruncation.ts b/packages/core/src/tracing/ai/messageTruncation.ts index 9c8718387404..f5c040892dcf 100644 --- a/packages/core/src/tracing/ai/messageTruncation.ts +++ b/packages/core/src/tracing/ai/messageTruncation.ts @@ -294,11 +294,17 @@ function truncatePartsMessage(message: PartsMessage, maxBytes: number): unknown[ * @returns Array containing the truncated message, or empty array if truncation fails */ function truncateSingleMessage(message: unknown, maxBytes: number): unknown[] { - /* c8 ignore start - unreachable */ - if (!message || typeof message !== 'object') { + if (!message) return []; + + // Handle plain strings (e.g., embeddings input) + if (typeof message === 'string') { + const truncated = truncateTextByBytes(message, maxBytes); + return truncated ? [truncated] : []; + } + + if (typeof message !== 'object') { return []; } - /* c8 ignore stop */ if (isContentMessage(message)) { return truncateContentMessage(message, maxBytes); @@ -374,19 +380,19 @@ function stripInlineMediaFromMessages(messages: unknown[]): unknown[] { * Truncate an array of messages to fit within a byte limit. * * Strategy: - * - Keeps the newest messages (from the end of the array) - * - Uses O(n) algorithm: precompute sizes once, then find largest suffix under budget - * - If no complete messages fit, attempts to truncate the newest single message + * - Always keeps only the last (newest) message + * - Strips inline media from the message + * - Truncates the message content if it exceeds the byte limit * * @param messages - Array of messages to truncate - * @param maxBytes - Maximum total byte limit for all messages - * @returns Truncated array of messages + * @param maxBytes - Maximum total byte limit for the message + * @returns Array containing only the last message (possibly truncated) * * @example * ```ts * const messages = [msg1, msg2, msg3, msg4]; // newest is msg4 * const truncated = truncateMessagesByBytes(messages, 10000); - * // Returns [msg3, msg4] if they fit, or [msg4] if only it fits, etc. + * // Returns [msg4] (truncated if needed) * ``` */ function truncateMessagesByBytes(messages: unknown[], maxBytes: number): unknown[] { @@ -395,46 +401,21 @@ function truncateMessagesByBytes(messages: unknown[], maxBytes: number): unknown return messages; } - // strip inline media first. This will often get us below the threshold, - // while preserving human-readable information about messages sent. - const stripped = stripInlineMediaFromMessages(messages); - - // Fast path: if all messages fit, return as-is - const totalBytes = jsonBytes(stripped); - if (totalBytes <= maxBytes) { - return stripped; - } - - // Precompute each message's JSON size once for efficiency - const messageSizes = stripped.map(jsonBytes); + // Always keep only the last message + const lastMessage = messages[messages.length - 1]; - // Find the largest suffix (newest messages) that fits within the budget - let bytesUsed = 0; - let startIndex = stripped.length; // Index where the kept suffix starts + // Strip inline media from the single message + const stripped = stripInlineMediaFromMessages([lastMessage]); + const strippedMessage = stripped[0]; - for (let i = stripped.length - 1; i >= 0; i--) { - const messageSize = messageSizes[i]; - - if (messageSize && bytesUsed + messageSize > maxBytes) { - // Adding this message would exceed the budget - break; - } - - if (messageSize) { - bytesUsed += messageSize; - } - startIndex = i; - } - - // If no complete messages fit, try truncating just the newest message - if (startIndex === stripped.length) { - // we're truncating down to one message, so all others dropped. - const newestMessage = stripped[stripped.length - 1]; - return truncateSingleMessage(newestMessage, maxBytes); + // Check if it fits + const messageBytes = jsonBytes(strippedMessage); + if (messageBytes <= maxBytes) { + return stripped; } - // Return the suffix that fits - return stripped.slice(startIndex); + // Truncate the single message if needed + return truncateSingleMessage(strippedMessage, maxBytes); } /** diff --git a/packages/core/src/tracing/openai/index.ts b/packages/core/src/tracing/openai/index.ts index 6789f5fca3ce..51ac5ac4901b 100644 --- a/packages/core/src/tracing/openai/index.ts +++ b/packages/core/src/tracing/openai/index.ts @@ -5,6 +5,7 @@ import { SPAN_STATUS_ERROR } from '../../tracing'; import { startSpan, startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; import { + GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, @@ -12,6 +13,7 @@ import { GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, + OPENAI_OPERATIONS, } from '../ai/gen-ai-attributes'; import { getTruncatedJsonString } from '../ai/utils'; import { instrumentStream } from './streaming'; @@ -107,16 +109,50 @@ function addResponseAttributes(span: Span, result: unknown, recordOutputs?: bool } // Extract and record AI request inputs, if present. This is intentionally separate from response attributes. -function addRequestAttributes(span: Span, params: Record): void { - const src = 'input' in params ? params.input : 'messages' in params ? params.messages : undefined; - // typically an array, but can be other types. skip if an empty array. - const length = Array.isArray(src) ? src.length : undefined; - if (src && length !== 0) { - const truncatedInput = getTruncatedJsonString(src); - span.setAttribute(GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, truncatedInput); - if (length) { - span.setAttribute(GEN_AI_REQUEST_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, length); +function addRequestAttributes(span: Span, params: Record, operationName: string): void { + // Store embeddings input on a separate attribute and do not truncate it + if (operationName === OPENAI_OPERATIONS.EMBEDDINGS && 'input' in params) { + const input = params.input; + + // No input provided + if (input == null) { + return; + } + + // Empty input string + if (typeof input === 'string' && input.length === 0) { + return; } + + // Empty array input + if (Array.isArray(input) && input.length === 0) { + return; + } + + // Store strings as-is, arrays/objects as JSON + span.setAttribute(GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, typeof input === 'string' ? input : JSON.stringify(input)); + return; + } + + // Apply truncation to chat completions / responses API inputs + const src = 'input' in params ? params.input : 'messages' in params ? params.messages : undefined; + + // No input/messages provided + if (!src) { + return; + } + + // Empty array input + if (Array.isArray(src) && src.length === 0) { + return; + } + + const truncatedInput = getTruncatedJsonString(src); + span.setAttribute(GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, truncatedInput); + + // Record original length if it's an array + if (Array.isArray(src)) { + span.setAttribute(GEN_AI_REQUEST_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, src.length); } } @@ -150,7 +186,7 @@ function instrumentMethod( async (span: Span) => { try { if (options.recordInputs && params) { - addRequestAttributes(span, params); + addRequestAttributes(span, params, operationName); } const result = await originalMethod.apply(context, args); @@ -189,7 +225,7 @@ function instrumentMethod( async (span: Span) => { try { if (options.recordInputs && params) { - addRequestAttributes(span, params); + addRequestAttributes(span, params, operationName); } const result = await originalMethod.apply(context, args); diff --git a/packages/core/test/lib/tracing/ai-message-truncation.test.ts b/packages/core/test/lib/tracing/ai-message-truncation.test.ts index 968cd2308bb7..8a8cefaffa5b 100644 --- a/packages/core/test/lib/tracing/ai-message-truncation.test.ts +++ b/packages/core/test/lib/tracing/ai-message-truncation.test.ts @@ -96,33 +96,8 @@ describe('message truncation utilities', () => { // original messages objects must not be mutated expect(JSON.stringify(messages, null, 2)).toBe(messagesJson); + // only the last message should be kept (with media stripped) expect(result).toStrictEqual([ - { - role: 'user', - content: [ - { - type: 'image', - source: { - type: 'base64', - media_type: 'image/png', - data: removed, - }, - }, - ], - }, - { - role: 'user', - content: { - image_url: removed, - }, - }, - { - role: 'agent', - type: 'image', - content: { - b64_json: removed, - }, - }, { role: 'system', inlineData: { @@ -177,39 +152,35 @@ describe('message truncation utilities', () => { const giant = 'this is a long string '.repeat(1_000); const big = 'this is a long string '.repeat(100); - it('drops older messages to fit in the limit', () => { + it('keeps only the last message without truncation when it fits the limit', () => { + // Multiple messages that together exceed 20KB, but last message is small const messages = [ - `0 ${giant}`, - { type: 'text', content: `1 ${big}` }, - { type: 'text', content: `2 ${big}` }, - { type: 'text', content: `3 ${giant}` }, - { type: 'text', content: `4 ${big}` }, - `5 ${big}`, - { type: 'text', content: `6 ${big}` }, - { type: 'text', content: `7 ${big}` }, - { type: 'text', content: `8 ${big}` }, - { type: 'text', content: `9 ${big}` }, - { type: 'text', content: `10 ${big}` }, - { type: 'text', content: `11 ${big}` }, - { type: 'text', content: `12 ${big}` }, + { content: `1 ${humongous}` }, + { content: `2 ${humongous}` }, + { content: `3 ${big}` }, // last message - small enough to fit ]; - const messagesJson = JSON.stringify(messages, null, 2); const result = truncateGenAiMessages(messages); - // should not mutate original messages list - expect(JSON.stringify(messages, null, 2)).toBe(messagesJson); - // just retain the messages that fit in the budget - expect(result).toStrictEqual([ - `5 ${big}`, - { type: 'text', content: `6 ${big}` }, - { type: 'text', content: `7 ${big}` }, - { type: 'text', content: `8 ${big}` }, - { type: 'text', content: `9 ${big}` }, - { type: 'text', content: `10 ${big}` }, - { type: 'text', content: `11 ${big}` }, - { type: 'text', content: `12 ${big}` }, - ]); + // Should only keep the last message, unchanged + expect(result).toStrictEqual([{ content: `3 ${big}` }]); + }); + + it('keeps only the last message with truncation when it does not fit the limit', () => { + const messages = [{ content: `1 ${humongous}` }, { content: `2 ${humongous}` }, { content: `3 ${humongous}` }]; + const result = truncateGenAiMessages(messages); + const truncLen = 20_000 - JSON.stringify({ content: '' }).length; + expect(result).toStrictEqual([{ content: `3 ${humongous}`.substring(0, truncLen) }]); + }); + + it('drops if last message cannot be safely truncated', () => { + const messages = [ + { content: `1 ${humongous}` }, + { content: `2 ${humongous}` }, + { what_even_is_this: `? ${humongous}` }, + ]; + const result = truncateGenAiMessages(messages); + expect(result).toStrictEqual([]); }); it('fully drops message if content cannot be made to fit', () => { @@ -315,22 +286,5 @@ describe('message truncation utilities', () => { }, ]); }); - - it('truncates first message if none fit', () => { - const messages = [{ content: `1 ${humongous}` }, { content: `2 ${humongous}` }, { content: `3 ${humongous}` }]; - const result = truncateGenAiMessages(messages); - const truncLen = 20_000 - JSON.stringify({ content: '' }).length; - expect(result).toStrictEqual([{ content: `3 ${humongous}`.substring(0, truncLen) }]); - }); - - it('drops if first message cannot be safely truncated', () => { - const messages = [ - { content: `1 ${humongous}` }, - { content: `2 ${humongous}` }, - { what_even_is_this: `? ${humongous}` }, - ]; - const result = truncateGenAiMessages(messages); - expect(result).toStrictEqual([]); - }); }); });