From 218c42e0c806cc10a1184e6d243beeb3a21306bc Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 12 Jan 2026 16:42:42 -0700 Subject: [PATCH 1/3] Reapply "Add wrapTools() for LangChain4j tool instrumentation" This reverts commit 4c26f5bdec3c98a3b1332969b06485d05075f2bd. --- README.md | 59 +++++ build.gradle | 2 + examples/build.gradle | 14 ++ .../LangchainToolWrappingExample.java | 158 +++++++++++++ .../langchain/BraintrustLangchain.java | 22 ++ .../langchain/ByteBuddyToolWrapper.java | 119 ++++++++++ .../langchain/BraintrustLangchainTest.java | 216 ++++++++++++++++++ .../instrumentation/langchain/TestTools.java | 22 ++ 8 files changed, 612 insertions(+) create mode 100644 examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java create mode 100644 src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java create mode 100644 src/test/java/dev/braintrust/instrumentation/langchain/TestTools.java diff --git a/README.md b/README.md index 0d33aaa..7bd307f 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,65 @@ var request = var response = openAIClient.chat().completions().create(request); ``` +### LangChain4j Instrumentation + +```java +var braintrust = Braintrust.get(); +var openTelemetry = braintrust.openTelemetryCreate(); + +// Wrap the chat model to trace LLM calls +ChatModel model = BraintrustLangchain.wrap( + openTelemetry, + OpenAiChatModel.builder() + .apiKey(System.getenv("OPENAI_API_KEY")) + .modelName("gpt-4o-mini") + .temperature(0.0)); + +var response = model.chat("What is the capital of France?"); +``` + +#### Tool Wrapping + +Use `BraintrustLangchain.wrapTools()` to automatically trace tool executions in your LangChain4j agents: + +```java +// Create your tool class +public class WeatherTools { + @Tool("Get current weather for a location") + public String getWeather(String location) { + return "The weather in " + location + " is sunny."; + } +} + +// Wrap tools to create spans for each tool execution +WeatherTools tools = new WeatherTools(); +WeatherTools instrumentedTools = BraintrustLangchain.wrapTools(openTelemetry, tools); + +// Use instrumented tools in your AI service +Assistant assistant = AiServices.builder(Assistant.class) + .chatModel(model) + .tools(instrumentedTools) + .build(); +``` + +Each tool call will automatically create an OpenTelemetry span in Braintrust with: +- Tool name and parameters +- Execution duration +- Return values +- Any exceptions thrown + +**Note:** For proper display in the Braintrust UI, ensure parent spans (conversation, turn, etc.) also set the required Braintrust attributes: +```java +var span = tracer.spanBuilder("my-span").startSpan(); +span.setAttribute("braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"my-span\"}"); +span.setAttribute("braintrust.input_json", "{\"user_message\":\"...\"}"); +// ... do work ... +span.setAttribute("braintrust.output_json", "{\"result\":\"...\"}"); +span.end(); +``` + +See [LangchainToolWrappingExample.java](./examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java) for a complete example with proper span hierarchy. + ## Running Examples Example source code can be found [here](./examples/src/main/java/dev/braintrust/examples) diff --git a/build.gradle b/build.gradle index ad1b078..90d9c9f 100644 --- a/build.gradle +++ b/build.gradle @@ -95,6 +95,8 @@ dependencies { testImplementation "dev.langchain4j:langchain4j:${langchainVersion}" testImplementation "dev.langchain4j:langchain4j-http-client:${langchainVersion}" testImplementation "dev.langchain4j:langchain4j-open-ai:${langchainVersion}" + + implementation 'net.bytebuddy:byte-buddy:1.14.11' // ByteBuddy for LangChain4j tool wrapping } /** diff --git a/examples/build.gradle b/examples/build.gradle index f4e9c20..553af25 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -172,3 +172,17 @@ task runLangchain(type: JavaExec) { suspend = false } } + +task runLangchainToolWrapping(type: JavaExec) { + group = 'Braintrust SDK Examples' + description = 'Run the LangChain4j tool wrapping example. NOTE: this requires OPENAI_API_KEY to be exported and will make a small call to openai, using your tokens' + classpath = sourceSets.main.runtimeClasspath + mainClass = 'dev.braintrust.examples.LangchainToolWrappingExample' + systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel + debugOptions { + enabled = true + port = 5566 + server = true + suspend = false + } +} diff --git a/examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java b/examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java new file mode 100644 index 0000000..102880d --- /dev/null +++ b/examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java @@ -0,0 +1,158 @@ +package dev.braintrust.examples; + +import dev.braintrust.Braintrust; +import dev.braintrust.instrumentation.langchain.BraintrustLangchain; +import dev.langchain4j.agent.tool.Tool; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.openai.OpenAiChatModel; +import dev.langchain4j.service.AiServices; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; + +/** + * Demonstrates how to use BraintrustLangchain.wrapTools() to automatically trace tool executions. + * + *

This example shows: + * + *

+ */ +public class LangchainToolWrappingExample { + + /** Example tool class with weather-related methods */ + public static class WeatherTools { + @Tool("Get current weather for a location") + public String getWeather(String location) { + // Simulate a weather API call + return String.format("The weather in %s is sunny with 72°F temperature.", location); + } + + @Tool("Get weather forecast for next N days") + public String getForecast(String location, int days) { + // Simulate a forecast API call + return String.format( + "The %d-day forecast for %s: Mostly sunny with temperatures between 65-75°F.", + days, location); + } + } + + /** AI Service interface for the assistant */ + interface Assistant { + String chat(String userMessage); + } + + public static void main(String[] args) throws Exception { + if (null == System.getenv("OPENAI_API_KEY")) { + System.err.println( + "\nWARNING envar OPENAI_API_KEY not found. This example will likely fail.\n"); + } + + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); + var tracer = openTelemetry.getTracer("langchain-tool-wrapping"); + + System.out.println("\n=== LangChain4j Tool Wrapping Example ===\n"); + + // Create root span for the conversation + var conversationSpan = tracer.spanBuilder("weather-assistant-conversation").startSpan(); + conversationSpan.setAttribute( + "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"conversation\"}"); + conversationSpan.setAttribute( + "braintrust.input_json", + "{\"description\":\"Weather assistant with tool wrapping\",\"turns\":2}"); + + try (var ignored = conversationSpan.makeCurrent()) { + // Wrap the LLM with Braintrust instrumentation + ChatModel model = + BraintrustLangchain.wrap( + openTelemetry, + OpenAiChatModel.builder() + .apiKey(System.getenv("OPENAI_API_KEY")) + .modelName("gpt-4o-mini") + .temperature(0.0)); + + // Create tools and wrap them with Braintrust instrumentation + WeatherTools tools = new WeatherTools(); + WeatherTools instrumentedTools = BraintrustLangchain.wrapTools(openTelemetry, tools); + + System.out.println("Tools wrapped with Braintrust instrumentation"); + System.out.println("Each tool call will create a span in Braintrust\n"); + + // Create AI service with the instrumented tools + Assistant assistant = + AiServices.builder(Assistant.class) + .chatModel(model) + .tools(instrumentedTools) + .build(); + + // Example 1: Single tool call + System.out.println("--- Turn 1: Single Tool Call ---"); + String query1 = "What's the weather in San Francisco?"; + System.out.println("User: " + query1); + Span turn1 = tracer.spanBuilder("turn_1").startSpan(); + turn1.setAttribute( + "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"turn_1\"}"); + turn1.setAttribute("braintrust.input_json", "{\"user_message\":\"" + query1 + "\"}"); + String response1; + try (Scope scope = turn1.makeCurrent()) { + response1 = assistant.chat(query1); + System.out.println("Assistant: " + response1); + turn1.setAttribute( + "braintrust.output_json", + "{\"assistant_message\":\"" + response1.replace("\"", "\\\"") + "\"}"); + } finally { + turn1.end(); + } + System.out.println(); + + // Example 2: Tool with multiple parameters + System.out.println("--- Turn 2: Multiple Parameters ---"); + String query2 = "What's the 5-day forecast for Tokyo?"; + System.out.println("User: " + query2); + Span turn2 = tracer.spanBuilder("turn_2").startSpan(); + turn2.setAttribute( + "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"turn_2\"}"); + turn2.setAttribute("braintrust.input_json", "{\"user_message\":\"" + query2 + "\"}"); + String response2; + try (Scope scope = turn2.makeCurrent()) { + response2 = assistant.chat(query2); + System.out.println("Assistant: " + response2); + turn2.setAttribute( + "braintrust.output_json", + "{\"assistant_message\":\"" + response2.replace("\"", "\\\"") + "\"}"); + } finally { + turn2.end(); + } + System.out.println(); + + } finally { + conversationSpan.setAttribute( + "braintrust.output_json", "{\"status\":\"completed\",\"turns\":2}"); + conversationSpan.end(); + } + + // Flush traces before exit + if (openTelemetry instanceof OpenTelemetrySdk) { + ((OpenTelemetrySdk) openTelemetry).close(); + } + + var url = + braintrust.projectUri() + + "/logs?r=%s&s=%s" + .formatted( + conversationSpan.getSpanContext().getTraceId(), + conversationSpan.getSpanContext().getSpanId()); + + System.out.println("Example complete!"); + System.out.println("\nIn Braintrust, you'll see:"); + System.out.println(" • Root conversation span"); + System.out.println(" • Nested turn spans for each user interaction"); + System.out.println(" • LLM call spans (from BraintrustLangchain.wrap())"); + System.out.println(" • Tool execution spans (from BraintrustLangchain.wrapTools())"); + System.out.println("\n View your traces: " + url + "\n"); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java b/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java index a3aafa8..9645f88 100644 --- a/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java +++ b/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java @@ -49,6 +49,28 @@ public static OpenAiStreamingChatModel wrap( } } + /** + * Wrap a tools object to instrument @Tool method executions with Braintrust traces. Returns a + * proxy that intercepts all @Tool annotated methods and creates OpenTelemetry spans. + * + *

Usage: StoryTools tools = new StoryTools(); StoryTools instrumented = + * BraintrustLangchain.wrapTools(openTelemetry, tools); + * AiServices.builder(Assistant.class).chatModel(model).tools(instrumented).build() + * + * @param otel OpenTelemetry instance from braintrust.openTelemetryCreate() + * @param tools Tool object with @Tool annotated methods + * @return Proxied tool object that creates spans for each @Tool method call + */ + @SuppressWarnings("unchecked") + public static T wrapTools(OpenTelemetry otel, T tools) { + try { + return (T) new ByteBuddyToolWrapper(otel).wrap(tools); + } catch (Exception e) { + log.warn("Failed to wrap tools with instrumentation, returning original", e); + return tools; + } + } + private static HttpClientBuilder wrap( OpenTelemetry otel, HttpClientBuilder builder, Options options) { return new WrappedHttpClientBuilder(otel, builder, options); diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java b/src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java new file mode 100644 index 0000000..0c4ba25 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java @@ -0,0 +1,119 @@ +package dev.braintrust.instrumentation.langchain; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.agent.tool.Tool; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.*; +import net.bytebuddy.matcher.ElementMatchers; + +/** + * Uses ByteBuddy to create runtime subclass proxies that intercept @Tool methods and add + * OpenTelemetry spans with Braintrust attributes. + */ +@Slf4j +class ByteBuddyToolWrapper { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private final Tracer tracer; + + ByteBuddyToolWrapper(OpenTelemetry otel) { + this.tracer = otel.getTracer("braintrust"); + } + + @SuppressWarnings("unchecked") + public T wrap(T originalTools) throws Exception { + Class toolClass = originalTools.getClass(); + + // Create subclass with interceptor, preserving all annotations + Class proxyClass = + new ByteBuddy() + .subclass(toolClass) + .method(ElementMatchers.isAnnotatedWith(Tool.class)) + .intercept( + MethodDelegation.to( + new ToolMethodInterceptor(tracer, originalTools))) + .attribute( + net.bytebuddy.implementation.attribute.MethodAttributeAppender + .ForInstrumentedMethod.INCLUDING_RECEIVER) + .make() + .load(toolClass.getClassLoader()) + .getLoaded(); + + // Create instance - ByteBuddy subclass will delegate to original + return (T) proxyClass.getDeclaredConstructor().newInstance(); + } + + /** Interceptor that wraps @Tool method calls with OpenTelemetry spans */ + public static class ToolMethodInterceptor { + private final Tracer tracer; + private final Object originalTools; + + public ToolMethodInterceptor(Tracer tracer, Object originalTools) { + this.tracer = tracer; + this.originalTools = originalTools; + } + + @RuntimeType + public Object intercept( + @Origin Method method, @AllArguments Object[] args, @SuperCall Callable zuper) + throws Exception { + + String toolName = method.getName(); + + // Build input map from parameters + Map inputMap = new HashMap<>(); + var parameters = method.getParameters(); + for (int i = 0; i < parameters.length && i < args.length; i++) { + inputMap.put(parameters[i].getName(), args[i]); + } + + Span span = tracer.spanBuilder(toolName).startSpan(); + try (Scope scope = span.makeCurrent()) { + // Set Braintrust span attributes + span.setAttribute( + "braintrust.span_attributes", + json(Map.of("type", "tool", "name", toolName))); + + // Set input + span.setAttribute("braintrust.input_json", json(inputMap)); + + // Execute method and measure time + long startTime = System.nanoTime(); + Object result = zuper.call(); // Call original method + long endTime = System.nanoTime(); + + // Set output + span.setAttribute("braintrust.output_json", json(result)); + + // Set metrics + double executionTime = (endTime - startTime) / 1_000_000_000.0; + span.setAttribute( + "braintrust.metrics", json(Map.of("execution_time", executionTime))); + + return result; + } catch (Throwable t) { + span.setStatus(StatusCode.ERROR, t.getMessage()); + span.recordException(t); + throw t; + } finally { + span.end(); + } + } + + @SneakyThrows + private static String json(Object o) { + return JSON_MAPPER.writeValueAsString(o); + } + } +} diff --git a/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java b/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java index e2555bd..67d0927 100644 --- a/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java +++ b/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java @@ -13,6 +13,8 @@ import dev.langchain4j.model.openai.OpenAiChatModel; import dev.langchain4j.model.openai.OpenAiStreamingChatModel; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.StatusCode; +import java.util.List; import java.util.concurrent.CompletableFuture; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; @@ -215,4 +217,218 @@ public void onError(Throwable error) { "Output should contain the complete streamed response"); assertNotNull(choice.get("finish_reason"), "Output should have finish_reason"); } + + @Test + @SneakyThrows + void testToolWrapping() { + // Create and wrap tools + TestTools tools = new TestTools(); + TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); + + // Call wrapped tool method directly + String result = wrappedTools.getWeather("Paris"); + assertNotNull(result); + assertTrue(result.contains("Paris")); + + // Verify span was created + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size(), "Expected one span for tool execution"); + var span = spans.get(0); + + // Verify span name + assertEquals("getWeather", span.getName()); + + // Verify span type + var attributes = span.getAttributes(); + String spanAttrsJson = attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson); + assertEquals("tool", spanAttrs.get("type").asText()); + assertEquals("getWeather", spanAttrs.get("name").asText()); + + // Verify input (parameter names may be arg0, arg1, etc without -parameters flag) + String inputJson = attributes.get(AttributeKey.stringKey("braintrust.input_json")); + assertNotNull(inputJson); + JsonNode input = JSON_MAPPER.readTree(inputJson); + assertTrue(input.isObject(), "Input should be an object"); + assertTrue(input.size() > 0, "Input should have at least one parameter"); + // Check if parameter value is present (either as "location" or "arg0") + String paramValue = + input.has("location") + ? input.get("location").asText() + : input.elements().next().asText(); + assertEquals("Paris", paramValue); + + // Verify output + String outputJson = attributes.get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson); + JsonNode output = JSON_MAPPER.readTree(outputJson); + assertTrue(output.asText().contains("Paris")); + assertTrue(output.asText().contains("72")); + + // Verify metrics + String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); + JsonNode metrics = JSON_MAPPER.readTree(metricsJson); + assertTrue(metrics.has("execution_time")); + assertTrue(metrics.get("execution_time").asDouble() >= 0); + } + + @Test + @SneakyThrows + void testToolWrappingWithException() { + TestTools tools = new TestTools(); + TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); + + // Execute and expect exception + assertThrows( + RuntimeException.class, + () -> { + wrappedTools.throwError(); + }); + + // Verify span with error status + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + var span = spans.get(0); + + assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); + assertTrue( + span.getEvents().stream().anyMatch(e -> e.getName().equals("exception")), + "Span should have exception event"); + } + + @Test + @SneakyThrows + void testToolWrappingWithMultipleCalls() { + TestTools tools = new TestTools(); + TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); + + // Call multiple tool methods + wrappedTools.getWeather("Tokyo"); + int sum = wrappedTools.calculateSum(5, 7); + assertEquals(12, sum); + + // Verify two spans created + var spans = testHarness.awaitExportedSpans(); + assertEquals(2, spans.size(), "Expected two spans"); + + // Verify first span (getWeather) + var weatherSpan = spans.get(0); + assertEquals("getWeather", weatherSpan.getName()); + + // Verify second span (calculateSum) + var sumSpan = spans.get(1); + assertEquals("calculateSum", sumSpan.getName()); + String sumOutput = + sumSpan.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertEquals("12", sumOutput); + } + + @Test + @SneakyThrows + void testToolWrappingAllBraintrustAttributesPresent() { + // This test verifies ALL required Braintrust attributes are present + // to catch issues that would cause UI problems + TestTools tools = new TestTools(); + TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); + + wrappedTools.getWeather("London"); + + var spans = testHarness.awaitExportedSpans(); + var span = spans.get(0); + var attributes = span.getAttributes(); + + // CRITICAL: These attributes MUST be present for Braintrust UI to display properly + assertNotNull( + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")), + "braintrust.span_attributes is required for UI"); + assertNotNull( + attributes.get(AttributeKey.stringKey("braintrust.input_json")), + "braintrust.input_json is required for UI to show inputs"); + assertNotNull( + attributes.get(AttributeKey.stringKey("braintrust.output_json")), + "braintrust.output_json is required for UI to show outputs"); + assertNotNull( + attributes.get(AttributeKey.stringKey("braintrust.metrics")), + "braintrust.metrics is required for UI to show metrics"); + + // Verify span_attributes has correct structure + String spanAttrsJson = attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson); + assertTrue(spanAttrs.has("type"), "span_attributes must have 'type' field"); + assertTrue(spanAttrs.has("name"), "span_attributes must have 'name' field"); + assertEquals("tool", spanAttrs.get("type").asText(), "Tool spans must have type='tool'"); + } + + @Test + @SneakyThrows + void testToolWrappingIntegrationWithConversationHierarchy() { + // This test simulates a realistic usage pattern like the example + // and verifies ALL spans in the hierarchy have input/output for UI + var tracer = testHarness.openTelemetry().getTracer("test"); + TestTools tools = new TestTools(); + TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); + + // Create conversation span (like example does) + var conversationSpan = tracer.spanBuilder("conversation").startSpan(); + conversationSpan.setAttribute( + "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"conversation\"}"); + conversationSpan.setAttribute( + "braintrust.input_json", "{\"description\":\"test conversation\"}"); + + try (var ignored = conversationSpan.makeCurrent()) { + // Create turn span (like example does) + var turnSpan = tracer.spanBuilder("turn_1").startSpan(); + turnSpan.setAttribute( + "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"turn_1\"}"); + turnSpan.setAttribute("braintrust.input_json", "{\"user_message\":\"test query\"}"); + + try (var turnScope = turnSpan.makeCurrent()) { + // Call tool within turn (tool wrapper should create tool span) + String result = wrappedTools.getWeather("Paris"); + turnSpan.setAttribute( + "braintrust.output_json", "{\"assistant_message\":\"" + result + "\"}"); + } finally { + turnSpan.end(); + } + } finally { + conversationSpan.setAttribute("braintrust.output_json", "{\"status\":\"completed\"}"); + conversationSpan.end(); + } + + // Verify all 3 spans exist and have required attributes + var spans = testHarness.awaitExportedSpans(); + assertEquals(3, spans.size(), "Expected 3 spans: conversation, turn, and tool"); + + // Find each span type + var toolSpan = + spans.stream().filter(s -> s.getName().equals("getWeather")).findFirst().get(); + var turnSpanData = + spans.stream().filter(s -> s.getName().equals("turn_1")).findFirst().get(); + var convSpan = + spans.stream().filter(s -> s.getName().equals("conversation")).findFirst().get(); + + // CRITICAL: Every span must have input/output for UI to display properly + for (var span : List.of(toolSpan, turnSpanData, convSpan)) { + var attrs = span.getAttributes(); + assertNotNull( + attrs.get(AttributeKey.stringKey("braintrust.span_attributes")), + span.getName() + " missing braintrust.span_attributes"); + assertNotNull( + attrs.get(AttributeKey.stringKey("braintrust.input_json")), + span.getName() + " missing braintrust.input_json - UI won't show input!"); + assertNotNull( + attrs.get(AttributeKey.stringKey("braintrust.output_json")), + span.getName() + " missing braintrust.output_json - UI won't show output!"); + } + + // Verify span hierarchy (parent-child relationships) + assertEquals( + convSpan.getSpanId(), + turnSpanData.getParentSpanId(), + "Turn should be child of conversation"); + assertEquals( + turnSpanData.getSpanId(), + toolSpan.getParentSpanId(), + "Tool should be child of turn"); + } } diff --git a/src/test/java/dev/braintrust/instrumentation/langchain/TestTools.java b/src/test/java/dev/braintrust/instrumentation/langchain/TestTools.java new file mode 100644 index 0000000..b289611 --- /dev/null +++ b/src/test/java/dev/braintrust/instrumentation/langchain/TestTools.java @@ -0,0 +1,22 @@ +package dev.braintrust.instrumentation.langchain; + +import dev.langchain4j.agent.tool.Tool; + +public class TestTools { + + @Tool("Get weather for a location") + public String getWeather(String location) { + return String.format( + "{\"location\":\"%s\",\"temperature\":72,\"condition\":\"sunny\"}", location); + } + + @Tool("Calculate sum") + public int calculateSum(int a, int b) { + return a + b; + } + + @Tool("Tool that throws exception") + public String throwError() { + throw new RuntimeException("Intentional error for testing"); + } +} From 8f864d390ba8283fc7ce0ce1a38e73a9dab14e01 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 12 Jan 2026 22:04:09 -0700 Subject: [PATCH 2/3] record vcr --- ...4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json} | 2 +- ...-68182ecb-89f2-4d6d-9d6a-49cb3f036357.txt} | 17 +++--- ...4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json} | 24 ++++----- ...68182ecb-89f2-4d6d-9d6a-49cb3f036357.json} | 24 ++++----- ...47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json} | 4 +- ...f1d892bb-4783-475f-9320-1a5800d4f293.json} | 4 +- ...47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json} | 12 ++--- ...f1d892bb-4783-475f-9320-1a5800d4f293.json} | 12 ++--- ...13e69bf3-e90d-497f-bb66-47b448dc86bd.json} | 6 +-- ...4706df90-aa52-4bd3-97c0-3448a4e0005f.json} | 6 +-- ...s-541b9306-6880-41df-bb44-69ea82f417f8.txt | 22 ++++++++ ...s-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt | 22 -------- ...-9016066f-1bbd-49ec-a403-4311b29fe2af.txt} | 20 +++---- ...-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json | 54 +++++++++++++++++++ ...-d6eda521-a7dd-4216-911e-4a916e45fca1.json | 36 +++++++++++++ ...13e69bf3-e90d-497f-bb66-47b448dc86bd.json} | 22 ++++---- ...4706df90-aa52-4bd3-97c0-3448a4e0005f.json} | 22 ++++---- ...541b9306-6880-41df-bb44-69ea82f417f8.json} | 22 ++++---- ...9016066f-1bbd-49ec-a403-4311b29fe2af.json} | 20 +++---- ...-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json | 50 +++++++++++++++++ ...-d6eda521-a7dd-4216-911e-4a916e45fca1.json | 50 +++++++++++++++++ 21 files changed, 322 insertions(+), 129 deletions(-) rename src/test/resources/cassettes/anthropic/__files/{v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json => v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json} (70%) rename src/test/resources/cassettes/anthropic/__files/{v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt => v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.txt} (70%) rename src/test/resources/cassettes/anthropic/mappings/{v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json => v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json} (66%) rename src/test/resources/cassettes/anthropic/mappings/{v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json => v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.json} (68%) rename src/test/resources/cassettes/google/__files/{v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json => v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json} (87%) rename src/test/resources/cassettes/google/__files/{v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json => v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json} (87%) rename src/test/resources/cassettes/google/mappings/{v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json => v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json} (79%) rename src/test/resources/cassettes/google/mappings/{v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json => v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json} (79%) rename src/test/resources/cassettes/openai/__files/{chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json => chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json} (86%) rename src/test/resources/cassettes/openai/__files/{chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json => chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json} (86%) create mode 100644 src/test/resources/cassettes/openai/__files/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.txt delete mode 100644 src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt rename src/test/resources/cassettes/openai/__files/{chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt => chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.txt} (50%) create mode 100644 src/test/resources/cassettes/openai/__files/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json create mode 100644 src/test/resources/cassettes/openai/__files/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json rename src/test/resources/cassettes/openai/mappings/{chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json => chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json} (62%) rename src/test/resources/cassettes/openai/mappings/{chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json => chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json} (62%) rename src/test/resources/cassettes/openai/mappings/{chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json => chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.json} (64%) rename src/test/resources/cassettes/openai/mappings/{chat_completions-06107a5c-8505-436f-850b-598871b122f1.json => chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.json} (65%) create mode 100644 src/test/resources/cassettes/openai/mappings/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json create mode 100644 src/test/resources/cassettes/openai/mappings/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json diff --git a/src/test/resources/cassettes/anthropic/__files/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json b/src/test/resources/cassettes/anthropic/__files/v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json similarity index 70% rename from src/test/resources/cassettes/anthropic/__files/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json rename to src/test/resources/cassettes/anthropic/__files/v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json index a174e92..4481d62 100644 --- a/src/test/resources/cassettes/anthropic/__files/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json +++ b/src/test/resources/cassettes/anthropic/__files/v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json @@ -1 +1 @@ -{"model":"claude-3-5-haiku-20241022","id":"msg_01EW7UJJzqRS5ugXCRgvUaHR","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard"}} \ No newline at end of file +{"model":"claude-3-5-haiku-20241022","id":"msg_01MhSBYGdz56JjxtCkyqLNMi","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard"}} \ No newline at end of file diff --git a/src/test/resources/cassettes/anthropic/__files/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt b/src/test/resources/cassettes/anthropic/__files/v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.txt similarity index 70% rename from src/test/resources/cassettes/anthropic/__files/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt rename to src/test/resources/cassettes/anthropic/__files/v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.txt index ac0aa3b..65b3b61 100644 --- a/src/test/resources/cassettes/anthropic/__files/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt +++ b/src/test/resources/cassettes/anthropic/__files/v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.txt @@ -1,21 +1,24 @@ event: message_start -data: {"type":"message_start","message":{"model":"claude-3-5-haiku-20241022","id":"msg_01647tewtck1BoYBQHe7DP6o","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard"}} } +data: {"type":"message_start","message":{"model":"claude-3-5-haiku-20241022","id":"msg_01YTLrex9iVq8MvNEBcvw3cG","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard"}} } event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + +event: ping +data: {"type": "ping"} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The capital of France is"} } +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The capital of France is"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Paris."} } +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Paris."} } event: content_block_stop -data: {"type":"content_block_stop","index":0 } +data: {"type":"content_block_stop","index":0} event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":10} } +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":19,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":10} } event: message_stop -data: {"type":"message_stop" } +data: {"type":"message_stop" } diff --git a/src/test/resources/cassettes/anthropic/mappings/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json b/src/test/resources/cassettes/anthropic/mappings/v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json similarity index 66% rename from src/test/resources/cassettes/anthropic/mappings/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json rename to src/test/resources/cassettes/anthropic/mappings/v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json index 253a39d..7ad4058 100644 --- a/src/test/resources/cassettes/anthropic/mappings/v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json +++ b/src/test/resources/cassettes/anthropic/mappings/v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json @@ -1,5 +1,5 @@ { - "id" : "0f8340d1-48b6-43f6-adf6-f92f304db769", + "id" : "4c99ad9a-3bc5-4f73-bd08-84f892478c4d", "name" : "v1_messages", "request" : { "url" : "/v1/messages", @@ -12,38 +12,38 @@ "bodyPatterns" : [ { "equalToJson" : "{\"max_tokens\":50,\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"claude-3-5-haiku-20241022\",\"system\":\"You are a helpful assistant\",\"temperature\":0.0}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "v1_messages-0f8340d1-48b6-43f6-adf6-f92f304db769.json", + "bodyFileName" : "v1_messages-4c99ad9a-3bc5-4f73-bd08-84f892478c4d.json", "headers" : { - "Date" : "Wed, 31 Dec 2025 05:49:53 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:40 GMT", "Content-Type" : "application/json", "anthropic-ratelimit-requests-limit" : "10000", "anthropic-ratelimit-requests-remaining" : "9999", - "anthropic-ratelimit-requests-reset" : "2025-12-31T05:49:52Z", + "anthropic-ratelimit-requests-reset" : "2026-01-13T18:55:39Z", "anthropic-ratelimit-input-tokens-limit" : "5000000", "anthropic-ratelimit-input-tokens-remaining" : "5000000", - "anthropic-ratelimit-input-tokens-reset" : "2025-12-31T05:49:52Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-01-13T18:55:39Z", "anthropic-ratelimit-output-tokens-limit" : "1000000", "anthropic-ratelimit-output-tokens-remaining" : "1000000", - "anthropic-ratelimit-output-tokens-reset" : "2025-12-31T05:49:52Z", + "anthropic-ratelimit-output-tokens-reset" : "2026-01-13T18:55:40Z", "anthropic-ratelimit-tokens-limit" : "6000000", "anthropic-ratelimit-tokens-remaining" : "6000000", - "anthropic-ratelimit-tokens-reset" : "2025-12-31T05:49:52Z", - "request-id" : "req_011CWeEvg23RAPm3zhy2ymhs", + "anthropic-ratelimit-tokens-reset" : "2026-01-13T18:55:39Z", + "request-id" : "req_011CX5tMzY85axeEQMUkdDni", "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", "Server" : "cloudflare", - "x-envoy-upstream-service-time" : "556", + "x-envoy-upstream-service-time" : "538", "cf-cache-status" : "DYNAMIC", "X-Robots-Tag" : "none", - "CF-RAY" : "9b677f027e99a37b-SEA" + "CF-RAY" : "9bd71befef2c76b2-SEA" } }, - "uuid" : "0f8340d1-48b6-43f6-adf6-f92f304db769", + "uuid" : "4c99ad9a-3bc5-4f73-bd08-84f892478c4d", "persistent" : true, "insertionIndex" : 1 } \ No newline at end of file diff --git a/src/test/resources/cassettes/anthropic/mappings/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json b/src/test/resources/cassettes/anthropic/mappings/v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.json similarity index 68% rename from src/test/resources/cassettes/anthropic/mappings/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json rename to src/test/resources/cassettes/anthropic/mappings/v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.json index 8af192d..9e01635 100644 --- a/src/test/resources/cassettes/anthropic/mappings/v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.json +++ b/src/test/resources/cassettes/anthropic/mappings/v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.json @@ -1,5 +1,5 @@ { - "id" : "db51abc2-016e-4cd2-b863-3c55bd20dc10", + "id" : "68182ecb-89f2-4d6d-9d6a-49cb3f036357", "name" : "v1_messages", "request" : { "url" : "/v1/messages", @@ -12,39 +12,39 @@ "bodyPatterns" : [ { "equalToJson" : "{\"max_tokens\":50,\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"claude-3-5-haiku-20241022\",\"system\":\"You are a helpful assistant\",\"temperature\":0.0,\"stream\":true}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "v1_messages-db51abc2-016e-4cd2-b863-3c55bd20dc10.txt", + "bodyFileName" : "v1_messages-68182ecb-89f2-4d6d-9d6a-49cb3f036357.txt", "headers" : { - "Date" : "Wed, 31 Dec 2025 05:49:53 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:41 GMT", "Content-Type" : "text/event-stream; charset=utf-8", "Cache-Control" : "no-cache", "anthropic-ratelimit-requests-limit" : "10000", "anthropic-ratelimit-requests-remaining" : "9999", - "anthropic-ratelimit-requests-reset" : "2025-12-31T05:49:53Z", + "anthropic-ratelimit-requests-reset" : "2026-01-13T18:55:40Z", "anthropic-ratelimit-input-tokens-limit" : "5000000", "anthropic-ratelimit-input-tokens-remaining" : "5000000", - "anthropic-ratelimit-input-tokens-reset" : "2025-12-31T05:49:53Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-01-13T18:55:40Z", "anthropic-ratelimit-output-tokens-limit" : "1000000", "anthropic-ratelimit-output-tokens-remaining" : "1000000", - "anthropic-ratelimit-output-tokens-reset" : "2025-12-31T05:49:53Z", + "anthropic-ratelimit-output-tokens-reset" : "2026-01-13T18:55:40Z", "anthropic-ratelimit-tokens-limit" : "6000000", "anthropic-ratelimit-tokens-remaining" : "6000000", - "anthropic-ratelimit-tokens-reset" : "2025-12-31T05:49:53Z", - "request-id" : "req_011CWeEvkNUico2HESGkzLt2", + "anthropic-ratelimit-tokens-reset" : "2026-01-13T18:55:40Z", + "request-id" : "req_011CX5tN5bEBPFm9bcjPAmud", "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", "Server" : "cloudflare", - "x-envoy-upstream-service-time" : "461", + "x-envoy-upstream-service-time" : "371", "cf-cache-status" : "DYNAMIC", "X-Robots-Tag" : "none", - "CF-RAY" : "9b677f08d98ca3c8-SEA" + "CF-RAY" : "9bd71bf7499f347d-SEA" } }, - "uuid" : "db51abc2-016e-4cd2-b863-3c55bd20dc10", + "uuid" : "68182ecb-89f2-4d6d-9d6a-49cb3f036357", "persistent" : true, "insertionIndex" : 2 } \ No newline at end of file diff --git a/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json similarity index 87% rename from src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json rename to src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json index 1184072..40afc75 100644 --- a/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json +++ b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json @@ -10,7 +10,7 @@ "role": "model" }, "finishReason": "STOP", - "avgLogprobs": -0.01313322534163793 + "avgLogprobs": -0.0095387622714042664 } ], "usageMetadata": { @@ -31,5 +31,5 @@ ] }, "modelVersion": "gemini-2.0-flash-lite", - "responseId": "g7lUaYzsB8DcqtsPgrSB2A0" + "responseId": "L5VmaYSPFpmdz7IP2PWQmAc" } diff --git a/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json similarity index 87% rename from src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json rename to src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json index f797f8b..8a7e387 100644 --- a/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json +++ b/src/test/resources/cassettes/google/__files/v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json @@ -10,7 +10,7 @@ "role": "model" }, "finishReason": "STOP", - "avgLogprobs": -0.060282785445451736 + "avgLogprobs": -0.050756204873323441 } ], "usageMetadata": { @@ -31,5 +31,5 @@ ] }, "modelVersion": "gemini-2.0-flash-lite", - "responseId": "grlUacDXF4mP6dkP1NLR6Q0" + "responseId": "LZVmadvKL7O6mtkPxbjguAg" } diff --git a/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json similarity index 79% rename from src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json rename to src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json index 3e74f82..60e578d 100644 --- a/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json +++ b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json @@ -1,5 +1,5 @@ { - "id" : "4559bbc0-1b54-4479-b477-bbe376bcb21b", + "id" : "47e92aee-8b68-4d7a-bd8a-f07c4ce231c2", "name" : "v1beta_models_gemini-20-flash-litegeneratecontent", "request" : { "url" : "/v1beta/models/gemini-2.0-flash-lite:generateContent", @@ -12,25 +12,25 @@ "bodyPatterns" : [ { "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"What is the capital of France?\"}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0,\"maxOutputTokens\":50}}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "v1beta_models_gemini-20-flash-litegeneratecontent-4559bbc0-1b54-4479-b477-bbe376bcb21b.json", + "bodyFileName" : "v1beta_models_gemini-20-flash-litegeneratecontent-47e92aee-8b68-4d7a-bd8a-f07c4ce231c2.json", "headers" : { "Content-Type" : "application/json; charset=UTF-8", "Vary" : [ "Origin", "X-Origin", "Referer" ], - "Date" : "Wed, 31 Dec 2025 05:49:55 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:43 GMT", "Server" : "scaffolding on HTTPServer2", "X-XSS-Protection" : "0", "X-Frame-Options" : "SAMEORIGIN", "X-Content-Type-Options" : "nosniff", - "Server-Timing" : "gfet4t7; dur=389", + "Server-Timing" : "gfet4t7; dur=490", "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" } }, - "uuid" : "4559bbc0-1b54-4479-b477-bbe376bcb21b", + "uuid" : "47e92aee-8b68-4d7a-bd8a-f07c4ce231c2", "persistent" : true, "insertionIndex" : 2 } \ No newline at end of file diff --git a/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json similarity index 79% rename from src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json rename to src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json index 7afb3fa..4c5b0c8 100644 --- a/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json +++ b/src/test/resources/cassettes/google/mappings/v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json @@ -1,5 +1,5 @@ { - "id" : "25a72094-2c7c-43cf-9084-a8cd6ab34382", + "id" : "f1d892bb-4783-475f-9320-1a5800d4f293", "name" : "v1beta_models_gemini-20-flash-litegeneratecontent", "request" : { "url" : "/v1beta/models/gemini-2.0-flash-lite:generateContent", @@ -12,25 +12,25 @@ "bodyPatterns" : [ { "equalToJson" : "{\"contents\":[{\"parts\":[{\"text\":\"What is the capital of Germany?\"}],\"role\":\"user\"}],\"generationConfig\":{\"temperature\":0.0,\"maxOutputTokens\":50}}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "v1beta_models_gemini-20-flash-litegeneratecontent-25a72094-2c7c-43cf-9084-a8cd6ab34382.json", + "bodyFileName" : "v1beta_models_gemini-20-flash-litegeneratecontent-f1d892bb-4783-475f-9320-1a5800d4f293.json", "headers" : { "Content-Type" : "application/json; charset=UTF-8", "Vary" : [ "Origin", "X-Origin", "Referer" ], - "Date" : "Wed, 31 Dec 2025 05:49:54 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:43 GMT", "Server" : "scaffolding on HTTPServer2", "X-XSS-Protection" : "0", "X-Frame-Options" : "SAMEORIGIN", "X-Content-Type-Options" : "nosniff", - "Server-Timing" : "gfet4t7; dur=410", + "Server-Timing" : "gfet4t7; dur=1344", "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" } }, - "uuid" : "25a72094-2c7c-43cf-9084-a8cd6ab34382", + "uuid" : "f1d892bb-4783-475f-9320-1a5800d4f293", "persistent" : true, "insertionIndex" : 1 } \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json b/src/test/resources/cassettes/openai/__files/chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json similarity index 86% rename from src/test/resources/cassettes/openai/__files/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json rename to src/test/resources/cassettes/openai/__files/chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json index 91ccdd5..1c89a82 100644 --- a/src/test/resources/cassettes/openai/__files/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json +++ b/src/test/resources/cassettes/openai/__files/chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json @@ -1,7 +1,7 @@ { - "id": "chatcmpl-CsjNnOWy4wIVUedC2PGW1KWTecWsP", + "id": "chatcmpl-CxdqPl3mdEMREYg8YcrJrp2tmqRTJ", "object": "chat.completion", - "created": 1767160199, + "created": 1768330549, "model": "gpt-4o-mini-2024-07-18", "choices": [ { @@ -32,5 +32,5 @@ } }, "service_tier": "default", - "system_fingerprint": "fp_29330a9688" + "system_fingerprint": "fp_c4585b5b9c" } diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json b/src/test/resources/cassettes/openai/__files/chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json similarity index 86% rename from src/test/resources/cassettes/openai/__files/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json rename to src/test/resources/cassettes/openai/__files/chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json index 008b614..1cc12fa 100644 --- a/src/test/resources/cassettes/openai/__files/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json +++ b/src/test/resources/cassettes/openai/__files/chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json @@ -1,7 +1,7 @@ { - "id": "chatcmpl-CsjNlSbOA9QyFPqBHuOJ7rNGTbHGS", + "id": "chatcmpl-CxdqNcT26jSeKfx2QAWkScbokWofM", "object": "chat.completion", - "created": 1767160197, + "created": 1768330547, "model": "gpt-4o-mini-2024-07-18", "choices": [ { @@ -32,5 +32,5 @@ } }, "service_tier": "default", - "system_fingerprint": "fp_c4585b5b9c" + "system_fingerprint": "fp_29330a9688" } diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.txt b/src/test/resources/cassettes/openai/__files/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.txt new file mode 100644 index 0000000..32380a2 --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"i0DEsxe0d"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"f9lVHuPX"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"LJo"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4Ush7Uov"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"w29M"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"oGMJtqrv"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"oulmu"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Q7jCyF6KeB"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Hy3sv"} + +data: {"id":"chatcmpl-CxdqO7Gy8Lx4oY0QTk40yFak8aXlQ","object":"chat.completion.chunk","created":1768330548,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[],"usage":{"prompt_tokens":23,"completion_tokens":7,"total_tokens":30,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"8yIDij1lDhf"} + +data: [DONE] + diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt b/src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt deleted file mode 100644 index e9253bd..0000000 --- a/src/test/resources/cassettes/openai/__files/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt +++ /dev/null @@ -1,22 +0,0 @@ -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"3GIsvqZNS"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"eqO4TiLK"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"jU7"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"21NQowmO"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"dCF0"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"rQJNUmqX"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"aJGDm"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"940tkEiuxP"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"kRqGD"} - -data: {"id":"chatcmpl-CsjNnQijT3p3D6pLvZB8pz12OW0vO","object":"chat.completion.chunk","created":1767160199,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c4585b5b9c","choices":[],"usage":{"prompt_tokens":23,"completion_tokens":7,"total_tokens":30,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"q8zbjlCR66C"} - -data: [DONE] - diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt b/src/test/resources/cassettes/openai/__files/chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.txt similarity index 50% rename from src/test/resources/cassettes/openai/__files/chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt rename to src/test/resources/cassettes/openai/__files/chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.txt index cad7b80..40cd446 100644 --- a/src/test/resources/cassettes/openai/__files/chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt +++ b/src/test/resources/cassettes/openai/__files/chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.txt @@ -1,22 +1,22 @@ -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"beUIaVqte"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"zhm3yj623"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"dxVBy03k"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"AhGJkfcK"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"oYL"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"5JD"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"75W3y0x3"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"9E3BimxD"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"1z0o"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"I0ue"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"vfaJFMjg"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"zhZ6meZ6"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"zlAvb"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"RtrPa"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"H9SDfImM9v"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"0SIqKB167Z"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"JI4WH"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"wtORO"} -data: {"id":"chatcmpl-CsjNkMGDx0rBrk7q0RvudXXsa1li7","object":"chat.completion.chunk","created":1767160196,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":7,"total_tokens":21,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"q9aPLCw6oBo"} +data: {"id":"chatcmpl-CxdqKsEuotLHjXL9yh6KxVGkZFeSq","object":"chat.completion.chunk","created":1768330544,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_29330a9688","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":7,"total_tokens":21,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"yNAPCwHY0Pf"} data: [DONE] diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json b/src/test/resources/cassettes/openai/__files/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json new file mode 100644 index 0000000..e9fca78 --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json @@ -0,0 +1,54 @@ +{ + "id": "chatcmpl-CxdqLHL6kBkvILQ1S5QOt1qeWw3b5", + "object": "chat.completion", + "created": 1768330545, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_O0vPA17lB2sSTvNui5KkMr7E", + "type": "function", + "function": { + "name": "getWeather", + "arguments": "{\"arg0\": \"Paris\"}" + } + }, + { + "id": "call_gVz1cYvAomaPcTrheam72S8H", + "type": "function", + "function": { + "name": "getWeather", + "arguments": "{\"arg0\": \"New York\"}" + } + } + ], + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 84, + "completion_tokens": 47, + "total_tokens": 131, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_c4585b5b9c" +} diff --git a/src/test/resources/cassettes/openai/__files/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json b/src/test/resources/cassettes/openai/__files/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json new file mode 100644 index 0000000..148499c --- /dev/null +++ b/src/test/resources/cassettes/openai/__files/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-CxdqMP3DDF5hyKQhHBRTrBCTtd0BE", + "object": "chat.completion", + "created": 1768330546, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The current temperature is the same in both Paris and New York, at 72°F, and both cities are experiencing sunny weather.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 177, + "completion_tokens": 27, + "total_tokens": 204, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_c4585b5b9c" +} diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json b/src/test/resources/cassettes/openai/mappings/chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json similarity index 62% rename from src/test/resources/cassettes/openai/mappings/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json rename to src/test/resources/cassettes/openai/mappings/chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json index 64e5b49..3041718 100644 --- a/src/test/resources/cassettes/openai/mappings/chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json @@ -1,5 +1,5 @@ { - "id" : "9b6e1d09-eeff-4c08-a479-9a1cd5b9913a", + "id" : "13e69bf3-e90d-497f-bb66-47b448dc86bd", "name" : "chat_completions", "request" : { "url" : "/chat/completions", @@ -12,39 +12,39 @@ "bodyPatterns" : [ { "equalToJson" : "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"temperature\":0.0}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "chat_completions-9b6e1d09-eeff-4c08-a479-9a1cd5b9913a.json", + "bodyFileName" : "chat_completions-13e69bf3-e90d-497f-bb66-47b448dc86bd.json", "headers" : { - "Date" : "Wed, 31 Dec 2025 05:50:00 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:49 GMT", "Content-Type" : "application/json", "access-control-expose-headers" : "X-Request-ID", "openai-organization" : "braintrust-data", - "openai-processing-ms" : "370", + "openai-processing-ms" : "414", "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version" : "2020-10-01", - "x-envoy-upstream-service-time" : "393", + "x-envoy-upstream-service-time" : "553", "x-ratelimit-limit-requests" : "30000", "x-ratelimit-limit-tokens" : "150000000", "x-ratelimit-remaining-requests" : "29999", "x-ratelimit-remaining-tokens" : "149999982", "x-ratelimit-reset-requests" : "2ms", "x-ratelimit-reset-tokens" : "0s", - "x-request-id" : "req_94fe822d95b044a58cd2eacccc013d5c", + "x-request-id" : "req_bc56bf172e1b41cda76cb06a5ca73ed0", "x-openai-proxy-wasm" : "v0.1", "cf-cache-status" : "DYNAMIC", - "Set-Cookie" : [ "__cf_bm=juuIFMD6qt989prqR27aGApTMecE9BUuvTAD5dGhPho-1767160200-1.0.1.1-bRt_lonT2SPEwJm2Bi9EWUnJFZnsrG_3QbGf8P4OOf_f_EqfTyG0NVAh_WmPpJ1t0u039SVg1XyUd1FEjz_oE3ZlIzf3VE8oCktjReFCkKg; path=/; expires=Wed, 31-Dec-25 06:20:00 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=x5FoQ1I.0RMSh8W0juh1zeyYg.Oi1sc2Da4deU27gUM-1767160200105-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Set-Cookie" : [ "__cf_bm=KnxsdeOSd50kaFk76pJ9ETxarjO99vDibkW9gcRSZF0-1768330549-1.0.1.1-WjfuxAjZSzqtvv3lnyq9bnx3pxzRUbCjfXzeXwBJKRzcvutTMQfCl6z4L1pE42fFq93ODxppcSpBpRt.84GY4apv7AufWbLfugBz5mnyZp0; path=/; expires=Tue, 13-Jan-26 19:25:49 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=ACDXQn3j61tpIi9gPK9jM.jFHQdh0jpS7JKxLogWKpY-1768330549707-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", "X-Content-Type-Options" : "nosniff", "Server" : "cloudflare", - "CF-RAY" : "9b677f2fc891eb6b-SEA", + "CF-RAY" : "9bd71c2bbc74b9c4-SEA", "alt-svc" : "h3=\":443\"; ma=86400" } }, - "uuid" : "9b6e1d09-eeff-4c08-a479-9a1cd5b9913a", + "uuid" : "13e69bf3-e90d-497f-bb66-47b448dc86bd", "persistent" : true, - "insertionIndex" : 4 + "insertionIndex" : 6 } \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json b/src/test/resources/cassettes/openai/mappings/chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json similarity index 62% rename from src/test/resources/cassettes/openai/mappings/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json rename to src/test/resources/cassettes/openai/mappings/chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json index 11e4d79..f67468f 100644 --- a/src/test/resources/cassettes/openai/mappings/chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json @@ -1,5 +1,5 @@ { - "id" : "657d1294-4a6e-41ca-b54a-4ab699f336bc", + "id" : "4706df90-aa52-4bd3-97c0-3448a4e0005f", "name" : "chat_completions", "request" : { "url" : "/chat/completions", @@ -12,39 +12,39 @@ "bodyPatterns" : [ { "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"What is the capital of France?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : false\n}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "chat_completions-657d1294-4a6e-41ca-b54a-4ab699f336bc.json", + "bodyFileName" : "chat_completions-4706df90-aa52-4bd3-97c0-3448a4e0005f.json", "headers" : { - "Date" : "Wed, 31 Dec 2025 05:49:58 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:47 GMT", "Content-Type" : "application/json", "access-control-expose-headers" : "X-Request-ID", "openai-organization" : "braintrust-data", - "openai-processing-ms" : "790", + "openai-processing-ms" : "403", "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version" : "2020-10-01", - "x-envoy-upstream-service-time" : "803", + "x-envoy-upstream-service-time" : "421", "x-ratelimit-limit-requests" : "30000", "x-ratelimit-limit-tokens" : "150000000", "x-ratelimit-remaining-requests" : "29999", "x-ratelimit-remaining-tokens" : "149999990", "x-ratelimit-reset-requests" : "2ms", "x-ratelimit-reset-tokens" : "0s", - "x-request-id" : "req_3494dddb4ff04d71b4e04c8bdd849238", + "x-request-id" : "req_e6ad6e08adaf4bafbb57a63d4f06081f", "x-openai-proxy-wasm" : "v0.1", "cf-cache-status" : "DYNAMIC", - "Set-Cookie" : [ "__cf_bm=xs7eo1iqAzPmMEWkRX2as3txeen9VmfNgdgv4NUA5Gk-1767160198-1.0.1.1-N41Aszum8iDWzU8bHrVs1qe0sTi_604F4U.i0IuTVFzIcaimCZU8aStvawypaumbIqZsNXtcgdBq68oJc1OOYyp5.g5O4g_Hma.Vbnu_kPs; path=/; expires=Wed, 31-Dec-25 06:19:58 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=fQPpTctWYUE6rgjuAT4gZWj9pkxzQhXeDAVonakGCw0-1767160198559-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Set-Cookie" : [ "__cf_bm=Kf0PbpZlmshlH.OMcmGSPEKIneXZrbLpMkYluWZ53rM-1768330547-1.0.1.1-dyEibCy4JMqWjO47uv.WMSRPxMrD4E.v7QDrQ9e9D1qyPfRzJoKmz2v5zMtvapNpQ5Xv.RYHD0MIgE2GXDk6vYpwO_sTmRvtYdbtEIiEcuU; path=/; expires=Tue, 13-Jan-26 19:25:47 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=a2Iqf.uw0VrBcSHQkzxsmfmy9XzkFde5gjk5LB7Tpb8-1768330547777-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", "X-Content-Type-Options" : "nosniff", "Server" : "cloudflare", - "CF-RAY" : "9b677f237ecaba4c-SEA", + "CF-RAY" : "9bd71c206d2f280d-SEA", "alt-svc" : "h3=\":443\"; ma=86400" } }, - "uuid" : "657d1294-4a6e-41ca-b54a-4ab699f336bc", + "uuid" : "4706df90-aa52-4bd3-97c0-3448a4e0005f", "persistent" : true, - "insertionIndex" : 2 + "insertionIndex" : 4 } \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json b/src/test/resources/cassettes/openai/mappings/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.json similarity index 64% rename from src/test/resources/cassettes/openai/mappings/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json rename to src/test/resources/cassettes/openai/mappings/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.json index d78f766..4a08405 100644 --- a/src/test/resources/cassettes/openai/mappings/chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.json +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.json @@ -1,5 +1,5 @@ { - "id" : "81d3ca7f-0bf4-4a03-b89a-105ccb33de39", + "id" : "541b9306-6880-41df-bb44-69ea82f417f8", "name" : "chat_completions", "request" : { "url" : "/chat/completions", @@ -12,39 +12,39 @@ "bodyPatterns" : [ { "equalToJson" : "{\"messages\":[{\"content\":\"You are a helpful assistant\",\"role\":\"system\"},{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"stream_options\":{\"include_usage\":true},\"temperature\":0.0,\"stream\":true}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "chat_completions-81d3ca7f-0bf4-4a03-b89a-105ccb33de39.txt", + "bodyFileName" : "chat_completions-541b9306-6880-41df-bb44-69ea82f417f8.txt", "headers" : { - "Date" : "Wed, 31 Dec 2025 05:49:59 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:48 GMT", "Content-Type" : "text/event-stream; charset=utf-8", "access-control-expose-headers" : "X-Request-ID", "openai-organization" : "braintrust-data", - "openai-processing-ms" : "239", + "openai-processing-ms" : "183", "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version" : "2020-10-01", - "x-envoy-upstream-service-time" : "253", + "x-envoy-upstream-service-time" : "196", "x-ratelimit-limit-requests" : "30000", "x-ratelimit-limit-tokens" : "150000000", "x-ratelimit-remaining-requests" : "29999", "x-ratelimit-remaining-tokens" : "149999982", "x-ratelimit-reset-requests" : "2ms", "x-ratelimit-reset-tokens" : "0s", - "x-request-id" : "req_a5095aecf5044a0db2dcf6a3352b1384", + "x-request-id" : "req_a5315e01514f4b9298b1727d17a7a63e", "x-openai-proxy-wasm" : "v0.1", "cf-cache-status" : "DYNAMIC", - "Set-Cookie" : [ "__cf_bm=gc1.e.r5u9NPTh_gRsIeNuihBexPsRGwWt_owdt34Oo-1767160199-1.0.1.1-En_zK2nPi53SQrT1waW6yOpzYUWvD2ekMQfus0FutgB83AqAz5tppvvOfgR8HYQPWOrFKi6Ctpu_uYJ31r1Snrelh46NfhJaLNPXR4HsMpI; path=/; expires=Wed, 31-Dec-25 06:19:59 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=MBmlbrM8g3JiQhLdt5SsuDZQLoC3UhIOApeILIfdRcY-1767160199298-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Set-Cookie" : [ "__cf_bm=JXEJvLpTetK2tdCv6rCOsWm9XsCh35xX80RYSdbKlOQ-1768330548-1.0.1.1-LU.AUqqeHIJMar9S7gWLrEIivCcP4MJUEnfNDdc0mSsxnUNyUxhElI0Hk9_JklQpx0ceQrZ0v8T5VBY5twFxgLeJ9Rws3dzYTIx0I2L2Y_U; path=/; expires=Tue, 13-Jan-26 19:25:48 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=.9AGf0FnX2Yz1lpFeCSgRDTEkbpoZxJQOmHuxZaOYiU-1768330548584-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", "X-Content-Type-Options" : "nosniff", "Server" : "cloudflare", - "CF-RAY" : "9b677f2b8fc5ba4b-SEA", + "CF-RAY" : "9bd71c270a48d7dd-SEA", "alt-svc" : "h3=\":443\"; ma=86400" } }, - "uuid" : "81d3ca7f-0bf4-4a03-b89a-105ccb33de39", + "uuid" : "541b9306-6880-41df-bb44-69ea82f417f8", "persistent" : true, - "insertionIndex" : 3 + "insertionIndex" : 5 } \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-06107a5c-8505-436f-850b-598871b122f1.json b/src/test/resources/cassettes/openai/mappings/chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.json similarity index 65% rename from src/test/resources/cassettes/openai/mappings/chat_completions-06107a5c-8505-436f-850b-598871b122f1.json rename to src/test/resources/cassettes/openai/mappings/chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.json index 33641c3..d7ff5b9 100644 --- a/src/test/resources/cassettes/openai/mappings/chat_completions-06107a5c-8505-436f-850b-598871b122f1.json +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.json @@ -1,5 +1,5 @@ { - "id" : "06107a5c-8505-436f-850b-598871b122f1", + "id" : "9016066f-1bbd-49ec-a403-4311b29fe2af", "name" : "chat_completions", "request" : { "url" : "/chat/completions", @@ -12,39 +12,39 @@ "bodyPatterns" : [ { "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"What is the capital of France?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : true,\n \"stream_options\" : {\n \"include_usage\" : true\n }\n}", "ignoreArrayOrder" : true, - "ignoreExtraElements" : true + "ignoreExtraElements" : false } ] }, "response" : { "status" : 200, - "bodyFileName" : "chat_completions-06107a5c-8505-436f-850b-598871b122f1.txt", + "bodyFileName" : "chat_completions-9016066f-1bbd-49ec-a403-4311b29fe2af.txt", "headers" : { - "Date" : "Wed, 31 Dec 2025 05:49:56 GMT", + "Date" : "Tue, 13 Jan 2026 18:55:44 GMT", "Content-Type" : "text/event-stream; charset=utf-8", "access-control-expose-headers" : "X-Request-ID", "openai-organization" : "braintrust-data", - "openai-processing-ms" : "330", + "openai-processing-ms" : "163", "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version" : "2020-10-01", - "x-envoy-upstream-service-time" : "495", + "x-envoy-upstream-service-time" : "295", "x-ratelimit-limit-requests" : "30000", "x-ratelimit-limit-tokens" : "150000000", "x-ratelimit-remaining-requests" : "29999", "x-ratelimit-remaining-tokens" : "149999990", "x-ratelimit-reset-requests" : "2ms", "x-ratelimit-reset-tokens" : "0s", - "x-request-id" : "req_574478be32734066986a8074185255ae", + "x-request-id" : "req_54de4d24315b4956916c34f431127db5", "x-openai-proxy-wasm" : "v0.1", "cf-cache-status" : "DYNAMIC", - "Set-Cookie" : [ "__cf_bm=55HipGfK7WWBg.Vg8L4q42mu6bXkWWZ6H8lWdLjrA_U-1767160196-1.0.1.1-vqM_5zFTfWPhcNo7owkGT2ZgRcpK4VuJgaigtqiGUgEBE0o07NsPpCmiOCkwf4rn1gXKY9W.yW.fyEdmJIJXR0w9BHXnQ08BD.OCtnU9tdg; path=/; expires=Wed, 31-Dec-25 06:19:56 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=rbxezynouHlFuKWzjsrbwclTwM7KvHixfgpIOc5h2vY-1767160196378-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Set-Cookie" : [ "__cf_bm=_UjMnl_QDwQVpxokprrHLdnoEskB5r7JY8twdegieGo-1768330544-1.0.1.1-J0MsKAdxFAJi7gXyTdIY5xGElzghGXqWT3jZB7J_idg15FnZdRUgy.ZylsUo74UEwCL2AQDf4nu0oRUBxH7c8.DgsbIiU8E4QPM84OyZgrE; path=/; expires=Tue, 13-Jan-26 19:25:44 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=yPo_7QFyEr5D0p4gVCxgxZusm3_C.Ms7TeOjEsGEjZM-1768330544555-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", "X-Content-Type-Options" : "nosniff", "Server" : "cloudflare", - "CF-RAY" : "9b677f17a95d7630-SEA", + "CF-RAY" : "9bd71c0d1f2f7538-SEA", "alt-svc" : "h3=\":443\"; ma=86400" } }, - "uuid" : "06107a5c-8505-436f-850b-598871b122f1", + "uuid" : "9016066f-1bbd-49ec-a403-4311b29fe2af", "persistent" : true, "insertionIndex" : 1 } \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json b/src/test/resources/cassettes/openai/mappings/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json new file mode 100644 index 0000000..29775d6 --- /dev/null +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json @@ -0,0 +1,50 @@ +{ + "id" : "d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"is it hotter in Paris or New York right now?\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : false,\n \"tools\" : [ {\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"getForecast\",\n \"description\" : \"Get weather forecast for next N days\",\n \"parameters\" : {\n \"type\" : \"object\",\n \"properties\" : {\n \"arg0\" : {\n \"type\" : \"string\"\n },\n \"arg1\" : {\n \"type\" : \"integer\"\n }\n },\n \"required\" : [ \"arg0\", \"arg1\" ]\n }\n }\n }, {\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"getWeather\",\n \"description\" : \"Get current weather for a location\",\n \"parameters\" : {\n \"type\" : \"object\",\n \"properties\" : {\n \"arg0\" : {\n \"type\" : \"string\"\n }\n },\n \"required\" : [ \"arg0\" ]\n }\n }\n } ]\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b.json", + "headers" : { + "Date" : "Tue, 13 Jan 2026 18:55:46 GMT", + "Content-Type" : "application/json", + "access-control-expose-headers" : "X-Request-ID", + "openai-organization" : "braintrust-data", + "openai-processing-ms" : "1118", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version" : "2020-10-01", + "x-envoy-upstream-service-time" : "1134", + "x-ratelimit-limit-requests" : "30000", + "x-ratelimit-limit-tokens" : "150000000", + "x-ratelimit-remaining-requests" : "29999", + "x-ratelimit-remaining-tokens" : "149999987", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-reset-tokens" : "0s", + "x-request-id" : "req_986e2d3165064709ac5ed23dc003dbd7", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "Set-Cookie" : [ "__cf_bm=BUHB2FaXynubGP92D6G2IvABiHu2eEGLvn8wFbWPmSo-1768330546-1.0.1.1-T6ZEZ.yw9bAhRfralFWrawOZlupeGnKzWk2CEzY7ok6AOXGdnWY5SL1.mewnFFHj6vLfMOIn0CPdULLrBjhYGOfb02DSZ4XJoHpYosO3KUU; path=/; expires=Tue, 13-Jan-26 19:25:46 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=xY.tpG.J8nfj5Le3HLJpq.g736HU9coaVEmWUc98mx4-1768330546132-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options" : "nosniff", + "Server" : "cloudflare", + "CF-RAY" : "9bd71c11cdf8c39a-SEA", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "d0ddbec9-910b-4f8b-a0b8-a605e6a23b4b", + "persistent" : true, + "insertionIndex" : 2 +} \ No newline at end of file diff --git a/src/test/resources/cassettes/openai/mappings/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json b/src/test/resources/cassettes/openai/mappings/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json new file mode 100644 index 0000000..c745847 --- /dev/null +++ b/src/test/resources/cassettes/openai/mappings/chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json @@ -0,0 +1,50 @@ +{ + "id" : "d6eda521-a7dd-4216-911e-4a916e45fca1", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\n \"model\" : \"gpt-4o-mini\",\n \"messages\" : [ {\n \"role\" : \"user\",\n \"content\" : \"is it hotter in Paris or New York right now?\"\n }, {\n \"role\" : \"assistant\",\n \"tool_calls\" : [ {\n \"id\" : \"call_O0vPA17lB2sSTvNui5KkMr7E\",\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"getWeather\",\n \"arguments\" : \"{\\\"arg0\\\": \\\"Paris\\\"}\"\n }\n }, {\n \"id\" : \"call_gVz1cYvAomaPcTrheam72S8H\",\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"getWeather\",\n \"arguments\" : \"{\\\"arg0\\\": \\\"New York\\\"}\"\n }\n } ]\n }, {\n \"role\" : \"tool\",\n \"tool_call_id\" : \"call_O0vPA17lB2sSTvNui5KkMr7E\",\n \"content\" : \"The weather in Paris is sunny with 72°F temperature.\"\n }, {\n \"role\" : \"tool\",\n \"tool_call_id\" : \"call_gVz1cYvAomaPcTrheam72S8H\",\n \"content\" : \"The weather in New York is sunny with 72°F temperature.\"\n } ],\n \"temperature\" : 0.0,\n \"stream\" : false,\n \"tools\" : [ {\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"getForecast\",\n \"description\" : \"Get weather forecast for next N days\",\n \"parameters\" : {\n \"type\" : \"object\",\n \"properties\" : {\n \"arg0\" : {\n \"type\" : \"string\"\n },\n \"arg1\" : {\n \"type\" : \"integer\"\n }\n },\n \"required\" : [ \"arg0\", \"arg1\" ]\n }\n }\n }, {\n \"type\" : \"function\",\n \"function\" : {\n \"name\" : \"getWeather\",\n \"description\" : \"Get current weather for a location\",\n \"parameters\" : {\n \"type\" : \"object\",\n \"properties\" : {\n \"arg0\" : {\n \"type\" : \"string\"\n }\n },\n \"required\" : [ \"arg0\" ]\n }\n }\n } ]\n}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-d6eda521-a7dd-4216-911e-4a916e45fca1.json", + "headers" : { + "Date" : "Tue, 13 Jan 2026 18:55:47 GMT", + "Content-Type" : "application/json", + "access-control-expose-headers" : "X-Request-ID", + "openai-organization" : "braintrust-data", + "openai-processing-ms" : "725", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "openai-version" : "2020-10-01", + "x-envoy-upstream-service-time" : "782", + "x-ratelimit-limit-requests" : "30000", + "x-ratelimit-limit-tokens" : "150000000", + "x-ratelimit-remaining-requests" : "29999", + "x-ratelimit-remaining-tokens" : "149999955", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-reset-tokens" : "0s", + "x-request-id" : "req_3fd434fe36e5457387641e91bddd872e", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "Set-Cookie" : [ "__cf_bm=GqbfmxZoOlS1Ff1BF3Pbb7tWbU6gizh68j2IIgjH.CE-1768330547-1.0.1.1-X3.JfC8E_qdMD_um.SLG7_uLhhxfj3letN8jGjFgDK2ZPDWk.lQYUWqkOUoshTf8UIxpi7UBmB2_C68nkATEom3o5SMBkYQ6I9bBYW32Y00; path=/; expires=Tue, 13-Jan-26 19:25:47 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "_cfuvid=bEAnkV1ZOc_LuPfi3M1oNKr3OyHkhoGrgkd6IxqpeK0-1768330547136-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" ], + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options" : "nosniff", + "Server" : "cloudflare", + "CF-RAY" : "9bd71c1a3ce5d44f-SEA", + "alt-svc" : "h3=\":443\"; ma=86400" + } + }, + "uuid" : "d6eda521-a7dd-4216-911e-4a916e45fca1", + "persistent" : true, + "insertionIndex" : 3 +} \ No newline at end of file From c4f4249c619e37b57aadda0c8d1fd5114687b8ad Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Tue, 13 Jan 2026 11:55:18 -0700 Subject: [PATCH 3/3] Instrument Langchain4j AI Services --- build.gradle | 2 - examples/build.gradle | 10 +- .../examples/LangchainAIServicesExample.java | 71 ++++ ...ample.java => LangchainSimpleExample.java} | 5 +- .../LangchainToolWrappingExample.java | 158 -------- .../langchain/BraintrustLangchain.java | 194 +++++++--- .../langchain/ByteBuddyToolWrapper.java | 119 ------ .../langchain/OtelContextPassingExecutor.java | 31 ++ .../langchain/TracingProxy.java | 50 +++ .../langchain/TracingToolExecutor.java | 78 ++++ .../langchain/WrappedHttpClient.java | 35 +- src/test/java/dev/braintrust/VCR.java | 6 +- .../langchain/BraintrustLangchainTest.java | 343 +++++++----------- 13 files changed, 540 insertions(+), 562 deletions(-) create mode 100644 examples/src/main/java/dev/braintrust/examples/LangchainAIServicesExample.java rename examples/src/main/java/dev/braintrust/examples/{LangchainExample.java => LangchainSimpleExample.java} (93%) delete mode 100644 examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java delete mode 100644 src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java create mode 100644 src/main/java/dev/braintrust/instrumentation/langchain/OtelContextPassingExecutor.java create mode 100644 src/main/java/dev/braintrust/instrumentation/langchain/TracingProxy.java create mode 100644 src/main/java/dev/braintrust/instrumentation/langchain/TracingToolExecutor.java diff --git a/build.gradle b/build.gradle index 90d9c9f..ad1b078 100644 --- a/build.gradle +++ b/build.gradle @@ -95,8 +95,6 @@ dependencies { testImplementation "dev.langchain4j:langchain4j:${langchainVersion}" testImplementation "dev.langchain4j:langchain4j-http-client:${langchainVersion}" testImplementation "dev.langchain4j:langchain4j-open-ai:${langchainVersion}" - - implementation 'net.bytebuddy:byte-buddy:1.14.11' // ByteBuddy for LangChain4j tool wrapping } /** diff --git a/examples/build.gradle b/examples/build.gradle index 553af25..ad342e4 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -159,11 +159,11 @@ task runRemoteEval(type: JavaExec) { } } -task runLangchain(type: JavaExec) { +task runLangchainSimple(type: JavaExec) { group = 'Braintrust SDK Examples' description = 'Run the LangChain4j instrumentation example. NOTE: this requires OPENAI_API_KEY to be exported and will make a small call to openai, using your tokens' classpath = sourceSets.main.runtimeClasspath - mainClass = 'dev.braintrust.examples.LangchainExample' + mainClass = 'dev.braintrust.examples.LangchainSimpleExample' systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel debugOptions { enabled = true @@ -173,11 +173,11 @@ task runLangchain(type: JavaExec) { } } -task runLangchainToolWrapping(type: JavaExec) { +task runLangchainAIServices(type: JavaExec) { group = 'Braintrust SDK Examples' - description = 'Run the LangChain4j tool wrapping example. NOTE: this requires OPENAI_API_KEY to be exported and will make a small call to openai, using your tokens' + description = 'Run the LangChain4j AI Services example. NOTE: this requires OPENAI_API_KEY to be exported and will make a small call to openai, using your tokens' classpath = sourceSets.main.runtimeClasspath - mainClass = 'dev.braintrust.examples.LangchainToolWrappingExample' + mainClass = 'dev.braintrust.examples.LangchainAIServicesExample' systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel debugOptions { enabled = true diff --git a/examples/src/main/java/dev/braintrust/examples/LangchainAIServicesExample.java b/examples/src/main/java/dev/braintrust/examples/LangchainAIServicesExample.java new file mode 100644 index 0000000..e911b10 --- /dev/null +++ b/examples/src/main/java/dev/braintrust/examples/LangchainAIServicesExample.java @@ -0,0 +1,71 @@ +package dev.braintrust.examples; + +import dev.braintrust.Braintrust; +import dev.braintrust.instrumentation.langchain.BraintrustLangchain; +import dev.langchain4j.agent.tool.Tool; +import dev.langchain4j.model.openai.OpenAiChatModel; +import dev.langchain4j.service.AiServices; + +public class LangchainAIServicesExample { + + public static void main(String[] args) throws Exception { + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); + + Assistant assistant = + BraintrustLangchain.wrap( + openTelemetry, + AiServices.builder(Assistant.class) + .chatModel( + OpenAiChatModel.builder() + .apiKey(System.getenv("OPENAI_API_KEY")) + .modelName("gpt-4o-mini") + .temperature(0.0) + .build()) + .tools(new WeatherTools()) + .executeToolsConcurrently()); + + var rootSpan = + openTelemetry + .getTracer("my-instrumentation") + .spanBuilder("langchain4j-ai-services-example") + .startSpan(); + try (var ignored = rootSpan.makeCurrent()) { + // response 1 should do a concurrent tool call + var response1 = assistant.chat("is it hotter in Paris or New York right now?"); + System.out.println("response1: " + response1); + var response2 = assistant.chat("what's the five day forecast for San Francisco?"); + System.out.println("response2: " + response2); + } finally { + rootSpan.end(); + } + var url = + braintrust.projectUri() + + "/logs?r=%s&s=%s" + .formatted( + rootSpan.getSpanContext().getTraceId(), + rootSpan.getSpanContext().getSpanId()); + System.out.println( + "\n\n Example complete! View your data in Braintrust: %s\n".formatted(url)); + } + + /** AI Service interface for the assistant */ + interface Assistant { + String chat(String userMessage); + } + + /** Example tool class with weather-related methods */ + public static class WeatherTools { + @Tool("Get current weather for a location") + public String getWeather(String location) { + return String.format("The weather in %s is sunny with 72°F temperature.", location); + } + + @Tool("Get weather forecast for next N days") + public String getForecast(String location, int days) { + return String.format( + "The %d-day forecast for %s: Mostly sunny with temperatures between 65-75°F.", + days, location); + } + } +} diff --git a/examples/src/main/java/dev/braintrust/examples/LangchainExample.java b/examples/src/main/java/dev/braintrust/examples/LangchainSimpleExample.java similarity index 93% rename from examples/src/main/java/dev/braintrust/examples/LangchainExample.java rename to examples/src/main/java/dev/braintrust/examples/LangchainSimpleExample.java index 117272f..9d6fd2b 100644 --- a/examples/src/main/java/dev/braintrust/examples/LangchainExample.java +++ b/examples/src/main/java/dev/braintrust/examples/LangchainSimpleExample.java @@ -7,7 +7,7 @@ import dev.langchain4j.model.openai.OpenAiChatModel; /** Basic OTel + LangChain4j instrumentation example */ -public class LangchainExample { +public class LangchainSimpleExample { public static void main(String[] args) throws Exception { if (null == System.getenv("OPENAI_API_KEY")) { @@ -46,8 +46,7 @@ public static void main(String[] args) throws Exception { } private static void chatExample(ChatModel model) { - var message = UserMessage.from("What is the capital of France?"); - var response = model.chat(message); + var response = model.chat(UserMessage.from("What is the capital of France?")); System.out.println( "\n~~~ LANGCHAIN4J CHAT RESPONSE: %s\n".formatted(response.aiMessage().text())); } diff --git a/examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java b/examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java deleted file mode 100644 index 102880d..0000000 --- a/examples/src/main/java/dev/braintrust/examples/LangchainToolWrappingExample.java +++ /dev/null @@ -1,158 +0,0 @@ -package dev.braintrust.examples; - -import dev.braintrust.Braintrust; -import dev.braintrust.instrumentation.langchain.BraintrustLangchain; -import dev.langchain4j.agent.tool.Tool; -import dev.langchain4j.model.chat.ChatModel; -import dev.langchain4j.model.openai.OpenAiChatModel; -import dev.langchain4j.service.AiServices; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Scope; -import io.opentelemetry.sdk.OpenTelemetrySdk; - -/** - * Demonstrates how to use BraintrustLangchain.wrapTools() to automatically trace tool executions. - * - *

This example shows: - * - *

- */ -public class LangchainToolWrappingExample { - - /** Example tool class with weather-related methods */ - public static class WeatherTools { - @Tool("Get current weather for a location") - public String getWeather(String location) { - // Simulate a weather API call - return String.format("The weather in %s is sunny with 72°F temperature.", location); - } - - @Tool("Get weather forecast for next N days") - public String getForecast(String location, int days) { - // Simulate a forecast API call - return String.format( - "The %d-day forecast for %s: Mostly sunny with temperatures between 65-75°F.", - days, location); - } - } - - /** AI Service interface for the assistant */ - interface Assistant { - String chat(String userMessage); - } - - public static void main(String[] args) throws Exception { - if (null == System.getenv("OPENAI_API_KEY")) { - System.err.println( - "\nWARNING envar OPENAI_API_KEY not found. This example will likely fail.\n"); - } - - var braintrust = Braintrust.get(); - var openTelemetry = braintrust.openTelemetryCreate(); - var tracer = openTelemetry.getTracer("langchain-tool-wrapping"); - - System.out.println("\n=== LangChain4j Tool Wrapping Example ===\n"); - - // Create root span for the conversation - var conversationSpan = tracer.spanBuilder("weather-assistant-conversation").startSpan(); - conversationSpan.setAttribute( - "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"conversation\"}"); - conversationSpan.setAttribute( - "braintrust.input_json", - "{\"description\":\"Weather assistant with tool wrapping\",\"turns\":2}"); - - try (var ignored = conversationSpan.makeCurrent()) { - // Wrap the LLM with Braintrust instrumentation - ChatModel model = - BraintrustLangchain.wrap( - openTelemetry, - OpenAiChatModel.builder() - .apiKey(System.getenv("OPENAI_API_KEY")) - .modelName("gpt-4o-mini") - .temperature(0.0)); - - // Create tools and wrap them with Braintrust instrumentation - WeatherTools tools = new WeatherTools(); - WeatherTools instrumentedTools = BraintrustLangchain.wrapTools(openTelemetry, tools); - - System.out.println("Tools wrapped with Braintrust instrumentation"); - System.out.println("Each tool call will create a span in Braintrust\n"); - - // Create AI service with the instrumented tools - Assistant assistant = - AiServices.builder(Assistant.class) - .chatModel(model) - .tools(instrumentedTools) - .build(); - - // Example 1: Single tool call - System.out.println("--- Turn 1: Single Tool Call ---"); - String query1 = "What's the weather in San Francisco?"; - System.out.println("User: " + query1); - Span turn1 = tracer.spanBuilder("turn_1").startSpan(); - turn1.setAttribute( - "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"turn_1\"}"); - turn1.setAttribute("braintrust.input_json", "{\"user_message\":\"" + query1 + "\"}"); - String response1; - try (Scope scope = turn1.makeCurrent()) { - response1 = assistant.chat(query1); - System.out.println("Assistant: " + response1); - turn1.setAttribute( - "braintrust.output_json", - "{\"assistant_message\":\"" + response1.replace("\"", "\\\"") + "\"}"); - } finally { - turn1.end(); - } - System.out.println(); - - // Example 2: Tool with multiple parameters - System.out.println("--- Turn 2: Multiple Parameters ---"); - String query2 = "What's the 5-day forecast for Tokyo?"; - System.out.println("User: " + query2); - Span turn2 = tracer.spanBuilder("turn_2").startSpan(); - turn2.setAttribute( - "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"turn_2\"}"); - turn2.setAttribute("braintrust.input_json", "{\"user_message\":\"" + query2 + "\"}"); - String response2; - try (Scope scope = turn2.makeCurrent()) { - response2 = assistant.chat(query2); - System.out.println("Assistant: " + response2); - turn2.setAttribute( - "braintrust.output_json", - "{\"assistant_message\":\"" + response2.replace("\"", "\\\"") + "\"}"); - } finally { - turn2.end(); - } - System.out.println(); - - } finally { - conversationSpan.setAttribute( - "braintrust.output_json", "{\"status\":\"completed\",\"turns\":2}"); - conversationSpan.end(); - } - - // Flush traces before exit - if (openTelemetry instanceof OpenTelemetrySdk) { - ((OpenTelemetrySdk) openTelemetry).close(); - } - - var url = - braintrust.projectUri() - + "/logs?r=%s&s=%s" - .formatted( - conversationSpan.getSpanContext().getTraceId(), - conversationSpan.getSpanContext().getSpanId()); - - System.out.println("Example complete!"); - System.out.println("\nIn Braintrust, you'll see:"); - System.out.println(" • Root conversation span"); - System.out.println(" • Nested turn spans for each user interaction"); - System.out.println(" • LLM call spans (from BraintrustLangchain.wrap())"); - System.out.println(" • Tool execution spans (from BraintrustLangchain.wrapTools())"); - System.out.println("\n View your traces: " + url + "\n"); - } -} diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java b/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java index 9645f88..6947bd4 100644 --- a/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java +++ b/src/main/java/dev/braintrust/instrumentation/langchain/BraintrustLangchain.java @@ -1,88 +1,178 @@ package dev.braintrust.instrumentation.langchain; -import dev.langchain4j.http.client.HttpClientBuilder; -import dev.langchain4j.http.client.HttpClientBuilderLoader; import dev.langchain4j.model.openai.OpenAiChatModel; import dev.langchain4j.model.openai.OpenAiStreamingChatModel; +import dev.langchain4j.service.AiServiceContext; +import dev.langchain4j.service.AiServices; +import dev.langchain4j.service.tool.ToolExecutor; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import java.util.Map; import lombok.extern.slf4j.Slf4j; /** Braintrust LangChain4j client instrumentation. */ @Slf4j public final class BraintrustLangchain { + + private static final String INSTRUMENTATION_NAME = "braintrust-langchain4j"; + + @SuppressWarnings("unchecked") + public static T wrap(OpenTelemetry openTelemetry, AiServices aiServices) { + try { + AiServiceContext context = getPrivateField(aiServices, "context"); + Tracer tracer = openTelemetry.getTracer(INSTRUMENTATION_NAME); + + // ////// CREATE A LLM SPAN FOR EACH CALL TO AI PROVIDER + var chatModel = context.chatModel; + var streamingChatModel = context.streamingChatModel; + if (chatModel != null) { + if (chatModel instanceof OpenAiChatModel oaiModel) { + aiServices.chatModel(wrap(openTelemetry, oaiModel)); + } else { + log.warn( + "unsupported model: {}. LLM calls will not be instrumented", + chatModel.getClass().getName()); + } + // intentional fall-through + } else if (streamingChatModel != null) { + if (streamingChatModel instanceof OpenAiStreamingChatModel oaiModel) { + aiServices.streamingChatModel(wrap(openTelemetry, oaiModel)); + } else { + log.warn( + "unsupported model: {}. LLM calls will not be instrumented", + streamingChatModel.getClass().getName()); + } + // intentional fall-through + } else { + // langchain is going to fail to build. don't apply instrumentation. + throw new RuntimeException("model or chat model must be set"); + } + + if (context.toolService != null) { + // ////// CREATE A SPAN FOR EACH TOOL CALL + for (Map.Entry entry : + context.toolService.toolExecutors().entrySet()) { + String toolName = entry.getKey(); + ToolExecutor original = entry.getValue(); + entry.setValue(new TracingToolExecutor(original, toolName, tracer)); + } + + // ////// LINK SPANS ACROSS CONCURRENT TOOL CALLS + var underlyingExecutor = context.toolService.executor(); + if (underlyingExecutor != null) { + aiServices.executeToolsConcurrently( + new OtelContextPassingExecutor(underlyingExecutor)); + } + } + + // ////// CREATE A SPAN ON SERVICE METHOD INVOKE + T service = aiServices.build(); + Class serviceInterface = (Class) context.aiServiceClass; + return TracingProxy.create(serviceInterface, service, tracer); + } catch (Exception e) { + log.warn("failed to apply langchain AI services instrumentation", e); + return aiServices.build(); + } + } + /** Instrument langchain openai chat model with braintrust traces */ public static OpenAiChatModel wrap( OpenTelemetry otel, OpenAiChatModel.OpenAiChatModelBuilder builder) { + return wrap(otel, builder.build()); + } + + private static OpenAiChatModel wrap(OpenTelemetry otel, OpenAiChatModel model) { try { - HttpClientBuilder underlyingHttpClient = getPrivateField(builder, "httpClientBuilder"); - if (underlyingHttpClient == null) { - underlyingHttpClient = HttpClientBuilderLoader.loadHttpClientBuilder(); + // Get the internal OpenAiClient from the chat model + Object internalClient = getPrivateField(model, "client"); + + // Get the HttpClient from the internal client + dev.langchain4j.http.client.HttpClient httpClient = + getPrivateField(internalClient, "httpClient"); + + if (httpClient instanceof WrappedHttpClient) { + log.debug("model already instrumented. skipping: {}", httpClient.getClass()); + return model; } - HttpClientBuilder wrappedHttpClient = - wrap(otel, underlyingHttpClient, new Options("openai")); - return builder.httpClientBuilder(wrappedHttpClient).build(); + + // Wrap the HttpClient with our instrumented version + dev.langchain4j.http.client.HttpClient wrappedHttpClient = + new WrappedHttpClient(otel, httpClient, new Options("openai")); + + setPrivateField(internalClient, "httpClient", wrappedHttpClient); + + return model; } catch (Exception e) { - log.warn( - "Braintrust instrumentation could not be applied to OpenAiChatModel builder", - e); - return builder.build(); + log.warn("failed to instrument OpenAiChatModel", e); + return model; } } /** Instrument langchain openai chat model with braintrust traces */ public static OpenAiStreamingChatModel wrap( OpenTelemetry otel, OpenAiStreamingChatModel.OpenAiStreamingChatModelBuilder builder) { - try { - HttpClientBuilder underlyingHttpClient = getPrivateField(builder, "httpClientBuilder"); - if (underlyingHttpClient == null) { - underlyingHttpClient = HttpClientBuilderLoader.loadHttpClientBuilder(); - } - HttpClientBuilder wrappedHttpClient = - wrap(otel, underlyingHttpClient, new Options("openai")); - return builder.httpClientBuilder(wrappedHttpClient).build(); - } catch (Exception e) { - log.warn( - "Braintrust instrumentation could not be applied to OpenAiStreamingChatModel" - + " builder", - e); - return builder.build(); - } + return wrap(otel, builder.build()); } - /** - * Wrap a tools object to instrument @Tool method executions with Braintrust traces. Returns a - * proxy that intercepts all @Tool annotated methods and creates OpenTelemetry spans. - * - *

Usage: StoryTools tools = new StoryTools(); StoryTools instrumented = - * BraintrustLangchain.wrapTools(openTelemetry, tools); - * AiServices.builder(Assistant.class).chatModel(model).tools(instrumented).build() - * - * @param otel OpenTelemetry instance from braintrust.openTelemetryCreate() - * @param tools Tool object with @Tool annotated methods - * @return Proxied tool object that creates spans for each @Tool method call - */ - @SuppressWarnings("unchecked") - public static T wrapTools(OpenTelemetry otel, T tools) { + public static OpenAiStreamingChatModel wrap( + OpenTelemetry otel, OpenAiStreamingChatModel model) { try { - return (T) new ByteBuddyToolWrapper(otel).wrap(tools); + // Get the internal OpenAiClient from the streaming chat model + Object internalClient = getPrivateField(model, "client"); + + // Get the HttpClient from the internal client + dev.langchain4j.http.client.HttpClient httpClient = + getPrivateField(internalClient, "httpClient"); + + if (httpClient instanceof WrappedHttpClient) { + log.debug("model already instrumented. skipping: {}", httpClient.getClass()); + return model; + } + + // Wrap the HttpClient with our instrumented version + dev.langchain4j.http.client.HttpClient wrappedHttpClient = + new WrappedHttpClient(otel, httpClient, new Options("openai")); + + setPrivateField(internalClient, "httpClient", wrappedHttpClient); + + return model; } catch (Exception e) { - log.warn("Failed to wrap tools with instrumentation, returning original", e); - return tools; + log.warn("failed to instrument OpenAiStreamingChatModel", e); + return model; } } - private static HttpClientBuilder wrap( - OpenTelemetry otel, HttpClientBuilder builder, Options options) { - return new WrappedHttpClientBuilder(otel, builder, options); - } - public record Options(String providerName) {} @SuppressWarnings("unchecked") private static T getPrivateField(Object obj, String fieldName) throws ReflectiveOperationException { - java.lang.reflect.Field field = obj.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - return (T) field.get(obj); + Class clazz = obj.getClass(); + while (clazz != null) { + try { + java.lang.reflect.Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(obj); + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + private static void setPrivateField(Object obj, String fieldName, Object value) + throws ReflectiveOperationException { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + java.lang.reflect.Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(obj, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); } } diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java b/src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java deleted file mode 100644 index 0c4ba25..0000000 --- a/src/main/java/dev/braintrust/instrumentation/langchain/ByteBuddyToolWrapper.java +++ /dev/null @@ -1,119 +0,0 @@ -package dev.braintrust.instrumentation.langchain; - -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.langchain4j.agent.tool.Tool; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Scope; -import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Callable; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.implementation.MethodDelegation; -import net.bytebuddy.implementation.bind.annotation.*; -import net.bytebuddy.matcher.ElementMatchers; - -/** - * Uses ByteBuddy to create runtime subclass proxies that intercept @Tool methods and add - * OpenTelemetry spans with Braintrust attributes. - */ -@Slf4j -class ByteBuddyToolWrapper { - private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - private final Tracer tracer; - - ByteBuddyToolWrapper(OpenTelemetry otel) { - this.tracer = otel.getTracer("braintrust"); - } - - @SuppressWarnings("unchecked") - public T wrap(T originalTools) throws Exception { - Class toolClass = originalTools.getClass(); - - // Create subclass with interceptor, preserving all annotations - Class proxyClass = - new ByteBuddy() - .subclass(toolClass) - .method(ElementMatchers.isAnnotatedWith(Tool.class)) - .intercept( - MethodDelegation.to( - new ToolMethodInterceptor(tracer, originalTools))) - .attribute( - net.bytebuddy.implementation.attribute.MethodAttributeAppender - .ForInstrumentedMethod.INCLUDING_RECEIVER) - .make() - .load(toolClass.getClassLoader()) - .getLoaded(); - - // Create instance - ByteBuddy subclass will delegate to original - return (T) proxyClass.getDeclaredConstructor().newInstance(); - } - - /** Interceptor that wraps @Tool method calls with OpenTelemetry spans */ - public static class ToolMethodInterceptor { - private final Tracer tracer; - private final Object originalTools; - - public ToolMethodInterceptor(Tracer tracer, Object originalTools) { - this.tracer = tracer; - this.originalTools = originalTools; - } - - @RuntimeType - public Object intercept( - @Origin Method method, @AllArguments Object[] args, @SuperCall Callable zuper) - throws Exception { - - String toolName = method.getName(); - - // Build input map from parameters - Map inputMap = new HashMap<>(); - var parameters = method.getParameters(); - for (int i = 0; i < parameters.length && i < args.length; i++) { - inputMap.put(parameters[i].getName(), args[i]); - } - - Span span = tracer.spanBuilder(toolName).startSpan(); - try (Scope scope = span.makeCurrent()) { - // Set Braintrust span attributes - span.setAttribute( - "braintrust.span_attributes", - json(Map.of("type", "tool", "name", toolName))); - - // Set input - span.setAttribute("braintrust.input_json", json(inputMap)); - - // Execute method and measure time - long startTime = System.nanoTime(); - Object result = zuper.call(); // Call original method - long endTime = System.nanoTime(); - - // Set output - span.setAttribute("braintrust.output_json", json(result)); - - // Set metrics - double executionTime = (endTime - startTime) / 1_000_000_000.0; - span.setAttribute( - "braintrust.metrics", json(Map.of("execution_time", executionTime))); - - return result; - } catch (Throwable t) { - span.setStatus(StatusCode.ERROR, t.getMessage()); - span.recordException(t); - throw t; - } finally { - span.end(); - } - } - - @SneakyThrows - private static String json(Object o) { - return JSON_MAPPER.writeValueAsString(o); - } - } -} diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/OtelContextPassingExecutor.java b/src/main/java/dev/braintrust/instrumentation/langchain/OtelContextPassingExecutor.java new file mode 100644 index 0000000..cdf54d9 --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/langchain/OtelContextPassingExecutor.java @@ -0,0 +1,31 @@ +package dev.braintrust.instrumentation.langchain; + +import io.opentelemetry.context.Context; +import java.util.concurrent.Executor; +import org.jspecify.annotations.NonNull; + +/** + * An executor that links open telemetry spans across threads + * + *

Any tasks submitted to the executor will point to the parent context that was present at the + * time of task submission. Or, if no parent context was present tasks will create spans as they + * normally would (or would not) without this executor. + */ +class OtelContextPassingExecutor implements Executor { + private final Executor underlying; + + public OtelContextPassingExecutor(Executor executor) { + this.underlying = executor; + } + + @Override + public void execute(@NonNull Runnable command) { + var context = Context.current(); + underlying.execute( + () -> { + try (var ignored = context.makeCurrent()) { + command.run(); + } + }); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/TracingProxy.java b/src/main/java/dev/braintrust/instrumentation/langchain/TracingProxy.java new file mode 100644 index 0000000..5bbb4cd --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/langchain/TracingProxy.java @@ -0,0 +1,50 @@ +package dev.braintrust.instrumentation.langchain; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import org.jspecify.annotations.NonNull; + +class TracingProxy { + /** + * Use a java {@link Proxy} to wrap a service interface methods with spans + * + *

Each interface method will create a span with the same name as the method + */ + @SuppressWarnings("unchecked") + public static @NonNull T create(Class serviceInterface, T service, Tracer tracer) { + return (T) + Proxy.newProxyInstance( + serviceInterface.getClassLoader(), + new Class[] {serviceInterface}, + (proxy, method, args) -> { + // Skip Object methods (equals, hashCode, toString) + if (method.getDeclaringClass() == Object.class) { + return method.invoke(service, args); + } + + Span span = tracer.spanBuilder(method.getName()).startSpan(); + try (Scope ignored = span.makeCurrent()) { + // Use setAccessible to handle non-public interfaces + method.setAccessible(true); + return method.invoke(service, args); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + span.setStatus(StatusCode.ERROR, cause.getMessage()); + span.recordException(cause); + throw cause; + } catch (Exception e) { + span.setStatus(StatusCode.ERROR, e.getMessage()); + span.recordException(e); + throw e; + } finally { + span.end(); + } + }); + } + + private TracingProxy() {} +} diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/TracingToolExecutor.java b/src/main/java/dev/braintrust/instrumentation/langchain/TracingToolExecutor.java new file mode 100644 index 0000000..dd6dd5e --- /dev/null +++ b/src/main/java/dev/braintrust/instrumentation/langchain/TracingToolExecutor.java @@ -0,0 +1,78 @@ +package dev.braintrust.instrumentation.langchain; + +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.invocation.InvocationContext; +import dev.langchain4j.service.tool.ToolExecutionResult; +import dev.langchain4j.service.tool.ToolExecutor; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +/** A ToolExecutor wrapper that creates a span around tool execution */ +@Slf4j +class TracingToolExecutor implements ToolExecutor { + static final String TYPE_TOOL_JSON = "{\"type\":\"tool\"}"; + + private final ToolExecutor delegate; + private final String toolName; + private final Tracer tracer; + + TracingToolExecutor(ToolExecutor delegate, String toolName, Tracer tracer) { + this.delegate = delegate; + this.toolName = toolName; + this.tracer = tracer; + } + + @Override + public String execute(ToolExecutionRequest request, Object memoryId) { + Span span = tracer.spanBuilder(toolName).startSpan(); + try (Scope ignored = span.makeCurrent()) { + String result = delegate.execute(request, memoryId); + setSpanAttributes(span, request, result); + return result; + } catch (Exception e) { + span.setStatus(StatusCode.ERROR, e.getMessage()); + span.recordException(e); + throw e; + } finally { + span.end(); + } + } + + @Override + public ToolExecutionResult executeWithContext( + ToolExecutionRequest request, InvocationContext context) { + Span span = tracer.spanBuilder(toolName).startSpan(); + try (Scope ignored = span.makeCurrent()) { + ToolExecutionResult result = delegate.executeWithContext(request, context); + setSpanAttributes(span, request, result.resultText()); + return result; + } catch (Exception e) { + span.setStatus(StatusCode.ERROR, e.getMessage()); + span.recordException(e); + throw e; + } finally { + span.end(); + } + } + + private void setSpanAttributes( + Span span, ToolExecutionRequest request, @Nullable String toolCallResult) { + try { + span.setAttribute("braintrust.span_attributes", TYPE_TOOL_JSON); + + String args = request.arguments(); + if (args != null && !args.isEmpty()) { + span.setAttribute("braintrust.input_json", args); + } + if (toolCallResult != null) { + span.setAttribute("braintrust.output", toolCallResult); + } + } catch (Exception e) { + log.debug("Failed to set tool span attributes", e); + } + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/langchain/WrappedHttpClient.java b/src/main/java/dev/braintrust/instrumentation/langchain/WrappedHttpClient.java index d9215f7..54668d7 100644 --- a/src/main/java/dev/braintrust/instrumentation/langchain/WrappedHttpClient.java +++ b/src/main/java/dev/braintrust/instrumentation/langchain/WrappedHttpClient.java @@ -71,7 +71,7 @@ public void execute(HttpRequest request, ServerSentEventListener listener) { ProviderInfo providerInfo = new ProviderInfo(options.providerName(), extractEndpoint(request)); Span span = startNewSpan(getSpanName(providerInfo)); - try (Scope scope = span.makeCurrent()) { + try (Scope ignored = span.makeCurrent()) { tagSpan(span, request, providerInfo); underlying.execute( request, new WrappedServerSentEventListener(listener, span, providerInfo)); @@ -94,7 +94,7 @@ public void execute( ProviderInfo providerInfo = new ProviderInfo(options.providerName(), extractEndpoint(request)); Span span = startNewSpan(getSpanName(providerInfo)); - try { + try (Scope ignored = span.makeCurrent()) { tagSpan(span, request, providerInfo); underlying.execute( request, @@ -235,6 +235,7 @@ private static class WrappedServerSentEventListener implements ServerSentEventLi private long firstTokenTime = 0; private final long startTime; private JsonNode usageData = null; + private String finishReason = null; WrappedServerSentEventListener( ServerSentEventListener delegate, Span span, ProviderInfo providerInfo) { @@ -246,19 +247,25 @@ private static class WrappedServerSentEventListener implements ServerSentEventLi @Override public void onOpen(SuccessfulHttpResponse response) { - delegate.onOpen(response); + try (Scope ignored = span.makeCurrent()) { + delegate.onOpen(response); + } } @Override public void onEvent(ServerSentEvent event, ServerSentEventContext context) { - instrumentEvent(event); - delegate.onEvent(event, context); + try (Scope ignored = span.makeCurrent()) { + instrumentEvent(event); + delegate.onEvent(event, context); + } } @Override public void onEvent(ServerSentEvent event) { - instrumentEvent(event); - delegate.onEvent(event); + try (Scope ignored = span.makeCurrent()) { + instrumentEvent(event); + delegate.onEvent(event); + } } private void instrumentEvent(ServerSentEvent event) { @@ -287,6 +294,10 @@ private void instrumentEvent(ServerSentEvent event) { outputBuffer.append(content); } } + // Capture finish_reason when present (usually in the last chunk) + if (choice.has("finish_reason") && !choice.get("finish_reason").isNull()) { + finishReason = choice.get("finish_reason").asText(); + } } // Extract usage data if present (usually in the last chunk) @@ -300,7 +311,7 @@ private void instrumentEvent(ServerSentEvent event) { @Override public void onError(Throwable error) { - try { + try (Scope ignored = span.makeCurrent()) { delegate.onError(error); } finally { tagSpan(span, error); @@ -311,7 +322,7 @@ public void onError(Throwable error) { @Override public void onClose() { - try { + try (Scope ignored = span.makeCurrent()) { delegate.onClose(); } finally { finalizeSpan(); @@ -332,12 +343,14 @@ private void finalizeSpan() { // Reconstruct output as a choices array for streaming // Format: [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", // "content": "..."}}] - if (outputBuffer.length() > 0) { + if (outputBuffer.length() > 0 || finishReason != null) { try { // Create a proper choice object matching OpenAI API format var choiceBuilder = JSON_MAPPER.createObjectNode(); choiceBuilder.put("index", 0); - choiceBuilder.put("finish_reason", "stop"); + if (finishReason != null) { + choiceBuilder.put("finish_reason", finishReason); + } var messageNode = JSON_MAPPER.createObjectNode(); messageNode.put("role", "assistant"); diff --git a/src/test/java/dev/braintrust/VCR.java b/src/test/java/dev/braintrust/VCR.java index 61b8f09..880fffb 100644 --- a/src/test/java/dev/braintrust/VCR.java +++ b/src/test/java/dev/braintrust/VCR.java @@ -137,7 +137,11 @@ private void startRecording(String targetBaseUrl) { .captureHeader("Content-Type") // .captureHeader("Authorization", true) .extractTextBodiesOver(0) // Always extract bodies - .makeStubsPersistent(true); // Save to disk + .makeStubsPersistent(true) // Save to disk + // Use JSON matching: + // - ignoreArrayOrder=true + // - ignoreExtraElements=false + .matchRequestBodyWithEqualToJson(true, false); wireMock.startRecording(recordSpec); } diff --git a/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java b/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java index 67d0927..153bcdf 100644 --- a/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java +++ b/src/test/java/dev/braintrust/instrumentation/langchain/BraintrustLangchainTest.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import dev.braintrust.TestHarness; +import dev.langchain4j.agent.tool.Tool; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.ChatModel; import dev.langchain4j.model.chat.StreamingChatModel; @@ -12,10 +13,14 @@ import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; import dev.langchain4j.model.openai.OpenAiChatModel; import dev.langchain4j.model.openai.OpenAiStreamingChatModel; +import dev.langchain4j.service.AiServices; import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.sdk.trace.data.SpanData; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,6 +36,14 @@ void beforeEach() { testHarness = TestHarness.setup(); } + @Test + @SneakyThrows + void typeToolJsonCorrect() { + assertEquals( + JSON_MAPPER.writeValueAsString(Map.of("type", "tool")), + TracingToolExecutor.TYPE_TOOL_JSON); + } + @Test @SneakyThrows void testSyncChatCompletion() { @@ -114,7 +127,8 @@ void testSyncChatCompletion() { @Test @SneakyThrows void testStreamingChatCompletion() { - // Create LangChain4j streaming client with Braintrust instrumentation using VCR + var tracer = testHarness.openTelemetry().getTracer("test-tracer"); + StreamingChatModel model = BraintrustLangchain.wrap( testHarness.openTelemetry(), @@ -127,12 +141,19 @@ void testStreamingChatCompletion() { // Execute streaming chat request var future = new CompletableFuture(); var responseBuilder = new StringBuilder(); + var callbackCount = new AtomicInteger(0); model.chat( "What is the capital of France?", new StreamingChatResponseHandler() { @Override public void onPartialResponse(String token) { + // Create a child span during the callback to verify parenting + Span childSpan = + tracer.spanBuilder( + "callback-span-" + callbackCount.incrementAndGet()) + .startSpan(); + childSpan.end(); responseBuilder.append(token); } @@ -154,16 +175,44 @@ public void onError(Throwable error) { assertNotNull(response); assertFalse(responseBuilder.toString().isEmpty(), "Response should not be empty"); - // Verify spans were exported - var spans = testHarness.awaitExportedSpans(1); - assertEquals(1, spans.size(), "Expected one span for streaming chat completion"); - var span = spans.get(0); + // We expect at least 2 spans: 1 LLM span + at least 1 callback span + int expectedMinSpans = 1 + callbackCount.get(); + var spans = testHarness.awaitExportedSpans(expectedMinSpans); + assertTrue( + spans.size() >= expectedMinSpans, + "Expected at least " + expectedMinSpans + " spans, got " + spans.size()); + + // Find the LLM span and callback spans + SpanData llmSpan = null; + List callbackSpans = new java.util.ArrayList<>(); + + for (var span : spans) { + if (span.getName().equals("Chat Completion")) { + llmSpan = span; + } else if (span.getName().startsWith("callback-span-")) { + callbackSpans.add(span); + } + } - // Verify span name - assertEquals("Chat Completion", span.getName(), "Span name should be 'Chat Completion'"); + assertNotNull(llmSpan, "Should have an LLM span named 'Chat Completion'"); + assertEquals( + callbackCount.get(), + callbackSpans.size(), + "Should have one callback span per onPartialResponse invocation"); + + // Verify all callback spans are parented under the LLM span + String llmSpanId = llmSpan.getSpanId(); + for (var callbackSpan : callbackSpans) { + assertEquals( + llmSpanId, + callbackSpan.getParentSpanId(), + "Callback span '" + + callbackSpan.getName() + + "' should be parented under LLM span"); + } - // Verify span attributes - var attributes = span.getAttributes(); + // Verify LLM span attributes + var attributes = llmSpan.getAttributes(); var braintrustSpanAttributesJson = attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); @@ -220,215 +269,87 @@ public void onError(Throwable error) { @Test @SneakyThrows - void testToolWrapping() { - // Create and wrap tools - TestTools tools = new TestTools(); - TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); - - // Call wrapped tool method directly - String result = wrappedTools.getWeather("Paris"); - assertNotNull(result); - assertTrue(result.contains("Paris")); - - // Verify span was created - var spans = testHarness.awaitExportedSpans(); - assertEquals(1, spans.size(), "Expected one span for tool execution"); - var span = spans.get(0); - - // Verify span name - assertEquals("getWeather", span.getName()); - - // Verify span type - var attributes = span.getAttributes(); - String spanAttrsJson = attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); - JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson); - assertEquals("tool", spanAttrs.get("type").asText()); - assertEquals("getWeather", spanAttrs.get("name").asText()); - - // Verify input (parameter names may be arg0, arg1, etc without -parameters flag) - String inputJson = attributes.get(AttributeKey.stringKey("braintrust.input_json")); - assertNotNull(inputJson); - JsonNode input = JSON_MAPPER.readTree(inputJson); - assertTrue(input.isObject(), "Input should be an object"); - assertTrue(input.size() > 0, "Input should have at least one parameter"); - // Check if parameter value is present (either as "location" or "arg0") - String paramValue = - input.has("location") - ? input.get("location").asText() - : input.elements().next().asText(); - assertEquals("Paris", paramValue); - - // Verify output - String outputJson = attributes.get(AttributeKey.stringKey("braintrust.output_json")); - assertNotNull(outputJson); - JsonNode output = JSON_MAPPER.readTree(outputJson); - assertTrue(output.asText().contains("Paris")); - assertTrue(output.asText().contains("72")); - - // Verify metrics - String metricsJson = attributes.get(AttributeKey.stringKey("braintrust.metrics")); - JsonNode metrics = JSON_MAPPER.readTree(metricsJson); - assertTrue(metrics.has("execution_time")); - assertTrue(metrics.get("execution_time").asDouble() >= 0); - } - - @Test - @SneakyThrows - void testToolWrappingWithException() { - TestTools tools = new TestTools(); - TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); - - // Execute and expect exception - assertThrows( - RuntimeException.class, - () -> { - wrappedTools.throwError(); - }); - - // Verify span with error status - var spans = testHarness.awaitExportedSpans(); - assertEquals(1, spans.size()); - var span = spans.get(0); - - assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); - assertTrue( - span.getEvents().stream().anyMatch(e -> e.getName().equals("exception")), - "Span should have exception event"); - } - - @Test - @SneakyThrows - void testToolWrappingWithMultipleCalls() { - TestTools tools = new TestTools(); - TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); + void testAiServicesWithTools() { + Assistant assistant = + BraintrustLangchain.wrap( + testHarness.openTelemetry(), + AiServices.builder(Assistant.class) + .chatModel( + OpenAiChatModel.builder() + .apiKey(testHarness.openAiApiKey()) + .baseUrl(testHarness.openAiBaseUrl()) + .modelName("gpt-4o-mini") + .temperature(0.0) + .build()) + .tools(new WeatherTools()) + .executeToolsConcurrently()); + + // This should trigger two (concurrent) tool calls + var response = assistant.chat("is it hotter in Paris or New York right now?"); - // Call multiple tool methods - wrappedTools.getWeather("Tokyo"); - int sum = wrappedTools.calculateSum(5, 7); - assertEquals(12, sum); + // Verify the response + assertNotNull(response); - // Verify two spans created - var spans = testHarness.awaitExportedSpans(); - assertEquals(2, spans.size(), "Expected two spans"); - - // Verify first span (getWeather) - var weatherSpan = spans.get(0); - assertEquals("getWeather", weatherSpan.getName()); - - // Verify second span (calculateSum) - var sumSpan = spans.get(1); - assertEquals("calculateSum", sumSpan.getName()); - String sumOutput = - sumSpan.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); - assertEquals("12", sumOutput); + // Verify spans were exported - should have at least: + // - one AI service method span ("chat") + // - at least two LLM calls (one to request the tool calls, and another to analyze) + // - at least two tool call spans + var spans = testHarness.awaitExportedSpans(3); + assertTrue(spans.size() >= 3, "Expected at least 3 spans for AI Services with tools"); + + // Verify we have the expected span types + int numServiceMethodSpans = 0; + int numLLMSpans = 0; + int numToolCallSpans = 0; + + for (var span : spans) { + String spanName = span.getName(); + var attributes = span.getAttributes(); + + if (spanName.equals("chat")) { + numServiceMethodSpans++; + } else if (spanName.equals("Chat Completion")) { + numLLMSpans++; + // Verify LLM span has proper attributes + var spanAttributesJson = + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(spanAttributesJson, "LLM span should have span_attributes"); + JsonNode spanAttributes = JSON_MAPPER.readTree(spanAttributesJson); + assertEquals( + "llm", spanAttributes.get("type").asText(), "Span type should be 'llm'"); + } else if (spanName.equals("getWeather")) { + numToolCallSpans++; + // Verify tool span has proper attributes + var spanAttributesJson = + attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(spanAttributesJson, "Tool span should have span_attributes"); + JsonNode spanAttributes = JSON_MAPPER.readTree(spanAttributesJson); + assertEquals( + "tool", spanAttributes.get("type").asText(), "Span type should be 'tool'"); + } + } + assertEquals(1, numServiceMethodSpans, "should be exactly one service call"); + assertTrue(numLLMSpans >= 2, "should be at least two llm spans"); + assertTrue(numToolCallSpans >= 2, "should be at least two tool call spans"); } - @Test - @SneakyThrows - void testToolWrappingAllBraintrustAttributesPresent() { - // This test verifies ALL required Braintrust attributes are present - // to catch issues that would cause UI problems - TestTools tools = new TestTools(); - TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); - - wrappedTools.getWeather("London"); - - var spans = testHarness.awaitExportedSpans(); - var span = spans.get(0); - var attributes = span.getAttributes(); - - // CRITICAL: These attributes MUST be present for Braintrust UI to display properly - assertNotNull( - attributes.get(AttributeKey.stringKey("braintrust.span_attributes")), - "braintrust.span_attributes is required for UI"); - assertNotNull( - attributes.get(AttributeKey.stringKey("braintrust.input_json")), - "braintrust.input_json is required for UI to show inputs"); - assertNotNull( - attributes.get(AttributeKey.stringKey("braintrust.output_json")), - "braintrust.output_json is required for UI to show outputs"); - assertNotNull( - attributes.get(AttributeKey.stringKey("braintrust.metrics")), - "braintrust.metrics is required for UI to show metrics"); - - // Verify span_attributes has correct structure - String spanAttrsJson = attributes.get(AttributeKey.stringKey("braintrust.span_attributes")); - JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson); - assertTrue(spanAttrs.has("type"), "span_attributes must have 'type' field"); - assertTrue(spanAttrs.has("name"), "span_attributes must have 'name' field"); - assertEquals("tool", spanAttrs.get("type").asText(), "Tool spans must have type='tool'"); + /** AI Service interface for the assistant */ + interface Assistant { + String chat(String userMessage); } - @Test - @SneakyThrows - void testToolWrappingIntegrationWithConversationHierarchy() { - // This test simulates a realistic usage pattern like the example - // and verifies ALL spans in the hierarchy have input/output for UI - var tracer = testHarness.openTelemetry().getTracer("test"); - TestTools tools = new TestTools(); - TestTools wrappedTools = BraintrustLangchain.wrapTools(testHarness.openTelemetry(), tools); - - // Create conversation span (like example does) - var conversationSpan = tracer.spanBuilder("conversation").startSpan(); - conversationSpan.setAttribute( - "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"conversation\"}"); - conversationSpan.setAttribute( - "braintrust.input_json", "{\"description\":\"test conversation\"}"); - - try (var ignored = conversationSpan.makeCurrent()) { - // Create turn span (like example does) - var turnSpan = tracer.spanBuilder("turn_1").startSpan(); - turnSpan.setAttribute( - "braintrust.span_attributes", "{\"type\":\"task\",\"name\":\"turn_1\"}"); - turnSpan.setAttribute("braintrust.input_json", "{\"user_message\":\"test query\"}"); - - try (var turnScope = turnSpan.makeCurrent()) { - // Call tool within turn (tool wrapper should create tool span) - String result = wrappedTools.getWeather("Paris"); - turnSpan.setAttribute( - "braintrust.output_json", "{\"assistant_message\":\"" + result + "\"}"); - } finally { - turnSpan.end(); - } - } finally { - conversationSpan.setAttribute("braintrust.output_json", "{\"status\":\"completed\"}"); - conversationSpan.end(); + /** Example tool class with weather-related methods */ + public static class WeatherTools { + @Tool("Get current weather for a location") + public String getWeather(String location) { + return String.format("The weather in %s is sunny with 72°F temperature.", location); } - // Verify all 3 spans exist and have required attributes - var spans = testHarness.awaitExportedSpans(); - assertEquals(3, spans.size(), "Expected 3 spans: conversation, turn, and tool"); - - // Find each span type - var toolSpan = - spans.stream().filter(s -> s.getName().equals("getWeather")).findFirst().get(); - var turnSpanData = - spans.stream().filter(s -> s.getName().equals("turn_1")).findFirst().get(); - var convSpan = - spans.stream().filter(s -> s.getName().equals("conversation")).findFirst().get(); - - // CRITICAL: Every span must have input/output for UI to display properly - for (var span : List.of(toolSpan, turnSpanData, convSpan)) { - var attrs = span.getAttributes(); - assertNotNull( - attrs.get(AttributeKey.stringKey("braintrust.span_attributes")), - span.getName() + " missing braintrust.span_attributes"); - assertNotNull( - attrs.get(AttributeKey.stringKey("braintrust.input_json")), - span.getName() + " missing braintrust.input_json - UI won't show input!"); - assertNotNull( - attrs.get(AttributeKey.stringKey("braintrust.output_json")), - span.getName() + " missing braintrust.output_json - UI won't show output!"); + @Tool("Get weather forecast for next N days") + public String getForecast(String location, int days) { + return String.format( + "The %d-day forecast for %s: Mostly sunny with temperatures between 65-75°F.", + days, location); } - - // Verify span hierarchy (parent-child relationships) - assertEquals( - convSpan.getSpanId(), - turnSpanData.getParentSpanId(), - "Turn should be child of conversation"); - assertEquals( - turnSpanData.getSpanId(), - toolSpan.getParentSpanId(), - "Tool should be child of turn"); } }