Skip to content

Commit 2b95212

Browse files
realarkrealark
committed
initial support for remote evals
1 parent 0495e7e commit 2b95212

16 files changed

Lines changed: 2633 additions & 4 deletions

File tree

‎examples/build.gradle‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,18 @@ task runSpringAI(type: JavaExec) {
141141
suspend = false
142142
}
143143
}
144+
145+
146+
task runRemoteEval(type: JavaExec) {
147+
group = 'Braintrust SDK Examples'
148+
description = 'Run the remote eval example'
149+
classpath = sourceSets.main.runtimeClasspath
150+
mainClass = 'dev.braintrust.examples.RemoteEvalExample'
151+
systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel
152+
debugOptions {
153+
enabled = true
154+
port = 5566
155+
server = true
156+
suspend = false
157+
}
158+
}

‎examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package dev.braintrust.examples;
2+
3+
import com.openai.client.okhttp.OpenAIOkHttpClient;
4+
import com.openai.models.ChatModel;
5+
import com.openai.models.chat.completions.ChatCompletionCreateParams;
6+
import dev.braintrust.Braintrust;
7+
import dev.braintrust.devserver.Devserver;
8+
import dev.braintrust.devserver.RemoteEval;
9+
import dev.braintrust.eval.Scorer;
10+
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
11+
import java.util.List;
12+
13+
/** Simple Dev Server for Remote Evals */
14+
public class RemoteEvalExample {
15+
public static void main(String[] args) throws Exception {
16+
var braintrust = Braintrust.get();
17+
var openTelemetry = braintrust.openTelemetryCreate();
18+
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
19+
20+
RemoteEval<String, String> foodTypeEval =
21+
RemoteEval.<String, String>builder()
22+
.name("food-type-classifier")
23+
.taskFunction(
24+
food -> {
25+
var request =
26+
ChatCompletionCreateParams.builder()
27+
.model(ChatModel.GPT_4O_MINI)
28+
.addSystemMessage("Return a one word answer")
29+
.addUserMessage(
30+
"What kind of food is " + food + "?")
31+
.maxTokens(50L)
32+
.temperature(0.0)
33+
.build();
34+
var response =
35+
openAIClient.chat().completions().create(request);
36+
return response.choices()
37+
.get(0)
38+
.message()
39+
.content()
40+
.orElse("")
41+
.toLowerCase();
42+
})
43+
.scorers(
44+
List.of(
45+
Scorer.of("static_scorer", (expected, result) -> 0.7),
46+
Scorer.of(
47+
"close_enough_match",
48+
(expected, result) ->
49+
expected.trim()
50+
.equalsIgnoreCase(
51+
result.trim())
52+
? 1.0
53+
: 0.0)))
54+
.build();
55+
56+
Devserver devserver =
57+
Devserver.builder()
58+
.config(braintrust.config())
59+
.registerEval(foodTypeEval)
60+
.host("localhost") // set to 0.0.0.0 to bind all interfaces
61+
.port(8301)
62+
.build();
63+
64+
Runtime.getRuntime()
65+
.addShutdownHook(
66+
new Thread(
67+
() -> {
68+
System.out.println("Shutting down...");
69+
devserver.stop();
70+
System.out.flush();
71+
System.err.flush();
72+
}));
73+
System.out.println("Starting Braintrust dev server");
74+
devserver.start();
75+
}
76+
}

‎src/main/java/dev/braintrust/BraintrustUtils.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
import dev.braintrust.api.BraintrustApiClient;
44
import java.net.URI;
55
import java.net.URISyntaxException;
6+
import java.util.ArrayList;
7+
import java.util.Arrays;
8+
import java.util.List;
69
import javax.annotation.Nonnull;
710

811
public class BraintrustUtils {
@@ -42,4 +45,18 @@ public String toParentValue() {
4245
return type + ":" + id;
4346
}
4447
}
48+
49+
public static List<String> parseCsv(String csv) {
50+
if (csv == null || csv.isBlank()) {
51+
return List.of();
52+
}
53+
54+
return Arrays.stream(csv.split("\\s*,\\s*")).toList();
55+
}
56+
57+
public static <T> List<T> append(List<T> list, T value) {
58+
List<T> result = new ArrayList<>(list);
59+
result.add(value);
60+
return result;
61+
}
4562
}

‎src/main/java/dev/braintrust/config/BraintrustConfig.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ public final class BraintrustConfig extends BaseConfig {
5151
private final boolean exportSpansInMemoryForUnitTest =
5252
getConfig("BRAINTRUST_JAVA_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", false);
5353

54+
/** CORS origins to allow when running remote eval devserver */
55+
private final String devserverCorsOriginWhitelistCsv =
56+
getConfig(
57+
"BRAINTRUST_DEVSERVER_CORS_ORIGIN_WHITELIST_CSV",
58+
"https://www.braintrust.dev,https://www.braintrustdata.com,http://localhost:3000");
59+
5460
public static BraintrustConfig fromEnvironment() {
5561
return of();
5662
}
@@ -192,8 +198,8 @@ Builder experimentalOtelLogs(boolean value) {
192198
return this;
193199
}
194200

195-
// hiding visibility. only used for testing
196-
Builder exportSpansInMemoryForUnitTest(boolean value) {
201+
// only used for testing
202+
public Builder exportSpansInMemoryForUnitTest(boolean value) {
197203
envOverrides.put(
198204
"BRAINTRUST_JAVA_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", String.valueOf(value));
199205
return this;
@@ -209,6 +215,11 @@ public Builder x509TrustManager(X509TrustManager value) {
209215
return this;
210216
}
211217

218+
public Builder devserverCorsOriginWhitelistCsv(String csv) {
219+
envOverrides.put("BRAINTRUST_DEVSERVER_CORS_ORIGIN_WHITELIST_CSV", csv);
220+
return this;
221+
}
222+
212223
public BraintrustConfig build() {
213224
return new BraintrustConfig(envOverrides, sslContext, x509TrustManager);
214225
}

0 commit comments

Comments
 (0)