From cc3ca9384229f05243affcef86c53364f7e2eb9a Mon Sep 17 00:00:00 2001 From: Matt Perpick Date: Fri, 9 Jan 2026 10:44:22 -0500 Subject: [PATCH 1/2] handle prompts, templating, etc. --- .claude/settings.json | 47 ++ examples/prompt.rb | 95 ++++ lib/braintrust.rb | 1 + lib/braintrust/api/functions.rb | 8 + lib/braintrust/prompt.rb | 172 +++++++ test/braintrust/api/functions_test.rb | 51 ++ test/braintrust/prompt_test.rb | 203 ++++++++ test/fixtures/vcr_cassettes/functions/get.yml | 449 ++++++++++++++++++ test/fixtures/vcr_cassettes/prompt/load.yml | 389 +++++++++++++++ .../vcr_cassettes/prompt/load_not_found.yml | 143 ++++++ 10 files changed, 1558 insertions(+) create mode 100644 .claude/settings.json create mode 100644 examples/prompt.rb create mode 100644 lib/braintrust/prompt.rb create mode 100644 test/braintrust/prompt_test.rb create mode 100644 test/fixtures/vcr_cassettes/functions/get.yml create mode 100644 test/fixtures/vcr_cassettes/prompt/load.yml create mode 100644 test/fixtures/vcr_cassettes/prompt/load_not_found.yml diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..21984fd --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,47 @@ +{ + "permissions": { + "allow": [ + "Bash(git log:*)", + "Bash(git branch:*)", + "Bash(git remote:*)", + "Bash(git fetch:*)", + "Bash(git stash:*)", + "Bash(git diff:*)", + "Bash(git status:*)", + "Bash(git show:*)", + "Bash(gh repo view:*)", + "Bash(gh run list:*)", + "Bash(gh run view:*)", + "Bash(gh pr view:*)", + "Bash(gh pr diff:*)", + "Bash(gh pr list:*)", + "Bash(gh issue view:*)", + "Bash(gh release:*)", + "Bash(gh api:*)", + "Bash(rake -T:*)", + "Bash(bundle show:*)", + "Bash(bundle list:*)", + "Bash(bundle platform:*)", + "Bash(gem search:*)", + "Bash(gem list:*)", + "Bash(tree:*)", + "Bash(find:*)", + "Bash(echo:*)", + "Bash(sort:*)", + "Bash(cat:*)", + "Bash(head:*)", + "Bash(tail:*)", + "Bash(less:*)", + "Bash(wc:*)", + "Bash(ls:*)", + "Bash(pwd:*)", + "Bash(which:*)", + "Bash(type:*)", + "Bash(file:*)", + "Bash(VCR_MODE=new_episodes bundle exec ruby:*)", + "Bash(VCR_MODE=new_episodes bundle exec rake:*)", + "Bash(VCR_MODE=new_episodes rake test:*)" + ], + "deny": [] + } +} diff --git a/examples/prompt.rb b/examples/prompt.rb new file mode 100644 index 0000000..32a51e3 --- /dev/null +++ b/examples/prompt.rb @@ -0,0 +1,95 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Example: Loading and using prompts from Braintrust +# +# This example demonstrates how to: +# 1. Create a prompt (function) on the Braintrust server +# 2. Load it using Prompt.load +# 3. Build the prompt with variable substitution +# 4. Use the built prompt with an LLM client +# +# Benefits of loading prompts: +# - Centralized prompt management in Braintrust UI +# - Version control and A/B testing for prompts +# - No code deployment needed for prompt changes +# - Works with any LLM client (OpenAI, Anthropic, etc.) + +require "bundler/setup" +require "braintrust" + +# Initialize Braintrust +Braintrust.init + +project_name = "ruby-sdk-examples" +prompt_slug = "greeting-prompt-#{Time.now.to_i}" + +# First, create a prompt on the server +# In practice, you would create prompts via the Braintrust UI +puts "Creating prompt..." + +api = Braintrust::API.new +api.functions.create( + project_name: project_name, + slug: prompt_slug, + function_data: {type: "prompt"}, + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "system", + content: "You are a friendly assistant who speaks {{language}}." + }, + { + role: "user", + content: "Say hello to {{name}} and wish them a great {{time_of_day}}!" + } + ] + }, + options: { + model: "gpt-4o-mini", + params: {temperature: 0.7, max_tokens: 100} + } + } +) +puts "Created prompt: #{prompt_slug}" + +# Load the prompt using Prompt.load +puts "\nLoading prompt..." +prompt = Braintrust::Prompt.load(project: project_name, slug: prompt_slug) + +puts " ID: #{prompt.id}" +puts " Name: #{prompt.name}" +puts " Model: #{prompt.model}" +puts " Messages: #{prompt.messages.length}" + +# Build the prompt with variable substitution +puts "\nBuilding prompt with variables..." +params = prompt.build( + name: "Alice", + language: "Spanish", + time_of_day: "morning" +) + +puts " Model: #{params[:model]}" +puts " Temperature: #{params[:temperature]}" +puts " Max tokens: #{params[:max_tokens]}" +puts " Messages:" +params[:messages].each do |msg| + puts " [#{msg[:role]}] #{msg[:content]}" +end + +# The params hash is ready to pass to any LLM client: +# +# With OpenAI: +# client.chat.completions.create(**params) +# +# With Anthropic: +# client.messages.create(**params) + +puts "\nPrompt is ready to use with any LLM client!" + +# Clean up - delete the test prompt +api.functions.delete(id: prompt.id) +puts "Cleaned up test prompt." diff --git a/lib/braintrust.rb b/lib/braintrust.rb index 469cde1..7e6b676 100644 --- a/lib/braintrust.rb +++ b/lib/braintrust.rb @@ -5,6 +5,7 @@ require_relative "braintrust/state" require_relative "braintrust/trace" require_relative "braintrust/api" +require_relative "braintrust/prompt" require_relative "braintrust/internal/experiments" require_relative "braintrust/eval" diff --git a/lib/braintrust/api/functions.rb b/lib/braintrust/api/functions.rb index ccfea31..66d82b0 100644 --- a/lib/braintrust/api/functions.rb +++ b/lib/braintrust/api/functions.rb @@ -85,6 +85,14 @@ def invoke(id:, input:) http_post_json("/v1/function/#{id}/invoke", payload) end + # Get a function by ID (includes full prompt_data) + # GET /v1/function/{id} + # @param id [String] Function UUID + # @return [Hash] Full function data including prompt_data + def get(id:) + http_get("/v1/function/#{id}") + end + # Delete a function by ID # DELETE /v1/function/{id} # @param id [String] Function UUID diff --git a/lib/braintrust/prompt.rb b/lib/braintrust/prompt.rb new file mode 100644 index 0000000..d417e95 --- /dev/null +++ b/lib/braintrust/prompt.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: true + +module Braintrust + # Prompt class for loading and building prompts from Braintrust + # + # @example Load and use a prompt + # prompt = Braintrust::Prompt.load(project: "my-project", slug: "summarizer") + # params = prompt.build(text: "Article to summarize...") + # client.messages.create(**params) + class Prompt + attr_reader :id, :name, :slug, :project_id + + # Load a prompt from Braintrust + # + # @param project [String] Project name + # @param slug [String] Prompt slug + # @param version [String, nil] Specific version (default: latest) + # @param defaults [Hash] Default variable values for build() + # @param state [State, nil] Braintrust state (default: global) + # @return [Prompt] + def self.load(project:, slug:, version: nil, defaults: {}, state: nil) + state ||= Braintrust.current_state + raise Error, "No state available - call Braintrust.init first" unless state + + api = API.new(state: state) + + # Find the function by project + slug + result = api.functions.list(project_name: project, slug: slug) + function = result.dig("objects")&.first + raise Error, "Prompt '#{slug}' not found in project '#{project}'" unless function + + # Fetch full function data including prompt_data + full_data = api.functions.get(id: function["id"]) + + new(full_data, defaults: defaults) + end + + # Initialize a Prompt from function data + # + # @param data [Hash] Function data from API + # @param defaults [Hash] Default variable values for build() + def initialize(data, defaults: {}) + @data = data + @defaults = stringify_keys(defaults) + + @id = data["id"] + @name = data["name"] + @slug = data["slug"] + @project_id = data["project_id"] + end + + # Get the raw prompt definition + # @return [Hash, nil] + def prompt + @data.dig("prompt_data", "prompt") + end + + # Get the prompt messages + # @return [Array] + def messages + prompt&.dig("messages") || [] + end + + # Get the model name + # @return [String, nil] + def model + @data.dig("prompt_data", "options", "model") + end + + # Get model options + # @return [Hash] + def options + @data.dig("prompt_data", "options") || {} + end + + # Build the prompt with variable substitution + # + # Returns a hash ready to pass to an LLM client: + # {model: "...", messages: [...], temperature: ..., ...} + # + # @param variables [Hash] Variables to substitute (e.g., {name: "Alice"}) + # @param strict [Boolean] Raise error on missing variables (default: false) + # @return [Hash] Built prompt ready for LLM client + # + # @example With keyword arguments + # prompt.build(name: "Alice", task: "coding") + # + # @example With explicit hash + # prompt.build({name: "Alice"}, strict: true) + def build(variables = nil, strict: false, **kwargs) + # Support both explicit hash and keyword arguments + variables_hash = variables.is_a?(Hash) ? variables : {} + vars = @defaults.merge(stringify_keys(variables_hash)).merge(stringify_keys(kwargs)) + + # Substitute variables in messages + built_messages = messages.map do |msg| + { + role: msg["role"].to_sym, + content: substitute_variables(msg["content"], vars, strict: strict) + } + end + + # Build result with model and messages + result = { + model: model, + messages: built_messages + } + + # Add params (temperature, max_tokens, etc.) to top level + params = options.dig("params") + if params.is_a?(Hash) + params.each do |key, value| + result[key.to_sym] = value + end + end + + result + end + + private + + # Substitute {{variable}} placeholders with values + def substitute_variables(text, variables, strict:) + return text unless text.is_a?(String) + + # Find all {{variable}} patterns + missing = [] + + result = text.gsub(/\{\{([^}]+)\}\}/) do |match| + var_path = ::Regexp.last_match(1).strip + value = resolve_variable(var_path, variables) + + if value.nil? + missing << var_path + match # Keep original placeholder + else + value.to_s + end + end + + if strict && missing.any? + raise Error, "Missing required variables: #{missing.join(", ")}" + end + + result + end + + # Resolve a variable path like "user.name" from variables hash + def resolve_variable(path, variables) + parts = path.split(".") + value = variables + + parts.each do |part| + return nil unless value.is_a?(Hash) + # Try both string and symbol keys + value = value[part] || value[part.to_sym] + return nil if value.nil? + end + + value + end + + # Convert hash keys to strings (handles both symbol and string keys) + def stringify_keys(hash) + return {} unless hash.is_a?(Hash) + + hash.transform_keys(&:to_s).transform_values do |v| + v.is_a?(Hash) ? stringify_keys(v) : v + end + end + end +end diff --git a/test/braintrust/api/functions_test.rb b/test/braintrust/api/functions_test.rb index 753c1d2..bdb6e5b 100644 --- a/test/braintrust/api/functions_test.rb +++ b/test/braintrust/api/functions_test.rb @@ -378,4 +378,55 @@ def test_helper_validates_prompt_data_has_prompt_key end end end + + def test_functions_get_by_id + VCR.use_cassette("functions/get") do + api = get_test_api + # This test verifies that we can get the full function data by ID, + # including prompt_data which is needed for Prompt.load() + function_slug = "test-ruby-sdk-get-func" + + # Create a function with prompt_data + create_response = api.functions.create( + project_name: @project_name, + slug: function_slug, + function_data: {type: "prompt"}, + prompt_data: { + prompt: { + type: "chat", + messages: [ + {role: "system", content: "You are a helpful assistant."}, + {role: "user", content: "Hello {{name}}"} + ] + }, + options: { + model: "gpt-4o-mini", + params: {temperature: 0.7} + } + } + ) + function_id = create_response["id"] + + # Get the full function data + result = api.functions.get(id: function_id) + + assert_instance_of Hash, result + assert_equal function_id, result["id"] + assert_equal function_slug, result["slug"] + + # Verify prompt_data is included + assert result.key?("prompt_data") + prompt_data = result["prompt_data"] + assert prompt_data.key?("prompt") + assert prompt_data["prompt"].key?("messages") + assert_equal 2, prompt_data["prompt"]["messages"].length + + # Verify options are included + assert prompt_data.key?("options") + assert_equal "gpt-4o-mini", prompt_data["options"]["model"] + + # Clean up + api.functions.delete(id: function_id) + end + end end diff --git a/test/braintrust/prompt_test.rb b/test/braintrust/prompt_test.rb new file mode 100644 index 0000000..14ed7d1 --- /dev/null +++ b/test/braintrust/prompt_test.rb @@ -0,0 +1,203 @@ +# frozen_string_literal: true + +require "test_helper" + +class Braintrust::PromptTest < Minitest::Test + def setup + # Sample function data as returned by API + @function_data = { + "id" => "func-123", + "name" => "Test Prompt", + "slug" => "test-prompt", + "project_id" => "proj-456", + "prompt_data" => { + "prompt" => { + "type" => "chat", + "messages" => [ + {"role" => "system", "content" => "You are a helpful assistant."}, + {"role" => "user", "content" => "Hello {{name}}, please help with {{task}}."} + ] + }, + "options" => { + "model" => "claude-3-5-sonnet", + "params" => {"temperature" => 0.7, "max_tokens" => 1000} + } + } + } + end + + def test_prompt_initialization + prompt = Braintrust::Prompt.new(@function_data) + + assert_equal "func-123", prompt.id + assert_equal "Test Prompt", prompt.name + assert_equal "test-prompt", prompt.slug + assert_equal "proj-456", prompt.project_id + end + + def test_prompt_messages + prompt = Braintrust::Prompt.new(@function_data) + messages = prompt.messages + + assert_equal 2, messages.length + assert_equal "system", messages[0]["role"] + assert_equal "You are a helpful assistant.", messages[0]["content"] + assert_equal "user", messages[1]["role"] + end + + def test_prompt_model + prompt = Braintrust::Prompt.new(@function_data) + + assert_equal "claude-3-5-sonnet", prompt.model + end + + def test_prompt_options + prompt = Braintrust::Prompt.new(@function_data) + options = prompt.options + + assert_equal "claude-3-5-sonnet", options["model"] + assert_equal 0.7, options["params"]["temperature"] + assert_equal 1000, options["params"]["max_tokens"] + end + + def test_prompt_raw_returns_prompt_definition + prompt = Braintrust::Prompt.new(@function_data) + raw = prompt.prompt + + assert_equal "chat", raw["type"] + assert raw.key?("messages") + end + + def test_build_substitutes_variables + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build(name: "Alice", task: "coding") + + assert_equal "claude-3-5-sonnet", result[:model] + assert_equal 2, result[:messages].length + assert_equal "You are a helpful assistant.", result[:messages][0][:content] + assert_equal "Hello Alice, please help with coding.", result[:messages][1][:content] + end + + def test_build_with_string_keys + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build("name" => "Bob", "task" => "writing") + + assert_equal "Hello Bob, please help with writing.", result[:messages][1][:content] + end + + def test_build_includes_params + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build(name: "Alice", task: "coding") + + assert_equal 0.7, result[:temperature] + assert_equal 1000, result[:max_tokens] + end + + def test_build_with_defaults + prompt = Braintrust::Prompt.new(@function_data, defaults: {name: "Default User"}) + result = prompt.build(task: "testing") + + assert_equal "Hello Default User, please help with testing.", result[:messages][1][:content] + end + + def test_build_overrides_defaults + prompt = Braintrust::Prompt.new(@function_data, defaults: {name: "Default User"}) + result = prompt.build(name: "Override User", task: "testing") + + assert_equal "Hello Override User, please help with testing.", result[:messages][1][:content] + end + + def test_build_strict_raises_on_missing_variable + prompt = Braintrust::Prompt.new(@function_data) + + error = assert_raises(Braintrust::Error) do + prompt.build({name: "Alice"}, strict: true) + end + + assert_match(/missing.*task/i, error.message) + end + + def test_build_non_strict_leaves_missing_variables + prompt = Braintrust::Prompt.new(@function_data) + result = prompt.build(name: "Alice") + + assert_equal "Hello Alice, please help with {{task}}.", result[:messages][1][:content] + end + + def test_build_handles_nested_variables + data = @function_data.dup + data["prompt_data"]["prompt"]["messages"] = [ + {"role" => "user", "content" => "User: {{user.name}}, Email: {{user.email}}"} + ] + prompt = Braintrust::Prompt.new(data) + + result = prompt.build(user: {name: "Alice", email: "alice@example.com"}) + + assert_equal "User: Alice, Email: alice@example.com", result[:messages][0][:content] + end +end + +class Braintrust::PromptLoadTest < Minitest::Test + def setup + flunk "BRAINTRUST_API_KEY not set" unless ENV["BRAINTRUST_API_KEY"] + @project_name = "ruby-sdk-test" + end + + def test_prompt_load + VCR.use_cassette("prompt/load") do + Braintrust.init(blocking_login: true) + + # Create a prompt first + api = Braintrust::API.new + slug = "test-prompt-load" + + api.functions.create( + project_name: @project_name, + slug: slug, + function_data: {type: "prompt"}, + prompt_data: { + prompt: { + type: "chat", + messages: [ + {role: "user", content: "Say hello to {{name}}"} + ] + }, + options: { + model: "gpt-4o-mini" + } + } + ) + + # Load the prompt using Prompt.load + prompt = Braintrust::Prompt.load(project: @project_name, slug: slug) + + assert_instance_of Braintrust::Prompt, prompt + assert_equal slug, prompt.slug + assert_equal "gpt-4o-mini", prompt.model + assert_equal 1, prompt.messages.length + + # Test build + result = prompt.build(name: "World") + assert_equal "Say hello to World", result[:messages][0][:content] + + # Clean up + api.functions.delete(id: prompt.id) + ensure + OpenTelemetry.tracer_provider.shutdown + end + end + + def test_prompt_load_not_found + VCR.use_cassette("prompt/load_not_found") do + Braintrust.init(blocking_login: true) + + error = assert_raises(Braintrust::Error) do + Braintrust::Prompt.load(project: @project_name, slug: "nonexistent-prompt-xyz") + end + + assert_match(/not found/i, error.message) + ensure + OpenTelemetry.tracer_provider.shutdown + end + end +end diff --git a/test/fixtures/vcr_cassettes/functions/get.yml b/test/fixtures/vcr_cassettes/functions/get.yml new file mode 100644 index 0000000..908caac --- /dev/null +++ b/test/fixtures/vcr_cassettes/functions/get.yml @@ -0,0 +1,449 @@ +--- +http_interactions: +- request: + method: get + uri: https://api.braintrust.dev/v1/function/eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '650' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:22:45 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 999f045e-52bf-431a-ab23-4231e84c599b + X-Bt-Internal-Trace-Id: + - 69611d45000000001d09e277b8e05ffb + X-Amz-Apigw-Id: + - W7GC7HqrIAMEBEQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"28a-o+EOs8vizZBvL0+kfKefNOgxPc0" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611d45-406960f37e85c137106930c0;Parent=0943949cf2b37855;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 d9b04a822e1c215374729ec159356140.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - UeZO45ONY1S230A3ccsinel_eSg2umCVj5z3YObzFX1Y_j2Q3IXy6w== + body: + encoding: ASCII-8BIT + string: '{"id":"eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216","_xact_id":"1000196458014160880","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-ruby-sdk-get-func","slug":"test-ruby-sdk-get-func","description":null,"created":"2026-01-09T15:06:49.196Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}' + recorded_at: Fri, 09 Jan 2026 15:22:45 GMT +- request: + method: delete + uri: https://api.braintrust.dev/v1/function/eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '282' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:22:47 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - b46eee9c-fcb5-4035-897c-fae539611a0e + X-Bt-Internal-Trace-Id: + - 69611d47000000005cc7572f079de445 + X-Amz-Apigw-Id: + - W7GDKGoaoAMEg5g= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"11a-P+3A9L3licMg9JgEGiq9t1aKens" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611d47-4a348ed101c9c9fc6fdb1b86;Parent=14e14957b2cba183;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 b601959712c1f21193a489b5759f70ba.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - VsS0_f9QROIT-ckZpbiAmMpiT6skqE6pHr-hGzREkFyqEXIihNwgNA== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","id":"eaa1d7fa-ddfc-487e-bdf1-8c83c9e5e216","created":"2026-01-09T15:22:47.106Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196458076954719"}' + recorded_at: Fri, 09 Jan 2026 15:22:47 GMT +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MzQ5OGZkZjYtMmJlOC00NDY2LWE1MzAtODY5N2ViZjI5YzFl'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 09 Jan 2026 15:26:29 GMT + Etag: + - '"ubzjf1iqqj75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - MzQ5OGZkZjYtMmJlOC00NDY2LWE1MzAtODY5N2ViZjI5YzFl + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::vhxvp-1767972389112-bcc3207d4501 + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Fri, 09 Jan 2026 15:26:29 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/project?project_name=ruby-sdk-test + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '269' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:29 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - df8c7dca-c9f8-4554-bc08-c4003c5faaaa + X-Bt-Internal-Trace-Id: + - 69611e25000000002d2feed003d7844a + X-Amz-Apigw-Id: + - W7Gl4FNZIAMET2Q= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"10d-jHt6t+s3DyzhLf+d20I2XKvIjGQ" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e25-0e71a6817e098cd12d46df1e;Parent=55ae8824648bf68a;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 f25b89e7ef738cb8bb7e28e041d8fe54.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - Z8Zuf3NfTPdgGMykdiyvlEx-Hfpr-8BgDvslAhNlF-FBzdse37eiWQ== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","description":null,"created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}]}' + recorded_at: Fri, 09 Jan 2026 15:26:29 GMT +- request: + method: post + uri: https://api.braintrust.dev/v1/function + body: + encoding: UTF-8 + string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","name":"test-ruby-sdk-get-func","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}}}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '545' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:29 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 601a8e03-eb46-49c4-8d2a-facdecc0274a + X-Bt-Internal-Trace-Id: + - 69611e25000000004e9aa6fbc35c2222 + X-Amz-Apigw-Id: + - W7Gl8EE0IAMEBuw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"221-MYmA2S4f3IZPIdPfajiNHs1EESA" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e25-605279d552cb95a6336cea62;Parent=3d0b812115b7dd19;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 c4d0da6268789cfda9bb5da1f3f8fc58.cloudfront.net (CloudFront), 1.1 537c1727cc67e6d2567bb61ae0478182.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 5P3DoYCNcIvlRYIrkJ6CQY-dCflpP0eCy6XrAmn2kJObAN6B1_kgsA== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","name":"test-ruby-sdk-get-func","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}},"id":"9c763edd-0474-4034-8869-8bbe307fcd32","created":"2026-01-09T15:26:29.742Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196458091495183"}' + recorded_at: Fri, 09 Jan 2026 15:26:29 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function/9c763edd-0474-4034-8869-8bbe307fcd32 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '650' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:30 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 9a293bcc-24ab-4ec0-9b3f-842c0fce7b76 + X-Bt-Internal-Trace-Id: + - 69611e250000000002effecf6f118ea1 + X-Amz-Apigw-Id: + - W7Gl_FW4IAMEI8w= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"28a-dA3pKxLHq2B0teWLcR3HocUdXLs" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e25-2ebc8ae9137051f60738d543;Parent=66f232e4b293ec11;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 ab734ad5d81cc9d470b6176a05dd968e.cloudfront.net (CloudFront), 1.1 8ca36406fe3aa11c1641e5bc917c8a74.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - vOXNwjsPTHAO-Am1MlqnlLjUiJfihdYMdlRvPf-vQW1B3CP7ogmIAw== + body: + encoding: ASCII-8BIT + string: '{"id":"9c763edd-0474-4034-8869-8bbe307fcd32","_xact_id":"1000196458091495183","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-ruby-sdk-get-func","slug":"test-ruby-sdk-get-func","description":null,"created":"2026-01-09T15:26:29.742Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + are a helpful assistant."},{"role":"user","content":"Hello {{name}}"}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0.7}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}' + recorded_at: Fri, 09 Jan 2026 15:26:30 GMT +- request: + method: delete + uri: https://api.braintrust.dev/v1/function/9c763edd-0474-4034-8869-8bbe307fcd32 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '282' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:26:30 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - d1b4a95f-c311-4b54-8332-a8d893a10e09 + X-Bt-Internal-Trace-Id: + - 69611e260000000011fd6ec6f9aad01f + X-Amz-Apigw-Id: + - W7GmDGdooAMERvQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"11a-G6BnvsesfXj6YiI3nDotOemHZaM" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611e26-6993aa0d4516e67113188a3e;Parent=7678979c42d17017;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 b5fe18267507cb61755963d8928a60f4.cloudfront.net (CloudFront), 1.1 b601959712c1f21193a489b5759f70ba.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 8gF7-nfICr1kX_ArRuSfyuBcWIjLCSubLVjOW_gAi3mvyB_fNxsaxw== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-get-func","id":"9c763edd-0474-4034-8869-8bbe307fcd32","created":"2026-01-09T15:26:30.444Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196458091561660"}' + recorded_at: Fri, 09 Jan 2026 15:26:30 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/prompt/load.yml b/test/fixtures/vcr_cassettes/prompt/load.yml new file mode 100644 index 0000000..9df95d7 --- /dev/null +++ b/test/fixtures/vcr_cassettes/prompt/load.yml @@ -0,0 +1,389 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MGQzNDRiYjctMzcyZC00ODEzLTg0MmYtYjYwNmFiZWY0Nzc5'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 09 Jan 2026 15:31:57 GMT + Etag: + - '"ubzjf1iqqj75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - MGQzNDRiYjctMzcyZC00ODEzLTg0MmYtYjYwNmFiZWY0Nzc5 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::478kl-1767972717018-8f1c4449dd71 + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Fri, 09 Jan 2026 15:31:57 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/project?project_name=ruby-sdk-test + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '269' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:57 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 6308ae43-6aef-4e2a-b677-6eeb03896599 + X-Bt-Internal-Trace-Id: + - 69611f6d000000007e25f8ac6b23187b + X-Amz-Apigw-Id: + - W7HZIGr1IAMEjxA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"10d-jHt6t+s3DyzhLf+d20I2XKvIjGQ" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6d-6e2659e44897bfc601214fab;Parent=168da3c03f62beb4;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 ad22d4e4410fd07809425488bf6e79be.cloudfront.net (CloudFront), 1.1 f391dfb0806f29cccc5f1df3e1ae836e.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - dEzh8blLXvYXHkMRSBV5zi5DiwpekBQFkgXMWtmtiirn4jCsm4dU8w== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","description":null,"created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}]}' + recorded_at: Fri, 09 Jan 2026 15:31:57 GMT +- request: + method: post + uri: https://api.braintrust.dev/v1/function + body: + encoding: UTF-8 + string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-prompt-load","name":"test-prompt-load","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}}}' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Content-Type: + - application/json + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '452' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:57 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 411104c9-b444-450f-b49c-d7cd030b9b18 + X-Bt-Internal-Trace-Id: + - 69611f6d000000007281a26ff26ab68f + X-Amz-Apigw-Id: + - W7HZKEbxIAMEDJQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1c4-HTfX1Q46VKOG3lasMWxS8J85LCQ" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6d-27b6279359f4794e35dcdbc5;Parent=5158f835fa228c4a;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6ea9421ec132e3640100792ef9535494.cloudfront.net (CloudFront), 1.1 d3041c3025b9205db460853b5b9626bc.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - hL_HuFng0i2Jeib3N83UPHx5g9XfA0nnGHSST1ARKxpzagyH3dd-2g== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-prompt-load","name":"test-prompt-load","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}},"id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","created":"2026-01-09T15:31:57.524Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196458112984917"}' + recorded_at: Fri, 09 Jan 2026 15:31:57 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=test-prompt-load + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '571' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:58 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - af75d70c-ba8f-4244-a61e-fb6e643abd74 + X-Bt-Internal-Trace-Id: + - 69611f6d0000000079da7c9d7c78e29e + X-Amz-Apigw-Id: + - W7HZNF8ooAMErzg= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"23b-zKRi5DEcuekPd/TsGth74hG9tXw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6d-17dd36184ab7c9202e705efa;Parent=4103abe6bc9216e0;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 ef73a156d5c211fdbb7e4231f2a0edca.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 78ntxSmfza39FAmaCa-exUtRnNXyNWRnHxcQ77_UZoLQ1D-gcWcmHg== + body: + encoding: ASCII-8BIT + string: '{"objects":[{"id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","_xact_id":"1000196458112984917","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-prompt-load","slug":"test-prompt-load","description":null,"created":"2026-01-09T15:31:57.524Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' + recorded_at: Fri, 09 Jan 2026 15:31:58 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function/2ee042ed-2971-47f4-a73c-7abe38ab1a38 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '557' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:58 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 9e3c6ab3-0160-4a1e-94a9-35ab9f19913e + X-Bt-Internal-Trace-Id: + - 69611f6e0000000071b8a71148470d4f + X-Amz-Apigw-Id: + - W7HZTHd7oAMESTw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"22d-ma3D4t+NO2VLj1JEXruVnj40R7s" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6e-71c865c4585fcb7f648bd594;Parent=407f3261d889f0cb;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 f9aa0e4086fcbefc20f307d96a8e3b44.cloudfront.net (CloudFront), 1.1 b601959712c1f21193a489b5759f70ba.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - ccO14e2OKjR_YWWVAAiYlvQ148Mh72XPFQUJP7-mfkXI8h4tmQjX1g== + body: + encoding: ASCII-8BIT + string: '{"id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","_xact_id":"1000196458112984917","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-prompt-load","slug":"test-prompt-load","description":null,"created":"2026-01-09T15:31:57.524Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Say + hello to {{name}}"}]},"options":{"model":"gpt-4o-mini"}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}' + recorded_at: Fri, 09 Jan 2026 15:31:58 GMT +- request: + method: delete + uri: https://api.braintrust.dev/v1/function/2ee042ed-2971-47f4-a73c-7abe38ab1a38 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '276' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:31:59 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 2fd59f3d-1ab3-4711-a36d-cc8f4a571f38 + X-Bt-Internal-Trace-Id: + - 69611f6e000000006407a3df038879bd + X-Amz-Apigw-Id: + - W7HZYFE2oAMEF7g= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"114-DQ3SOr2UfgKZ6FbXE8ta7C0iQ+0" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f6e-4980fe2e0131dc962a5a453b;Parent=21b6b931be060230;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 38bc9c97daf30f968ccac44ef89e14e0.cloudfront.net (CloudFront), 1.1 92672fff57a11d8cf4f64313a69242d0.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - t19MgdmAEgAvifEHWTWB_Eg5GrWxHuhgmBzUCXkGWFh0UPmXSQsLKA== + body: + encoding: ASCII-8BIT + string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-prompt-load","id":"2ee042ed-2971-47f4-a73c-7abe38ab1a38","created":"2026-01-09T15:31:58.990Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196458113118216"}' + recorded_at: Fri, 09 Jan 2026 15:31:59 GMT +recorded_with: VCR 6.4.0 diff --git a/test/fixtures/vcr_cassettes/prompt/load_not_found.yml b/test/fixtures/vcr_cassettes/prompt/load_not_found.yml new file mode 100644 index 0000000..aeafd82 --- /dev/null +++ b/test/fixtures/vcr_cassettes/prompt/load_not_found.yml @@ -0,0 +1,143 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZDE5MmFhODgtNjU5Ny00OWVmLTgzZjAtNDRmMDJlYTY5MmQ1'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 09 Jan 2026 15:32:03 GMT + Etag: + - '"ubzjf1iqqj75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - ZDE5MmFhODgtNjU5Ny00OWVmLTgzZjAtNDRmMDJlYTY5MmQ1 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::c7rlk-1767972723551-c064f07e7248 + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Fri, 09 Jan 2026 15:32:03 GMT +- request: + method: get + uri: https://api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=nonexistent-prompt-xyz + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '14' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P2 + - JFK50-P5 + Date: + - Fri, 09 Jan 2026 15:32:03 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - e525a4b0-3de5-43ce-9b4f-fedef9b6d867 + X-Bt-Internal-Trace-Id: + - 69611f73000000005ccf167cee9e9974 + X-Amz-Apigw-Id: + - W7HaJFBAIAMEF8w= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"e-xZKibKAiOxxBbzTm2byfFNRkvtA" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-69611f73-2656d761732ae2725b2c54f8;Parent=0f5f3e756009b8f2;Sampled=0;Lineage=1:24be3d11:0 + Via: + - 1.1 ab734ad5d81cc9d470b6176a05dd968e.cloudfront.net (CloudFront), 1.1 baec235d174153a8f2e92ea724643824.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - CcRV9GjE2YYavLYeGWNDevFa9mY2YldGSV-TUg4E5QrLLfsnxL3cJQ== + body: + encoding: ASCII-8BIT + string: '{"objects":[]}' + recorded_at: Fri, 09 Jan 2026 15:32:03 GMT +recorded_with: VCR 6.4.0 From 7e77515f30a16fa0e2da762ac455fec7d8600126 Mon Sep 17 00:00:00 2001 From: Matt Perpick Date: Fri, 9 Jan 2026 11:27:22 -0500 Subject: [PATCH 2/2] use mustache templates --- .claude/settings.json | 1 + Gemfile.lock | 2 ++ braintrust.gemspec | 1 + examples/prompt.rb | 41 ++++++++++++++++++--------------- lib/braintrust/prompt.rb | 42 +++++++++++++++++----------------- test/braintrust/prompt_test.rb | 5 ++-- 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 21984fd..bba6f84 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -25,6 +25,7 @@ "Bash(gem search:*)", "Bash(gem list:*)", "Bash(tree:*)", + "Bash(grep:*)", "Bash(find:*)", "Bash(echo:*)", "Bash(sort:*)", diff --git a/Gemfile.lock b/Gemfile.lock index cff826e..4ec76b3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,6 +2,7 @@ PATH remote: . specs: braintrust (0.0.12) + mustache (~> 1.0) openssl (~> 3.3.1) opentelemetry-exporter-otlp (~> 0.28) opentelemetry-sdk (~> 1.3) @@ -49,6 +50,7 @@ GEM builder minitest (>= 5.0) ruby-progressbar + mustache (1.1.1) openssl (3.3.1) opentelemetry-api (1.7.0) opentelemetry-common (0.23.0) diff --git a/braintrust.gemspec b/braintrust.gemspec index 6f1fd68..e7b36fa 100644 --- a/braintrust.gemspec +++ b/braintrust.gemspec @@ -31,6 +31,7 @@ Gem::Specification.new do |spec| # Runtime dependencies spec.add_runtime_dependency "opentelemetry-sdk", "~> 1.3" spec.add_runtime_dependency "opentelemetry-exporter-otlp", "~> 0.28" + spec.add_runtime_dependency "mustache", "~> 1.0" # OpenSSL 3.3.1+ fixes macOS CRL (Certificate Revocation List) verification issues # that occur with OpenSSL 3.6 + Ruby (certificate verify failed: unable to get certificate CRL). diff --git a/examples/prompt.rb b/examples/prompt.rb index 32a51e3..4c0e837 100644 --- a/examples/prompt.rb +++ b/examples/prompt.rb @@ -1,26 +1,31 @@ #!/usr/bin/env ruby # frozen_string_literal: true -# Example: Loading and using prompts from Braintrust +# Example: Loading and executing prompts from Braintrust # # This example demonstrates how to: # 1. Create a prompt (function) on the Braintrust server # 2. Load it using Prompt.load -# 3. Build the prompt with variable substitution -# 4. Use the built prompt with an LLM client +# 3. Build the prompt with Mustache variable substitution +# 4. Execute the prompt with OpenAI and get a response # # Benefits of loading prompts: # - Centralized prompt management in Braintrust UI # - Version control and A/B testing for prompts # - No code deployment needed for prompt changes # - Works with any LLM client (OpenAI, Anthropic, etc.) +# - Uses standard Mustache templating ({{variable}}, {{object.property}}) require "bundler/setup" require "braintrust" +require "openai" -# Initialize Braintrust +# Initialize Braintrust with tracing Braintrust.init +# Wrap OpenAI client for tracing +openai = Braintrust::Trace::OpenAI.wrap(OpenAI::Client.new) + project_name = "ruby-sdk-examples" prompt_slug = "greeting-prompt-#{Time.now.to_i}" @@ -39,7 +44,7 @@ messages: [ { role: "system", - content: "You are a friendly assistant who speaks {{language}}." + content: "You are a friendly assistant. Respond in {{language}}. Keep responses brief (1-2 sentences)." }, { role: "user", @@ -60,11 +65,10 @@ prompt = Braintrust::Prompt.load(project: project_name, slug: prompt_slug) puts " ID: #{prompt.id}" -puts " Name: #{prompt.name}" +puts " Slug: #{prompt.slug}" puts " Model: #{prompt.model}" -puts " Messages: #{prompt.messages.length}" -# Build the prompt with variable substitution +# Build the prompt with Mustache variable substitution puts "\nBuilding prompt with variables..." params = prompt.build( name: "Alice", @@ -74,22 +78,23 @@ puts " Model: #{params[:model]}" puts " Temperature: #{params[:temperature]}" -puts " Max tokens: #{params[:max_tokens]}" puts " Messages:" params[:messages].each do |msg| puts " [#{msg[:role]}] #{msg[:content]}" end -# The params hash is ready to pass to any LLM client: -# -# With OpenAI: -# client.chat.completions.create(**params) -# -# With Anthropic: -# client.messages.create(**params) +# Execute the prompt with OpenAI +puts "\nExecuting prompt with OpenAI..." +response = openai.chat.completions.create(**params) -puts "\nPrompt is ready to use with any LLM client!" +puts "\nResponse:" +content = response.choices.first.message.content +puts " #{content}" # Clean up - delete the test prompt +puts "\nCleaning up..." api.functions.delete(id: prompt.id) -puts "Cleaned up test prompt." +puts "Done!" + +# Flush traces +OpenTelemetry.tracer_provider.shutdown diff --git a/lib/braintrust/prompt.rb b/lib/braintrust/prompt.rb index d417e95..dc162ff 100644 --- a/lib/braintrust/prompt.rb +++ b/lib/braintrust/prompt.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "mustache" + module Braintrust # Prompt class for loading and building prompts from Braintrust # @@ -92,11 +94,11 @@ def build(variables = nil, strict: false, **kwargs) variables_hash = variables.is_a?(Hash) ? variables : {} vars = @defaults.merge(stringify_keys(variables_hash)).merge(stringify_keys(kwargs)) - # Substitute variables in messages + # Render Mustache templates in messages built_messages = messages.map do |msg| { role: msg["role"].to_sym, - content: substitute_variables(msg["content"], vars, strict: strict) + content: render_template(msg["content"], vars, strict: strict) } end @@ -119,33 +121,31 @@ def build(variables = nil, strict: false, **kwargs) private - # Substitute {{variable}} placeholders with values - def substitute_variables(text, variables, strict:) + # Render Mustache template with variables + def render_template(text, variables, strict:) return text unless text.is_a?(String) - # Find all {{variable}} patterns - missing = [] - - result = text.gsub(/\{\{([^}]+)\}\}/) do |match| - var_path = ::Regexp.last_match(1).strip - value = resolve_variable(var_path, variables) - - if value.nil? - missing << var_path - match # Keep original placeholder - else - value.to_s + if strict + # Check for missing variables before rendering + missing = find_missing_variables(text, variables) + if missing.any? + raise Error, "Missing required variables: #{missing.join(", ")}" end end - if strict && missing.any? - raise Error, "Missing required variables: #{missing.join(", ")}" - end + Mustache.render(text, variables) + end - result + # Find variables in template that are not provided + def find_missing_variables(text, variables) + # Extract {{variable}} and {{variable.path}} patterns + # Mustache uses {{name}} syntax + text.scan(/\{\{([^}#^\/!>]+)\}\}/).flatten.map(&:strip).uniq.reject do |var| + resolve_variable(var, variables) + end end - # Resolve a variable path like "user.name" from variables hash + # Check if a variable path exists in the variables hash def resolve_variable(path, variables) parts = path.split(".") value = variables diff --git a/test/braintrust/prompt_test.rb b/test/braintrust/prompt_test.rb index 14ed7d1..247de90 100644 --- a/test/braintrust/prompt_test.rb +++ b/test/braintrust/prompt_test.rb @@ -117,11 +117,12 @@ def test_build_strict_raises_on_missing_variable assert_match(/missing.*task/i, error.message) end - def test_build_non_strict_leaves_missing_variables + def test_build_non_strict_replaces_missing_variables_with_empty + # Mustache standard behavior: missing variables become empty strings prompt = Braintrust::Prompt.new(@function_data) result = prompt.build(name: "Alice") - assert_equal "Hello Alice, please help with {{task}}.", result[:messages][1][:content] + assert_equal "Hello Alice, please help with .", result[:messages][1][:content] end def test_build_handles_nested_variables