Auto-instrument LLM libraries with tracing

Appraisals

@@ -63,3 +63,25 @@ appraise "opentelemetry-latest" do
   gem "opentelemetry-sdk", ">= 1.10"
   gem "opentelemetry-exporter-otlp", ">= 0.31"
 end
+
+# Both OpenAI gems installed - tests that loaded? correctly distinguishes them
+appraise "openai-ruby-openai" do
+  gem "openai", ">= 0.34"
+  gem "ruby-openai", ">= 8.0"
+end
+
+# LLM libraries that can coexist - for demos that use multiple LLM SDKs
+# Note: ruby-openai is excluded because it conflicts with the official openai gem
+# (they share the same namespace). Use openai-ruby-openai appraisal to test both.
+appraise "contrib" do
+  gem "openai", ">= 0.34"
+  gem "anthropic", ">= 1.11"
+  gem "ruby_llm", ">= 1.9"
+  gem "base64" # needed for openai gem on Ruby 3.4+
+end
+
+# Rails integration testing (minimal dependencies)
+appraise "rails" do
+  gem "activesupport", "~> 8.0"
+  gem "railties", "~> 8.0"
+end

CONTRIBUTING.md

@@ -2,9 +2,18 @@
 
 Thanks for contributing! Follow TDD practices: write tests first, implement minimal code, refactor.
 
+- [Quick Setup](#quick-setup)
+  - [Local development (on host)](#local-development-on-host)
+  - [Local development (in Docker)](#local-development-in-docker)
+- [Development](#development)
+  - [Running tasks](#running-tasks)
+  - [Adding new integrations for AI Libraries](#adding-new-integrations-for-ai-libraries)
+- [Testing](#testing)
+  - [With different Ruby versions](#with-different-ruby-versions)
+
 ## Quick Setup
 
-### Option 1: Local Development
+### Local development (on host)
 
 ```bash
 # Clone repo
@@ -31,7 +40,7 @@ cp .env.example .env
 rake
 ```
 
-### Option 2: Docker Container
+### Local development (in Docker)
 
 Use the dev container if you prefer not to install Ruby tooling on your host machine. Your IDE edits files on the host; all tools run in the container.
 
@@ -62,12 +71,158 @@ rake
 
 ## Development
 
+### Running tasks
+
 All of our common dev tasks are in rake.
 
 ```bash
 rake -T 
 ```
 
+We use VCR for making http tests fast. You can run tests with these enabled,
+off, etc. If you add new tests you'll need to record new cassettes. See this
+for more details.
+
+```bash
+rake -T test:vcr
+```
+
+### Adding new integrations for AI Libraries
+
+
+To add instrumentation support for a new library, use the integration generator:
+
+```bash
+rake contrib:generate NAME=trustybrain_llm AUTO_REGISTER=true
+```
+
+This will create the integration structure and optionally register it. You can also specify additional options:
+
+```bash
+rake contrib:generate NAME=trustybrain_llm \
+  GEM_NAMES=trustybrain_llm,trustybrain \
+  REQUIRE_PATHS=trustybrain \
+  MIN_VERSION=1.0.0 \
+  MAX_VERSION=2.0.0 \
+  AUTO_REGISTER=true
+```
+
+#### Manual Setup
+
+If you prefer to create the integration manually, follow these steps:
+
+##### 1. Create the integration directory structure
+
+```bash
+mkdir -p lib/braintrust/contrib/trustybrain_llm
+mkdir -p test/braintrust/contrib/trustybrain_llm
+```
+
+##### 2. Define the integration
+
+Create `lib/braintrust/contrib/trustybrain_llm/integration.rb`:
+
+```ruby
+# frozen_string_literal: true
+
+require_relative "../integration"
+
+module Braintrust
+  module Contrib
+    module TrustybrainLLM
+      class Integration
+        include Braintrust::Contrib::Integration
+
+        def self.integration_name
+          :trustybrain_llm
+        end
+
+        def self.gem_names
+          ["trustybrain_llm"]
+        end
+
+        def self.loaded?
+          defined?(::TrustybrainLLM::Client) ? true : false
+        end
+
+        def self.patchers
+          require_relative "patcher"
+          [Patcher]
+        end
+      end
+    end
+  end
+end
+```
+
+##### 3. Create a patcher
+
+Create `lib/braintrust/contrib/trustybrain_llm/patcher.rb`:
+
+```ruby
+# frozen_string_literal: true
+
+require_relative "../patcher"
+
+module Braintrust
+  module Contrib
+    module TrustybrainLLM
+      class Patcher < Braintrust::Contrib::Patcher
+        class << self
+          def applicable?
+            defined?(::TrustybrainLLM::Client)
+          end
+
+          def perform_patch(**options)
+            ::TrustybrainLLM::Client.prepend(Instrumentation)
+          end
+        end
+
+        module Instrumentation
+          def chat(*args, **kwargs, &block)
+            Braintrust::Contrib.tracer_for(self).in_span("trustybrain_llm.chat") do
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end
+```
+
+##### 4. Register it
+
+Add to `lib/braintrust/contrib.rb`:
+
+```ruby
+require_relative "contrib/trustybrain_llm/integration"
+
+# At the bottom:
+Contrib::TrustybrainLLM::Integration.register!
+```
+
+##### 5. Add tests
+
+Create test files in `test/braintrust/contrib/trustybrain_llm/`:
+
+```ruby
+# test/braintrust/contrib/trustybrain_llm/integration_test.rb
+require "test_helper"
+
+class Braintrust::Contrib::TrustybrainLLM::IntegrationTest < Minitest::Test
+  def test_integration_basics
+    integration = Braintrust::Contrib::TrustybrainLLM::Integration
+    assert_equal :trustybrain_llm, integration.integration_name
+    assert_equal ["trustybrain_llm"], integration.gem_names
+  end
+
+  # TODO: Add tests for patchers, availability, compatibility, and instrumentation
+end
+```
+
+See existing tests in `test/braintrust/contrib/` for complete examples of testing integrations, patchers, and the registry.
+
 ## Testing
 
 We use VCR for making http tests fast. You can run tests with these enabled,
@@ -78,7 +233,7 @@ for more details.
 rake -T test:vcr
 ```
 
-### Testing with Different Ruby Versions
+### With different Ruby versions
 
 CI tests against Ruby 3.2, 3.3, and 3.4. To test locally with a different Ruby version:
 

Gemfile

@@ -7,3 +7,5 @@ gemspec
 
 # Additional development tools not in gemspec (optional/convenience)
 gem "minitest-reporters", "~> 1.6"
+gem "minitest-stub-const", "~> 0.6"
+gem "climate_control", "~> 1.2"

Gemfile.lock

@@ -9,47 +9,43 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.7)
-      public_suffix (>= 2.0.2, < 7.0)
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
     ansi (1.5.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     ast (2.4.3)
-    bigdecimal (3.3.1)
+    bigdecimal (4.0.1)
     builder (3.3.0)
+    climate_control (1.2.0)
     crack (1.0.1)
       bigdecimal
       rexml
     docile (1.4.1)
-    google-protobuf (4.33.0-aarch64-linux-gnu)
+    google-protobuf (4.33.2-arm64-darwin)
       bigdecimal
       rake (>= 13)
-    google-protobuf (4.33.0-arm64-darwin)
-      bigdecimal
-      rake (>= 13)
-    google-protobuf (4.33.0-x64-mingw-ucrt)
-      bigdecimal
-      rake (>= 13)
-    google-protobuf (4.33.0-x86_64-linux-gnu)
+    google-protobuf (4.33.2-x86_64-linux-gnu)
       bigdecimal
       rake (>= 13)
     googleapis-common-protos-types (1.22.0)
       google-protobuf (~> 4.26)
     hashdiff (1.2.1)
-    json (2.15.1)
+    json (2.18.0)
     kramdown (2.5.1)
       rexml (>= 3.3.9)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
-    minitest (5.26.0)
+    minitest (5.27.0)
     minitest-reporters (1.7.1)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    openssl (3.3.1)
+    minitest-stub-const (0.6)
+    openssl (3.3.2)
     opentelemetry-api (1.7.0)
     opentelemetry-common (0.23.0)
       opentelemetry-api (~> 1.0)
@@ -70,79 +66,76 @@ GEM
     opentelemetry-semantic_conventions (1.36.0)
       opentelemetry-api (~> 1.0)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.6.0)
-    public_suffix (6.0.2)
+    prism (1.7.0)
+    public_suffix (7.0.2)
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.3.0)
+    rake (13.3.1)
     regexp_parser (2.11.3)
     rexml (3.4.4)
-    rubocop (1.80.2)
+    rubocop (1.81.7)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.49.0)
       parser (>= 3.3.7.2)
-      prism (~> 1.4)
-    rubocop-performance (1.25.0)
+      prism (~> 1.7)
+    rubocop-performance (1.26.1)
       lint_roller (~> 1.1)
       rubocop (>= 1.75.0, < 2.0)
-      rubocop-ast (>= 1.38.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
     ruby-progressbar (1.13.0)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    standard (1.51.1)
+    standard (1.52.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.80.2)
+      rubocop (~> 1.81.7)
       standard-custom (~> 1.0.0)
       standard-performance (~> 1.8)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.8.0)
+    standard-performance (1.9.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.25.0)
+      rubocop-performance (~> 1.26.0)
     thor (1.4.0)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
-    unicode-emoji (4.1.0)
+    unicode-emoji (4.2.0)
     vcr (6.4.0)
-    webmock (3.25.1)
+    webmock (3.26.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    yard (0.9.37)
+    yard (0.9.38)
 
 PLATFORMS
-  aarch64-linux
-  arm64-darwin-23
-  arm64-darwin-24
-  arm64-darwin-25
-  x64-mingw
-  x64-mingw-ucrt
+  arm64-darwin
   x86_64-linux
 
 DEPENDENCIES
   appraisal (~> 2.5)
   braintrust!
+  climate_control (~> 1.2)
   kramdown (~> 2.0)
   minitest (~> 5.0)
   minitest-reporters (~> 1.6)
+  minitest-stub-const (~> 0.6)
   rake (~> 13.0)
   simplecov (~> 0.22)
   standard (~> 1.0)