From 33f4975e7b95e787b23323716d2aeeb5f260ebb9 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 15 Apr 2026 09:27:37 -0500 Subject: [PATCH 01/23] feat!: Add per-execution runId and at-most-once event tracking - Each tracker now carries a run_id (UUIDv4) included in all emitted events, scoping every metric to a single execution - At-most-once semantics: duplicate calls to track_duration, track_tokens, track_success/track_error, track_feedback, and track_time_to_first_token on the same tracker are dropped with a warning Co-Authored-By: Claude Sonnet 4.6 --- lib/server/ai/ai_config_tracker.rb | 39 +++++++++++ spec/server/ai/config_tracker_spec.rb | 99 +++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 7 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index c603e46..74e18d3 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'ldclient-rb' +require 'securerandom' module LaunchDarkly module Server @@ -64,6 +65,13 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p @provider_name = provider_name @context = context @summary = MetricSummary.new + @run_id = SecureRandom.uuid + @tracked_duration = false + @tracked_time_to_first_token = false + @tracked_tokens = false + @tracked_success = nil + @tracked_feedback = false + @logger = LaunchDarkly::Server::AI.default_logger end # @@ -72,6 +80,11 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p # @param duration [Integer] The duration in milliseconds # def track_duration(duration) + if @tracked_duration + @logger&.warn("Duration has already been tracked for this execution.") + return + end + @tracked_duration = true @summary.duration = duration @ld_client.track( '$ld:ai:duration:total', @@ -101,6 +114,11 @@ def track_duration_of(&block) # @param duration [Integer] The duration in milliseconds # def track_time_to_first_token(time_to_first_token) + if @tracked_time_to_first_token + @logger&.warn("Time to first token has already been tracked for this execution.") + return + end + @tracked_time_to_first_token = true @summary.time_to_first_token = time_to_first_token @ld_client.track( '$ld:ai:tokens:ttf', @@ -116,6 +134,11 @@ def track_time_to_first_token(time_to_first_token) # @param kind [Symbol] The kind of feedback (:positive or :negative) # def track_feedback(kind:) + if @tracked_feedback + @logger&.warn("Feedback has already been tracked for this execution.") + return + end + @tracked_feedback = true @summary.feedback = kind event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' @ld_client.track( @@ -130,6 +153,11 @@ def track_feedback(kind:) # Track a successful AI generation # def track_success + unless @tracked_success.nil? + @logger&.warn("Success or error has already been tracked for this execution.") + return + end + @tracked_success = true @summary.success = true @ld_client.track( '$ld:ai:generation:success', @@ -143,6 +171,11 @@ def track_success # Track an error in AI generation # def track_error + unless @tracked_success.nil? + @logger&.warn("Success or error has already been tracked for this execution.") + return + end + @tracked_success = false @summary.success = false @ld_client.track( '$ld:ai:generation:error', @@ -158,6 +191,11 @@ def track_error # @param token_usage [TokenUsage] An object containing token usage details # def track_tokens(token_usage) + if @tracked_tokens + @logger&.warn("Tokens have already been tracked for this execution.") + return + end + @tracked_tokens = true @summary.usage = token_usage if token_usage.total.positive? @ld_client.track( @@ -223,6 +261,7 @@ def track_bedrock_converse_metrics(&block) private def flag_data { + runId: @run_id, variationKey: @variation_key, configKey: @config_key, version: @version, diff --git a/spec/server/ai/config_tracker_spec.rb b/spec/server/ai/config_tracker_spec.rb index ef470e4..824319c 100644 --- a/spec/server/ai/config_tracker_spec.rb +++ b/spec/server/ai/config_tracker_spec.rb @@ -27,7 +27,7 @@ end let(:context) { LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) } - let(:tracker_flag_data) { { variationKey: 'test-variation', configKey: 'test-config', version: 1, modelName: 'fakeModel', providerName: 'fakeProvider' } } + let(:tracker_flag_data) { { runId: kind_of(String), variationKey: 'test-variation', configKey: 'test-config', version: 1, modelName: 'fakeModel', providerName: 'fakeProvider' } } let(:tracker) do described_class.new( ld_client: ld_client, @@ -322,25 +322,107 @@ expect(tracker.summary.success).to be false end - it 'overwrites success with error if both are tracked' do + it 'does not track error if success has already been tracked' do expect(ld_client).to receive(:track).with( '$ld:ai:generation:success', context, tracker_flag_data, 1 ) + + tracker.track_success + expect(tracker.summary.success).to be true + tracker.track_error + expect(tracker.summary.success).to be true + end + end + + describe 'at-most-once tracking' do + it 'only tracks duration once' do expect(ld_client).to receive(:track).with( - '$ld:ai:generation:error', + '$ld:ai:duration:total', context, tracker_flag_data, - 1 - ) + 100 + ).once + tracker.track_duration(100) + tracker.track_duration(200) + expect(tracker.summary.duration).to eq(100) + end + + it 'only tracks time to first token once' do + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:ttf', + context, + tracker_flag_data, + 100 + ).once + tracker.track_time_to_first_token(100) + tracker.track_time_to_first_token(200) + expect(tracker.summary.time_to_first_token).to eq(100) + end + + it 'only tracks tokens once' do + tokens1 = LaunchDarkly::Server::AI::TokenUsage.new(total: 300, input: 200, output: 100) + tokens2 = LaunchDarkly::Server::AI::TokenUsage.new(total: 600, input: 400, output: 200) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:total', + context, + tracker_flag_data, + 300 + ).once + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:input', + context, + tracker_flag_data, + 200 + ).once + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:output', + context, + tracker_flag_data, + 100 + ).once + tracker.track_tokens(tokens1) + tracker.track_tokens(tokens2) + expect(tracker.summary.usage).to eq(tokens1) + end + it 'only tracks success once' do + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:success', + context, + tracker_flag_data, + 1 + ).once + tracker.track_success tracker.track_success expect(tracker.summary.success).to be true + end + + it 'only tracks error once' do + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + tracker_flag_data, + 1 + ).once + tracker.track_error tracker.track_error expect(tracker.summary.success).to be false end + + it 'only tracks feedback once' do + expect(ld_client).to receive(:track).with( + '$ld:ai:feedback:user:positive', + context, + tracker_flag_data, + 1 + ).once + tracker.track_feedback(kind: :positive) + tracker.track_feedback(kind: :negative) + expect(tracker.summary.feedback).to eq(:positive) + end end describe '#summary' do @@ -370,11 +452,14 @@ end describe '#flag_data' do - it 'includes model_name and provider_name in flag data' do - expect(tracker.send(:flag_data)).to include( + it 'includes runId, model_name, and provider_name in flag data' do + flag_data = tracker.send(:flag_data) + expect(flag_data).to include( + runId: kind_of(String), modelName: 'fakeModel', providerName: 'fakeProvider' ) + expect(flag_data[:runId]).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i) end end From e2d74cf8e895bdcfd0d76e3f654b5cab32b9bdf1 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 15 Apr 2026 12:36:26 -0500 Subject: [PATCH 02/23] feat!: Add per-execution runId, at-most-once tracking, and cross-process tracker resumption Co-Authored-By: Claude Opus 4.6 --- launchdarkly-server-sdk-ai.gemspec | 1 + lib/server/ai/ai_config_tracker.rb | 49 +++++++++++++- lib/server/ai/client.rb | 57 +++++++++++++---- spec/server/ai/client_spec.rb | 78 +++++++++++++++++++++-- spec/server/ai/config_tracker_spec.rb | 92 +++++++++++++++++++++++++++ 5 files changed, 258 insertions(+), 19 deletions(-) diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index dd290f2..dc7d749 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -19,6 +19,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.required_ruby_version = '>= 3.1.0' + spec.add_dependency 'base64' spec.add_dependency 'launchdarkly-server-sdk', '~> 8.5' spec.add_dependency 'logger' spec.add_dependency 'mustache', '~> 1.1' diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 74e18d3..d8c74ed 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'base64' +require 'json' require 'ldclient-rb' require 'securerandom' @@ -56,7 +58,7 @@ class AIConfigTracker # @param provider_name [String] The name of the AI provider # @param context [LDContext] The context used for the flag evaluation # - def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, provider_name:, context:) + def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, provider_name:, context:, run_id:) @ld_client = ld_client @variation_key = variation_key @config_key = config_key @@ -65,7 +67,7 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p @provider_name = provider_name @context = context @summary = MetricSummary.new - @run_id = SecureRandom.uuid + @run_id = run_id @tracked_duration = false @tracked_time_to_first_token = false @tracked_tokens = false @@ -74,6 +76,49 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p @logger = LaunchDarkly::Server::AI.default_logger end + # + # Returns a URL-safe Base64-encoded JSON token that can be used to reconstruct + # a tracker in a different process (e.g. for deferred feedback). + # + # The token contains: runId, configKey, variationKey, version. + # modelName and providerName are NOT included. + # + # @return [String] the resumption token + # + def resumption_token + payload = { + runId: @run_id, + configKey: @config_key, + variationKey: @variation_key, + version: @version, + } + Base64.urlsafe_encode64(JSON.generate(payload), padding: false) + end + + # + # Reconstructs a tracker from a resumption token. + # + # @param token [String] A URL-safe Base64-encoded JSON resumption token + # @param ld_client [LDClient] The LaunchDarkly client instance + # @param context [LDContext] The context for track events + # @return [AIConfigTracker] A new tracker instance + # + def self.from_resumption_token(token:, ld_client:, context:) + json = Base64.urlsafe_decode64(token) + payload = JSON.parse(json) + + new( + ld_client: ld_client, + run_id: payload['runId'], + config_key: payload['configKey'], + variation_key: payload.fetch('variationKey', ''), + version: payload['version'], + model_name: '', + provider_name: '', + context: context + ) + end + # # Track the duration of an AI operation # diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 87d49e7..7f5a051 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -2,6 +2,7 @@ require 'ldclient-rb' require 'mustache' +require 'securerandom' require_relative 'ai_config_tracker' require_relative 'sdk_info' @@ -103,16 +104,26 @@ def to_h # The AIConfig class represents an AI configuration. # class AIConfig - attr_reader :enabled, :messages, :tracker, :model, :provider + attr_reader :enabled, :messages, :model, :provider - def initialize(enabled: nil, model: nil, messages: nil, tracker: nil, provider: nil) + def initialize(enabled: nil, model: nil, messages: nil, tracker_factory: nil, provider: nil) @enabled = enabled @messages = messages - @tracker = tracker + @tracker_factory = tracker_factory @model = model @provider = provider end + # + # Creates a new tracker with a fresh runId for tracking a single AI execution. + # Returns nil when the config has no tracker factory (e.g. a static disabled config). + # + # @return [AIConfigTracker, nil] a new tracker instance, or nil + # + def create_tracker + @tracker_factory&.call + end + # # Returns a new disabled AIConfig instance. # @@ -182,6 +193,18 @@ def completion_config(key:, context:, default: nil, variables: nil) _completion_config(key:, context:, default: default || AIConfig.disabled, variables:) end + # + # Reconstructs a tracker from a resumption token, allowing deferred tracking + # (e.g. feedback from a different process). + # + # @param token [String] A resumption token obtained from AIConfigTracker#resumption_token + # @param context [LDContext] The context for track events + # @return [AIConfigTracker] A new tracker instance + # + def create_tracker(token:, context:) + AIConfigTracker.from_resumption_token(token: token, ld_client: @ld_client, context: context) + end + # @deprecated Use {#completion_config} instead. def config(key:, context:, default: nil, variables: nil) warn '[DEPRECATION] `config` is deprecated. Use `completion_config` instead.' @@ -226,20 +249,28 @@ def _completion_config(key:, context:, default:, variables: nil) ) end - tracker = LaunchDarkly::Server::AI::AIConfigTracker.new( - ld_client: @ld_client, - variation_key: variation.dig(:_ldMeta, :variationKey) || '', - config_key: key, - version: variation.dig(:_ldMeta, :version) || 1, - model_name: model&.name || '', - provider_name: provider_config&.name || '', - context: - ) + variation_key = variation.dig(:_ldMeta, :variationKey) || '' + version = variation.dig(:_ldMeta, :version) || 1 + model_name = model&.name || '' + provider_name = provider_config&.name || '' + + tracker_factory = lambda { + LaunchDarkly::Server::AI::AIConfigTracker.new( + ld_client: @ld_client, + run_id: SecureRandom.uuid, + variation_key: variation_key, + config_key: key, + version: version, + model_name: model_name, + provider_name: provider_name, + context: context + ) + } AIConfig.new( enabled: variation.dig(:_ldMeta, :enabled) || false, messages:, - tracker:, + tracker_factory:, model:, provider: provider_config ) diff --git a/spec/server/ai/client_spec.rb b/spec/server/ai/client_spec.rb index 43fdb35..5783787 100644 --- a/spec/server/ai/client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -227,8 +227,9 @@ expect(config.provider).not_to be_nil expect(config.provider.name).to eq('fakeProvider') - expect(config.tracker).not_to be_nil - expect(config.tracker.send(:flag_data)).to include( + tracker = config.create_tracker + expect(tracker).not_to be_nil + expect(tracker.send(:flag_data)).to include( modelName: 'fakeModel', providerName: 'fakeProvider' ) @@ -353,8 +354,9 @@ expect(config.model).to be_nil expect(config.messages).to be_nil expect(config.provider).to be_nil - expect(config.tracker).not_to be_nil - expect(config.tracker.send(:flag_data)).to include( + tracker = config.create_tracker + expect(tracker).not_to be_nil + expect(tracker.send(:flag_data)).to include( modelName: '', providerName: '' ) @@ -375,6 +377,74 @@ expect(config.enabled).to be false end + + it 'create_tracker returns a new tracker with a fresh runId each time' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' }) + + tracker1 = config.create_tracker + tracker2 = config.create_tracker + + expect(tracker1).to be_a(LaunchDarkly::Server::AI::AIConfigTracker) + expect(tracker2).to be_a(LaunchDarkly::Server::AI::AIConfigTracker) + expect(tracker1).not_to equal(tracker2) + + run_id1 = tracker1.send(:flag_data)[:runId] + run_id2 = tracker2.send(:flag_data)[:runId] + expect(run_id1).not_to eq(run_id2) + expect(run_id1).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i) + expect(run_id2).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i) + end + + it 'create_tracker preserves flag metadata across calls' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' }) + + tracker = config.create_tracker + flag_data = tracker.send(:flag_data) + + expect(flag_data[:configKey]).to eq('model-config') + expect(flag_data[:variationKey]).to eq('abcd') + expect(flag_data[:version]).to eq(1) + expect(flag_data[:modelName]).to eq('fakeModel') + expect(flag_data[:providerName]).to eq('fakeProvider') + end + + it 'create_tracker returns nil for static disabled config' do + config = LaunchDarkly::Server::AI::AIConfig.disabled + expect(config.create_tracker).to be_nil + end + + it 'round-trips a tracker through a resumption token via client create_tracker' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' }) + + tracker = config.create_tracker + token = tracker.resumption_token + original_run_id = tracker.send(:flag_data)[:runId] + + restored = ai_client.create_tracker(token: token, context: context) + + expect(restored).to be_a(LaunchDarkly::Server::AI::AIConfigTracker) + expect(restored.send(:flag_data)[:runId]).to eq(original_run_id) + expect(restored.send(:flag_data)[:configKey]).to eq('model-config') + expect(restored.send(:flag_data)[:modelName]).to eq('') + expect(restored.send(:flag_data)[:providerName]).to eq('') + end + + it 'each tracker has independent at-most-once tracking' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + config = ai_client.completion_config(key: 'model-config', context:, variables: { 'name' => 'World' }) + + tracker1 = config.create_tracker + tracker2 = config.create_tracker + + tracker1.track_duration(100) + tracker2.track_duration(200) + + expect(tracker1.summary.duration).to eq(100) + expect(tracker2.summary.duration).to eq(200) + end end describe LaunchDarkly::Server::AI::AIConfig do diff --git a/spec/server/ai/config_tracker_spec.rb b/spec/server/ai/config_tracker_spec.rb index 824319c..c0413ed 100644 --- a/spec/server/ai/config_tracker_spec.rb +++ b/spec/server/ai/config_tracker_spec.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'base64' +require 'json' +require 'securerandom' require 'launchdarkly-server-sdk' require 'launchdarkly-server-sdk-ai' @@ -31,6 +34,7 @@ let(:tracker) do described_class.new( ld_client: ld_client, + run_id: SecureRandom.uuid, config_key: tracker_flag_data[:configKey], context: context, variation_key: tracker_flag_data[:variationKey], @@ -463,6 +467,94 @@ end end + describe '#resumption_token' do + it 'returns a URL-safe base64-encoded JSON string' do + token = tracker.resumption_token + decoded = JSON.parse(Base64.urlsafe_decode64(token)) + + expect(decoded['runId']).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i) + expect(decoded['configKey']).to eq('test-config') + expect(decoded['variationKey']).to eq('test-variation') + expect(decoded['version']).to eq(1) + end + + it 'does not include modelName or providerName' do + token = tracker.resumption_token + decoded = JSON.parse(Base64.urlsafe_decode64(token)) + + expect(decoded).not_to have_key('modelName') + expect(decoded).not_to have_key('providerName') + end + + it 'contains the same runId as the tracker flag data' do + flag_data = tracker.send(:flag_data) + token = tracker.resumption_token + decoded = JSON.parse(Base64.urlsafe_decode64(token)) + + expect(decoded['runId']).to eq(flag_data[:runId]) + end + end + + describe '.from_resumption_token' do + it 'reconstructs a tracker that uses the same runId' do + original_token = tracker.resumption_token + original_run_id = tracker.send(:flag_data)[:runId] + + restored = described_class.from_resumption_token( + token: original_token, + ld_client: ld_client, + context: context + ) + + expect(restored.send(:flag_data)[:runId]).to eq(original_run_id) + expect(restored.send(:flag_data)[:configKey]).to eq('test-config') + expect(restored.send(:flag_data)[:variationKey]).to eq('test-variation') + expect(restored.send(:flag_data)[:version]).to eq(1) + end + + it 'sets modelName and providerName to empty strings' do + token = tracker.resumption_token + + restored = described_class.from_resumption_token( + token: token, + ld_client: ld_client, + context: context + ) + + expect(restored.send(:flag_data)[:modelName]).to eq('') + expect(restored.send(:flag_data)[:providerName]).to eq('') + end + + it 'can track events with the restored tracker' do + token = tracker.resumption_token + original_run_id = tracker.send(:flag_data)[:runId] + + restored = described_class.from_resumption_token( + token: token, + ld_client: ld_client, + context: context + ) + + expected_data = { + runId: original_run_id, + variationKey: 'test-variation', + configKey: 'test-config', + version: 1, + modelName: '', + providerName: '', + } + + expect(ld_client).to receive(:track).with( + '$ld:ai:feedback:user:positive', + context, + expected_data, + 1 + ) + + restored.track_feedback(kind: :positive) + end + end + describe 'completion_config method tracking' do it 'calls track with correct parameters when completion_config is called' do allow(ld_client).to receive(:track) From a78313f6bb31e0ffd3522d2503b1841d7618d149 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 13:39:51 -0500 Subject: [PATCH 03/23] chore: Remove unused require securerandom from tracker Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/ai_config_tracker.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index d8c74ed..67a5960 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -3,7 +3,6 @@ require 'base64' require 'json' require 'ldclient-rb' -require 'securerandom' module LaunchDarkly module Server From 417e336b8706204bba34d9ff8432d1015ccb3e7f Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:26:01 -0500 Subject: [PATCH 04/23] refactor: Replace @tracked_* booleans with @summary nil checks Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/ai_config_tracker.rb | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 67a5960..595d05d 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -67,11 +67,6 @@ def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, p @context = context @summary = MetricSummary.new @run_id = run_id - @tracked_duration = false - @tracked_time_to_first_token = false - @tracked_tokens = false - @tracked_success = nil - @tracked_feedback = false @logger = LaunchDarkly::Server::AI.default_logger end @@ -124,11 +119,10 @@ def self.from_resumption_token(token:, ld_client:, context:) # @param duration [Integer] The duration in milliseconds # def track_duration(duration) - if @tracked_duration + unless @summary.duration.nil? @logger&.warn("Duration has already been tracked for this execution.") return end - @tracked_duration = true @summary.duration = duration @ld_client.track( '$ld:ai:duration:total', @@ -158,11 +152,10 @@ def track_duration_of(&block) # @param duration [Integer] The duration in milliseconds # def track_time_to_first_token(time_to_first_token) - if @tracked_time_to_first_token + unless @summary.time_to_first_token.nil? @logger&.warn("Time to first token has already been tracked for this execution.") return end - @tracked_time_to_first_token = true @summary.time_to_first_token = time_to_first_token @ld_client.track( '$ld:ai:tokens:ttf', @@ -178,11 +171,10 @@ def track_time_to_first_token(time_to_first_token) # @param kind [Symbol] The kind of feedback (:positive or :negative) # def track_feedback(kind:) - if @tracked_feedback + unless @summary.feedback.nil? @logger&.warn("Feedback has already been tracked for this execution.") return end - @tracked_feedback = true @summary.feedback = kind event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' @ld_client.track( @@ -197,11 +189,10 @@ def track_feedback(kind:) # Track a successful AI generation # def track_success - unless @tracked_success.nil? + unless @summary.success.nil? @logger&.warn("Success or error has already been tracked for this execution.") return end - @tracked_success = true @summary.success = true @ld_client.track( '$ld:ai:generation:success', @@ -215,11 +206,10 @@ def track_success # Track an error in AI generation # def track_error - unless @tracked_success.nil? + unless @summary.success.nil? @logger&.warn("Success or error has already been tracked for this execution.") return end - @tracked_success = false @summary.success = false @ld_client.track( '$ld:ai:generation:error', @@ -235,11 +225,10 @@ def track_error # @param token_usage [TokenUsage] An object containing token usage details # def track_tokens(token_usage) - if @tracked_tokens + unless @summary.usage.nil? @logger&.warn("Tokens have already been tracked for this execution.") return end - @tracked_tokens = true @summary.usage = token_usage if token_usage.total.positive? @ld_client.track( From bcc58bf485d7b7eb313fca4a4f483c686d20d7c4 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:37:06 -0500 Subject: [PATCH 05/23] refactor: Reorder tracker initialize params to match js-core spec Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/ai_config_tracker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 595d05d..08ea90c 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -57,7 +57,7 @@ class AIConfigTracker # @param provider_name [String] The name of the AI provider # @param context [LDContext] The context used for the flag evaluation # - def initialize(ld_client:, variation_key:, config_key:, version:, model_name:, provider_name:, context:, run_id:) + def initialize(ld_client:, run_id:, config_key:, variation_key:, version:, model_name:, provider_name:, context:) @ld_client = ld_client @variation_key = variation_key @config_key = config_key From 3e335bb4c946ee94a7f47b4adc4f6a413f5d5c0e Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:49:34 -0500 Subject: [PATCH 06/23] fix: Reorder context param and omit empty variationKey from resumption token Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/ai_config_tracker.rb | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 08ea90c..3d8ee73 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -57,7 +57,7 @@ class AIConfigTracker # @param provider_name [String] The name of the AI provider # @param context [LDContext] The context used for the flag evaluation # - def initialize(ld_client:, run_id:, config_key:, variation_key:, version:, model_name:, provider_name:, context:) + def initialize(ld_client:, run_id:, config_key:, variation_key:, version:, context:, model_name:, provider_name:) @ld_client = ld_client @variation_key = variation_key @config_key = config_key @@ -80,12 +80,9 @@ def initialize(ld_client:, run_id:, config_key:, variation_key:, version:, model # @return [String] the resumption token # def resumption_token - payload = { - runId: @run_id, - configKey: @config_key, - variationKey: @variation_key, - version: @version, - } + payload = { runId: @run_id, configKey: @config_key } + payload[:variationKey] = @variation_key if @variation_key && !@variation_key.empty? + payload[:version] = @version Base64.urlsafe_encode64(JSON.generate(payload), padding: false) end @@ -105,11 +102,11 @@ def self.from_resumption_token(token:, ld_client:, context:) ld_client: ld_client, run_id: payload['runId'], config_key: payload['configKey'], - variation_key: payload.fetch('variationKey', ''), + variation_key: payload.fetch('variationKey', nil), version: payload['version'], + context: context, model_name: '', - provider_name: '', - context: context + provider_name: '' ) end From 793f5402ccfdaa931703ff9be57b0eafe6965dea Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 17:04:14 -0500 Subject: [PATCH 07/23] chore: Include flag_data in tracker at-most-once warning logs Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/ai_config_tracker.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 3d8ee73..047a58a 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -117,7 +117,7 @@ def self.from_resumption_token(token:, ld_client:, context:) # def track_duration(duration) unless @summary.duration.nil? - @logger&.warn("Duration has already been tracked for this execution.") + @logger&.warn("Duration has already been tracked for this execution. #{flag_data}") return end @summary.duration = duration @@ -150,7 +150,7 @@ def track_duration_of(&block) # def track_time_to_first_token(time_to_first_token) unless @summary.time_to_first_token.nil? - @logger&.warn("Time to first token has already been tracked for this execution.") + @logger&.warn("Time to first token has already been tracked for this execution. #{flag_data}") return end @summary.time_to_first_token = time_to_first_token @@ -169,7 +169,7 @@ def track_time_to_first_token(time_to_first_token) # def track_feedback(kind:) unless @summary.feedback.nil? - @logger&.warn("Feedback has already been tracked for this execution.") + @logger&.warn("Feedback has already been tracked for this execution. #{flag_data}") return end @summary.feedback = kind @@ -187,7 +187,7 @@ def track_feedback(kind:) # def track_success unless @summary.success.nil? - @logger&.warn("Success or error has already been tracked for this execution.") + @logger&.warn("Success or error has already been tracked for this execution. #{flag_data}") return end @summary.success = true @@ -204,7 +204,7 @@ def track_success # def track_error unless @summary.success.nil? - @logger&.warn("Success or error has already been tracked for this execution.") + @logger&.warn("Success or error has already been tracked for this execution. #{flag_data}") return end @summary.success = false @@ -223,7 +223,7 @@ def track_error # def track_tokens(token_usage) unless @summary.usage.nil? - @logger&.warn("Tokens have already been tracked for this execution.") + @logger&.warn("Tokens have already been tracked for this execution. #{flag_data}") return end @summary.usage = token_usage From a32b6a6b0e428d24d2e98e5cc0309f2317a6d132 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 17:20:02 -0500 Subject: [PATCH 08/23] fix: Omit variationKey from flag_data when nil or empty Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/ai_config_tracker.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 047a58a..efcc0ff 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -290,14 +290,15 @@ def track_bedrock_converse_metrics(&block) end private def flag_data - { + data = { runId: @run_id, - variationKey: @variation_key, configKey: @config_key, version: @version, modelName: @model_name, providerName: @provider_name, } + data[:variationKey] = @variation_key if @variation_key && !@variation_key.empty? + data end private def openai_to_token_usage(usage) From 1bf0b6f15e788ccc6f4cce09dc5d58a5e4864379 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 09:31:02 -0500 Subject: [PATCH 09/23] fix: create_tracker must always return a tracker, never nil Per AICONF spec 1.2.7.1, create_tracker always returns a new tracker instance. The evaluation path always sets a factory, so the safe navigator is unnecessary. Updated test to verify disabled configs from evaluation still produce a tracker. Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/client.rb | 5 ++--- spec/server/ai/client_spec.rb | 10 +++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 7f5a051..d5a265d 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -116,12 +116,11 @@ def initialize(enabled: nil, model: nil, messages: nil, tracker_factory: nil, pr # # Creates a new tracker with a fresh runId for tracking a single AI execution. - # Returns nil when the config has no tracker factory (e.g. a static disabled config). # - # @return [AIConfigTracker, nil] a new tracker instance, or nil + # @return [AIConfigTracker] a new tracker instance # def create_tracker - @tracker_factory&.call + @tracker_factory.call end # diff --git a/spec/server/ai/client_spec.rb b/spec/server/ai/client_spec.rb index 5783787..9e87bc9 100644 --- a/spec/server/ai/client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -410,9 +410,13 @@ expect(flag_data[:providerName]).to eq('fakeProvider') end - it 'create_tracker returns nil for static disabled config' do - config = LaunchDarkly::Server::AI::AIConfig.disabled - expect(config.create_tracker).to be_nil + it 'create_tracker returns a tracker even for disabled configs from evaluation' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + config = ai_client.completion_config(key: 'off-config', context:) + + expect(config.enabled).to be false + tracker = config.create_tracker + expect(tracker).to be_a(LaunchDarkly::Server::AI::AIConfigTracker) end it 'round-trips a tracker through a resumption token via client create_tracker' do From 0aec0187bcc2bd49ad67bb53cb3da79c2a57f688 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 15:33:05 -0500 Subject: [PATCH 10/23] fix: Update examples to use create_tracker and resumption token pattern Each ask_agent invocation now creates its own tracker via create_tracker, returns a resumption token alongside the response, and deferred feedback is correlated back to the specific invocation by reconstructing the tracker from the token at the call site. Co-Authored-By: Claude Sonnet 4.6 --- examples/chatbot/aws-bedrock/hello_bedrock.rb | 28 ++++++++++++++----- examples/chatbot/openai/hello_openai.rb | 28 ++++++++++++++----- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/examples/chatbot/aws-bedrock/hello_bedrock.rb b/examples/chatbot/aws-bedrock/hello_bedrock.rb index ecc5699..f2d73de 100644 --- a/examples/chatbot/aws-bedrock/hello_bedrock.rb +++ b/examples/chatbot/aws-bedrock/hello_bedrock.rb @@ -29,10 +29,13 @@ def initialize(ai_config, bedrock_client) @bedrock_client = bedrock_client end + # Returns [response_content, resumption_token]. The resumption token can be + # used to reconstruct a tracker for deferred operations like feedback. def ask_agent(question) @messages << LaunchDarkly::Server::AI::Message.new('user', question) + tracker = @ai_config.create_tracker begin - response = ai_config.tracker.track_bedrock_converse_metrics do + response = tracker.track_bedrock_converse_metrics do @bedrock_client.converse( map_converse_arguments( ai_config.model.name, @@ -41,15 +44,17 @@ def ask_agent(question) ) end @messages << LaunchDarkly::Server::AI::Message.new('assistant', response.output.message.content[0].text) - response.output.message.content[0].text + [response.output.message.content[0].text, tracker.resumption_token] rescue StandardError => e - "An error occured: #{e.message}" + ["An error occured: #{e.message}", nil] end end - def agent_was_helpful(helpful) + def agent_was_helpful(helpful, tracker) + return if tracker.nil? + kind = helpful ? :positive : :negative - ai_config.tracker.track_feedback(kind: kind) + tracker.track_feedback(kind: kind) end def map_converse_arguments(model_id, messages) @@ -105,18 +110,27 @@ def map_converse_arguments(model_id, messages) chatbot = BedrockChatbot.new(ai_config, bedrock_client) +last_resumption_token = nil + loop do print "Ask a question (or type 'exit'): " question = gets&.chomp break if question.nil? || question.strip.downcase == 'exit' - response = chatbot.ask_agent(question) + response, last_resumption_token = chatbot.ask_agent(question) puts "AI Response: #{response}" end print "Was the chat helpful? [yes/no]: " feedback = gets&.chomp -chatbot.agent_was_helpful(feedback == 'yes') unless feedback.nil? +unless feedback.nil? || last_resumption_token.nil? + tracker = LaunchDarkly::Server::AI::AIConfigTracker.from_resumption_token( + token: last_resumption_token, + ld_client:, + context: + ) + chatbot.agent_was_helpful(feedback == 'yes', tracker) +end ld_client.close diff --git a/examples/chatbot/openai/hello_openai.rb b/examples/chatbot/openai/hello_openai.rb index 39ca137..a3372b6 100644 --- a/examples/chatbot/openai/hello_openai.rb +++ b/examples/chatbot/openai/hello_openai.rb @@ -35,10 +35,13 @@ def initialize(ai_config, openai_client) @openai_client = openai_client end + # Returns [response_content, resumption_token]. The resumption token can be + # used to reconstruct a tracker for deferred operations like feedback. def ask_agent(question) @messages << LaunchDarkly::Server::AI::Message.new('user', question) + tracker = @ai_config.create_tracker begin - completion = ai_config.tracker.track_openai_metrics do + completion = tracker.track_openai_metrics do @openai_client.chat.completions.create( model: ai_config.model.name, messages: @messages.map(&:to_h) @@ -46,15 +49,17 @@ def ask_agent(question) end response_content = completion[:choices][0][:message][:content] @messages << LaunchDarkly::Server::AI::Message.new('assistant', response_content) - response_content + [response_content, tracker.resumption_token] rescue StandardError => e - "An error occurred: #{e.message}" + ["An error occurred: #{e.message}", nil] end end - def agent_was_helpful(helpful) + def agent_was_helpful(helpful, tracker) + return if tracker.nil? + kind = helpful ? :positive : :negative - ai_config.tracker.track_feedback(kind: kind) + tracker.track_feedback(kind: kind) end end @@ -94,18 +99,27 @@ def agent_was_helpful(helpful) chatbot = Chatbot.new(ai_config, OpenAI::Client.new(api_key: openai_api_key)) +last_resumption_token = nil + loop do print "Ask a question (or type 'exit'): " question = gets&.chomp break if question.nil? || question.strip.downcase == 'exit' - response = chatbot.ask_agent(question) + response, last_resumption_token = chatbot.ask_agent(question) puts "AI Response: #{response}" end print "Was the chat helpful? [yes/no]: " feedback = gets&.chomp -chatbot.agent_was_helpful(feedback == 'yes') unless feedback.nil? +unless feedback.nil? || last_resumption_token.nil? + tracker = LaunchDarkly::Server::AI::AIConfigTracker.from_resumption_token( + token: last_resumption_token, + ld_client:, + context: + ) + chatbot.agent_was_helpful(feedback == 'yes', tracker) +end ld_client.close From 9af767b5cfa6a95ea339fee3ef42e52deb0a7743 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 17:45:42 -0500 Subject: [PATCH 11/23] feat!: Remove public AIConfig.disabled class method Disabled configs are an internal client concern. AIConfig.disabled had no tracker factory, which could cause errors if create_tracker was called on it. The client now uses a private DISABLED_AI_CONFIG_DEFAULT hash constant as its internal fallback. Co-Authored-By: Claude Sonnet 4.6 --- lib/server/ai/client.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index d5a265d..8d2811c 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -123,15 +123,6 @@ def create_tracker @tracker_factory.call end - # - # Returns a new disabled AIConfig instance. - # - # @return [AIConfig] a new disabled config - # - def self.disabled - new(enabled: false) - end - def to_h { _ldMeta: { @@ -150,6 +141,9 @@ def to_h TRACK_SDK_INFO = '$ld:ai:sdk:info' TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config' + # Internal fallback used when no default config is provided. Not part of the public API. + DISABLED_AI_CONFIG_DEFAULT = { _ldMeta: { enabled: false } }.freeze + INIT_TRACK_CONTEXT = LaunchDarkly::LDContext.create({ kind: 'ld_ai', key: 'ld-internal-tracking', @@ -189,7 +183,7 @@ def initialize(ld_client) def completion_config(key:, context:, default: nil, variables: nil) @ld_client.track(TRACK_USAGE_COMPLETION_CONFIG, context, key, 1) - _completion_config(key:, context:, default: default || AIConfig.disabled, variables:) + _completion_config(key:, context:, default: default || DISABLED_AI_CONFIG_DEFAULT, variables:) end # From b4c331217daa4e9b948382ec7967a37031586171 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 20 Apr 2026 16:44:58 -0500 Subject: [PATCH 12/23] feat: Add AIConfigDefault type for user-provided fallback configs Split AIConfig into two types to match the js-core pattern: - AIConfigDefault: user-facing fallback type with enabled, model, messages, provider (no tracker factory). Used as default: parameter. - AIConfig: SDK-returned type with tracker_factory as a required kwarg, ensuring create_tracker is always available. Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/client.rb | 39 ++++++++++++++++++++++++++++++--- spec/server/ai/client_spec.rb | 41 ++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 8d2811c..24d579e 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -101,12 +101,45 @@ def to_h end # - # The AIConfig class represents an AI configuration. + # The AIConfigDefault class represents a user-provided fallback AI configuration. + # + # Pass an instance of this class as the +default:+ parameter to + # {Client#completion_config} to control the fallback values when a flag + # is not found or cannot be evaluated. + # + class AIConfigDefault + attr_reader :enabled, :messages, :model, :provider + + def initialize(enabled: false, model: nil, messages: nil, provider: nil) + @enabled = enabled + @messages = messages + @model = model + @provider = provider + end + + def to_h + { + _ldMeta: { + enabled: @enabled || false, + }, + messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, + model: @model&.to_h, + provider: @provider&.to_h, + } + end + end + + # + # The AIConfig class represents an AI configuration returned by the SDK. + # + # Instances are created by {Client#completion_config} and always include + # a {#create_tracker} factory. Do not instantiate directly — use + # {AIConfigDefault} for fallback values. # class AIConfig attr_reader :enabled, :messages, :model, :provider - def initialize(enabled: nil, model: nil, messages: nil, tracker_factory: nil, provider: nil) + def initialize(enabled: nil, model: nil, messages: nil, tracker_factory:, provider: nil) @enabled = enabled @messages = messages @tracker_factory = tracker_factory @@ -176,7 +209,7 @@ def initialize(ld_client) # # @param key [String] The key of the configuration flag # @param context [LDContext] The context used when evaluating the flag - # @param default [AIConfig] The default value to use if the flag is not found + # @param default [AIConfigDefault] The default value to use if the flag is not found # @param variables [Hash] Optional variables for rendering messages # @return [AIConfig] An AIConfig instance containing the configuration data # diff --git a/spec/server/ai/client_spec.rb b/spec/server/ai/client_spec.rb index 9e87bc9..1aa3e2d 100644 --- a/spec/server/ai/client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -153,7 +153,7 @@ temperature: 0.5, maxTokens: 4096 }) messages = [LaunchDarkly::Server::AI::Message.new('system', 'Hello, {{name}}!')] - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: model, messages: messages @@ -174,7 +174,7 @@ it 'interpolates variables in model config messages' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel'), messages: [LaunchDarkly::Server::AI::Message.new('system', 'Hello, {{name}}!')] @@ -195,7 +195,7 @@ it 'returns config with messages interpolated as empty when no variables are provided' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel'), messages: [] @@ -216,7 +216,7 @@ it 'handles provider config correctly' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -237,7 +237,7 @@ it 'interpolates context variables in messages using ldctx' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy', last: 'Beaches' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -263,7 +263,7 @@ org_context = LaunchDarkly::LDContext.create({ key: 'org-key', kind: 'org', name: 'LaunchDarkly', shortname: 'LD' }) context = LaunchDarkly::LDContext.create_multi([user_context, org_context]) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -286,7 +286,7 @@ it 'handles multiple messages and variable interpolation' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -309,7 +309,7 @@ it 'returns disabled config when flag is off' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -326,7 +326,7 @@ it 'returns disabled config with nil model/messages/provider when initial config is disabled' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: true, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -342,7 +342,7 @@ it 'returns enabled config with nil model/messages/provider when initial config is enabled' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default = LaunchDarkly::Server::AI::AIConfig.new( + default = LaunchDarkly::Server::AI::AIConfigDefault.new( enabled: false, model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] @@ -451,19 +451,26 @@ end end - describe LaunchDarkly::Server::AI::AIConfig do - it 'disabled class method returns a disabled AIConfig' do - config = described_class.disabled + describe LaunchDarkly::Server::AI::AIConfigDefault do + it 'defaults to a disabled configuration' do + config = described_class.new expect(config).to be_a(described_class) expect(config.enabled).to be false expect(config.messages).to be_nil expect(config.model).to be_nil + expect(config.provider).to be_nil end - it 'disabled class method returns a new instance each call' do - first = described_class.disabled - second = described_class.disabled - expect(first).not_to be(second) + it 'serializes to a hash matching the variation format' do + model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'test-model') + messages = [LaunchDarkly::Server::AI::Message.new('system', 'Hello')] + config = described_class.new(enabled: true, model: model, messages: messages) + hash = config.to_h + + expect(hash[:_ldMeta][:enabled]).to be true + expect(hash[:model][:name]).to eq('test-model') + expect(hash[:messages].length).to eq(1) + expect(hash[:messages][0][:content]).to eq('Hello') end end end From ddfb73898a48bf104afa0606d6f6246c1b5611fa Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 20 Apr 2026 17:06:47 -0500 Subject: [PATCH 13/23] feat: Add AIConfigDefault.disabled convenience class method Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/client.rb | 4 ++++ spec/server/ai/client_spec.rb | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 24d579e..6bc5395 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -110,6 +110,10 @@ def to_h class AIConfigDefault attr_reader :enabled, :messages, :model, :provider + def self.disabled + new(enabled: false) + end + def initialize(enabled: false, model: nil, messages: nil, provider: nil) @enabled = enabled @messages = messages diff --git a/spec/server/ai/client_spec.rb b/spec/server/ai/client_spec.rb index 1aa3e2d..4c2f91f 100644 --- a/spec/server/ai/client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -461,6 +461,21 @@ expect(config.provider).to be_nil end + it 'disabled class method returns a disabled AIConfigDefault' do + config = described_class.disabled + expect(config).to be_a(described_class) + expect(config.enabled).to be false + expect(config.messages).to be_nil + expect(config.model).to be_nil + expect(config.provider).to be_nil + end + + it 'disabled class method returns a new instance each call' do + first = described_class.disabled + second = described_class.disabled + expect(first).not_to be(second) + end + it 'serializes to a hash matching the variation format' do model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'test-model') messages = [LaunchDarkly::Server::AI::Message.new('system', 'Hello')] From 4394bfce73fe676f05c3d35781e89ec2d495663a Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 21 Apr 2026 08:57:56 -0500 Subject: [PATCH 14/23] refactor: Replace DISABLED_AI_CONFIG_DEFAULT hash with AIConfigDefault.disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The constant was only referenced once — inline AIConfigDefault.disabled in completion_config and remove the private hash constant entirely. Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/client.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 6bc5395..38c347e 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -178,9 +178,6 @@ def to_h TRACK_SDK_INFO = '$ld:ai:sdk:info' TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config' - # Internal fallback used when no default config is provided. Not part of the public API. - DISABLED_AI_CONFIG_DEFAULT = { _ldMeta: { enabled: false } }.freeze - INIT_TRACK_CONTEXT = LaunchDarkly::LDContext.create({ kind: 'ld_ai', key: 'ld-internal-tracking', @@ -220,7 +217,7 @@ def initialize(ld_client) def completion_config(key:, context:, default: nil, variables: nil) @ld_client.track(TRACK_USAGE_COMPLETION_CONFIG, context, key, 1) - _completion_config(key:, context:, default: default || DISABLED_AI_CONFIG_DEFAULT, variables:) + _completion_config(key:, context:, default: default || AIConfigDefault.disabled, variables:) end # From 222c264b5ae67a02398978bc0faae644d656f2ae Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 21 Apr 2026 09:20:28 -0500 Subject: [PATCH 15/23] fix: Move required tracker_factory kwarg before optional params Fixes Style/KeywordParametersOrder rubocop offense by placing the required keyword parameter first in AIConfig#initialize. Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 38c347e..fa2aceb 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -143,7 +143,7 @@ def to_h class AIConfig attr_reader :enabled, :messages, :model, :provider - def initialize(enabled: nil, model: nil, messages: nil, tracker_factory:, provider: nil) + def initialize(tracker_factory:, enabled: nil, model: nil, messages: nil, provider: nil) @enabled = enabled @messages = messages @tracker_factory = tracker_factory From 8d1a66bd939a757f110b0eaf7bc2ea166d30e60d Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 21 Apr 2026 18:04:30 -0500 Subject: [PATCH 16/23] refactor: Make AIConfig inherit from AIConfigDefault AIConfigDefault serves as the base class holding the shared attr_readers (enabled, messages, model, provider) and to_h. AIConfig extends it with tracker_factory and create_tracker only. Co-Authored-By: Claude Opus 4.6 --- lib/server/ai/client.rb | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index fa2aceb..7d4141a 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -140,15 +140,10 @@ def to_h # a {#create_tracker} factory. Do not instantiate directly — use # {AIConfigDefault} for fallback values. # - class AIConfig - attr_reader :enabled, :messages, :model, :provider - + class AIConfig < AIConfigDefault def initialize(tracker_factory:, enabled: nil, model: nil, messages: nil, provider: nil) - @enabled = enabled - @messages = messages + super(enabled: enabled, model: model, messages: messages, provider: provider) @tracker_factory = tracker_factory - @model = model - @provider = provider end # @@ -159,17 +154,6 @@ def initialize(tracker_factory:, enabled: nil, model: nil, messages: nil, provid def create_tracker @tracker_factory.call end - - def to_h - { - _ldMeta: { - enabled: @enabled || false, - }, - messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, - model: @model&.to_h, - provider: @provider&.to_h, - } - end end # From 322bb4c409a5aeadfb7fc94587735e36add03c34 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:25:55 -0500 Subject: [PATCH 17/23] refactor: Clarify already-tracked warnings as skips with remedy Reword the six at-most-once warning messages to lead with the method name ("Skipping :") and tell the user how to recover ("Call create_tracker on the AI Config for a new run"). Matches the wording applied to the Go SDK in PR #363. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/server/ai/ai_config_tracker.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index efcc0ff..3833fe8 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -117,7 +117,7 @@ def self.from_resumption_token(token:, ld_client:, context:) # def track_duration(duration) unless @summary.duration.nil? - @logger&.warn("Duration has already been tracked for this execution. #{flag_data}") + @logger&.warn("Skipping track_duration: duration already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.duration = duration @@ -150,7 +150,7 @@ def track_duration_of(&block) # def track_time_to_first_token(time_to_first_token) unless @summary.time_to_first_token.nil? - @logger&.warn("Time to first token has already been tracked for this execution. #{flag_data}") + @logger&.warn("Skipping track_time_to_first_token: time-to-first-token already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.time_to_first_token = time_to_first_token @@ -169,7 +169,7 @@ def track_time_to_first_token(time_to_first_token) # def track_feedback(kind:) unless @summary.feedback.nil? - @logger&.warn("Feedback has already been tracked for this execution. #{flag_data}") + @logger&.warn("Skipping track_feedback: feedback already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.feedback = kind @@ -187,7 +187,7 @@ def track_feedback(kind:) # def track_success unless @summary.success.nil? - @logger&.warn("Success or error has already been tracked for this execution. #{flag_data}") + @logger&.warn("Skipping track_success: success/error already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.success = true @@ -204,7 +204,7 @@ def track_success # def track_error unless @summary.success.nil? - @logger&.warn("Success or error has already been tracked for this execution. #{flag_data}") + @logger&.warn("Skipping track_error: success/error already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.success = false @@ -223,7 +223,7 @@ def track_error # def track_tokens(token_usage) unless @summary.usage.nil? - @logger&.warn("Tokens have already been tracked for this execution. #{flag_data}") + @logger&.warn("Skipping track_tokens: token usage already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.usage = token_usage From 7e19374574b71b7113c9d4693d3f8ff5f7d5212c Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:27:26 -0500 Subject: [PATCH 18/23] docs: Clarify runId purpose and per-method tracker semantics Expand the AIConfigTracker class doc and AIConfig#create_tracker doc to explain runId, how all events emitted by a tracker share a runId so they correlate in metrics views, and that a resumption token preserves the runId across processes. Add per-method "Records at most once per Tracker" paragraphs to track_duration, track_time_to_first_token, track_feedback, and track_tokens. Note shared-state behavior on track_success and track_error. Note that track_openai_metrics and track_bedrock_converse_metrics will re-run the inner block but emit no additional metric events on repeat calls. Sweep remaining "execution" wording in doc-comments and replace with "AI run". Public method names and on-the-wire event names are unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/server/ai/ai_config_tracker.rb | 46 +++++++++++++++++++++++++----- lib/server/ai/client.rb | 5 +++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 3833fe8..48c9293 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -41,7 +41,15 @@ def initialize end # - # The AIConfigTracker class is used to track AI configuration usage. + # The AIConfigTracker records metrics for a single AI run. Unless + # otherwise noted, the tracker's methods are not safe for concurrent use. + # + # All events a tracker emits share a runId (a UUIDv4) so LaunchDarkly can + # correlate them in metrics views. See individual track methods for their + # specific semantics. Call create_tracker on the AI Config to start a new + # run. A resumption token preserves the runId, so events emitted by a + # tracker reconstructed in another process correlate with the original + # run. # class AIConfigTracker attr_reader :ld_client, :config_key, :context, :variation_key, :version, :summary, :model_name, :provider_name @@ -111,7 +119,9 @@ def self.from_resumption_token(token:, ld_client:, context:) end # - # Track the duration of an AI operation + # Track the duration of an AI run. + # + # Records at most once per Tracker; further calls are ignored. # # @param duration [Integer] The duration in milliseconds # @@ -144,7 +154,9 @@ def track_duration_of(&block) end # - # Track time to first token + # Track time to first token. + # + # Records at most once per Tracker; further calls are ignored. # # @param duration [Integer] The duration in milliseconds # @@ -163,7 +175,9 @@ def track_time_to_first_token(time_to_first_token) end # - # Track user feedback + # Track user feedback. + # + # Records at most once per Tracker; further calls are ignored. # # @param kind [Symbol] The kind of feedback (:positive or :negative) # @@ -183,7 +197,11 @@ def track_feedback(kind:) end # - # Track a successful AI generation + # Track a successful AI generation. + # + # Records at most once per Tracker. track_success and track_error share + # state; only one of the two can record per Tracker, and subsequent + # calls are ignored. # def track_success unless @summary.success.nil? @@ -200,7 +218,11 @@ def track_success end # - # Track an error in AI generation + # Track an error in AI generation. + # + # Records at most once per Tracker. track_success and track_error share + # state; only one of the two can record per Tracker, and subsequent + # calls are ignored. # def track_error unless @summary.success.nil? @@ -217,7 +239,9 @@ def track_error end # - # Track token usage + # Track token usage. + # + # Records at most once per Tracker; further calls are ignored. # # @param token_usage [TokenUsage] An object containing token usage details # @@ -259,6 +283,10 @@ def track_tokens(token_usage) # If the provided block raises, this method will also raise. # A failed operation will not have any token usage data. # + # Because each inner metric is at-most-once per Tracker, calling this + # twice on the same Tracker will run the inner block again but produce + # no additional metric events. + # # @yield The block to track. # @return The result of the tracked block. # @@ -276,6 +304,10 @@ def track_openai_metrics(&block) # Track AWS Bedrock conversation operations. # This method tracks the duration, token usage, and success/error status. # + # Because each inner metric is at-most-once per Tracker, calling this + # twice on the same Tracker will run the inner block again but produce + # no additional metric events. + # # @yield The block to track. # @return [Hash] The original response hash. # diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 7d4141a..1c59373 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -147,7 +147,10 @@ def initialize(tracker_factory:, enabled: nil, model: nil, messages: nil, provid end # - # Creates a new tracker with a fresh runId for tracking a single AI execution. + # Creates a new tracker for a fresh AI run. Each call mints a new + # runId (a UUIDv4) that LaunchDarkly uses to correlate the run's + # events in metrics views. Call this once per AI run; metrics from + # different runIds cannot be combined. # # @return [AIConfigTracker] a new tracker instance # From 8201b0ad597b1dc893e7d704c78a66d7e6fd1ff9 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:30:06 -0500 Subject: [PATCH 19/23] style: Wrap long track_time_to_first_token warning to satisfy rubocop The reworded warning exceeded the project's 180-character line limit. Split it into two adjacent string literals; the resulting string is unchanged at runtime. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/server/ai/ai_config_tracker.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 48c9293..3c512a4 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -162,7 +162,8 @@ def track_duration_of(&block) # def track_time_to_first_token(time_to_first_token) unless @summary.time_to_first_token.nil? - @logger&.warn("Skipping track_time_to_first_token: time-to-first-token already recorded on this tracker. Call create_tracker on the AI Config for a new run. #{flag_data}") + @logger&.warn("Skipping track_time_to_first_token: time-to-first-token already recorded on this tracker. " \ + "Call create_tracker on the AI Config for a new run. #{flag_data}") return end @summary.time_to_first_token = time_to_first_token From 3c1a3623c7f7690b890c6a9dee3bf22692374e6b Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:57:42 -0500 Subject: [PATCH 20/23] docs: Avoid "run...runId" duplicate phrasing in resumption docs Rephrase the two doc sentences that placed "run" and "runId" next to each other. The AIConfigTracker class doc now says a reconstructed tracker "shares the original runId" instead of "correlates with the original run". The AIConfig#create_tracker doc now describes the runId as correlating "the tracker's events" rather than "the run's events". Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/server/ai/ai_config_tracker.rb | 3 +-- lib/server/ai/client.rb | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 3c512a4..27ae7f9 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -48,8 +48,7 @@ def initialize # correlate them in metrics views. See individual track methods for their # specific semantics. Call create_tracker on the AI Config to start a new # run. A resumption token preserves the runId, so events emitted by a - # tracker reconstructed in another process correlate with the original - # run. + # tracker reconstructed in another process share the original runId. # class AIConfigTracker attr_reader :ld_client, :config_key, :context, :variation_key, :version, :summary, :model_name, :provider_name diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 1c59373..562eee6 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -148,7 +148,7 @@ def initialize(tracker_factory:, enabled: nil, model: nil, messages: nil, provid # # Creates a new tracker for a fresh AI run. Each call mints a new - # runId (a UUIDv4) that LaunchDarkly uses to correlate the run's + # runId (a UUIDv4) that LaunchDarkly uses to correlate the tracker's # events in metrics views. Call this once per AI run; metrics from # different runIds cannot be combined. # From 2877606f1b2b5f9f48489e4553f0bf4f1cbb518e Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 14 May 2026 10:17:38 -0500 Subject: [PATCH 21/23] fix: Preserve empty variation_key across resumption-token round-trip A tracker built from _completion_config defaults variation_key to '' when _ldMeta.variationKey is absent. resumption_token correctly omits empty strings from the encoded payload, but from_resumption_token previously defaulted the missing key to nil, so the reconstructed tracker exposed variation_key as nil instead of ''. The wire payload was unaffected because flag_data guards on both nil and empty, but the asymmetry was visible via the public attr_reader and brittle: any future simplification of either guard would diverge the two paths. Default the decode to '' to match how fresh trackers are constructed in client.rb, and to mirror the existing empty-string defaults for model_name and provider_name in the same method. Add a spec covering the empty-variation_key round-trip. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/server/ai/ai_config_tracker.rb | 2 +- spec/server/ai/config_tracker_spec.rb | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 27ae7f9..76db5b5 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -109,7 +109,7 @@ def self.from_resumption_token(token:, ld_client:, context:) ld_client: ld_client, run_id: payload['runId'], config_key: payload['configKey'], - variation_key: payload.fetch('variationKey', nil), + variation_key: payload.fetch('variationKey', ''), version: payload['version'], context: context, model_name: '', diff --git a/spec/server/ai/config_tracker_spec.rb b/spec/server/ai/config_tracker_spec.rb index c0413ed..a11a365 100644 --- a/spec/server/ai/config_tracker_spec.rb +++ b/spec/server/ai/config_tracker_spec.rb @@ -553,6 +553,27 @@ restored.track_feedback(kind: :positive) end + + it 'round-trips an empty variation_key as an empty string' do + empty_tracker = described_class.new( + ld_client: ld_client, + run_id: SecureRandom.uuid, + config_key: 'test-config', + context: context, + variation_key: '', + version: 1, + model_name: 'fakeModel', + provider_name: 'fakeProvider' + ) + + restored = described_class.from_resumption_token( + token: empty_tracker.resumption_token, + ld_client: ld_client, + context: context + ) + + expect(restored.variation_key).to eq('') + end end describe 'completion_config method tracking' do From 822734b537dc7abcfc4ca43cd603caaac932d4b5 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 14 May 2026 10:31:27 -0500 Subject: [PATCH 22/23] refactor: Make AIConfig and AIConfigDefault independent classes AIConfig and AIConfigDefault are now parallel hierarchies rather than parent/child. The two types serve different roles -- AIConfigDefault is an application-supplied fallback value passed into the SDK, AIConfig is what the SDK returns and always carries a working tracker factory -- and inheritance was producing footguns at the boundary: * AIConfig inherited AIConfigDefault.disabled, which dispatched to AIConfig.new(enabled: false) and raised ArgumentError because tracker_factory is required. * AIConfigDefault.disabled.create_tracker raised NoMethodError because AIConfigDefault has no create_tracker. Both crashes are now impossible by construction: AIConfig has no disabled class method, and AIConfigDefault is never expected to mint trackers. The shared fields (enabled, model, messages, provider) and to_h are intentionally duplicated -- the type separation is the point. Update the commented fallback examples in hello_bedrock.rb and hello_openai.rb to use AIConfigDefault.new(...) (the public way to build a fallback). Add three specs locking in the new invariants: AIConfig does not respond to .disabled, AIConfig.new requires tracker_factory, and AIConfig is not in AIConfigDefault.ancestors. The inheritance was only introduced earlier on this branch (commit 8d1a66b) and never shipped, so no released API is broken. Co-Authored-By: Claude Opus 4.7 (1M context) --- examples/chatbot/aws-bedrock/hello_bedrock.rb | 2 +- examples/chatbot/openai/hello_openai.rb | 2 +- lib/server/ai/client.rb | 39 ++++++++++++++++--- spec/server/ai/client_spec.rb | 14 +++++++ 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/examples/chatbot/aws-bedrock/hello_bedrock.rb b/examples/chatbot/aws-bedrock/hello_bedrock.rb index f2d73de..e88e911 100644 --- a/examples/chatbot/aws-bedrock/hello_bedrock.rb +++ b/examples/chatbot/aws-bedrock/hello_bedrock.rb @@ -94,7 +94,7 @@ def map_converse_arguments(model_id, messages) # Pass a default for improved resiliency when the flag is unavailable or LaunchDarkly is unreachable; omit for a disabled default. # Example: -# default = LaunchDarkly::Server::AI::AIConfig.new( +# default = LaunchDarkly::Server::AI::AIConfigDefault.new( # enabled: true, # model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'my-model'), # provider: LaunchDarkly::Server::AI::ProviderConfig.new(name: 'my-provider'), diff --git a/examples/chatbot/openai/hello_openai.rb b/examples/chatbot/openai/hello_openai.rb index a3372b6..bf7bfe2 100644 --- a/examples/chatbot/openai/hello_openai.rb +++ b/examples/chatbot/openai/hello_openai.rb @@ -83,7 +83,7 @@ def agent_was_helpful(helpful, tracker) # Pass a default for improved resiliency when the flag is unavailable or LaunchDarkly is unreachable; omit for a disabled default. # Example: -# default = LaunchDarkly::Server::AI::AIConfig.new( +# default = LaunchDarkly::Server::AI::AIConfigDefault.new( # enabled: true, # model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'my-model'), # provider: LaunchDarkly::Server::AI::ProviderConfig.new(name: 'my-provider'), diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 562eee6..63eb6ba 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -101,15 +101,26 @@ def to_h end # - # The AIConfigDefault class represents a user-provided fallback AI configuration. + # The AIConfigDefault class represents a user-provided fallback AI + # configuration. # # Pass an instance of this class as the +default:+ parameter to # {Client#completion_config} to control the fallback values when a flag # is not found or cannot be evaluated. # + # This is an input-only type: it is what an application supplies to the + # SDK, never what the SDK returns. The SDK always returns an + # {AIConfig}, which carries a tracker factory; AIConfigDefault does not. + # class AIConfigDefault attr_reader :enabled, :messages, :model, :provider + # + # Returns a new AIConfigDefault with enabled: false and no model, + # messages, or provider. + # + # @return [AIConfigDefault] a new disabled fallback config + # def self.disabled new(enabled: false) end @@ -136,16 +147,32 @@ def to_h # # The AIConfig class represents an AI configuration returned by the SDK. # - # Instances are created by {Client#completion_config} and always include - # a {#create_tracker} factory. Do not instantiate directly — use - # {AIConfigDefault} for fallback values. + # Instances are created by {Client#completion_config} and always carry a + # tracker factory; see {#create_tracker}. Do not instantiate directly. + # For application-supplied fallback values, use {AIConfigDefault}. # - class AIConfig < AIConfigDefault + class AIConfig + attr_reader :enabled, :messages, :model, :provider + def initialize(tracker_factory:, enabled: nil, model: nil, messages: nil, provider: nil) - super(enabled: enabled, model: model, messages: messages, provider: provider) + @enabled = enabled + @messages = messages + @model = model + @provider = provider @tracker_factory = tracker_factory end + def to_h + { + _ldMeta: { + enabled: @enabled || false, + }, + messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, + model: @model&.to_h, + provider: @provider&.to_h, + } + end + # # Creates a new tracker for a fresh AI run. Each call mints a new # runId (a UUIDv4) that LaunchDarkly uses to correlate the tracker's diff --git a/spec/server/ai/client_spec.rb b/spec/server/ai/client_spec.rb index 4c2f91f..0a5e6e3 100644 --- a/spec/server/ai/client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -451,6 +451,20 @@ end end + describe LaunchDarkly::Server::AI::AIConfig do + it 'does not expose a public disabled class method' do + expect(described_class).not_to respond_to(:disabled) + end + + it 'requires a tracker_factory to construct' do + expect { described_class.new(enabled: false) }.to raise_error(ArgumentError, /tracker_factory/) + end + + it 'is not a subclass of AIConfigDefault' do + expect(described_class.ancestors).not_to include(LaunchDarkly::Server::AI::AIConfigDefault) + end + end + describe LaunchDarkly::Server::AI::AIConfigDefault do it 'defaults to a disabled configuration' do config = described_class.new From 86bcf1f1f7c8f1935ea8ebacc53d29e3e5f4dbe5 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 14 May 2026 11:33:52 -0500 Subject: [PATCH 23/23] docs: Clarify composite-tracker re-call wording The previous wording on track_openai_metrics and track_bedrock_converse_metrics claimed a second call produces "no additional metric events". That is wrong for partial-failure cases: if the first call recorded duration and success but the inner block had no usage data, a second call where the block returns usage can still emit token metrics, because track_tokens has not yet recorded on this Tracker. Rephrase both doc paragraphs to describe the actual behavior -- only metrics not already recorded are emitted -- and point users at create_tracker for a clean run, matching the call-to-action in the at-most-once warnings. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/server/ai/ai_config_tracker.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/server/ai/ai_config_tracker.rb b/lib/server/ai/ai_config_tracker.rb index 76db5b5..e2277b7 100644 --- a/lib/server/ai/ai_config_tracker.rb +++ b/lib/server/ai/ai_config_tracker.rb @@ -283,9 +283,9 @@ def track_tokens(token_usage) # If the provided block raises, this method will also raise. # A failed operation will not have any token usage data. # - # Because each inner metric is at-most-once per Tracker, calling this - # twice on the same Tracker will run the inner block again but produce - # no additional metric events. + # Subsequent calls re-run the inner block but emit only metrics not + # already recorded on this Tracker. Call create_tracker on the AI + # Config to start a new run. # # @yield The block to track. # @return The result of the tracked block. @@ -304,9 +304,9 @@ def track_openai_metrics(&block) # Track AWS Bedrock conversation operations. # This method tracks the duration, token usage, and success/error status. # - # Because each inner metric is at-most-once per Tracker, calling this - # twice on the same Tracker will run the inner block again but produce - # no additional metric events. + # Subsequent calls re-run the inner block but emit only metrics not + # already recorded on this Tracker. Call create_tracker on the AI + # Config to start a new run. # # @yield The block to track. # @return [Hash] The original response hash.