From ceec84c4051dd14f47d442347f61438f228711a4 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 28 May 2025 22:15:54 +0000 Subject: [PATCH 01/20] feat: Support the LDAIClient --- .gitignore | 2 +- .rspec | 3 +++ Gemfile | 15 +++++++++++++++ bin/console | 11 +++++++++++ bin/setup | 8 ++++++++ launchdarkly-server-sdk-ai.gemspec | 3 ++- lib/launchdarkly-server-sdk-ai.rb | 3 --- lib/launchdarkly_server_sdk_ai.rb | 3 +++ lib/ldclient-ai.rb | 25 +++++++++++++++++++++++++ lib/ldclient-ai/ld_ai_client.rb | 22 ++++++++++++++++++++++ lib/ldclient-ai/version.rb | 7 +++++++ spec/ld_ai_client_spec.rb | 14 ++++++++++++++ spec/ldclient_ai_spec.rb | 15 +++++++++++++++ spec/spec_helper.rb | 15 +++++++++++++++ 14 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 .rspec create mode 100644 Gemfile create mode 100755 bin/console create mode 100755 bin/setup delete mode 100644 lib/launchdarkly-server-sdk-ai.rb create mode 100644 lib/launchdarkly_server_sdk_ai.rb create mode 100644 lib/ldclient-ai.rb create mode 100644 lib/ldclient-ai/ld_ai_client.rb create mode 100644 lib/ldclient-ai/version.rb create mode 100644 spec/ld_ai_client_spec.rb create mode 100644 spec/ldclient_ai_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/.gitignore b/.gitignore index 8593096..c683bde 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ doc/ /pkg/ /spec/reports/ /tmp/ - +.DS_Store # rspec failure tracking .rspec_status diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..34c5164 --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..46d9bab --- /dev/null +++ b/Gemfile @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in launchdarkly-server-sdk-ai.gemspec +gemspec + +gem "rake", "~> 13.0" + +gem "rspec", "~> 3.0" + +gem "rubocop", "~> 1.21" +gem "rubocop-performance", "~> 1.15" +gem "rubocop-rake", "~> 0.6" +gem "rubocop-rspec", "~> 2.27" diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..295cd1b --- /dev/null +++ b/bin/console @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bundler/setup' +require 'launchdarkly_server_sdk_ai' + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +require 'irb' +IRB.start(__FILE__) diff --git a/bin/setup b/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index b0eb6d0..70aabb7 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -19,5 +19,6 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = ">= 3.0.0" - spec.add_runtime_dependency "launchdarkly-server-sdk", "~> 8.4.0" + spec.add_dependency "launchdarkly-server-sdk", "~> 8.4.0" + spec.add_dependency "logger" end \ No newline at end of file diff --git a/lib/launchdarkly-server-sdk-ai.rb b/lib/launchdarkly-server-sdk-ai.rb deleted file mode 100644 index 22c8157..0000000 --- a/lib/launchdarkly-server-sdk-ai.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -raise "Reserved for LaunchDarkly" \ No newline at end of file diff --git a/lib/launchdarkly_server_sdk_ai.rb b/lib/launchdarkly_server_sdk_ai.rb new file mode 100644 index 0000000..12bbb2c --- /dev/null +++ b/lib/launchdarkly_server_sdk_ai.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative 'ldclient-ai' diff --git a/lib/ldclient-ai.rb b/lib/ldclient-ai.rb new file mode 100644 index 0000000..3c52d5e --- /dev/null +++ b/lib/ldclient-ai.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "ldclient-ai/ld_ai_client" +require 'ldclient-ai/version' +require 'logger' + +module LaunchDarkly + # + # Namespace for the LaunchDarkly AI SDK. + # + module AI + # + # @return [Logger] the Rails logger if in Rails, or a default Logger at WARN level otherwise + # + def self.default_logger + if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger + Rails.logger + else + log = ::Logger.new($stdout) + log.level = ::Logger::WARN + log + end + end + end +end diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb new file mode 100644 index 0000000..cdf4a3a --- /dev/null +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'ldclient-rb' + +module LaunchDarkly + # + # Namespace for the LaunchDarkly AI SDK. + # + module AI + class LDAIClient + attr_reader :logger + attr_reader :ld_client + + def initialize(ld_client) + raise ArgumentError, 'LDClient instance is required' unless ld_client.is_a?(LaunchDarkly::LDClient) + + @ld_client = ld_client + @logger = LaunchDarkly::AI.default_logger + end + end + end +end \ No newline at end of file diff --git a/lib/ldclient-ai/version.rb b/lib/ldclient-ai/version.rb new file mode 100644 index 0000000..cd7e001 --- /dev/null +++ b/lib/ldclient-ai/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module LaunchDarkly + module AI + VERSION = '0.0.0' # x-release-please-version + end +end diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb new file mode 100644 index 0000000..a39b2ed --- /dev/null +++ b/spec/ld_ai_client_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'launchdarkly_server_sdk_ai' + +RSpec.describe LaunchDarkly::AI::LDAIClient do + it '1.2.1 The SDK MUST provide access to an LDAIClient' do + ld_client = LaunchDarkly::LDClient.new('sdk-key-123abc') + ai_client = LaunchDarkly::AI::LDAIClient.new(ld_client) + expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) + end + it '1.2.2 The LDAIClient MUST be provided a fully configured LDClient instance at instantiation.' do + expect { LaunchDarkly::AI::LDAIClient.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') + end +end diff --git a/spec/ldclient_ai_spec.rb b/spec/ldclient_ai_spec.rb new file mode 100644 index 0000000..cf969e8 --- /dev/null +++ b/spec/ldclient_ai_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'launchdarkly_server_sdk_ai' + +RSpec.describe LaunchDarkly::AI do + it 'has a version number' do + expect(LaunchDarkly::AI::VERSION).not_to be nil + end + + it 'returns a logger' do + logger = LaunchDarkly::AI.default_logger + expect(logger).to be_a(Logger) + expect(logger.level).to eq(Logger::WARN) + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..7975e16 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# require 'ldclient-ai' + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = '.rspec_status' + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end From 9dbe7356d7da8ab230fb44365bd35f567f7aad1a Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 29 May 2025 14:10:44 +0000 Subject: [PATCH 02/20] giving cursor a go at developing the library --- launchdarkly-server-sdk-ai.gemspec | 7 + lib/ldclient-ai.rb | 7 +- lib/ldclient-ai/ld_ai_client.rb | 45 ++++ lib/ldclient-ai/ld_ai_config_tracker.rb | 167 +++++++++++++ spec/ld_ai_client_spec.rb | 86 ++++++- spec/ld_ai_config_tracker_spec.rb | 309 ++++++++++++++++++++++++ 6 files changed, 613 insertions(+), 8 deletions(-) create mode 100644 lib/ldclient-ai/ld_ai_config_tracker.rb create mode 100644 spec/ld_ai_config_tracker_spec.rb diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index 70aabb7..101692d 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -21,4 +21,11 @@ Gem::Specification.new do |spec| spec.add_dependency "launchdarkly-server-sdk", "~> 8.4.0" spec.add_dependency "logger" + spec.add_dependency "mustache", "~> 1.1" + + spec.add_development_dependency 'bundler', '~> 2.0' + spec.add_development_dependency 'rake', '~> 13.0' + spec.add_development_dependency 'rspec', '~> 3.0' + spec.add_development_dependency 'rubocop', '~> 1.0' + spec.add_development_dependency 'rubocop-rspec', '~> 2.0' end \ No newline at end of file diff --git a/lib/ldclient-ai.rb b/lib/ldclient-ai.rb index 3c52d5e..255a716 100644 --- a/lib/ldclient-ai.rb +++ b/lib/ldclient-ai.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true -require "ldclient-ai/ld_ai_client" -require 'ldclient-ai/version' require 'logger' +require 'mustache' + +require 'ldclient-ai/version' +require 'ldclient-ai/ld_ai_client' +require 'ldclient-ai/ld_ai_config_tracker' module LaunchDarkly # diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb index cdf4a3a..ec75999 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'ldclient-rb' +require 'mustache' module LaunchDarkly # @@ -17,6 +18,50 @@ def initialize(ld_client) @ld_client = ld_client @logger = LaunchDarkly::AI.default_logger end + + # Retrieves a configuration and returns a tracker for monitoring its usage + # @param config_key [String] The key of the configuration flag + # @param context [LDContext] The context used when evaluating the flag + # @return [LDAIConfigTracker] A tracker instance for monitoring the configuration usage + def config(config_key, context) + result = @ld_client.json_variation(config_key, context, {}) + unless result.is_a?(Hash) + raise ArgumentError, 'Result must be a dictionary' + end + + unless result.key?('_ldMeta') + raise ArgumentError, 'Result must contain _ldMeta' + end + + unless result['_ldMeta'].key?('enabled') + raise ArgumentError, 'Result must contain _ldMeta.enabled' + end + + # Build variables dictionary for rendering messages + variables = {} + result.each do |key, value| + next if key == '_ldMeta' + variables[key] = value + end + + # Render messages if they exist + if result.key?('messages') + result['messages'] = result['messages'].map do |message| + if message.key?('content') + message['content'] = Mustache.render(message['content'], variables) + end + message + end + end + + LDAIConfigTracker.new( + ld_client: @ld_client, + config_key: config_key, + context: context, + variation_key: config_key, + version: 1 + ) + end end end end \ No newline at end of file diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb new file mode 100644 index 0000000..1d7b84e --- /dev/null +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require 'ldclient-rb' + +module LaunchDarkly + module AI + class LDAIConfigTracker + attr_reader :ld_client, :config_key, :context, :variation_key, :version + + def initialize(ld_client:, config_key:, context:, variation_key:, version:) + @ld_client = ld_client + @config_key = config_key + @context = context + @variation_key = variation_key + @version = version + @duration = nil + @feedback = nil + @tokens = nil + @success = nil + @time_to_first_token = nil + end + + def track_duration(duration) + @duration = duration + @ld_client.track( + '$ld:ai:duration:total', + @context, + { variationKey: @variation_key, configKey: @config_key }, + duration + ) + end + + def track_duration_of + start_time = Time.now + result = yield + duration = ((Time.now - start_time) * 1000).to_i + track_duration(duration) + result + end + + def track_feedback(kind:) + @feedback = kind + event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' + @ld_client.track( + event_name, + @context, + { variationKey: @variation_key, configKey: @config_key }, + 1 + ) + end + + def track_success + @success = true + @ld_client.track( + '$ld:ai:generation', + @context, + { variationKey: @variation_key, configKey: @config_key }, + 1 + ) + @ld_client.track( + '$ld:ai:generation:success', + @context, + { variationKey: @variation_key, configKey: @config_key }, + 1 + ) + end + + def track_error + @success = false + @ld_client.track( + '$ld:ai:generation', + @context, + { variationKey: @variation_key, configKey: @config_key }, + 1 + ) + @ld_client.track( + '$ld:ai:generation:error', + @context, + { variationKey: @variation_key, configKey: @config_key }, + 1 + ) + end + + def track_tokens(total: nil, input: nil, output: nil) + @tokens = { total: total, input: input, output: output } + if total + @ld_client.track( + '$ld:ai:tokens:total', + @context, + { variationKey: @variation_key, configKey: @config_key }, + total + ) + end + if input + @ld_client.track( + '$ld:ai:tokens:input', + @context, + { variationKey: @variation_key, configKey: @config_key }, + input + ) + end + if output + @ld_client.track( + '$ld:ai:tokens:output', + @context, + { variationKey: @variation_key, configKey: @config_key }, + output + ) + end + end + + def track_openai_metrics + start_time = Time.now + result = yield + duration = ((Time.now - start_time) * 1000).to_i + track_duration(duration) + track_success + if result.usage + track_tokens( + total: result.usage.total_tokens, + input: result.usage.prompt_tokens, + output: result.usage.completion_tokens + ) + end + result + rescue StandardError => e + track_error + raise e + end + + def track_bedrock_metrics + start_time = Time.now + result = yield + duration = ((Time.now - start_time) * 1000).to_i + track_duration(duration) + if result.usage + track_tokens( + total: result.usage.total_tokens, + input: result.usage.input_tokens, + output: result.usage.output_tokens + ) + end + result + end + + def track_time_to_first_token(duration) + @time_to_first_token = duration + @ld_client.track( + '$ld:ai:tokens:ttf', + @context, + { variationKey: @variation_key, configKey: @config_key }, + duration + ) + end + + def get_summary + { + duration: @duration, + feedback: @feedback, + tokens: @tokens, + success: @success, + time_to_first_token: @time_to_first_token + } + end + end + end +end \ No newline at end of file diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index a39b2ed..ce3ba3b 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -3,12 +3,86 @@ require 'launchdarkly_server_sdk_ai' RSpec.describe LaunchDarkly::AI::LDAIClient do - it '1.2.1 The SDK MUST provide access to an LDAIClient' do - ld_client = LaunchDarkly::LDClient.new('sdk-key-123abc') - ai_client = LaunchDarkly::AI::LDAIClient.new(ld_client) - expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) + let(:ld_client) { instance_double(LaunchDarkly::LDClient) } + let(:context) { instance_double(LaunchDarkly::LDContext) } + let(:client) { described_class.new(ld_client) } + + describe '#initialize' do + it 'raises an error if LDClient is not provided' do + expect { described_class.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') + end + + it 'raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do + expect { described_class.new('not a client') }.to raise_error(ArgumentError, 'LDClient instance is required') + end + + it 'initializes with a valid LDClient instance' do + expect(client.ld_client).to eq(ld_client) + expect(client.logger).to eq(LaunchDarkly::AI.default_logger) + end end - it '1.2.2 The LDAIClient MUST be provided a fully configured LDClient instance at instantiation.' do - expect { LaunchDarkly::AI::LDAIClient.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') + + describe '#config' do + let(:config_key) { 'test-config' } + let(:context) { instance_double(LaunchDarkly::LDContext) } + let(:config_data) do + { + 'model' => 'gpt-4', + 'temperature' => 0.7, + 'max_tokens' => 100, + 'messages' => [ + { 'role' => 'system', 'content' => 'You are a helpful assistant' } + ], + '_ldMeta' => { 'enabled' => true } + } + end + + before do + allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return(config_data) + end + + it 'returns a tracker with the correct configuration' do + tracker = client.config(config_key, context) + expect(tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) + expect(tracker.ld_client).to eq(ld_client) + expect(tracker.config_key).to eq(config_key) + expect(tracker.context).to eq(context) + expect(tracker.variation_key).to eq(config_key) + expect(tracker.version).to eq(1) + end + + it 'raises an error if the result is not a dictionary' do + allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return('not a dict') + expect { client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must be a dictionary') + end + + it 'raises an error if _ldMeta is missing' do + allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({}) + expect { client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta') + end + + it 'raises an error if _ldMeta.enabled is missing' do + allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({ '_ldMeta' => {} }) + expect { client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta.enabled') + end + + it 'renders messages with variables' do + config_data['messages'] = [ + { 'role' => 'system', 'content' => 'Hello {{name}}!' } + ] + config_data['name'] = 'World' + + tracker = client.config(config_key, context) + expect(config_data['messages'][0]['content']).to eq('Hello World!') + end + + it 'preserves messages without content' do + config_data['messages'] = [ + { 'role' => 'system' } + ] + + tracker = client.config(config_key, context) + expect(config_data['messages'][0]).to eq({ 'role' => 'system' }) + end end end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb new file mode 100644 index 0000000..341f705 --- /dev/null +++ b/spec/ld_ai_config_tracker_spec.rb @@ -0,0 +1,309 @@ +# frozen_string_literal: true + +require 'launchdarkly_server_sdk_ai' + +RSpec.describe LaunchDarkly::AI::LDAIConfigTracker do + let(:ld_client) { instance_double(LaunchDarkly::LDClient) } + let(:context) { instance_double(LaunchDarkly::LDContext) } + let(:tracker) do + described_class.new( + ld_client: ld_client, + config_key: 'test-config', + context: context, + variation_key: 'test-variation', + version: 1 + ) + end + + describe '#track_duration' do + it 'tracks duration with correct event name and data' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 100 + ) + tracker.track_duration(100) + end + end + + describe '#track_duration_of' do + it 'tracks duration of a block and returns its result' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + kind_of(Integer) + ) + result = tracker.track_duration_of { 'test result' } + expect(result).to eq('test result') + end + end + + describe '#track_feedback' do + it 'tracks positive feedback' do + expect(ld_client).to receive(:track).with( + '$ld:ai:feedback:user:positive', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + tracker.track_feedback(kind: :positive) + end + + it 'tracks negative feedback' do + expect(ld_client).to receive(:track).with( + '$ld:ai:feedback:user:negative', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + tracker.track_feedback(kind: :negative) + end + end + + describe '#track_success' do + it 'tracks generation and success events' do + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:success', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + tracker.track_success + end + end + + describe '#track_error' do + it 'tracks generation and error events' do + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + tracker.track_error + end + end + + describe '#track_tokens' do + it 'tracks total tokens' do + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 100 + ) + tracker.track_tokens(total: 100) + end + + it 'tracks input tokens' do + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:input', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + tracker.track_tokens(input: 50) + end + + it 'tracks output tokens' do + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:output', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + tracker.track_tokens(output: 50) + end + + it 'tracks all token types' do + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 100 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:input', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:output', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + tracker.track_tokens(total: 100, input: 50, output: 50) + end + end + + describe '#track_openai_metrics' do + let(:openai_result) do + double('OpenAIResult', usage: double( + total_tokens: 100, + prompt_tokens: 50, + completion_tokens: 50 + )) + end + + it 'tracks duration and tokens for successful operation' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + kind_of(Integer) + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 100 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:input', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:output', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:success', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + + result = tracker.track_openai_metrics { openai_result } + expect(result).to eq(openai_result) + end + + it 'tracks error for failed operation' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + kind_of(Integer) + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + + expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') + end + end + + describe '#track_bedrock_metrics' do + let(:bedrock_result) do + double('BedrockResult', usage: double( + total_tokens: 100, + input_tokens: 50, + output_tokens: 50 + )) + end + + it 'tracks duration and tokens' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + kind_of(Integer) + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 100 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:input', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:output', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 50 + ) + + result = tracker.track_bedrock_metrics { bedrock_result } + expect(result).to eq(bedrock_result) + end + end + + describe '#track_time_to_first_token' do + it 'tracks time to first token' do + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:ttf', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 100 + ) + tracker.track_time_to_first_token(100) + end + end + + describe '#get_summary' do + it 'returns a summary of tracked metrics' do + tracker.track_duration(100) + tracker.track_feedback(kind: :positive) + tracker.track_tokens(total: 100, input: 50, output: 50) + tracker.track_success + tracker.track_time_to_first_token(50) + + expect(tracker.get_summary).to eq({ + duration: 100, + feedback: :positive, + tokens: { total: 100, input: 50, output: 50 }, + success: true, + time_to_first_token: 50 + }) + end + + it 'returns nil for untracked metrics' do + expect(tracker.get_summary).to eq({ + duration: nil, + feedback: nil, + tokens: nil, + success: nil, + time_to_first_token: nil + }) + end + end +end \ No newline at end of file From c246ead2867acec9e4b16b1ba15806914df65182 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 30 May 2025 12:47:07 +0000 Subject: [PATCH 03/20] Working to get running tests, still failing --- lib/ldclient-ai/ld_ai_client.rb | 82 ++++++++++++++-------- spec/ld_ai_client_spec.rb | 113 +++++++++++++++++------------- spec/ld_ai_config_tracker_spec.rb | 70 +++++++++--------- 3 files changed, 154 insertions(+), 111 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb index ec75999..d1a27d1 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -8,9 +8,33 @@ module LaunchDarkly # Namespace for the LaunchDarkly AI SDK. # module AI + # The LDAIConfigTracker class is used to track AI configuration. + class AIConfig + attr_reader :enabled, :messages, :variables, :tracker + + def initialize(enabled: false, messages: nil, variables: nil, tracker: nil) + @enabled = enabled + @messages = messages + @variables = variables + @tracker = tracker + end + + def default + AIConfig.new(enabled: false) + end + + def to_h + { + enabled: @enabled || false, + messages: @messages, + variables: @variables, + tracker: @tracker + } + end + end + class LDAIClient - attr_reader :logger - attr_reader :ld_client + attr_reader :logger, :ld_client def initialize(ld_client) raise ArgumentError, 'LDClient instance is required' unless ld_client.is_a?(LaunchDarkly::LDClient) @@ -19,49 +43,49 @@ def initialize(ld_client) @logger = LaunchDarkly::AI.default_logger end - # Retrieves a configuration and returns a tracker for monitoring its usage - # @param config_key [String] The key of the configuration flag + # Retrieves the AIConfig + # @param key [String] The key of the configuration flag # @param context [LDContext] The context used when evaluating the flag - # @return [LDAIConfigTracker] A tracker instance for monitoring the configuration usage - def config(config_key, context) - result = @ld_client.json_variation(config_key, context, {}) - unless result.is_a?(Hash) - raise ArgumentError, 'Result must be a dictionary' - end - - unless result.key?('_ldMeta') - raise ArgumentError, 'Result must contain _ldMeta' - end - - unless result['_ldMeta'].key?('enabled') - raise ArgumentError, 'Result must contain _ldMeta.enabled' - end + # @param default_value [AIConfig] + # @param variables [] Optional variables for rendering messages + # @return [AIConfig] An AIConfig instance containing the configuration data + def config(key, context, default_value, variables: nil) + variation = @ld_client.variation(key, context, default_value.to_h) # Build variables dictionary for rendering messages - variables = {} - result.each do |key, value| - next if key == '_ldMeta' - variables[key] = value + all_variables = {} + + if variables.nil? + variables.each do |key, value| + all_variables[key] = value + end end + all_variables['ldctx'] = context.to_h # Render messages if they exist - if result.key?('messages') - result['messages'] = result['messages'].map do |message| - if message.key?('content') - message['content'] = Mustache.render(message['content'], variables) - end + messages = nil + if variation.key?('messages') + messages = variation['messages'].map do |message| + message['content'] = Mustache.render(message['content'], variables) if message.key?('content') message end end - LDAIConfigTracker.new( + tracker = LDAIConfigTracker.new( ld_client: @ld_client, config_key: config_key, context: context, variation_key: config_key, version: 1 ) + + AIConfig.new( + enabled: result['_ldMeta']['enabled'], + messages: messages, + variables: variables, + tracker: tracker + ) end end end -end \ No newline at end of file +end diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index ce3ba3b..9edd3d3 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -5,25 +5,31 @@ RSpec.describe LaunchDarkly::AI::LDAIClient do let(:ld_client) { instance_double(LaunchDarkly::LDClient) } let(:context) { instance_double(LaunchDarkly::LDContext) } - let(:client) { described_class.new(ld_client) } + before { + allow(ld_client).to receive(:is_a?).with(LaunchDarkly::LDClient).and_return(true) + allow(context).to receive(:is_a?).with(LaunchDarkly::LDContext).and_return(true) + } describe '#initialize' do - it 'raises an error if LDClient is not provided' do - expect { described_class.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') + it '1.2.1: initializes with a valid LDClient instance' do + ai_client = described_class.new(ld_client) + expect { ai_client }.not_to raise_error + expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) + expect(ai_client.ld_client).to eq(ld_client) + expect(ai_client.logger).to be_a(Logger) end - it 'raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do - expect { described_class.new('not a client') }.to raise_error(ArgumentError, 'LDClient instance is required') + it '1.2.2: raises an error if LDClient is not provided' do + expect { described_class.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') end - it 'initializes with a valid LDClient instance' do - expect(client.ld_client).to eq(ld_client) - expect(client.logger).to eq(LaunchDarkly::AI.default_logger) + it '1.2.2: raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do + expect { described_class.new('not a client') }.to raise_error(ArgumentError, 'LDClient instance is required') end end describe '#config' do - let(:config_key) { 'test-config' } + let(:key) { 'key-123' } let(:context) { instance_double(LaunchDarkly::LDContext) } let(:config_data) do { @@ -37,52 +43,65 @@ } end - before do - allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return(config_data) - end + it 'The LDAIClient MUST provide a method for retrieving a model config' do + ai_client = described_class.new(ld_client) + ai_config = LaunchDarkly::AI::AIConfig.new + allow(ld_client).to receive(:variation).with(key, context, ai_config.to_h).and_return(config_data) - it 'returns a tracker with the correct configuration' do - tracker = client.config(config_key, context) - expect(tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) - expect(tracker.ld_client).to eq(ld_client) - expect(tracker.config_key).to eq(config_key) - expect(tracker.context).to eq(context) - expect(tracker.variation_key).to eq(config_key) - expect(tracker.version).to eq(1) + + config = ai_client.config(key, context, ai_config) + expect(config).to be_a(LaunchDarkly::AI::AIConfig) + expect(config.enabled).to be false end - it 'raises an error if the result is not a dictionary' do - allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return('not a dict') - expect { client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must be a dictionary') - end + # it 'returns a tracker with the correct configuration' do + # ai_client = described_class.new(ld_client) + # tracker = ai_client.config(config_key, context) + # expect(tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) + # expect(tracker.ld_client).to eq(ld_client) + # expect(tracker.config_key).to eq(config_key) + # expect(tracker.context).to eq(context) + # expect(tracker.variation_key).to eq(config_key) + # expect(tracker.version).to eq(1) + # end - it 'raises an error if _ldMeta is missing' do - allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({}) - expect { client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta') - end + # it 'raises an error if the result is not a dictionary' do + # ai_client = described_class.new(ld_client) + # allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return('not a dict') + # expect { ai_client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must be a dictionary') + # end - it 'raises an error if _ldMeta.enabled is missing' do - allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({ '_ldMeta' => {} }) - expect { client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta.enabled') - end + # it 'raises an error if _ldMeta is missing' do + # ai_client = described_class.new(ld_client) + # allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({}) + # expect { ai_client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta') + # end - it 'renders messages with variables' do - config_data['messages'] = [ - { 'role' => 'system', 'content' => 'Hello {{name}}!' } - ] - config_data['name'] = 'World' + # it 'raises an error if _ldMeta.enabled is missing' do + # ai_client = described_class.new(ld_client) + # allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({ '_ldMeta' => {} }) + # expect { ai_client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta.enabled') + # end + + # it 'renders messages with variables' do + # ai_client = described_class.new(ld_client) + # config_data['messages'] = [ + # { 'role' => 'system', 'content' => 'Hello {{name}}!' } + # ] + # config_data['name'] = 'World' - tracker = client.config(config_key, context) - expect(config_data['messages'][0]['content']).to eq('Hello World!') - end + # tracker = ai_client.config(config_key, context) + # expect(config_data['messages'][0]['content']).to eq('Hello World!') + # end - it 'preserves messages without content' do - config_data['messages'] = [ - { 'role' => 'system' } - ] + # it 'preserves messages without content' do + # ai_client = described_class.new(ld_client) + # config_data['messages'] = [ + # { 'role' => 'system' } + # ] - tracker = client.config(config_key, context) - expect(config_data['messages'][0]).to eq({ 'role' => 'system' }) - end + # tracker = ai_client.config(config_key, context) + # expect(config_data['messages'][0]).to eq({ 'role' => 'system' }) + # end end end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index 341f705..f7fbf82 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -203,28 +203,28 @@ expect(result).to eq(openai_result) end - it 'tracks error for failed operation' do - expect(ld_client).to receive(:track).with( - '$ld:ai:duration:total', - context, - { variationKey: 'test-variation', configKey: 'test-config' }, - kind_of(Integer) - ) - expect(ld_client).to receive(:track).with( - '$ld:ai:generation', - context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 1 - ) - expect(ld_client).to receive(:track).with( - '$ld:ai:generation:error', - context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 1 - ) + # it 'tracks error for failed operation' do + # expect(ld_client).to receive(:track).with( + # '$ld:ai:duration:total', + # context, + # { variationKey: 'test-variation', configKey: 'test-config' }, + # kind_of(Integer) + # ) + # expect(ld_client).to receive(:track).with( + # '$ld:ai:generation', + # context, + # { variationKey: 'test-variation', configKey: 'test-config' }, + # 1 + # ) + # expect(ld_client).to receive(:track).with( + # '$ld:ai:generation:error', + # context, + # { variationKey: 'test-variation', configKey: 'test-config' }, + # 1 + # ) - expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') - end + # expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') + # end end describe '#track_bedrock_metrics' do @@ -280,21 +280,21 @@ end describe '#get_summary' do - it 'returns a summary of tracked metrics' do - tracker.track_duration(100) - tracker.track_feedback(kind: :positive) - tracker.track_tokens(total: 100, input: 50, output: 50) - tracker.track_success - tracker.track_time_to_first_token(50) + # it 'returns a summary of tracked metrics' do + # tracker.track_duration(100) + # tracker.track_feedback(kind: :positive) + # tracker.track_tokens(total: 100, input: 50, output: 50) + # tracker.track_success + # tracker.track_time_to_first_token(50) - expect(tracker.get_summary).to eq({ - duration: 100, - feedback: :positive, - tokens: { total: 100, input: 50, output: 50 }, - success: true, - time_to_first_token: 50 - }) - end + # expect(tracker.get_summary).to eq({ + # duration: 100, + # feedback: :positive, + # tokens: { total: 100, input: 50, output: 50 }, + # success: true, + # time_to_first_token: 50 + # }) + # end it 'returns nil for untracked metrics' do expect(tracker.get_summary).to eq({ From b946c2e611b62c657a197b8e7b12b50da222f21e Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 30 May 2025 12:47:31 +0000 Subject: [PATCH 04/20] Letting cursor have another go --- lib/ldclient-ai/ld_ai_client.rb | 50 +++++--- lib/ldclient-ai/ld_ai_config_tracker.rb | 24 ++++ spec/ld_ai_client_spec.rb | 149 +++++++++++++----------- spec/ld_ai_config_tracker_spec.rb | 70 +++++------ 4 files changed, 174 insertions(+), 119 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb index d1a27d1..f43ec58 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -8,7 +8,7 @@ module LaunchDarkly # Namespace for the LaunchDarkly AI SDK. # module AI - # The LDAIConfigTracker class is used to track AI configuration. + # The AIConfig class represents an AI configuration. class AIConfig attr_reader :enabled, :messages, :variables, :tracker @@ -19,7 +19,7 @@ def initialize(enabled: false, messages: nil, variables: nil, tracker: nil) @tracker = tracker end - def default + def self.default AIConfig.new(enabled: false) end @@ -33,6 +33,28 @@ def to_h end end + # The LDAIConfigTracker class is used to track AI configuration. + class LDAIConfigTracker + attr_reader :ld_client, :config_key, :context, :variation_key, :version + + def initialize(ld_client:, config_key:, context:, variation_key:, version:) + @ld_client = ld_client + @config_key = config_key + @context = context + @variation_key = variation_key + @version = version + end + + def track + @ld_client.track('ai_config_used', @context, nil, { + configKey: @config_key, + variationKey: @variation_key, + version: @version + }) + end + end + + # The LDAIClient class is the main entry point for the LaunchDarkly AI SDK. class LDAIClient attr_reader :logger, :ld_client @@ -46,43 +68,43 @@ def initialize(ld_client) # Retrieves the AIConfig # @param key [String] The key of the configuration flag # @param context [LDContext] The context used when evaluating the flag - # @param default_value [AIConfig] - # @param variables [] Optional variables for rendering messages + # @param default_value [AIConfig] The default value to use if the flag is not found + # @param variables [Hash] Optional variables for rendering messages # @return [AIConfig] An AIConfig instance containing the configuration data - def config(key, context, default_value, variables: nil) + def config(key, context, default_value = AIConfig.default, variables: nil) variation = @ld_client.variation(key, context, default_value.to_h) # Build variables dictionary for rendering messages all_variables = {} - if variables.nil? - variables.each do |key, value| - all_variables[key] = value + unless variables.nil? + variables.each do |k, v| + all_variables[k] = v end end - all_variables['ldctx'] = context.to_h + all_variables['ldctx'] = context.keys # Render messages if they exist messages = nil if variation.key?('messages') messages = variation['messages'].map do |message| - message['content'] = Mustache.render(message['content'], variables) if message.key?('content') + message['content'] = Mustache.render(message['content'], all_variables) if message.key?('content') message end end tracker = LDAIConfigTracker.new( ld_client: @ld_client, - config_key: config_key, + config_key: key, context: context, - variation_key: config_key, + variation_key: key, version: 1 ) AIConfig.new( - enabled: result['_ldMeta']['enabled'], + enabled: variation['enabled'] || false, messages: messages, - variables: variables, + variables: all_variables, tracker: tracker ) end diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb index 1d7b84e..5ed48ac 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -4,6 +4,7 @@ module LaunchDarkly module AI + # The LDAIConfigTracker class is used to track AI configuration usage. class LDAIConfigTracker attr_reader :ld_client, :config_key, :context, :variation_key, :version @@ -20,6 +21,8 @@ def initialize(ld_client:, config_key:, context:, variation_key:, version:) @time_to_first_token = nil end + # Track the duration of an AI operation + # @param duration [Integer] The duration in milliseconds def track_duration(duration) @duration = duration @ld_client.track( @@ -30,6 +33,9 @@ def track_duration(duration) ) end + # Track the duration of a block of code + # @yield The block to measure + # @return The result of the block def track_duration_of start_time = Time.now result = yield @@ -38,6 +44,8 @@ def track_duration_of result end + # Track user feedback + # @param kind [Symbol] The kind of feedback (:positive or :negative) def track_feedback(kind:) @feedback = kind event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' @@ -49,6 +57,7 @@ def track_feedback(kind:) ) end + # Track a successful AI generation def track_success @success = true @ld_client.track( @@ -65,6 +74,7 @@ def track_success ) end + # Track an error in AI generation def track_error @success = false @ld_client.track( @@ -81,6 +91,10 @@ def track_error ) end + # Track token usage + # @param total [Integer] Total number of tokens + # @param input [Integer] Number of input tokens + # @param output [Integer] Number of output tokens def track_tokens(total: nil, input: nil, output: nil) @tokens = { total: total, input: input, output: output } if total @@ -109,6 +123,9 @@ def track_tokens(total: nil, input: nil, output: nil) end end + # Track OpenAI metrics + # @yield The block to measure + # @return The result of the block def track_openai_metrics start_time = Time.now result = yield @@ -128,6 +145,9 @@ def track_openai_metrics raise e end + # Track Bedrock metrics + # @yield The block to measure + # @return The result of the block def track_bedrock_metrics start_time = Time.now result = yield @@ -143,6 +163,8 @@ def track_bedrock_metrics result end + # Track time to first token + # @param duration [Integer] The duration in milliseconds def track_time_to_first_token(duration) @time_to_first_token = duration @ld_client.track( @@ -153,6 +175,8 @@ def track_time_to_first_token(duration) ) end + # Get a summary of all tracked metrics + # @return [Hash] A hash containing all tracked metrics def get_summary { duration: @duration, diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index 9edd3d3..849d8dc 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -1,107 +1,116 @@ # frozen_string_literal: true -require 'launchdarkly_server_sdk_ai' +require 'ldclient-rb' +require 'ldclient-ai' RSpec.describe LaunchDarkly::AI::LDAIClient do - let(:ld_client) { instance_double(LaunchDarkly::LDClient) } - let(:context) { instance_double(LaunchDarkly::LDContext) } - before { + let(:td) { LaunchDarkly::Integrations::TestData.data_source() } + let(:config) { LaunchDarkly::Config.new({data_source: td}) } + let(:ld_client) { LaunchDarkly::LDClient.new('key', config) } + let(:ai_client) { described_class.new(ld_client) } + let(:default_config) { LaunchDarkly::AI::AIConfig.default } + + before do allow(ld_client).to receive(:is_a?).with(LaunchDarkly::LDClient).and_return(true) - allow(context).to receive(:is_a?).with(LaunchDarkly::LDContext).and_return(true) - } + end describe '#initialize' do - it '1.2.1: initializes with a valid LDClient instance' do + it 'initializes with a valid LDClient instance' do ai_client = described_class.new(ld_client) - expect { ai_client }.not_to raise_error expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) expect(ai_client.ld_client).to eq(ld_client) - expect(ai_client.logger).to be_a(Logger) end - it '1.2.2: raises an error if LDClient is not provided' do + it 'raises an error if LDClient is not provided' do expect { described_class.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') end - it '1.2.2: raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do + it 'raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do expect { described_class.new('not a client') }.to raise_error(ArgumentError, 'LDClient instance is required') end end describe '#config' do - let(:key) { 'key-123' } - let(:context) { instance_double(LaunchDarkly::LDContext) } let(:config_data) do { + 'enabled' => true, 'model' => 'gpt-4', 'temperature' => 0.7, 'max_tokens' => 100, 'messages' => [ { 'role' => 'system', 'content' => 'You are a helpful assistant' } - ], - '_ldMeta' => { 'enabled' => true } + ] } end - it 'The LDAIClient MUST provide a method for retrieving a model config' do - ai_client = described_class.new(ld_client) - ai_config = LaunchDarkly::AI::AIConfig.new - allow(ld_client).to receive(:variation).with(key, context, ai_config.to_h).and_return(config_data) + it 'returns an AIConfig instance with the correct data' do + context = LaunchDarkly::LDContext.create('user-key') + config = ai_client.config(key, context) + expect(config).to be_a(LaunchDarkly::AI::AIConfig) + expect(config.enabled).to be true + expect(config.variables['model']).to eq('gpt-4') + expect(config.variables['temperature']).to eq(0.7) + expect(config.variables['max_tokens']).to eq(100) + expect(config.messages[0]['role']).to eq('system') + expect(config.messages[0]['content']).to eq('You are a helpful assistant') + end + + it 'renders message content with variables' do + config_data['messages'] = [ + { 'role' => 'system', 'content' => 'Hello {{name}}!' } + ] + config_data['name'] = 'World' + context = LaunchDarkly::LDContext.create('user-key') + + config = ai_client.config(key, context) + expect(config.messages[0]['content']).to eq('Hello World!') + end + + it 'preserves messages without content' do + config_data['messages'] = [ + { 'role' => 'system' } + ] + + context = LaunchDarkly::LDContext.create('user-key') + config = ai_client.config(key, context) + expect(config.messages[0]).to eq({ 'role' => 'system' }) + end + + it 'includes context in variables' do + context = LaunchDarkly::LDContext.create('user-key') + + config = ai_client.config(key, context) + expect(config.variables['ldctx']).to eq({ key: 'test-user' }) + end + + it 'handles custom variables' do + custom_vars = { 'custom' => 'value' } + context = LaunchDarkly::LDContext.create('user-key') - - config = ai_client.config(key, context, ai_config) + config = ai_client.config(key, context, default_config, variables: custom_vars) + expect(config.variables['custom']).to eq('value') + end + + it 'returns default config when variation returns nil' do + context = LaunchDarkly::LDContext.create('user-key') + + config = ai_client.config(key, context) expect(config).to be_a(LaunchDarkly::AI::AIConfig) expect(config.enabled).to be false + expect(config.messages).to be_nil + expect(config.variables).to be_nil end - # it 'returns a tracker with the correct configuration' do - # ai_client = described_class.new(ld_client) - # tracker = ai_client.config(config_key, context) - # expect(tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) - # expect(tracker.ld_client).to eq(ld_client) - # expect(tracker.config_key).to eq(config_key) - # expect(tracker.context).to eq(context) - # expect(tracker.variation_key).to eq(config_key) - # expect(tracker.version).to eq(1) - # end - - # it 'raises an error if the result is not a dictionary' do - # ai_client = described_class.new(ld_client) - # allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return('not a dict') - # expect { ai_client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must be a dictionary') - # end - - # it 'raises an error if _ldMeta is missing' do - # ai_client = described_class.new(ld_client) - # allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({}) - # expect { ai_client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta') - # end - - # it 'raises an error if _ldMeta.enabled is missing' do - # ai_client = described_class.new(ld_client) - # allow(ld_client).to receive(:json_variation).with(config_key, context, {}).and_return({ '_ldMeta' => {} }) - # expect { ai_client.config(config_key, context) }.to raise_error(ArgumentError, 'Result must contain _ldMeta.enabled') - # end - - # it 'renders messages with variables' do - # ai_client = described_class.new(ld_client) - # config_data['messages'] = [ - # { 'role' => 'system', 'content' => 'Hello {{name}}!' } - # ] - # config_data['name'] = 'World' - - # tracker = ai_client.config(config_key, context) - # expect(config_data['messages'][0]['content']).to eq('Hello World!') - # end - - # it 'preserves messages without content' do - # ai_client = described_class.new(ld_client) - # config_data['messages'] = [ - # { 'role' => 'system' } - # ] - - # tracker = ai_client.config(config_key, context) - # expect(config_data['messages'][0]).to eq({ 'role' => 'system' }) - # end + it 'creates a tracker with the correct configuration' do + context = LaunchDarkly::LDContext.create('user-key') + + config = ai_client.config(key, context) + expect(config.tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) + expect(config.tracker.ld_client).to eq(ld_client) + expect(config.tracker.config_key).to eq(key) + expect(config.tracker.context).to eq(context) + expect(config.tracker.variation_key).to eq(key) + expect(config.tracker.version).to eq(1) + end end end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index f7fbf82..341f705 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -203,28 +203,28 @@ expect(result).to eq(openai_result) end - # it 'tracks error for failed operation' do - # expect(ld_client).to receive(:track).with( - # '$ld:ai:duration:total', - # context, - # { variationKey: 'test-variation', configKey: 'test-config' }, - # kind_of(Integer) - # ) - # expect(ld_client).to receive(:track).with( - # '$ld:ai:generation', - # context, - # { variationKey: 'test-variation', configKey: 'test-config' }, - # 1 - # ) - # expect(ld_client).to receive(:track).with( - # '$ld:ai:generation:error', - # context, - # { variationKey: 'test-variation', configKey: 'test-config' }, - # 1 - # ) + it 'tracks error for failed operation' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + kind_of(Integer) + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) - # expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') - # end + expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') + end end describe '#track_bedrock_metrics' do @@ -280,21 +280,21 @@ end describe '#get_summary' do - # it 'returns a summary of tracked metrics' do - # tracker.track_duration(100) - # tracker.track_feedback(kind: :positive) - # tracker.track_tokens(total: 100, input: 50, output: 50) - # tracker.track_success - # tracker.track_time_to_first_token(50) + it 'returns a summary of tracked metrics' do + tracker.track_duration(100) + tracker.track_feedback(kind: :positive) + tracker.track_tokens(total: 100, input: 50, output: 50) + tracker.track_success + tracker.track_time_to_first_token(50) - # expect(tracker.get_summary).to eq({ - # duration: 100, - # feedback: :positive, - # tokens: { total: 100, input: 50, output: 50 }, - # success: true, - # time_to_first_token: 50 - # }) - # end + expect(tracker.get_summary).to eq({ + duration: 100, + feedback: :positive, + tokens: { total: 100, input: 50, output: 50 }, + success: true, + time_to_first_token: 50 + }) + end it 'returns nil for untracked metrics' do expect(tracker.get_summary).to eq({ From 01f150328ba51f61daee76a34a3db1cd56803104 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 30 May 2025 14:15:13 +0000 Subject: [PATCH 05/20] fix the data source created by cursor --- spec/ld_ai_client_spec.rb | 159 +++++++++++++++++++++++++++----------- 1 file changed, 114 insertions(+), 45 deletions(-) diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index 849d8dc..70cfef7 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -4,16 +4,96 @@ require 'ldclient-ai' RSpec.describe LaunchDarkly::AI::LDAIClient do - let(:td) { LaunchDarkly::Integrations::TestData.data_source() } - let(:config) { LaunchDarkly::Config.new({data_source: td}) } - let(:ld_client) { LaunchDarkly::LDClient.new('key', config) } - let(:ai_client) { described_class.new(ld_client) } - let(:default_config) { LaunchDarkly::AI::AIConfig.default } - - before do - allow(ld_client).to receive(:is_a?).with(LaunchDarkly::LDClient).and_return(true) + let(:td) do + data_source = LaunchDarkly::Integrations::TestData.data_source + data_source.update(data_source.flag('model-config') + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}}, + 'provider': {'name': 'fakeProvider'}, + 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], + '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, + }, + "green", + ) + .variation_for_all(0)) + + data_source.update(data_source.flag('multiple-messages') + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.7, 'maxTokens': 8192}}, + 'messages': [ + {'role': 'system', 'content': 'Hello, {{name}}!'}, + {'role': 'user', 'content': 'The day is, {{day}}!'}, + ], + '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, + }, + "green", + ) + .variation_for_all(0)) + + data_source.update(data_source.flag('ctx-interpolation') + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}}, + 'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.name}}! Is your last name {{ldctx.last}}?'}], + '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, + } + ) + .variation_for_all(0)) + + data_source.update(data_source.flag('multi-ctx-interpolation') + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}}, + 'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.user.name}}! Do you work for {{ldctx.org.shortname}}?'}], + '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, + } + ) + .variation_for_all(0)) + + data_source.update(data_source.flag('off-config') + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.1}}, + 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], + '_ldMeta': {'enabled': false, 'variationKey': 'abcd', 'version': 1}, + } + ) + .variation_for_all(0)) + + data_source.update(data_source.flag('initial-config-disabled') + .variations( + { + '_ldMeta': {'enabled': false}, + }, + { + '_ldMeta': {'enabled': true}, + } + ) + .variation_for_all(0)) + + data_source.update(data_source.flag('initial-config-enabled') + .variations( + { + '_ldMeta': {'enabled': false}, + }, + { + '_ldMeta': {'enabled': true}, + } + ) + .variation_for_all(1)) + + data_source end + let(:sdk_key) { 'sdk-key' } + let(:config) { LaunchDarkly::Config.new(data_source: td) } + let(:ld_client) { LaunchDarkly::LDClient.new(sdk_key, config) } + let(:context) { LaunchDarkly::LDContext.create({ key: 'test-user' }) } + let(:key) { 'model-config' } + let(:default_config) { LaunchDarkly::AI::AIConfig.default } + describe '#initialize' do it 'initializes with a valid LDClient instance' do ai_client = described_class.new(ld_client) @@ -31,23 +111,11 @@ end describe '#config' do - let(:config_data) do - { - 'enabled' => true, - 'model' => 'gpt-4', - 'temperature' => 0.7, - 'max_tokens' => 100, - 'messages' => [ - { 'role' => 'system', 'content' => 'You are a helpful assistant' } - ] - } - end - it 'returns an AIConfig instance with the correct data' do - context = LaunchDarkly::LDContext.create('user-key') + ai_client = described_class.new(ld_client) config = ai_client.config(key, context) expect(config).to be_a(LaunchDarkly::AI::AIConfig) - expect(config.enabled).to be true + expect(config.enabled).to be false expect(config.variables['model']).to eq('gpt-4') expect(config.variables['temperature']).to eq(0.7) expect(config.variables['max_tokens']).to eq(100) @@ -56,45 +124,33 @@ end it 'renders message content with variables' do - config_data['messages'] = [ - { 'role' => 'system', 'content' => 'Hello {{name}}!' } - ] - config_data['name'] = 'World' - context = LaunchDarkly::LDContext.create('user-key') - - config = ai_client.config(key, context) + ai_client = described_class.new(ld_client) + config = ai_client.config('model-config-message-content', context) expect(config.messages[0]['content']).to eq('Hello World!') end it 'preserves messages without content' do - config_data['messages'] = [ - { 'role' => 'system' } - ] - - context = LaunchDarkly::LDContext.create('user-key') - config = ai_client.config(key, context) + ai_client = described_class.new(ld_client) + config = ai_client.config('model-config-no-content', context) expect(config.messages[0]).to eq({ 'role' => 'system' }) end it 'includes context in variables' do - context = LaunchDarkly::LDContext.create('user-key') - + ai_client = described_class.new(ld_client) config = ai_client.config(key, context) expect(config.variables['ldctx']).to eq({ key: 'test-user' }) end it 'handles custom variables' do custom_vars = { 'custom' => 'value' } - context = LaunchDarkly::LDContext.create('user-key') - + ai_client = described_class.new(ld_client) config = ai_client.config(key, context, default_config, variables: custom_vars) expect(config.variables['custom']).to eq('value') end - it 'returns default config when variation returns nil' do - context = LaunchDarkly::LDContext.create('user-key') - - config = ai_client.config(key, context) + it 'returns default config when flag is off' do + ai_client = described_class.new(ld_client) + config = ai_client.config('model-config-default', context) expect(config).to be_a(LaunchDarkly::AI::AIConfig) expect(config.enabled).to be false expect(config.messages).to be_nil @@ -102,8 +158,7 @@ end it 'creates a tracker with the correct configuration' do - context = LaunchDarkly::LDContext.create('user-key') - + ai_client = described_class.new(ld_client) config = ai_client.config(key, context) expect(config.tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) expect(config.tracker.ld_client).to eq(ld_client) @@ -112,5 +167,19 @@ expect(config.tracker.variation_key).to eq(key) expect(config.tracker.version).to eq(1) end + + it 'handles fallthrough variation' do + ai_client = described_class.new(ld_client) + config = ai_client.config('model-config-fallthrough', context) + expect(config.enabled).to be false + expect(config.variables['model']).to eq('gpt-3.5-turbo') + end + + it 'handles multiple variations' do + ai_client = described_class.new(ld_client) + config = ai_client.config('model-config-multiple', context) + expect(config.enabled).to be true + expect(config.variables['model']).to eq('claude-2') + end end end From 5dfbbe8f0655f2146d489f5941d31bf95178fe19 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 4 Jun 2025 22:20:25 +0000 Subject: [PATCH 06/20] finish adding all the client tests and have them passing --- lib/ldclient-ai/ld_ai_client.rb | 173 ++++++++---- lib/ldclient-ai/ld_ai_config_tracker.rb | 24 +- spec/ld_ai_client_spec.rb | 356 +++++++++++++++++------- spec/ld_ai_config_tracker_spec.rb | 67 +++-- 4 files changed, 431 insertions(+), 189 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb index f43ec58..3461573 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -2,55 +2,110 @@ require 'ldclient-rb' require 'mustache' +require_relative 'ld_ai_config_tracker' module LaunchDarkly # # Namespace for the LaunchDarkly AI SDK. # module AI - # The AIConfig class represents an AI configuration. - class AIConfig - attr_reader :enabled, :messages, :variables, :tracker + # Holds AI role and content. + class LDMessage + attr_reader :role, :content + + # TODO: Do we need to validate the role to only be 'system', 'user', or 'assistant'? + def initialize(role, content) + @role = role + @content = content + end - def initialize(enabled: false, messages: nil, variables: nil, tracker: nil) - @enabled = enabled - @messages = messages - @variables = variables - @tracker = tracker + def to_h + { + role: @role, + content: @content + } + end + end + + # The ModelConfig class represents an AI model configuration. + class ModelConfig + attr_reader :name, :parameters, :custom + + def initialize(name:, parameters: {}, custom: {}) + @name = name + @parameters = parameters + @custom = custom + end + + # Retrieve model-specific parameters. + # + # Accessing a named, typed attribute (e.g. name) will result in the call + # being delegated to the appropriate property. + # + # @param key [String] The parameter key to retrieve + # @return [Object] The parameter value or nil if not found + def get_parameter(key) + return @name if key == 'name' + return nil if @parameters.nil? + + @parameters[key] end - def self.default - AIConfig.new(enabled: false) + # Retrieve customer provided data. + # + # @param key [String] The custom key to retrieve + # @return [Object] The custom value or nil if not found + def get_custom(key) + return nil if @custom.nil? + + @custom[key] end def to_h { - enabled: @enabled || false, - messages: @messages, - variables: @variables, - tracker: @tracker + name: @name, + parameters: @parameters, + custom: @custom } end end - # The LDAIConfigTracker class is used to track AI configuration. - class LDAIConfigTracker - attr_reader :ld_client, :config_key, :context, :variation_key, :version + # Configuration related to the provider. + class ProviderConfig + attr_reader :name - def initialize(ld_client:, config_key:, context:, variation_key:, version:) - @ld_client = ld_client - @config_key = config_key - @context = context - @variation_key = variation_key - @version = version + def initialize(name) + @name = name + end + + def to_h + { + name: @name + } + end + end + + # The AIConfig class represents an AI configuration. + class AIConfig + attr_reader :enabled, :messages, :variables, :tracker, :model, :provider + + def initialize(enabled: nil, model: nil, messages: nil, tracker: nil, provider: nil) + @enabled = enabled + @messages = messages + @tracker = tracker + @model = model + @provider = provider end - def track - @ld_client.track('ai_config_used', @context, nil, { - configKey: @config_key, - variationKey: @variation_key, - version: @version - }) + def to_h + { + _ldMeta: { + enabled: @enabled || false + }, + messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, + model: @model&.to_h, + provider: @provider&.to_h + } end end @@ -66,46 +121,58 @@ def initialize(ld_client) end # Retrieves the AIConfig - # @param key [String] The key of the configuration flag + # @param config_key [String] The key of the configuration flag # @param context [LDContext] The context used when evaluating the flag # @param default_value [AIConfig] The default value to use if the flag is not found # @param variables [Hash] Optional variables for rendering messages # @return [AIConfig] An AIConfig instance containing the configuration data - def config(key, context, default_value = AIConfig.default, variables: nil) - variation = @ld_client.variation(key, context, default_value.to_h) + def config(config_key, context, default_value = nil, variables = nil) + variation = @ld_client.variation( + config_key, + context, + default_value.respond_to?(:to_h) ? default_value.to_h : nil + ) + puts("Config variation: #{variation.inspect}") - # Build variables dictionary for rendering messages - all_variables = {} + variables ||= {} + variables[:ldctx] = context.to_h - unless variables.nil? - variables.each do |k, v| - all_variables[k] = v + messages = variation.fetch(:messages, nil) + if messages.is_a?(Array) && messages.all? { |msg| msg.is_a?(Hash) } + messages = messages.map do |message| + message[:content] = Mustache.render(message[:content], variables) if message[:content].is_a?(String) + message end end - all_variables['ldctx'] = context.keys - # Render messages if they exist - messages = nil - if variation.key?('messages') - messages = variation['messages'].map do |message| - message['content'] = Mustache.render(message['content'], all_variables) if message.key?('content') - message - end + if (provider_config = variation.fetch(:provider, nil)) && provider_config.is_a?(Hash) + provider_config = ProviderConfig.new(provider_config.fetch(:name, '')) + end + + if (model = variation.fetch(:model, nil)) && model.is_a?(Hash) + parameters = variation[:model][:parameters] + custom = variation[:model][:custom] + model = ModelConfig.new( + name: variation[:model][:name], + parameters: parameters, + custom: custom + ) end - tracker = LDAIConfigTracker.new( + tracker = LaunchDarkly::AI::LDAIConfigTracker.new( ld_client: @ld_client, - config_key: key, - context: context, - variation_key: key, - version: 1 + variation_key: variation.dig(:_ldMeta, :variationKey) || '', + config_key: config_key, + version: variation.dig(:_ldMeta, :version) || 1, + context: context ) AIConfig.new( - enabled: variation['enabled'] || false, + enabled: variation.dig(:_ldMeta, :enabled) || false, messages: messages, - variables: all_variables, - tracker: tracker + tracker: tracker, + model: model, + provider: provider_config ) end end diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb index 5ed48ac..d7d5b06 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -8,12 +8,12 @@ module AI class LDAIConfigTracker attr_reader :ld_client, :config_key, :context, :variation_key, :version - def initialize(ld_client:, config_key:, context:, variation_key:, version:) + def initialize(ld_client:, variation_key:, config_key:, version:, context:) @ld_client = ld_client - @config_key = config_key - @context = context @variation_key = variation_key + @config_key = config_key @version = version + @context = context @duration = nil @feedback = nil @tokens = nil @@ -113,14 +113,14 @@ def track_tokens(total: nil, input: nil, output: nil) input ) end - if output - @ld_client.track( - '$ld:ai:tokens:output', - @context, - { variationKey: @variation_key, configKey: @config_key }, - output - ) - end + return unless output + + @ld_client.track( + '$ld:ai:tokens:output', + @context, + { variationKey: @variation_key, configKey: @config_key }, + output + ) end # Track OpenAI metrics @@ -188,4 +188,4 @@ def get_summary end end end -end \ No newline at end of file +end diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index 70cfef7..e041638 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -8,95 +8,100 @@ data_source = LaunchDarkly::Integrations::TestData.data_source data_source.update(data_source.flag('model-config') .variations( - { - 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}}, - 'provider': {'name': 'fakeProvider'}, - 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], - '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, - }, - "green", - ) + { + model: { name: 'fakeModel', parameters: { temperature: 0.5, maxTokens: 4096 }, + custom: { 'extra-attribute': 'value' } }, + provider: { name: 'fakeProvider' }, + messages: [{ role: 'system', content: 'Hello, {{name}}!' }], + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + }, + :green + ) .variation_for_all(0)) data_source.update(data_source.flag('multiple-messages') .variations( - { - 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.7, 'maxTokens': 8192}}, - 'messages': [ - {'role': 'system', 'content': 'Hello, {{name}}!'}, - {'role': 'user', 'content': 'The day is, {{day}}!'}, - ], - '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, - }, - "green", - ) + { + model: { name: 'fakeModel', parameters: { temperature: 0.7, maxTokens: 8192 } }, + messages: [ + { role: 'system', content: 'Hello, {{name}}!' }, + { role: 'user', content: 'The day is, {{day}}!' } + ], + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + }, + :green + ) .variation_for_all(0)) data_source.update(data_source.flag('ctx-interpolation') .variations( - { - 'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}}, - 'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.name}}! Is your last name {{ldctx.last}}?'}], - '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, - } - ) + { + model: { name: 'fakeModel', + parameters: { 'extra-attribute': 'I can be anything I set my mind/type to' } }, + messages: [{ role: 'system', content: 'Hello, {{ldctx.name}}! Is your last name {{ldctx.last}}?' }], + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + } + ) .variation_for_all(0)) data_source.update(data_source.flag('multi-ctx-interpolation') .variations( - { - 'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}}, - 'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.user.name}}! Do you work for {{ldctx.org.shortname}}?'}], - '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, - } + { + model: { name: 'fakeModel', + parameters: { 'extra-attribute': 'I can be anything I set my mind/type to' } }, + messages: [{ role: 'system', + content: 'Hello, {{ldctx.user.name}}! Do you work for {{ldctx.org.shortname}}?' }], + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + } ) .variation_for_all(0)) data_source.update(data_source.flag('off-config') .variations( - { - 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.1}}, - 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], - '_ldMeta': {'enabled': false, 'variationKey': 'abcd', 'version': 1}, - } + { + model: { name: 'fakeModel', parameters: { temperature: 0.1 } }, + messages: [{ role: 'system', content: 'Hello, {{name}}!' }], + _ldMeta: { enabled: false, variationKey: 'abcd', version: 1 } + } ) .variation_for_all(0)) data_source.update(data_source.flag('initial-config-disabled') .variations( - { - '_ldMeta': {'enabled': false}, - }, - { - '_ldMeta': {'enabled': true}, - } + { + _ldMeta: { enabled: false } + }, + { + _ldMeta: { enabled: true } + } ) .variation_for_all(0)) data_source.update(data_source.flag('initial-config-enabled') .variations( - { - '_ldMeta': {'enabled': false}, - }, - { - '_ldMeta': {'enabled': true}, - } + { + _ldMeta: { enabled: false } + }, + { + _ldMeta: { enabled: true } + } ) .variation_for_all(1)) data_source end - let(:sdk_key) { 'sdk-key' } - let(:config) { LaunchDarkly::Config.new(data_source: td) } - let(:ld_client) { LaunchDarkly::LDClient.new(sdk_key, config) } + let(:ld_client) do + config = LaunchDarkly::Config.new(data_source: td, send_events: false) + LaunchDarkly::LDClient.new('sdk-key', config) + end + let(:ai_client) { LaunchDarkly::AI::LDAIClient.new(ld_client) } let(:context) { LaunchDarkly::LDContext.create({ key: 'test-user' }) } let(:key) { 'model-config' } let(:default_config) { LaunchDarkly::AI::AIConfig.default } describe '#initialize' do it 'initializes with a valid LDClient instance' do - ai_client = described_class.new(ld_client) expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) expect(ai_client.ld_client).to eq(ld_client) end @@ -111,75 +116,224 @@ end describe '#config' do - it 'returns an AIConfig instance with the correct data' do - ai_client = described_class.new(ld_client) - config = ai_client.config(key, context) - expect(config).to be_a(LaunchDarkly::AI::AIConfig) - expect(config.enabled).to be false - expect(config.variables['model']).to eq('gpt-4') - expect(config.variables['temperature']).to eq(0.7) - expect(config.variables['max_tokens']).to eq(100) - expect(config.messages[0]['role']).to eq('system') - expect(config.messages[0]['content']).to eq('You are a helpful assistant') + it 'delegates to properties' do + model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) + expect(model.name).to eq('fakeModel') + expect(model.get_parameter(:'extra-attribute')).to eq('value') + expect(model.get_parameter('non-existent')).to be_nil + expect(model.get_parameter('name')).to eq('fakeModel') end - it 'renders message content with variables' do - ai_client = described_class.new(ld_client) - config = ai_client.config('model-config-message-content', context) - expect(config.messages[0]['content']).to eq('Hello World!') + it 'handles custom attributes' do + model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', custom: { 'extra-attribute': 'value' }) + expect(model.name).to eq('fakeModel') + expect(model.get_custom(:'extra-attribute')).to eq('value') + expect(model.get_custom('non-existent')).to be_nil + expect(model.get_custom('name')).to be_nil end - it 'preserves messages without content' do - ai_client = described_class.new(ld_client) - config = ai_client.config('model-config-no-content', context) - expect(config.messages[0]).to eq({ 'role' => 'system' }) + it 'uses default config on invalid flag' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', + parameters: { + temperature: 0.5, maxTokens: 4096 + }) + messages = [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] + default_config = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: model, + messages: messages + ) + variables = { 'name' => 'World' } + + config = ai_client.config('missing-flag', context, default_config, variables) + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.5) + expect(config.model.get_parameter(:maxTokens)).to eq(4096) end - it 'includes context in variables' do - ai_client = described_class.new(ld_client) - config = ai_client.config(key, context) - expect(config.variables['ldctx']).to eq({ key: 'test-user' }) + it 'interpolates variables in model config messages' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), + messages: [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('model-config', context, default_value, variables) + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.5) + expect(config.model.get_parameter(:maxTokens)).to eq(4096) end - it 'handles custom variables' do - custom_vars = { 'custom' => 'value' } - ai_client = described_class.new(ld_client) - config = ai_client.config(key, context, default_config, variables: custom_vars) - expect(config.variables['custom']).to eq('value') + it 'returns config with messages interpolated as empty when no variables are provided' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), + messages: [] + ) + + config = ai_client.config('model-config', context, default_value, {}) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, !') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.5) + expect(config.model.get_parameter(:maxTokens)).to eq(4096) end - it 'returns default config when flag is off' do - ai_client = described_class.new(ld_client) - config = ai_client.config('model-config-default', context) - expect(config).to be_a(LaunchDarkly::AI::AIConfig) - expect(config.enabled).to be false - expect(config.messages).to be_nil - expect(config.variables).to be_nil + it 'handles provider config correctly' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('model-config', context, default_value, variables) + + expect(config.provider).not_to be_nil + expect(config.provider.name).to eq('fakeProvider') + end + + it 'interpolates context variables in messages using ldctx' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy', last: 'Beaches' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('ctx-interpolation', context, default_value, variables) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, Sandy! Is your last name Beaches?') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to be_nil + expect(config.model.get_parameter(:maxTokens)).to be_nil + expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') + end + + it 'interpolates variables from multiple contexts in messages using ldctx' do + user_context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) + org_context = LaunchDarkly::LDContext.create({ key: 'org-key', kind: 'org', name: 'LaunchDarkly', + shortname: 'LD' }) + context = LaunchDarkly::LDContext.create_multi([user_context, org_context]) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('multi-ctx-interpolation', context, default_value, variables) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, Sandy! Do you work for LD?') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to be_nil + expect(config.model.get_parameter(:maxTokens)).to be_nil + expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') end - it 'creates a tracker with the correct configuration' do - ai_client = described_class.new(ld_client) - config = ai_client.config(key, context) - expect(config.tracker).to be_a(LaunchDarkly::AI::LDAIConfigTracker) - expect(config.tracker.ld_client).to eq(ld_client) - expect(config.tracker.config_key).to eq(key) - expect(config.tracker.context).to eq(context) - expect(config.tracker.variation_key).to eq(key) - expect(config.tracker.version).to eq(1) + it 'handles multiple messages and variable interpolation' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World', 'day' => 'Monday' } + + config = ai_client.config('multiple-messages', context, default_value, variables) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.messages[1][:content]).to eq('The day is, Monday!') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.7) + expect(config.model.get_parameter(:maxTokens)).to eq(8192) end - it 'handles fallthrough variation' do - ai_client = described_class.new(ld_client) - config = ai_client.config('model-config-fallthrough', context) + it 'returns disabled config when flag is off' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: false, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + + config = ai_client.config('off-config', context, default_value, {}) + + expect(config.model).not_to be_nil expect(config.enabled).to be false - expect(config.variables['model']).to eq('gpt-3.5-turbo') + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.1) + expect(config.model.get_parameter(:maxTokens)).to be_nil end - it 'handles multiple variations' do - ai_client = described_class.new(ld_client) - config = ai_client.config('model-config-multiple', context) + it 'returns disabled config with nil model/messages/provider when initial config is disabled' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: false, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + + config = ai_client.config('initial-config-disabled', context, default_value, {}) + + expect(config.enabled).to be false + expect(config.model).to be_nil + expect(config.messages).to be_nil + expect(config.provider).to be_nil + end + + it 'returns enabled config with nil model/messages/provider when initial config is enabled' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: false, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + + config = ai_client.config('initial-config-enabled', context, default_value, {}) + expect(config.enabled).to be true - expect(config.variables['model']).to eq('claude-2') + expect(config.model).to be_nil + expect(config.messages).to be_nil + expect(config.provider).to be_nil end end end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index 341f705..9a72f42 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -1,9 +1,30 @@ # frozen_string_literal: true -require 'launchdarkly_server_sdk_ai' +require 'ldclient-rb' +require 'ldclient-ai' RSpec.describe LaunchDarkly::AI::LDAIConfigTracker do - let(:ld_client) { instance_double(LaunchDarkly::LDClient) } + let(:td) do + LaunchDarkly::Integrations::TestData.data_source().update( + LaunchDarkly::Integrations::TestData.data_source().flag('model-config') + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}}, + 'provider': {'name': 'fakeProvider'}, + 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], + '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, + }, + "green", + ) + .variation_for_all(0) + ) + end + + let(:ld_client) do + config = LaunchDarkly::Config.new(data_source: td, send_events: false) + LaunchDarkly::LDClient.new('sdk-key', config) + end + let(:context) { instance_double(LaunchDarkly::LDContext) } let(:tracker) do described_class.new( @@ -203,28 +224,28 @@ expect(result).to eq(openai_result) end - it 'tracks error for failed operation' do - expect(ld_client).to receive(:track).with( - '$ld:ai:duration:total', - context, - { variationKey: 'test-variation', configKey: 'test-config' }, - kind_of(Integer) - ) - expect(ld_client).to receive(:track).with( - '$ld:ai:generation', - context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 1 - ) - expect(ld_client).to receive(:track).with( - '$ld:ai:generation:error', - context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 1 - ) + # it 'tracks error for failed operation' do + # expect(ld_client).to receive(:track).with( + # '$ld:ai:duration:total', + # context, + # { variationKey: 'test-variation', configKey: 'test-config' }, + # kind_of(Integer) + # ) + # expect(ld_client).to receive(:track).with( + # '$ld:ai:generation', + # context, + # { variationKey: 'test-variation', configKey: 'test-config' }, + # 1 + # ) + # expect(ld_client).to receive(:track).with( + # '$ld:ai:generation:error', + # context, + # { variationKey: 'test-variation', configKey: 'test-config' }, + # 1 + # ) - expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') - end + # expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') + # end end describe '#track_bedrock_metrics' do From ca59de7d2b4bfdeb8f6d890da4e2d5d6ac70111c Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 5 Jun 2025 14:40:38 +0000 Subject: [PATCH 07/20] all ai client tests are passing --- lib/ldclient-ai/ld_ai_config_tracker.rb | 33 +- spec/ld_ai_client_spec.rb | 419 ++++++++++++------------ spec/ld_ai_config_tracker_spec.rb | 89 ++--- 3 files changed, 283 insertions(+), 258 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb index d7d5b06..c830066 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -4,6 +4,33 @@ module LaunchDarkly module AI + # Tracks token usage for AI operations. + class TokenUsage + attr_reader :total, :input, :output + + # @param total [Integer] Total number of tokens used. + # @param input [Integer] Number of tokens in the prompt. + # @param output [Integer] Number of tokens in the completion. + def initialize(total: nil, input: nil, output: nil) + @total = total + @input = input + @output = output + end + end + + # Summary of metrics which have been tracked. + class LDAIMetricSummary + attr_reader :duration, :success, :feedback, :usage, :time_to_first_token + + def initialize + @duration = nil + @success = nil + @feedback = nil + @usage = nil + @time_to_first_token = nil + end + end + # The LDAIConfigTracker class is used to track AI configuration usage. class LDAIConfigTracker attr_reader :ld_client, :config_key, :context, :variation_key, :version @@ -14,11 +41,7 @@ def initialize(ld_client:, variation_key:, config_key:, version:, context:) @config_key = config_key @version = version @context = context - @duration = nil - @feedback = nil - @tokens = nil - @success = nil - @time_to_first_token = nil + @summary = LDAIMetricSummary.new end # Track the duration of an AI operation diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index e041638..fa053b9 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'ldclient-rb' require 'ldclient-ai' +require 'ldclient-rb' -RSpec.describe LaunchDarkly::AI::LDAIClient do +RSpec.describe LaunchDarkly::AI do let(:td) do data_source = LaunchDarkly::Integrations::TestData.data_source data_source.update(data_source.flag('model-config') @@ -96,26 +96,8 @@ LaunchDarkly::LDClient.new('sdk-key', config) end let(:ai_client) { LaunchDarkly::AI::LDAIClient.new(ld_client) } - let(:context) { LaunchDarkly::LDContext.create({ key: 'test-user' }) } - let(:key) { 'model-config' } - let(:default_config) { LaunchDarkly::AI::AIConfig.default } - - describe '#initialize' do - it 'initializes with a valid LDClient instance' do - expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) - expect(ai_client.ld_client).to eq(ld_client) - end - - it 'raises an error if LDClient is not provided' do - expect { described_class.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') - end - - it 'raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do - expect { described_class.new('not a client') }.to raise_error(ArgumentError, 'LDClient instance is required') - end - end - describe '#config' do + describe LaunchDarkly::AI::ModelConfig do it 'delegates to properties' do model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') @@ -131,209 +113,228 @@ expect(model.get_custom('non-existent')).to be_nil expect(model.get_custom('name')).to be_nil end + end - it 'uses default config on invalid flag' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', - parameters: { - temperature: 0.5, maxTokens: 4096 - }) - messages = [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] - default_config = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: model, - messages: messages - ) - variables = { 'name' => 'World' } - - config = ai_client.config('missing-flag', context, default_config, variables) - expect(config.messages).not_to be_nil - expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, World!') - expect(config.enabled).to be true - - expect(config.model).not_to be_nil - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.5) - expect(config.model.get_parameter(:maxTokens)).to eq(4096) - end - - it 'interpolates variables in model config messages' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), - messages: [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] - ) - variables = { 'name' => 'World' } - - config = ai_client.config('model-config', context, default_value, variables) - expect(config.messages).not_to be_nil - expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, World!') - expect(config.enabled).to be true - - expect(config.model).not_to be_nil - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.5) - expect(config.model.get_parameter(:maxTokens)).to eq(4096) - end - - it 'returns config with messages interpolated as empty when no variables are provided' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), - messages: [] - ) - - config = ai_client.config('model-config', context, default_value, {}) - - expect(config.messages).not_to be_nil - expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, !') - expect(config.enabled).to be true - - expect(config.model).not_to be_nil - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.5) - expect(config.model.get_parameter(:maxTokens)).to eq(4096) - end - - it 'handles provider config correctly' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) - variables = { 'name' => 'World' } - - config = ai_client.config('model-config', context, default_value, variables) - - expect(config.provider).not_to be_nil - expect(config.provider.name).to eq('fakeProvider') - end - - it 'interpolates context variables in messages using ldctx' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy', last: 'Beaches' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) - variables = { 'name' => 'World' } - - config = ai_client.config('ctx-interpolation', context, default_value, variables) - - expect(config.messages).not_to be_nil - expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, Sandy! Is your last name Beaches?') - expect(config.enabled).to be true - - expect(config.model).not_to be_nil - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to be_nil - expect(config.model.get_parameter(:maxTokens)).to be_nil - expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') - end - - it 'interpolates variables from multiple contexts in messages using ldctx' do - user_context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) - org_context = LaunchDarkly::LDContext.create({ key: 'org-key', kind: 'org', name: 'LaunchDarkly', - shortname: 'LD' }) - context = LaunchDarkly::LDContext.create_multi([user_context, org_context]) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) - variables = { 'name' => 'World' } - - config = ai_client.config('multi-ctx-interpolation', context, default_value, variables) + describe LaunchDarkly::AI::LDAIClient do + describe '#initialize' do + it 'initializes with a valid LDClient instance' do + expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) + expect(ai_client.ld_client).to eq(ld_client) + end - expect(config.messages).not_to be_nil - expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, Sandy! Do you work for LD?') - expect(config.enabled).to be true + it 'raises an error if LDClient is not provided' do + expect { described_class.new(nil) }.to raise_error(ArgumentError, 'LDClient instance is required') + end - expect(config.model).not_to be_nil - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to be_nil - expect(config.model.get_parameter(:maxTokens)).to be_nil - expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') + it 'raises an error if LDClient is not an instance of LaunchDarkly::LDClient' do + expect { described_class.new('not a client') }.to raise_error(ArgumentError, 'LDClient instance is required') + end end - it 'handles multiple messages and variable interpolation' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) - variables = { 'name' => 'World', 'day' => 'Monday' } - - config = ai_client.config('multiple-messages', context, default_value, variables) - - expect(config.messages).not_to be_nil - expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, World!') - expect(config.messages[1][:content]).to eq('The day is, Monday!') - expect(config.enabled).to be true + describe '#config' do + it 'uses default config on invalid flag' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', + parameters: { + temperature: 0.5, maxTokens: 4096 + }) + messages = [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] + default_config = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: model, + messages: messages + ) + variables = { 'name' => 'World' } + + config = ai_client.config('missing-flag', context, default_config, variables) + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.5) + expect(config.model.get_parameter(:maxTokens)).to eq(4096) + end + + it 'interpolates variables in model config messages' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), + messages: [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('model-config', context, default_value, variables) + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.5) + expect(config.model.get_parameter(:maxTokens)).to eq(4096) + end + + it 'returns config with messages interpolated as empty when no variables are provided' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), + messages: [] + ) - expect(config.model).not_to be_nil - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.7) - expect(config.model.get_parameter(:maxTokens)).to eq(8192) - end + config = ai_client.config('model-config', context, default_value, {}) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, !') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.5) + expect(config.model.get_parameter(:maxTokens)).to eq(4096) + end + + it 'handles provider config correctly' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World' } - it 'returns disabled config when flag is off' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: false, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) + config = ai_client.config('model-config', context, default_value, variables) - config = ai_client.config('off-config', context, default_value, {}) + expect(config.provider).not_to be_nil + expect(config.provider.name).to eq('fakeProvider') + end - expect(config.model).not_to be_nil - expect(config.enabled).to be false - expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.1) - expect(config.model.get_parameter(:maxTokens)).to be_nil - end + it 'interpolates context variables in messages using ldctx' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy', last: 'Beaches' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('ctx-interpolation', context, default_value, variables) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, Sandy! Is your last name Beaches?') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to be_nil + expect(config.model.get_parameter(:maxTokens)).to be_nil + expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') + end + + it 'interpolates variables from multiple contexts in messages using ldctx' do + user_context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) + org_context = LaunchDarkly::LDContext.create({ key: 'org-key', kind: 'org', name: 'LaunchDarkly', + shortname: 'LD' }) + context = LaunchDarkly::LDContext.create_multi([user_context, org_context]) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World' } + + config = ai_client.config('multi-ctx-interpolation', context, default_value, variables) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, Sandy! Do you work for LD?') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to be_nil + expect(config.model.get_parameter(:maxTokens)).to be_nil + expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') + end + + it 'handles multiple messages and variable interpolation' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) + variables = { 'name' => 'World', 'day' => 'Monday' } + + config = ai_client.config('multiple-messages', context, default_value, variables) + + expect(config.messages).not_to be_nil + expect(config.messages.length).to be > 0 + expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.messages[1][:content]).to eq('The day is, Monday!') + expect(config.enabled).to be true + + expect(config.model).not_to be_nil + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.7) + expect(config.model.get_parameter(:maxTokens)).to eq(8192) + end + + it 'returns disabled config when flag is off' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) - it 'returns disabled config with nil model/messages/provider when initial config is disabled' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: false, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) + config = ai_client.config('off-config', context, default_value, {}) + + expect(config.model).not_to be_nil + expect(config.enabled).to be false + expect(config.model.name).to eq('fakeModel') + expect(config.model.get_parameter(:temperature)).to eq(0.1) + expect(config.model.get_parameter(:maxTokens)).to be_nil + end + + it 'returns disabled config with nil model/messages/provider when initial config is disabled' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: true, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) - config = ai_client.config('initial-config-disabled', context, default_value, {}) + config = ai_client.config('initial-config-disabled', context, default_value, {}) - expect(config.enabled).to be false - expect(config.model).to be_nil - expect(config.messages).to be_nil - expect(config.provider).to be_nil - end + expect(config.enabled).to be false + expect(config.model).to be_nil + expect(config.messages).to be_nil + expect(config.provider).to be_nil + end - it 'returns enabled config with nil model/messages/provider when initial config is enabled' do - context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( - enabled: false, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), - messages: [] - ) + it 'returns enabled config with nil model/messages/provider when initial config is enabled' do + context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) + default_value = LaunchDarkly::AI::AIConfig.new( + enabled: false, + model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + messages: [] + ) - config = ai_client.config('initial-config-enabled', context, default_value, {}) + config = ai_client.config('initial-config-enabled', context, default_value, {}) - expect(config.enabled).to be true - expect(config.model).to be_nil - expect(config.messages).to be_nil - expect(config.provider).to be_nil + expect(config.enabled).to be true + expect(config.model).to be_nil + expect(config.messages).to be_nil + expect(config.provider).to be_nil + end end end end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index 9a72f42..9dd0aaf 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -5,27 +5,28 @@ RSpec.describe LaunchDarkly::AI::LDAIConfigTracker do let(:td) do - LaunchDarkly::Integrations::TestData.data_source().update( - LaunchDarkly::Integrations::TestData.data_source().flag('model-config') + LaunchDarkly::Integrations::TestData.data_source.update( + LaunchDarkly::Integrations::TestData.data_source.flag('model_config') .variations( { - 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}}, - 'provider': {'name': 'fakeProvider'}, - 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], - '_ldMeta': {'enabled': true, 'variationKey': 'abcd', 'version': 1}, + model: { name: 'fakeModel', parameters: { temperature: 0.5, maxTokens: 4096 }, + custom: { 'extra-attribute': 'value' } }, + provider: { name: 'fakeProvider' }, + messages: [{ role: 'system', content: 'Hello, {{name}}!' }], + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } }, - "green", + 'green' ) .variation_for_all(0) ) end let(:ld_client) do - config = LaunchDarkly::Config.new(data_source: td, send_events: false) - LaunchDarkly::LDClient.new('sdk-key', config) + config = LaunchDarkly::Config.new(data_source: td, send_events: false) + LaunchDarkly::LDClient.new('sdk-key', config) end - let(:context) { instance_double(LaunchDarkly::LDContext) } + let(:context) { LaunchDarkly::LDContext.create({ key: 'test-user', kind: 'user' }) } let(:tracker) do described_class.new( ld_client: ld_client, @@ -224,28 +225,28 @@ expect(result).to eq(openai_result) end - # it 'tracks error for failed operation' do - # expect(ld_client).to receive(:track).with( - # '$ld:ai:duration:total', - # context, - # { variationKey: 'test-variation', configKey: 'test-config' }, - # kind_of(Integer) - # ) - # expect(ld_client).to receive(:track).with( - # '$ld:ai:generation', - # context, - # { variationKey: 'test-variation', configKey: 'test-config' }, - # 1 - # ) - # expect(ld_client).to receive(:track).with( - # '$ld:ai:generation:error', - # context, - # { variationKey: 'test-variation', configKey: 'test-config' }, - # 1 - # ) + it 'tracks error for failed operation' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + kind_of(Integer) + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + { variationKey: 'test-variation', configKey: 'test-config' }, + 1 + ) - # expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') - # end + expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') + end end describe '#track_bedrock_metrics' do @@ -309,22 +310,22 @@ tracker.track_time_to_first_token(50) expect(tracker.get_summary).to eq({ - duration: 100, - feedback: :positive, - tokens: { total: 100, input: 50, output: 50 }, - success: true, - time_to_first_token: 50 - }) + duration: 100, + feedback: :positive, + tokens: { total: 100, input: 50, output: 50 }, + success: true, + time_to_first_token: 50 + }) end it 'returns nil for untracked metrics' do expect(tracker.get_summary).to eq({ - duration: nil, - feedback: nil, - tokens: nil, - success: nil, - time_to_first_token: nil - }) + duration: nil, + feedback: nil, + tokens: nil, + success: nil, + time_to_first_token: nil + }) end end -end \ No newline at end of file +end From ec5409382c66e655ffa38c7cd7c6d832290844c0 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 5 Jun 2025 17:12:35 +0000 Subject: [PATCH 08/20] fixing issues in the tracker --- lib/ldclient-ai/ld_ai_config_tracker.rb | 157 ++++++++++++------------ spec/ld_ai_config_tracker_spec.rb | 104 ++++++++-------- 2 files changed, 130 insertions(+), 131 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb index c830066..630a187 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -20,7 +20,7 @@ def initialize(total: nil, input: nil, output: nil) # Summary of metrics which have been tracked. class LDAIMetricSummary - attr_reader :duration, :success, :feedback, :usage, :time_to_first_token + attr_accessor :duration, :success, :feedback, :usage, :time_to_first_token def initialize @duration = nil @@ -33,7 +33,7 @@ def initialize # The LDAIConfigTracker class is used to track AI configuration usage. class LDAIConfigTracker - attr_reader :ld_client, :config_key, :context, :variation_key, :version + attr_reader :ld_client, :config_key, :context, :variation_key, :version, :summary def initialize(ld_client:, variation_key:, config_key:, version:, context:) @ld_client = ld_client @@ -47,11 +47,11 @@ def initialize(ld_client:, variation_key:, config_key:, version:, context:) # Track the duration of an AI operation # @param duration [Integer] The duration in milliseconds def track_duration(duration) - @duration = duration + @summary.duration = duration @ld_client.track( '$ld:ai:duration:total', @context, - { variationKey: @variation_key, configKey: @config_key }, + flag_data, duration ) end @@ -61,106 +61,113 @@ def track_duration(duration) # @return The result of the block def track_duration_of start_time = Time.now - result = yield + yield + ensure duration = ((Time.now - start_time) * 1000).to_i track_duration(duration) - result + end + + # Track time to first token + # @param duration [Integer] The duration in milliseconds + def track_time_to_first_token(time_to_first_token) + @summary.time_to_first_token = time_to_first_token + @ld_client.track( + '$ld:ai:tokens:ttf', + @context, + flag_data, + time_to_first_token + ) end # Track user feedback # @param kind [Symbol] The kind of feedback (:positive or :negative) def track_feedback(kind:) - @feedback = kind + @summary.feedback = kind event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' @ld_client.track( event_name, @context, - { variationKey: @variation_key, configKey: @config_key }, + flag_data, 1 ) end # Track a successful AI generation def track_success - @success = true + @summary.success = true @ld_client.track( '$ld:ai:generation', @context, - { variationKey: @variation_key, configKey: @config_key }, + flag_data, 1 ) @ld_client.track( '$ld:ai:generation:success', @context, - { variationKey: @variation_key, configKey: @config_key }, + flag_data, 1 ) end # Track an error in AI generation def track_error - @success = false + @summary.success = false @ld_client.track( '$ld:ai:generation', @context, - { variationKey: @variation_key, configKey: @config_key }, + flag_data, 1 ) @ld_client.track( '$ld:ai:generation:error', @context, - { variationKey: @variation_key, configKey: @config_key }, + flag_data, 1 ) end # Track token usage - # @param total [Integer] Total number of tokens - # @param input [Integer] Number of input tokens - # @param output [Integer] Number of output tokens - def track_tokens(total: nil, input: nil, output: nil) - @tokens = { total: total, input: input, output: output } - if total + # @param token_usage [TokenUsage] An object containing token usage details + def track_tokens(token_usage) + @summary.usage = token_usage + if token_usage.total.positive? @ld_client.track( '$ld:ai:tokens:total', @context, - { variationKey: @variation_key, configKey: @config_key }, - total + flag_data, + token_usage.total ) end - if input + if token_usage.input.positive? @ld_client.track( '$ld:ai:tokens:input', @context, - { variationKey: @variation_key, configKey: @config_key }, - input + flag_data, + token_usage.input ) end - return unless output + return unless token_usage.output.positive? @ld_client.track( '$ld:ai:tokens:output', @context, - { variationKey: @variation_key, configKey: @config_key }, - output + flag_data, + token_usage.output ) end - # Track OpenAI metrics - # @yield The block to measure - # @return The result of the block - def track_openai_metrics - start_time = Time.now - result = yield - duration = ((Time.now - start_time) * 1000).to_i - track_duration(duration) + # Track OpenAI-specific operations. + # This method tracks the duration, token usage, and success/error status. + # If the provided block raises, this method will also raise. + # A failed operation will not have any token usage data. + # + # @yield The block to track. + # @return The result of the tracked block. + def track_openai_metrics(&block) + result = track_duration_of(&block) track_success - if result.usage - track_tokens( - total: result.usage.total_tokens, - input: result.usage.prompt_tokens, - output: result.usage.completion_tokens - ) + if result.respond_to?(:usage) && result.usage.respond_to?(:to_h) + track_tokens(openai_to_token_usage(result.usage.to_h)) end result rescue StandardError => e @@ -168,46 +175,44 @@ def track_openai_metrics raise e end - # Track Bedrock metrics - # @yield The block to measure - # @return The result of the block - def track_bedrock_metrics - start_time = Time.now - result = yield - duration = ((Time.now - start_time) * 1000).to_i - track_duration(duration) - if result.usage - track_tokens( - total: result.usage.total_tokens, - input: result.usage.input_tokens, - output: result.usage.output_tokens - ) + # Track AWS Bedrock conversation operations. + # This method tracks the duration, token usage, and success/error status. + # @param res [Hash] Response hash from Bedrock. + # @return [Hash] The original response hash. + def track_bedrock_converse_metrics(res) + status_code = res.dig('$metadata', 'httpStatusCode') || 0 + if status_code == 200 + track_success + elsif status_code >= 400 + track_error end - result + + track_duration(res['metrics']['latencyMs']) if res.dig('metrics', 'latencyMs') + track_tokens(bedrock_to_token_usage(res['usage'])) if res['usage'] + + res end - # Track time to first token - # @param duration [Integer] The duration in milliseconds - def track_time_to_first_token(duration) - @time_to_first_token = duration - @ld_client.track( - '$ld:ai:tokens:ttf', - @context, - { variationKey: @variation_key, configKey: @config_key }, - duration + private + + def flag_data + { variationKey: @variation_key, configKey: @config_key, version: @version } + end + + def openai_to_token_usage(usage) + TokenUsage.new( + total: usage[:total_tokens] || usage['total_tokens'], + input: usage[:prompt_tokens] || usage['prompt_tokens'], + output: usage[:completion_tokens] || usage['completion_tokens'] ) end - # Get a summary of all tracked metrics - # @return [Hash] A hash containing all tracked metrics - def get_summary - { - duration: @duration, - feedback: @feedback, - tokens: @tokens, - success: @success, - time_to_first_token: @time_to_first_token - } + def bedrock_to_token_usage(usage) + TokenUsage.new( + total: usage[:total_tokens] || usage['total_tokens'], + input: usage[:input_tokens] || usage['input_tokens'], + output: usage[:output_tokens] || usage['output_tokens'] + ) end end end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index 9dd0aaf..1ce68e2 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -42,7 +42,7 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 100 ) tracker.track_duration(100) @@ -54,7 +54,7 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, kind_of(Integer) ) result = tracker.track_duration_of { 'test result' } @@ -67,7 +67,7 @@ expect(ld_client).to receive(:track).with( '$ld:ai:feedback:user:positive', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) tracker.track_feedback(kind: :positive) @@ -77,7 +77,7 @@ expect(ld_client).to receive(:track).with( '$ld:ai:feedback:user:negative', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) tracker.track_feedback(kind: :negative) @@ -89,13 +89,13 @@ expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation:success', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) tracker.track_success @@ -107,13 +107,13 @@ expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation:error', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) tracker.track_error @@ -125,49 +125,43 @@ expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 100 + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + 300 ) - tracker.track_tokens(total: 100) - end - - it 'tracks input tokens' do expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 50 + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + 200 ) - tracker.track_tokens(input: 50) - end - - it 'tracks output tokens' do expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config' }, - 50 + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + 100 ) - tracker.track_tokens(output: 50) + tokens = LaunchDarkly::AI::TokenUsage.new(total: 300, input: 200, output: 100) + tracker.track_tokens(tokens) + expect(tracker.summary.usage).to eq(tokens) end it 'tracks all token types' do expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 100 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 50 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 50 ) tracker.track_tokens(total: 100, input: 50, output: 50) @@ -187,37 +181,37 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, kind_of(Integer) ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 100 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 50 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 50 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation:success', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) @@ -229,19 +223,19 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, kind_of(Integer) ) expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation:error', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 1 ) @@ -262,25 +256,25 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, kind_of(Integer) ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 100 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 50 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 50 ) @@ -294,14 +288,14 @@ expect(ld_client).to receive(:track).with( '$ld:ai:tokens:ttf', context, - { variationKey: 'test-variation', configKey: 'test-config' }, + { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, 100 ) tracker.track_time_to_first_token(100) end end - describe '#get_summary' do + describe '#summary' do it 'returns a summary of tracked metrics' do tracker.track_duration(100) tracker.track_feedback(kind: :positive) @@ -309,23 +303,23 @@ tracker.track_success tracker.track_time_to_first_token(50) - expect(tracker.get_summary).to eq({ - duration: 100, - feedback: :positive, - tokens: { total: 100, input: 50, output: 50 }, - success: true, - time_to_first_token: 50 - }) + expect(tracker.summary).to eq({ + duration: 100, + feedback: :positive, + tokens: { total: 100, input: 50, output: 50 }, + success: true, + time_to_first_token: 50 + }) end it 'returns nil for untracked metrics' do - expect(tracker.get_summary).to eq({ - duration: nil, - feedback: nil, - tokens: nil, - success: nil, - time_to_first_token: nil - }) + expect(tracker.summary).to eq({ + duration: nil, + feedback: nil, + tokens: nil, + success: nil, + time_to_first_token: nil + }) end end end From 1d7d6f710f5dd3575a6e60f26f057a7b43ac416b Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 6 Jun 2025 21:48:05 +0000 Subject: [PATCH 09/20] fix remaining tests for the tracker --- lib/ldclient-ai/ld_ai_client.rb | 1 - lib/ldclient-ai/ld_ai_config_tracker.rb | 10 +- spec/ld_ai_config_tracker_spec.rb | 348 ++++++++++++++++-------- 3 files changed, 236 insertions(+), 123 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb index 3461573..8f1d099 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -132,7 +132,6 @@ def config(config_key, context, default_value = nil, variables = nil) context, default_value.respond_to?(:to_h) ? default_value.to_h : nil ) - puts("Config variation: #{variation.inspect}") variables ||= {} variables[:ldctx] = context.to_h diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb index 630a187..90ff060 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -166,9 +166,7 @@ def track_tokens(token_usage) def track_openai_metrics(&block) result = track_duration_of(&block) track_success - if result.respond_to?(:usage) && result.usage.respond_to?(:to_h) - track_tokens(openai_to_token_usage(result.usage.to_h)) - end + track_tokens(openai_to_token_usage(result[:usage])) if result[:usage] result rescue StandardError => e track_error @@ -180,15 +178,15 @@ def track_openai_metrics(&block) # @param res [Hash] Response hash from Bedrock. # @return [Hash] The original response hash. def track_bedrock_converse_metrics(res) - status_code = res.dig('$metadata', 'httpStatusCode') || 0 + status_code = res.dig(:'$metadata', :httpStatusCode) || 0 if status_code == 200 track_success elsif status_code >= 400 track_error end - track_duration(res['metrics']['latencyMs']) if res.dig('metrics', 'latencyMs') - track_tokens(bedrock_to_token_usage(res['usage'])) if res['usage'] + track_duration(res[:metrics][:latency_ms]) if res.dig(:metrics, :latency_ms) + track_tokens(bedrock_to_token_usage(res[:usage])) if res[:usage] res end diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index 1ce68e2..fba410f 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -26,14 +26,15 @@ LaunchDarkly::LDClient.new('sdk-key', config) end - let(:context) { LaunchDarkly::LDContext.create({ key: 'test-user', kind: 'user' }) } + let(:context) { LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) } + let(:tracker_flag_data) { { variationKey: 'test-variation', configKey: 'test-config', version: 1 } } let(:tracker) do described_class.new( ld_client: ld_client, - config_key: 'test-config', + config_key: tracker_flag_data[:configKey], context: context, - variation_key: 'test-variation', - version: 1 + variation_key: tracker_flag_data[:variationKey], + version: tracker_flag_data[:version] ) end @@ -42,10 +43,11 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 100 ) tracker.track_duration(100) + expect(tracker.summary.duration).to eq(100) end end @@ -54,244 +56,360 @@ expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, kind_of(Integer) ) - result = tracker.track_duration_of { 'test result' } - expect(result).to eq('test result') + result = tracker.track_duration_of { sleep(0.01) } + expect(result).to be_within(10).of(0) # Allow some tolerance for sleep timing + expect(tracker.summary.duration).to be_within(1000).of(10) # Allow some tolerance for sleep timing end - end - describe '#track_feedback' do - it 'tracks positive feedback' do + it 'tracks duration even when an exception is raised' do expect(ld_client).to receive(:track).with( - '$ld:ai:feedback:user:positive', + '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 1 + tracker_flag_data, + kind_of(Integer) ) - tracker.track_feedback(kind: :positive) + + expect do + tracker.track_duration_of do + sleep(0.01) + raise 'Something went wrong' + end + end.to raise_error('Something went wrong') + expect(tracker.summary.duration).to be_within(1000).of(10) # Allow some tolerance for sleep timing end + end - it 'tracks negative feedback' do + describe '#track_time_to_first_token' do + it 'tracks time to first token' do expect(ld_client).to receive(:track).with( - '$ld:ai:feedback:user:negative', + '$ld:ai:tokens:ttf', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 1 + tracker_flag_data, + 100 ) - tracker.track_feedback(kind: :negative) + tracker.track_time_to_first_token(100) + expect(tracker.summary.time_to_first_token).to eq(100) end end - describe '#track_success' do - it 'tracks generation and success events' do + describe '#track_tokens' do + it 'tracks token usage' do expect(ld_client).to receive(:track).with( - '$ld:ai:generation', + '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 1 + tracker_flag_data, + 300 ) expect(ld_client).to receive(:track).with( - '$ld:ai:generation:success', + '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 1 + tracker_flag_data, + 200 ) - tracker.track_success + expect(ld_client).to receive(:track).with( + '$ld:ai:tokens:output', + context, + tracker_flag_data, + 100 + ) + tokens = LaunchDarkly::AI::TokenUsage.new(total: 300, input: 200, output: 100) + tracker.track_tokens(tokens) + expect(tracker.summary.usage).to eq(tokens) end end - describe '#track_error' do - it 'tracks generation and error events' do + describe '#track_bedrock_metrics' do + it 'tracks duration and tokens' do + # Verify the $metadata field in bedrock result. I don't see anything like this in the docs. + bedrock_result = { + '$metadata': { httpStatusCode: 200 }, + usage: { + total_tokens: 300, + input_tokens: 200, + output_tokens: 100 + }, + metrics: { + latency_ms: 50 + } + } expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 1 ) expect(ld_client).to receive(:track).with( - '$ld:ai:generation:error', + '$ld:ai:generation:success', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 1 ) - tracker.track_error - end - end - - describe '#track_tokens' do - it 'tracks total tokens' do + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + tracker_flag_data, + kind_of(Integer) + ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 300 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 200 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 100 ) - tokens = LaunchDarkly::AI::TokenUsage.new(total: 300, input: 200, output: 100) - tracker.track_tokens(tokens) - expect(tracker.summary.usage).to eq(tokens) + + result = tracker.track_bedrock_converse_metrics(bedrock_result) + expect(result).to eq(bedrock_result) + expect(tracker.summary).to be_a(LaunchDarkly::AI::LDAIMetricSummary) + expect(tracker.summary.usage).to be_a(LaunchDarkly::AI::TokenUsage) + expect(tracker.summary.usage.total).to eq(300) + expect(tracker.summary.usage.input).to eq(200) + expect(tracker.summary.usage.output).to eq(100) + expect(tracker.summary.duration).to eq(50) + expect(tracker.summary.success).to be true end - it 'tracks all token types' do + it 'tracks duration and tokens with error' do + # Verify the $metadata field in bedrock result. I don't see anything like this in the docs. + bedrock_result = { + '$metadata': { httpStatusCode: 400 }, + usage: { + total_tokens: 300, + input_tokens: 200, + output_tokens: 100 + }, + metrics: { + latency_ms: 50 + } + } + expect(ld_client).to receive(:track).with( + '$ld:ai:generation', + context, + tracker_flag_data, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + tracker_flag_data, + 1 + ) + expect(ld_client).to receive(:track).with( + '$ld:ai:duration:total', + context, + tracker_flag_data, + kind_of(Integer) + ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 100 + tracker_flag_data, + 300 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 50 + tracker_flag_data, + 200 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 50 + tracker_flag_data, + 100 ) - tracker.track_tokens(total: 100, input: 50, output: 50) + + result = tracker.track_bedrock_converse_metrics(bedrock_result) + expect(result).to eq(bedrock_result) + expect(tracker.summary).to be_a(LaunchDarkly::AI::LDAIMetricSummary) + expect(tracker.summary.usage).to be_a(LaunchDarkly::AI::TokenUsage) + expect(tracker.summary.usage.total).to eq(300) + expect(tracker.summary.usage.input).to eq(200) + expect(tracker.summary.usage.output).to eq(100) + expect(tracker.summary.duration).to eq(50) + expect(tracker.summary.success).to be false end end describe '#track_openai_metrics' do let(:openai_result) do - double('OpenAIResult', usage: double( - total_tokens: 100, - prompt_tokens: 50, - completion_tokens: 50 - )) + { + usage: { + total_tokens: 300, + prompt_tokens: 200, + completion_tokens: 100 + } + } end it 'tracks duration and tokens for successful operation' do expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, kind_of(Integer) ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 100 + tracker_flag_data, + 300 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:input', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 50 + tracker_flag_data, + 200 ) expect(ld_client).to receive(:track).with( '$ld:ai:tokens:output', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 50 + tracker_flag_data, + 100 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 1 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation:success', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 1 ) result = tracker.track_openai_metrics { openai_result } expect(result).to eq(openai_result) + expect(tracker.summary.usage.total).to eq(300) + expect(tracker.summary.usage.input).to eq(200) + expect(tracker.summary.usage.output).to eq(100) + expect(tracker.summary.success).to be true end it 'tracks error for failed operation' do expect(ld_client).to receive(:track).with( '$ld:ai:duration:total', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, kind_of(Integer) ) expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 1 ) expect(ld_client).to receive(:track).with( '$ld:ai:generation:error', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, + tracker_flag_data, 1 ) expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') + expect(tracker.summary.usage).to eq(nil) end end - describe '#track_bedrock_metrics' do - let(:bedrock_result) do - double('BedrockResult', usage: double( - total_tokens: 100, - input_tokens: 50, - output_tokens: 50 - )) + describe '#track_feedback' do + it 'tracks positive feedback' do + expect(ld_client).to receive(:track).with( + '$ld:ai:feedback:user:positive', + context, + tracker_flag_data, + 1 + ) + tracker.track_feedback(kind: :positive) end - it 'tracks duration and tokens' do + it 'tracks negative feedback' do expect(ld_client).to receive(:track).with( - '$ld:ai:duration:total', + '$ld:ai:feedback:user:negative', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - kind_of(Integer) + tracker_flag_data, + 1 ) + tracker.track_feedback(kind: :negative) + end + end + + describe '#track_success' do + it 'tracks generation and success events' do expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:total', + '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 100 + tracker_flag_data, + 1 ) expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:input', + '$ld:ai:generation:success', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 50 + tracker_flag_data, + 1 ) + tracker.track_success + expect(tracker.summary.success).to be true + end + end + + describe '#track_error' do + it 'tracks generation and error events' do expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:output', + '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 50 + tracker_flag_data, + 1 ) - - result = tracker.track_bedrock_metrics { bedrock_result } - expect(result).to eq(bedrock_result) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + tracker_flag_data, + 1 + ) + tracker.track_error + expect(tracker.summary.success).to be false end - end - describe '#track_time_to_first_token' do - it 'tracks time to first token' do + it 'overwrites success with error if both are tracked' do expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:ttf', + '$ld:ai:generation', context, - { variationKey: 'test-variation', configKey: 'test-config', version: 1 }, - 100 + tracker_flag_data, + 1 + ).twice + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:success', + context, + tracker_flag_data, + 1 ) - tracker.track_time_to_first_token(100) + expect(ld_client).to receive(:track).with( + '$ld:ai:generation:error', + context, + tracker_flag_data, + 1 + ) + + tracker.track_success + expect(tracker.summary.success).to be true + tracker.track_error + expect(tracker.summary.success).to be false end end @@ -299,27 +417,25 @@ it 'returns a summary of tracked metrics' do tracker.track_duration(100) tracker.track_feedback(kind: :positive) - tracker.track_tokens(total: 100, input: 50, output: 50) + tracker.track_tokens(LaunchDarkly::AI::TokenUsage.new(total: 100, input: 50, output: 50)) tracker.track_success tracker.track_time_to_first_token(50) - expect(tracker.summary).to eq({ - duration: 100, - feedback: :positive, - tokens: { total: 100, input: 50, output: 50 }, - success: true, - time_to_first_token: 50 - }) + expect(tracker.summary.duration).to eq(100) + expect(tracker.summary.feedback).to eq(:positive) + expect(tracker.summary.usage.total).to eq(100) + expect(tracker.summary.usage.input).to eq(50) + expect(tracker.summary.usage.output).to eq(50) + expect(tracker.summary.success).to be true + expect(tracker.summary.time_to_first_token).to eq(50) end it 'returns nil for untracked metrics' do - expect(tracker.summary).to eq({ - duration: nil, - feedback: nil, - tokens: nil, - success: nil, - time_to_first_token: nil - }) + expect(tracker.summary.duration).to be_nil + expect(tracker.summary.feedback).to be_nil + expect(tracker.summary.usage).to be_nil + expect(tracker.summary.success).to be_nil + expect(tracker.summary.time_to_first_token).to be_nil end end end From 6399b69828f7c1db3dd391bf99431c7edac5ae54 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 10 Jun 2025 15:39:08 +0000 Subject: [PATCH 10/20] gemfile updates --- launchdarkly-server-sdk-ai.gemspec | 35 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index 101692d..bc7e2d7 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -1,31 +1,32 @@ # frozen_string_literal: true -require_relative "lib/ldclient-ai/version" +require_relative 'lib/ldclient-ai/version' Gem::Specification.new do |spec| - spec.name = "launchdarkly-server-sdk-ai" + spec.name = 'launchdarkly-server-sdk-ai' spec.version = LaunchDarkly::AI::VERSION - spec.authors = ["LaunchDarkly"] - spec.email = ["team@launchdarkly.com"] - spec.summary = "LaunchDarkly AI SDK for Ruby" - spec.description = "LaunchDarkly SDK AI Configs integration for the Ruby server side SDK" - spec.license = "Apache-2.0" - spec.homepage = "https://github.com/launchdarkly/ruby-server-sdk-ai" - spec.metadata["source_code_uri"] = "https://github.com/launchdarkly/ruby-server-sdk-ai" - spec.metadata["changelog_uri"] = "https://github.com/launchdarkly/ruby-server-sdk-ai/blob/main/CHANGELOG.md" + spec.authors = ['LaunchDarkly'] + spec.email = ['team@launchdarkly.com'] + spec.summary = 'LaunchDarkly AI SDK for Ruby' + spec.description = 'LaunchDarkly SDK AI Configs integration for the Ruby server side SDK' + spec.license = 'Apache-2.0' + spec.homepage = 'https://github.com/launchdarkly/ruby-server-sdk-ai' + spec.metadata['source_code_uri'] = 'https://github.com/launchdarkly/ruby-server-sdk-ai' + spec.metadata['changelog_uri'] = 'https://github.com/launchdarkly/ruby-server-sdk-ai/blob/main/CHANGELOG.md' - spec.files = Dir["{lib}/**/*.rb", "bin/*", "LICENSE", "*.md"] + spec.files = Dir['{lib}/**/*.rb', 'bin/*', 'LICENSE', '*.md'] spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } - spec.require_paths = ["lib"] - spec.required_ruby_version = ">= 3.0.0" + spec.require_paths = ['lib'] + spec.required_ruby_version = '>= 3.0.0' - spec.add_dependency "launchdarkly-server-sdk", "~> 8.4.0" - spec.add_dependency "logger" - spec.add_dependency "mustache", "~> 1.1" + spec.add_dependency 'launchdarkly-server-sdk', '~> 8.5.0' + spec.add_dependency 'logger' + spec.add_dependency 'mustache', '~> 1.1' spec.add_development_dependency 'bundler', '~> 2.0' + spec.add_development_dependency 'debug', '~> 1.0' spec.add_development_dependency 'rake', '~> 13.0' spec.add_development_dependency 'rspec', '~> 3.0' spec.add_development_dependency 'rubocop', '~> 1.0' spec.add_development_dependency 'rubocop-rspec', '~> 2.0' -end \ No newline at end of file +end From ec553d5c1a101461afaa0a29198597f9945aa79e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 21:31:35 +0000 Subject: [PATCH 11/20] Address code review feedback - Add role validation to LDMessage class with VALID_ROLES constant - Improve nil safety in track_tokens method using safe navigation operator - Remove TODO comment by implementing proper role validation - Add type checking for TokenUsage parameter in track_tokens method Co-Authored-By: jbailey@launchdarkly.com --- lib/ldclient-ai/ld_ai_client.rb | 5 ++++- lib/ldclient-ai/ld_ai_config_tracker.rb | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/ld_ai_client.rb index 8f1d099..ae32c42 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/ld_ai_client.rb @@ -13,8 +13,11 @@ module AI class LDMessage attr_reader :role, :content - # TODO: Do we need to validate the role to only be 'system', 'user', or 'assistant'? + VALID_ROLES = %w[system user assistant].freeze + def initialize(role, content) + raise ArgumentError, "Invalid role: #{role}. Must be one of: #{VALID_ROLES.join(', ')}" unless VALID_ROLES.include?(role.to_s) + @role = role @content = content end diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/ld_ai_config_tracker.rb index 90ff060..2dd0e4c 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/ld_ai_config_tracker.rb @@ -129,8 +129,10 @@ def track_error # Track token usage # @param token_usage [TokenUsage] An object containing token usage details def track_tokens(token_usage) + return unless token_usage.is_a?(TokenUsage) + @summary.usage = token_usage - if token_usage.total.positive? + if token_usage.total&.positive? @ld_client.track( '$ld:ai:tokens:total', @context, @@ -138,7 +140,7 @@ def track_tokens(token_usage) token_usage.total ) end - if token_usage.input.positive? + if token_usage.input&.positive? @ld_client.track( '$ld:ai:tokens:input', @context, @@ -146,7 +148,7 @@ def track_tokens(token_usage) token_usage.input ) end - return unless token_usage.output.positive? + return unless token_usage.output&.positive? @ld_client.track( '$ld:ai:tokens:output', From 1bb2f01b690160b6817cbf7f9005c788fd0f8cf9 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 11 Jun 2025 13:48:02 +0000 Subject: [PATCH 12/20] Addressing feedback --- Gemfile | 11 +--- launchdarkly-server-sdk-ai.gemspec | 2 + lib/ldclient-ai.rb | 4 +- .../{ld_ai_client.rb => client.rb} | 42 ++++++++++----- ...ai_config_tracker.rb => config_tracker.rb} | 52 ++++++++++++++----- spec/ld_ai_client_spec.rb | 44 ++++++++-------- spec/ld_ai_config_tracker_spec.rb | 2 +- 7 files changed, 97 insertions(+), 60 deletions(-) rename lib/ldclient-ai/{ld_ai_client.rb => client.rb} (87%) rename lib/ldclient-ai/{ld_ai_config_tracker.rb => config_tracker.rb} (91%) diff --git a/Gemfile b/Gemfile index 46d9bab..6d92065 100644 --- a/Gemfile +++ b/Gemfile @@ -1,15 +1,6 @@ # frozen_string_literal: true -source "https://rubygems.org" +source 'https://rubygems.org' # Specify your gem's dependencies in launchdarkly-server-sdk-ai.gemspec gemspec - -gem "rake", "~> 13.0" - -gem "rspec", "~> 3.0" - -gem "rubocop", "~> 1.21" -gem "rubocop-performance", "~> 1.15" -gem "rubocop-rake", "~> 0.6" -gem "rubocop-rspec", "~> 2.27" diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index bc7e2d7..0a5dc74 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -28,5 +28,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'rake', '~> 13.0' spec.add_development_dependency 'rspec', '~> 3.0' spec.add_development_dependency 'rubocop', '~> 1.0' + spec.add_development_dependency 'rubocop-performance', '~> 1.15' + spec.add_development_dependency 'rubocop-rake', '~> 0.6' spec.add_development_dependency 'rubocop-rspec', '~> 2.0' end diff --git a/lib/ldclient-ai.rb b/lib/ldclient-ai.rb index 255a716..c8a21fe 100644 --- a/lib/ldclient-ai.rb +++ b/lib/ldclient-ai.rb @@ -4,8 +4,8 @@ require 'mustache' require 'ldclient-ai/version' -require 'ldclient-ai/ld_ai_client' -require 'ldclient-ai/ld_ai_config_tracker' +require 'ldclient-ai/client' +require 'ldclient-ai/config_tracker' module LaunchDarkly # diff --git a/lib/ldclient-ai/ld_ai_client.rb b/lib/ldclient-ai/client.rb similarity index 87% rename from lib/ldclient-ai/ld_ai_client.rb rename to lib/ldclient-ai/client.rb index ae32c42..dd385b2 100644 --- a/lib/ldclient-ai/ld_ai_client.rb +++ b/lib/ldclient-ai/client.rb @@ -2,22 +2,20 @@ require 'ldclient-rb' require 'mustache' -require_relative 'ld_ai_config_tracker' +require_relative 'config_tracker' module LaunchDarkly # # Namespace for the LaunchDarkly AI SDK. # module AI + # # Holds AI role and content. + # class LDMessage attr_reader :role, :content - VALID_ROLES = %w[system user assistant].freeze - def initialize(role, content) - raise ArgumentError, "Invalid role: #{role}. Must be one of: #{VALID_ROLES.join(', ')}" unless VALID_ROLES.include?(role.to_s) - @role = role @content = content end @@ -30,9 +28,11 @@ def to_h end end + # # The ModelConfig class represents an AI model configuration. + # class ModelConfig - attr_reader :name, :parameters, :custom + attr_reader :name def initialize(name:, parameters: {}, custom: {}) @name = name @@ -40,6 +40,7 @@ def initialize(name:, parameters: {}, custom: {}) @custom = custom end + # # Retrieve model-specific parameters. # # Accessing a named, typed attribute (e.g. name) will result in the call @@ -47,19 +48,22 @@ def initialize(name:, parameters: {}, custom: {}) # # @param key [String] The parameter key to retrieve # @return [Object] The parameter value or nil if not found - def get_parameter(key) + # + def parameter(key) return @name if key == 'name' - return nil if @parameters.nil? + return nil unless @parameters.is_a?(Hash) @parameters[key] end + # # Retrieve customer provided data. # # @param key [String] The custom key to retrieve # @return [Object] The custom value or nil if not found - def get_custom(key) - return nil if @custom.nil? + # + def custom(key) + return nil unless @custom.is_a?(Hash) @custom[key] end @@ -73,7 +77,9 @@ def to_h end end + # # Configuration related to the provider. + # class ProviderConfig attr_reader :name @@ -88,7 +94,9 @@ def to_h end end + # # The AIConfig class represents an AI configuration. + # class AIConfig attr_reader :enabled, :messages, :variables, :tracker, :model, :provider @@ -112,7 +120,9 @@ def to_h end end + # # The LDAIClient class is the main entry point for the LaunchDarkly AI SDK. + # class LDAIClient attr_reader :logger, :ld_client @@ -123,12 +133,15 @@ def initialize(ld_client) @logger = LaunchDarkly::AI.default_logger end + # # Retrieves the AIConfig + # # @param config_key [String] The key of the configuration flag # @param context [LDContext] The context used when evaluating the flag # @param default_value [AIConfig] The default value to use if the flag is not found # @param variables [Hash] Optional variables for rendering messages # @return [AIConfig] An AIConfig instance containing the configuration data + # def config(config_key, context, default_value = nil, variables = nil) variation = @ld_client.variation( config_key, @@ -139,7 +152,10 @@ def config(config_key, context, default_value = nil, variables = nil) variables ||= {} variables[:ldctx] = context.to_h - messages = variation.fetch(:messages, nil) + # + # Process messages and provider configuration + # + messages = variation[:messages] if messages.is_a?(Array) && messages.all? { |msg| msg.is_a?(Hash) } messages = messages.map do |message| message[:content] = Mustache.render(message[:content], variables) if message[:content].is_a?(String) @@ -147,11 +163,11 @@ def config(config_key, context, default_value = nil, variables = nil) end end - if (provider_config = variation.fetch(:provider, nil)) && provider_config.is_a?(Hash) + if (provider_config = variation[:provider]) && provider_config.is_a?(Hash) provider_config = ProviderConfig.new(provider_config.fetch(:name, '')) end - if (model = variation.fetch(:model, nil)) && model.is_a?(Hash) + if (model = variation[:model]) && model.is_a?(Hash) parameters = variation[:model][:parameters] custom = variation[:model][:custom] model = ModelConfig.new( diff --git a/lib/ldclient-ai/ld_ai_config_tracker.rb b/lib/ldclient-ai/config_tracker.rb similarity index 91% rename from lib/ldclient-ai/ld_ai_config_tracker.rb rename to lib/ldclient-ai/config_tracker.rb index 2dd0e4c..12560f3 100644 --- a/lib/ldclient-ai/ld_ai_config_tracker.rb +++ b/lib/ldclient-ai/config_tracker.rb @@ -4,13 +4,17 @@ module LaunchDarkly module AI + # # Tracks token usage for AI operations. + # class TokenUsage attr_reader :total, :input, :output + # # @param total [Integer] Total number of tokens used. # @param input [Integer] Number of tokens in the prompt. # @param output [Integer] Number of tokens in the completion. + # def initialize(total: nil, input: nil, output: nil) @total = total @input = input @@ -18,7 +22,9 @@ def initialize(total: nil, input: nil, output: nil) end end + # # Summary of metrics which have been tracked. + # class LDAIMetricSummary attr_accessor :duration, :success, :feedback, :usage, :time_to_first_token @@ -31,7 +37,9 @@ def initialize end end + # # The LDAIConfigTracker class is used to track AI configuration usage. + # class LDAIConfigTracker attr_reader :ld_client, :config_key, :context, :variation_key, :version, :summary @@ -44,8 +52,11 @@ def initialize(ld_client:, variation_key:, config_key:, version:, context:) @summary = LDAIMetricSummary.new end + # # Track the duration of an AI operation + # # @param duration [Integer] The duration in milliseconds + # def track_duration(duration) @summary.duration = duration @ld_client.track( @@ -56,9 +67,12 @@ def track_duration(duration) ) end + # # Track the duration of a block of code + # # @yield The block to measure # @return The result of the block + # def track_duration_of start_time = Time.now yield @@ -67,8 +81,11 @@ def track_duration_of track_duration(duration) end + # # Track time to first token + # # @param duration [Integer] The duration in milliseconds + # def track_time_to_first_token(time_to_first_token) @summary.time_to_first_token = time_to_first_token @ld_client.track( @@ -79,8 +96,11 @@ def track_time_to_first_token(time_to_first_token) ) end + # # Track user feedback + # # @param kind [Symbol] The kind of feedback (:positive or :negative) + # def track_feedback(kind:) @summary.feedback = kind event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' @@ -92,7 +112,9 @@ def track_feedback(kind:) ) end + # # Track a successful AI generation + # def track_success @summary.success = true @ld_client.track( @@ -109,7 +131,9 @@ def track_success ) end + # # Track an error in AI generation + # def track_error @summary.success = false @ld_client.track( @@ -126,13 +150,14 @@ def track_error ) end + # # Track token usage + # # @param token_usage [TokenUsage] An object containing token usage details + # def track_tokens(token_usage) - return unless token_usage.is_a?(TokenUsage) - @summary.usage = token_usage - if token_usage.total&.positive? + if token_usage.total.positive? @ld_client.track( '$ld:ai:tokens:total', @context, @@ -140,7 +165,7 @@ def track_tokens(token_usage) token_usage.total ) end - if token_usage.input&.positive? + if token_usage.input.positive? @ld_client.track( '$ld:ai:tokens:input', @context, @@ -148,7 +173,7 @@ def track_tokens(token_usage) token_usage.input ) end - return unless token_usage.output&.positive? + return unless token_usage.output.positive? @ld_client.track( '$ld:ai:tokens:output', @@ -158,6 +183,7 @@ def track_tokens(token_usage) ) end + # # Track OpenAI-specific operations. # This method tracks the duration, token usage, and success/error status. # If the provided block raises, this method will also raise. @@ -165,20 +191,24 @@ def track_tokens(token_usage) # # @yield The block to track. # @return The result of the tracked block. + # def track_openai_metrics(&block) result = track_duration_of(&block) track_success track_tokens(openai_to_token_usage(result[:usage])) if result[:usage] result - rescue StandardError => e + rescue StandardError track_error - raise e + raise end + # # Track AWS Bedrock conversation operations. # This method tracks the duration, token usage, and success/error status. + # # @param res [Hash] Response hash from Bedrock. # @return [Hash] The original response hash. + # def track_bedrock_converse_metrics(res) status_code = res.dig(:'$metadata', :httpStatusCode) || 0 if status_code == 200 @@ -193,13 +223,11 @@ def track_bedrock_converse_metrics(res) res end - private - - def flag_data + private def flag_data { variationKey: @variation_key, configKey: @config_key, version: @version } end - def openai_to_token_usage(usage) + private def openai_to_token_usage(usage) TokenUsage.new( total: usage[:total_tokens] || usage['total_tokens'], input: usage[:prompt_tokens] || usage['prompt_tokens'], @@ -207,7 +235,7 @@ def openai_to_token_usage(usage) ) end - def bedrock_to_token_usage(usage) + private def bedrock_to_token_usage(usage) TokenUsage.new( total: usage[:total_tokens] || usage['total_tokens'], input: usage[:input_tokens] || usage['input_tokens'], diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index fa053b9..a1537b8 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -101,17 +101,17 @@ it 'delegates to properties' do model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') - expect(model.get_parameter(:'extra-attribute')).to eq('value') - expect(model.get_parameter('non-existent')).to be_nil - expect(model.get_parameter('name')).to eq('fakeModel') + expect(model.parameter(:'extra-attribute')).to eq('value') + expect(model.parameter('non-existent')).to be_nil + expect(model.parameter('name')).to eq('fakeModel') end it 'handles custom attributes' do model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', custom: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') - expect(model.get_custom(:'extra-attribute')).to eq('value') - expect(model.get_custom('non-existent')).to be_nil - expect(model.get_custom('name')).to be_nil + expect(model.custom(:'extra-attribute')).to eq('value') + expect(model.custom('non-existent')).to be_nil + expect(model.custom('name')).to be_nil end end @@ -154,8 +154,8 @@ expect(config.model).not_to be_nil expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.5) - expect(config.model.get_parameter(:maxTokens)).to eq(4096) + expect(config.model.parameter(:temperature)).to eq(0.5) + expect(config.model.parameter(:maxTokens)).to eq(4096) end it 'interpolates variables in model config messages' do @@ -175,8 +175,8 @@ expect(config.model).not_to be_nil expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.5) - expect(config.model.get_parameter(:maxTokens)).to eq(4096) + expect(config.model.parameter(:temperature)).to eq(0.5) + expect(config.model.parameter(:maxTokens)).to eq(4096) end it 'returns config with messages interpolated as empty when no variables are provided' do @@ -196,8 +196,8 @@ expect(config.model).not_to be_nil expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.5) - expect(config.model.get_parameter(:maxTokens)).to eq(4096) + expect(config.model.parameter(:temperature)).to eq(0.5) + expect(config.model.parameter(:maxTokens)).to eq(4096) end it 'handles provider config correctly' do @@ -233,9 +233,9 @@ expect(config.model).not_to be_nil expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to be_nil - expect(config.model.get_parameter(:maxTokens)).to be_nil - expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') + expect(config.model.parameter(:temperature)).to be_nil + expect(config.model.parameter(:maxTokens)).to be_nil + expect(config.model.parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') end it 'interpolates variables from multiple contexts in messages using ldctx' do @@ -259,9 +259,9 @@ expect(config.model).not_to be_nil expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to be_nil - expect(config.model.get_parameter(:maxTokens)).to be_nil - expect(config.model.get_parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') + expect(config.model.parameter(:temperature)).to be_nil + expect(config.model.parameter(:maxTokens)).to be_nil + expect(config.model.parameter(:'extra-attribute')).to eq('I can be anything I set my mind/type to') end it 'handles multiple messages and variable interpolation' do @@ -283,8 +283,8 @@ expect(config.model).not_to be_nil expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.7) - expect(config.model.get_parameter(:maxTokens)).to eq(8192) + expect(config.model.parameter(:temperature)).to eq(0.7) + expect(config.model.parameter(:maxTokens)).to eq(8192) end it 'returns disabled config when flag is off' do @@ -300,8 +300,8 @@ expect(config.model).not_to be_nil expect(config.enabled).to be false expect(config.model.name).to eq('fakeModel') - expect(config.model.get_parameter(:temperature)).to eq(0.1) - expect(config.model.get_parameter(:maxTokens)).to be_nil + expect(config.model.parameter(:temperature)).to eq(0.1) + expect(config.model.parameter(:maxTokens)).to be_nil end it 'returns disabled config with nil model/messages/provider when initial config is disabled' do diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/ld_ai_config_tracker_spec.rb index fba410f..cbfb8e6 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/ld_ai_config_tracker_spec.rb @@ -123,7 +123,7 @@ describe '#track_bedrock_metrics' do it 'tracks duration and tokens' do - # Verify the $metadata field in bedrock result. I don't see anything like this in the docs. + # TODO: Verify the $metadata field in bedrock result. I don't see anything like this in the docs. bedrock_result = { '$metadata': { httpStatusCode: 200 }, usage: { From 3585f8420c730271207905aa3a7d6838b7277333 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 11 Jun 2025 17:25:11 +0000 Subject: [PATCH 13/20] don't modify passed in objects --- lib/ldclient-ai/client.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/ldclient-ai/client.rb b/lib/ldclient-ai/client.rb index dd385b2..9eb44d0 100644 --- a/lib/ldclient-ai/client.rb +++ b/lib/ldclient-ai/client.rb @@ -149,17 +149,21 @@ def config(config_key, context, default_value = nil, variables = nil) default_value.respond_to?(:to_h) ? default_value.to_h : nil ) - variables ||= {} - variables[:ldctx] = context.to_h + all_variables = variables ? variables.dup : {} + all_variables[:ldctx] = context.to_h # # Process messages and provider configuration # - messages = variation[:messages] - if messages.is_a?(Array) && messages.all? { |msg| msg.is_a?(Hash) } - messages = messages.map do |message| - message[:content] = Mustache.render(message[:content], variables) if message[:content].is_a?(String) - message + messages = nil + if variation[:messages].is_a?(Array) && variation[:messages].all? { |msg| msg.is_a?(Hash) } + messages = variation[:messages].map do |message| + next unless message[:content].is_a?(String) + + LDMessage.new( + message[:role], + Mustache.render(message[:content], variables) + ) end end From 6acd89b5c12b895501349973aafcc89a8305babf Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 16 Jun 2025 13:35:23 +0000 Subject: [PATCH 14/20] Use the proper variables for rendering --- launchdarkly-server-sdk-ai.gemspec | 2 +- lib/ldclient-ai/client.rb | 2 +- spec/ld_ai_client_spec.rb | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index 0a5dc74..b2d8694 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.required_ruby_version = '>= 3.0.0' - spec.add_dependency 'launchdarkly-server-sdk', '~> 8.5.0' + spec.add_dependency 'launchdarkly-server-sdk', '~> 8.5' spec.add_dependency 'logger' spec.add_dependency 'mustache', '~> 1.1' diff --git a/lib/ldclient-ai/client.rb b/lib/ldclient-ai/client.rb index 9eb44d0..62228b6 100644 --- a/lib/ldclient-ai/client.rb +++ b/lib/ldclient-ai/client.rb @@ -162,7 +162,7 @@ def config(config_key, context, default_value = nil, variables = nil) LDMessage.new( message[:role], - Mustache.render(message[:content], variables) + Mustache.render(message[:content], all_variables) ) end end diff --git a/spec/ld_ai_client_spec.rb b/spec/ld_ai_client_spec.rb index a1537b8..7d27840 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/ld_ai_client_spec.rb @@ -149,7 +149,7 @@ config = ai_client.config('missing-flag', context, default_config, variables) expect(config.messages).not_to be_nil expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.messages[0].content).to eq('Hello, World!') expect(config.enabled).to be true expect(config.model).not_to be_nil @@ -170,7 +170,7 @@ config = ai_client.config('model-config', context, default_value, variables) expect(config.messages).not_to be_nil expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, World!') + expect(config.messages[0].content).to eq('Hello, World!') expect(config.enabled).to be true expect(config.model).not_to be_nil @@ -191,7 +191,7 @@ expect(config.messages).not_to be_nil expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, !') + expect(config.messages[0].content).to eq('Hello, !') expect(config.enabled).to be true expect(config.model).not_to be_nil @@ -228,7 +228,7 @@ expect(config.messages).not_to be_nil expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, Sandy! Is your last name Beaches?') + expect(config.messages[0].content).to eq('Hello, Sandy! Is your last name Beaches?') expect(config.enabled).to be true expect(config.model).not_to be_nil @@ -254,7 +254,7 @@ expect(config.messages).not_to be_nil expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, Sandy! Do you work for LD?') + expect(config.messages[0].content).to eq('Hello, Sandy! Do you work for LD?') expect(config.enabled).to be true expect(config.model).not_to be_nil @@ -277,8 +277,8 @@ expect(config.messages).not_to be_nil expect(config.messages.length).to be > 0 - expect(config.messages[0][:content]).to eq('Hello, World!') - expect(config.messages[1][:content]).to eq('The day is, Monday!') + expect(config.messages[0].content).to eq('Hello, World!') + expect(config.messages[1].content).to eq('The day is, Monday!') expect(config.enabled).to be true expect(config.model).not_to be_nil From d8972ee3db3162a7325328b4e66476519b76acbc Mon Sep 17 00:00:00 2001 From: Jason Bailey Date: Mon, 16 Jun 2025 08:36:19 -0500 Subject: [PATCH 15/20] Apply suggestions from code review Co-authored-by: Matthew M. Keeler --- lib/ldclient-ai/client.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ldclient-ai/client.rb b/lib/ldclient-ai/client.rb index 62228b6..ad80f3d 100644 --- a/lib/ldclient-ai/client.rb +++ b/lib/ldclient-ai/client.rb @@ -47,7 +47,7 @@ def initialize(name:, parameters: {}, custom: {}) # being delegated to the appropriate property. # # @param key [String] The parameter key to retrieve - # @return [Object] The parameter value or nil if not found + # @return [Object, nil] The parameter value or nil if not found # def parameter(key) return @name if key == 'name' @@ -60,7 +60,7 @@ def parameter(key) # Retrieve customer provided data. # # @param key [String] The custom key to retrieve - # @return [Object] The custom value or nil if not found + # @return [Object, nil] The custom value or nil if not found # def custom(key) return nil unless @custom.is_a?(Hash) From f6df663efda2530ccb2ad9c9fe237e84194ce74d Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 16 Jun 2025 14:35:34 +0000 Subject: [PATCH 16/20] Update module naming to align more closely to package --- CONTRIBUTING.md | 4 +- examples/hello-bedrock/Gemfile | 6 + launchdarkly-server-sdk-ai.gemspec | 4 +- lib/launchdarkly-server-sdk-ai.rb | 30 +++ lib/launchdarkly_server_sdk_ai.rb | 3 - lib/ldclient-ai.rb | 28 -- lib/ldclient-ai/client.rb | 202 -------------- lib/ldclient-ai/config_tracker.rb | 247 ----------------- lib/ldclient-ai/version.rb | 7 - lib/server/ai/client.rb | 207 +++++++++++++++ lib/server/ai/config_tracker.rb | 249 ++++++++++++++++++ lib/server/ai/version.rb | 9 + spec/ldclient_ai_spec.rb | 15 -- spec/server/ai/ai_spec.rb | 15 ++ .../ai/client_spec.rb} | 62 ++--- .../ai/config_tracker_spec.rb} | 18 +- 16 files changed, 560 insertions(+), 546 deletions(-) create mode 100644 examples/hello-bedrock/Gemfile create mode 100644 lib/launchdarkly-server-sdk-ai.rb delete mode 100644 lib/launchdarkly_server_sdk_ai.rb delete mode 100644 lib/ldclient-ai.rb delete mode 100644 lib/ldclient-ai/client.rb delete mode 100644 lib/ldclient-ai/config_tracker.rb delete mode 100644 lib/ldclient-ai/version.rb create mode 100644 lib/server/ai/client.rb create mode 100644 lib/server/ai/config_tracker.rb create mode 100644 lib/server/ai/version.rb delete mode 100644 spec/ldclient_ai_spec.rb create mode 100644 spec/server/ai/ai_spec.rb rename spec/{ld_ai_client_spec.rb => server/ai/client_spec.rb} (84%) rename spec/{ld_ai_config_tracker_spec.rb => server/ai/config_tracker_spec.rb} (94%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index caa5422..4283e4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,12 +43,12 @@ The output will appear in `docs/build/html`. ## Code organization -A special case is the namespace `LaunchDarkly::AI::Impl`, and any namespaces within it. Everything under `Impl` is considered a private implementation detail: all files there are excluded from the generated documentation, and are considered subject to change at any time and not supported for direct use by application developers. We do this because Ruby's scope/visibility system is somewhat limited compared to other languages: a method can be `private` or `protected` within a class, but there is no way to make it visible to other classes in the library yet invisible to code outside of the library, and there is similarly no way to hide a class. +A special case is the namespace `LaunchDarkly::Server::AI::Impl`, and any namespaces within it. Everything under `Impl` is considered a private implementation detail: all files there are excluded from the generated documentation, and are considered subject to change at any time and not supported for direct use by application developers. We do this because Ruby's scope/visibility system is somewhat limited compared to other languages: a method can be `private` or `protected` within a class, but there is no way to make it visible to other classes in the library yet invisible to code outside of the library, and there is similarly no way to hide a class. So, if there is a class whose existence is entirely an implementation detail, it should be in `Impl`. Similarly, classes that are _not_ in `Impl` must not expose any public members that are not meant to be part of the supported public API. This is important because of our guarantee of backward compatibility for all public APIs within a major version: we want to be able to change our implementation details to suit the needs of the code, without worrying about breaking a customer's code. Due to how the language works, we can't actually prevent an application developer from referencing those classes in their code, but this convention makes it clear that such use is discouraged and unsupported. ## Documenting types and methods -All classes and public methods outside of `LaunchDarkly::AI::Impl` should have documentation comments. These are used to build the API documentation that is published at https://launchdarkly.github.io/ruby-server-sdk-ai/ and https://www.rubydoc.info/gems/launchdarkly-server-sdk-ai. The documentation generator is YARD; see https://yardoc.org/ for the comment format it uses. +All classes and public methods outside of `LaunchDarkly::Server::AI::Impl` should have documentation comments. These are used to build the API documentation that is published at https://launchdarkly.github.io/ruby-server-sdk-ai/ and https://www.rubydoc.info/gems/launchdarkly-server-sdk-ai. The documentation generator is YARD; see https://yardoc.org/ for the comment format it uses. Please try to make the style and terminology in documentation comments consistent with other documentation comments in the library. Also, if a class or method is being added that has an equivalent in other libraries, and if we have described it in a consistent away in those other libraries, please reuse the text whenever possible (with adjustments for anything language-specific) rather than writing new text. diff --git a/examples/hello-bedrock/Gemfile b/examples/hello-bedrock/Gemfile new file mode 100644 index 0000000..83b1714 --- /dev/null +++ b/examples/hello-bedrock/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'aws-sdk-bedrockruntime' +gem 'launchdarkly-server-sdk-ai', path: '../../..' diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index b2d8694..9bf0af8 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -1,10 +1,10 @@ # frozen_string_literal: true -require_relative 'lib/ldclient-ai/version' +require_relative 'lib/server/ai/version' Gem::Specification.new do |spec| spec.name = 'launchdarkly-server-sdk-ai' - spec.version = LaunchDarkly::AI::VERSION + spec.version = LaunchDarkly::Server::AI::VERSION spec.authors = ['LaunchDarkly'] spec.email = ['team@launchdarkly.com'] spec.summary = 'LaunchDarkly AI SDK for Ruby' diff --git a/lib/launchdarkly-server-sdk-ai.rb b/lib/launchdarkly-server-sdk-ai.rb new file mode 100644 index 0000000..ff5d9a4 --- /dev/null +++ b/lib/launchdarkly-server-sdk-ai.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'logger' +require 'mustache' + +require 'server/ai/version' +require 'server/ai/client' +require 'server/ai/config_tracker' + +module LaunchDarkly + module Server + # + # Namespace for the LaunchDarkly AI SDK. + # + module AI + # + # @return [Logger] the Rails logger if in Rails, or a default Logger at WARN level otherwise + # + def self.default_logger + if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger + Rails.logger + else + log = ::Logger.new($stdout) + log.level = ::Logger::WARN + log + end + end + end + end +end diff --git a/lib/launchdarkly_server_sdk_ai.rb b/lib/launchdarkly_server_sdk_ai.rb deleted file mode 100644 index 12bbb2c..0000000 --- a/lib/launchdarkly_server_sdk_ai.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -require_relative 'ldclient-ai' diff --git a/lib/ldclient-ai.rb b/lib/ldclient-ai.rb deleted file mode 100644 index c8a21fe..0000000 --- a/lib/ldclient-ai.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require 'logger' -require 'mustache' - -require 'ldclient-ai/version' -require 'ldclient-ai/client' -require 'ldclient-ai/config_tracker' - -module LaunchDarkly - # - # Namespace for the LaunchDarkly AI SDK. - # - module AI - # - # @return [Logger] the Rails logger if in Rails, or a default Logger at WARN level otherwise - # - def self.default_logger - if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger - Rails.logger - else - log = ::Logger.new($stdout) - log.level = ::Logger::WARN - log - end - end - end -end diff --git a/lib/ldclient-ai/client.rb b/lib/ldclient-ai/client.rb deleted file mode 100644 index ad80f3d..0000000 --- a/lib/ldclient-ai/client.rb +++ /dev/null @@ -1,202 +0,0 @@ -# frozen_string_literal: true - -require 'ldclient-rb' -require 'mustache' -require_relative 'config_tracker' - -module LaunchDarkly - # - # Namespace for the LaunchDarkly AI SDK. - # - module AI - # - # Holds AI role and content. - # - class LDMessage - attr_reader :role, :content - - def initialize(role, content) - @role = role - @content = content - end - - def to_h - { - role: @role, - content: @content - } - end - end - - # - # The ModelConfig class represents an AI model configuration. - # - class ModelConfig - attr_reader :name - - def initialize(name:, parameters: {}, custom: {}) - @name = name - @parameters = parameters - @custom = custom - end - - # - # Retrieve model-specific parameters. - # - # Accessing a named, typed attribute (e.g. name) will result in the call - # being delegated to the appropriate property. - # - # @param key [String] The parameter key to retrieve - # @return [Object, nil] The parameter value or nil if not found - # - def parameter(key) - return @name if key == 'name' - return nil unless @parameters.is_a?(Hash) - - @parameters[key] - end - - # - # Retrieve customer provided data. - # - # @param key [String] The custom key to retrieve - # @return [Object, nil] The custom value or nil if not found - # - def custom(key) - return nil unless @custom.is_a?(Hash) - - @custom[key] - end - - def to_h - { - name: @name, - parameters: @parameters, - custom: @custom - } - end - end - - # - # Configuration related to the provider. - # - class ProviderConfig - attr_reader :name - - def initialize(name) - @name = name - end - - def to_h - { - name: @name - } - end - end - - # - # The AIConfig class represents an AI configuration. - # - class AIConfig - attr_reader :enabled, :messages, :variables, :tracker, :model, :provider - - def initialize(enabled: nil, model: nil, messages: nil, tracker: nil, provider: nil) - @enabled = enabled - @messages = messages - @tracker = tracker - @model = model - @provider = provider - end - - def to_h - { - _ldMeta: { - enabled: @enabled || false - }, - messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, - model: @model&.to_h, - provider: @provider&.to_h - } - end - end - - # - # The LDAIClient class is the main entry point for the LaunchDarkly AI SDK. - # - class LDAIClient - attr_reader :logger, :ld_client - - def initialize(ld_client) - raise ArgumentError, 'LDClient instance is required' unless ld_client.is_a?(LaunchDarkly::LDClient) - - @ld_client = ld_client - @logger = LaunchDarkly::AI.default_logger - end - - # - # Retrieves the AIConfig - # - # @param config_key [String] The key of the configuration flag - # @param context [LDContext] The context used when evaluating the flag - # @param default_value [AIConfig] The default value to use if the flag is not found - # @param variables [Hash] Optional variables for rendering messages - # @return [AIConfig] An AIConfig instance containing the configuration data - # - def config(config_key, context, default_value = nil, variables = nil) - variation = @ld_client.variation( - config_key, - context, - default_value.respond_to?(:to_h) ? default_value.to_h : nil - ) - - all_variables = variables ? variables.dup : {} - all_variables[:ldctx] = context.to_h - - # - # Process messages and provider configuration - # - messages = nil - if variation[:messages].is_a?(Array) && variation[:messages].all? { |msg| msg.is_a?(Hash) } - messages = variation[:messages].map do |message| - next unless message[:content].is_a?(String) - - LDMessage.new( - message[:role], - Mustache.render(message[:content], all_variables) - ) - end - end - - if (provider_config = variation[:provider]) && provider_config.is_a?(Hash) - provider_config = ProviderConfig.new(provider_config.fetch(:name, '')) - end - - if (model = variation[:model]) && model.is_a?(Hash) - parameters = variation[:model][:parameters] - custom = variation[:model][:custom] - model = ModelConfig.new( - name: variation[:model][:name], - parameters: parameters, - custom: custom - ) - end - - tracker = LaunchDarkly::AI::LDAIConfigTracker.new( - ld_client: @ld_client, - variation_key: variation.dig(:_ldMeta, :variationKey) || '', - config_key: config_key, - version: variation.dig(:_ldMeta, :version) || 1, - context: context - ) - - AIConfig.new( - enabled: variation.dig(:_ldMeta, :enabled) || false, - messages: messages, - tracker: tracker, - model: model, - provider: provider_config - ) - end - end - end -end diff --git a/lib/ldclient-ai/config_tracker.rb b/lib/ldclient-ai/config_tracker.rb deleted file mode 100644 index 12560f3..0000000 --- a/lib/ldclient-ai/config_tracker.rb +++ /dev/null @@ -1,247 +0,0 @@ -# frozen_string_literal: true - -require 'ldclient-rb' - -module LaunchDarkly - module AI - # - # Tracks token usage for AI operations. - # - class TokenUsage - attr_reader :total, :input, :output - - # - # @param total [Integer] Total number of tokens used. - # @param input [Integer] Number of tokens in the prompt. - # @param output [Integer] Number of tokens in the completion. - # - def initialize(total: nil, input: nil, output: nil) - @total = total - @input = input - @output = output - end - end - - # - # Summary of metrics which have been tracked. - # - class LDAIMetricSummary - attr_accessor :duration, :success, :feedback, :usage, :time_to_first_token - - def initialize - @duration = nil - @success = nil - @feedback = nil - @usage = nil - @time_to_first_token = nil - end - end - - # - # The LDAIConfigTracker class is used to track AI configuration usage. - # - class LDAIConfigTracker - attr_reader :ld_client, :config_key, :context, :variation_key, :version, :summary - - def initialize(ld_client:, variation_key:, config_key:, version:, context:) - @ld_client = ld_client - @variation_key = variation_key - @config_key = config_key - @version = version - @context = context - @summary = LDAIMetricSummary.new - end - - # - # Track the duration of an AI operation - # - # @param duration [Integer] The duration in milliseconds - # - def track_duration(duration) - @summary.duration = duration - @ld_client.track( - '$ld:ai:duration:total', - @context, - flag_data, - duration - ) - end - - # - # Track the duration of a block of code - # - # @yield The block to measure - # @return The result of the block - # - def track_duration_of - start_time = Time.now - yield - ensure - duration = ((Time.now - start_time) * 1000).to_i - track_duration(duration) - end - - # - # Track time to first token - # - # @param duration [Integer] The duration in milliseconds - # - def track_time_to_first_token(time_to_first_token) - @summary.time_to_first_token = time_to_first_token - @ld_client.track( - '$ld:ai:tokens:ttf', - @context, - flag_data, - time_to_first_token - ) - end - - # - # Track user feedback - # - # @param kind [Symbol] The kind of feedback (:positive or :negative) - # - def track_feedback(kind:) - @summary.feedback = kind - event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' - @ld_client.track( - event_name, - @context, - flag_data, - 1 - ) - end - - # - # Track a successful AI generation - # - def track_success - @summary.success = true - @ld_client.track( - '$ld:ai:generation', - @context, - flag_data, - 1 - ) - @ld_client.track( - '$ld:ai:generation:success', - @context, - flag_data, - 1 - ) - end - - # - # Track an error in AI generation - # - def track_error - @summary.success = false - @ld_client.track( - '$ld:ai:generation', - @context, - flag_data, - 1 - ) - @ld_client.track( - '$ld:ai:generation:error', - @context, - flag_data, - 1 - ) - end - - # - # Track token usage - # - # @param token_usage [TokenUsage] An object containing token usage details - # - def track_tokens(token_usage) - @summary.usage = token_usage - if token_usage.total.positive? - @ld_client.track( - '$ld:ai:tokens:total', - @context, - flag_data, - token_usage.total - ) - end - if token_usage.input.positive? - @ld_client.track( - '$ld:ai:tokens:input', - @context, - flag_data, - token_usage.input - ) - end - return unless token_usage.output.positive? - - @ld_client.track( - '$ld:ai:tokens:output', - @context, - flag_data, - token_usage.output - ) - end - - # - # Track OpenAI-specific operations. - # This method tracks the duration, token usage, and success/error status. - # If the provided block raises, this method will also raise. - # A failed operation will not have any token usage data. - # - # @yield The block to track. - # @return The result of the tracked block. - # - def track_openai_metrics(&block) - result = track_duration_of(&block) - track_success - track_tokens(openai_to_token_usage(result[:usage])) if result[:usage] - result - rescue StandardError - track_error - raise - end - - # - # Track AWS Bedrock conversation operations. - # This method tracks the duration, token usage, and success/error status. - # - # @param res [Hash] Response hash from Bedrock. - # @return [Hash] The original response hash. - # - def track_bedrock_converse_metrics(res) - status_code = res.dig(:'$metadata', :httpStatusCode) || 0 - if status_code == 200 - track_success - elsif status_code >= 400 - track_error - end - - track_duration(res[:metrics][:latency_ms]) if res.dig(:metrics, :latency_ms) - track_tokens(bedrock_to_token_usage(res[:usage])) if res[:usage] - - res - end - - private def flag_data - { variationKey: @variation_key, configKey: @config_key, version: @version } - end - - private def openai_to_token_usage(usage) - TokenUsage.new( - total: usage[:total_tokens] || usage['total_tokens'], - input: usage[:prompt_tokens] || usage['prompt_tokens'], - output: usage[:completion_tokens] || usage['completion_tokens'] - ) - end - - private def bedrock_to_token_usage(usage) - TokenUsage.new( - total: usage[:total_tokens] || usage['total_tokens'], - input: usage[:input_tokens] || usage['input_tokens'], - output: usage[:output_tokens] || usage['output_tokens'] - ) - end - end - end -end diff --git a/lib/ldclient-ai/version.rb b/lib/ldclient-ai/version.rb deleted file mode 100644 index cd7e001..0000000 --- a/lib/ldclient-ai/version.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module LaunchDarkly - module AI - VERSION = '0.0.0' # x-release-please-version - end -end diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb new file mode 100644 index 0000000..815e64a --- /dev/null +++ b/lib/server/ai/client.rb @@ -0,0 +1,207 @@ +# frozen_string_literal: true + +require 'ldclient-rb' +require 'mustache' +require_relative 'config_tracker' + +module LaunchDarkly + # + # Namespace for the LaunchDarkly Server SDK + # + module Server + # + # Namespace for the LaunchDarkly Server AI SDK. + # + module AI + # + # Holds AI role and content. + # + class Message + attr_reader :role, :content + + def initialize(role, content) + @role = role + @content = content + end + + def to_h + { + role: @role, + content: @content + } + end + end + + # + # The ModelConfig class represents an AI model configuration. + # + class ModelConfig + attr_reader :name + + def initialize(name:, parameters: {}, custom: {}) + @name = name + @parameters = parameters + @custom = custom + end + + # + # Retrieve model-specific parameters. + # + # Accessing a named, typed attribute (e.g. name) will result in the call + # being delegated to the appropriate property. + # + # @param key [String] The parameter key to retrieve + # @return [Object, nil] The parameter value or nil if not found + # + def parameter(key) + return @name if key == 'name' + return nil unless @parameters.is_a?(Hash) + + @parameters[key] + end + + # + # Retrieve customer provided data. + # + # @param key [String] The custom key to retrieve + # @return [Object, nil] The custom value or nil if not found + # + def custom(key) + return nil unless @custom.is_a?(Hash) + + @custom[key] + end + + def to_h + { + name: @name, + parameters: @parameters, + custom: @custom + } + end + end + + # + # Configuration related to the provider. + # + class ProviderConfig + attr_reader :name + + def initialize(name) + @name = name + end + + def to_h + { + name: @name + } + end + end + + # + # The AIConfig class represents an AI configuration. + # + class AIConfig + attr_reader :enabled, :messages, :variables, :tracker, :model, :provider + + def initialize(enabled: nil, model: nil, messages: nil, tracker: nil, provider: nil) + @enabled = enabled + @messages = messages + @tracker = tracker + @model = model + @provider = provider + end + + def to_h + { + _ldMeta: { + enabled: @enabled || false + }, + messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, + model: @model&.to_h, + provider: @provider&.to_h + } + end + end + + # + # The Client class is the main entry point for the LaunchDarkly AI SDK. + # + class Client + attr_reader :logger, :ld_client + + def initialize(ld_client) + raise ArgumentError, 'LDClient instance is required' unless ld_client.is_a?(LaunchDarkly::LDClient) + + @ld_client = ld_client + @logger = LaunchDarkly::Server::AI.default_logger + end + + # + # Retrieves the AIConfig + # + # @param config_key [String] The key of the configuration flag + # @param context [LDContext] The context used when evaluating the flag + # @param default_value [AIConfig] The default value to use if the flag is not found + # @param variables [Hash] Optional variables for rendering messages + # @return [AIConfig] An AIConfig instance containing the configuration data + # + def config(config_key, context, default_value = nil, variables = nil) + variation = @ld_client.variation( + config_key, + context, + default_value.respond_to?(:to_h) ? default_value.to_h : nil + ) + + all_variables = variables ? variables.dup : {} + all_variables[:ldctx] = context.to_h + + # + # Process messages and provider configuration + # + messages = nil + if variation[:messages].is_a?(Array) && variation[:messages].all? { |msg| msg.is_a?(Hash) } + messages = variation[:messages].map do |message| + next unless message[:content].is_a?(String) + + Message.new( + message[:role], + Mustache.render(message[:content], all_variables) + ) + end + end + + if (provider_config = variation[:provider]) && provider_config.is_a?(Hash) + provider_config = ProviderConfig.new(provider_config.fetch(:name, '')) + end + + if (model = variation[:model]) && model.is_a?(Hash) + parameters = variation[:model][:parameters] + custom = variation[:model][:custom] + model = ModelConfig.new( + name: variation[:model][:name], + parameters: parameters, + custom: custom + ) + end + + tracker = LaunchDarkly::Server::AI::AIConfigTracker.new( + ld_client: @ld_client, + variation_key: variation.dig(:_ldMeta, :variationKey) || '', + config_key: config_key, + version: variation.dig(:_ldMeta, :version) || 1, + context: context + ) + + AIConfig.new( + enabled: variation.dig(:_ldMeta, :enabled) || false, + messages: messages, + tracker: tracker, + model: model, + provider: provider_config + ) + end + end + end + end +end diff --git a/lib/server/ai/config_tracker.rb b/lib/server/ai/config_tracker.rb new file mode 100644 index 0000000..6f68a89 --- /dev/null +++ b/lib/server/ai/config_tracker.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +require 'ldclient-rb' + +module LaunchDarkly + module Server + module AI + # + # Tracks token usage for AI operations. + # + class TokenUsage + attr_reader :total, :input, :output + + # + # @param total [Integer] Total number of tokens used. + # @param input [Integer] Number of tokens in the prompt. + # @param output [Integer] Number of tokens in the completion. + # + def initialize(total: nil, input: nil, output: nil) + @total = total + @input = input + @output = output + end + end + + # + # Summary of metrics which have been tracked. + # + class MetricSummary + attr_accessor :duration, :success, :feedback, :usage, :time_to_first_token + + def initialize + @duration = nil + @success = nil + @feedback = nil + @usage = nil + @time_to_first_token = nil + end + end + + # + # The AIConfigTracker class is used to track AI configuration usage. + # + class AIConfigTracker + attr_reader :ld_client, :config_key, :context, :variation_key, :version, :summary + + def initialize(ld_client:, variation_key:, config_key:, version:, context:) + @ld_client = ld_client + @variation_key = variation_key + @config_key = config_key + @version = version + @context = context + @summary = MetricSummary.new + end + + # + # Track the duration of an AI operation + # + # @param duration [Integer] The duration in milliseconds + # + def track_duration(duration) + @summary.duration = duration + @ld_client.track( + '$ld:ai:duration:total', + @context, + flag_data, + duration + ) + end + + # + # Track the duration of a block of code + # + # @yield The block to measure + # @return The result of the block + # + def track_duration_of + start_time = Time.now + yield + ensure + duration = ((Time.now - start_time) * 1000).to_i + track_duration(duration) + end + + # + # Track time to first token + # + # @param duration [Integer] The duration in milliseconds + # + def track_time_to_first_token(time_to_first_token) + @summary.time_to_first_token = time_to_first_token + @ld_client.track( + '$ld:ai:tokens:ttf', + @context, + flag_data, + time_to_first_token + ) + end + + # + # Track user feedback + # + # @param kind [Symbol] The kind of feedback (:positive or :negative) + # + def track_feedback(kind:) + @summary.feedback = kind + event_name = kind == :positive ? '$ld:ai:feedback:user:positive' : '$ld:ai:feedback:user:negative' + @ld_client.track( + event_name, + @context, + flag_data, + 1 + ) + end + + # + # Track a successful AI generation + # + def track_success + @summary.success = true + @ld_client.track( + '$ld:ai:generation', + @context, + flag_data, + 1 + ) + @ld_client.track( + '$ld:ai:generation:success', + @context, + flag_data, + 1 + ) + end + + # + # Track an error in AI generation + # + def track_error + @summary.success = false + @ld_client.track( + '$ld:ai:generation', + @context, + flag_data, + 1 + ) + @ld_client.track( + '$ld:ai:generation:error', + @context, + flag_data, + 1 + ) + end + + # + # Track token usage + # + # @param token_usage [TokenUsage] An object containing token usage details + # + def track_tokens(token_usage) + @summary.usage = token_usage + if token_usage.total.positive? + @ld_client.track( + '$ld:ai:tokens:total', + @context, + flag_data, + token_usage.total + ) + end + if token_usage.input.positive? + @ld_client.track( + '$ld:ai:tokens:input', + @context, + flag_data, + token_usage.input + ) + end + return unless token_usage.output.positive? + + @ld_client.track( + '$ld:ai:tokens:output', + @context, + flag_data, + token_usage.output + ) + end + + # + # Track OpenAI-specific operations. + # This method tracks the duration, token usage, and success/error status. + # If the provided block raises, this method will also raise. + # A failed operation will not have any token usage data. + # + # @yield The block to track. + # @return The result of the tracked block. + # + def track_openai_metrics(&block) + result = track_duration_of(&block) + track_success + track_tokens(openai_to_token_usage(result[:usage])) if result[:usage] + result + rescue StandardError + track_error + raise + end + + # + # Track AWS Bedrock conversation operations. + # This method tracks the duration, token usage, and success/error status. + # + # @param res [Hash] Response hash from Bedrock. + # @return [Hash] The original response hash. + # + def track_bedrock_converse_metrics(res) + status_code = res.dig(:'$metadata', :httpStatusCode) || 0 + if status_code == 200 + track_success + elsif status_code >= 400 + track_error + end + + track_duration(res[:metrics][:latency_ms]) if res.dig(:metrics, :latency_ms) + track_tokens(bedrock_to_token_usage(res[:usage])) if res[:usage] + + res + end + + private def flag_data + { variationKey: @variation_key, configKey: @config_key, version: @version } + end + + private def openai_to_token_usage(usage) + TokenUsage.new( + total: usage[:total_tokens] || usage['total_tokens'], + input: usage[:prompt_tokens] || usage['prompt_tokens'], + output: usage[:completion_tokens] || usage['completion_tokens'] + ) + end + + private def bedrock_to_token_usage(usage) + TokenUsage.new( + total: usage[:total_tokens] || usage['total_tokens'], + input: usage[:input_tokens] || usage['input_tokens'], + output: usage[:output_tokens] || usage['output_tokens'] + ) + end + end + end + end +end diff --git a/lib/server/ai/version.rb b/lib/server/ai/version.rb new file mode 100644 index 0000000..0fcda51 --- /dev/null +++ b/lib/server/ai/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module LaunchDarkly + module Server + module AI + VERSION = '0.0.0' # x-release-please-version + end + end +end diff --git a/spec/ldclient_ai_spec.rb b/spec/ldclient_ai_spec.rb deleted file mode 100644 index cf969e8..0000000 --- a/spec/ldclient_ai_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -require 'launchdarkly_server_sdk_ai' - -RSpec.describe LaunchDarkly::AI do - it 'has a version number' do - expect(LaunchDarkly::AI::VERSION).not_to be nil - end - - it 'returns a logger' do - logger = LaunchDarkly::AI.default_logger - expect(logger).to be_a(Logger) - expect(logger.level).to eq(Logger::WARN) - end -end diff --git a/spec/server/ai/ai_spec.rb b/spec/server/ai/ai_spec.rb new file mode 100644 index 0000000..87858c4 --- /dev/null +++ b/spec/server/ai/ai_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'launchdarkly-server-sdk-ai' + +RSpec.describe LaunchDarkly::Server::AI do + it 'has a version number' do + expect(LaunchDarkly::Server::AI::VERSION).not_to be nil + end + + it 'returns a logger' do + logger = LaunchDarkly::Server::AI.default_logger + expect(logger).to be_a(Logger) + expect(logger.level).to eq(Logger::WARN) + end +end diff --git a/spec/ld_ai_client_spec.rb b/spec/server/ai/client_spec.rb similarity index 84% rename from spec/ld_ai_client_spec.rb rename to spec/server/ai/client_spec.rb index 7d27840..6fcd00c 100644 --- a/spec/ld_ai_client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'ldclient-ai' -require 'ldclient-rb' +require 'launchdarkly-server-sdk' +require 'launchdarkly-server-sdk-ai' -RSpec.describe LaunchDarkly::AI do +RSpec.describe LaunchDarkly::Server::AI do let(:td) do data_source = LaunchDarkly::Integrations::TestData.data_source data_source.update(data_source.flag('model-config') @@ -95,11 +95,11 @@ config = LaunchDarkly::Config.new(data_source: td, send_events: false) LaunchDarkly::LDClient.new('sdk-key', config) end - let(:ai_client) { LaunchDarkly::AI::LDAIClient.new(ld_client) } + let(:ai_client) { LaunchDarkly::Server::AI::Client.new(ld_client) } - describe LaunchDarkly::AI::ModelConfig do + describe LaunchDarkly::Server::AI::ModelConfig do it 'delegates to properties' do - model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) + model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') expect(model.parameter(:'extra-attribute')).to eq('value') expect(model.parameter('non-existent')).to be_nil @@ -107,7 +107,7 @@ end it 'handles custom attributes' do - model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', custom: { 'extra-attribute': 'value' }) + model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel', custom: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') expect(model.custom(:'extra-attribute')).to eq('value') expect(model.custom('non-existent')).to be_nil @@ -115,10 +115,10 @@ end end - describe LaunchDarkly::AI::LDAIClient do + describe LaunchDarkly::Server::AI::Client do describe '#initialize' do it 'initializes with a valid LDClient instance' do - expect(ai_client).to be_a(LaunchDarkly::AI::LDAIClient) + expect(ai_client).to be_a(LaunchDarkly::Server::AI::Client) expect(ai_client.ld_client).to eq(ld_client) end @@ -134,12 +134,12 @@ describe '#config' do it 'uses default config on invalid flag' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - model = LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel', + model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel', parameters: { temperature: 0.5, maxTokens: 4096 }) - messages = [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] - default_config = LaunchDarkly::AI::AIConfig.new( + messages = [LaunchDarkly::Server::AI::Message.new('system', 'Hello, {{name}}!')] + default_config = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, model: model, messages: messages @@ -160,10 +160,10 @@ it 'interpolates variables in model config messages' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), - messages: [LaunchDarkly::AI::LDMessage.new('system', 'Hello, {{name}}!')] + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel'), + messages: [LaunchDarkly::Server::AI::Message.new('system', 'Hello, {{name}}!')] ) variables = { 'name' => 'World' } @@ -181,9 +181,9 @@ it 'returns config with messages interpolated as empty when no variables are provided' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fakeModel'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel'), messages: [] ) @@ -202,9 +202,9 @@ it 'handles provider config correctly' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) variables = { 'name' => 'World' } @@ -217,9 +217,9 @@ it 'interpolates context variables in messages using ldctx' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user', name: 'Sandy', last: 'Beaches' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) variables = { 'name' => 'World' } @@ -243,9 +243,9 @@ org_context = LaunchDarkly::LDContext.create({ key: 'org-key', kind: 'org', name: 'LaunchDarkly', shortname: 'LD' }) context = LaunchDarkly::LDContext.create_multi([user_context, org_context]) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) variables = { 'name' => 'World' } @@ -266,9 +266,9 @@ it 'handles multiple messages and variable interpolation' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) variables = { 'name' => 'World', 'day' => 'Monday' } @@ -289,9 +289,9 @@ it 'returns disabled config when flag is off' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) @@ -306,9 +306,9 @@ it 'returns disabled config with nil model/messages/provider when initial config is disabled' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: true, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) @@ -322,9 +322,9 @@ it 'returns enabled config with nil model/messages/provider when initial config is enabled' do context = LaunchDarkly::LDContext.create({ key: 'user-key', kind: 'user' }) - default_value = LaunchDarkly::AI::AIConfig.new( + default_value = LaunchDarkly::Server::AI::AIConfig.new( enabled: false, - model: LaunchDarkly::AI::ModelConfig.new(name: 'fake-model'), + model: LaunchDarkly::Server::AI::ModelConfig.new(name: 'fake-model'), messages: [] ) diff --git a/spec/ld_ai_config_tracker_spec.rb b/spec/server/ai/config_tracker_spec.rb similarity index 94% rename from spec/ld_ai_config_tracker_spec.rb rename to spec/server/ai/config_tracker_spec.rb index cbfb8e6..4920de0 100644 --- a/spec/ld_ai_config_tracker_spec.rb +++ b/spec/server/ai/config_tracker_spec.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'ldclient-rb' -require 'ldclient-ai' +require 'launchdarkly-server-sdk' +require 'launchdarkly-server-sdk-ai' -RSpec.describe LaunchDarkly::AI::LDAIConfigTracker do +RSpec.describe LaunchDarkly::Server::AI::AIConfigTracker do let(:td) do LaunchDarkly::Integrations::TestData.data_source.update( LaunchDarkly::Integrations::TestData.data_source.flag('model_config') @@ -115,7 +115,7 @@ tracker_flag_data, 100 ) - tokens = LaunchDarkly::AI::TokenUsage.new(total: 300, input: 200, output: 100) + tokens = LaunchDarkly::Server::AI::TokenUsage.new(total: 300, input: 200, output: 100) tracker.track_tokens(tokens) expect(tracker.summary.usage).to eq(tokens) end @@ -174,8 +174,8 @@ result = tracker.track_bedrock_converse_metrics(bedrock_result) expect(result).to eq(bedrock_result) - expect(tracker.summary).to be_a(LaunchDarkly::AI::LDAIMetricSummary) - expect(tracker.summary.usage).to be_a(LaunchDarkly::AI::TokenUsage) + expect(tracker.summary).to be_a(LaunchDarkly::Server::AI::MetricSummary) + expect(tracker.summary.usage).to be_a(LaunchDarkly::Server::AI::TokenUsage) expect(tracker.summary.usage.total).to eq(300) expect(tracker.summary.usage.input).to eq(200) expect(tracker.summary.usage.output).to eq(100) @@ -235,8 +235,8 @@ result = tracker.track_bedrock_converse_metrics(bedrock_result) expect(result).to eq(bedrock_result) - expect(tracker.summary).to be_a(LaunchDarkly::AI::LDAIMetricSummary) - expect(tracker.summary.usage).to be_a(LaunchDarkly::AI::TokenUsage) + expect(tracker.summary).to be_a(LaunchDarkly::Server::AI::MetricSummary) + expect(tracker.summary.usage).to be_a(LaunchDarkly::Server::AI::TokenUsage) expect(tracker.summary.usage.total).to eq(300) expect(tracker.summary.usage.input).to eq(200) expect(tracker.summary.usage.output).to eq(100) @@ -417,7 +417,7 @@ it 'returns a summary of tracked metrics' do tracker.track_duration(100) tracker.track_feedback(kind: :positive) - tracker.track_tokens(LaunchDarkly::AI::TokenUsage.new(total: 100, input: 50, output: 50)) + tracker.track_tokens(LaunchDarkly::Server::AI::TokenUsage.new(total: 100, input: 50, output: 50)) tracker.track_success tracker.track_time_to_first_token(50) From 305483c80cb1a783079c6171ea1b9640c5a7b7ad Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 16 Jun 2025 19:33:48 +0000 Subject: [PATCH 17/20] fix some style issues --- launchdarkly-server-sdk-ai.gemspec | 4 ++-- lib/server/ai/client.rb | 12 ++++++------ spec/server/ai/ai_spec.rb | 4 ++-- spec/server/ai/client_spec.rb | 26 +++++++++++++------------- spec/server/ai/config_tracker_spec.rb | 20 ++++++++++---------- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/launchdarkly-server-sdk-ai.gemspec b/launchdarkly-server-sdk-ai.gemspec index 9bf0af8..fba31a0 100644 --- a/launchdarkly-server-sdk-ai.gemspec +++ b/launchdarkly-server-sdk-ai.gemspec @@ -27,8 +27,8 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'debug', '~> 1.0' spec.add_development_dependency 'rake', '~> 13.0' spec.add_development_dependency 'rspec', '~> 3.0' - spec.add_development_dependency 'rubocop', '~> 1.0' + spec.add_development_dependency 'rubocop', '~> 1.21' spec.add_development_dependency 'rubocop-performance', '~> 1.15' spec.add_development_dependency 'rubocop-rake', '~> 0.6' - spec.add_development_dependency 'rubocop-rspec', '~> 2.0' + spec.add_development_dependency 'rubocop-rspec', '~> 3.6' end diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 815e64a..84aae76 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -7,7 +7,7 @@ module LaunchDarkly # # Namespace for the LaunchDarkly Server SDK - # + # module Server # # Namespace for the LaunchDarkly Server AI SDK. @@ -27,7 +27,7 @@ def initialize(role, content) def to_h { role: @role, - content: @content + content: @content, } end end @@ -76,7 +76,7 @@ def to_h { name: @name, parameters: @parameters, - custom: @custom + custom: @custom, } end end @@ -93,7 +93,7 @@ def initialize(name) def to_h { - name: @name + name: @name, } end end @@ -115,11 +115,11 @@ def initialize(enabled: nil, model: nil, messages: nil, tracker: nil, provider: def to_h { _ldMeta: { - enabled: @enabled || false + enabled: @enabled || false, }, messages: @messages.is_a?(Array) ? @messages.map { |msg| msg&.to_h } : nil, model: @model&.to_h, - provider: @provider&.to_h + provider: @provider&.to_h, } end end diff --git a/spec/server/ai/ai_spec.rb b/spec/server/ai/ai_spec.rb index 87858c4..e0733a6 100644 --- a/spec/server/ai/ai_spec.rb +++ b/spec/server/ai/ai_spec.rb @@ -4,11 +4,11 @@ RSpec.describe LaunchDarkly::Server::AI do it 'has a version number' do - expect(LaunchDarkly::Server::AI::VERSION).not_to be nil + expect(LaunchDarkly::Server::AI::VERSION).not_to be_nil end it 'returns a logger' do - logger = LaunchDarkly::Server::AI.default_logger + logger = described_class.default_logger expect(logger).to be_a(Logger) expect(logger.level).to eq(Logger::WARN) end diff --git a/spec/server/ai/client_spec.rb b/spec/server/ai/client_spec.rb index 6fcd00c..d5ae0dd 100644 --- a/spec/server/ai/client_spec.rb +++ b/spec/server/ai/client_spec.rb @@ -13,7 +13,7 @@ custom: { 'extra-attribute': 'value' } }, provider: { name: 'fakeProvider' }, messages: [{ role: 'system', content: 'Hello, {{name}}!' }], - _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 }, }, :green ) @@ -25,9 +25,9 @@ model: { name: 'fakeModel', parameters: { temperature: 0.7, maxTokens: 8192 } }, messages: [ { role: 'system', content: 'Hello, {{name}}!' }, - { role: 'user', content: 'The day is, {{day}}!' } + { role: 'user', content: 'The day is, {{day}}!' }, ], - _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 }, }, :green ) @@ -39,7 +39,7 @@ model: { name: 'fakeModel', parameters: { 'extra-attribute': 'I can be anything I set my mind/type to' } }, messages: [{ role: 'system', content: 'Hello, {{ldctx.name}}! Is your last name {{ldctx.last}}?' }], - _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 }, } ) .variation_for_all(0)) @@ -51,7 +51,7 @@ parameters: { 'extra-attribute': 'I can be anything I set my mind/type to' } }, messages: [{ role: 'system', content: 'Hello, {{ldctx.user.name}}! Do you work for {{ldctx.org.shortname}}?' }], - _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 }, } ) .variation_for_all(0)) @@ -61,7 +61,7 @@ { model: { name: 'fakeModel', parameters: { temperature: 0.1 } }, messages: [{ role: 'system', content: 'Hello, {{name}}!' }], - _ldMeta: { enabled: false, variationKey: 'abcd', version: 1 } + _ldMeta: { enabled: false, variationKey: 'abcd', version: 1 }, } ) .variation_for_all(0)) @@ -69,10 +69,10 @@ data_source.update(data_source.flag('initial-config-disabled') .variations( { - _ldMeta: { enabled: false } + _ldMeta: { enabled: false }, }, { - _ldMeta: { enabled: true } + _ldMeta: { enabled: true }, } ) .variation_for_all(0)) @@ -80,10 +80,10 @@ data_source.update(data_source.flag('initial-config-enabled') .variations( { - _ldMeta: { enabled: false } + _ldMeta: { enabled: false }, }, { - _ldMeta: { enabled: true } + _ldMeta: { enabled: true }, } ) .variation_for_all(1)) @@ -99,7 +99,7 @@ describe LaunchDarkly::Server::AI::ModelConfig do it 'delegates to properties' do - model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) + model = described_class.new(name: 'fakeModel', parameters: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') expect(model.parameter(:'extra-attribute')).to eq('value') expect(model.parameter('non-existent')).to be_nil @@ -107,7 +107,7 @@ end it 'handles custom attributes' do - model = LaunchDarkly::Server::AI::ModelConfig.new(name: 'fakeModel', custom: { 'extra-attribute': 'value' }) + model = described_class.new(name: 'fakeModel', custom: { 'extra-attribute': 'value' }) expect(model.name).to eq('fakeModel') expect(model.custom(:'extra-attribute')).to eq('value') expect(model.custom('non-existent')).to be_nil @@ -118,7 +118,7 @@ describe LaunchDarkly::Server::AI::Client do describe '#initialize' do it 'initializes with a valid LDClient instance' do - expect(ai_client).to be_a(LaunchDarkly::Server::AI::Client) + expect(ai_client).to be_a(described_class) expect(ai_client.ld_client).to eq(ld_client) end diff --git a/spec/server/ai/config_tracker_spec.rb b/spec/server/ai/config_tracker_spec.rb index 4920de0..462aad2 100644 --- a/spec/server/ai/config_tracker_spec.rb +++ b/spec/server/ai/config_tracker_spec.rb @@ -13,7 +13,7 @@ custom: { 'extra-attribute': 'value' } }, provider: { name: 'fakeProvider' }, messages: [{ role: 'system', content: 'Hello, {{name}}!' }], - _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 } + _ldMeta: { enabled: true, variationKey: 'abcd', version: 1 }, }, 'green' ) @@ -129,11 +129,11 @@ usage: { total_tokens: 300, input_tokens: 200, - output_tokens: 100 + output_tokens: 100, }, metrics: { - latency_ms: 50 - } + latency_ms: 50, + }, } expect(ld_client).to receive(:track).with( '$ld:ai:generation', @@ -190,11 +190,11 @@ usage: { total_tokens: 300, input_tokens: 200, - output_tokens: 100 + output_tokens: 100, }, metrics: { - latency_ms: 50 - } + latency_ms: 50, + }, } expect(ld_client).to receive(:track).with( '$ld:ai:generation', @@ -251,8 +251,8 @@ usage: { total_tokens: 300, prompt_tokens: 200, - completion_tokens: 100 - } + completion_tokens: 100, + }, } end @@ -323,7 +323,7 @@ ) expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') - expect(tracker.summary.usage).to eq(nil) + expect(tracker.summary.usage).to be_nil end end From ac5b07b8e357363fabe0edd3ca3d895f0c650e70 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 16 Jun 2025 21:42:51 +0000 Subject: [PATCH 18/20] align bedrock with spec --- lib/server/ai/config_tracker.rb | 24 ++++------ spec/server/ai/config_tracker_spec.rb | 66 ++++++++------------------- 2 files changed, 28 insertions(+), 62 deletions(-) diff --git a/lib/server/ai/config_tracker.rb b/lib/server/ai/config_tracker.rb index 6f68a89..2cb9337 100644 --- a/lib/server/ai/config_tracker.rb +++ b/lib/server/ai/config_tracker.rb @@ -74,7 +74,7 @@ def track_duration(duration) # @yield The block to measure # @return The result of the block # - def track_duration_of + def track_duration_of(&block) start_time = Time.now yield ensure @@ -207,21 +207,17 @@ def track_openai_metrics(&block) # Track AWS Bedrock conversation operations. # This method tracks the duration, token usage, and success/error status. # - # @param res [Hash] Response hash from Bedrock. + # @yield The block to track. # @return [Hash] The original response hash. # - def track_bedrock_converse_metrics(res) - status_code = res.dig(:'$metadata', :httpStatusCode) || 0 - if status_code == 200 - track_success - elsif status_code >= 400 - track_error - end - - track_duration(res[:metrics][:latency_ms]) if res.dig(:metrics, :latency_ms) - track_tokens(bedrock_to_token_usage(res[:usage])) if res[:usage] - - res + def track_bedrock_converse_metrics(&block) + result = track_duration_of(&block) + track_success + track_tokens(bedrock_to_token_usage(result[:usage])) if result[:usage] + result + rescue StandardError + track_error + raise end private def flag_data diff --git a/spec/server/ai/config_tracker_spec.rb b/spec/server/ai/config_tracker_spec.rb index 462aad2..8fed843 100644 --- a/spec/server/ai/config_tracker_spec.rb +++ b/spec/server/ai/config_tracker_spec.rb @@ -122,19 +122,17 @@ end describe '#track_bedrock_metrics' do - it 'tracks duration and tokens' do - # TODO: Verify the $metadata field in bedrock result. I don't see anything like this in the docs. - bedrock_result = { - '$metadata': { httpStatusCode: 200 }, + let(:bedrock_result) do + { usage: { total_tokens: 300, input_tokens: 200, output_tokens: 100, }, - metrics: { - latency_ms: 50, - }, } + end + + it 'tracks duration and tokens' do expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, @@ -172,30 +170,19 @@ 100 ) - result = tracker.track_bedrock_converse_metrics(bedrock_result) + result = tracker.track_bedrock_converse_metrics { bedrock_result } expect(result).to eq(bedrock_result) expect(tracker.summary).to be_a(LaunchDarkly::Server::AI::MetricSummary) expect(tracker.summary.usage).to be_a(LaunchDarkly::Server::AI::TokenUsage) expect(tracker.summary.usage.total).to eq(300) expect(tracker.summary.usage.input).to eq(200) expect(tracker.summary.usage.output).to eq(100) - expect(tracker.summary.duration).to eq(50) + expect(tracker.summary.duration).to be_a(Integer) + expect(tracker.summary.duration).to be >= 0 expect(tracker.summary.success).to be true end - it 'tracks duration and tokens with error' do - # Verify the $metadata field in bedrock result. I don't see anything like this in the docs. - bedrock_result = { - '$metadata': { httpStatusCode: 400 }, - usage: { - total_tokens: 300, - input_tokens: 200, - output_tokens: 100, - }, - metrics: { - latency_ms: 50, - }, - } + it 'tracks error for failed operation' do expect(ld_client).to receive(:track).with( '$ld:ai:generation', context, @@ -214,33 +201,11 @@ tracker_flag_data, kind_of(Integer) ) - expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:total', - context, - tracker_flag_data, - 300 - ) - expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:input', - context, - tracker_flag_data, - 200 - ) - expect(ld_client).to receive(:track).with( - '$ld:ai:tokens:output', - context, - tracker_flag_data, - 100 - ) - result = tracker.track_bedrock_converse_metrics(bedrock_result) - expect(result).to eq(bedrock_result) - expect(tracker.summary).to be_a(LaunchDarkly::Server::AI::MetricSummary) - expect(tracker.summary.usage).to be_a(LaunchDarkly::Server::AI::TokenUsage) - expect(tracker.summary.usage.total).to eq(300) - expect(tracker.summary.usage.input).to eq(200) - expect(tracker.summary.usage.output).to eq(100) - expect(tracker.summary.duration).to eq(50) + expect { tracker.track_bedrock_converse_metrics { raise 'test error' } }.to raise_error('test error') + expect(tracker.summary.usage).to be_nil + expect(tracker.summary.duration).to be_a(Integer) + expect(tracker.summary.duration).to be >= 0 expect(tracker.summary.success).to be false end end @@ -299,6 +264,8 @@ expect(tracker.summary.usage.total).to eq(300) expect(tracker.summary.usage.input).to eq(200) expect(tracker.summary.usage.output).to eq(100) + expect(tracker.summary.duration).to be_a(Integer) + expect(tracker.summary.duration).to be >= 0 expect(tracker.summary.success).to be true end @@ -324,6 +291,9 @@ expect { tracker.track_openai_metrics { raise 'test error' } }.to raise_error('test error') expect(tracker.summary.usage).to be_nil + expect(tracker.summary.duration).to be_a(Integer) + expect(tracker.summary.duration).to be >= 0 + expect(tracker.summary.success).to be false end end From 91359450521b1f98d2817a2454b811fb33cbdd70 Mon Sep 17 00:00:00 2001 From: Jason Bailey Date: Tue, 17 Jun 2025 08:39:38 -0500 Subject: [PATCH 19/20] Update lib/server/ai/client.rb Co-authored-by: Matthew M. Keeler --- lib/server/ai/client.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 84aae76..24405da 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -156,9 +156,7 @@ def config(config_key, context, default_value = nil, variables = nil) all_variables = variables ? variables.dup : {} all_variables[:ldctx] = context.to_h - # # Process messages and provider configuration - # messages = nil if variation[:messages].is_a?(Array) && variation[:messages].all? { |msg| msg.is_a?(Hash) } messages = variation[:messages].map do |message| From 2c1f8b19d05974c6cafbb23379b26a8977db1961 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 17 Jun 2025 13:43:41 +0000 Subject: [PATCH 20/20] align file name with class name --- lib/launchdarkly-server-sdk-ai.rb | 2 +- lib/server/ai/{config_tracker.rb => ai_config_tracker.rb} | 0 lib/server/ai/client.rb | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename lib/server/ai/{config_tracker.rb => ai_config_tracker.rb} (100%) diff --git a/lib/launchdarkly-server-sdk-ai.rb b/lib/launchdarkly-server-sdk-ai.rb index ff5d9a4..1abaeec 100644 --- a/lib/launchdarkly-server-sdk-ai.rb +++ b/lib/launchdarkly-server-sdk-ai.rb @@ -5,7 +5,7 @@ require 'server/ai/version' require 'server/ai/client' -require 'server/ai/config_tracker' +require 'server/ai/ai_config_tracker' module LaunchDarkly module Server diff --git a/lib/server/ai/config_tracker.rb b/lib/server/ai/ai_config_tracker.rb similarity index 100% rename from lib/server/ai/config_tracker.rb rename to lib/server/ai/ai_config_tracker.rb diff --git a/lib/server/ai/client.rb b/lib/server/ai/client.rb index 24405da..073f092 100644 --- a/lib/server/ai/client.rb +++ b/lib/server/ai/client.rb @@ -2,7 +2,7 @@ require 'ldclient-rb' require 'mustache' -require_relative 'config_tracker' +require_relative 'ai_config_tracker' module LaunchDarkly #