From 2f128cb1e3b6b3ad9abb82c74059711788755cc0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 8 Jul 2026 03:13:57 +0000 Subject: [PATCH 1/3] feat(ldai): stamp modelKey and modelVersion on AI usage events (AIC-2850) Read modelKey and modelVersion from the AI Config variation payload and expose them on Config. Stamp both fields on every metric event alongside existing modelName and providerName. modelVersion defaults to 1 when absent; modelKey is omitted when empty. Neither field is included in the resumption token. Additive and backward compatible with older payloads. Co-authored-by: Anthony Torns II --- ldai/client.go | 4 +++ ldai/client_test.go | 68 ++++++++++++++++++++++++++++++++++++ ldai/config.go | 30 ++++++++++++++++ ldai/datamodel/datamodel.go | 6 ++++ ldai/tracker.go | 15 +++++--- ldai/tracker_test.go | 69 +++++++++++++++++++++++++++++++++++-- 6 files changed, 184 insertions(+), 8 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index 76eddc5e..afebf6be 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -156,12 +156,16 @@ func (c *Client) evaluateConfig( builder := NewConfig(). WithModelName(parsed.Model.Name). + WithModelKey(parsed.Model.Key). WithProviderName(parsed.Provider.Name). WithEnabled(parsed.Meta.Enabled). WithMode(parsed.Mode). WithEvaluationMetricKey(parsed.EvaluationMetricKey). WithEvaluationMetricKeys(parsed.EvaluationMetricKeys). WithJudgeConfiguration(parsed.JudgeConfiguration) + if parsed.Model.Version != nil { + builder.WithModelVersion(*parsed.Model.Version) + } for k, v := range parsed.Model.Parameters { builder.WithModelParam(k, v) diff --git a/ldai/client_test.go b/ldai/client_test.go index 01805f1e..30847178 100644 --- a/ldai/client_test.go +++ b/ldai/client_test.go @@ -150,6 +150,72 @@ func TestParseModelName(t *testing.T) { } } +func TestParseModelKeyAndVersion(t *testing.T) { + tests := []struct { + name string + json []byte + expectedKey string + expectedVersion int + }{ + { + name: "missing", + json: []byte(`{"model": {"name": "gpt-4"}}`), + expectedKey: "", + expectedVersion: 1, + }, + { + name: "modelKey and modelVersion set", + json: []byte(`{"model": {"name": "gpt-4", "modelKey": "my-model", "modelVersion": 2}}`), + expectedKey: "my-model", + expectedVersion: 2, + }, + { + name: "modelVersion only", + json: []byte(`{"model": {"name": "gpt-4", "modelVersion": 3}}`), + expectedKey: "", + expectedVersion: 3, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, err := NewClient(newMockSDK(test.json, nil)) + require.NoError(t, err) + require.NotNil(t, client) + + defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() + cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) + + assert.Equal(t, test.expectedKey, cfg.ModelKey()) + assert.Equal(t, test.expectedVersion, cfg.ModelVersion()) + }) + } +} + +func TestCreateTrackerStampsModelKeyAndVersionOnTrackData(t *testing.T) { + configJSON := []byte(`{ + "_ldMeta": {"variationKey": "var-1", "enabled": true, "version": 1}, + "model": {"name": "gpt-4", "modelKey": "my-model", "modelVersion": 2}, + "provider": {"name": "openai"}, + "messages": [{"content": "hello", "role": "user"}] + }`) + + mockSDK := newMockSDK(configJSON, nil) + client, err := NewClient(mockSDK) + require.NoError(t, err) + mockSDK.events = nil + + cfg := client.CompletionConfig("my-config", ldcontext.New("user"), Disabled(), nil) + tracker := cfg.CreateTracker() + require.NotNil(t, tracker) + assert.NoError(t, tracker.TrackSuccess()) + + require.NotEmpty(t, mockSDK.events) + data := mockSDK.events[len(mockSDK.events)-1].data + assert.Equal(t, "my-model", data.GetByKey("modelKey").StringValue()) + assert.Equal(t, 2, data.GetByKey("modelVersion").IntValue()) +} + func TestParseProviderName(t *testing.T) { tests := []struct { name string @@ -1009,6 +1075,8 @@ func TestClient_CreateTracker_RoundTrip(t *testing.T) { // modelName and providerName should be empty on reconstructed tracker assert.Equal(t, "", feedbackEvent.data.GetByKey("modelName").StringValue()) assert.Equal(t, "", feedbackEvent.data.GetByKey("providerName").StringValue()) + assert.False(t, feedbackEvent.data.GetByKey("modelKey").IsDefined()) + assert.Equal(t, 1, feedbackEvent.data.GetByKey("modelVersion").IntValue()) } func TestClient_CreateTracker_InvalidToken(t *testing.T) { diff --git a/ldai/config.go b/ldai/config.go index 8fa58bfd..39e455c6 100644 --- a/ldai/config.go +++ b/ldai/config.go @@ -49,6 +49,20 @@ func (c *Config) ModelName() string { return c.c.Model.Name } +// ModelKey returns the stable, unique key of the model (used for direct lookup; distinct from +// ModelName, which is not guaranteed unique). +func (c *Config) ModelKey() string { + return c.c.Model.Key +} + +// ModelVersion returns the pinned version of the model that this config variation references. +func (c *Config) ModelVersion() int { + if c.c.Model.Version == nil { + return 1 + } + return *c.c.Model.Version +} + // ModelParam returns the model parameter named by key. The second parameter is true if the key exists. func (c *Config) ModelParam(key string) (ldvalue.Value, bool) { val, ok := c.c.Model.Parameters[key] @@ -113,6 +127,8 @@ type ConfigBuilder struct { enabled bool providerName string modelName string + modelKey string + modelVersion *int modelParams map[string]ldvalue.Value modelCustomParams map[string]ldvalue.Value mode string @@ -165,6 +181,18 @@ func (cb *ConfigBuilder) WithModelName(modelName string) *ConfigBuilder { return cb } +// WithModelKey sets the stable, unique key of the model associated with the config. +func (cb *ConfigBuilder) WithModelKey(modelKey string) *ConfigBuilder { + cb.modelKey = modelKey + return cb +} + +// WithModelVersion sets the pinned version of the model associated with the config. +func (cb *ConfigBuilder) WithModelVersion(modelVersion int) *ConfigBuilder { + cb.modelVersion = &modelVersion + return cb +} + // WithProviderName sets the provider name associated with the config. func (cb *ConfigBuilder) WithProviderName(providerName string) *ConfigBuilder { cb.providerName = providerName @@ -235,6 +263,8 @@ func (cb *ConfigBuilder) Build() Config { }, Model: datamodel.Model{ Name: cb.modelName, + Key: cb.modelKey, + Version: cb.modelVersion, Parameters: maps.Clone(cb.modelParams), Custom: maps.Clone(cb.modelCustomParams), }, diff --git a/ldai/datamodel/datamodel.go b/ldai/datamodel/datamodel.go index b5de1961..1f27aa6f 100644 --- a/ldai/datamodel/datamodel.go +++ b/ldai/datamodel/datamodel.go @@ -19,6 +19,12 @@ type Model struct { // Name identifies the model. Name string `json:"name"` + // Key is the model's stable, unique key (distinct from Name, which is not guaranteed unique). + Key string `json:"modelKey,omitempty"` + + // Version is the pinned version of the model that the variation references. + Version *int `json:"modelVersion,omitempty"` + // Parameters are the model parameters, generally provided by LaunchDarkly. Parameters map[string]ldvalue.Value `json:"parameters,omitempty"` diff --git a/ldai/tracker.go b/ldai/tracker.go index 66473315..caa825ff 100644 --- a/ldai/tracker.go +++ b/ldai/tracker.go @@ -203,7 +203,11 @@ func newTrackerWithStopwatch( Set("configKey", ldvalue.String(key)). Set("version", ldvalue.Int(version)). Set("providerName", ldvalue.String(config.ProviderName())). - Set("modelName", ldvalue.String(config.ModelName())) + Set("modelName", ldvalue.String(config.ModelName())). + Set("modelVersion", ldvalue.Int(config.ModelVersion())) + if config.ModelKey() != "" { + builder.Set("modelKey", ldvalue.String(config.ModelKey())) + } if variationKey != "" { builder.Set("variationKey", ldvalue.String(variationKey)) } @@ -230,7 +234,7 @@ func (t *Tracker) logWarning(format string, args ...interface{}) { // ResumptionToken returns a URL-safe Base64-encoded token that can be used to reconstruct a tracker // in a different process (e.g., for deferred feedback). The token contains the runId, configKey, -// variationKey, and version. It does not contain modelName or providerName. +// variationKey, and version. It does not contain modelName, providerName, modelKey, or modelVersion. func (t *Tracker) ResumptionToken() string { payload := resumptionPayload{ RunID: t.runID, @@ -245,8 +249,8 @@ func (t *Tracker) ResumptionToken() string { // TrackerFromResumptionToken reconstructs a Tracker from a resumption token and the given context. // This is used for cross-process scenarios (e.g., deferred feedback) where the original tracker // is no longer available but its runId must be reused. The token is obtained from Tracker.ResumptionToken(). -// The reconstructed tracker will have empty modelName and providerName since these are not included -// in the token. +// The reconstructed tracker will have empty modelName, providerName, and modelKey, and modelVersion +// defaults to 1, since these are not included in the token. func TrackerFromResumptionToken(token string, sdk ServerSDK, context ldcontext.Context) (*Tracker, error) { decoded, err := base64.RawURLEncoding.DecodeString(token) if err != nil { @@ -262,7 +266,8 @@ func TrackerFromResumptionToken(token string, sdk ServerSDK, context ldcontext.C Set("configKey", ldvalue.String(payload.ConfigKey)). Set("version", ldvalue.Int(payload.Version)). Set("providerName", ldvalue.String("")). - Set("modelName", ldvalue.String("")) + Set("modelName", ldvalue.String("")). + Set("modelVersion", ldvalue.Int(1)) if payload.VariationKey != "" { builder.Set("variationKey", ldvalue.String(payload.VariationKey)) } diff --git a/ldai/tracker_test.go b/ldai/tracker_test.go index ef7eac15..f1b42fe3 100644 --- a/ldai/tracker_test.go +++ b/ldai/tracker_test.go @@ -55,7 +55,11 @@ func makeTrackData(configKey, variationKey string, version int, config *Config, Set("configKey", ldvalue.String(configKey)). Set("version", ldvalue.Int(version)). Set("providerName", ldvalue.String(config.ProviderName())). - Set("modelName", ldvalue.String(config.ModelName())) + Set("modelName", ldvalue.String(config.ModelName())). + Set("modelVersion", ldvalue.Int(config.ModelVersion())) + if config.ModelKey() != "" { + builder.Set("modelKey", ldvalue.String(config.ModelKey())) + } if variationKey != "" { builder.Set("variationKey", ldvalue.String(variationKey)) } @@ -579,9 +583,14 @@ func TestTracker_ResumptionToken(t *testing.T) { assert.Equal(t, 3, payload.Version) }) - t.Run("does not include modelName or providerName", func(t *testing.T) { + t.Run("does not include modelName, providerName, modelKey, or modelVersion", func(t *testing.T) { events := newMockEvents() - config := NewConfig().WithModelName("gpt-4").WithProviderName("openai").Build() + config := NewConfig(). + WithModelName("gpt-4"). + WithProviderName("openai"). + WithModelKey("my-model"). + WithModelVersion(2). + Build() tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) token := tracker.ResumptionToken() @@ -593,7 +602,61 @@ func TestTracker_ResumptionToken(t *testing.T) { _, hasModel := raw["modelName"] _, hasProvider := raw["providerName"] + _, hasModelKey := raw["modelKey"] + _, hasModelVersion := raw["modelVersion"] assert.False(t, hasModel, "token should not contain modelName") assert.False(t, hasProvider, "token should not contain providerName") + assert.False(t, hasModelKey, "token should not contain modelKey") + assert.False(t, hasModelVersion, "token should not contain modelVersion") + }) +} + +func TestTracker_TrackDataIncludesModelKeyAndVersion(t *testing.T) { + t.Run("includes modelKey and modelVersion when set on config", func(t *testing.T) { + events := newMockEvents() + config := NewConfig(). + WithModelName("gpt-4"). + WithModelKey("my-model"). + WithModelVersion(2). + Build() + tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) + assert.NoError(t, tracker.TrackSuccess()) + + require.Len(t, events.events, 1) + data := events.events[0].data + assert.Equal(t, "my-model", data.GetByKey("modelKey").StringValue()) + assert.Equal(t, 2, data.GetByKey("modelVersion").IntValue()) }) + + t.Run("omits modelKey when empty but still includes modelVersion", func(t *testing.T) { + events := newMockEvents() + config := NewConfig().WithModelName("gpt-4").Build() + tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) + assert.NoError(t, tracker.TrackSuccess()) + + require.Len(t, events.events, 1) + data := events.events[0].data + assert.False(t, data.GetByKey("modelKey").IsDefined()) + assert.Equal(t, 1, data.GetByKey("modelVersion").IntValue()) + }) +} + +func TestTrackerFromResumptionToken_ModelKeyAndVersionDefaults(t *testing.T) { + mockSDK := newMockSDK(nil, nil) + config := NewConfig(). + WithModelName("gpt-4"). + WithModelKey("my-model"). + WithModelVersion(2). + Build() + tracker := newTracker(mockSDK, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, mockSDK.log.Loggers) + + token := tracker.ResumptionToken() + reconstructed, err := TrackerFromResumptionToken(token, mockSDK, ldcontext.New("key")) + require.NoError(t, err) + assert.NoError(t, reconstructed.TrackSuccess()) + + require.Len(t, mockSDK.events, 1) + data := mockSDK.events[0].data + assert.False(t, data.GetByKey("modelKey").IsDefined()) + assert.Equal(t, 1, data.GetByKey("modelVersion").IntValue()) } From 19e52dbc4e3c00e70d5585c91db7b5cb51a5bd9e Mon Sep 17 00:00:00 2001 From: Anthony Torns II Date: Wed, 15 Jul 2026 16:14:24 -0700 Subject: [PATCH 2/3] Read modelKey/modelVersion from _ldMeta, matching gonfalon PR 67230 Gonfalon moved these two fields from the payload's model object to _ldMeta (launchdarkly/gonfalon#67230) to avoid modelVersion reading as the underlying LLM's own version. Field names are unchanged; only the JSON location moves. The SDK's public Config.ModelKey()/ModelVersion() and the tracker's stamped event fields are untouched. Co-Authored-By: Claude Fable 5 --- ldai/client.go | 6 +++--- ldai/client_test.go | 8 ++++---- ldai/config.go | 12 ++++++------ ldai/datamodel/datamodel.go | 12 ++++++------ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index afebf6be..8a0df8d9 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -156,15 +156,15 @@ func (c *Client) evaluateConfig( builder := NewConfig(). WithModelName(parsed.Model.Name). - WithModelKey(parsed.Model.Key). + WithModelKey(parsed.Meta.ModelKey). WithProviderName(parsed.Provider.Name). WithEnabled(parsed.Meta.Enabled). WithMode(parsed.Mode). WithEvaluationMetricKey(parsed.EvaluationMetricKey). WithEvaluationMetricKeys(parsed.EvaluationMetricKeys). WithJudgeConfiguration(parsed.JudgeConfiguration) - if parsed.Model.Version != nil { - builder.WithModelVersion(*parsed.Model.Version) + if parsed.Meta.ModelVersion != nil { + builder.WithModelVersion(*parsed.Meta.ModelVersion) } for k, v := range parsed.Model.Parameters { diff --git a/ldai/client_test.go b/ldai/client_test.go index 30847178..ad2b7313 100644 --- a/ldai/client_test.go +++ b/ldai/client_test.go @@ -165,13 +165,13 @@ func TestParseModelKeyAndVersion(t *testing.T) { }, { name: "modelKey and modelVersion set", - json: []byte(`{"model": {"name": "gpt-4", "modelKey": "my-model", "modelVersion": 2}}`), + json: []byte(`{"model": {"name": "gpt-4"}, "_ldMeta": {"modelKey": "my-model", "modelVersion": 2}}`), expectedKey: "my-model", expectedVersion: 2, }, { name: "modelVersion only", - json: []byte(`{"model": {"name": "gpt-4", "modelVersion": 3}}`), + json: []byte(`{"model": {"name": "gpt-4"}, "_ldMeta": {"modelVersion": 3}}`), expectedKey: "", expectedVersion: 3, }, @@ -194,8 +194,8 @@ func TestParseModelKeyAndVersion(t *testing.T) { func TestCreateTrackerStampsModelKeyAndVersionOnTrackData(t *testing.T) { configJSON := []byte(`{ - "_ldMeta": {"variationKey": "var-1", "enabled": true, "version": 1}, - "model": {"name": "gpt-4", "modelKey": "my-model", "modelVersion": 2}, + "_ldMeta": {"variationKey": "var-1", "enabled": true, "version": 1, "modelKey": "my-model", "modelVersion": 2}, + "model": {"name": "gpt-4"}, "provider": {"name": "openai"}, "messages": [{"content": "hello", "role": "user"}] }`) diff --git a/ldai/config.go b/ldai/config.go index 39e455c6..ff246e9c 100644 --- a/ldai/config.go +++ b/ldai/config.go @@ -52,15 +52,15 @@ func (c *Config) ModelName() string { // ModelKey returns the stable, unique key of the model (used for direct lookup; distinct from // ModelName, which is not guaranteed unique). func (c *Config) ModelKey() string { - return c.c.Model.Key + return c.c.Meta.ModelKey } // ModelVersion returns the pinned version of the model that this config variation references. func (c *Config) ModelVersion() int { - if c.c.Model.Version == nil { + if c.c.Meta.ModelVersion == nil { return 1 } - return *c.c.Model.Version + return *c.c.Meta.ModelVersion } // ModelParam returns the model parameter named by key. The second parameter is true if the key exists. @@ -259,12 +259,12 @@ func (cb *ConfigBuilder) Build() Config { c: datamodel.Config{ Messages: slices.Clone(cb.messages), Meta: datamodel.Meta{ - Enabled: cb.enabled, + Enabled: cb.enabled, + ModelKey: cb.modelKey, + ModelVersion: cb.modelVersion, }, Model: datamodel.Model{ Name: cb.modelName, - Key: cb.modelKey, - Version: cb.modelVersion, Parameters: maps.Clone(cb.modelParams), Custom: maps.Clone(cb.modelCustomParams), }, diff --git a/ldai/datamodel/datamodel.go b/ldai/datamodel/datamodel.go index 1f27aa6f..8a149f53 100644 --- a/ldai/datamodel/datamodel.go +++ b/ldai/datamodel/datamodel.go @@ -12,6 +12,12 @@ type Meta struct { // Version is the version of the Variation. Version *int `json:"version,omitempty"` + + // ModelKey is the model's stable, unique key (distinct from Model.Name, which is not guaranteed unique). + ModelKey string `json:"modelKey,omitempty"` + + // ModelVersion is the pinned version of the model that the variation references. + ModelVersion *int `json:"modelVersion,omitempty"` } // Model defines the serialization format for a model. @@ -19,12 +25,6 @@ type Model struct { // Name identifies the model. Name string `json:"name"` - // Key is the model's stable, unique key (distinct from Name, which is not guaranteed unique). - Key string `json:"modelKey,omitempty"` - - // Version is the pinned version of the model that the variation references. - Version *int `json:"modelVersion,omitempty"` - // Parameters are the model parameters, generally provided by LaunchDarkly. Parameters map[string]ldvalue.Value `json:"parameters,omitempty"` From 5cb642364f3dbca6db7b92f400de2202a099c785 Mon Sep 17 00:00:00 2001 From: Anthony Torns II Date: Fri, 17 Jul 2026 13:00:37 -0700 Subject: [PATCH 3/3] fix(ldai): stop exposing ModelKey/ModelVersion on the public Config modelKey/modelVersion should only be visible via the tracker's stamped event data, mirroring how VariationKey/Version are treated. They now flow from the already-parsed _ldMeta fields straight into newTracker as explicit parameters instead of being read off Config. Co-Authored-By: Claude Sonnet 5 --- ldai/client.go | 14 ++++--- ldai/client_test.go | 18 +++++++-- ldai/config.go | 32 +--------------- ldai/tracker.go | 13 +++++-- ldai/tracker_test.go | 91 +++++++++++++++++++++----------------------- 5 files changed, 76 insertions(+), 92 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index 8a0df8d9..c8d85f33 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -114,7 +114,7 @@ func (c *Client) CreateTracker(token string, context ldcontext.Context) (*Tracke // returns the resulting Config. Used for all error-path returns in evaluateConfig. func (c *Client) returnDefault(key string, context ldcontext.Context, def Config) Config { def.trackerFactory = func() *Tracker { - return newTracker(c.sdk, newRunID(), key, "", 1, context, &def, c.logger) + return newTracker(c.sdk, newRunID(), key, "", 1, "", 1, context, &def, c.logger) } return def } @@ -156,16 +156,12 @@ func (c *Client) evaluateConfig( builder := NewConfig(). WithModelName(parsed.Model.Name). - WithModelKey(parsed.Meta.ModelKey). WithProviderName(parsed.Provider.Name). WithEnabled(parsed.Meta.Enabled). WithMode(parsed.Mode). WithEvaluationMetricKey(parsed.EvaluationMetricKey). WithEvaluationMetricKeys(parsed.EvaluationMetricKeys). WithJudgeConfiguration(parsed.JudgeConfiguration) - if parsed.Meta.ModelVersion != nil { - builder.WithModelVersion(*parsed.Meta.ModelVersion) - } for k, v := range parsed.Model.Parameters { builder.WithModelParam(k, v) @@ -193,9 +189,15 @@ func (c *Client) evaluateConfig( version = *parsed.Meta.Version } + modelVersion := 1 + if parsed.Meta.ModelVersion != nil { + modelVersion = *parsed.Meta.ModelVersion + } + variationKey := parsed.Meta.VariationKey + modelKey := parsed.Meta.ModelKey cfg.trackerFactory = func() *Tracker { - return newTracker(c.sdk, newRunID(), key, variationKey, version, context, &cfg, c.logger) + return newTracker(c.sdk, newRunID(), key, variationKey, version, modelKey, modelVersion, context, &cfg, c.logger) } return cfg diff --git a/ldai/client_test.go b/ldai/client_test.go index ad2b7313..04fc88d9 100644 --- a/ldai/client_test.go +++ b/ldai/client_test.go @@ -151,6 +151,9 @@ func TestParseModelName(t *testing.T) { } func TestParseModelKeyAndVersion(t *testing.T) { + // modelKey/modelVersion are intentionally not exposed on Config (they'd read as properties of + // the LLM itself, e.g. a version like "5.4"); the only place they surface is the tracker's + // stamped event data, mirroring variationKey/version. tests := []struct { name string json []byte @@ -179,15 +182,22 @@ func TestParseModelKeyAndVersion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - client, err := NewClient(newMockSDK(test.json, nil)) + mockSDK := newMockSDK(test.json, nil) + client, err := NewClient(mockSDK) require.NoError(t, err) require.NotNil(t, client) + mockSDK.events = nil defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - assert.Equal(t, test.expectedKey, cfg.ModelKey()) - assert.Equal(t, test.expectedVersion, cfg.ModelVersion()) + tracker := cfg.CreateTracker() + require.NotNil(t, tracker) + assert.NoError(t, tracker.TrackSuccess()) + + require.NotEmpty(t, mockSDK.events) + data := mockSDK.events[len(mockSDK.events)-1].data + assert.Equal(t, test.expectedKey, data.GetByKey("modelKey").StringValue()) + assert.Equal(t, test.expectedVersion, data.GetByKey("modelVersion").IntValue()) }) } } diff --git a/ldai/config.go b/ldai/config.go index ff246e9c..8fa58bfd 100644 --- a/ldai/config.go +++ b/ldai/config.go @@ -49,20 +49,6 @@ func (c *Config) ModelName() string { return c.c.Model.Name } -// ModelKey returns the stable, unique key of the model (used for direct lookup; distinct from -// ModelName, which is not guaranteed unique). -func (c *Config) ModelKey() string { - return c.c.Meta.ModelKey -} - -// ModelVersion returns the pinned version of the model that this config variation references. -func (c *Config) ModelVersion() int { - if c.c.Meta.ModelVersion == nil { - return 1 - } - return *c.c.Meta.ModelVersion -} - // ModelParam returns the model parameter named by key. The second parameter is true if the key exists. func (c *Config) ModelParam(key string) (ldvalue.Value, bool) { val, ok := c.c.Model.Parameters[key] @@ -127,8 +113,6 @@ type ConfigBuilder struct { enabled bool providerName string modelName string - modelKey string - modelVersion *int modelParams map[string]ldvalue.Value modelCustomParams map[string]ldvalue.Value mode string @@ -181,18 +165,6 @@ func (cb *ConfigBuilder) WithModelName(modelName string) *ConfigBuilder { return cb } -// WithModelKey sets the stable, unique key of the model associated with the config. -func (cb *ConfigBuilder) WithModelKey(modelKey string) *ConfigBuilder { - cb.modelKey = modelKey - return cb -} - -// WithModelVersion sets the pinned version of the model associated with the config. -func (cb *ConfigBuilder) WithModelVersion(modelVersion int) *ConfigBuilder { - cb.modelVersion = &modelVersion - return cb -} - // WithProviderName sets the provider name associated with the config. func (cb *ConfigBuilder) WithProviderName(providerName string) *ConfigBuilder { cb.providerName = providerName @@ -259,9 +231,7 @@ func (cb *ConfigBuilder) Build() Config { c: datamodel.Config{ Messages: slices.Clone(cb.messages), Meta: datamodel.Meta{ - Enabled: cb.enabled, - ModelKey: cb.modelKey, - ModelVersion: cb.modelVersion, + Enabled: cb.enabled, }, Model: datamodel.Model{ Name: cb.modelName, diff --git a/ldai/tracker.go b/ldai/tracker.go index caa825ff..b951c480 100644 --- a/ldai/tracker.go +++ b/ldai/tracker.go @@ -174,11 +174,14 @@ func newTracker( key string, variationKey string, version int, + modelKey string, + modelVersion int, ctx ldcontext.Context, config *Config, loggers interfaces.LDLoggers, ) *Tracker { - return newTrackerWithStopwatch(events, runID, key, variationKey, version, ctx, config, loggers, &defaultStopwatch{}) + return newTrackerWithStopwatch( + events, runID, key, variationKey, version, modelKey, modelVersion, ctx, config, loggers, &defaultStopwatch{}) } // newTrackerWithStopwatch creates a new Tracker with the specified runID, key, event sink, config, context, loggers, @@ -189,6 +192,8 @@ func newTrackerWithStopwatch( key string, variationKey string, version int, + modelKey string, + modelVersion int, ctx ldcontext.Context, config *Config, loggers interfaces.LDLoggers, @@ -204,9 +209,9 @@ func newTrackerWithStopwatch( Set("version", ldvalue.Int(version)). Set("providerName", ldvalue.String(config.ProviderName())). Set("modelName", ldvalue.String(config.ModelName())). - Set("modelVersion", ldvalue.Int(config.ModelVersion())) - if config.ModelKey() != "" { - builder.Set("modelKey", ldvalue.String(config.ModelKey())) + Set("modelVersion", ldvalue.Int(modelVersion)) + if modelKey != "" { + builder.Set("modelKey", ldvalue.String(modelKey)) } if variationKey != "" { builder.Set("variationKey", ldvalue.String(variationKey)) diff --git a/ldai/tracker_test.go b/ldai/tracker_test.go index f1b42fe3..33ef0bec 100644 --- a/ldai/tracker_test.go +++ b/ldai/tracker_test.go @@ -39,26 +39,32 @@ func (m *mockEvents) TrackMetric(eventName string, context ldcontext.Context, me func TestTracker_NewPanicsWithNilConfig(t *testing.T) { assert.Panics(t, func() { - newTracker(newMockEvents(), newRunID(), "key", "variationKey", 1, ldcontext.New("key"), nil, nil) + newTracker(newMockEvents(), newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), nil, nil) }) } func TestTracker_NewDoesNotPanicWithConfig(t *testing.T) { assert.NotPanics(t, func() { - newTracker(newMockEvents(), newRunID(), "key", "variationKey", 1, ldcontext.New("key"), &Config{}, nil) + newTracker(newMockEvents(), newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), &Config{}, nil) }) } func makeTrackData(configKey, variationKey string, version int, config *Config, runId string) ldvalue.Value { + return makeTrackDataWithModel(configKey, variationKey, version, "", 1, config, runId) +} + +func makeTrackDataWithModel( + configKey, variationKey string, version int, modelKey string, modelVersion int, config *Config, runId string, +) ldvalue.Value { builder := ldvalue.ObjectBuild(). Set("runId", ldvalue.String(runId)). Set("configKey", ldvalue.String(configKey)). Set("version", ldvalue.Int(version)). Set("providerName", ldvalue.String(config.ProviderName())). Set("modelName", ldvalue.String(config.ModelName())). - Set("modelVersion", ldvalue.Int(config.ModelVersion())) - if config.ModelKey() != "" { - builder.Set("modelKey", ldvalue.String(config.ModelKey())) + Set("modelVersion", ldvalue.Int(modelVersion)) + if modelKey != "" { + builder.Set("modelKey", ldvalue.String(modelKey)) } if variationKey != "" { builder.Set("variationKey", ldvalue.String(variationKey)) @@ -77,7 +83,7 @@ func extractRunId(t *testing.T, events *mockEvents) string { func TestTracker_TrackSuccess(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackSuccess()) runId := extractRunId(t, events) @@ -96,7 +102,7 @@ func TestTracker_TrackSuccess(t *testing.T) { func TestTracker_TrackError(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 2, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 2, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackError()) runId := extractRunId(t, events) @@ -115,7 +121,7 @@ func TestTracker_TrackError(t *testing.T) { func TestTracker_TrackRequest(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 3, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 3, "", 1, ldcontext.New("key"), config, nil) expectedResponse := ProviderResponse{ Usage: TokenUsage{ @@ -177,7 +183,7 @@ func TestTracker_TrackRequestReceivesConfig(t *testing.T) { Enable(). Build() - tracker := newTracker(events, newRunID(), "key", "variationKey", 4, ldcontext.New("key"), &expectedConfig, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 4, "", 1, ldcontext.New("key"), &expectedConfig, nil) var gotConfig *Config _, _ = tracker.TrackRequest(func(c *Config) (ProviderResponse, error) { @@ -201,7 +207,7 @@ func TestTracker_LatencyMeasuredIfNotProvided(t *testing.T) { config := &Config{} tracker := newTrackerWithStopwatch( - events, newRunID(), "key", "variationKey", 5, ldcontext.New("key"), config, nil, mockStopwatch(42*time.Millisecond)) + events, newRunID(), "key", "variationKey", 5, "", 1, ldcontext.New("key"), config, nil, mockStopwatch(42*time.Millisecond)) expectedResponse := ProviderResponse{ Usage: TokenUsage{ @@ -225,7 +231,7 @@ func TestTracker_LatencyMeasuredIfNotProvided(t *testing.T) { func TestTracker_TrackDuration(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 6, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 6, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackDuration(time.Millisecond*10)) @@ -244,7 +250,7 @@ func TestTracker_TrackFeedback(t *testing.T) { t.Run("positive feedback", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 7, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 7, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackFeedback(FeedbackPositive)) @@ -262,7 +268,7 @@ func TestTracker_TrackFeedback(t *testing.T) { t.Run("negative feedback", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 7, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 7, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackFeedback(FeedbackNegative)) @@ -280,7 +286,7 @@ func TestTracker_TrackFeedback(t *testing.T) { t.Run("invalid feedback returns error", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 7, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 7, "", 1, ldcontext.New("key"), config, nil) assert.Error(t, tracker.TrackFeedback("not a valid feedback value")) assert.Empty(t, events.events) @@ -291,7 +297,7 @@ func TestTracker_TrackTokens(t *testing.T) { t.Run("only one field set, only one event", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 8, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 8, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackTokens(TokenUsage{ Total: 42, @@ -311,7 +317,7 @@ func TestTracker_TrackTokens(t *testing.T) { t.Run("all fields set, all events", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 9, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 9, "", 1, ldcontext.New("key"), config, nil) assert.NoError(t, tracker.TrackTokens(TokenUsage{ Total: 42, @@ -348,7 +354,7 @@ func TestTracker_TrackTokens(t *testing.T) { func TestTracker_GetSummary(t *testing.T) { t.Run("empty summary when nothing tracked", func(t *testing.T) { events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 10, ldcontext.New("key"), &Config{}, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 10, "", 1, ldcontext.New("key"), &Config{}, nil) summary := tracker.GetSummary() @@ -361,7 +367,7 @@ func TestTracker_GetSummary(t *testing.T) { t.Run("first duration is returned", func(t *testing.T) { events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 11, ldcontext.New("key"), &Config{}, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 11, "", 1, ldcontext.New("key"), &Config{}, events.log.Loggers) _ = tracker.TrackDuration(time.Millisecond * 10) _ = tracker.TrackDuration(time.Millisecond * 20) @@ -374,7 +380,7 @@ func TestTracker_GetSummary(t *testing.T) { t.Run("first feedback is returned", func(t *testing.T) { events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 12, ldcontext.New("key"), &Config{}, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 12, "", 1, ldcontext.New("key"), &Config{}, events.log.Loggers) _ = tracker.TrackFeedback(FeedbackPositive) _ = tracker.TrackFeedback(FeedbackNegative) @@ -387,7 +393,7 @@ func TestTracker_GetSummary(t *testing.T) { t.Run("success status tracked correctly", func(t *testing.T) { events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 13, ldcontext.New("key"), &Config{}, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 13, "", 1, ldcontext.New("key"), &Config{}, nil) _ = tracker.TrackSuccess() @@ -399,7 +405,7 @@ func TestTracker_GetSummary(t *testing.T) { t.Run("time to first token is returned", func(t *testing.T) { events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 14, ldcontext.New("key"), &Config{}, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 14, "", 1, ldcontext.New("key"), &Config{}, nil) duration := time.Millisecond * 30 _ = tracker.TrackTimeToFirstToken(duration) @@ -412,7 +418,7 @@ func TestTracker_GetSummary(t *testing.T) { t.Run("token usage is returned", func(t *testing.T) { events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 15, ldcontext.New("key"), &Config{}, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 15, "", 1, ldcontext.New("key"), &Config{}, nil) usage := TokenUsage{ Total: 100, @@ -431,7 +437,7 @@ func TestTracker_GetSummary(t *testing.T) { func TestTracker_RunIdPresentInTrackData(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, nil) _ = tracker.TrackSuccess() require.NotEmpty(t, events.events) @@ -444,7 +450,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackDuration only tracks once", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackDuration(10*time.Millisecond)) assert.NoError(t, tracker.TrackDuration(20*time.Millisecond)) @@ -461,7 +467,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackTimeToFirstToken only tracks once", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackTimeToFirstToken(10*time.Millisecond)) assert.NoError(t, tracker.TrackTimeToFirstToken(20*time.Millisecond)) @@ -478,7 +484,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackTokens only tracks once", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackTokens(TokenUsage{Total: 10})) assert.NoError(t, tracker.TrackTokens(TokenUsage{Total: 20})) @@ -495,7 +501,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackFeedback only tracks once", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackFeedback(FeedbackPositive)) assert.NoError(t, tracker.TrackFeedback(FeedbackNegative)) @@ -512,7 +518,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackSuccess only tracks once", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackSuccess()) assert.NoError(t, tracker.TrackSuccess()) @@ -529,7 +535,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackError only tracks once", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackError()) assert.NoError(t, tracker.TrackError()) @@ -546,7 +552,7 @@ func TestTracker_AtMostOnce(t *testing.T) { t.Run("TrackSuccess then TrackError only tracks success", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) + tracker := newTracker(events, newRunID(), "key", "variationKey", 1, "", 1, ldcontext.New("key"), config, events.log.Loggers) assert.NoError(t, tracker.TrackSuccess()) assert.NoError(t, tracker.TrackError()) @@ -560,7 +566,7 @@ func TestTracker_ResumptionToken(t *testing.T) { t.Run("produces valid base64url-encoded token", func(t *testing.T) { events := newMockEvents() config := &Config{} - tracker := newTracker(events, newRunID(), "my-config", "var-1", 3, ldcontext.New("key"), config, nil) + tracker := newTracker(events, newRunID(), "my-config", "var-1", 3, "", 1, ldcontext.New("key"), config, nil) token := tracker.ResumptionToken() assert.NotEmpty(t, token) @@ -588,10 +594,8 @@ func TestTracker_ResumptionToken(t *testing.T) { config := NewConfig(). WithModelName("gpt-4"). WithProviderName("openai"). - WithModelKey("my-model"). - WithModelVersion(2). Build() - tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) + tracker := newTracker(events, newRunID(), "key", "var", 1, "my-model", 2, ldcontext.New("key"), &config, nil) token := tracker.ResumptionToken() decoded, err := base64.RawURLEncoding.DecodeString(token) @@ -614,12 +618,8 @@ func TestTracker_ResumptionToken(t *testing.T) { func TestTracker_TrackDataIncludesModelKeyAndVersion(t *testing.T) { t.Run("includes modelKey and modelVersion when set on config", func(t *testing.T) { events := newMockEvents() - config := NewConfig(). - WithModelName("gpt-4"). - WithModelKey("my-model"). - WithModelVersion(2). - Build() - tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) + config := NewConfig().WithModelName("gpt-4").Build() + tracker := newTracker(events, newRunID(), "key", "var", 1, "my-model", 2, ldcontext.New("key"), &config, nil) assert.NoError(t, tracker.TrackSuccess()) require.Len(t, events.events, 1) @@ -631,7 +631,7 @@ func TestTracker_TrackDataIncludesModelKeyAndVersion(t *testing.T) { t.Run("omits modelKey when empty but still includes modelVersion", func(t *testing.T) { events := newMockEvents() config := NewConfig().WithModelName("gpt-4").Build() - tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) + tracker := newTracker(events, newRunID(), "key", "var", 1, "", 1, ldcontext.New("key"), &config, nil) assert.NoError(t, tracker.TrackSuccess()) require.Len(t, events.events, 1) @@ -643,12 +643,9 @@ func TestTracker_TrackDataIncludesModelKeyAndVersion(t *testing.T) { func TestTrackerFromResumptionToken_ModelKeyAndVersionDefaults(t *testing.T) { mockSDK := newMockSDK(nil, nil) - config := NewConfig(). - WithModelName("gpt-4"). - WithModelKey("my-model"). - WithModelVersion(2). - Build() - tracker := newTracker(mockSDK, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, mockSDK.log.Loggers) + config := NewConfig().WithModelName("gpt-4").Build() + tracker := newTracker( + mockSDK, newRunID(), "key", "var", 1, "my-model", 2, ldcontext.New("key"), &config, mockSDK.log.Loggers) token := tracker.ResumptionToken() reconstructed, err := TrackerFromResumptionToken(token, mockSDK, ldcontext.New("key"))