From 0bea1073ad3549345ad87647f9a6499a841897cd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 8 Jul 2026 03:24:19 +0000 Subject: [PATCH] feat(server-ai): stamp modelKey and modelVersion on AI usage events (AIC-2856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read modelKey and modelVersion from the AI Config variation payload and stamp them on all LdAiConfigTracker metric event payloads alongside existing modelName/providerName fields. - Add ModelKey/ModelVersion to ModelConfig - Parse new fields from variation payload in ConfigFactory - Include modelVersion unconditionally (default 1) and modelKey when non-empty in track data; exclude both from resumption token - Additive/backward compatible — older payloads without the new fields continue to work Part of AIC-2849. Depends on AIC-2876 (backend payload). Co-authored-by: Anthony Torns II --- .../sdk/server-ai/src/Config/ConfigFactory.cs | 7 +- .../server-ai/src/Config/LdAiConfigTypes.cs | 20 ++- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 13 +- .../server-ai/test/ConfigFactoryParserTest.cs | 114 ++++++++++++++++++ .../test/LdAiAgentGraphConfigTest.cs | 2 +- pkgs/sdk/server-ai/test/LdAiClientTest.cs | 4 +- .../server-ai/test/LdAiConfigTrackerTest.cs | 78 ++++++++++++ 7 files changed, 231 insertions(+), 7 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs index e6eee49d..c2aff1cf 100644 --- a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs +++ b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs @@ -305,6 +305,8 @@ private Func TrackerFactoryFor(Context context, context, cfg.Model?.Name, cfg.Provider?.Name, + cfg.Model?.ModelKey, + cfg.Model?.ModelVersion ?? 1, graphKey); } @@ -326,7 +328,10 @@ private static LdAiConfigTypes.ModelConfig ParseModel(LdValue modelValue) var name = modelValue.Get("name").AsString ?? ""; var parameters = LdValueObjectToDictionary(modelValue.Get("parameters")); var custom = LdValueObjectToDictionary(modelValue.Get("custom")); - return new LdAiConfigTypes.ModelConfig(name, parameters, custom); + var modelKey = modelValue.Get("modelKey").AsString; + var modelVersionValue = modelValue.Get("modelVersion"); + var modelVersion = modelVersionValue.IsNull ? 1 : modelVersionValue.AsInt; + return new LdAiConfigTypes.ModelConfig(name, parameters, custom, modelKey, modelVersion); } private static LdAiConfigTypes.ProviderConfig ParseProvider(LdValue providerValue) diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs index a4278ab7..25115bba 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs @@ -71,7 +71,23 @@ public sealed record ModelConfig /// public readonly IReadOnlyDictionary Custom; - internal ModelConfig(string name, IReadOnlyDictionary parameters, IReadOnlyDictionary custom) + /// + /// The stable, unique key of the model (used for direct lookup; distinct from + /// , which is not guaranteed unique). + /// + public readonly string ModelKey; + + /// + /// The pinned version of the model that this config variation references. + /// + public readonly int ModelVersion; + + internal ModelConfig( + string name, + IReadOnlyDictionary parameters, + IReadOnlyDictionary custom, + string modelKey = null, + int modelVersion = 1) { Name = name; // Materialize into an ImmutableDictionary so a consumer that downcasts to @@ -80,6 +96,8 @@ internal ModelConfig(string name, IReadOnlyDictionary parameter // NotSupportedException at runtime, matching the typed read-only promise. Parameters = parameters?.ToImmutableDictionary() ?? ImmutableDictionary.Empty; Custom = custom?.ToImmutableDictionary() ?? ImmutableDictionary.Empty; + ModelKey = modelKey; + ModelVersion = modelVersion; } } diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 70a2182e..8ad6721e 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -76,7 +76,7 @@ public class LdAiConfigTracker : ILdAiConfigTracker /// internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string configKey, string variationKey, int version, Context context, string modelName, string providerName, - string graphKey = null) + string modelKey = null, int modelVersion = 1, string graphKey = null) { _client = client ?? throw new ArgumentNullException(nameof(client)); _configKey = configKey ?? throw new ArgumentNullException(nameof(configKey)); @@ -96,6 +96,7 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf { "version", LdValue.Of(_version) }, { "modelName", LdValue.Of(_modelName) }, { "providerName", LdValue.Of(_providerName) }, + { "modelVersion", LdValue.Of(modelVersion) }, }; if (!string.IsNullOrEmpty(_graphKey)) { @@ -105,6 +106,10 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf { trackDataBuilder.Add("variationKey", LdValue.Of(_variationKey)); } + if (!string.IsNullOrEmpty(modelKey)) + { + trackDataBuilder.Add("modelKey", LdValue.Of(modelKey)); + } _trackData = LdValue.ObjectFrom(trackDataBuilder); _resumptionToken = new Lazy(BuildResumptionToken); @@ -118,6 +123,7 @@ private string BuildResumptionToken() // Utf8JsonWriter gives stable key ordering and avoids the runtime cost of // anonymous-type reflection. The wire format omits empty optional fields so that // resumption tokens round-trip exactly for configs that never carried them. + // modelName, providerName, modelKey, and modelVersion are not included. using var stream = new MemoryStream(); using (var writer = new Utf8JsonWriter(stream)) { @@ -426,7 +432,8 @@ private LdValue MergeTrackData(string key, LdValue value) /// process or at a later time. /// /// The reconstructed tracker will have empty model and provider names, as these are not - /// included in the resumption token. + /// included in the resumption token. modelKey and modelVersion are also not included; + /// modelVersion defaults to 1 on reconstructed trackers. /// /// the resumption token obtained from /// the LaunchDarkly client @@ -466,7 +473,7 @@ public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyC } return new LdAiConfigTracker(client, payload.RunId, payload.ConfigKey, - payload.VariationKey, payload.Version ?? 1, context, "", "", payload.GraphKey); + payload.VariationKey, payload.Version ?? 1, context, "", "", graphKey: payload.GraphKey); } private class ResumptionPayload diff --git a/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs b/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs index 004cebce..ec04ee15 100644 --- a/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs +++ b/pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using Moq; using Xunit; namespace LaunchDarkly.Sdk.Server.Ai; @@ -204,4 +206,116 @@ public void ParseAllFields_WithNoNewFields_ReturnsNullOrEmptyWithoutError() Assert.Null(judgeConfig); Assert.Null(evaluationMetricKey); } + + [Fact] + public void CompletionConfigReadsModelKeyAndVersionFromFlag() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => + x.JsonVariation("model-config-with-key-version", It.IsAny(), It.IsAny())) + .Returns(LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("v1"), + ["version"] = LdValue.Of(1) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4"), + ["modelKey"] = LdValue.Of("my-model"), + ["modelVersion"] = LdValue.Of(2) + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var result = client.CompletionConfig("model-config-with-key-version", Context.New("user-key")); + + Assert.Equal("my-model", result.Model.ModelKey); + Assert.Equal(2, result.Model.ModelVersion); + } + + [Fact] + public void CompletionConfigDefaultsModelVersionWhenAbsent() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => + x.JsonVariation("model-config-no-version", It.IsAny(), It.IsAny())) + .Returns(LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("v1"), + ["version"] = LdValue.Of(1) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4") + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var result = client.CompletionConfig("model-config-no-version", Context.New("user-key")); + + Assert.Null(result.Model.ModelKey); + Assert.Equal(1, result.Model.ModelVersion); + } + + [Fact] + public void CreateTrackerStampsModelKeyAndVersionOnTrackData() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => + x.JsonVariation("my-config-key", It.IsAny(), It.IsAny())) + .Returns(LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("var-abc"), + ["version"] = LdValue.Of(7) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4"), + ["modelKey"] = LdValue.Of("my-model"), + ["modelVersion"] = LdValue.Of(2) + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var context = Context.New("user-key"); + var config = client.CompletionConfig("my-config-key", context); + var tracker = config.CreateTracker(); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelKey").AsString == "my-model" && + d.Get("modelVersion").AsInt == 2), + 1.0f), Times.Once); + } } diff --git a/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs b/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs index 7e47c846..b5ba1550 100644 --- a/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs @@ -24,7 +24,7 @@ private static LdAiConfigTracker MakeTracker(ILaunchDarklyClient client, Context string graphKey = null) { return new LdAiConfigTracker(client, Guid.NewGuid().ToString(), "config-key", - "v1", 1, context, "model", "provider", graphKey); + "v1", 1, context, "model", "provider", graphKey: graphKey); } // Test 1: Tracker created with graphKey → events include graphKey in data diff --git a/pkgs/sdk/server-ai/test/LdAiClientTest.cs b/pkgs/sdk/server-ai/test/LdAiClientTest.cs index dc4034f1..7f133f27 100644 --- a/pkgs/sdk/server-ai/test/LdAiClientTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiClientTest.cs @@ -668,7 +668,9 @@ public void CreateTrackerFromResumptionTokenSetsEmptyModelAndProvider() d.Get("variationKey").AsString == "var-1" && d.Get("version").AsInt == 2 && d.Get("modelName").AsString == "" && - d.Get("providerName").AsString == ""), + d.Get("providerName").AsString == "" && + d.Get("modelVersion").AsInt == 1 && + d.Get("modelKey").IsNull), 1.0f), Times.Once); } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 0bd7793a..f701ad25 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -686,6 +686,84 @@ public void ResumptionTokenContainsExpectedFields() // modelName and providerName should NOT be in the token Assert.False(doc.RootElement.TryGetProperty("modelName", out _)); Assert.False(doc.RootElement.TryGetProperty("providerName", out _)); + Assert.False(doc.RootElement.TryGetProperty("modelKey", out _)); + Assert.False(doc.RootElement.TryGetProperty("modelVersion", out _)); + } + + [Fact] + public void TrackDataIncludesModelVersionByDefault() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "config-key", "variation-key", 3, context, "fakeModel", "fakeProvider"); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => d.Get("modelVersion").AsInt == 1), 1.0f), Times.Once); + } + + [Fact] + public void TrackDataIncludesModelKeyWhenSet() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "config-key", "variation-key", 3, context, "fakeModel", "fakeProvider", + modelKey: "my-model", modelVersion: 2); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelKey").AsString == "my-model" && + d.Get("modelVersion").AsInt == 2), + 1.0f), Times.Once); + } + + [Fact] + public void TrackDataOmitsModelKeyWhenEmpty() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "config-key", "variation-key", 3, context, "fakeModel", "fakeProvider", + modelKey: "", modelVersion: 3); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelVersion").AsInt == 3 && + d.Get("modelKey").IsNull), + 1.0f), Times.Once); + } + + [Fact] + public void FromResumptionTokenDefaultsModelVersionAndOmitsModelKey() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var payload = JsonSerializer.Serialize(new + { + runId = "test-run-id", + configKey = "test-key", + variationKey = "var-1", + version = 2, + }); + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)); + var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + + var tracker = LdAiConfigTracker.FromResumptionToken(token, mockClient.Object, context); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("modelVersion").AsInt == 1 && + d.Get("modelKey").IsNull), + 1.0f), Times.Once); } [Fact]