Skip to content

Commit 792d33c

Browse files
committed
fix: catch all exceptions in Judge.evaluate() to prevent blank metricKey from escaping
1 parent a6a1ca1 commit 792d33c

1 file changed

Lines changed: 49 additions & 52 deletions

File tree

  • lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai

lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/Judge.java

Lines changed: 49 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -104,69 +104,66 @@ public JudgeResult evaluate(String input, String output, double samplingRate) {
104104
.build();
105105
}
106106

107-
String formatted = "MESSAGE HISTORY:\n" + input + "\n\nRESPONSE TO EVALUATE:\n" + output;
108-
LDAIConfigTracker tracker = config.createTracker();
109-
110-
RunnerResult result;
111107
try {
112-
result = tracker.trackMetricsOf(RunnerResult::getMetrics, () -> runner.run(formatted, EVALUATION_SCHEMA));
113-
} catch (Exception ex) {
114-
return JudgeResult.builder()
108+
String formatted = "MESSAGE HISTORY:\n" + input + "\n\nRESPONSE TO EVALUATE:\n" + output;
109+
LDAIConfigTracker tracker = config.createTracker();
110+
111+
RunnerResult result = tracker.trackMetricsOf(RunnerResult::getMetrics, () -> runner.run(formatted, EVALUATION_SCHEMA));
112+
113+
Map<String, Object> parsed = result.getParsed();
114+
if (parsed == null) {
115+
if (logger != null) logger.warn("Judge {}: runner returned null parsed output", config.getKey());
116+
return JudgeResult.builder()
117+
.sampled(true)
118+
.success(false)
119+
.judgeConfigKey(config.getKey())
120+
.metricKey(config.getEvaluationMetricKey())
121+
.build();
122+
}
123+
124+
Object scoreRaw = parsed.get("score");
125+
if (!(scoreRaw instanceof Number)) {
126+
if (logger != null) logger.warn("Judge {}: parsed output missing numeric score", config.getKey());
127+
return JudgeResult.builder()
128+
.sampled(true)
129+
.success(false)
130+
.judgeConfigKey(config.getKey())
131+
.metricKey(config.getEvaluationMetricKey())
132+
.build();
133+
}
134+
double score = ((Number) scoreRaw).doubleValue();
135+
if (!Double.isFinite(score) || score < 0.0 || score > 1.0) {
136+
if (logger != null) logger.warn("Judge {}: score {} is outside [0.0, 1.0]", config.getKey(), score);
137+
return JudgeResult.builder()
138+
.sampled(true)
139+
.success(false)
140+
.judgeConfigKey(config.getKey())
141+
.metricKey(config.getEvaluationMetricKey())
142+
.build();
143+
}
144+
145+
JudgeResult.Builder resultBuilder = JudgeResult.builder()
115146
.sampled(true)
116-
.success(false)
147+
.success(true)
117148
.judgeConfigKey(config.getKey())
118149
.metricKey(config.getEvaluationMetricKey())
119-
.errorMessage(ex.getMessage())
120-
.build();
121-
}
150+
.score(score);
122151

123-
Map<String, Object> parsed = result.getParsed();
124-
if (parsed == null) {
125-
if (logger != null) logger.warn("Judge {}: runner returned null parsed output", config.getKey());
126-
return JudgeResult.builder()
127-
.sampled(true)
128-
.success(false)
129-
.judgeConfigKey(config.getKey())
130-
.metricKey(config.getEvaluationMetricKey())
131-
.build();
132-
}
152+
Object reasoningRaw = parsed.get("reasoning");
153+
if (reasoningRaw instanceof String) {
154+
resultBuilder.reasoning((String) reasoningRaw);
155+
} else if (reasoningRaw != null) {
156+
if (logger != null) logger.warn("Judge {}: reasoning is not a string, ignoring", config.getKey());
157+
}
133158

134-
Object scoreRaw = parsed.get("score");
135-
if (!(scoreRaw instanceof Number)) {
136-
if (logger != null) logger.warn("Judge {}: parsed output missing numeric score", config.getKey());
137-
return JudgeResult.builder()
138-
.sampled(true)
139-
.success(false)
140-
.judgeConfigKey(config.getKey())
141-
.metricKey(config.getEvaluationMetricKey())
142-
.build();
143-
}
144-
double score = ((Number) scoreRaw).doubleValue();
145-
if (!Double.isFinite(score) || score < 0.0 || score > 1.0) {
146-
if (logger != null) logger.warn("Judge {}: score {} is outside [0.0, 1.0]", config.getKey(), score);
159+
return resultBuilder.build();
160+
} catch (Exception ex) {
147161
return JudgeResult.builder()
148162
.sampled(true)
149163
.success(false)
150-
.judgeConfigKey(config.getKey())
151-
.metricKey(config.getEvaluationMetricKey())
164+
.errorMessage(ex.getMessage())
152165
.build();
153166
}
154-
155-
JudgeResult.Builder resultBuilder = JudgeResult.builder()
156-
.sampled(true)
157-
.success(true)
158-
.judgeConfigKey(config.getKey())
159-
.metricKey(config.getEvaluationMetricKey())
160-
.score(score);
161-
162-
Object reasoningRaw = parsed.get("reasoning");
163-
if (reasoningRaw instanceof String) {
164-
resultBuilder.reasoning((String) reasoningRaw);
165-
} else if (reasoningRaw != null) {
166-
if (logger != null) logger.warn("Judge {}: reasoning is not a string, ignoring", config.getKey());
167-
}
168-
169-
return resultBuilder.build();
170167
}
171168

172169
/**

0 commit comments

Comments
 (0)