@@ -104,69 +104,66 @@ public JudgeResult evaluate(String input, String output, double samplingRate) {
104104 .build ();
105105 }
106106
107- String formatted = "MESSAGE HISTORY:\n " + input + "\n \n RESPONSE TO EVALUATE:\n " + output ;
108- LDAIConfigTracker tracker = config .createTracker ();
109-
110- RunnerResult result ;
111107 try {
112- result = tracker .trackMetricsOf (RunnerResult ::getMetrics , () -> runner .run (formatted , EVALUATION_SCHEMA ));
113- } catch (Exception ex ) {
114- return JudgeResult .builder ()
108+ String formatted = "MESSAGE HISTORY:\n " + input + "\n \n RESPONSE TO EVALUATE:\n " + output ;
109+ LDAIConfigTracker tracker = config .createTracker ();
110+
111+ RunnerResult result = tracker .trackMetricsOf (RunnerResult ::getMetrics , () -> runner .run (formatted , EVALUATION_SCHEMA ));
112+
113+ Map <String , Object > parsed = result .getParsed ();
114+ if (parsed == null ) {
115+ if (logger != null ) logger .warn ("Judge {}: runner returned null parsed output" , config .getKey ());
116+ return JudgeResult .builder ()
117+ .sampled (true )
118+ .success (false )
119+ .judgeConfigKey (config .getKey ())
120+ .metricKey (config .getEvaluationMetricKey ())
121+ .build ();
122+ }
123+
124+ Object scoreRaw = parsed .get ("score" );
125+ if (!(scoreRaw instanceof Number )) {
126+ if (logger != null ) logger .warn ("Judge {}: parsed output missing numeric score" , config .getKey ());
127+ return JudgeResult .builder ()
128+ .sampled (true )
129+ .success (false )
130+ .judgeConfigKey (config .getKey ())
131+ .metricKey (config .getEvaluationMetricKey ())
132+ .build ();
133+ }
134+ double score = ((Number ) scoreRaw ).doubleValue ();
135+ if (!Double .isFinite (score ) || score < 0.0 || score > 1.0 ) {
136+ if (logger != null ) logger .warn ("Judge {}: score {} is outside [0.0, 1.0]" , config .getKey (), score );
137+ return JudgeResult .builder ()
138+ .sampled (true )
139+ .success (false )
140+ .judgeConfigKey (config .getKey ())
141+ .metricKey (config .getEvaluationMetricKey ())
142+ .build ();
143+ }
144+
145+ JudgeResult .Builder resultBuilder = JudgeResult .builder ()
115146 .sampled (true )
116- .success (false )
147+ .success (true )
117148 .judgeConfigKey (config .getKey ())
118149 .metricKey (config .getEvaluationMetricKey ())
119- .errorMessage (ex .getMessage ())
120- .build ();
121- }
150+ .score (score );
122151
123- Map <String , Object > parsed = result .getParsed ();
124- if (parsed == null ) {
125- if (logger != null ) logger .warn ("Judge {}: runner returned null parsed output" , config .getKey ());
126- return JudgeResult .builder ()
127- .sampled (true )
128- .success (false )
129- .judgeConfigKey (config .getKey ())
130- .metricKey (config .getEvaluationMetricKey ())
131- .build ();
132- }
152+ Object reasoningRaw = parsed .get ("reasoning" );
153+ if (reasoningRaw instanceof String ) {
154+ resultBuilder .reasoning ((String ) reasoningRaw );
155+ } else if (reasoningRaw != null ) {
156+ if (logger != null ) logger .warn ("Judge {}: reasoning is not a string, ignoring" , config .getKey ());
157+ }
133158
134- Object scoreRaw = parsed .get ("score" );
135- if (!(scoreRaw instanceof Number )) {
136- if (logger != null ) logger .warn ("Judge {}: parsed output missing numeric score" , config .getKey ());
137- return JudgeResult .builder ()
138- .sampled (true )
139- .success (false )
140- .judgeConfigKey (config .getKey ())
141- .metricKey (config .getEvaluationMetricKey ())
142- .build ();
143- }
144- double score = ((Number ) scoreRaw ).doubleValue ();
145- if (!Double .isFinite (score ) || score < 0.0 || score > 1.0 ) {
146- if (logger != null ) logger .warn ("Judge {}: score {} is outside [0.0, 1.0]" , config .getKey (), score );
159+ return resultBuilder .build ();
160+ } catch (Exception ex ) {
147161 return JudgeResult .builder ()
148162 .sampled (true )
149163 .success (false )
150- .judgeConfigKey (config .getKey ())
151- .metricKey (config .getEvaluationMetricKey ())
164+ .errorMessage (ex .getMessage ())
152165 .build ();
153166 }
154-
155- JudgeResult .Builder resultBuilder = JudgeResult .builder ()
156- .sampled (true )
157- .success (true )
158- .judgeConfigKey (config .getKey ())
159- .metricKey (config .getEvaluationMetricKey ())
160- .score (score );
161-
162- Object reasoningRaw = parsed .get ("reasoning" );
163- if (reasoningRaw instanceof String ) {
164- resultBuilder .reasoning ((String ) reasoningRaw );
165- } else if (reasoningRaw != null ) {
166- if (logger != null ) logger .warn ("Judge {}: reasoning is not a string, ignoring" , config .getKey ());
167- }
168-
169- return resultBuilder .build ();
170167 }
171168
172169 /**
0 commit comments