Better align with braintrust-spec

Author: aswink

js/dev/server.ts

@@ -8,6 +8,7 @@ import {
   EvalScorer,
   EvaluatorDef,
   OneOrMoreScores,
+  classifierName,
   scorerName,
 } from "../src/framework";
 import { errorHandler } from "./errorHandler";
@@ -120,6 +121,9 @@ export function runDevServer(
           scores: (evaluator.scores ?? []).map((score, idx) => ({
             name: scorerName(score, idx),
           })),
+          classifiers: (evaluator.classifiers ?? []).map((classifier, idx) => ({
+            name: classifierName(classifier, idx),
+          })),
         };
       }
 

js/dev/types.ts

@@ -111,6 +111,7 @@ export type SerializedParametersContainer = z.infer<
 export const evaluatorDefinitionSchema = z.object({
   parameters: serializedParametersContainerSchema.optional(),
   scores: z.array(z.object({ name: z.string() })).optional(),
+  classifiers: z.array(z.object({ name: z.string() })).optional(),
 });
 export type EvaluatorDefinition = z.infer<typeof evaluatorDefinitionSchema>;
 

js/src/cli/functions/infer-source.ts

@@ -86,7 +86,9 @@ export async function findCodeDefinition({
       fn =
         location.position.type === "task"
           ? evaluator.task
-          : (evaluator.scores ?? [])[location.position.index];
+          : location.position.type === "scorer"
+            ? (evaluator.scores ?? [])[location.position.index]
+            : (evaluator.classifiers ?? [])[location.position.index];
     }
   } else if (location.type === "function") {
     fn = outFileModule.functions[location.index].handler;

js/src/cli/functions/upload.test.ts

@@ -84,4 +84,37 @@ describe("buildBundledFunctionEntry", () => {
 
     expect(entry.tags).toBeUndefined();
   });
+
+  test("preserves classifier experiment locations", async () => {
+    const entry = await buildBundledFunctionEntry({
+      spec: {
+        project_id: "proj-123",
+        name: "test-classifier",
+        slug: "test-classifier",
+        description: "Test classifier",
+        location: {
+          type: "experiment" as const,
+          eval_name: "eval-1",
+          position: {
+            type: "classifier" as const,
+            index: 0,
+          },
+        },
+        function_type: "classifier" as const,
+      },
+      runtime_context: { runtime: "node", version: "22.0.0" },
+      bundleId: "bundle-123",
+      sourceMapContext: undefined,
+    });
+
+    expect(entry.function_type).toBe("classifier");
+    expect(entry.function_data.data.location).toEqual({
+      type: "experiment",
+      eval_name: "eval-1",
+      position: {
+        type: "classifier",
+        index: 0,
+      },
+    });
+  });
 });

js/src/cli/functions/upload.ts

@@ -4,7 +4,7 @@ import {
   type IfExistsType as IfExists,
 } from "../../generated_types";
 import type { BuildSuccess, EvaluatorState, FileHandle } from "../types";
-import { scorerName, warning } from "../../framework";
+import { classifierName, scorerName, warning } from "../../framework";
 import {
   _internalGetGlobalState,
   Experiment,
@@ -200,6 +200,23 @@ export async function uploadHandleBundles({
             };
           },
         ),
+        ...(evaluator.evaluator.classifiers ?? []).map(
+          (classifier, i): BundledFunctionSpec => {
+            const name = classifierName(classifier, i);
+            return {
+              ...baseInfo,
+              ...formatNameAndSlug(["eval", namePrefix, "classifier", name]),
+              description: `Classifier ${name} for eval ${namePrefix}`,
+              location: {
+                type: "experiment",
+                eval_name: evaluator.evaluator.evalName,
+                position: { type: "classifier", index: i },
+              },
+              function_type: "classifier",
+              origin,
+            };
+          },
+        ),
       ];
 
       bundleSpecs.push(...fileSpecs);
@@ -225,6 +242,11 @@ export async function uploadHandleBundles({
           scores: (evaluator.evaluator.scores ?? []).map((score, i) => ({
             name: scorerName(score, i),
           })),
+          classifiers: (evaluator.evaluator.classifiers ?? []).map(
+            (classifier, i) => ({
+              name: classifierName(classifier, i),
+            }),
+          ),
         };
 
         bundleSpecs.push({

js/src/framework.test.ts

@@ -186,9 +186,9 @@ describe("runEvaluator", () => {
         undefined,
       );
 
-      expect(out.results.every((r) => Object.keys(r.scores).length === 0)).toBe(
-        true,
-      );
+      expect(
+        out.results.every((r) => Object.keys(r.scores ?? {}).length === 0),
+      ).toBe(true);
     });
 
     describe("errorScoreHandler", () => {
@@ -216,8 +216,8 @@ describe("runEvaluator", () => {
           expect(
             out.results.every(
               (r) =>
-                Object.keys(r.scores).length === 3 &&
-                Object.values(r.scores).every((v) => v === 0),
+                Object.keys(r.scores ?? {}).length === 3 &&
+                Object.values(r.scores ?? {}).every((v) => v === 0),
             ),
           ).toBe(true);
         });
@@ -245,10 +245,10 @@ describe("runEvaluator", () => {
           expect(
             out.results.every(
               (r) =>
-                Object.keys(r.scores).length === 3 &&
-                r.scores.scorer_0 === 0 &&
-                r.scores.scorer_1 === 1 &&
-                r.scores.scorer_2 === 1,
+                Object.keys(r.scores ?? {}).length === 3 &&
+                r.scores?.scorer_0 === 0 &&
+                r.scores?.scorer_1 === 1 &&
+                r.scores?.scorer_2 === 1,
             ),
           ).toBe(true);
         });
@@ -276,7 +276,7 @@ describe("runEvaluator", () => {
           );
 
           expect(
-            out.results.every((r) => Object.keys(r.scores).length === 0),
+            out.results.every((r) => Object.keys(r.scores ?? {}).length === 0),
           ).toBe(true);
         });
 
@@ -303,8 +303,8 @@ describe("runEvaluator", () => {
           expect(
             out.results.every(
               (r) =>
-                Object.keys(r.scores).length === 1 &&
-                r.scores.error_score === 1,
+                Object.keys(r.scores ?? {}).length === 1 &&
+                r.scores?.error_score === 1,
             ),
           ).toBe(true);
         });
@@ -579,13 +579,13 @@ test("Eval with noSendLogs: true runs locally without creating experiment", asyn
   expect(result.results).toHaveLength(2);
   expect(result.results[0].input).toBe("hello");
   expect(result.results[0].output).toBe("hello world");
-  expect(result.results[0].scores.exact_match).toBe(1);
-  expect(result.results[0].scores.simple_scorer).toBe(0.8);
+  expect(result.results[0].scores?.exact_match).toBe(1);
+  expect(result.results[0].scores?.simple_scorer).toBe(0.8);
 
   expect(result.results[1].input).toBe("test");
   expect(result.results[1].output).toBe("test world");
-  expect(result.results[1].scores.exact_match).toBe(1);
-  expect(result.results[1].scores.simple_scorer).toBe(0.8);
+  expect(result.results[1].scores?.exact_match).toBe(1);
+  expect(result.results[1].scores?.simple_scorer).toBe(0.8);
 
   // Verify it builds a local summary (no experimentUrl means local run)
   expect(result.summary.projectName).toBe("test-no-logs");
@@ -660,10 +660,10 @@ test("Eval with returnResults: true collects all results", async () => {
   expect(result.results).toHaveLength(2);
   expect(result.results[0].input).toBe("hello");
   expect(result.results[0].output).toBe("hello world");
-  expect(result.results[0].scores.exact_match).toBe(1);
+  expect(result.results[0].scores?.exact_match).toBe(1);
   expect(result.results[1].input).toBe("test");
   expect(result.results[1].output).toBe("test world");
-  expect(result.results[1].scores.exact_match).toBe(1);
+  expect(result.results[1].scores?.exact_match).toBe(1);
 
   // Summary should also be correct
   expect(result.summary.scores.exact_match.score).toBe(1);
@@ -862,7 +862,7 @@ test("scorer spans have purpose='scorer' attribute", async () => {
   );
 
   expect(result.results).toHaveLength(1);
-  expect(result.results[0].scores.simple_scorer).toBe(1);
+  expect(result.results[0].scores?.simple_scorer).toBe(1);
 
   await memoryLogger.flush();
   const logs = await memoryLogger.drain();
@@ -1635,6 +1635,7 @@ test("classifier-only evaluator populates classifications field", async () => {
 
   expect(result.results).toHaveLength(1);
   const r = result.results[0];
+  expect(r.scores).toBeUndefined();
   expect(r.classifications?.category).toEqual([
     {
       id: "greeting",
@@ -1661,7 +1662,7 @@ test("scorer-only evaluator populates scores field", async () => {
   );
 
   expect(result.results).toHaveLength(1);
-  expect(result.results[0].scores.exact_match).toBe(1);
+  expect(result.results[0].scores?.exact_match).toBe(1);
   expect(result.results[0].classifications).toBeUndefined();
 });
 
@@ -1713,27 +1714,8 @@ test("mixed evaluator populates both scores and classifications", async () => {
   );
 
   expect(result.results).toHaveLength(1);
-  expect(result.results[0].scores.exact_match).toBe(1);
+  expect(result.results[0].scores?.exact_match).toBe(1);
   expect(result.results[0].classifications?.category).toEqual([
     { id: "greeting", label: "Greeting" },
   ]);
 });
-
-test("malformed classifier output fails clearly", async () => {
-  const result = await Eval(
-    "test-invalid-classifier-output",
-    {
-      data: [{ input: "hello" }],
-      task: (input) => input,
-      classifiers: [() => ({}) as never],
-    },
-    { noSendLogs: true, returnResults: true },
-  );
-
-  expect(result.results).toHaveLength(1);
-  expect((result.results[0] as any).metadata?.classifier_errors).toMatchObject({
-    classifier_0: expect.stringMatching(
-      /must return classifications with a non-empty string name/,
-    ),
-  });
-});