Fix #85: [discussion] explore: CLI: Structured JSON Diff for Verifier Failures

packages/verifier-runtime/src/verify.test.ts

@@ -0,0 +1,92 @@
+import { expect, test } from 'bun:test';
+import { verify } from './verify.js';
+
+test('verify returns passed=true when output matches expected', () => {
+  const result = verify({
+    taskId: 'normalize_simple',
+    input: { a: 1, b: null },
+    output: { a: 1, b: null },
+    constraints: { sort_keys: true, strip_nulls: false, flatten: null },
+    operationType: 'normalize',
+  });
+  expect(result.passed).toBe(true);
+  expect(result.expected_hash).toBe(result.actual_hash);
+  expect(result.failure_reason).toBeUndefined();
+  expect(result.failure_diff).toBeUndefined();
+});
+
+test('verify returns passed=false with failure_diff when output differs', () => {
+  const result = verify({
+    taskId: 'normalize_simple',
+    input: { a: 1, b: 2 },
+    output: { a: 1, b: 3 },
+    constraints: { sort_keys: true, strip_nulls: false, flatten: null },
+    operationType: 'normalize',
+  });
+  expect(result.passed).toBe(false);
+  expect(result.expected_hash).not.toBe(result.actual_hash);
+  expect(result.failure_reason).toBeDefined();
+  expect(result.failure_diff).toBeDefined();
+  // failure_diff should be an array of ops
+  expect(Array.isArray(result.failure_diff)).toBe(true);
+  if (Array.isArray(result.failure_diff)) {
+    expect(result.failure_diff.length).toBeGreaterThan(0);
+    // Should contain a replace operation for /b
+    expect(result.failure_diff[0].op).toBe('replace');
+    expect(result.failure_diff[0].path).toBe('/b');
+  }
+});
+
+test('verify returns failure_diff for deeply nested object mismatch', () => {
+  const result = verify({
+    taskId: 'normalize_deep',
+    input: { nested: { deep: { value: 1, extra: 'x' } } },
+    output: { nested: { deep: { value: 2, extra: 'x' } } },
+    constraints: { sort_keys: true, strip_nulls: false, flatten: null },
+    operationType: 'normalize',
+  });
+  expect(result.passed).toBe(false);
+  expect(result.failure_diff).toBeDefined();
+  if (Array.isArray(result.failure_diff)) {
+    expect(result.failure_diff[0].path).toBe('/nested/deep/value');
+  }
+});
+
+test('verify returns failure_diff for array element mismatch', () => {
+  const result = verify({
+    taskId: 'normalize_array',
+    input: { list: [1, 2, 3] },
+    output: { list: [1, 4, 3] },
+    constraints: { sort_keys: true, strip_nulls: false, flatten: null },
+    operationType: 'normalize',
+  });
+  expect(result.passed).toBe(false);
+  expect(result.failure_diff).toBeDefined();
+  if (Array.isArray(result.failure_diff)) {
+    expect(result.failure_diff[0].path).toBe('/list/1');
+  }
+});
+
+test('verify returns failure_diff for diff operation', () => {
+  const result = verify({
+    taskId: 'diff_test',
+    input: { source: { a: 1, b: 2 }, target: { a: 1, b: 3 } },
+    output: [{ op: 'replace', path: '/b', value: 3 }],
+    constraints: { format: 'ops', array_indices: true, max_depth: 10 },
+    operationType: 'diff',
+  });
+  expect(result.passed).toBe(false);
+  expect(result.failure_diff).toBeDefined();
+});
+
+test('verify on diff operation passes for correct output', () => {
+  const result = verify({
+    taskId: 'diff_test',
+    input: { source: { a: 1, b: 2 }, target: { a: 1, b: 3 } },
+    output: [{ op: 'replace', path: '/b', value: 3 }],
+    constraints: { format: 'ops', array_indices: true, max_depth: 10 },
+    operationType: 'diff',
+  });
+  expect(result.passed).toBe(true);
+  expect(result.failure_diff).toBeUndefined();
+});

packages/verifier-runtime/src/verify.ts

@@ -25,14 +25,38 @@ export interface VerifyResult {
   expected_hash: string;
   actual_hash: string;
   failure_reason?: string;
+  /**
+   * Structured semantic diff between the expected output and the solver's
+   * actual output, computed when the verification fails. The diff uses the
+   * same ops-format as `diff_patch.diff()` (a list of add/replace/remove
+   * operations with JSON Pointer paths), which directly visualizes the gap
+   * between expected and actual values. Absent when the verification passes.
+   */
+  failure_diff?: unknown;
 }
 
+/**
+ * Default diff patch constraints used for computing failure diffs.
+ * These match the schema defaults: ops format, array-indices granularity,
+ * and a max depth of 10 levels.
+ */
+const FAILURE_DIFF_CONSTRAINTS = {
+  format: 'ops' as const,
+  array_indices: true,
+  max_depth: 10,
+};
+
 /**
  * Deterministic verifier oracle for all json_transform operations.
  *
  * For normalize, diff, patch, merge, and migrate operations, a submission passes
  * iff its structural hash equals the reference implementation's structural hash.
  * Object key order is irrelevant (see `stableStringify`); array order is significant.
+ *
+ * When the verification fails, the result includes a `failure_diff` field
+ * containing a structured operation-level diff between the expected output
+ * and the solver's actual output. This allows debugging tools to surface
+ * exact property insertions, deletions, and modifications.
  */
 export function verify(input: VerifyInput): VerifyResult {
   const operationType = input.operationType || inferOperationType(input.taskId);
@@ -79,7 +103,32 @@ export function verify(input: VerifyInput): VerifyResult {
     actual_hash: actualHash,
   };
 
-  return passed ? base : { ...base, failure_reason: failureReason };
+  if (passed) {
+    return base;
+  }
+
+  // Compute a structured semantic diff between expected and actual output
+  // when verification fails. This enables the CLI and other tools to show
+  // exact property insertions, deletions, and modifications.
+  const failureDiff = computeFailureDiff(expected, input.output);
+
+  return { ...base, failure_reason: failureReason, failure_diff: failureDiff };
+}
+
+/**
+ * Compute a structured diff between two JSON values for failure reporting.
+ *
+ * Uses the ops-format diff_patch diff with default constraints. If the diff
+ * computation itself fails (e.g. due to depth limits), returns a descriptive
+ * sentinel string.
+ */
+function computeFailureDiff(expected: unknown, actual: unknown): unknown {
+  try {
+    const constraints = parseDiffPatchConstraints(FAILURE_DIFF_CONSTRAINTS);
+    return diff(expected, actual, constraints);
+  } catch {
+    return '__DIFF_UNAVAILABLE__';
+  }
 }
 
 /**