diff --git a/packages/verifier-runtime/src/verify.test.ts b/packages/verifier-runtime/src/verify.test.ts new file mode 100644 index 0000000..4770b7b --- /dev/null +++ b/packages/verifier-runtime/src/verify.test.ts @@ -0,0 +1,92 @@ +import { expect, test } from 'bun:test'; +import { verify } from './verify.js'; + +test('verify returns passed=true when output matches expected', () => { + const result = verify({ + taskId: 'normalize_simple', + input: { a: 1, b: null }, + output: { a: 1, b: null }, + constraints: { sort_keys: true, strip_nulls: false, flatten: null }, + operationType: 'normalize', + }); + expect(result.passed).toBe(true); + expect(result.expected_hash).toBe(result.actual_hash); + expect(result.failure_reason).toBeUndefined(); + expect(result.failure_diff).toBeUndefined(); +}); + +test('verify returns passed=false with failure_diff when output differs', () => { + const result = verify({ + taskId: 'normalize_simple', + input: { a: 1, b: 2 }, + output: { a: 1, b: 3 }, + constraints: { sort_keys: true, strip_nulls: false, flatten: null }, + operationType: 'normalize', + }); + expect(result.passed).toBe(false); + expect(result.expected_hash).not.toBe(result.actual_hash); + expect(result.failure_reason).toBeDefined(); + expect(result.failure_diff).toBeDefined(); + // failure_diff should be an array of ops + expect(Array.isArray(result.failure_diff)).toBe(true); + if (Array.isArray(result.failure_diff)) { + expect(result.failure_diff.length).toBeGreaterThan(0); + // Should contain a replace operation for /b + expect(result.failure_diff[0].op).toBe('replace'); + expect(result.failure_diff[0].path).toBe('/b'); + } +}); + +test('verify returns failure_diff for deeply nested object mismatch', () => { + const result = verify({ + taskId: 'normalize_deep', + input: { nested: { deep: { value: 1, extra: 'x' } } }, + output: { nested: { deep: { value: 2, extra: 'x' } } }, + constraints: { sort_keys: true, strip_nulls: false, flatten: null }, + operationType: 'normalize', + }); + expect(result.passed).toBe(false); + expect(result.failure_diff).toBeDefined(); + if (Array.isArray(result.failure_diff)) { + expect(result.failure_diff[0].path).toBe('/nested/deep/value'); + } +}); + +test('verify returns failure_diff for array element mismatch', () => { + const result = verify({ + taskId: 'normalize_array', + input: { list: [1, 2, 3] }, + output: { list: [1, 4, 3] }, + constraints: { sort_keys: true, strip_nulls: false, flatten: null }, + operationType: 'normalize', + }); + expect(result.passed).toBe(false); + expect(result.failure_diff).toBeDefined(); + if (Array.isArray(result.failure_diff)) { + expect(result.failure_diff[0].path).toBe('/list/1'); + } +}); + +test('verify returns failure_diff for diff operation', () => { + const result = verify({ + taskId: 'diff_test', + input: { source: { a: 1, b: 2 }, target: { a: 1, b: 3 } }, + output: [{ op: 'replace', path: '/b', value: 3 }], + constraints: { format: 'ops', array_indices: true, max_depth: 10 }, + operationType: 'diff', + }); + expect(result.passed).toBe(false); + expect(result.failure_diff).toBeDefined(); +}); + +test('verify on diff operation passes for correct output', () => { + const result = verify({ + taskId: 'diff_test', + input: { source: { a: 1, b: 2 }, target: { a: 1, b: 3 } }, + output: [{ op: 'replace', path: '/b', value: 3 }], + constraints: { format: 'ops', array_indices: true, max_depth: 10 }, + operationType: 'diff', + }); + expect(result.passed).toBe(true); + expect(result.failure_diff).toBeUndefined(); +}); diff --git a/packages/verifier-runtime/src/verify.ts b/packages/verifier-runtime/src/verify.ts index 5faf336..01f8ee6 100644 --- a/packages/verifier-runtime/src/verify.ts +++ b/packages/verifier-runtime/src/verify.ts @@ -25,14 +25,38 @@ export interface VerifyResult { expected_hash: string; actual_hash: string; failure_reason?: string; + /** + * Structured semantic diff between the expected output and the solver's + * actual output, computed when the verification fails. The diff uses the + * same ops-format as `diff_patch.diff()` (a list of add/replace/remove + * operations with JSON Pointer paths), which directly visualizes the gap + * between expected and actual values. Absent when the verification passes. + */ + failure_diff?: unknown; } +/** + * Default diff patch constraints used for computing failure diffs. + * These match the schema defaults: ops format, array-indices granularity, + * and a max depth of 10 levels. + */ +const FAILURE_DIFF_CONSTRAINTS = { + format: 'ops' as const, + array_indices: true, + max_depth: 10, +}; + /** * Deterministic verifier oracle for all json_transform operations. * * For normalize, diff, patch, merge, and migrate operations, a submission passes * iff its structural hash equals the reference implementation's structural hash. * Object key order is irrelevant (see `stableStringify`); array order is significant. + * + * When the verification fails, the result includes a `failure_diff` field + * containing a structured operation-level diff between the expected output + * and the solver's actual output. This allows debugging tools to surface + * exact property insertions, deletions, and modifications. */ export function verify(input: VerifyInput): VerifyResult { const operationType = input.operationType || inferOperationType(input.taskId); @@ -79,7 +103,32 @@ export function verify(input: VerifyInput): VerifyResult { actual_hash: actualHash, }; - return passed ? base : { ...base, failure_reason: failureReason }; + if (passed) { + return base; + } + + // Compute a structured semantic diff between expected and actual output + // when verification fails. This enables the CLI and other tools to show + // exact property insertions, deletions, and modifications. + const failureDiff = computeFailureDiff(expected, input.output); + + return { ...base, failure_reason: failureReason, failure_diff: failureDiff }; +} + +/** + * Compute a structured diff between two JSON values for failure reporting. + * + * Uses the ops-format diff_patch diff with default constraints. If the diff + * computation itself fails (e.g. due to depth limits), returns a descriptive + * sentinel string. + */ +function computeFailureDiff(expected: unknown, actual: unknown): unknown { + try { + const constraints = parseDiffPatchConstraints(FAILURE_DIFF_CONSTRAINTS); + return diff(expected, actual, constraints); + } catch { + return '__DIFF_UNAVAILABLE__'; + } } /**