diff --git a/CHANGELOG.md b/CHANGELOG.md index b6380d6..6055c04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.5.0 - 2026-09-08 + +- Classify Node's native spec reporter pass, fail, skip, aggregate, and timing + records while retaining failure diagnostics and honestly escalating unknown + output. +- Treat native dot-reporter progress as structure, so repetitive successful + verification does not become hash-per-line model evidence. +- Preserve the canonical audit packet and semantic model-evidence projection as + distinct artifacts with exact UTF-8 byte accounting. + ## 0.4.0 - 2026-09-06 - Added the compatible `opsle.context-firewall.model-evidence/v1` semantic-only diff --git a/README.md b/README.md index 77cd51f..4be7bb0 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ safely not see?** This repository does not yet answer it. ## Prototype scope -Version 0.4.0 is a dependency-free Node.js reference reducer for a documented -flat TAP-compatible test-output subset. It: +Version 0.5.0 is a dependency-free Node.js reference reducer for a documented +flat TAP-compatible subset plus Node's native spec and dot reporters. It: - reads caller-supplied stdout and stderr bytes plus process metadata; - derives test verdict and pass, fail, and skip counts; diff --git a/SPEC.md b/SPEC.md index 1318dd6..3746b50 100644 --- a/SPEC.md +++ b/SPEC.md @@ -7,6 +7,9 @@ Packet version: `opsle.context-firewall.evidence-packet/v1`. Model-evidence projection version: `opsle.context-firewall.model-evidence/v1`. +Reducer policy revision: `test-output-policy/v2`. It recognizes the documented +TAP subset and Node's native spec and dot reporter records. + ## Compatibility boundary The reference primitive accepts generic test-run bytes and emits generic diff --git a/fixtures/corpus.js b/fixtures/corpus.js index d9a1a28..4e1ba26 100644 --- a/fixtures/corpus.js +++ b/fixtures/corpus.js @@ -25,6 +25,26 @@ function tap({ passed = [], failed = [], skipped = [], extras = [], finalNewline return `${lines.join('\n')}${finalNewline ? '\n' : ''}`; } +function nodeSpec({ passed = [], failed = [], skipped = [], extras = [] }) { + const lines = []; + for (const name of passed) lines.push(`✔ ${name} (1.25ms)`); + for (const name of skipped) lines.push(`﹣ ${name} (0.1ms) # SKIP`); + for (const failure of failed) { + lines.push(`✖ ${failure.name} (2.5ms)`); + lines.push(...(failure.details ?? []).map((detail) => ` ${detail}`)); + } + lines.push(...extras); + lines.push(`ℹ tests ${passed.length + failed.length + skipped.length}`); + lines.push('ℹ suites 0'); + lines.push(`ℹ pass ${passed.length}`); + lines.push(`ℹ fail ${failed.length}`); + lines.push('ℹ cancelled 0'); + lines.push(`ℹ skipped ${skipped.length}`); + lines.push('ℹ todo 0'); + lines.push('ℹ duration_ms 12.5'); + return `${lines.join('\n')}\n`; +} + function invocation({ stdout = '', stderr = '', exitCode = 0, interrupted = false, source = true, rawRef = true, operationId = true, durationMs = 12.5, @@ -45,6 +65,8 @@ function invocation({ const manyPasses = Array.from({ length: 1500 }, (_, index) => `case-${String(index + 1).padStart(4, '0')}`); const repetitivePasses = Array.from({ length: 400 }, () => 'repetitive-success'); +const task16Passes = Array.from({ length: 112 }, (_, index) => + `task-16-case-${index + 1}-${'repetitive-success-detail-'.repeat(3)}`); const longName = `very-long-${'x'.repeat(40_000)}`; const longFailure = `message: ${'critical'.repeat(4_000)}`; @@ -53,7 +75,10 @@ export const corpus = Object.freeze([ { name: 'normal/large-all-pass', input: invocation({ stdout: tap({ passed: manyPasses }) }), expected: { status: 'passed', disposition: 'SUFFICIENT', counts: [1500, 0, 0], substantialReduction: true } }, { name: 'normal/repetitive-success', input: invocation({ stdout: tap({ passed: repetitivePasses }) }), expected: { status: 'passed', counts: [400, 0, 0], substantialReduction: true } }, { name: 'normal/skipped-tests', input: invocation({ stdout: tap({ passed: ['runs'], skipped: ['not-applicable', 'platform-only'] }) }), expected: { status: 'passed', counts: [1, 0, 2] } }, + { name: 'normal/node-spec-task-16', input: invocation({ stdout: nodeSpec({ passed: task16Passes, skipped: ['external-a', 'external-b', 'external-c'], extras: ['> package@1.0.0 test', '> node --test', 'ℹ {"bounded":"diagnostic"}', 'Switched to a new branch \'feature\''] }) }), options: { maxOutputBytes: 12_000 }, expected: { status: 'passed', disposition: 'SUFFICIENT', counts: [112, 0, 3], substantialReduction: true } }, + { name: 'normal/node-dot-progress', input: invocation({ stdout: nodeSpec({ passed: ['alpha', 'beta'], extras: ['..'] }) }), expected: { status: 'passed', disposition: 'SUFFICIENT', counts: [2, 0, 0] } }, { name: 'failure/one', input: invocation({ stdout: tap({ failed: [{ name: 'adds values', details: ['message: expected two', 'expected: 2', 'actual: 3'] }] }), exitCode: 1 }), expected: { status: 'failed', failures: 1, contains: ['adds values', 'expected two'] } }, + { name: 'failure/node-spec', input: invocation({ stdout: nodeSpec({ passed: ['green'], failed: [{ name: 'adds values', details: ['error: expected two', 'expected: 2', 'actual: 3', 'at test (file:///public/test.js:4:5)'] }] }), exitCode: 1 }), expected: { status: 'failed', disposition: 'SUFFICIENT', counts: [1, 1, 0], failures: 1, contains: ['adds values', 'expected two', 'file:///public/test.js:4:5'] } }, { name: 'failure/several', input: invocation({ stdout: tap({ failed: [{ name: 'first', details: ['message: first broke'] }, { name: 'second', details: ['message: second broke'] }, { name: 'third', details: ['message: third broke'] }] }), exitCode: 1 }), expected: { status: 'failed', failures: 3, contains: ['first broke', 'second broke', 'third broke'] } }, { name: 'failure/mixed-pass-fail', input: invocation({ stdout: tap({ passed: ['green-a', 'green-b'], failed: [{ name: 'red', details: ['message: broken'] }] }), exitCode: 1 }), expected: { status: 'failed', counts: [2, 1, 0], failures: 1 } }, { name: 'failure/assertion-difference', input: invocation({ stdout: tap({ failed: [{ name: 'diffs objects', details: ['operator: deepStrictEqual', 'expected: {a: 1}', 'actual: {a: 2}', 'diff: -1 +2'] }] }), exitCode: 1 }), expected: { status: 'failed', contains: ['deepStrictEqual', 'diff: -1 +2'] } }, diff --git a/package.json b/package.json index 4730820..9a77052 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@opsle/context-firewall", - "version": "0.4.0", + "version": "0.5.0", "private": true, "type": "module", "bin": { diff --git a/src/reducer.js b/src/reducer.js index 51827e6..bdc2d5d 100644 --- a/src/reducer.js +++ b/src/reducer.js @@ -6,8 +6,8 @@ export const INPUT_PROTOCOL = 'opsle.context-firewall.test-run-input/v1'; export const PACKET_PROTOCOL = 'opsle.context-firewall.evidence-packet/v1'; export const MODEL_EVIDENCE_PROTOCOL = 'opsle.context-firewall.model-evidence/v1'; export const REDUCER_NAME = '@opsle/context-firewall/test-output'; -export const REDUCER_VERSION = '0.4.0'; -export const POLICY_REVISION = 'tap-subset-policy/v1'; +export const REDUCER_VERSION = '0.5.0'; +export const POLICY_REVISION = 'test-output-policy/v2'; const ANSI_PATTERN = /[\u001b\u009b][[\]()#;?]*(?:(?:(?:[a-zA-Z\d]*(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*)?\u0007)|(?:(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))/g; const utf8Decoder = new TextDecoder('utf-8', { fatal: true }); @@ -191,19 +191,26 @@ function parseTranscript(streams) { const line = { stream: stream.name, line: index + 1, text: rawLine }; const clean = rawLine.replace(/\r$/, '').replace(ANSI_PATTERN, ''); const marker = clean.match(/^\s*(not ok|ok)\b(?:\s+\d+)?(?:\s*-\s*)?(.*)$/); - const summary = clean.match(/^\s*#\s*(tests|pass|fail|skipped)\s+(\d+)\s*$/); + const nodeMarker = clean.match(/^\s*([✔✖﹣])\s+(.+?)(?:\s+\([^)]*ms\))?(?:\s+#\s*(SKIP|TODO)\b.*)?$/u); + const summary = clean.match(/^\s*(?:#|ℹ)\s*(tests|pass|fail|skipped)\s+(\d+)\s*$/u); - if (marker) { + if (marker || nodeMarker) { currentFailure = null; - const directive = marker[2].match(/\s+#\s*(SKIP|TODO)\b.*$/i); - const identity = marker[2].replace(/\s+#\s*(?:SKIP|TODO)\b.*$/i, '').trim() || '(unnamed test)'; - if (marker[1] === 'ok' && directive?.[1].toUpperCase() === 'SKIP') { + const tapDirective = marker?.[2].match(/\s+#\s*(SKIP|TODO)\b.*$/i); + const directive = tapDirective?.[1] || nodeMarker?.[3]; + const identity = marker + ? marker[2].replace(/\s+#\s*(?:SKIP|TODO)\b.*$/i, '').trim() || '(unnamed test)' + : nodeMarker[2].trim() || '(unnamed test)'; + const successful = marker ? marker[1] === 'ok' : nodeMarker[1] === '✔'; + const skipped = nodeMarker?.[1] === '﹣' + || (successful && directive?.toUpperCase() === 'SKIP'); + if (skipped) { line.kind = 'skipped_test'; observed.skipped += 1; - } else if (marker[1] === 'ok') { + } else if (successful) { line.kind = 'successful_test'; observed.passed += 1; - } else if (directive?.[1].toUpperCase() === 'TODO') { + } else if (directive?.toUpperCase() === 'TODO') { line.kind = 'unclassified'; unclassified.push(line); } else { @@ -242,8 +249,12 @@ function parseTranscript(streams) { } else if (warning) { line.kind = 'abnormal_warning'; warnings.push(line); - } else if (/^\s*#\s*(?:duration_ms\s+\d+(?:\.\d+)?|note:\s*.*)\s*$/.test(clean)) { + } else if (/^\s*(?:#|ℹ)\s*(?:duration_ms\s+\d+(?:\.\d+)?|(?:suites|cancelled|todo)\s+\d+|note:\s*.*)\s*$/u.test(clean)) { line.kind = clean.includes('duration_ms') ? 'duration_source' : 'informational'; + } else if (/^\s*(?:>\s+\S.*|ℹ\s+.*|Switched to (?:a new branch|branch) .*)$/u.test(clean)) { + line.kind = 'informational'; + } else if (/^\s*[.X]+\s*$/.test(clean)) { + line.kind = 'progress'; } else if (/^\s*$/.test(clean)) { line.kind = 'blank'; } else { diff --git a/tests/reducer.test.js b/tests/reducer.test.js index b0ded04..6e21a6d 100644 --- a/tests/reducer.test.js +++ b/tests/reducer.test.js @@ -95,6 +95,29 @@ test('large all-pass output is substantially reduced with correct aggregates', ( assert.equal(packet.receipt.suppressed.categories.successful_test, 1500); }); +test('Node spec output retains counts and diagnostics without retaining passing repetition', () => { + const task16 = fixture('normal/node-spec-task-16'); + const packet = reduceTestRun(task16.input, task16.options); + const packetBytes = serializePacket(packet).length; + const modelBytes = serializeModelEvidence(modelEvidenceForPacket(packet)).length; + assert.deepEqual(packet.decision_evidence.counts, + { passed: 112, failed: 0, skipped: 3, total: 115 }); + assert.equal(packet.decision_evidence.status, 'passed'); + assert.equal(packet.decision_evidence.disposition, 'SUFFICIENT'); + assert.equal(packet.decision_evidence.unclassified_evidence.length, 0); + assert.equal(packet.receipt.suppressed.categories.successful_test, 112); + assert.equal(packet.receipt.suppressed.categories.skipped_test, 3); + assert.ok(packetBytes <= 12_000); + assert.ok(modelBytes < packetBytes); + assert.ok(packetBytes < packet.receipt.measurements.original_bytes); + + const failure = reduceTestRun(fixture('failure/node-spec').input); + assert.equal(failure.decision_evidence.status, 'failed'); + assert.equal(failure.decision_evidence.failures[0].identity, 'adds values'); + assert.ok(failure.decision_evidence.failures[0].details + .some(item => item.text.includes('file:///public/test.js:4:5'))); +}); + test('a failed test retains identity, message, assertion, and location', () => { const packet = reduceTestRun(fixture('failure/one').input); const [failure] = packet.decision_evidence.failures; @@ -365,7 +388,7 @@ test('value receipt preserves expansion instead of fabricating avoided bytes', ( assert.equal(avoided.delta, avoided.result - avoided.baseline); assert.equal( formatContextFirewallIndicator(valueReceipt), - '[Context Firewall] 104 B -> 1,824 B | 1,720 B expansion | escalation: no', + '[Context Firewall] 104 B -> 1,825 B | 1,721 B expansion | escalation: no', ); }); @@ -373,7 +396,7 @@ test('value receipt exposes reduction and the exact named operator indicator', ( const { valueReceipt } = reduceWithValueReceipt(fixture('normal/large-all-pass').input); assert.equal( formatContextFirewallIndicator(valueReceipt), - '[Context Firewall] 28,981 B -> 1,846 B | 27,135 B initially avoided (93.63%) | escalation: no', + '[Context Firewall] 28,981 B -> 1,847 B | 27,134 B initially avoided (93.63%) | escalation: no', ); }); @@ -505,9 +528,9 @@ test('CLI keeps an impossible-ceiling error off model-visible stdout', () => { assert.equal(error.disposition, 'NEEDS_RAW_EVIDENCE'); }); -test('all 30 synthetic fixtures conform', () => { +test('all 33 synthetic fixtures conform', () => { const report = conformanceReport(); - assert.equal(report.fixture_count, 30); + assert.equal(report.fixture_count, 33); assert.equal(report.conformance, 'PASS'); assert.equal(report.fixtures.every((item) => item.conformance === 'PASS'), true); });