From 07441d47d86085f32e788d72c1eea58cacf8e0ea Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:19:35 -0700 Subject: [PATCH 1/2] test(formal): expose missing per-phase evidence retention Expected hosted RED before implementation. No local project execution. Delegated-by: Dylan Roy Actual-executors: OpenAI Codex root and scoped test subagent Scientific-effect: NONE Organizational-independence: 0 --- .github/workflows/formal-verification.yml | 4 + tests/test_formal_evidence_retention.py | 373 ++++++++++++++++++++++ 2 files changed, 377 insertions(+) create mode 100644 tests/test_formal_evidence_retention.py diff --git a/.github/workflows/formal-verification.yml b/.github/workflows/formal-verification.yml index ee62c4a0e..d9a76dc3b 100644 --- a/.github/workflows/formal-verification.yml +++ b/.github/workflows/formal-verification.yml @@ -59,6 +59,10 @@ jobs: echo 'PIPELINE_FAILURE_PROPAGATION_PASS' - name: Shell regression controls run: python -B -S -m unittest discover -s tests -p test_formal_workflow_shell.py -v + - name: Phase custody regression controls in both interpreter modes + run: | + python -B -S -m unittest discover -s tests -p test_formal_evidence_retention.py -v + python -B -O -S -m unittest discover -s tests -p test_formal_evidence_retention.py -v - name: Source-only identity gate (no Lean yet; fails closed before any download) run: python -B -S tools/formal_gate_check.py - name: Install elan (pinned release script, digest checked) and the pinned toolchain diff --git a/tests/test_formal_evidence_retention.py b/tests/test_formal_evidence_retention.py new file mode 100644 index 000000000..78f0cad36 --- /dev/null +++ b/tests/test_formal_evidence_retention.py @@ -0,0 +1,373 @@ +"""Custody controls for three real formal executions; stdlib, no Lean fixtures. + +The workflow-order regression runs on the test-only RED baseline. CLI controls +explicitly skip until the retention implementation exists. These fixtures are +for hosted execution; writing this source is not an execution claim. +""" +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / 'tools/formal_evidence_history.py' +WORKFLOW = ROOT / '.github/workflows/formal-verification.yml' +PHASES = ('initial', 'normal', 'optimized') + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def workflow_commands(text): + """Read actual run scalars in the repository's existing step indentation.""" + commands = [] + for block in re.split(r'(?=^ - )', text, flags=re.MULTILINE): + if not block.startswith(' - '): + continue + match = re.search(r'^ run: (.*)$', block, re.MULTILINE) + if match is None: + continue + value = match.group(1) + if value in ('|', '|-', '>', '>-'): + lines = [] + for line in block[match.end():].splitlines()[1:]: + if line and not line.startswith(' '): + break + lines.append(line[10:]) + value = '\n'.join(lines) + commands.append((value, block)) + return commands + + +class WorkflowRetentionOrderTests(unittest.TestCase): + def test_every_real_producer_is_retained_before_its_evidence_can_be_reused(self): + commands = workflow_commands(WORKFLOW.read_text(encoding='utf-8')) + + def locate(pattern): + found = [(i, block) for i, (body, block) in enumerate(commands) + if re.search(pattern, body, re.MULTILINE)] + self.assertEqual(len(found), 1, 'expected one executable workflow command: '+pattern) + return found[0] + + prepare = locate(r'tools/formal_evidence_history\.py\s+prepare\b') + initial = locate(r'tools/formal_gate_check\.py\s+--run-lean\b') + normal = locate(r'python\S*\s+-B\s+-S\s+-m\s+unittest\s+tests\.test_formal_gate\b') + optimized = locate(r'python\S*\s+-B\s+-O\s+-S\s+-m\s+unittest\s+tests\.test_formal_gate\b') + binder = locate(r'tools/required_formal_check\.py\s+receipt\b') + captures = {phase: locate(r'tools/formal_evidence_history\.py\s+capture\s+'+phase+r'\b') + for phase in PHASES} + verify = locate(r'tools/formal_evidence_history\.py\s+verify\b') + order = [prepare[0], initial[0], captures['initial'][0], normal[0], + captures['normal'][0], optimized[0], binder[0], + captures['optimized'][0], verify[0]] + self.assertTrue(all(left < right for left, right in zip(order, order[1:])), + 'capture initial/normal before the next producer; retain optimized after binding') + for phase, producer in zip(PHASES, (initial, normal, optimized)): + with self.subTest(phase=phase): + block = captures[phase][1] + self.assertRegex(block, r'(?m)^ if:\s*(?:\$\{\{\s*)?always\(\)') + producer_id = re.search(r'^ id:\s*(\S+)\s*$', producer[1], re.MULTILINE) + self.assertIsNotNone(producer_id, 'capture needs the actual producing step outcome') + self.assertIn('steps.'+producer_id.group(1)+'.outcome', block) + self.assertIn('--binding-outcome', captures['optimized'][1]) + self.assertIn('steps.required-binding.outcome', captures['optimized'][1]) + + +@unittest.skipUnless(SCRIPT.is_file(), 'implementation absent in RED baseline') +class FormalEvidenceRetentionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.source = self.root / 'formal/.lake/formal-evidence' + self.phases = self.root / 'formal-evidence/phases' + self.manifest = self.root / 'formal/manifest.json' + self.manifest.parent.mkdir() + self.manifest.write_bytes(b'{"fixture": "custody only, not a Lean manifest"}\n') + self.env = dict(os.environ) + for name in ('GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR'): + self.env.pop(name, None) + self.git('init', '-q') + self.git('-c', 'user.name=Formal custody fixture', + '-c', 'user.email=formal-fixture@example.invalid', + '-c', 'commit.gpgsign=false', '-c', 'core.hooksPath=/dev/null', + 'commit', '--allow-empty', '-qm', 'disposable custody fixture') + self.head = self.git('rev-parse', 'HEAD').stdout.strip() + self.env.update(GITHUB_SHA=self.head, GITHUB_REPOSITORY='d6g8k5htny-coder/main', + GITHUB_RUN_ID='226001', GITHUB_RUN_ATTEMPT='2') + self.context = {'checked_commit': self.head, 'repository': 'd6g8k5htny-coder/main', + 'run_id': '226001', 'run_attempt': '2'} + + def git(self, *args): + result = subprocess.run(['git', *args], cwd=self.root, env=self.env, + capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stdout+result.stderr) + return result + + def cli(self, *args, succeeds=True): + mode = ['-O'] if sys.flags.optimize else [] + result = subprocess.run([sys.executable, '-B', *mode, '-S', str(SCRIPT), *args], + cwd=self.root, env=self.env, capture_output=True, + text=True, timeout=20) + if succeeds: + self.assertEqual(result.returncode, 0, result.stdout+result.stderr) + else: + self.assertNotEqual(result.returncode, 0, 'unsafe custody operation succeeded') + return result + + def payload(self, label, receipt=True): + files = {'build.log': (label+' build\n').encode(), + 'version.log': (label+' version observation\n').encode(), + 'Audit.lean': b'-- inert custody fixture, never compiled\n'} + if receipt: + record = {'schema_version': 1, 'scientific_effect': 'NONE', + 'scientific_status_authority': False, 'checked_commit': self.head, + 'repository': self.context['repository'], + 'workflow_run_id': self.context['run_id'], + 'formalization_status': 'kernel-checked', + 'alignment_status': 'PENDING_INDEPENDENT_REVIEW', + 'manifest_sha256': digest(self.manifest.read_bytes()), + 'logs': {name: digest(raw) for name, raw in files.items() + if name.endswith('.log')}} + files['receipt.json'] = (json.dumps(record, sort_keys=True)+'\n').encode() + self.source.mkdir(parents=True, exist_ok=True) + for name, raw in files.items(): + (self.source/name).write_bytes(raw) + return files + + def observation(self, phase): + return json.loads((self.phases/phase/'observation.json').read_text(encoding='utf-8')) + + def assert_snapshot(self, phase, expected, outcome=None, source_present=True): + observed = self.observation(phase) + self.assertEqual(observed['phase'], phase) + self.assertEqual(observed['context'], self.context) + self.assertIs(observed['source_present'], source_present) + self.assertIn('binding_outcome', observed) + if outcome is not None: + self.assertEqual(observed['outcome'], outcome) + self.assertEqual(observed['files'], {name: {'sha256': digest(raw), 'size': len(raw)} + for name, raw in expected.items()}) + raw_dir = self.phases/phase/'raw' + actual = {p.relative_to(raw_dir).as_posix(): p.read_bytes() + for p in raw_dir.rglob('*') if p.is_file()} + self.assertEqual(actual, expected) + return observed + + def finish_skipped(self, *phases): + for phase in phases: + self.cli('capture', phase, 'skipped') + + def complete_fixture(self): + self.cli('prepare') + expected = {} + for phase in PHASES: + expected[phase] = self.payload(phase) + self.cli('capture', phase, 'success') + self.cli('verify') + return expected + + def test_prepare_preserves_preexisting_bytes_and_removes_them_from_next_phase(self): + old = self.payload('old unrelated execution') + self.cli('prepare') + self.assert_snapshot('preexisting', old, 'preexisting') + self.assertFalse(self.source.exists()) + self.finish_skipped(*PHASES) + self.cli('verify') + + def test_three_distinct_executions_keep_exact_bytes_and_optimized_stays_canonical(self): + expected = self.complete_fixture() + for phase in PHASES: + self.assert_snapshot(phase, expected[phase], 'success') + self.assertEqual(len({expected[p]['receipt.json'] for p in PHASES}), 3) + self.assertEqual({p.name: p.read_bytes() for p in self.source.iterdir()}, expected['optimized']) + + def test_failed_missing_next_phase_cannot_inherit_previous_success(self): + self.cli('prepare') + initial = self.payload('initial') + self.cli('capture', 'initial', 'success') + self.assertFalse(self.source.exists()) + self.cli('capture', 'normal', 'failure') + self.assert_snapshot('normal', {}, 'failure', source_present=False) + self.assert_snapshot('initial', initial, 'success') + self.finish_skipped('optimized') + self.cli('verify') + + def test_success_without_new_evidence_is_recorded_but_refused(self): + self.cli('prepare') + self.cli('capture', 'initial', 'success', succeeds=False) + self.assert_snapshot('initial', {}, 'success', source_present=False) + + def test_skipped_phase_records_no_source_even_when_stale_canonical_bytes_exist(self): + self.cli('prepare') + stale = self.payload('unconsumed old source') + self.cli('capture', 'initial', 'skipped') + self.assert_snapshot('initial', {}, 'skipped', source_present=False) + self.assertEqual({p.name: p.read_bytes() for p in self.source.iterdir()}, stale) + self.finish_skipped('normal', 'optimized') + self.cli('verify') + + def test_partial_failed_execution_is_preserved_without_manufacturing_receipt(self): + self.cli('prepare') + partial = self.payload('interrupted build', receipt=False) + self.cli('capture', 'initial', 'failure') + self.assert_snapshot('initial', partial, 'failure') + self.assertFalse(self.source.exists()) + self.assertNotIn('receipt.json', self.observation('initial')['files']) + self.finish_skipped('normal', 'optimized') + self.cli('verify') + + def test_gate_receipt_survives_later_suite_failure_without_relabeling_the_phase(self): + self.cli('prepare') + self.finish_skipped('initial') + succeeded_gate = self.payload('gate succeeded before suite assertion failed') + self.cli('capture', 'normal', 'failure') + self.assert_snapshot('normal', succeeded_gate, 'failure') + self.finish_skipped('optimized') + self.cli('verify') + + def test_actual_required_binder_consumes_only_optimized_logs_and_receipt(self): + spec = importlib.util.spec_from_file_location('custody_required_check', ROOT/'tools/required_formal_check.py') + binder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(binder) + self.cli('prepare') + initial = self.payload('initial') + self.cli('capture', 'initial', 'success') + self.payload('normal') + self.cli('capture', 'normal', 'success') + optimized = self.payload('optimized') + result = binder.bind_receipt(self.source/'receipt.json', self.manifest, self.env, self.head) + self.assertEqual(result['receipt_sha256'], digest(optimized['receipt.json'])) + self.assertNotEqual(result['receipt_sha256'], digest(initial['receipt.json'])) + binding_raw = (json.dumps(result, sort_keys=True)+'\n').encode() + (self.source/'required-check-binding.json').write_bytes(binding_raw) + optimized['required-check-binding.json'] = binding_raw + self.cli('capture', 'optimized', 'success', '--binding-outcome', 'success') + observed = self.assert_snapshot('optimized', optimized, 'success') + self.assertEqual(observed['binding_outcome'], 'success') + self.assertEqual((self.source/'receipt.json').read_bytes(), optimized['receipt.json']) + self.cli('verify') + (self.source/'receipt.json').write_bytes(initial['receipt.json']) + with self.assertRaises(ValueError): + binder.bind_receipt(self.source/'receipt.json', self.manifest, self.env, self.head) + + def test_duplicate_capture_refuses_to_overwrite_archive_or_consume_new_source(self): + self.cli('prepare') + first = self.payload('first') + self.cli('capture', 'initial', 'success') + observation_raw = (self.phases/'initial/observation.json').read_bytes() + second = self.payload('second') + self.cli('capture', 'initial', 'success', succeeds=False) + self.assert_snapshot('initial', first, 'success') + self.assertEqual((self.phases/'initial/observation.json').read_bytes(), observation_raw) + self.assertEqual({p.name: p.read_bytes() for p in self.source.iterdir()}, second) + + def test_prepare_refuses_duplicate_preexisting_archive(self): + self.cli('prepare') + raw = (self.phases/'preexisting/observation.json').read_bytes() + old = self.payload('would overwrite prior observation') + self.cli('prepare', succeeds=False) + self.assertEqual((self.phases/'preexisting/observation.json').read_bytes(), raw) + self.assertEqual({p.name: p.read_bytes() for p in self.source.iterdir()}, old) + + def test_source_symlink_is_refused_without_reading_or_moving_its_target(self): + self.cli('prepare') + outside = self.root/'outside'; outside.mkdir() + (outside/'secret.log').write_bytes(b'outside sentinel\n') + self.source.parent.mkdir(parents=True, exist_ok=True) + self.source.symlink_to(outside, target_is_directory=True) + self.cli('capture', 'initial', 'success', succeeds=False) + self.assertTrue(self.source.is_symlink()) + self.assertEqual((outside/'secret.log').read_bytes(), b'outside sentinel\n') + self.assertFalse((self.phases/'initial/raw/secret.log').exists()) + + def test_linked_archive_parent_is_refused(self): + outside = self.root/'outside'; outside.mkdir() + (self.root/'formal-evidence').symlink_to(outside, target_is_directory=True) + self.cli('prepare', succeeds=False) + self.assertEqual(list(outside.iterdir()), []) + + def test_linked_source_parent_is_refused(self): + outside = self.root/'outside'; outside.mkdir() + (self.root/'formal/.lake').symlink_to(outside, target_is_directory=True) + self.cli('prepare', succeeds=False) + self.assertEqual(list(outside.iterdir()), []) + + def test_payload_symlink_is_refused_without_consuming_originals(self): + self.cli('prepare') + self.source.mkdir(parents=True) + outside = self.root/'sentinel'; outside.write_bytes(b'external bytes\n') + link = self.source/'linked.log'; link.symlink_to(outside) + self.cli('capture', 'initial', 'failure', succeeds=False) + self.assertTrue(link.is_symlink()) + self.assertEqual(outside.read_bytes(), b'external bytes\n') + + def test_fifo_is_refused_without_consuming_source(self): + if not hasattr(os, 'mkfifo'): + self.skipTest('FIFO control requires hosted POSIX filesystem') + self.cli('prepare') + self.source.mkdir(parents=True) + fifo = self.source/'pipe.log'; os.mkfifo(fifo) + self.cli('capture', 'initial', 'failure', succeeds=False) + self.assertTrue(fifo.exists()) + + def test_verify_rejects_missing_phase_observation(self): + self.cli('prepare') + self.finish_skipped('initial', 'normal') + self.cli('verify', succeeds=False) + + def test_verify_rejects_changed_missing_and_unlisted_payloads(self): + expected = self.complete_fixture() + log = self.phases/'initial/raw/build.log' + original = expected['initial']['build.log'] + for mutation in ('changed', 'missing', 'unlisted'): + with self.subTest(mutation=mutation): + extra = self.phases/'initial/raw/not-in-inventory.log' + if mutation == 'changed': + log.write_bytes(b'forged earlier output\n') + elif mutation == 'missing': + log.unlink() + else: + extra.write_bytes(b'unbound extra output\n') + self.cli('verify', succeeds=False) + log.write_bytes(original) + if extra.exists(): + extra.unlink() + self.cli('verify') + + def test_verify_rejects_inventory_digest_size_and_path_substitution(self): + self.complete_fixture() + path = self.phases/'normal/observation.json' + original = path.read_bytes() + for mutation in ('digest', 'size', 'path'): + with self.subTest(mutation=mutation): + record = json.loads(original) + if mutation == 'digest': + record['files']['build.log']['sha256'] = '0'*64 + elif mutation == 'size': + record['files']['build.log']['size'] += 1 + else: + record['files']['../outside.log'] = record['files'].pop('build.log') + path.write_text(json.dumps(record), encoding='utf-8') + self.cli('verify', succeeds=False) + path.write_bytes(original) + self.cli('verify') + + def test_verify_rejects_replaced_snapshot_link_even_when_target_bytes_match(self): + expected = self.complete_fixture() + log = self.phases/'normal/raw/build.log' + outside = self.root/'same-bytes.log'; outside.write_bytes(expected['normal']['build.log']) + log.unlink(); log.symlink_to(outside) + self.cli('verify', succeeds=False) + + +if __name__ == '__main__': + unittest.main() From aae377652d26617ee6dc5e3cb36d89318bd2d2ba Mon Sep 17 00:00:00 2001 From: Dylan Roy <249851642+d6g8k5htny-coder@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:24:57 -0700 Subject: [PATCH 2/2] fix(formal): retain distinct real-execution evidence before reuse Rotate initial and normal raw outputs; copy optimized evidence after unchanged binding. Preserve native failed/skipped outcomes and verify complete file inventories. Hosted RED: run37979500128 job113986110507. Candidate execution pending; no local project execution. Delegated-by: Dylan Roy Actual-executor: OpenAI Codex root01a0bbb5 Scientific-effect: NONE Organizational-independence: 0 --- .github/workflows/formal-verification.yml | 36 ++++- docs/FORMAL_REQUIRED_CHECKS.md | 42 ++++++ tools/formal_evidence_history.py | 164 ++++++++++++++++++++++ 3 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 tools/formal_evidence_history.py diff --git a/.github/workflows/formal-verification.yml b/.github/workflows/formal-verification.yml index d9a76dc3b..c5f4f6331 100644 --- a/.github/workflows/formal-verification.yml +++ b/.github/workflows/formal-verification.yml @@ -43,6 +43,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false + - name: Parse formal workflow with the hosted YAML parser + run: | + ruby -rpsych -e 'Psych.safe_load(File.read(ARGV.fetch(0)), aliases: false); puts "FORMAL_WORKFLOW_YAML_PARSED"' .github/workflows/formal-verification.yml - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: '3.11' @@ -65,6 +68,8 @@ jobs: python -B -O -S -m unittest discover -s tests -p test_formal_evidence_retention.py -v - name: Source-only identity gate (no Lean yet; fails closed before any download) run: python -B -S tools/formal_gate_check.py + - name: Separate preexisting evidence from this execution + run: python -B -S tools/formal_evidence_history.py prepare - name: Install elan (pinned release script, digest checked) and the pinned toolchain run: | set -euo pipefail @@ -74,24 +79,44 @@ jobs: echo "$HOME/.elan/bin" >> "$GITHUB_PATH" "$HOME/.elan/bin/elan" toolchain install "$(cat formal/lean-toolchain)" - name: Kernel lane — build, leanchecker, axiom audit, negative controls, receipt + id: initial env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ID: ${{ github.run_id }} run: python -B -S tools/formal_gate_check.py --run-lean | tee /tmp/run-lean.log - - name: Gate controls in both interpreter modes - run: | - python -B -S -m unittest tests.test_formal_gate -v 2>&1 | tee /tmp/tests-normal.log - python -B -O -S -m unittest tests.test_formal_gate -v 2>&1 | tee /tmp/tests-optimized.log + - name: Retain initial evidence before the normal execution + if: always() + env: + PHASE_OUTCOME: ${{ steps.initial.outcome }} + run: python -B -S tools/formal_evidence_history.py capture initial "$PHASE_OUTCOME" + - name: Gate controls in normal interpreter mode + id: normal + run: python -B -S -m unittest tests.test_formal_gate -v 2>&1 | tee /tmp/tests-normal.log + - name: Retain normal evidence before the optimized execution + if: always() + env: + PHASE_OUTCOME: ${{ steps.normal.outcome }} + run: python -B -S tools/formal_evidence_history.py capture normal "$PHASE_OUTCOME" + - name: Gate controls in optimized interpreter mode + id: optimized + run: python -B -O -S -m unittest tests.test_formal_gate -v 2>&1 | tee /tmp/tests-optimized.log - name: Bind current-run receipt to required check id: required-binding run: python3 -B -S tools/required_formal_check.py receipt + - name: Retain optimized evidence and the actual binding outcome + if: always() + env: + PHASE_OUTCOME: ${{ steps.optimized.outcome }} + BINDING_OUTCOME: ${{ steps.required-binding.outcome }} + run: python -B -S tools/formal_evidence_history.py capture optimized "$PHASE_OUTCOME" --binding-outcome "$BINDING_OUTCOME" - name: Collect evidence if: always() run: | mkdir -p formal-evidence git rev-parse HEAD > formal-evidence/commit.txt cp /tmp/run-lean.log /tmp/tests-normal.log /tmp/tests-optimized.log formal-evidence/ 2>/dev/null || true - if [ -d formal/.lake/formal-evidence ]; then cp -r formal/.lake/formal-evidence formal-evidence/receipt; fi + python -B -S tools/formal_evidence_history.py verify + if [ -d formal-evidence/phases/optimized/raw ]; then cp -r formal-evidence/phases/optimized/raw formal-evidence/receipt; fi ls -R formal-evidence - name: Describe coverage if: always() @@ -107,5 +132,6 @@ jobs: with: name: formal-verification-${{ github.run_id }} path: formal-evidence + include-hidden-files: true if-no-files-found: error retention-days: 30 diff --git a/docs/FORMAL_REQUIRED_CHECKS.md b/docs/FORMAL_REQUIRED_CHECKS.md index 06765589e..466ed8d7d 100644 --- a/docs/FORMAL_REQUIRED_CHECKS.md +++ b/docs/FORMAL_REQUIRED_CHECKS.md @@ -26,6 +26,48 @@ PR-head evidence with base/head integration checks. Changing a base changes the tested merge; the existing strict rulesets handle revalidation. Manual diagnostic runs do not substitute for PR checks. This is not merge-queue configuration. +## Per-phase execution custody + +The main formal workflow still executes the real gate three times: the initial +CLI, the normal test suite's real-package control, and the optimized test suite's +real-package control. Their raw outputs are retained separately under +`formal-evidence/phases/{initial,normal,optimized}/raw`. The initial and normal +directories are moved out of the shared producer path before the next execution; +the optimized directory remains at `formal/.lake/formal-evidence` for the +unchanged required-check binder. Its snapshot is copied after the binding step, +including any binding file actually produced. The artifact's existing `receipt/` +view comes from that verified optimized snapshot, never an earlier fallback. + +Each phase's `observation.json` records its native step outcome, producing +commit/repository/run/attempt, source presence and complete file-size/SHA-256 +inventory. Optimized observations also record the actual binding-step outcome. +These are custody records, not new gate receipts or status authorities. Receipt +presence does not establish phase success: a suite may fail after its real gate +has written a valid receipt. Skipped phases adopt no leftover canonical files. +Partial failed output and a failed phase with no output are recorded as such. +Any evidence already present before the first execution is preserved separately +as `preexisting`; it receives no fresh execution credit. + +The standard-library helper refuses duplicate destinations, linked ancestors, +symlinks, hard-linked files and special files, then verifies snapshot membership +and bytes. Capture or verification failure fails the formal job. Always-run +collection/upload preserves available evidence; job cancellation, timeout or +upload failure may prevent full retention and must not be reported as complete. +The gate writes a subprocess log only after that subprocess returns, so this +change cannot supply a log that the gate never wrote. This is sequential custody +inside an owned job workspace, not protection against a concurrent hostile writer. +Outside this workflow, the gate's existing fixed-directory behavior is unchanged. + +The historical C225 initial log bytes were overwritten and remain unavailable. +New runs produce new evidence; this change does not recover those bytes. Earlier +phase records do not replace the final required receipt, independent statement +alignment, source review, or scientific acceptance. + +Regression controls are `tests/test_formal_evidence_retention.py`, run normally +and with `-O` before toolchain installation. Fixtures test custody and actual +receipt binding without compiling Lean; hosted real gate/suite execution remains +a separate required check. The existing shell failure controls remain in place. + ## Current and future agents Coordinate in the current source-bound PR. The earlier discussion is in diff --git a/tools/formal_evidence_history.py b/tools/formal_evidence_history.py new file mode 100644 index 000000000..28a73b7c3 --- /dev/null +++ b/tools/formal_evidence_history.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Retain sequential formal workflow observations; no scientific authority. + +The producing steps must have stopped before capture. This is byte custody in +an owned job workspace, not a sandbox against a concurrently hostile writer. +Gate receipts are never edited. Only the optimized directory remains canonical. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import sys + +from required_formal_check import context, require, strict_json + + +SOURCE = Path('formal/.lake/formal-evidence') +PHASES = Path('formal-evidence/phases') +NAMES = ('preexisting', 'initial', 'normal', 'optimized') +OUTCOMES = ('success', 'failure', 'cancelled', 'skipped') + + +def safe_path(path): + """Reject linked ancestors before any traversal or directory creation.""" + require(not path.is_absolute() and '..' not in path.parts, 'unsafe path') + for parent in reversed(path.parents): + require(not parent.is_symlink(), 'symlinked parent: '+str(parent)) + require(not parent.exists() or parent.is_dir(), 'non-directory parent') + require(not path.is_symlink(), 'symlinked path: '+str(path)) + + +def inventory(folder): + safe_path(folder) + if not folder.exists(): + return {} + require(folder.is_dir(), 'evidence root must be a directory') + result = {} + for path in sorted(folder.rglob('*')): + safe_path(path) + info = path.lstat() + if stat.S_ISDIR(info.st_mode): + continue + require(stat.S_ISREG(info.st_mode), 'nonregular evidence: '+str(path)) + require(info.st_nlink == 1, 'hard-linked evidence: '+str(path)) + value = hashlib.sha256() + with path.open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + value.update(chunk) + result[path.relative_to(folder).as_posix()] = { + 'sha256': value.hexdigest(), 'size': info.st_size} + return result + + +def capture(phase, outcome, binding_outcome=None): + ctx = context(dict(os.environ)) + destination = PHASES / phase + safe_path(destination) + require(not destination.exists(), 'phase destination already exists: '+phase) + # A skipped producer never owns whatever may remain at the canonical path. + present = False + files = {} + if outcome != 'skipped': + safe_path(SOURCE) + files = inventory(SOURCE) + present = SOURCE.exists() + destination.mkdir(parents=True) + if present: + if phase == 'optimized': + shutil.copytree(SOURCE, destination / 'raw') + else: + SOURCE.rename(destination / 'raw') + require(inventory(destination / 'raw') == files, 'capture changed evidence bytes') + record = { + 'schema_version': 1, 'scientific_effect': 'NONE', + 'scientific_status_authority': False, 'context': ctx, + 'phase': phase, 'outcome': outcome, 'binding_outcome': binding_outcome, + 'source_present': present, 'files': files, + } + with (destination / 'observation.json').open('x', encoding='utf-8') as stream: + stream.write(json.dumps(record, sort_keys=True, indent=2)+'\n') + # Preserve the observation even when the claimed success has no evidence. + require(outcome != 'success' or present, 'successful phase produced no evidence') + return record + + +def verify(): + ctx = context(dict(os.environ)) + safe_path(PHASES) + require(PHASES.is_dir(), 'missing phase observations') + require({p.name for p in PHASES.iterdir()} == set(NAMES), 'incomplete or extra phases') + for phase in NAMES: + folder = PHASES / phase + safe_path(folder) + record_path = folder / 'observation.json' + safe_path(record_path) + require(record_path.is_file() and record_path.stat().st_nlink == 1, + 'missing or aliased observation') + record = strict_json(record_path.read_text(encoding='utf-8')) + require(isinstance(record, dict), 'observation must be an object') + require(record.get('schema_version') == 1 and record.get('phase') == phase, + 'observation identity mismatch') + require(record.get('context') == ctx, 'observation execution context mismatch') + require(record.get('scientific_effect') == 'NONE' and + record.get('scientific_status_authority') is False, 'unexpected authority') + allowed = ('preexisting',) if phase == 'preexisting' else OUTCOMES + require(record.get('outcome') in allowed, 'invalid phase outcome') + require(record.get('binding_outcome') in (None, *OUTCOMES), 'invalid binding outcome') + present = record.get('source_present') + require(type(present) is bool, 'invalid source presence') + raw = folder / 'raw' + files = inventory(raw) + require(raw.exists() == present, 'source presence mismatch') + require(record.get('files') == files, 'snapshot inventory mismatch: '+phase) + require({p.name for p in folder.iterdir()} == + ({'observation.json', 'raw'} if present else {'observation.json'}), + 'unlisted observation material') + require(record['outcome'] != 'skipped' or not present, 'skipped phase adopted evidence') + require(record['outcome'] != 'success' or present, 'successful phase lacks evidence') + if phase != 'optimized': + require(record['binding_outcome'] is None, 'binding belongs to optimized phase') + if record['binding_outcome'] == 'success': + require(phase == 'optimized' and record['outcome'] == 'success', + 'binding cannot succeed for an unsuccessful phase') + binding = strict_json((raw / 'required-check-binding.json').read_text(encoding='utf-8')) + require(isinstance(binding, dict) and all(binding.get(k) == v for k, v in ctx.items()), + 'retained binding context mismatch') + require(binding.get('receipt_sha256') == + hashlib.sha256((raw / 'receipt.json').read_bytes()).hexdigest(), + 'retained binding receipt mismatch') + return {'custody': 'verified', 'phases': list(NAMES), 'context': ctx, + 'scientific_effect': 'NONE', 'scientific_status_authority': False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='operation', required=True) + commands.add_parser('prepare') + command = commands.add_parser('capture') + command.add_argument('phase', choices=NAMES[1:]) + command.add_argument('outcome', choices=OUTCOMES) + command.add_argument('--binding-outcome', choices=OUTCOMES) + commands.add_parser('verify') + args = parser.parse_args() + try: + if args.operation == 'prepare': + result = capture('preexisting', 'preexisting') + elif args.operation == 'capture': + require(args.phase == 'optimized' or args.binding_outcome is None, + 'binding outcome only applies to optimized phase') + result = capture(args.phase, args.outcome, args.binding_outcome) + else: + result = verify() + print(json.dumps(result, sort_keys=True)) + return 0 + except (ValueError, OSError) as exc: + print('FORMAL_EVIDENCE_CUSTODY_FAILED: '+str(exc), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main())