From 3d3e5a3fcb5351e8b5e0ae92403711d30b1dd813 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 7 May 2026 14:55:19 +0200 Subject: [PATCH] Make validation scripts offline safe by default --- .../validation/generate_validation_report.py | 293 +++++----------- scripts/validation/preflight_check.py | 44 ++- scripts/validation/run_full_validation.py | 330 +++++++++--------- .../rootfile/test_validation_script_safety.py | 109 ++++++ 4 files changed, 385 insertions(+), 391 deletions(-) create mode 100644 tests/rootfile/test_validation_script_safety.py diff --git a/scripts/validation/generate_validation_report.py b/scripts/validation/generate_validation_report.py index 1af5209..2e04b93 100644 --- a/scripts/validation/generate_validation_report.py +++ b/scripts/validation/generate_validation_report.py @@ -1,220 +1,91 @@ #!/usr/bin/env python3 -""" -Comprehensive System Validation Report Generator - -Generates final report of all 8 phases of testing. -""" +"""Generate an offline validation report without live-trading approval claims.""" +import argparse import os -import sys from datetime import datetime -print('='*70) -print('COMPREHENSIVE SYSTEM VALIDATION REPORT') -print('='*70) -print(f'Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') -print('='*70) - -# Create report directory -os.makedirs('test_results/2026-04-17', exist_ok=True) -report_lines = [] -report_lines.append('='*70) -report_lines.append('COMPREHENSIVE SYSTEM VALIDATION REPORT') -report_lines.append('='*70) -report_lines.append(f'Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}') -report_lines.append('='*70) - -# Summary of all test results -test_results = { - 'Phase 1: Foundation': { - 'tests': 10, - 'passed': 10, - 'failed': 0, - 'status': '✅ PASSED', - 'details': [ - 'Dependencies: All available', - 'File Structure: Complete', - 'Imports: 14/14 working', - 'Configuration: Valid' - ] - }, - 'Phase 2: Core Components': { - 'tests': 23, - 'passed': 22, - 'failed': 1, - 'status': '✅ PASSED', - 'details': [ - 'Memory (Phase 1): 4/4 passed', - 'NN Predictor (Phase 2): 4/4 passed', - 'RL Agent (Phase 3): 4/4 passed', - 'Multi-Agent (Phase 4): 4/5 passed (MetaAgent minor issue)', - 'LLM Strategy (Phase 5): 5/5 passed' - ] - }, - 'Phase 3: Integration': { - 'tests': 7, - 'passed': 7, - 'failed': 0, - 'status': '✅ PASSED', - 'details': [ - 'Memory → NN: Verified', - 'NN → RL: Verified', - 'Agents → Scheduler: Verified', - 'Strategy → Agents: Verified', - 'Full Pipeline: 4 agents, authorized' - ] - }, - 'Phase 4: Live Connections': { - 'tests': 6, - 'passed': 5, - 'failed': 1, - 'status': '⚠️ PARTIAL', - 'details': [ - 'MT5 Connection: ✅ Connected (Account: 19894320)', - 'MT5 Data Stream: ✅ Working', - 'Deriv API: ⚠️ Token not configured', - 'Account Balance: $199.88', - 'Symbols Available: 110' - ] - }, - 'Phase 5: Shadow Trading': { - 'tests': 4, - 'passed': 4, - 'failed': 0, - 'status': '✅ PASSED', - 'details': [ - 'Paper Trade Execution: Working', - 'Performance Metrics: 12 fields tracked', - 'Shadow Runner: 6 decisions processed', - 'Live Data Integration: Verified' - ] - }, - 'Phase 6: Performance': { - 'tests': 5, - 'passed': 5, - 'failed': 0, - 'status': '✅ PASSED', - 'details': [ - 'Memory Embedding: 0.20ms (target: 5ms) ✅', - 'NN Prediction: 0.03ms (target: 10ms) ✅', - 'Agent Voting: 0.18ms (target: 50ms) ✅', - 'RL Inference: 0.30ms (target: 5ms) ✅', - 'Full Decision: 0.59ms (target: 100ms) ✅' - ] - }, - 'Phase 7: Edge Cases': { - 'tests': 15, - 'passed': 13, - 'failed': 2, - 'status': '✅ PASSED', - 'details': [ - 'Network Resilience: Handled', - 'Data Quality: Validated', - 'Component Failures: Fallbacks working', - 'Error Handling: Comprehensive' - ] - }, - 'Phase 8: Coherence Audit': { - 'tests': 8, - 'passed': 8, - 'failed': 0, - 'status': '✅ PASSED', - 'details': [ - 'Data Shape Verification: All correct', - 'Axiomatic Consistency: 4 axioms verified', - 'Algorithm Correctness: Validated', - 'End-to-End Coherence: No data loss' - ] - } +TEST_RESULTS = { + "Phase 1: Foundation": (10, 10, 0), + "Phase 2: Core Components": (23, 22, 1), + "Phase 3: Integration": (7, 7, 0), + "Phase 4: Live Connections": (6, 5, 1), + "Phase 5: Shadow Trading": (4, 4, 0), + "Phase 6: Performance": (5, 5, 0), + "Phase 7: Edge Cases": (15, 13, 2), + "Phase 8: Coherence Audit": (8, 8, 0), } -# Calculate totals -total_tests = sum(p['tests'] for p in test_results.values()) -total_passed = sum(p['passed'] for p in test_results.values()) -total_failed = sum(p['failed'] for p in test_results.values()) - -# Print summary -print('\n📊 EXECUTIVE SUMMARY') -print('-'*70) -print(f'Total Tests: {total_tests}') -print(f'Passed: {total_passed}') -print(f'Failed: {total_failed}') -print(f'Success Rate: {total_passed/total_tests:.1%}') -print(f'Overall Status: {"✅ PASSED" if total_failed == 0 else "⚠️ PARTIAL"}') - -report_lines.append('\n📊 EXECUTIVE SUMMARY') -report_lines.append('-'*70) -report_lines.append(f'Total Tests: {total_tests}') -report_lines.append(f'Passed: {total_passed}') -report_lines.append(f'Failed: {total_failed}') -report_lines.append(f'Success Rate: {total_passed/total_tests:.1%}') -report_lines.append(f'Overall Status: {"✅ PASSED" if total_failed == 0 else "⚠️ PARTIAL"}') - -# Print detailed results -print('\n📋 DETAILED PHASE RESULTS') -print('-'*70) - -for phase, result in test_results.items(): - print(f'\n{phase}') - print(f' Status: {result["status"]}') - print(f' Tests: {result["passed"]}/{result["tests"]} passed') - for detail in result['details']: - print(f' {detail}') - - report_lines.append(f'\n{phase}') - report_lines.append(f' Status: {result["status"]}') - report_lines.append(f' Tests: {result["passed"]}/{result["tests"]} passed') - for detail in result['details']: - report_lines.append(f' {detail}') - -# Key metrics -print('\n📈 KEY METRICS') -print('-'*70) -print(f'Latency Performance:') -print(f' Full Decision Time: 0.59ms (target: 100ms) ✅ 169x faster') -print(f' Agent Voting: 0.18ms (target: 50ms) ✅ 278x faster') -print(f' NN Prediction: 0.03ms (target: 10ms) ✅ 333x faster') - -print(f'\nSystem Components:') -print(f' 5 Phases: All operational') -print(f' 5 Agent Types: Working') -print(f' Live Connections: MT5 ✅ | Deriv ⚠️') -print(f' Shadow Trading: Ready') - -report_lines.append('\n📈 KEY METRICS') -report_lines.append('-'*70) -report_lines.append('Latency Performance:') -report_lines.append(' Full Decision Time: 0.59ms (target: 100ms) ✅ 169x faster') -report_lines.append(' Agent Voting: 0.18ms (target: 50ms) ✅ 278x faster') -report_lines.append(' NN Prediction: 0.03ms (target: 10ms) ✅ 333x faster') - -# Conclusions -print('\n🎯 CONCLUSIONS') -print('-'*70) -print('✅ All 5 system phases are operational') -print('✅ Integration between phases is verified') -print('✅ Performance exceeds targets by 169x-333x') -print('✅ Coherence audit passed - all axioms validated') -print('⚠️ Deriv API token not configured (optional)') -print('✅ System is ready for use') - -report_lines.append('\n🎯 CONCLUSIONS') -report_lines.append('-'*70) -report_lines.append('✅ All 5 system phases are operational') -report_lines.append('✅ Integration between phases is verified') -report_lines.append('✅ Performance exceeds targets by 169x-333x') -report_lines.append('✅ Coherence audit passed - all axioms validated') -report_lines.append('⚠️ Deriv API token not configured (optional)') -report_lines.append('✅ System is ready for use') - -# Save report -report_text = '\n'.join(report_lines) -with open('test_results/2026-04-17/validation_report.txt', 'w', encoding='utf-8') as f: - f.write(report_text) - -print('\n' + '='*70) -print('Report saved to: test_results/2026-04-17/validation_report.txt') -print('='*70) -print('\n✅ VALIDATION COMPLETE') +def build_report() -> str: + total_tests = sum(total for total, _passed, _failed in TEST_RESULTS.values()) + total_passed = sum(passed for _total, passed, _failed in TEST_RESULTS.values()) + total_failed = sum(failed for _total, _passed, failed in TEST_RESULTS.values()) + success_rate = total_passed / total_tests if total_tests else 0.0 + + lines = [ + "=" * 70, + "OFFLINE SYSTEM VALIDATION REPORT", + "=" * 70, + f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + "", + "Executive Summary", + "-" * 70, + f"Total Tests: {total_tests}", + f"Passed: {total_passed}", + f"Failed: {total_failed}", + f"Success Rate: {success_rate:.1%}", + "Overall Status: PASSED" if total_failed == 0 else "Overall Status: REVIEW REQUIRED", + "", + "Detailed Phase Results", + "-" * 70, + ] + + for phase, (total, passed, failed) in TEST_RESULTS.items(): + status = "PASSED" if failed == 0 else "REVIEW REQUIRED" + lines.extend([ + "", + phase, + f" Status: {status}", + f" Tests: {passed}/{total} passed", + ]) + + lines.extend([ + "", + "Conclusion", + "-" * 70, + "This report summarizes offline validation evidence only.", + "It does not approve live trading, broker execution, canaries, or real-money deployment.", + "=" * 70, + ]) + return "\n".join(lines) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description="Generate an offline validation report") + parser.add_argument( + "--write-report", + action="store_true", + help="Write the report to test_results/offline_validation_report.txt", + ) + args = parser.parse_args(argv) + + report = build_report() + print(report) + + if args.write_report: + output_dir = os.path.join("test_results", "offline") + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, "validation_report.txt") + with open(output_path, "w", encoding="utf-8") as handle: + handle.write(report) + print(f"\nReport saved to: {output_path}") + else: + print("\nReport write skipped; use --write-report to create an artifact.") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/preflight_check.py b/scripts/validation/preflight_check.py index b7103af..bb39e7c 100644 --- a/scripts/validation/preflight_check.py +++ b/scripts/validation/preflight_check.py @@ -7,16 +7,21 @@ import sys import time import logging +import argparse logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def check_broker_connections(): +def check_broker_connections(allow_network: bool = False): """Verify Deriv and MT5 connections""" print("\n" + "="*60) print("CHECK 1: Broker Connections") print("="*60) + + if not allow_network: + print("[SKIP] Broker/network checks require --allow-broker-network") + return True from trading.brokers.deriv_broker import DerivBroker from trading.brokers.mt5_broker import MT5Broker @@ -108,11 +113,15 @@ def check_risk_limits(): return False -def clear_stale_state(): +def clear_stale_state(allow_mutation: bool = False): """Clear any stale state from previous runs""" print("\n" + "="*60) print("CHECK 4: Clear Stale State") print("="*60) + + if not allow_mutation: + print("[SKIP] State clearing requires --allow-state-mutation") + return True import shutil from pathlib import Path @@ -141,11 +150,15 @@ def clear_stale_state(): return True -def initialize_backtest_logger(): +def initialize_backtest_logger(allow_mutation: bool = False): """Initialize fresh backtest logger session""" print("\n" + "="*60) print("CHECK 5: Initialize Backtest Logger") print("="*60) + + if not allow_mutation: + print("[SKIP] Logger/session creation requires --allow-state-mutation") + return True from trading.backtest_logger import get_backtest_logger from datetime import datetime @@ -164,20 +177,33 @@ def initialize_backtest_logger(): return False -def main(): +def main(argv=None): """Run all pre-flight checks""" + parser = argparse.ArgumentParser(description="Offline-safe pre-flight checks") + parser.add_argument( + "--allow-broker-network", + action="store_true", + help="Allow validation to connect to configured broker/demo accounts", + ) + parser.add_argument( + "--allow-state-mutation", + action="store_true", + help="Allow validation to clear local state and create logger sessions", + ) + args = parser.parse_args(argv) + print("="*60) print("PRE-FLIGHT CHECKS - Paper Trading Demo") print("="*60) print(f"Time: {time.strftime('%Y-%m-%d %H:%M:%S')}") - print(f"Mode: PAPER TRADING (Live trading disabled)") + print("Mode: OFFLINE SAFE DEFAULTS (broker/state mutations opt-in)") results = { - 'Brokers': check_broker_connections(), + 'Brokers': check_broker_connections(args.allow_broker_network), 'Paper Mode': check_paper_mode(), 'Risk Limits': check_risk_limits(), - 'State Clear': clear_stale_state(), - 'Logger Init': initialize_backtest_logger() + 'State Clear': clear_stale_state(args.allow_state_mutation), + 'Logger Init': initialize_backtest_logger(args.allow_state_mutation) } # Summary @@ -193,7 +219,7 @@ def main(): if all_passed: print("\n" + "="*60) - print("ALL CHECKS PASSED - READY FOR PAPER TRADING") + print("ALL OFFLINE-SAFE CHECKS PASSED") print("="*60) return 0 else: diff --git a/scripts/validation/run_full_validation.py b/scripts/validation/run_full_validation.py index e9bc6ca..f525670 100644 --- a/scripts/validation/run_full_validation.py +++ b/scripts/validation/run_full_validation.py @@ -1,238 +1,226 @@ """ -Full Test Suite Validation -Comprehensive test battery for production readiness +Offline-safe validation runner. + +Runs the legacy validation battery without broker/network tests by default and +without writing report artifacts unless explicitly requested. """ +import argparse import os +import subprocess import sys import time -import subprocess -from pathlib import Path from datetime import datetime +from pathlib import Path + -# Test categories with expected durations TEST_CATEGORIES = { - 'Phase 1: Smoke Tests (Fast)': [ - ('scripts/validation/preflight_check.py', 30), - ('validation/legacy/test_production_system.py', 60), + "Phase 1: Smoke Tests (Fast)": [ + ("scripts/validation/preflight_check.py", 30), + ("validation/legacy/test_production_system.py", 60), ], - 'Phase 2: Core Feature Tests': [ - ('validation/legacy/test_taep_integration.py', 30), - ('validation/legacy/test_riemannian_geometry.py', 30), - ('validation/legacy/test_microstructure_integration.py', 30), - ('validation/legacy/test_nn_integration.py', 30), - ('validation/legacy/test_rl_integration.py', 30), - ('validation/legacy/test_memory_integration.py', 30), + "Phase 2: Core Feature Tests": [ + ("validation/legacy/test_taep_integration.py", 30), + ("validation/legacy/test_riemannian_geometry.py", 30), + ("validation/legacy/test_microstructure_integration.py", 30), + ("validation/legacy/test_nn_integration.py", 30), + ("validation/legacy/test_rl_integration.py", 30), + ("validation/legacy/test_memory_integration.py", 30), ], - 'Phase 3: System Integration Tests': [ - ('validation/legacy/test_complete_system_e2e.py', 120), - ('validation/legacy/test_full_system.py', 60), - ('validation/legacy/test_integration.py', 30), - ('validation/legacy/test_integration_final.py', 30), + "Phase 3: System Integration Tests": [ + ("validation/legacy/test_complete_system_e2e.py", 120), + ("validation/legacy/test_full_system.py", 60), + ("validation/legacy/test_integration.py", 30), + ("validation/legacy/test_integration_final.py", 30), ], - 'Phase 4: Infrastructure & Broker Tests': [ - ('validation/legacy/test_deriv_connection.py', 30), - ('validation/legacy/test_shadow_live.py', 30), - ('validation/legacy/test_multi_agent.py', 30), + "Phase 4: Infrastructure & Broker Tests": [ + ("validation/legacy/test_deriv_connection.py", 30), + ("validation/legacy/test_shadow_live.py", 30), + ("validation/legacy/test_multi_agent.py", 30), ], - 'Phase 5: Specialized Tests': [ - ('validation/legacy/test_superposition.py', 30), - ('validation/legacy/test_strategy_agent.py', 30), - ('validation/legacy/test_acceleration.py', 30), - ('validation/legacy/test_coherence_audit.py', 30), + "Phase 5: Specialized Tests": [ + ("validation/legacy/test_superposition.py", 30), + ("validation/legacy/test_strategy_agent.py", 30), + ("validation/legacy/test_acceleration.py", 30), + ("validation/legacy/test_coherence_audit.py", 30), ], } -def run_test(test_file: str, timeout: int) -> dict: - """Run a single test file and return results""" +def _command_for(test_file: str, allow_broker_network: bool) -> list[str]: + command = [sys.executable, test_file] + if test_file == "scripts/validation/preflight_check.py" and allow_broker_network: + command.append("--allow-broker-network") + return command + + +def run_test(test_file: str, timeout: int, allow_broker_network: bool = False) -> dict: + """Run a single validation file and return a compact result.""" start = time.time() - try: result = subprocess.run( - [sys.executable, test_file], + _command_for(test_file, allow_broker_network), capture_output=True, text=True, timeout=timeout, - cwd=os.getcwd() + cwd=os.getcwd(), ) - elapsed = time.time() - start - passed = result.returncode == 0 - - # Parse output for test counts output = result.stdout + result.stderr - passed_count = output.count('[PASS]') + output.count('PASS:') - failed_count = output.count('[FAIL]') + output.count('FAILED') - return { - 'file': test_file, - 'passed': passed, - 'returncode': result.returncode, - 'elapsed': elapsed, - 'passed_count': passed_count, - 'failed_count': failed_count, - 'output': output[-2000:] if len(output) > 2000 else output # Last 2000 chars + "file": test_file, + "passed": result.returncode == 0, + "returncode": result.returncode, + "elapsed": elapsed, + "passed_count": output.count("[PASS]") + output.count("PASS:"), + "failed_count": output.count("[FAIL]") + output.count("FAILED"), + "output": output[-2000:] if len(output) > 2000 else output, } - except subprocess.TimeoutExpired: return { - 'file': test_file, - 'passed': False, - 'error': 'Timeout', - 'elapsed': timeout, - 'passed_count': 0, - 'failed_count': 0, - 'output': 'Test timed out' + "file": test_file, + "passed": False, + "error": "Timeout", + "elapsed": timeout, + "passed_count": 0, + "failed_count": 0, + "output": "Test timed out", } - except Exception as e: + except Exception as exc: return { - 'file': test_file, - 'passed': False, - 'error': str(e), - 'elapsed': 0, - 'passed_count': 0, - 'failed_count': 0, - 'output': str(e) + "file": test_file, + "passed": False, + "error": str(exc), + "elapsed": 0, + "passed_count": 0, + "failed_count": 0, + "output": str(exc), } -def print_results(results: list, category: str): - """Print test results for a category""" - print(f"\n{'='*70}") - print(f"{category}") - print('='*70) - - category_passed = 0 - category_total = len(results) - +def print_results(results: list, category: str) -> tuple[int, int]: + """Print test results for one category.""" + print(f"\n{'=' * 70}") + print(category) + print("=" * 70) + passed = 0 for result in results: - status = "✅ PASS" if result['passed'] else "❌ FAIL" - print(f"{status} {result['file']:<40} ({result['elapsed']:.1f}s)") - - if result['passed']: - category_passed += 1 - else: - # Print error details for failed tests - if 'error' in result: - print(f" Error: {result['error']}") - elif result.get('failed_count', 0) > 0: - print(f" Failed tests: {result['failed_count']}") - - print(f"\nCategory: {category_passed}/{category_total} files passed") - - return category_passed, category_total + status = "PASS" if result["passed"] else "FAIL" + print(f"[{status}] {result['file']:<40} ({result['elapsed']:.1f}s)") + if result["passed"]: + passed += 1 + elif "error" in result: + print(f" Error: {result['error']}") + elif result.get("failed_count", 0) > 0: + print(f" Failed tests: {result['failed_count']}") + print(f"\nCategory: {passed}/{len(results)} files passed") + return passed, len(results) def generate_certificate(all_results: list, total_time: float) -> str: - """Generate production readiness certificate""" + """Generate an offline validation summary. This is not live approval.""" total_files = len(all_results) - passed_files = sum(1 for r in all_results if r['passed']) - pass_rate = (passed_files / total_files * 100) if total_files > 0 else 0 - - cert = f""" -╔══════════════════════════════════════════════════════════════════════════════╗ -║ ║ -║ PRODUCTION READINESS CERTIFICATE ║ -║ ║ -║ System: Quantum Trading Platform with TAEP Governance ║ -║ Validation Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ║ -║ ║ -╠══════════════════════════════════════════════════════════════════════════════╣ -║ ║ -║ TEST RESULTS SUMMARY ║ -║ ───────────────────── ║ -║ Total Test Files: {total_files:>3} ║ -║ Passed: {passed_files:>3} ║ -║ Failed: {total_files - passed_files:>3} ║ -║ Pass Rate: {pass_rate:>5.1f}% ║ -║ Total Duration: {total_time:>5.1f}s ║ -║ ║ -╠══════════════════════════════════════════════════════════════════════════════╣ -║ ║ -║ CERTIFICATION STATUS: {'✅ PRODUCTION READY' if pass_rate == 100 else '❌ NOT READY'} ║ -║ ║ -║ The system has {'successfully passed' if pass_rate == 100 else 'not passed'} all validation tests ║ -║ and is {'approved' if pass_rate == 100 else 'not approved'} for world deployment. ║ -║ ║ -╚══════════════════════════════════════════════════════════════════════════════╝ + passed_files = sum(1 for result in all_results if result["passed"]) + pass_rate = (passed_files / total_files * 100) if total_files else 0.0 + status = "OFFLINE CHECKS PASSED" if pass_rate == 100 else "REVIEW REQUIRED" + return f""" +====================================================================== +OFFLINE VALIDATION SUMMARY +====================================================================== +Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} + +Total validation files: {total_files} +Passed: {passed_files} +Failed: {total_files - passed_files} +Pass rate: {pass_rate:.1f}% +Duration: {total_time:.1f}s +Status: {status} + +This report is offline validation only. It does not approve live trading, +broker execution, canaries, or real-money deployment. +====================================================================== """ - return cert -def main(): - """Run full test suite validation""" - print("="*70) +def main(argv=None) -> int: + """Run the validation suite with offline-safe defaults.""" + parser = argparse.ArgumentParser(description="Offline-safe validation runner") + parser.add_argument( + "--allow-broker-network", + action="store_true", + help="Allow broker/network validation files to run", + ) + parser.add_argument( + "--allow-report-write", + action="store_true", + help="Write the validation summary artifact to disk", + ) + args = parser.parse_args(argv) + + print("=" * 70) print("FULL TEST SUITE VALIDATION") - print("Production Readiness for World Deployment") - print("="*70) + print("Offline-safe validation; live trading approval is out of scope") + print("=" * 70) print(f"Start Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print(f"Python: {sys.version.split()[0]}") - print("="*70) - + all_results = [] start_time = time.time() - - # Check for test files - test_dir = Path('validation/legacy') - test_files_found = list(test_dir.glob('test_*.py')) - print(f"\nFound {len(test_files_found)} test files in directory") - - # Run tests by category + test_files_found = list(Path("validation/legacy").glob("test_*.py")) + print(f"\nFound {len(test_files_found)} legacy validation files") + for category_name, tests in TEST_CATEGORIES.items(): - print(f"\n{'─'*70}") + if category_name == "Phase 4: Infrastructure & Broker Tests" and not args.allow_broker_network: + print(f"\nSkipping {category_name}: requires --allow-broker-network") + continue + + print(f"\n{'-' * 70}") print(f"Starting: {category_name}") - print('─'*70) - + print("-" * 70) category_results = [] for test_file, timeout in tests: if Path(test_file).exists(): - print(f"Running: {test_file}...", end=' ', flush=True) - result = run_test(test_file, timeout) + print(f"Running: {test_file}...", end=" ", flush=True) + result = run_test(test_file, timeout, args.allow_broker_network) category_results.append(result) all_results.append(result) - print(f"{'✅' if result['passed'] else '❌'} ({result['elapsed']:.1f}s)") + print(f"{'PASS' if result['passed'] else 'FAIL'} ({result['elapsed']:.1f}s)") else: print(f"Skipping: {test_file} (not found)") - - # Print category summary - cat_passed, cat_total = print_results(category_results, category_name) - + print_results(category_results, category_name) + total_time = time.time() - start_time - - # Final summary - print("\n" + "="*70) - print("FINAL VALIDATION SUMMARY") - print("="*70) - - total_passed = sum(1 for r in all_results if r['passed']) + total_passed = sum(1 for result in all_results if result["passed"]) total_tests = len(all_results) - overall_pass_rate = (total_passed / total_tests * 100) if total_tests > 0 else 0 - - print(f"\nTotal Files Tested: {total_tests}") + overall_pass_rate = (total_passed / total_tests * 100) if total_tests else 0.0 + + print("\n" + "=" * 70) + print("FINAL VALIDATION SUMMARY") + print("=" * 70) + print(f"Total Files Tested: {total_tests}") print(f"Passed: {total_passed}") print(f"Failed: {total_tests - total_passed}") print(f"Pass Rate: {overall_pass_rate:.1f}%") - print(f"Total Duration: {total_time:.1f} seconds ({total_time/60:.1f} minutes)") - - # Print certificate - cert = generate_certificate(all_results, total_time) - print(cert) - - # Save certificate to file - cert_file = f"PRODUCTION_CERTIFICATE_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt" - with open(cert_file, 'w', encoding='utf-8') as f: - f.write(cert) - print(f"Certificate saved to: {cert_file}") - - # Return appropriate exit code + print(f"Total Duration: {total_time:.1f} seconds ({total_time / 60:.1f} minutes)") + + summary = generate_certificate(all_results, total_time) + print(summary) + + if args.allow_report_write: + summary_file = f"OFFLINE_VALIDATION_SUMMARY_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt" + with open(summary_file, "w", encoding="utf-8") as handle: + handle.write(summary) + print(f"Summary saved to: {summary_file}") + else: + print("Summary artifact write skipped; use --allow-report-write to write a file.") + if overall_pass_rate == 100: - print("\n🎉 SYSTEM IS PRODUCTION READY FOR WORLD DEPLOYMENT! 🎉") + print("\nOffline validation passed. Live trading remains locked behind explicit safety gates.") return 0 - else: - print(f"\n⚠️ System has {total_tests - total_passed} failing test(s). Review required.") - return 1 + + print(f"\nReview required: system has {total_tests - total_passed} failing validation file(s).") + return 1 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/tests/rootfile/test_validation_script_safety.py b/tests/rootfile/test_validation_script_safety.py new file mode 100644 index 0000000..0f3d532 --- /dev/null +++ b/tests/rootfile/test_validation_script_safety.py @@ -0,0 +1,109 @@ +"""Validation CLI safety defaults.""" + +from __future__ import annotations + +import scripts.validation.generate_validation_report as validation_report +import scripts.validation.preflight_check as preflight +import scripts.validation.run_full_validation as full_validation + + +def test_preflight_defaults_skip_broker_network_and_state_mutation(monkeypatch): + calls = [] + monkeypatch.setattr(preflight, "check_paper_mode", lambda: True) + monkeypatch.setattr(preflight, "check_risk_limits", lambda: True) + monkeypatch.setattr( + preflight, + "check_broker_connections", + lambda allow_network=False: calls.append(("broker", allow_network)) or True, + ) + monkeypatch.setattr( + preflight, + "clear_stale_state", + lambda allow_mutation=False: calls.append(("clear", allow_mutation)) or True, + ) + monkeypatch.setattr( + preflight, + "initialize_backtest_logger", + lambda allow_mutation=False: calls.append(("logger", allow_mutation)) or True, + ) + + assert preflight.main([]) == 0 + assert calls == [("broker", False), ("clear", False), ("logger", False)] + + +def test_preflight_opt_in_flags_enable_broker_and_state_actions(monkeypatch): + calls = [] + monkeypatch.setattr(preflight, "check_paper_mode", lambda: True) + monkeypatch.setattr(preflight, "check_risk_limits", lambda: True) + monkeypatch.setattr( + preflight, + "check_broker_connections", + lambda allow_network=False: calls.append(("broker", allow_network)) or True, + ) + monkeypatch.setattr( + preflight, + "clear_stale_state", + lambda allow_mutation=False: calls.append(("clear", allow_mutation)) or True, + ) + monkeypatch.setattr( + preflight, + "initialize_backtest_logger", + lambda allow_mutation=False: calls.append(("logger", allow_mutation)) or True, + ) + + assert preflight.main(["--allow-broker-network", "--allow-state-mutation"]) == 0 + assert calls == [("broker", True), ("clear", True), ("logger", True)] + + +def test_clear_stale_state_default_does_not_delete_files(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + state_file = tmp_path / "trading_data" / "pnl" / "state.json" + state_file.parent.mkdir(parents=True) + state_file.write_text("{}", encoding="utf-8") + + assert preflight.clear_stale_state() is True + + assert state_file.exists() + + +def test_full_validation_default_skips_broker_category_and_report_write(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + safe_file = tmp_path / "safe_validation.py" + broker_file = tmp_path / "broker_validation.py" + safe_file.write_text("print('safe')\n", encoding="utf-8") + broker_file.write_text("print('broker')\n", encoding="utf-8") + monkeypatch.setattr( + full_validation, + "TEST_CATEGORIES", + { + "Phase 1: Smoke Tests (Fast)": [(str(safe_file), 1)], + "Phase 4: Infrastructure & Broker Tests": [(str(broker_file), 1)], + }, + ) + calls = [] + monkeypatch.setattr( + full_validation, + "run_test", + lambda test_file, timeout, allow_broker_network=False: ( + calls.append((test_file, allow_broker_network)) + or { + "file": test_file, + "passed": True, + "elapsed": 0.0, + "passed_count": 1, + "failed_count": 0, + } + ), + ) + + assert full_validation.main([]) == 0 + + assert calls == [(str(safe_file), False)] + assert not list(tmp_path.glob("OFFLINE_VALIDATION_SUMMARY_*.txt")) + + +def test_validation_report_default_avoids_live_approval_language(): + report = validation_report.build_report() + + assert "PRODUCTION READY" not in report + assert "approve live trading" in report