Skip to content

Make validation scripts offline safe by default #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mopati123 merged 1 commit into from
May 7, 2026
Merged

Make validation scripts offline safe by default #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
293 changes: 82 additions & 211 deletions scripts/validation/generate_validation_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,220 +1,91 @@
#!/usr/bin/env python3
"""
Comprehensive System Validation Report Generator

Generates final report of all 8 phases of testing.
"""
"""Generate an offline validation report without live-trading approval claims."""

import argparse
import os
import sys
from datetime import datetime

print('='*70)
print('COMPREHENSIVE SYSTEM VALIDATION REPORT')
print('='*70)
print(f'Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
print('='*70)

# Create report directory
os.makedirs('test_results/2026-04-17', exist_ok=True)

report_lines = []
report_lines.append('='*70)
report_lines.append('COMPREHENSIVE SYSTEM VALIDATION REPORT')
report_lines.append('='*70)
report_lines.append(f'Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
report_lines.append('='*70)

# Summary of all test results
test_results = {
'Phase 1: Foundation': {
'tests': 10,
'passed': 10,
'failed': 0,
'status': '✅ PASSED',
'details': [
'Dependencies: All available',
'File Structure: Complete',
'Imports: 14/14 working',
'Configuration: Valid'
]
},
'Phase 2: Core Components': {
'tests': 23,
'passed': 22,
'failed': 1,
'status': '✅ PASSED',
'details': [
'Memory (Phase 1): 4/4 passed',
'NN Predictor (Phase 2): 4/4 passed',
'RL Agent (Phase 3): 4/4 passed',
'Multi-Agent (Phase 4): 4/5 passed (MetaAgent minor issue)',
'LLM Strategy (Phase 5): 5/5 passed'
]
},
'Phase 3: Integration': {
'tests': 7,
'passed': 7,
'failed': 0,
'status': '✅ PASSED',
'details': [
'Memory → NN: Verified',
'NN → RL: Verified',
'Agents → Scheduler: Verified',
'Strategy → Agents: Verified',
'Full Pipeline: 4 agents, authorized'
]
},
'Phase 4: Live Connections': {
'tests': 6,
'passed': 5,
'failed': 1,
'status': '⚠️ PARTIAL',
'details': [
'MT5 Connection: ✅ Connected (Account: 19894320)',
'MT5 Data Stream: ✅ Working',
'Deriv API: ⚠️ Token not configured',
'Account Balance: $199.88',
'Symbols Available: 110'
]
},
'Phase 5: Shadow Trading': {
'tests': 4,
'passed': 4,
'failed': 0,
'status': '✅ PASSED',
'details': [
'Paper Trade Execution: Working',
'Performance Metrics: 12 fields tracked',
'Shadow Runner: 6 decisions processed',
'Live Data Integration: Verified'
]
},
'Phase 6: Performance': {
'tests': 5,
'passed': 5,
'failed': 0,
'status': '✅ PASSED',
'details': [
'Memory Embedding: 0.20ms (target: 5ms) ✅',
'NN Prediction: 0.03ms (target: 10ms) ✅',
'Agent Voting: 0.18ms (target: 50ms) ✅',
'RL Inference: 0.30ms (target: 5ms) ✅',
'Full Decision: 0.59ms (target: 100ms) ✅'
]
},
'Phase 7: Edge Cases': {
'tests': 15,
'passed': 13,
'failed': 2,
'status': '✅ PASSED',
'details': [
'Network Resilience: Handled',
'Data Quality: Validated',
'Component Failures: Fallbacks working',
'Error Handling: Comprehensive'
]
},
'Phase 8: Coherence Audit': {
'tests': 8,
'passed': 8,
'failed': 0,
'status': '✅ PASSED',
'details': [
'Data Shape Verification: All correct',
'Axiomatic Consistency: 4 axioms verified',
'Algorithm Correctness: Validated',
'End-to-End Coherence: No data loss'
]
}
TEST_RESULTS = {
"Phase 1: Foundation": (10, 10, 0),
"Phase 2: Core Components": (23, 22, 1),
"Phase 3: Integration": (7, 7, 0),
"Phase 4: Live Connections": (6, 5, 1),
"Phase 5: Shadow Trading": (4, 4, 0),
"Phase 6: Performance": (5, 5, 0),
"Phase 7: Edge Cases": (15, 13, 2),
"Phase 8: Coherence Audit": (8, 8, 0),
}

# Calculate totals
total_tests = sum(p['tests'] for p in test_results.values())
total_passed = sum(p['passed'] for p in test_results.values())
total_failed = sum(p['failed'] for p in test_results.values())

# Print summary
print('\n📊 EXECUTIVE SUMMARY')
print('-'*70)
print(f'Total Tests: {total_tests}')
print(f'Passed: {total_passed}')
print(f'Failed: {total_failed}')
print(f'Success Rate: {total_passed/total_tests:.1%}')
print(f'Overall Status: {"✅ PASSED" if total_failed == 0 else "⚠️ PARTIAL"}')

report_lines.append('\n📊 EXECUTIVE SUMMARY')
report_lines.append('-'*70)
report_lines.append(f'Total Tests: {total_tests}')
report_lines.append(f'Passed: {total_passed}')
report_lines.append(f'Failed: {total_failed}')
report_lines.append(f'Success Rate: {total_passed/total_tests:.1%}')
report_lines.append(f'Overall Status: {"✅ PASSED" if total_failed == 0 else "⚠️ PARTIAL"}')

# Print detailed results
print('\n📋 DETAILED PHASE RESULTS')
print('-'*70)

for phase, result in test_results.items():
print(f'\n{phase}')
print(f' Status: {result["status"]}')
print(f' Tests: {result["passed"]}/{result["tests"]} passed')
for detail in result['details']:
print(f' {detail}')

report_lines.append(f'\n{phase}')
report_lines.append(f' Status: {result["status"]}')
report_lines.append(f' Tests: {result["passed"]}/{result["tests"]} passed')
for detail in result['details']:
report_lines.append(f' {detail}')

# Key metrics
print('\n📈 KEY METRICS')
print('-'*70)
print(f'Latency Performance:')
print(f' Full Decision Time: 0.59ms (target: 100ms) ✅ 169x faster')
print(f' Agent Voting: 0.18ms (target: 50ms) ✅ 278x faster')
print(f' NN Prediction: 0.03ms (target: 10ms) ✅ 333x faster')

print(f'\nSystem Components:')
print(f' 5 Phases: All operational')
print(f' 5 Agent Types: Working')
print(f' Live Connections: MT5 ✅ | Deriv ⚠️')
print(f' Shadow Trading: Ready')

report_lines.append('\n📈 KEY METRICS')
report_lines.append('-'*70)
report_lines.append('Latency Performance:')
report_lines.append(' Full Decision Time: 0.59ms (target: 100ms) ✅ 169x faster')
report_lines.append(' Agent Voting: 0.18ms (target: 50ms) ✅ 278x faster')
report_lines.append(' NN Prediction: 0.03ms (target: 10ms) ✅ 333x faster')

# Conclusions
print('\n🎯 CONCLUSIONS')
print('-'*70)
print('✅ All 5 system phases are operational')
print('✅ Integration between phases is verified')
print('✅ Performance exceeds targets by 169x-333x')
print('✅ Coherence audit passed - all axioms validated')
print('⚠️ Deriv API token not configured (optional)')
print('✅ System is ready for use')

report_lines.append('\n🎯 CONCLUSIONS')
report_lines.append('-'*70)
report_lines.append('✅ All 5 system phases are operational')
report_lines.append('✅ Integration between phases is verified')
report_lines.append('✅ Performance exceeds targets by 169x-333x')
report_lines.append('✅ Coherence audit passed - all axioms validated')
report_lines.append('⚠️ Deriv API token not configured (optional)')
report_lines.append('✅ System is ready for use')

# Save report
report_text = '\n'.join(report_lines)
with open('test_results/2026-04-17/validation_report.txt', 'w', encoding='utf-8') as f:
f.write(report_text)

print('\n' + '='*70)
print('Report saved to: test_results/2026-04-17/validation_report.txt')
print('='*70)

print('\n✅ VALIDATION COMPLETE')
def build_report() -> str:
total_tests = sum(total for total, _passed, _failed in TEST_RESULTS.values())
total_passed = sum(passed for _total, passed, _failed in TEST_RESULTS.values())
total_failed = sum(failed for _total, _passed, failed in TEST_RESULTS.values())
success_rate = total_passed / total_tests if total_tests else 0.0

lines = [
"=" * 70,
"OFFLINE SYSTEM VALIDATION REPORT",
"=" * 70,
f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
"",
"Executive Summary",
"-" * 70,
f"Total Tests: {total_tests}",
f"Passed: {total_passed}",
f"Failed: {total_failed}",
f"Success Rate: {success_rate:.1%}",
"Overall Status: PASSED" if total_failed == 0 else "Overall Status: REVIEW REQUIRED",
"",
"Detailed Phase Results",
"-" * 70,
]

for phase, (total, passed, failed) in TEST_RESULTS.items():
status = "PASSED" if failed == 0 else "REVIEW REQUIRED"
lines.extend([
"",
phase,
f" Status: {status}",
f" Tests: {passed}/{total} passed",
])

lines.extend([
"",
"Conclusion",
"-" * 70,
"This report summarizes offline validation evidence only.",
"It does not approve live trading, broker execution, canaries, or real-money deployment.",
"=" * 70,
])
return "\n".join(lines)


def main(argv=None) -> int:
parser = argparse.ArgumentParser(description="Generate an offline validation report")
parser.add_argument(
"--write-report",
action="store_true",
help="Write the report to test_results/offline_validation_report.txt",
)
args = parser.parse_args(argv)

report = build_report()
print(report)

if args.write_report:
output_dir = os.path.join("test_results", "offline")
os.makedirs(output_dir, exist_ok=True)
output_path = os.path.join(output_dir, "validation_report.txt")
with open(output_path, "w", encoding="utf-8") as handle:
handle.write(report)
print(f"\nReport saved to: {output_path}")
else:
print("\nReport write skipped; use --write-report to create an artifact.")

return 0


if __name__ == "__main__":
raise SystemExit(main())
Loading