From 193ba7c148be81e4af4e89cc8da08cbe0ddb29f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 04:07:44 +0000 Subject: [PATCH 1/4] Initial plan From f0eeba1c98bdd118da0c9c86349b61127a59c0b9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 04:10:06 +0000 Subject: [PATCH 2/4] Remove .uvx/ directory from git and add to .gitignore Co-authored-by: SomeRandmGuyy <127457986+SomeRandmGuyy@users.noreply.github.com> --- .gitignore | 1 + .uvx/config.toml | 7 - .uvx/scripts/run_performance_tests.sh | 873 -------------------------- .uvx/scripts/run_tests.sh | 524 ---------------- .uvx/scripts/setup.sh | 233 ------- 5 files changed, 1 insertion(+), 1637 deletions(-) delete mode 100644 .uvx/config.toml delete mode 100644 .uvx/scripts/run_performance_tests.sh delete mode 100644 .uvx/scripts/run_tests.sh delete mode 100644 .uvx/scripts/setup.sh diff --git a/.gitignore b/.gitignore index 4fcc5a2..64e2f8e 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.uvx/ # Spyder project settings .spyderproject diff --git a/.uvx/config.toml b/.uvx/config.toml deleted file mode 100644 index 1c7f4f2..0000000 --- a/.uvx/config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[uvx] -python-version = "3.11" -cache-dir = ".uvx/cache" -env-dir = ".uvx/envs" - -[pip] -index-url = "https://pypi.org/simple" diff --git a/.uvx/scripts/run_performance_tests.sh b/.uvx/scripts/run_performance_tests.sh deleted file mode 100644 index e001e14..0000000 --- a/.uvx/scripts/run_performance_tests.sh +++ /dev/null @@ -1,873 +0,0 @@ -#!/bin/bash -# FSL Continuum UVX Performance Testing Script -# Complete performance testing using UVX virtual environments - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -log_step() { - echo -e "${PURPLE}[STEP]${NC} $1" -} - -log_benchmark() { - echo -e "${CYAN}[BENCHMARK]${NC} $1" -} - -# Configuration -PYTHON_VERSION="3.11" -PROJECT_NAME="fsl-continuum" -PERFORMANCE_OUTPUT_DIR="performance_results" -BENCHMARK_DIR=".benchmark" - -# Performance thresholds -BAML_PARSING_THRESHOLD=0.1 # 100ms per document -PARETO_LANG_PARSING_THRESHOLD=0.1 # 100ms per document -XML_TRANSFORMATION_THRESHOLD=0.5 # 500ms per transformation -AI_PROCESSING_THRESHOLD=2.0 # 2 seconds per AI operation - -# Memory thresholds -BAML_PARSING_MEMORY_THRESHOLD=50 # 50MB -PARETO_LANG_PARSING_MEMORY_THRESHOLD=50 # 50MB -XML_TRANSFORMATION_MEMORY_THRESHOLD=100 # 100MB -AI_PROCESSING_MEMORY_THRESHOLD=200 # 200MB - -# Performance counters -TOTAL_BENCHMARKS=0 -PASSED_BENCHMARKS=0 -FAILED_BENCHMARKS=0 -REGRESSION_DETECTED=0 - -# Function to run performance benchmark -run_performance_benchmark() { - local test_name="$1" - local test_file="$2" - local test_description="$3" - local threshold="$4" - local memory_threshold="$5" - - log_benchmark "Running ${test_name} performance benchmark..." - log_info "${test_description}" - - # Create output directory - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export PYTEST_CURRENT_TEST="${test_name}_performance" - export PYTEST_BENCHMARK_ENABLED=true - export PYTEST_PLUGINS="pytest-cov pytest-mock pytest-benchmark" - - # Execute benchmark - if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ - pytest "${test_file}" -v -m performance \ - --benchmark-only \ - --benchmark-json="${PERFORMANCE_OUTPUT_DIR}/${test_name}/${test_name}_benchmark.json" \ - --benchmark-sort=min \ - --benchmark-min-rounds=5 \ - --benchmark-max-time=30 \ - --benchmark-warmup=true \ - --benchmark-warmup-shutdown=true \ - --benchmark-storage="${BENCHMARK_DIR}" \ - 2>&1 | \ - tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/${test_name}_benchmark.log"; then - - log_success "${test_name} performance benchmark completed!" - PASSED_BENCHMARKS=$((PASSED_BENCHMARKS + 1)) - - # Analyze benchmark results - analyze_benchmark_results "${test_name}" "${threshold}" "${memory_threshold}" - - return 0 - else - log_error "${test_name} performance benchmark failed!" - FAILED_BENCHMARKS=$((FAILED_BENCHMARKS + 1)) - return 1 - fi -} - -# Function to analyze benchmark results -analyze_benchmark_results() { - local test_name="$1" - local threshold="$2" - local memory_threshold="$3" - - local benchmark_file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/${test_name}_benchmark.json" - - if [ ! -f "${benchmark_file}" ]; then - log_warning "Benchmark file not found: ${benchmark_file}" - return 1 - fi - - # Parse benchmark results using Python - python3 -c " -import json -import sys -import os - -try: - with open('${benchmark_file}', 'r') as f: - data = json.load(f) - - if 'benchmarks' in data: - benchmarks = data['benchmarks'] - else: - benchmarks = data - - regression_detected = False - - for benchmark_name, metrics in benchmarks.items(): - if isinstance(metrics, dict) and 'mean' in metrics: - mean_time = metrics['mean'] - min_time = metrics.get('min', mean_time) - max_time = metrics.get('max', mean_time) - - print(f' {benchmark_name}:') - print(f' Mean: {mean_time:.6f}s') - print(f' Min: {min_time:.6f}s') - print(f' Max: {max_time:.6f}s') - - # Check threshold - if mean_time > ${threshold}: - print(f' โŒ Exceeds threshold (${threshold}s)') - regression_detected = True - else: - print(f' โœ… Within threshold (${threshold}s)') - - # Check memory if available - if 'memory' in metrics: - memory_usage = metrics['memory'] - print(f' Memory: {memory_usage:.2f}MB') - - if memory_usage > ${memory_threshold}: - print(f' โŒ Exceeds memory threshold (${memory_threshold}MB)') - regression_detected = True - else: - print(f' โœ… Within memory threshold (${memory_threshold}MB)') - - print() - - if regression_detected: - print(f'๐Ÿšจ REGRESSION DETECTED in ${test_name}!') - sys.exit(1) - else: - print(f'โœ… No regressions detected in ${test_name}!') - sys.exit(0) - -except Exception as e: - print(f'โŒ Error analyzing benchmark: {e}') - sys.exit(1) -" - - local result=$? - if [ $result -eq 1 ]; then - REGRESSION_DETECTED=$((REGRESSION_DETECTED + 1)) - log_warning "Performance regression detected in ${test_name}" - fi -} - -# Function to run memory profiling -run_memory_profiling() { - local test_name="$1" - local test_file="$2" - local test_description="$3" - - log_step "Running ${test_name} memory profiling..." - log_info "${test_description}" - - # Create output directory - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export MEMORY_PROFILER_ENABLED=true - - # Execute memory profiling - if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ - python -m memory_profiler \ - --output-file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_profile.txt" \ - -c " -import sys -sys.path.insert(0, 'src') -from memory_profiler import profile - -@profile -def run_memory_test(): - # Import and run test - exec(open('${test_file}').read()) - -if __name__ == '__main__': - run_memory_test() -" 2>&1 | \ - tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_profiling.log"; then - - log_success "${test_name} memory profiling completed!" - - # Generate memory profile report - generate_memory_profile_report "${test_name}" - - return 0 - else - log_error "${test_name} memory profiling failed!" - return 1 - fi -} - -# Function to generate memory profile report -generate_memory_profile_report() { - local test_name="$1" - - local memory_profile_file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_profile.txt" - - if [ ! -f "${memory_profile_file}" ]; then - log_warning "Memory profile file not found: ${memory_profile_file}" - return 1 - fi - - # Parse memory profile and generate report - python3 -c " -import re -import os - -try: - with open('${memory_profile_file}', 'r') as f: - content = f.read() - - # Extract memory usage information - lines = content.split('\n') - memory_usage = [] - - for line in lines: - if 'MiB' in line: - match = re.search(r'(\d+\.\d+)\s+MiB', line) - if match: - memory_usage.append(float(match.group(1))) - - if memory_usage: - max_memory = max(memory_usage) - avg_memory = sum(memory_usage) / len(memory_usage) - - print(f'Memory Profile Analysis for ${test_name}:') - print(f' Max Memory: {max_memory:.2f} MiB') - print(f' Avg Memory: {avg_memory:.2f} MiB') - print(f' Samples: {len(memory_usage)}') - - # Check memory thresholds - if test_name == 'baml_parsing' and max_memory > ${BAML_PARSING_MEMORY_THRESHOLD}: - print(f' โŒ Exceeds memory threshold (${BAML_PARSING_MEMORY_THRESHOLD} MiB)') - elif test_name == 'pareto_lang_parsing' and max_memory > ${PARETO_LANG_PARSING_MEMORY_THRESHOLD}: - print(f' โŒ Exceeds memory threshold (${PARETO_LANG_PARSING_MEMORY_THRESHOLD} MiB)') - elif test_name == 'xml_transformation' and max_memory > ${XML_TRANSFORMATION_MEMORY_THRESHOLD}: - print(f' โŒ Exceeds memory threshold (${XML_TRANSFORMATION_MEMORY_THRESHOLD} MiB)') - elif test_name == 'ai_processing' and max_memory > ${AI_PROCESSING_MEMORY_THRESHOLD}: - print(f' โŒ Exceeds memory threshold (${AI_PROCESSING_MEMORY_THRESHOLD} MiB)') - else: - print(f' โœ… Within memory thresholds') - - # Generate memory profile report - with open('${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_report.json', 'w') as f: - import json - report = { - 'test_name': '${test_name}', - 'max_memory': max_memory, - 'avg_memory': avg_memory, - 'samples': len(memory_usage) - } - json.dump(report, f, indent=2) - - print(f'Memory profile report generated: ${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_report.json') - -except Exception as e: - print(f'โŒ Error generating memory profile report: {e}') -" -} - -# Function to run CPU profiling -run_cpu_profiling() { - local test_name="$1" - local test_file="$2" - local test_description="$3" - - log_step "Running ${test_name} CPU profiling..." - log_info "${test_description}" - - # Create output directory - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export CPU_PROFILER_ENABLED=true - - # Execute CPU profiling - if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ - python -m cProfile \ - -o "${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_profile.prof" \ - -c " -import sys -sys.path.insert(0, 'src') - -# Import and run test -exec(open('${test_file}').read()) -" 2>&1 | \ - tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_profiling.log"; then - - log_success "${test_name} CPU profiling completed!" - - # Generate CPU profile report - generate_cpu_profile_report "${test_name}" - - return 0 - else - log_error "${test_name} CPU profiling failed!" - return 1 - fi -} - -# Function to generate CPU profile report -generate_cpu_profile_report() { - local test_name="$1" - - local cpu_profile_file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_profile.prof" - - if [ ! -f "${cpu_profile_file}" ]; then - log_warning "CPU profile file not found: ${cpu_profile_file}" - return 1 - fi - - # Generate CPU profile report - if command -v snakeviz &> /dev/null; then - snakeviz "${cpu_profile_file}" --hostname=0.0.0.0 --port=8080 & - local snakeviz_pid=$! - - # Wait a moment for snakeviz to start - sleep 2 - - log_info "CPU profile visualization available at: http://localhost:8080" - echo "Press Ctrl+C to stop the visualization server" - - # Wait for user to stop the server - wait $snakeviz_pid - else - # Fallback to pstats - uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ - python -c " -import pstats -import sys -import json - -try: - stats = pstats.Stats('${cpu_profile_file}') - stats.sort_stats('cumulative') - - # Get top 10 functions by cumulative time - top_functions = stats.get_stats_profile().func_profiles[:10] - - report = { - 'test_name': '${test_name}', - 'top_functions': [] - } - - for func, cc, nc, tt, ct, callers in top_functions: - func_info = { - 'function': func, - 'call_count': cc, - 'cumulative_time': ct, - 'total_time': tt - } - report['top_functions'].append(func_info) - - with open('${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_report.json', 'w') as f: - json.dump(report, f, indent=2) - - print(f'CPU profile report generated: ${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_report.json') - -except Exception as e: - print(f'โŒ Error generating CPU profile report: {e}') -" - fi -} - -# Function to run load testing -run_load_testing() { - local test_name="$1" - local test_file="$2" - local test_description="$3" - local concurrency="$4" - local duration="$5" - - log_step "Running ${test_name} load testing..." - log_info "${test_description}" - log_info "Concurrency: ${concurrency}, Duration: ${duration}s" - - # Create output directory - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}/load" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export LOAD_TESTING_ENABLED=true - - # Execute load testing - if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ - python -c " -import concurrent.futures -import time -import sys -import json -import os -sys.path.insert(0, 'src') - -def run_single_test(): - '''Run a single test iteration''' - try: - # Import and run test - with open('${test_file}', 'r') as f: - exec(f.read()) - return {'success': True, 'timestamp': time.time()} - except Exception as e: - return {'success': False, 'error': str(e), 'timestamp': time.time()} - -def run_load_test(): - '''Run load test''' - start_time = time.time() - results = [] - - with concurrent.futures.ThreadPoolExecutor(max_workers=${concurrency}) as executor: - futures = [] - - # Submit tasks for the specified duration - while time.time() - start_time < ${duration}: - future = executor.submit(run_single_test) - futures.append(future) - - # Limit concurrent tasks - if len(futures) >= ${concurrency} * 2: - # Wait for some tasks to complete - completed_futures = [f for f in futures if f.done()] - for completed_future in completed_futures: - results.append(completed_future.result()) - futures.remove(completed_future) - - # Wait for remaining tasks to complete - for future in futures: - results.append(future.result()) - - end_time = time.time() - - # Analyze results - successful_tests = [r for r in results if r['success']] - failed_tests = [r for r in results if not r['success']] - - total_tests = len(results) - success_rate = len(successful_tests) / total_tests * 100 - test_rate = total_tests / (end_time - start_time) - - print(f'Load Test Results for ${test_name}:') - print(f' Total Tests: {total_tests}') - print(f' Successful: {len(successful_tests)}') - print(f' Failed: {len(failed_tests)}') - print(f' Success Rate: {success_rate:.2f}%') - print(f' Test Rate: {test_rate:.2f} tests/sec') - print(f' Duration: {end_time - start_time:.2f}s') - print(f' Concurrency: ${concurrency}') - - # Save results - report = { - 'test_name': '${test_name}', - 'total_tests': total_tests, - 'successful_tests': len(successful_tests), - 'failed_tests': len(failed_tests), - 'success_rate': success_rate, - 'test_rate': test_rate, - 'duration': end_time - start_time, - 'concurrency': ${concurrency} - } - - with open('${PERFORMANCE_OUTPUT_DIR}/${test_name}/load/load_report.json', 'w') as f: - json.dump(report, f, indent=2) - - print(f'Load test report generated: ${PERFORMANCE_OUTPUT_DIR}/${test_name}/load/load_report.json') - - return success_rate >= 95 # Require 95% success rate - -if __name__ == '__main__': - success = run_load_test() - sys.exit(0 if success else 1) -" 2>&1 | \ - tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/load/load_testing.log"; then - - log_success "${test_name} load testing completed!" - return 0 - else - log_error "${test_name} load testing failed!" - return 1 - fi -} - -# Check UVX installation -check_uvx() { - log_info "Checking UVX installation..." - - if ! command -v uvx &> /dev/null; then - log_error "UVX is not installed. Please install UVX first:" - log_error "pip install uvx" - exit 1 - fi - - UVX_VERSION=$(uvx --version) - log_success "UVX installed: ${UVX_VERSION}" -} - -# Check UVX performance environment -check_uvx_performance_environment() { - log_info "Checking UVX performance environment..." - - # Check if performance environment exists - if ! uvx list "${PROJECT_NAME}-performance" &> /dev/null; then - log_warning "UVX performance environment not found. Creating..." - uvx create "${PROJECT_NAME}-performance" --python "${PYTHON_VERSION}" --performance --description "FSL Continuum Performance Testing Environment" - uvx install "${PROJECT_NAME}" --python "${PYTHON_VERSION}" --performance --environment "${PROJECT_NAME}-performance" - fi - - log_success "UVX performance environment ready" -} - -# Set up performance testing environment -setup_performance_environment() { - log_step "Setting up performance testing environment..." - - # Create performance output directories - mkdir -p "${PERFORMANCE_OUTPUT_DIR}" - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/baml_parsing" - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/pareto_lang_parsing" - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/xml_transformation" - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/ai_processing" - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/concurrent_processing" - mkdir -p "${PERFORMANCE_OUTPUT_DIR}/load_testing" - mkdir -p "${BENCHMARK_DIR}" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export PYTEST_CURRENT_TEST="performance" - export PYTEST_BENCHMARK_ENABLED=true - export MEMORY_PROFILER_ENABLED=false - export CPU_PROFILER_ENABLED=false - export LOAD_TESTING_ENABLED=false - - log_success "Performance testing environment setup complete" -} - -# Run all performance benchmarks -run_all_performance_benchmarks() { - log_step "Running all performance benchmarks..." - - # BAML parsing performance - run_performance_benchmark "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing performance benchmark" "${BAML_PARSING_THRESHOLD}" "${BAML_PARSING_MEMORY_THRESHOLD}" - TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) - - # Pareto-Lang parsing performance - run_performance_benchmark "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing performance benchmark" "${PARETO_LANG_PARSING_THRESHOLD}" "${PARETO_LANG_PARSING_MEMORY_THRESHOLD}" - TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) - - # XML transformation performance - run_performance_benchmark "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation performance benchmark" "${XML_TRANSFORMATION_THRESHOLD}" "${XML_TRANSFORMATION_MEMORY_THRESHOLD}" - TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) - - # AI processing performance - run_performance_benchmark "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing performance benchmark" "${AI_PROCESSING_THRESHOLD}" "${AI_PROCESSING_MEMORY_THRESHOLD}" - TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) - - # Concurrent processing performance - run_performance_benchmark "concurrent_processing" "src/tests/performance/test_concurrent_processing_performance.py" "Concurrent processing performance benchmark" "1.0" "150" - TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) -} - -# Run memory profiling -run_all_memory_profiling() { - log_step "Running memory profiling..." - - # BAML parsing memory profiling - run_memory_profiling "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing memory profiling" - - # Pareto-Lang parsing memory profiling - run_memory_profiling "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing memory profiling" - - # XML transformation memory profiling - run_memory_profiling "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation memory profiling" - - # AI processing memory profiling - run_memory_profiling "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing memory profiling" -} - -# Run CPU profiling -run_all_cpu_profiling() { - log_step "Running CPU profiling..." - - # BAML parsing CPU profiling - run_cpu_profiling "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing CPU profiling" - - # Pareto-Lang parsing CPU profiling - run_cpu_profiling "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing CPU profiling" - - # XML transformation CPU profiling - run_cpu_profiling "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation CPU profiling" - - # AI processing CPU profiling - run_cpu_profiling "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing CPU profiling" -} - -# Run load testing -run_all_load_testing() { - log_step "Running load testing..." - - # BAML parsing load testing - run_load_testing "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing load testing" 10 30 - - # Pareto-Lang parsing load testing - run_load_testing "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing load testing" 10 30 - - # XML transformation load testing - run_load_testing "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation load testing" 5 30 - - # AI processing load testing - run_load_testing "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing load testing" 3 30 -} - -# Generate combined performance report -generate_performance_report() { - log_step "Generating combined performance report..." - - # Combine benchmark results - python3 -c " -import json -import glob -import os -import statistics - -benchmarks = {} -benchmark_files = glob.glob('${PERFORMANCE_OUTPUT_DIR}/*/*_benchmark.json') - -for benchmark_file in benchmark_files: - if os.path.exists(benchmark_file): - with open(benchmark_file, 'r') as f: - data = json.load(f) - test_name = os.path.basename(benchmark_file).replace('_benchmark.json', '') - benchmarks[test_name] = data - -# Generate combined report -combined_report = { - 'timestamp': os.path.getmtime('${PERFORMANCE_OUTPUT_DIR}'), - 'benchmarks': benchmarks, - 'summary': { - 'total_benchmarks': len(benchmarks), - 'passed_benchmarks': ${PASSED_BENCHMARKS}, - 'failed_benchmarks': ${FAILED_BENCHMARKS}, - 'regressions_detected': ${REGRESSION_DETECTED} - } -} - -# Save combined report -with open('${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json', 'w') as f: - json.dump(combined_report, f, indent=2) - -# Generate performance summary -print('๐Ÿ“Š Combined Performance Report:') -print('=' * 60) -print(f'Total Benchmarks: {len(benchmarks)}') -print(f'Passed Benchmarks: ${PASSED_BENCHMARKS}') -print(f'Failed Benchmarks: ${FAILED_BENCHMARKS}') -print(f'Regressions Detected: ${REGRESSION_DETECTED}') -print() - -print('Performance Results:') -for test_name, data in benchmarks.items(): - if 'benchmarks' in data: - for benchmark_name, metrics in data['benchmarks'].items(): - if isinstance(metrics, dict) and 'mean' in metrics: - print(f' {test_name}.{benchmark_name}: {metrics[\"mean\"]:.6f}s') - elif 'mean' in data: - print(f' {test_name}: {data[\"mean\"]:.6f}s') - -print() -print('Performance Summary Report:') -print(' Generated: ${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json') -print(' Benchmarks: ${len(benchmarks)}') -" - - log_success "Combined performance report generated: ${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json" -} - -# Generate performance summary -generate_performance_summary() { - log_step "Generating performance summary..." - - TOTAL_TESTS=$((PASSED_BENCHMARKS + FAILED_BENCHMARKS)) - - echo "" - echo "==================================" - echo "๐Ÿš€ FSL CONTINUUM PERFORMANCE SUMMARY" - echo "==================================" - echo "Total Benchmarks: ${TOTAL_TESTS}" - echo "Passed Benchmarks: ${PASSED_BENCHMARKS}" - echo "Failed Benchmarks: ${FAILED_BENCHMARKS}" - echo "Regressions Detected: ${REGRESSION_DETECTED}" - echo "" - - if [ ${FAILED_BENCHMARKS} -eq 0 ] && [ ${REGRESSION_DETECTED} -eq 0 ]; then - echo "โœ… ALL PERFORMANCE BENCHMARKS PASSED!" - echo "๐Ÿš€ FSL Continuum performance is optimal!" - elif [ ${FAILED_BENCHMARKS} -eq 0 ] && [ ${REGRESSION_DETECTED} -gt 0 ]; then - echo "โš ๏ธ PERFORMANCE REGRESSIONS DETECTED!" - echo "๐Ÿ”ง Please review regressions and optimize performance." - else - echo "โŒ SOME PERFORMANCE BENCHMARKS FAILED!" - echo "๐Ÿ”ง Please review failed benchmarks and fix issues." - fi - - echo "==================================" - echo "" - - # Performance results location - echo "๐Ÿ“Š Performance Results Location:" - echo " Performance Logs: ${PERFORMANCE_OUTPUT_DIR}/" - echo " Benchmark Results: ${PERFORMANCE_OUTPUT_DIR}/*_benchmark.json" - echo " Memory Profiles: ${PERFORMANCE_OUTPUT_DIR}/*/memory/" - echo " CPU Profiles: ${PERFORMANCE_OUTPUT_DIR}/*/cpu/" - echo " Load Tests: ${PERFORMANCE_OUTPUT_DIR}/*/load/" - echo " Combined Report: ${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json" - echo "" - - # Performance thresholds status - echo "๐Ÿ“ˆ Performance Thresholds Status:" - echo " BAML Parsing Threshold: ${BAML_PARSING_THRESHOLD}s" - echo " Pareto-Lang Parsing Threshold: ${PARETO_LANG_PARSING_THRESHOLD}s" - echo " XML Transformation Threshold: ${XML_TRANSFORMATION_THRESHOLD}s" - echo " AI Processing Threshold: ${AI_PROCESSING_THRESHOLD}s" - echo "" - - echo "๐Ÿง  Memory Thresholds Status:" - echo " BAML Parsing Memory Threshold: ${BAML_PARSING_MEMORY_THRESHOLD}MB" - echo " Pareto-Lang Parsing Memory Threshold: ${PARETO_LANG_PARSING_MEMORY_THRESHOLD}MB" - echo " XML Transformation Memory Threshold: ${XML_TRANSFORMATION_MEMORY_THRESHOLD}MB" - echo " AI Processing Memory Threshold: ${AI_PROCESSING_MEMORY_THRESHOLD}MB" - echo "" -} - -# Cleanup function -cleanup() { - log_step "Cleaning up performance testing environment..." - - # Clean up temporary files - find "${PERFORMANCE_OUTPUT_DIR}" -name "*.pyc" -delete 2>/dev/null || true - find "${PERFORMANCE_OUTPUT_DIR}" -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true - - log_success "Performance testing environment cleaned up" -} - -# Main function -main() { - local test_type="${1:-all}" - - echo "โšก FSL Continuum UVX Performance Testing" - echo "===================================" - echo "Python Version: ${PYTHON_VERSION}" - echo "Project: ${PROJECT_NAME}" - echo "Test Type: ${test_type}" - echo "" - - # Set up environment - check_uvx - check_uvx_performance_environment - setup_performance_environment - - # Set up cleanup trap - trap cleanup EXIT - - # Run performance tests based on type - case "${test_type}" in - "benchmarks") - run_all_performance_benchmarks - ;; - "memory") - run_all_memory_profiling - ;; - "cpu") - run_all_cpu_profiling - ;; - "load") - run_all_load_testing - ;; - "quick") - run_performance_benchmark "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "Quick BAML parsing benchmark" "${BAML_PARSING_THRESHOLD}" "${BAML_PARSING_MEMORY_THRESHOLD}" - TOTAL_BENCHMARKS=1 - ;; - "all"|*) - run_all_performance_benchmarks - run_all_memory_profiling - run_all_cpu_profiling - run_all_load_testing - ;; - esac - - # Generate reports - generate_performance_report - generate_performance_summary - - # Exit with appropriate code - if [ ${FAILED_BENCHMARKS} -eq 0 ] && [ ${REGRESSION_DETECTED} -eq 0 ]; then - exit 0 - else - exit 1 - fi -} - -# Show usage -usage() { - echo "Usage: $0 [test_type]" - echo "" - echo "Performance Test Types:" - echo " benchmarks - Run performance benchmarks only" - echo " memory - Run memory profiling only" - echo " cpu - Run CPU profiling only" - echo " load - Run load testing only" - echo " quick - Run quick performance test" - echo " all - Run all performance tests (default)" - echo "" - echo "Examples:" - echo " $0 # Run all performance tests" - echo " $0 benchmarks # Run performance benchmarks only" - echo " $0 memory # Run memory profiling only" - echo " $0 quick # Run quick performance test" -} - -# Parse command line arguments -if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then - usage - exit 0 -fi - -# Run main function -main "$@" diff --git a/.uvx/scripts/run_tests.sh b/.uvx/scripts/run_tests.sh deleted file mode 100644 index f1a2fa2..0000000 --- a/.uvx/scripts/run_tests.sh +++ /dev/null @@ -1,524 +0,0 @@ -#!/bin/bash -# FSL Continuum UVX Test Execution Script -# Complete test execution using UVX virtual environments - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -log_step() { - echo -e "${PURPLE}[STEP]${NC} $1" -} - -log_test() { - echo -e "${CYAN}[TEST]${NC} $1" -} - -# Configuration -PYTHON_VERSION="3.11" -PROJECT_NAME="fsl-continuum" -TEST_OUTPUT_DIR="test_results" -COVERAGE_DIR="htmlcov" - -# Test execution counters -TOTAL_TESTS=0 -PASSED_TESTS=0 -FAILED_TESTS=0 -SKIPPED_TESTS=0 - -# Function to run UVX test -run_uvx_test() { - local test_name="$1" - local test_command="$2" - local test_description="$3" - local test_type="$4" - - log_step "Running ${test_name} tests..." - log_info "${test_description}" - - # Create output directory - mkdir -p "${TEST_OUTPUT_DIR}/${test_type}" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export PYTEST_CURRENT_TEST="${test_name}" - export PYTEST_PLUGINS="pytest-cov pytest-mock pytest-benchmark" - - # Execute test - if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" ${test_command} 2>&1 | \ - tee "${TEST_OUTPUT_DIR}/${test_type}/${test_name}.log"; then - - log_success "${test_name} tests passed!" - PASSED_TESTS=$((PASSED_TESTS + 1)) - - # Check if coverage was generated - if [ -f "coverage.xml" ]; then - cp coverage.xml "${TEST_OUTPUT_DIR}/${test_type}/${test_name}_coverage.xml" - fi - - # Check if benchmarks were generated - if [ -f "benchmark.json" ]; then - cp benchmark.json "${TEST_OUTPUT_DIR}/${test_type}/${test_name}_benchmark.json" - fi - - return 0 - else - log_error "${test_name} tests failed!" - FAILED_TESTS=$((FAILED_TESTS + 1)) - return 1 - fi -} - -# Function to run tests with specific markers -run_marked_tests() { - local test_name="$1" - local markers="$2" - local test_description="$3" - - run_uvx_test "${test_name}" "pytest src/tests/ -v -m '${markers}' --cov=src --cov-report=html --cov-report=xml --cov-report=term-missing --junitxml=${TEST_OUTPUT_DIR}/${test_name}.xml" "${test_description}" "${test_name}" -} - -# Function to run specific test files -run_test_files() { - local test_name="$1" - local test_files="$2" - local test_description="$3" - - run_uvx_test "${test_name}" "pytest ${test_files} -v --cov=src --cov-report=html --cov-report=xml --cov-report=term-missing --junitxml=${TEST_OUTPUT_DIR}/${test_name}.xml" "${test_description}" "${test_name}" -} - -# Function to run performance tests -run_performance_tests() { - local test_name="$1" - local test_files="$2" - local test_description="$3" - - run_uvx_test "${test_name}" "pytest ${test_files} -v -m performance --benchmark-only --benchmark-json=${TEST_OUTPUT_DIR}/performance/${test_name}_benchmark.json --benchmark-sort=min" "${test_description}" "performance" -} - -# Check UVX installation -check_uvx() { - log_info "Checking UVX installation..." - - if ! command -v uvx &> /dev/null; then - log_error "UVX is not installed. Please install UVX first:" - log_error "pip install uvx" - exit 1 - fi - - UVX_VERSION=$(uvx --version) - log_success "UVX installed: ${UVX_VERSION}" -} - -# Check UVX environment -check_uvx_environment() { - log_info "Checking UVX environment..." - - # Check if test environment exists - if ! uvx list "${PROJECT_NAME}-test" &> /dev/null; then - log_warning "UVX test environment not found. Creating..." - uvx create "${PROJECT_NAME}-test" --python "${PYTHON_VERSION}" --test --description "FSL Continuum Testing Environment" - uvx install "${PROJECT_NAME}" --python "${PYTHON_VERSION}" --test --environment "${PROJECT_NAME}-test" - fi - - # Check if test package is installed - if ! uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" python -c "import fsl_continuum" &> /dev/null; then - log_warning "FSL Continuum not installed in test environment. Installing..." - uvx install "${PROJECT_NAME}" --python "${PYTHON_VERSION}" --test --environment "${PROJECT_NAME}-test" - fi - - log_success "UVX test environment ready" -} - -# Set up test environment -setup_test_environment() { - log_step "Setting up test environment..." - - # Create test output directories - mkdir -p "${TEST_OUTPUT_DIR}" - mkdir -p "${TEST_OUTPUT_DIR}/unit" - mkdir -p "${TEST_OUTPUT_DIR}/integration" - mkdir -p "${TEST_OUTPUT_DIR}/performance" - mkdir -p "${TEST_OUTPUT_DIR}/ai_processing" - mkdir -p "${TEST_OUTPUT_DIR}/xml_transformation" - mkdir -p "${TEST_OUTPUT_DIR}/semantic_languages" - - # Set environment variables - export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" - export PYTEST_CURRENT_TEST="fsl_continuum" - export PYTEST_PLUGINS="pytest-cov pytest-mock pytest-benchmark" - - # Create pytest configuration - if [ ! -f "pytest.ini" ]; then - log_warning "pytest.ini not found. Using default configuration." - fi - - log_success "Test environment setup complete" -} - -# Run unit tests -run_unit_tests() { - log_step "Running unit tests..." - - run_marked_tests "unit" "unit" "Running unit tests for individual components" - - # Run specific unit test categories - run_test_files "baml_unit" "src/tests/unit/semantic_languages/test_baml_*.py" "Running BAML unit tests" - run_test_files "pareto_lang_unit" "src/tests/unit/semantic_languages/test_pareto_lang_*.py" "Running Pareto-Lang unit tests" - run_test_files "xml_processing_unit" "src/tests/unit/xml_processing/test_*.py" "Running XML processing unit tests" - run_test_files "ai_integration_unit" "src/tests/unit/ai_integration/test_*.py" "Running AI integration unit tests" -} - -# Run integration tests -run_integration_tests() { - log_step "Running integration tests..." - - run_marked_tests "integration" "integration" "Running integration tests for component interactions" - - # Run specific integration test categories - run_test_files "semantic_language_integration" "src/tests/integration/test_semantic_language_integration.py" "Running semantic language integration tests" - run_test_files "xml_integration" "src/tests/integration/test_xml_integration.py" "Running XML integration tests" - run_test_files "ai_integration" "src/tests/integration/test_ai_integration.py" "Running AI integration tests" -} - -# Run performance tests -run_performance_tests_all() { - log_step "Running performance tests..." - - run_performance_tests "semantic_language_performance" "src/tests/performance/test_semantic_language_performance.py" "Running semantic language performance tests" - run_performance_tests "xml_performance" "src/tests/performance/test_xml_performance.py" "Running XML transformation performance tests" - run_performance_tests "ai_performance" "src/tests/performance/test_ai_performance.py" "Running AI processing performance tests" -} - -# Run AI processing tests -run_ai_processing_tests() { - log_step "Running AI processing tests..." - - run_marked_tests "ai_processing" "ai_processing" "Running AI processing and validation tests" - - # Run specific AI test categories - run_test_files "semantic_ai_processing" "src/tests/ai_processing/test_semantic_ai_processor.py" "Running semantic AI processing tests" - run_test_files "ai_optimization" "src/tests/ai_processing/test_ai_optimizer.py" "Running AI optimization tests" -} - -# Run XML transformation tests -run_xml_transformation_tests() { - log_step "Running XML transformation tests..." - - run_marked_tests "xml_transformation" "xml_transformation" "Running XML transformation tests" - - # Run specific XML test categories - run_test_files "xml_processor" "src/tests/xml_transformation/test_xml_processor.py" "Running XML processor tests" - run_test_files "xml_validator" "src/tests/xml_transformation/test_xml_validator.py" "Running XML validator tests" - run_test_files "xml_transformer" "src/tests/xml_transformation/test_xml_transformer.py" "Running XML transformer tests" -} - -# Run semantic language tests -run_semantic_language_tests() { - log_step "Running semantic language tests..." - - run_marked_tests "semantic_languages" "semantic_languages" "Running semantic language integration tests" - - # Run specific semantic language test categories - run_test_files "baml_tests" "src/tests/semantic_languages/test_baml_*.py" "Running BAML tests" - run_test_files "pareto_lang_tests" "src/tests/semantic_languages/test_pareto_lang_*.py" "Running Pareto-Lang tests" - run_test_files "semantic_bridge" "src/tests/semantic_languages/test_semantic_bridge.py" "Running semantic bridge tests" -} - -# Run smoke tests -run_smoke_tests() { - log_step "Running smoke tests..." - - run_marked_tests "smoke" "smoke" "Running smoke tests for basic functionality" - - # Run basic smoke tests - run_test_files "basic_smoke" "src/tests/smoke/test_basic_smoke.py" "Running basic smoke tests" -} - -# Run regression tests -run_regression_tests() { - log_step "Running regression tests..." - - run_marked_tests "regression" "regression" "Running regression tests" - - # Run specific regression tests - run_test_files "semantic_regression" "src/tests/regression/test_semantic_regression.py" "Running semantic regression tests" -} - -# Generate coverage report -generate_coverage_report() { - log_step "Generating combined coverage report..." - - # Combine coverage data - if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage combine 2>/dev/null; then - log_success "Coverage data combined" - else - log_warning "Failed to combine coverage data" - fi - - # Generate HTML coverage report - if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage html --directory="${COVERAGE_DIR}" 2>/dev/null; then - log_success "HTML coverage report generated: ${COVERAGE_DIR}/index.html" - else - log_warning "Failed to generate HTML coverage report" - fi - - # Generate XML coverage report - if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage xml 2>/dev/null; then - log_success "XML coverage report generated: coverage.xml" - else - log_warning "Failed to generate XML coverage report" - fi - - # Generate terminal coverage report - if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage report 2>/dev/null; then - log_success "Terminal coverage report generated" - else - log_warning "Failed to generate terminal coverage report" - fi -} - -# Generate benchmark report -generate_benchmark_report() { - log_step "Generating benchmark report..." - - # Combine benchmark data - python3 -c " -import json -import glob -import os - -benchmarks = {} -benchmark_files = glob.glob('test_results/performance/*_benchmark.json') - -for benchmark_file in benchmark_files: - if os.path.exists(benchmark_file): - with open(benchmark_file, 'r') as f: - data = json.load(f) - test_name = os.path.basename(benchmark_file).replace('_benchmark.json', '') - benchmarks[test_name] = data - -with open('combined_benchmark.json', 'w') as f: - json.dump(benchmarks, f, indent=2) - -print('Combined benchmark report generated: combined_benchmark.json') -print(f'Found {len(benchmarks)} benchmark results') -" -} - -# Generate test summary -generate_test_summary() { - log_step "Generating test summary..." - - TOTAL_TESTS=$((PASSED_TESTS + FAILED_TESTS + SKIPPED_TESTS)) - - echo "" - echo "==================================" - echo "๐ŸŽฏ FSL CONTINUUM TEST SUMMARY" - echo "==================================" - echo "Total Tests Run: ${TOTAL_TESTS}" - echo "Passed Tests: ${PASSED_TESTS}" - echo "Failed Tests: ${FAILED_TESTS}" - echo "Skipped Tests: ${SKIPPED_TESTS}" - echo "" - - if [ ${FAILED_TESTS} -eq 0 ]; then - echo "โœ… ALL TESTS PASSED!" - echo "๐Ÿš€ FSL Continuum is ready for production!" - else - echo "โŒ SOME TESTS FAILED!" - echo "๐Ÿ”ง Please review the failed tests and fix the issues." - fi - - echo "==================================" - echo "" - - # Test results location - echo "๐Ÿ“Š Test Results Location:" - echo " Test Logs: ${TEST_OUTPUT_DIR}/" - echo " Coverage Report: ${COVERAGE_DIR}/index.html" - echo " Coverage XML: coverage.xml" - echo " Benchmark Report: combined_benchmark.json" - echo "" - - # Coverage summary - if [ -f "coverage.xml" ]; then - echo "๐Ÿ“ˆ Coverage Summary:" - uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage report | tail -n 1 - echo "" - fi - - # Performance summary - if [ -f "combined_benchmark.json" ]; then - echo "โšก Performance Summary:" - python3 -c " -import json -import os - -if os.path.exists('combined_benchmark.json'): - with open('combined_benchmark.json', 'r') as f: - benchmarks = json.load(f) - - print(f' Benchmarks: {len(benchmarks)}') - for test_name, data in benchmarks.items(): - if 'benchmarks' in data: - for benchmark, metrics in data['benchmarks'].items(): - if 'mean' in metrics: - print(f' {test_name}.{benchmark}: {metrics[\"mean\"]:.3f}s') - elif 'mean' in data: - print(f' {test_name}: {data[\"mean\"]:.3f}s') -" - echo "" - fi -} - -# Cleanup function -cleanup() { - log_step "Cleaning up test environment..." - - # Clean up temporary files - find "${TEST_OUTPUT_DIR}" -name "*.pyc" -delete 2>/dev/null || true - find "${TEST_OUTPUT_DIR}" -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true - - log_success "Test environment cleaned up" -} - -# Main function -main() { - local test_type="${1:-all}" - - echo "๐Ÿงช FSL Continuum UVX Test Execution" - echo "==================================" - echo "Python Version: ${PYTHON_VERSION}" - echo "Project: ${PROJECT_NAME}" - echo "Test Type: ${test_type}" - echo "" - - # Set up environment - check_uvx - check_uvx_environment - setup_test_environment - - # Set up cleanup trap - trap cleanup EXIT - - # Run tests based on type - case "${test_type}" in - "unit") - run_unit_tests - ;; - "integration") - run_integration_tests - ;; - "performance") - run_performance_tests_all - ;; - "ai_processing") - run_ai_processing_tests - ;; - "xml_transformation") - run_xml_transformation_tests - ;; - "semantic_languages") - run_semantic_language_tests - ;; - "smoke") - run_smoke_tests - ;; - "regression") - run_regression_tests - ;; - "quick") - run_smoke_tests - run_unit_tests - ;; - "full") - run_smoke_tests - run_unit_tests - run_integration_tests - run_ai_processing_tests - run_xml_transformation_tests - run_semantic_language_tests - run_performance_tests_all - ;; - "all"|*) - run_smoke_tests - run_unit_tests - run_integration_tests - run_ai_processing_tests - run_xml_transformation_tests - run_semantic_language_tests - run_performance_tests_all - run_regression_tests - ;; - esac - - # Generate reports - generate_coverage_report - generate_benchmark_report - generate_test_summary - - # Exit with appropriate code - if [ ${FAILED_TESTS} -eq 0 ]; then - exit 0 - else - exit 1 - fi -} - -# Show usage -usage() { - echo "Usage: $0 [test_type]" - echo "" - echo "Test Types:" - echo " unit - Run unit tests only" - echo " integration - Run integration tests only" - echo " performance - Run performance tests only" - echo " ai_processing - Run AI processing tests only" - echo " xml_transformation - Run XML transformation tests only" - echo " semantic_languages - Run semantic language tests only" - echo " smoke - Run smoke tests only" - echo " regression - Run regression tests only" - echo " quick - Run smoke and unit tests" - echo " full - Run all tests except regression" - echo " all - Run all tests (default)" - echo "" - echo "Examples:" - echo " $0 # Run all tests" - echo " $0 unit # Run unit tests only" - echo " $0 performance # Run performance tests only" - echo " $0 quick # Run quick tests" -} - -# Parse command line arguments -if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then - usage - exit 0 -fi - -# Run main function -main "$@" diff --git a/.uvx/scripts/setup.sh b/.uvx/scripts/setup.sh deleted file mode 100644 index ab8803f..0000000 --- a/.uvx/scripts/setup.sh +++ /dev/null @@ -1,233 +0,0 @@ -#!/bin/bash -# FSL Continuum UVX Environment Setup Script -# Complete UVX virtual environment setup for testing and development - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Check if UVX is installed -check_uvx() { - log_info "Checking UVX installation..." - - if ! command -v uvx &> /dev/null; then - log_error "UVX is not installed. Please install UVX first:" - log_error "pip install uvx" - exit 1 - fi - - UVX_VERSION=$(uvx --version) - log_success "UVX installed: $UVX_VERSION" -} - -# Create UVX directory structure -create_directories() { - log_info "Creating UVX directory structure..." - - directories=( - ".uvx" - ".uvx/cache" - ".uvx/environments" - ".uvx/logs" - ".uvx/scripts" - ".uvx/artifacts" - ) - - for dir in "${directories[@]}"; do - if [ ! -d "$dir" ]; then - mkdir -p "$dir" - log_success "Created directory: $dir" - else - log_info "Directory already exists: $dir" - fi - done -} - -# Create UVX environments -create_environments() { - log_info "Creating UVX environments..." - - # Create production environment - log_info "Creating fsl-continuum production environment..." - if ! uvx list fsl-continuum &> /dev/null; then - uvx create fsl-continuum --python 3.11 --description "FSL Continuum Production Environment" - log_success "Created fsl-continuum production environment" - else - log_info "fsl-continuum production environment already exists" - fi - - # Create development environment - log_info "Creating fsl-continuum-dev development environment..." - if ! uvx list fsl-continuum-dev &> /dev/null; then - uvx create fsl-continuum-dev --python 3.11 --dev --description "FSL Continuum Development Environment" - log_success "Created fsl-continuum-dev development environment" - else - log_info "fsl-continuum-dev development environment already exists" - fi - - # Create testing environment - log_info "Creating fsl-continuum-test testing environment..." - if ! uvx list fsl-continuum-test &> /dev/null; then - uvx create fsl-continuum-test --python 3.11 --test --description "FSL Continuum Testing Environment" - log_success "Created fsl-continuum-test testing environment" - else - log_info "fsl-continuum-test testing environment already exists" - fi - - # Create coverage environment - log_info "Creating fsl-continuum-coverage coverage environment..." - if ! uvx list fsl-continuum-coverage &> /dev/null; then - uvx create fsl-continuum-coverage --python 3.11 --coverage --description "FSL Continuum Coverage Testing Environment" - log_success "Created fsl-continuum-coverage coverage environment" - else - log_info "fsl-continuum-coverage coverage environment already exists" - fi - - # Create performance environment - log_info "Creating fsl-continuum-performance performance environment..." - if ! uvx list fsl-continuum-performance &> /dev/null; then - uvx create fsl-continuum-performance --python 3.11 --performance --description "FSL Continuum Performance Testing Environment" - log_success "Created fsl-continuum-performance performance environment" - else - log_info "fsl-continuum-performance performance environment already exists" - fi -} - -# Install FSL Continuum in environments -install_fsl_continuum() { - log_info "Installing FSL Continuum in UVX environments..." - - # Install in production environment - log_info "Installing FSL Continuum in production environment..." - uvx install fsl-continuum --python 3.11 --environment fsl-continuum - log_success "Installed FSL Continuum in production environment" - - # Install in development environment - log_info "Installing FSL Continuum in development environment..." - uvx install fsl-continuum --python 3.11 --dev --environment fsl-continuum-dev - log_success "Installed FSL Continuum in development environment" - - # Install in testing environment - log_info "Installing FSL Continuum in testing environment..." - uvx install fsl-continuum --python 3.11 --test --environment fsl-continuum-test - log_success "Installed FSL Continuum in testing environment" - - # Install in coverage environment - log_info "Installing FSL Continuum in coverage environment..." - uvx install fsl-continuum --python 3.11 --coverage --environment fsl-continuum-coverage - log_success "Installed FSL Continuum in coverage environment" - - # Install in performance environment - log_info "Installing FSL Continuum in performance environment..." - uvx install fsl-continuum --python 3.11 --performance --environment fsl-continuum-performance - log_success "Installed FSL Continuum in performance environment" -} - -# Validate UVX setup -validate_setup() { - log_info "Validating UVX setup..." - - # Check if all environments exist - environments=("fsl-continuum" "fsl-continuum-dev" "fsl-continuum-test" "fsl-continuum-coverage" "fsl-continuum-performance") - - for env in "${environments[@]}"; do - if uvx list "$env" &> /dev/null; then - log_success "Environment $env exists" - else - log_error "Environment $env does not exist" - exit 1 - fi - done - - # Test import in production environment - log_info "Testing FSL Continuum import in production environment..." - if uvx run --python 3.11 fsl-continuum python -c "import fsl_continuum; print(fsl_continuum.get_version())" &> /dev/null; then - log_success "FSL Continuum import works in production environment" - else - log_error "FSL Continuum import failed in production environment" - exit 1 - fi - - # Test import in testing environment - log_info "Testing FSL Continuum import in testing environment..." - if uvx run --python 3.11 --test fsl-continuum python -c "import fsl_continuum; print('Testing import successful')" &> /dev/null; then - log_success "FSL Continuum import works in testing environment" - else - log_error "FSL Continuum import failed in testing environment" - exit 1 - fi -} - -# Create UVX scripts -create_scripts() { - log_info "Creating UVX scripts..." - - # Make scripts executable - chmod +x .uvx/scripts/*.sh - - log_success "Created UVX scripts" -} - -# Main function -main() { - echo "๐Ÿ”ง FSL Continuum UVX Environment Setup" - echo "==================================" - - # Change to project root - cd "$(dirname "$0")/../.." - - log_info "Setting up UVX environments for FSL Continuum..." - - check_uvx - create_directories - create_environments - install_fsl_continuum - validate_setup - create_scripts - - echo "==================================" - log_success "UVX environment setup complete!" - echo "" - log_info "Available UVX environments:" - uvx list - - echo "" - log_info "Usage examples:" - echo " uvx run --python 3.11 fsl-continuum # Production environment" - echo " uvx run --python 3.11 --dev fsl-continuum # Development environment" - echo " uvx run --python 3.11 --test fsl-continuum # Testing environment" - echo " uvx run --python 3.11 --coverage fsl-continuum # Coverage environment" - echo " uvx run --python 3.11 --performance fsl-continuum # Performance environment" - echo "" - echo " .uvx/scripts/run_tests.sh # Run all tests" - echo " .uvx/scripts/run_performance_tests.sh # Run performance tests" - echo " .uvx/scripts/run_coverage_tests.sh # Run coverage tests" - - echo "" - log_success "Ready to run tests with UVX!" -} - -# Run main function -main "$@" From 7ff0eaea0d31bd91f380357124705db287ae4199 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 05:00:16 +0000 Subject: [PATCH 3/4] Fix .gitignore conflict by including both .fsl_test_env/ and .uvx/ Co-authored-by: SomeRandmGuyy <127457986+SomeRandmGuyy@users.noreply.github.com> --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 64e2f8e..3a55d05 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.fsl_test_env/ .uvx/ # Spyder project settings From cd2803476e52f0f120a76fc77d468761a99bacf3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Nov 2025 05:07:12 +0000 Subject: [PATCH 4/4] Restore .uvx/ directory and remove from .gitignore for testing Co-authored-by: SomeRandmGuyy <127457986+SomeRandmGuyy@users.noreply.github.com> --- .gitignore | 1 - .uvx/config.toml | 7 + .uvx/scripts/run_performance_tests.sh | 873 ++++++++++++++++++++++++++ .uvx/scripts/run_tests.sh | 524 ++++++++++++++++ .uvx/scripts/setup.sh | 233 +++++++ 5 files changed, 1637 insertions(+), 1 deletion(-) create mode 100644 .uvx/config.toml create mode 100644 .uvx/scripts/run_performance_tests.sh create mode 100644 .uvx/scripts/run_tests.sh create mode 100644 .uvx/scripts/setup.sh diff --git a/.gitignore b/.gitignore index 3a55d05..82405b4 100644 --- a/.gitignore +++ b/.gitignore @@ -111,7 +111,6 @@ ENV/ env.bak/ venv.bak/ .fsl_test_env/ -.uvx/ # Spyder project settings .spyderproject diff --git a/.uvx/config.toml b/.uvx/config.toml new file mode 100644 index 0000000..1c7f4f2 --- /dev/null +++ b/.uvx/config.toml @@ -0,0 +1,7 @@ +[uvx] +python-version = "3.11" +cache-dir = ".uvx/cache" +env-dir = ".uvx/envs" + +[pip] +index-url = "https://pypi.org/simple" diff --git a/.uvx/scripts/run_performance_tests.sh b/.uvx/scripts/run_performance_tests.sh new file mode 100644 index 0000000..e001e14 --- /dev/null +++ b/.uvx/scripts/run_performance_tests.sh @@ -0,0 +1,873 @@ +#!/bin/bash +# FSL Continuum UVX Performance Testing Script +# Complete performance testing using UVX virtual environments + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${PURPLE}[STEP]${NC} $1" +} + +log_benchmark() { + echo -e "${CYAN}[BENCHMARK]${NC} $1" +} + +# Configuration +PYTHON_VERSION="3.11" +PROJECT_NAME="fsl-continuum" +PERFORMANCE_OUTPUT_DIR="performance_results" +BENCHMARK_DIR=".benchmark" + +# Performance thresholds +BAML_PARSING_THRESHOLD=0.1 # 100ms per document +PARETO_LANG_PARSING_THRESHOLD=0.1 # 100ms per document +XML_TRANSFORMATION_THRESHOLD=0.5 # 500ms per transformation +AI_PROCESSING_THRESHOLD=2.0 # 2 seconds per AI operation + +# Memory thresholds +BAML_PARSING_MEMORY_THRESHOLD=50 # 50MB +PARETO_LANG_PARSING_MEMORY_THRESHOLD=50 # 50MB +XML_TRANSFORMATION_MEMORY_THRESHOLD=100 # 100MB +AI_PROCESSING_MEMORY_THRESHOLD=200 # 200MB + +# Performance counters +TOTAL_BENCHMARKS=0 +PASSED_BENCHMARKS=0 +FAILED_BENCHMARKS=0 +REGRESSION_DETECTED=0 + +# Function to run performance benchmark +run_performance_benchmark() { + local test_name="$1" + local test_file="$2" + local test_description="$3" + local threshold="$4" + local memory_threshold="$5" + + log_benchmark "Running ${test_name} performance benchmark..." + log_info "${test_description}" + + # Create output directory + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export PYTEST_CURRENT_TEST="${test_name}_performance" + export PYTEST_BENCHMARK_ENABLED=true + export PYTEST_PLUGINS="pytest-cov pytest-mock pytest-benchmark" + + # Execute benchmark + if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ + pytest "${test_file}" -v -m performance \ + --benchmark-only \ + --benchmark-json="${PERFORMANCE_OUTPUT_DIR}/${test_name}/${test_name}_benchmark.json" \ + --benchmark-sort=min \ + --benchmark-min-rounds=5 \ + --benchmark-max-time=30 \ + --benchmark-warmup=true \ + --benchmark-warmup-shutdown=true \ + --benchmark-storage="${BENCHMARK_DIR}" \ + 2>&1 | \ + tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/${test_name}_benchmark.log"; then + + log_success "${test_name} performance benchmark completed!" + PASSED_BENCHMARKS=$((PASSED_BENCHMARKS + 1)) + + # Analyze benchmark results + analyze_benchmark_results "${test_name}" "${threshold}" "${memory_threshold}" + + return 0 + else + log_error "${test_name} performance benchmark failed!" + FAILED_BENCHMARKS=$((FAILED_BENCHMARKS + 1)) + return 1 + fi +} + +# Function to analyze benchmark results +analyze_benchmark_results() { + local test_name="$1" + local threshold="$2" + local memory_threshold="$3" + + local benchmark_file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/${test_name}_benchmark.json" + + if [ ! -f "${benchmark_file}" ]; then + log_warning "Benchmark file not found: ${benchmark_file}" + return 1 + fi + + # Parse benchmark results using Python + python3 -c " +import json +import sys +import os + +try: + with open('${benchmark_file}', 'r') as f: + data = json.load(f) + + if 'benchmarks' in data: + benchmarks = data['benchmarks'] + else: + benchmarks = data + + regression_detected = False + + for benchmark_name, metrics in benchmarks.items(): + if isinstance(metrics, dict) and 'mean' in metrics: + mean_time = metrics['mean'] + min_time = metrics.get('min', mean_time) + max_time = metrics.get('max', mean_time) + + print(f' {benchmark_name}:') + print(f' Mean: {mean_time:.6f}s') + print(f' Min: {min_time:.6f}s') + print(f' Max: {max_time:.6f}s') + + # Check threshold + if mean_time > ${threshold}: + print(f' โŒ Exceeds threshold (${threshold}s)') + regression_detected = True + else: + print(f' โœ… Within threshold (${threshold}s)') + + # Check memory if available + if 'memory' in metrics: + memory_usage = metrics['memory'] + print(f' Memory: {memory_usage:.2f}MB') + + if memory_usage > ${memory_threshold}: + print(f' โŒ Exceeds memory threshold (${memory_threshold}MB)') + regression_detected = True + else: + print(f' โœ… Within memory threshold (${memory_threshold}MB)') + + print() + + if regression_detected: + print(f'๐Ÿšจ REGRESSION DETECTED in ${test_name}!') + sys.exit(1) + else: + print(f'โœ… No regressions detected in ${test_name}!') + sys.exit(0) + +except Exception as e: + print(f'โŒ Error analyzing benchmark: {e}') + sys.exit(1) +" + + local result=$? + if [ $result -eq 1 ]; then + REGRESSION_DETECTED=$((REGRESSION_DETECTED + 1)) + log_warning "Performance regression detected in ${test_name}" + fi +} + +# Function to run memory profiling +run_memory_profiling() { + local test_name="$1" + local test_file="$2" + local test_description="$3" + + log_step "Running ${test_name} memory profiling..." + log_info "${test_description}" + + # Create output directory + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export MEMORY_PROFILER_ENABLED=true + + # Execute memory profiling + if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ + python -m memory_profiler \ + --output-file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_profile.txt" \ + -c " +import sys +sys.path.insert(0, 'src') +from memory_profiler import profile + +@profile +def run_memory_test(): + # Import and run test + exec(open('${test_file}').read()) + +if __name__ == '__main__': + run_memory_test() +" 2>&1 | \ + tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_profiling.log"; then + + log_success "${test_name} memory profiling completed!" + + # Generate memory profile report + generate_memory_profile_report "${test_name}" + + return 0 + else + log_error "${test_name} memory profiling failed!" + return 1 + fi +} + +# Function to generate memory profile report +generate_memory_profile_report() { + local test_name="$1" + + local memory_profile_file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_profile.txt" + + if [ ! -f "${memory_profile_file}" ]; then + log_warning "Memory profile file not found: ${memory_profile_file}" + return 1 + fi + + # Parse memory profile and generate report + python3 -c " +import re +import os + +try: + with open('${memory_profile_file}', 'r') as f: + content = f.read() + + # Extract memory usage information + lines = content.split('\n') + memory_usage = [] + + for line in lines: + if 'MiB' in line: + match = re.search(r'(\d+\.\d+)\s+MiB', line) + if match: + memory_usage.append(float(match.group(1))) + + if memory_usage: + max_memory = max(memory_usage) + avg_memory = sum(memory_usage) / len(memory_usage) + + print(f'Memory Profile Analysis for ${test_name}:') + print(f' Max Memory: {max_memory:.2f} MiB') + print(f' Avg Memory: {avg_memory:.2f} MiB') + print(f' Samples: {len(memory_usage)}') + + # Check memory thresholds + if test_name == 'baml_parsing' and max_memory > ${BAML_PARSING_MEMORY_THRESHOLD}: + print(f' โŒ Exceeds memory threshold (${BAML_PARSING_MEMORY_THRESHOLD} MiB)') + elif test_name == 'pareto_lang_parsing' and max_memory > ${PARETO_LANG_PARSING_MEMORY_THRESHOLD}: + print(f' โŒ Exceeds memory threshold (${PARETO_LANG_PARSING_MEMORY_THRESHOLD} MiB)') + elif test_name == 'xml_transformation' and max_memory > ${XML_TRANSFORMATION_MEMORY_THRESHOLD}: + print(f' โŒ Exceeds memory threshold (${XML_TRANSFORMATION_MEMORY_THRESHOLD} MiB)') + elif test_name == 'ai_processing' and max_memory > ${AI_PROCESSING_MEMORY_THRESHOLD}: + print(f' โŒ Exceeds memory threshold (${AI_PROCESSING_MEMORY_THRESHOLD} MiB)') + else: + print(f' โœ… Within memory thresholds') + + # Generate memory profile report + with open('${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_report.json', 'w') as f: + import json + report = { + 'test_name': '${test_name}', + 'max_memory': max_memory, + 'avg_memory': avg_memory, + 'samples': len(memory_usage) + } + json.dump(report, f, indent=2) + + print(f'Memory profile report generated: ${PERFORMANCE_OUTPUT_DIR}/${test_name}/memory/memory_report.json') + +except Exception as e: + print(f'โŒ Error generating memory profile report: {e}') +" +} + +# Function to run CPU profiling +run_cpu_profiling() { + local test_name="$1" + local test_file="$2" + local test_description="$3" + + log_step "Running ${test_name} CPU profiling..." + log_info "${test_description}" + + # Create output directory + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export CPU_PROFILER_ENABLED=true + + # Execute CPU profiling + if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ + python -m cProfile \ + -o "${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_profile.prof" \ + -c " +import sys +sys.path.insert(0, 'src') + +# Import and run test +exec(open('${test_file}').read()) +" 2>&1 | \ + tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_profiling.log"; then + + log_success "${test_name} CPU profiling completed!" + + # Generate CPU profile report + generate_cpu_profile_report "${test_name}" + + return 0 + else + log_error "${test_name} CPU profiling failed!" + return 1 + fi +} + +# Function to generate CPU profile report +generate_cpu_profile_report() { + local test_name="$1" + + local cpu_profile_file="${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_profile.prof" + + if [ ! -f "${cpu_profile_file}" ]; then + log_warning "CPU profile file not found: ${cpu_profile_file}" + return 1 + fi + + # Generate CPU profile report + if command -v snakeviz &> /dev/null; then + snakeviz "${cpu_profile_file}" --hostname=0.0.0.0 --port=8080 & + local snakeviz_pid=$! + + # Wait a moment for snakeviz to start + sleep 2 + + log_info "CPU profile visualization available at: http://localhost:8080" + echo "Press Ctrl+C to stop the visualization server" + + # Wait for user to stop the server + wait $snakeviz_pid + else + # Fallback to pstats + uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ + python -c " +import pstats +import sys +import json + +try: + stats = pstats.Stats('${cpu_profile_file}') + stats.sort_stats('cumulative') + + # Get top 10 functions by cumulative time + top_functions = stats.get_stats_profile().func_profiles[:10] + + report = { + 'test_name': '${test_name}', + 'top_functions': [] + } + + for func, cc, nc, tt, ct, callers in top_functions: + func_info = { + 'function': func, + 'call_count': cc, + 'cumulative_time': ct, + 'total_time': tt + } + report['top_functions'].append(func_info) + + with open('${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_report.json', 'w') as f: + json.dump(report, f, indent=2) + + print(f'CPU profile report generated: ${PERFORMANCE_OUTPUT_DIR}/${test_name}/cpu/cpu_report.json') + +except Exception as e: + print(f'โŒ Error generating CPU profile report: {e}') +" + fi +} + +# Function to run load testing +run_load_testing() { + local test_name="$1" + local test_file="$2" + local test_description="$3" + local concurrency="$4" + local duration="$5" + + log_step "Running ${test_name} load testing..." + log_info "${test_description}" + log_info "Concurrency: ${concurrency}, Duration: ${duration}s" + + # Create output directory + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/${test_name}/load" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export LOAD_TESTING_ENABLED=true + + # Execute load testing + if uvx run --python "${PYTHON_VERSION}" --performance "${PROJECT_NAME}" \ + python -c " +import concurrent.futures +import time +import sys +import json +import os +sys.path.insert(0, 'src') + +def run_single_test(): + '''Run a single test iteration''' + try: + # Import and run test + with open('${test_file}', 'r') as f: + exec(f.read()) + return {'success': True, 'timestamp': time.time()} + except Exception as e: + return {'success': False, 'error': str(e), 'timestamp': time.time()} + +def run_load_test(): + '''Run load test''' + start_time = time.time() + results = [] + + with concurrent.futures.ThreadPoolExecutor(max_workers=${concurrency}) as executor: + futures = [] + + # Submit tasks for the specified duration + while time.time() - start_time < ${duration}: + future = executor.submit(run_single_test) + futures.append(future) + + # Limit concurrent tasks + if len(futures) >= ${concurrency} * 2: + # Wait for some tasks to complete + completed_futures = [f for f in futures if f.done()] + for completed_future in completed_futures: + results.append(completed_future.result()) + futures.remove(completed_future) + + # Wait for remaining tasks to complete + for future in futures: + results.append(future.result()) + + end_time = time.time() + + # Analyze results + successful_tests = [r for r in results if r['success']] + failed_tests = [r for r in results if not r['success']] + + total_tests = len(results) + success_rate = len(successful_tests) / total_tests * 100 + test_rate = total_tests / (end_time - start_time) + + print(f'Load Test Results for ${test_name}:') + print(f' Total Tests: {total_tests}') + print(f' Successful: {len(successful_tests)}') + print(f' Failed: {len(failed_tests)}') + print(f' Success Rate: {success_rate:.2f}%') + print(f' Test Rate: {test_rate:.2f} tests/sec') + print(f' Duration: {end_time - start_time:.2f}s') + print(f' Concurrency: ${concurrency}') + + # Save results + report = { + 'test_name': '${test_name}', + 'total_tests': total_tests, + 'successful_tests': len(successful_tests), + 'failed_tests': len(failed_tests), + 'success_rate': success_rate, + 'test_rate': test_rate, + 'duration': end_time - start_time, + 'concurrency': ${concurrency} + } + + with open('${PERFORMANCE_OUTPUT_DIR}/${test_name}/load/load_report.json', 'w') as f: + json.dump(report, f, indent=2) + + print(f'Load test report generated: ${PERFORMANCE_OUTPUT_DIR}/${test_name}/load/load_report.json') + + return success_rate >= 95 # Require 95% success rate + +if __name__ == '__main__': + success = run_load_test() + sys.exit(0 if success else 1) +" 2>&1 | \ + tee "${PERFORMANCE_OUTPUT_DIR}/${test_name}/load/load_testing.log"; then + + log_success "${test_name} load testing completed!" + return 0 + else + log_error "${test_name} load testing failed!" + return 1 + fi +} + +# Check UVX installation +check_uvx() { + log_info "Checking UVX installation..." + + if ! command -v uvx &> /dev/null; then + log_error "UVX is not installed. Please install UVX first:" + log_error "pip install uvx" + exit 1 + fi + + UVX_VERSION=$(uvx --version) + log_success "UVX installed: ${UVX_VERSION}" +} + +# Check UVX performance environment +check_uvx_performance_environment() { + log_info "Checking UVX performance environment..." + + # Check if performance environment exists + if ! uvx list "${PROJECT_NAME}-performance" &> /dev/null; then + log_warning "UVX performance environment not found. Creating..." + uvx create "${PROJECT_NAME}-performance" --python "${PYTHON_VERSION}" --performance --description "FSL Continuum Performance Testing Environment" + uvx install "${PROJECT_NAME}" --python "${PYTHON_VERSION}" --performance --environment "${PROJECT_NAME}-performance" + fi + + log_success "UVX performance environment ready" +} + +# Set up performance testing environment +setup_performance_environment() { + log_step "Setting up performance testing environment..." + + # Create performance output directories + mkdir -p "${PERFORMANCE_OUTPUT_DIR}" + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/baml_parsing" + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/pareto_lang_parsing" + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/xml_transformation" + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/ai_processing" + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/concurrent_processing" + mkdir -p "${PERFORMANCE_OUTPUT_DIR}/load_testing" + mkdir -p "${BENCHMARK_DIR}" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export PYTEST_CURRENT_TEST="performance" + export PYTEST_BENCHMARK_ENABLED=true + export MEMORY_PROFILER_ENABLED=false + export CPU_PROFILER_ENABLED=false + export LOAD_TESTING_ENABLED=false + + log_success "Performance testing environment setup complete" +} + +# Run all performance benchmarks +run_all_performance_benchmarks() { + log_step "Running all performance benchmarks..." + + # BAML parsing performance + run_performance_benchmark "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing performance benchmark" "${BAML_PARSING_THRESHOLD}" "${BAML_PARSING_MEMORY_THRESHOLD}" + TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) + + # Pareto-Lang parsing performance + run_performance_benchmark "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing performance benchmark" "${PARETO_LANG_PARSING_THRESHOLD}" "${PARETO_LANG_PARSING_MEMORY_THRESHOLD}" + TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) + + # XML transformation performance + run_performance_benchmark "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation performance benchmark" "${XML_TRANSFORMATION_THRESHOLD}" "${XML_TRANSFORMATION_MEMORY_THRESHOLD}" + TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) + + # AI processing performance + run_performance_benchmark "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing performance benchmark" "${AI_PROCESSING_THRESHOLD}" "${AI_PROCESSING_MEMORY_THRESHOLD}" + TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) + + # Concurrent processing performance + run_performance_benchmark "concurrent_processing" "src/tests/performance/test_concurrent_processing_performance.py" "Concurrent processing performance benchmark" "1.0" "150" + TOTAL_BENCHMARKS=$((TOTAL_BENCHMARKS + 1)) +} + +# Run memory profiling +run_all_memory_profiling() { + log_step "Running memory profiling..." + + # BAML parsing memory profiling + run_memory_profiling "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing memory profiling" + + # Pareto-Lang parsing memory profiling + run_memory_profiling "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing memory profiling" + + # XML transformation memory profiling + run_memory_profiling "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation memory profiling" + + # AI processing memory profiling + run_memory_profiling "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing memory profiling" +} + +# Run CPU profiling +run_all_cpu_profiling() { + log_step "Running CPU profiling..." + + # BAML parsing CPU profiling + run_cpu_profiling "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing CPU profiling" + + # Pareto-Lang parsing CPU profiling + run_cpu_profiling "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing CPU profiling" + + # XML transformation CPU profiling + run_cpu_profiling "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation CPU profiling" + + # AI processing CPU profiling + run_cpu_profiling "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing CPU profiling" +} + +# Run load testing +run_all_load_testing() { + log_step "Running load testing..." + + # BAML parsing load testing + run_load_testing "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "BAML parsing load testing" 10 30 + + # Pareto-Lang parsing load testing + run_load_testing "pareto_lang_parsing" "src/tests/performance/test_pareto_lang_parsing_performance.py" "Pareto-Lang parsing load testing" 10 30 + + # XML transformation load testing + run_load_testing "xml_transformation" "src/tests/performance/test_xml_transformation_performance.py" "XML transformation load testing" 5 30 + + # AI processing load testing + run_load_testing "ai_processing" "src/tests/performance/test_ai_processing_performance.py" "AI processing load testing" 3 30 +} + +# Generate combined performance report +generate_performance_report() { + log_step "Generating combined performance report..." + + # Combine benchmark results + python3 -c " +import json +import glob +import os +import statistics + +benchmarks = {} +benchmark_files = glob.glob('${PERFORMANCE_OUTPUT_DIR}/*/*_benchmark.json') + +for benchmark_file in benchmark_files: + if os.path.exists(benchmark_file): + with open(benchmark_file, 'r') as f: + data = json.load(f) + test_name = os.path.basename(benchmark_file).replace('_benchmark.json', '') + benchmarks[test_name] = data + +# Generate combined report +combined_report = { + 'timestamp': os.path.getmtime('${PERFORMANCE_OUTPUT_DIR}'), + 'benchmarks': benchmarks, + 'summary': { + 'total_benchmarks': len(benchmarks), + 'passed_benchmarks': ${PASSED_BENCHMARKS}, + 'failed_benchmarks': ${FAILED_BENCHMARKS}, + 'regressions_detected': ${REGRESSION_DETECTED} + } +} + +# Save combined report +with open('${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json', 'w') as f: + json.dump(combined_report, f, indent=2) + +# Generate performance summary +print('๐Ÿ“Š Combined Performance Report:') +print('=' * 60) +print(f'Total Benchmarks: {len(benchmarks)}') +print(f'Passed Benchmarks: ${PASSED_BENCHMARKS}') +print(f'Failed Benchmarks: ${FAILED_BENCHMARKS}') +print(f'Regressions Detected: ${REGRESSION_DETECTED}') +print() + +print('Performance Results:') +for test_name, data in benchmarks.items(): + if 'benchmarks' in data: + for benchmark_name, metrics in data['benchmarks'].items(): + if isinstance(metrics, dict) and 'mean' in metrics: + print(f' {test_name}.{benchmark_name}: {metrics[\"mean\"]:.6f}s') + elif 'mean' in data: + print(f' {test_name}: {data[\"mean\"]:.6f}s') + +print() +print('Performance Summary Report:') +print(' Generated: ${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json') +print(' Benchmarks: ${len(benchmarks)}') +" + + log_success "Combined performance report generated: ${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json" +} + +# Generate performance summary +generate_performance_summary() { + log_step "Generating performance summary..." + + TOTAL_TESTS=$((PASSED_BENCHMARKS + FAILED_BENCHMARKS)) + + echo "" + echo "==================================" + echo "๐Ÿš€ FSL CONTINUUM PERFORMANCE SUMMARY" + echo "==================================" + echo "Total Benchmarks: ${TOTAL_TESTS}" + echo "Passed Benchmarks: ${PASSED_BENCHMARKS}" + echo "Failed Benchmarks: ${FAILED_BENCHMARKS}" + echo "Regressions Detected: ${REGRESSION_DETECTED}" + echo "" + + if [ ${FAILED_BENCHMARKS} -eq 0 ] && [ ${REGRESSION_DETECTED} -eq 0 ]; then + echo "โœ… ALL PERFORMANCE BENCHMARKS PASSED!" + echo "๐Ÿš€ FSL Continuum performance is optimal!" + elif [ ${FAILED_BENCHMARKS} -eq 0 ] && [ ${REGRESSION_DETECTED} -gt 0 ]; then + echo "โš ๏ธ PERFORMANCE REGRESSIONS DETECTED!" + echo "๐Ÿ”ง Please review regressions and optimize performance." + else + echo "โŒ SOME PERFORMANCE BENCHMARKS FAILED!" + echo "๐Ÿ”ง Please review failed benchmarks and fix issues." + fi + + echo "==================================" + echo "" + + # Performance results location + echo "๐Ÿ“Š Performance Results Location:" + echo " Performance Logs: ${PERFORMANCE_OUTPUT_DIR}/" + echo " Benchmark Results: ${PERFORMANCE_OUTPUT_DIR}/*_benchmark.json" + echo " Memory Profiles: ${PERFORMANCE_OUTPUT_DIR}/*/memory/" + echo " CPU Profiles: ${PERFORMANCE_OUTPUT_DIR}/*/cpu/" + echo " Load Tests: ${PERFORMANCE_OUTPUT_DIR}/*/load/" + echo " Combined Report: ${PERFORMANCE_OUTPUT_DIR}/combined_performance_report.json" + echo "" + + # Performance thresholds status + echo "๐Ÿ“ˆ Performance Thresholds Status:" + echo " BAML Parsing Threshold: ${BAML_PARSING_THRESHOLD}s" + echo " Pareto-Lang Parsing Threshold: ${PARETO_LANG_PARSING_THRESHOLD}s" + echo " XML Transformation Threshold: ${XML_TRANSFORMATION_THRESHOLD}s" + echo " AI Processing Threshold: ${AI_PROCESSING_THRESHOLD}s" + echo "" + + echo "๐Ÿง  Memory Thresholds Status:" + echo " BAML Parsing Memory Threshold: ${BAML_PARSING_MEMORY_THRESHOLD}MB" + echo " Pareto-Lang Parsing Memory Threshold: ${PARETO_LANG_PARSING_MEMORY_THRESHOLD}MB" + echo " XML Transformation Memory Threshold: ${XML_TRANSFORMATION_MEMORY_THRESHOLD}MB" + echo " AI Processing Memory Threshold: ${AI_PROCESSING_MEMORY_THRESHOLD}MB" + echo "" +} + +# Cleanup function +cleanup() { + log_step "Cleaning up performance testing environment..." + + # Clean up temporary files + find "${PERFORMANCE_OUTPUT_DIR}" -name "*.pyc" -delete 2>/dev/null || true + find "${PERFORMANCE_OUTPUT_DIR}" -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true + + log_success "Performance testing environment cleaned up" +} + +# Main function +main() { + local test_type="${1:-all}" + + echo "โšก FSL Continuum UVX Performance Testing" + echo "===================================" + echo "Python Version: ${PYTHON_VERSION}" + echo "Project: ${PROJECT_NAME}" + echo "Test Type: ${test_type}" + echo "" + + # Set up environment + check_uvx + check_uvx_performance_environment + setup_performance_environment + + # Set up cleanup trap + trap cleanup EXIT + + # Run performance tests based on type + case "${test_type}" in + "benchmarks") + run_all_performance_benchmarks + ;; + "memory") + run_all_memory_profiling + ;; + "cpu") + run_all_cpu_profiling + ;; + "load") + run_all_load_testing + ;; + "quick") + run_performance_benchmark "baml_parsing" "src/tests/performance/test_baml_parsing_performance.py" "Quick BAML parsing benchmark" "${BAML_PARSING_THRESHOLD}" "${BAML_PARSING_MEMORY_THRESHOLD}" + TOTAL_BENCHMARKS=1 + ;; + "all"|*) + run_all_performance_benchmarks + run_all_memory_profiling + run_all_cpu_profiling + run_all_load_testing + ;; + esac + + # Generate reports + generate_performance_report + generate_performance_summary + + # Exit with appropriate code + if [ ${FAILED_BENCHMARKS} -eq 0 ] && [ ${REGRESSION_DETECTED} -eq 0 ]; then + exit 0 + else + exit 1 + fi +} + +# Show usage +usage() { + echo "Usage: $0 [test_type]" + echo "" + echo "Performance Test Types:" + echo " benchmarks - Run performance benchmarks only" + echo " memory - Run memory profiling only" + echo " cpu - Run CPU profiling only" + echo " load - Run load testing only" + echo " quick - Run quick performance test" + echo " all - Run all performance tests (default)" + echo "" + echo "Examples:" + echo " $0 # Run all performance tests" + echo " $0 benchmarks # Run performance benchmarks only" + echo " $0 memory # Run memory profiling only" + echo " $0 quick # Run quick performance test" +} + +# Parse command line arguments +if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then + usage + exit 0 +fi + +# Run main function +main "$@" diff --git a/.uvx/scripts/run_tests.sh b/.uvx/scripts/run_tests.sh new file mode 100644 index 0000000..f1a2fa2 --- /dev/null +++ b/.uvx/scripts/run_tests.sh @@ -0,0 +1,524 @@ +#!/bin/bash +# FSL Continuum UVX Test Execution Script +# Complete test execution using UVX virtual environments + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +PURPLE='\033[0;35m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${PURPLE}[STEP]${NC} $1" +} + +log_test() { + echo -e "${CYAN}[TEST]${NC} $1" +} + +# Configuration +PYTHON_VERSION="3.11" +PROJECT_NAME="fsl-continuum" +TEST_OUTPUT_DIR="test_results" +COVERAGE_DIR="htmlcov" + +# Test execution counters +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 +SKIPPED_TESTS=0 + +# Function to run UVX test +run_uvx_test() { + local test_name="$1" + local test_command="$2" + local test_description="$3" + local test_type="$4" + + log_step "Running ${test_name} tests..." + log_info "${test_description}" + + # Create output directory + mkdir -p "${TEST_OUTPUT_DIR}/${test_type}" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export PYTEST_CURRENT_TEST="${test_name}" + export PYTEST_PLUGINS="pytest-cov pytest-mock pytest-benchmark" + + # Execute test + if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" ${test_command} 2>&1 | \ + tee "${TEST_OUTPUT_DIR}/${test_type}/${test_name}.log"; then + + log_success "${test_name} tests passed!" + PASSED_TESTS=$((PASSED_TESTS + 1)) + + # Check if coverage was generated + if [ -f "coverage.xml" ]; then + cp coverage.xml "${TEST_OUTPUT_DIR}/${test_type}/${test_name}_coverage.xml" + fi + + # Check if benchmarks were generated + if [ -f "benchmark.json" ]; then + cp benchmark.json "${TEST_OUTPUT_DIR}/${test_type}/${test_name}_benchmark.json" + fi + + return 0 + else + log_error "${test_name} tests failed!" + FAILED_TESTS=$((FAILED_TESTS + 1)) + return 1 + fi +} + +# Function to run tests with specific markers +run_marked_tests() { + local test_name="$1" + local markers="$2" + local test_description="$3" + + run_uvx_test "${test_name}" "pytest src/tests/ -v -m '${markers}' --cov=src --cov-report=html --cov-report=xml --cov-report=term-missing --junitxml=${TEST_OUTPUT_DIR}/${test_name}.xml" "${test_description}" "${test_name}" +} + +# Function to run specific test files +run_test_files() { + local test_name="$1" + local test_files="$2" + local test_description="$3" + + run_uvx_test "${test_name}" "pytest ${test_files} -v --cov=src --cov-report=html --cov-report=xml --cov-report=term-missing --junitxml=${TEST_OUTPUT_DIR}/${test_name}.xml" "${test_description}" "${test_name}" +} + +# Function to run performance tests +run_performance_tests() { + local test_name="$1" + local test_files="$2" + local test_description="$3" + + run_uvx_test "${test_name}" "pytest ${test_files} -v -m performance --benchmark-only --benchmark-json=${TEST_OUTPUT_DIR}/performance/${test_name}_benchmark.json --benchmark-sort=min" "${test_description}" "performance" +} + +# Check UVX installation +check_uvx() { + log_info "Checking UVX installation..." + + if ! command -v uvx &> /dev/null; then + log_error "UVX is not installed. Please install UVX first:" + log_error "pip install uvx" + exit 1 + fi + + UVX_VERSION=$(uvx --version) + log_success "UVX installed: ${UVX_VERSION}" +} + +# Check UVX environment +check_uvx_environment() { + log_info "Checking UVX environment..." + + # Check if test environment exists + if ! uvx list "${PROJECT_NAME}-test" &> /dev/null; then + log_warning "UVX test environment not found. Creating..." + uvx create "${PROJECT_NAME}-test" --python "${PYTHON_VERSION}" --test --description "FSL Continuum Testing Environment" + uvx install "${PROJECT_NAME}" --python "${PYTHON_VERSION}" --test --environment "${PROJECT_NAME}-test" + fi + + # Check if test package is installed + if ! uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" python -c "import fsl_continuum" &> /dev/null; then + log_warning "FSL Continuum not installed in test environment. Installing..." + uvx install "${PROJECT_NAME}" --python "${PYTHON_VERSION}" --test --environment "${PROJECT_NAME}-test" + fi + + log_success "UVX test environment ready" +} + +# Set up test environment +setup_test_environment() { + log_step "Setting up test environment..." + + # Create test output directories + mkdir -p "${TEST_OUTPUT_DIR}" + mkdir -p "${TEST_OUTPUT_DIR}/unit" + mkdir -p "${TEST_OUTPUT_DIR}/integration" + mkdir -p "${TEST_OUTPUT_DIR}/performance" + mkdir -p "${TEST_OUTPUT_DIR}/ai_processing" + mkdir -p "${TEST_OUTPUT_DIR}/xml_transformation" + mkdir -p "${TEST_OUTPUT_DIR}/semantic_languages" + + # Set environment variables + export PYTHONPATH="${PYTHONPATH}:$(pwd)/src" + export PYTEST_CURRENT_TEST="fsl_continuum" + export PYTEST_PLUGINS="pytest-cov pytest-mock pytest-benchmark" + + # Create pytest configuration + if [ ! -f "pytest.ini" ]; then + log_warning "pytest.ini not found. Using default configuration." + fi + + log_success "Test environment setup complete" +} + +# Run unit tests +run_unit_tests() { + log_step "Running unit tests..." + + run_marked_tests "unit" "unit" "Running unit tests for individual components" + + # Run specific unit test categories + run_test_files "baml_unit" "src/tests/unit/semantic_languages/test_baml_*.py" "Running BAML unit tests" + run_test_files "pareto_lang_unit" "src/tests/unit/semantic_languages/test_pareto_lang_*.py" "Running Pareto-Lang unit tests" + run_test_files "xml_processing_unit" "src/tests/unit/xml_processing/test_*.py" "Running XML processing unit tests" + run_test_files "ai_integration_unit" "src/tests/unit/ai_integration/test_*.py" "Running AI integration unit tests" +} + +# Run integration tests +run_integration_tests() { + log_step "Running integration tests..." + + run_marked_tests "integration" "integration" "Running integration tests for component interactions" + + # Run specific integration test categories + run_test_files "semantic_language_integration" "src/tests/integration/test_semantic_language_integration.py" "Running semantic language integration tests" + run_test_files "xml_integration" "src/tests/integration/test_xml_integration.py" "Running XML integration tests" + run_test_files "ai_integration" "src/tests/integration/test_ai_integration.py" "Running AI integration tests" +} + +# Run performance tests +run_performance_tests_all() { + log_step "Running performance tests..." + + run_performance_tests "semantic_language_performance" "src/tests/performance/test_semantic_language_performance.py" "Running semantic language performance tests" + run_performance_tests "xml_performance" "src/tests/performance/test_xml_performance.py" "Running XML transformation performance tests" + run_performance_tests "ai_performance" "src/tests/performance/test_ai_performance.py" "Running AI processing performance tests" +} + +# Run AI processing tests +run_ai_processing_tests() { + log_step "Running AI processing tests..." + + run_marked_tests "ai_processing" "ai_processing" "Running AI processing and validation tests" + + # Run specific AI test categories + run_test_files "semantic_ai_processing" "src/tests/ai_processing/test_semantic_ai_processor.py" "Running semantic AI processing tests" + run_test_files "ai_optimization" "src/tests/ai_processing/test_ai_optimizer.py" "Running AI optimization tests" +} + +# Run XML transformation tests +run_xml_transformation_tests() { + log_step "Running XML transformation tests..." + + run_marked_tests "xml_transformation" "xml_transformation" "Running XML transformation tests" + + # Run specific XML test categories + run_test_files "xml_processor" "src/tests/xml_transformation/test_xml_processor.py" "Running XML processor tests" + run_test_files "xml_validator" "src/tests/xml_transformation/test_xml_validator.py" "Running XML validator tests" + run_test_files "xml_transformer" "src/tests/xml_transformation/test_xml_transformer.py" "Running XML transformer tests" +} + +# Run semantic language tests +run_semantic_language_tests() { + log_step "Running semantic language tests..." + + run_marked_tests "semantic_languages" "semantic_languages" "Running semantic language integration tests" + + # Run specific semantic language test categories + run_test_files "baml_tests" "src/tests/semantic_languages/test_baml_*.py" "Running BAML tests" + run_test_files "pareto_lang_tests" "src/tests/semantic_languages/test_pareto_lang_*.py" "Running Pareto-Lang tests" + run_test_files "semantic_bridge" "src/tests/semantic_languages/test_semantic_bridge.py" "Running semantic bridge tests" +} + +# Run smoke tests +run_smoke_tests() { + log_step "Running smoke tests..." + + run_marked_tests "smoke" "smoke" "Running smoke tests for basic functionality" + + # Run basic smoke tests + run_test_files "basic_smoke" "src/tests/smoke/test_basic_smoke.py" "Running basic smoke tests" +} + +# Run regression tests +run_regression_tests() { + log_step "Running regression tests..." + + run_marked_tests "regression" "regression" "Running regression tests" + + # Run specific regression tests + run_test_files "semantic_regression" "src/tests/regression/test_semantic_regression.py" "Running semantic regression tests" +} + +# Generate coverage report +generate_coverage_report() { + log_step "Generating combined coverage report..." + + # Combine coverage data + if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage combine 2>/dev/null; then + log_success "Coverage data combined" + else + log_warning "Failed to combine coverage data" + fi + + # Generate HTML coverage report + if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage html --directory="${COVERAGE_DIR}" 2>/dev/null; then + log_success "HTML coverage report generated: ${COVERAGE_DIR}/index.html" + else + log_warning "Failed to generate HTML coverage report" + fi + + # Generate XML coverage report + if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage xml 2>/dev/null; then + log_success "XML coverage report generated: coverage.xml" + else + log_warning "Failed to generate XML coverage report" + fi + + # Generate terminal coverage report + if uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage report 2>/dev/null; then + log_success "Terminal coverage report generated" + else + log_warning "Failed to generate terminal coverage report" + fi +} + +# Generate benchmark report +generate_benchmark_report() { + log_step "Generating benchmark report..." + + # Combine benchmark data + python3 -c " +import json +import glob +import os + +benchmarks = {} +benchmark_files = glob.glob('test_results/performance/*_benchmark.json') + +for benchmark_file in benchmark_files: + if os.path.exists(benchmark_file): + with open(benchmark_file, 'r') as f: + data = json.load(f) + test_name = os.path.basename(benchmark_file).replace('_benchmark.json', '') + benchmarks[test_name] = data + +with open('combined_benchmark.json', 'w') as f: + json.dump(benchmarks, f, indent=2) + +print('Combined benchmark report generated: combined_benchmark.json') +print(f'Found {len(benchmarks)} benchmark results') +" +} + +# Generate test summary +generate_test_summary() { + log_step "Generating test summary..." + + TOTAL_TESTS=$((PASSED_TESTS + FAILED_TESTS + SKIPPED_TESTS)) + + echo "" + echo "==================================" + echo "๐ŸŽฏ FSL CONTINUUM TEST SUMMARY" + echo "==================================" + echo "Total Tests Run: ${TOTAL_TESTS}" + echo "Passed Tests: ${PASSED_TESTS}" + echo "Failed Tests: ${FAILED_TESTS}" + echo "Skipped Tests: ${SKIPPED_TESTS}" + echo "" + + if [ ${FAILED_TESTS} -eq 0 ]; then + echo "โœ… ALL TESTS PASSED!" + echo "๐Ÿš€ FSL Continuum is ready for production!" + else + echo "โŒ SOME TESTS FAILED!" + echo "๐Ÿ”ง Please review the failed tests and fix the issues." + fi + + echo "==================================" + echo "" + + # Test results location + echo "๐Ÿ“Š Test Results Location:" + echo " Test Logs: ${TEST_OUTPUT_DIR}/" + echo " Coverage Report: ${COVERAGE_DIR}/index.html" + echo " Coverage XML: coverage.xml" + echo " Benchmark Report: combined_benchmark.json" + echo "" + + # Coverage summary + if [ -f "coverage.xml" ]; then + echo "๐Ÿ“ˆ Coverage Summary:" + uvx run --python "${PYTHON_VERSION}" --test "${PROJECT_NAME}" coverage report | tail -n 1 + echo "" + fi + + # Performance summary + if [ -f "combined_benchmark.json" ]; then + echo "โšก Performance Summary:" + python3 -c " +import json +import os + +if os.path.exists('combined_benchmark.json'): + with open('combined_benchmark.json', 'r') as f: + benchmarks = json.load(f) + + print(f' Benchmarks: {len(benchmarks)}') + for test_name, data in benchmarks.items(): + if 'benchmarks' in data: + for benchmark, metrics in data['benchmarks'].items(): + if 'mean' in metrics: + print(f' {test_name}.{benchmark}: {metrics[\"mean\"]:.3f}s') + elif 'mean' in data: + print(f' {test_name}: {data[\"mean\"]:.3f}s') +" + echo "" + fi +} + +# Cleanup function +cleanup() { + log_step "Cleaning up test environment..." + + # Clean up temporary files + find "${TEST_OUTPUT_DIR}" -name "*.pyc" -delete 2>/dev/null || true + find "${TEST_OUTPUT_DIR}" -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true + + log_success "Test environment cleaned up" +} + +# Main function +main() { + local test_type="${1:-all}" + + echo "๐Ÿงช FSL Continuum UVX Test Execution" + echo "==================================" + echo "Python Version: ${PYTHON_VERSION}" + echo "Project: ${PROJECT_NAME}" + echo "Test Type: ${test_type}" + echo "" + + # Set up environment + check_uvx + check_uvx_environment + setup_test_environment + + # Set up cleanup trap + trap cleanup EXIT + + # Run tests based on type + case "${test_type}" in + "unit") + run_unit_tests + ;; + "integration") + run_integration_tests + ;; + "performance") + run_performance_tests_all + ;; + "ai_processing") + run_ai_processing_tests + ;; + "xml_transformation") + run_xml_transformation_tests + ;; + "semantic_languages") + run_semantic_language_tests + ;; + "smoke") + run_smoke_tests + ;; + "regression") + run_regression_tests + ;; + "quick") + run_smoke_tests + run_unit_tests + ;; + "full") + run_smoke_tests + run_unit_tests + run_integration_tests + run_ai_processing_tests + run_xml_transformation_tests + run_semantic_language_tests + run_performance_tests_all + ;; + "all"|*) + run_smoke_tests + run_unit_tests + run_integration_tests + run_ai_processing_tests + run_xml_transformation_tests + run_semantic_language_tests + run_performance_tests_all + run_regression_tests + ;; + esac + + # Generate reports + generate_coverage_report + generate_benchmark_report + generate_test_summary + + # Exit with appropriate code + if [ ${FAILED_TESTS} -eq 0 ]; then + exit 0 + else + exit 1 + fi +} + +# Show usage +usage() { + echo "Usage: $0 [test_type]" + echo "" + echo "Test Types:" + echo " unit - Run unit tests only" + echo " integration - Run integration tests only" + echo " performance - Run performance tests only" + echo " ai_processing - Run AI processing tests only" + echo " xml_transformation - Run XML transformation tests only" + echo " semantic_languages - Run semantic language tests only" + echo " smoke - Run smoke tests only" + echo " regression - Run regression tests only" + echo " quick - Run smoke and unit tests" + echo " full - Run all tests except regression" + echo " all - Run all tests (default)" + echo "" + echo "Examples:" + echo " $0 # Run all tests" + echo " $0 unit # Run unit tests only" + echo " $0 performance # Run performance tests only" + echo " $0 quick # Run quick tests" +} + +# Parse command line arguments +if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then + usage + exit 0 +fi + +# Run main function +main "$@" diff --git a/.uvx/scripts/setup.sh b/.uvx/scripts/setup.sh new file mode 100644 index 0000000..ab8803f --- /dev/null +++ b/.uvx/scripts/setup.sh @@ -0,0 +1,233 @@ +#!/bin/bash +# FSL Continuum UVX Environment Setup Script +# Complete UVX virtual environment setup for testing and development + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if UVX is installed +check_uvx() { + log_info "Checking UVX installation..." + + if ! command -v uvx &> /dev/null; then + log_error "UVX is not installed. Please install UVX first:" + log_error "pip install uvx" + exit 1 + fi + + UVX_VERSION=$(uvx --version) + log_success "UVX installed: $UVX_VERSION" +} + +# Create UVX directory structure +create_directories() { + log_info "Creating UVX directory structure..." + + directories=( + ".uvx" + ".uvx/cache" + ".uvx/environments" + ".uvx/logs" + ".uvx/scripts" + ".uvx/artifacts" + ) + + for dir in "${directories[@]}"; do + if [ ! -d "$dir" ]; then + mkdir -p "$dir" + log_success "Created directory: $dir" + else + log_info "Directory already exists: $dir" + fi + done +} + +# Create UVX environments +create_environments() { + log_info "Creating UVX environments..." + + # Create production environment + log_info "Creating fsl-continuum production environment..." + if ! uvx list fsl-continuum &> /dev/null; then + uvx create fsl-continuum --python 3.11 --description "FSL Continuum Production Environment" + log_success "Created fsl-continuum production environment" + else + log_info "fsl-continuum production environment already exists" + fi + + # Create development environment + log_info "Creating fsl-continuum-dev development environment..." + if ! uvx list fsl-continuum-dev &> /dev/null; then + uvx create fsl-continuum-dev --python 3.11 --dev --description "FSL Continuum Development Environment" + log_success "Created fsl-continuum-dev development environment" + else + log_info "fsl-continuum-dev development environment already exists" + fi + + # Create testing environment + log_info "Creating fsl-continuum-test testing environment..." + if ! uvx list fsl-continuum-test &> /dev/null; then + uvx create fsl-continuum-test --python 3.11 --test --description "FSL Continuum Testing Environment" + log_success "Created fsl-continuum-test testing environment" + else + log_info "fsl-continuum-test testing environment already exists" + fi + + # Create coverage environment + log_info "Creating fsl-continuum-coverage coverage environment..." + if ! uvx list fsl-continuum-coverage &> /dev/null; then + uvx create fsl-continuum-coverage --python 3.11 --coverage --description "FSL Continuum Coverage Testing Environment" + log_success "Created fsl-continuum-coverage coverage environment" + else + log_info "fsl-continuum-coverage coverage environment already exists" + fi + + # Create performance environment + log_info "Creating fsl-continuum-performance performance environment..." + if ! uvx list fsl-continuum-performance &> /dev/null; then + uvx create fsl-continuum-performance --python 3.11 --performance --description "FSL Continuum Performance Testing Environment" + log_success "Created fsl-continuum-performance performance environment" + else + log_info "fsl-continuum-performance performance environment already exists" + fi +} + +# Install FSL Continuum in environments +install_fsl_continuum() { + log_info "Installing FSL Continuum in UVX environments..." + + # Install in production environment + log_info "Installing FSL Continuum in production environment..." + uvx install fsl-continuum --python 3.11 --environment fsl-continuum + log_success "Installed FSL Continuum in production environment" + + # Install in development environment + log_info "Installing FSL Continuum in development environment..." + uvx install fsl-continuum --python 3.11 --dev --environment fsl-continuum-dev + log_success "Installed FSL Continuum in development environment" + + # Install in testing environment + log_info "Installing FSL Continuum in testing environment..." + uvx install fsl-continuum --python 3.11 --test --environment fsl-continuum-test + log_success "Installed FSL Continuum in testing environment" + + # Install in coverage environment + log_info "Installing FSL Continuum in coverage environment..." + uvx install fsl-continuum --python 3.11 --coverage --environment fsl-continuum-coverage + log_success "Installed FSL Continuum in coverage environment" + + # Install in performance environment + log_info "Installing FSL Continuum in performance environment..." + uvx install fsl-continuum --python 3.11 --performance --environment fsl-continuum-performance + log_success "Installed FSL Continuum in performance environment" +} + +# Validate UVX setup +validate_setup() { + log_info "Validating UVX setup..." + + # Check if all environments exist + environments=("fsl-continuum" "fsl-continuum-dev" "fsl-continuum-test" "fsl-continuum-coverage" "fsl-continuum-performance") + + for env in "${environments[@]}"; do + if uvx list "$env" &> /dev/null; then + log_success "Environment $env exists" + else + log_error "Environment $env does not exist" + exit 1 + fi + done + + # Test import in production environment + log_info "Testing FSL Continuum import in production environment..." + if uvx run --python 3.11 fsl-continuum python -c "import fsl_continuum; print(fsl_continuum.get_version())" &> /dev/null; then + log_success "FSL Continuum import works in production environment" + else + log_error "FSL Continuum import failed in production environment" + exit 1 + fi + + # Test import in testing environment + log_info "Testing FSL Continuum import in testing environment..." + if uvx run --python 3.11 --test fsl-continuum python -c "import fsl_continuum; print('Testing import successful')" &> /dev/null; then + log_success "FSL Continuum import works in testing environment" + else + log_error "FSL Continuum import failed in testing environment" + exit 1 + fi +} + +# Create UVX scripts +create_scripts() { + log_info "Creating UVX scripts..." + + # Make scripts executable + chmod +x .uvx/scripts/*.sh + + log_success "Created UVX scripts" +} + +# Main function +main() { + echo "๐Ÿ”ง FSL Continuum UVX Environment Setup" + echo "==================================" + + # Change to project root + cd "$(dirname "$0")/../.." + + log_info "Setting up UVX environments for FSL Continuum..." + + check_uvx + create_directories + create_environments + install_fsl_continuum + validate_setup + create_scripts + + echo "==================================" + log_success "UVX environment setup complete!" + echo "" + log_info "Available UVX environments:" + uvx list + + echo "" + log_info "Usage examples:" + echo " uvx run --python 3.11 fsl-continuum # Production environment" + echo " uvx run --python 3.11 --dev fsl-continuum # Development environment" + echo " uvx run --python 3.11 --test fsl-continuum # Testing environment" + echo " uvx run --python 3.11 --coverage fsl-continuum # Coverage environment" + echo " uvx run --python 3.11 --performance fsl-continuum # Performance environment" + echo "" + echo " .uvx/scripts/run_tests.sh # Run all tests" + echo " .uvx/scripts/run_performance_tests.sh # Run performance tests" + echo " .uvx/scripts/run_coverage_tests.sh # Run coverage tests" + + echo "" + log_success "Ready to run tests with UVX!" +} + +# Run main function +main "$@"