enhanced_integration_example.py

#!/usr/bin/env python3
"""
Enhanced Trading Platform Configuration Integration Example

This script demonstrates the advanced configuration features including:
- Environment variable precedence (env > CLI > JSON)
- .env file support and auto-discovery
- Multiple credential source methods
- CLI override capabilities
- Security features (secret masking, safe export)
- Comprehensive precedence logging

Precedence Order:
1. Environment variables (highest precedence)
2. CLI flag overrides (medium precedence)
3. JSON configuration defaults (lowest precedence)

Usage:
    python enhanced_integration_example.py --config trading_platform.example.json
    python enhanced_integration_example.py --config config.json --demo-env-vars
    python enhanced_integration_example.py --config config.json --env-file custom.env

Author: Trading Platform Team
Version: 2.0.0
"""

import argparse
import json
import os
import sys
from pathlib import Path
from typing import Any, Dict, Optional

# Import our enhanced configuration loader
from config_loader import (
    ConfigurationLoader,
    EnvironmentType,
    TradingPlatformConfig,
    create_default_config,
)


def demonstrate_configuration_precedence():
    """
    Demonstrate configuration precedence with environment variables and CLI overrides
    """
    print("\n🔄 === Configuration Precedence Demonstration ===")
    print("Precedence Order: Environment Variables > CLI Overrides > JSON Defaults")

    config_file = "trading_platform.example.json"

    # Ensure we have a config file
    if not Path(config_file).exists():
        print(f"⚠️  Config file {config_file} not found, creating example...")
        default_config = create_default_config()
        loader = ConfigurationLoader()
        loader._config = default_config
        loader.export_config(config_file)
        print(f"✅ Created example configuration: {config_file}")

    # Set some demo environment variables to show precedence
    demo_env_vars = {
        "TP_ENVIRONMENT": "development",
        "TP_DATA_OPENBB_API_KEY": "env_override_openbb_key_123456789",
        "OPENBB_API_KEY": "template_openbb_key_987654321",
        "TP_DATA_BINANCE_API_KEY": "env_override_binance_key",
        "DATABASE_URL": "postgresql://env_user:env_pass@localhost/env_db",
    }

    print("\n🔧 Setting demo environment variables:")
    for var, value in demo_env_vars.items():
        os.environ[var] = value
        # Mask sensitive values in output
        if any(
            sensitive in var.lower()
            for sensitive in ["key", "secret", "password", "token"]
        ):
            masked_value = f"{value[:4]}...{value[-4:]}" if len(value) > 8 else "***"
            print(f"   {var} = {masked_value}")
        else:
            print(f"   {var} = {value}")

    # Demonstrate CLI overrides
    cli_overrides = {
        "metadata": {"environment": "staging"},
        "data": {
            "storage": {"cache": {"host": "redis-cli-override.internal", "port": 6380}}
        },
        "system": {
            "feature_flags": {"enable_paper_trading": False, "enable_dry_run": True}
        },
    }

    print("\n⚙️  CLI overrides to be applied:")
    print(json.dumps(cli_overrides, indent=2))

    # Load configuration with all precedence sources
    print("\n📋 Loading configuration with precedence handling...")
    loader = ConfigurationLoader(config_file)
    config = loader.load_config(cli_overrides=cli_overrides)

    print("✅ Configuration loaded successfully!")

    # Show final resolved values to demonstrate precedence
    print("\n🎯 Final resolved configuration values:")

    # Environment setting (CLI should override JSON, but env should override CLI)
    final_env = config.metadata.environment if config.metadata else "Unknown"
    print(f"   Environment: {final_env}")
    print(f"     JSON: development → CLI: staging → ENV: development = {final_env}")

    # Show precedence summary
    precedence_info = loader.get_precedence_info()
    print(f"\n📊 Precedence Summary:")
    print(
        f"   Environment overrides applied: {len(precedence_info['environment_overrides'])}"
    )
    print(f"   CLI overrides applied: {len(precedence_info['cli_overrides'])}")

    # Cleanup demo environment variables
    for var in demo_env_vars:
        if var in os.environ:
            del os.environ[var]

    return loader, config


def demonstrate_environment_variable_methods(config: TradingPlatformConfig):
    """
    Demonstrate different environment variable configuration methods
    """
    print("\n🔧 === Environment Variable Methods Demonstration ===")

    print("1. Direct TP_* Prefix Overrides:")
    print("   Format: TP_<SECTION>_<SUBSECTION>_<FIELD>")

    # Check current environment for TP_* variables
    tp_vars = {k: v for k, v in os.environ.items() if k.startswith("TP_")}
    if tp_vars:
        print("   Currently set TP_* variables:")
        for var, value in tp_vars.items():
            if any(s in var.lower() for s in ["key", "secret", "password", "token"]):
                masked = f"{value[:4]}...{value[-4:]}" if len(value) > 8 else "***"
                print(f"     {var} = {masked}")
            else:
                print(f"     {var} = {value}")
    else:
        print("   No TP_* variables currently set")

    print("\n2. JSON Template Variables (${VAR} syntax):")
    print('   Example in JSON: "api_key": "${OPENBB_API_KEY}"')
    template_vars = ["OPENBB_API_KEY", "BINANCE_API_KEY", "DATABASE_PASSWORD"]
    for var in template_vars:
        value = os.getenv(var)
        if value:
            masked = f"{value[:4]}...{value[-4:]}" if len(value) > 8 else "***"
            print(f"     {var} = {masked}")
        else:
            print(f"     {var} = Not set")

    print("\n3. Environment Prefix Auto-Discovery:")
    print('   Example: env_prefix: "ALPACA" looks for ALPACA_API_KEY, ALPACA_SECRET')

    # Demo some common prefixes
    prefixes = ["ALPACA", "BINANCE", "COINBASE", "POLYGON"]
    for prefix in prefixes:
        key_var = f"{prefix}_API_KEY"
        secret_var = f"{prefix}_SECRET"

        key_value = os.getenv(key_var)
        secret_value = os.getenv(secret_var)

        if key_value or secret_value:
            print(f"   {prefix}:")
            if key_value:
                print(f"     {key_var} = {key_value[:4]}...{key_value[-4:]}")
            if secret_value:
                print(f"     {secret_var} = {secret_value[:4]}...{secret_value[-4:]}")
        else:
            print(f"   {prefix}: No credentials found")

    print("\n4. Default Value Syntax (${VAR:-default}):")
    print('   Example: "host": "${REDIS_HOST:-localhost}"')
    print("   Allows fallback to default if environment variable not set")


def demonstrate_dotenv_file_support():
    """
    Demonstrate .env file support and auto-discovery
    """
    print("\n📁 === .env File Support Demonstration ===")

    # Create a demo .env file
    demo_env_content = """# Demo .env file for Trading Platform
# This demonstrates .env file loading with precedence

# Environment setting
TP_ENVIRONMENT=production

# API Keys (demo values)
OPENBB_API_KEY=dotenv_openbb_demo_key_12345678
BINANCE_API_KEY=dotenv_binance_demo_key_87654321
DATABASE_PASSWORD=dotenv_secure_password_2023

# Cache settings with defaults
REDIS_HOST=localhost
REDIS_PORT=6379

# Feature flags
ENABLE_PAPER_TRADING=true
ENABLE_DRY_RUN=false
"""

    demo_env_file = Path(".env.demo")
    with open(demo_env_file, "w") as f:
        f.write(demo_env_content)

    print(f"✅ Created demo .env file: {demo_env_file}")
    print(f"📖 .env file search order:")
    print(f"   1. Explicitly provided env_file path")
    print(f"   2. .env file in same directory as config file")
    print(f"   3. .env file in current working directory")
    print(f"   4. .env file in parent directories (up to 3 levels)")

    # Load configuration with .env file
    loader = ConfigurationLoader(
        config_path="trading_platform.example.json", env_file=demo_env_file
    )

    print(f"\n🔄 Loading configuration with .env file...")
    try:
        config = loader.load_config()
        print(f"✅ Configuration loaded with .env support")

        # Show .env file impact
        precedence_info = loader.get_precedence_info()
        if precedence_info["env_file_loaded"]:
            print(f"✅ .env file support is available")
        else:
            print(f"⚠️  .env file support not available (python-dotenv not installed)")

    except Exception as e:
        print(f"❌ Failed to load with .env file: {e}")

    # Cleanup
    demo_env_file.unlink(missing_ok=True)
    print(f"🧹 Cleaned up demo .env file")


def demonstrate_security_features():
    """
    Demonstrate security features like secret masking and safe export
    """
    print("\n🔒 === Security Features Demonstration ===")

    # Set some demo sensitive environment variables
    sensitive_vars = {
        "DEMO_API_KEY": "ak_live_1234567890abcdef",
        "DEMO_SECRET": "secret_live_abcdefghijklmnop",
        "DEMO_TOKEN": "token_live_zyxwvutsrqponmlk",
        "DEMO_PASSWORD": "super_secure_password_2023",
    }

    print("🔧 Setting demo sensitive environment variables...")
    for var, value in sensitive_vars.items():
        os.environ[var] = value
        masked = f"{value[:4]}...{value[-4:]}" if len(value) > 8 else "***"
        print(f"   {var} = {masked}")

    # Load configuration and demonstrate safe export
    loader = ConfigurationLoader("trading_platform.example.json")
    config = loader.load_config()

    print("\n💾 Demonstrating safe configuration export...")

    # Export with sensitive data masked
    safe_export_file = "safe_config_demo.json"
    loader.export_config(safe_export_file, include_sensitive=False)

    print(f"✅ Exported configuration with masked secrets to: {safe_export_file}")

    # Show sample of masked data
    with open(safe_export_file, "r") as f:
        safe_data = json.load(f)

    print("📖 Sample of automatically masked sensitive data:")

    # Look for any credentials in the exported data
    def find_credentials(data, path=""):
        if isinstance(data, dict):
            for key, value in data.items():
                new_path = f"{path}.{key}" if path else key
                if any(
                    s in key.lower() for s in ["key", "secret", "password", "token"]
                ):
                    print(f"   {new_path}: {value}")
                else:
                    find_credentials(value, new_path)

    find_credentials(safe_data)

    # Show precedence information with masked values
    precedence_info = loader.get_precedence_info()
    if precedence_info["environment_overrides"]:
        print(f"\n🎭 Environment overrides (automatically masked):")
        for var, value in precedence_info["environment_overrides"].items():
            print(f"   {var} = {value}")

    # Cleanup
    Path(safe_export_file).unlink(missing_ok=True)
    for var in sensitive_vars:
        if var in os.environ:
            del os.environ[var]

    print("🧹 Cleaned up demo data and files")


def demonstrate_error_handling_and_validation():
    """
    Demonstrate configuration error handling and validation features
    """
    print("\n🛡️ === Error Handling and Validation Demonstration ===")

    print("1. Missing Environment Variable Handling:")

    # Create a config that requires a missing environment variable
    test_config_content = {
        "$schema": "https://schemas.trading-platform.com/config/v1/trading-platform.schema.json",
        "metadata": {"environment": "${MISSING_ENV_VAR}"},
        "data": {
            "providers": {
                "openbb": {
                    "credentials": {
                        "environment_variable": "${REQUIRED_BUT_MISSING_KEY}"
                    }
                }
            }
        },
    }

    test_config_file = "test_missing_vars.json"
    with open(test_config_file, "w") as f:
        json.dump(test_config_content, f, indent=2)

    print(f"   Created test config requiring missing env vars...")

    try:
        loader = ConfigurationLoader(test_config_file)
        config = loader.load_config()
        print("❌ Should have failed due to missing environment variables")
    except ValueError as e:
        print(f"✅ Correctly caught missing environment variable: {e}")
    except Exception as e:
        print(f"⚠️  Unexpected error type: {e}")

    print("\n2. Default Value Syntax:")

    # Test default value syntax
    test_config_with_defaults = {
        "$schema": "https://schemas.trading-platform.com/config/v1/trading-platform.schema.json",
        "metadata": {"environment": "${TEST_ENVIRONMENT:-development}"},
        "data": {
            "storage": {
                "cache": {
                    "host": "${REDIS_HOST:-localhost}",
                    "port": "${REDIS_PORT:-6379}",
                }
            }
        },
    }

    test_defaults_file = "test_defaults.json"
    with open(test_defaults_file, "w") as f:
        json.dump(test_config_with_defaults, f, indent=2)

    try:
        loader = ConfigurationLoader(test_defaults_file)
        config = loader.load_config()
        print(f"✅ Default values work correctly:")
        print(f"   Environment: {config.metadata.environment}")
        if config.data and config.data.storage and config.data.storage.cache:
            cache = config.data.storage.cache
            print(f"   Cache host: {cache.host}")
            print(f"   Cache port: {cache.port}")
    except Exception as e:
        print(f"❌ Default value syntax failed: {e}")

    print("\n3. Invalid Environment Variable Pattern:")

    test_invalid_pattern = {
        "data": {
            "providers": {
                "openbb": {
                    "credentials": {
                        "environment_variable": "${invalid-pattern-with-dashes}"
                    }
                }
            }
        }
    }

    test_invalid_file = "test_invalid_pattern.json"
    with open(test_invalid_file, "w") as f:
        json.dump(test_invalid_pattern, f, indent=2)

    try:
        loader = ConfigurationLoader(test_invalid_file)
        config = loader.load_config()
        print("❌ Should have failed due to invalid env var pattern")
    except Exception as e:
        print(f"✅ Correctly caught invalid pattern: {e}")

    # Cleanup test files
    for file in [test_config_file, test_defaults_file, test_invalid_file]:
        Path(file).unlink(missing_ok=True)

    print("🧹 Cleaned up test configuration files")


def main():
    """
    Main demonstration function
    """
    print("🚀 Trading Platform Enhanced Configuration Features Demo")
    print("=" * 65)
    print("Features: Environment Variables | CLI Overrides | .env Files | Security")
    print("Precedence: Environment Variables > CLI Overrides > JSON Defaults")

    # Parse command line arguments
    parser = argparse.ArgumentParser(
        description="Enhanced Configuration Integration Demo",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
    # Basic demo with precedence
    python enhanced_integration_example.py --config trading_platform.example.json

    # Demo with environment variables
    python enhanced_integration_example.py --demo-env-vars

    # Demo with custom .env file
    python enhanced_integration_example.py --env-file custom.env

    # Demo all features
    python enhanced_integration_example.py --demo-env-vars --all-features
        """,
    )

    parser.add_argument(
        "--config",
        "-c",
        default="trading_platform.example.json",
        help="Configuration file path",
    )

    parser.add_argument("--env-file", "-e", help="Custom .env file path")

    parser.add_argument(
        "--demo-env-vars", action="store_true", help="Set demo environment variables"
    )

    parser.add_argument(
        "--all-features", action="store_true", help="Demonstrate all enhanced features"
    )

    args = parser.parse_args()

    # Set demo environment variables if requested
    if args.demo_env_vars:
        demo_vars = {
            "TP_ENVIRONMENT": "development",
            "TP_DATA_OPENBB_API_KEY": "demo_key_123456789012345678901234567890",
            "OPENBB_API_KEY": "template_key_098765432109876543210987654321",
            "BINANCE_API_KEY": "binance_demo_key_abcdefghijklmnopqr",
            "DATABASE_URL": "postgresql://demo_user:demo_pass@localhost:5432/demo_db",
        }

        print("\n🔧 Setting demo environment variables for testing...")
        for var, value in demo_vars.items():
            os.environ[var] = value
            if any(s in var.lower() for s in ["key", "secret", "password", "token"]):
                masked = f"{value[:4]}...{value[-4:]}" if len(value) > 8 else "***"
                print(f"   {var} = {masked}")
            else:
                print(f"   {var} = {value}")

    try:
        # Core demonstration - configuration precedence
        loader, config = demonstrate_configuration_precedence()

        if not config:
            print("❌ Cannot continue without valid configuration")
            return 1

        # Demonstrate environment variable methods
        demonstrate_environment_variable_methods(config)

        # Demonstrate .env file support
        demonstrate_dotenv_file_support()

        # Demonstrate security features
        demonstrate_security_features()

        # Demonstrate error handling if all features requested
        if args.all_features:
            demonstrate_error_handling_and_validation()

        # Final summary
        print("\n🎯 === Enhanced Configuration Demo Summary ===")
        print("✅ Configuration precedence (Env > CLI > JSON)")
        print("✅ Environment variable integration (TP_*, ${VAR}, prefixes)")
        print("✅ .env file support and auto-discovery")
        print("✅ CLI override capabilities")
        print("✅ Security features (masking, safe export)")
        print("✅ Comprehensive error handling and validation")

        if args.all_features:
            print("✅ Advanced error handling and validation patterns")

        precedence_info = loader.get_precedence_info()
        print(f"\n📊 Final Configuration State:")
        print(
            f"   Environment overrides: {len(precedence_info['environment_overrides'])}"
        )
        print(f"   CLI overrides: {len(precedence_info['cli_overrides'])}")
        print(
            f"   .env file support: {'Available' if precedence_info['env_file_loaded'] else 'Not available'}"
        )

        print("\n🎉 All enhanced configuration features demonstrated successfully!")

        return 0

    except KeyboardInterrupt:
        print("\n\n🛑 Demo interrupted by user")
        return 1
    except Exception as e:
        print(f"\n❌ Unexpected error during demo: {e}")
        return 1

    finally:
        # Cleanup any remaining demo environment variables
        if args.demo_env_vars:
            demo_vars = [
                "TP_ENVIRONMENT",
                "TP_DATA_OPENBB_API_KEY",
                "OPENBB_API_KEY",
                "BINANCE_API_KEY",
                "DATABASE_URL",
            ]
            for var in demo_vars:
                if var in os.environ:
                    del os.environ[var]
            print("\n🧹 Cleaned up demo environment variables")


if __name__ == "__main__":
    sys.exit(main())