config_loader.py

"""
Configuration Loader for Trading Platform

This module provides Pydantic-based configuration loading, validation, and
typed object exposure for a comprehensive trading platform supporting multiple
asset classes, data sources, brokers, models, risk management, and execution.

Integrates with:
- OpenBB Platform
- Third-party data providers (Polygon, crypto exchanges)
- Broker APIs (Alpaca, Interactive Brokers)
- Machine learning models
- Risk management systems
- Execution engines

Author: Trading Platform Team
Version: 1.0.0
"""

import json
import os
import sys
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from pydantic import (
    AnyUrl,
    BaseModel,
    ConfigDict,
    Field,
    SecretStr,
    field_validator,
    model_validator,
)

# Import python-dotenv for .env file support
try:
    from dotenv import load_dotenv

    DOTENV_AVAILABLE = True
except ImportError:
    DOTENV_AVAILABLE = False
    load_dotenv = None


class EnvironmentType(str, Enum):
    """Environment configuration types"""

    DEVELOPMENT = "development"
    TESTING = "testing"
    STAGING = "staging"
    PRODUCTION = "production"


class BackoffStrategy(str, Enum):
    """Rate limiting backoff strategies"""

    EXPONENTIAL = "exponential"
    LINEAR = "linear"
    FIXED = "fixed"


class LogLevel(str, Enum):
    """Logging levels"""

    DEBUG = "DEBUG"
    INFO = "INFO"
    WARNING = "WARNING"
    ERROR = "ERROR"
    CRITICAL = "CRITICAL"


class LogFormat(str, Enum):
    """Log output formats"""

    JSON = "json"
    TEXT = "text"


class OrderType(str, Enum):
    """Order types for trading"""

    MARKET = "MARKET"
    LIMIT = "LIMIT"
    STOP = "STOP"
    STOP_LIMIT = "STOP_LIMIT"
    TRAIL = "TRAIL"
    OCO = "OCO"


class TimeInForce(str, Enum):
    """Time in force options"""

    DAY = "DAY"
    GTC = "GTC"
    IOC = "IOC"
    FOK = "FOK"


class CryptoType(str, Enum):
    """Cryptocurrency trading types"""

    SPOT = "spot"
    MARGIN = "margin"
    FUTURE = "future"


# Core Configuration Models


class CredentialsConfig(BaseModel):
    """API Credentials configuration with environment variable support

    Supports multiple ways to provide credentials with precedence order:
    1. Environment variables (highest precedence)
    2. Direct values from CLI/config override
    3. JSON configuration defaults (lowest precedence)
    """

    api_key: Optional[str] = Field(None, min_length=8, max_length=128)
    secret: Optional[SecretStr] = Field(None, min_length=16, max_length=256)
    passphrase: Optional[SecretStr] = Field(None, min_length=8, max_length=64)
    environment_variable: Optional[str] = None
    # New fields for enhanced secrets handling
    env_prefix: Optional[str] = Field(
        None, description="Environment variable prefix for auto-discovery"
    )
    vault_path: Optional[str] = Field(None, description="HashiCorp Vault secret path")

    @model_validator(mode="before")
    @classmethod
    def validate_credentials(cls, values):
        """Ensure either direct credentials or environment variable is provided

        Handles precedence order:
        1. Environment variables (resolved from ${VAR} patterns)
        2. Direct credential values
        3. Fallback validation
        """
        if isinstance(values, dict):
            api_key = values.get("api_key")
            secret = values.get("secret")
            env_var = values.get("environment_variable")
            env_prefix = values.get("env_prefix")

            # If environment_variable contains the ${} pattern, treat it as unresolved
            # If it's a regular string, treat it as resolved credential
            if env_var and env_var.startswith("${") and env_var.endswith("}"):
                # Validate environment variable pattern
                import re

                if not re.match(r"^\$\{[A-Z_][A-Z0-9_]*\}$", env_var):
                    raise ValueError(f"Invalid environment variable pattern: {env_var}")
            elif env_var:
                # Environment variable has been resolved - treat as api_key
                values["api_key"] = env_var
                values["environment_variable"] = None
            elif env_prefix:
                # Try to auto-discover credentials using prefix
                prefix_upper = env_prefix.upper()
                if not api_key:
                    api_key = os.getenv(f"{prefix_upper}_API_KEY") or os.getenv(
                        f"{prefix_upper}_KEY"
                    )
                    if api_key:
                        values["api_key"] = api_key
                if not secret:
                    secret_val = os.getenv(f"{prefix_upper}_SECRET") or os.getenv(
                        f"{prefix_upper}_API_SECRET"
                    )
                    if secret_val:
                        values["secret"] = secret_val
                passphrase_val = values.get("passphrase")
                if not passphrase_val:
                    passphrase_val = os.getenv(f"{prefix_upper}_PASSPHRASE")
                    if passphrase_val:
                        values["passphrase"] = passphrase_val
            elif not (api_key and secret):
                raise ValueError(
                    "Either environment_variable, env_prefix, or both api_key and secret must be provided"
                )
        return values


class RateLimitConfig(BaseModel):
    """Rate limiting configuration"""

    requests_per_second: float = Field(default=10.0, gt=0.1, le=1000)
    requests_per_hour: int = Field(default=1000, ge=1, le=100000)
    burst_limit: int = Field(default=5, ge=1, le=100)
    backoff_strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL


class ConnectionConfig(BaseModel):
    """Connection configuration parameters"""

    timeout: int = Field(default=30, ge=1, le=300)
    retry_attempts: int = Field(default=3, ge=0, le=10)
    retry_delay: float = Field(default=1.0, ge=0.1, le=60.0)
    heartbeat_interval: int = Field(default=30, ge=5, le=300)
    reconnect_delay: int = Field(default=5, ge=1, le=60)


class RiskLimitsConfig(BaseModel):
    """Risk management limits configuration"""

    max_position_size: float = Field(default=0.05, gt=0.001, le=1.0)
    max_daily_loss: float = Field(default=0.02, gt=0.001, le=1.0)
    stop_loss: Optional[float] = Field(default=0.015, gt=0.001, le=0.5)
    take_profit: Optional[float] = Field(default=0.03, gt=0.001, le=1.0)
    max_drawdown: Optional[float] = Field(default=0.1, gt=0.001, le=1.0)


class LogConfig(BaseModel):
    """Logging configuration"""

    level: LogLevel = LogLevel.INFO
    format: LogFormat = LogFormat.JSON
    file_path: Optional[str] = Field(None, pattern=r"^[\w\-\.\/\\:]+\.(log|txt)$")
    max_file_size: str = Field(default="100MB", pattern=r"^\d+[KMGT]?B$")
    backup_count: int = Field(default=10, ge=1, le=100)


# Data Provider Models


class OpenBBConfig(BaseModel):
    """OpenBB Platform integration configuration"""

    credentials: CredentialsConfig
    websocket_url: AnyUrl = "wss://websocket.openbb.co"
    rate_limits: Optional[RateLimitConfig] = None
    connection: Optional[ConnectionConfig] = None
    symbols: Optional[Dict[str, List[str]]] = None


class PolygonConfig(BaseModel):
    """Polygon.io market data configuration"""

    credentials: CredentialsConfig
    websocket_url: AnyUrl = "wss://socket.polygon.io"
    rate_limits: Optional[RateLimitConfig] = None


class CryptoExchangeConfig(BaseModel):
    """Base cryptocurrency exchange configuration"""

    credentials: CredentialsConfig
    sandbox: bool = True
    symbols: Optional[List[str]] = None


class BinanceConfig(CryptoExchangeConfig):
    """Binance exchange configuration"""

    default_type: CryptoType = CryptoType.SPOT


class CoinbaseConfig(CryptoExchangeConfig):
    """Coinbase exchange configuration"""

    pass


class BybitConfig(CryptoExchangeConfig):
    """Bybit exchange configuration"""

    pass


class CryptoExchangesConfig(BaseModel):
    """Cryptocurrency exchanges configuration"""

    binance: Optional[BinanceConfig] = None
    coinbase: Optional[CoinbaseConfig] = None
    bybit: Optional[BybitConfig] = None


class AlternativeDataConfig(BaseModel):
    """Alternative data sources configuration"""

    defi_protocols: Optional[Dict[str, Any]] = None
    social_sentiment: Optional[Dict[str, Any]] = None


class DataProvidersConfig(BaseModel):
    """Data providers configuration"""

    openbb: Optional[OpenBBConfig] = None
    polygon: Optional[PolygonConfig] = None
    crypto_exchanges: Optional[CryptoExchangesConfig] = None
    alternative_data: Optional[AlternativeDataConfig] = None


class DatabaseConfig(BaseModel):
    """Database configuration"""

    type: str = "postgresql"
    connection_string: str = Field(..., min_length=10)
    pool_size: int = Field(default=20, ge=1, le=100)
    timeout: int = Field(default=60, ge=1, le=300)


class CacheConfig(BaseModel):
    """Cache configuration"""

    type: str = "redis"
    host: str = "localhost"
    port: int = Field(default=6379, ge=1, le=65535)
    ttl: int = Field(default=1800, ge=1)


class StorageConfig(BaseModel):
    """Storage configuration"""

    database: Optional[DatabaseConfig] = None
    cache: Optional[CacheConfig] = None


class DataConfig(BaseModel):
    """Complete data configuration"""

    providers: Optional[DataProvidersConfig] = None
    storage: Optional[StorageConfig] = None


# Broker Models


class AlpacaConfig(BaseModel):
    """Alpaca broker configuration"""

    credentials: CredentialsConfig
    base_url: AnyUrl = "https://paper-api.alpaca.markets"
    supported_assets: List[str] = ["stocks", "etfs", "options", "crypto"]


class InteractiveBrokersConfig(BaseModel):
    """Interactive Brokers configuration"""

    account_id: str = Field(..., min_length=5, max_length=20)
    gateway_host: str = "localhost"
    gateway_port: int = Field(default=7497, ge=1, le=65535)
    client_id: int = Field(default=1, ge=1, le=32)
    supported_assets: List[str] = ["stocks", "options", "futures", "forex", "bonds"]


class BrokerConnectionsConfig(BaseModel):
    """Broker connections configuration"""

    alpaca: Optional[AlpacaConfig] = None
    interactive_brokers: Optional[InteractiveBrokersConfig] = None


class OAuthConfig(BaseModel):
    """OAuth configuration"""

    provider: str = "auth0"
    client_id: str = Field(..., min_length=10)
    redirect_uri: AnyUrl
    scopes: List[str] = ["read:portfolio", "write:trades", "read:market_data"]


class AuthenticationConfig(BaseModel):
    """Authentication configuration"""

    oauth: Optional[OAuthConfig] = None


class BrokersConfig(BaseModel):
    """Complete brokers configuration"""

    connections: Optional[BrokerConnectionsConfig] = None
    authentication: Optional[AuthenticationConfig] = None


# Models Configuration


class XGBoostModelConfig(BaseModel):
    """XGBoost model configuration"""

    weight: float = Field(default=0.35, gt=0.0, le=1.0)
    params: Dict[str, Any] = {
        "objective": "reg:squarederror",
        "max_depth": 8,
        "learning_rate": 0.08,
        "n_estimators": 1500,
    }


class EnsembleConfig(BaseModel):
    """Ensemble model configuration"""

    prediction_horizon: str = "1h"
    models: Dict[str, Any] = {}
    feature_engineering: Dict[str, Any] = {}


class PPOConfig(BaseModel):
    """PPO (Proximal Policy Optimization) configuration"""

    training_iterations: int = Field(default=2000, ge=100)
    checkpoint_frequency: int = Field(default=200, ge=10)
    config: Dict[str, Any] = {"lr": 0.0005, "gamma": 0.995, "clip_param": 0.2}


class ReinforcementLearningConfig(BaseModel):
    """Reinforcement learning configuration"""

    ppo: Optional[PPOConfig] = None


class SentimentModelConfig(BaseModel):
    """Individual sentiment model configuration"""

    model_name: str = Field(..., min_length=5)
    weight: float = Field(default=1.0, gt=0.0, le=1.0)


class SentimentAnalysisConfig(BaseModel):
    """Sentiment analysis configuration"""

    models: Dict[str, SentimentModelConfig] = {}
    processing: Dict[str, Any] = {"max_length": 512, "batch_size": 64}


class MachineLearningConfig(BaseModel):
    """Machine learning models configuration"""

    ensemble: Optional[EnsembleConfig] = None
    reinforcement_learning: Optional[ReinforcementLearningConfig] = None
    sentiment_analysis: Optional[SentimentAnalysisConfig] = None


class DerivativesPricingConfig(BaseModel):
    """Derivatives pricing models configuration"""

    black_scholes: Dict[str, float] = {"risk_free_rate": 0.05, "dividend_yield": 0.015}
    monte_carlo: Dict[str, int] = {"simulations": 50000, "time_steps": 252}


class PricingConfig(BaseModel):
    """Pricing models configuration"""

    derivatives: Optional[DerivativesPricingConfig] = None


class ModelsConfig(BaseModel):
    """Complete models configuration"""

    machine_learning: Optional[MachineLearningConfig] = None
    pricing: Optional[PricingConfig] = None


# Risk Management Models


class KillSwitchConfig(BaseModel):
    """Emergency kill switch configuration"""

    enabled: bool = True
    threshold: float = Field(default=0.08, gt=0.0, le=1.0)
    check_interval: int = Field(default=5, ge=1, le=60)


class RiskMonitoringConfig(BaseModel):
    """Risk monitoring configuration"""

    real_time: bool = True
    update_interval: int = Field(default=15, ge=1, le=300)
    kill_switch: Optional[KillSwitchConfig] = None


class KYCAMLConfig(BaseModel):
    """KYC/AML compliance configuration"""

    enabled: bool = True
    provider: str = "jumio"
    transaction_monitoring: bool = True


class ComplianceReportingConfig(BaseModel):
    """Compliance reporting configuration"""

    trade_reporting: bool = True
    position_reporting: bool = True
    frequency: str = "daily"


class ComplianceConfig(BaseModel):
    """Compliance configuration"""

    kyc_aml: Optional[KYCAMLConfig] = None
    reporting: Optional[ComplianceReportingConfig] = None


class RiskLimitsGroupConfig(BaseModel):
    """Risk limits group configuration"""

    portfolio_level: Optional[RiskLimitsConfig] = None
    asset_class_level: Optional[Dict[str, RiskLimitsConfig]] = None
    position_level: Optional[Dict[str, Any]] = None


class RiskConfig(BaseModel):
    """Complete risk management configuration"""

    limits: Optional[RiskLimitsGroupConfig] = None
    monitoring: Optional[RiskMonitoringConfig] = None
    compliance: Optional[ComplianceConfig] = None


# Execution Models


class RetryConfig(BaseModel):
    """Retry configuration for orders"""

    max_attempts: int = Field(default=3, ge=1, le=10)
    retry_delay: float = Field(default=1.5, ge=0.1, le=60.0)


class OrderManagementConfig(BaseModel):
    """Order management configuration"""

    order_types: List[OrderType] = [OrderType.MARKET, OrderType.LIMIT, OrderType.STOP]
    time_in_force: TimeInForce = TimeInForce.DAY
    retry_config: Optional[RetryConfig] = None


class SlippageControlConfig(BaseModel):
    """Slippage control configuration"""

    max_slippage_warning: float = Field(default=0.015, gt=0.0, le=1.0)
    tolerance: float = Field(default=0.008, gt=0.0, le=1.0)


class TWAPConfig(BaseModel):
    """TWAP algorithm configuration"""

    enabled: bool = True
    time_window: int = Field(default=120, ge=1, le=1440)


class VWAPConfig(BaseModel):
    """VWAP algorithm configuration"""

    enabled: bool = True
    participation_rate: float = Field(default=0.15, gt=0.0, le=1.0)


class AlgorithmsConfig(BaseModel):
    """Trading algorithms configuration"""

    twap: Optional[TWAPConfig] = None
    vwap: Optional[VWAPConfig] = None


class ExecutionConfig(BaseModel):
    """Complete execution configuration"""

    order_management: Optional[OrderManagementConfig] = None
    slippage_control: Optional[SlippageControlConfig] = None
    algorithms: Optional[AlgorithmsConfig] = None


# Backtesting Models


class BenchmarkConfig(BaseModel):
    """Benchmark configuration for backtesting"""

    symbol: str = "SPY"
    enabled: bool = True


class BacktestEngineConfig(BaseModel):
    """Backtesting engine configuration"""

    framework: str = "custom"
    performance_metrics: List[str] = [
        "sharpe",
        "sortino",
        "max_drawdown",
        "calmar",
        "win_rate",
        "profit_factor",
    ]
    benchmark: Optional[BenchmarkConfig] = None


class HistoricalRangeConfig(BaseModel):
    """Historical data range configuration"""

    training_days: int = Field(default=90, ge=1)
    min_data_points: int = Field(default=2000, ge=100)


class DataHandlingConfig(BaseModel):
    """Backtesting data handling configuration"""

    historical_range: Optional[HistoricalRangeConfig] = None
    survivorship_bias: bool = True


class CrossValidationConfig(BaseModel):
    """Cross-validation configuration"""

    enabled: bool = True
    folds: int = Field(default=7, ge=2, le=20)


class WalkForwardConfig(BaseModel):
    """Walk-forward validation configuration"""

    enabled: bool = True
    window_size: int = Field(default=120, ge=1)
    step_size: int = Field(default=14, ge=1)


class ValidationConfig(BaseModel):
    """Validation configuration"""

    cross_validation: Optional[CrossValidationConfig] = None
    walk_forward: Optional[WalkForwardConfig] = None


class BacktestConfig(BaseModel):
    """Complete backtesting configuration"""

    engine: Optional[BacktestEngineConfig] = None
    data_handling: Optional[DataHandlingConfig] = None
    validation: Optional[ValidationConfig] = None
    symbols: Optional[List[str]] = None


# System Configuration Models


class ComputeConfig(BaseModel):
    """Compute resources configuration"""

    cpu_threads: int = Field(default=16, ge=1, le=128)
    worker_processes: int = Field(default=8, ge=1, le=64)
    memory_limit: str = Field(default="16GB", pattern=r"^\d+[KMGT]?B$")


class CORSConfig(BaseModel):
    """CORS configuration"""

    allowed_origins: List[str] = [
        "http://localhost:3000",
        "http://localhost:8080",
        "https://trading.example.com",
    ]
    allowed_methods: List[str] = ["GET", "POST", "PUT", "DELETE", "OPTIONS"]


class NetworkingConfig(BaseModel):
    """Networking configuration"""

    cors: Optional[CORSConfig] = None


class InfrastructureConfig(BaseModel):
    """Infrastructure configuration"""

    compute: Optional[ComputeConfig] = None
    networking: Optional[NetworkingConfig] = None


class ComponentLogConfig(BaseModel):
    """Individual component logging configuration"""

    level: LogLevel = LogLevel.INFO
    format: LogFormat = LogFormat.JSON
    file_path: str = Field(..., pattern=r"^[\w\-\.\/\\:]+\.(log|txt)$")
    max_file_size: str = Field(default="100MB", pattern=r"^\d+[KMGT]?B$")
    backup_count: int = Field(default=10, ge=1, le=100)


class LoggingConfig(BaseModel):
    """System logging configuration"""

    application: Optional[ComponentLogConfig] = None
    components: Optional[Dict[str, ComponentLogConfig]] = None


class PrometheusConfig(BaseModel):
    """Prometheus monitoring configuration"""

    enabled: bool = True
    port: int = Field(default=9090, ge=1, le=65535)
    scrape_interval: str = "10s"


class GrafanaConfig(BaseModel):
    """Grafana dashboard configuration"""

    enabled: bool = True
    port: int = Field(default=3000, ge=1, le=65535)
    admin_credentials: Optional[CredentialsConfig] = None


class MonitoringConfig(BaseModel):
    """System monitoring configuration"""

    prometheus: Optional[PrometheusConfig] = None
    grafana: Optional[GrafanaConfig] = None


class VaultConfig(BaseModel):
    """HashiCorp Vault configuration"""

    enabled: bool = True
    url: AnyUrl = "https://vault.trading-platform.internal:8200"
    auth_method: str = "kubernetes"


class EncryptionConfig(BaseModel):
    """Encryption configuration"""

    at_rest: bool = True
    in_transit: bool = True
    algorithm: str = "AES-256"


class SecurityConfig(BaseModel):
    """Security configuration"""

    vault: Optional[VaultConfig] = None
    encryption: Optional[EncryptionConfig] = None


class SlackConfig(BaseModel):
    """Slack notifications configuration"""

    enabled: bool = True
    credentials: CredentialsConfig
    channels: Dict[str, str] = {
        "critical": "#trading-critical",
        "executions": "#trading-executions",
        "risk": "#trading-risk-alerts",
    }


class TelegramConfig(BaseModel):
    """Telegram notifications configuration"""

    enabled: bool = True
    credentials: CredentialsConfig
    chat_id: str = Field(..., min_length=5)


class NotificationsConfig(BaseModel):
    """Notifications configuration"""

    slack: Optional[SlackConfig] = None
    telegram: Optional[TelegramConfig] = None


class FeatureFlagsConfig(BaseModel):
    """Feature flags configuration"""

    enable_paper_trading: bool = True
    enable_backtesting_mode: bool = True
    enable_dry_run: bool = True
    enable_position_sizing: bool = True
    enable_slippage_control: bool = True


class SystemConfig(BaseModel):
    """Complete system configuration"""

    infrastructure: Optional[InfrastructureConfig] = None
    logging: Optional[LoggingConfig] = None
    monitoring: Optional[MonitoringConfig] = None
    security: Optional[SecurityConfig] = None
    notifications: Optional[NotificationsConfig] = None
    environment: EnvironmentType = EnvironmentType.DEVELOPMENT
    feature_flags: Optional[FeatureFlagsConfig] = None


# Main Configuration Model


class ConfigMetadata(BaseModel):
    """Configuration metadata"""

    version: str = Field(default="1.0.0", pattern=r"^\d+\.\d+\.\d+$")
    name: Optional[str] = Field(None, min_length=1, max_length=100)
    environment: EnvironmentType = EnvironmentType.DEVELOPMENT
    created_at: Optional[datetime] = None
    updated_at: Optional[datetime] = None


class TradingPlatformConfig(BaseModel):
    """
    Main configuration model for the trading platform

    This model represents the complete configuration structure for a
    multi-asset trading platform with comprehensive validation rules.
    """

    model_config = ConfigDict(
        extra="forbid",
        validate_assignment=True,
        str_strip_whitespace=True,
        use_enum_values=True,
    )

    schema_: Optional[str] = Field(None, alias="$schema")
    metadata: Optional[ConfigMetadata] = None
    data: Optional[DataConfig] = None
    brokers: Optional[BrokersConfig] = None
    models: Optional[ModelsConfig] = None
    risk: Optional[RiskConfig] = None
    execution: Optional[ExecutionConfig] = None
    backtest: Optional[BacktestConfig] = None
    system: Optional[SystemConfig] = None

    @field_validator("system")
    @classmethod
    def validate_production_requirements(cls, v, info):
        """Ensure production environments have proper security and monitoring"""
        if hasattr(info, "data") and info.data:
            metadata = info.data.get("metadata")
            if (
                metadata
                and hasattr(metadata, "environment")
                and metadata.environment == EnvironmentType.PRODUCTION
            ):
                if not v or not v.security or not v.monitoring:
                    raise ValueError(
                        "Production environment requires security and monitoring configuration"
                    )
        return v


# Configuration Loader Class


class ConfigurationLoader:
    """
    Configuration loader with validation and environment-specific handling

    Provides methods to load, validate, and expose configuration settings
    as strongly-typed Pydantic objects with integration support for OpenBB
    and other financial data platforms.

    Supports configuration precedence order:
    1. Environment variables (highest precedence)
    2. CLI flag overrides (medium precedence)
    3. JSON configuration defaults (lowest precedence)
    """

    def __init__(
        self,
        config_path: Optional[Union[str, Path]] = None,
        env_file: Optional[Union[str, Path]] = None,
    ):
        """
        Initialize configuration loader

        Args:
            config_path: Path to configuration file (JSON/YAML)
            env_file: Path to .env file for environment variable loading
        """
        self.config_path = Path(config_path) if config_path else None
        self.env_file = Path(env_file) if env_file else None
        self._config: Optional[TradingPlatformConfig] = None
        self._env_overrides: Dict[str, Any] = {}
        self._cli_overrides: Dict[str, Any] = {}

        # Load .env file if available
        self._load_env_file()

    def load_config(
        self,
        config_path: Optional[Union[str, Path]] = None,
        validate: bool = True,
        cli_overrides: Optional[Dict[str, Any]] = None,
    ) -> TradingPlatformConfig:
        """
        Load configuration from file with environment and CLI overrides

        Applies configuration in precedence order:
        1. Environment variables (highest precedence)
        2. CLI flag overrides (medium precedence)
        3. JSON configuration defaults (lowest precedence)

        Args:
            config_path: Path to configuration file
            validate: Whether to validate the configuration
            cli_overrides: Dictionary of CLI flag overrides

        Returns:
            Validated TradingPlatformConfig object

        Raises:
            FileNotFoundError: If config file doesn't exist
            ValueError: If configuration is invalid
            json.JSONDecodeError: If JSON is malformed
        """
        if config_path:
            self.config_path = Path(config_path)

        # Store CLI overrides
        if cli_overrides:
            self._cli_overrides = cli_overrides

        if not self.config_path or not self.config_path.exists():
            raise FileNotFoundError(f"Configuration file not found: {self.config_path}")

        # Load configuration data (step 3 - lowest precedence)
        config_data = self._load_config_file()

        # Apply CLI overrides (step 2 - medium precedence)
        config_data = self._apply_cli_overrides(config_data)

        # Resolve environment variables (step 1 - highest precedence)
        config_data = self._resolve_environment_variables(config_data)

        # Apply direct environment variable overrides
        config_data = self._apply_environment_overrides(config_data)

        # Create and validate configuration
        try:
            self._config = TradingPlatformConfig(**config_data)

            if validate:
                self._validate_integrations()
                self._log_precedence_info()

            return self._config

        except Exception as e:
            raise ValueError(f"Configuration validation failed: {str(e)}")

    def _load_config_file(self) -> Dict[str, Any]:
        """Load configuration from JSON file"""
        try:
            with open(self.config_path, "r", encoding="utf-8") as file:
                return json.load(file)
        except json.JSONDecodeError as e:
            raise json.JSONDecodeError(
                f"Invalid JSON in config file: {str(e)}", e.doc, e.pos
            )

    def _load_env_file(self):
        """
        Load environment variables from .env file if available

        Searches for .env files in the following order:
        1. Explicitly provided env_file path
        2. .env file in the same directory as config file
        3. .env file in current working directory
        4. .env file in parent directories (up to 3 levels)
        """
        if not DOTENV_AVAILABLE:
            return

        env_files_to_try = []

        # Add explicitly provided env file
        if self.env_file and self.env_file.exists():
            env_files_to_try.append(self.env_file)

        # Add .env file in config directory
        if self.config_path:
            config_dir_env = self.config_path.parent / ".env"
            if config_dir_env.exists():
                env_files_to_try.append(config_dir_env)

        # Add .env file in current directory and parent directories
        current_dir = Path.cwd()
        for i in range(4):  # Check current + 3 parent levels
            env_file = current_dir / ".env"
            if env_file.exists() and env_file not in env_files_to_try:
                env_files_to_try.append(env_file)
            current_dir = current_dir.parent
            if current_dir == current_dir.parent:  # Reached filesystem root
                break

        # Load the first found .env file
        for env_file in env_files_to_try:
            try:
                load_dotenv(
                    env_file, override=False
                )  # Don't override existing env vars
                print(f"✅ Loaded environment variables from: {env_file}")
                break
            except Exception as e:
                print(f"⚠️  Failed to load .env file {env_file}: {e}")
                continue

    def _apply_cli_overrides(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """
        Apply CLI flag overrides to configuration data

        Args:
            data: Configuration dictionary

        Returns:
            Configuration with CLI overrides applied
        """
        if not self._cli_overrides:
            return data

        def deep_update(base_dict, update_dict):
            """Recursively update nested dictionary"""
            for key, value in update_dict.items():
                if (
                    key in base_dict
                    and isinstance(base_dict[key], dict)
                    and isinstance(value, dict)
                ):
                    deep_update(base_dict[key], value)
                else:
                    base_dict[key] = value

        # Apply CLI overrides with deep merge
        deep_update(data, self._cli_overrides)
        return data

    def _apply_environment_overrides(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """
        Apply direct environment variable overrides for common sensitive fields

        This method provides a convenient way to override sensitive configuration
        fields directly via environment variables without requiring ${VAR} syntax
        in the JSON configuration.

        Environment variable naming convention:
        - TP_<SECTION>_<SUBSECTION>_<FIELD>
        - Example: TP_DATA_OPENBB_API_KEY

        Args:
            data: Configuration dictionary

        Returns:
            Configuration with environment overrides applied
        """
        # Define mapping of environment variables to config paths
        env_mappings = {
            # Data provider API keys
            "TP_DATA_OPENBB_API_KEY": [
                "data",
                "providers",
                "openbb",
                "credentials",
                "api_key",
            ],
            "TP_DATA_POLYGON_API_KEY": [
                "data",
                "providers",
                "polygon",
                "credentials",
                "api_key",
            ],
            "TP_DATA_BINANCE_API_KEY": [
                "data",
                "providers",
                "crypto_exchanges",
                "binance",
                "credentials",
                "api_key",
            ],
            "TP_DATA_BINANCE_SECRET": [
                "data",
                "providers",
                "crypto_exchanges",
                "binance",
                "credentials",
                "secret",
            ],
            "TP_DATA_COINBASE_API_KEY": [
                "data",
                "providers",
                "crypto_exchanges",
                "coinbase",
                "credentials",
                "api_key",
            ],
            "TP_DATA_COINBASE_SECRET": [
                "data",
                "providers",
                "crypto_exchanges",
                "coinbase",
                "credentials",
                "secret",
            ],
            "TP_DATA_BYBIT_API_KEY": [
                "data",
                "providers",
                "crypto_exchanges",
                "bybit",
                "credentials",
                "api_key",
            ],
            "TP_DATA_BYBIT_SECRET": [
                "data",
                "providers",
                "crypto_exchanges",
                "bybit",
                "credentials",
                "secret",
            ],
            # Broker credentials
            "TP_BROKERS_ALPACA_API_KEY": [
                "brokers",
                "connections",
                "alpaca",
                "credentials",
                "api_key",
            ],
            "TP_BROKERS_ALPACA_SECRET": [
                "brokers",
                "connections",
                "alpaca",
                "credentials",
                "secret",
            ],
            # Database credentials
            "TP_DATABASE_PASSWORD": [
                "data",
                "storage",
                "database",
                "connection_string",
            ],
            "TP_DATABASE_URL": ["data", "storage", "database", "connection_string"],
            # Notification credentials
            "TP_SLACK_BOT_TOKEN": [
                "system",
                "notifications",
                "slack",
                "credentials",
                "api_key",
            ],
            "TP_TELEGRAM_BOT_TOKEN": [
                "system",
                "notifications",
                "telegram",
                "credentials",
                "api_key",
            ],
            # Monitoring credentials
            "TP_GRAFANA_ADMIN_PASSWORD": [
                "system",
                "monitoring",
                "grafana",
                "admin_credentials",
                "api_key",
            ],
            # OAuth credentials
            "TP_OAUTH_CLIENT_ID": ["brokers", "authentication", "oauth", "client_id"],
            "TP_OAUTH_CLIENT_SECRET": [
                "brokers",
                "authentication",
                "oauth",
                "client_secret",
            ],
            # Environment override
            "TP_ENVIRONMENT": ["metadata", "environment"],
        }

        # Apply environment variable overrides
        for env_var, config_path in env_mappings.items():
            env_value = os.getenv(env_var)
            if env_value:
                # Navigate to the nested dictionary location
                current_dict = data
                for key in config_path[:-1]:
                    if key not in current_dict:
                        current_dict[key] = {}
                    current_dict = current_dict[key]

                # Set the value
                final_key = config_path[-1]
                current_dict[final_key] = env_value

                # Log override (mask sensitive values)
                if any(
                    sensitive in env_var.lower()
                    for sensitive in ["key", "secret", "password", "token"]
                ):
                    masked_value = (
                        f"{env_value[:4]}...{env_value[-4:]}"
                        if len(env_value) > 8
                        else "***"
                    )
                    self._env_overrides[env_var] = masked_value
                else:
                    self._env_overrides[env_var] = env_value

        return data

    def _resolve_environment_variables(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """
        Recursively resolve environment variables in configuration using ${VAR} syntax

        Args:
            data: Configuration dictionary

        Returns:
            Configuration with resolved environment variables
        """
        if isinstance(data, dict):
            return {k: self._resolve_environment_variables(v) for k, v in data.items()}
        elif isinstance(data, list):
            return [self._resolve_environment_variables(item) for item in data]
        elif isinstance(data, str) and data.startswith("${") and data.endswith("}"):
            # Extract environment variable name
            env_var = data[2:-1]
            value = os.getenv(env_var)
            if value is None:
                # Check for default value syntax: ${VAR:-default}
                if ":-" in env_var:
                    var_name, default_value = env_var.split(":-", 1)
                    value = os.getenv(var_name, default_value)
                else:
                    raise ValueError(f"Environment variable '{env_var}' not found")
            return value
        return data

    def _log_precedence_info(self):
        """
        Log information about configuration precedence and applied overrides
        """
        print("\n📋 Configuration Precedence Information:")
        print("   1. Environment variables (highest precedence)")
        print("   2. CLI flag overrides (medium precedence)")
        print("   3. JSON configuration defaults (lowest precedence)")

        if self._env_overrides:
            print("\n🔧 Environment variable overrides applied:")
            for env_var, value in self._env_overrides.items():
                print(f"   {env_var} = {value}")

        if self._cli_overrides:
            print("\n⚙️  CLI overrides applied:")
            for key, value in self._cli_overrides.items():
                print(f"   {key} = {value}")

        if not self._env_overrides and not self._cli_overrides:
            print("\n✅ No overrides applied - using JSON configuration defaults")

    def _validate_integrations(self):
        """Validate OpenBB and other integrations"""
        if not self._config:
            return

        # Validate OpenBB configuration
        if (
            self._config.data
            and self._config.data.providers
            and self._config.data.providers.openbb
        ):
            openbb_config = self._config.data.providers.openbb
            if openbb_config.credentials.environment_variable:
                env_var = openbb_config.credentials.environment_variable[
                    2:-1
                ]  # Remove ${}
                if not os.getenv(env_var):
                    print(
                        f"Warning: OpenBB API key environment variable '{env_var}' not set"
                    )

    def get_config(self) -> Optional[TradingPlatformConfig]:
        """Get current configuration object"""
        return self._config

    def get_openbb_config(self) -> Optional[OpenBBConfig]:
        """Get OpenBB-specific configuration"""
        if self._config and self._config.data and self._config.data.providers:
            return self._config.data.providers.openbb
        return None

    def get_broker_config(
        self, broker_name: str
    ) -> Optional[Union[AlpacaConfig, InteractiveBrokersConfig]]:
        """
        Get broker-specific configuration

        Args:
            broker_name: Name of the broker ('alpaca', 'interactive_brokers')

        Returns:
            Broker configuration object or None
        """
        if self._config and self._config.brokers and self._config.brokers.connections:
            connections = self._config.brokers.connections
            return getattr(connections, broker_name, None)
        return None

    def get_risk_limits(
        self, asset_class: Optional[str] = None
    ) -> Optional[RiskLimitsConfig]:
        """
        Get risk limits for specific asset class or portfolio level

        Args:
            asset_class: Asset class name (e.g., 'equities', 'crypto')

        Returns:
            Risk limits configuration
        """
        if not (self._config and self._config.risk and self._config.risk.limits):
            return None

        limits = self._config.risk.limits

        if asset_class and limits.asset_class_level:
            return limits.asset_class_level.get(asset_class)

        return limits.portfolio_level

    def is_production(self) -> bool:
        """Check if running in production environment"""
        return (
            self._config
            and self._config.metadata
            and self._config.metadata.environment == EnvironmentType.PRODUCTION
        )

    def export_config(
        self,
        output_path: Union[str, Path],
        format: str = "json",
        include_sensitive: bool = False,
    ) -> None:
        """
        Export current configuration to file with option to mask sensitive data

        Args:
            output_path: Output file path
            format: Output format ('json' or 'dict')
            include_sensitive: Whether to include sensitive data (default: False for security)
        """
        if not self._config:
            raise ValueError("No configuration loaded")

        output_path = Path(output_path)
        # Omit unset and None values in exported artifacts to avoid nulls/empties
        config_dict = self._config.model_dump(
            by_alias=True, exclude_none=True, exclude_unset=True
        )

        # Mask sensitive data if requested
        if not include_sensitive:
            config_dict = self._mask_sensitive_data(config_dict)

        if format.lower() == "json":
            with open(output_path, "w", encoding="utf-8") as file:
                json.dump(config_dict, file, indent=2, default=str)
        else:
            raise ValueError(f"Unsupported export format: {format}")

    def _mask_sensitive_data(self, data: Any) -> Any:
        """
        Recursively mask sensitive data in configuration

        Args:
            data: Configuration data

        Returns:
            Configuration with sensitive data masked
        """
        if isinstance(data, dict):
            masked_data = {}
            for key, value in data.items():
                # Check if key indicates sensitive data
                if any(
                    sensitive in key.lower()
                    for sensitive in ["key", "secret", "password", "token"]
                ):
                    if isinstance(value, str) and len(value) > 8:
                        masked_data[key] = f"{value[:4]}...{value[-4:]}"
                    else:
                        masked_data[key] = "***"
                else:
                    masked_data[key] = self._mask_sensitive_data(value)
            return masked_data
        elif isinstance(data, list):
            return [self._mask_sensitive_data(item) for item in data]
        else:
            return data

    def get_precedence_info(self) -> Dict[str, Any]:
        """
        Get information about configuration precedence and applied overrides

        Returns:
            Dictionary containing precedence information
        """
        return {
            "precedence_order": [
                "Environment variables (highest)",
                "CLI flag overrides (medium)",
                "JSON configuration defaults (lowest)",
            ],
            "environment_overrides": dict(self._env_overrides),
            "cli_overrides": dict(self._cli_overrides),
            "env_file_loaded": DOTENV_AVAILABLE,
            "config_file_path": str(self.config_path) if self.config_path else None,
        }


# Utility Functions


def load_config(config_path: Union[str, Path]) -> TradingPlatformConfig:
    """
    Convenience function to load and validate configuration

    Args:
        config_path: Path to configuration file

    Returns:
        Validated configuration object
    """
    loader = ConfigurationLoader(config_path)
    return loader.load_config()


def create_default_config() -> TradingPlatformConfig:
    """
    Create a default configuration with minimal required fields

    Returns:
        Default configuration object
    """
    return TradingPlatformConfig(
        metadata=ConfigMetadata(
            name="Default Trading Platform Configuration",
            environment=EnvironmentType.DEVELOPMENT,
        ),
        system=SystemConfig(
            environment=EnvironmentType.DEVELOPMENT, feature_flags=FeatureFlagsConfig()
        ),
    )


if __name__ == "__main__":
    """Example usage and testing"""
    import argparse

    parser = argparse.ArgumentParser(
        description="Trading Platform Configuration Loader"
    )
    parser.add_argument("--config", "-c", help="Configuration file path")
    parser.add_argument(
        "--validate", "-v", action="store_true", help="Validate configuration only"
    )
    parser.add_argument("--export", "-e", help="Export configuration to file")

    args = parser.parse_args()

    if args.config:
        try:
            loader = ConfigurationLoader(args.config)
            config = loader.load_config()
            print(f"✅ Configuration loaded successfully from {args.config}")
            print(
                f"Environment: {config.metadata.environment if config.metadata else 'Unknown'}"
            )

            # Display key configuration sections
            sections = []
            if config.data:
                sections.append("Data Sources")
            if config.brokers:
                sections.append("Brokers")
            if config.models:
                sections.append("Models")
            if config.risk:
                sections.append("Risk Management")
            if config.execution:
                sections.append("Execution")
            if config.backtest:
                sections.append("Backtesting")
            if config.system:
                sections.append("System")

            print(f"Configured sections: {', '.join(sections)}")

            if args.export:
                loader.export_config(args.export)
                print(f"✅ Configuration exported to {args.export}")

        except Exception as e:
            print(f"❌ Configuration error: {str(e)}")
            sys.exit(1)
    else:
        # Create and display default configuration
        default_config = create_default_config()
        print("✅ Default configuration created")
        print("Use --config to load a specific configuration file")