Refactor core foundation (exceptions and bootstrap)

CIME/SystemTests/test_mods.py

@@ -1,7 +1,7 @@
 import logging
 import os
 
-from CIME.utils import CIMEError
+from CIME.core.exceptions import CIMEError
 from CIME.XML.files import Files
 
 logger = logging.getLogger(__name__)

CIME/XML/compsets.py

@@ -6,7 +6,7 @@
 from CIME.XML.generic_xml import GenericXML
 from CIME.XML.entry_id import EntryID
 from CIME.XML.files import Files
-from CIME.utils import CIMEError
+from CIME.core.exceptions import CIMEError
 
 logger = logging.getLogger(__name__)
 

CIME/XML/machines.py

@@ -5,7 +5,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.XML.generic_xml import GenericXML
 from CIME.XML.files import Files
-from CIME.utils import CIMEError, expect, convert_to_unknown_type, get_cime_config
+from CIME.core.exceptions import CIMEError
+from CIME.utils import expect, convert_to_unknown_type, get_cime_config
 
 import re
 import logging

CIME/XML/namelist_definition.py

@@ -21,7 +21,7 @@
     Namelist,
     get_fortran_name_only,
 )
-from CIME.utils import CIMEError
+from CIME.core.exceptions import CIMEError
 from CIME.XML.standard_module_setup import *
 from CIME.XML.entry_id import EntryID
 from CIME.XML.files import Files
@@ -32,7 +32,6 @@
 
 
 class CaseInsensitiveDict(dict):
-
     """Basic case insensitive dict with strings only keys.
     From https://stackoverflow.com/a/27890005"""
 
@@ -65,7 +64,6 @@ def __setitem__(self, k, v):
 
 
 class NamelistDefinition(EntryID):
-
     """Class representing variable definitions for a namelist.
     This class inherits from `EntryID`, and supports most inherited methods;
     however, `set_value` is unsupported.

CIME/XML/tests.py

@@ -1,11 +1,13 @@
 """
 Interface to the config_tests.xml file.  This class inherits from GenericEntry
 """
+
 from CIME.XML.standard_module_setup import *
 
 from CIME.XML.generic_xml import GenericXML
 from CIME.XML.files import Files
-from CIME.utils import find_system_test, CIMEError
+from CIME.core.exceptions import CIMEError
+from CIME.utils import find_system_test
 from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo
 from CIME.SystemTests.system_tests_compare_n import SystemTestsCompareN
 

CIME/case/case_run.py

@@ -5,7 +5,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.config import Config
 from CIME.utils import gzip_existing_file, new_lid
-from CIME.utils import run_sub_or_cmd, safe_copy, model_log, CIMEError
+from CIME.core.exceptions import CIMEError
+from CIME.utils import run_sub_or_cmd, safe_copy, model_log
 from CIME.utils import batch_jobid, is_comp_standalone
 from CIME.status import append_status, run_and_log_case_status
 from CIME.get_timing import get_timing

CIME/case/case_submit.py

@@ -6,9 +6,11 @@
 jobs.
 submit, check_case and check_da_settings are members of class Case in file case.py
 """
+
 import configparser
 from CIME.XML.standard_module_setup import *
-from CIME.utils import expect, CIMEError, get_time_in_seconds
+from CIME.core.exceptions import CIMEError
+from CIME.utils import expect, get_time_in_seconds
 from CIME.status import run_and_log_case_status
 from CIME.locked_files import (
     unlock_file,

CIME/compare_namelists.py

@@ -1,10 +1,12 @@
 import os, re, logging
-from CIME.utils import expect, CIMEError
+from CIME.core.exceptions import CIMEError
+from CIME.utils import expect
 
 logger = logging.getLogger(__name__)
 
 # pragma pylint: disable=unsubscriptable-object
 
+
 ###############################################################################
 def _normalize_lists(value_str):
     ###############################################################################

CIME/core/__init__.py

@@ -0,0 +1,7 @@
+"""
+CIME core package — Dependency Injection (DI) based abstractions for filesystem, process, environment,
+clock, and configuration bootstrap.
+
+These protocols are designed for constructor injection so that business logic
+can be tested with lightweight mocks instead of hitting real OS resources.
+"""

CIME/core/config/__init__.py

@@ -0,0 +1 @@
+"""Configuration bootstrap and loading utilities."""

CIME/core/config/bootstrap.py

@@ -0,0 +1,182 @@
+"""
+Centralized sys.path management for CIME.
+
+CIME is not pip-installed; it relies on sys.path manipulation so that
+scripts executed via symlinks (e.g. case.setup, xmlchange) can find
+the CIME package. This module consolidates the various sys.path.insert
+patterns scattered across the codebase into one place.
+
+**This module is standalone for Slice 1** — existing call-sites are NOT
+modified yet. Migration will happen incrementally in later slices.
+
+Typical usage (future, after wiring)::
+
+    from CIME.core.config.bootstrap import bootstrap_cime
+    bootstrap_cime()                     # auto-detect CIMEROOT
+    bootstrap_cime(cimeroot="/explicit") # explicit root
+"""
+
+import os
+import sys
+import warnings
+from typing import List, Optional, Sequence
+
+
+def find_cimeroot(starting_dir: Optional[str] = None) -> str:
+    """Locate the CIME root directory.
+
+    Resolution order:
+    1. ``starting_dir`` argument (if provided)
+    2. ``CIMEROOT`` environment variable
+    3. Walk upward from this file to find the directory containing ``CIME/``
+
+    Returns:
+        Absolute path to the CIME root directory.
+
+    Raises:
+        RuntimeError: If CIMEROOT cannot be determined.
+    """
+    if starting_dir is not None:
+        resolved = os.path.abspath(starting_dir)
+        if _is_cimeroot(resolved):
+            return resolved
+        raise RuntimeError(f"Specified directory is not a valid CIMEROOT: {resolved}")
+
+    env_root = os.environ.get("CIMEROOT")
+    if env_root and _is_cimeroot(env_root):
+        return os.path.abspath(env_root)
+
+    # Walk up from this file: CIME/core/config/bootstrap.py -> CIME/core/config -> CIME/core -> CIME -> root
+    candidate = os.path.abspath(
+        os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..")
+    )
+    if _is_cimeroot(candidate):
+        return candidate
+
+    raise RuntimeError(
+        "Cannot determine CIMEROOT. Set the CIMEROOT environment variable "
+        "or pass cimeroot explicitly."
+    )
+
+
+def _is_cimeroot(path: str) -> bool:
+    """Check whether a directory looks like a valid CIMEROOT."""
+    return os.path.isdir(os.path.join(path, "CIME"))
+
+
+def bootstrap_cime(
+    cimeroot: Optional[str] = None,
+    extra_paths: Optional[Sequence[str]] = None,
+    set_env: bool = True,
+) -> str:
+    """Set up sys.path for CIME and return the resolved CIMEROOT.
+
+    This is the single function that should be called to prepare the
+    Python environment for CIME imports. It:
+
+    1. Resolves CIMEROOT
+    2. Inserts CIMEROOT at the front of sys.path (if not already present)
+    3. Inserts CIME/Tools at position 1 (if not already present)
+    4. Inserts any extra_paths
+    5. Optionally sets the CIMEROOT environment variable
+
+    Args:
+        cimeroot: Explicit CIMEROOT path. If None, auto-detected.
+        extra_paths: Additional paths to insert after CIMEROOT and Tools.
+        set_env: Whether to set ``os.environ["CIMEROOT"]``.
+
+    Returns:
+        The resolved CIMEROOT path.
+    """
+    root = find_cimeroot(cimeroot)
+    tools_path = get_tools_path(root)
+
+    paths_to_add: List[str] = [root, tools_path]
+    if extra_paths:
+        paths_to_add.extend(str(p) for p in extra_paths)
+
+    _prepend_sys_path(paths_to_add)
+
+    if set_env:
+        os.environ["CIMEROOT"] = root
+
+    return root
+
+
+def _prepend_sys_path(paths: Sequence[str]) -> None:
+    """Prepend paths to the front of ``sys.path``, order-preserving and deduped.
+
+    Each path is converted to an absolute path and placed at the front of
+    ``sys.path`` such that ``paths[0]`` ends up at index 0, ``paths[1]`` at
+    index 1, and so on. If a path is already present it is removed first, so
+    the entry is repositioned to the front rather than duplicated.
+
+    Duplicate entries within ``paths`` itself are collapsed to the first
+    occurrence before any insertion, so positions are stable regardless of
+    how many times a path appears in the input. A ``UserWarning`` is issued
+    when duplicates are detected, since they may indicate a caller bug that
+    would silently affect import precedence.
+
+    Args:
+        paths: Paths to place at the front of ``sys.path``, highest priority
+            first.
+
+    Warns:
+        UserWarning: If ``paths`` contains duplicate entries.
+
+    Example:
+        Given ``sys.path = [A, B, C]`` and ``paths = [C, D, C]``, the
+        duplicate ``C`` is collapsed to its first occurrence, giving an
+        effective prepend list of ``[C, D]``.  The result is::
+
+            sys.path == [C, D, A, B]
+    """
+    # Deduplicate while preserving first-occurrence order.
+    abs_paths = [os.path.abspath(p) for p in paths]
+    unique = list(dict.fromkeys(abs_paths))
+    duplicates = [p for p in unique if abs_paths.count(p) > 1]
+    if duplicates:
+        warnings.warn(
+            f"Duplicate paths passed to _prepend_sys_path: {duplicates}. "
+            "First occurrence takes precedence; this may affect import order.",
+            UserWarning,
+            stacklevel=2,
+        )
+    for i, absp in enumerate(unique):
+        # Remove any existing entry so the path is repositioned, not duplicated.
+        if absp in sys.path:
+            sys.path.remove(absp)
+        sys.path.insert(i, absp)
+
+
+def get_tools_path(cimeroot: Optional[str] = None) -> str:
+    """Return the CIME/Tools directory path."""
+    root = cimeroot or find_cimeroot()
+    return os.path.join(root, "CIME", "Tools")
+
+
+def check_minimum_python_version(
+    major: int = 3, minor: int = 9, warn_only: bool = False
+) -> None:
+    """Check that the running Python meets minimum version requirements.
+
+    Consolidated from CIME/Tools/standard_script_setup.py.
+
+    Args:
+        major: Required major version.
+        minor: Required minimum minor version.
+        warn_only: If True, print warning instead of raising.
+
+    Raises:
+        RuntimeError: If version is insufficient and warn_only is False.
+    """
+    if sys.version_info >= (major, minor):
+        return
+    msg = (
+        f"Python {major}.{minor} is {'recommended' if warn_only else 'required'} "
+        f"to run CIME. You have {sys.version_info[0]}.{sys.version_info[1]}"
+    )
+    if warn_only:
+        print(msg + ".", file=sys.stderr)
+        return
+    raise RuntimeError(msg + " - please use a newer version of Python.")

CIME/core/exceptions.py

@@ -0,0 +1,81 @@
+"""
+Typed exception hierarchy for CIME.
+
+CIMEError inherits from both SystemExit and Exception so that:
+- Tracebacks are suppressed in normal usage (SystemExit behavior)
+- Exceptions are still catchable (Exception behavior)
+- Users can run with --debug to see full tracebacks
+
+All CIME-specific exceptions should inherit from CIMEError.
+"""
+
+
+class CIMEError(SystemExit, Exception):
+    """Base exception for all CIME errors.
+
+    Inherits from SystemExit to suppress tracebacks in normal usage,
+    and from Exception to remain catchable.
+    """
+
+
+class ConfigurationError(CIMEError):
+    """Invalid or missing configuration (XML files, env vars, etc.)."""
+
+
+class BuildError(CIMEError):
+    """Failure during model build."""
+
+
+class SubmitError(CIMEError):
+    """Failure during job submission."""
+
+
+class SystemTestError(CIMEError):
+    """Failure in a CIME system test."""
+
+
+class LockingError(CIMEError):
+    """Case locking or unlocking failure."""
+
+
+class ExternalCommandError(CIMEError):
+    """An external command (subprocess) failed.
+
+    Modeled after subprocess.CalledProcessError for consistency.
+
+    Attributes:
+        returncode: Exit code of the command.
+        cmd: The command that was run (string or list).
+        output: Standard output captured from the command, if any.
+        stderr: Standard error captured from the command, if any.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        returncode: int = 1,
+        cmd: str = "",
+        output: str = "",
+        stderr: str = "",
+        # Deprecated: use cmd instead
+        command: str = "",
+    ):
+        super().__init__(message)
+        self.returncode = returncode
+        # Support both 'cmd' (preferred) and 'command' (deprecated) for compatibility
+        self.cmd = cmd or command
+        self.output = output
+        self.stderr = stderr
+
+    @property
+    def command(self) -> str:
+        """Deprecated: Use cmd instead."""
+        return self.cmd
+
+
+class RetryableExternalCommandError(ExternalCommandError):
+    """An external command failed but may succeed on retry (transient error)."""
+
+
+class InputError(CIMEError):
+    """Invalid user input (bad arguments, missing required values)."""

CIME/get_tests.py

@@ -356,7 +356,7 @@ def infer_arch_from_tests(testargs):
     ('melvin', ['gnu'])
     >>> infer_arch_from_tests(["NCK.f19_g16.A.melvin_gnu9", "NCK.f19_g16.A.melvin_gnu"])
     ('melvin', ['gnu9', 'gnu'])
-    >>> infer_arch_from_tests(["NCK.f19_g16.A.melvin_gnu", "NCK.f19_g16.A.mappy_gnu"])
+    >>> infer_arch_from_tests(["NCK.f19_g16.A.melvin_gnu", "NCK.f19_g16.A.mappy_gnu"])  # doctest: +IGNORE_EXCEPTION_DETAIL
     Traceback (most recent call last):
         ...
     CIME.utils.CIMEError: ERROR: Must have consistent machine 'melvin' != 'mappy'