From 0a33367b7305985f8d28024ce4c9a1c05377a7c8 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 6 Nov 2025 16:21:35 +0100 Subject: [PATCH 1/9] ewoks install: support pixi, conda, poetry, uv and pipenv --- src/ewoks/__main__.py | 2 +- src/ewoks/_requirements/__init__.py | 77 ++++++++++ src/ewoks/_requirements/managers/__init__.py | 0 src/ewoks/_requirements/managers/conda.py | 30 ++++ src/ewoks/_requirements/managers/pip.py | 56 ++++++++ src/ewoks/_requirements/managers/pipenv.py | 36 +++++ src/ewoks/_requirements/managers/pixi.py | 29 ++++ src/ewoks/_requirements/managers/poetry.py | 22 +++ .../_requirements/managers/utils/__init__.py | 0 .../_requirements/managers/utils/base.py | 114 +++++++++++++++ .../_requirements/managers/utils/commands.py | 35 +++++ .../_requirements/managers/utils/detect.py | 91 ++++++++++++ .../_requirements/managers/utils/supported.py | 134 ++++++++++++++++++ src/ewoks/_requirements/managers/uv.py | 22 +++ src/ewoks/_requirements/metadata/__init__.py | 0 .../_requirements/metadata/gather/__init__.py | 15 ++ .../_requirements/metadata/gather/current.py | 35 +++++ .../metadata/gather/installed/__init__.py | 88 ++++++++++++ .../metadata/gather/installed/git.py | 73 ++++++++++ .../gather/last_resort.py} | 36 ++--- .../metadata/gather/pip_freeze.py | 11 ++ .../_requirements/metadata/gather/unknown.py | 31 ++++ src/ewoks/_requirements/metadata/parse.py | 20 +++ .../sanitize.py => metadata/pip_freeze.py} | 89 +++++++++++- src/ewoks/_requirements/models/__init__.py | 27 ++++ src/ewoks/_requirements/models/base.py | 39 +++++ src/ewoks/_requirements/models/conda.py | 13 ++ src/ewoks/_requirements/models/distro.py | 24 ++++ src/ewoks/_requirements/models/pip.py | 18 +++ src/ewoks/_requirements/models/pipenv.py | 18 +++ src/ewoks/_requirements/models/pixi.py | 13 ++ src/ewoks/_requirements/models/poetry.py | 14 ++ src/ewoks/_requirements/models/uv.py | 18 +++ src/ewoks/_requirements/pip/__init__.py | 1 - src/ewoks/_requirements/pip/install.py | 15 -- src/ewoks/bindings.py | 53 ++++--- src/ewoks/cli_utils/cli_install_utils.py | 6 +- src/ewoks/tests/requirements/__init__.py | 0 src/ewoks/tests/requirements/pip/__init__.py | 0 .../pip/test_freeze.py} | 14 +- .../requirements/pip/test_install_cli.py | 116 +++++++++++++++ src/ewoks/tests/requirements/utils.py | 13 ++ src/ewoks/tests/test_convert_cli.py | 22 +-- src/ewoks/tests/test_execute_cli.py | 22 +-- src/ewoks/tests/test_install_cli.py | 55 ------- 45 files changed, 1387 insertions(+), 160 deletions(-) create mode 100644 src/ewoks/_requirements/managers/__init__.py create mode 100644 src/ewoks/_requirements/managers/conda.py create mode 100644 src/ewoks/_requirements/managers/pip.py create mode 100644 src/ewoks/_requirements/managers/pipenv.py create mode 100644 src/ewoks/_requirements/managers/pixi.py create mode 100644 src/ewoks/_requirements/managers/poetry.py create mode 100644 src/ewoks/_requirements/managers/utils/__init__.py create mode 100644 src/ewoks/_requirements/managers/utils/base.py create mode 100644 src/ewoks/_requirements/managers/utils/commands.py create mode 100644 src/ewoks/_requirements/managers/utils/detect.py create mode 100644 src/ewoks/_requirements/managers/utils/supported.py create mode 100644 src/ewoks/_requirements/managers/uv.py create mode 100644 src/ewoks/_requirements/metadata/__init__.py create mode 100644 src/ewoks/_requirements/metadata/gather/__init__.py create mode 100644 src/ewoks/_requirements/metadata/gather/current.py create mode 100644 src/ewoks/_requirements/metadata/gather/installed/__init__.py create mode 100644 src/ewoks/_requirements/metadata/gather/installed/git.py rename src/ewoks/_requirements/{pip/extract.py => metadata/gather/last_resort.py} (59%) create mode 100644 src/ewoks/_requirements/metadata/gather/pip_freeze.py create mode 100644 src/ewoks/_requirements/metadata/gather/unknown.py create mode 100644 src/ewoks/_requirements/metadata/parse.py rename src/ewoks/_requirements/{pip/sanitize.py => metadata/pip_freeze.py} (63%) create mode 100644 src/ewoks/_requirements/models/__init__.py create mode 100644 src/ewoks/_requirements/models/base.py create mode 100644 src/ewoks/_requirements/models/conda.py create mode 100644 src/ewoks/_requirements/models/distro.py create mode 100644 src/ewoks/_requirements/models/pip.py create mode 100644 src/ewoks/_requirements/models/pipenv.py create mode 100644 src/ewoks/_requirements/models/pixi.py create mode 100644 src/ewoks/_requirements/models/poetry.py create mode 100644 src/ewoks/_requirements/models/uv.py delete mode 100644 src/ewoks/_requirements/pip/__init__.py delete mode 100644 src/ewoks/_requirements/pip/install.py create mode 100644 src/ewoks/tests/requirements/__init__.py create mode 100644 src/ewoks/tests/requirements/pip/__init__.py rename src/ewoks/tests/{test_requirements.py => requirements/pip/test_freeze.py} (87%) create mode 100644 src/ewoks/tests/requirements/pip/test_install_cli.py create mode 100644 src/ewoks/tests/requirements/utils.py delete mode 100644 src/ewoks/tests/test_install_cli.py diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 3436fa9..3a86e8f 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -159,7 +159,7 @@ def command_install( cli_install_utils.parse_install_arguments(cli_args, shell=shell) for workflow, graph in zip(cli_args.workflows, cli_args.graphs): try: - install_graph(graph, cli_args.yes, cli_args.python) + install_graph(graph, cli_args.yes, command=cli_args.manager_command) except CalledProcessError as e: print(f"Install failed for {workflow}: {e}") except AbortException: diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index afa8fed..24c7477 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -1 +1,78 @@ """Workflow requirements.""" + +import logging +from typing import Tuple + +from ewokscore.graph import TaskGraph + +from .managers.utils.base import BaseRequirements +from .managers.utils.detect import get_manager +from .metadata import parse +from .metadata.gather import last_resort + +logger = logging.getLogger(__file__) + + +def add_requirements(graph: TaskGraph, command: Tuple[str, ...] = tuple()) -> None: + """Add requirements to a workflow definition in-place.""" + manager = get_manager(command=command) + requirements = manager.gather_requirements() + graph.graph.graph["requirements"] = requirements.model_dump() + + +def get_requirements(graph: TaskGraph) -> BaseRequirements: + """Extract requirements from a workflow definition.""" + requirements = graph.graph.graph.get("requirements", None) + no_requirements = not requirements + + if no_requirements: + logger.warning( + "BaseRequirements field is empty. Trying to extract requirements automatically..." + ) + requirements = last_resort.last_resort_requirements(graph) + + requirements = parse.parse_requirements(requirements) + + if no_requirements: + logger.info(f"Extracted the following requirements: {requirements.__info__()}") + + return requirements + + +def install_requirements( + requirements: BaseRequirements, command: Tuple[str, ...] = tuple() +) -> None: + """Install workflow requirements.""" + manager = get_manager(manager_name=requirements.manager.name, command=command) + manager.install_requirements(requirements) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + import time + from pprint import pprint + + t0 = time.perf_counter() + + try: + manager = get_manager(manager_name=None) + requirements = manager.gather_requirements() + + print() + print("Model:") + pprint(requirements.model_dump()) + finally: + print("Freeze time:", time.perf_counter() - t0) + + pip_freeze = requirements.manager.freeze + + dists_freeze = manager.freeze_distributions(requirements) + dists_freeze = [s for s in dists_freeze if not s.startswith("#")] + + print() + print("pip freeze has these extra's:") + pprint(set(pip_freeze) - set(dists_freeze)) + + print() + print("native freeze has these extra's:") + pprint(set(dists_freeze) - set(pip_freeze)) diff --git a/src/ewoks/_requirements/managers/__init__.py b/src/ewoks/_requirements/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/_requirements/managers/conda.py b/src/ewoks/_requirements/managers/conda.py new file mode 100644 index 0000000..a973405 --- /dev/null +++ b/src/ewoks/_requirements/managers/conda.py @@ -0,0 +1,30 @@ +import logging + +import yaml + +from ..metadata.gather import gather_requirements +from ..models.conda import CondaRequirements +from .utils.base import BaseManager + +logger = logging.getLogger(__name__) + + +class CondaManager(BaseManager): + NAME = "conda" + + def _gather_requirements(self, manager_version: str) -> CondaRequirements: + output = self._check_output("env", "export") + environment = yaml.safe_load(output) + environment.pop("name", None) + environment.pop("prefix", None) + + return gather_requirements( + manager_name="conda", + manager_version=manager_version, + environment=environment, + ) + + def install_requirements(self, requirements: CondaRequirements) -> None: + text = yaml.safe_dump(requirements.environment) + with self._temporary_file(text, ".yml") as tmp_path: + self._check_call("env", "update", "-f", tmp_path) diff --git a/src/ewoks/_requirements/managers/pip.py b/src/ewoks/_requirements/managers/pip.py new file mode 100644 index 0000000..b2100b6 --- /dev/null +++ b/src/ewoks/_requirements/managers/pip.py @@ -0,0 +1,56 @@ +import logging +from typing import List + +from ..metadata import pip_freeze +from ..metadata.gather import gather_requirements +from ..models.pip import PipRequirements +from .utils.base import BaseManager + +logger = logging.getLogger(__name__) + + +class PipManager(BaseManager): + NAME = "pip" + + def _gather_requirements(self, manager_version: str) -> PipRequirements: + freeze_output = self._check_output("freeze").strip().splitlines() + return gather_requirements( + manager_name=self.NAME, + manager_version=manager_version, + freeze=freeze_output, + ) + + def _install_requirements(self, requirements: PipRequirements) -> None: + freeze = requirements.manager.freeze + + if freeze: + arguments = self._arguments(freeze) + try: + self._check_call("install", "--no-cache-dir", *arguments) + return + except Exception: + if not requirements.distributions: + raise + + freeze = self.freeze_distributions(requirements) + if freeze: + arguments = self._arguments(freeze) + self._check_call("install", "--no-cache-dir", *arguments) + return + + raise ValueError("No distibutions provided to install") + + def freeze_distributions(self, requirements: PipRequirements) -> List[str]: + freeze = [] + for dist in requirements.distributions: + lines, warnings = pip_freeze.freeze_distribution(dist) + for warning in warnings: + logger.warning(warning) + freeze.extend(lines) + return freeze + + def _arguments(self, freeze: List[str]) -> List[str]: + arguments, warnings = pip_freeze.sanitize_freeze(freeze) + for warning in warnings: + logger.warning(warning) + return arguments diff --git a/src/ewoks/_requirements/managers/pipenv.py b/src/ewoks/_requirements/managers/pipenv.py new file mode 100644 index 0000000..4f5a161 --- /dev/null +++ b/src/ewoks/_requirements/managers/pipenv.py @@ -0,0 +1,36 @@ +import json + +from ..metadata.gather import gather_requirements +from ..models.pipenv import PipenvRequirements +from .utils.base import BaseManager + + +class PipenvManager(BaseManager): + NAME = "pipenv" + + def _gather_requirements(self, manager_version: str) -> PipenvRequirements: + output = self._check_output("lock", "--requirements") + requirements = output.strip().splitlines() + + return gather_requirements( + manager_name=self.NAME, + manager_version=manager_version, + requirements=requirements, + ) + + def _install_requirements(self, requirements: PipenvRequirements) -> None: + lock_data = { + "_meta": {"hash": {"sha256": "dummy"}}, # minimal metadata + "default": { + pkg.split("==")[0]: {"version": pkg.split("==")[1]} + for pkg in requirements.requirements + }, + "develop": { + pkg.split("==")[0]: {"version": pkg.split("==")[1]} + for pkg in getattr(requirements, "dev_requirements", []) + }, + } + text = json.dumps(lock_data, indent=2) + + with self._temporary_file(text, ".lock") as tmp_path: + self._check_call("sync", "--ignore-pipfile", "-f", tmp_path) diff --git a/src/ewoks/_requirements/managers/pixi.py b/src/ewoks/_requirements/managers/pixi.py new file mode 100644 index 0000000..04b79ae --- /dev/null +++ b/src/ewoks/_requirements/managers/pixi.py @@ -0,0 +1,29 @@ +import os + +from ..metadata.gather import gather_requirements +from ..models.pixi import PixiRequirements +from .utils.base import BaseManager + + +class PixiManager(BaseManager): + NAME = "pixi" + + def _gather_requirements(self, manager_version: str) -> PixiRequirements: + if os.path.exists("pixi.lock"): + with open("pixi.lock", "r", encoding="utf-8") as f: + lock_content = f.read() + elif os.path.exists("pixi.toml"): + with open("pixi.toml", "r", encoding="utf-8") as f: + lock_content = f.read() + else: + raise RuntimeError("No pixi.lock or pixi.toml file found") + + return gather_requirements( + manager_name=self.NAME, + manager_version=manager_version, + lockfile=lock_content, + ) + + def _install_requirements(self, requirements: PixiRequirements) -> None: + with self._temporary_file(requirements.lockfile, ".lock") as tmp_path: + self._check_call("install", cwd=os.path.dirname(tmp_path)) diff --git a/src/ewoks/_requirements/managers/poetry.py b/src/ewoks/_requirements/managers/poetry.py new file mode 100644 index 0000000..6260295 --- /dev/null +++ b/src/ewoks/_requirements/managers/poetry.py @@ -0,0 +1,22 @@ +from ..metadata.gather import gather_requirements +from ..models.poetry import PoetryRequirements +from .utils.base import BaseManager + + +class PoetryManager(BaseManager): + NAME = "poetry" + + def _gather_requirements(self, manager_version: str) -> PoetryRequirements: + output = self._check_output("export", "--without-hashes") + requirements = output.strip().splitlines() + + return gather_requirements( + manager_name=self.NAME, + manager_version=manager_version, + requirements=requirements, + ) + + def _install_requirements(self, requirements: PoetryRequirements) -> None: + text = "\n".join(requirements.requirements) + with self._temporary_file(text, ".txt") as tmp_path: + self._check_call("add", "--lock", "--file", tmp_path) diff --git a/src/ewoks/_requirements/managers/utils/__init__.py b/src/ewoks/_requirements/managers/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/_requirements/managers/utils/base.py b/src/ewoks/_requirements/managers/utils/base.py new file mode 100644 index 0000000..b0080b5 --- /dev/null +++ b/src/ewoks/_requirements/managers/utils/base.py @@ -0,0 +1,114 @@ +import logging +import os +import subprocess +import tempfile +from abc import abstractmethod +from contextlib import contextmanager +from typing import Generator +from typing import List +from typing import Optional + +from ...models.base import BaseRequirements +from .commands import get_manager_command + +logger = logging.getLogger(__name__) + + +class BaseManager: + """Defines the interface all package managers must implement. + + If `MyManager` is an impementation of this interface then + to get the Ewoks workflow requirements like this: + + .. code-block:: python + + manager = MyManager() + requirements = manager.gather_requirements() + + Ewoks workflow requirements can be installed like this: + + .. code-block:: python + + manager = MyManager() + install_requirements.install_requirements(requirements) + """ + + NAME = NotImplemented + + def __init__(self, *command: str) -> None: + if not command: + command = get_manager_command(self.NAME) + self._cmd_args = command + + def gather_requirements(self) -> Optional[BaseRequirements]: + """Return requirements generated from the current python environment.""" + from .supported import get_supported_managers + + manager_version = get_supported_managers()[self.NAME].version + if not manager_version: + raise RuntimeError(f"{self.NAME!r} is not installed") + + try: + return self._gather_requirements(manager_version) + except Exception as ex: + logger.error( + "%s: failed to generate requirements (%s)", type(self).__name__, ex + ) + return None + + def install_requirements(self, requirements: BaseRequirements) -> None: + """Install requirements into the current python environment.""" + try: + return self._install_requirements(requirements) + except Exception as ex: + logger.error( + "%s: failed to install requirements (%s)", type(self).__name__, ex + ) + raise + + @abstractmethod + def _gather_requirements(self, manager_version: str) -> BaseRequirements: + pass + + @abstractmethod + def _install_requirements(self, requirements: BaseRequirements) -> None: + pass + + def _check_output(self, *args) -> str: + return _check_output([*self._cmd_args, *args]) + + def _check_call(self, *args, raw: bool = False) -> int: + if raw: + return _check_call([*args]) + return _check_call([*self._cmd_args, *args]) + + @contextmanager + def _temporary_file(self, text: str, suffix: str) -> Generator[str, None, None]: + tmp_path = None + try: + with tempfile.NamedTemporaryFile("w", suffix=suffix, delete=False) as tmp: + tmp.write(text) + tmp_path = tmp.name + + yield tmp_path + + finally: + if tmp_path: + try: + os.remove(tmp_path) + except OSError: + logger.debug("Could not delete temporary file: %s", tmp_path) + + +def _check_output(args: List[str]) -> str: + try: + return subprocess.check_output(args, text=True) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex + + +def _check_call(args: List[str]) -> int: + try: + return subprocess.check_call(args) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex diff --git a/src/ewoks/_requirements/managers/utils/commands.py b/src/ewoks/_requirements/managers/utils/commands.py new file mode 100644 index 0000000..b78b22a --- /dev/null +++ b/src/ewoks/_requirements/managers/utils/commands.py @@ -0,0 +1,35 @@ +import subprocess +import sys +from functools import lru_cache +from typing import Dict +from typing import Tuple + + +def get_manager_command(manager_name: str) -> Tuple[str, ...]: + return manager_commands()[manager_name] + + +@lru_cache +def manager_commands() -> Dict[str, Tuple[str, ...]]: + return dict( + pip=(sys.executable, "-m", "pip"), + poetry=(sys.executable, "-m", "poetry"), + pipenv=(sys.executable, "-m", "pipenv"), + conda=_get_conda_command(), + pixi=("pixi",), + uv=("uv",), + ) + + +def _get_conda_command() -> Tuple[str, ...]: + try: + _ = subprocess.check_output(["mamba", "--version"], text=True) + return ("mamba",) + except Exception: + pass + try: + _ = subprocess.check_output(["micromamba", "--version"], text=True) + return ("micromamba",) + except Exception: + pass + return ("conda",) diff --git a/src/ewoks/_requirements/managers/utils/detect.py b/src/ewoks/_requirements/managers/utils/detect.py new file mode 100644 index 0000000..1f18ae1 --- /dev/null +++ b/src/ewoks/_requirements/managers/utils/detect.py @@ -0,0 +1,91 @@ +import logging +from collections import Counter +from functools import lru_cache +from typing import Dict +from typing import Optional +from typing import Tuple + +from ...metadata.gather import installed +from .base import BaseManager +from .supported import ManagerInfo +from .supported import get_supported_managers + +logger = logging.getLogger(__name__) + + +def get_manager( + manager_name: Optional[str] = None, + command: Tuple[str, ...] = tuple(), +) -> BaseManager: + """ + :raise ValueError: package manager not support + :raise RuntimeError: no package manager available + """ + if manager_name: + managers = get_supported_managers() + + info = managers.get(manager_name) + if info is None: + raise ValueError(f"Package manager {manager_name!r} is not supported") + + return info.manager_type(*command) + + info = _detect_manager() + if info is None: + raise RuntimeError("No known package manager installed or available") + + return info.manager_type(*command) + + +def _detect_manager() -> Optional[ManagerInfo]: + # Available package managers + available_managers = { + name: info for name, info in get_supported_managers().items() if info.version + } + if not available_managers: + return None + + # Select the active manager with the highest priority + active_managers = { + name: info for name, info in available_managers.items() if info.is_active + } + if active_managers: + name = max(active_managers, key=lambda name: active_managers[name].priority) + info = active_managers[name] + logger.debug( + "Detected active %r package manager\n available = %s\n active = %s", + name, + list(available_managers), + list(active_managers), + ) + return info + + # Infer most likely package manager + counts = _installer_distribution_count() + if set(counts) & set(available_managers): + # Use the number of installed distibutions as the score + crit = "distribution count" + scores = {name: counts.get(name, -1) for name in available_managers} + else: + # Use the package manager priority as the score + crit = "priority" + scores = {name: info.priority for name, info in available_managers.items()} + + name = max(scores, key=scores.get) + logger.debug( + "Package manager selection based on %s\n %s", + crit, + "\n ".join( + f"{k} = {v} {'(SELECTED)' if k == name else ''}" for k, v in scores.items() + ), + ) + return available_managers[name] + + +@lru_cache +def _installer_distribution_count() -> Dict[str, int]: + counts: Counter = Counter() + for dist in installed.distributions(): + if dist.installer: + counts[dist.installer] += 1 + return dict(counts) diff --git a/src/ewoks/_requirements/managers/utils/supported.py b/src/ewoks/_requirements/managers/utils/supported.py new file mode 100644 index 0000000..90c572d --- /dev/null +++ b/src/ewoks/_requirements/managers/utils/supported.py @@ -0,0 +1,134 @@ +import importlib.metadata +import os +import subprocess +import sys +from functools import lru_cache +from typing import Dict +from typing import NamedTuple +from typing import Optional +from typing import Type + +from ..pip import PipManager +from .base import BaseManager + +# from ..conda import CondaManager +# from ..pipenv import PipenvManager +# from ..pixi import PixiManager +# from ..poetry import PoetryManager +# from ..uv import UvManager + + +class ManagerInfo(NamedTuple): + manager_type: Type[BaseManager] + version: Optional[str] + is_active: bool + priority: int + + +@lru_cache +def get_supported_managers() -> Dict[str, ManagerInfo]: + return dict( + pip=ManagerInfo( + manager_type=PipManager, + version=_get_pip_version(), + is_active=False, + priority=0, + ), + # uv=ManagerInfo( + # manager_type=UvManager, + # version=_get_uv_version(), + # is_active=False, + # priority=1, + # ), + # poetry=ManagerInfo( + # manager_type=PoetryManager, + # version=_get_poetry_version(), + # is_active=_is_poetry_active(), + # priority=2, + # ), + # pipenv=ManagerInfo( + # manager_type=PipenvManager, + # version=_get_pipenv_version(), + # is_active=_is_pipenv_active(), + # priority=3, + # ), + # conda=ManagerInfo( + # manager_type=CondaManager, + # version=_get_conda_version(), + # is_active=_is_conda_active(), + # priority=4, + # ), + # pixi=ManagerInfo( + # manager_type=PixiManager, + # version=_get_pixi_version(), + # is_active=_is_pixi_active(), + # priority=5, + # ), + ) + + +def _get_pip_version() -> Optional[str]: + try: + return importlib.metadata.version("pip") + except importlib.metadata.PackageNotFoundError: + return None + + +def _get_poetry_version() -> Optional[str]: + try: + return importlib.metadata.version("poetry") + except importlib.metadata.PackageNotFoundError: + return None + + +def _get_pipenv_version() -> Optional[str]: + try: + return importlib.metadata.version("pipenv") + except importlib.metadata.PackageNotFoundError: + return None + + +def _get_conda_version() -> Optional[str]: + try: + output = subprocess.check_output(["conda", "--version"], text=True) + return output.strip().split(" ")[-1] + except Exception: + return None + + +def _get_pixi_version() -> Optional[str]: + try: + output = subprocess.check_output(["pixi", "--version"], text=True) + return output.strip().split(" ")[-1] + except Exception: + return None + + +def _get_uv_version() -> Optional[str]: + try: + output = subprocess.check_output(["uv", "--version"], text=True) + return output.strip().split(" ")[-1] + except Exception: + return None + + +def _is_conda_active() -> bool: + return "CONDA_PREFIX" in os.environ or os.path.exists( + os.path.join(sys.prefix, "conda-meta") + ) + + +def _is_pixi_active() -> bool: + return "PIXI_PROJECT_ROOT" in os.environ + + +def _is_poetry_active() -> bool: + return "POETRY_ACTIVE" in os.environ + + +def _is_pipenv_active() -> bool: + return "PIPENV_ACTIVE" in os.environ + + +def _in_virtual_environment() -> bool: + return sys.prefix != sys.base_prefix diff --git a/src/ewoks/_requirements/managers/uv.py b/src/ewoks/_requirements/managers/uv.py new file mode 100644 index 0000000..cd5dcc9 --- /dev/null +++ b/src/ewoks/_requirements/managers/uv.py @@ -0,0 +1,22 @@ +from ..metadata.gather import gather_requirements +from ..models.uv import UvRequirements +from .utils.base import BaseManager + + +class UvManager(BaseManager): + NAME = "uv" + + def _gather_requirements(self, manager_version: str) -> UvRequirements: + output = self._check_output("pip", "freeze") + requirements = output.strip().splitlines() + + return gather_requirements( + manager_name=self.NAME, + manager_version=manager_version, + requirements=requirements, + ) + + def _install_requirements(self, requirements: UvRequirements) -> None: + text = "\n".join(requirements.requirements) + with self._temporary_file(text, ".txt") as tmp_path: + self._check_call("add", "-r", tmp_path) diff --git a/src/ewoks/_requirements/metadata/__init__.py b/src/ewoks/_requirements/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/_requirements/metadata/gather/__init__.py b/src/ewoks/_requirements/metadata/gather/__init__.py new file mode 100644 index 0000000..18d1c90 --- /dev/null +++ b/src/ewoks/_requirements/metadata/gather/__init__.py @@ -0,0 +1,15 @@ +from ...models import get_model +from ...models.base import BaseRequirements +from .current import current_requirements + + +def gather_requirements( + manager_name: str, manager_version: str, **parameters +) -> BaseRequirements: + """ + :raises ValueError: unknown package manager + :raises ValidationError: wrong parameters + """ + model_cls = get_model(manager_name) + manager = dict(name=manager_name, version=manager_version, **parameters) + return model_cls(manager=manager, **current_requirements()) diff --git a/src/ewoks/_requirements/metadata/gather/current.py b/src/ewoks/_requirements/metadata/gather/current.py new file mode 100644 index 0000000..3723765 --- /dev/null +++ b/src/ewoks/_requirements/metadata/gather/current.py @@ -0,0 +1,35 @@ +import platform +from functools import lru_cache +from typing import Any +from typing import Dict + +from . import installed + + +@lru_cache +def current_requirements() -> Dict[str, Any]: + return dict( + system=_system_metadata(), + python=_python_metadata(), + distributions=installed.distributions(), + ) + + +def _system_metadata() -> Dict[str, Any]: + uname = platform.uname() + return dict( + system=uname.system, + release=uname.release, + version=uname.version, + machine=uname.machine, + processor=uname.processor, + ) + + +def _python_metadata() -> Dict[str, Any]: + return dict( + version=platform.python_version(), + implementation=platform.python_implementation(), + compiler=platform.python_compiler(), + build=", ".join(platform.python_build()), + ) diff --git a/src/ewoks/_requirements/metadata/gather/installed/__init__.py b/src/ewoks/_requirements/metadata/gather/installed/__init__.py new file mode 100644 index 0000000..61c5dc6 --- /dev/null +++ b/src/ewoks/_requirements/metadata/gather/installed/__init__.py @@ -0,0 +1,88 @@ +import json +import logging +from functools import lru_cache +from importlib import metadata +from pathlib import Path +from typing import List + +from ....models.distro import ArchiveInfo +from ....models.distro import Distribution +from ....models.distro import GitInfo +from . import git + +logger = logging.getLogger(__name__) + + +@lru_cache() +def distributions() -> List[Distribution]: + """ + Return installed distributions. + """ + return [_distribution_from_metadata(dist) for dist in metadata.distributions()] + + +def _distribution_from_metadata(dist: metadata.Distribution) -> Distribution: + name = dist.metadata["Name"] + version = dist.version + + try: + installer = dist.read_text("INSTALLER") or "" + installer = installer.strip() + if not installer: + installer = None + except Exception: + installer = None + + # PEP 610: Direct URL Origin of installed distributions + # https://packaging.python.org/en/latest/specifications/direct-url-data-structure/ + git_info = None + archive_info = None + try: + direct_url_text = dist.read_text("direct_url.json") + direct_url = json.loads(direct_url_text) if direct_url_text else {} + except Exception: + direct_url = {} + + url = direct_url.get("url", "") + has_info = False + + if not has_info and "vcs_info" in direct_url: + vcs_info = direct_url["vcs_info"] + if vcs_info.get("vcs") == "git": + commit_id = vcs_info.get("commit_id") + if commit_id: + git_info = GitInfo( + commit=commit_id, remote=url, uncomitted_changes=False + ) + has_info = True + + if not has_info and "dir_info" in direct_url: + if url.startswith("file://") or "://" not in url: + path = Path(url.replace("file://", "")) + if path.exists(): + git_info = git.git_info_from_path(path) + has_info = git_info is not None + + if not has_info and "archive_info" in direct_url: + if not url.startswith("file://") and "://" in url: + archive = direct_url["archive_info"] + + if "hashes" in archive: + hashes = archive["hashes"] + elif "hash" in archive: + # deprecated + algo, value = archive["hash"].split("=", 1) + hashes = {algo: value} + else: + hashes = {} + + archive_info = ArchiveInfo(url=url, hashes=hashes) + has_info = True + + return Distribution( + name=name, + version=version, + git=git_info, + archive=archive_info, + installer=installer, + ) diff --git a/src/ewoks/_requirements/metadata/gather/installed/git.py b/src/ewoks/_requirements/metadata/gather/installed/git.py new file mode 100644 index 0000000..918134e --- /dev/null +++ b/src/ewoks/_requirements/metadata/gather/installed/git.py @@ -0,0 +1,73 @@ +import subprocess +from pathlib import Path +from typing import List +from typing import Optional + +from ....models.distro import GitInfo + + +def git_info_from_path(path: Path) -> Optional[GitInfo]: + """ + Return GitInfo for a local path, or None if not a git repository. + """ + try: + commit = _git(["rev-parse", "HEAD"], path) + except Exception: + return None + + try: + uncomitted_changes = bool(_git(["status", "--porcelain"], path)) + except Exception: + uncomitted_changes = False + + remote_name = _find_remote_for_commit(path, commit) + if remote_name: + try: + remote_url = _git(["remote", "get-url", remote_name], path) + except Exception: + remote_url = None + else: + remote_url = None + + return GitInfo( + commit=commit, remote=remote_url, uncomitted_changes=uncomitted_changes + ) + + +def _find_remote_for_commit(path: Path, commit: str) -> Optional[str]: + """ + Return the name of a remote that contains the given commit. + """ + try: + remotes = _git_lines(["remote"], path) + except Exception: + return None + + if not remotes: + return None + + if "origin" in remotes: + remotes = ["origin"] + [r for r in remotes if r != "origin"] + + try: + branches = _git_lines(["branch", "-r", "--contains", commit], path) + except Exception: + return None + + for remote in remotes: + prefix = f"{remote}/" + if any(b.startswith(prefix) for b in branches): + return remote + + return None + + +def _git(cmd: List[str], repo: Path) -> str: + return subprocess.check_output( + ["git"] + cmd, cwd=repo, stderr=subprocess.DEVNULL, text=True + ).strip() + + +def _git_lines(cmd: List[str], repo: Path) -> List[str]: + out = _git(cmd, repo) + return [line.strip() for line in out.splitlines() if line.strip()] diff --git a/src/ewoks/_requirements/pip/extract.py b/src/ewoks/_requirements/metadata/gather/last_resort.py similarity index 59% rename from src/ewoks/_requirements/pip/extract.py rename to src/ewoks/_requirements/metadata/gather/last_resort.py index 2cdce7d..e5a8890 100644 --- a/src/ewoks/_requirements/pip/extract.py +++ b/src/ewoks/_requirements/metadata/gather/last_resort.py @@ -1,15 +1,19 @@ import logging -import subprocess -import sys -from typing import List +from typing import Any +from typing import Dict from ewokscore.graph import TaskGraph -logger = logging.getLogger(__file__) +from . import pip_freeze +logger = logging.getLogger(__name__) -def extract_pip_requirements(graph: TaskGraph) -> List[str]: - imports: set[str] = set() + +def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: + """Last resort when installing a workflow that does not have requirements: + guess the requirements from the workflow nodes. + """ + freeze: set[str] = set() for node_id, node in graph.graph.nodes.items(): task_identifier = node["task_identifier"] @@ -23,13 +27,13 @@ def extract_pip_requirements(graph: TaskGraph) -> List[str]: ) continue - imports.add(package) + freeze.add(package) elif task_type == "notebook": logger.warning( f"Requirement extraction may be incomplete for node {node_id}: {task_type} is only partially supported." ) - imports.add("ewokscore[notebook]") + freeze.add("ewokscore[notebook]") elif task_type == "script": logger.warning( @@ -40,18 +44,4 @@ def extract_pip_requirements(graph: TaskGraph) -> List[str]: f"Could not extract requirements for node {node_id}: unsupported task type {task_type}." ) - return list(imports) - - -def add_current_env_pip_requirements(graph: TaskGraph) -> TaskGraph: - try: - freeze_output = subprocess.check_output( - [sys.executable, "-m", "pip", "freeze"], text=True - ) - except subprocess.CalledProcessError as ex: - logger.warning("Cannot generate list of requirements with 'pip' (%s).", ex) - return graph - - requirements = freeze_output.strip().split("\n") - graph.graph.graph["requirements"] = requirements - return graph + return pip_freeze.pip_freeze_requirements(list(freeze)) diff --git a/src/ewoks/_requirements/metadata/gather/pip_freeze.py b/src/ewoks/_requirements/metadata/gather/pip_freeze.py new file mode 100644 index 0000000..9f0e2bd --- /dev/null +++ b/src/ewoks/_requirements/metadata/gather/pip_freeze.py @@ -0,0 +1,11 @@ +from typing import Any +from typing import Dict +from typing import List + +from . import unknown + + +def pip_freeze_requirements(freeze: List[str]) -> Dict[str, Any]: + metadata = unknown.unknown_requirements() + metadata["manager"] = dict(name="pip", version="", freeze=freeze) + return metadata diff --git a/src/ewoks/_requirements/metadata/gather/unknown.py b/src/ewoks/_requirements/metadata/gather/unknown.py new file mode 100644 index 0000000..0c47d2b --- /dev/null +++ b/src/ewoks/_requirements/metadata/gather/unknown.py @@ -0,0 +1,31 @@ +from functools import lru_cache +from typing import Any +from typing import Dict + + +@lru_cache +def unknown_requirements() -> Dict[str, Any]: + return dict( + system=_unknown_system_metadata(), + python=_unknown_python_metadata(), + distributions=list(), + ) + + +def _unknown_system_metadata() -> Dict[str, Any]: + return dict( + system="", + release="", + version="", + machine="", + processor="", + ) + + +def _unknown_python_metadata() -> Dict[str, Any]: + return dict( + version="", + implementation="", + compiler="", + build="", + ) diff --git a/src/ewoks/_requirements/metadata/parse.py b/src/ewoks/_requirements/metadata/parse.py new file mode 100644 index 0000000..89ccea7 --- /dev/null +++ b/src/ewoks/_requirements/metadata/parse.py @@ -0,0 +1,20 @@ +from typing import List +from typing import Union + +from ..models import get_model +from ..models.base import BaseRequirements +from .gather.pip_freeze import pip_freeze_requirements + + +def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: + if isinstance(requirements, list): + # Legacy 'pip freeze' list + requirements = pip_freeze_requirements(requirements) + + if not isinstance(requirements, dict): + raise TypeError( + f"Graph requirements must be a list or dictionary (type: {type(requirements)})" + ) + + manager = requirements.get("manager", dict()).get("name") + return get_model(manager)(**requirements) diff --git a/src/ewoks/_requirements/pip/sanitize.py b/src/ewoks/_requirements/metadata/pip_freeze.py similarity index 63% rename from src/ewoks/_requirements/pip/sanitize.py rename to src/ewoks/_requirements/metadata/pip_freeze.py index b71d23e..032d92e 100644 --- a/src/ewoks/_requirements/pip/sanitize.py +++ b/src/ewoks/_requirements/metadata/pip_freeze.py @@ -11,10 +11,95 @@ from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement +from ..models.distro import Distribution -def sanitize_requirements(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: + +def freeze_distribution( + dist: Distribution, +) -> Tuple[List[str], List[str]]: + """ + Return the pip freeze of the distribution with associated warnings regarding reproducibility. + """ + lines = [] + warnings = [] + + if dist.version: + pypi_req = f"{dist.name}=={dist.version}" + else: + pypi_req = dist.name + + if dist.archive: + archive_req = f"{dist.name} @ {dist.archive.url}" + + if dist.archive.hashes: + for algo, value in dist.archive.hashes.items(): + archive_req += f" --hash={algo}:{value}" + + lines.append(archive_req) + return lines, warnings + + if dist.git: + warning_fmt = "Non-reproducible Ewoks workflow requirement: {}@{} {}" + + if dist.git.remote: + url = _normalize_git_url(dist.git.remote) + git_req = f"{dist.name} @ {url}@{dist.git.commit}" + else: + git_req = pypi_req + warning = warning_fmt.format( + dist.name, dist.git.commit, "has no remote repository" + ) + warnings.append(warning) + lines.append(f"# {warning}") + + if dist.git.uncomitted_changes: + warning = warning_fmt.format( + dist.name, dist.git.commit, "has uncommited changes" + ) + warnings.append(warning) + lines.append(f"# {warning}") + + lines.append(git_req) + return lines, warnings + + lines.append(pypi_req) + return lines, warnings + + +def _normalize_git_url(url: str, preserve_ssh: bool = False) -> str: + """ + Return a PEP 508-compatible VCS URL, prefixed with 'git+'. + """ + # SCP-like SSH syntax: git@host:group/repo.git + if url.startswith("git@"): + host, path = url[len("git@") :].split(":", 1) + if preserve_ssh: + return f"git+ssh://git@{host}/{path}" + return f"git+https://{host}/{path}" + + # Explicit SSH URL + if url.startswith("ssh://"): + if preserve_ssh: + return f"git+{url}" + + # ssh://git@host/group/repo.git -> https://host/group/repo.git + without_scheme = url[len("ssh://") :] + if without_scheme.startswith("git@"): + without_scheme = without_scheme[len("git@") :] + return f"git+https://{without_scheme}" + + # HTTP(S) + if url.startswith("http://") or url.startswith("https://"): + return f"git+{url}" + + # Unknown / already normalized + return url + + +def sanitize_freeze(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: """Sanitize a list of requirements coming from 'pip freeze'. - Returns a sanitized with warnings regarding applied changes. + + Returns a sanitized list with warnings regarding applied changes. """ sanitized = [] warnings = [] diff --git a/src/ewoks/_requirements/models/__init__.py b/src/ewoks/_requirements/models/__init__.py new file mode 100644 index 0000000..7365aee --- /dev/null +++ b/src/ewoks/_requirements/models/__init__.py @@ -0,0 +1,27 @@ +from typing import Type + +from ..models.base import BaseRequirements +from ..models.conda import CondaRequirements +from ..models.pip import PipRequirements +from ..models.pipenv import PipenvRequirements +from ..models.pixi import PixiRequirements +from ..models.poetry import PoetryRequirements +from ..models.uv import UvRequirements + +_REQUIREMENT_MODELS = { + "pip": PipRequirements, + "poetry": PoetryRequirements, + "pipenv": PipenvRequirements, + "uv": UvRequirements, + "conda": CondaRequirements, + "pixi": PixiRequirements, +} + + +def get_model(manager_name: str) -> Type[BaseRequirements]: + """ + :raises ValueError: unknown package manager + """ + if manager_name not in _REQUIREMENT_MODELS: + raise ValueError(f"{manager_name!r} is not a valid package manager") + return _REQUIREMENT_MODELS[manager_name] diff --git a/src/ewoks/_requirements/models/base.py b/src/ewoks/_requirements/models/base.py new file mode 100644 index 0000000..0305bc2 --- /dev/null +++ b/src/ewoks/_requirements/models/base.py @@ -0,0 +1,39 @@ +from typing import List + +from pydantic import BaseModel + +from .distro import Distribution + + +class SystemInfo(BaseModel): + system: str + release: str + version: str + machine: str + processor: str + + +class PythonInfo(BaseModel): + version: str + implementation: str + compiler: str + build: str + + +class BaseManagerInfo(BaseModel): + name: str + version: str + + +class BaseRequirements(BaseModel): + system: SystemInfo + python: PythonInfo + distributions: List[Distribution] + manager: BaseManagerInfo + + def __info__(self) -> str: + return ( + f"Manager: {self.manager.name} ({self.manager.version}) " + f"python={self.python.version}) " + f"distributions={len(self.distributions)}" + ) diff --git a/src/ewoks/_requirements/models/conda.py b/src/ewoks/_requirements/models/conda.py new file mode 100644 index 0000000..4c936ef --- /dev/null +++ b/src/ewoks/_requirements/models/conda.py @@ -0,0 +1,13 @@ +from typing import Literal + +from .base import BaseManagerInfo +from .base import BaseRequirements + + +class CondaManagerInfo(BaseManagerInfo): + name: Literal["conda"] = "conda" + + +class CondaRequirements(BaseRequirements): + manager: CondaManagerInfo + environment: dict diff --git a/src/ewoks/_requirements/models/distro.py b/src/ewoks/_requirements/models/distro.py new file mode 100644 index 0000000..f394c82 --- /dev/null +++ b/src/ewoks/_requirements/models/distro.py @@ -0,0 +1,24 @@ +from typing import Dict +from typing import Optional + +from pydantic import BaseModel +from pydantic import Field + + +class GitInfo(BaseModel): + commit: str + remote: Optional[str] = None + uncomitted_changes: bool = Field(default=False, description="Uncommited changes") + + +class ArchiveInfo(BaseModel): + url: str + hashes: Dict[str, str] + + +class Distribution(BaseModel): + name: str + version: str + git: Optional[GitInfo] = None + archive: Optional[ArchiveInfo] = None + installer: Optional[str] = None diff --git a/src/ewoks/_requirements/models/pip.py b/src/ewoks/_requirements/models/pip.py new file mode 100644 index 0000000..acd5333 --- /dev/null +++ b/src/ewoks/_requirements/models/pip.py @@ -0,0 +1,18 @@ +from typing import List +from typing import Literal + +from .base import BaseManagerInfo +from .base import BaseRequirements + + +class PipManagerInfo(BaseManagerInfo): + name: Literal["pip"] = "pip" + freeze: List[str] + + +class PipRequirements(BaseRequirements): + manager: PipManagerInfo + + def __info__(self) -> str: + freeze = "\n ".join(self.manager.freeze) + return f"{super().__info__()}\nRequirements:\n {freeze}" diff --git a/src/ewoks/_requirements/models/pipenv.py b/src/ewoks/_requirements/models/pipenv.py new file mode 100644 index 0000000..8043cc8 --- /dev/null +++ b/src/ewoks/_requirements/models/pipenv.py @@ -0,0 +1,18 @@ +from typing import List +from typing import Literal + +from .base import BaseManagerInfo +from .base import BaseRequirements + + +class PipenvManagerInfo(BaseManagerInfo): + name: Literal["pipenv"] = "pipenv" + requirements: List[str] + + +class PipenvRequirements(BaseRequirements): + manager: PipenvManagerInfo + + def __info__(self) -> str: + requirements = "\n ".join(self.manager.requirements) + return f"{super().__info__()}\nRequirements:\n {requirements}" diff --git a/src/ewoks/_requirements/models/pixi.py b/src/ewoks/_requirements/models/pixi.py new file mode 100644 index 0000000..7d2a0fe --- /dev/null +++ b/src/ewoks/_requirements/models/pixi.py @@ -0,0 +1,13 @@ +from typing import Literal + +from .base import BaseManagerInfo +from .base import BaseRequirements + + +class PixiManagerInfo(BaseManagerInfo): + name: Literal["pixi"] = "pixi" + lockfile: str + + +class PixiRequirements(BaseRequirements): + manager: PixiManagerInfo diff --git a/src/ewoks/_requirements/models/poetry.py b/src/ewoks/_requirements/models/poetry.py new file mode 100644 index 0000000..88f42dd --- /dev/null +++ b/src/ewoks/_requirements/models/poetry.py @@ -0,0 +1,14 @@ +from typing import List +from typing import Literal + +from .base import BaseManagerInfo +from .base import BaseRequirements + + +class PoetryManagerInfo(BaseManagerInfo): + name: Literal["poetry"] = "pip" + requirements: List[str] + + +class PoetryRequirements(BaseRequirements): + manager: PoetryManagerInfo diff --git a/src/ewoks/_requirements/models/uv.py b/src/ewoks/_requirements/models/uv.py new file mode 100644 index 0000000..5a38f3b --- /dev/null +++ b/src/ewoks/_requirements/models/uv.py @@ -0,0 +1,18 @@ +from typing import List +from typing import Literal + +from .base import BaseManagerInfo +from .base import BaseRequirements + + +class UvManagerInfo(BaseManagerInfo): + name: Literal["uv"] = "uv" + requirements: List[str] + + +class UvRequirements(BaseRequirements): + manager: UvManagerInfo + + def __info__(self) -> str: + requirements = "\n ".join(self.manager.requirements) + return f"{super().__info__()}\nRequirements:\n {requirements}" diff --git a/src/ewoks/_requirements/pip/__init__.py b/src/ewoks/_requirements/pip/__init__.py deleted file mode 100644 index 312a112..0000000 --- a/src/ewoks/_requirements/pip/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Workflow requirements managed by `pip`.""" diff --git a/src/ewoks/_requirements/pip/install.py b/src/ewoks/_requirements/pip/install.py deleted file mode 100644 index d71a622..0000000 --- a/src/ewoks/_requirements/pip/install.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -import subprocess -from typing import Sequence - -from .sanitize import sanitize_requirements - -logger = logging.getLogger(__name__) - - -def pip_install(requirements: Sequence[str], python_path: str) -> int: - requirements, warnings = sanitize_requirements(requirements) - for warning in warnings: - logger.warning(warning) - # https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program - return subprocess.check_call([python_path, "-m", "pip", "install", *requirements]) diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index 3f27d03..07dd5a4 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -1,6 +1,8 @@ import datetime import logging import os +import re +import shlex import sys from contextlib import contextmanager from pathlib import Path @@ -9,6 +11,7 @@ from typing import Generator from typing import List from typing import Optional +from typing import Tuple from typing import Union from ewokscore.events.contexts import RawExecInfoType @@ -18,10 +21,8 @@ from tabulate import tabulate from . import _engines +from . import _requirements from . import graph_cache -from ._requirements.pip.extract import add_current_env_pip_requirements -from ._requirements.pip.extract import extract_pip_requirements -from ._requirements.pip.install import pip_install from .errors import AbortException try: @@ -205,7 +206,10 @@ def convert_graph( save_options = dict() graph = load_graph(source, inputs=inputs, **load_options) if save_requirements: - graph = add_current_env_pip_requirements(graph) + try: + _requirements.add_requirements(graph) + except Exception: + logger.exception("Continue after failure to add workflow requirements") return save_graph(graph, destination, **save_options) @@ -249,34 +253,43 @@ def _print_graph( def install_graph( source, skip_prompt: bool = False, - python_path: Optional[str] = None, + command: Union[None, str, Tuple[str, ...]] = None, load_options: Optional[dict] = None, -): +) -> None: if load_options is None: load_options = dict() graph = load_graph(source, **load_options) - requirements = graph.requirements - if requirements is None: - logger.warning( - "Requirements field is empty. Trying to extract requirements automatically..." - ) - requirements = extract_pip_requirements(graph) - logger.info(f"Extracted the following requirements: {requirements}") + requirements = _requirements.get_requirements(graph) - if python_path is None: - python_path = sys.executable + if not command: + command = tuple() + elif isinstance(command, str): + command = _split_command(command) if skip_prompt: - pip_install(requirements, python_path) + _requirements.install_requirements(requirements, command=command) return - requirements_as_str = "\n".join(requirements) - answer = input( - f"{requirements_as_str}\nThis will install the above packages via {python_path} -m pip install. Do you want to proceed (y/N)?" + f"{requirements.__info__()}\n\nThis will install the packages above. Do you want to proceed (y/N)?" ) if answer.lower() == "y" or answer.lower() == "yes": - pip_install(requirements, python_path) + _requirements.install_requirements(requirements, command=command) else: raise AbortException() + + +def _split_command(command: str) -> Tuple[str, ...]: + if sys.platform == "win32": + return tuple(_shlex_split(command)) + return tuple(shlex.split(command)) + + +def _shlex_split(command: str) -> Tuple[str, ...]: + r""" + Split a string on spaces, but treat escaped spaces (\ ) as part of the token. + """ + parts = re.split(r"(? None: + assert distribution_names, "no names provides" + assert "requirements" in graph.graph.graph, "no requirements" + requirements = parse_requirements(graph.graph.graph["requirements"]) + + existing = {distribution.name for distribution in requirements.distributions} + not_existing = set(distribution_names) - existing + assert not not_existing, f"{sorted(not_existing)} not in requirements" diff --git a/src/ewoks/tests/test_convert_cli.py b/src/ewoks/tests/test_convert_cli.py index 48ca344..eaf8db3 100644 --- a/src/ewoks/tests/test_convert_cli.py +++ b/src/ewoks/tests/test_convert_cli.py @@ -4,25 +4,15 @@ import pytest from ewokscore import load_graph -from ewokscore.graph import TaskGraph from ewokscore.task import Task from ewokscore.tests.examples.graphs import graph_names from ewoksutils.import_utils import import_qualname from orangewidget.widget import OWBaseWidget -from ewoks.__main__ import main -from ewoks.tests.utils import has_default_input -from ewoks.tests.utils import no_widget_registry - - -def _ewokscore_in_graph_requirements(graph: TaskGraph) -> bool: - ewokscore_in_req = False - for requirement in graph.graph.graph["requirements"]: - if "ewokscore" in requirement: - ewokscore_in_req = True - break - - return ewokscore_in_req +from ..__main__ import main +from .requirements.utils import assert_in_graph_requirements +from .utils import has_default_input +from .utils import no_widget_registry @pytest.mark.parametrize("graph_name", graph_names()) @@ -41,8 +31,8 @@ def test_convert_to_json(graph_name, tmpdir): assert os.path.exists(destination) graph = load_graph(destination) - assert graph.graph.graph["requirements"] is not None - assert _ewokscore_in_graph_requirements(graph) + + assert_in_graph_requirements(graph, "ewokscore") def test_convert_with_all_inputs(tmpdir): diff --git a/src/ewoks/tests/test_execute_cli.py b/src/ewoks/tests/test_execute_cli.py index 5b07dcc..859feb9 100644 --- a/src/ewoks/tests/test_execute_cli.py +++ b/src/ewoks/tests/test_execute_cli.py @@ -3,23 +3,13 @@ import pytest from ewokscore import load_graph -from ewokscore.graph import TaskGraph from ewokscore.tests.examples.graphs import get_graph from ewokscore.tests.examples.graphs import graph_names from ewokscore.tests.utils.results import assert_execute_graph_default_result -from ewoks.__main__ import main -from ewoks.tests.utils import has_default_input - - -def _ewokscore_in_graph_requirements(graph: TaskGraph) -> bool: - ewokscore_in_req = False - for requirement in graph.graph.graph["requirements"]: - if "ewokscore" in requirement: - ewokscore_in_req = True - break - - return ewokscore_in_req +from ..__main__ import main +from .requirements.utils import assert_in_graph_requirements +from .utils import has_default_input @pytest.mark.parametrize("graph_name", graph_names()) @@ -80,8 +70,7 @@ def test_execute_with_convert_destination(tmpdir): task1_node = graph.graph.nodes["task1"] assert has_default_input(task1_node, "b", 42) - assert graph.graph.graph["requirements"] is not None - assert _ewokscore_in_graph_requirements(graph) + assert_in_graph_requirements(graph, "ewokscore") def test_execute_with_convert_destination_inputs_all(tmpdir): @@ -106,5 +95,4 @@ def test_execute_with_convert_destination_inputs_all(tmpdir): for node in graph.graph.nodes.values(): assert has_default_input(node, "b", 42) - assert graph.graph.graph["requirements"] is not None - assert _ewokscore_in_graph_requirements(graph) + assert_in_graph_requirements(graph, "ewokscore") diff --git a/src/ewoks/tests/test_install_cli.py b/src/ewoks/tests/test_install_cli.py deleted file mode 100644 index b2920a9..0000000 --- a/src/ewoks/tests/test_install_cli.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -import subprocess - -import pytest - - -def test_install(venv): - with pytest.raises(Exception, match="package is not installed"): - venv.get_version("ewoksdata") - - subprocess.check_call( - [ - "ewoks", - "install", - "--yes", - '{"graph": {"id": "test_install", "requirements": ["ewoksdata"]}}', - "-p", - f"{venv.python}", - ] - ) - - assert venv.get_version("ewoksdata") is not None - - -def test_install_with_extract(venv): - with pytest.raises(Exception, match="package is not installed"): - venv.get_version("ewoksdata") - - nodes = [ - { - "id": 1, - "task_identifier": 'ewoksdata.tasks.normalization.Normalization"', - "task_type": "class", - }, - { - "id": 2, - "task_identifier": "path/to/my/script", - "task_type": "script", - }, # Check that unsupported task type goes though without error - ] - - graph = {"graph": {"id": "test_install"}, "nodes": nodes} - - subprocess.check_call( - [ - "ewoks", - "install", - "--yes", - json.dumps(graph), - "-p", - f"{venv.python}", - ] - ) - - assert venv.get_version("ewoksdata") is not None From e82bc4f8c3fcb9c6b4cb4a1fba7a14a2c97e4186 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 4 Jun 2026 15:55:26 +0200 Subject: [PATCH 2/9] merge ManagerInfo in BaseManager --- src/ewoks/_requirements/managers/conda.py | 37 +++++ src/ewoks/_requirements/managers/pip.py | 20 +++ src/ewoks/_requirements/managers/pipenv.py | 21 +++ src/ewoks/_requirements/managers/pixi.py | 19 +++ src/ewoks/_requirements/managers/poetry.py | 22 +++ .../_requirements/managers/utils/base.py | 59 ++++---- .../_requirements/managers/utils/commands.py | 35 ----- .../_requirements/managers/utils/detect.py | 37 ++--- .../_requirements/managers/utils/supported.py | 131 ++---------------- src/ewoks/_requirements/managers/uv.py | 20 +++ .../_requirements/metadata/gather/current.py | 2 +- .../metadata/gather/installed/__init__.py | 2 +- .../_requirements/metadata/gather/unknown.py | 2 +- 13 files changed, 205 insertions(+), 202 deletions(-) delete mode 100644 src/ewoks/_requirements/managers/utils/commands.py diff --git a/src/ewoks/_requirements/managers/conda.py b/src/ewoks/_requirements/managers/conda.py index a973405..ef5c9bc 100644 --- a/src/ewoks/_requirements/managers/conda.py +++ b/src/ewoks/_requirements/managers/conda.py @@ -1,4 +1,8 @@ import logging +import os +import sys +from typing import Optional +from typing import Tuple import yaml @@ -11,6 +15,26 @@ class CondaManager(BaseManager): NAME = "conda" + PRIORITY = 4 + + def __init__(self, *command: str) -> None: + if not command: + command = self._get_conda_command() + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output("--version") + except RuntimeError: + return None + return output.strip().split(" ")[-1] + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "CONDA_PREFIX" in os.environ or os.path.exists( + os.path.join(sys.prefix, "conda-meta") + ) def _gather_requirements(self, manager_version: str) -> CondaRequirements: output = self._check_output("env", "export") @@ -28,3 +52,16 @@ def install_requirements(self, requirements: CondaRequirements) -> None: text = yaml.safe_dump(requirements.environment) with self._temporary_file(text, ".yml") as tmp_path: self._check_call("env", "update", "-f", tmp_path) + + def _get_conda_command(self) -> Tuple[str, ...]: + try: + _ = self._check_output_raw("mamba", "--version") + return ("mamba",) + except Exception: + pass + try: + _ = self._check_output_raw("micromamba", "--version") + return ("micromamba",) + except Exception: + pass + return ("conda",) diff --git a/src/ewoks/_requirements/managers/pip.py b/src/ewoks/_requirements/managers/pip.py index b2100b6..fc3d562 100644 --- a/src/ewoks/_requirements/managers/pip.py +++ b/src/ewoks/_requirements/managers/pip.py @@ -1,5 +1,8 @@ +import importlib.metadata import logging +import sys from typing import List +from typing import Optional from ..metadata import pip_freeze from ..metadata.gather import gather_requirements @@ -11,6 +14,23 @@ class PipManager(BaseManager): NAME = "pip" + PRIORITY = 0 + + def __init__(self, *command: str) -> None: + if not command: + command = sys.executable, "-m", "pip" + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("pip") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return False def _gather_requirements(self, manager_version: str) -> PipRequirements: freeze_output = self._check_output("freeze").strip().splitlines() diff --git a/src/ewoks/_requirements/managers/pipenv.py b/src/ewoks/_requirements/managers/pipenv.py index 4f5a161..7f45a79 100644 --- a/src/ewoks/_requirements/managers/pipenv.py +++ b/src/ewoks/_requirements/managers/pipenv.py @@ -1,4 +1,8 @@ +import importlib.metadata import json +import os +import sys +from typing import Optional from ..metadata.gather import gather_requirements from ..models.pipenv import PipenvRequirements @@ -7,6 +11,23 @@ class PipenvManager(BaseManager): NAME = "pipenv" + PRIORITY = 3 + + def __init__(self, *command: str) -> None: + if not command: + command = sys.executable, "-m", "pipenv" + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("pipenv") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "PIPENV_ACTIVE" in os.environ def _gather_requirements(self, manager_version: str) -> PipenvRequirements: output = self._check_output("lock", "--requirements") diff --git a/src/ewoks/_requirements/managers/pixi.py b/src/ewoks/_requirements/managers/pixi.py index 04b79ae..3544df3 100644 --- a/src/ewoks/_requirements/managers/pixi.py +++ b/src/ewoks/_requirements/managers/pixi.py @@ -1,4 +1,5 @@ import os +from typing import Optional from ..metadata.gather import gather_requirements from ..models.pixi import PixiRequirements @@ -7,6 +8,24 @@ class PixiManager(BaseManager): NAME = "pixi" + PRIORITY = 5 + + def __init__(self, *command: str) -> None: + if not command: + command = ("pixi",) + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output("--version", text=True) + return output.strip().split(" ")[-1] + except Exception: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "PIXI_PROJECT_ROOT" in os.environ def _gather_requirements(self, manager_version: str) -> PixiRequirements: if os.path.exists("pixi.lock"): diff --git a/src/ewoks/_requirements/managers/poetry.py b/src/ewoks/_requirements/managers/poetry.py index 6260295..4d4ea14 100644 --- a/src/ewoks/_requirements/managers/poetry.py +++ b/src/ewoks/_requirements/managers/poetry.py @@ -1,3 +1,8 @@ +import importlib.metadata +import os +import sys +from typing import Optional + from ..metadata.gather import gather_requirements from ..models.poetry import PoetryRequirements from .utils.base import BaseManager @@ -5,6 +10,23 @@ class PoetryManager(BaseManager): NAME = "poetry" + PRIORITY = 2 + + def __init__(self, *command: str) -> None: + if not command: + command = sys.executable, "-m", "poetry" + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("poetry") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "POETRY_ACTIVE" in os.environ def _gather_requirements(self, manager_version: str) -> PoetryRequirements: output = self._check_output("export", "--without-hashes") diff --git a/src/ewoks/_requirements/managers/utils/base.py b/src/ewoks/_requirements/managers/utils/base.py index b0080b5..8b81759 100644 --- a/src/ewoks/_requirements/managers/utils/base.py +++ b/src/ewoks/_requirements/managers/utils/base.py @@ -5,11 +5,9 @@ from abc import abstractmethod from contextlib import contextmanager from typing import Generator -from typing import List from typing import Optional from ...models.base import BaseRequirements -from .commands import get_manager_command logger = logging.getLogger(__name__) @@ -34,18 +32,17 @@ class BaseManager: """ NAME = NotImplemented + PRIORITY = NotImplemented def __init__(self, *command: str) -> None: if not command: - command = get_manager_command(self.NAME) + raise ValueError(f"{type(self).__name__} needs an associated shell command") self._cmd_args = command def gather_requirements(self) -> Optional[BaseRequirements]: """Return requirements generated from the current python environment.""" - from .supported import get_supported_managers - - manager_version = get_supported_managers()[self.NAME].version - if not manager_version: + manager_version = self.version() + if manager_version is None: raise RuntimeError(f"{self.NAME!r} is not installed") try: @@ -56,6 +53,16 @@ def gather_requirements(self) -> Optional[BaseRequirements]: ) return None + @abstractmethod + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + pass + + @abstractmethod + def is_active(self) -> bool: + """Manager is explicitly active.""" + pass + def install_requirements(self, requirements: BaseRequirements) -> None: """Install requirements into the current python environment.""" try: @@ -74,13 +81,25 @@ def _gather_requirements(self, manager_version: str) -> BaseRequirements: def _install_requirements(self, requirements: BaseRequirements) -> None: pass - def _check_output(self, *args) -> str: - return _check_output([*self._cmd_args, *args]) + def _check_output(self, *args: str) -> str: + return self._check_output_raw(*[*self._cmd_args, *args]) - def _check_call(self, *args, raw: bool = False) -> int: - if raw: - return _check_call([*args]) - return _check_call([*self._cmd_args, *args]) + def _check_call(self, *args: str) -> int: + return self._check_call_raw(*[*self._cmd_args, *args]) + + @staticmethod + def _check_output_raw(*args: str) -> str: + try: + return subprocess.check_output(args, text=True) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex + + @staticmethod + def _check_call_raw(*args: str) -> int: + try: + return subprocess.check_call(args) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex @contextmanager def _temporary_file(self, text: str, suffix: str) -> Generator[str, None, None]: @@ -98,17 +117,3 @@ def _temporary_file(self, text: str, suffix: str) -> Generator[str, None, None]: os.remove(tmp_path) except OSError: logger.debug("Could not delete temporary file: %s", tmp_path) - - -def _check_output(args: List[str]) -> str: - try: - return subprocess.check_output(args, text=True) - except Exception as ex: - raise RuntimeError(f"Command failed: {args}") from ex - - -def _check_call(args: List[str]) -> int: - try: - return subprocess.check_call(args) - except Exception as ex: - raise RuntimeError(f"Command failed: {args}") from ex diff --git a/src/ewoks/_requirements/managers/utils/commands.py b/src/ewoks/_requirements/managers/utils/commands.py deleted file mode 100644 index b78b22a..0000000 --- a/src/ewoks/_requirements/managers/utils/commands.py +++ /dev/null @@ -1,35 +0,0 @@ -import subprocess -import sys -from functools import lru_cache -from typing import Dict -from typing import Tuple - - -def get_manager_command(manager_name: str) -> Tuple[str, ...]: - return manager_commands()[manager_name] - - -@lru_cache -def manager_commands() -> Dict[str, Tuple[str, ...]]: - return dict( - pip=(sys.executable, "-m", "pip"), - poetry=(sys.executable, "-m", "poetry"), - pipenv=(sys.executable, "-m", "pipenv"), - conda=_get_conda_command(), - pixi=("pixi",), - uv=("uv",), - ) - - -def _get_conda_command() -> Tuple[str, ...]: - try: - _ = subprocess.check_output(["mamba", "--version"], text=True) - return ("mamba",) - except Exception: - pass - try: - _ = subprocess.check_output(["micromamba", "--version"], text=True) - return ("micromamba",) - except Exception: - pass - return ("conda",) diff --git a/src/ewoks/_requirements/managers/utils/detect.py b/src/ewoks/_requirements/managers/utils/detect.py index 1f18ae1..bd0fec1 100644 --- a/src/ewoks/_requirements/managers/utils/detect.py +++ b/src/ewoks/_requirements/managers/utils/detect.py @@ -7,7 +7,6 @@ from ...metadata.gather import installed from .base import BaseManager -from .supported import ManagerInfo from .supported import get_supported_managers logger = logging.getLogger(__name__) @@ -22,43 +21,47 @@ def get_manager( :raise RuntimeError: no package manager available """ if manager_name: - managers = get_supported_managers() + managers = get_supported_managers(*command) - info = managers.get(manager_name) - if info is None: + manager = managers.get(manager_name) + if manager is None: raise ValueError(f"Package manager {manager_name!r} is not supported") - return info.manager_type(*command) + return manager - info = _detect_manager() - if info is None: + manager = _detect_manager() + if manager is None: raise RuntimeError("No known package manager installed or available") - return info.manager_type(*command) + return manager -def _detect_manager() -> Optional[ManagerInfo]: +def _detect_manager() -> Optional[BaseManager]: # Available package managers available_managers = { - name: info for name, info in get_supported_managers().items() if info.version + name: manager + for name, manager in get_supported_managers().items() + if manager.version() } if not available_managers: return None # Select the active manager with the highest priority active_managers = { - name: info for name, info in available_managers.items() if info.is_active + name: manager + for name, manager in available_managers.items() + if manager.is_active() } if active_managers: - name = max(active_managers, key=lambda name: active_managers[name].priority) - info = active_managers[name] + name = max(active_managers, key=lambda name: active_managers[name].PRIORITY) + manager = active_managers[name] logger.debug( "Detected active %r package manager\n available = %s\n active = %s", name, list(available_managers), list(active_managers), ) - return info + return manager # Infer most likely package manager counts = _installer_distribution_count() @@ -69,7 +72,9 @@ def _detect_manager() -> Optional[ManagerInfo]: else: # Use the package manager priority as the score crit = "priority" - scores = {name: info.priority for name, info in available_managers.items()} + scores = { + name: manager.PRIORITY for name, manager in available_managers.items() + } name = max(scores, key=scores.get) logger.debug( @@ -82,7 +87,7 @@ def _detect_manager() -> Optional[ManagerInfo]: return available_managers[name] -@lru_cache +@lru_cache(1) def _installer_distribution_count() -> Dict[str, int]: counts: Counter = Counter() for dist in installed.distributions(): diff --git a/src/ewoks/_requirements/managers/utils/supported.py b/src/ewoks/_requirements/managers/utils/supported.py index 90c572d..9c9fd5b 100644 --- a/src/ewoks/_requirements/managers/utils/supported.py +++ b/src/ewoks/_requirements/managers/utils/supported.py @@ -1,12 +1,5 @@ -import importlib.metadata -import os -import subprocess -import sys from functools import lru_cache from typing import Dict -from typing import NamedTuple -from typing import Optional -from typing import Type from ..pip import PipManager from .base import BaseManager @@ -18,117 +11,13 @@ # from ..uv import UvManager -class ManagerInfo(NamedTuple): - manager_type: Type[BaseManager] - version: Optional[str] - is_active: bool - priority: int - - -@lru_cache -def get_supported_managers() -> Dict[str, ManagerInfo]: - return dict( - pip=ManagerInfo( - manager_type=PipManager, - version=_get_pip_version(), - is_active=False, - priority=0, - ), - # uv=ManagerInfo( - # manager_type=UvManager, - # version=_get_uv_version(), - # is_active=False, - # priority=1, - # ), - # poetry=ManagerInfo( - # manager_type=PoetryManager, - # version=_get_poetry_version(), - # is_active=_is_poetry_active(), - # priority=2, - # ), - # pipenv=ManagerInfo( - # manager_type=PipenvManager, - # version=_get_pipenv_version(), - # is_active=_is_pipenv_active(), - # priority=3, - # ), - # conda=ManagerInfo( - # manager_type=CondaManager, - # version=_get_conda_version(), - # is_active=_is_conda_active(), - # priority=4, - # ), - # pixi=ManagerInfo( - # manager_type=PixiManager, - # version=_get_pixi_version(), - # is_active=_is_pixi_active(), - # priority=5, - # ), - ) - - -def _get_pip_version() -> Optional[str]: - try: - return importlib.metadata.version("pip") - except importlib.metadata.PackageNotFoundError: - return None - - -def _get_poetry_version() -> Optional[str]: - try: - return importlib.metadata.version("poetry") - except importlib.metadata.PackageNotFoundError: - return None - - -def _get_pipenv_version() -> Optional[str]: - try: - return importlib.metadata.version("pipenv") - except importlib.metadata.PackageNotFoundError: - return None - - -def _get_conda_version() -> Optional[str]: - try: - output = subprocess.check_output(["conda", "--version"], text=True) - return output.strip().split(" ")[-1] - except Exception: - return None - - -def _get_pixi_version() -> Optional[str]: - try: - output = subprocess.check_output(["pixi", "--version"], text=True) - return output.strip().split(" ")[-1] - except Exception: - return None - - -def _get_uv_version() -> Optional[str]: - try: - output = subprocess.check_output(["uv", "--version"], text=True) - return output.strip().split(" ")[-1] - except Exception: - return None - - -def _is_conda_active() -> bool: - return "CONDA_PREFIX" in os.environ or os.path.exists( - os.path.join(sys.prefix, "conda-meta") - ) - - -def _is_pixi_active() -> bool: - return "PIXI_PROJECT_ROOT" in os.environ - - -def _is_poetry_active() -> bool: - return "POETRY_ACTIVE" in os.environ - - -def _is_pipenv_active() -> bool: - return "PIPENV_ACTIVE" in os.environ - - -def _in_virtual_environment() -> bool: - return sys.prefix != sys.base_prefix +@lru_cache(maxsize=None) +def get_supported_managers(*command: str) -> Dict[str, BaseManager]: + managers = [ + PipManager(*command), # UvManager(*command), + # PoetryManager(*command), + # PipenvManager(*command), + # CondaManager(*command), + # PixiManager(*command), + ] + return {manager.NAME: manager for manager in managers} diff --git a/src/ewoks/_requirements/managers/uv.py b/src/ewoks/_requirements/managers/uv.py index cd5dcc9..43b33c4 100644 --- a/src/ewoks/_requirements/managers/uv.py +++ b/src/ewoks/_requirements/managers/uv.py @@ -1,3 +1,5 @@ +from typing import Optional + from ..metadata.gather import gather_requirements from ..models.uv import UvRequirements from .utils.base import BaseManager @@ -5,6 +7,24 @@ class UvManager(BaseManager): NAME = "uv" + PRIORITY = 1 + + def __init__(self, *command: str) -> None: + if not command: + command = ("uv",) + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output("--version") + return output.strip().split(" ")[-1] + except RuntimeError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + pass def _gather_requirements(self, manager_version: str) -> UvRequirements: output = self._check_output("pip", "freeze") diff --git a/src/ewoks/_requirements/metadata/gather/current.py b/src/ewoks/_requirements/metadata/gather/current.py index 3723765..3eab86a 100644 --- a/src/ewoks/_requirements/metadata/gather/current.py +++ b/src/ewoks/_requirements/metadata/gather/current.py @@ -6,7 +6,7 @@ from . import installed -@lru_cache +@lru_cache(1) def current_requirements() -> Dict[str, Any]: return dict( system=_system_metadata(), diff --git a/src/ewoks/_requirements/metadata/gather/installed/__init__.py b/src/ewoks/_requirements/metadata/gather/installed/__init__.py index 61c5dc6..f79b442 100644 --- a/src/ewoks/_requirements/metadata/gather/installed/__init__.py +++ b/src/ewoks/_requirements/metadata/gather/installed/__init__.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -@lru_cache() +@lru_cache(1) def distributions() -> List[Distribution]: """ Return installed distributions. diff --git a/src/ewoks/_requirements/metadata/gather/unknown.py b/src/ewoks/_requirements/metadata/gather/unknown.py index 0c47d2b..892da64 100644 --- a/src/ewoks/_requirements/metadata/gather/unknown.py +++ b/src/ewoks/_requirements/metadata/gather/unknown.py @@ -3,7 +3,7 @@ from typing import Dict -@lru_cache +@lru_cache(1) def unknown_requirements() -> Dict[str, Any]: return dict( system=_unknown_system_metadata(), From cd02d86dcb84a9e234439e950db3a022f483ddc0 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 4 Jun 2026 16:19:48 +0200 Subject: [PATCH 3/9] fix typos --- src/ewoks/_requirements/managers/utils/base.py | 6 +++--- src/ewoks/_requirements/managers/utils/detect.py | 2 +- .../_requirements/metadata/gather/installed/__init__.py | 2 +- src/ewoks/_requirements/metadata/gather/installed/git.py | 6 +++--- src/ewoks/_requirements/metadata/pip_freeze.py | 2 +- src/ewoks/_requirements/models/distro.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ewoks/_requirements/managers/utils/base.py b/src/ewoks/_requirements/managers/utils/base.py index 8b81759..e6cdf07 100644 --- a/src/ewoks/_requirements/managers/utils/base.py +++ b/src/ewoks/_requirements/managers/utils/base.py @@ -15,8 +15,8 @@ class BaseManager: """Defines the interface all package managers must implement. - If `MyManager` is an impementation of this interface then - to get the Ewoks workflow requirements like this: + If `MyManager` is an implementation of this interface then + Ewoks workflow requirements can be obtained like this: .. code-block:: python @@ -28,7 +28,7 @@ class BaseManager: .. code-block:: python manager = MyManager() - install_requirements.install_requirements(requirements) + manager.install_requirements(requirements) """ NAME = NotImplemented diff --git a/src/ewoks/_requirements/managers/utils/detect.py b/src/ewoks/_requirements/managers/utils/detect.py index bd0fec1..6d2ad91 100644 --- a/src/ewoks/_requirements/managers/utils/detect.py +++ b/src/ewoks/_requirements/managers/utils/detect.py @@ -66,7 +66,7 @@ def _detect_manager() -> Optional[BaseManager]: # Infer most likely package manager counts = _installer_distribution_count() if set(counts) & set(available_managers): - # Use the number of installed distibutions as the score + # Use the number of installed distributions as the score crit = "distribution count" scores = {name: counts.get(name, -1) for name in available_managers} else: diff --git a/src/ewoks/_requirements/metadata/gather/installed/__init__.py b/src/ewoks/_requirements/metadata/gather/installed/__init__.py index f79b442..333f7bd 100644 --- a/src/ewoks/_requirements/metadata/gather/installed/__init__.py +++ b/src/ewoks/_requirements/metadata/gather/installed/__init__.py @@ -52,7 +52,7 @@ def _distribution_from_metadata(dist: metadata.Distribution) -> Distribution: commit_id = vcs_info.get("commit_id") if commit_id: git_info = GitInfo( - commit=commit_id, remote=url, uncomitted_changes=False + commit=commit_id, remote=url, uncommitted_changes=False ) has_info = True diff --git a/src/ewoks/_requirements/metadata/gather/installed/git.py b/src/ewoks/_requirements/metadata/gather/installed/git.py index 918134e..4eefc29 100644 --- a/src/ewoks/_requirements/metadata/gather/installed/git.py +++ b/src/ewoks/_requirements/metadata/gather/installed/git.py @@ -16,9 +16,9 @@ def git_info_from_path(path: Path) -> Optional[GitInfo]: return None try: - uncomitted_changes = bool(_git(["status", "--porcelain"], path)) + uncommitted_changes = bool(_git(["status", "--porcelain"], path)) except Exception: - uncomitted_changes = False + uncommitted_changes = False remote_name = _find_remote_for_commit(path, commit) if remote_name: @@ -30,7 +30,7 @@ def git_info_from_path(path: Path) -> Optional[GitInfo]: remote_url = None return GitInfo( - commit=commit, remote=remote_url, uncomitted_changes=uncomitted_changes + commit=commit, remote=remote_url, uncommitted_changes=uncommitted_changes ) diff --git a/src/ewoks/_requirements/metadata/pip_freeze.py b/src/ewoks/_requirements/metadata/pip_freeze.py index 032d92e..9f93ef8 100644 --- a/src/ewoks/_requirements/metadata/pip_freeze.py +++ b/src/ewoks/_requirements/metadata/pip_freeze.py @@ -52,7 +52,7 @@ def freeze_distribution( warnings.append(warning) lines.append(f"# {warning}") - if dist.git.uncomitted_changes: + if dist.git.uncommitted_changes: warning = warning_fmt.format( dist.name, dist.git.commit, "has uncommited changes" ) diff --git a/src/ewoks/_requirements/models/distro.py b/src/ewoks/_requirements/models/distro.py index f394c82..a8cf3f7 100644 --- a/src/ewoks/_requirements/models/distro.py +++ b/src/ewoks/_requirements/models/distro.py @@ -8,7 +8,7 @@ class GitInfo(BaseModel): commit: str remote: Optional[str] = None - uncomitted_changes: bool = Field(default=False, description="Uncommited changes") + uncommitted_changes: bool = Field(default=False, description="Uncommited changes") class ArchiveInfo(BaseModel): From f881d06397ae780c4239cc796aefcdec2cb44ccb Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 4 Jun 2026 20:24:17 +0200 Subject: [PATCH 4/9] use custom manager command only when targeting a specific manager --- .../_requirements/managers/utils/detect.py | 16 +++++++++++----- .../_requirements/managers/utils/supported.py | 18 ++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/ewoks/_requirements/managers/utils/detect.py b/src/ewoks/_requirements/managers/utils/detect.py index 6d2ad91..fd020c7 100644 --- a/src/ewoks/_requirements/managers/utils/detect.py +++ b/src/ewoks/_requirements/managers/utils/detect.py @@ -21,13 +21,16 @@ def get_manager( :raise RuntimeError: no package manager available """ if manager_name: - managers = get_supported_managers(*command) + managers = get_supported_managers() - manager = managers.get(manager_name) - if manager is None: + manager_cls = managers.get(manager_name) + if manager_cls is None: raise ValueError(f"Package manager {manager_name!r} is not supported") - return manager + return manager_cls(*command) + else: + if command: + raise ValueError(f"Provide 'manager_name' associated to command {command}") manager = _detect_manager() if manager is None: @@ -38,9 +41,12 @@ def get_manager( def _detect_manager() -> Optional[BaseManager]: # Available package managers + available_managers = { + name: manager_cls() for name, manager_cls in get_supported_managers().items() + } available_managers = { name: manager - for name, manager in get_supported_managers().items() + for name, manager in available_managers.items() if manager.version() } if not available_managers: diff --git a/src/ewoks/_requirements/managers/utils/supported.py b/src/ewoks/_requirements/managers/utils/supported.py index 9c9fd5b..10ca223 100644 --- a/src/ewoks/_requirements/managers/utils/supported.py +++ b/src/ewoks/_requirements/managers/utils/supported.py @@ -1,5 +1,6 @@ from functools import lru_cache from typing import Dict +from typing import Type from ..pip import PipManager from .base import BaseManager @@ -11,13 +12,14 @@ # from ..uv import UvManager -@lru_cache(maxsize=None) -def get_supported_managers(*command: str) -> Dict[str, BaseManager]: +@lru_cache(1) +def get_supported_managers() -> Dict[str, Type[BaseManager]]: managers = [ - PipManager(*command), # UvManager(*command), - # PoetryManager(*command), - # PipenvManager(*command), - # CondaManager(*command), - # PixiManager(*command), + PipManager, + # UvManager, + # PoetryManager, + # PipenvManager, + # CondaManager, + # PixiManager, ] - return {manager.NAME: manager for manager in managers} + return {manager_cls.NAME: manager_cls for manager_cls in managers} From 3878cdc627e7eafd0650945e9c9bcb541c868461 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 4 Jun 2026 21:19:53 +0200 Subject: [PATCH 5/9] rearrange requirements --- src/ewoks/_requirements/__init__.py | 8 +-- .../_requirements/{managers => }/conda.py | 27 ++++++---- src/ewoks/_requirements/metadata/__init__.py | 0 .../_requirements/metadata/gather/__init__.py | 15 ------ .../_requirements/metadata/gather/current.py | 35 ------------- src/ewoks/_requirements/metadata/parse.py | 20 -------- src/ewoks/_requirements/models/__init__.py | 27 ---------- src/ewoks/_requirements/models/base.py | 39 --------------- src/ewoks/_requirements/models/conda.py | 13 ----- src/ewoks/_requirements/models/pip.py | 18 ------- src/ewoks/_requirements/models/pipenv.py | 18 ------- src/ewoks/_requirements/models/pixi.py | 13 ----- src/ewoks/_requirements/models/poetry.py | 14 ------ src/ewoks/_requirements/models/uv.py | 18 ------- src/ewoks/_requirements/{managers => }/pip.py | 34 +++++++++---- .../_requirements/{managers => }/pipenv.py | 32 ++++++++---- .../_requirements/{managers => }/pixi.py | 27 ++++++---- .../_requirements/{managers => }/poetry.py | 28 +++++++---- .../{managers => utils}/__init__.py | 0 .../utils/base.py => utils/base_manager.py} | 33 +++++++++++-- .../{managers => }/utils/detect.py | 6 +-- .../utils => utils/metadata}/__init__.py | 0 .../metadata/from_pip_freeze.py} | 0 .../metadata/from_python}/__init__.py | 49 +++++++++++++++---- .../metadata/from_python}/git.py | 6 +-- .../gather => utils/metadata}/last_resort.py | 4 +- .../metadata/metadata_models.py} | 15 ++++++ .../gather => utils/metadata}/unknown.py | 0 src/ewoks/_requirements/utils/parse.py | 25 ++++++++++ .../{metadata => utils}/pip_freeze.py | 4 +- .../{managers => }/utils/supported.py | 2 +- src/ewoks/_requirements/{managers => }/uv.py | 32 ++++++++---- .../tests/requirements/pip/test_freeze.py | 2 +- .../requirements/pip/test_install_cli.py | 2 +- src/ewoks/tests/requirements/utils.py | 2 +- 35 files changed, 252 insertions(+), 316 deletions(-) rename src/ewoks/_requirements/{managers => }/conda.py (75%) delete mode 100644 src/ewoks/_requirements/metadata/__init__.py delete mode 100644 src/ewoks/_requirements/metadata/gather/__init__.py delete mode 100644 src/ewoks/_requirements/metadata/gather/current.py delete mode 100644 src/ewoks/_requirements/metadata/parse.py delete mode 100644 src/ewoks/_requirements/models/__init__.py delete mode 100644 src/ewoks/_requirements/models/base.py delete mode 100644 src/ewoks/_requirements/models/conda.py delete mode 100644 src/ewoks/_requirements/models/pip.py delete mode 100644 src/ewoks/_requirements/models/pipenv.py delete mode 100644 src/ewoks/_requirements/models/pixi.py delete mode 100644 src/ewoks/_requirements/models/poetry.py delete mode 100644 src/ewoks/_requirements/models/uv.py rename src/ewoks/_requirements/{managers => }/pip.py (74%) rename src/ewoks/_requirements/{managers => }/pipenv.py (67%) rename src/ewoks/_requirements/{managers => }/pixi.py (70%) rename src/ewoks/_requirements/{managers => }/poetry.py (65%) rename src/ewoks/_requirements/{managers => utils}/__init__.py (100%) rename src/ewoks/_requirements/{managers/utils/base.py => utils/base_manager.py} (77%) rename src/ewoks/_requirements/{managers => }/utils/detect.py (95%) rename src/ewoks/_requirements/{managers/utils => utils/metadata}/__init__.py (100%) rename src/ewoks/_requirements/{metadata/gather/pip_freeze.py => utils/metadata/from_pip_freeze.py} (100%) rename src/ewoks/_requirements/{metadata/gather/installed => utils/metadata/from_python}/__init__.py (65%) rename src/ewoks/_requirements/{metadata/gather/installed => utils/metadata/from_python}/git.py (92%) rename src/ewoks/_requirements/{metadata/gather => utils/metadata}/last_resort.py (93%) rename src/ewoks/_requirements/{models/distro.py => utils/metadata/metadata_models.py} (70%) rename src/ewoks/_requirements/{metadata/gather => utils/metadata}/unknown.py (100%) create mode 100644 src/ewoks/_requirements/utils/parse.py rename src/ewoks/_requirements/{metadata => utils}/pip_freeze.py (98%) rename src/ewoks/_requirements/{managers => }/utils/supported.py (93%) rename src/ewoks/_requirements/{managers => }/uv.py (56%) diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index 24c7477..2b8804a 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -5,10 +5,10 @@ from ewokscore.graph import TaskGraph -from .managers.utils.base import BaseRequirements -from .managers.utils.detect import get_manager -from .metadata import parse -from .metadata.gather import last_resort +from .utils import parse +from .utils.base_manager import BaseRequirements +from .utils.detect import get_manager +from .utils.metadata import last_resort logger = logging.getLogger(__file__) diff --git a/src/ewoks/_requirements/managers/conda.py b/src/ewoks/_requirements/conda.py similarity index 75% rename from src/ewoks/_requirements/managers/conda.py rename to src/ewoks/_requirements/conda.py index ef5c9bc..7ee6340 100644 --- a/src/ewoks/_requirements/managers/conda.py +++ b/src/ewoks/_requirements/conda.py @@ -1,21 +1,34 @@ import logging import os import sys +from typing import Any +from typing import Dict +from typing import Literal from typing import Optional from typing import Tuple import yaml -from ..metadata.gather import gather_requirements -from ..models.conda import CondaRequirements -from .utils.base import BaseManager +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements logger = logging.getLogger(__name__) +class CondaManagerInfo(BaseManagerInfo): + name: Literal["conda"] = "conda" + + +class CondaRequirements(BaseRequirements): + manager: CondaManagerInfo + environment: dict + + class CondaManager(BaseManager): NAME = "conda" PRIORITY = 4 + REQUIREMENTS_MODEL = CondaRequirements def __init__(self, *command: str) -> None: if not command: @@ -36,17 +49,13 @@ def is_active(self) -> bool: os.path.join(sys.prefix, "conda-meta") ) - def _gather_requirements(self, manager_version: str) -> CondaRequirements: + def _gather_requirements(self) -> Dict[str, Any]: output = self._check_output("env", "export") environment = yaml.safe_load(output) environment.pop("name", None) environment.pop("prefix", None) - return gather_requirements( - manager_name="conda", - manager_version=manager_version, - environment=environment, - ) + return {"environment": environment} def install_requirements(self, requirements: CondaRequirements) -> None: text = yaml.safe_dump(requirements.environment) diff --git a/src/ewoks/_requirements/metadata/__init__.py b/src/ewoks/_requirements/metadata/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/ewoks/_requirements/metadata/gather/__init__.py b/src/ewoks/_requirements/metadata/gather/__init__.py deleted file mode 100644 index 18d1c90..0000000 --- a/src/ewoks/_requirements/metadata/gather/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from ...models import get_model -from ...models.base import BaseRequirements -from .current import current_requirements - - -def gather_requirements( - manager_name: str, manager_version: str, **parameters -) -> BaseRequirements: - """ - :raises ValueError: unknown package manager - :raises ValidationError: wrong parameters - """ - model_cls = get_model(manager_name) - manager = dict(name=manager_name, version=manager_version, **parameters) - return model_cls(manager=manager, **current_requirements()) diff --git a/src/ewoks/_requirements/metadata/gather/current.py b/src/ewoks/_requirements/metadata/gather/current.py deleted file mode 100644 index 3eab86a..0000000 --- a/src/ewoks/_requirements/metadata/gather/current.py +++ /dev/null @@ -1,35 +0,0 @@ -import platform -from functools import lru_cache -from typing import Any -from typing import Dict - -from . import installed - - -@lru_cache(1) -def current_requirements() -> Dict[str, Any]: - return dict( - system=_system_metadata(), - python=_python_metadata(), - distributions=installed.distributions(), - ) - - -def _system_metadata() -> Dict[str, Any]: - uname = platform.uname() - return dict( - system=uname.system, - release=uname.release, - version=uname.version, - machine=uname.machine, - processor=uname.processor, - ) - - -def _python_metadata() -> Dict[str, Any]: - return dict( - version=platform.python_version(), - implementation=platform.python_implementation(), - compiler=platform.python_compiler(), - build=", ".join(platform.python_build()), - ) diff --git a/src/ewoks/_requirements/metadata/parse.py b/src/ewoks/_requirements/metadata/parse.py deleted file mode 100644 index 89ccea7..0000000 --- a/src/ewoks/_requirements/metadata/parse.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import List -from typing import Union - -from ..models import get_model -from ..models.base import BaseRequirements -from .gather.pip_freeze import pip_freeze_requirements - - -def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: - if isinstance(requirements, list): - # Legacy 'pip freeze' list - requirements = pip_freeze_requirements(requirements) - - if not isinstance(requirements, dict): - raise TypeError( - f"Graph requirements must be a list or dictionary (type: {type(requirements)})" - ) - - manager = requirements.get("manager", dict()).get("name") - return get_model(manager)(**requirements) diff --git a/src/ewoks/_requirements/models/__init__.py b/src/ewoks/_requirements/models/__init__.py deleted file mode 100644 index 7365aee..0000000 --- a/src/ewoks/_requirements/models/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import Type - -from ..models.base import BaseRequirements -from ..models.conda import CondaRequirements -from ..models.pip import PipRequirements -from ..models.pipenv import PipenvRequirements -from ..models.pixi import PixiRequirements -from ..models.poetry import PoetryRequirements -from ..models.uv import UvRequirements - -_REQUIREMENT_MODELS = { - "pip": PipRequirements, - "poetry": PoetryRequirements, - "pipenv": PipenvRequirements, - "uv": UvRequirements, - "conda": CondaRequirements, - "pixi": PixiRequirements, -} - - -def get_model(manager_name: str) -> Type[BaseRequirements]: - """ - :raises ValueError: unknown package manager - """ - if manager_name not in _REQUIREMENT_MODELS: - raise ValueError(f"{manager_name!r} is not a valid package manager") - return _REQUIREMENT_MODELS[manager_name] diff --git a/src/ewoks/_requirements/models/base.py b/src/ewoks/_requirements/models/base.py deleted file mode 100644 index 0305bc2..0000000 --- a/src/ewoks/_requirements/models/base.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import List - -from pydantic import BaseModel - -from .distro import Distribution - - -class SystemInfo(BaseModel): - system: str - release: str - version: str - machine: str - processor: str - - -class PythonInfo(BaseModel): - version: str - implementation: str - compiler: str - build: str - - -class BaseManagerInfo(BaseModel): - name: str - version: str - - -class BaseRequirements(BaseModel): - system: SystemInfo - python: PythonInfo - distributions: List[Distribution] - manager: BaseManagerInfo - - def __info__(self) -> str: - return ( - f"Manager: {self.manager.name} ({self.manager.version}) " - f"python={self.python.version}) " - f"distributions={len(self.distributions)}" - ) diff --git a/src/ewoks/_requirements/models/conda.py b/src/ewoks/_requirements/models/conda.py deleted file mode 100644 index 4c936ef..0000000 --- a/src/ewoks/_requirements/models/conda.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Literal - -from .base import BaseManagerInfo -from .base import BaseRequirements - - -class CondaManagerInfo(BaseManagerInfo): - name: Literal["conda"] = "conda" - - -class CondaRequirements(BaseRequirements): - manager: CondaManagerInfo - environment: dict diff --git a/src/ewoks/_requirements/models/pip.py b/src/ewoks/_requirements/models/pip.py deleted file mode 100644 index acd5333..0000000 --- a/src/ewoks/_requirements/models/pip.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import List -from typing import Literal - -from .base import BaseManagerInfo -from .base import BaseRequirements - - -class PipManagerInfo(BaseManagerInfo): - name: Literal["pip"] = "pip" - freeze: List[str] - - -class PipRequirements(BaseRequirements): - manager: PipManagerInfo - - def __info__(self) -> str: - freeze = "\n ".join(self.manager.freeze) - return f"{super().__info__()}\nRequirements:\n {freeze}" diff --git a/src/ewoks/_requirements/models/pipenv.py b/src/ewoks/_requirements/models/pipenv.py deleted file mode 100644 index 8043cc8..0000000 --- a/src/ewoks/_requirements/models/pipenv.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import List -from typing import Literal - -from .base import BaseManagerInfo -from .base import BaseRequirements - - -class PipenvManagerInfo(BaseManagerInfo): - name: Literal["pipenv"] = "pipenv" - requirements: List[str] - - -class PipenvRequirements(BaseRequirements): - manager: PipenvManagerInfo - - def __info__(self) -> str: - requirements = "\n ".join(self.manager.requirements) - return f"{super().__info__()}\nRequirements:\n {requirements}" diff --git a/src/ewoks/_requirements/models/pixi.py b/src/ewoks/_requirements/models/pixi.py deleted file mode 100644 index 7d2a0fe..0000000 --- a/src/ewoks/_requirements/models/pixi.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import Literal - -from .base import BaseManagerInfo -from .base import BaseRequirements - - -class PixiManagerInfo(BaseManagerInfo): - name: Literal["pixi"] = "pixi" - lockfile: str - - -class PixiRequirements(BaseRequirements): - manager: PixiManagerInfo diff --git a/src/ewoks/_requirements/models/poetry.py b/src/ewoks/_requirements/models/poetry.py deleted file mode 100644 index 88f42dd..0000000 --- a/src/ewoks/_requirements/models/poetry.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List -from typing import Literal - -from .base import BaseManagerInfo -from .base import BaseRequirements - - -class PoetryManagerInfo(BaseManagerInfo): - name: Literal["poetry"] = "pip" - requirements: List[str] - - -class PoetryRequirements(BaseRequirements): - manager: PoetryManagerInfo diff --git a/src/ewoks/_requirements/models/uv.py b/src/ewoks/_requirements/models/uv.py deleted file mode 100644 index 5a38f3b..0000000 --- a/src/ewoks/_requirements/models/uv.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import List -from typing import Literal - -from .base import BaseManagerInfo -from .base import BaseRequirements - - -class UvManagerInfo(BaseManagerInfo): - name: Literal["uv"] = "uv" - requirements: List[str] - - -class UvRequirements(BaseRequirements): - manager: UvManagerInfo - - def __info__(self) -> str: - requirements = "\n ".join(self.manager.requirements) - return f"{super().__info__()}\nRequirements:\n {requirements}" diff --git a/src/ewoks/_requirements/managers/pip.py b/src/ewoks/_requirements/pip.py similarity index 74% rename from src/ewoks/_requirements/managers/pip.py rename to src/ewoks/_requirements/pip.py index fc3d562..468b9d0 100644 --- a/src/ewoks/_requirements/managers/pip.py +++ b/src/ewoks/_requirements/pip.py @@ -1,20 +1,37 @@ import importlib.metadata import logging import sys +from typing import Any +from typing import Dict from typing import List +from typing import Literal from typing import Optional -from ..metadata import pip_freeze -from ..metadata.gather import gather_requirements -from ..models.pip import PipRequirements -from .utils.base import BaseManager +from .utils import pip_freeze +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements logger = logging.getLogger(__name__) +class PipManagerInfo(BaseManagerInfo): + name: Literal["pip"] = "pip" + freeze: List[str] + + +class PipRequirements(BaseRequirements): + manager: PipManagerInfo + + def __info__(self) -> str: + freeze = "\n ".join(self.manager.freeze) + return f"{super().__info__()}\nRequirements:\n {freeze}" + + class PipManager(BaseManager): NAME = "pip" PRIORITY = 0 + REQUIREMENTS_MODEL = PipRequirements def __init__(self, *command: str) -> None: if not command: @@ -32,13 +49,10 @@ def is_active(self) -> bool: """Manager is explicitly active.""" return False - def _gather_requirements(self, manager_version: str) -> PipRequirements: + def _gather_requirements(self) -> Dict[str, Any]: freeze_output = self._check_output("freeze").strip().splitlines() - return gather_requirements( - manager_name=self.NAME, - manager_version=manager_version, - freeze=freeze_output, - ) + + return {"freeze": freeze_output} def _install_requirements(self, requirements: PipRequirements) -> None: freeze = requirements.manager.freeze diff --git a/src/ewoks/_requirements/managers/pipenv.py b/src/ewoks/_requirements/pipenv.py similarity index 67% rename from src/ewoks/_requirements/managers/pipenv.py rename to src/ewoks/_requirements/pipenv.py index 7f45a79..4be8eb5 100644 --- a/src/ewoks/_requirements/managers/pipenv.py +++ b/src/ewoks/_requirements/pipenv.py @@ -2,16 +2,34 @@ import json import os import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Literal from typing import Optional -from ..metadata.gather import gather_requirements -from ..models.pipenv import PipenvRequirements -from .utils.base import BaseManager +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class PipenvManagerInfo(BaseManagerInfo): + name: Literal["pipenv"] = "pipenv" + requirements: List[str] + + +class PipenvRequirements(BaseRequirements): + manager: PipenvManagerInfo + + def __info__(self) -> str: + requirements = "\n ".join(self.manager.requirements) + return f"{super().__info__()}\nRequirements:\n {requirements}" class PipenvManager(BaseManager): NAME = "pipenv" PRIORITY = 3 + REQUIREMENTS_MODEL = PipenvRequirements def __init__(self, *command: str) -> None: if not command: @@ -29,15 +47,11 @@ def is_active(self) -> bool: """Manager is explicitly active.""" return "PIPENV_ACTIVE" in os.environ - def _gather_requirements(self, manager_version: str) -> PipenvRequirements: + def _gather_requirements(self) -> Dict[str, Any]: output = self._check_output("lock", "--requirements") requirements = output.strip().splitlines() - return gather_requirements( - manager_name=self.NAME, - manager_version=manager_version, - requirements=requirements, - ) + return {"requirements": requirements} def _install_requirements(self, requirements: PipenvRequirements) -> None: lock_data = { diff --git a/src/ewoks/_requirements/managers/pixi.py b/src/ewoks/_requirements/pixi.py similarity index 70% rename from src/ewoks/_requirements/managers/pixi.py rename to src/ewoks/_requirements/pixi.py index 3544df3..b8866f7 100644 --- a/src/ewoks/_requirements/managers/pixi.py +++ b/src/ewoks/_requirements/pixi.py @@ -1,14 +1,27 @@ import os +from typing import Any +from typing import Dict +from typing import Literal from typing import Optional -from ..metadata.gather import gather_requirements -from ..models.pixi import PixiRequirements -from .utils.base import BaseManager +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class PixiManagerInfo(BaseManagerInfo): + name: Literal["pixi"] = "pixi" + lockfile: str + + +class PixiRequirements(BaseRequirements): + manager: PixiManagerInfo class PixiManager(BaseManager): NAME = "pixi" PRIORITY = 5 + REQUIREMENTS_MODEL = PixiRequirements def __init__(self, *command: str) -> None: if not command: @@ -27,7 +40,7 @@ def is_active(self) -> bool: """Manager is explicitly active.""" return "PIXI_PROJECT_ROOT" in os.environ - def _gather_requirements(self, manager_version: str) -> PixiRequirements: + def _gather_requirements(self) -> Dict[str, Any]: if os.path.exists("pixi.lock"): with open("pixi.lock", "r", encoding="utf-8") as f: lock_content = f.read() @@ -37,11 +50,7 @@ def _gather_requirements(self, manager_version: str) -> PixiRequirements: else: raise RuntimeError("No pixi.lock or pixi.toml file found") - return gather_requirements( - manager_name=self.NAME, - manager_version=manager_version, - lockfile=lock_content, - ) + return {"lockfile": lock_content} def _install_requirements(self, requirements: PixiRequirements) -> None: with self._temporary_file(requirements.lockfile, ".lock") as tmp_path: diff --git a/src/ewoks/_requirements/managers/poetry.py b/src/ewoks/_requirements/poetry.py similarity index 65% rename from src/ewoks/_requirements/managers/poetry.py rename to src/ewoks/_requirements/poetry.py index 4d4ea14..2c7e358 100644 --- a/src/ewoks/_requirements/managers/poetry.py +++ b/src/ewoks/_requirements/poetry.py @@ -1,16 +1,30 @@ import importlib.metadata import os import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Literal from typing import Optional -from ..metadata.gather import gather_requirements -from ..models.poetry import PoetryRequirements -from .utils.base import BaseManager +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class PoetryManagerInfo(BaseManagerInfo): + name: Literal["poetry"] = "pip" + requirements: List[str] + + +class PoetryRequirements(BaseRequirements): + manager: PoetryManagerInfo class PoetryManager(BaseManager): NAME = "poetry" PRIORITY = 2 + REQUIREMENTS_MODEL = PoetryRequirements def __init__(self, *command: str) -> None: if not command: @@ -28,15 +42,11 @@ def is_active(self) -> bool: """Manager is explicitly active.""" return "POETRY_ACTIVE" in os.environ - def _gather_requirements(self, manager_version: str) -> PoetryRequirements: + def _gather_requirements(self) -> Dict[str, Any]: output = self._check_output("export", "--without-hashes") requirements = output.strip().splitlines() - return gather_requirements( - manager_name=self.NAME, - manager_version=manager_version, - requirements=requirements, - ) + return {"requirements": requirements} def _install_requirements(self, requirements: PoetryRequirements) -> None: text = "\n".join(requirements.requirements) diff --git a/src/ewoks/_requirements/managers/__init__.py b/src/ewoks/_requirements/utils/__init__.py similarity index 100% rename from src/ewoks/_requirements/managers/__init__.py rename to src/ewoks/_requirements/utils/__init__.py diff --git a/src/ewoks/_requirements/managers/utils/base.py b/src/ewoks/_requirements/utils/base_manager.py similarity index 77% rename from src/ewoks/_requirements/managers/utils/base.py rename to src/ewoks/_requirements/utils/base_manager.py index e6cdf07..2d12c2a 100644 --- a/src/ewoks/_requirements/managers/utils/base.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -4,14 +4,37 @@ import tempfile from abc import abstractmethod from contextlib import contextmanager +from typing import Any +from typing import Dict from typing import Generator +from typing import List from typing import Optional -from ...models.base import BaseRequirements +from .metadata import metadata_models +from .metadata.from_python import current_requirements logger = logging.getLogger(__name__) +class BaseManagerInfo(metadata_models.BaseModel): + name: str + version: str + + +class BaseRequirements(metadata_models.BaseModel): + system: metadata_models.SystemInfo + python: metadata_models.PythonInfo + distributions: List[metadata_models.Distribution] + manager: BaseManagerInfo + + def __info__(self) -> str: + return ( + f"Manager: {self.manager.name} ({self.manager.version}) " + f"python={self.python.version}) " + f"distributions={len(self.distributions)}" + ) + + class BaseManager: """Defines the interface all package managers must implement. @@ -33,6 +56,7 @@ class BaseManager: NAME = NotImplemented PRIORITY = NotImplemented + REQUIREMENTS_MODEL = NotImplemented def __init__(self, *command: str) -> None: if not command: @@ -46,13 +70,16 @@ def gather_requirements(self) -> Optional[BaseRequirements]: raise RuntimeError(f"{self.NAME!r} is not installed") try: - return self._gather_requirements(manager_version) + parameters = self._gather_requirements() except Exception as ex: logger.error( "%s: failed to generate requirements (%s)", type(self).__name__, ex ) return None + manager = dict(name=self.NAME, version=manager_version, **parameters) + return self.REQUIREMENTS_MODEL(manager=manager, **current_requirements()) + @abstractmethod def version(self) -> Optional[str]: """Returns None when this manager is not available.""" @@ -74,7 +101,7 @@ def install_requirements(self, requirements: BaseRequirements) -> None: raise @abstractmethod - def _gather_requirements(self, manager_version: str) -> BaseRequirements: + def _gather_requirements(self) -> Dict[str, Any]: pass @abstractmethod diff --git a/src/ewoks/_requirements/managers/utils/detect.py b/src/ewoks/_requirements/utils/detect.py similarity index 95% rename from src/ewoks/_requirements/managers/utils/detect.py rename to src/ewoks/_requirements/utils/detect.py index fd020c7..63eee4b 100644 --- a/src/ewoks/_requirements/managers/utils/detect.py +++ b/src/ewoks/_requirements/utils/detect.py @@ -5,8 +5,8 @@ from typing import Optional from typing import Tuple -from ...metadata.gather import installed -from .base import BaseManager +from .base_manager import BaseManager +from .metadata.from_python import current_requirements from .supported import get_supported_managers logger = logging.getLogger(__name__) @@ -96,7 +96,7 @@ def _detect_manager() -> Optional[BaseManager]: @lru_cache(1) def _installer_distribution_count() -> Dict[str, int]: counts: Counter = Counter() - for dist in installed.distributions(): + for dist in current_requirements()["distributions"]: if dist.installer: counts[dist.installer] += 1 return dict(counts) diff --git a/src/ewoks/_requirements/managers/utils/__init__.py b/src/ewoks/_requirements/utils/metadata/__init__.py similarity index 100% rename from src/ewoks/_requirements/managers/utils/__init__.py rename to src/ewoks/_requirements/utils/metadata/__init__.py diff --git a/src/ewoks/_requirements/metadata/gather/pip_freeze.py b/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py similarity index 100% rename from src/ewoks/_requirements/metadata/gather/pip_freeze.py rename to src/ewoks/_requirements/utils/metadata/from_pip_freeze.py diff --git a/src/ewoks/_requirements/metadata/gather/installed/__init__.py b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py similarity index 65% rename from src/ewoks/_requirements/metadata/gather/installed/__init__.py rename to src/ewoks/_requirements/utils/metadata/from_python/__init__.py index 333f7bd..00f25ff 100644 --- a/src/ewoks/_requirements/metadata/gather/installed/__init__.py +++ b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py @@ -1,27 +1,56 @@ import json import logging +import platform from functools import lru_cache from importlib import metadata from pathlib import Path -from typing import List +from typing import Any +from typing import Dict -from ....models.distro import ArchiveInfo -from ....models.distro import Distribution -from ....models.distro import GitInfo +from .. import metadata_models from . import git logger = logging.getLogger(__name__) @lru_cache(1) -def distributions() -> List[Distribution]: +def current_requirements() -> Dict[str, Any]: """ Return installed distributions. """ - return [_distribution_from_metadata(dist) for dist in metadata.distributions()] + distributions = [ + _distribution_from_metadata(dist) for dist in metadata.distributions() + ] + return dict( + system=_system_metadata(), + python=_python_metadata(), + distributions=distributions, + ) + + +def _system_metadata() -> Dict[str, Any]: + uname = platform.uname() + return dict( + system=uname.system, + release=uname.release, + version=uname.version, + machine=uname.machine, + processor=uname.processor, + ) + + +def _python_metadata() -> Dict[str, Any]: + return dict( + version=platform.python_version(), + implementation=platform.python_implementation(), + compiler=platform.python_compiler(), + build=", ".join(platform.python_build()), + ) -def _distribution_from_metadata(dist: metadata.Distribution) -> Distribution: +def _distribution_from_metadata( + dist: metadata.Distribution, +) -> metadata_models.Distribution: name = dist.metadata["Name"] version = dist.version @@ -51,7 +80,7 @@ def _distribution_from_metadata(dist: metadata.Distribution) -> Distribution: if vcs_info.get("vcs") == "git": commit_id = vcs_info.get("commit_id") if commit_id: - git_info = GitInfo( + git_info = metadata_models.GitInfo( commit=commit_id, remote=url, uncommitted_changes=False ) has_info = True @@ -76,10 +105,10 @@ def _distribution_from_metadata(dist: metadata.Distribution) -> Distribution: else: hashes = {} - archive_info = ArchiveInfo(url=url, hashes=hashes) + archive_info = metadata_models.ArchiveInfo(url=url, hashes=hashes) has_info = True - return Distribution( + return metadata_models.Distribution( name=name, version=version, git=git_info, diff --git a/src/ewoks/_requirements/metadata/gather/installed/git.py b/src/ewoks/_requirements/utils/metadata/from_python/git.py similarity index 92% rename from src/ewoks/_requirements/metadata/gather/installed/git.py rename to src/ewoks/_requirements/utils/metadata/from_python/git.py index 4eefc29..489f117 100644 --- a/src/ewoks/_requirements/metadata/gather/installed/git.py +++ b/src/ewoks/_requirements/utils/metadata/from_python/git.py @@ -3,10 +3,10 @@ from typing import List from typing import Optional -from ....models.distro import GitInfo +from .. import metadata_models -def git_info_from_path(path: Path) -> Optional[GitInfo]: +def git_info_from_path(path: Path) -> Optional[metadata_models.GitInfo]: """ Return GitInfo for a local path, or None if not a git repository. """ @@ -29,7 +29,7 @@ def git_info_from_path(path: Path) -> Optional[GitInfo]: else: remote_url = None - return GitInfo( + return metadata_models.GitInfo( commit=commit, remote=remote_url, uncommitted_changes=uncommitted_changes ) diff --git a/src/ewoks/_requirements/metadata/gather/last_resort.py b/src/ewoks/_requirements/utils/metadata/last_resort.py similarity index 93% rename from src/ewoks/_requirements/metadata/gather/last_resort.py rename to src/ewoks/_requirements/utils/metadata/last_resort.py index e5a8890..ffe0def 100644 --- a/src/ewoks/_requirements/metadata/gather/last_resort.py +++ b/src/ewoks/_requirements/utils/metadata/last_resort.py @@ -4,7 +4,7 @@ from ewokscore.graph import TaskGraph -from . import pip_freeze +from .from_pip_freeze import pip_freeze_requirements logger = logging.getLogger(__name__) @@ -44,4 +44,4 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: f"Could not extract requirements for node {node_id}: unsupported task type {task_type}." ) - return pip_freeze.pip_freeze_requirements(list(freeze)) + return pip_freeze_requirements(list(freeze)) diff --git a/src/ewoks/_requirements/models/distro.py b/src/ewoks/_requirements/utils/metadata/metadata_models.py similarity index 70% rename from src/ewoks/_requirements/models/distro.py rename to src/ewoks/_requirements/utils/metadata/metadata_models.py index a8cf3f7..c446438 100644 --- a/src/ewoks/_requirements/models/distro.py +++ b/src/ewoks/_requirements/utils/metadata/metadata_models.py @@ -5,6 +5,21 @@ from pydantic import Field +class SystemInfo(BaseModel): + system: str + release: str + version: str + machine: str + processor: str + + +class PythonInfo(BaseModel): + version: str + implementation: str + compiler: str + build: str + + class GitInfo(BaseModel): commit: str remote: Optional[str] = None diff --git a/src/ewoks/_requirements/metadata/gather/unknown.py b/src/ewoks/_requirements/utils/metadata/unknown.py similarity index 100% rename from src/ewoks/_requirements/metadata/gather/unknown.py rename to src/ewoks/_requirements/utils/metadata/unknown.py diff --git a/src/ewoks/_requirements/utils/parse.py b/src/ewoks/_requirements/utils/parse.py new file mode 100644 index 0000000..da784cf --- /dev/null +++ b/src/ewoks/_requirements/utils/parse.py @@ -0,0 +1,25 @@ +from typing import List +from typing import Union + +from .base_manager import BaseRequirements +from .metadata.from_pip_freeze import pip_freeze_requirements +from .supported import get_supported_managers + + +def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: + if isinstance(requirements, list): + # Legacy 'pip freeze' list + requirements = pip_freeze_requirements(requirements) + + if not isinstance(requirements, dict): + raise TypeError( + f"Graph requirements must be a list or dictionary (type: {type(requirements)})" + ) + + manager_name = requirements.get("manager", dict()).get("name") + + managers = get_supported_managers() + if manager_name not in managers: + raise ValueError(f"{manager_name!r} is not a valid package manager") + + return managers[manager_name].REQUIREMENTS_MODEL(**requirements) diff --git a/src/ewoks/_requirements/metadata/pip_freeze.py b/src/ewoks/_requirements/utils/pip_freeze.py similarity index 98% rename from src/ewoks/_requirements/metadata/pip_freeze.py rename to src/ewoks/_requirements/utils/pip_freeze.py index 9f93ef8..a93d7d5 100644 --- a/src/ewoks/_requirements/metadata/pip_freeze.py +++ b/src/ewoks/_requirements/utils/pip_freeze.py @@ -11,11 +11,11 @@ from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement -from ..models.distro import Distribution +from .metadata import metadata_models def freeze_distribution( - dist: Distribution, + dist: metadata_models.Distribution, ) -> Tuple[List[str], List[str]]: """ Return the pip freeze of the distribution with associated warnings regarding reproducibility. diff --git a/src/ewoks/_requirements/managers/utils/supported.py b/src/ewoks/_requirements/utils/supported.py similarity index 93% rename from src/ewoks/_requirements/managers/utils/supported.py rename to src/ewoks/_requirements/utils/supported.py index 10ca223..a9bec1f 100644 --- a/src/ewoks/_requirements/managers/utils/supported.py +++ b/src/ewoks/_requirements/utils/supported.py @@ -3,7 +3,7 @@ from typing import Type from ..pip import PipManager -from .base import BaseManager +from .base_manager import BaseManager # from ..conda import CondaManager # from ..pipenv import PipenvManager diff --git a/src/ewoks/_requirements/managers/uv.py b/src/ewoks/_requirements/uv.py similarity index 56% rename from src/ewoks/_requirements/managers/uv.py rename to src/ewoks/_requirements/uv.py index 43b33c4..4ab32c8 100644 --- a/src/ewoks/_requirements/managers/uv.py +++ b/src/ewoks/_requirements/uv.py @@ -1,13 +1,31 @@ +from typing import Any +from typing import Dict +from typing import List +from typing import Literal from typing import Optional -from ..metadata.gather import gather_requirements -from ..models.uv import UvRequirements -from .utils.base import BaseManager +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class UvManagerInfo(BaseManagerInfo): + name: Literal["uv"] = "uv" + requirements: List[str] + + +class UvRequirements(BaseRequirements): + manager: UvManagerInfo + + def __info__(self) -> str: + requirements = "\n ".join(self.manager.requirements) + return f"{super().__info__()}\nRequirements:\n {requirements}" class UvManager(BaseManager): NAME = "uv" PRIORITY = 1 + REQUIREMENTS_MODEL = UvRequirements def __init__(self, *command: str) -> None: if not command: @@ -26,15 +44,11 @@ def is_active(self) -> bool: """Manager is explicitly active.""" pass - def _gather_requirements(self, manager_version: str) -> UvRequirements: + def _gather_requirements(self) -> Dict[str, Any]: output = self._check_output("pip", "freeze") requirements = output.strip().splitlines() - return gather_requirements( - manager_name=self.NAME, - manager_version=manager_version, - requirements=requirements, - ) + return {"requirements": requirements} def _install_requirements(self, requirements: UvRequirements) -> None: text = "\n".join(requirements.requirements) diff --git a/src/ewoks/tests/requirements/pip/test_freeze.py b/src/ewoks/tests/requirements/pip/test_freeze.py index f99a3fc..61b0b7f 100644 --- a/src/ewoks/tests/requirements/pip/test_freeze.py +++ b/src/ewoks/tests/requirements/pip/test_freeze.py @@ -1,6 +1,6 @@ import pytest -from ...._requirements.metadata.pip_freeze import sanitize_freeze +from ...._requirements.utils.pip_freeze import sanitize_freeze def test_normal_requirement(): diff --git a/src/ewoks/tests/requirements/pip/test_install_cli.py b/src/ewoks/tests/requirements/pip/test_install_cli.py index 0666614..a7aadea 100644 --- a/src/ewoks/tests/requirements/pip/test_install_cli.py +++ b/src/ewoks/tests/requirements/pip/test_install_cli.py @@ -3,7 +3,7 @@ import pytest -from ...._requirements.metadata.gather import unknown +from ...._requirements.utils.metadata import unknown def test_install_pip_with_freeze(venv): diff --git a/src/ewoks/tests/requirements/utils.py b/src/ewoks/tests/requirements/utils.py index 74145b8..651d016 100644 --- a/src/ewoks/tests/requirements/utils.py +++ b/src/ewoks/tests/requirements/utils.py @@ -1,6 +1,6 @@ from ewokscore.graph import TaskGraph -from ..._requirements.metadata.parse import parse_requirements +from ..._requirements.utils.parse import parse_requirements def assert_in_graph_requirements(graph: TaskGraph, *distribution_names) -> None: From 37d089ec38465f8a4c8babfd5e2ffa0dba7f38db Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 4 Jun 2026 22:21:11 +0200 Subject: [PATCH 6/9] refactor manager native installation with python distribution fallback --- src/ewoks/_requirements/__init__.py | 2 +- src/ewoks/_requirements/conda.py | 6 +- src/ewoks/_requirements/pip.py | 34 +++++----- src/ewoks/_requirements/pipenv.py | 5 +- src/ewoks/_requirements/pixi.py | 7 +- src/ewoks/_requirements/poetry.py | 7 +- src/ewoks/_requirements/utils/base_manager.py | 66 +++++++++++++++---- .../utils/metadata/from_python/__init__.py | 2 +- src/ewoks/_requirements/uv.py | 7 +- 9 files changed, 99 insertions(+), 37 deletions(-) diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index 2b8804a..eab5f8e 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -66,7 +66,7 @@ def install_requirements( pip_freeze = requirements.manager.freeze - dists_freeze = manager.freeze_distributions(requirements) + dists_freeze = manager._freeze_distributions(requirements) dists_freeze = [s for s in dists_freeze if not s.startswith("#")] print() diff --git a/src/ewoks/_requirements/conda.py b/src/ewoks/_requirements/conda.py index 7ee6340..d4eacaf 100644 --- a/src/ewoks/_requirements/conda.py +++ b/src/ewoks/_requirements/conda.py @@ -57,10 +57,14 @@ def _gather_requirements(self) -> Dict[str, Any]: return {"environment": environment} - def install_requirements(self, requirements: CondaRequirements) -> None: + def _install_native_requirements(self, requirements: CondaRequirements) -> bool: text = yaml.safe_dump(requirements.environment) with self._temporary_file(text, ".yml") as tmp_path: self._check_call("env", "update", "-f", tmp_path) + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") def _get_conda_command(self) -> Tuple[str, ...]: try: diff --git a/src/ewoks/_requirements/pip.py b/src/ewoks/_requirements/pip.py index 468b9d0..f1bd48d 100644 --- a/src/ewoks/_requirements/pip.py +++ b/src/ewoks/_requirements/pip.py @@ -54,27 +54,29 @@ def _gather_requirements(self) -> Dict[str, Any]: return {"freeze": freeze_output} - def _install_requirements(self, requirements: PipRequirements) -> None: + def _install_native_requirements(self, requirements: PipRequirements) -> bool: freeze = requirements.manager.freeze - if freeze: - arguments = self._arguments(freeze) - try: - self._check_call("install", "--no-cache-dir", *arguments) - return - except Exception: - if not requirements.distributions: - raise + if not freeze: + return False - freeze = self.freeze_distributions(requirements) - if freeze: - arguments = self._arguments(freeze) - self._check_call("install", "--no-cache-dir", *arguments) - return + arguments = self._arguments(freeze) + self._check_call("install", "--no-cache-dir", *arguments) + return True - raise ValueError("No distibutions provided to install") + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + freeze = self._freeze_distributions(requirements) + if not freeze: + return False - def freeze_distributions(self, requirements: PipRequirements) -> List[str]: + arguments = self._arguments(freeze) + self._check_call("install", "--no-cache-dir", *arguments) + return True + + def _freeze_distributions(self, requirements: BaseRequirements) -> List[str]: + """ + Pip freeze argument from list of distributions. + """ freeze = [] for dist in requirements.distributions: lines, warnings = pip_freeze.freeze_distribution(dist) diff --git a/src/ewoks/_requirements/pipenv.py b/src/ewoks/_requirements/pipenv.py index 4be8eb5..7153576 100644 --- a/src/ewoks/_requirements/pipenv.py +++ b/src/ewoks/_requirements/pipenv.py @@ -53,7 +53,7 @@ def _gather_requirements(self) -> Dict[str, Any]: return {"requirements": requirements} - def _install_requirements(self, requirements: PipenvRequirements) -> None: + def _install_native_requirements(self, requirements: PipenvRequirements) -> bool: lock_data = { "_meta": {"hash": {"sha256": "dummy"}}, # minimal metadata "default": { @@ -69,3 +69,6 @@ def _install_requirements(self, requirements: PipenvRequirements) -> None: with self._temporary_file(text, ".lock") as tmp_path: self._check_call("sync", "--ignore-pipfile", "-f", tmp_path) + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/pixi.py b/src/ewoks/_requirements/pixi.py index b8866f7..ebdb2cb 100644 --- a/src/ewoks/_requirements/pixi.py +++ b/src/ewoks/_requirements/pixi.py @@ -52,6 +52,11 @@ def _gather_requirements(self) -> Dict[str, Any]: return {"lockfile": lock_content} - def _install_requirements(self, requirements: PixiRequirements) -> None: + def _install_requirements(self, requirements: PixiRequirements) -> bool: with self._temporary_file(requirements.lockfile, ".lock") as tmp_path: self._check_call("install", cwd=os.path.dirname(tmp_path)) + + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/poetry.py b/src/ewoks/_requirements/poetry.py index 2c7e358..b9190d9 100644 --- a/src/ewoks/_requirements/poetry.py +++ b/src/ewoks/_requirements/poetry.py @@ -48,7 +48,12 @@ def _gather_requirements(self) -> Dict[str, Any]: return {"requirements": requirements} - def _install_requirements(self, requirements: PoetryRequirements) -> None: + def _install_native_requirements(self, requirements: PoetryRequirements) -> bool: text = "\n".join(requirements.requirements) with self._temporary_file(text, ".txt") as tmp_path: self._check_call("add", "--lock", "--file", tmp_path) + + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 2d12c2a..9ee6d80 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -63,11 +63,25 @@ def __init__(self, *command: str) -> None: raise ValueError(f"{type(self).__name__} needs an associated shell command") self._cmd_args = command + @abstractmethod + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + pass + + @abstractmethod + def is_active(self) -> bool: + """Manager is explicitly active.""" + pass + def gather_requirements(self) -> Optional[BaseRequirements]: - """Return requirements generated from the current python environment.""" + """ + Return requirements associated to the current python environment. + + :raises RuntimeError: package manager not available + """ manager_version = self.version() if manager_version is None: - raise RuntimeError(f"{self.NAME!r} is not installed") + raise RuntimeError(f"{self.NAME!r} is not available") try: parameters = self._gather_requirements() @@ -80,18 +94,12 @@ def gather_requirements(self) -> Optional[BaseRequirements]: manager = dict(name=self.NAME, version=manager_version, **parameters) return self.REQUIREMENTS_MODEL(manager=manager, **current_requirements()) - @abstractmethod - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - pass - - @abstractmethod - def is_active(self) -> bool: - """Manager is explicitly active.""" - pass - def install_requirements(self, requirements: BaseRequirements) -> None: - """Install requirements into the current python environment.""" + """ + Install requirements into the current environment. + + :raises ValueError: no distibutions provided to install + """ try: return self._install_requirements(requirements) except Exception as ex: @@ -100,12 +108,42 @@ def install_requirements(self, requirements: BaseRequirements) -> None: ) raise + def _install_requirements(self, requirements: BaseRequirements) -> None: + reraise = None + + if isinstance(requirements, self.REQUIREMENTS_MODEL): + try: + if self._install_native_requirements(requirements): + return + except Exception as ex: + reraise = ex + logger.debug( + ( + "Failed installing requirements native to package %s. " + "Try installing python distributions." + ), + self.NAME, + ex, + ) + pass + + if self._install_base_requirements(requirements): + return + + if reraise: + raise reraise + raise ValueError("No distibutions provided to install") + @abstractmethod def _gather_requirements(self) -> Dict[str, Any]: pass @abstractmethod - def _install_requirements(self, requirements: BaseRequirements) -> None: + def _install_native_requirements(self, requirements: BaseRequirements) -> bool: + pass + + @abstractmethod + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: pass def _check_output(self, *args: str) -> str: diff --git a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py index 00f25ff..f50f0f8 100644 --- a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py +++ b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py @@ -16,7 +16,7 @@ @lru_cache(1) def current_requirements() -> Dict[str, Any]: """ - Return installed distributions. + Return installed python distributions. """ distributions = [ _distribution_from_metadata(dist) for dist in metadata.distributions() diff --git a/src/ewoks/_requirements/uv.py b/src/ewoks/_requirements/uv.py index 4ab32c8..f5269f3 100644 --- a/src/ewoks/_requirements/uv.py +++ b/src/ewoks/_requirements/uv.py @@ -50,7 +50,12 @@ def _gather_requirements(self) -> Dict[str, Any]: return {"requirements": requirements} - def _install_requirements(self, requirements: UvRequirements) -> None: + def _install_native_requirements(self, requirements: UvRequirements) -> bool: text = "\n".join(requirements.requirements) with self._temporary_file(text, ".txt") as tmp_path: self._check_call("add", "-r", tmp_path) + + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") From 86b59932cb3000814520234954ae2499afaaeadb Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 4 Jun 2026 23:15:18 +0200 Subject: [PATCH 7/9] ewoks convert/install arguments: package manager name and command --- CHANGELOG.md | 9 +++++ src/ewoks/__main__.py | 7 +++- src/ewoks/_requirements/__init__.py | 33 ++++++++++++++--- src/ewoks/_requirements/utils/base_manager.py | 3 ++ src/ewoks/_requirements/utils/detect.py | 33 +++++++++++------ src/ewoks/bindings.py | 37 +++++++++++++++---- src/ewoks/cli_utils/cli_convert_utils.py | 17 +++++++++ src/ewoks/cli_utils/cli_install_utils.py | 10 ++++- .../requirements/pip/test_install_cli.py | 30 +++++++++++++-- src/ewoks/tests/test_cli.py | 3 ++ 10 files changed, 151 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96f99a0..832a822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `ewoks install`: add `--package-manager-name` and `--package-manager-command` arguments. +- `ewoks convert`: add `--package-manager-name` and `--package-manager-command` arguments. + +### Removed + +- `ewoks install`: remove `-p/--python` argument. + ## [5.0.0] - 2026-04-27 - Upgrade `ewoksdask` to 3.* diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 3a86e8f..416a896 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -159,7 +159,12 @@ def command_install( cli_install_utils.parse_install_arguments(cli_args, shell=shell) for workflow, graph in zip(cli_args.workflows, cli_args.graphs): try: - install_graph(graph, cli_args.yes, command=cli_args.manager_command) + install_graph( + graph, + skip_prompt=cli_args.yes, + package_manager_name=cli_args.package_manager_name, + package_manager_command=cli_args.package_manager_command, + ) except CalledProcessError as e: print(f"Install failed for {workflow}: {e}") except AbortException: diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index eab5f8e..6aaa98b 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -1,7 +1,9 @@ """Workflow requirements.""" import logging +from typing import Optional from typing import Tuple +from typing import Union from ewokscore.graph import TaskGraph @@ -13,9 +15,13 @@ logger = logging.getLogger(__file__) -def add_requirements(graph: TaskGraph, command: Tuple[str, ...] = tuple()) -> None: +def add_requirements( + graph: TaskGraph, + manager_name: Optional[str] = None, + manager_command: Tuple[str, ...] = tuple(), +) -> None: """Add requirements to a workflow definition in-place.""" - manager = get_manager(command=command) + manager = get_manager(manager_name=manager_name, manager_command=manager_command) requirements = manager.gather_requirements() graph.graph.graph["requirements"] = requirements.model_dump() @@ -40,10 +46,27 @@ def get_requirements(graph: TaskGraph) -> BaseRequirements: def install_requirements( - requirements: BaseRequirements, command: Tuple[str, ...] = tuple() + requirements: BaseRequirements, + manager_name: Optional[str] = None, + manager_command: Union[None, str, Tuple[str, ...]] = None, ) -> None: """Install workflow requirements.""" - manager = get_manager(manager_name=requirements.manager.name, command=command) + + if manager_command and not manager_name: + raise ValueError( + f"Provide 'manager_name' associated to command {manager_command}" + ) + + try: + if manager_name: + raise ValueError("Ignore package manager used to generate the requirements") + else: + manager = get_manager(manager_name=requirements.manager.name) + except ValueError: + manager = get_manager( + manager_name=manager_name, manager_command=manager_command + ) + manager.install_requirements(requirements) @@ -55,7 +78,7 @@ def install_requirements( t0 = time.perf_counter() try: - manager = get_manager(manager_name=None) + manager = get_manager(manager_name="pip") requirements = manager.gather_requirements() print() diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 9ee6d80..42178b8 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -63,6 +63,9 @@ def __init__(self, *command: str) -> None: raise ValueError(f"{type(self).__name__} needs an associated shell command") self._cmd_args = command + def __repr__(self): + return f"{type(self).__name__}({', '.join(self._cmd_args)})" + @abstractmethod def version(self) -> Optional[str]: """Returns None when this manager is not available.""" diff --git a/src/ewoks/_requirements/utils/detect.py b/src/ewoks/_requirements/utils/detect.py index 63eee4b..93e18d1 100644 --- a/src/ewoks/_requirements/utils/detect.py +++ b/src/ewoks/_requirements/utils/detect.py @@ -14,31 +14,42 @@ def get_manager( manager_name: Optional[str] = None, - command: Tuple[str, ...] = tuple(), + manager_command: Tuple[str, ...] = tuple(), ) -> BaseManager: """ - :raise ValueError: package manager not support + :raise ValueError: package manager not support or not available :raise RuntimeError: no package manager available """ if manager_name: - managers = get_supported_managers() + return _select_manager(manager_name, manager_command) - manager_cls = managers.get(manager_name) - if manager_cls is None: - raise ValueError(f"Package manager {manager_name!r} is not supported") - - return manager_cls(*command) - else: - if command: - raise ValueError(f"Provide 'manager_name' associated to command {command}") + if manager_command: + raise ValueError( + f"Provide 'manager_name' associated to command {manager_command}" + ) manager = _detect_manager() + if manager is None: raise RuntimeError("No known package manager installed or available") return manager +def _select_manager(manager_name: str, manager_command: Tuple[str, ...]) -> BaseManager: + managers = get_supported_managers() + + manager_cls = managers.get(manager_name) + if manager_cls is None: + raise ValueError(f"Package manager {manager_name!r} is not supported") + + manager = manager_cls(*manager_command) + if not manager.version: + raise ValueError(f"Package manager {manager_name!r} is not available") + + return manager + + def _detect_manager() -> Optional[BaseManager]: # Available package managers available_managers = { diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index 07dd5a4..e55b97d 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -199,15 +199,27 @@ def convert_graph( load_options: Optional[dict] = None, save_options: Optional[dict] = None, save_requirements: bool = True, + package_manager_name: Optional[str] = None, + package_manager_command: Union[None, str, Tuple[str, ...]] = None, ) -> Union[str, dict]: if load_options is None: load_options = dict() if save_options is None: save_options = dict() graph = load_graph(source, inputs=inputs, **load_options) + if save_requirements: + if not package_manager_command: + package_manager_command = tuple() + elif isinstance(package_manager_command, str): + package_manager_command = _split_command(package_manager_command) + try: - _requirements.add_requirements(graph) + _requirements.add_requirements( + graph, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) except Exception: logger.exception("Continue after failure to add workflow requirements") return save_graph(graph, destination, **save_options) @@ -253,7 +265,8 @@ def _print_graph( def install_graph( source, skip_prompt: bool = False, - command: Union[None, str, Tuple[str, ...]] = None, + package_manager_name: Optional[str] = None, + package_manager_command: Union[None, str, Tuple[str, ...]] = None, load_options: Optional[dict] = None, ) -> None: if load_options is None: @@ -262,20 +275,28 @@ def install_graph( requirements = _requirements.get_requirements(graph) - if not command: - command = tuple() - elif isinstance(command, str): - command = _split_command(command) + if not package_manager_command: + package_manager_command = tuple() + elif isinstance(package_manager_command, str): + package_manager_command = _split_command(package_manager_command) if skip_prompt: - _requirements.install_requirements(requirements, command=command) + _requirements.install_requirements( + requirements, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) return answer = input( f"{requirements.__info__()}\n\nThis will install the packages above. Do you want to proceed (y/N)?" ) if answer.lower() == "y" or answer.lower() == "yes": - _requirements.install_requirements(requirements, command=command) + _requirements.install_requirements( + requirements, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) else: raise AbortException() diff --git a/src/ewoks/cli_utils/cli_convert_utils.py b/src/ewoks/cli_utils/cli_convert_utils.py index 1c128a6..caf520c 100644 --- a/src/ewoks/cli_utils/cli_convert_utils.py +++ b/src/ewoks/cli_utils/cli_convert_utils.py @@ -61,6 +61,18 @@ def convert_arguments( action="store_true", help="Do not include the packages of the current Python environment as requirements in the destination workflow.", ), + CLIArg( + "package_manager_name", + ["--package-manager-name"], + type=str, + help='Package manager name to generate requirements. For example "pip"', + ), + CLIArg( + "package_manager_command", + ["--package-manager-command"], + type=str, + help='Package manager command to generate requirements. For example "python -m pip"', + ), ] return args_list @@ -92,4 +104,9 @@ def parse_convert_arguments(cli_args: Namespace, shell: bool = False) -> None: } if cli_args.exclude_requirements: convert_options["save_requirements"] = False + else: + convert_options["save_requirements"] = True + convert_options["package_manager_name"] = cli_args.package_manager_name + convert_options["package_manager_command"] = cli_args.package_manager_command + cli_args.convert_options = convert_options diff --git a/src/ewoks/cli_utils/cli_install_utils.py b/src/ewoks/cli_utils/cli_install_utils.py index a8952d2..ca2bfdd 100644 --- a/src/ewoks/cli_utils/cli_install_utils.py +++ b/src/ewoks/cli_utils/cli_install_utils.py @@ -27,8 +27,14 @@ def install_arguments( help="Automatically accept installation prompts.", ), CLIArg( - "manager_command", - ["-m", "--manager"], + "package_manager_name", + ["--package-manager-name"], + type=str, + help='Package manager name. For example "pip"', + ), + CLIArg( + "package_manager_command", + ["--package-manager-command"], type=str, help='Package manager command. For example "python -m pip"', ), diff --git a/src/ewoks/tests/requirements/pip/test_install_cli.py b/src/ewoks/tests/requirements/pip/test_install_cli.py index a7aadea..e2b11f3 100644 --- a/src/ewoks/tests/requirements/pip/test_install_cli.py +++ b/src/ewoks/tests/requirements/pip/test_install_cli.py @@ -22,7 +22,16 @@ def test_install_pip_with_freeze(venv): } subprocess.check_call( - ["ewoks", "install", "--yes", json.dumps(graph), "-m", f"{venv.python} -m pip"] + [ + "ewoks", + "install", + "--yes", + json.dumps(graph), + "--package-manager-name", + "pip", + "--package-manager-command", + f"{venv.python} -m pip", + ] ) assert venv.get_version("ewoksdata") @@ -47,7 +56,16 @@ def test_install_pip_without_freeze(venv): } subprocess.check_call( - ["ewoks", "install", "--yes", json.dumps(graph), "-m", f"{venv.python} -m pip"] + [ + "ewoks", + "install", + "--yes", + json.dumps(graph), + "--package-manager-name", + "pip", + "--package-manager-command", + f"{venv.python} -m pip", + ] ) assert venv.get_version("ewoksdata") @@ -72,7 +90,9 @@ def test_install_legacy_pip_freeze(venv): "install", "--yes", json.dumps(graph), - "-m", + "--package-manager-name", + "pip", + "--package-manager-command", f"{venv.python} -m pip", ] ) @@ -108,7 +128,9 @@ def test_install_without_requirements(venv): "install", "--yes", json.dumps(graph), - "-m", + "--package-manager-name", + "pip", + "--package-manager-command", f"{venv.python} -m pip", ] ) diff --git a/src/ewoks/tests/test_cli.py b/src/ewoks/tests/test_cli.py index f0cd582..ac8ea4e 100644 --- a/src/ewoks/tests/test_cli.py +++ b/src/ewoks/tests/test_cli.py @@ -33,6 +33,9 @@ def test_cli_convert(cli_interface): # noqa F811 ], "load_options": {"representation": "test_core"}, "save_options": {"representation": "json"}, + "package_manager_command": None, + "package_manager_name": None, + "save_requirements": True, } assert cli_args.convert_options == convert_options From 1c2d1def71fdf7be67bfadfe9638620e4ef91635 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 5 Jun 2026 00:03:03 +0200 Subject: [PATCH 8/9] make some modules private --- .../utils/{supported.py => _supported.py} | 0 src/ewoks/_requirements/utils/base_manager.py | 12 ++++++------ src/ewoks/_requirements/utils/detect.py | 2 +- .../utils/metadata/{unknown.py => _unknown.py} | 0 .../utils/metadata/from_pip_freeze.py | 4 ++-- .../utils/metadata/from_python/__init__.py | 14 +++++++------- .../utils/metadata/from_python/{git.py => _git.py} | 6 +++--- .../metadata/{metadata_models.py => models.py} | 0 src/ewoks/_requirements/utils/parse.py | 2 +- src/ewoks/_requirements/utils/pip_freeze.py | 7 ++++--- .../tests/requirements/pip/test_install_cli.py | 8 +++----- 11 files changed, 27 insertions(+), 28 deletions(-) rename src/ewoks/_requirements/utils/{supported.py => _supported.py} (100%) rename src/ewoks/_requirements/utils/metadata/{unknown.py => _unknown.py} (100%) rename src/ewoks/_requirements/utils/metadata/from_python/{git.py => _git.py} (92%) rename src/ewoks/_requirements/utils/metadata/{metadata_models.py => models.py} (100%) diff --git a/src/ewoks/_requirements/utils/supported.py b/src/ewoks/_requirements/utils/_supported.py similarity index 100% rename from src/ewoks/_requirements/utils/supported.py rename to src/ewoks/_requirements/utils/_supported.py diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 42178b8..bf1e79d 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -10,21 +10,21 @@ from typing import List from typing import Optional -from .metadata import metadata_models +from .metadata import models from .metadata.from_python import current_requirements logger = logging.getLogger(__name__) -class BaseManagerInfo(metadata_models.BaseModel): +class BaseManagerInfo(models.BaseModel): name: str version: str -class BaseRequirements(metadata_models.BaseModel): - system: metadata_models.SystemInfo - python: metadata_models.PythonInfo - distributions: List[metadata_models.Distribution] +class BaseRequirements(models.BaseModel): + system: models.SystemInfo + python: models.PythonInfo + distributions: List[models.Distribution] manager: BaseManagerInfo def __info__(self) -> str: diff --git a/src/ewoks/_requirements/utils/detect.py b/src/ewoks/_requirements/utils/detect.py index 93e18d1..bce99e1 100644 --- a/src/ewoks/_requirements/utils/detect.py +++ b/src/ewoks/_requirements/utils/detect.py @@ -5,9 +5,9 @@ from typing import Optional from typing import Tuple +from ._supported import get_supported_managers from .base_manager import BaseManager from .metadata.from_python import current_requirements -from .supported import get_supported_managers logger = logging.getLogger(__name__) diff --git a/src/ewoks/_requirements/utils/metadata/unknown.py b/src/ewoks/_requirements/utils/metadata/_unknown.py similarity index 100% rename from src/ewoks/_requirements/utils/metadata/unknown.py rename to src/ewoks/_requirements/utils/metadata/_unknown.py diff --git a/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py b/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py index 9f0e2bd..d1a77a6 100644 --- a/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py +++ b/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py @@ -2,10 +2,10 @@ from typing import Dict from typing import List -from . import unknown +from . import _unknown def pip_freeze_requirements(freeze: List[str]) -> Dict[str, Any]: - metadata = unknown.unknown_requirements() + metadata = _unknown.unknown_requirements() metadata["manager"] = dict(name="pip", version="", freeze=freeze) return metadata diff --git a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py index f50f0f8..faaf394 100644 --- a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py +++ b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py @@ -7,8 +7,8 @@ from typing import Any from typing import Dict -from .. import metadata_models -from . import git +from .. import models +from . import _git logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def _python_metadata() -> Dict[str, Any]: def _distribution_from_metadata( dist: metadata.Distribution, -) -> metadata_models.Distribution: +) -> models.Distribution: name = dist.metadata["Name"] version = dist.version @@ -80,7 +80,7 @@ def _distribution_from_metadata( if vcs_info.get("vcs") == "git": commit_id = vcs_info.get("commit_id") if commit_id: - git_info = metadata_models.GitInfo( + git_info = models.GitInfo( commit=commit_id, remote=url, uncommitted_changes=False ) has_info = True @@ -89,7 +89,7 @@ def _distribution_from_metadata( if url.startswith("file://") or "://" not in url: path = Path(url.replace("file://", "")) if path.exists(): - git_info = git.git_info_from_path(path) + git_info = _git.git_info_from_path(path) has_info = git_info is not None if not has_info and "archive_info" in direct_url: @@ -105,10 +105,10 @@ def _distribution_from_metadata( else: hashes = {} - archive_info = metadata_models.ArchiveInfo(url=url, hashes=hashes) + archive_info = models.ArchiveInfo(url=url, hashes=hashes) has_info = True - return metadata_models.Distribution( + return models.Distribution( name=name, version=version, git=git_info, diff --git a/src/ewoks/_requirements/utils/metadata/from_python/git.py b/src/ewoks/_requirements/utils/metadata/from_python/_git.py similarity index 92% rename from src/ewoks/_requirements/utils/metadata/from_python/git.py rename to src/ewoks/_requirements/utils/metadata/from_python/_git.py index 489f117..15f6293 100644 --- a/src/ewoks/_requirements/utils/metadata/from_python/git.py +++ b/src/ewoks/_requirements/utils/metadata/from_python/_git.py @@ -3,10 +3,10 @@ from typing import List from typing import Optional -from .. import metadata_models +from .. import models -def git_info_from_path(path: Path) -> Optional[metadata_models.GitInfo]: +def git_info_from_path(path: Path) -> Optional[models.GitInfo]: """ Return GitInfo for a local path, or None if not a git repository. """ @@ -29,7 +29,7 @@ def git_info_from_path(path: Path) -> Optional[metadata_models.GitInfo]: else: remote_url = None - return metadata_models.GitInfo( + return models.GitInfo( commit=commit, remote=remote_url, uncommitted_changes=uncommitted_changes ) diff --git a/src/ewoks/_requirements/utils/metadata/metadata_models.py b/src/ewoks/_requirements/utils/metadata/models.py similarity index 100% rename from src/ewoks/_requirements/utils/metadata/metadata_models.py rename to src/ewoks/_requirements/utils/metadata/models.py diff --git a/src/ewoks/_requirements/utils/parse.py b/src/ewoks/_requirements/utils/parse.py index da784cf..91e2a5d 100644 --- a/src/ewoks/_requirements/utils/parse.py +++ b/src/ewoks/_requirements/utils/parse.py @@ -1,9 +1,9 @@ from typing import List from typing import Union +from ._supported import get_supported_managers from .base_manager import BaseRequirements from .metadata.from_pip_freeze import pip_freeze_requirements -from .supported import get_supported_managers def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: diff --git a/src/ewoks/_requirements/utils/pip_freeze.py b/src/ewoks/_requirements/utils/pip_freeze.py index a93d7d5..38697ad 100644 --- a/src/ewoks/_requirements/utils/pip_freeze.py +++ b/src/ewoks/_requirements/utils/pip_freeze.py @@ -11,14 +11,15 @@ from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement -from .metadata import metadata_models +from .metadata import models def freeze_distribution( - dist: metadata_models.Distribution, + dist: models.Distribution, ) -> Tuple[List[str], List[str]]: """ - Return the pip freeze of the distribution with associated warnings regarding reproducibility. + Return the pip freeze argument corresponding to the distribution with + associated warnings regarding reproducibility. """ lines = [] warnings = [] diff --git a/src/ewoks/tests/requirements/pip/test_install_cli.py b/src/ewoks/tests/requirements/pip/test_install_cli.py index e2b11f3..bab1336 100644 --- a/src/ewoks/tests/requirements/pip/test_install_cli.py +++ b/src/ewoks/tests/requirements/pip/test_install_cli.py @@ -3,15 +3,14 @@ import pytest -from ...._requirements.utils.metadata import unknown +from ...._requirements.utils.metadata import from_pip_freeze def test_install_pip_with_freeze(venv): with pytest.raises(Exception, match="package is not installed"): _ = venv.get_version("ewoksdata") - requirements = unknown.unknown_requirements() - requirements["manager"] = {"name": "pip", "version": "", "freeze": ["ewoksdata"]} + requirements = from_pip_freeze.pip_freeze_requirements(["ewoksdata"]) graph = { "graph": { @@ -41,8 +40,7 @@ def test_install_pip_without_freeze(venv): with pytest.raises(Exception, match="package is not installed"): _ = venv.get_version("ewoksdata") - requirements = unknown.unknown_requirements() - requirements["manager"] = {"name": "pip", "version": "", "freeze": []} + requirements = from_pip_freeze.pip_freeze_requirements(["ewoksdata"]) requirements["distributions"] = [ {"name": "ewoksdata", "version": ""}, ] From 19dcca92a19effb68f9c2f0e24b39fd3ca54ab97 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 5 Jun 2026 00:09:41 +0200 Subject: [PATCH 9/9] add distributions to last-resort requirements when installing graphs that do not have requirements --- src/ewoks/_requirements/utils/metadata/last_resort.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ewoks/_requirements/utils/metadata/last_resort.py b/src/ewoks/_requirements/utils/metadata/last_resort.py index ffe0def..fb650ec 100644 --- a/src/ewoks/_requirements/utils/metadata/last_resort.py +++ b/src/ewoks/_requirements/utils/metadata/last_resort.py @@ -1,6 +1,7 @@ import logging from typing import Any from typing import Dict +from typing import Set from ewokscore.graph import TaskGraph @@ -13,7 +14,8 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: """Last resort when installing a workflow that does not have requirements: guess the requirements from the workflow nodes. """ - freeze: set[str] = set() + freeze: Set[str] = set() + distributions: Set[str] = set() for node_id, node in graph.graph.nodes.items(): task_identifier = node["task_identifier"] @@ -28,6 +30,7 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: continue freeze.add(package) + distributions.add(package) elif task_type == "notebook": logger.warning( @@ -44,4 +47,8 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: f"Could not extract requirements for node {node_id}: unsupported task type {task_type}." ) - return pip_freeze_requirements(list(freeze)) + requirements = pip_freeze_requirements(sorted(freeze)) + requirements["distributions"] = [ + {"name": name, "version": ""} for name in sorted(distributions) + ] + return requirements