diff --git a/CHANGELOG.md b/CHANGELOG.md index 96f99a0..832a822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `ewoks install`: add `--package-manager-name` and `--package-manager-command` arguments. +- `ewoks convert`: add `--package-manager-name` and `--package-manager-command` arguments. + +### Removed + +- `ewoks install`: remove `-p/--python` argument. + ## [5.0.0] - 2026-04-27 - Upgrade `ewoksdask` to 3.* diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 3436fa9..416a896 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -159,7 +159,12 @@ def command_install( cli_install_utils.parse_install_arguments(cli_args, shell=shell) for workflow, graph in zip(cli_args.workflows, cli_args.graphs): try: - install_graph(graph, cli_args.yes, cli_args.python) + install_graph( + graph, + skip_prompt=cli_args.yes, + package_manager_name=cli_args.package_manager_name, + package_manager_command=cli_args.package_manager_command, + ) except CalledProcessError as e: print(f"Install failed for {workflow}: {e}") except AbortException: diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index afa8fed..6aaa98b 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -1 +1,101 @@ """Workflow requirements.""" + +import logging +from typing import Optional +from typing import Tuple +from typing import Union + +from ewokscore.graph import TaskGraph + +from .utils import parse +from .utils.base_manager import BaseRequirements +from .utils.detect import get_manager +from .utils.metadata import last_resort + +logger = logging.getLogger(__file__) + + +def add_requirements( + graph: TaskGraph, + manager_name: Optional[str] = None, + manager_command: Tuple[str, ...] = tuple(), +) -> None: + """Add requirements to a workflow definition in-place.""" + manager = get_manager(manager_name=manager_name, manager_command=manager_command) + requirements = manager.gather_requirements() + graph.graph.graph["requirements"] = requirements.model_dump() + + +def get_requirements(graph: TaskGraph) -> BaseRequirements: + """Extract requirements from a workflow definition.""" + requirements = graph.graph.graph.get("requirements", None) + no_requirements = not requirements + + if no_requirements: + logger.warning( + "BaseRequirements field is empty. Trying to extract requirements automatically..." + ) + requirements = last_resort.last_resort_requirements(graph) + + requirements = parse.parse_requirements(requirements) + + if no_requirements: + logger.info(f"Extracted the following requirements: {requirements.__info__()}") + + return requirements + + +def install_requirements( + requirements: BaseRequirements, + manager_name: Optional[str] = None, + manager_command: Union[None, str, Tuple[str, ...]] = None, +) -> None: + """Install workflow requirements.""" + + if manager_command and not manager_name: + raise ValueError( + f"Provide 'manager_name' associated to command {manager_command}" + ) + + try: + if manager_name: + raise ValueError("Ignore package manager used to generate the requirements") + else: + manager = get_manager(manager_name=requirements.manager.name) + except ValueError: + manager = get_manager( + manager_name=manager_name, manager_command=manager_command + ) + + manager.install_requirements(requirements) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + import time + from pprint import pprint + + t0 = time.perf_counter() + + try: + manager = get_manager(manager_name="pip") + requirements = manager.gather_requirements() + + print() + print("Model:") + pprint(requirements.model_dump()) + finally: + print("Freeze time:", time.perf_counter() - t0) + + pip_freeze = requirements.manager.freeze + + dists_freeze = manager._freeze_distributions(requirements) + dists_freeze = [s for s in dists_freeze if not s.startswith("#")] + + print() + print("pip freeze has these extra's:") + pprint(set(pip_freeze) - set(dists_freeze)) + + print() + print("native freeze has these extra's:") + pprint(set(dists_freeze) - set(pip_freeze)) diff --git a/src/ewoks/_requirements/conda.py b/src/ewoks/_requirements/conda.py new file mode 100644 index 0000000..d4eacaf --- /dev/null +++ b/src/ewoks/_requirements/conda.py @@ -0,0 +1,80 @@ +import logging +import os +import sys +from typing import Any +from typing import Dict +from typing import Literal +from typing import Optional +from typing import Tuple + +import yaml + +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + +logger = logging.getLogger(__name__) + + +class CondaManagerInfo(BaseManagerInfo): + name: Literal["conda"] = "conda" + + +class CondaRequirements(BaseRequirements): + manager: CondaManagerInfo + environment: dict + + +class CondaManager(BaseManager): + NAME = "conda" + PRIORITY = 4 + REQUIREMENTS_MODEL = CondaRequirements + + def __init__(self, *command: str) -> None: + if not command: + command = self._get_conda_command() + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output("--version") + except RuntimeError: + return None + return output.strip().split(" ")[-1] + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "CONDA_PREFIX" in os.environ or os.path.exists( + os.path.join(sys.prefix, "conda-meta") + ) + + def _gather_requirements(self) -> Dict[str, Any]: + output = self._check_output("env", "export") + environment = yaml.safe_load(output) + environment.pop("name", None) + environment.pop("prefix", None) + + return {"environment": environment} + + def _install_native_requirements(self, requirements: CondaRequirements) -> bool: + text = yaml.safe_dump(requirements.environment) + with self._temporary_file(text, ".yml") as tmp_path: + self._check_call("env", "update", "-f", tmp_path) + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") + + def _get_conda_command(self) -> Tuple[str, ...]: + try: + _ = self._check_output_raw("mamba", "--version") + return ("mamba",) + except Exception: + pass + try: + _ = self._check_output_raw("micromamba", "--version") + return ("micromamba",) + except Exception: + pass + return ("conda",) diff --git a/src/ewoks/_requirements/pip.py b/src/ewoks/_requirements/pip.py new file mode 100644 index 0000000..f1bd48d --- /dev/null +++ b/src/ewoks/_requirements/pip.py @@ -0,0 +1,92 @@ +import importlib.metadata +import logging +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Literal +from typing import Optional + +from .utils import pip_freeze +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + +logger = logging.getLogger(__name__) + + +class PipManagerInfo(BaseManagerInfo): + name: Literal["pip"] = "pip" + freeze: List[str] + + +class PipRequirements(BaseRequirements): + manager: PipManagerInfo + + def __info__(self) -> str: + freeze = "\n ".join(self.manager.freeze) + return f"{super().__info__()}\nRequirements:\n {freeze}" + + +class PipManager(BaseManager): + NAME = "pip" + PRIORITY = 0 + REQUIREMENTS_MODEL = PipRequirements + + def __init__(self, *command: str) -> None: + if not command: + command = sys.executable, "-m", "pip" + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("pip") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return False + + def _gather_requirements(self) -> Dict[str, Any]: + freeze_output = self._check_output("freeze").strip().splitlines() + + return {"freeze": freeze_output} + + def _install_native_requirements(self, requirements: PipRequirements) -> bool: + freeze = requirements.manager.freeze + + if not freeze: + return False + + arguments = self._arguments(freeze) + self._check_call("install", "--no-cache-dir", *arguments) + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + freeze = self._freeze_distributions(requirements) + if not freeze: + return False + + arguments = self._arguments(freeze) + self._check_call("install", "--no-cache-dir", *arguments) + return True + + def _freeze_distributions(self, requirements: BaseRequirements) -> List[str]: + """ + Pip freeze argument from list of distributions. + """ + freeze = [] + for dist in requirements.distributions: + lines, warnings = pip_freeze.freeze_distribution(dist) + for warning in warnings: + logger.warning(warning) + freeze.extend(lines) + return freeze + + def _arguments(self, freeze: List[str]) -> List[str]: + arguments, warnings = pip_freeze.sanitize_freeze(freeze) + for warning in warnings: + logger.warning(warning) + return arguments diff --git a/src/ewoks/_requirements/pip/__init__.py b/src/ewoks/_requirements/pip/__init__.py deleted file mode 100644 index 312a112..0000000 --- a/src/ewoks/_requirements/pip/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Workflow requirements managed by `pip`.""" diff --git a/src/ewoks/_requirements/pip/install.py b/src/ewoks/_requirements/pip/install.py deleted file mode 100644 index d71a622..0000000 --- a/src/ewoks/_requirements/pip/install.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -import subprocess -from typing import Sequence - -from .sanitize import sanitize_requirements - -logger = logging.getLogger(__name__) - - -def pip_install(requirements: Sequence[str], python_path: str) -> int: - requirements, warnings = sanitize_requirements(requirements) - for warning in warnings: - logger.warning(warning) - # https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program - return subprocess.check_call([python_path, "-m", "pip", "install", *requirements]) diff --git a/src/ewoks/_requirements/pipenv.py b/src/ewoks/_requirements/pipenv.py new file mode 100644 index 0000000..7153576 --- /dev/null +++ b/src/ewoks/_requirements/pipenv.py @@ -0,0 +1,74 @@ +import importlib.metadata +import json +import os +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Literal +from typing import Optional + +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class PipenvManagerInfo(BaseManagerInfo): + name: Literal["pipenv"] = "pipenv" + requirements: List[str] + + +class PipenvRequirements(BaseRequirements): + manager: PipenvManagerInfo + + def __info__(self) -> str: + requirements = "\n ".join(self.manager.requirements) + return f"{super().__info__()}\nRequirements:\n {requirements}" + + +class PipenvManager(BaseManager): + NAME = "pipenv" + PRIORITY = 3 + REQUIREMENTS_MODEL = PipenvRequirements + + def __init__(self, *command: str) -> None: + if not command: + command = sys.executable, "-m", "pipenv" + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("pipenv") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "PIPENV_ACTIVE" in os.environ + + def _gather_requirements(self) -> Dict[str, Any]: + output = self._check_output("lock", "--requirements") + requirements = output.strip().splitlines() + + return {"requirements": requirements} + + def _install_native_requirements(self, requirements: PipenvRequirements) -> bool: + lock_data = { + "_meta": {"hash": {"sha256": "dummy"}}, # minimal metadata + "default": { + pkg.split("==")[0]: {"version": pkg.split("==")[1]} + for pkg in requirements.requirements + }, + "develop": { + pkg.split("==")[0]: {"version": pkg.split("==")[1]} + for pkg in getattr(requirements, "dev_requirements", []) + }, + } + text = json.dumps(lock_data, indent=2) + + with self._temporary_file(text, ".lock") as tmp_path: + self._check_call("sync", "--ignore-pipfile", "-f", tmp_path) + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/pixi.py b/src/ewoks/_requirements/pixi.py new file mode 100644 index 0000000..ebdb2cb --- /dev/null +++ b/src/ewoks/_requirements/pixi.py @@ -0,0 +1,62 @@ +import os +from typing import Any +from typing import Dict +from typing import Literal +from typing import Optional + +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class PixiManagerInfo(BaseManagerInfo): + name: Literal["pixi"] = "pixi" + lockfile: str + + +class PixiRequirements(BaseRequirements): + manager: PixiManagerInfo + + +class PixiManager(BaseManager): + NAME = "pixi" + PRIORITY = 5 + REQUIREMENTS_MODEL = PixiRequirements + + def __init__(self, *command: str) -> None: + if not command: + command = ("pixi",) + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output("--version", text=True) + return output.strip().split(" ")[-1] + except Exception: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "PIXI_PROJECT_ROOT" in os.environ + + def _gather_requirements(self) -> Dict[str, Any]: + if os.path.exists("pixi.lock"): + with open("pixi.lock", "r", encoding="utf-8") as f: + lock_content = f.read() + elif os.path.exists("pixi.toml"): + with open("pixi.toml", "r", encoding="utf-8") as f: + lock_content = f.read() + else: + raise RuntimeError("No pixi.lock or pixi.toml file found") + + return {"lockfile": lock_content} + + def _install_requirements(self, requirements: PixiRequirements) -> bool: + with self._temporary_file(requirements.lockfile, ".lock") as tmp_path: + self._check_call("install", cwd=os.path.dirname(tmp_path)) + + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/poetry.py b/src/ewoks/_requirements/poetry.py new file mode 100644 index 0000000..b9190d9 --- /dev/null +++ b/src/ewoks/_requirements/poetry.py @@ -0,0 +1,59 @@ +import importlib.metadata +import os +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Literal +from typing import Optional + +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class PoetryManagerInfo(BaseManagerInfo): + name: Literal["poetry"] = "pip" + requirements: List[str] + + +class PoetryRequirements(BaseRequirements): + manager: PoetryManagerInfo + + +class PoetryManager(BaseManager): + NAME = "poetry" + PRIORITY = 2 + REQUIREMENTS_MODEL = PoetryRequirements + + def __init__(self, *command: str) -> None: + if not command: + command = sys.executable, "-m", "poetry" + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("poetry") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return "POETRY_ACTIVE" in os.environ + + def _gather_requirements(self) -> Dict[str, Any]: + output = self._check_output("export", "--without-hashes") + requirements = output.strip().splitlines() + + return {"requirements": requirements} + + def _install_native_requirements(self, requirements: PoetryRequirements) -> bool: + text = "\n".join(requirements.requirements) + with self._temporary_file(text, ".txt") as tmp_path: + self._check_call("add", "--lock", "--file", tmp_path) + + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/utils/__init__.py b/src/ewoks/_requirements/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/_requirements/utils/_supported.py b/src/ewoks/_requirements/utils/_supported.py new file mode 100644 index 0000000..a9bec1f --- /dev/null +++ b/src/ewoks/_requirements/utils/_supported.py @@ -0,0 +1,25 @@ +from functools import lru_cache +from typing import Dict +from typing import Type + +from ..pip import PipManager +from .base_manager import BaseManager + +# from ..conda import CondaManager +# from ..pipenv import PipenvManager +# from ..pixi import PixiManager +# from ..poetry import PoetryManager +# from ..uv import UvManager + + +@lru_cache(1) +def get_supported_managers() -> Dict[str, Type[BaseManager]]: + managers = [ + PipManager, + # UvManager, + # PoetryManager, + # PipenvManager, + # CondaManager, + # PixiManager, + ] + return {manager_cls.NAME: manager_cls for manager_cls in managers} diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py new file mode 100644 index 0000000..bf1e79d --- /dev/null +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -0,0 +1,187 @@ +import logging +import os +import subprocess +import tempfile +from abc import abstractmethod +from contextlib import contextmanager +from typing import Any +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional + +from .metadata import models +from .metadata.from_python import current_requirements + +logger = logging.getLogger(__name__) + + +class BaseManagerInfo(models.BaseModel): + name: str + version: str + + +class BaseRequirements(models.BaseModel): + system: models.SystemInfo + python: models.PythonInfo + distributions: List[models.Distribution] + manager: BaseManagerInfo + + def __info__(self) -> str: + return ( + f"Manager: {self.manager.name} ({self.manager.version}) " + f"python={self.python.version}) " + f"distributions={len(self.distributions)}" + ) + + +class BaseManager: + """Defines the interface all package managers must implement. + + If `MyManager` is an implementation of this interface then + Ewoks workflow requirements can be obtained like this: + + .. code-block:: python + + manager = MyManager() + requirements = manager.gather_requirements() + + Ewoks workflow requirements can be installed like this: + + .. code-block:: python + + manager = MyManager() + manager.install_requirements(requirements) + """ + + NAME = NotImplemented + PRIORITY = NotImplemented + REQUIREMENTS_MODEL = NotImplemented + + def __init__(self, *command: str) -> None: + if not command: + raise ValueError(f"{type(self).__name__} needs an associated shell command") + self._cmd_args = command + + def __repr__(self): + return f"{type(self).__name__}({', '.join(self._cmd_args)})" + + @abstractmethod + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + pass + + @abstractmethod + def is_active(self) -> bool: + """Manager is explicitly active.""" + pass + + def gather_requirements(self) -> Optional[BaseRequirements]: + """ + Return requirements associated to the current python environment. + + :raises RuntimeError: package manager not available + """ + manager_version = self.version() + if manager_version is None: + raise RuntimeError(f"{self.NAME!r} is not available") + + try: + parameters = self._gather_requirements() + except Exception as ex: + logger.error( + "%s: failed to generate requirements (%s)", type(self).__name__, ex + ) + return None + + manager = dict(name=self.NAME, version=manager_version, **parameters) + return self.REQUIREMENTS_MODEL(manager=manager, **current_requirements()) + + def install_requirements(self, requirements: BaseRequirements) -> None: + """ + Install requirements into the current environment. + + :raises ValueError: no distibutions provided to install + """ + try: + return self._install_requirements(requirements) + except Exception as ex: + logger.error( + "%s: failed to install requirements (%s)", type(self).__name__, ex + ) + raise + + def _install_requirements(self, requirements: BaseRequirements) -> None: + reraise = None + + if isinstance(requirements, self.REQUIREMENTS_MODEL): + try: + if self._install_native_requirements(requirements): + return + except Exception as ex: + reraise = ex + logger.debug( + ( + "Failed installing requirements native to package %s. " + "Try installing python distributions." + ), + self.NAME, + ex, + ) + pass + + if self._install_base_requirements(requirements): + return + + if reraise: + raise reraise + raise ValueError("No distibutions provided to install") + + @abstractmethod + def _gather_requirements(self) -> Dict[str, Any]: + pass + + @abstractmethod + def _install_native_requirements(self, requirements: BaseRequirements) -> bool: + pass + + @abstractmethod + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + pass + + def _check_output(self, *args: str) -> str: + return self._check_output_raw(*[*self._cmd_args, *args]) + + def _check_call(self, *args: str) -> int: + return self._check_call_raw(*[*self._cmd_args, *args]) + + @staticmethod + def _check_output_raw(*args: str) -> str: + try: + return subprocess.check_output(args, text=True) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex + + @staticmethod + def _check_call_raw(*args: str) -> int: + try: + return subprocess.check_call(args) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex + + @contextmanager + def _temporary_file(self, text: str, suffix: str) -> Generator[str, None, None]: + tmp_path = None + try: + with tempfile.NamedTemporaryFile("w", suffix=suffix, delete=False) as tmp: + tmp.write(text) + tmp_path = tmp.name + + yield tmp_path + + finally: + if tmp_path: + try: + os.remove(tmp_path) + except OSError: + logger.debug("Could not delete temporary file: %s", tmp_path) diff --git a/src/ewoks/_requirements/utils/detect.py b/src/ewoks/_requirements/utils/detect.py new file mode 100644 index 0000000..bce99e1 --- /dev/null +++ b/src/ewoks/_requirements/utils/detect.py @@ -0,0 +1,113 @@ +import logging +from collections import Counter +from functools import lru_cache +from typing import Dict +from typing import Optional +from typing import Tuple + +from ._supported import get_supported_managers +from .base_manager import BaseManager +from .metadata.from_python import current_requirements + +logger = logging.getLogger(__name__) + + +def get_manager( + manager_name: Optional[str] = None, + manager_command: Tuple[str, ...] = tuple(), +) -> BaseManager: + """ + :raise ValueError: package manager not support or not available + :raise RuntimeError: no package manager available + """ + if manager_name: + return _select_manager(manager_name, manager_command) + + if manager_command: + raise ValueError( + f"Provide 'manager_name' associated to command {manager_command}" + ) + + manager = _detect_manager() + + if manager is None: + raise RuntimeError("No known package manager installed or available") + + return manager + + +def _select_manager(manager_name: str, manager_command: Tuple[str, ...]) -> BaseManager: + managers = get_supported_managers() + + manager_cls = managers.get(manager_name) + if manager_cls is None: + raise ValueError(f"Package manager {manager_name!r} is not supported") + + manager = manager_cls(*manager_command) + if not manager.version: + raise ValueError(f"Package manager {manager_name!r} is not available") + + return manager + + +def _detect_manager() -> Optional[BaseManager]: + # Available package managers + available_managers = { + name: manager_cls() for name, manager_cls in get_supported_managers().items() + } + available_managers = { + name: manager + for name, manager in available_managers.items() + if manager.version() + } + if not available_managers: + return None + + # Select the active manager with the highest priority + active_managers = { + name: manager + for name, manager in available_managers.items() + if manager.is_active() + } + if active_managers: + name = max(active_managers, key=lambda name: active_managers[name].PRIORITY) + manager = active_managers[name] + logger.debug( + "Detected active %r package manager\n available = %s\n active = %s", + name, + list(available_managers), + list(active_managers), + ) + return manager + + # Infer most likely package manager + counts = _installer_distribution_count() + if set(counts) & set(available_managers): + # Use the number of installed distributions as the score + crit = "distribution count" + scores = {name: counts.get(name, -1) for name in available_managers} + else: + # Use the package manager priority as the score + crit = "priority" + scores = { + name: manager.PRIORITY for name, manager in available_managers.items() + } + + name = max(scores, key=scores.get) + logger.debug( + "Package manager selection based on %s\n %s", + crit, + "\n ".join( + f"{k} = {v} {'(SELECTED)' if k == name else ''}" for k, v in scores.items() + ), + ) + return available_managers[name] + + +@lru_cache(1) +def _installer_distribution_count() -> Dict[str, int]: + counts: Counter = Counter() + for dist in current_requirements()["distributions"]: + if dist.installer: + counts[dist.installer] += 1 + return dict(counts) diff --git a/src/ewoks/_requirements/utils/metadata/__init__.py b/src/ewoks/_requirements/utils/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/_requirements/utils/metadata/_unknown.py b/src/ewoks/_requirements/utils/metadata/_unknown.py new file mode 100644 index 0000000..892da64 --- /dev/null +++ b/src/ewoks/_requirements/utils/metadata/_unknown.py @@ -0,0 +1,31 @@ +from functools import lru_cache +from typing import Any +from typing import Dict + + +@lru_cache(1) +def unknown_requirements() -> Dict[str, Any]: + return dict( + system=_unknown_system_metadata(), + python=_unknown_python_metadata(), + distributions=list(), + ) + + +def _unknown_system_metadata() -> Dict[str, Any]: + return dict( + system="", + release="", + version="", + machine="", + processor="", + ) + + +def _unknown_python_metadata() -> Dict[str, Any]: + return dict( + version="", + implementation="", + compiler="", + build="", + ) diff --git a/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py b/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py new file mode 100644 index 0000000..d1a77a6 --- /dev/null +++ b/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py @@ -0,0 +1,11 @@ +from typing import Any +from typing import Dict +from typing import List + +from . import _unknown + + +def pip_freeze_requirements(freeze: List[str]) -> Dict[str, Any]: + metadata = _unknown.unknown_requirements() + metadata["manager"] = dict(name="pip", version="", freeze=freeze) + return metadata diff --git a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py new file mode 100644 index 0000000..faaf394 --- /dev/null +++ b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py @@ -0,0 +1,117 @@ +import json +import logging +import platform +from functools import lru_cache +from importlib import metadata +from pathlib import Path +from typing import Any +from typing import Dict + +from .. import models +from . import _git + +logger = logging.getLogger(__name__) + + +@lru_cache(1) +def current_requirements() -> Dict[str, Any]: + """ + Return installed python distributions. + """ + distributions = [ + _distribution_from_metadata(dist) for dist in metadata.distributions() + ] + return dict( + system=_system_metadata(), + python=_python_metadata(), + distributions=distributions, + ) + + +def _system_metadata() -> Dict[str, Any]: + uname = platform.uname() + return dict( + system=uname.system, + release=uname.release, + version=uname.version, + machine=uname.machine, + processor=uname.processor, + ) + + +def _python_metadata() -> Dict[str, Any]: + return dict( + version=platform.python_version(), + implementation=platform.python_implementation(), + compiler=platform.python_compiler(), + build=", ".join(platform.python_build()), + ) + + +def _distribution_from_metadata( + dist: metadata.Distribution, +) -> models.Distribution: + name = dist.metadata["Name"] + version = dist.version + + try: + installer = dist.read_text("INSTALLER") or "" + installer = installer.strip() + if not installer: + installer = None + except Exception: + installer = None + + # PEP 610: Direct URL Origin of installed distributions + # https://packaging.python.org/en/latest/specifications/direct-url-data-structure/ + git_info = None + archive_info = None + try: + direct_url_text = dist.read_text("direct_url.json") + direct_url = json.loads(direct_url_text) if direct_url_text else {} + except Exception: + direct_url = {} + + url = direct_url.get("url", "") + has_info = False + + if not has_info and "vcs_info" in direct_url: + vcs_info = direct_url["vcs_info"] + if vcs_info.get("vcs") == "git": + commit_id = vcs_info.get("commit_id") + if commit_id: + git_info = models.GitInfo( + commit=commit_id, remote=url, uncommitted_changes=False + ) + has_info = True + + if not has_info and "dir_info" in direct_url: + if url.startswith("file://") or "://" not in url: + path = Path(url.replace("file://", "")) + if path.exists(): + git_info = _git.git_info_from_path(path) + has_info = git_info is not None + + if not has_info and "archive_info" in direct_url: + if not url.startswith("file://") and "://" in url: + archive = direct_url["archive_info"] + + if "hashes" in archive: + hashes = archive["hashes"] + elif "hash" in archive: + # deprecated + algo, value = archive["hash"].split("=", 1) + hashes = {algo: value} + else: + hashes = {} + + archive_info = models.ArchiveInfo(url=url, hashes=hashes) + has_info = True + + return models.Distribution( + name=name, + version=version, + git=git_info, + archive=archive_info, + installer=installer, + ) diff --git a/src/ewoks/_requirements/utils/metadata/from_python/_git.py b/src/ewoks/_requirements/utils/metadata/from_python/_git.py new file mode 100644 index 0000000..15f6293 --- /dev/null +++ b/src/ewoks/_requirements/utils/metadata/from_python/_git.py @@ -0,0 +1,73 @@ +import subprocess +from pathlib import Path +from typing import List +from typing import Optional + +from .. import models + + +def git_info_from_path(path: Path) -> Optional[models.GitInfo]: + """ + Return GitInfo for a local path, or None if not a git repository. + """ + try: + commit = _git(["rev-parse", "HEAD"], path) + except Exception: + return None + + try: + uncommitted_changes = bool(_git(["status", "--porcelain"], path)) + except Exception: + uncommitted_changes = False + + remote_name = _find_remote_for_commit(path, commit) + if remote_name: + try: + remote_url = _git(["remote", "get-url", remote_name], path) + except Exception: + remote_url = None + else: + remote_url = None + + return models.GitInfo( + commit=commit, remote=remote_url, uncommitted_changes=uncommitted_changes + ) + + +def _find_remote_for_commit(path: Path, commit: str) -> Optional[str]: + """ + Return the name of a remote that contains the given commit. + """ + try: + remotes = _git_lines(["remote"], path) + except Exception: + return None + + if not remotes: + return None + + if "origin" in remotes: + remotes = ["origin"] + [r for r in remotes if r != "origin"] + + try: + branches = _git_lines(["branch", "-r", "--contains", commit], path) + except Exception: + return None + + for remote in remotes: + prefix = f"{remote}/" + if any(b.startswith(prefix) for b in branches): + return remote + + return None + + +def _git(cmd: List[str], repo: Path) -> str: + return subprocess.check_output( + ["git"] + cmd, cwd=repo, stderr=subprocess.DEVNULL, text=True + ).strip() + + +def _git_lines(cmd: List[str], repo: Path) -> List[str]: + out = _git(cmd, repo) + return [line.strip() for line in out.splitlines() if line.strip()] diff --git a/src/ewoks/_requirements/pip/extract.py b/src/ewoks/_requirements/utils/metadata/last_resort.py similarity index 59% rename from src/ewoks/_requirements/pip/extract.py rename to src/ewoks/_requirements/utils/metadata/last_resort.py index 2cdce7d..fb650ec 100644 --- a/src/ewoks/_requirements/pip/extract.py +++ b/src/ewoks/_requirements/utils/metadata/last_resort.py @@ -1,15 +1,21 @@ import logging -import subprocess -import sys -from typing import List +from typing import Any +from typing import Dict +from typing import Set from ewokscore.graph import TaskGraph -logger = logging.getLogger(__file__) +from .from_pip_freeze import pip_freeze_requirements +logger = logging.getLogger(__name__) -def extract_pip_requirements(graph: TaskGraph) -> List[str]: - imports: set[str] = set() + +def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: + """Last resort when installing a workflow that does not have requirements: + guess the requirements from the workflow nodes. + """ + freeze: Set[str] = set() + distributions: Set[str] = set() for node_id, node in graph.graph.nodes.items(): task_identifier = node["task_identifier"] @@ -23,13 +29,14 @@ def extract_pip_requirements(graph: TaskGraph) -> List[str]: ) continue - imports.add(package) + freeze.add(package) + distributions.add(package) elif task_type == "notebook": logger.warning( f"Requirement extraction may be incomplete for node {node_id}: {task_type} is only partially supported." ) - imports.add("ewokscore[notebook]") + freeze.add("ewokscore[notebook]") elif task_type == "script": logger.warning( @@ -40,18 +47,8 @@ def extract_pip_requirements(graph: TaskGraph) -> List[str]: f"Could not extract requirements for node {node_id}: unsupported task type {task_type}." ) - return list(imports) - - -def add_current_env_pip_requirements(graph: TaskGraph) -> TaskGraph: - try: - freeze_output = subprocess.check_output( - [sys.executable, "-m", "pip", "freeze"], text=True - ) - except subprocess.CalledProcessError as ex: - logger.warning("Cannot generate list of requirements with 'pip' (%s).", ex) - return graph - - requirements = freeze_output.strip().split("\n") - graph.graph.graph["requirements"] = requirements - return graph + requirements = pip_freeze_requirements(sorted(freeze)) + requirements["distributions"] = [ + {"name": name, "version": ""} for name in sorted(distributions) + ] + return requirements diff --git a/src/ewoks/_requirements/utils/metadata/models.py b/src/ewoks/_requirements/utils/metadata/models.py new file mode 100644 index 0000000..c446438 --- /dev/null +++ b/src/ewoks/_requirements/utils/metadata/models.py @@ -0,0 +1,39 @@ +from typing import Dict +from typing import Optional + +from pydantic import BaseModel +from pydantic import Field + + +class SystemInfo(BaseModel): + system: str + release: str + version: str + machine: str + processor: str + + +class PythonInfo(BaseModel): + version: str + implementation: str + compiler: str + build: str + + +class GitInfo(BaseModel): + commit: str + remote: Optional[str] = None + uncommitted_changes: bool = Field(default=False, description="Uncommited changes") + + +class ArchiveInfo(BaseModel): + url: str + hashes: Dict[str, str] + + +class Distribution(BaseModel): + name: str + version: str + git: Optional[GitInfo] = None + archive: Optional[ArchiveInfo] = None + installer: Optional[str] = None diff --git a/src/ewoks/_requirements/utils/parse.py b/src/ewoks/_requirements/utils/parse.py new file mode 100644 index 0000000..91e2a5d --- /dev/null +++ b/src/ewoks/_requirements/utils/parse.py @@ -0,0 +1,25 @@ +from typing import List +from typing import Union + +from ._supported import get_supported_managers +from .base_manager import BaseRequirements +from .metadata.from_pip_freeze import pip_freeze_requirements + + +def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: + if isinstance(requirements, list): + # Legacy 'pip freeze' list + requirements = pip_freeze_requirements(requirements) + + if not isinstance(requirements, dict): + raise TypeError( + f"Graph requirements must be a list or dictionary (type: {type(requirements)})" + ) + + manager_name = requirements.get("manager", dict()).get("name") + + managers = get_supported_managers() + if manager_name not in managers: + raise ValueError(f"{manager_name!r} is not a valid package manager") + + return managers[manager_name].REQUIREMENTS_MODEL(**requirements) diff --git a/src/ewoks/_requirements/pip/sanitize.py b/src/ewoks/_requirements/utils/pip_freeze.py similarity index 63% rename from src/ewoks/_requirements/pip/sanitize.py rename to src/ewoks/_requirements/utils/pip_freeze.py index b71d23e..38697ad 100644 --- a/src/ewoks/_requirements/pip/sanitize.py +++ b/src/ewoks/_requirements/utils/pip_freeze.py @@ -11,10 +11,96 @@ from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement +from .metadata import models -def sanitize_requirements(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: + +def freeze_distribution( + dist: models.Distribution, +) -> Tuple[List[str], List[str]]: + """ + Return the pip freeze argument corresponding to the distribution with + associated warnings regarding reproducibility. + """ + lines = [] + warnings = [] + + if dist.version: + pypi_req = f"{dist.name}=={dist.version}" + else: + pypi_req = dist.name + + if dist.archive: + archive_req = f"{dist.name} @ {dist.archive.url}" + + if dist.archive.hashes: + for algo, value in dist.archive.hashes.items(): + archive_req += f" --hash={algo}:{value}" + + lines.append(archive_req) + return lines, warnings + + if dist.git: + warning_fmt = "Non-reproducible Ewoks workflow requirement: {}@{} {}" + + if dist.git.remote: + url = _normalize_git_url(dist.git.remote) + git_req = f"{dist.name} @ {url}@{dist.git.commit}" + else: + git_req = pypi_req + warning = warning_fmt.format( + dist.name, dist.git.commit, "has no remote repository" + ) + warnings.append(warning) + lines.append(f"# {warning}") + + if dist.git.uncommitted_changes: + warning = warning_fmt.format( + dist.name, dist.git.commit, "has uncommited changes" + ) + warnings.append(warning) + lines.append(f"# {warning}") + + lines.append(git_req) + return lines, warnings + + lines.append(pypi_req) + return lines, warnings + + +def _normalize_git_url(url: str, preserve_ssh: bool = False) -> str: + """ + Return a PEP 508-compatible VCS URL, prefixed with 'git+'. + """ + # SCP-like SSH syntax: git@host:group/repo.git + if url.startswith("git@"): + host, path = url[len("git@") :].split(":", 1) + if preserve_ssh: + return f"git+ssh://git@{host}/{path}" + return f"git+https://{host}/{path}" + + # Explicit SSH URL + if url.startswith("ssh://"): + if preserve_ssh: + return f"git+{url}" + + # ssh://git@host/group/repo.git -> https://host/group/repo.git + without_scheme = url[len("ssh://") :] + if without_scheme.startswith("git@"): + without_scheme = without_scheme[len("git@") :] + return f"git+https://{without_scheme}" + + # HTTP(S) + if url.startswith("http://") or url.startswith("https://"): + return f"git+{url}" + + # Unknown / already normalized + return url + + +def sanitize_freeze(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: """Sanitize a list of requirements coming from 'pip freeze'. - Returns a sanitized with warnings regarding applied changes. + + Returns a sanitized list with warnings regarding applied changes. """ sanitized = [] warnings = [] diff --git a/src/ewoks/_requirements/uv.py b/src/ewoks/_requirements/uv.py new file mode 100644 index 0000000..f5269f3 --- /dev/null +++ b/src/ewoks/_requirements/uv.py @@ -0,0 +1,61 @@ +from typing import Any +from typing import Dict +from typing import List +from typing import Literal +from typing import Optional + +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements + + +class UvManagerInfo(BaseManagerInfo): + name: Literal["uv"] = "uv" + requirements: List[str] + + +class UvRequirements(BaseRequirements): + manager: UvManagerInfo + + def __info__(self) -> str: + requirements = "\n ".join(self.manager.requirements) + return f"{super().__info__()}\nRequirements:\n {requirements}" + + +class UvManager(BaseManager): + NAME = "uv" + PRIORITY = 1 + REQUIREMENTS_MODEL = UvRequirements + + def __init__(self, *command: str) -> None: + if not command: + command = ("uv",) + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + output = self._check_output("--version") + return output.strip().split(" ")[-1] + except RuntimeError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + pass + + def _gather_requirements(self) -> Dict[str, Any]: + output = self._check_output("pip", "freeze") + requirements = output.strip().splitlines() + + return {"requirements": requirements} + + def _install_native_requirements(self, requirements: UvRequirements) -> bool: + text = "\n".join(requirements.requirements) + with self._temporary_file(text, ".txt") as tmp_path: + self._check_call("add", "-r", tmp_path) + + return True + + def _install_base_requirements(self, requirements: BaseRequirements) -> bool: + raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index 3f27d03..e55b97d 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -1,6 +1,8 @@ import datetime import logging import os +import re +import shlex import sys from contextlib import contextmanager from pathlib import Path @@ -9,6 +11,7 @@ from typing import Generator from typing import List from typing import Optional +from typing import Tuple from typing import Union from ewokscore.events.contexts import RawExecInfoType @@ -18,10 +21,8 @@ from tabulate import tabulate from . import _engines +from . import _requirements from . import graph_cache -from ._requirements.pip.extract import add_current_env_pip_requirements -from ._requirements.pip.extract import extract_pip_requirements -from ._requirements.pip.install import pip_install from .errors import AbortException try: @@ -198,14 +199,29 @@ def convert_graph( load_options: Optional[dict] = None, save_options: Optional[dict] = None, save_requirements: bool = True, + package_manager_name: Optional[str] = None, + package_manager_command: Union[None, str, Tuple[str, ...]] = None, ) -> Union[str, dict]: if load_options is None: load_options = dict() if save_options is None: save_options = dict() graph = load_graph(source, inputs=inputs, **load_options) + if save_requirements: - graph = add_current_env_pip_requirements(graph) + if not package_manager_command: + package_manager_command = tuple() + elif isinstance(package_manager_command, str): + package_manager_command = _split_command(package_manager_command) + + try: + _requirements.add_requirements( + graph, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) + except Exception: + logger.exception("Continue after failure to add workflow requirements") return save_graph(graph, destination, **save_options) @@ -249,34 +265,52 @@ def _print_graph( def install_graph( source, skip_prompt: bool = False, - python_path: Optional[str] = None, + package_manager_name: Optional[str] = None, + package_manager_command: Union[None, str, Tuple[str, ...]] = None, load_options: Optional[dict] = None, -): +) -> None: if load_options is None: load_options = dict() graph = load_graph(source, **load_options) - requirements = graph.requirements - if requirements is None: - logger.warning( - "Requirements field is empty. Trying to extract requirements automatically..." - ) - requirements = extract_pip_requirements(graph) - logger.info(f"Extracted the following requirements: {requirements}") + requirements = _requirements.get_requirements(graph) - if python_path is None: - python_path = sys.executable + if not package_manager_command: + package_manager_command = tuple() + elif isinstance(package_manager_command, str): + package_manager_command = _split_command(package_manager_command) if skip_prompt: - pip_install(requirements, python_path) + _requirements.install_requirements( + requirements, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) return - requirements_as_str = "\n".join(requirements) - answer = input( - f"{requirements_as_str}\nThis will install the above packages via {python_path} -m pip install. Do you want to proceed (y/N)?" + f"{requirements.__info__()}\n\nThis will install the packages above. Do you want to proceed (y/N)?" ) if answer.lower() == "y" or answer.lower() == "yes": - pip_install(requirements, python_path) + _requirements.install_requirements( + requirements, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) else: raise AbortException() + + +def _split_command(command: str) -> Tuple[str, ...]: + if sys.platform == "win32": + return tuple(_shlex_split(command)) + return tuple(shlex.split(command)) + + +def _shlex_split(command: str) -> Tuple[str, ...]: + r""" + Split a string on spaces, but treat escaped spaces (\ ) as part of the token. + """ + parts = re.split(r"(? None: } if cli_args.exclude_requirements: convert_options["save_requirements"] = False + else: + convert_options["save_requirements"] = True + convert_options["package_manager_name"] = cli_args.package_manager_name + convert_options["package_manager_command"] = cli_args.package_manager_command + cli_args.convert_options = convert_options diff --git a/src/ewoks/cli_utils/cli_install_utils.py b/src/ewoks/cli_utils/cli_install_utils.py index 20db4c5..ca2bfdd 100644 --- a/src/ewoks/cli_utils/cli_install_utils.py +++ b/src/ewoks/cli_utils/cli_install_utils.py @@ -27,10 +27,16 @@ def install_arguments( help="Automatically accept installation prompts.", ), CLIArg( - "python", - ["-p", "--python"], + "package_manager_name", + ["--package-manager-name"], type=str, - help="Python interpreter of the environment where the packages should be installed. Default: current environment Python.", + help='Package manager name. For example "pip"', + ), + CLIArg( + "package_manager_command", + ["--package-manager-command"], + type=str, + help='Package manager command. For example "python -m pip"', ), ] return args_list diff --git a/src/ewoks/tests/requirements/__init__.py b/src/ewoks/tests/requirements/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/tests/requirements/pip/__init__.py b/src/ewoks/tests/requirements/pip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ewoks/tests/test_requirements.py b/src/ewoks/tests/requirements/pip/test_freeze.py similarity index 87% rename from src/ewoks/tests/test_requirements.py rename to src/ewoks/tests/requirements/pip/test_freeze.py index 268c012..61b0b7f 100644 --- a/src/ewoks/tests/test_requirements.py +++ b/src/ewoks/tests/requirements/pip/test_freeze.py @@ -1,11 +1,11 @@ import pytest -from .._requirements.pip.sanitize import sanitize_requirements +from ...._requirements.utils.pip_freeze import sanitize_freeze def test_normal_requirement(): req = ["ewoks==1.1.0"] - sanitized, warnings = sanitize_requirements(req) + sanitized, warnings = sanitize_freeze(req) assert sanitized == ["ewoks==1.1.0"] assert warnings == [] @@ -18,7 +18,7 @@ def test_editable_ssh_vcs_url_normalized(): f"-e git+ssh://git@{ssh_project_url}@{ssh_project_commit}#egg={ssh_project_name}" ] - sanitized, warnings = sanitize_requirements(req) + sanitized, warnings = sanitize_freeze(req) expected_sanitized = [ f"{ssh_project_name} @ git+https://{ssh_project_url}@{ssh_project_commit}" @@ -44,7 +44,7 @@ def test_editable_local_path_with_comment_replacement(tmp_path, exists): replacement = "project_name==1.0.0" req = [comment, f"-e {path}"] - sanitized, warnings = sanitize_requirements(req) + sanitized, warnings = sanitize_freeze(req) assert sanitized == [replacement] assert warnings == [f"Replaced editable install '{path}' with '{replacement}'."] @@ -61,7 +61,7 @@ def test_editable_local_path_without_comment_replacement(tmp_path, exists): req = [f"-e {path}"] - sanitized, warnings = sanitize_requirements(req) + sanitized, warnings = sanitize_freeze(req) assert sanitized == [f"-e {path}"] if exists: @@ -77,7 +77,7 @@ def test_branch_specified_requirement(): req = [f"{project_name}@ git+https://{project_url}@{project_branch}"] - sanitized, warnings = sanitize_requirements(req) + sanitized, warnings = sanitize_freeze(req) # No warnings expected here (assuming valid format) assert sanitized == [f"{project_name}@ git+https://{project_url}@{project_branch}"] @@ -89,7 +89,7 @@ def test_invalid_requirement_warning(): req = [f"git+https://{project_url}"] - sanitized, warnings = sanitize_requirements(req) + sanitized, warnings = sanitize_freeze(req) assert sanitized == [f"git+https://{project_url}"] assert any("Possibly invalid requirement format" in w for w in warnings) diff --git a/src/ewoks/tests/requirements/pip/test_install_cli.py b/src/ewoks/tests/requirements/pip/test_install_cli.py new file mode 100644 index 0000000..bab1336 --- /dev/null +++ b/src/ewoks/tests/requirements/pip/test_install_cli.py @@ -0,0 +1,136 @@ +import json +import subprocess + +import pytest + +from ...._requirements.utils.metadata import from_pip_freeze + + +def test_install_pip_with_freeze(venv): + with pytest.raises(Exception, match="package is not installed"): + _ = venv.get_version("ewoksdata") + + requirements = from_pip_freeze.pip_freeze_requirements(["ewoksdata"]) + + graph = { + "graph": { + "schema_version": "1.1", + "id": "test_install", + "requirements": requirements, + } + } + + subprocess.check_call( + [ + "ewoks", + "install", + "--yes", + json.dumps(graph), + "--package-manager-name", + "pip", + "--package-manager-command", + f"{venv.python} -m pip", + ] + ) + + assert venv.get_version("ewoksdata") + + +def test_install_pip_without_freeze(venv): + with pytest.raises(Exception, match="package is not installed"): + _ = venv.get_version("ewoksdata") + + requirements = from_pip_freeze.pip_freeze_requirements(["ewoksdata"]) + requirements["distributions"] = [ + {"name": "ewoksdata", "version": ""}, + ] + + graph = { + "graph": { + "schema_version": "1.1", + "id": "test_install", + "requirements": requirements, + } + } + + subprocess.check_call( + [ + "ewoks", + "install", + "--yes", + json.dumps(graph), + "--package-manager-name", + "pip", + "--package-manager-command", + f"{venv.python} -m pip", + ] + ) + + assert venv.get_version("ewoksdata") + + +def test_install_legacy_pip_freeze(venv): + with pytest.raises(Exception, match="package is not installed"): + _ = venv.get_version("ewoksdata") + + requirements = ["ewoksdata"] + graph = { + "graph": { + "schema_version": "1.1", + "id": "test_install", + "requirements": requirements, + } + } + + subprocess.check_call( + [ + "ewoks", + "install", + "--yes", + json.dumps(graph), + "--package-manager-name", + "pip", + "--package-manager-command", + f"{venv.python} -m pip", + ] + ) + + assert venv.get_version("ewoksdata") + + +def test_install_without_requirements(venv): + with pytest.raises(Exception, match="package is not installed"): + _ = venv.get_version("ewoksdata") + + nodes = [ + { + "id": 1, + "task_identifier": 'ewoksdata.tasks.normalization.Normalization"', + "task_type": "class", + }, + { + "id": 2, + "task_identifier": "path/to/my/script", + "task_type": "script", + }, # Check that unsupported task type goes through without error + ] + + graph = { + "graph": {"schema_version": "1.1", "id": "test_install"}, + "nodes": nodes, + } + + subprocess.check_call( + [ + "ewoks", + "install", + "--yes", + json.dumps(graph), + "--package-manager-name", + "pip", + "--package-manager-command", + f"{venv.python} -m pip", + ] + ) + + assert venv.get_version("ewoksdata") diff --git a/src/ewoks/tests/requirements/utils.py b/src/ewoks/tests/requirements/utils.py new file mode 100644 index 0000000..651d016 --- /dev/null +++ b/src/ewoks/tests/requirements/utils.py @@ -0,0 +1,13 @@ +from ewokscore.graph import TaskGraph + +from ..._requirements.utils.parse import parse_requirements + + +def assert_in_graph_requirements(graph: TaskGraph, *distribution_names) -> None: + assert distribution_names, "no names provides" + assert "requirements" in graph.graph.graph, "no requirements" + requirements = parse_requirements(graph.graph.graph["requirements"]) + + existing = {distribution.name for distribution in requirements.distributions} + not_existing = set(distribution_names) - existing + assert not not_existing, f"{sorted(not_existing)} not in requirements" diff --git a/src/ewoks/tests/test_cli.py b/src/ewoks/tests/test_cli.py index f0cd582..ac8ea4e 100644 --- a/src/ewoks/tests/test_cli.py +++ b/src/ewoks/tests/test_cli.py @@ -33,6 +33,9 @@ def test_cli_convert(cli_interface): # noqa F811 ], "load_options": {"representation": "test_core"}, "save_options": {"representation": "json"}, + "package_manager_command": None, + "package_manager_name": None, + "save_requirements": True, } assert cli_args.convert_options == convert_options diff --git a/src/ewoks/tests/test_convert_cli.py b/src/ewoks/tests/test_convert_cli.py index 48ca344..eaf8db3 100644 --- a/src/ewoks/tests/test_convert_cli.py +++ b/src/ewoks/tests/test_convert_cli.py @@ -4,25 +4,15 @@ import pytest from ewokscore import load_graph -from ewokscore.graph import TaskGraph from ewokscore.task import Task from ewokscore.tests.examples.graphs import graph_names from ewoksutils.import_utils import import_qualname from orangewidget.widget import OWBaseWidget -from ewoks.__main__ import main -from ewoks.tests.utils import has_default_input -from ewoks.tests.utils import no_widget_registry - - -def _ewokscore_in_graph_requirements(graph: TaskGraph) -> bool: - ewokscore_in_req = False - for requirement in graph.graph.graph["requirements"]: - if "ewokscore" in requirement: - ewokscore_in_req = True - break - - return ewokscore_in_req +from ..__main__ import main +from .requirements.utils import assert_in_graph_requirements +from .utils import has_default_input +from .utils import no_widget_registry @pytest.mark.parametrize("graph_name", graph_names()) @@ -41,8 +31,8 @@ def test_convert_to_json(graph_name, tmpdir): assert os.path.exists(destination) graph = load_graph(destination) - assert graph.graph.graph["requirements"] is not None - assert _ewokscore_in_graph_requirements(graph) + + assert_in_graph_requirements(graph, "ewokscore") def test_convert_with_all_inputs(tmpdir): diff --git a/src/ewoks/tests/test_execute_cli.py b/src/ewoks/tests/test_execute_cli.py index 5b07dcc..859feb9 100644 --- a/src/ewoks/tests/test_execute_cli.py +++ b/src/ewoks/tests/test_execute_cli.py @@ -3,23 +3,13 @@ import pytest from ewokscore import load_graph -from ewokscore.graph import TaskGraph from ewokscore.tests.examples.graphs import get_graph from ewokscore.tests.examples.graphs import graph_names from ewokscore.tests.utils.results import assert_execute_graph_default_result -from ewoks.__main__ import main -from ewoks.tests.utils import has_default_input - - -def _ewokscore_in_graph_requirements(graph: TaskGraph) -> bool: - ewokscore_in_req = False - for requirement in graph.graph.graph["requirements"]: - if "ewokscore" in requirement: - ewokscore_in_req = True - break - - return ewokscore_in_req +from ..__main__ import main +from .requirements.utils import assert_in_graph_requirements +from .utils import has_default_input @pytest.mark.parametrize("graph_name", graph_names()) @@ -80,8 +70,7 @@ def test_execute_with_convert_destination(tmpdir): task1_node = graph.graph.nodes["task1"] assert has_default_input(task1_node, "b", 42) - assert graph.graph.graph["requirements"] is not None - assert _ewokscore_in_graph_requirements(graph) + assert_in_graph_requirements(graph, "ewokscore") def test_execute_with_convert_destination_inputs_all(tmpdir): @@ -106,5 +95,4 @@ def test_execute_with_convert_destination_inputs_all(tmpdir): for node in graph.graph.nodes.values(): assert has_default_input(node, "b", 42) - assert graph.graph.graph["requirements"] is not None - assert _ewokscore_in_graph_requirements(graph) + assert_in_graph_requirements(graph, "ewokscore") diff --git a/src/ewoks/tests/test_install_cli.py b/src/ewoks/tests/test_install_cli.py deleted file mode 100644 index b2920a9..0000000 --- a/src/ewoks/tests/test_install_cli.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -import subprocess - -import pytest - - -def test_install(venv): - with pytest.raises(Exception, match="package is not installed"): - venv.get_version("ewoksdata") - - subprocess.check_call( - [ - "ewoks", - "install", - "--yes", - '{"graph": {"id": "test_install", "requirements": ["ewoksdata"]}}', - "-p", - f"{venv.python}", - ] - ) - - assert venv.get_version("ewoksdata") is not None - - -def test_install_with_extract(venv): - with pytest.raises(Exception, match="package is not installed"): - venv.get_version("ewoksdata") - - nodes = [ - { - "id": 1, - "task_identifier": 'ewoksdata.tasks.normalization.Normalization"', - "task_type": "class", - }, - { - "id": 2, - "task_identifier": "path/to/my/script", - "task_type": "script", - }, # Check that unsupported task type goes though without error - ] - - graph = {"graph": {"id": "test_install"}, "nodes": nodes} - - subprocess.check_call( - [ - "ewoks", - "install", - "--yes", - json.dumps(graph), - "-p", - f"{venv.python}", - ] - ) - - assert venv.get_version("ewoksdata") is not None