diff --git a/doc/source/admin/container_resolvers.rst b/doc/source/admin/container_resolvers.rst index 153c5053c605..4743bfa25b0d 100644 --- a/doc/source/admin/container_resolvers.rst +++ b/doc/source/admin/container_resolvers.rst @@ -238,6 +238,21 @@ return a container description that points to the image file in the yield the path (even if non existent, i.e. before the 1st tool run or the caching was triggered). +The optional ``namespace`` parameter strips ``docker://quay.io/NAMESPACE/`` +from the image identifier when constructing the local cache path. This yields a +flat filename (e.g. ``bwa:0.7.17--h7132678_9``) instead of a nested path, which +is required when pointing ``cache_directory`` at a pre-populated flat cache such +as ``/cvmfs/singularity.galaxyproject.org/all/``: + +.. code-block:: yaml + + - type: cached_explicit_singularity + cache_directory: /cvmfs/singularity.galaxyproject.org/all + namespace: biocontainers + +Images whose identifier does not start with ``docker://quay.io/NAMESPACE/`` +(e.g. ``shub://`` URIs) are not affected by this setting. + 2. Mulled resolvers """"""""""""""""""" diff --git a/lib/galaxy/config/sample/container_resolvers.yml.sample b/lib/galaxy/config/sample/container_resolvers.yml.sample index dc68dcdce100..9d253efc55bd 100644 --- a/lib/galaxy/config/sample/container_resolvers.yml.sample +++ b/lib/galaxy/config/sample/container_resolvers.yml.sample @@ -19,6 +19,20 @@ #- type: cached_explicit_singularity # set the cache directory for storing images #cache_directory: database/container_cache/singularity/explicit + # + # When namespace is set, the prefix docker://quay.io/NAMESPACE/ is stripped + # from the image identifier before constructing the cache path, yielding a + # flat filename (e.g. "bwa:0.7.17--h7132678_9") instead of a nested path. + # This is useful when pointing cache_directory at a pre-populated flat cache + # such as /cvmfs/singularity.galaxyproject.org/all/. + #namespace: biocontainers + # + # Caching strategy for the directory listing used when namespace is set. + # "uncached" (default) re-reads the directory on every resolve. + # "dir_mtime" caches the full directory listing and only re-reads it when + # the directory mtime changes; recommended for large network-mounted caches + # such as CVMFS where repeated directory scans are expensive. + #cache_directory_cacher_type: uncached # Mulled container resolvers # ========================== diff --git a/lib/galaxy/tool_util/deps/container_resolvers/explicit.py b/lib/galaxy/tool_util/deps/container_resolvers/explicit.py index f8e4d4d8205c..c1690e1124e5 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/explicit.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/explicit.py @@ -11,7 +11,13 @@ from galaxy.util.commands import shell from . import ContainerResolver -from .mulled import CliContainerResolver +from .mulled import ( + CacheDirectory, + CachedMulledImageSingleTarget, + CliContainerResolver, + get_cache_directory_cacher, + identifier_to_cached_target, +) from ..container_classes import SingularityContainer from ..requirements import ContainerDescription @@ -85,6 +91,10 @@ def __init__(self, app_info: "AppInfo", **kwargs) -> None: assert self.app_info.container_image_cache_path cache_directory_path = os.path.join(self.app_info.container_image_cache_path, "singularity", "explicit") self.cache_directory_path = cache_directory_path + self.namespace = kwargs.get("namespace") + cache_directory_cacher_type = kwargs.get("cache_directory_cacher_type") + cacher_class = get_cache_directory_cacher(cache_directory_cacher_type) + self.cache_directory: CacheDirectory = cacher_class(self.cache_directory_path) os.makedirs(self.cache_directory_path, exist_ok=True) def resolve( @@ -103,11 +113,37 @@ def resolve( container_description.identifier = f"docker://{container_description.identifier}" if not self._container_type_enabled(container_description, enabled_container_types): return None - if not self.cli_available: - return container_description image_id = container_description.identifier - cache_path = os.path.normpath(os.path.join(self.cache_directory_path, image_id)) + if self.namespace: + prefix = f"docker://quay.io/{self.namespace}/" + if image_id.startswith(prefix): + image_id = image_id[len(prefix) :] + parsed = identifier_to_cached_target(image_id, "v2") + if parsed and isinstance(parsed, CachedMulledImageSingleTarget): + cached_images = self.cache_directory.list_cached_mulled_images_from_path() + for cached in cached_images: + if ( + isinstance(cached, CachedMulledImageSingleTarget) + and cached.package_name == parsed.package_name + and cached.version == parsed.version + ): + container_description.identifier = os.path.join( + self.cache_directory_path, cached.image_identifier + ) + return container_description + if not install: + return None + else: + if not install and not os.path.exists(os.path.join(self.cache_directory_path, image_id)): + return None + cache_path = os.path.join(self.cache_directory_path, image_id) + else: + if not self.cli_available: + return container_description + cache_path = os.path.normpath(os.path.join(self.cache_directory_path, image_id)) if install and not os.path.exists(cache_path): + if not self.cli_available: + return None destination_info = {} destination_for_container_type = kwds.get("destination_for_container_type") if destination_for_container_type: @@ -122,6 +158,7 @@ def resolve( ) command = container.build_singularity_pull_command(cache_path=cache_path) shell(command) + self.cache_directory.invalidate_cache() # Point to container in the cache in stead. container_description.identifier = cache_path return container_description @@ -129,7 +166,12 @@ def resolve( return None def __str__(self): - return f"CachedExplicitSingularityContainerResolver[cache_directory={self.cache_directory_path}]" + return ( + f"CachedExplicitSingularityContainerResolver[" + f"cache_directory={self.cache_directory_path}," + f"namespace={self.namespace}," + f"cache_directory_cacher_type={self.cache_directory.cacher_type}]" + ) class BaseAdminConfiguredContainerResolver(ContainerResolver): diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index 5f9165b867d3..d907abd8851e 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -106,6 +106,8 @@ def package_hash(self) -> str: class CacheDirectory(metaclass=ABCMeta): + cacher_type: str + def __init__(self, path: str, hash_func: Literal["v1", "v2"] = "v2") -> None: self.path = path self.hash_func = hash_func diff --git a/test/integration/test_container_resolvers.py b/test/integration/test_container_resolvers.py index 4e3fdcfb4ca8..0e75b1feaa6c 100644 --- a/test/integration/test_container_resolvers.py +++ b/test/integration/test_container_resolvers.py @@ -1217,3 +1217,61 @@ class TestCachedExplicitSingularityContainerResolverWithSingularityRequirement( def handle_galaxy_config_kwds(cls, config) -> None: super().handle_galaxy_config_kwds(config) config["container_resolvers"] = cls.container_resolvers_config + + +class TestCachedExplicitSingularityContainerResolverWithNamespace( + SingularityContainerResolverTestCase, ContainerResolverTestCases, ExplicitTestCase +): + """ + test cached_explicit_singularity container resolver with namespace stripping + + when namespace="biocontainers" is configured, the resolver strips + docker://quay.io/biocontainers/ from the image identifier so that the + cached image is stored as a flat filename (e.g. bwa:0.7.17--h7132678_9) + compatible with CVMFS-hosted biocontainer caches. + """ + + _image_name = ExplicitTestCase.mulled_hash.rsplit("/", 1)[-1] + + container_resolvers_config: list[dict[str, Any]] = [ + {"type": "cached_explicit_singularity", "namespace": "biocontainers"}, + ] + assumptions: dict[str, Any] = { + "run": { + "output": [ + "Program: bwa (alignment via Burrows-Wheeler transformation)", + "Version: 0.7.17-r1188", + ], + "cached": True, + "resolver_type": "cached_explicit_singularity", + "cache_name": _image_name, + "cache_namespace": "biocontainers", + }, + # With namespace set the resolver returns None when the image is not yet + # cached, so listing resolves to NullDependency before any install. + "list": [ + {"unresolved": True}, + {"unresolved": True}, + ], + "build": [ + { + "resolver_type": "cached_explicit_singularity", + "identifier": f"/tmp/.*/singularity/explicit/{_image_name}", + "cached": True, + "cache_name": _image_name, + "cache_namespace": "biocontainers", + }, + { + "resolver_type": "cached_explicit_singularity", + "identifier": f"/tmp/.*/singularity/explicit/{_image_name}", + "cached": True, + "cache_name": _image_name, + "cache_namespace": "biocontainers", + }, + ], + } + + @classmethod + def handle_galaxy_config_kwds(cls, config) -> None: + super().handle_galaxy_config_kwds(config) + config["container_resolvers"] = cls.container_resolvers_config