[26.1] Add namespace stripping to CachedExplicitSingularityContainerResolver

doc/source/admin/container_resolvers.rst

@@ -238,6 +238,21 @@ return a container description that points to the image file in the
    yield the path (even if non existent, i.e. before the 1st tool run or the
    caching was triggered).
 
+The optional ``namespace`` parameter strips ``docker://quay.io/NAMESPACE/``
+from the image identifier when constructing the local cache path. This yields a
+flat filename (e.g. ``bwa:0.7.17--h7132678_9``) instead of a nested path, which
+is required when pointing ``cache_directory`` at a pre-populated flat cache such
+as ``/cvmfs/singularity.galaxyproject.org/all/``:
+
+.. code-block:: yaml
+
+   - type: cached_explicit_singularity
+     cache_directory: /cvmfs/singularity.galaxyproject.org/all
+     namespace: biocontainers
+
+Images whose identifier does not start with ``docker://quay.io/NAMESPACE/``
+(e.g. ``shub://`` URIs) are not affected by this setting.
+
 2. Mulled resolvers
 """""""""""""""""""
 

lib/galaxy/config/sample/container_resolvers.yml.sample

@@ -19,6 +19,20 @@
 #- type: cached_explicit_singularity
   # set the cache directory for storing images
   #cache_directory: database/container_cache/singularity/explicit
+  #
+  # When namespace is set, the prefix docker://quay.io/NAMESPACE/ is stripped
+  # from the image identifier before constructing the cache path, yielding a
+  # flat filename (e.g. "bwa:0.7.17--h7132678_9") instead of a nested path.
+  # This is useful when pointing cache_directory at a pre-populated flat cache
+  # such as /cvmfs/singularity.galaxyproject.org/all/.
+  #namespace: biocontainers
+  #
+  # Caching strategy for the directory listing used when namespace is set.
+  # "uncached" (default) re-reads the directory on every resolve.
+  # "dir_mtime" caches the full directory listing and only re-reads it when
+  # the directory mtime changes; recommended for large network-mounted caches
+  # such as CVMFS where repeated directory scans are expensive.
+  #cache_directory_cacher_type: uncached
 
 # Mulled container resolvers
 # ==========================

lib/galaxy/tool_util/deps/container_resolvers/explicit.py

@@ -11,7 +11,13 @@
 
 from galaxy.util.commands import shell
 from . import ContainerResolver
-from .mulled import CliContainerResolver
+from .mulled import (
+    CacheDirectory,
+    CachedMulledImageSingleTarget,
+    CliContainerResolver,
+    get_cache_directory_cacher,
+    identifier_to_cached_target,
+)
 from ..container_classes import SingularityContainer
 from ..requirements import ContainerDescription
 
@@ -85,6 +91,10 @@ def __init__(self, app_info: "AppInfo", **kwargs) -> None:
             assert self.app_info.container_image_cache_path
             cache_directory_path = os.path.join(self.app_info.container_image_cache_path, "singularity", "explicit")
         self.cache_directory_path = cache_directory_path
+        self.namespace = kwargs.get("namespace")
+        cache_directory_cacher_type = kwargs.get("cache_directory_cacher_type")
+        cacher_class = get_cache_directory_cacher(cache_directory_cacher_type)
+        self.cache_directory: CacheDirectory = cacher_class(self.cache_directory_path)
         os.makedirs(self.cache_directory_path, exist_ok=True)
 
     def resolve(
@@ -103,11 +113,37 @@ def resolve(
                 container_description.identifier = f"docker://{container_description.identifier}"
             if not self._container_type_enabled(container_description, enabled_container_types):
                 return None
-            if not self.cli_available:
-                return container_description
             image_id = container_description.identifier
-            cache_path = os.path.normpath(os.path.join(self.cache_directory_path, image_id))
+            if self.namespace:
+                prefix = f"docker://quay.io/{self.namespace}/"
+                if image_id.startswith(prefix):
+                    image_id = image_id[len(prefix) :]
+                parsed = identifier_to_cached_target(image_id, "v2")
+                if parsed and isinstance(parsed, CachedMulledImageSingleTarget):
+                    cached_images = self.cache_directory.list_cached_mulled_images_from_path()
+                    for cached in cached_images:
+                        if (
+                            isinstance(cached, CachedMulledImageSingleTarget)
+                            and cached.package_name == parsed.package_name
+                            and cached.version == parsed.version
+                        ):
+                            container_description.identifier = os.path.join(
+                                self.cache_directory_path, cached.image_identifier
+                            )
+                            return container_description
+                    if not install:
+                        return None
+                else:
+                    if not install and not os.path.exists(os.path.join(self.cache_directory_path, image_id)):
+                        return None
+                cache_path = os.path.join(self.cache_directory_path, image_id)
+            else:
+                if not self.cli_available:
+                    return container_description
+                cache_path = os.path.normpath(os.path.join(self.cache_directory_path, image_id))
             if install and not os.path.exists(cache_path):
+                if not self.cli_available:
+                    return None
                 destination_info = {}
                 destination_for_container_type = kwds.get("destination_for_container_type")
                 if destination_for_container_type:
@@ -122,14 +158,20 @@ def resolve(
                 )
                 command = container.build_singularity_pull_command(cache_path=cache_path)
                 shell(command)
+                self.cache_directory.invalidate_cache()
             # Point to container in the cache in stead.
             container_description.identifier = cache_path
             return container_description
         else:  # No container descriptions found
             return None
 
     def __str__(self):
-        return f"CachedExplicitSingularityContainerResolver[cache_directory={self.cache_directory_path}]"
+        return (
+            f"CachedExplicitSingularityContainerResolver["
+            f"cache_directory={self.cache_directory_path},"
+            f"namespace={self.namespace},"
+            f"cache_directory_cacher_type={self.cache_directory.cacher_type}]"
+        )
 
 
 class BaseAdminConfiguredContainerResolver(ContainerResolver):

lib/galaxy/tool_util/deps/container_resolvers/mulled.py

@@ -106,6 +106,8 @@ def package_hash(self) -> str:
 
 
 class CacheDirectory(metaclass=ABCMeta):
+    cacher_type: str
+
     def __init__(self, path: str, hash_func: Literal["v1", "v2"] = "v2") -> None:
         self.path = path
         self.hash_func = hash_func

test/integration/test_container_resolvers.py

@@ -1217,3 +1217,61 @@ class TestCachedExplicitSingularityContainerResolverWithSingularityRequirement(
     def handle_galaxy_config_kwds(cls, config) -> None:
         super().handle_galaxy_config_kwds(config)
         config["container_resolvers"] = cls.container_resolvers_config
+
+
+class TestCachedExplicitSingularityContainerResolverWithNamespace(
+    SingularityContainerResolverTestCase, ContainerResolverTestCases, ExplicitTestCase
+):
+    """
+    test cached_explicit_singularity container resolver with namespace stripping
+
+    when namespace="biocontainers" is configured, the resolver strips
+    docker://quay.io/biocontainers/ from the image identifier so that the
+    cached image is stored as a flat filename (e.g. bwa:0.7.17--h7132678_9)
+    compatible with CVMFS-hosted biocontainer caches.
+    """
+
+    _image_name = ExplicitTestCase.mulled_hash.rsplit("/", 1)[-1]
+
+    container_resolvers_config: list[dict[str, Any]] = [
+        {"type": "cached_explicit_singularity", "namespace": "biocontainers"},
+    ]
+    assumptions: dict[str, Any] = {
+        "run": {
+            "output": [
+                "Program: bwa (alignment via Burrows-Wheeler transformation)",
+                "Version: 0.7.17-r1188",
+            ],
+            "cached": True,
+            "resolver_type": "cached_explicit_singularity",
+            "cache_name": _image_name,
+            "cache_namespace": "biocontainers",
+        },
+        # With namespace set the resolver returns None when the image is not yet
+        # cached, so listing resolves to NullDependency before any install.
+        "list": [
+            {"unresolved": True},
+            {"unresolved": True},
+        ],
+        "build": [
+            {
+                "resolver_type": "cached_explicit_singularity",
+                "identifier": f"/tmp/.*/singularity/explicit/{_image_name}",
+                "cached": True,
+                "cache_name": _image_name,
+                "cache_namespace": "biocontainers",
+            },
+            {
+                "resolver_type": "cached_explicit_singularity",
+                "identifier": f"/tmp/.*/singularity/explicit/{_image_name}",
+                "cached": True,
+                "cache_name": _image_name,
+                "cache_namespace": "biocontainers",
+            },
+        ],
+    }
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config) -> None:
+        super().handle_galaxy_config_kwds(config)
+        config["container_resolvers"] = cls.container_resolvers_config