Merge branch 'main' into fix_mask_equal_2

valeriupredoi · web-flow · commit 22443ce5ec23 · 2026-04-10T13:01:42.000+01:00
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,20 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  # Enable version updates for GitHub Actions
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`
+    # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7
+    labels:
+      - dependencies
+      - github_actions
+      - automatedPR
diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml
@@ -16,11 +16,11 @@ jobs:
       # IMPORTANT: this permission is mandatory for trusted publishing on PyPI or Test PyPI
       id-token: write
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Set up Python 3.14
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: "3.14"
       - name: Install pep517
diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -20,7 +20,7 @@ jobs:
     name: Create and verify conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
@@ -64,7 +64,7 @@ jobs:
       # Automated PR
       # see https://github.com/marketplace/actions/create-pull-request
       - name: Create Automated PR if conda lock file has changed
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v8
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Updating condalock file
diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml
@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
diff --git a/.github/workflows/run-s3-test-push.yml b/.github/workflows/run-s3-test-push.yml
@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Export proxy
diff --git a/.github/workflows/run-test-push.yml b/.github/workflows/run-test-push.yml
@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
@@ -33,4 +33,4 @@ jobs:
       - run: conda list
       - run: flake8 --exclude tests,doc --max-line-length 120 --ignore F405,F401
       - run: pytest -n 2 -m "not slow" --junitxml=report-1.xml
-      - uses: codecov/codecov-action@v5
+      - uses: codecov/codecov-action@v6
diff --git a/.github/workflows/run-tests-condaforge.yml b/.github/workflows/run-tests-condaforge.yml
@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
@@ -54,7 +54,7 @@ jobs:
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
@@ -46,7 +46,7 @@ jobs:
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - uses: conda-incubator/setup-miniconda@v3
diff --git a/.github/workflows/test_s3_minio.yml b/.github/workflows/test_s3_minio.yml
@@ -23,7 +23,7 @@ jobs:
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Export proxy
diff --git a/activestorage/active.py b/activestorage/active.py
@@ -191,7 +191,8 @@ def __init__(self,
                  interface_type: str = None,
                  max_threads: int = 100,
                  storage_options: dict = None,
-                 active_storage_url: str = None) -> None:
+                 active_storage_url: str = None,
+                 option_disable_chunk_cache: bool = False) -> None:
         """
         Instantiate with a NetCDF4 dataset URI and the variable of interest within that file.
         (We need the variable, because we need variable specific metadata from within that
@@ -200,6 +201,7 @@ def __init__(self,
 
         :param storage_options: s3fs.S3FileSystem options
         :param active_storage_url: Reductionist server URL
+        :param option_disable_chunk_cache: flag to disable chunk caching
         """
         self.ds = None
         input_variable = False
@@ -257,6 +259,9 @@ def __init__(self,
         self.storage_options = storage_options
         self.active_storage_url = active_storage_url
 
+        # turn off chunk caching
+        self.option_disable_chunk_cache = option_disable_chunk_cache
+
         # basic check on file
         if not input_variable:
             if not os.path.isfile(self.uri) and not self.interface_type:
@@ -515,21 +520,22 @@ def _from_storage(self, ds, indexer, chunks, out_shape, out_dtype,
         # Create a shared session object.
         if self.interface_type == "s3" and self._version == 2:
             if self.storage_options is not None:
-                key, secret = None, None
                 if self.storage_options.get("anon", None) is True:
                     print("Reductionist session for Anon S3 bucket.")
                     session = reductionist.get_session(
                         None, None, S3_ACTIVE_STORAGE_CACERT)
-                if "key" in self.storage_options:
-                    key = self.storage_options["key"]
-                if "secret" in self.storage_options:
-                    secret = self.storage_options["secret"]
-                if key and secret:
-                    session = reductionist.get_session(
-                        key, secret, S3_ACTIVE_STORAGE_CACERT)
                 else:
-                    session = reductionist.get_session(
-                        S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT)
+                    key, secret = None, None
+                    if "key" in self.storage_options:
+                        key = self.storage_options["key"]
+                    if "secret" in self.storage_options:
+                        secret = self.storage_options["secret"]
+                    if key and secret:
+                        session = reductionist.get_session(
+                            key, secret, S3_ACTIVE_STORAGE_CACERT)
+                    else:
+                        session = reductionist.get_session(
+                            S3_ACCESS_KEY, S3_SECRET_KEY, S3_ACTIVE_STORAGE_CACERT)
             else:
                 session = reductionist.get_session(S3_ACCESS_KEY,
                                                    S3_SECRET_KEY,
@@ -661,6 +667,9 @@ def _process_chunk(self,
         # Axes over which to apply a reduction
         axis = self._axis
 
+        # turn off chunk caching
+        chunk_caching = self.option_disable_chunk_cache
+
         if self.interface_type == 's3' and self._version == 1:
             tmp, count = reduce_opens3_chunk(ds._fh,
                                              offset,
@@ -703,7 +712,8 @@ def _process_chunk(self,
                                                        ds._order,
                                                        chunk_selection,
                                                        axis,
-                                                       operation=self._method)
+                                                       operation=self._method,
+                                                       option_disable_chunk_cache=chunk_caching,)
             else:
                 if self.storage_options.get("anon", None) is True:
                     bucket = os.path.dirname(parsed_url.path)
@@ -723,7 +733,8 @@ def _process_chunk(self,
                     ds._order,
                     chunk_selection,
                     axis,
-                    operation=self._method)
+                    operation=self._method,
+                    option_disable_chunk_cache=chunk_caching,)
         elif self.interface_type == "https" and self._version == 2:
             tmp, count = reductionist.reduce_chunk(session,
                                                    self.active_storage_url,
@@ -739,7 +750,8 @@ def _process_chunk(self,
                                                    chunk_selection,
                                                    axis,
                                                    operation=self._method,
-                                                   interface_type="https")
+                                                   interface_type="https",
+                                                   option_disable_chunk_cache=chunk_caching,)
 
         elif self.interface_type == 'ActivePosix' and self.version == 2:
             # This is where the DDN Fuse and Infinia wrappers go
diff --git a/activestorage/reductionist.py b/activestorage/reductionist.py
@@ -24,13 +24,10 @@ def get_session(username: str, password: str,
     :returns: a client session object.
     """
     session = requests.Session()
-    # TODO Stack-HPC
-    # we need to allow Anon buckets. though this
-    # will break connection to data server
-    # if username is None and password is None:
-    #     return session
-    session.auth = (username, password)
     session.verify = cacert or False
+    if username is None and password is None:
+        return session
+    session.auth = (username, password)
     return session
 
 
@@ -48,7 +45,8 @@ def reduce_chunk(session,
                  chunk_selection,
                  axis,
                  operation,
-                 interface_type=None):
+                 interface_type=None,
+                 option_disable_chunk_cache=False):
     """Perform a reduction on a chunk using Reductionist.
 
     :param server: Reductionist server URL
@@ -71,6 +69,7 @@ def reduce_chunk(session,
     :param axis: tuple of the axes to be reduced (non-negative integers)
     :param operation: name of operation to perform
     :param interface_type: optional testing flag to allow HTTPS reduction
+    :param option_disable_chunk_cache: optional turn off chunk cache
     :returns: the reduced data as a numpy array or scalar
     :raises ReductionistError: if the request to Reductionist fails
     """
@@ -86,7 +85,8 @@ def reduce_chunk(session,
                                       order,
                                       chunk_selection,
                                       axis,
-                                      interface_type=interface_type)
+                                      interface_type=interface_type,
+                                      option_disable_chunk_cache=option_disable_chunk_cache)
     if DEBUG:
         print(f"Reductionist request data dictionary: {request_data}")
     api_operation = "sum" if operation == "mean" else operation or "select"
@@ -184,7 +184,8 @@ def build_request_data(url: str,
                        order,
                        selection,
                        axis,
-                       interface_type=None) -> dict:
+                       interface_type=None,
+                       option_disable_chunk_cache=False) -> dict:
     """Build request data for Reductionist API."""
     request_data = {
         'interface_type': interface_type if interface_type else "s3",
@@ -208,6 +209,8 @@ def build_request_data(url: str,
         request_data["filters"] = encode_filters(filters)
     if any(missing):
         request_data["missing"] = encode_missing(missing)
+    if option_disable_chunk_cache:
+        request_data["option_disable_chunk_cache"] = True
 
     if axis is not None:
         request_data['axis'] = axis
@@ -255,8 +258,13 @@ def __init__(self, status_code, error):
 
 def decode_and_raise_error(response):
     """Decode an error response and raise ReductionistError."""
-    try:
-        error = json.dumps(response.json())
+    if response.status_code == http.client.INTERNAL_SERVER_ERROR:
+        try:
+            error = json.dumps(response.json())
+        except requests.exceptions.JSONDecodeError as exc:
+            error = http.client.responses.get(response.status_code, "-")
+            raise ReductionistError(response.status_code, error) from exc
         raise ReductionistError(response.status_code, error)
-    except requests.exceptions.JSONDecodeError as exc:
-        raise ReductionistError(response.status_code, "-") from exc
+
+    error = http.client.responses.get(response.status_code, "-")
+    raise ReductionistError(response.status_code, error)
diff --git a/activestorage/storage.py b/activestorage/storage.py
@@ -16,7 +16,8 @@ def reduce_chunk(rfile,
                  order,
                  chunk_selection,
                  axis,
-                 method=None):
+                 method=None,
+                 option_disable_chunk_cache=False,):
     """
     We do our own read of chunks and decoding etc
 
diff --git a/conda-linux-64.lock b/conda-linux-64.lock
diff --git a/tests/test_real_https.py b/tests/test_real_https.py
diff --git a/tests/test_real_s3.py b/tests/test_real_s3.py
diff --git a/tests/test_real_s3_with_axes.py b/tests/test_real_s3_with_axes.py
diff --git a/tests/unit/test_reductionist.py b/tests/unit/test_reductionist.py
diff --git a/tests/unit/test_storage_types.py b/tests/unit/test_storage_types.py
