From e47b20d74bbf38633bebc144525cdbc9690cc2c5 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 4 May 2026 10:42:55 +0100 Subject: [PATCH 1/5] Fix memray CI --- .github/workflows/tests.yaml | 6 +++--- distributed/diagnostics/tests/test_memray.py | 2 +- pyproject.toml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 886cf69ffe1..246755d5078 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -84,10 +84,10 @@ jobs: partition: "not ci1" - os: ubuntu-latest - environment: mindeps + environment: "3.10" label: memray - extra_packages: [memray] - partition: "extra_packages" + extra_packages: [memray] # Not available on Windows + partition: memray # Uncomment to stress-test the test suite for random failures. # Must also change `export TEST_ID` in first step below. diff --git a/distributed/diagnostics/tests/test_memray.py b/distributed/diagnostics/tests/test_memray.py index 05c9576254a..b7de905181d 100644 --- a/distributed/diagnostics/tests/test_memray.py +++ b/distributed/diagnostics/tests/test_memray.py @@ -4,7 +4,7 @@ memray = pytest.importorskip("memray") -pytestmark = pytest.mark.extra_packages +pytestmark = pytest.mark.memray from distributed import Client, LocalCluster from distributed.diagnostics.memray import memray_scheduler, memray_workers diff --git a/pyproject.toml b/pyproject.toml index b07c4e54d0e..7070490d7fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,12 +184,12 @@ filterwarnings = [ minversion = "6" markers = [ "ci1: marks tests as belonging to 1 out of 2 partitions to run on CI ('-m \"not ci1\"' for second partition)", - "extra_packages: marks tests that require a special dependency to run.", - "slow: marks tests as slow (deselected by default", # select with '--runslow') + "memray: marks tests that require memray to run (available on Linux/MacOS only)", + "slow: marks tests as slow (deselected by default; select with '--runslow')", "avoid_ci: marks tests as flaky or broken on CI on all OSs", "ipython: marks tests as exercising IPython", "gpu: marks tests we want to run on GPUs", - "leaking: ignore leaked resources", # see pytest_resourceleaks.py for usage + "leaking: ignore leaked resources", # see pytest_resourceleaks.py for usage "workerstate: deterministic test for the worker state machine. Automatically applied to all tests that use the 'ws' fixture.", ] # pytest-timeout settings From ab29a15a3908cbc5f802e02482974f2972ca45be Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 4 May 2026 10:59:22 +0100 Subject: [PATCH 2/5] Cosmetic --- .github/workflows/tests.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 246755d5078..ff3dbbbcdc0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -53,35 +53,35 @@ jobs: - os: ubuntu-latest environment: "3.10" label: no_queue - partition: "ci1" + partition: ci1 - os: ubuntu-latest environment: "3.10" label: no_queue - partition: "not ci1" + partition: not ci1 # dask.array P2P shuffle - os: ubuntu-latest environment: mindeps label: numpy extra_packages: [numpy=1.24] - partition: "ci1" + partition: ci1 - os: ubuntu-latest environment: mindeps label: numpy extra_packages: [numpy=1.24] - partition: "not ci1" + partition: not ci1 # dask.dataframe P2P shuffle - os: ubuntu-latest environment: mindeps label: pandas extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0] - partition: "ci1" + partition: ci1 - os: ubuntu-latest environment: mindeps label: pandas extra_packages: [numpy=1.24, pandas=2.0, pyarrow=16.0] - partition: "not ci1" + partition: not ci1 - os: ubuntu-latest environment: "3.10" From 7f6364b2bc5715998d44323a498b9761c5fb248a Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 4 May 2026 11:00:35 +0100 Subject: [PATCH 3/5] Cache extra installs --- .github/workflows/tests.yaml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ff3dbbbcdc0..94060b46c16 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -150,7 +150,7 @@ jobs: uses: actions/cache@v5 with: path: ${{ env.CONDA }}/envs - key: conda-${{ matrix.os }}-${{ matrix.environment }}-${{ matrix.label }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(env.CONDA_FILE) }}-${{ env.CACHE_NUMBER }} + key: conda-${{ matrix.os }}-${{ matrix.environment }}-${{ matrix.label }}-${{ matrix.extra_packages }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(env.CONDA_FILE) }}-${{ env.CACHE_NUMBER }} env: # Increase this value to reset cache if # continuous_integration/environment-${{ matrix.environment }}.yaml has not @@ -168,16 +168,23 @@ jobs: || steps.cache.outputs.cache-hit != 'true' ) + - name: Extra Installs + if: | + matrix.extra_packages + && ( + steps.skip-caching.outputs.trigger-found == 'true' + || (github.event_name == 'pull_request' + && contains(github.event.pull_request.labels.*.name, 'skip-caching')) + || steps.cache.outputs.cache-hit != 'true' + ) + shell: bash -l {0} + run: mamba install -y ${{ join(matrix.extra_packages, ' ') }} + - name: Install shell: bash -l {0} run: | python -m pip install --no-deps -e . - - name: Extra Installs - if: ${{ matrix.extra_packages }} - shell: bash -l {0} - run: mamba install -y ${{ join(matrix.extra_packages, ' ') }} - - name: mamba list shell: bash -l {0} run: mamba list From 58dcfb6daf0940db8818c9760e601e7d7a1f30ce Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 4 May 2026 11:38:55 +0100 Subject: [PATCH 4/5] Use 3.11 as a base --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 94060b46c16..1fe8975d9df 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -84,7 +84,8 @@ jobs: partition: not ci1 - os: ubuntu-latest - environment: "3.10" + # Don't disturb the various GPU-only deps in the 3.10 environment + environment: "3.11" label: memray extra_packages: [memray] # Not available on Windows partition: memray From 1b17d74eb67ed3c9cff9600d79f5601b31e7a222 Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 4 May 2026 11:01:18 +0100 Subject: [PATCH 5/5] Replace mamba with conda --- .github/workflows/test-report.yaml | 5 ++--- .github/workflows/tests.yaml | 15 +++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-report.yaml b/.github/workflows/test-report.yaml index 6b9f14ded75..09e645a0728 100644 --- a/.github/workflows/test-report.yaml +++ b/.github/workflows/test-report.yaml @@ -25,15 +25,14 @@ jobs: with: miniforge-version: latest condarc-file: continuous_integration/condarc - use-mamba: true environment-file: continuous_integration/scripts/test-report-environment.yml activate-environment: dask-distributed - name: Show conda options run: conda config --show - - name: mamba list - run: mamba list + - name: conda list + run: conda list - uses: actions/cache@v5 id: cache diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1fe8975d9df..866d75858a8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -122,7 +122,6 @@ jobs: with: miniforge-version: latest condarc-file: continuous_integration/condarc - use-mamba: true activate-environment: dask-distributed - name: Show conda options @@ -160,7 +159,7 @@ jobs: id: cache - name: Update environment - run: mamba env update -n dask-distributed -f ${{ env.CONDA_FILE }} + run: conda env update -n dask-distributed -f ${{ env.CONDA_FILE }} if: | ( steps.skip-caching.outputs.trigger-found == 'true' @@ -179,22 +178,22 @@ jobs: || steps.cache.outputs.cache-hit != 'true' ) shell: bash -l {0} - run: mamba install -y ${{ join(matrix.extra_packages, ' ') }} + run: conda install -y ${{ join(matrix.extra_packages, ' ') }} - name: Install shell: bash -l {0} run: | python -m pip install --no-deps -e . - - name: mamba list + - name: conda list shell: bash -l {0} - run: mamba list + run: conda list - - name: mamba env export + - name: conda env export shell: bash -l {0} run: | - echo -e "--\n--Conda Environment (re-create this with \`mamba env create --name -f \`)\n--" - mamba env export | grep -E -v '^prefix:.*$' + echo -e "--\n--Conda Environment (re-create this with \`conda env create --name -f \`)\n--" + conda env export | grep -E -v '^prefix:.*$' - name: Setup SSH shell: bash -l {0}