From 26587c68d772dfd42385302c96f98db5931c6e3b Mon Sep 17 00:00:00 2001 From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:57:21 +0100 Subject: [PATCH 1/7] update bluesky-plugins to be minimal --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f14966b..45303d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "event-model", "mongoquery", "pytz", - "tiled[client] >=0.2.15", + "tiled[minimal-client] >=0.2.15", "tzlocal", ] From 343b9f0a95e69c1493f719ee0660183b765db3fa Mon Sep 17 00:00:00 2001 From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:49:49 +0100 Subject: [PATCH 2/7] Declare runtime deps needed with tiled[minimal-client] The plugin imports xarray, pandas, pyarrow and numpy at module level, and tiled.ndslice needs ndindex, which tiled's xarray extra omits. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 45303d9..b9a9bfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,12 @@ dependencies = [ "dask", "event-model", "mongoquery", + "ndindex", # imported by tiled.ndslice; not in tiled's xarray extra + "numpy", + "pandas", + "pyarrow", "pytz", - "tiled[minimal-client] >=0.2.15", + "tiled[minimal-client,xarray] >=0.2.15", "tzlocal", ] From fe032d9d94f78082e0d90c90ae33868a0f62549e Mon Sep 17 00:00:00 2001 From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:56:46 +0100 Subject: [PATCH 3/7] Depend on xarray directly instead of tiled's xarray extra Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b9a9bfd..90a3233 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,13 +37,14 @@ dependencies = [ "dask", "event-model", "mongoquery", - "ndindex", # imported by tiled.ndslice; not in tiled's xarray extra + "ndindex", # imported by tiled.ndslice "numpy", "pandas", "pyarrow", "pytz", - "tiled[minimal-client,xarray] >=0.2.15", + "tiled[minimal-client] >=0.2.15", "tzlocal", + "xarray", ] From 7482a6be422d9d7ce29509a8764d2875e2be8eec Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 14 Oct 2026 15:20:14 +1300 Subject: [PATCH 4/7] Add ragged/all extras and raise an actionable error for missing ragged deps With tiled[minimal-client], the awkward and ragged packages are no longer installed, so writing ragged (variable-length) array data fails at the lazy import inside TiledWriter. Declare the optional dependencies as published extras -- bluesky-tiled-plugins[ragged] (pins matching the tiled[client] extra) and an [all] aggregate -- and wrap the lazy import so the mid-scan failure tells the user which extra to install. Also bump the minimum tiled to 0.2.19 (runtime and test group). --- pyproject.toml | 16 ++++++++++++++-- .../writing/tiled_writer.py | 9 ++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 90a3233..30d23f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,11 +42,23 @@ dependencies = [ "pandas", "pyarrow", "pytz", - "tiled[minimal-client] >=0.2.15", + "tiled[minimal-client] >=0.2.19", "tzlocal", "xarray", ] +[project.optional-dependencies] +# Writing ragged (variable-length) array data; imported lazily by TiledWriter. +# Pins match the tiled[client] extra. +ragged = [ + "awkward >=2.4.3", + "ragged", +] +# All optional features. +all = [ + "bluesky-tiled-plugins[ragged]", +] + [project.urls] Homepage = "https://github.com/bluesky/bluesky-tiled-plugins" @@ -76,7 +88,7 @@ test = [ "redis-json-dict", "suitcase-mongo", "tifffile", - "tiled[server] >=0.2.15", + "tiled[server] >=0.2.19", ] dev = [{ include-group = "test" }] docs = [ diff --git a/src/bluesky_tiled_plugins/writing/tiled_writer.py b/src/bluesky_tiled_plugins/writing/tiled_writer.py index be436df..3d6e494 100644 --- a/src/bluesky_tiled_plugins/writing/tiled_writer.py +++ b/src/bluesky_tiled_plugins/writing/tiled_writer.py @@ -816,7 +816,14 @@ def _write_internal_data( # 2. Write internal ragged array data, if any; remove it from the tabular data for key in self._int_ragged_array_keys[desc_name]: - from tiled.structures.ragged import make_ragged_array + try: + from tiled.structures.ragged import make_ragged_array + except ImportError as err: + raise ImportError( + "Writing ragged (variable-length) array data requires optional " + "dependencies. Install them with: " + "pip install bluesky-tiled-plugins[ragged]" + ) from err arr_lst = [row.pop(key) for row in data_cache if key in row] shape = (len(arr_lst), *self.data_keys[key].get("shape", ())) From f684635a04481df447fe6bd895dcecbc5d62372a Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 14 Oct 2026 15:36:14 +1300 Subject: [PATCH 5/7] Add a CI job verifying the minimal install is sufficient The test dependency group includes tiled[server], which transitively provides awkward, ragged, and more -- so the regular test jobs can not catch a missing runtime dependency. Add a minimal-install job that installs the package with no extras and no dependency groups, then runs tests/minimal_smoke.py: it import-walks every submodule (except the server-only routers.validator), exercises the document-processing machinery without a server, and asserts the optional ragged dependencies are genuinely absent so the slimmed dependency footprint can not silently regress. The smoke script is not collected by pytest (no test_ prefix); it fails by design in a full dev environment. --- .github/workflows/ci.yml | 33 +++++++++++ tests/minimal_smoke.py | 116 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 tests/minimal_smoke.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0809d01..30ca751 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,39 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + # Verify that the declared runtime dependencies are sufficient on their own: + # install the package with no extras and no dev/test dependency groups (so + # e.g. tiled[server] in the test group does not mask a missing runtime + # dependency) and smoke test it. See tests/minimal_smoke.py. + minimal-install: + name: Minimal install / Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + needs: [pre-commit] + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.13"] + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - uses: astral-sh/setup-uv@v7 + + - name: Install package without extras or dependency groups + run: | + uv venv --python ${{ matrix.python-version }} + uv pip install . + + - name: Smoke test the minimal installation + run: .venv/bin/python tests/minimal_smoke.py + # Windows and macOS jobs run without Mongo/Redis. Tests gated on # TEST_MONGO_URI / TEST_REDIS_URI skip themselves automatically. other-checks: diff --git a/tests/minimal_smoke.py b/tests/minimal_smoke.py new file mode 100644 index 0000000..7008f53 --- /dev/null +++ b/tests/minimal_smoke.py @@ -0,0 +1,116 @@ +"""Smoke test for a MINIMAL installation of bluesky-tiled-plugins. + +Run with plain python (not pytest) in an environment where only the package and +its declared runtime dependencies are installed -- no extras, no dev/test +dependency groups: + + uv venv && uv pip install . && .venv/bin/python tests/minimal_smoke.py + +It verifies that the declared runtime dependencies are sufficient: every +submodule imports, the client-side writing machinery works, and the optional +ragged dependencies are genuinely absent (so this script will fail by design +if run in a full dev/test environment). +""" + +import importlib +import pkgutil +import traceback + +import bluesky_tiled_plugins as btp + +# Modules that intentionally require tiled[server] and are only imported in +# server deployments (e.g. mounted via the server config). +SERVER_ONLY = {"bluesky_tiled_plugins.routers.validator"} + +failures = [] +for mod in pkgutil.walk_packages(btp.__path__, btp.__name__ + "."): + if mod.name in SERVER_ONLY: + continue + try: + importlib.import_module(mod.name) + except Exception: + failures.append((mod.name, traceback.format_exc())) + +for name, tb in failures: + print(f"FAILED IMPORT: {name}\n{tb}") +if failures: + raise SystemExit(f"{len(failures)} module(s) failed to import; see above.") + +# The public API is usable. +from bluesky_tiled_plugins import TiledInserter, TiledWriter # noqa: E402, F401 +from bluesky_tiled_plugins.writing.consolidators import ( # noqa: E402 + consolidator_factory, +) +from bluesky_tiled_plugins.writing.tiled_writer import RunNormalizer # noqa: E402 + +# The document-processing machinery works without a server. +received = [] +normalizer = RunNormalizer() +normalizer.subscribe(lambda name, doc: received.append(name)) +normalizer("start", {"uid": "u1", "time": 0.0}) +normalizer( + "descriptor", + { + "uid": "d1", + "run_start": "u1", + "time": 0.0, + "name": "primary", + "data_keys": {"x": {"source": "s", "dtype": "number", "shape": []}}, + "object_keys": {"det": ["x"]}, + "configuration": {}, + "hints": {}, + }, +) +normalizer( + "event", + { + "uid": "e1", + "descriptor": "d1", + "time": 1.0, + "seq_num": 1, + "data": {"x": 1.0}, + "timestamps": {"x": 1.0}, + "filled": {}, + }, +) +normalizer( + "stop", + { + "uid": "s1", + "run_start": "u1", + "time": 2.0, + "exit_status": "success", + "reason": "", + "num_events": {"primary": 1}, + }, +) +assert "start" in received and "stop" in received, received + +consolidator = consolidator_factory( + { + "data_key": "img", + "uri": "file:///x", + "mimetype": "multipart/related;type=image/tiff", + "parameters": {"chunk_shape": (1, 5, 7), "template": "f_{:06d}.tiff"}, + }, + {"data_keys": {"img": {"shape": [3, 5, 7], "dtype": "array", "dtype_numpy": "|u1"}}}, +) +consolidator.consume_stream_datum( + {"indices": {"start": 0, "stop": 1}, "seq_nums": {"start": 1, "stop": 2}} +) +assert consolidator.shape and consolidator.assets +assert consolidator.structure() is not None + +# The optional ragged dependencies must NOT be present in a minimal install; +# this guards against them sneaking back in through a transitive dependency +# (install bluesky-tiled-plugins[ragged] to get them). +try: + import tiled.structures.ragged # noqa: F401 +except ImportError: + pass +else: + raise SystemExit( + "tiled.structures.ragged is importable: the environment is not minimal." + ) + +print("Minimal installation smoke test: OK") From dd0eb91fa2ccbfefcf65a11c928e60d39ff5668a Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 14 Oct 2026 15:42:56 +1300 Subject: [PATCH 6/7] Move the minimal-install smoke script out of tests/ The name-tests-test pre-commit hook requires files under tests/ to match test_*.py, and the smoke script must not be collected by pytest (it fails by design in a full dev environment). Place it under .github/scripts/, next to the workflow that runs it. --- {tests => .github/scripts}/minimal_smoke.py | 8 ++++++-- .github/workflows/ci.yml | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) rename {tests => .github/scripts}/minimal_smoke.py (94%) diff --git a/tests/minimal_smoke.py b/.github/scripts/minimal_smoke.py similarity index 94% rename from tests/minimal_smoke.py rename to .github/scripts/minimal_smoke.py index 7008f53..38cd537 100644 --- a/tests/minimal_smoke.py +++ b/.github/scripts/minimal_smoke.py @@ -4,7 +4,7 @@ its declared runtime dependencies are installed -- no extras, no dev/test dependency groups: - uv venv && uv pip install . && .venv/bin/python tests/minimal_smoke.py + uv venv && uv pip install . && .venv/bin/python .github/scripts/minimal_smoke.py It verifies that the declared runtime dependencies are sufficient: every submodule imports, the client-side writing machinery works, and the optional @@ -93,7 +93,11 @@ "mimetype": "multipart/related;type=image/tiff", "parameters": {"chunk_shape": (1, 5, 7), "template": "f_{:06d}.tiff"}, }, - {"data_keys": {"img": {"shape": [3, 5, 7], "dtype": "array", "dtype_numpy": "|u1"}}}, + { + "data_keys": { + "img": {"shape": [3, 5, 7], "dtype": "array", "dtype_numpy": "|u1"} + } + }, ) consolidator.consume_stream_datum( {"indices": {"start": 0, "stop": 1}, "seq_nums": {"start": 1, "stop": 2}} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30ca751..5dab7d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Verify that the declared runtime dependencies are sufficient on their own: # install the package with no extras and no dev/test dependency groups (so # e.g. tiled[server] in the test group does not mask a missing runtime - # dependency) and smoke test it. See tests/minimal_smoke.py. + # dependency) and smoke test it. See .github/scripts/minimal_smoke.py. minimal-install: name: Minimal install / Python ${{ matrix.python-version }} runs-on: ubuntu-latest @@ -121,7 +121,7 @@ jobs: uv pip install . - name: Smoke test the minimal installation - run: .venv/bin/python tests/minimal_smoke.py + run: .venv/bin/python .github/scripts/minimal_smoke.py # Windows and macOS jobs run without Mongo/Redis. Tests gated on # TEST_MONGO_URI / TEST_REDIS_URI skip themselves automatically. From 05d5c67cd9911a2ca50ca91354dc68857307d704 Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 14 Oct 2026 15:53:26 +1300 Subject: [PATCH 7/7] Restrict GITHUB_TOKEN to contents: read in the CI workflow The CI jobs only check out the repository and run tests; the codecov upload authenticates with its own CODECOV_TOKEN secret. Setting an explicit minimal permissions block follows the practice already used in cd.yml and resolves the GitHub Actions security warning. --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5dab7d6..4561fa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,11 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# The jobs only check out the repo and run tests; the codecov upload uses its +# own CODECOV_TOKEN secret, not the GITHUB_TOKEN. +permissions: + contents: read + env: # Many color libraries just need this variable to be set to any value. # Set it to 3 to support 8-bit color graphics (256 colors per channel)