From a931238e99283ee74df853bddd4879a2aee598f6 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 10:58:07 +0000 Subject: [PATCH 01/67] Unify movable_logic/flyable_logic into a single logic A Device that both moves and flies (Motor) had to expose the same logic object under two property names. Each mix-in now declares `logic` with its own required type, so one implementation satisfies both and the type checker verifies it against each mix-in independently -- the intersection is expressed by the logic class's own MRO. Also fixes the abstractness of the declaration. `functools.cached_property` does not forward `__isabstractmethod__`, so `@cached_property` over `@abstractmethod` registered nothing: `StandardMovable.__abstractmethods__` was empty and a missing logic surfaced as `AttributeError: 'NoneType' object has no attribute 'readback'` from inside `Device.__init__`. The new `abstract_cached_property` sets the flag on the descriptor so the metaclass collects it, giving a TypeError at instantiation instead. `StandardFlyable` is consequently abstract, so `FlyableLogic.with_device` builds a concrete `_EphemeralFlyable` to attach the logic to. `movable_logic` and `flyable_logic` remain as deprecated properties. Co-Authored-By: Claude Opus 5 (cherry picked from commit e98799b3877b7c55f2835177a763fca6105887c2) --- src/ophyd_async/core/__init__.py | 2 + src/ophyd_async/core/_flyable.py | 60 ++++++++++++++----- src/ophyd_async/core/_movable.py | 59 +++++++++++------- src/ophyd_async/core/_utils.py | 23 ++++++- src/ophyd_async/epics/demo/_motor.py | 2 +- src/ophyd_async/epics/motor.py | 14 +---- src/ophyd_async/sim/_motor.py | 2 +- src/ophyd_async/tango/demo/_motor.py | 2 +- tests/unit_tests/core/test_flyer.py | 14 +---- tests/unit_tests/core/test_movable.py | 16 ++--- .../core/test_multi_derived_signal.py | 4 +- 11 files changed, 125 insertions(+), 73 deletions(-) diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index b9770c804b..1c84360f95 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -154,6 +154,7 @@ SubsetEnum, SupersetEnum, WatcherUpdate, + abstract_cached_property, error_if_none, gather_dict, get_dtype, @@ -323,6 +324,7 @@ def __getattr__(name): "make_datakey", "wait_for_connection", "Ignore", + "abstract_cached_property", "non_zero", "simulate_move", # Derived signal diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index 3eb68c076a..a6410eae25 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -1,4 +1,5 @@ import asyncio +import warnings from abc import abstractmethod from collections.abc import AsyncIterator from enum import Enum @@ -17,6 +18,7 @@ CalculatableTimeout, ConfinedModel, WatcherUpdate, + abstract_cached_property, ) #: The per-scan info passed to `FlyableLogic.on_prepare` (often a @@ -82,10 +84,12 @@ async def on_unstage(self) -> None: """Clean the flyer up on `unstage()`. Defaults to `stop`.""" await self.stop() - def with_device(self, name: str = "") -> "StandardFlyable": + def with_device(self, name: str = "") -> "StandardFlyable[PrepareT, CtxT]": """Wrap this logic in an ephemeral `StandardFlyable` for use in a plan.""" - flyer = StandardFlyable(name=name) - flyer.flyable_logic = self + flyer = _EphemeralFlyable[PrepareT, CtxT](name=name) + # cached_property writes straight to the instance __dict__, so this + # satisfies the abstract declaration without a subclass per logic type. + flyer.logic = self return flyer @@ -146,7 +150,7 @@ class StandardFlyable( ): """Device that provides standard logic for flying. - This class must be inherited and have a `flyable_logic` @cached_property. + This class must be inherited and have a `logic` @cached_property. For an ephemeral flyer in a plan, call `FlyableLogic.with_device` instead of inheriting. It owns the context threaded between the logic's stages and enforces prepare -> kickoff -> complete ordering. `stage()`/`unstage()` run @@ -165,18 +169,29 @@ def __init__(self, *args, **kwargs) -> None: self._fly_ctx: CtxT = cast(CtxT, None) self._fly_stage = _FlyStage.IDLE - @cached_property - def flyable_logic(self) -> FlyableLogic[PrepareT, CtxT]: + @abstract_cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: """The logic object that describes how this device flies. A static flyer (e.g. `Motor`) provides this as a `@cached_property` that builds a `FlyableLogic` from its signals. An ephemeral flyer created by `FlyableLogic.with_device` has it set directly on the instance. + + A Device that is flyable *and* movable implements this once with a logic + object inheriting both `FlyableLogic` and `MovableLogic`; each mix-in + declares `logic` with its own required type, so the type + checker verifies the one implementation against both. """ - raise NotImplementedError( - f"{type(self).__name__} has no flyable_logic; override it as a " - "@cached_property or create the flyer via FlyableLogic.with_device()" + raise NotImplementedError + + # Back compat - delete before 1.0 + @property + def flyable_logic(self) -> FlyableLogic[PrepareT, CtxT]: + warnings.warn( + DeprecationWarning("Use `logic` instead of `flyable_logic`"), + stacklevel=2, ) + return self.logic def _reset_fly_state(self) -> None: self._fly_ctx = cast(CtxT, None) @@ -184,18 +199,18 @@ def _reset_fly_state(self) -> None: @AsyncStatus.wrap async def _on_stage(self) -> None: - await self.flyable_logic.on_stage() + await self.logic.on_stage() self._reset_fly_state() @AsyncStatus.wrap async def _on_unstage(self) -> None: - await self.flyable_logic.on_unstage() + await self.logic.on_unstage() self._reset_fly_state() @AsyncStatus.wrap async def prepare(self, value: PrepareT) -> None: """Move to the start and set up the fly scan.""" - self._fly_ctx = await self.flyable_logic.on_prepare(value) + self._fly_ctx = await self.logic.on_prepare(value) self._fly_stage = _FlyStage.PREPARED @AsyncStatus.wrap @@ -205,7 +220,7 @@ async def kickoff(self) -> None: raise RuntimeError( f"{self.name}: prepare() must be called before kickoff()" ) - self._fly_ctx = await self.flyable_logic.on_kickoff(self._fly_ctx) + self._fly_ctx = await self.logic.on_kickoff(self._fly_ctx) self._fly_stage = _FlyStage.KICKED_OFF @WatchableAsyncStatus.wrap @@ -221,7 +236,7 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: raise RuntimeError( f"{self.name}: kickoff() must be called before complete()" ) - logic = self.flyable_logic + logic = self.logic if isinstance(logic, MovableLogic): initial, target, (units, precision) = await asyncio.gather( logic.readback.get_value(), @@ -241,5 +256,20 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: precision=precision, ) else: - await self.flyable_logic.on_complete(self._fly_ctx) + await self.logic.on_complete(self._fly_ctx) self._fly_stage = _FlyStage.IDLE + + +class _EphemeralFlyable(StandardFlyable[PrepareT, CtxT]): + """A concrete `StandardFlyable` whose logic is set on the instance. + + `StandardFlyable.logic` is abstract, so the class itself cannot be + instantiated; `FlyableLogic.with_device` needs something concrete to attach + a logic object to. + """ + + @cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: + raise NotImplementedError( + f"{self.name}: no logic set; create this via FlyableLogic.with_device()" + ) diff --git a/src/ophyd_async/core/_movable.py b/src/ophyd_async/core/_movable.py index beeb1a1ee8..bf0a9b3f35 100644 --- a/src/ophyd_async/core/_movable.py +++ b/src/ophyd_async/core/_movable.py @@ -1,10 +1,9 @@ import asyncio import time -from abc import abstractmethod +import warnings from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from functools import cached_property from typing import Generic from bluesky.protocols import ( @@ -26,6 +25,7 @@ CalculatableTimeout, Callback, WatcherUpdate, + abstract_cached_property, ) @@ -111,9 +111,9 @@ async def connect(self, device: "StandardMovable") -> None: """Mock signals to do an instant move on setpoint write.""" def _instant_move(value): - set_mock_value(device.movable_logic.readback, value) # Arrive instantly + set_mock_value(device.logic.readback, value) # Arrive instantly - callback_on_mock_put(device.movable_logic.setpoint, _instant_move) + callback_on_mock_put(device.logic.setpoint, _instant_move) class StopState(Enum): @@ -133,7 +133,7 @@ class StandardMovable( ): """Device that provides standard logic for moving. - This class must be inherited and have a `movable_logic` @cached_property. + This class must be inherited and have a `logic` @cached_property. If stop is called while moving, it will raise a RuntimeError. """ @@ -142,20 +142,35 @@ class StandardMovable( _stop_state: Enum = StopState.NONE _move_status: AsyncStatus | None = None - @cached_property - @abstractmethod - def movable_logic(self) -> MovableLogic: + @abstract_cached_property + def logic(self) -> MovableLogic[SignalDatatypeT]: """The logic object that describes how this device moves. This is intentionally public so that mock helpers (e.g. `InstantMovableMock`) and subclasses can access the `setpoint` and `readback` signals directly. Subclasses must implement this as a - `@cached_property` that returns a `MovableLogic` instance. + `@cached_property` returning a `MovableLogic`. + + A Device that is movable *and* flyable (e.g. `Motor`) implements this + once with a logic object that inherits both `MovableLogic` and + `FlyableLogic`; each mix-in declares `logic` with its own + required type, so the type checker verifies the one implementation + against both. """ + raise NotImplementedError + + # Back compat - delete before 1.0 + @property + def movable_logic(self) -> MovableLogic[SignalDatatypeT]: + warnings.warn( + DeprecationWarning("Use `logic` instead of `movable_logic`"), + stacklevel=2, + ) + return self.logic async def check_value(self, value: SignalDatatypeT) -> None: """Check the move is valid before doing it.""" - await self.movable_logic.check_move(value) + await self.logic.check_move(value) @WatchableAsyncStatus.wrap async def set( @@ -166,13 +181,13 @@ async def set( """Move to the given value.""" self._stop_state = StopState.NONE old_position, (units, precision) = await asyncio.gather( - self.movable_logic.readback.get_value(), - self.movable_logic.get_units_precision(), + self.logic.readback.get_value(), + self.logic.get_units_precision(), ) - await self.movable_logic.check_move(new_position) + await self.logic.check_move(new_position) if timeout == CALCULATE_TIMEOUT: - move_timeout = await self.movable_logic.calculate_timeout( + move_timeout = await self.logic.calculate_timeout( old_position, new_position ) else: @@ -180,12 +195,12 @@ async def set( try: async with AsyncStatus( - self.movable_logic.move( + self.logic.move( new_position=new_position, timeout=MoveTimeout(move_timeout) ) ) as self._move_status: async for current_position in observe_value( - self.movable_logic.readback, + self.logic.readback, done_status=self._move_status, ): yield WatcherUpdate( @@ -215,20 +230,20 @@ async def set( async def stop(self, success=False): """Request to stop moving and return immediately.""" self._stop_state = StopState.USER_SUCCESS if success else StopState.USER_FAILURE - await self.movable_logic.stop() + await self.logic.stop() if self._move_status: self._move_status.task.cancel() def set_name(self, name: str, *, child_name_separator: str | None = None) -> None: super().set_name(name, child_name_separator=child_name_separator) # Readback should be named the same as its parent in read() - self.movable_logic.readback.set_name(name) + self.logic.readback.set_name(name) async def locate(self) -> Location[SignalDatatypeT]: """Return the current setpoint and readback of the device.""" setpoint, readback = await asyncio.gather( - self.movable_logic.setpoint.get_value(), - self.movable_logic.readback.get_value(), + self.logic.setpoint.get_value(), + self.logic.readback.get_value(), ) return Location(setpoint=setpoint, readback=readback) @@ -236,7 +251,7 @@ def subscribe_reading( self, function: Callback[dict[str, Reading[SignalDatatypeT]]] ) -> None: """Subscribe to reading.""" - self.movable_logic.readback.subscribe_reading(function) + self.logic.readback.subscribe_reading(function) subscribe = subscribe_reading @@ -244,4 +259,4 @@ def clear_sub( self, function: Callback[dict[str, Reading[SignalDatatypeT]]] ) -> None: """Unsubscribe.""" - self.movable_logic.readback.clear_sub(function) + self.logic.readback.clear_sub(function) diff --git a/src/ophyd_async/core/_utils.py b/src/ophyd_async/core/_utils.py index 6acd93551c..477ba8da5e 100644 --- a/src/ophyd_async/core/_utils.py +++ b/src/ophyd_async/core/_utils.py @@ -13,7 +13,7 @@ ) from dataclasses import dataclass from enum import Enum, EnumMeta, StrEnum -from functools import lru_cache +from functools import cached_property, lru_cache from inspect import isawaitable from typing import ( Any, @@ -439,6 +439,27 @@ def non_zero(value): return int(value) != 0 +def abstract_cached_property(func: Callable[[Any], T]) -> cached_property[T]: + """A `cached_property` that `ABCMeta` treats as abstract. + + `functools.cached_property` does not forward `__isabstractmethod__` from the + function it wraps, so `@cached_property` stacked on `@abstractmethod` + registers nothing: the class stays instantiable and a missing implementation + surfaces as an `AttributeError` on `None` from deep inside `__init__`. Setting + the flag on the descriptor itself makes `Device`'s metaclass collect it, so + the omission is a `TypeError` at instantiation instead. + + The declared return type is deliberately `cached_property[T]` rather than a + subclass, so that a subclass overriding it with a plain `@cached_property` + still gets checked for covariance by the type checker. + + :param func: The property body, which should raise `NotImplementedError`. + """ + prop = cached_property(func) + prop.__isabstractmethod__ = True # type: ignore[attr-defined] + return prop + + async def _wait_for(coro: Awaitable[T], timeout: float | None, source: str) -> T: try: return await asyncio.wait_for(coro, timeout) diff --git a/src/ophyd_async/epics/demo/_motor.py b/src/ophyd_async/epics/demo/_motor.py index 308bc4f99a..40be2edcfa 100644 --- a/src/ophyd_async/epics/demo/_motor.py +++ b/src/ophyd_async/epics/demo/_motor.py @@ -55,7 +55,7 @@ class DemoMotor(EpicsDevice, StandardReadable, StandardMovable[float]): stop_: A[TriggerableCommand, PvSuffix("Stop.PROC")] @cached_property - def movable_logic(self) -> MovableLogic: + def logic(self) -> MovableLogic: return DemoMotorMoveLogic( readback=self.readback, setpoint=self.setpoint, diff --git a/src/ophyd_async/epics/motor.py b/src/ophyd_async/epics/motor.py index bb89b38982..b4a1903bb9 100644 --- a/src/ophyd_async/epics/motor.py +++ b/src/ophyd_async/epics/motor.py @@ -103,7 +103,7 @@ class MotorFlyableMovableLogic( ): """Combined move + fly logic for a motor record. - A single logic object backs a `Motor`'s `movable_logic` and `flyable_logic`: + A single logic object backs a `Motor`'s `logic`: it implements `MovableLogic` (the motor-record move, limit and timeout logic) and `FlyableLogic` (the fly hooks, which reuse `move`/`check_move` and carry all per-scan state in a `MotorFlyCtx` rather than on the logic). @@ -343,8 +343,8 @@ def __init__(self, prefix: str, name="") -> None: super().__init__(name) @cached_property - def _logic(self) -> MotorFlyableMovableLogic: - """The combined move + fly logic, shared by movable_logic and flyable_logic.""" + def logic(self) -> MotorFlyableMovableLogic: + """The combined move + fly logic, satisfying both mix-ins at once.""" return MotorFlyableMovableLogic( readback=self.user_readback, setpoint=self.user_setpoint, @@ -359,11 +359,3 @@ def _logic(self) -> MotorFlyableMovableLogic: max_velocity=self.max_velocity, motor_egu=self.motor_egu, ) - - @cached_property - def movable_logic(self) -> MotorFlyableMovableLogic: - return self._logic - - @cached_property - def flyable_logic(self) -> MotorFlyableMovableLogic: - return self._logic diff --git a/src/ophyd_async/sim/_motor.py b/src/ophyd_async/sim/_motor.py index 41697fd2e1..ba31504658 100644 --- a/src/ophyd_async/sim/_motor.py +++ b/src/ophyd_async/sim/_motor.py @@ -83,7 +83,7 @@ def __init__( super().__init__(name=name) @cached_property - def movable_logic(self): + def logic(self): return SimMotorMoveLogic( readback=self.user_readback, readback_set=self._user_readback_set, diff --git a/src/ophyd_async/tango/demo/_motor.py b/src/ophyd_async/tango/demo/_motor.py index f1194763f0..6fd504d4df 100644 --- a/src/ophyd_async/tango/demo/_motor.py +++ b/src/ophyd_async/tango/demo/_motor.py @@ -52,7 +52,7 @@ class DemoMotor(TangoDevice, StandardReadable, StandardMovable[float]): stop_: TriggerableCommand @cached_property - def movable_logic(self) -> MovableLogic: + def logic(self) -> MovableLogic: return DemoMotorMoveLogic( readback=self.readback, setpoint=self.setpoint, diff --git a/tests/unit_tests/core/test_flyer.py b/tests/unit_tests/core/test_flyer.py index e267c4f500..084db28c9c 100644 --- a/tests/unit_tests/core/test_flyer.py +++ b/tests/unit_tests/core/test_flyer.py @@ -87,17 +87,9 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def _logic(self) -> MovableFlyableLogic: + def logic(self) -> MovableFlyableLogic: return MovableFlyableLogic(setpoint=self.setpoint, readback=self.readback) - @cached_property - def movable_logic(self) -> MovableFlyableLogic: - return self._logic - - @cached_property - def flyable_logic(self) -> MovableFlyableLogic: - return self._logic - @pytest.fixture async def recording_flyer(): @@ -114,7 +106,7 @@ async def test_ephemeral_flyable_drives_logic_hooks(recording_flyer): assert flyer.name == "flyer" assert isinstance(flyer, StandardFlyable) # The ephemeral device exposes the same logic instance - assert flyer.flyable_logic is logic + assert flyer.logic is logic await flyer.prepare(5) await flyer.kickoff() @@ -250,7 +242,7 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def flyable_logic(self) -> FlyableLogic[int, None]: + def logic(self) -> FlyableLogic[int, None]: return logic async with init_devices(mock=True): diff --git a/tests/unit_tests/core/test_movable.py b/tests/unit_tests/core/test_movable.py index c3df8fd096..7ac786f6b0 100644 --- a/tests/unit_tests/core/test_movable.py +++ b/tests/unit_tests/core/test_movable.py @@ -50,7 +50,7 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def movable_logic(self) -> MovableLogic[float]: + def logic(self) -> MovableLogic[float]: return MovableLogic(setpoint=self.setpoint, readback=self.readback) @@ -62,16 +62,16 @@ async def movable() -> StandardMovableImpl: def test_movable_logic_is_cached(movable: StandardMovableImpl): - logic = movable.movable_logic - logic2 = movable.movable_logic + logic = movable.logic + logic2 = movable.logic assert logic == logic2 async def test_movable_check_value(movable: StandardMovableImpl): - movable.movable_logic.check_move = AsyncMock() + movable.logic.check_move = AsyncMock() await movable.check_value(5) - movable.movable_logic.check_move.assert_awaited_once_with(5) + movable.logic.check_move.assert_awaited_once_with(5) async def test_locatable(movable: StandardMovableImpl) -> None: @@ -133,12 +133,12 @@ async def test_movable_moving_stopped(movable: StandardMovableImpl): async def test_movable_set_calls_movable_logic_check_move_and_calculate_timeout( movable: StandardMovableImpl, ): - mock_check_move = movable.movable_logic.check_move = AsyncMock() + mock_check_move = movable.logic.check_move = AsyncMock() timeout = 5 - mock_calculate_timeout = movable.movable_logic.calculate_timeout = AsyncMock( + mock_calculate_timeout = movable.logic.calculate_timeout = AsyncMock( return_value=timeout ) - mock_move = movable.movable_logic.move = AsyncMock() + mock_move = movable.logic.move = AsyncMock() with patch("ophyd_async.core._movable.MoveTimeout") as move_timeout: pos = 10 diff --git a/tests/unit_tests/core/test_multi_derived_signal.py b/tests/unit_tests/core/test_multi_derived_signal.py index dd252af6fa..3f8b038301 100644 --- a/tests/unit_tests/core/test_multi_derived_signal.py +++ b/tests/unit_tests/core/test_multi_derived_signal.py @@ -298,8 +298,8 @@ async def _put(val: float) -> None: derived_sig = derived_signal_rw(_get, _put, m1=m1, m2=m2) await derived_sig.set(6) - assert await m1.movable_logic.readback.get_value() == 3 - assert await m2.movable_logic.readback.get_value() == 3 + assert await m1.logic.readback.get_value() == 3 + assert await m2.logic.readback.get_value() == 3 assert await derived_sig.get_value() == 6 From ca56d0d851de1d9d40a5175cb1b24b36a54c3ce3 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 11:04:28 +0000 Subject: [PATCH 02/67] Key the readable registry by device so formats can change at runtime Fixes #1394. `add_readables` baked bound methods into four tuples, discarding which device each contribution came from. Nothing could then be found, removed or re-formatted, so there was no equivalent of ophyd v1's runtime `kind`. The registry is now a tuple of `(device, format)` pairs and the callables for each verb are derived on demand, which makes `set_readable_format(dev, format)` a whole-tuple swap. Passing `None` drops the device entirely. Staging is derived the same way, so a format changed between runs is picked up by the next `stage()`. `add_readables` stays as a thin wrapper, so the 85 in-tree call sites and all downstream Devices are unaffected. It has "set" semantics now: registering the same device twice replaces rather than duplicates. The deprecated `ConfigSignal`/`HintedSignal` markers announce themselves by comparing equal to their target, which relied on the `match` running at registration; formats are normalised to real enum members on the way in so the warning still fires there rather than on first read(). Tests that asserted on the private tuples are rewritten onto the public verbs, per the testing conventions in CLAUDE.md. Co-Authored-By: Claude Opus 5 (cherry picked from commit 1a43d10ddf0af8d02062fb77958041774c8c5cb2) --- src/ophyd_async/core/_readable.py | 217 ++++++++++++++++------ tests/unit_tests/core/test_readable.py | 248 +++++++++++++++---------- tests/unit_tests/epics/pvi/test_pvi.py | 2 +- 3 files changed, 312 insertions(+), 155 deletions(-) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index efb18d966a..aa8f34e399 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -1,5 +1,6 @@ +import asyncio import warnings -from collections.abc import Awaitable, Callable, Generator, Sequence +from collections.abc import Awaitable, Callable, Generator, Iterator, Sequence from contextlib import contextmanager from enum import Enum from typing import Any, cast @@ -11,9 +12,45 @@ from ._protocol import AsyncConfigurable, AsyncReadable, AsyncStageable from ._signal import SignalR from ._standard_base import _StandardBase +from ._status import AsyncStatus from ._utils import merge_gathered_dicts +class _Verb(Enum): + """The four bluesky verbs a registered child can contribute to.""" + + DESCRIBE_CONFIG = "DESCRIBE_CONFIG" + READ_CONFIG = "READ_CONFIG" + DESCRIBE = "DESCRIBE" + READ = "READ" + + +_CONFIG_VERBS = frozenset({_Verb.DESCRIBE_CONFIG, _Verb.READ_CONFIG}) +_READ_VERBS = frozenset({_Verb.DESCRIBE, _Verb.READ}) + + +def _as_signal_r(device: Device) -> SignalR: + if not isinstance(device, SignalR): + raise TypeError(f"{device} is not a SignalR") + return device + + +def _normalise_format(format: "StandardReadableFormat") -> "StandardReadableFormat": + """Resolve a deprecated `ConfigSignal`/`HintedSignal` marker to a real member. + + The registry stores real enum members so that the format can be compared, + serialised and reported later. The deprecated markers only announce + themselves when compared, so the comparison has to happen here, at + registration, rather than each time a verb is called. + """ + if isinstance(format, StandardReadableFormat): + return format + for member in StandardReadableFormat: + if format == member: # _WarningMatcher.__eq__ raises the DeprecationWarning + return member + raise TypeError(f"{format} is not a StandardReadableFormat") + + class StandardReadableFormat(Enum): """Declare how a `Device` should contribute to the `StandardReadable` verbs.""" @@ -89,35 +126,112 @@ class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints - Provide their value in `read_configuration()` and `describe_configuration() - Select a value to appear in `hints` - The behavior is customized with a [](#StandardReadableFormat) + The behavior is customized with a [](#StandardReadableFormat), which can be + changed at runtime with [](#StandardReadable.set_readable_format). """ - # These must be immutable types to avoid accidental sharing between - # different instances of the class - _describe_config_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () - _read_config_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () - _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () - _read_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () - _has_hints: tuple[HasHints, ...] = () + # The registered children, in registration order. Immutable so that it cannot + # be accidentally shared between instances of the class, and so that a runtime + # format change swaps the whole tuple rather than mutating shared state. + _readables: tuple[tuple[Device, StandardReadableFormat], ...] = () + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Staging is derived from _readables on each call rather than registered + # up-front, so a format changed between runs is picked up on next stage(). + self._stage_funcs += (self._stage_readables,) + self._unstage_funcs += (self._unstage_readables,) + + @AsyncStatus.wrap + async def _stage_readables(self) -> None: + await asyncio.gather(*(sig.stage().task for sig in self._signals_to_stage())) + + @AsyncStatus.wrap + async def _unstage_readables(self) -> None: + await asyncio.gather(*(sig.unstage().task for sig in self._signals_to_stage())) + + def _signals_to_stage(self) -> Iterator[AsyncStageable]: + for device, format in self._readables: + if format is StandardReadableFormat.HINTED_SIGNAL and isinstance( + device, AsyncStageable + ): + # Caching is set up on stage so that read() is fast + yield device + elif format is StandardReadableFormat.CHILD and isinstance( + device, AsyncStageable + ): + yield device + + def _funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: + """Derive the callables contributing to one verb from the registry.""" + for device, format in self._readables: + match format: + case StandardReadableFormat.CHILD: + if verb in _CONFIG_VERBS and isinstance(device, AsyncConfigurable): + yield ( + device.describe_configuration + if verb is _Verb.DESCRIBE_CONFIG + else device.read_configuration + ) + elif verb in _READ_VERBS and isinstance(device, AsyncReadable): + yield device.describe if verb is _Verb.DESCRIBE else device.read + case StandardReadableFormat.CONFIG_SIGNAL: + signal = _as_signal_r(device) + if verb is _Verb.DESCRIBE_CONFIG: + yield signal.describe + elif verb is _Verb.READ_CONFIG: + yield signal.read + case StandardReadableFormat.HINTED_SIGNAL: + signal = _as_signal_r(device) + if verb is _Verb.DESCRIBE: + yield signal.describe + elif verb is _Verb.READ: + yield signal.read + case ( + StandardReadableFormat.UNCACHED_SIGNAL + | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + ): + signal = _as_signal_r(device) + if verb is _Verb.DESCRIBE: + yield signal.describe + elif verb is _Verb.READ: + yield _UncachedRead(signal) async def describe_configuration(self) -> dict[str, DataKey]: return await merge_gathered_dicts( - [func() for func in self._describe_config_funcs] + [func() for func in self._funcs_for(_Verb.DESCRIBE_CONFIG)] ) async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts([func() for func in self._read_config_funcs]) + return await merge_gathered_dicts( + [func() for func in self._funcs_for(_Verb.READ_CONFIG)] + ) async def describe(self) -> dict[str, DataKey]: - return await merge_gathered_dicts([func() for func in self._describe_funcs]) + return await merge_gathered_dicts( + [func() for func in self._funcs_for(_Verb.DESCRIBE)] + ) async def read(self) -> dict[str, Reading]: - return await merge_gathered_dicts([func() for func in self._read_funcs]) + return await merge_gathered_dicts( + [func() for func in self._funcs_for(_Verb.READ)] + ) + + def _hint_sources(self) -> Iterator[HasHints]: + for device, format in self._readables: + match format: + case StandardReadableFormat.CHILD if isinstance(device, HasHints): + yield device + case ( + StandardReadableFormat.HINTED_SIGNAL + | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + ): + yield _HintsFromName(device) @property def hints(self) -> Hints: hints: Hints = {} - for new_hint in self._has_hints: + for new_hint in self._hint_sources(): # Merge the existing and new hints, based on the type of the value. # This avoids default dict merge behavior that overrides the values; # we want to combine them when they are Sequences, and ensure they are @@ -194,6 +308,41 @@ def add_children_as_readables( new_devices = list(filter(lambda x: isinstance(x, Device), flattened_values)) self.add_readables(new_devices, format) + def set_readable_format( + self, device: Device, format: StandardReadableFormat | None + ) -> None: + """Set how a child Device contributes to this Device's bluesky verbs. + + This is the runtime equivalent of the `Kind` attribute in ophyd v1: it + can be called after construction to move a child between configuration, + hinted and uncached reads, or to drop it from the verbs entirely. + + Call it **between runs**. The descriptor for a run is emitted at its + start, and [](#StandardReadableFormat.HINTED_SIGNAL) sets up caching in + `stage()`, so changing a format part way through a run would make + `describe()` and `read()` disagree. + + :param device: The Device to set the format of, normally a child of this one + :param format: + The format to give it, or `None` to stop it contributing at all. + Replaces any format the device already has, rather than adding a + second contribution. + :raises TypeError: If a signal-only format is given a non-`SignalR` + """ + if format is not None: + format = _normalise_format(format) + if format is not StandardReadableFormat.CHILD: + _as_signal_r(device) + kept = tuple((d, f) for d, f in self._readables if d is not device) + self._readables = kept if format is None else (*kept, (device, format)) + + def get_readable_format(self, device: Device) -> StandardReadableFormat | None: + """Return the format of a child Device, or `None` if it does not contribute.""" + for registered, format in self._readables: + if registered is device: + return format + return None + def add_readables( self, devices: Sequence[Device], @@ -214,46 +363,8 @@ def add_readables( Determines which of the devices functions are added to which verb as per the [](#StandardReadableFormat) documentation """ - - def assert_device_is_signalr(device: Device) -> SignalR: - if not isinstance(device, SignalR): - raise TypeError(f"{device} is not a SignalR") - return device - for device in devices: - match format: - case StandardReadableFormat.CHILD: - if isinstance(device, AsyncConfigurable): - self._describe_config_funcs += (device.describe_configuration,) - self._read_config_funcs += (device.read_configuration,) - if isinstance(device, AsyncReadable): - self._describe_funcs += (device.describe,) - self._read_funcs += (device.read,) - if isinstance(device, AsyncStageable): - self._stage_funcs += (device.stage,) - self._unstage_funcs += (device.unstage,) - if isinstance(device, HasHints): - self._has_hints += (device,) - case StandardReadableFormat.CONFIG_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_config_funcs += (signalr_device.describe,) - self._read_config_funcs += (signalr_device.read,) - case StandardReadableFormat.HINTED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (signalr_device.read,) - self._stage_funcs += (signalr_device.stage,) - self._unstage_funcs += (signalr_device.unstage,) - self._has_hints += (_HintsFromName(signalr_device),) - case StandardReadableFormat.UNCACHED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (_UncachedRead(signalr_device),) - case StandardReadableFormat.HINTED_UNCACHED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (_UncachedRead(signalr_device),) - self._has_hints += (_HintsFromName(signalr_device),) + self.set_readable_format(device, format) class _UncachedRead: diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 75a98b424b..1d4b027a2f 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -4,9 +4,6 @@ from bluesky.protocols import HasHints from ophyd_async.core import ( - AsyncConfigurable, - AsyncReadable, - AsyncStageable, ConfigSignal, Device, DeviceMap, @@ -60,10 +57,7 @@ def test_standard_readable_hints_raises_when_overriding_string_literal(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"gridding": "a different string"} - sr._has_hints = ( - hint1, - hint2, - ) + sr.add_readables([hint1, hint2]) with pytest.raises(RuntimeError, match=r"Hints key .* value may not be overridden"): sr.hints # noqa: B018 @@ -78,10 +72,7 @@ def test_standard_readable_hints_raises_when_overriding_sequence(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"fields": ["field2"]} - sr._has_hints = ( - hint1, - hint2, - ) + sr.add_readables([hint1, hint2]) with pytest.raises(RuntimeError, match=r"Hint fields .* overrides existing hint"): sr.hints # noqa: B018 @@ -94,7 +85,7 @@ def test_standard_readable_hints_invalid_types(invalid_type): hint1 = MagicMock(spec=HasHints) hint1.hints = {"test": invalid_type} - sr._has_hints = (hint1,) + sr.add_readables([hint1]) with pytest.raises(TypeError, match=r"Unknown type for value .* for key .*"): sr.hints # noqa: B018 @@ -158,70 +149,69 @@ def test_standard_readable_add_children_cm_filters_non_devices(): assert set(mock.call_args.args[0]) == {sr.a, sr.b} -def assert_sr_has_attrs(sr: StandardReadable, expected_attrs: dict[str, tuple]): - attrs_to_check = ( - "_describe_config_funcs", - "_read_config_funcs", - "_describe_funcs", - "_read_funcs", - "_stage_funcs", - "_unstage_funcs", - "_has_hints", - ) - actual = {attr: getattr(sr, attr) for attr in attrs_to_check} - expected = {attr: expected_attrs.get(attr, ()) for attr in attrs_to_check} - assert actual == expected - - -signal_r = MagicMock(spec=SignalR) -async_readable = MagicMock(spec=AsyncReadable) -async_configurable = MagicMock(spec=AsyncConfigurable) -async_stageable = MagicMock(spec=AsyncStageable) -has_hints = MagicMock(spec=HasHints) +async def assert_contributes( + sr: StandardReadable, + *, + read: set[str] = frozenset(), + config: set[str] = frozenset(), + hints: set[str] = frozenset(), +) -> None: + """Assert which keys a StandardReadable contributes to each verb.""" + assert set(await sr.read()) == set(read) + assert set(await sr.describe()) == set(read) + assert set(await sr.read_configuration()) == set(config) + assert set(await sr.describe_configuration()) == set(config) + assert set(sr.hints.get("fields", [])) == set(hints) @pytest.mark.parametrize( - "readable, expected_attrs", + "format, in_read, in_config, in_hints", [ - ( - signal_r, - { - "_read_funcs": (signal_r.read,), - "_describe_funcs": (signal_r.describe,), - "_stage_funcs": (signal_r.stage,), - "_unstage_funcs": (signal_r.unstage,), - }, - ), - ( - async_readable, - { - "_read_funcs": (async_readable.read,), - "_describe_funcs": (async_readable.describe,), - }, - ), - ( - async_configurable, - { - "_read_config_funcs": (async_configurable.read_configuration,), - "_describe_config_funcs": (async_configurable.describe_configuration,), - }, - ), - ( - async_stageable, - { - "_stage_funcs": (async_stageable.stage,), - "_unstage_funcs": (async_stageable.unstage,), - }, - ), - (has_hints, {"_has_hints": (has_hints,)}), + (Format.CONFIG_SIGNAL, False, True, False), + (Format.HINTED_SIGNAL, True, False, True), + (Format.UNCACHED_SIGNAL, True, False, False), + (Format.HINTED_UNCACHED_SIGNAL, True, False, True), ], ) -def test_standard_readable_add_readables_adds_to_expected_attrs( - readable, expected_attrs: dict[str, tuple] +async def test_standard_readable_signal_formats_route_to_expected_verbs( + format, in_read, in_config, in_hints ): - sr = StandardReadable() - sr.add_readables([readable]) - assert_sr_has_attrs(sr, expected_attrs) + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + sr.set_readable_format(sr.sig, format) + name = {sr.sig.name} + await assert_contributes( + sr, + read=name if in_read else set(), + config=name if in_config else set(), + hints=name if in_hints else set(), + ) + + +async def test_standard_readable_child_format_uses_child_verbs(): + inner = StandardReadable(name="inner") + inner.conf, _ = soft_signal_r_and_setter(int, 0, name="conf") + inner.val, _ = soft_signal_r_and_setter(int, 0, name="val") + inner.set_readable_format(inner.conf, Format.CONFIG_SIGNAL) + inner.set_readable_format(inner.val, Format.HINTED_SIGNAL) + + outer = StandardReadable(name="outer") + outer.inner = inner + outer.set_readable_format(inner, Format.CHILD) + + await assert_contributes( + outer, + read={inner.val.name}, + config={inner.conf.name}, + hints={inner.val.name}, + ) + + +async def test_standard_readable_child_format_ignores_plain_device(): + sr = StandardReadable(name="sr") + sr.child = Device(name="child") + sr.set_readable_format(sr.child, Format.CHILD) + await assert_contributes(sr) @pytest.mark.parametrize( @@ -243,40 +233,36 @@ def test_standard_readable_add_readables_raises_signalr_typeerror(format) -> Non sr.add_readables([mock_device], format=format) -def test_standard_readable_config_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.CONFIG_SIGNAL) - assert sr._describe_config_funcs == (signal_r.describe,) - assert sr._read_config_funcs == (signal_r.read,) - - -def test_standard_readable_hinted_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.HINTED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs == (signal_r.read,) - assert sr._stage_funcs == (signal_r.stage,) - assert sr._unstage_funcs == (signal_r.unstage,) - assert sr._has_hints[0].device == signal_r - - -def test_standard_readable_uncached_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.UNCACHED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs[0].signal == signal_r +@pytest.mark.parametrize( + "format, staged", + [ + (Format.HINTED_SIGNAL, True), + (Format.CHILD, True), + (Format.CONFIG_SIGNAL, False), + (Format.UNCACHED_SIGNAL, False), + (Format.HINTED_UNCACHED_SIGNAL, False), + ], +) +async def test_standard_readable_stage_starts_caching_only_for_cached_formats( + format, staged +): + # stage() exists to start monitoring, and reading cached raises if it isn't + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + await sr.sig.connect(mock=False) + sr.set_readable_format(sr.sig, format) + await sr.stage() + if staged: + assert await sr.sig.read(cached=True) + else: + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) -def test_standard_readable_hinted_uncached_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.HINTED_UNCACHED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs[0].signal == signal_r - assert sr._has_hints[0].device == signal_r + # unstage tears the monitor back down + await sr.unstage() + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) def test_standard_readable_add_children_multi_nested(): @@ -305,3 +291,63 @@ def __init__(self, name): with pytest.raises(KeyError): DummyDerivedDevice("test_duplicates") + + +async def test_set_readable_format_changes_verb_at_runtime(): + # The ophyd v1 `kind` use case from #1394: a signal that is config for one + # technique and hinted for another, without redefining the Device. + sr = StandardReadable(name="sr") + sr.energy, _ = soft_signal_r_and_setter(float, 0.0, name="energy") + + sr.set_readable_format(sr.energy, Format.CONFIG_SIGNAL) + await assert_contributes(sr, config={sr.energy.name}) + + sr.set_readable_format(sr.energy, Format.HINTED_SIGNAL) + await assert_contributes(sr, read={sr.energy.name}, hints={sr.energy.name}) + + sr.set_readable_format(sr.energy, None) + await assert_contributes(sr) + + +async def test_set_readable_format_replaces_rather_than_duplicates(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + + assert sr.hints == {"fields": [sr.sig.name]} + assert sr.get_readable_format(sr.sig) is Format.HINTED_SIGNAL + + +async def test_get_readable_format_returns_none_when_unregistered(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + assert sr.get_readable_format(sr.sig) is None + sr.set_readable_format(sr.sig, Format.CONFIG_SIGNAL) + assert sr.get_readable_format(sr.sig) is Format.CONFIG_SIGNAL + + +async def test_set_readable_format_affects_staging_of_later_runs(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + await sr.sig.connect(mock=False) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + + await sr.stage() + assert await sr.sig.read(cached=True) + await sr.unstage() + + # Dropping the signal means the next run does not monitor it + sr.set_readable_format(sr.sig, None) + await sr.stage() + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) + await sr.unstage() + + +def test_set_readable_format_rejects_non_signal_for_signal_format(): + sr = StandardReadable(name="sr") + sr.child = Device(name="child") + with pytest.raises(TypeError, match="is not a SignalR"): + sr.set_readable_format(sr.child, Format.CONFIG_SIGNAL) diff --git a/tests/unit_tests/epics/pvi/test_pvi.py b/tests/unit_tests/epics/pvi/test_pvi.py index 7dcafdd3e4..17aa12137b 100644 --- a/tests/unit_tests/epics/pvi/test_pvi.py +++ b/tests/unit_tests/epics/pvi/test_pvi.py @@ -155,7 +155,7 @@ async def test_device_create_children_from_annotations_with_device_vectors(): await device.connect(mock=True) block_1_device = device.device - assert block_1_device in device._has_hints + assert device.get_readable_format(block_1_device) is Format.CHILD block_2_device_vector = device.device_vector assert device.device_vector[1].name == "test_device-device_vector-1" From 54979f8ea948e5e0e3ee19eb49beb0b0f5bbd08d Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 11:12:44 +0000 Subject: [PATCH 03/67] Add load/save of readable formats Which signals are hinted, config or omitted is now part of a Device's state, so it needs to be storable alongside signal values. A beamline that switches technique wants to save "energy is config" and "energy is hinted" as two profiles and load whichever applies. Stored as {path of StandardReadable: {path of child: format}}, using the same dotted attribute paths as store_settings, with "" for the root. Values are plain strings so the yaml stays readable and hand-editable. apply_readable_formats replaces the registered children of each named StandardReadable rather than merging into them, so loading a profile drops what the previous one registered instead of accumulating the union. It resolves every path before changing anything, so a bad path cannot leave the tree half applied. Kept out of store_settings: values and formats are stored under separate names, since they change on different cadences. Co-Authored-By: Claude Opus 5 (cherry picked from commit fb74d300537f3965b5eb4545632dea29251fde8e) --- src/ophyd_async/core/__init__.py | 6 + src/ophyd_async/core/_readable.py | 94 ++++++++++++++- src/ophyd_async/plan_stubs/__init__.py | 4 + src/ophyd_async/plan_stubs/_settings.py | 50 ++++++++ tests/unit_tests/plan_stubs/test_settings.py | 114 ++++++++++++++++++- 5 files changed, 266 insertions(+), 2 deletions(-) diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 1c84360f95..96663a4576 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -101,8 +101,11 @@ from ._readable import ( ConfigSignal, HintedSignal, + ReadableFormats, StandardReadable, StandardReadableFormat, + apply_readable_formats, + walk_readable_formats, ) from ._settings import Settings, SettingsProvider from ._signal import ( @@ -272,6 +275,9 @@ def __getattr__(name): # Readable "StandardReadable", "StandardReadableFormat", + "ReadableFormats", + "walk_readable_formats", + "apply_readable_formats", # Detector "DetectorTrigger", "TriggerInfo", diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index aa8f34e399..9fd92e3b50 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -10,7 +10,7 @@ from ._device import Device, DeviceMap, DeviceVector from ._protocol import AsyncConfigurable, AsyncReadable, AsyncStageable -from ._signal import SignalR +from ._signal import SignalR, walk_devices from ._standard_base import _StandardBase from ._status import AsyncStatus from ._utils import merge_gathered_dicts @@ -343,6 +343,14 @@ def get_readable_format(self, device: Device) -> StandardReadableFormat | None: return format return None + def readable_children(self) -> Iterator[tuple[Device, StandardReadableFormat]]: + """Iterate over the registered children and their formats, in order.""" + yield from self._readables + + def clear_readables(self) -> None: + """Drop every registered child, so this Device contributes nothing.""" + self._readables = () + def add_readables( self, devices: Sequence[Device], @@ -387,3 +395,87 @@ def name(self) -> str: def hints(self) -> Hints: fields = [self.name] if self.name else [] return {"fields": fields} + + +#: The readable formats of a Device tree in a form that can be stored and +#: retrieved: `{path of the StandardReadable: {path of the child: format}}`. +#: Paths are dotted attribute paths from the root Device, as produced by +#: [](#walk_devices), with `""` meaning the root Device itself. +ReadableFormats = dict[str, dict[str, StandardReadableFormat]] + + +def _paths_to_devices(device: Device) -> dict[str, Device]: + return {"": device, **walk_devices(device)} + + +def walk_readable_formats(device: Device) -> ReadableFormats: + """Retrieve the readable format of every registered child in a Device tree. + + Used as part of saving a Device, so that which signals are hinted, config + or omitted can be restored later. Every `StandardReadable` in the tree gets + an entry, including those with nothing registered, so that applying the + result is a complete description rather than a partial overlay. + + :param device: The root Device to walk. + :return: A [](#ReadableFormats) suitable for storing. + """ + device_to_path = {dev: path for path, dev in _paths_to_devices(device).items()} + formats: ReadableFormats = {} + for path, dev in _paths_to_devices(device).items(): + if not isinstance(dev, StandardReadable): + continue + entries: dict[str, StandardReadableFormat] = {} + for child, format in dev.readable_children(): + child_path = device_to_path.get(child) + if child_path is None: + # Registered something that is not in this tree, so it has no + # stable path to store it against + warnings.warn( + f"{dev.name or dev}: cannot store the format of " + f"{child.name or child} as it is not within " + f"{device.name or device}", + stacklevel=2, + ) + else: + entries[child_path] = format + formats[path] = entries + return formats + + +def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: + """Set the readable format of children in a Device tree. + + This **replaces** the registered children of each `StandardReadable` named + in `formats`, rather than adding to them, so that applying a stored set + switches a Device between techniques rather than accumulating the union of + both. `StandardReadable`s not named in `formats` are left alone. + + :param device: The root Device the paths in `formats` are relative to. + :param formats: A [](#ReadableFormats), e.g. from [](#walk_readable_formats). + :raises KeyError: If a path does not name a Device in the tree. + :raises TypeError: If a path does not name a `StandardReadable`. + """ + devices = _paths_to_devices(device) + resolved: list[tuple[StandardReadable, list[tuple[Device, StandardReadableFormat]]]] + resolved = [] + # Resolve everything before changing anything, so a bad path cannot leave + # the tree half applied + for owner_path, entries in formats.items(): + owner = devices.get(owner_path) + if owner is None: + raise KeyError(f"No Device at {owner_path!r} in {device.name or device}") + if not isinstance(owner, StandardReadable): + raise TypeError(f"{owner_path!r} is not a StandardReadable, got {owner}") + readables = [] + for child_path, format in entries.items(): + child = devices.get(child_path) + if child is None: + raise KeyError( + f"No Device at {child_path!r} in {device.name or device}" + ) + readables.append((child, StandardReadableFormat(format))) + resolved.append((owner, readables)) + for owner, readables in resolved: + owner.clear_readables() + for child, format in readables: + owner.set_readable_format(child, format) diff --git a/src/ophyd_async/plan_stubs/__init__.py b/src/ophyd_async/plan_stubs/__init__.py index ea1824dec4..2218bb2f83 100644 --- a/src/ophyd_async/plan_stubs/__init__.py +++ b/src/ophyd_async/plan_stubs/__init__.py @@ -5,7 +5,9 @@ apply_settings, apply_settings_if_different, get_current_settings, + retrieve_readable_formats, retrieve_settings, + store_readable_formats, store_settings, ) @@ -14,6 +16,8 @@ "apply_settings", "apply_settings_if_different", "get_current_settings", + "retrieve_readable_formats", "retrieve_settings", + "store_readable_formats", "store_settings", ] diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 5e75c6eb83..2086d930e3 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -10,11 +10,14 @@ from ophyd_async.core import ( Device, + ReadableFormats, Settings, SettingsProvider, SignalRW, + StandardReadableFormat, Table, walk_config_signals, + walk_readable_formats, walk_rw_signals, ) @@ -150,3 +153,50 @@ def _is_different(current, required) -> bool: lambda sig: _is_different(current_settings[sig], settings[sig]) ) yield from apply_plan(settings_to_change) + + +@plan +def store_readable_formats( + provider: SettingsProvider, name: str, device: Device +) -> MsgGenerator[None]: + """Walk a Device for readable formats and store them. + + Stores which children of each `StandardReadable` are hinted, configuration + or omitted, so that a Device can be switched between techniques. This is + stored separately from [](#store_settings), which stores signal *values*; + give them different names. + + :param provider: The provider to store the formats with. + :param name: The name to store the formats under. + :param device: The Device to walk. + """ + formats = walk_readable_formats(device) + serialisable = { + path: {child: format.value for child, format in entries.items()} + for path, entries in formats.items() + } + yield from wait_for_awaitable(provider.store(name, serialisable)) + + +@plan +def retrieve_readable_formats( + provider: SettingsProvider, name: str, device: Device +) -> MsgGenerator[ReadableFormats]: + """Retrieve named readable formats for a Device from a provider. + + Apply the result with [](#apply_readable_formats), which replaces rather + than merges, so that switching technique drops the formats of the previous + one. Formats take effect on the next `stage()`, so retrieve and apply them + between runs. + + :param provider: The provider to retrieve the formats from. + :param name: The name the formats were stored under. + :param device: The Device the stored paths are relative to. + """ + stored = yield from wait_for_awaitable(provider.retrieve(name)) + return { + path: { + child: StandardReadableFormat(format) for child, format in entries.items() + } + for path, entries in stored.items() + } diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index c64f2a36c3..7e5cfbffa0 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -6,12 +6,22 @@ import pytest import yaml -from ophyd_async.core import Settings, YamlSettingsProvider, get_mock +from ophyd_async.core import ( + Settings, + StandardReadable, + YamlSettingsProvider, + apply_readable_formats, + get_mock, + soft_signal_r_and_setter, +) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.plan_stubs import ( apply_settings, apply_settings_if_different, get_current_settings, + retrieve_readable_formats, retrieve_settings, + store_readable_formats, store_settings, ) from ophyd_async.testing import ( @@ -179,3 +189,105 @@ def my_plan(): assert m.mock_calls == [call.sig_rw.put("foo")] RE(my_plan()) + + +@pytest.fixture +def technique_device() -> StandardReadable: + """A Device whose energy signal is config for one technique, hinted for another.""" + device = StandardReadable(name="mono") + device.energy, _ = soft_signal_r_and_setter(float, 7.0) + device.temperature, _ = soft_signal_r_and_setter(float, 20.0) + device.set_name("mono") + device.set_readable_format(device.energy, Format.CONFIG_SIGNAL) + device.set_readable_format(device.temperature, Format.CONFIG_SIGNAL) + return device + + +async def test_store_and_retrieve_readable_formats(RE, technique_device, tmp_path): + provider = YamlSettingsProvider(tmp_path) + + def my_plan(): + yield from store_readable_formats(provider, "fixed_energy", technique_device) + with open(tmp_path / "fixed_energy.yaml") as f: + # Stored as plain strings against dotted paths, so it stays editable + assert yaml.safe_load(f) == { + "": {"energy": "CONFIG_SIGNAL", "temperature": "CONFIG_SIGNAL"} + } + + retrieved = yield from retrieve_readable_formats( + provider, "fixed_energy", technique_device + ) + assert retrieved == { + "": { + "energy": Format.CONFIG_SIGNAL, + "temperature": Format.CONFIG_SIGNAL, + } + } + + RE(my_plan()) + + +async def test_readable_formats_round_trip_switches_technique( + RE, technique_device, tmp_path +): + provider = YamlSettingsProvider(tmp_path) + energy, temperature = technique_device.energy, technique_device.temperature + + def my_plan(): + yield from store_readable_formats(provider, "fixed_energy", technique_device) + + # Switch to a technique that scans energy and drops temperature entirely + technique_device.set_readable_format(energy, Format.HINTED_SIGNAL) + technique_device.set_readable_format(temperature, None) + yield from store_readable_formats(provider, "scan_energy", technique_device) + + # Going back restores both, including the one that was dropped + fixed = yield from retrieve_readable_formats( + provider, "fixed_energy", technique_device + ) + apply_readable_formats(technique_device, fixed) + assert technique_device.get_readable_format(energy) is Format.CONFIG_SIGNAL + assert technique_device.get_readable_format(temperature) is Format.CONFIG_SIGNAL + + # And forward again drops temperature rather than merging the two + scanning = yield from retrieve_readable_formats( + provider, "scan_energy", technique_device + ) + apply_readable_formats(technique_device, scanning) + assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL + assert technique_device.get_readable_format(temperature) is None + + RE(my_plan()) + + +async def test_store_readable_formats_walks_nested_devices(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + inner = StandardReadable(name="inner") + inner.sig, _ = soft_signal_r_and_setter(float, 1.0) + outer = StandardReadable(name="outer") + outer.inner = inner + outer.top, _ = soft_signal_r_and_setter(float, 2.0) + outer.set_name("outer") + inner.set_readable_format(inner.sig, Format.HINTED_SIGNAL) + outer.set_readable_format(outer.top, Format.CONFIG_SIGNAL) + outer.set_readable_format(inner, Format.CHILD) + + def my_plan(): + yield from store_readable_formats(provider, "nested", outer) + with open(tmp_path / "nested.yaml") as f: + assert yaml.safe_load(f) == { + "": {"top": "CONFIG_SIGNAL", "inner": "CHILD"}, + "inner": {"inner.sig": "HINTED_SIGNAL"}, + } + + RE(my_plan()) + + +def test_apply_readable_formats_rejects_unknown_path(technique_device): + with pytest.raises(KeyError, match="No Device at 'nope'"): + apply_readable_formats(technique_device, {"nope": {}}) + + +def test_apply_readable_formats_rejects_non_readable(technique_device): + with pytest.raises(TypeError, match="is not a StandardReadable"): + apply_readable_formats(technique_device, {"energy": {}}) From 6f5de1fabed180b8f75485fbb807d45fd28c4464 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 11:21:08 +0000 Subject: [PATCH 04/67] Document the readable format and logic changes Adds ADR 0021 for keying the readable registry by Device, covering the designs that were rejected (chain parsing, one pluggable registry for formats and logic objects both), and ADR 0022 for the single `logic` property, which supersedes part of ADR 0017. Numbering starts at 21 to leave 20 for the unmerged ADR in #1367. Adds a how-to for changing format at runtime and for saving/restoring sets of formats as technique profiles. `tox -e docs` already failed on main with three warnings. Retargeting the write-a-flyer literalinclude at `Motor.logic` fixes one of them; the other two are TypeVars documented on StandardFlyable in #1359, added to the existing nitpick ignore list so the build is green. Co-Authored-By: Claude Opus 5 (cherry picked from commit f0ddc99d9755a2457ebf477c1f4f76f67b29e439) --- docs/conf.py | 3 + .../decisions/0021-runtime-readable-format.md | 130 ++++++++++++++++++ .../decisions/0022-single-logic-property.md | 75 ++++++++++ docs/how-to/change-readable-format.md | 90 ++++++++++++ docs/how-to/choose-right-baseclass.md | 2 +- docs/how-to/write-a-flyer.md | 19 +-- 6 files changed, 304 insertions(+), 15 deletions(-) create mode 100644 docs/explanations/decisions/0021-runtime-readable-format.md create mode 100644 docs/explanations/decisions/0022-single-logic-property.md create mode 100644 docs/how-to/change-readable-format.md diff --git a/docs/conf.py b/docs/conf.py index 9b3c96a9ed..a8efd17b34 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -190,6 +190,9 @@ def setup(app: application.Sphinx): "0.001", "1.0", "bluesky.protocols.T_co", + # TypeVars documented on StandardFlyable, added in #1359 + "PrepareT", + "CtxT", ] nitpick_ignore = [] for var in obj_ignore: diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md new file mode 100644 index 0000000000..3aa467d796 --- /dev/null +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -0,0 +1,130 @@ +# 21. Make readable format a runtime setting keyed by Device + +Date: 2026-08-18 + +## Status + +Accepted + +## Context + +In ophyd v1 a `Component`'s `kind` attribute could be changed at runtime, so a beamline +could reuse one Device across techniques: a monochromator's energy signal is +configuration for a fixed-energy technique and a hinted read for one that scans it. + +`StandardReadable` had no equivalent. `add_readables(devices, format)` appended bound +methods into four tuples: + +```python +case StandardReadableFormat.CONFIG_SIGNAL: + self._describe_config_funcs += (signal.describe,) + self._read_config_funcs += (signal.read,) +``` + +Once appended, the tuples no longer recorded which device a contribution came from, so +nothing could be found, removed or re-formatted afterwards. The only ways to change what +a device reported were to edit its class definition or to keep two near-identical Device +classes differing in one signal's format. This is issue #1394. + +The same gap showed up from the other direction in #1395. `StandardDetector` did not +inherit `StandardReadable` and had its own `add_config_signals` instead, so detectors and +every other Device disagreed about how a signal is nominated for `read_configuration()`. +Reviewers of that PR were unwilling to have areaDetector plugins registered automatically, +because a detector typically carries far more plugins (16 ROIs, 16 stats) than are wired +into its chain, and reporting all of them gives no way to tell which values are +meaningful. Per-plugin opt-in needs a per-device call that can be made after construction +— which is the same primitive #1394 asks for. + +Three designs were considered. + +**Chain parsing.** Inspect the areaDetector plugin graph and register what is actually +wired up. Rejected as too large, and it breaks down once gather/scatter and PVA transfer +plugins are used between IOCs, since it requires every plugin to be exposed to ophyd. + +**A single pluggable registry** holding both formats and logic objects, so that +`StandardReadable`, `StandardMovable`, `StandardFlyable` and `StandardDetector` all +register their behaviour the same way. Rejected: a format entry is keyed by child, +unbounded in number and merged by union, whereas a logic object is keyed by role, at most +one per role, and conflicts are an error. A single registry would have to branch on entry +kind at every insert. Runtime mutation also means different things for the two — changing +a format alters the contents of `describe()`, while changing a data logic alters which +bluesky protocols the Device satisfies. Finally the declarative annotation form only fits +formats: `A[SignalRW[float], PvSuffix.rbv("AcquireTime"), Format.CONFIG_SIGNAL]` works +because a format needs exactly `(parent, child)`, whereas `ADHDFDataLogic` needs a prefix, +a driver, a plugin list and a path provider. + +**Key the format registry by device**, and leave logic objects alone. Chosen. + +## Decision + +### The registry holds `(device, format)` pairs + +`StandardReadable._readables` is a tuple of pairs in registration order, and the callables +for each verb are derived on demand. `add_readables` becomes a thin wrapper over a new +`set_readable_format(device, format)`, and `format=None` removes the device. + +Deriving on demand also fixes staging: `stage()` walks the current registry rather than a +tuple captured at registration, so a format changed between runs takes effect on the next +`stage()`. + +Registration is "set" rather than "add": registering the same device twice replaces its +format rather than contributing it twice. This is a behaviour change, and the previous +behaviour was not deliberate. + +Formats are normalised to real enum members on the way in. The deprecated `ConfigSignal` +and `HintedSignal` markers announce themselves by comparing equal to their target, which +relied on the `match` statement running during `add_readables`; normalising on entry keeps +the warning at registration rather than moving it to the first `read()`. + +### Format changes apply between runs, not within one + +A run's descriptor is emitted at its start, and `HINTED_SIGNAL` sets up monitoring in +`stage()`. Changing a format part way through a run would make `describe()` and `read()` +disagree. This is documented rather than enforced: the check would have to know about run +boundaries, which a Device does not. + +### `StandardDetector` inherits `StandardReadable` + +A detector's step-scan signals are registered exactly like any other Device's, and +`read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. +`add_config_signals` becomes a deprecated wrapper. + +Plugins are **not** registered automatically. `AreaDetector` registers only the driver's +`acquire_time` and `acquire_period`, plus whatever the caller passes as `config_sigs`. +Registering a plugin is a `set_readable_format()` call at the call site, which is the +explicit opt-in the #1395 reviewers asked for, now expressible per plugin and reversible +at runtime. + +### Formats are stored and retrieved separately from settings + +`walk_readable_formats` and `apply_readable_formats` serialise the registry as +`{path of StandardReadable: {path of child: format}}`, using the same dotted attribute +paths as `store_settings`, with `""` for the root Device. Values are plain strings so the +YAML stays hand-editable. + +`apply_readable_formats` **replaces** the registered children of each named +`StandardReadable` rather than merging into them, so that loading a technique profile +drops what the previous one registered instead of accumulating the union of both. It +resolves every path before changing anything, so a bad path cannot leave the tree half +applied. + +Formats are stored under a different name from `store_settings`, not folded into it: +values and formats change on different cadences, and `Settings` is a mapping of +`SignalRW` to value, whereas a format applies to `SignalR`s and whole Devices too. + +## Consequences + +- A Device can be switched between techniques at runtime, and the switch can be saved and + restored, closing #1394. +- `add_readables` keeps working unchanged for all 85 in-tree call sites and for downstream + Devices, but gains "set" semantics. +- Tests that asserted on `_read_funcs`, `_has_hints` and friends had to be rewritten onto + the public verbs. Those assertions were already against the testing conventions in + `CLAUDE.md`, and the derived-on-demand design makes them impossible rather than merely + discouraged. +- Deriving callables per call costs a pass over the registry on every verb. The registries + are small (tens of entries), and it is what makes the format mutable. +- Registering a device that is not within the tree still works, but its format cannot be + stored, since there is no stable path for it. `walk_readable_formats` warns and skips. +- This ADR is numbered 21 rather than 20 to leave 20 for the unmerged bounded-data-logic + ADR in #1367. diff --git a/docs/explanations/decisions/0022-single-logic-property.md b/docs/explanations/decisions/0022-single-logic-property.md new file mode 100644 index 0000000000..ff130e1497 --- /dev/null +++ b/docs/explanations/decisions/0022-single-logic-property.md @@ -0,0 +1,75 @@ +# 22. One `logic` property per Device + +Date: 2026-08-18 + +## Status + +Accepted + +Supersedes part of [](./0017-standard-movable.md), which introduced `movable_logic`. + +## Context + +A Device that both moves and flies had to expose the same logic object twice. `Motor` +built one `MotorFlyableMovableLogic` and returned it from a `movable_logic` property for +[](#StandardMovable) and a `flyable_logic` property for [](#StandardFlyable), plus a +private `_logic` to avoid building it twice. Every future mix-in would add another name. + +The obstacle is that Python has no intersection type: there is no way to write "the logic +must be both a `MovableLogic[float]` and a `FlyableLogic[FlyMotorInfo, MotorFlyCtx]`" as a +single annotation. + +ADR 0017 also chose `@cached_property` stacked on `@abstractmethod` over an +`add_movable_logic()` method, on the grounds that the property is "checkable by static +analysis". That turned out not to hold. `functools.cached_property` does not forward +`__isabstractmethod__` from the function it wraps, so `StandardMovable.__abstractmethods__` +was empty, pyright reported nothing, and a Device that forgot its logic failed with +`AttributeError: 'NoneType' object has no attribute 'readback'` from inside +`Device.__init__`. + +## Decision + +Each mix-in declares the **same** attribute name, `logic`, with **its own** +required type: + +```python +class StandardMovable(Device, ..., Generic[SignalDatatypeT]): + @abstract_cached_property + def logic(self) -> MovableLogic[SignalDatatypeT]: ... + +class StandardFlyable(_StandardBase, ..., Generic[PrepareT, CtxT]): + @abstract_cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: ... +``` + +A Device inheriting both provides one implementation, and the type checker verifies it +against each base independently. The intersection is expressed by the logic class's own +MRO: `MotorFlyableMovableLogic` inherits both, so it satisfies both declarations, and the +concrete type is what callers see back. + +This catches, at type-check time, a logic that satisfies one mix-in but not the other, a +wrong datatype parameter, and a Device with two mix-ins and no implementation. A Device +with a *single* mix-in and no implementation is caught at instantiation instead, by +`abstract_cached_property`, a `cached_property` that sets `__isabstractmethod__` on the +descriptor so that `Device`'s metaclass collects it. It is declared as returning +`cached_property[T]` rather than as a subclass, because a subclass breaks the covariance +check on the override and would reject the correct case too. + +Because [](#StandardFlyable) is now genuinely abstract, `FlyableLogic.with_device` builds +a concrete `_EphemeralFlyable` to attach the logic to. + +An alternative of a single `logics` collection searched by type was rejected: membership +cannot be proved statically, so it would reintroduce exactly the runtime-only failure +this change fixes. + +## Consequences + +- One property per Device however many mix-ins it combines; `Motor` loses two properties. +- Where the move and fly logic are genuinely unrelated implementations they must be + combined into one class, since one property returns one object. Nothing in-tree pays + this cost, as `MotorFlyableMovableLogic` was already that shape. +- Forgetting the logic now fails with `TypeError: Can't instantiate abstract class ... + with abstract method logic` rather than an `AttributeError` on `None`. +- `movable_logic` and `flyable_logic` remain as deprecated properties. +- The same mechanism is available to `StandardDetector` if it later inherits + [](#StandardFlyable); its logic would need to satisfy `FlyableLogic` too. diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md new file mode 100644 index 0000000000..4234c5993b --- /dev/null +++ b/docs/how-to/change-readable-format.md @@ -0,0 +1,90 @@ +# How to change what a Device reports at runtime + +A [](#StandardReadable) decides what appears in `read()`, `read_configuration()` and +`hints` from the [](#StandardReadableFormat) each of its children is registered with. +Normally that is fixed when the Device class is written, but it can also be changed on a +live Device with [](#StandardReadable.set_readable_format). + +This is the equivalent of changing a `Component`'s `kind` in ophyd v1. + +## Changing one signal + +A beamline with two techniques might scan a monochromator's energy in one and hold it +fixed in the other. The same Device serves both: + +```python +from ophyd_async.core import StandardReadableFormat as Format + +# Scanning technique: energy is a hinted read +mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + +# Fixed technique: energy is configuration, recorded once per run +mono.set_readable_format(mono.energy, Format.CONFIG_SIGNAL) + +# Not interesting for this technique at all +mono.set_readable_format(mono.energy, None) +``` + +[](#StandardReadable.get_readable_format) reports the current format, returning `None` if +the child does not contribute. + +```{note} +Change formats **between runs**, not inside one. A run's descriptor is emitted when the +run starts, and [](#StandardReadableFormat.HINTED_SIGNAL) begins monitoring in `stage()`, +so a format changed part way through a run would leave `describe()` and `read()` +disagreeing. +``` + +## Opting a detector plugin in + +A [](#StandardDetector) is a [](#StandardReadable), so the same call works on one. This +matters most for areaDetector, where a detector often carries many more plugins than are +wired into its chain — so plugins are never registered automatically, and you opt in to +the ones that are actually producing meaningful values: + +```python +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) +``` + +Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can +write a file *and* report a plugin scalar in a step scan. + +## Saving and restoring a set of formats + +Formats are part of a Device's state, so they can be stored and reloaded like settings +are. Use [](#store_readable_formats) and [](#retrieve_readable_formats) with any +[](#SettingsProvider), then [](#apply_readable_formats): + +```python +def save_technique(det): + provider = YamlSettingsProvider("/path/to/profiles") + yield from store_readable_formats(provider, "fixed_energy", det) + + +def load_technique(det, technique: str): + provider = YamlSettingsProvider("/path/to/profiles") + formats = yield from retrieve_readable_formats(provider, technique, det) + apply_readable_formats(det, formats) +``` + +The stored yaml maps each [](#StandardReadable)'s path to its children's formats, with +`""` for the root Device, so it stays readable and can be edited by hand: + +```yaml +'': {energy: CONFIG_SIGNAL, temperature: CONFIG_SIGNAL} +stats: {stats.total: HINTED_UNCACHED_SIGNAL} +``` + +```{note} +[](#apply_readable_formats) **replaces** the children registered on each Device it names, +rather than merging into them. That is what makes switching technique work: loading a +profile drops what the previous one registered instead of accumulating both. +``` + +Formats are stored under a different name from [](#store_settings), which stores signal +*values*. The two change on different cadences, so keep them in separate files. + +```{seealso} +[](../explanations/decisions/0021-runtime-readable-format.md) for why the registry is +keyed by Device, and [](./store-and-retrieve.md) for storing signal values. +``` diff --git a/docs/how-to/choose-right-baseclass.md b/docs/how-to/choose-right-baseclass.md index bbea3b6df6..83b7224a95 100644 --- a/docs/how-to/choose-right-baseclass.md +++ b/docs/how-to/choose-right-baseclass.md @@ -10,7 +10,7 @@ When writing a new Device there are several base classes to choose from that wil There are some utility baseclasses that allow you to create a Device pre-populated with the right verbs to work in bluesky plans: -- [](#StandardReadable) allows you to compose the values of child Signals and Devices together so that you can `read()` the Device during a step scan. +- [](#StandardReadable) allows you to compose the values of child Signals and Devices together so that you can `read()` the Device during a step scan. What each child contributes can also be changed on a live Device, see [](./change-readable-format.md). - [](#StandardMovable) allows devices that move to a target value and wait for a readback to confirm arrival (motors, temperature controllers, piezo stages, and similar) to be implemented by composing a [](#MovableLogic) instance, rather than directly implementing the `Locatable`, `Stoppable`, and `Subscribable` protocols. See [](../explanations/when-to-extend-movable.md) for more details. - [](#StandardDetector) allows file-writing detectors to be used within both step and fly scans, reporting periodic references to the data that has been written so far. Instances of [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) are composed together to provide this functionality. This separation of concerns makes it easy to mix and match different trigger modes, acquisition strategies, and data outputs. - [](#StandardFlyable) allows actuators (like a motor controller) to be used within a fly scan, by composing a [](#FlyableLogic) instance rather than directly implementing the `Preparable`, `Flyable`, and `Stageable` protocols. Use it as a mix-in on a Device, or call [](#FlyableLogic.with_device) to wrap a bare logic object in an ephemeral flyer within a plan. diff --git a/docs/how-to/write-a-flyer.md b/docs/how-to/write-a-flyer.md index a794ab47f4..db33d32e89 100644 --- a/docs/how-to/write-a-flyer.md +++ b/docs/how-to/write-a-flyer.md @@ -77,23 +77,14 @@ Because the logic is also a `MovableLogic`, `complete()` reports progress to wat movable (see the next section) just blocks with no progress updates. See [](../explanations/when-to-extend-movable.md) for more on `MovableLogic`. -Wire the logic into the Device with a `@cached_property`. `Motor` builds one shared -instance and returns it from both `movable_logic` and `flyable_logic` so the move and -fly paths act on the same signals: +Wire the logic into the Device with a `@cached_property` called `logic`. Each +mix-in declares that same name with its own required type, so one logic object that +inherits both `MovableLogic` and `FlyableLogic` satisfies both, and the move and fly +paths act on the same signals: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: Motor._logic -``` - -```python -@cached_property -def movable_logic(self) -> MotorFlyableMovableLogic: - return self._logic - -@cached_property -def flyable_logic(self) -> MotorFlyableMovableLogic: - return self._logic +:pyobject: Motor.logic ``` ## An ephemeral flyer created in a plan From 3e46aadb817ec2e3940b11c17210d6dadb01ecf4 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 14:48:11 +0000 Subject: [PATCH 05/67] Store readable formats in the settings file under a reserved key Values and formats are applied on the same cadence -- a technique change wants both -- so they belong in one file and one apply_settings call rather than two of each. Settings gains a readable_formats attribute, so one retrieve_settings returns the whole stored state. Values stay flat, formats go under "". That key cannot collide with a dotted attribute path, because "<" cannot begin a Python identifier, so no store-time guard is needed; and unlike "*FORMATS*" it is not a YAML indicator, so it needs no quoting. "" replaces "" as the root owner for the same reason, and because an empty-string key makes PyYAML emit its hard-to-read "? '' :" explicit key form. "" is reserved but unwritten so storing names later needs no migration. No version marker: a file written before formats existed simply lacks the key, which under replacement semantics correctly means "clear no owner". Each part being optional also makes a hand-written formats-only profile work, switching technique without writing a value to hardware. Settings.partition copies formats to both halves. apply_panda_settings applies only the halves and never the original Settings, so putting them on one half would let that plan silently decide whether formats were restored. Applying them twice touches no hardware and is idempotent. apply_settings applies formats after values, so a failed write leaves the registry alone. store_readable_formats and retrieve_readable_formats are removed; they only existed earlier in this branch. Co-Authored-By: Claude Opus 5 (cherry picked from commit 586f79e8ac40b5a48ccb74734957c35f300bdc48) --- .../decisions/0021-runtime-readable-format.md | 64 ++++- docs/how-to/change-readable-format.md | 52 ++-- docs/how-to/store-and-retrieve.md | 2 + src/ophyd_async/core/__init__.py | 6 + src/ophyd_async/core/_readable.py | 36 ++- src/ophyd_async/core/_settings.py | 29 ++- src/ophyd_async/plan_stubs/__init__.py | 4 - src/ophyd_async/plan_stubs/_settings.py | 103 ++++---- tests/unit_tests/plan_stubs/test_settings.py | 147 +++++++++--- .../test_data/test_yaml_config_save.yaml | 76 +++--- .../unit_tests/test_data/test_yaml_save.yaml | 226 ++++++++++-------- 11 files changed, 468 insertions(+), 277 deletions(-) diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index 3aa467d796..1f0a26cfa2 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -95,22 +95,60 @@ Registering a plugin is a `set_readable_format()` call at the call site, which i explicit opt-in the #1395 reviewers asked for, now expressible per plugin and reversible at runtime. -### Formats are stored and retrieved separately from settings +### Formats are stored in the same file as settings, under a reserved key + +Values and formats are applied on the same cadence — a technique change wants both — so +they belong in one file and one `apply_settings` call. `store_settings` keeps values flat +and adds one reserved key: + +```yaml +energy: 7.0 +temperature: 20.0 +: + : + energy: CONFIG_SIGNAL + temperature: CONFIG_SIGNAL +``` -`walk_readable_formats` and `apply_readable_formats` serialise the registry as -`{path of StandardReadable: {path of child: format}}`, using the same dotted attribute -paths as `store_settings`, with `""` for the root Device. Values are plain strings so the -YAML stays hand-editable. +`Settings` gains a `readable_formats` attribute, so one `retrieve_settings` returns the +whole stored state. + +**No version marker is needed.** A file written before formats existed simply lacks the +reserved key, which under replacement semantics reads correctly as "no owner is listed, so +clear no owner" — leave formats alone. Present-but-empty for an owner (`: {}`) means +"clear that owner", which is what a freshly-saved Device with nothing registered produces. +The two states are distinguishable without a version, and each section being independently +optional also makes a hand-written formats-only profile work, changing technique without +writing a value to hardware. + +**The reserved keys cannot collide.** `` and `` start with +`<`, which cannot begin a Python identifier, so no attribute assignment can produce a +colliding path — no store-time guard is required. `<` is also not a YAML indicator, so +unlike `*FORMATS*` or `%FORMATS%` the key needs no quoting. `` replaces `""` as the +root owner for the same reason and because an empty-string key makes PyYAML emit its +hard-to-read explicit `? '' :` form. `` is reserved but unwritten, so storing +names later needs no migration. + +**The formats section is two levels while values are flat.** A value is intrinsic to one +path; a format is a fact about an (owner, child) *pair*, and the same child can be +registered on two owners with different formats — which #1395 creates directly, since +`NDStatsIO` will declare `total` on itself while a detector may also register `stats.total`. +Recording the owner rather than inferring it also means the layout needs no "must be a +direct child" rule, so `AreaDetector` can keep registering `driver.acquire_time` and +`DeviceVector` elements keep working, with no dependency on #1395 and no second migration +later. `apply_readable_formats` **replaces** the registered children of each named -`StandardReadable` rather than merging into them, so that loading a technique profile -drops what the previous one registered instead of accumulating the union of both. It -resolves every path before changing anything, so a bad path cannot leave the tree half -applied. - -Formats are stored under a different name from `store_settings`, not folded into it: -values and formats change on different cadences, and `Settings` is a mapping of -`SignalRW` to value, whereas a format applies to `SignalR`s and whole Devices too. +`StandardReadable` rather than merging into them, so that loading a technique profile drops +what the previous one registered instead of accumulating the union of both. It resolves +every path before changing anything, so a bad path cannot leave the tree half applied. + +`Settings.partition` copies the formats to **both** halves. A device-specific apply plan +normally applies only the halves and never the original `Settings` — `apply_panda_settings` +partitions and calls `apply_settings` twice — so putting formats on one half would let the +plan silently decide whether they were restored at all. Applying them more than once is +harmless: it touches no hardware and is idempotent. `apply_settings` applies formats after +values, so a failed value write leaves the readable registry untouched. ## Consequences diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index 4234c5993b..d38d9a2917 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -49,42 +49,60 @@ det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can write a file *and* report a plugin scalar in a step scan. -## Saving and restoring a set of formats +## Saving and restoring a technique -Formats are part of a Device's state, so they can be stored and reloaded like settings -are. Use [](#store_readable_formats) and [](#retrieve_readable_formats) with any -[](#SettingsProvider), then [](#apply_readable_formats): +Formats are part of a Device's state, and they change on the same cadence as signal +values, so [](#store_settings) stores both and [](#apply_settings) applies both: ```python def save_technique(det): provider = YamlSettingsProvider("/path/to/profiles") - yield from store_readable_formats(provider, "fixed_energy", det) + yield from store_settings(provider, "fixed_energy", det) def load_technique(det, technique: str): provider = YamlSettingsProvider("/path/to/profiles") - formats = yield from retrieve_readable_formats(provider, technique, det) - apply_readable_formats(det, formats) + settings = yield from retrieve_settings(provider, technique, det) + yield from apply_settings(settings) ``` -The stored yaml maps each [](#StandardReadable)'s path to its children's formats, with -`""` for the root Device, so it stays readable and can be edited by hand: +Values stay flat in the yaml, and formats go under the reserved +[](#READABLE_FORMATS_KEY), so the file stays readable and hand-editable: ```yaml -'': {energy: CONFIG_SIGNAL, temperature: CONFIG_SIGNAL} -stats: {stats.total: HINTED_UNCACHED_SIGNAL} +energy: 7.0 +temperature: 20.0 +: + : + energy: CONFIG_SIGNAL + temperature: CONFIG_SIGNAL + stats: + stats.total: HINTED_UNCACHED_SIGNAL ``` +The outer key of each formats entry is the [](#StandardReadable) the child is registered +on, with [](#ROOT_DEVICE_KEY) for the Device you stored. It is recorded rather than inferred +because the same child can be registered on more than one Device with different formats. + ```{note} -[](#apply_readable_formats) **replaces** the children registered on each Device it names, -rather than merging into them. That is what makes switching technique work: loading a -profile drops what the previous one registered instead of accumulating both. +Applying **replaces** the children registered on each Device the file names, rather than +merging into them. That is what makes switching technique work: loading a profile drops +what the previous one registered instead of accumulating both. ``` -Formats are stored under a different name from [](#store_settings), which stores signal -*values*. The two change on different cadences, so keep them in separate files. +Because each part is optional, a file with no `` key leaves formats +alone — so settings files saved before formats were storable keep working untouched. The +reverse is useful too: a hand-written formats-only profile switches technique without +writing a single value to hardware. + +```yaml +: + : + energy: HINTED_SIGNAL +``` ```{seealso} [](../explanations/decisions/0021-runtime-readable-format.md) for why the registry is -keyed by Device, and [](./store-and-retrieve.md) for storing signal values. +keyed by Device and how the file is laid out, and [](./store-and-retrieve.md) for the +wider store/retrieve workflow. ``` diff --git a/docs/how-to/store-and-retrieve.md b/docs/how-to/store-and-retrieve.md index e91ec08dd5..3e1159e76a 100644 --- a/docs/how-to/store-and-retrieve.md +++ b/docs/how-to/store-and-retrieve.md @@ -35,6 +35,8 @@ def load_panda(panda1: HDFPanda): yield from apply_panda_settings(settings) ``` +As well as signal values, [](#store_settings) records how each child contributes to the Device's bluesky verbs, under the reserved [](#READABLE_FORMATS_KEY), and [](#apply_settings) restores that too. This means a stored file can switch a Device between techniques as well as putting its values back — see [](./change-readable-format.md). Files stored before this existed simply have no such key, and load unchanged. + In the situation where the time taken to read a set of SignalRWs of a Device is less than the time taken to set those signals, the [](#apply_settings_if_different) stub should be used instead. This will take the SignalRWs included in the Settings, read them from the connected Device, and set those that differ from the stored value: ``` diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 96663a4576..9d8b3b0de1 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -99,6 +99,9 @@ Watcher, ) from ._readable import ( + DEVICE_NAMES_KEY, + READABLE_FORMATS_KEY, + ROOT_DEVICE_KEY, ConfigSignal, HintedSignal, ReadableFormats, @@ -278,6 +281,9 @@ def __getattr__(name): "ReadableFormats", "walk_readable_formats", "apply_readable_formats", + "READABLE_FORMATS_KEY", + "DEVICE_NAMES_KEY", + "ROOT_DEVICE_KEY", # Detector "DetectorTrigger", "TriggerInfo", diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 9fd92e3b50..d347e60aaf 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -397,15 +397,35 @@ def hints(self) -> Hints: return {"fields": fields} +#: Reserved key under which [](#store_settings) writes readable formats. Chosen +#: so that it cannot be a dotted attribute path: `<` cannot start a Python +#: identifier, so no attribute assignment can produce a colliding key, and +#: unlike `*FORMATS*` it needs no quoting in yaml. +READABLE_FORMATS_KEY = "" + +#: Reserved key for storing Device names. Reserved now so that adding it later +#: needs no migration; nothing writes it yet. +DEVICE_NAMES_KEY = "" + +#: The path standing in for the root Device itself in a [](#ReadableFormats). +#: Not a valid Python identifier, so no attribute assignment can produce a +#: colliding path, and unlike `""` it does not make yaml fall back to its +#: hard-to-read explicit key syntax. +ROOT_DEVICE_KEY = "" + #: The readable formats of a Device tree in a form that can be stored and #: retrieved: `{path of the StandardReadable: {path of the child: format}}`. #: Paths are dotted attribute paths from the root Device, as produced by -#: [](#walk_devices), with `""` meaning the root Device itself. +#: [](#walk_devices), with [](#ROOT_DEVICE_KEY) meaning the root Device itself. +#: +#: The outer key is needed because a format is a fact about an (owner, child) +#: *pair*, not about a single Device: the same child can be registered on more +#: than one `StandardReadable` with a different format each time. ReadableFormats = dict[str, dict[str, StandardReadableFormat]] def _paths_to_devices(device: Device) -> dict[str, Device]: - return {"": device, **walk_devices(device)} + return {ROOT_DEVICE_KEY: device, **walk_devices(device)} def walk_readable_formats(device: Device) -> ReadableFormats: @@ -429,11 +449,15 @@ def walk_readable_formats(device: Device) -> ReadableFormats: child_path = device_to_path.get(child) if child_path is None: # Registered something that is not in this tree, so it has no - # stable path to store it against + # stable path to store it against. Its value is not stored + # either, as store_settings only walks the tree, so it will not + # survive a store/apply round trip at all. See + # https://github.com/bluesky/ophyd-async/issues/1402 warnings.warn( - f"{dev.name or dev}: cannot store the format of " - f"{child.name or child} as it is not within " - f"{device.name or device}", + f"{dev.name or dev}: {child.name or child} is not within " + f"{device.name or device}, so neither its readable format " + "nor its value will be stored, and applying stored settings " + "will not restore it", stacklevel=2, ) else: diff --git a/src/ophyd_async/core/_settings.py b/src/ophyd_async/core/_settings.py index 4edc87b2e5..948516e2b3 100644 --- a/src/ophyd_async/core/_settings.py +++ b/src/ophyd_async/core/_settings.py @@ -5,6 +5,7 @@ from typing import Any, Generic from ._device import Device, DeviceT +from ._readable import ReadableFormats from ._signal import SignalRW from ._signal_backend import SignalDatatypeT @@ -33,11 +34,20 @@ class Settings(MutableMapping[SignalRW[Any], Any], Generic[DeviceT]): """ def __init__( - self, device: DeviceT, settings: MutableMapping[SignalRW, Any] | None = None + self, + device: DeviceT, + settings: MutableMapping[SignalRW, Any] | None = None, + readable_formats: ReadableFormats | None = None, ): self.device = device self._settings = {} self.update(settings or {}) + self.readable_formats: ReadableFormats = dict(readable_formats or {}) + """How the Device's children contribute to its bluesky verbs. + + Stored and applied alongside the signal values, as the two change on + the same cadence. Applied by [](#apply_settings). + """ def __getitem__(self, key: SignalRW[SignalDatatypeT]) -> SignalDatatypeT: return self._settings[key] @@ -73,7 +83,11 @@ def __or__(self, other: MutableMapping[SignalRW, Any]) -> Settings[DeviceT]: """Create a new Settings that is the union of self overridden by other.""" if isinstance(other, Settings) and not self._is_in_device(other.device): raise ValueError(f"{other.device} is not a child of {self.device}") - return Settings(self.device, self._settings | dict(other)) + formats = dict(self.readable_formats) + if isinstance(other, Settings): + # Merge per owner, so that other only replaces the owners it mentions + formats |= other.readable_formats + return Settings(self.device, self._settings | dict(other), formats) def partition( self, predicate: Callable[[SignalRW], bool] @@ -88,13 +102,22 @@ def partition( The first contains the signals for which the predicate returned True, and the second contains the signals for which the predicate returned False. + Both halves carry the *same* `readable_formats`. Formats are not a value + that can differ, and a device specific apply plan normally applies only + the halves and never the original Settings (see `apply_panda_settings`), + so putting them on one half would mean whichever half that plan applied + first silently decided whether formats were restored at all. Applying + them more than once is harmless: it writes to no hardware and is + idempotent. + :example: ```python settings = Settings(device, {device.special: 1, device.sig: 2}) specials, others = settings.partition(lambda sig: "special" in sig.name) ``` """ - where_true, where_false = Settings(self.device), Settings(self.device) + where_true = Settings(self.device, readable_formats=self.readable_formats) + where_false = Settings(self.device, readable_formats=self.readable_formats) for signal, value in self.items(): dest = where_true if predicate(signal) else where_false dest[signal] = value diff --git a/src/ophyd_async/plan_stubs/__init__.py b/src/ophyd_async/plan_stubs/__init__.py index 2218bb2f83..ea1824dec4 100644 --- a/src/ophyd_async/plan_stubs/__init__.py +++ b/src/ophyd_async/plan_stubs/__init__.py @@ -5,9 +5,7 @@ apply_settings, apply_settings_if_different, get_current_settings, - retrieve_readable_formats, retrieve_settings, - store_readable_formats, store_settings, ) @@ -16,8 +14,6 @@ "apply_settings", "apply_settings_if_different", "get_current_settings", - "retrieve_readable_formats", "retrieve_settings", - "store_readable_formats", "store_settings", ] diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 2086d930e3..8b61effcd0 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -9,6 +9,8 @@ from bluesky.utils import MsgGenerator, plan from ophyd_async.core import ( + DEVICE_NAMES_KEY, + READABLE_FORMATS_KEY, Device, ReadableFormats, Settings, @@ -16,6 +18,7 @@ SignalRW, StandardReadableFormat, Table, + apply_readable_formats, walk_config_signals, walk_readable_formats, walk_rw_signals, @@ -56,21 +59,35 @@ def get_current_settings( def store_settings( provider: SettingsProvider, name: str, device: Device, only_config: bool = False ) -> MsgGenerator[None]: - """Walk a Device for SignalRWs and store their values. + """Walk a Device for SignalRWs and store their values and readable formats. If `only_config` is True, store only configuration settings on `Configurable`. + Readable formats are stored in both cases, since they are what *determines* + which signals are configuration. + + Values are stored flat, keyed by dotted attribute path. Readable formats go + under the reserved [](#READABLE_FORMATS_KEY), which is not a valid Python + identifier and so cannot collide with a path. :param provider: The provider to store the settings with. :param name: The name to store the settings under. :param device: The Device to walk for SignalRWs. - :param only_config: If True, store only configuration settings. + :param only_config: If True, store only configuration signal values. """ if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: signals = walk_rw_signals(device) - named_values = yield from _get_values_of_signals(signals) - yield from wait_for_awaitable(provider.store(name, named_values)) + data: dict[str, Any] = yield from _get_values_of_signals(signals) + formats = walk_readable_formats(device) + if formats: + # Omitted entirely when there is nothing to say, so that a settings only + # file stays exactly as it was before formats were storable + data[READABLE_FORMATS_KEY] = { + owner: {child: format.value for child, format in entries.items()} + for owner, entries in formats.items() + } + yield from wait_for_awaitable(provider.store(name, data)) @plan @@ -81,26 +98,45 @@ def retrieve_settings( If `only_config` is True, retrieve only configuration settings on `Configurable`. + A file stored before readable formats existed simply has no + [](#READABLE_FORMATS_KEY), which reads correctly as "do not change any + formats" -- so no migration or version marker is needed. + :param provider: The provider to retrieve the settings from. :param name: The name of the settings to retrieve. :param device: The Device to retrieve the settings for. :param only_config: If True, retrieve only configuration settings. """ - named_values = yield from wait_for_awaitable(provider.retrieve(name)) + data = yield from wait_for_awaitable(provider.retrieve(name)) + # Reserved keys first, so what is left is signal paths and the unknown name + # check below still catches genuine typos + stored_formats = data.pop(READABLE_FORMATS_KEY, {}) + data.pop(DEVICE_NAMES_KEY, None) + readable_formats: ReadableFormats = { + owner: { + child: StandardReadableFormat(format) for child, format in entries.items() + } + for owner, entries in stored_formats.items() + } if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: signals = walk_rw_signals(device) - unknown_names = set(named_values) - set(signals) + unknown_names = set(data) - set(signals) if unknown_names: raise NameError(f"Unknown signal names {sorted(unknown_names)}") - signal_values = {signals[name]: value for name, value in named_values.items()} - return Settings(device, signal_values) + signal_values = {signals[name]: value for name, value in data.items()} + return Settings(device, signal_values, readable_formats) @plan def apply_settings(settings: Settings) -> MsgGenerator[None]: - """Set every SignalRW to the given value in Settings. If value is None ignore it.""" + """Set every SignalRW to the given value in Settings. If value is None ignore it. + + Then apply any readable formats the Settings carries. Formats go last + because they touch no hardware and cannot fail, so a failed value write + leaves the Device's readable registry untouched. + """ signal_values = { signal: value for signal, value in settings.items() if value is not None } @@ -108,6 +144,8 @@ def apply_settings(settings: Settings) -> MsgGenerator[None]: for signal, value in signal_values.items(): yield from bps.abs_set(signal, value, group="apply_settings") yield from bps.wait("apply_settings") + if settings.readable_formats: + apply_readable_formats(settings.device, settings.readable_formats) @plan @@ -153,50 +191,3 @@ def _is_different(current, required) -> bool: lambda sig: _is_different(current_settings[sig], settings[sig]) ) yield from apply_plan(settings_to_change) - - -@plan -def store_readable_formats( - provider: SettingsProvider, name: str, device: Device -) -> MsgGenerator[None]: - """Walk a Device for readable formats and store them. - - Stores which children of each `StandardReadable` are hinted, configuration - or omitted, so that a Device can be switched between techniques. This is - stored separately from [](#store_settings), which stores signal *values*; - give them different names. - - :param provider: The provider to store the formats with. - :param name: The name to store the formats under. - :param device: The Device to walk. - """ - formats = walk_readable_formats(device) - serialisable = { - path: {child: format.value for child, format in entries.items()} - for path, entries in formats.items() - } - yield from wait_for_awaitable(provider.store(name, serialisable)) - - -@plan -def retrieve_readable_formats( - provider: SettingsProvider, name: str, device: Device -) -> MsgGenerator[ReadableFormats]: - """Retrieve named readable formats for a Device from a provider. - - Apply the result with [](#apply_readable_formats), which replaces rather - than merges, so that switching technique drops the formats of the previous - one. Formats take effect on the next `stage()`, so retrieve and apply them - between runs. - - :param provider: The provider to retrieve the formats from. - :param name: The name the formats were stored under. - :param device: The Device the stored paths are relative to. - """ - stored = yield from wait_for_awaitable(provider.retrieve(name)) - return { - path: { - child: StandardReadableFormat(format) for child, format in entries.items() - } - for path, entries in stored.items() - } diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index 7e5cfbffa0..be6555031e 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -12,16 +12,14 @@ YamlSettingsProvider, apply_readable_formats, get_mock, - soft_signal_r_and_setter, + soft_signal_rw, ) from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.plan_stubs import ( apply_settings, apply_settings_if_different, get_current_settings, - retrieve_readable_formats, retrieve_settings, - store_readable_formats, store_settings, ) from ophyd_async.testing import ( @@ -195,89 +193,149 @@ def my_plan(): def technique_device() -> StandardReadable: """A Device whose energy signal is config for one technique, hinted for another.""" device = StandardReadable(name="mono") - device.energy, _ = soft_signal_r_and_setter(float, 7.0) - device.temperature, _ = soft_signal_r_and_setter(float, 20.0) + device.energy = soft_signal_rw(float, 7.0) + device.temperature = soft_signal_rw(float, 20.0) device.set_name("mono") device.set_readable_format(device.energy, Format.CONFIG_SIGNAL) device.set_readable_format(device.temperature, Format.CONFIG_SIGNAL) return device -async def test_store_and_retrieve_readable_formats(RE, technique_device, tmp_path): +async def test_store_settings_writes_formats_under_reserved_key( + RE, technique_device, tmp_path +): provider = YamlSettingsProvider(tmp_path) def my_plan(): - yield from store_readable_formats(provider, "fixed_energy", technique_device) + yield from store_settings(provider, "fixed_energy", technique_device) with open(tmp_path / "fixed_energy.yaml") as f: - # Stored as plain strings against dotted paths, so it stays editable - assert yaml.safe_load(f) == { - "": {"energy": "CONFIG_SIGNAL", "temperature": "CONFIG_SIGNAL"} - } - - retrieved = yield from retrieve_readable_formats( - provider, "fixed_energy", technique_device - ) - assert retrieved == { - "": { - "energy": Format.CONFIG_SIGNAL, - "temperature": Format.CONFIG_SIGNAL, - } + stored = yaml.safe_load(f) + # Values stay flat, formats live under a key that cannot be a path + assert stored == { + "energy": 7.0, + "temperature": 20.0, + "": { + "": { + "energy": "CONFIG_SIGNAL", + "temperature": "CONFIG_SIGNAL", + } + }, } RE(my_plan()) -async def test_readable_formats_round_trip_switches_technique( - RE, technique_device, tmp_path -): +async def test_reserved_keys_need_no_quoting_in_yaml(RE, technique_device, tmp_path): + provider = YamlSettingsProvider(tmp_path) + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + text = (tmp_path / "fixed_energy.yaml").read_text() + assert ":" in text + assert ":" in text + # An empty string root would make yaml fall back to "? ''" explicit keys + assert "?" not in text + + RE(my_plan()) + + +async def test_settings_round_trip_switches_technique(RE, technique_device, tmp_path): provider = YamlSettingsProvider(tmp_path) energy, temperature = technique_device.energy, technique_device.temperature def my_plan(): - yield from store_readable_formats(provider, "fixed_energy", technique_device) + yield from store_settings(provider, "fixed_energy", technique_device) # Switch to a technique that scans energy and drops temperature entirely technique_device.set_readable_format(energy, Format.HINTED_SIGNAL) technique_device.set_readable_format(temperature, None) - yield from store_readable_formats(provider, "scan_energy", technique_device) + yield from bps.abs_set(energy, 9.0, wait=True) + yield from store_settings(provider, "scan_energy", technique_device) - # Going back restores both, including the one that was dropped - fixed = yield from retrieve_readable_formats( - provider, "fixed_energy", technique_device - ) - apply_readable_formats(technique_device, fixed) + # Going back restores both the value and the formats in one apply + fixed = yield from retrieve_settings(provider, "fixed_energy", technique_device) + yield from apply_settings(fixed) assert technique_device.get_readable_format(energy) is Format.CONFIG_SIGNAL assert technique_device.get_readable_format(temperature) is Format.CONFIG_SIGNAL + assert (yield from bps.rd(energy)) == 7.0 # And forward again drops temperature rather than merging the two - scanning = yield from retrieve_readable_formats( + scanning = yield from retrieve_settings( provider, "scan_energy", technique_device ) - apply_readable_formats(technique_device, scanning) + yield from apply_settings(scanning) assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL assert technique_device.get_readable_format(temperature) is None + assert (yield from bps.rd(energy)) == 9.0 + + RE(my_plan()) + + +async def test_settings_file_without_formats_leaves_them_alone( + RE, technique_device, tmp_path +): + """A file stored before formats existed must not clear them.""" + provider = YamlSettingsProvider(tmp_path) + energy = technique_device.energy + (tmp_path / "old_style.yaml").write_text("energy: 3.0\ntemperature: 4.0\n") + + def my_plan(): + settings = yield from retrieve_settings(provider, "old_style", technique_device) + assert settings.readable_formats == {} + yield from apply_settings(settings) + # Values applied, formats untouched + assert (yield from bps.rd(energy)) == 3.0 + assert technique_device.get_readable_format(energy) is Format.CONFIG_SIGNAL RE(my_plan()) -async def test_store_readable_formats_walks_nested_devices(RE, tmp_path): +async def test_formats_only_file_applies_without_touching_values( + RE, technique_device, tmp_path +): + """A hand written formats only profile changes no hardware.""" + provider = YamlSettingsProvider(tmp_path) + energy = technique_device.energy + (tmp_path / "hinted.yaml").write_text( + ":\n :\n energy: HINTED_SIGNAL\n" + ) + + def my_plan(): + settings = yield from retrieve_settings(provider, "hinted", technique_device) + assert dict(settings) == {} + yield from apply_settings(settings) + assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL + # temperature was not mentioned, so the root's registry was replaced + assert ( + technique_device.get_readable_format(technique_device.temperature) is None + ) + assert (yield from bps.rd(energy)) == 7.0 + + RE(my_plan()) + + +async def test_store_settings_walks_nested_devices(RE, tmp_path): provider = YamlSettingsProvider(tmp_path) inner = StandardReadable(name="inner") - inner.sig, _ = soft_signal_r_and_setter(float, 1.0) + inner.sig = soft_signal_rw(float, 1.0) outer = StandardReadable(name="outer") outer.inner = inner - outer.top, _ = soft_signal_r_and_setter(float, 2.0) + outer.top = soft_signal_rw(float, 2.0) outer.set_name("outer") inner.set_readable_format(inner.sig, Format.HINTED_SIGNAL) outer.set_readable_format(outer.top, Format.CONFIG_SIGNAL) outer.set_readable_format(inner, Format.CHILD) def my_plan(): - yield from store_readable_formats(provider, "nested", outer) + yield from store_settings(provider, "nested", outer) with open(tmp_path / "nested.yaml") as f: assert yaml.safe_load(f) == { - "": {"top": "CONFIG_SIGNAL", "inner": "CHILD"}, - "inner": {"inner.sig": "HINTED_SIGNAL"}, + "top": 2.0, + "inner.sig": 1.0, + "": { + "": {"top": "CONFIG_SIGNAL", "inner": "CHILD"}, + "inner": {"inner.sig": "HINTED_SIGNAL"}, + }, } RE(my_plan()) @@ -291,3 +349,16 @@ def test_apply_readable_formats_rejects_unknown_path(technique_device): def test_apply_readable_formats_rejects_non_readable(technique_device): with pytest.raises(TypeError, match="is not a StandardReadable"): apply_readable_formats(technique_device, {"energy": {}}) + + +def test_settings_partition_carries_formats_to_both_halves(technique_device): + # apply_panda_settings applies only the halves, never the original, so + # formats must survive on whichever half is applied + settings = Settings( + technique_device, + {technique_device.energy: 1.0, technique_device.temperature: 2.0}, + {"": {"energy": Format.CONFIG_SIGNAL}}, + ) + a, b = settings.partition(lambda sig: "energy" in sig.name) + assert a.readable_formats == settings.readable_formats + assert b.readable_formats == settings.readable_formats diff --git a/tests/unit_tests/test_data/test_yaml_config_save.yaml b/tests/unit_tests/test_data/test_yaml_config_save.yaml index 99fb4ac43c..f2245dda43 100644 --- a/tests/unit_tests/test_data/test_yaml_config_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_config_save.yaml @@ -1,3 +1,27 @@ +: + : + a_bool: CONFIG_SIGNAL + a_enum: CONFIG_SIGNAL + a_float: CONFIG_SIGNAL + a_int: CONFIG_SIGNAL + a_str: CONFIG_SIGNAL + a_subset_enum: CONFIG_SIGNAL + a_superset_enum: CONFIG_SIGNAL + boola: CONFIG_SIGNAL + enuma: CONFIG_SIGNAL + float32a: CONFIG_SIGNAL + float64a: CONFIG_SIGNAL + int16a: CONFIG_SIGNAL + int32a: CONFIG_SIGNAL + int64a: CONFIG_SIGNAL + int8a: CONFIG_SIGNAL + ndarray: CONFIG_SIGNAL + stra: CONFIG_SIGNAL + table: CONFIG_SIGNAL + uint16a: CONFIG_SIGNAL + uint32a: CONFIG_SIGNAL + uint64a: CONFIG_SIGNAL + uint8a: CONFIG_SIGNAL a_bool: true a_enum: Bbb a_float: 1.234 @@ -7,53 +31,35 @@ a_subset_enum: Bbb a_superset_enum: Bbb boola: [false, false, true] enuma: - - Aaa - - Ccc -float32a: - [ - -3.4028234663852886e+38, - 3.4028234663852886e+38, - 1.1754943508222875e-38, - 1.401298464324817e-45, - 0.0, - 1.2339999675750732, - 234000.0, - 3.4499998946557753e-06, - ] -float64a: - [ - -1.7976931348623157e+308, - 1.7976931348623157e+308, - 2.2250738585072014e-308, - 5.0e-324, - 0.0, - 1.234, - 234000.0, - 3.45e-06, - ] +- Aaa +- Ccc +float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, + 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] +float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, + 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] int16a: [-32768, 32767, 0, 1, 2, 3, 4] int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] int8a: [-128, 127, 0, 1, 2, 3, 4] ndarray: [[1, 2, 3], [4, 5, 6]] stra: - - one - - two - - three +- one +- two +- three table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar uint16a: [0, 65535, 0, 1, 2, 3, 4] uint32a: [0, 4294967295, 0, 1, 2, 3, 4] uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] diff --git a/tests/unit_tests/test_data/test_yaml_save.yaml b/tests/unit_tests/test_data/test_yaml_save.yaml index 900837389f..af0e7ab207 100644 --- a/tests/unit_tests/test_data/test_yaml_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_save.yaml @@ -1,3 +1,73 @@ +: + child: + child.a_bool: CONFIG_SIGNAL + child.a_enum: CONFIG_SIGNAL + child.a_float: CONFIG_SIGNAL + child.a_int: CONFIG_SIGNAL + child.a_str: CONFIG_SIGNAL + child.a_subset_enum: CONFIG_SIGNAL + child.a_superset_enum: CONFIG_SIGNAL + child.boola: CONFIG_SIGNAL + child.enuma: CONFIG_SIGNAL + child.float32a: CONFIG_SIGNAL + child.float64a: CONFIG_SIGNAL + child.int16a: CONFIG_SIGNAL + child.int32a: CONFIG_SIGNAL + child.int64a: CONFIG_SIGNAL + child.int8a: CONFIG_SIGNAL + child.ndarray: CONFIG_SIGNAL + child.stra: CONFIG_SIGNAL + child.table: CONFIG_SIGNAL + child.uint16a: CONFIG_SIGNAL + child.uint32a: CONFIG_SIGNAL + child.uint64a: CONFIG_SIGNAL + child.uint8a: CONFIG_SIGNAL + vector.1: + vector.1.a_bool: CONFIG_SIGNAL + vector.1.a_enum: CONFIG_SIGNAL + vector.1.a_float: CONFIG_SIGNAL + vector.1.a_int: CONFIG_SIGNAL + vector.1.a_str: CONFIG_SIGNAL + vector.1.a_subset_enum: CONFIG_SIGNAL + vector.1.a_superset_enum: CONFIG_SIGNAL + vector.1.boola: CONFIG_SIGNAL + vector.1.enuma: CONFIG_SIGNAL + vector.1.float32a: CONFIG_SIGNAL + vector.1.float64a: CONFIG_SIGNAL + vector.1.int16a: CONFIG_SIGNAL + vector.1.int32a: CONFIG_SIGNAL + vector.1.int64a: CONFIG_SIGNAL + vector.1.int8a: CONFIG_SIGNAL + vector.1.ndarray: CONFIG_SIGNAL + vector.1.stra: CONFIG_SIGNAL + vector.1.table: CONFIG_SIGNAL + vector.1.uint16a: CONFIG_SIGNAL + vector.1.uint32a: CONFIG_SIGNAL + vector.1.uint64a: CONFIG_SIGNAL + vector.1.uint8a: CONFIG_SIGNAL + vector.3: + vector.3.a_bool: CONFIG_SIGNAL + vector.3.a_enum: CONFIG_SIGNAL + vector.3.a_float: CONFIG_SIGNAL + vector.3.a_int: CONFIG_SIGNAL + vector.3.a_str: CONFIG_SIGNAL + vector.3.a_subset_enum: CONFIG_SIGNAL + vector.3.a_superset_enum: CONFIG_SIGNAL + vector.3.boola: CONFIG_SIGNAL + vector.3.enuma: CONFIG_SIGNAL + vector.3.float32a: CONFIG_SIGNAL + vector.3.float64a: CONFIG_SIGNAL + vector.3.int16a: CONFIG_SIGNAL + vector.3.int32a: CONFIG_SIGNAL + vector.3.int64a: CONFIG_SIGNAL + vector.3.int8a: CONFIG_SIGNAL + vector.3.ndarray: CONFIG_SIGNAL + vector.3.stra: CONFIG_SIGNAL + vector.3.table: CONFIG_SIGNAL + vector.3.uint16a: CONFIG_SIGNAL + vector.3.uint32a: CONFIG_SIGNAL + vector.3.uint64a: CONFIG_SIGNAL + vector.3.uint8a: CONFIG_SIGNAL _sig_rw: Top level private SignalRW child.a_bool: true child.a_enum: Bbb @@ -8,53 +78,35 @@ child.a_subset_enum: Bbb child.a_superset_enum: Bbb child.boola: [false, false, true] child.enuma: - - Aaa - - Ccc -child.float32a: - [ - -3.4028234663852886e+38, - 3.4028234663852886e+38, - 1.1754943508222875e-38, - 1.401298464324817e-45, - 0.0, - 1.2339999675750732, - 234000.0, - 3.4499998946557753e-06, - ] -child.float64a: - [ - -1.7976931348623157e+308, - 1.7976931348623158e+308, - 2.2250738585072014e-308, - 5.0e-324, - 0.0, - 1.234, - 234000.0, - 3.45e-06, - ] +- Aaa +- Ccc +child.float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, + 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] +child.float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, + 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] child.int16a: [-32768, 32767, 0, 1, 2, 3, 4] child.int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] child.int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] child.int8a: [-128, 127, 0, 1, 2, 3, 4] child.ndarray: [[1, 2, 3], [4, 5, 6]] child.stra: - - one - - two - - three +- one +- two +- three child.table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar child.uint16a: [0, 65535, 0, 1, 2, 3, 4] child.uint32a: [0, 4294967295, 0, 1, 2, 3, 4] child.uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] @@ -69,53 +121,35 @@ vector.1.a_subset_enum: Bbb vector.1.a_superset_enum: Bbb vector.1.boola: [false, false, true] vector.1.enuma: - - Aaa - - Ccc -vector.1.float32a: - [ - -3.4028234663852886e+38, - 3.4028234663852886e+38, - 1.1754943508222875e-38, - 1.401298464324817e-45, - 0.0, - 1.2339999675750732, - 234000.0, - 3.4499998946557753e-06, - ] -vector.1.float64a: - [ - -1.7976931348623157e+308, - 1.7976931348623157e+308, - 2.2250738585072014e-308, - 5.0e-324, - 0.0, - 1.234, - 234000.0, - 3.45e-06, - ] +- Aaa +- Ccc +vector.1.float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, + 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] +vector.1.float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, + 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] vector.1.int16a: [-32768, 32767, 0, 1, 2, 3, 4] vector.1.int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] vector.1.int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] vector.1.int8a: [-128, 127, 0, 1, 2, 3, 4] vector.1.ndarray: [[1, 2, 3], [4, 5, 6]] vector.1.stra: - - one - - two - - three +- one +- two +- three vector.1.table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar vector.1.uint16a: [0, 65535, 0, 1, 2, 3, 4] vector.1.uint32a: [0, 4294967295, 0, 1, 2, 3, 4] vector.1.uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] @@ -129,53 +163,35 @@ vector.3.a_subset_enum: Bbb vector.3.a_superset_enum: Bbb vector.3.boola: [false, false, true] vector.3.enuma: - - Aaa - - Ccc -vector.3.float32a: - [ - -3.4028234663852886e+38, - 3.4028234663852886e+38, - 1.1754943508222875e-38, - 1.401298464324817e-45, - 0.0, - 1.2339999675750732, - 234000.0, - 3.4499998946557753e-06, - ] -vector.3.float64a: - [ - -1.7976931348623157e+308, - 1.7976931348623157e+308, - 2.2250738585072014e-308, - 5.0e-324, - 0.0, - 1.234, - 234000.0, - 3.45e-06, - ] +- Aaa +- Ccc +vector.3.float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, + 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] +vector.3.float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, + 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] vector.3.int16a: [-32768, 32767, 0, 1, 2, 3, 4] vector.3.int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] vector.3.int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] vector.3.int8a: [-128, 127, 0, 1, 2, 3, 4] vector.3.ndarray: [[1, 2, 3], [4, 5, 6]] vector.3.stra: - - one - - two - - three +- one +- two +- three vector.3.table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar vector.3.uint16a: [0, 65535, 0, 1, 2, 3, 4] vector.3.uint32a: [0, 4294967295, 0, 1, 2, 3, 4] vector.3.uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] From 2b44ed28b4d0c55516dc6a48992e5b0f34c0b511 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 14:57:26 +0000 Subject: [PATCH 06/67] Add the formats key to the tango system test golden file TangoTestDevice is a StandardReadable that registers nothing, so walk_readable_formats returns {"": {}} for it -- one owner with no children, which means "this owner registers nothing" and has to be stored so that applying the file clears it. The EPICS golden files are untouched because EpicsTestCaDevice is not a StandardReadable, so no formats key is written for it at all. Co-Authored-By: Claude Opus 5 (cherry picked from commit 2f764b259bf78732aa1bcf5c25d391ad6a1a29fb) --- tests/system_tests/tango/core/test_yaml_save.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system_tests/tango/core/test_yaml_save.yaml b/tests/system_tests/tango/core/test_yaml_save.yaml index c7103cfd3b..7b02aaaa60 100644 --- a/tests/system_tests/tango/core/test_yaml_save.yaml +++ b/tests/system_tests/tango/core/test_yaml_save.yaml @@ -7,3 +7,5 @@ float64_image: [[-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585 int32_spectrum: [-2147483648, 2147483647, 0, 1, 2, 3, 4] my_state: INIT strenum: BBB +: + : {} From 0bbe294d472be8abcf78af317f719c5a02f40486 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 18 Aug 2026 15:08:20 +0000 Subject: [PATCH 07/67] Cover the empty-owner formats case in unit tests An owner entry with no children means "this owner registers nothing", so it has to be stored -- otherwise applying a profile that registers nothing would silently leave the previous profile's registrations in place. That is what made the tango golden file need the key. The distinction was only exercised by a system test needing a live Tango device server, so both shapes are now unit tested: a StandardReadable that registers nothing stores {"": {}} and applying it clears, while a Device with no StandardReadable in its tree stores no key at all. Co-Authored-By: Claude Opus 5 (cherry picked from commit 1681595e843acad13129aa299d2042f1f5e11349) --- tests/unit_tests/plan_stubs/test_settings.py | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index be6555031e..93420169e4 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -7,6 +7,7 @@ import yaml from ophyd_async.core import ( + Device, Settings, StandardReadable, YamlSettingsProvider, @@ -362,3 +363,57 @@ def test_settings_partition_carries_formats_to_both_halves(technique_device): a, b = settings.partition(lambda sig: "energy" in sig.name) assert a.readable_formats == settings.readable_formats assert b.readable_formats == settings.readable_formats + + +async def test_store_settings_records_a_readable_that_registers_nothing(RE, tmp_path): + """An empty owner entry means "clear this owner", so it must be stored. + + This is the shape of TangoTestDevice, and it is why applying a profile that + registers nothing drops what the previous profile registered rather than + silently leaving it in place. + """ + provider = YamlSettingsProvider(tmp_path) + device = StandardReadable(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + + def my_plan(): + yield from store_settings(provider, "empty", device) + with open(tmp_path / "empty.yaml") as f: + assert yaml.safe_load(f) == { + "sig": 1.0, + "": {"": {}}, + } + + # Register something, then apply the stored file back + device.set_readable_format(device.sig, Format.CONFIG_SIGNAL) + settings = yield from retrieve_settings(provider, "empty", device) + yield from apply_settings(settings) + assert device.get_readable_format(device.sig) is None + + RE(my_plan()) + + +async def test_store_settings_omits_the_key_for_a_device_with_no_readables( + RE, tmp_path +): + """A Device with no StandardReadable in its tree has nothing to clear. + + This is the shape of EpicsTestCaDevice, and is why the EPICS golden files + needed no regeneration. + """ + provider = YamlSettingsProvider(tmp_path) + + class Plain(Device): + pass + + device = Plain(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + + def my_plan(): + yield from store_settings(provider, "plain", device) + with open(tmp_path / "plain.yaml") as f: + assert yaml.safe_load(f) == {"sig": 1.0} + + RE(my_plan()) From 8d5d900f20bdb17e77daa3c1b9036c4fc4afc640 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 25 Aug 2026 20:33:59 +0000 Subject: [PATCH 08/67] Address review: drop deprecated shims, hook into the base classes, merge formats Detector-side changes from the original commit move to the StandardDetector group, so this PR stays core-only. --- .../decisions/0021-runtime-readable-format.md | 55 +++++-- .../decisions/0022-single-logic-property.md | 5 +- docs/how-to/change-readable-format.md | 57 +------ docs/how-to/store-and-retrieve.md | 38 ++++- docs/reference/environment-variables.md | 2 +- src/ophyd_async/core/__init__.py | 4 - src/ophyd_async/core/_flyable.py | 30 ++-- src/ophyd_async/core/_movable.py | 10 -- src/ophyd_async/core/_readable.py | 139 +++++++++--------- src/ophyd_async/plan_stubs/_settings.py | 9 +- .../tango/core/test_yaml_save.yaml | 2 - tests/unit_tests/core/test_readable.py | 9 +- tests/unit_tests/plan_stubs/test_settings.py | 36 +++-- 13 files changed, 193 insertions(+), 203 deletions(-) diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index 1f0a26cfa2..4c29328e2f 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -71,10 +71,11 @@ Registration is "set" rather than "add": registering the same device twice repla format rather than contributing it twice. This is a behaviour change, and the previous behaviour was not deliberate. -Formats are normalised to real enum members on the way in. The deprecated `ConfigSignal` -and `HintedSignal` markers announce themselves by comparing equal to their target, which -relied on the `match` statement running during `add_readables`; normalising on entry keeps -the warning at registration rather than moving it to the first `read()`. +The deprecated `ConfigSignal` and `HintedSignal` markers are removed rather than carried +across. They worked by comparing equal to their target, which relied on the `match` +statement running during `add_readables`; deriving the callables on demand moves that +comparison to the first `read()` instead. Rather than keep a shim alive to preserve where +the warning fires, they go — they have been deprecated since well before this change. ### Format changes apply between runs, not within one @@ -89,6 +90,17 @@ A detector's step-scan signals are registered exactly like any other Device's, a `read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. `add_config_signals` becomes a deprecated wrapper. +The detector does not *reimplement* any of those verbs to achieve it. `StandardReadable` +grows two hooks, `_extra_funcs_for(verb)` and `_extra_hint_sources()`, that contribute +from somewhere other than the registry, and `StandardDetector` overrides those to add what +its data logics produce. Likewise it registers its stage and unstage work into +`_stage_funcs`/`_unstage_funcs`, the way `StandardFlyable` already does, rather than +replacing `stage()`. Overriding the verbs meant re-deriving the registered children's +contributions by hand in each one, which is exactly what would drift. + +One consequence: a detector with no hinted fields now reports `hints` as `{}` rather than +`{"fields": []}`, because it no longer special cases what `StandardReadable` already does. + Plugins are **not** registered automatically. `AreaDetector` registers only the driver's `acquire_time` and `acquire_period`, plus whatever the caller passes as `config_sigs`. Registering a plugin is a `set_readable_format()` call at the call site, which is the @@ -114,12 +126,9 @@ temperature: 20.0 whole stored state. **No version marker is needed.** A file written before formats existed simply lacks the -reserved key, which under replacement semantics reads correctly as "no owner is listed, so -clear no owner" — leave formats alone. Present-but-empty for an owner (`: {}`) means -"clear that owner", which is what a freshly-saved Device with nothing registered produces. -The two states are distinguishable without a version, and each section being independently -optional also makes a hand-written formats-only profile work, changing technique without -writing a value to hardware. +reserved key, which means "change no formats". Each part being independently optional also +makes a hand-written formats-only profile work, changing technique without writing a value +to hardware. **The reserved keys cannot collide.** `` and `` start with `<`, which cannot begin a Python identifier, so no attribute assignment can produce a @@ -138,10 +147,28 @@ direct child" rule, so `AreaDetector` can keep registering `driver.acquire_time` `DeviceVector` elements keep working, with no dependency on #1395 and no second migration later. -`apply_readable_formats` **replaces** the registered children of each named -`StandardReadable` rather than merging into them, so that loading a technique profile drops -what the previous one registered instead of accumulating the union of both. It resolves -every path before changing anything, so a bad path cannot leave the tree half applied. +### Applying merges, and unregistering is explicit + +`apply_readable_formats` **merges** into what is already registered rather than replacing +it: a child the file does not mention keeps whatever format it has. + +Replacement was tried first, because it makes technique switching automatic — loading a +profile drops what the previous one registered. It was rejected because it breaks the case +that matters more: store a file, add a signal to the Device in a later version, apply the +old file, and the new signal is silently unregistered. Signal *values* already behave the +other way round — applying a stored file leaves signals it has never heard of alone — and +formats living in the same file should not behave differently from the values beside them. + +A child is dropped by giving it a format of `None`, a null in the stored file. Nothing +writes those nulls automatically, because storing a Device records only what it currently +registers and cannot know what some other profile registered; a profile that needs a child +dropped has to say so. The cost is that round-tripping A -> B -> A no longer drops +automatically, which is the price of not silently unregistering signals added since a file +was written. + +It follows that a `StandardReadable` registering nothing gets no entry at all, since an +empty entry would be a no-op. `apply_readable_formats` resolves every path before changing +anything, so a bad path cannot leave the tree half applied. `Settings.partition` copies the formats to **both** halves. A device-specific apply plan normally applies only the halves and never the original `Settings` — `apply_panda_settings` diff --git a/docs/explanations/decisions/0022-single-logic-property.md b/docs/explanations/decisions/0022-single-logic-property.md index ff130e1497..77a5fcb6de 100644 --- a/docs/explanations/decisions/0022-single-logic-property.md +++ b/docs/explanations/decisions/0022-single-logic-property.md @@ -70,6 +70,9 @@ this change fixes. this cost, as `MotorFlyableMovableLogic` was already that shape. - Forgetting the logic now fails with `TypeError: Can't instantiate abstract class ... with abstract method logic` rather than an `AttributeError` on `None`. -- `movable_logic` and `flyable_logic` remain as deprecated properties. +- `movable_logic` and `flyable_logic` are removed rather than deprecated. A shim would + only have helped code that *reads* the attribute: a subclass that still *defines* + `movable_logic` would override the shim and leave `logic` unimplemented, so the + Device would fail at instantiation anyway. - The same mechanism is available to `StandardDetector` if it later inherits [](#StandardFlyable); its logic would need to satisfy `FlyableLogic` too. diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index d38d9a2917..3f4a6c108a 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -49,60 +49,9 @@ det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can write a file *and* report a plugin scalar in a step scan. -## Saving and restoring a technique - -Formats are part of a Device's state, and they change on the same cadence as signal -values, so [](#store_settings) stores both and [](#apply_settings) applies both: - -```python -def save_technique(det): - provider = YamlSettingsProvider("/path/to/profiles") - yield from store_settings(provider, "fixed_energy", det) - - -def load_technique(det, technique: str): - provider = YamlSettingsProvider("/path/to/profiles") - settings = yield from retrieve_settings(provider, technique, det) - yield from apply_settings(settings) -``` - -Values stay flat in the yaml, and formats go under the reserved -[](#READABLE_FORMATS_KEY), so the file stays readable and hand-editable: - -```yaml -energy: 7.0 -temperature: 20.0 -: - : - energy: CONFIG_SIGNAL - temperature: CONFIG_SIGNAL - stats: - stats.total: HINTED_UNCACHED_SIGNAL -``` - -The outer key of each formats entry is the [](#StandardReadable) the child is registered -on, with [](#ROOT_DEVICE_KEY) for the Device you stored. It is recorded rather than inferred -because the same child can be registered on more than one Device with different formats. - -```{note} -Applying **replaces** the children registered on each Device the file names, rather than -merging into them. That is what makes switching technique work: loading a profile drops -what the previous one registered instead of accumulating both. -``` - -Because each part is optional, a file with no `` key leaves formats -alone — so settings files saved before formats were storable keep working untouched. The -reverse is useful too: a hand-written formats-only profile switches technique without -writing a single value to hardware. - -```yaml -: - : - energy: HINTED_SIGNAL -``` - ```{seealso} +[](./store-and-retrieve.md) for saving a set of formats alongside signal values and +loading them back, and [](../explanations/decisions/0021-runtime-readable-format.md) for why the registry is -keyed by Device and how the file is laid out, and [](./store-and-retrieve.md) for the -wider store/retrieve workflow. +keyed by Device. ``` diff --git a/docs/how-to/store-and-retrieve.md b/docs/how-to/store-and-retrieve.md index 3e1159e76a..f138fb071b 100644 --- a/docs/how-to/store-and-retrieve.md +++ b/docs/how-to/store-and-retrieve.md @@ -35,8 +35,6 @@ def load_panda(panda1: HDFPanda): yield from apply_panda_settings(settings) ``` -As well as signal values, [](#store_settings) records how each child contributes to the Device's bluesky verbs, under the reserved [](#READABLE_FORMATS_KEY), and [](#apply_settings) restores that too. This means a stored file can switch a Device between techniques as well as putting its values back — see [](./change-readable-format.md). Files stored before this existed simply have no such key, and load unchanged. - In the situation where the time taken to read a set of SignalRWs of a Device is less than the time taken to set those signals, the [](#apply_settings_if_different) stub should be used instead. This will take the SignalRWs included in the Settings, read them from the connected Device, and set those that differ from the stored value: ``` @@ -45,3 +43,39 @@ def load_panda(panda1: HDFPanda): settings = yield from retrieve_settings(provider, "yaml_file_name", panda1) yield from apply_settings_if_different(settings, apply_panda_settings) ``` + +## Storing readable formats too + +As well as signal values, [](#store_settings) records how each child contributes to the Device's bluesky verbs — which signals are hinted, which are configuration — and [](#apply_settings) restores that alongside the values. The two change on the same cadence, so they live in one file and are applied by one call. This means a stored file can switch a Device between techniques as well as putting its values back; see [](./change-readable-format.md) for changing a format directly. + +Values stay flat in the yaml, and formats go under the reserved [](#READABLE_FORMATS_KEY), so the file stays readable and can be edited by hand: + +```yaml +energy: 7.0 +temperature: 20.0 +: + : + energy: CONFIG_SIGNAL + temperature: CONFIG_SIGNAL + stats: + stats.total: HINTED_UNCACHED_SIGNAL +``` + +The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with [](#ROOT_DEVICE_KEY) standing for the Device you stored. It is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. + +```{note} +Applying **merges** into what is already registered: a child the file does not mention keeps whatever format it has. This matches how values behave — applying a stored file leaves signals it has never heard of alone — so a file stored against an older version of a Device does not silently unregister signals that have been added since. +``` + +To stop a child contributing, the file has to say so explicitly with a null: + +```yaml +: + : + energy: HINTED_SIGNAL + temperature: null +``` + +Nothing writes those nulls for you, because storing a Device records only what it currently registers and cannot know what some other profile registered. A profile that needs a child dropped has to spell it out. + +Both parts are optional. A file with no `` key changes no formats, so settings files saved before formats were storable keep working untouched. A file with *only* that key switches technique without writing a single value to hardware. diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 8ab91db8d8..78afba8c51 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -13,7 +13,7 @@ effect on ophyd-async itself. Controls the implicit prepare performed by [`StandardDetector.trigger()`](#ophyd_async.core.StandardDetector.trigger) when [`prepare()`](#ophyd_async.core.StandardDetector.prepare) has not been called since -the last [`stage()`](#ophyd_async.core.StandardDetector.stage). +the last `stage()`. | Value | Behaviour | |-------|-----------| diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 9d8b3b0de1..e5c0328637 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -102,8 +102,6 @@ DEVICE_NAMES_KEY, READABLE_FORMATS_KEY, ROOT_DEVICE_KEY, - ConfigSignal, - HintedSignal, ReadableFormats, StandardReadable, StandardReadableFormat, @@ -347,8 +345,6 @@ def __getattr__(name): "DerivedSignalFactory", "merge_gathered_dicts", # Back compat - delete before 1.0 - "ConfigSignal", - "HintedSignal", # Standard enums "EnabledDisabled", "EnableDisable", diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index a6410eae25..548f9a8bd1 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -1,5 +1,4 @@ import asyncio -import warnings from abc import abstractmethod from collections.abc import AsyncIterator from enum import Enum @@ -86,11 +85,7 @@ async def on_unstage(self) -> None: def with_device(self, name: str = "") -> "StandardFlyable[PrepareT, CtxT]": """Wrap this logic in an ephemeral `StandardFlyable` for use in a plan.""" - flyer = _EphemeralFlyable[PrepareT, CtxT](name=name) - # cached_property writes straight to the instance __dict__, so this - # satisfies the abstract declaration without a subclass per logic type. - flyer.logic = self - return flyer + return _EphemeralFlyable(self, name=name) class FlyMotorInfo(ConfinedModel): @@ -184,15 +179,6 @@ def logic(self) -> FlyableLogic[PrepareT, CtxT]: """ raise NotImplementedError - # Back compat - delete before 1.0 - @property - def flyable_logic(self) -> FlyableLogic[PrepareT, CtxT]: - warnings.warn( - DeprecationWarning("Use `logic` instead of `flyable_logic`"), - stacklevel=2, - ) - return self.logic - def _reset_fly_state(self) -> None: self._fly_ctx = cast(CtxT, None) self._fly_stage = _FlyStage.IDLE @@ -261,15 +247,17 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: class _EphemeralFlyable(StandardFlyable[PrepareT, CtxT]): - """A concrete `StandardFlyable` whose logic is set on the instance. + """A concrete `StandardFlyable` built around a logic object. `StandardFlyable.logic` is abstract, so the class itself cannot be - instantiated; `FlyableLogic.with_device` needs something concrete to attach - a logic object to. + instantiated; `FlyableLogic.with_device` needs something concrete to wrap a + logic object in. """ + def __init__(self, logic: FlyableLogic[PrepareT, CtxT], name: str = "") -> None: + self._logic = logic + super().__init__(name=name) + @cached_property def logic(self) -> FlyableLogic[PrepareT, CtxT]: - raise NotImplementedError( - f"{self.name}: no logic set; create this via FlyableLogic.with_device()" - ) + return self._logic diff --git a/src/ophyd_async/core/_movable.py b/src/ophyd_async/core/_movable.py index bf0a9b3f35..9d5d293f2a 100644 --- a/src/ophyd_async/core/_movable.py +++ b/src/ophyd_async/core/_movable.py @@ -1,6 +1,5 @@ import asyncio import time -import warnings from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum @@ -159,15 +158,6 @@ def logic(self) -> MovableLogic[SignalDatatypeT]: """ raise NotImplementedError - # Back compat - delete before 1.0 - @property - def movable_logic(self) -> MovableLogic[SignalDatatypeT]: - warnings.warn( - DeprecationWarning("Use `logic` instead of `movable_logic`"), - stacklevel=2, - ) - return self.logic - async def check_value(self, value: SignalDatatypeT) -> None: """Check the move is valid before doing it.""" await self.logic.check_move(value) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index d347e60aaf..5a842314f8 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -3,7 +3,6 @@ from collections.abc import Awaitable, Callable, Generator, Iterator, Sequence from contextlib import contextmanager from enum import Enum -from typing import Any, cast from bluesky.protocols import HasHints, Hints, Reading from event_model import DataKey @@ -35,22 +34,6 @@ def _as_signal_r(device: Device) -> SignalR: return device -def _normalise_format(format: "StandardReadableFormat") -> "StandardReadableFormat": - """Resolve a deprecated `ConfigSignal`/`HintedSignal` marker to a real member. - - The registry stores real enum members so that the format can be compared, - serialised and reported later. The deprecated markers only announce - themselves when compared, so the comparison has to happen here, at - registration, rather than each time a verb is called. - """ - if isinstance(format, StandardReadableFormat): - return format - for member in StandardReadableFormat: - if format == member: # _WarningMatcher.__eq__ raises the DeprecationWarning - return member - raise TypeError(f"{format} is not a StandardReadableFormat") - - class StandardReadableFormat(Enum): """Declare how a `Device` should contribute to the `StandardReadable` verbs.""" @@ -89,34 +72,6 @@ def __call__(self, parent: Device, child: Device): parent.add_readables([child], self) -# Back compat -class _WarningMatcher: - def __init__(self, name: str, target: StandardReadableFormat): - self._name = name - self._target = target - - def __eq__(self, value: object) -> bool: - warnings.warn( - DeprecationWarning( - f"Use `StandardReadableFormat.{self._target.name}` " - f"instead of `{self._name}`" - ), - stacklevel=2, - ) - return value == self._target - - -def _compat_format(name: str, target: StandardReadableFormat) -> StandardReadableFormat: - return cast(StandardReadableFormat, _WarningMatcher(name, target)) - - -ConfigSignal = _compat_format("ConfigSignal", StandardReadableFormat.CONFIG_SIGNAL) -HintedSignal: Any = _compat_format("HintedSignal", StandardReadableFormat.HINTED_SIGNAL) -HintedSignal.uncached = _compat_format( - "HintedSignal.uncached", StandardReadableFormat.HINTED_UNCACHED_SIGNAL -) - - class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints): """Device that provides selected child Device values in `read()`. @@ -162,8 +117,19 @@ def _signals_to_stage(self) -> Iterator[AsyncStageable]: ): yield device + def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: + """Contribute to a verb from something other than the registry. + + Subclasses that produce data from somewhere other than a registered + child override this rather than the verb methods themselves, so that + registered children keep working without being reimplemented. + `StandardDetector` uses it for the data its data logics produce. + """ + return iter(()) + def _funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: """Derive the callables contributing to one verb from the registry.""" + yield from self._extra_funcs_for(verb) for device, format in self._readables: match format: case StandardReadableFormat.CHILD: @@ -217,7 +183,15 @@ async def read(self) -> dict[str, Reading]: [func() for func in self._funcs_for(_Verb.READ)] ) + def _extra_hint_sources(self) -> Iterator[HasHints]: + """Contribute hints from something other than the registry. + + The counterpart of [](#StandardReadable._extra_funcs_for) for `hints`. + """ + return iter(()) + def _hint_sources(self) -> Iterator[HasHints]: + yield from self._extra_hint_sources() for device, format in self._readables: match format: case StandardReadableFormat.CHILD if isinstance(device, HasHints): @@ -330,7 +304,8 @@ def set_readable_format( :raises TypeError: If a signal-only format is given a non-`SignalR` """ if format is not None: - format = _normalise_format(format) + if not isinstance(format, StandardReadableFormat): + raise TypeError(f"{format} is not a StandardReadableFormat") if format is not StandardReadableFormat.CHILD: _as_signal_r(device) kept = tuple((d, f) for d, f in self._readables if d is not device) @@ -347,10 +322,6 @@ def readable_children(self) -> Iterator[tuple[Device, StandardReadableFormat]]: """Iterate over the registered children and their formats, in order.""" yield from self._readables - def clear_readables(self) -> None: - """Drop every registered child, so this Device contributes nothing.""" - self._readables = () - def add_readables( self, devices: Sequence[Device], @@ -383,6 +354,21 @@ async def __call__(self) -> dict[str, Reading]: return await self.signal.read(cached=False) +class _HintedFields(HasHints): + """Present a fixed list of field names as `hints`.""" + + def __init__(self, fields: Sequence[str]) -> None: + self._fields = list(fields) + + @property + def name(self) -> str: + return "" + + @property + def hints(self) -> Hints: + return {"fields": self._fields} + + class _HintsFromName(HasHints): def __init__(self, device: Device) -> None: self.device = device @@ -414,14 +400,15 @@ def hints(self) -> Hints: ROOT_DEVICE_KEY = "" #: The readable formats of a Device tree in a form that can be stored and -#: retrieved: `{path of the StandardReadable: {path of the child: format}}`. +#: retrieved: `{path of the StandardReadable: {path of the child: format}}`, +#: where a format of `None` means "stop this child contributing". #: Paths are dotted attribute paths from the root Device, as produced by #: [](#walk_devices), with [](#ROOT_DEVICE_KEY) meaning the root Device itself. #: #: The outer key is needed because a format is a fact about an (owner, child) #: *pair*, not about a single Device: the same child can be registered on more #: than one `StandardReadable` with a different format each time. -ReadableFormats = dict[str, dict[str, StandardReadableFormat]] +ReadableFormats = dict[str, dict[str, StandardReadableFormat | None]] def _paths_to_devices(device: Device) -> dict[str, Device]: @@ -431,10 +418,10 @@ def _paths_to_devices(device: Device) -> dict[str, Device]: def walk_readable_formats(device: Device) -> ReadableFormats: """Retrieve the readable format of every registered child in a Device tree. - Used as part of saving a Device, so that which signals are hinted, config - or omitted can be restored later. Every `StandardReadable` in the tree gets - an entry, including those with nothing registered, so that applying the - result is a complete description rather than a partial overlay. + Used as part of saving a Device, so that which signals are hinted or + config can be restored later. Only `StandardReadable`s that actually + register something get an entry: applying merges rather than replaces, so + an empty entry would do nothing. :param device: The root Device to walk. :return: A [](#ReadableFormats) suitable for storing. @@ -444,7 +431,7 @@ def walk_readable_formats(device: Device) -> ReadableFormats: for path, dev in _paths_to_devices(device).items(): if not isinstance(dev, StandardReadable): continue - entries: dict[str, StandardReadableFormat] = {} + entries: dict[str, StandardReadableFormat | None] = {} for child, format in dev.readable_children(): child_path = device_to_path.get(child) if child_path is None: @@ -462,17 +449,33 @@ def walk_readable_formats(device: Device) -> ReadableFormats: ) else: entries[child_path] = format - formats[path] = entries + if entries: + formats[path] = entries return formats def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: """Set the readable format of children in a Device tree. - This **replaces** the registered children of each `StandardReadable` named - in `formats`, rather than adding to them, so that applying a stored set - switches a Device between techniques rather than accumulating the union of - both. `StandardReadable`s not named in `formats` are left alone. + This **merges** into what is already registered rather than replacing it: a + child the formats do not mention keeps whatever format it has. That matches + how signal values behave, where applying a stored file leaves signals it has + never heard of alone -- so a file stored against an older version of a + Device does not silently unregister signals added since. + + To stop a child contributing, give it a format of `None` explicitly, which + in a stored file is a null: + + ```yaml + : + : + energy: HINTED_SIGNAL + temperature: null + ``` + + Nothing writes those nulls for you, because storing a Device records only + what it currently registers and cannot know what some other profile + registered. A profile that needs a child dropped has to say so. :param device: The root Device the paths in `formats` are relative to. :param formats: A [](#ReadableFormats), e.g. from [](#walk_readable_formats). @@ -480,8 +483,9 @@ def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: :raises TypeError: If a path does not name a `StandardReadable`. """ devices = _paths_to_devices(device) - resolved: list[tuple[StandardReadable, list[tuple[Device, StandardReadableFormat]]]] - resolved = [] + resolved: list[ + tuple[StandardReadable, list[tuple[Device, StandardReadableFormat | None]]] + ] = [] # Resolve everything before changing anything, so a bad path cannot leave # the tree half applied for owner_path, entries in formats.items(): @@ -490,16 +494,17 @@ def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: raise KeyError(f"No Device at {owner_path!r} in {device.name or device}") if not isinstance(owner, StandardReadable): raise TypeError(f"{owner_path!r} is not a StandardReadable, got {owner}") - readables = [] + readables: list[tuple[Device, StandardReadableFormat | None]] = [] for child_path, format in entries.items(): child = devices.get(child_path) if child is None: raise KeyError( f"No Device at {child_path!r} in {device.name or device}" ) - readables.append((child, StandardReadableFormat(format))) + readables.append( + (child, None if format is None else StandardReadableFormat(format)) + ) resolved.append((owner, readables)) for owner, readables in resolved: - owner.clear_readables() for child, format in readables: owner.set_readable_format(child, format) diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 8b61effcd0..50594f058a 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -84,7 +84,10 @@ def store_settings( # Omitted entirely when there is nothing to say, so that a settings only # file stays exactly as it was before formats were storable data[READABLE_FORMATS_KEY] = { - owner: {child: format.value for child, format in entries.items()} + owner: { + child: None if format is None else format.value + for child, format in entries.items() + } for owner, entries in formats.items() } yield from wait_for_awaitable(provider.store(name, data)) @@ -114,7 +117,9 @@ def retrieve_settings( data.pop(DEVICE_NAMES_KEY, None) readable_formats: ReadableFormats = { owner: { - child: StandardReadableFormat(format) for child, format in entries.items() + # A null means "stop this child contributing", so it is not coerced + child: None if format is None else StandardReadableFormat(format) + for child, format in entries.items() } for owner, entries in stored_formats.items() } diff --git a/tests/system_tests/tango/core/test_yaml_save.yaml b/tests/system_tests/tango/core/test_yaml_save.yaml index 7b02aaaa60..c7103cfd3b 100644 --- a/tests/system_tests/tango/core/test_yaml_save.yaml +++ b/tests/system_tests/tango/core/test_yaml_save.yaml @@ -7,5 +7,3 @@ float64_image: [[-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585 int32_spectrum: [-2147483648, 2147483647, 0, 1, 2, 3, 4] my_state: INIT strenum: BBB -: - : {} diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 1d4b027a2f..f4c71e61fb 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -4,11 +4,9 @@ from bluesky.protocols import HasHints from ophyd_async.core import ( - ConfigSignal, Device, DeviceMap, DeviceVector, - HintedSignal, MockSignalBackend, SignalR, StandardReadable, @@ -18,11 +16,10 @@ from ophyd_async.core import StandardReadableFormat as Format -@pytest.mark.parametrize("wrapper", [HintedSignal, HintedSignal.uncached, ConfigSignal]) -def test_standard_readable_wrappers_raise_deprecation_warning(wrapper): +def test_standard_readable_rejects_a_non_format(): sr = StandardReadable() - with pytest.deprecated_call(): - sr.add_readables([soft_signal_rw(int)], wrapper) + with pytest.raises(TypeError, match="is not a StandardReadableFormat"): + sr.add_readables([soft_signal_rw(int)], "CONFIG_SIGNAL") # type: ignore def test_standard_readable_hints(): diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index 93420169e4..a1d15906e9 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -266,7 +266,8 @@ def my_plan(): ) yield from apply_settings(scanning) assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL - assert technique_device.get_readable_format(temperature) is None + # Merge, so temperature keeps the format the previous profile gave it + assert technique_device.get_readable_format(temperature) is Format.CONFIG_SIGNAL assert (yield from bps.rd(energy)) == 9.0 RE(my_plan()) @@ -294,11 +295,18 @@ def my_plan(): async def test_formats_only_file_applies_without_touching_values( RE, technique_device, tmp_path ): - """A hand written formats only profile changes no hardware.""" + """A hand written formats only profile changes no hardware. + + Also covers the explicit null, which is the only way to unregister a child, + since applying otherwise merges. + """ provider = YamlSettingsProvider(tmp_path) energy = technique_device.energy (tmp_path / "hinted.yaml").write_text( - ":\n :\n energy: HINTED_SIGNAL\n" + ":\n" + " :\n" + " energy: HINTED_SIGNAL\n" + " temperature: null\n" ) def my_plan(): @@ -306,7 +314,7 @@ def my_plan(): assert dict(settings) == {} yield from apply_settings(settings) assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL - # temperature was not mentioned, so the root's registry was replaced + # An explicit null drops it, where omitting it would have left it alone assert ( technique_device.get_readable_format(technique_device.temperature) is None ) @@ -365,12 +373,11 @@ def test_settings_partition_carries_formats_to_both_halves(technique_device): assert b.readable_formats == settings.readable_formats -async def test_store_settings_records_a_readable_that_registers_nothing(RE, tmp_path): - """An empty owner entry means "clear this owner", so it must be stored. +async def test_store_settings_omits_a_readable_that_registers_nothing(RE, tmp_path): + """An owner with no registered children is a no-op when applied. - This is the shape of TangoTestDevice, and it is why applying a profile that - registers nothing drops what the previous profile registered rather than - silently leaving it in place. + This is the shape of TangoTestDevice, and is why its golden file needs no + formats key. """ provider = YamlSettingsProvider(tmp_path) device = StandardReadable(name="dev") @@ -380,16 +387,7 @@ async def test_store_settings_records_a_readable_that_registers_nothing(RE, tmp_ def my_plan(): yield from store_settings(provider, "empty", device) with open(tmp_path / "empty.yaml") as f: - assert yaml.safe_load(f) == { - "sig": 1.0, - "": {"": {}}, - } - - # Register something, then apply the stored file back - device.set_readable_format(device.sig, Format.CONFIG_SIGNAL) - settings = yield from retrieve_settings(provider, "empty", device) - yield from apply_settings(settings) - assert device.get_readable_format(device.sig) is None + assert yaml.safe_load(f) == {"sig": 1.0} RE(my_plan()) From 4fba82cdd5dce01645bade44cd3e330dee83a19c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 10:48:15 +0000 Subject: [PATCH 09/67] Replace the readable format accessors with one get_readable_formats() get_readable_format(device) was only ever used from tests, and readable_children() returned an iterator of pairs that every caller immediately wanted to look up by device. One get_readable_formats() -> dict[Device, StandardReadableFormat] covers both: a Device that does not contribute is simply absent, so .get(child) replaces the singular accessor, and the dict keeps registration order. Device.__eq__ is object identity and DeviceVector / DeviceMap hash by id, so the dict is identity keyed exactly like the tuple it is built from. Co-Authored-By: Claude Opus 5 (cherry picked from commit 3517dbcc1f1854463ef97953df618d89054e5b74) --- docs/how-to/change-readable-format.md | 8 ++++-- src/ophyd_async/core/_readable.py | 17 +++++------- tests/unit_tests/core/test_readable.py | 17 +++++++++--- tests/unit_tests/epics/pvi/test_pvi.py | 2 +- tests/unit_tests/plan_stubs/test_settings.py | 29 +++++++++++++++----- 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index 3f4a6c108a..2726a18c6c 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -25,8 +25,12 @@ mono.set_readable_format(mono.energy, Format.CONFIG_SIGNAL) mono.set_readable_format(mono.energy, None) ``` -[](#StandardReadable.get_readable_format) reports the current format, returning `None` if -the child does not contribute. +[](#StandardReadable.get_readable_formats) reports what is currently registered, keyed by +child Device, so a child that does not contribute is simply absent: + +```python +mono.get_readable_formats().get(mono.energy) +``` ```{note} Change formats **between runs**, not inside one. A run's descriptor is emitted when the diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 5a842314f8..9dfff7d0e4 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -311,16 +311,13 @@ def set_readable_format( kept = tuple((d, f) for d, f in self._readables if d is not device) self._readables = kept if format is None else (*kept, (device, format)) - def get_readable_format(self, device: Device) -> StandardReadableFormat | None: - """Return the format of a child Device, or `None` if it does not contribute.""" - for registered, format in self._readables: - if registered is device: - return format - return None + def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: + """Return the registered children and their formats, in registration order. - def readable_children(self) -> Iterator[tuple[Device, StandardReadableFormat]]: - """Iterate over the registered children and their formats, in order.""" - yield from self._readables + A Device that does not contribute is absent, so + `get_readable_formats().get(child)` gives its format or `None`. + """ + return dict(self._readables) def add_readables( self, @@ -432,7 +429,7 @@ def walk_readable_formats(device: Device) -> ReadableFormats: if not isinstance(dev, StandardReadable): continue entries: dict[str, StandardReadableFormat | None] = {} - for child, format in dev.readable_children(): + for child, format in dev.get_readable_formats().items(): child_path = device_to_path.get(child) if child_path is None: # Registered something that is not in this tree, so it has no diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index f4c71e61fb..7ea0d2d45e 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -314,15 +314,24 @@ async def test_set_readable_format_replaces_rather_than_duplicates(): sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) assert sr.hints == {"fields": [sr.sig.name]} - assert sr.get_readable_format(sr.sig) is Format.HINTED_SIGNAL + assert sr.get_readable_formats() == {sr.sig: Format.HINTED_SIGNAL} -async def test_get_readable_format_returns_none_when_unregistered(): +async def test_get_readable_formats_omits_unregistered_children(): sr = StandardReadable(name="sr") sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") - assert sr.get_readable_format(sr.sig) is None + sr.other, _ = soft_signal_r_and_setter(int, 0, name="other") + assert sr.get_readable_formats() == {} sr.set_readable_format(sr.sig, Format.CONFIG_SIGNAL) - assert sr.get_readable_format(sr.sig) is Format.CONFIG_SIGNAL + sr.set_readable_format(sr.other, Format.HINTED_SIGNAL) + # Registration order is preserved, and an unregistered child is absent + assert sr.get_readable_formats() == { + sr.sig: Format.CONFIG_SIGNAL, + sr.other: Format.HINTED_SIGNAL, + } + assert list(sr.get_readable_formats()) == [sr.sig, sr.other] + sr.set_readable_format(sr.sig, None) + assert sr.get_readable_formats() == {sr.other: Format.HINTED_SIGNAL} async def test_set_readable_format_affects_staging_of_later_runs(): diff --git a/tests/unit_tests/epics/pvi/test_pvi.py b/tests/unit_tests/epics/pvi/test_pvi.py index 17aa12137b..f6bdb29007 100644 --- a/tests/unit_tests/epics/pvi/test_pvi.py +++ b/tests/unit_tests/epics/pvi/test_pvi.py @@ -155,7 +155,7 @@ async def test_device_create_children_from_annotations_with_device_vectors(): await device.connect(mock=True) block_1_device = device.device - assert device.get_readable_format(block_1_device) is Format.CHILD + assert device.get_readable_formats().get(block_1_device) is Format.CHILD block_2_device_vector = device.device_vector assert device.device_vector[1].name == "test_device-device_vector-1" diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index a1d15906e9..025cb086fe 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -256,8 +256,13 @@ def my_plan(): # Going back restores both the value and the formats in one apply fixed = yield from retrieve_settings(provider, "fixed_energy", technique_device) yield from apply_settings(fixed) - assert technique_device.get_readable_format(energy) is Format.CONFIG_SIGNAL - assert technique_device.get_readable_format(temperature) is Format.CONFIG_SIGNAL + assert ( + technique_device.get_readable_formats().get(energy) is Format.CONFIG_SIGNAL + ) + assert ( + technique_device.get_readable_formats().get(temperature) + is Format.CONFIG_SIGNAL + ) assert (yield from bps.rd(energy)) == 7.0 # And forward again drops temperature rather than merging the two @@ -265,9 +270,14 @@ def my_plan(): provider, "scan_energy", technique_device ) yield from apply_settings(scanning) - assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL + assert ( + technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL + ) # Merge, so temperature keeps the format the previous profile gave it - assert technique_device.get_readable_format(temperature) is Format.CONFIG_SIGNAL + assert ( + technique_device.get_readable_formats().get(temperature) + is Format.CONFIG_SIGNAL + ) assert (yield from bps.rd(energy)) == 9.0 RE(my_plan()) @@ -287,7 +297,9 @@ def my_plan(): yield from apply_settings(settings) # Values applied, formats untouched assert (yield from bps.rd(energy)) == 3.0 - assert technique_device.get_readable_format(energy) is Format.CONFIG_SIGNAL + assert ( + technique_device.get_readable_formats().get(energy) is Format.CONFIG_SIGNAL + ) RE(my_plan()) @@ -313,10 +325,13 @@ def my_plan(): settings = yield from retrieve_settings(provider, "hinted", technique_device) assert dict(settings) == {} yield from apply_settings(settings) - assert technique_device.get_readable_format(energy) is Format.HINTED_SIGNAL + assert ( + technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL + ) # An explicit null drops it, where omitting it would have left it alone assert ( - technique_device.get_readable_format(technique_device.temperature) is None + technique_device.get_readable_formats().get(technique_device.temperature) + is None ) assert (yield from bps.rd(energy)) == 7.0 From 132fee860396619dec17d06b9ca1d11d7d0dfdea Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 10:48:52 +0000 Subject: [PATCH 10/67] Merge readable formats per child in Settings.__or__, and cover the warning Settings.__or__ merged formats with `formats |= other.readable_formats`, a dict update keyed by owner, so the whole owner entry was replaced: {"": {a: CONFIG, b: CONFIG}} | {"": {a: HINTED}} dropped b. Values merge per signal and apply_readable_formats merges per child -- deliberately, so a stored file never silently unregisters a signal added since it was written -- so __or__ must merge at the same granularity. Two things fell out of writing the test. __or__ rejected `other` being a Settings for the *same* Device, because _is_in_device is really "is a strict child of", which made the whole formats path unreachable. And a Settings for a Device *below* this one has paths relative to itself, so and every dotted path would come to mean something else once merged; re-rooting them is not implemented as nothing needs it, so that now raises rather than silently attaching them to the wrong Device. Settings.__or__ has no callers in tree and had no tests either way. Also covers walk_readable_formats' warning for a readable registered outside the tree (#1402) -- the branch whose message says the value is not stored either, which nothing exercised. Co-Authored-By: Claude Opus 5 (cherry picked from commit 1e9cc9f3ce8783c2bbc54358ed1586dac035e266) --- src/ophyd_async/core/_settings.py | 37 ++++++-- tests/unit_tests/plan_stubs/test_settings.py | 97 ++++++++++++++++++++ 2 files changed, 128 insertions(+), 6 deletions(-) diff --git a/src/ophyd_async/core/_settings.py b/src/ophyd_async/core/_settings.py index 948516e2b3..83ae4a9aac 100644 --- a/src/ophyd_async/core/_settings.py +++ b/src/ophyd_async/core/_settings.py @@ -80,13 +80,38 @@ def __len__(self) -> int: return len(self._settings) def __or__(self, other: MutableMapping[SignalRW, Any]) -> Settings[DeviceT]: - """Create a new Settings that is the union of self overridden by other.""" - if isinstance(other, Settings) and not self._is_in_device(other.device): - raise ValueError(f"{other.device} is not a child of {self.device}") - formats = dict(self.readable_formats) + """Create a new Settings that is the union of self overridden by other. + + Readable formats merge per (owner, child), the same granularity that + values merge at and that [](#apply_readable_formats) applies at, so + `other` only replaces the children it actually mentions. + + :raises ValueError: + If `other` is a `Settings` for a Device outside this one, or for a + Device below this one while carrying readable formats -- their paths + would be relative to a different root. + """ + formats = { + owner: dict(entries) for owner, entries in self.readable_formats.items() + } if isinstance(other, Settings): - # Merge per owner, so that other only replaces the owners it mentions - formats |= other.readable_formats + if other.device is not self.device: + if not self._is_in_device(other.device): + msg = f"{other.device} is not a child of {self.device}" + raise ValueError(msg) + if other.readable_formats: + # Paths in a ReadableFormats are relative to its own device, + # so and every dotted path would mean something + # different once merged. Re-rooting them is not implemented + # as nothing needs it yet; better to say so than to silently + # attach them to the wrong Device. + msg = ( + f"Cannot merge readable formats from {other.device}, whose " + f"paths are relative to it rather than to {self.device}" + ) + raise ValueError(msg) + for owner, entries in other.readable_formats.items(): + formats.setdefault(owner, {}).update(entries) return Settings(self.device, self._settings | dict(other), formats) def partition( diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index 025cb086fe..0ab658f66a 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -14,6 +14,7 @@ apply_readable_formats, get_mock, soft_signal_rw, + walk_readable_formats, ) from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.plan_stubs import ( @@ -375,6 +376,68 @@ def test_apply_readable_formats_rejects_non_readable(technique_device): apply_readable_formats(technique_device, {"energy": {}}) +def test_settings_or_merges_formats_per_child(technique_device): + # Values merge per signal and apply_readable_formats merges per child, so + # `other` must only replace the children it mentions -- overriding a whole + # owner would drop children it has nothing to say about + base = Settings( + technique_device, + {technique_device.energy: 1.0}, + { + "": { + "energy": Format.CONFIG_SIGNAL, + "temperature": Format.CHILD, + } + }, + ) + override = Settings( + technique_device, + {technique_device.temperature: 2.0}, + {"": {"energy": Format.HINTED_SIGNAL}}, + ) + merged = base | override + assert merged.readable_formats == { + "": { + "energy": Format.HINTED_SIGNAL, # overridden + "temperature": Format.CHILD, # untouched, not dropped + } + } + assert dict(merged) == { + technique_device.energy: 1.0, + technique_device.temperature: 2.0, + } + # Neither operand is mutated + assert base.readable_formats[""]["energy"] is Format.CONFIG_SIGNAL + assert override.readable_formats[""] == { + "energy": Format.HINTED_SIGNAL + } + + +def test_settings_or_keeps_owners_the_other_does_not_mention(technique_device): + base = Settings(technique_device, {}, {"a": {"x": Format.CONFIG_SIGNAL}}) + override = Settings(technique_device, {}, {"b": {"y": Format.HINTED_SIGNAL}}) + assert (base | override).readable_formats == { + "a": {"x": Format.CONFIG_SIGNAL}, + "b": {"y": Format.HINTED_SIGNAL}, + } + + +def test_settings_or_rejects_formats_from_a_sub_device(technique_device): + # A sub-device's paths are relative to itself, so would silently come + # to mean the parent + parent = StandardReadable(name="parent") + parent.child = technique_device + parent.set_name("parent") + base = Settings(parent) + sub = Settings( + technique_device, {}, {"": {"energy": Format.HINTED_SIGNAL}} + ) + with pytest.raises(ValueError, match="paths are relative to it"): + base | sub + # Without formats it is just a value merge, which is fine + assert (base | Settings(technique_device)).readable_formats == {} + + def test_settings_partition_carries_formats_to_both_halves(technique_device): # apply_panda_settings applies only the halves, never the original, so # formats must survive on whichever half is applied @@ -407,6 +470,40 @@ def my_plan(): RE(my_plan()) +async def test_store_settings_warns_and_skips_a_readable_outside_the_tree(RE, tmp_path): + """A child registered from outside the tree has no path to store it against. + + Its value is not stored either, since store_settings only walks the tree, so + it cannot survive a round trip at all. See + https://github.com/bluesky/ophyd-async/issues/1402. + """ + provider = YamlSettingsProvider(tmp_path) + device = StandardReadable(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + outsider = soft_signal_rw(float, 2.0, name="outsider") + device.set_readable_format(device.sig, Format.CONFIG_SIGNAL) + device.set_readable_format(outsider, Format.CONFIG_SIGNAL) + + match = "dev: outsider is not within dev" + with pytest.warns(UserWarning, match=match): + assert walk_readable_formats(device) == { + "": {"sig": Format.CONFIG_SIGNAL} + } + + def my_plan(): + with pytest.warns(UserWarning, match=match): + yield from store_settings(provider, "outsider", device) + with open(tmp_path / "outsider.yaml") as f: + # Neither the outsider's format nor its value is stored + assert yaml.safe_load(f) == { + "sig": 1.0, + "": {"": {"sig": "CONFIG_SIGNAL"}}, + } + + RE(my_plan()) + + async def test_store_settings_omits_the_key_for_a_device_with_no_readables( RE, tmp_path ): From 46fc22ec2d59f8bb0b4144ba985b2d363f585594 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 11:40:49 +0000 Subject: [PATCH 11/67] Tidy the readable format registry and settings round trip Five small things from review, no behaviour change except the reserved key handling: - retrieve_settings copied the dict from the provider before popping the reserved keys out of it, so a provider that caches what it returns no longer loses the formats on a second retrieve. - DEVICE_NAMES_KEY is gone. It was exported in core.__all__ but nothing wrote it and only a defensive pop read it, so it was public API for a feature that does not exist. retrieve_settings now drops *every* key rather than the ones it knows: no path can look like that, so a key written by a newer version is ignored instead of being reported as an unknown signal name, and adding one later needs neither a migration nor a constant reserved up front. Import directionality forbids plan_stubs reaching into core._readable, so keeping the constant module private was not an option anyway. - walk_readable_formats walked the tree twice, once to build the path map and once to invert it. - _signals_to_stage had the same condition written out twice; the two staging formats are now a named frozenset. - The warning stacklevel is left at 2 deliberately: it correctly blames the caller of walk_readable_formats, which is the public entry point. Co-Authored-By: Claude Opus 5 (cherry picked from commit fa2b8b280841d2c6a761004c10a06db24df19f78) --- .../decisions/0021-runtime-readable-format.md | 18 ++++++---- src/ophyd_async/core/__init__.py | 2 -- src/ophyd_async/core/_readable.py | 28 +++++++-------- src/ophyd_async/plan_stubs/_settings.py | 12 ++++--- tests/unit_tests/plan_stubs/test_settings.py | 34 +++++++++++++++++++ 5 files changed, 67 insertions(+), 27 deletions(-) diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index 4c29328e2f..6c88ce7f73 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -130,13 +130,17 @@ reserved key, which means "change no formats". Each part being independently opt makes a hand-written formats-only profile work, changing technique without writing a value to hardware. -**The reserved keys cannot collide.** `` and `` start with -`<`, which cannot begin a Python identifier, so no attribute assignment can produce a -colliding path — no store-time guard is required. `<` is also not a YAML indicator, so -unlike `*FORMATS*` or `%FORMATS%` the key needs no quoting. `` replaces `""` as the -root owner for the same reason and because an empty-string key makes PyYAML emit its -hard-to-read explicit `? '' :` form. `` is reserved but unwritten, so storing -names later needs no migration. +**The reserved keys cannot collide.** `` starts with `<`, which cannot +begin a Python identifier, so no attribute assignment can produce a colliding path — no +store-time guard is required. `<` is also not a YAML indicator, so unlike `*FORMATS*` or +`%FORMATS%` the key needs no quoting. `` replaces `""` as the root owner for the same +reason and because an empty-string key makes PyYAML emit its hard-to-read explicit +`? '' :` form. + +Because no path can look like `<...>`, `retrieve_settings` drops *every* such key rather +than only the one it understands. So a key written by a newer version — device names, say +— is ignored by an older one instead of being reported as an unknown signal, and adding +one later needs no migration and no constant reserved up front. **The formats section is two levels while values are flat.** A value is intrinsic to one path; a format is a fact about an (owner, child) *pair*, and the same child can be diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index e5c0328637..1563695a29 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -99,7 +99,6 @@ Watcher, ) from ._readable import ( - DEVICE_NAMES_KEY, READABLE_FORMATS_KEY, ROOT_DEVICE_KEY, ReadableFormats, @@ -280,7 +279,6 @@ def __getattr__(name): "walk_readable_formats", "apply_readable_formats", "READABLE_FORMATS_KEY", - "DEVICE_NAMES_KEY", "ROOT_DEVICE_KEY", # Detector "DetectorTrigger", diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 9dfff7d0e4..ee64321262 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -72,6 +72,13 @@ def __call__(self, parent: Device, child: Device): parent.add_readables([child], self) +#: The formats whose devices take part in `stage()`/`unstage()`, if they are +#: `Stageable` at all. +_STAGED_FORMATS = frozenset( + {StandardReadableFormat.HINTED_SIGNAL, StandardReadableFormat.CHILD} +) + + class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints): """Device that provides selected child Device values in `read()`. @@ -106,15 +113,11 @@ async def _unstage_readables(self) -> None: await asyncio.gather(*(sig.unstage().task for sig in self._signals_to_stage())) def _signals_to_stage(self) -> Iterator[AsyncStageable]: + # HINTED_SIGNAL stages so that caching is set up and read() is fast; a + # CHILD stages because it may have staging of its own. The uncached and + # config formats deliberately do not. for device, format in self._readables: - if format is StandardReadableFormat.HINTED_SIGNAL and isinstance( - device, AsyncStageable - ): - # Caching is set up on stage so that read() is fast - yield device - elif format is StandardReadableFormat.CHILD and isinstance( - device, AsyncStageable - ): + if format in _STAGED_FORMATS and isinstance(device, AsyncStageable): yield device def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: @@ -386,10 +389,6 @@ def hints(self) -> Hints: #: unlike `*FORMATS*` it needs no quoting in yaml. READABLE_FORMATS_KEY = "" -#: Reserved key for storing Device names. Reserved now so that adding it later -#: needs no migration; nothing writes it yet. -DEVICE_NAMES_KEY = "" - #: The path standing in for the root Device itself in a [](#ReadableFormats). #: Not a valid Python identifier, so no attribute assignment can produce a #: colliding path, and unlike `""` it does not make yaml fall back to its @@ -423,9 +422,10 @@ def walk_readable_formats(device: Device) -> ReadableFormats: :param device: The root Device to walk. :return: A [](#ReadableFormats) suitable for storing. """ - device_to_path = {dev: path for path, dev in _paths_to_devices(device).items()} + paths_to_devices = _paths_to_devices(device) + device_to_path = {dev: path for path, dev in paths_to_devices.items()} formats: ReadableFormats = {} - for path, dev in _paths_to_devices(device).items(): + for path, dev in paths_to_devices.items(): if not isinstance(dev, StandardReadable): continue entries: dict[str, StandardReadableFormat | None] = {} diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 50594f058a..3a4b65e5d3 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -9,7 +9,6 @@ from bluesky.utils import MsgGenerator, plan from ophyd_async.core import ( - DEVICE_NAMES_KEY, READABLE_FORMATS_KEY, Device, ReadableFormats, @@ -110,11 +109,16 @@ def retrieve_settings( :param device: The Device to retrieve the settings for. :param only_config: If True, retrieve only configuration settings. """ - data = yield from wait_for_awaitable(provider.retrieve(name)) + # Copy, as the provider may hand back a dict it holds on to and the reserved + # keys are popped out of it below + data = dict((yield from wait_for_awaitable(provider.retrieve(name)))) # Reserved keys first, so what is left is signal paths and the unknown name - # check below still catches genuine typos + # check below still catches genuine typos. Any key is dropped, + # not just the one understood here: a path can never look like that, so a + # key written by a newer version is ignored rather than reported as a typo. stored_formats = data.pop(READABLE_FORMATS_KEY, {}) - data.pop(DEVICE_NAMES_KEY, None) + for key in [k for k in data if k.startswith("<") and k.endswith(">")]: + del data[key] readable_formats: ReadableFormats = { owner: { # A null means "stop this child contributing", so it is not coerced diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index 0ab658f66a..72fb808ed8 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -339,6 +339,40 @@ def my_plan(): RE(my_plan()) +async def test_retrieve_settings_ignores_unknown_reserved_keys( + RE, technique_device, tmp_path +): + """A key from a newer version is skipped, not reported as an unknown signal. + + No signal path can look like <...>, so there is nothing to reserve up front. + """ + provider = YamlSettingsProvider(tmp_path) + (tmp_path / "newer.yaml").write_text( + "energy: 3.0\n:\n : mono\n: 1\n" + ) + + def my_plan(): + settings = yield from retrieve_settings(provider, "newer", technique_device) + assert dict(settings) == {technique_device.energy: 3.0} + yield from apply_settings(settings) + assert (yield from bps.rd(technique_device.energy)) == 3.0 + + RE(my_plan()) + + +async def test_retrieve_settings_still_rejects_a_genuine_typo( + RE, technique_device, tmp_path +): + provider = YamlSettingsProvider(tmp_path) + (tmp_path / "typo.yaml").write_text("energyy: 3.0\n") + + def my_plan(): + with pytest.raises(NameError, match=r"Unknown signal names \['energyy'\]"): + yield from retrieve_settings(provider, "typo", technique_device) + + RE(my_plan()) + + async def test_store_settings_walks_nested_devices(RE, tmp_path): provider = YamlSettingsProvider(tmp_path) inner = StandardReadable(name="inner") From 11b22b1e5a104763510a4156bf290696897e43e7 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 11:48:01 +0000 Subject: [PATCH 12/67] Store the readable registry in a dict rather than a tuple of pairs set_readable_format rebuilt the whole tuple on every call, scanning it for the device first, so add_readables of n children was O(n^2): 2000 signals cost 77ms at construction. A dict makes it O(1) per call and linear overall -- 2000 signals now cost 13ms, 4000 cost 25ms. Keying by Device is identity keying, which is what a registry of children needs. Device does not override __eq__; DeviceVector and DeviceMap do inherit a value based __eq__ from Mapping, but they hash by id(), which is injective over live objects, so two distinct devices never share a hash and __eq__ is never reached. It is a cached_property rather than an attribute assigned in __init__, matching Device._child_devices: add_children_as_readables and set_readable_format are routinely called *before* super().__init__() (AreaDetector does exactly this), so anything __init__ assigned would discard them. One behaviour change: re-formatting an already registered child now keeps its position instead of moving it to the end, so changing a format no longer reorders hints. Both behaviours are new in this PR, so nothing downstream depends on the old one. Co-Authored-By: Claude Opus 5 (cherry picked from commit a6d2aa8c31acd3dcd3dd77140dec5d864a175165) --- src/ophyd_async/core/_readable.py | 36 +++++++++++++++++++------- tests/unit_tests/core/test_readable.py | 32 +++++++++++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index ee64321262..66840bcf84 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -3,6 +3,7 @@ from collections.abc import Awaitable, Callable, Generator, Iterator, Sequence from contextlib import contextmanager from enum import Enum +from functools import cached_property from bluesky.protocols import HasHints, Hints, Reading from event_model import DataKey @@ -92,10 +93,23 @@ class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints changed at runtime with [](#StandardReadable.set_readable_format). """ - # The registered children, in registration order. Immutable so that it cannot - # be accidentally shared between instances of the class, and so that a runtime - # format change swaps the whole tuple rather than mutating shared state. - _readables: tuple[tuple[Device, StandardReadableFormat], ...] = () + @cached_property + def _readables(self) -> dict[Device, StandardReadableFormat]: + """The registered children and their formats, in registration order. + + A `cached_property` rather than an attribute set in `__init__`, because + `add_children_as_readables` and `set_readable_format` are routinely + called *before* `super().__init__()` (e.g. `AreaDetector`), so anything + `__init__` assigned would discard them. It also keeps each instance's + registry its own, where a mutable class attribute would be shared. + + Keying by Device is identity keying, which is what a registry of + children needs: `Device` does not override `__eq__`, and while + `DeviceVector` and `DeviceMap` inherit a value based `__eq__` from + `Mapping`, they hash by `id()`. That is injective over live objects, so + two distinct devices never share a hash and `__eq__` is never reached. + """ + return {} def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -116,7 +130,7 @@ def _signals_to_stage(self) -> Iterator[AsyncStageable]: # HINTED_SIGNAL stages so that caching is set up and read() is fast; a # CHILD stages because it may have staging of its own. The uncached and # config formats deliberately do not. - for device, format in self._readables: + for device, format in self._readables.items(): if format in _STAGED_FORMATS and isinstance(device, AsyncStageable): yield device @@ -133,7 +147,7 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] def _funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: """Derive the callables contributing to one verb from the registry.""" yield from self._extra_funcs_for(verb) - for device, format in self._readables: + for device, format in self._readables.items(): match format: case StandardReadableFormat.CHILD: if verb in _CONFIG_VERBS and isinstance(device, AsyncConfigurable): @@ -195,7 +209,7 @@ def _extra_hint_sources(self) -> Iterator[HasHints]: def _hint_sources(self) -> Iterator[HasHints]: yield from self._extra_hint_sources() - for device, format in self._readables: + for device, format in self._readables.items(): match format: case StandardReadableFormat.CHILD if isinstance(device, HasHints): yield device @@ -311,8 +325,12 @@ def set_readable_format( raise TypeError(f"{format} is not a StandardReadableFormat") if format is not StandardReadableFormat.CHILD: _as_signal_r(device) - kept = tuple((d, f) for d, f in self._readables if d is not device) - self._readables = kept if format is None else (*kept, (device, format)) + if format is None: + self._readables.pop(device, None) + else: + # Re-formatting an already registered child keeps its position, so + # changing a format does not reorder `hints` + self._readables[device] = format def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: """Return the registered children and their formats, in registration order. diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 7ea0d2d45e..b681fb9ef5 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -334,6 +334,38 @@ async def test_get_readable_formats_omits_unregistered_children(): assert sr.get_readable_formats() == {sr.other: Format.HINTED_SIGNAL} +async def test_readables_registered_before_super_init_survive(): + # AreaDetector registers config signals before calling super().__init__(), + # so the registry must not be created there. It must also be per instance. + class Dev(StandardReadable): + def __init__(self, name=""): + self.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + self.set_readable_format(self.sig, Format.HINTED_SIGNAL) + super().__init__(name=name) + + a, b = Dev(name="a"), Dev(name="b") + assert a.get_readable_formats() == {a.sig: Format.HINTED_SIGNAL} + assert b.get_readable_formats() == {b.sig: Format.HINTED_SIGNAL} + assert a.get_readable_formats() != b.get_readable_formats() + + b.set_readable_format(b.sig, None) + assert a.get_readable_formats() == {a.sig: Format.HINTED_SIGNAL} + assert b.get_readable_formats() == {} + + +async def test_set_readable_format_keeps_position_when_reformatting(): + # Changing a format must not reorder hints + sr = StandardReadable(name="sr") + sr.one, _ = soft_signal_r_and_setter(int, 0, name="one") + sr.two, _ = soft_signal_r_and_setter(int, 0, name="two") + sr.set_readable_format(sr.one, Format.HINTED_SIGNAL) + sr.set_readable_format(sr.two, Format.HINTED_SIGNAL) + assert sr.hints == {"fields": [sr.one.name, sr.two.name]} + + sr.set_readable_format(sr.one, Format.HINTED_UNCACHED_SIGNAL) + assert sr.hints == {"fields": [sr.one.name, sr.two.name]} + + async def test_set_readable_format_affects_staging_of_later_runs(): sr = StandardReadable(name="sr") sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") From ea55c715770a8674de450b6567bca169c10ab9c2 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Thu, 20 Aug 2026 15:52:36 +0000 Subject: [PATCH 13/67] Add reset_readable_formats() to undo runtime format changes @oliwenmandiamond's concern was that a class declaring sig_a as hinted, then having that changed at runtime, makes the class definition misleading, and proposed either locking class-declared formats or a separate StandardDynamicReadable. Both were rejected: static-by-default blocks #1394's own motivating case, a monochromator whose energy is *declared in the class* as config and which one technique wants hinted. @jwlodek made the same call from the v1 side, where changing kind at runtime is used extensively and behaviour differences cost NSLS-II. This is their counter-proposal instead: put the Device back to what its class declared, so a plan that retunes one can undo it without recording what it changed. The baseline is sealed by a metaclass rather than at the end of StandardReadable.__init__. Formats can be declared in three places: annotations, applied inside Device.__init__; registration a subclass does before its super().__init__(), which is what all 24 in-tree Devices do; and registration it does after, which nothing in tree does but which is legal. type.__call__ runs the whole __init__ chain before returning and is the only point after all three -- an end-of-__init__ snapshot was measured to miss the third, and the symptom would have been reset_readable_formats silently dropping a child the class declared. Device already has a metaclass (_ProtocolMeta, from the HasName Protocol), so this adds no new machinery. Per Device rather than per tree, pairing with set_readable_format rather than apply_readable_formats, and the baseline is the class declaration rather than the last applied settings file -- so it discards a stored technique too. Both documented, with a one-liner for the tree case. Co-Authored-By: Claude Opus 5 (cherry picked from commit e798387ad9a37033ecfe8f87adac928f01ab2d96) --- .../decisions/0021-runtime-readable-format.md | 34 +++++++++ docs/how-to/change-readable-format.md | 28 +++++++ src/ophyd_async/core/_readable.py | 71 ++++++++++++++++- tests/unit_tests/core/test_readable.py | 76 +++++++++++++++++++ 4 files changed, 208 insertions(+), 1 deletion(-) diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index 6c88ce7f73..9c20198162 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -77,6 +77,40 @@ statement running during `add_readables`; deriving the callables on demand moves comparison to the first `read()` instead. Rather than keep a shim alive to preserve where the warning fires, they go — they have been deprecated since well before this change. +### Runtime changes can be undone, and the baseline is sealed by a metaclass + +@oliwenmandiamond raised that a class declaring `sig_a` as hinted, then having that +changed at runtime, makes the class definition misleading. Two safeguards were proposed: +making class-declared formats immutable unless opted out, or a separate +`StandardDynamicReadable`. + +Both were rejected, because static-by-default blocks the case this ADR exists for: #1394's +motivating example is a monochromator whose `energy` is *declared in the class* as +configuration and which one technique wants hinted. Locking it precisely because it was +declared would mean a beamline had to get the Device class changed to switch technique, and +the "allow dynamic" decision would be made by whoever is furthest from the technique. +@jwlodek made the same call from the other side: ophyd v1 has always been mutable, v1 +codebases at NSLS-II make extensive use of changing `kind` at runtime, and behaviour +differences between the two libraries are a cost they do not want. + +What was adopted instead is @oliwenmandiamond's and @jwlodek's counter-proposal: +[](#StandardReadable.reset_readable_formats) puts a Device back to what its class declared, +so a plan that retunes a Device can undo it without recording what it changed. + +**The baseline is sealed by a metaclass**, not at the end of `StandardReadable.__init__`. +Formats can be declared in three places: annotations, applied inside `Device.__init__`; +registration a subclass does *before* its `super().__init__()`, which is what all 24 +in-tree Devices do; and registration it does *after*, which nothing in tree does but which +is legal. `type.__call__` runs the whole `__init__` chain before returning, and is the only +point after all three. Snapshotting at the end of `StandardReadable.__init__` was measured +to miss the third, and the symptom would be `reset_readable_formats` silently dropping a +child the class had declared. `Device` already has a metaclass (`_ProtocolMeta`, from the +`HasName` Protocol), so this adds no new machinery to the hierarchy. + +The reset is per-Device rather than per-tree, pairing with `set_readable_format` rather +than with `apply_readable_formats`, and its baseline is the class declaration rather than +the last applied settings file — so it discards a stored technique too. + ### Format changes apply between runs, not within one A run's descriptor is emitted at its start, and `HINTED_SIGNAL` sets up monitoring in diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index 2726a18c6c..aa11759fc6 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -32,6 +32,34 @@ child Device, so a child that does not contribute is simply absent: mono.get_readable_formats().get(mono.energy) ``` +## Putting it back + +[](#StandardReadable.reset_readable_formats) undoes every runtime change, back to +how the class declared things — annotations, `add_children_as_readables`, and +anything the Device's own `__init__` registered: + +```python +mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) # retune for a technique +... +mono.reset_readable_formats() # back to the class defaults +``` + +So a plan that retunes a Device can put it back without having to record what it +changed. It resets that Device only, not its children — to do a whole tree: + +```python +from ophyd_async.core import walk_devices + +for dev in walk_devices(detector).values(): + if isinstance(dev, StandardReadable): + dev.reset_readable_formats() +``` + +```{note} +The baseline is the *class declaration*, not the last file you loaded, so this +also discards a technique applied with [](#apply_settings). +``` + ```{note} Change formats **between runs**, not inside one. A run's descriptor is emitted when the run starts, and [](#StandardReadableFormat.HINTED_SIGNAL) begins monitoring in `stage()`, diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 66840bcf84..76471fc43d 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -80,7 +80,40 @@ def __call__(self, parent: Device, child: Device): ) -class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints): +class _SealDefaultFormatsMeta(type(_StandardBase)): # type: ignore[misc] + """Record the formats a `StandardReadable` was constructed with. + + `type.__call__` runs `__new__` and then the *whole* `__init__` chain before + returning, which is the only point that is after every place a format can + be declared: + + - annotations, applied by `create_children_from_annotations` inside + `Device.__init__` + - registration a subclass does *before* calling `super().__init__()`, which + is what all 24 in-tree Devices do + - registration a subclass does *after* it, which nothing in tree does but + which is perfectly legal + + Snapshotting at the end of `StandardReadable.__init__` would silently miss + the third, and the symptom would be + [](#StandardReadable.reset_readable_formats) dropping a child the class had + declared. Devices are already built on a metaclass (`_ProtocolMeta`, from + the `HasName` Protocol), so this adds no new machinery to the hierarchy. + """ + + def __call__(cls, *args, **kwargs): + self = super().__call__(*args, **kwargs) + self._default_readables = dict(self._readables) + return self + + +class StandardReadable( + _StandardBase, + AsyncReadable, + AsyncConfigurable, + HasHints, + metaclass=_SealDefaultFormatsMeta, +): """Device that provides selected child Device values in `read()`. Provides the ability for children to be registered to: @@ -111,6 +144,11 @@ def _readables(self) -> dict[Device, StandardReadableFormat]: """ return {} + # The formats declared during construction, sealed by the metaclass once + # __init__ has fully returned. Only ever replaced, never mutated in place, + # so the shared class level default is safe. + _default_readables: dict[Device, StandardReadableFormat] = {} + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Staging is derived from _readables on each call rather than registered @@ -332,6 +370,37 @@ def set_readable_format( # changing a format does not reorder `hints` self._readables[device] = format + def reset_readable_formats(self) -> None: + """Undo every runtime format change, back to how the class declared them. + + The baseline is everything registered by the time construction + finished: annotations, `add_children_as_readables`, and any + [](#StandardReadable.set_readable_format) an `__init__` made. Anything + done after that -- switching a child between config and hinted, + dropping one with a format of `None`, registering a new one -- is + undone. + + Intended for a Device that is retuned per technique, so a scan can put + it back without knowing what it changed. + + This resets **this Device only**, not its children; it is the + counterpart of `set_readable_format` rather than of + [](#apply_readable_formats). To reset a whole tree: + + ```python + for dev in walk_devices(detector).values(): + if isinstance(dev, StandardReadable): + dev.reset_readable_formats() + ``` + + The baseline is the *class declaration*, not the last stored settings + file, so this discards a technique loaded by [](#apply_settings) too. + """ + # Mutated rather than replaced, so the cached_property stays the same + # object and nothing holding a reference sees a stale registry + self._readables.clear() + self._readables.update(self._default_readables) + def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: """Return the registered children and their formats, in registration order. diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index b681fb9ef5..4a3cd38f5e 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -334,6 +334,82 @@ async def test_get_readable_formats_omits_unregistered_children(): assert sr.get_readable_formats() == {sr.other: Format.HINTED_SIGNAL} +async def test_reset_readable_formats_undoes_runtime_changes(): + class Dev(StandardReadable): + def __init__(self, name=""): + self.energy, _ = soft_signal_r_and_setter(float, 0.0, name="energy") + self.temp, _ = soft_signal_r_and_setter(float, 0.0, name="temp") + self.set_readable_format(self.energy, Format.CONFIG_SIGNAL) + self.set_readable_format(self.temp, Format.CONFIG_SIGNAL) + super().__init__(name=name) + + dev = Dev(name="dev") + dev.extra, _ = soft_signal_r_and_setter(float, 0.0, name="extra") + declared = dev.get_readable_formats() + + # Change one, drop one, add one that was never declared + dev.set_readable_format(dev.energy, Format.HINTED_SIGNAL) + dev.set_readable_format(dev.temp, None) + dev.set_readable_format(dev.extra, Format.HINTED_SIGNAL) + assert dev.get_readable_formats() == { + dev.energy: Format.HINTED_SIGNAL, + dev.extra: Format.HINTED_SIGNAL, + } + + dev.reset_readable_formats() + assert dev.get_readable_formats() == declared + # Repeatable, and the baseline is not consumed + dev.set_readable_format(dev.energy, None) + dev.reset_readable_formats() + assert dev.get_readable_formats() == declared + + +async def test_reset_readable_formats_keeps_registrations_after_super_init(): + # The reason the baseline is sealed by the metaclass rather than at the end + # of StandardReadable.__init__: this registration happens after super() and + # is still part of what the class declares + class Dev(StandardReadable): + def __init__(self, name=""): + self.before, _ = soft_signal_r_and_setter(int, 0, name="before") + self.set_readable_format(self.before, Format.CONFIG_SIGNAL) + super().__init__(name=name) + self.after, _ = soft_signal_r_and_setter(int, 0, name="after") + self.set_readable_format(self.after, Format.HINTED_SIGNAL) + + dev = Dev(name="dev") + expected = {dev.before: Format.CONFIG_SIGNAL, dev.after: Format.HINTED_SIGNAL} + assert dev.get_readable_formats() == expected + + dev.set_readable_format(dev.after, None) + dev.reset_readable_formats() + assert dev.get_readable_formats() == expected + + +async def test_reset_readable_formats_restores_annotated_declarations(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + # Nothing declared, so a runtime registration resets away entirely + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.reset_readable_formats() + assert sr.get_readable_formats() == {} + + +async def test_default_readable_formats_are_not_shared_between_instances(): + class Dev(StandardReadable): + def __init__(self, name=""): + self.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + self.set_readable_format(self.sig, Format.CONFIG_SIGNAL) + super().__init__(name=name) + + a, b = Dev(name="a"), Dev(name="b") + a.set_readable_format(a.sig, None) + a.reset_readable_formats() + b.reset_readable_formats() + assert a.get_readable_formats() == {a.sig: Format.CONFIG_SIGNAL} + assert b.get_readable_formats() == {b.sig: Format.CONFIG_SIGNAL} + assert StandardReadable._default_readables == {} + + async def test_readables_registered_before_super_init_survive(): # AreaDetector registers config signals before calling super().__init__(), # so the registry must not be created there. It must also be per instance. From 1d9417fcb42563e8b1524a73409e0c9592aaf9a1 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 28 Aug 2026 11:42:06 +0000 Subject: [PATCH 14/67] docs: clarify the reset baseline, the root-device key, and gate the detector Three review points on the readable-format docs: - ADR 0017 only had a backward reference from 0022; add the forward one, so a reader landing on 0017 learns that movable_logic is gone. - The note on reset_readable_formats said the baseline was "the class declaration, not the last file you loaded", which does not say what it affects. Say plainly that it discards formats that came from a stored settings file. - The stored-file how-to named the constant while the yaml above it showed the string. Name the string, and link the constant once. Also move the two sections that describe a StandardDetector inheriting StandardReadable -- one in ADR 0021, one in the how-to -- out of this branch. Nothing here makes that true yet; they belong with the change that does. --- .../decisions/0017-standard-movable.md | 3 +++ .../decisions/0021-runtime-readable-format.md | 23 ------------------- docs/how-to/change-readable-format.md | 18 ++------------- docs/how-to/store-and-retrieve.md | 2 +- src/ophyd_async/core/_readable.py | 9 ++++---- 5 files changed, 11 insertions(+), 44 deletions(-) diff --git a/docs/explanations/decisions/0017-standard-movable.md b/docs/explanations/decisions/0017-standard-movable.md index 3d70c39da9..f6de85ce27 100644 --- a/docs/explanations/decisions/0017-standard-movable.md +++ b/docs/explanations/decisions/0017-standard-movable.md @@ -6,6 +6,9 @@ Date: 2026-03-10 Accepted +Partly superseded by [](./0022-single-logic-property.md), which replaced `movable_logic` +with a single `logic` property. + ## Context Several device types share a common movement pattern: write a setpoint, then observe a diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index 9c20198162..df33bc7942 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -118,29 +118,6 @@ A run's descriptor is emitted at its start, and `HINTED_SIGNAL` sets up monitori disagree. This is documented rather than enforced: the check would have to know about run boundaries, which a Device does not. -### `StandardDetector` inherits `StandardReadable` - -A detector's step-scan signals are registered exactly like any other Device's, and -`read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. -`add_config_signals` becomes a deprecated wrapper. - -The detector does not *reimplement* any of those verbs to achieve it. `StandardReadable` -grows two hooks, `_extra_funcs_for(verb)` and `_extra_hint_sources()`, that contribute -from somewhere other than the registry, and `StandardDetector` overrides those to add what -its data logics produce. Likewise it registers its stage and unstage work into -`_stage_funcs`/`_unstage_funcs`, the way `StandardFlyable` already does, rather than -replacing `stage()`. Overriding the verbs meant re-deriving the registered children's -contributions by hand in each one, which is exactly what would drift. - -One consequence: a detector with no hinted fields now reports `hints` as `{}` rather than -`{"fields": []}`, because it no longer special cases what `StandardReadable` already does. - -Plugins are **not** registered automatically. `AreaDetector` registers only the driver's -`acquire_time` and `acquire_period`, plus whatever the caller passes as `config_sigs`. -Registering a plugin is a `set_readable_format()` call at the call site, which is the -explicit opt-in the #1395 reviewers asked for, now expressible per plugin and reversible -at runtime. - ### Formats are stored in the same file as settings, under a reserved key Values and formats are applied on the same cadence — a technique change wants both — so diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index aa11759fc6..941e644960 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -56,8 +56,8 @@ for dev in walk_devices(detector).values(): ``` ```{note} -The baseline is the *class declaration*, not the last file you loaded, so this -also discards a technique applied with [](#apply_settings). +The baseline is what the Device class declares, so this also discards formats that came +from a stored settings file — see [](./store-and-retrieve.md). ``` ```{note} @@ -67,20 +67,6 @@ so a format changed part way through a run would leave `describe()` and `read()` disagreeing. ``` -## Opting a detector plugin in - -A [](#StandardDetector) is a [](#StandardReadable), so the same call works on one. This -matters most for areaDetector, where a detector often carries many more plugins than are -wired into its chain — so plugins are never registered automatically, and you opt in to -the ones that are actually producing meaningful values: - -```python -det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) -``` - -Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can -write a file *and* report a plugin scalar in a step scan. - ```{seealso} [](./store-and-retrieve.md) for saving a set of formats alongside signal values and loading them back, and diff --git a/docs/how-to/store-and-retrieve.md b/docs/how-to/store-and-retrieve.md index f138fb071b..a10a890734 100644 --- a/docs/how-to/store-and-retrieve.md +++ b/docs/how-to/store-and-retrieve.md @@ -61,7 +61,7 @@ temperature: 20.0 stats.total: HINTED_UNCACHED_SIGNAL ``` -The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with [](#ROOT_DEVICE_KEY) standing for the Device you stored. It is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. +The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with `` ([](#ROOT_DEVICE_KEY)) standing for the Device you stored. It is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. ```{note} Applying **merges** into what is already registered: a child the file does not mention keeps whatever format it has. This matches how values behave — applying a stored file leaves signals it has never heard of alone — so a file stored against an older version of a Device does not silently unregister signals that have been added since. diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 76471fc43d..7d85cd22bc 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -476,10 +476,11 @@ def hints(self) -> Hints: #: unlike `*FORMATS*` it needs no quoting in yaml. READABLE_FORMATS_KEY = "" -#: The path standing in for the root Device itself in a [](#ReadableFormats). -#: Not a valid Python identifier, so no attribute assignment can produce a -#: colliding path, and unlike `""` it does not make yaml fall back to its -#: hard-to-read explicit key syntax. +#: The key standing in for the root Device itself in a [](#ReadableFormats), +#: where every other key is a dotted attribute path. Not a valid Python +#: identifier, so no attribute assignment can produce a colliding path, and +#: unlike `""` it does not make yaml fall back to its hard-to-read explicit +#: key syntax. ROOT_DEVICE_KEY = "" #: The readable formats of a Device tree in a form that can be stored and From bdf215f145ba7cc3937067d13899661c55258690 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 25 Aug 2026 20:36:55 +0000 Subject: [PATCH 15/67] Make StandardDetector a StandardReadable So that set_readable_format() works on a detector at all. read(), describe() and hints merge the registered children on top of whatever the data logics produce, and add_config_signals becomes a deprecated wrapper. The detector overrides _extra_funcs_for()/_extra_hint_sources() rather than reimplementing the verbs, so a detector with no hinted fields now reports hints as {} rather than {"fields": []}, matching StandardReadable. Split out of the runtime-readable-format work so that PR stays core-only. Co-Authored-By: Claude Opus 5 --- .../decisions/0021-runtime-readable-format.md | 23 +++- src/ophyd_async/core/_detector.py | 118 +++++++++++------- src/ophyd_async/epics/adcore/_detector.py | 14 ++- src/ophyd_async/sim/_blob_detector.py | 4 +- .../unit_tests/core/test_standard_detector.py | 32 ++++- .../unit_tests/fastcs/panda/test_hdf_panda.py | 4 +- 6 files changed, 142 insertions(+), 53 deletions(-) diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index df33bc7942..f545913485 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -57,11 +57,24 @@ a driver, a plugin list and a path provider. ## Decision -### The registry holds `(device, format)` pairs - -`StandardReadable._readables` is a tuple of pairs in registration order, and the callables -for each verb are derived on demand. `add_readables` becomes a thin wrapper over a new -`set_readable_format(device, format)`, and `format=None` removes the device. +### The registry is a `{device: format}` dict + +`StandardReadable._readables` maps each registered child to its format, in registration +order, and the callables for each verb are derived on demand. `add_readables` becomes a +thin wrapper over a new `set_readable_format(device, format)`, and `format=None` removes +the device. `get_readable_formats()` returns a copy. + +Keying by `Device` is identity keying, which is what a registry of children needs: `Device` +does not override `__eq__`, and while `DeviceVector` and `DeviceMap` inherit a value based +`__eq__` from `Mapping`, they hash by `id()` — injective over live objects, so two distinct +devices never share a hash and `__eq__` is never reached. + +A tuple of `(device, format)` pairs was tried first, for immutability. It made +`set_readable_format` rebuild and rescan the whole tuple, so registering *n* children was +O(n²) — 2000 signals cost 77 ms at construction, against 13 ms for the dict. The registry +is a `@cached_property`, matching `Device._child_devices`, because `AreaDetector` and +`add_children_as_readables` register children *before* calling `super().__init__()`, so +anything assigned in `__init__` would discard them. Deriving on demand also fixes staging: `stage()` walks the current registry rather than a tuple captured at registration, so a format changed between runs takes effect on the next diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index e56308cfa2..0d23b2aedc 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -4,8 +4,9 @@ import functools import os import time +import warnings from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence from dataclasses import dataclass from enum import Enum from functools import cached_property @@ -15,9 +16,7 @@ Collectable, Flyable, HasHints, - Hints, Preparable, - Reading, Stageable, StreamAsset, Triggerable, @@ -29,8 +28,12 @@ from ophyd_async.core._log import logger from ._data_providers import ReadableDataProvider, StreamableDataProvider -from ._device import Device -from ._protocol import AsyncConfigurable, AsyncReadable +from ._readable import ( + StandardReadable, + StandardReadableFormat, + _HintedFields, + _Verb, +) from ._settings import Settings from ._signal import ( SignalDict, @@ -353,28 +356,34 @@ class _KickoffCtx: class StandardDetector( - Device, + StandardReadable, Stageable, - AsyncConfigurable, - AsyncReadable, Triggerable, Preparable, Flyable, Collectable, WritesStreamAssets, - HasHints, ): """Detector base class for step and fly scanning detectors. Aggregates trigger, arm, reading or stream logic together. + + Signals read during a step scan are registered with + [](#StandardReadable.set_readable_format), exactly as on any other + `StandardReadable`; data produced by a `DetectorDataLogic` is added on top + of those in `read()` and `describe()`. + + `read()` and `describe()` require `prepare()` to have run, and raise + otherwise, so that a descriptor can never be emitted without the detector's + data keys. `trigger()` prepares implicitly, so a step scan never has to do + it explicitly. `read_configuration()` and `describe_configuration()` have no + such requirement. """ # Logic for the detector _trigger_logic: DetectorTriggerLogic | None = None _acquire_logic: DetectorAcquireLogic | None = None _data_logics: Sequence[DetectorDataLogic] = () - # Signals to include in read_configuration - _config_signals: Sequence[SignalR] = () # Context produced by prepare, used by trigger and kickoff _prepare_ctx: _PrepareCtx | None = None # Context produced by kickoff, used by complete @@ -382,6 +391,13 @@ class StandardDetector( # The triggers that are supported by the trigger logic _supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Contribute to _StandardBase's fan out rather than replacing stage(), + # so the readable children registered on this detector still get staged + self._stage_funcs += (self._stage_detector,) + self._unstage_funcs += (self._unstage_detector,) + # Report the number of events for the next kickoff @cached_property def events_to_kickoff(self) -> SignalRW[int]: @@ -410,7 +426,8 @@ def add_detector_logics( # Store the triggers that are supported self._supported_triggers = _get_supported_triggers(logic) # Add the config signals it needs - self.add_config_signals(*logic.config_sigs()) + for sig in logic.config_sigs(): + self.set_readable_format(sig, StandardReadableFormat.CONFIG_SIGNAL) elif isinstance(logic, DetectorAcquireLogic): if self._acquire_logic is not None: raise RuntimeError("Detector already has acquire logic") @@ -420,12 +437,21 @@ def add_detector_logics( else: raise TypeError(f"Unknown logic type: {type(logic)}") + # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: """Add a signal to read_configuration(). :param sig: The signal to add """ - self._config_signals = (*self._config_signals, *signals) + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(signal, StandardReadableFormat" + ".CONFIG_SIGNAL)` instead of `add_config_signals(signal)`" + ), + stacklevel=2, + ) + for signal in signals: + self.set_readable_format(signal, StandardReadableFormat.CONFIG_SIGNAL) async def get_trigger_deadtime( self, settings: Settings | None = None @@ -453,9 +479,9 @@ async def get_trigger_deadtime( return self._supported_triggers, deadtime @AsyncStatus.wrap - async def stage(self) -> None: + async def _stage_detector(self) -> None: """Make sure the detector is idle and ready to be used.""" - coros = [data_logic.stop() for data_logic in self._data_logics] + coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] if self._acquire_logic: coros.append(self._acquire_logic.ensure_ready()) await asyncio.gather(*coros) @@ -617,7 +643,7 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: """Trigger a single exposure. If [`prepare()`](#StandardDetector.prepare) has not been called since - the last [`stage()`](#StandardDetector.stage), an implicit prepare is + the last `stage()`, an implicit prepare is performed. When [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` [](#DetectorTriggerLogic.default_trigger_info) is called to read current hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is @@ -721,22 +747,37 @@ async def complete(self): ): yield update - async def describe_configuration(self) -> dict[str, DataKey]: - return await merge_gathered_dicts( - sig.describe() for sig in self._config_signals - ) + def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: + """Contribute what the data logics produce to read() and describe(). + + StandardReadable gathers these alongside the registered children, so + the verb methods themselves need no overriding. - async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts(sig.read() for sig in self._config_signals) + Raising when nothing has been prepared is deliberate, and predates + registered children being readable at all: a detector that has not been + prepared has no data keys, so a `describe()` that quietly succeeded + would emit a descriptor missing the detector's data. In practice + `trigger()` prepares implicitly, so `trigger_and_read` never reaches + this. `read_configuration()` and `describe_configuration()` are + unaffected, as this hook only feeds the data verbs. - async def describe(self) -> dict[str, DataKey]: + The cost is that the *registered children* are unreachable through + `read()`/`describe()` until then too, since this raises before the + registry is consulted. That is accepted rather than worked around: + reading an unprepared detector is already defined as an error. + """ + if verb not in (_Verb.DESCRIBE, _Verb.READ): + return ctx = error_if_none(self._prepare_ctx, "Prepare not run") - # Readable and Streamable data providers produce data during read - coros = [dp.make_datakeys() for dp in ctx.readable_data_providers] + [ - dp.make_datakeys(ctx.trigger_info.collections_per_event) - for dp in ctx.streamable_data_providers - ] - return await merge_gathered_dicts(coros) + for dp in ctx.readable_data_providers: + yield dp.make_datakeys if verb is _Verb.DESCRIBE else dp.make_readings + if verb is _Verb.DESCRIBE: + # Streamable providers describe their shape for a step scan, but + # produce their data through collect_asset_docs rather than read() + for sdp in ctx.streamable_data_providers: + yield functools.partial( + sdp.make_datakeys, ctx.trigger_info.collections_per_event + ) async def describe_collect(self) -> dict[str, DataKey]: ctx = error_if_none(self._prepare_ctx, "Prepare not run") @@ -747,18 +788,11 @@ async def describe_collect(self) -> dict[str, DataKey]: ] return await merge_gathered_dicts(coros) - @property - def hints(self) -> Hints: - fields: list[str] = [] + def _extra_hint_sources(self) -> Iterator[HasHints]: + """Contribute the data logics' hinted fields alongside the children's.""" for dl in self._data_logics: - fields.extend(dl.get_hinted_fields(self.name + dl.datakey_suffix)) - return Hints(fields=fields) - - async def read(self) -> dict[str, Reading]: - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - return await merge_gathered_dicts( - dp.make_readings() for dp in ctx.readable_data_providers - ) + if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): + yield _HintedFields(fields) async def collect_asset_docs( self, index: int | None = None @@ -784,9 +818,9 @@ async def get_index(self) -> int: return min_collections_written // ctx.trigger_info.collections_per_event @AsyncStatus.wrap - async def unstage(self) -> None: + async def _unstage_detector(self) -> None: """Stop the detector and file writing.""" - coros = [data_logic.stop() for data_logic in self._data_logics] + coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] if self._acquire_logic: coros.append(self._acquire_logic.ensure_stopped()) await asyncio.gather(*coros) diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index f624db83d4..372ae0f8b4 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -7,6 +7,7 @@ SignalR, StandardDetector, ) +from ophyd_async.core import StandardReadableFormat as Format from ._acquire_logic import ADContAcqAcquireLogic from ._data_logic import ADWriterFactory @@ -48,9 +49,16 @@ def __init__( writer, data_logic = factory(prefix, driver, plugin_list) setattr(self, factory.writer_name, writer) self.add_detector_logics(data_logic) - self.add_config_signals( - self.driver.acquire_period, self.driver.acquire_time, *config_sigs - ) + # Register the driver's timing signals, plus anything the caller asked + # for, as configuration. Plugins are deliberately not registered here: + # a detector normally carries far more plugins than are wired into the + # chain, so opt in per plugin with set_readable_format() instead. + for signal in ( + self.driver.acquire_period, + self.driver.acquire_time, + *config_sigs, + ): + self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) def get_plugin( diff --git a/src/ophyd_async/sim/_blob_detector.py b/src/ophyd_async/sim/_blob_detector.py index ed21968784..6760e088a1 100644 --- a/src/ophyd_async/sim/_blob_detector.py +++ b/src/ophyd_async/sim/_blob_detector.py @@ -1,6 +1,7 @@ from collections.abc import Sequence from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import StandardReadableFormat as Format from ._blob_acquire_logic import BlobAcquireLogic from ._blob_data_logic import BlobDataLogic @@ -26,5 +27,6 @@ def __init__( path_provider=path_provider, pattern_generator=self.pattern_generator ), ) - self.add_config_signals(*config_sigs) + for signal in config_sigs: + self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 02e8be2ac2..77074d9035 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -25,6 +25,7 @@ soft_signal_r_and_setter, soft_signal_rw, ) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.testing import ( assert_configuration, assert_reading, @@ -657,7 +658,7 @@ async def test_config_signals_in_describe_configuration( signal = soft_signal_rw( signal_type, initial_value=initial_value, name="test-config" ) - det.add_config_signals(signal) + det.set_readable_format(signal, Format.CONFIG_SIGNAL) await det.stage() @@ -1082,3 +1083,32 @@ async def test_data_logic_not_implemented_errors(): # stop() should not raise (has default implementation) await logic.stop() # Should pass + + +async def test_detector_readable_format_changes_at_runtime(): + # A StandardDetector is a StandardReadable, so a plugin signal can be moved + # between configuration and hinted reads without redefining the Device. + det = StandardDetector(name="det") + det.temperature, _ = soft_signal_r_and_setter(float, 20.0, name="temperature") + + name = det.temperature.name + det.set_readable_format(det.temperature, Format.CONFIG_SIGNAL) + assert set(await det.read_configuration()) == {name} + # Empty hints are {} rather than {"fields": []}, as for any StandardReadable + assert det.hints == {} + + det.set_readable_format(det.temperature, Format.HINTED_UNCACHED_SIGNAL) + assert set(await det.read_configuration()) == set() + assert det.hints == {"fields": [name]} + + det.set_readable_format(det.temperature, None) + assert set(await det.read_configuration()) == set() + assert det.hints == {} + + +async def test_detector_add_config_signals_is_deprecated(): + det = StandardDetector(name="det") + signal, _ = soft_signal_r_and_setter(float, 1.0, name="sig") + with pytest.deprecated_call(): + det.add_config_signals(signal) + assert set(await det.read_configuration()) == {signal.name} diff --git a/tests/unit_tests/fastcs/panda/test_hdf_panda.py b/tests/unit_tests/fastcs/panda/test_hdf_panda.py index 7b9e87b4e0..19a147a312 100644 --- a/tests/unit_tests/fastcs/panda/test_hdf_panda.py +++ b/tests/unit_tests/fastcs/panda/test_hdf_panda.py @@ -187,7 +187,9 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): "source": uri, }, } - assert hdf_panda.hints == {"fields": []} + # No hinted fields is {} rather than {"fields": []}, as for any + # StandardReadable -- the detector no longer special cases this + assert hdf_panda.hints == {} # Kick it off and start it completing await hdf_panda.kickoff() status = hdf_panda.complete() From d0374530312b69c9ccaeea0095dbba1d8afa1a60 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 11:49:27 -0400 Subject: [PATCH 16/67] Make standard detector extend from standard readable. Allow for pulling read/config/describe from children automatically (cherry picked from commit 32193035d7412c81bf42b930a2e275f6f7781df3) --- src/ophyd_async/core/_detector.py | 13 ++-- .../unit_tests/core/test_standard_detector.py | 59 ++++++++++++++++++- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 0d23b2aedc..a75f228234 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -136,18 +136,18 @@ class DetectorTriggerLogic: Subclasses must implement the appropriate `prepare_*` method for any trigger mode the detector supports, `get_deadtime` if it supports external - triggering, and `config_sigs` if the deadtime would vary according to + triggering, and `deadtime_sigs` if the deadtime would vary according to detector parameters. """ - def config_sigs(self) -> set[SignalR]: - """Return the signals that should appear in read_configuration.""" + def deadtime_sigs(self) -> set[SignalR]: + """Return the signals whose values are needed to compute deadtime.""" return set() def get_deadtime(self, config_values: SignalDict) -> float: """Return the deadtime in seconds for the detector. - :param config_values: the value of each signal in `config_sigs` + :param config_values: the value of each signal in `deadtime_sigs` """ raise NotImplementedError(self) @@ -425,9 +425,6 @@ def add_detector_logics( self._trigger_logic = logic # Store the triggers that are supported self._supported_triggers = _get_supported_triggers(logic) - # Add the config signals it needs - for sig in logic.config_sigs(): - self.set_readable_format(sig, StandardReadableFormat.CONFIG_SIGNAL) elif isinstance(logic, DetectorAcquireLogic): if self._acquire_logic is not None: raise RuntimeError("Detector already has acquire logic") @@ -465,7 +462,7 @@ async def get_trigger_deadtime( self._trigger_logic.get_deadtime ): config_values = SignalDict() - for sig in self._trigger_logic.config_sigs(): + for sig in self._trigger_logic.deadtime_sigs(): if settings and sig in settings: # Use value from settings if it is in there # cast to a SignalRW because settings can only contain those diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 77074d9035..38e4fd9355 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -18,6 +18,7 @@ SignalDict, SignalR, StandardDetector, + StandardReadable, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -98,8 +99,7 @@ class DeadtimeTriggerLogic(DetectorTriggerLogic): def __init__(self, deadtime_signal: SignalR[float]): self.deadtime_signal = deadtime_signal - def config_sigs(self) -> set[SignalR]: - """Return the deadtime signal as a config signal.""" + def deadtime_sigs(self) -> set[SignalR]: return {self.deadtime_signal} def get_deadtime(self, config_values: SignalDict) -> float: @@ -1051,6 +1051,61 @@ async def test_collect_asset_docs_with_explicit_index(tmp_path): ] +async def test_child_readable_config_signals_in_describe_configuration(): + """Child StandardReadable CONFIG_SIGNALs appear in describe_configuration.""" + child = StandardReadable(name="child") + config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") + child.add_readables([config_sig], Format.CONFIG_SIGNAL) + + det = StandardDetector(name="det") + det.add_readables([child], Format.CHILD) + det.add_detector_logics(ReadableOnlyDataLogic()) + await det.prepare(TriggerInfo()) + + config = await det.describe_configuration() + assert "child-exposure" in config + reading = await det.read_configuration() + assert "child-exposure" in reading + assert reading["child-exposure"]["value"] == 1.5 + + +async def test_child_readable_read_signals_in_read(): + """Child StandardReadable HINTED_SIGNALs appear in read/describe.""" + child = StandardReadable(name="child") + read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") + child.add_readables([read_sig], Format.HINTED_SIGNAL) + + det = StandardDetector(name="det") + det.add_readables([child], Format.CHILD) + det.add_detector_logics(ReadableOnlyDataLogic()) + await det.prepare(TriggerInfo()) + + desc = await det.describe() + assert "child-counts" in desc + assert "foo-value" in desc + + reading = await det.read() + assert "child-counts" in reading + assert reading["child-counts"]["value"] == 99 + # Data provider signal is also present + assert "foo-value" in reading + + +async def test_child_readable_hints_merged(): + """Child StandardReadable hints are merged with data logic hints.""" + child = StandardReadable(name="child") + hinted_sig = soft_signal_rw(float, name="child-intensity") + child.add_readables([hinted_sig], Format.HINTED_SIGNAL) + + det = StandardDetector(name="det") + det.add_readables([child], Format.CHILD) + det.add_detector_logics(ReadableOnlyDataLogic()) + + assert "fields" in det.hints + assert "child-intensity" in det.hints["fields"] + assert "foo-value" in det.hints["fields"] + + async def test_trigger_logic_not_implemented_errors(): """Test NotImplementedError for unimplemented DetectorTriggerLogic methods.""" logic = DetectorTriggerLogic() From 0566e9b476ee771e60fafcf0a75f761eb2d1d194 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 11:50:14 -0400 Subject: [PATCH 17/67] Update EPICS areaDetector classes to use standard readable child components for config/read (cherry picked from commit 64f4cd32b07ae6a7d6ee23e4c494ac56f8d812f8) --- src/ophyd_async/epics/adcore/__init__.py | 2 - src/ophyd_async/epics/adcore/_data_logic.py | 17 --- src/ophyd_async/epics/adcore/_detector.py | 20 ++-- src/ophyd_async/epics/adcore/_io.py | 110 +++++++++++++----- .../epics/adcore/test_data_logic.py | 28 ++--- .../unit_tests/epics/adcore/test_use_cases.py | 108 ++++++++--------- 6 files changed, 156 insertions(+), 129 deletions(-) diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index bf8a872021..960ca33146 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -9,7 +9,6 @@ ADMultipartDataLogic, ADWriterFactory, NDArrayDescription, - PluginSignalDataLogic, ) from ._detector import AreaDetector, ContAcqDetector from ._io import ( @@ -75,7 +74,6 @@ "ADContAcqAcquireLogic", # DataLogic "NDArrayDescription", - "PluginSignalDataLogic", "ADHDFDataLogic", "ADMultipartDataLogic", "ADWriterFactory", diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 9beb01237d..2542ed1838 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -12,7 +12,6 @@ EnableDisable, PathInfo, PathProvider, - SignalDataProvider, SignalR, StreamableDataProvider, StreamResourceDataProvider, @@ -35,22 +34,6 @@ from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType -@dataclass -class PluginSignalDataLogic(DetectorDataLogic): - driver: ADBaseIO - signal: SignalR - hinted: bool = True - - async def prepare_single(self, datakey_name: str) -> SignalDataProvider: - # Need to wait for all the plugins to have finished before we can read - # the plugin signal - await self.driver.wait_for_plugins.set(True) - return SignalDataProvider(self.signal) - - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return [self.signal.name] if self.hinted else [] - - @dataclass class NDArrayDescription: """Signals that describe the shape and data type of an NDArray frame. diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index 372ae0f8b4..ffd22ea6c1 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -49,15 +49,13 @@ def __init__( writer, data_logic = factory(prefix, driver, plugin_list) setattr(self, factory.writer_name, writer) self.add_detector_logics(data_logic) - # Register the driver's timing signals, plus anything the caller asked - # for, as configuration. Plugins are deliberately not registered here: - # a detector normally carries far more plugins than are wired into the - # chain, so opt in per plugin with set_readable_format() instead. - for signal in ( - self.driver.acquire_period, - self.driver.acquire_time, - *config_sigs, - ): + # The driver is always wired up, so register it whole and let ADBaseIO + # declare which of its signals are configuration. Plugins are + # deliberately *not* registered: a detector normally carries far more + # plugins than are wired into the chain, so opt in per plugin with + # `set_readable_format(det.stats, Format.CHILD)` instead. + self.set_readable_format(self.driver, Format.CHILD) + for signal in config_sigs: self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) @@ -76,7 +74,7 @@ def get_plugin( class ContAcqDetector(AreaDetector[ADBaseIO]): - """Create an ADSimDetector AreaDetector instance. + """Create an continuously acquiring AreaDetector instance. :param prefix: EPICS PV prefix for the detector :param writer_factories: Factories for file writer plugins and their data logics @@ -104,7 +102,7 @@ def __init__( *writer_factories, acquire_logic=ADContAcqAcquireLogic(driver, cb_plugin), trigger_logic=ADContAcqTriggerLogic(driver, cb_plugin), - plugins=(plugins or {}) | {"cb": cb_plugin}, + plugins=(plugins or {}) | {cb_plugin.name: cb_plugin}, config_sigs=config_sigs, name=name, ) diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index b48a0e6fef..3a8ee0bcc6 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -6,11 +6,13 @@ EnableDisable, SignalR, SignalRW, + StandardReadable, StrictEnum, SubsetEnum, SupersetEnum, non_zero, ) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.epics.core import EpicsDevice, EpicsOptions, PvSuffix # Common classes for drivers and plugins @@ -107,7 +109,7 @@ class ADState(StrictEnum): ADBaseIOT = TypeVar("ADBaseIOT", bound="ADBaseIO") -class ADBaseIO(NDArrayBaseIO): +class ADBaseIO(StandardReadable, NDArrayBaseIO): """Base class from which areaDetector drivers are derived. This mirrors the interface provided by ADCore/db/ADBase.template. @@ -118,10 +120,14 @@ class ADBaseIO(NDArrayBaseIO): compatibility with changes made in https://github.com/areaDetector/ADCore/blob/master/RELEASE.md#asynndarraydriver-addriver """ - acquire_time: A[SignalRW[float], PvSuffix.rbv("AcquireTime")] - acquire_period: A[SignalRW[float], PvSuffix.rbv("AcquirePeriod")] - num_images: A[SignalRW[int], PvSuffix.rbv("NumImages")] - image_mode: A[SignalRW[ADImageMode], PvSuffix.rbv("ImageMode")] + acquire_time: A[SignalRW[float], PvSuffix.rbv("AcquireTime"), Format.CONFIG_SIGNAL] + acquire_period: A[ + SignalRW[float], PvSuffix.rbv("AcquirePeriod"), Format.CONFIG_SIGNAL + ] + num_images: A[SignalRW[int], PvSuffix.rbv("NumImages"), Format.CONFIG_SIGNAL] + image_mode: A[ + SignalRW[ADImageMode], PvSuffix.rbv("ImageMode"), Format.CONFIG_SIGNAL + ] detector_state: A[SignalR[ADState], PvSuffix("DetectorState_RBV")] # The following signals have been moved from NDArrayBaseIO for backwards @@ -131,11 +137,13 @@ class ADBaseIO(NDArrayBaseIO): # There is no _RBV for this one wait_for_plugins: A[SignalRW[bool], PvSuffix("WaitForPlugins")] - manufacturer: A[SignalR[str], PvSuffix("Manufacturer_RBV")] - model: A[SignalR[str], PvSuffix("Model_RBV")] - serial_number: A[SignalR[str], PvSuffix("SerialNumber_RBV")] - sdk_version: A[SignalR[str], PvSuffix("SDKVersion_RBV")] - firmware_version: A[SignalR[str], PvSuffix("FirmwareVersion_RBV")] + manufacturer: A[SignalR[str], PvSuffix("Manufacturer_RBV"), Format.CONFIG_SIGNAL] + model: A[SignalR[str], PvSuffix("Model_RBV"), Format.CONFIG_SIGNAL] + serial_number: A[SignalR[str], PvSuffix("SerialNumber_RBV"), Format.CONFIG_SIGNAL] + sdk_version: A[SignalR[str], PvSuffix("SDKVersion_RBV"), Format.CONFIG_SIGNAL] + firmware_version: A[ + SignalR[str], PvSuffix("FirmwareVersion_RBV"), Format.CONFIG_SIGNAL + ] # Classes for plugins @@ -156,13 +164,15 @@ class NDPluginBaseIO(NDArrayBaseIO): queue_size: A[SignalRW[int], PvSuffix.rbv("QueueSize")] -class NDROIIO(NDPluginBaseIO): +class NDROIIO(StandardReadable, NDPluginBaseIO): """Plugin for taking a region of an NDArray. This mirrors the interface provided by ADCore/db/NDROI.template. See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROI.html """ + min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] size_z: A[SignalRW[int], PvSuffix.rbv("SizeZ")] @@ -171,6 +181,12 @@ class NDROIIO(NDPluginBaseIO): bin_y: A[SignalRW[int], PvSuffix.rbv("BinY")] bin_z: A[SignalRW[int], PvSuffix.rbv("BinZ")] + def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): + super().__init__(prefix, with_pvi, name) + self.add_readables( + [self.min_x, self.min_y, self.size_x, self.size_y], Format.CONFIG_SIGNAL + ) + class NDProcessIO(NDPluginBaseIO): """Plugin for performing processing on an NDArrray. @@ -183,7 +199,7 @@ class NDProcessIO(NDPluginBaseIO): scale: A[SignalRW[float], PvSuffix.rbv("Scale")] -class NDStatsIO(NDPluginBaseIO): +class NDStatsIO(StandardReadable, NDPluginBaseIO): """Plugin for computing statistics from an image or ROI within an image. This mirrors the interface provided by ADCore/db/NDStats.template. @@ -193,10 +209,24 @@ class NDStatsIO(NDPluginBaseIO): # Basic statistics compute_statistics: A[SignalRW[bool], PvSuffix.rbv("ComputeStatistics")] bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth")] + min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] + min_x: A[SignalR[float], PvSuffix("MinX_RBV")] + min_y: A[SignalR[float], PvSuffix("MinY_RBV")] + max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] + max_x: A[SignalR[float], PvSuffix("MaxX_RBV")] + max_y: A[SignalR[float], PvSuffix("MaxY_RBV")] + mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] + sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] total: A[SignalR[float], PvSuffix("Total_RBV")] + net: A[SignalR[float], PvSuffix("Net_RBV")] # Centroid statistics compute_centroid: A[SignalRW[bool], PvSuffix.rbv("ComputeCentroid")] centroid_threshold: A[SignalRW[float], PvSuffix.rbv("CentroidThreshold")] + centroid_x: A[SignalR[float], PvSuffix("CentroidX_RBV")] + centroid_y: A[SignalR[float], PvSuffix("CentroidY_RBV")] + sigma_x: A[SignalR[float], PvSuffix("SigmaX_RBV")] + sigma_y: A[SignalR[float], PvSuffix("SigmaY_RBV")] + sigma_xy: A[SignalR[float], PvSuffix("SigmaXY_RBV")] # X and Y Profiles compute_profiles: A[SignalRW[bool], PvSuffix.rbv("ComputeProfiles")] profile_size_x: A[SignalR[int], PvSuffix("ProfileSizeX_RBV")] @@ -209,27 +239,39 @@ class NDStatsIO(NDPluginBaseIO): hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin")] hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax")] + def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): + super().__init__(prefix, with_pvi, name) + self.add_readables( + [ + self.compute_statistics, + self.bgd_width, + self.compute_centroid, + self.centroid_threshold, + self.compute_profiles, + self.cursor_x, + self.cursor_y, + self.compute_histogram, + self.hist_size, + self.hist_min, + self.hist_max, + ], + Format.CONFIG_SIGNAL, + ) + self.add_readables( + [self.total, self.mean_value, self.min_value, self.max_value], + Format.HINTED_UNCACHED_SIGNAL, + ) + self.add_readables( + [self.min_x, self.min_y, self.max_x, self.max_y], + Format.UNCACHED_SIGNAL, + ) -class NDROIStatNIO(EpicsDevice): - """Defines the parameters for a single ROI used for statistics calculation. - Each instance represents a single ROI, with attributes for its position - (min_x, min_y) and size (size_x, size_y), as well as a name and use status. +class NDROIStatNIO(StandardReadable, EpicsDevice): + """Defines the parameters for a single ROI used for statistics calculation. This mirrors the interface provided by ADCore/db/NDROIStatN.template. See definition in ADApp/pluginSrc/NDPluginROIStat.h in https://github.com/areaDetector/ADCore. - - Attributes: - name: The name of the ROI. - use: Flag indicating whether the ROI is used. - min_x: The start X-coordinate of the ROI. - min_y: The start Y-coordinate of the ROI. - size_x: The width of the ROI. - size_y: The height of the ROI. - min_value: Minimum count value in the ROI. - max_value: Maximum count value in the ROI. - mean_value: Mean counts value in the ROI. - total: Total counts in the ROI. """ name_: A[SignalRW[str], PvSuffix("Name")] @@ -238,12 +280,22 @@ class NDROIStatNIO(EpicsDevice): min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] - # stats min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] total: A[SignalR[float], PvSuffix("Total_RBV")] + def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): + super().__init__(prefix, with_pvi, name) + self.add_readables( + [self.use, self.min_x, self.min_y, self.size_x, self.size_y], + Format.CONFIG_SIGNAL, + ) + self.add_readables( + [self.min_value, self.max_value, self.mean_value, self.total], + Format.HINTED_UNCACHED_SIGNAL, + ) + class NDROIStatIO(NDPluginBaseIO): """Plugin for calculating basic statistics for multiple ROIs. diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 4696f9ee98..9e940744b3 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -282,14 +282,12 @@ async def test_describe_different_color_modes( # Expected to give the right shape in the descriptor await hdf_det.prepare(TriggerInfo()) describe = await hdf_det.describe() - assert describe == { - "detector": { - "dtype": "array", - "dtype_numpy": " Date: Wed, 12 Aug 2026 11:50:39 -0400 Subject: [PATCH 18/67] Update implementations to move config_sigs -> deadtime_sigs for trigger logics (cherry picked from commit 404930bbdaa8648f63cdb7224ac120b6e85fa4cc) --- src/ophyd_async/epics/adaravis.py | 2 +- src/ophyd_async/epics/advimba.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index dcd360363a..7cdfb08d54 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -59,7 +59,7 @@ class AravisTriggerLogic(DetectorTriggerLogic): driver: AravisDriverIO override_deadtime: float | None = None - def config_sigs(self) -> set[SignalR]: + def deadtime_sigs(self) -> set[SignalR]: return {self.driver.model} def get_deadtime(self, config_values: SignalDict) -> float: diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index b44566b852..ba96a3564c 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -96,7 +96,7 @@ class VimbaTriggerLogic(DetectorTriggerLogic): driver: VimbaDriverIO override_deadtime: float | None = None - def config_sigs(self) -> set[SignalR]: + def deadtime_sigs(self) -> set[SignalR]: return {self.driver.model} def get_deadtime(self, config_values: SignalDict) -> float: From 09be57bf2eee265bb336842c29f2b755c4454f93 Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 12:11:15 -0400 Subject: [PATCH 19/67] Fix failing test (cherry picked from commit c80dc3620bc3838c81b07aad1bc0cc4e9eb6157d) --- src/ophyd_async/epics/adcore/_detector.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index ffd22ea6c1..c445904147 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -88,14 +88,15 @@ def __init__( self, prefix: str, *writer_factories: ADWriterFactory, - driver_suffix="cam1:", - cb_suffix="CB1:", + driver_suffix: str = "cam1:", + cb_suffix: str = "CB1:", + cb_plugin_name: str = "cb", plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), name: str = "", ) -> None: driver = ADBaseIO(prefix + driver_suffix) - cb_plugin = NDCircularBuffIO(prefix + cb_suffix) + cb_plugin = NDCircularBuffIO(prefix + cb_suffix, name=cb_plugin_name) super().__init__( driver, prefix, From ea13c0cb9b8685d54533a8831f1b3e20c2dfb8dd Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 12:18:11 -0400 Subject: [PATCH 20/67] Fix container test (cherry picked from commit dd36b7a64825d848f8ccf7792c3ba9d3c486399b) --- tests/container_tests/test_adsim_system.py | 58 ++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index 26fc83f3b7..084ad1b55a 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -196,10 +196,24 @@ def test_software_triggering( "data": { "adsim-driver-acquire_period": 0.005, "adsim-driver-acquire_time": 0.1, + "adsim-driver-num_images": 1, + "adsim-driver-image_mode": "Multiple", + "adsim-driver-manufacturer": "Simulated detector", + "adsim-driver-model": "Basic simulator", + "adsim-driver-serial_number": "No serial number", + "adsim-driver-sdk_version": ANY, + "adsim-driver-firmware_version": "No firmware", }, "timestamps": { "adsim-driver-acquire_period": ANY, "adsim-driver-acquire_time": ANY, + "adsim-driver-num_images": ANY, + "adsim-driver-image_mode": ANY, + "adsim-driver-manufacturer": ANY, + "adsim-driver-model": ANY, + "adsim-driver-serial_number": ANY, + "adsim-driver-sdk_version": ANY, + "adsim-driver-firmware_version": ANY, }, "data_keys": { "adsim-driver-acquire_period": { @@ -218,6 +232,50 @@ def test_software_triggering( "units": "", "precision": 3, }, + "adsim-driver-num_images": { + "dtype": "integer", + "shape": [], + "dtype_numpy": " Date: Wed, 19 Aug 2026 12:31:29 +0000 Subject: [PATCH 21/67] Opt plugins into read()/describe() explicitly in the AD use case tests #1395 registered every StandardReadable plugin as a CHILD readable in AreaDetector.__init__. That is dropped in the rebase onto #1400, for the reason @jwlodek gave on the PR himself: "what if we didn't add plugins as readables automatically when passed in at __init__, and instead explicitly required an add_readables call to include them?" and the reason @coretl gave for asking: a detector typically carries far more plugins (16 ROIs, 16 stats) than are wired into its chain, so reporting all of them gives no way to tell which values are meaningful. So the three tests that expected stats readings now say so, with set_readable_format(stat, Format.CHILD). NDStatsIO declares its own config and hinted signals, so opting the plugin in as a CHILD picks up exactly what these tests already assert. The wait_for_plugins assertions go with the AreaDetector override that set it, which both reviewers agreed was too hidden and belongs in an explicit data logic instead. Co-Authored-By: Claude Opus 5 (cherry picked from commit f56c52368a961fdb1053e888811d44a33190782d) --- tests/unit_tests/epics/adcore/test_use_cases.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 1e6496af3c..043b45564e 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -2,6 +2,7 @@ import pytest +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.core import ( StaticPathProvider, TriggerInfo, @@ -24,6 +25,8 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( plugins={"stats": stat}, ) temp = epics_signal_r(float, "SAMP:TEMP:RBV") + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) ndattributes = [ adcore.NDAttributeParam( name="det-sum", @@ -46,7 +49,6 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( set_mock_value(det.driver.array_size_y, 768) await det.stage() - assert await det.driver.wait_for_plugins.get_value() is False config_description = await det.describe_configuration() assert { "det-driver-acquire_period": { @@ -537,11 +539,11 @@ async def test_simdetector_with_stats_signal(): stat = adcore.NDStatsIO("PREFIX:STAT:") async with init_devices(mock=True): det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}) + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) set_mock_value(stat.total, 1.8) await det.stage() - assert await det.driver.wait_for_plugins.get_value() is False await det.trigger() - assert await det.driver.wait_for_plugins.get_value() is True description = await det.describe() assert "det-stats-total" in description assert description["det-stats-total"] == { @@ -568,6 +570,8 @@ async def test_step_scan_keep_numimages( adcore.ADWriterFactory.hdf(static_path_provider), plugins={"stats": stat}, ) + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) set_mock_value(det.driver.acquire_period, 0.1) set_mock_value(det.driver.acquire_time, 0.05) From 355a91872dd967f786c56dd61161d2efe2801a86 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 13:46:55 +0000 Subject: [PATCH 22/67] Set WaitForPlugins from ADAcquireLogic rather than hiding it in AreaDetector #1395 set driver.wait_for_plugins from an AreaDetector _update_prepare_context override, conditional on number_of_events == 1. Both reviewers called that too hidden, and the condition was unnecessary: WaitForPlugins gates the driver's AcquireBusy going to 0 when acquisition *completes*, so it never throttles frames. A flyscan does exactly one wait_for_idle, at the end, which is precisely when the plugins should have drained. It belongs on the acquire logic because that is what owns "poke this, then check acquire is done": wait_for_idle already blocks on the acquire put callback, which is the AcquireBusy that WaitForPlugins gates, so the existing wait needs no change at all. Defaults to on, since racing is silent and the wait costs nothing when a file writer is the slowest thing in the chain anyway. Off is a constructor argument rather than a subclass, matching how driver_armed_signal already parametrises this class; a detector needing genuinely custom arm/idle logic still subclasses instead. Set from ensure_ready, not start_acquiring: the PV is latched and has no _RBV, so it is written once per scan while the driver is known idle rather than on the latency path before every arm. Stop first, then always write it -- never left at whatever the IOC booted with, so a scan cannot depend on who touched the PV last. ADContAcqAcquireLogic deliberately gets no equivalent: the driver is always acquiring there, so there is no AcquireBusy transition to gate. Said so on the class so it does not look like an oversight. Co-Authored-By: Claude Opus 5 (cherry picked from commit fe787ce0cbf7ce97584c429265e39ea4b4f84231) --- .../epics/adcore/_acquire_logic.py | 41 ++++++++++++++++++- .../unit_tests/epics/adcore/test_use_cases.py | 21 ++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/ophyd_async/epics/adcore/_acquire_logic.py b/src/ophyd_async/epics/adcore/_acquire_logic.py index a5c0aaeabd..3ad36fd3d4 100644 --- a/src/ophyd_async/epics/adcore/_acquire_logic.py +++ b/src/ophyd_async/epics/adcore/_acquire_logic.py @@ -12,16 +12,46 @@ class ADAcquireLogic(DetectorAcquireLogic): + """Arm and disarm an areaDetector driver. + + :param driver: The areaDetector driver to arm. + :param driver_armed_signal: + The signal to watch for the driver having armed, if it is not `acquire`. + :param wait_for_plugins: + Whether the detector should wait for its plugins to drain before it + reports acquisition as complete. `WaitForPlugins` makes the driver's + `AcquireBusy` wait for the number of queued arrays to reach zero, and + `wait_for_idle` blocks on the `acquire` put callback, so turning it on + is what makes a step scan read plugin scalars for the frame it just + took rather than the one before. Defaults to on, since racing is + silent and the wait costs nothing when a file writer is the slowest + thing in the chain anyway. Turn it off for a detector whose plugin + chain must not gate acquisition. + """ + def __init__( - self, driver: ADBaseIO, driver_armed_signal: SignalR[bool] | None = None + self, + driver: ADBaseIO, + driver_armed_signal: SignalR[bool] | None = None, + wait_for_plugins: bool = True, ): self.driver = driver if driver_armed_signal is not None: self.driver_armed_signal = driver_armed_signal else: self.driver_armed_signal = driver.acquire + self.wait_for_plugins = wait_for_plugins self.acquire_status: AsyncStatus | None = None + async def ensure_ready(self): + # Stop first: WaitForPlugins is latched and has no _RBV, so set it once + # per scan while the driver is known to be idle rather than on the + # latency path before every arm. Always written, never left at whatever + # the IOC happened to boot with, so a scan does not depend on who + # touched the PV last. + await self.ensure_stopped() + await self.driver.wait_for_plugins.set(self.wait_for_plugins) + async def start_acquiring(self): self.acquire_status = await set_and_wait_for_other_value( set_signal=self.driver.acquire, @@ -46,6 +76,15 @@ async def ensure_stopped(self): class ADContAcqAcquireLogic(DetectorAcquireLogic): + """Start and stop capture on a circular buffer, leaving the driver acquiring. + + Deliberately has no `wait_for_plugins` equivalent to [](#ADAcquireLogic). + That setting gates the driver's `AcquireBusy` going to 0 at the end of an + acquisition, and here the driver is always acquiring -- there is no such + transition to wait on. `wait_for_idle` waits on the circular buffer's + capture instead. + """ + def __init__(self, driver: ADBaseIO, cb_plugin: NDCircularBuffIO): self.driver = driver self.cb_plugin = cb_plugin diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 043b45564e..e0c9a7c04d 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -49,6 +49,8 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( set_mock_value(det.driver.array_size_y, 768) await det.stage() + # ensure_ready sets WaitForPlugins so read() sees this frame's scalars + assert await det.driver.wait_for_plugins.get_value() is True config_description = await det.describe_configuration() assert { "det-driver-acquire_period": { @@ -543,6 +545,7 @@ async def test_simdetector_with_stats_signal(): det.set_readable_format(stat, Format.CHILD) set_mock_value(stat.total, 1.8) await det.stage() + assert await det.driver.wait_for_plugins.get_value() is True await det.trigger() description = await det.describe() assert "det-stats-total" in description @@ -558,6 +561,24 @@ async def test_simdetector_with_stats_signal(): assert "det-stats-mean_value" in det.hints["fields"] +@pytest.mark.parametrize("wait_for_plugins", [True, False]) +async def test_acquire_logic_writes_wait_for_plugins_on_stage(wait_for_plugins: bool): + # Always written, never left at whatever the IOC booted with, so a scan + # cannot depend on who touched the PV last + driver = adcore.ADBaseIO("PREFIX:DET:") + async with init_devices(mock=True): + det = adcore.AreaDetector( + driver, + acquire_logic=adcore.ADAcquireLogic( + driver, wait_for_plugins=wait_for_plugins + ), + name="det", + ) + set_mock_value(det.driver.wait_for_plugins, not wait_for_plugins) + await det.stage() + assert await det.driver.wait_for_plugins.get_value() is wait_for_plugins + + async def test_step_scan_keep_numimages( static_path_provider: StaticPathProvider, monkeypatch: pytest.MonkeyPatch, From c18fe5b9f1658959a584fa325376c2314d4618e5 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 14:20:09 +0000 Subject: [PATCH 23/67] Feed get_deadtime from the readable registry, dropping deadtime_sigs DetectorTriggerLogic no longer nominates the signals its deadtime depends on. get_trigger_deadtime passes the value of every signal the detector reports as configuration instead, so there is no second list to keep in step with the first: the values that determine deadtime are exactly the ones that ought to be recorded for the scan anyway. StandardReadable gains get_config_signals(), which walks the registry and recurses through CHILD-registered children. That recursion is what makes it work for areaDetector: AreaDetector registers the driver whole, and ADBaseIO declares model as CONFIG_SIGNAL, so the signal AravisTriggerLogic and VimbaTriggerLogic need arrives without either class saying anything. Both deadtime_sigs overrides are deleted. Unlike walk_config_signals it reads the registry rather than calling read_configuration(), so it does no I/O and includes read-only SignalRs -- driver.model is a SignalR, so the existing helper would have dropped precisely the signal that matters. Two consequences, both real: - A logic needing a signal must ensure it is declared as configuration. If it is not, get_deadtime gets a KeyError rather than a silently fetched value. - get_trigger_deadtime now reads every configuration signal, where a logic could previously declare none and pay nothing. Reads are gathered so it stays one round trip, but it does mean the call now needs a connected device even when deadtime is a constant -- test_adpilatus::test_deadtime had to start connecting its detector. Also moves every add_readables call in adcore/_io.py into the signal annotations, which removes three __init__ methods that existed only to declare formats. The remaining registration calls elsewhere cannot follow: they take caller-supplied config_sigs or a driver passed in as a constructor argument, neither of which is declarable. Co-Authored-By: Claude Opus 5 (cherry picked from commit c239654d0491b350bca0aaba2ed62db184129f84) --- docs/how-to/implement-ad-detector.md | 9 +- docs/tutorials/implementing-detectors.md | 6 +- src/ophyd_async/core/_detector.py | 40 ++++-- src/ophyd_async/core/_readable.py | 22 +++ src/ophyd_async/epics/adaravis.py | 3 - src/ophyd_async/epics/adcore/_io.py | 130 ++++++++---------- src/ophyd_async/epics/advimba.py | 3 - .../unit_tests/core/test_standard_detector.py | 14 +- tests/unit_tests/epics/test_adpilatus.py | 11 +- 9 files changed, 133 insertions(+), 105 deletions(-) diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 287b30e414..a50ad2394e 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -26,11 +26,14 @@ Implement methods for each trigger mode your detector supports: - `prepare_level(num)` - Setup for external level/gate triggering (high level duration determines exposure time) Only implement the prepare methods for trigger modes your detector actually supports. The detector will automatically report which trigger types are available based on which methods are implemented. -If the detector has configuration values that should be captured in the scan then implement: -- `config_sigs()` - Return the set of signals that should appear in read_configuration() +Configuration values that should be captured in the scan are declared on the IO class, +with [](#StandardReadableFormat.CONFIG_SIGNAL), exactly as on any other +[](#StandardReadable) — the trigger logic does not nominate them. If you support external triggering you should also implement: -- `get_deadtime(config_values)` - Calculate the minimum time between exposures based on configuration values +- `get_deadtime(config_values)` - Calculate the minimum time between exposures. It is + given the value of *every* signal the detector reports as configuration, so a signal + it needs must be declared with [](#StandardReadableFormat.CONFIG_SIGNAL). To preserve hardware state in plans like [`bp.count`](#bluesky.plans.count) and [`bps.trigger_and_read`](#bluesky.plan_stubs.trigger_and_read) that call `trigger()` diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index 4fa2fe93ea..1a4310fe13 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -199,8 +199,10 @@ If we wanted to support external triggering, we would also implement: - `prepare_level()` for external level/gate triggering (high level duration determines exposure) We could also implement: -- `get_deadtime()` to calculate the minimum time between exposures based on configuration -- `config_sigs()` to return signals that should appear in read_configuration() +- `get_deadtime()` to calculate the minimum time between exposures. It is given the value + of every signal the detector reports as configuration, so a signal it needs must be + declared with [](#StandardReadableFormat.CONFIG_SIGNAL) like on any other + [](#StandardReadable) — the trigger logic does not nominate signals itself - `default_trigger_info()` to return the [](#TriggerInfo) to use when `trigger()` is called without a preceding `prepare()` (governed by [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE)). ### `BlobAcquireLogic` diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index a75f228234..49c3c42933 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -135,19 +135,22 @@ class DetectorTriggerLogic: - Multi-exposure collection batching Subclasses must implement the appropriate `prepare_*` method for any trigger - mode the detector supports, `get_deadtime` if it supports external - triggering, and `deadtime_sigs` if the deadtime would vary according to - detector parameters. + mode the detector supports, and `get_deadtime` if it supports external + triggering. """ - def deadtime_sigs(self) -> set[SignalR]: - """Return the signals whose values are needed to compute deadtime.""" - return set() - def get_deadtime(self, config_values: SignalDict) -> float: """Return the deadtime in seconds for the detector. - :param config_values: the value of each signal in `deadtime_sigs` + :param config_values: + The value of every `Signal` the detector reports as configuration, + i.e. everything registered with + [](#StandardReadableFormat.CONFIG_SIGNAL), including those declared + on children registered as [](#StandardReadableFormat.CHILD). A + logic that needs a signal must make sure it is declared as + configuration, rather than nominating it separately: the values + that determine deadtime are exactly the ones that ought to be + recorded for the scan anyway. """ raise NotImplementedError(self) @@ -455,6 +458,12 @@ async def get_trigger_deadtime( ) -> tuple[set[DetectorTrigger], float | None]: """Get supported trigger types and deadtime for the detector. + The trigger logic is given the value of every signal this detector + reports as configuration, rather than nominating the ones it wants: + the values that determine deadtime are exactly the ones that ought to + be recorded for the scan anyway, so a second list to keep in step + earned nothing. + :param settings: Optional settings to use when getting configuration values :return: Tuple of supported trigger types and deadtime in seconds """ @@ -462,14 +471,23 @@ async def get_trigger_deadtime( self._trigger_logic.get_deadtime ): config_values = SignalDict() - for sig in self._trigger_logic.deadtime_sigs(): + to_read: list[SignalR] = [] + for sig in self.get_config_signals(): if settings and sig in settings: # Use value from settings if it is in there # cast to a SignalRW because settings can only contain those config_values[sig] = settings[cast(SignalRW, sig)] else: - # Get the value live - config_values[sig] = await sig.get_value() + to_read.append(sig) + # Read live values concurrently: this is now every configuration + # signal rather than a handful the logic named, so doing it in + # series would be one round trip per signal + for sig, value in zip( + to_read, + await asyncio.gather(*(sig.get_value() for sig in to_read)), + strict=True, + ): + config_values[sig] = value deadtime = self._trigger_logic.get_deadtime(config_values) else: deadtime = None diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 7d85cd22bc..bf0849802b 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -370,6 +370,28 @@ def set_readable_format( # changing a format does not reorder `hints` self._readables[device] = format + def get_config_signals(self) -> set[SignalR]: + """Return every `Signal` contributing to `read_configuration()`. + + Recurses into children registered as + [](#StandardReadableFormat.CHILD), so a Device that registers a + sub-Device whole picks up whatever that sub-Device declares as + configuration. + + Unlike [](#walk_config_signals) this reads the registry rather than + calling `read_configuration()`, so it does no I/O and includes + read-only `SignalR`s as well as `SignalRW`s. + """ + signals: set[SignalR] = set() + for device, format in self._readables.items(): + if format is StandardReadableFormat.CONFIG_SIGNAL: + signals.add(_as_signal_r(device)) + elif format is StandardReadableFormat.CHILD and isinstance( + device, StandardReadable + ): + signals |= device.get_config_signals() + return signals + def reset_readable_formats(self) -> None: """Undo every runtime format change, back to how the class declared them. diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index 7cdfb08d54..ffd072c91b 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -59,9 +59,6 @@ class AravisTriggerLogic(DetectorTriggerLogic): driver: AravisDriverIO override_deadtime: float | None = None - def deadtime_sigs(self) -> set[SignalR]: - return {self.driver.model} - def get_deadtime(self, config_values: SignalDict) -> float: return get_camera_deadtime( model=config_values[self.driver.model], diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 3a8ee0bcc6..6740536258 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -171,22 +171,16 @@ class NDROIIO(StandardReadable, NDPluginBaseIO): See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROI.html """ - min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] - min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] - size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] - size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] + min_x: A[SignalRW[int], PvSuffix.rbv("MinX"), Format.CONFIG_SIGNAL] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY"), Format.CONFIG_SIGNAL] + size_x: A[SignalRW[int], PvSuffix.rbv("SizeX"), Format.CONFIG_SIGNAL] + size_y: A[SignalRW[int], PvSuffix.rbv("SizeY"), Format.CONFIG_SIGNAL] size_z: A[SignalRW[int], PvSuffix.rbv("SizeZ")] bin_x: A[SignalRW[int], PvSuffix.rbv("BinX")] bin_y: A[SignalRW[int], PvSuffix.rbv("BinY")] bin_z: A[SignalRW[int], PvSuffix.rbv("BinZ")] - def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): - super().__init__(prefix, with_pvi, name) - self.add_readables( - [self.min_x, self.min_y, self.size_x, self.size_y], Format.CONFIG_SIGNAL - ) - class NDProcessIO(NDPluginBaseIO): """Plugin for performing processing on an NDArrray. @@ -207,64 +201,53 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): """ # Basic statistics - compute_statistics: A[SignalRW[bool], PvSuffix.rbv("ComputeStatistics")] - bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth")] - min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] - min_x: A[SignalR[float], PvSuffix("MinX_RBV")] - min_y: A[SignalR[float], PvSuffix("MinY_RBV")] - max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] - max_x: A[SignalR[float], PvSuffix("MaxX_RBV")] - max_y: A[SignalR[float], PvSuffix("MaxY_RBV")] - mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] + compute_statistics: A[ + SignalRW[bool], PvSuffix.rbv("ComputeStatistics"), Format.CONFIG_SIGNAL + ] + bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth"), Format.CONFIG_SIGNAL] + min_value: A[ + SignalR[float], PvSuffix("MinValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + min_x: A[SignalR[float], PvSuffix("MinX_RBV"), Format.UNCACHED_SIGNAL] + min_y: A[SignalR[float], PvSuffix("MinY_RBV"), Format.UNCACHED_SIGNAL] + max_value: A[ + SignalR[float], PvSuffix("MaxValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + max_x: A[SignalR[float], PvSuffix("MaxX_RBV"), Format.UNCACHED_SIGNAL] + max_y: A[SignalR[float], PvSuffix("MaxY_RBV"), Format.UNCACHED_SIGNAL] + mean_value: A[ + SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] - total: A[SignalR[float], PvSuffix("Total_RBV")] + total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] net: A[SignalR[float], PvSuffix("Net_RBV")] # Centroid statistics - compute_centroid: A[SignalRW[bool], PvSuffix.rbv("ComputeCentroid")] - centroid_threshold: A[SignalRW[float], PvSuffix.rbv("CentroidThreshold")] + compute_centroid: A[ + SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL + ] + centroid_threshold: A[ + SignalRW[float], PvSuffix.rbv("CentroidThreshold"), Format.CONFIG_SIGNAL + ] centroid_x: A[SignalR[float], PvSuffix("CentroidX_RBV")] centroid_y: A[SignalR[float], PvSuffix("CentroidY_RBV")] sigma_x: A[SignalR[float], PvSuffix("SigmaX_RBV")] sigma_y: A[SignalR[float], PvSuffix("SigmaY_RBV")] sigma_xy: A[SignalR[float], PvSuffix("SigmaXY_RBV")] # X and Y Profiles - compute_profiles: A[SignalRW[bool], PvSuffix.rbv("ComputeProfiles")] + compute_profiles: A[ + SignalRW[bool], PvSuffix.rbv("ComputeProfiles"), Format.CONFIG_SIGNAL + ] profile_size_x: A[SignalR[int], PvSuffix("ProfileSizeX_RBV")] profile_size_y: A[SignalR[int], PvSuffix("ProfileSizeY_RBV")] - cursor_x: A[SignalRW[int], PvSuffix.rbv("CursorX")] - cursor_y: A[SignalRW[int], PvSuffix.rbv("CursorY")] + cursor_x: A[SignalRW[int], PvSuffix.rbv("CursorX"), Format.CONFIG_SIGNAL] + cursor_y: A[SignalRW[int], PvSuffix.rbv("CursorY"), Format.CONFIG_SIGNAL] # Array Histogram - compute_histogram: A[SignalRW[bool], PvSuffix.rbv("ComputeHistogram")] - hist_size: A[SignalRW[int], PvSuffix.rbv("HistSize")] - hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin")] - hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax")] - - def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): - super().__init__(prefix, with_pvi, name) - self.add_readables( - [ - self.compute_statistics, - self.bgd_width, - self.compute_centroid, - self.centroid_threshold, - self.compute_profiles, - self.cursor_x, - self.cursor_y, - self.compute_histogram, - self.hist_size, - self.hist_min, - self.hist_max, - ], - Format.CONFIG_SIGNAL, - ) - self.add_readables( - [self.total, self.mean_value, self.min_value, self.max_value], - Format.HINTED_UNCACHED_SIGNAL, - ) - self.add_readables( - [self.min_x, self.min_y, self.max_x, self.max_y], - Format.UNCACHED_SIGNAL, - ) + compute_histogram: A[ + SignalRW[bool], PvSuffix.rbv("ComputeHistogram"), Format.CONFIG_SIGNAL + ] + hist_size: A[SignalRW[int], PvSuffix.rbv("HistSize"), Format.CONFIG_SIGNAL] + hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin"), Format.CONFIG_SIGNAL] + hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax"), Format.CONFIG_SIGNAL] class NDROIStatNIO(StandardReadable, EpicsDevice): @@ -275,26 +258,21 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): """ name_: A[SignalRW[str], PvSuffix("Name")] - use: A[SignalRW[bool], PvSuffix.rbv("Use")] - min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] - min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] - size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] - size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] - min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] - max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] - mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] - total: A[SignalR[float], PvSuffix("Total_RBV")] - - def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): - super().__init__(prefix, with_pvi, name) - self.add_readables( - [self.use, self.min_x, self.min_y, self.size_x, self.size_y], - Format.CONFIG_SIGNAL, - ) - self.add_readables( - [self.min_value, self.max_value, self.mean_value, self.total], - Format.HINTED_UNCACHED_SIGNAL, - ) + use: A[SignalRW[bool], PvSuffix.rbv("Use"), Format.CONFIG_SIGNAL] + min_x: A[SignalRW[int], PvSuffix.rbv("MinX"), Format.CONFIG_SIGNAL] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY"), Format.CONFIG_SIGNAL] + size_x: A[SignalRW[int], PvSuffix.rbv("SizeX"), Format.CONFIG_SIGNAL] + size_y: A[SignalRW[int], PvSuffix.rbv("SizeY"), Format.CONFIG_SIGNAL] + min_value: A[ + SignalR[float], PvSuffix("MinValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + max_value: A[ + SignalR[float], PvSuffix("MaxValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + mean_value: A[ + SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] class NDROIStatIO(NDPluginBaseIO): diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index ba96a3564c..5d776af560 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -96,9 +96,6 @@ class VimbaTriggerLogic(DetectorTriggerLogic): driver: VimbaDriverIO override_deadtime: float | None = None - def deadtime_sigs(self) -> set[SignalR]: - return {self.driver.model} - def get_deadtime(self, config_values: SignalDict) -> float: return get_camera_deadtime( model=config_values[self.driver.model], diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 38e4fd9355..4b764f5cb8 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -99,11 +99,12 @@ class DeadtimeTriggerLogic(DetectorTriggerLogic): def __init__(self, deadtime_signal: SignalR[float]): self.deadtime_signal = deadtime_signal - def deadtime_sigs(self) -> set[SignalR]: - return {self.deadtime_signal} - def get_deadtime(self, config_values: SignalDict) -> float: - """Return the deadtime from the signal value.""" + """Return the deadtime from the signal value. + + The signal reaches us because the detector registers it as + CONFIG_SIGNAL, not because this logic nominates it. + """ return config_values[self.deadtime_signal] async def prepare_internal(self, num: int, livetime: float, deadtime: float): @@ -232,6 +233,10 @@ async def test_get_trigger_deadtime( det = StandardDetector() if trigger_logic: det.add_detector_logics(trigger_logic) + if isinstance(trigger_logic, DeadtimeTriggerLogic): + # A logic that needs a signal declares it as configuration; the logic + # no longer nominates signals separately + det.set_readable_format(trigger_logic.deadtime_signal, Format.CONFIG_SIGNAL) triggers, deadtime = await det.get_trigger_deadtime() assert triggers == expected_triggers assert deadtime == expected_deadtime @@ -247,6 +252,7 @@ async def test_get_trigger_deadtime_with_settings(): det.sig = deadtime_signal tl = DeadtimeTriggerLogic(deadtime_signal) det.add_detector_logics(tl) + det.set_readable_format(deadtime_signal, Format.CONFIG_SIGNAL) # Verify initial deadtime from signal triggers, deadtime = await det.get_trigger_deadtime() diff --git a/tests/unit_tests/epics/test_adpilatus.py b/tests/unit_tests/epics/test_adpilatus.py index e3fb54cb83..d2355e2a2e 100644 --- a/tests/unit_tests/epics/test_adpilatus.py +++ b/tests/unit_tests/epics/test_adpilatus.py @@ -61,9 +61,14 @@ async def test_threshold_apply_triggers_command( ) async def test_deadtime(readout_time: adpilatus.PilatusReadoutTime, tmp_path): path_provider = StaticPathProvider(StaticFilenameProvider("data"), tmp_path) - pilatus = adpilatus.PilatusDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(path_provider), readout_time=readout_time - ) + # Connected because get_trigger_deadtime() now reads every configuration + # signal, even for a logic like this one whose deadtime is a constant + async with init_devices(mock=True): + pilatus = adpilatus.PilatusDetector( + "PREFIX:", + adcore.ADWriterFactory.hdf(path_provider), + readout_time=readout_time, + ) trigger_modes, deadtime = await pilatus.get_trigger_deadtime() assert trigger_modes == { DetectorTrigger.INTERNAL, From 494b6484c729f3157fca3c4492f1b788b57ee9ea Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 14:26:04 +0000 Subject: [PATCH 24/67] Make the config signal walk a private helper rather than public API get_config_signals was added as a public method on StandardReadable in the previous commit. #1367 restructures detector logic, and whether this should be a method, a function, or something else falls out of that, so it stays a module private _config_signals() in _readable.py until then. Nothing is added to core.__all__. Taking a Device rather than a StandardReadable also simplifies the recursion: a CHILD that is not a StandardReadable contributes nothing without the caller having to check. Co-Authored-By: Claude Opus 5 (cherry picked from commit ca5fcb7680eae179c98b0419eaa0f9653b0ddebb) --- src/ophyd_async/core/_detector.py | 3 +- src/ophyd_async/core/_readable.py | 49 +++++++++++++++++-------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 49c3c42933..1303060e70 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -31,6 +31,7 @@ from ._readable import ( StandardReadable, StandardReadableFormat, + _config_signals, _HintedFields, _Verb, ) @@ -472,7 +473,7 @@ async def get_trigger_deadtime( ): config_values = SignalDict() to_read: list[SignalR] = [] - for sig in self.get_config_signals(): + for sig in _config_signals(self): if settings and sig in settings: # Use value from settings if it is in there # cast to a SignalRW because settings can only contain those diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index bf0849802b..138086a6c5 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -370,28 +370,6 @@ def set_readable_format( # changing a format does not reorder `hints` self._readables[device] = format - def get_config_signals(self) -> set[SignalR]: - """Return every `Signal` contributing to `read_configuration()`. - - Recurses into children registered as - [](#StandardReadableFormat.CHILD), so a Device that registers a - sub-Device whole picks up whatever that sub-Device declares as - configuration. - - Unlike [](#walk_config_signals) this reads the registry rather than - calling `read_configuration()`, so it does no I/O and includes - read-only `SignalR`s as well as `SignalRW`s. - """ - signals: set[SignalR] = set() - for device, format in self._readables.items(): - if format is StandardReadableFormat.CONFIG_SIGNAL: - signals.add(_as_signal_r(device)) - elif format is StandardReadableFormat.CHILD and isinstance( - device, StandardReadable - ): - signals |= device.get_config_signals() - return signals - def reset_readable_formats(self) -> None: """Undo every runtime format change, back to how the class declared them. @@ -455,6 +433,33 @@ def add_readables( self.set_readable_format(device, format) +def _config_signals(device: Device) -> set[SignalR]: + """Return every `Signal` contributing to a Device's `read_configuration()`. + + Recurses into children registered as [](#StandardReadableFormat.CHILD), so + a Device that registers a sub-Device whole picks up whatever that + sub-Device declares as configuration. A Device that is not a + `StandardReadable` contributes nothing. + + Unlike `walk_config_signals` this reads the registry rather than calling + `read_configuration()`, so it does no I/O and includes read-only `SignalR`s + as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is + exactly the signal an areaDetector trigger logic needs for deadtime. + + Deliberately private and unexported: #1367 restructures detector logic, so + what the public shape of this should be is not settled yet. + """ + signals: set[SignalR] = set() + if not isinstance(device, StandardReadable): + return signals + for child, format in device._readables.items(): # noqa: SLF001 + if format is StandardReadableFormat.CONFIG_SIGNAL: + signals.add(_as_signal_r(child)) + elif format is StandardReadableFormat.CHILD: + signals |= _config_signals(child) + return signals + + class _UncachedRead: def __init__(self, signal: SignalR) -> None: self.signal = signal From 63bc24e40e884f0a4b111fd3b14fc88b25a01956 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 28 Aug 2026 11:52:37 +0000 Subject: [PATCH 25/67] Delete the prepare_single data logic tier A StandardDetector is a StandardReadable now, so a source that produces one value per event is a registered signal, not a data logic: set_readable_format puts it in read() and describe() for any number of collections, where a prepare_single logic dropped itself with a warning as soon as a scan asked for more than one. That leaves the tier with nothing of its own -- its only implementation, PluginSignalDataLogic, has already gone the same way -- so prepare_single, ReadableDataProvider and SignalDataProvider go rather than staying as a second way to get a scalar into an event. Also restores the two doc sections held back from the previous branch, since this is the change that makes them true, and points ADR 0012 at 0021. --- .../decisions/0012-detector-rewrite.md | 3 + .../decisions/0021-runtime-readable-format.md | 37 +++++++ docs/how-to/change-readable-format.md | 14 +++ docs/how-to/implement-ad-detector.md | 11 +- src/ophyd_async/core/__init__.py | 4 - src/ophyd_async/core/_data_providers.py | 27 +---- src/ophyd_async/core/_detector.py | 46 ++------ .../unit_tests/core/test_standard_detector.py | 102 ++++++------------ 8 files changed, 107 insertions(+), 137 deletions(-) diff --git a/docs/explanations/decisions/0012-detector-rewrite.md b/docs/explanations/decisions/0012-detector-rewrite.md index 3e1145b66b..db9fdf2006 100644 --- a/docs/explanations/decisions/0012-detector-rewrite.md +++ b/docs/explanations/decisions/0012-detector-rewrite.md @@ -5,6 +5,9 @@ Date: 2026-01-20 Accepted +Partly superseded by [](./0021-runtime-readable-format.md), which made `StandardDetector` a +`StandardReadable` and removed the `prepare_single` tier along with `ReadableDataProvider`. + ## Context The original `StandardDetector` implementation had several architectural issues: diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0021-runtime-readable-format.md index f545913485..cb53bebdcf 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0021-runtime-readable-format.md @@ -131,6 +131,43 @@ A run's descriptor is emitted at its start, and `HINTED_SIGNAL` sets up monitori disagree. This is documented rather than enforced: the check would have to know about run boundaries, which a Device does not. +### `StandardDetector` inherits `StandardReadable` + +A detector's step-scan signals are registered exactly like any other Device's, and +`read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. +`add_config_signals` becomes a deprecated wrapper. + +The detector does not *reimplement* any of those verbs to achieve it. `StandardReadable` +grows two hooks, `_extra_funcs_for(verb)` and `_extra_hint_sources()`, that contribute +from somewhere other than the registry, and `StandardDetector` overrides those to add what +its data logics produce. Likewise it registers its stage and unstage work into +`_stage_funcs`/`_unstage_funcs`, the way `StandardFlyable` already does, rather than +replacing `stage()`. Overriding the verbs meant re-deriving the registered children's +contributions by hand in each one, which is exactly what would drift. + +One consequence: a detector with no hinted fields now reports `hints` as `{}` rather than +`{"fields": []}`, because it no longer special cases what `StandardReadable` already does. + +Plugins are **not** registered automatically. `AreaDetector` registers only the driver's +`acquire_time` and `acquire_period`, plus whatever the caller passes as `config_sigs`. +Registering a plugin is a `set_readable_format()` call at the call site, which is the +explicit opt-in the #1395 reviewers asked for, now expressible per plugin and reversible +at runtime. + +### The `prepare_single` data logic tier goes + +ADR 0012 gave `DetectorDataLogic` a `prepare_single` tier, returning a +`ReadableDataProvider`, so that a source producing one value per event could reach +`read()` and `describe()`. Registering a signal now does that, for any Device and without +a data logic, so the tier has nothing left of its own: its only in-tree implementation was +`PluginSignalDataLogic`, wrapping a driver and one `SignalR`, and that is a +`set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL)` call. + +`prepare_single`, `ReadableDataProvider` and `SignalDataProvider` are therefore deleted +rather than kept alongside. Keeping them would leave two ways to get a scalar into an +event, one of which drops itself with a warning as soon as the scan asks for more than one +collection, while the other simply works. + ### Formats are stored in the same file as settings, under a reserved key Values and formats are applied on the same cadence — a technique change wants both — so diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index 941e644960..839ee5242a 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -67,6 +67,20 @@ so a format changed part way through a run would leave `describe()` and `read()` disagreeing. ``` +## Opting a detector plugin in + +A [](#StandardDetector) is a [](#StandardReadable), so the same call works on one. This +matters most for areaDetector, where a detector often carries many more plugins than are +wired into its chain — so plugins are never registered automatically, and you opt in to +the ones that are actually producing meaningful values: + +```python +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) +``` + +Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can +write a file *and* report a plugin scalar in a step scan. + ```{seealso} [](./store-and-retrieve.md) for saving a set of formats alongside signal values and loading them back, and diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index a50ad2394e..8f78000360 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -115,12 +115,15 @@ For example, for ADAravis: The composition-based architecture makes it possible to add multiple data outputs to a detector. After creating the detector, you can call `add_detector_logics()` to add additional data sources: ### Reading stats plugins alongside file writing +A plugin scalar needs no data logic: a detector is a [](#StandardReadable), so register +the signal and it appears in `read()` alongside whatever the file writer produces. + ```python -from ophyd_async.epics.adcore import PluginSignalDataLogic +from ophyd_async.core import StandardReadableFormat as Format -det = adaravis.AravisDetector(prefix, path_provider) -# Add stats total as a readable signal in events -det.add_detector_logics(adcore.PluginSignalDataLogic(det.driver, det.stats.total)) +det = adaravis.AravisDetector(prefix, path_provider, plugins={"stats": stats}) +# Add stats total as a hinted signal in events +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) ``` ### Multiple HDF writers for different ROIs diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 1563695a29..135743eefc 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -11,8 +11,6 @@ soft_command, ) from ._data_providers import ( - ReadableDataProvider, - SignalDataProvider, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -299,9 +297,7 @@ def __getattr__(name): "AutoMaxIncrementingPathProvider", "UUIDFilenameProvider", # Data Providers - "ReadableDataProvider", "StreamableDataProvider", - "SignalDataProvider", "StreamResourceInfo", "StreamResourceDataProvider", # Flyer diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index ad2983cb59..8faa7ba069 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -1,39 +1,14 @@ from abc import abstractmethod from collections.abc import AsyncIterator, Sequence -from dataclasses import dataclass from typing import Any -from bluesky.protocols import Reading, StreamAsset +from bluesky.protocols import StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange from ._signal import SignalR, SignalW from ._utils import ConfinedModel -class ReadableDataProvider: - @abstractmethod - async def make_datakeys(self) -> dict[str, DataKey]: - """Return a DataKey for each Readable that produces a Reading. - - Called before the first exposure is taken. - """ - - @abstractmethod - async def make_readings(self) -> dict[str, Reading]: - """Read the Signals and return their values.""" - - -@dataclass -class SignalDataProvider(ReadableDataProvider): - signal: SignalR - - async def make_datakeys(self) -> dict[str, DataKey]: - return await self.signal.describe() - - async def make_readings(self) -> dict[str, Reading]: - return await self.signal.read(cached=False) - - class StreamableDataProvider: collections_written_signal: SignalR[int] diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 1303060e70..9b49029c6d 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -25,9 +25,7 @@ from event_model import DataKey from pydantic import Field, NonNegativeInt, PositiveInt, computed_field -from ophyd_async.core._log import logger - -from ._data_providers import ReadableDataProvider, StreamableDataProvider +from ._data_providers import StreamableDataProvider from ._readable import ( StandardReadable, StandardReadableFormat, @@ -313,19 +311,16 @@ async def _get_collections_written( class DetectorDataLogic: """Abstract base class for detector data logic and handling. - Implementations must implement either prepare_unbounded for data sources - that work with step scans as well as flyscans, or prepare_single for those - that only work with step scans. + Implementations must override `prepare_unbounded`. A source that produces a + single value per event, like a plugin scalar, does not need a data logic at + all: register its signal with + [](#StandardReadable.set_readable_format) instead. """ #: Add this suffix to the detector name to specify the datakey. These need to be #: different for each DetectorDataLogic added to a detector datakey_suffix: str = "" - async def prepare_single(self, datakey_name: str) -> ReadableDataProvider: - """Provider can only work for a single event.""" - raise NotImplementedError(self) - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: """Provider can work for an unbounded number of collections.""" raise NotImplementedError(self) @@ -345,7 +340,6 @@ async def stop(self) -> None: @dataclass class _PrepareCtx: trigger_info: TriggerInfo - readable_data_providers: Sequence[ReadableDataProvider] streamable_data_providers: Sequence[StreamableDataProvider] collections_written: int @@ -516,7 +510,6 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: == trigger_info.collections_per_event ): # Reuse the existing data providers - readable_data_providers = self._prepare_ctx.readable_data_providers streamable_data_providers = self._prepare_ctx.streamable_data_providers else: # Stop the existing providers if there is a context and make new ones @@ -525,36 +518,19 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: await data_logic.stop() # Setup the data logic for the right number of collections streamable_coros: list[Awaitable[StreamableDataProvider]] = [] - readable_coros: list[Awaitable[ReadableDataProvider]] = [] for dl in self._data_logics: - if _data_logic_supported(dl.prepare_unbounded): - streamable_coros.append( - dl.prepare_unbounded(self.name + dl.datakey_suffix) - ) - elif _data_logic_supported(dl.prepare_single): - if trigger_info.number_of_collections == 1: - readable_coros.append( - dl.prepare_single(self.name + dl.datakey_suffix) - ) - else: - logger.warning( - f"DataLogic {dl} only supports a single collection, but " - "the detector was prepared for " - f"{trigger_info.number_of_collections} collections." - ) - else: + if not _data_logic_supported(dl.prepare_unbounded): msg = f"DataLogic hasn't overridden any prepare_* methods {dl}" raise RuntimeError(msg) - streamable_data_providers, readable_data_providers = await asyncio.gather( - asyncio.gather(*streamable_coros), - asyncio.gather(*readable_coros), - ) + streamable_coros.append( + dl.prepare_unbounded(self.name + dl.datakey_suffix) + ) + streamable_data_providers = await asyncio.gather(*streamable_coros) # Stash the prepare context so we can use it in trigger/kickoff self._prepare_ctx = _PrepareCtx( trigger_info=trigger_info, streamable_data_providers=streamable_data_providers, - readable_data_providers=readable_data_providers, collections_written=await _get_collections_written( streamable_data_providers ), @@ -785,8 +761,6 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] if verb not in (_Verb.DESCRIBE, _Verb.READ): return ctx = error_if_none(self._prepare_ctx, "Prepare not run") - for dp in ctx.readable_data_providers: - yield dp.make_datakeys if verb is _Verb.DESCRIBE else dp.make_readings if verb is _Verb.DESCRIBE: # Streamable providers describe their shape for a step scan, but # produce their data through collect_asset_docs rather than read() diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 4b764f5cb8..927777167a 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -12,9 +12,7 @@ DetectorDataLogic, DetectorTrigger, DetectorTriggerLogic, - ReadableDataProvider, Settings, - SignalDataProvider, SignalDict, SignalR, StandardDetector, @@ -131,26 +129,14 @@ async def ensure_stopped(self): self.disarm_count += 1 -class ReadableOnlyDataLogic(DetectorDataLogic): - """Produces only readable (non-streaming) data.""" - - def __init__(self): - self.signal = soft_signal_rw(int, initial_value=42, name="foo-value") - - async def prepare_single(self, datakey_name: str) -> ReadableDataProvider: - return SignalDataProvider(self.signal) - - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return ["foo-value"] - - class StreamableOnlyDataLogic(DetectorDataLogic): """Produces only streamable (file-based) data.""" - def __init__(self, tmp_path): + def __init__(self, tmp_path, datakey_suffix: str = ""): self.collections_written = soft_signal_rw(int) self.stop_count = 0 self.tmp_path = tmp_path + self.datakey_suffix = datakey_suffix async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: resource = StreamResourceInfo( @@ -439,10 +425,10 @@ async def test_arm_logic_called_on_stage(): assert al.armed is False -async def test_describe_before_prepare_raises(): +async def test_describe_before_prepare_raises(tmp_path): """Test that describe() fails before prepare().""" det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) + det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) with pytest.raises(RuntimeError, match="Prepare not run"): await det.describe() @@ -621,23 +607,21 @@ async def test_stage_resets_state(): assert await det.events_to_kickoff.get_value() == 0 -async def test_hints_from_single_data_logic(): +async def test_hints_from_single_data_logic(tmp_path): """Test that hints come from data logic.""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) + det = StandardDetector(name="bar") + det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) await det.prepare(TriggerInfo()) - assert det.hints == { - "fields": ["foo-value"] - } # ReadableOnlyDataLogic uses foo-value + assert det.hints == {"fields": ["bar"]} async def test_hints_from_multiple_data_logics(tmp_path): """Test that hints are aggregated from multiple data logics.""" det = StandardDetector(name="bar") - dl1 = ReadableOnlyDataLogic() - dl2 = StreamableOnlyDataLogic(tmp_path) + dl1 = StreamableOnlyDataLogic(tmp_path) + dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-extra") det.add_detector_logics(dl1, dl2) await det.prepare(TriggerInfo()) @@ -645,7 +629,7 @@ async def test_hints_from_multiple_data_logics(tmp_path): # Should include hints from both logics hints = det.hints assert "fields" in hints - assert hints["fields"] == ["foo-value", "bar"] + assert hints["fields"] == ["bar", "bar-extra"] @pytest.mark.parametrize( @@ -691,13 +675,13 @@ async def test_config_signals_in_describe_configuration( async def test_kickoff_without_streamable_data_raises(): """Test that kickoff() without streamable data fails.""" det = StandardDetector(name="foo") - det.add_detector_logics(JustInternalTriggerLogic(), ReadableOnlyDataLogic()) + det.add_detector_logics(JustInternalTriggerLogic()) - # Single event prepare for readable-only logic + # A detector with no data logic can still be triggered for a step scan await det.prepare(TriggerInfo()) await det.trigger() # This works - # Readable-only logic doesn't support kickoff + # ...but there is nothing to collect, so it cannot be flown await det.prepare(TriggerInfo(number_of_events=5)) with pytest.raises( ValueError, match="Detector foo is not streamable, so cannot kickoff" @@ -784,13 +768,13 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): async def test_read_returns_correct_values(): - """Test that read() returns values from readable providers.""" + """Test that read() returns the values of registered signals.""" det = StandardDetector(name="det") - dl = ReadableOnlyDataLogic() - det.add_detector_logics(dl) + det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") + det.set_readable_format(det.counts, Format.HINTED_SIGNAL) await det.trigger() - await assert_reading(det, {"foo-value": {"value": 42}}) + await assert_reading(det, {"det-counts": {"value": 42}}) async def test_detector_with_no_logics(): @@ -899,19 +883,6 @@ async def ensure_stopped(self): assert al.shutter_closed is True # unstage() must close the shutter -async def test_multiple_collections_with_single_only_logic_warns(caplog): - """Test that requesting multiple collections warns, leads to empty prepare ctx.""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) - - await det.prepare(TriggerInfo(number_of_events=5)) - - assert "only supports a single collection" in caplog.text - assert det._prepare_ctx is not None - assert det._prepare_ctx.readable_data_providers == [] - assert det._prepare_ctx.streamable_data_providers == [] - - async def test_data_logic_with_no_prepare_methods_raises(): """Test error when DataLogic doesn't override any prepare methods.""" @@ -976,16 +947,18 @@ async def test_different_collections_written_raises(tmp_path): await det.kickoff() -async def test_multiple_data_logics(tmp_path): - """Test detector with multiple data logics.""" +async def test_data_logic_and_registered_signal(tmp_path): + """A registered signal and a data logic both describe(), only one collects.""" det = StandardDetector(name="det") - dl1 = ReadableOnlyDataLogic() - dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl1, dl2) + det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") + det.set_readable_format(det.counts, Format.HINTED_SIGNAL) + det.add_detector_logics( + JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path) + ) await det.prepare(TriggerInfo()) - # Should have data from both logics + # Should have data from the data logic and the registered signal description = await det.describe() assert description == { "det": { @@ -995,11 +968,11 @@ async def test_multiple_data_logics(tmp_path): "shape": [1, 10, 15], "source": f"file://localhost/{tmp_path.as_posix().lstrip('/')}/test.h5", }, - "foo-value": { + "det-counts": { "dtype": "integer", "dtype_numpy": " Date: Fri, 17 Jul 2026 13:09:49 +0000 Subject: [PATCH 26/67] docs: add ADR 0020 for the bounded data logic tier #1248, #888, #1309 and #1364 all bottom out in the same two gaps: a data logic is told nothing about timing or extent at prepare time, and StandardDetector claims WritesStreamAssets unconditionally so the bluesky bundler will never call collect_pages on it. Record the design for a third `prepare_bounded` tier that closes both. It is one tier rather than two changes because the devices that cannot write files (MeasComp CTR-08, SIS3820 scaler, TetrAmm) and the ones that want rate-based chunking (Xspress, PandA) both need to be told what the scan looks like before it starts. Document the CTR-08 explicitly. It is not an areaDetector, and it is the case that puts pressure on the trigger/acquire/data split: one control both starts acquisition and erases the buffer, and one PV is both the exposure count and the buffer size. An areaDetector hides this by acquiring and capturing on separate PVs. The split holds -- it is a split of concerns, not of hardware -- but it forces the progress baseline to be stated rather than derived, since a buffer erased by the acquire control is erased after the data logic has been prepared. Decided ahead of any code so the tier design can be agreed on its own. (cherry picked from commit 97b72c6fc6902d682b4202744b61f84037870f70) --- .../decisions/0020-bounded-data-logic-tier.md | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 docs/explanations/decisions/0020-bounded-data-logic-tier.md diff --git a/docs/explanations/decisions/0020-bounded-data-logic-tier.md b/docs/explanations/decisions/0020-bounded-data-logic-tier.md new file mode 100644 index 0000000000..9d4dd16c9b --- /dev/null +++ b/docs/explanations/decisions/0020-bounded-data-logic-tier.md @@ -0,0 +1,283 @@ +# 20. Add a bounded `DetectorDataLogic` tier and select tiers by capability + +Date: 2026-07-17 + +## Status + +Accepted + +## Context + +ADR 0012 split data production out into `DetectorDataLogic`, with two tiers that a subclass +opts into by overriding the corresponding method: + +- `prepare_single(datakey_name) -> ReadableDataProvider` — a single event only, feeding + `read()` and `describe()` +- `prepare_unbounded(datakey_name) -> StreamableDataProvider` — any number of collections, + feeding `collect_asset_docs()` and `describe_collect()` + +Four open issues turn out to bottom out in the same two gaps. + +### The data logic is not told enough at prepare time + +Issue #1309 wants chunk size and flush rate configured from the scan parameters. An Xspress +running at 400 Hz with a 2 Hz flush rate wants 200 frames per chunk; the PandA wants a flush +period. Neither is expressible today, because a data logic is prepared for an *unbounded* number +of collections and so is told nothing about timing. Today's implementations paper over this: +`ADHDFDataLogic` reads `num_frames_chunks` back from the IOC and forces it to 1 if unset, the +PandA hardcodes `chunk_shape=(1024,)` behind a TODO, and Odin hardcodes `(1, *data_shape)`. + +The parameter that is actually needed is the frame **period** (`livetime + deadtime`), not the +number of frames — the requirement is a flush *rate*. This matters, because the period is known +for step scans as well as fly scans, whereas the total frame count is not. + +Two workarounds were considered and rejected. Subclassing `TriggerInfo` and `prepare()` to carry a +chunk size (as Malcolm did) puts a data-writing concern into the trigger path and defeats ADR +0012's rule that logic classes are handed only the fields they need. Computing the chunk size in +the `DetectorTriggerLogic` puts a data concern in the trigger logic. + +### `StandardDetector` unconditionally claims to write stream assets + +Issues #1248 and #888 both come from detectors that produce data but cannot write files: a Struck +SIS3820 scaler in a VME crate, a MeasComp CTR-08, a TetrAmm electrometer, an electron analyser. +These want the trigger, arm, prepare and stage machinery of `StandardDetector`, but need to emit +event pages rather than stream assets. + +These are worth describing concretely, because they are not areaDetectors and they put more +pressure on the trigger/acquire/data split than an areaDetector does. A MeasComp CTR-08 exposes: + +| PV | role | +|---|---| +| `MCS:EraseStart` | clears the arrays *and* starts counting — one control, no separate erase | +| `MCS:StopAll` | stop | +| `MCS:Acquiring` | idle status | +| `MCS:NuseAll` | number of channels to use — the buffer size, and the point at which it stops | +| `MCS:CurrentChannel` | how many points have been collected | +| `mca.VAL` | the growing waveform arrays that hold the data | + +The difference from an areaDetector is that there is no driver/plugin split in the hardware. An +areaDetector acquires with `driver.acquire` and captures with a *separate* `TSAcquire` or HDF +`capture`, so the acquire and data concerns land on different PVs and the split falls out of the +hardware. Here one control both gates acquisition and clears the data buffer, and one PV +(`NuseAll`) is both the exposure count and the buffer size. + +They cannot, because `StandardDetector` inherits `WritesStreamAssets` unconditionally, and the +bluesky bundler only calls `collect_pages()` on a device that is *not* a `WritesStreamAssets`. +The two protocols are mutually exclusive by design, to avoid the case where both produce blocks +of different sizes. + +#888 asks for the writer to be optional. The rewrite already made it optional in the literal +sense — `StandardDetector()` takes no writer at all. But the underlying request ("no HDF writer, +because I have PVs instead") is #1248, so #888 is a duplicate. + +### These are the same gap + +Both groups of devices must be told *how many* collections to expect at prepare time, because +they own a finite buffer that has to be sized before acquisition: the scaler's MCA array length +*is* the frame count, and the areaDetector stats plugin's `TSNumPoints` must be set before the +time series starts. A device with a finite buffer also naturally produces its data as pages at +the end of an event rather than as stream datums as it goes. + +So one new tier serves both, and #1309's period rides along with it. + +### The areaDetector stats duality + +An `NDPluginStats` exposes each statistic twice: as a scalar (`Total_RBV`) and as a time series +array (`TS:TSTotal`). Only the scalar is modelled today, by `PluginSignalDataLogic`. The scalar +can only ever serve a single collection; the time series can serve any number. Issue #1364 arises +directly from this: an AD detector with an HDF writer *and* a stats scalar cannot fly, because +the scalar is single-only. + +## Decision + +### A third tier: `prepare_bounded` + +```python +class DetectorDataLogic: + async def prepare_single( + self, datakey_name: str + ) -> ReadableDataProvider: ... + async def prepare_bounded( + self, datakey_name: str, num_collections: int, period: float + ) -> PageableDataProvider: ... + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: ... +``` + +`prepare_bounded` and `prepare_unbounded` are told the frame period; `prepare_single` is not, +since a single collection has no rate to size against. + +`num_collections` is always the **total** for the scan (`TriggerInfo.number_of_collections`), +never a per-kickoff figure. A finite buffer is fed by data callbacks and does not care how many +kickoffs span it, so per-kickoff sizing would mean resizing the buffer mid-scan. + +### `PageableDataProvider`, with readings derived from pages + +`prepare_bounded` returns a single provider type that emits pages. Readings are derived from +pages rather than being a separate code path: a step-scan prepare has `number_of_events == 1`, so +its page contains exactly one event, which extracts to a single reading. `collections_per_event` +appears in the datakey shape exactly as it does for `StreamableDataProvider`. + +### The period is resolved in `prepare()` + +`TriggerInfo.livetime` of 0 means "whatever is currently set on the detector", so the period +cannot simply be forwarded from the `TriggerInfo` — an explicit +`prepare(TriggerInfo(number_of_events=10))` would hand the data logic a period of 0. + +Instead `prepare()` resolves it. `DetectorTriggerLogic.default_trigger_info()` is extended to +return the current `livetime` and `deadtime` as well as the frame count, and `prepare()` uses it +to fill in a zero `livetime` after the trigger logic has been prepared. ADR 0012 already requires +that ordering, so the hardware holds the real values by that point. Data logics therefore never +see a period of 0, and the awkward case is handled once in core rather than by every implementer. + +### Tier selection is by capability, with no precedence rule + +Each data logic implements exactly one tier, so core does not choose between tiers — it only +asks whether the tier a logic implements can serve the requested number of collections +`n = TriggerInfo.number_of_collections`: + +| tier | serves | +|---|---| +| `prepare_unbounded` | always | +| `prepare_bounded` | `n != 0` (finite) | +| `prepare_single` | `n == 1` | + +A logic whose tier cannot serve `n` is dropped from the prepare context with a warning, rather +than raising. This generalises the rule agreed in #1364 to all three tiers: it is what lets an +AD detector carry an HDF writer plus a `PluginSignalDataLogic` on some unrelated PV (a +temperature, a ring current) and still fly, with only the scalar dropped. + +### Bounded providers are never reused + +`_update_prepare_context` currently reuses providers when `collections_per_event` is unchanged, +to avoid reopening files on every step-scan point. Two changes follow. + +The reuse check gains the period, since the period now determines chunk shape, and chunk shape is +baked into the `StreamResource` at provider construction. A period change therefore starts a new +file. This is inherent rather than incidental: areaDetector's `num_frames_chunks` can only be set +before `capture` starts. + +Bounded providers are excluded from reuse entirely, so `trigger()` re-prepares them on every +point. This is what re-arms a finite buffer for each event, and it needs no new API: `prepare()` +and `trigger()` both call `_update_prepare_context`, and `kickoff()` does not — which is exactly +the set of places a buffer should and should not be re-armed. + +Reuse consequently becomes a per-logic decision rather than an all-or-nothing branch, and +`_PrepareCtx` holds pageable providers in their own list — a re-armed bounded provider sitting at +0 alongside a monotonic HDF writer at 15 would otherwise trip the `_all_the_same` reducer. + +### A bounded provider's progress baseline is zero in `trigger()` + +`trigger()` takes a bounded provider's starting progress to be 0 rather than reading it back; +`kickoff()` continues to read it live. + +It is tempting to derive this instead of stating it, since `_update_prepare_context` refreshes +`collections_written` after re-preparing, and an areaDetector stats time series is erased by the +`TSAcquire` write inside `prepare_bounded` — so the refreshed read already returns zero. That +derivation only holds when the data logic performs the erase itself. Where the erase belongs to +the acquire control, as with the CTR-08's `EraseStart`, it happens *after* `_update_prepare_context` +has taken its reading, so the baseline would be the stale pre-erase value; the detector would then +wait for twice the buffer's length and hang. Stating the rule keeps the framework agnostic about +which component erases. + +`kickoff()` must not zero the baseline, because a fly scan erases once and accumulates across +kickoffs; zeroing there would make the second kickoff's target already satisfied and return +immediately. Since a detector may not mix bounded and unbounded logics, each call has providers of +only one kind and the rule reduces to a single choice per call site. + +### Erase and start controls belong to the acquire logic + +Where one control both starts acquisition and clears the data buffer, it belongs to the +`DetectorAcquireLogic`, alongside the stop control and the idle status. Nothing needs to declare +the erase to the framework: `prepare_bounded` is always called before `start_acquiring()`, so the +buffer is sized before the control fires, and the baseline rule above means the detector does not +care who erased it. + +The trigger/acquire/data split therefore holds for these devices, and the three logics share one +IO device exactly as the areaDetector logics share a `driver`. The split is a split of concerns, +not of hardware, so it survives the concerns landing on a single PV. + +### Logic objects fill exactly one role + +`add_detector_logics()` raises if an object satisfies more than one of the three logic protocols, +directing the author to pass separate objects. This is a hazard worth catching rather than a +hypothetical: a device whose concerns all live on one control is a natural candidate for a single +combined logic object, and the registration is a chain of `isinstance` tests, so such an object +would silently register as the first protocol it matched and its other roles would be dropped +without a word. + +### `collect_asset_docs` and `collect_pages` are exposed dynamically + +`StandardDetector` no longer inherits `WritesStreamAssets`. Instead `__getattr__` exposes +`collect_asset_docs` only when a data logic implementing `prepare_unbounded` is present, and +`collect_pages` only when one implementing `prepare_bounded` is present. Because bluesky's +protocols are `runtime_checkable`, an absent attribute is enough to make the isinstance check +fail, and the bundler then does the right thing. + +This keys off the data logics, which are fixed when the detector is constructed, rather than off +the prepare context, which does not exist until `prepare()` runs. + +This keeps a single detector class able to be either kind depending on its `__init__` arguments — +one `AravisDetector`, configured for files or for pages — which was the original requirement, and +it needs no change to bluesky. + +### Mixing bounded and unbounded logics raises + +`add_detector_logics()` raises if a detector is given both a bounded and an unbounded data logic, +since that is the one combination where both `collect_asset_docs` and `collect_pages` would be +exposed and both would produce data in a fly scan. Single-tier logics may coexist with either. + +### Stats with a file writer go via NDAttributes + +It follows that an areaDetector with an HDF writer does not use the stats time series. It does +not need to: `ADHDFDataLogic` already pulls NDAttribute datasets into the file, which is +areaDetector's own mechanism for exactly this. The time series tier is for detectors with no +writer at all. + +`PluginSignalDataLogic` (the scalar) therefore remains the right choice for a detector that +writes files and wants a stats value in a step scan, and a stats time series data logic is the +right choice for one that writes no files. + +### One stats data logic for many arrays + +An `NDPluginStats` has one time series control (`TSControl`, `TSNumPoints`, `TSAcquire`) shared +across roughly twenty arrays. A stats data logic therefore covers *many* arrays with the control +embedded, rather than one logic per array with a shared control object. + +This follows the grain of the existing design — a provider already emits many datakeys, as +`ADHDFDataLogic` does for the main dataset plus its NDAttributes — and it avoids the coordination +problem, since the detector prepares each data logic independently and several logics contending +over one `TSAcquire` would need reference counting or an ordering guarantee. Which statistics are +wanted becomes a constructor argument. + +## Consequences + +`DetectorDataLogic` implementations that override `prepare_unbounded` must take the new `period` +argument. This is a breaking change for out-of-tree data logics; the library is in alpha and no +compatibility shim is provided. In-tree, `ADHDFDataLogic` and `PandaHDFDataLogic` can then +compute chunk size and flush period from the rate, replacing the TODOs and hardcoded values that +stand in for it today. + +`DetectorTriggerLogic.default_trigger_info()` implementations should return the current +`livetime` and `deadtime` alongside the frame count. Those that do not will still work, but their +detectors will fall back to today's behaviour of reading the chunk size back from hardware. + +Detectors that cannot write files become expressible: a data logic implementing `prepare_bounded` +gives them `collect_pages()` and the whole of `StandardDetector` besides. This closes #1248 and +#888. + +Fly scanning an areaDetector that carries an HDF writer plus a single-tier data logic starts +working, with the single-tier logic dropped and a warning, closing #1364. + +Mixing bounded and unbounded data logics on one detector is not supported. If a use case appears +that genuinely needs a file *and* pages in one fly scan, it needs the bluesky bundler to allow +both protocols and error only on a size mismatch — a larger change, deliberately deferred. + +A device whose exposure count and buffer size are the same PV will have it written twice: once by +the trigger logic as `number_of_exposures`, and once by the data logic as `num_collections`. The +two agree only when `exposures_per_collection` is 1. This is latent rather than live, because such +devices do not average exposures and so cannot ask for anything else, but a detector that could do +both would need its data logic to read the value back rather than set it. It is the same +one-PV-two-concerns shape as the combined erase-and-start control, and the same answer applies: +the split is of concerns, not of PVs. From 847bbca1b18724bac80885898b5d1010efa39910 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 17 Jul 2026 15:33:02 +0000 Subject: [PATCH 27/67] fix: raise if one logic object fills two roles add_detector_logics dispatched with an if/elif chain, so an object that was both an AcquireLogic and a DataLogic registered as the first match only and had its other role silently dropped. A device whose concerns all live on one control is a natural candidate for a single combined logic object, so catch the ambiguity and direct the author to pass separate objects instead. Refs ADR 0020. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 6fe39e2691a0c7ccf20295cecfd0a96315e6745c) --- src/ophyd_async/core/_detector.py | 19 +++++++++++++++++++ .../unit_tests/core/test_standard_detector.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 9b49029c6d..305564c275 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -417,6 +417,25 @@ def add_detector_logics( :param logic: The logic to add """ for logic in logics: + # Each object must fill exactly one role. A single object that is both, + # say, an AcquireLogic and a DataLogic would otherwise register as only + # the first match and have its other role silently dropped, so require + # separate objects even when one device's concerns live on one control. + roles = [ + base + for base in ( + DetectorTriggerLogic, + DetectorAcquireLogic, + DetectorDataLogic, + ) + if isinstance(logic, base) + ] + if len(roles) > 1: + names = ", ".join(base.__name__ for base in roles) + raise TypeError( + f"{type(logic).__name__} is both {names}; pass a separate object " + "for each logic role" + ) if isinstance(logic, DetectorTriggerLogic): if self._trigger_logic is not None: raise RuntimeError("Detector already has trigger logic") diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 927777167a..26ecd94f62 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -840,6 +840,22 @@ class UnknownLogic: det.add_detector_logics(UnknownLogic()) +async def test_add_logic_filling_two_roles_raises(): + """An object satisfying two logic protocols must not be silently half-registered.""" + det = StandardDetector() + + class BothAcquireAndData(DetectorAcquireLogic, DetectorDataLogic): + async def start_acquiring(self): ... + async def wait_for_idle(self): ... + async def ensure_stopped(self): ... + + with pytest.raises( + TypeError, + match="is both DetectorAcquireLogic, DetectorDataLogic", + ): + det.add_detector_logics(BothAcquireAndData()) + + @pytest.mark.parametrize("initial_shutter_closed", [True, False]) async def test_ensure_ready_vs_ensure_stopped_hooks(initial_shutter_closed: bool): """ensure_ready and ensure_stopped are separate hooks. From 06de007dfb403583541a3f64600b71dc40a6921c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 17 Jul 2026 15:45:45 +0000 Subject: [PATCH 28/67] feat: add bounded DetectorDataLogic tier for finite buffers Adds prepare_bounded as a third data-logic tier alongside prepare_single and prepare_unbounded, returning a PageableDataProvider for data held in a finite buffer that must be sized before acquiring (a scaler MCA, an areaDetector stats time series). Such a provider emits event pages; read() derives a step-scan reading from the single-event page the buffer holds. Tier selection is capability-only: a data logic whose tier cannot serve the requested number of collections is dropped with a warning rather than raising, generalising #1364 to all three tiers. This replaces the previous hard error when a single-only logic met multiple collections. Bounded providers are never reused, so trigger() re-prepares them each point, which re-arms the buffer; kickoff() does not, so a fly scan arms once and accumulates across kickoffs. A bounded buffer's progress baseline is therefore 0 in trigger() and read live in kickoff(). Refs ADR 0020. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 7fa515d93e3a10d1aadfaf838390de421a08e083) --- src/ophyd_async/core/__init__.py | 2 + src/ophyd_async/core/_data_providers.py | 56 ++++- src/ophyd_async/core/_detector.py | 208 ++++++++++++++---- .../unit_tests/core/test_standard_detector.py | 131 ++++++++++- 4 files changed, 352 insertions(+), 45 deletions(-) diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 135743eefc..571552967e 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -11,6 +11,7 @@ soft_command, ) from ._data_providers import ( + PageableDataProvider, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -297,6 +298,7 @@ def __getattr__(name): "AutoMaxIncrementingPathProvider", "UUIDFilenameProvider", # Data Providers + "PageableDataProvider", "StreamableDataProvider", "StreamResourceInfo", "StreamResourceDataProvider", diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index 8faa7ba069..ab54379d65 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -2,8 +2,9 @@ from collections.abc import AsyncIterator, Sequence from typing import Any -from bluesky.protocols import StreamAsset +from bluesky.protocols import Reading, StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange +from event_model.documents import PartialEventPage from ._signal import SignalR, SignalW from ._utils import ConfinedModel @@ -33,6 +34,59 @@ async def make_stream_docs( yield +class PageableDataProvider: + """For bounded data held in a finite buffer, emitted as event pages. + + Used by data logics whose device must be told how many collections to + expect before it starts acquiring, because it holds a finite buffer that + has to be sized upfront: an areaDetector stats time series (`TSNumPoints`), + or a scaler whose MCA array length is the frame count. Such a device + produces its data as event pages at the end of an event rather than as + stream datums as it goes. + + A step-scan event has a single collection window, so `make_pages` yields + one page of one event which the base `make_readings` extracts to a single + reading; the same `make_pages` serves fly-scan collection. + """ + + collections_written_signal: SignalR[int] + + @abstractmethod + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + """Return a DataKey for each field this provider produces. + + Called before the first exposure is taken. + + :param collections_per_event: this should appear in the shape of each DataKey + """ + + @abstractmethod + def make_pages( + self, collections_written: int, collections_per_event: int + ) -> AsyncIterator[PartialEventPage]: + """Emit event pages for collections written since the last call. + + :param collections_written: how many collections have been written so far + :param collections_per_event: how many collections make up one event + """ + + async def make_readings(self, collections_per_event: int) -> dict[str, Reading]: + """Derive readings from the pages for the collections written so far. + + A step-scan prepare has a single event, so the page holds one event + whose per-key value is the `collections_per_event`-length array; this + extracts to a single reading per key. + """ + collections_written = await self.collections_written_signal.get_value() + readings: dict[str, Reading] = {} + async for page in self.make_pages(collections_written, collections_per_event): + times = page["time"] + for key, values in page["data"].items(): + for value, timestamp in zip(values, times, strict=True): + readings[key] = Reading(value=value, timestamp=timestamp) + return readings + + class StreamResourceInfo(ConfinedModel): """A description of a single StreamResource that should be emitted.""" diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 305564c275..f1fe543d13 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -23,9 +23,14 @@ WritesStreamAssets, ) from event_model import DataKey +from event_model.documents import PartialEventPage from pydantic import Field, NonNegativeInt, PositiveInt, computed_field -from ._data_providers import StreamableDataProvider +from ._data_providers import ( + PageableDataProvider, + StreamableDataProvider, +) +from ._log import logger from ._readable import ( StandardReadable, StandardReadableFormat, @@ -284,7 +289,7 @@ def _all_the_same(collections_written: set[int]) -> int: async def _get_collections_written( - data_providers: Sequence[StreamableDataProvider], + data_providers: Sequence[StreamableDataProvider | PageableDataProvider], reducer: Callable[[set[int]], int] = _all_the_same, ) -> int: """Return a single collections_written value for the given providers. @@ -311,9 +316,16 @@ async def _get_collections_written( class DetectorDataLogic: """Abstract base class for detector data logic and handling. - Implementations must override `prepare_unbounded`. A source that produces a - single value per event, like a plugin scalar, does not need a data logic at - all: register its signal with + Implementations must override exactly one of two tiers, according to how + many collections the data source can produce: + + - `prepare_bounded` for sources that must be told how many collections to + expect because they hold a finite buffer (step scans and flyscans) + - `prepare_unbounded` for sources that work for any number of collections + (step scans and flyscans) + + A source that produces a single value per event, like a plugin scalar, needs + no data logic at all: register its signal with [](#StandardReadable.set_readable_format) instead. """ @@ -321,6 +333,16 @@ class DetectorDataLogic: #: different for each DetectorDataLogic added to a detector datakey_suffix: str = "" + async def prepare_bounded( + self, datakey_name: str, num_collections: int, period: float + ) -> PageableDataProvider: + """Provider works for a known, finite number of collections. + + :param num_collections: total collections the buffer must be sized for + :param period: how long each collection takes, livetime + deadtime + """ + raise NotImplementedError(self) + async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: """Provider can work for an unbounded number of collections.""" raise NotImplementedError(self) @@ -337,17 +359,42 @@ async def stop(self) -> None: _data_logic_supported = functools.partial(_logic_supported, DetectorDataLogic) +def _data_logic_tier(dl: DetectorDataLogic) -> str: + """Return which prepare tier a data logic has overridden. + + Tiers are discovered by method-override detection, the same mechanism used + for trigger logic. A data logic must override exactly one. + """ + if _data_logic_supported(dl.prepare_unbounded): + return "unbounded" + if _data_logic_supported(dl.prepare_bounded): + return "bounded" + raise RuntimeError(f"DataLogic hasn't overridden any prepare_* methods {dl}") + + +def _tier_can_serve(tier: str, num_collections: int) -> bool: + """Whether a tier can serve the requested number of collections. + + - unbounded serves any number, including 0 (infinite) + - bounded serves any finite number, so not 0 + """ + if tier == "unbounded": + return True + return num_collections != 0 + + @dataclass class _PrepareCtx: trigger_info: TriggerInfo streamable_data_providers: Sequence[StreamableDataProvider] + pageable_data_providers: Sequence[PageableDataProvider] collections_written: int @dataclass class _KickoffCtx: trigger_info: TriggerInfo - data_providers: Sequence[StreamableDataProvider] + data_providers: Sequence[StreamableDataProvider | PageableDataProvider] collections_written: int collections_requested: int is_last_kickoff: bool @@ -519,45 +566,80 @@ async def _stage_detector(self) -> None: await self.events_to_kickoff.set(0) async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: - # The only thing that would stop us being able to reuse a provider is - # if the collections_per_event changes, as that would change the - # StreamResource shape. All other TriggerInfo parameters (exposures, livetime, - # etc.) don't affect the data provider configuration. - if ( - self._prepare_ctx + num_collections = trigger_info.number_of_collections + period = trigger_info.livetime + trigger_info.deadtime + # Classify each data logic by the tier it implements, and drop any whose + # tier cannot serve the requested number of collections. This is the rule + # from #1364: rather than raising, a data logic that cannot serve this scan + # is left out with a warning, so a detector carrying a bounded logic can + # still fly for an unbounded number of events with only that logic dropped. + serving: list[tuple[DetectorDataLogic, str]] = [] + for dl in self._data_logics: + tier = _data_logic_tier(dl) + if _tier_can_serve(tier, num_collections): + serving.append((dl, tier)) + else: + logger.warning( + "%s data logic %s cannot serve %d collections, " + "dropping it from this prepare", + tier, + self.name + dl.datakey_suffix, + num_collections, + ) + # Unbounded providers depend on collections_per_event (it sets the + # StreamResource shape), so may be reused across prepares when it is unchanged + # (this avoids reopening files on every step-scan point). Bounded providers are + # never reused: they hold a finite buffer that must be re-armed for each event, + # and re-calling prepare_bounded on every trigger() is what re-arms it. + # (Slice 5 will also invalidate reuse on an exposure-period change, once + # prepare_unbounded uses the period to size its chunks.) + reusable = ( + self._prepare_ctx is not None and self._prepare_ctx.trigger_info.collections_per_event == trigger_info.collections_per_event - ): - # Reuse the existing data providers + ) + if reusable and self._prepare_ctx is not None: streamable_data_providers = self._prepare_ctx.streamable_data_providers else: - # Stop the existing providers if there is a context and make new ones - if self._prepare_ctx: - for data_logic in self._data_logics: - await data_logic.stop() - # Setup the data logic for the right number of collections - streamable_coros: list[Awaitable[StreamableDataProvider]] = [] - for dl in self._data_logics: - if not _data_logic_supported(dl.prepare_unbounded): - msg = f"DataLogic hasn't overridden any prepare_* methods {dl}" - raise RuntimeError(msg) - streamable_coros.append( + # Stop the non-bounded logics we are replacing before making new ones + if self._prepare_ctx is not None: + await asyncio.gather( + *(dl.stop() for dl, tier in serving if tier != "bounded") + ) + streamable_data_providers = await asyncio.gather( + *( dl.prepare_unbounded(self.name + dl.datakey_suffix) + for dl, tier in serving + if tier == "unbounded" ) - streamable_data_providers = await asyncio.gather(*streamable_coros) - + ) + # Bounded providers are always (re)built, re-arming their buffers. + if self._prepare_ctx is not None: + await asyncio.gather( + *(dl.stop() for dl, tier in serving if tier == "bounded") + ) + pageable_data_providers = await asyncio.gather( + *( + dl.prepare_bounded( + self.name + dl.datakey_suffix, num_collections, period + ) + for dl, tier in serving + if tier == "bounded" + ) + ) # Stash the prepare context so we can use it in trigger/kickoff self._prepare_ctx = _PrepareCtx( trigger_info=trigger_info, streamable_data_providers=streamable_data_providers, + pageable_data_providers=pageable_data_providers, collections_written=await _get_collections_written( - streamable_data_providers + [*streamable_data_providers, *pageable_data_providers] ), ) async def _wait_for_index( self, - data_providers: Sequence[StreamableDataProvider], + data_providers: Sequence[StreamableDataProvider | PageableDataProvider], trigger_info: TriggerInfo, initial_collections_written: int, collections_requested: int, @@ -700,10 +782,18 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: # Start the detector acquiring and wait for it to finish. if self._acquire_logic: await self._acquire_logic.start_acquiring() + # A bounded provider has just been re-armed by the re-prepare above, so its + # buffer starts from zero; a streamable provider continues from wherever the + # prepared context left it. A detector never mixes the two. + collectable = [ + *ctx.streamable_data_providers, + *ctx.pageable_data_providers, + ] + initial = 0 if ctx.pageable_data_providers else ctx.collections_written async for update in self._wait_for_index( - data_providers=ctx.streamable_data_providers, + data_providers=collectable, trigger_info=ctx.trigger_info, - initial_collections_written=ctx.collections_written, + initial_collections_written=initial, collections_requested=ctx.trigger_info.collections_per_event, watcher_divisor=1, wait_for_idle=True, @@ -713,12 +803,21 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: @AsyncStatus.wrap async def kickoff(self): ctx = error_if_none(self._prepare_ctx, "Prepare not called") - if not ctx.streamable_data_providers: + # A fly scan collects from streamable providers (as stream datums) or from + # bounded providers (as event pages); either kind can be kicked off, but a + # detector with neither has nothing to collect. + collectable = [ + *ctx.streamable_data_providers, + *ctx.pageable_data_providers, + ] + if not collectable: raise ValueError( - f"Detector {self.name} is not streamable, so cannot kickoff" + f"Detector {self.name} has no collectable data, so cannot kickoff" ) + # Unlike trigger(), kickoff() does not re-arm a bounded buffer, so its + # progress is read live: a fly scan arms once and accumulates across kickoffs. collections_written, events_to_kickoff = await asyncio.gather( - _get_collections_written(ctx.streamable_data_providers), + _get_collections_written(collectable), self.events_to_kickoff.get_value(), ) collections_requested = ( @@ -736,7 +835,7 @@ async def kickoff(self): raise RuntimeError(msg) self._kickoff_ctx = _KickoffCtx( trigger_info=ctx.trigger_info, - data_providers=ctx.streamable_data_providers, + data_providers=collectable, collections_written=collections_written, collections_requested=collections_requested, is_last_kickoff=last_requested_collection == last_expected_collection, @@ -780,20 +879,25 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] if verb not in (_Verb.DESCRIBE, _Verb.READ): return ctx = error_if_none(self._prepare_ctx, "Prepare not run") + cpe = ctx.trigger_info.collections_per_event + # Bounded providers hold a single-event page for this step-scan point, + # which read() extracts back to a reading. + for pdp in ctx.pageable_data_providers: + yield functools.partial( + pdp.make_datakeys if verb is _Verb.DESCRIBE else pdp.make_readings, cpe + ) if verb is _Verb.DESCRIBE: # Streamable providers describe their shape for a step scan, but # produce their data through collect_asset_docs rather than read() for sdp in ctx.streamable_data_providers: - yield functools.partial( - sdp.make_datakeys, ctx.trigger_info.collections_per_event - ) + yield functools.partial(sdp.make_datakeys, cpe) async def describe_collect(self) -> dict[str, DataKey]: ctx = error_if_none(self._prepare_ctx, "Prepare not run") - # Only streamable data providers produce data during collect - coros = [ - dp.make_datakeys(ctx.trigger_info.collections_per_event) - for dp in ctx.streamable_data_providers + # Streamable providers collect stream datums, bounded providers collect pages + cpe = ctx.trigger_info.collections_per_event + coros = [dp.make_datakeys(cpe) for dp in ctx.streamable_data_providers] + [ + dp.make_datakeys(cpe) for dp in ctx.pageable_data_providers ] return await merge_gathered_dicts(coros) @@ -819,10 +923,28 @@ async def collect_asset_docs( ): yield doc + async def _collect_pages(self) -> AsyncIterator[PartialEventPage]: + # Collect event pages for all indices written. Exposed as collect_pages via + # __getattr__ only when a bounded data logic is present, so the bluesky + # bundler dispatches it in place of collect_asset_docs. + ctx = error_if_none(self._prepare_ctx, "Prepare not called") + cpe = ctx.trigger_info.collections_per_event + index = await self.get_index() + for data_provider in ctx.pageable_data_providers: + async for page in data_provider.make_pages( + collections_written=index * cpe, + collections_per_event=cpe, + ): + yield page + async def get_index(self) -> int: ctx = error_if_none(self._prepare_ctx, "Prepare not called") + collectable = [ + *ctx.streamable_data_providers, + *ctx.pageable_data_providers, + ] min_collections_written = await _get_collections_written( - ctx.streamable_data_providers, reducer=min + collectable, reducer=min ) return min_collections_written // ctx.trigger_info.collections_per_event diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 26ecd94f62..23f1b76c73 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -5,6 +5,8 @@ import numpy as np import pytest +from event_model import DataKey +from event_model.documents import PartialEventPage from ophyd_async.core import ( Array1D, @@ -12,6 +14,7 @@ DetectorDataLogic, DetectorTrigger, DetectorTriggerLogic, + PageableDataProvider, Settings, SignalDict, SignalR, @@ -187,6 +190,60 @@ async def test_stream_resource_data_provider_1d_single_collection_is_array(): } } +class MockPageableProvider(PageableDataProvider): + """A finite buffer emitted as event pages, one value per collection.""" + + def __init__(self, datakey_name: str, collections_written_signal: SignalR[int]): + self.datakey_name = datakey_name + self.collections_written_signal = collections_written_signal + self.last_emitted = 0 + + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + return { + self.datakey_name: DataKey( + source="mock", + shape=[collections_per_event], + dtype="array", + dtype_numpy=" self.last_emitted: + new = range(self.last_emitted, events) + page: PartialEventPage = { + "data": {self.datakey_name: [[0] * collections_per_event for _ in new]}, + "time": [0.0 for _ in new], + "timestamps": {self.datakey_name: [0.0 for _ in new]}, + } + self.last_emitted = events + yield page + + +class BoundedOnlyDataLogic(DetectorDataLogic): + """Produces bounded data held in a finite buffer, sized at prepare_bounded.""" + + def __init__(self): + self.collections_written = soft_signal_rw(int) + self.prepare_calls: list[tuple[int, float]] = [] + self.stop_count = 0 + + async def prepare_bounded( + self, datakey_name: str, num_collections: int, period: float + ) -> PageableDataProvider: + self.prepare_calls.append((num_collections, period)) + # A real buffer clears its progress counter when armed + await self.collections_written.set(0) + return MockPageableProvider(datakey_name, self.collections_written) + + async def stop(self) -> None: + self.stop_count += 1 + + def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: + return [datakey_name] + # Parameterized Tests for Logic Combinations @@ -684,7 +741,7 @@ async def test_kickoff_without_streamable_data_raises(): # ...but there is nothing to collect, so it cannot be flown await det.prepare(TriggerInfo(number_of_events=5)) with pytest.raises( - ValueError, match="Detector foo is not streamable, so cannot kickoff" + ValueError, match="Detector foo has no collectable data, so cannot kickoff" ): await det.kickoff() @@ -899,6 +956,78 @@ async def ensure_stopped(self): assert al.shutter_closed is True # unstage() must close the shutter +async def test_bounded_step_scan_reads_derived_from_page(): + """A bounded buffer describes as an array and read() derives one reading.""" + det = StandardDetector(name="foo") + dl = BoundedOnlyDataLogic() + det.add_detector_logics(JustInternalTriggerLogic(), dl) + + ti = TriggerInfo( + number_of_events=1, collections_per_event=5, livetime=0.1, deadtime=0.0 + ) + await det.prepare(ti) + desc = await det.describe() + assert desc["foo"]["shape"] == [5] + # prepare sized the buffer for 5 collections at the 0.1s period + assert dl.prepare_calls == [(5, pytest.approx(0.1))] + + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(5) + assert status.done + assert status.success + # trigger() re-armed the buffer, so prepare_bounded ran a second time + assert dl.prepare_calls == [(5, pytest.approx(0.1)), (5, pytest.approx(0.1))] + reading = await det.read() + assert reading["foo"]["value"] == [0, 0, 0, 0, 0] + + +async def test_bounded_fly_scan_accumulates_across_kickoffs(): + """A bounded buffer is armed once at prepare and not re-armed per kickoff.""" + det = StandardDetector(name="foo") + dl = BoundedOnlyDataLogic() + det.add_detector_logics(JustInternalTriggerLogic(), dl) + + await det.prepare( + TriggerInfo(number_of_events=10, collections_per_event=1, livetime=0.1) + ) + assert dl.prepare_calls == [(10, pytest.approx(0.1))] + + # First kickoff of 5 events, buffer fills 0 -> 5 + await det.events_to_kickoff.set(5) + await det.kickoff() + status = det.complete() + await dl.collections_written.set(5) + pages = [page async for page in det._collect_pages()] + assert pages[0]["data"]["foo"] == [[0]] * 5 + assert status.done + + # Second kickoff of 5 more, buffer continues 5 -> 10 (monotonic, no reset) + await det.events_to_kickoff.set(5) + await det.kickoff() + status = det.complete() + await dl.collections_written.set(10) + pages = [page async for page in det._collect_pages()] + assert pages[0]["data"]["foo"] == [[0]] * 5 + assert status.done + # kickoff() never re-arms: prepare_bounded was only ever called once + assert dl.prepare_calls == [(10, pytest.approx(0.1))] + + +async def test_bounded_dropped_for_infinite_events(caplog): + """A bounded buffer cannot serve an infinite scan, so is dropped with a warning.""" + det = StandardDetector(name="foo") + det.add_detector_logics(JustInternalTriggerLogic(), BoundedOnlyDataLogic()) + + with caplog.at_level("WARNING"): + await det.prepare(TriggerInfo(number_of_events=0)) + + assert "cannot serve 0 collections" in caplog.text + ctx = det._prepare_ctx + assert ctx is not None + assert ctx.pageable_data_providers == [] + + async def test_data_logic_with_no_prepare_methods_raises(): """Test error when DataLogic doesn't override any prepare methods.""" From 768c9f8a3301f80b7cb0a3b1aa0df509ce688a5f Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 17 Jul 2026 15:48:37 +0000 Subject: [PATCH 29/67] feat: resolve a zero livetime from the trigger logic in prepare() A TriggerInfo livetime of 0 means "use whatever the detector currently has set", so forwarding it to a data logic that sizes chunks or a buffer by the exposure period would hand it a period of 0. Resolve it in prepare(), after the trigger logic has been prepared (so the hardware holds the values), by reading the current livetime and deadtime back via default_trigger_info(). trigger_info_from_num_images now reads acquire_time and acquire_period alongside num_images so AD detectors report their real exposure period. Detectors whose exposure is unset (0) are unaffected. Refs ADR 0020. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 1c68afe1b51db19ea61ed72c64b2f662a95156b4) --- src/ophyd_async/core/_detector.py | 22 ++++++++++++ .../epics/adcore/_trigger_logic.py | 21 +++++++++-- .../unit_tests/core/test_standard_detector.py | 35 +++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index f1fe543d13..131f639af3 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -565,6 +565,27 @@ async def _stage_detector(self) -> None: self._kickoff_ctx = None await self.events_to_kickoff.set(0) + async def _resolve_period(self, value: TriggerInfo) -> TriggerInfo: + # A livetime of 0 means "use whatever the detector currently has set". A + # data logic that sizes chunks or a buffer by the exposure period needs a + # real value, so read the current livetime and deadtime back from the + # trigger logic and fill them in. This runs after the trigger logic has been + # prepared, so the hardware already holds the values we read. + if ( + value.livetime == 0.0 + and self._trigger_logic is not None + and _trigger_logic_supported(self._trigger_logic.default_trigger_info) + ): + current = await self._trigger_logic.default_trigger_info() + if current.livetime or current.deadtime: + value = value.model_copy( + update={ + "livetime": current.livetime, + "deadtime": current.deadtime, + } + ) + return value + async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: num_collections = trigger_info.number_of_collections period = trigger_info.livetime + trigger_info.deadtime @@ -724,6 +745,7 @@ async def prepare(self, value: TriggerInfo) -> None: ) # NOTE: this section must come after preparing the trigger logic as we may # use parameters from it to determine datatype for the streams + value = await self._resolve_period(value) await self._update_prepare_context(value) # Tell people how many collections we will acquire for await self.events_to_kickoff.set(value.number_of_events) diff --git a/src/ophyd_async/epics/adcore/_trigger_logic.py b/src/ophyd_async/epics/adcore/_trigger_logic.py index 72a3f5a0f4..3134caf2f9 100644 --- a/src/ophyd_async/epics/adcore/_trigger_logic.py +++ b/src/ophyd_async/epics/adcore/_trigger_logic.py @@ -31,9 +31,24 @@ async def prepare_exposures( async def trigger_info_from_num_images(driver: ADBaseIO) -> TriggerInfo: - """Default TriggerInfo for AD detectors, reading num_images from the driver.""" - num = await driver.num_images.get_value() - return TriggerInfo(collections_per_event=max(1, num)) + """Default TriggerInfo for AD detectors, read back from the driver. + + Reads num_images, acquire_time and acquire_period so the returned + TriggerInfo carries the current livetime and deadtime as well as the + collections per event. This lets prepare() fill in a livetime of 0 ("use + current") with the real exposure period, which a data logic needs to size + its chunks or buffer. + """ + num, livetime, period = await asyncio.gather( + driver.num_images.get_value(), + driver.acquire_time.get_value(), + driver.acquire_period.get_value(), + ) + return TriggerInfo( + collections_per_event=max(1, num), + livetime=livetime, + deadtime=max(0.0, period - livetime), + ) @dataclass diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 23f1b76c73..0942e47807 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -1014,6 +1014,41 @@ async def test_bounded_fly_scan_accumulates_across_kickoffs(): assert dl.prepare_calls == [(10, pytest.approx(0.1))] +@pytest.mark.parametrize( + "requested_livetime,default,expected_period", + [ + # livetime unset -> resolved from the trigger logic's current state + (0.0, TriggerInfo(livetime=0.1, deadtime=0.02), 0.12), + # livetime given -> used as-is, no readback substitution + (0.05, TriggerInfo(livetime=0.1, deadtime=0.02), 0.05), + # livetime unset but trigger logic has nothing set -> stays 0 + (0.0, TriggerInfo(), 0.0), + ], +) +async def test_prepare_resolves_zero_livetime_for_bounded_period( + requested_livetime, default, expected_period +): + """A livetime of 0 is filled in from the trigger logic before sizing a buffer.""" + + class PeriodTriggerLogic(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): + pass + + async def default_trigger_info(self) -> TriggerInfo: + return default + + det = StandardDetector(name="foo") + dl = BoundedOnlyDataLogic() + det.add_detector_logics(PeriodTriggerLogic(), dl) + + await det.prepare( + TriggerInfo( + number_of_events=1, collections_per_event=3, livetime=requested_livetime + ) + ) + assert dl.prepare_calls == [(3, pytest.approx(expected_period))] + + async def test_bounded_dropped_for_infinite_events(caplog): """A bounded buffer cannot serve an infinite scan, so is dropped with a warning.""" det = StandardDetector(name="foo") From 2bd1e473dd0b48a431bacee47b1b0fa2627311f5 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 17 Jul 2026 15:55:28 +0000 Subject: [PATCH 30/67] feat: expose collect_asset_docs/collect_pages by data-logic tier StandardDetector no longer inherits WritesStreamAssets. A detector writes stream assets or emits event pages depending on which data logics it carries, never both, and the bluesky bundler treats the two protocols as mutually exclusive. collect_asset_docs and collect_pages are now served by __getattr__ only when a data logic supporting that tier is present, so the bundler's structural isinstance check matches exactly the one that applies. add_detector_logics raises if a detector mixes bounded and unbounded data logics, the one combination that would expose both. A file writer that also wants stats should carry them in the file as NDAttributes. Refs ADR 0020. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit d67ec0408f494f53dc55d32993fe11da560633cb) --- src/ophyd_async/core/_detector.py | 50 ++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 131f639af3..af4f495d05 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -20,7 +20,6 @@ Stageable, StreamAsset, Triggerable, - WritesStreamAssets, ) from event_model import DataKey from event_model.documents import PartialEventPage @@ -407,7 +406,7 @@ class StandardDetector( Preparable, Flyable, Collectable, - WritesStreamAssets, + HasHints, ): """Detector base class for step and fly scanning detectors. @@ -423,6 +422,14 @@ class StandardDetector( data keys. `trigger()` prepares implicitly, so a step scan never has to do it explicitly. `read_configuration()` and `describe_configuration()` have no such requirement. + + `WritesStreamAssets` (and its `collect_asset_docs`) and + `EventPageCollectable` (and its `collect_pages`) are *not* inherited: a + detector writes stream assets or emits event pages depending on which data + logics it carries, never both, and the bluesky bundler treats the two as + mutually exclusive. The relevant method is exposed via `__getattr__` only + when a data logic supporting it is present, so the bundler's structural + isinstance check sees exactly the one that applies. """ # Logic for the detector @@ -497,6 +504,39 @@ def add_detector_logics( self._data_logics = (*self._data_logics, logic) else: raise TypeError(f"Unknown logic type: {type(logic)}") + # A detector may not mix bounded and unbounded data logics: it would expose + # both collect_pages and collect_asset_docs and produce data from both in a + # fly scan, which the bluesky bundler treats as mutually exclusive. A file + # writer that also wants stats should carry them in the file (as NDAttributes) + # rather than as a separate bounded logic. + has_bounded = any( + _data_logic_supported(dl.prepare_bounded) for dl in self._data_logics + ) + has_unbounded = any( + _data_logic_supported(dl.prepare_unbounded) for dl in self._data_logics + ) + if has_bounded and has_unbounded: + raise TypeError( + f"Detector {self.name} has both bounded and unbounded data logics; " + "these cannot be combined on one detector" + ) + + def __getattr__(self, name: str): + # Expose collect_asset_docs / collect_pages only when a data logic that + # produces that kind of document is present, so the bluesky bundler's + # structural isinstance checks (WritesStreamAssets vs EventPageCollectable) + # match exactly the one that applies. __getattr__ runs only for attributes + # not found normally, and _data_logics has a class-level default, so this + # never recurses. + if name == "collect_asset_docs" and any( + _data_logic_supported(dl.prepare_unbounded) for dl in self._data_logics + ): + return self._collect_asset_docs + if name == "collect_pages" and any( + _data_logic_supported(dl.prepare_bounded) for dl in self._data_logics + ): + return self._collect_pages + raise AttributeError(name) # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: @@ -929,10 +969,12 @@ def _extra_hint_sources(self) -> Iterator[HasHints]: if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): yield _HintedFields(fields) - async def collect_asset_docs( + async def _collect_asset_docs( self, index: int | None = None ) -> AsyncIterator[StreamAsset]: - # Collect stream datum documents for all indices written. + # Collect stream datum documents for all indices written. Exposed as + # collect_asset_docs via __getattr__ only when an unbounded data logic is + # present, so the bluesky bundler dispatches it in place of collect_pages. ctx = error_if_none(self._prepare_ctx, "Prepare not called") if index is None: # The index is optional, and provided for fly scans, if there is From e5896c97bbe8590f286dcee71c831a1dbf162ad1 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 20 Jul 2026 08:58:26 +0000 Subject: [PATCH 31/67] refactor: address review feedback on data-logic tiers Responds to review comments on PR #1367: - rename trigger_info_from_num_images -> trigger_info_from_driver, since it now reads acquire_time and acquire_period too, not just num_images - introduce a _DataLogicTier enum instead of bare tier strings, for tighter typing of _data_logic_tier / _tier_can_serve and their call sites - list the offending logics and their types in the bounded/unbounded mix TypeError - raise a standard ' object has no attribute ' AttributeError from __getattr__ (no base class defines __getattr__ to delegate to) Also finishes the dispatch slice with dedicated tests: an unbounded logic looks like WritesStreamAssets only, a bounded logic like EventPageCollectable only, a single/readable logic like neither, an unknown attribute raises a standard AttributeError, and mixing bounded with unbounded logics raises. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit dac4347840aa94360289be94eeaad319e65690f7) --- docs/how-to/implement-ad-detector.md | 5 +- src/ophyd_async/core/_detector.py | 69 +++++++++++++------ src/ophyd_async/epics/adandor.py | 4 +- src/ophyd_async/epics/adaravis.py | 4 +- src/ophyd_async/epics/adcore/__init__.py | 4 +- .../epics/adcore/_trigger_logic.py | 2 +- src/ophyd_async/epics/adkinetix.py | 4 +- src/ophyd_async/epics/admerlin.py | 4 +- src/ophyd_async/epics/adpilatus.py | 4 +- src/ophyd_async/epics/adsimdetector.py | 4 +- src/ophyd_async/epics/advimba.py | 4 +- .../unit_tests/core/test_standard_detector.py | 65 +++++++++++++++++ 12 files changed, 133 insertions(+), 40 deletions(-) diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 8f78000360..2b3590d744 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -39,8 +39,9 @@ To preserve hardware state in plans like [`bp.count`](#bluesky.plans.count) and [`bps.trigger_and_read`](#bluesky.plan_stubs.trigger_and_read) that call `trigger()` without a preceding `prepare()`, implement: - `default_trigger_info()` - Return the [](#TriggerInfo) to use for the implicit - prepare. For AD detectors call `await trigger_info_from_num_images(self.driver)` to - read back the current `num_images` from the driver rather than resetting it to 1. + prepare. For AD detectors call `await trigger_info_from_driver(self.driver)` to + read back the current `num_images`, `acquire_time` and `acquire_period` from the + driver rather than resetting them. diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index af4f495d05..85eab117e2 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -358,26 +358,37 @@ async def stop(self) -> None: _data_logic_supported = functools.partial(_logic_supported, DetectorDataLogic) -def _data_logic_tier(dl: DetectorDataLogic) -> str: +class _DataLogicTier(Enum): + """Which prepare tier a data logic implements. + + Discovered by method-override detection (see `_data_logic_tier`); a data + logic must override exactly one of the two `prepare_*` methods. + """ + + UNBOUNDED = "unbounded" + BOUNDED = "bounded" + + +def _data_logic_tier(dl: DetectorDataLogic) -> _DataLogicTier: """Return which prepare tier a data logic has overridden. Tiers are discovered by method-override detection, the same mechanism used for trigger logic. A data logic must override exactly one. """ if _data_logic_supported(dl.prepare_unbounded): - return "unbounded" + return _DataLogicTier.UNBOUNDED if _data_logic_supported(dl.prepare_bounded): - return "bounded" + return _DataLogicTier.BOUNDED raise RuntimeError(f"DataLogic hasn't overridden any prepare_* methods {dl}") -def _tier_can_serve(tier: str, num_collections: int) -> bool: +def _tier_can_serve(tier: _DataLogicTier, num_collections: int) -> bool: """Whether a tier can serve the requested number of collections. - unbounded serves any number, including 0 (infinite) - bounded serves any finite number, so not 0 """ - if tier == "unbounded": + if tier is _DataLogicTier.UNBOUNDED: return True return num_collections != 0 @@ -509,16 +520,23 @@ def add_detector_logics( # fly scan, which the bluesky bundler treats as mutually exclusive. A file # writer that also wants stats should carry them in the file (as NDAttributes) # rather than as a separate bounded logic. - has_bounded = any( - _data_logic_supported(dl.prepare_bounded) for dl in self._data_logics - ) - has_unbounded = any( - _data_logic_supported(dl.prepare_unbounded) for dl in self._data_logics - ) - if has_bounded and has_unbounded: + bounded = [ + dl for dl in self._data_logics if _data_logic_supported(dl.prepare_bounded) + ] + unbounded = [ + dl + for dl in self._data_logics + if _data_logic_supported(dl.prepare_unbounded) + ] + if bounded and unbounded: + + def _describe(logics: Sequence[DetectorDataLogic]) -> str: + return ", ".join(type(dl).__name__ for dl in logics) + raise TypeError( - f"Detector {self.name} has both bounded and unbounded data logics; " - "these cannot be combined on one detector" + f"Detector {self.name} has both bounded data logics " + f"({_describe(bounded)}) and unbounded data logics " + f"({_describe(unbounded)}); these cannot be combined on one detector" ) def __getattr__(self, name: str): @@ -536,7 +554,12 @@ def __getattr__(self, name: str): _data_logic_supported(dl.prepare_bounded) for dl in self._data_logics ): return self._collect_pages - raise AttributeError(name) + # No base class defines __getattr__, so there is nothing to delegate to; + # raise the same AttributeError the default attribute lookup would, so + # hasattr()/getattr() and error messages behave as normal. + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {name!r}" + ) # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: @@ -634,7 +657,7 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: # from #1364: rather than raising, a data logic that cannot serve this scan # is left out with a warning, so a detector carrying a bounded logic can # still fly for an unbounded number of events with only that logic dropped. - serving: list[tuple[DetectorDataLogic, str]] = [] + serving: list[tuple[DetectorDataLogic, _DataLogicTier]] = [] for dl in self._data_logics: tier = _data_logic_tier(dl) if _tier_can_serve(tier, num_collections): @@ -643,7 +666,7 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: logger.warning( "%s data logic %s cannot serve %d collections, " "dropping it from this prepare", - tier, + tier.value, self.name + dl.datakey_suffix, num_collections, ) @@ -665,19 +688,23 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: # Stop the non-bounded logics we are replacing before making new ones if self._prepare_ctx is not None: await asyncio.gather( - *(dl.stop() for dl, tier in serving if tier != "bounded") + *( + dl.stop() + for dl, tier in serving + if tier is not _DataLogicTier.BOUNDED + ) ) streamable_data_providers = await asyncio.gather( *( dl.prepare_unbounded(self.name + dl.datakey_suffix) for dl, tier in serving - if tier == "unbounded" + if tier is _DataLogicTier.UNBOUNDED ) ) # Bounded providers are always (re)built, re-arming their buffers. if self._prepare_ctx is not None: await asyncio.gather( - *(dl.stop() for dl, tier in serving if tier == "bounded") + *(dl.stop() for dl, tier in serving if tier is _DataLogicTier.BOUNDED) ) pageable_data_providers = await asyncio.gather( *( @@ -685,7 +712,7 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: self.name + dl.datakey_suffix, num_collections, period ) for dl, tier in serving - if tier == "bounded" + if tier is _DataLogicTier.BOUNDED ) ) # Stash the prepare context so we can use it in trigger/kickoff diff --git a/src/ophyd_async/epics/adandor.py b/src/ophyd_async/epics/adandor.py index 6ceb2cdf3e..66d72c7c30 100644 --- a/src/ophyd_async/epics/adandor.py +++ b/src/ophyd_async/epics/adandor.py @@ -21,7 +21,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from ophyd_async.epics.core import PvSuffix @@ -79,7 +79,7 @@ async def prepare_edge(self, num: int, livetime: float): await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE, livetime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class AndorDetector(AreaDetector[Andor2DriverIO]): diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index ffd072c91b..228364846f 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -23,7 +23,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .adgenicam import get_camera_deadtime from .core import PvSuffix @@ -80,7 +80,7 @@ async def prepare_edge(self, num: int, livetime: float): await prepare_exposures(self.driver, num, livetime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class AravisDetector(AreaDetector[AravisDriverIO]): diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 960ca33146..9a69cd5fc1 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -42,7 +42,7 @@ from ._trigger_logic import ( ADContAcqTriggerLogic, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) __all__ = [ @@ -68,7 +68,7 @@ # TriggerLogic "prepare_exposures", "ADContAcqTriggerLogic", - "trigger_info_from_num_images", + "trigger_info_from_driver", # AcquireLogic "ADAcquireLogic", "ADContAcqAcquireLogic", diff --git a/src/ophyd_async/epics/adcore/_trigger_logic.py b/src/ophyd_async/epics/adcore/_trigger_logic.py index 3134caf2f9..49fb3cab54 100644 --- a/src/ophyd_async/epics/adcore/_trigger_logic.py +++ b/src/ophyd_async/epics/adcore/_trigger_logic.py @@ -30,7 +30,7 @@ async def prepare_exposures( await asyncio.gather(*coros) -async def trigger_info_from_num_images(driver: ADBaseIO) -> TriggerInfo: +async def trigger_info_from_driver(driver: ADBaseIO) -> TriggerInfo: """Default TriggerInfo for AD detectors, read back from the driver. Reads num_images, acquire_time and acquire_period so the returned diff --git a/src/ophyd_async/epics/adkinetix.py b/src/ophyd_async/epics/adkinetix.py index 1d3774f0d2..72a81037d4 100644 --- a/src/ophyd_async/epics/adkinetix.py +++ b/src/ophyd_async/epics/adkinetix.py @@ -22,7 +22,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .core import PvSuffix @@ -81,7 +81,7 @@ async def prepare_level(self, num: int): await prepare_exposures(self.driver, num) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class KinetixDetector(AreaDetector[KinetixDriverIO]): diff --git a/src/ophyd_async/epics/admerlin.py b/src/ophyd_async/epics/admerlin.py index 04bba91b0e..401f6dbded 100644 --- a/src/ophyd_async/epics/admerlin.py +++ b/src/ophyd_async/epics/admerlin.py @@ -26,7 +26,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) __all__ = [ @@ -106,7 +106,7 @@ async def prepare_edge(self, num: int, livetime: float): await prepare_exposures(self.driver, num, livetime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class MerlinDetector(AreaDetector[MerlinDriverIO]): diff --git a/src/ophyd_async/epics/adpilatus.py b/src/ophyd_async/epics/adpilatus.py index 2b038cea81..6008daba78 100644 --- a/src/ophyd_async/epics/adpilatus.py +++ b/src/ophyd_async/epics/adpilatus.py @@ -25,7 +25,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .core import PvSuffix @@ -122,7 +122,7 @@ async def prepare_level(self, num: int): await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class PilatusDetector(AreaDetector[PilatusDriverIO]): diff --git a/src/ophyd_async/epics/adsimdetector.py b/src/ophyd_async/epics/adsimdetector.py index 1a47d7c895..c5ed33bb16 100644 --- a/src/ophyd_async/epics/adsimdetector.py +++ b/src/ophyd_async/epics/adsimdetector.py @@ -18,7 +18,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) __all__ = [ @@ -37,7 +37,7 @@ async def prepare_internal(self, num: int, livetime: float, deadtime: float): await prepare_exposures(self.driver, num, livetime, deadtime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class SimDetector(AreaDetector[ADBaseIO]): diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index 5d776af560..ac76100473 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -26,7 +26,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .adgenicam import get_camera_deadtime @@ -127,7 +127,7 @@ async def prepare_level(self, num: int): await prepare_exposures(self.driver, num) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class VimbaDetector(AreaDetector[VimbaDriverIO]): diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 0942e47807..097f7dddd1 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -5,6 +5,7 @@ import numpy as np import pytest +from bluesky.protocols import EventPageCollectable, WritesStreamAssets from event_model import DataKey from event_model.documents import PartialEventPage @@ -1321,3 +1322,67 @@ async def test_detector_add_config_signals_is_deprecated(): with pytest.deprecated_call(): det.add_config_signals(signal) assert set(await det.read_configuration()) == {signal.name} + + +async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): + """An unbounded logic exposes collect_asset_docs, and only that. + + The bluesky bundler picks up stream assets via a structural isinstance + against WritesStreamAssets (a hasattr check for collect_asset_docs). A + detector must look like exactly one of WritesStreamAssets / + EventPageCollectable so the bundler routes it down a single path. + """ + det = StandardDetector() + det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + + assert isinstance(det, WritesStreamAssets) + assert not isinstance(det, EventPageCollectable) + assert hasattr(det, "collect_asset_docs") + assert not hasattr(det, "collect_pages") + + +async def test_bounded_logic_looks_like_event_page_collectable(): + """A bounded logic exposes collect_pages, and only that.""" + det = StandardDetector() + det.add_detector_logics(BoundedOnlyDataLogic()) + + assert isinstance(det, EventPageCollectable) + assert not isinstance(det, WritesStreamAssets) + assert hasattr(det, "collect_pages") + assert not hasattr(det, "collect_asset_docs") + + +async def test_no_data_logic_looks_like_neither(): + """A detector with no data logic emits neither stream assets nor event pages.""" + det = StandardDetector() + det.add_detector_logics(JustInternalTriggerLogic()) + + assert not isinstance(det, WritesStreamAssets) + assert not isinstance(det, EventPageCollectable) + assert not hasattr(det, "collect_asset_docs") + assert not hasattr(det, "collect_pages") + + +async def test_missing_attribute_raises_standard_attribute_error(): + """__getattr__ falls through to a normal AttributeError for other names.""" + det = StandardDetector() + with pytest.raises( + AttributeError, + match=r"'StandardDetector' object has no attribute 'does_not_exist'", + ): + det.does_not_exist # noqa: B018 + + +async def test_mixing_bounded_and_unbounded_logics_raises(tmp_path): + """Bounded and unbounded logics are mutually exclusive on one detector.""" + det = StandardDetector() + with pytest.raises( + TypeError, + match=( + r"has both bounded data logics \(BoundedOnlyDataLogic\) and unbounded " + r"data logics \(StreamableOnlyDataLogic\)" + ), + ): + det.add_detector_logics( + BoundedOnlyDataLogic(), StreamableOnlyDataLogic(tmp_path) + ) From 3d93ceeaae432e52b1b4a36d85a2f24ccb5d6ea0 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 20 Jul 2026 09:10:16 +0000 Subject: [PATCH 32/67] feat: pass exposure period to prepare_unbounded, sizing HDF chunks (#1309) prepare_unbounded now takes the resolved frame period (livetime + deadtime), so a data logic can size its chunks or flush rate from the scan parameters instead of hardcoding them. The period is also added to the provider-reuse key: a period change starts a new file, because the chunk shape is baked into the StreamResource at construction and areaDetector's num_frames_chunks can only be set before capture starts. ADHDFDataLogic gains an opt-in flush_period: set it and the HDF chunk is sized so the file flushes at roughly that rate (e.g. 400 Hz frames with a 0.5s flush period -> 200 frames per chunk); left None it reads the chunk size back from the IOC as before, so existing detectors are unchanged. The other in-tree overrides (multipart AD, PandA, Odin, sim blob) accept the period but keep their current behaviour for now. test_step_scan_keep_numimages now expects a fresh StreamResource when the exposure changes, per ADR 0020's reuse rule. Fixes #1309 Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 66fd6f1a0c83e60e062544159d06f32cec9b69cc) --- src/ophyd_async/core/_detector.py | 29 ++++++++----- src/ophyd_async/epics/adcore/_data_logic.py | 43 +++++++++++++++---- src/ophyd_async/fastcs/odin/_data_logic.py | 6 ++- src/ophyd_async/fastcs/panda/_data_logic.py | 7 ++- src/ophyd_async/sim/_blob_data_logic.py | 6 ++- .../unit_tests/core/test_standard_detector.py | 6 ++- .../epics/adcore/test_data_logic.py | 41 ++++++++++++++++++ .../unit_tests/epics/adcore/test_use_cases.py | 21 ++++++--- 8 files changed, 129 insertions(+), 30 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 85eab117e2..f64a888838 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -342,8 +342,15 @@ async def prepare_bounded( """ raise NotImplementedError(self) - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - """Provider can work for an unbounded number of collections.""" + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: + """Provider can work for an unbounded number of collections. + + :param period: how long each collection takes, livetime + deadtime, so + the provider can size its chunks or set a flush rate. 0 means "use + whatever is currently set on the hardware". + """ raise NotImplementedError(self) def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @@ -671,16 +678,18 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: num_collections, ) # Unbounded providers depend on collections_per_event (it sets the - # StreamResource shape), so may be reused across prepares when it is unchanged - # (this avoids reopening files on every step-scan point). Bounded providers are + # StreamResource shape) and on the period (it sets the chunk shape), so may be + # reused across prepares when both are unchanged (this avoids reopening files on + # every step-scan point). A period change invalidates reuse because the chunk + # shape is baked into the StreamResource at construction and, for areaDetector, + # num_frames_chunks can only be set before capture starts. Bounded providers are # never reused: they hold a finite buffer that must be re-armed for each event, # and re-calling prepare_bounded on every trigger() is what re-arms it. - # (Slice 5 will also invalidate reuse on an exposure-period change, once - # prepare_unbounded uses the period to size its chunks.) + previous = self._prepare_ctx.trigger_info if self._prepare_ctx else None reusable = ( - self._prepare_ctx is not None - and self._prepare_ctx.trigger_info.collections_per_event - == trigger_info.collections_per_event + previous is not None + and previous.collections_per_event == trigger_info.collections_per_event + and previous.livetime + previous.deadtime == period ) if reusable and self._prepare_ctx is not None: streamable_data_providers = self._prepare_ctx.streamable_data_providers @@ -696,7 +705,7 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: ) streamable_data_providers = await asyncio.gather( *( - dl.prepare_unbounded(self.name + dl.datakey_suffix) + dl.prepare_unbounded(self.name + dl.datakey_suffix, period) for dl, tier in serving if tier is _DataLogicTier.UNBOUNDED ) diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 2542ed1838..af1ce143bd 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -164,17 +164,31 @@ class ADHDFDataLogic(DetectorDataLogic): plugins: Sequence[NDPluginBaseIO] = () datakey_suffix: str = "" hinted: bool = True - - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + #: Target seconds between HDF flushes. When set, the chunk is sized from + #: this and the frame period so the file flushes at roughly this rate + #: (#1309). Left as None, the chunk size is read back from the IOC, the + #: behaviour before #1309. + flush_period: float | None = None + + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: # Work out where to write path_info = self.path_provider(datakey_name) - # Determine number of frames that will be saved per HDF chunk. - # On a fresh IOC startup, this is set to zero until the first capture, - # so if it is zero, set it to 1. - frames_per_chunk = await self.writer.num_frames_chunks.get_value() - if frames_per_chunk == 0: - frames_per_chunk = 1 + # Size the HDF chunk from the frame period and the target flush period: + # e.g. 400 Hz frames (period 2.5 ms) with a 0.5 s flush period gives 200 + # frames per chunk, so the file is flushed at ~2 Hz (#1309). Only when a + # flush_period is configured and the period is known (non-zero); otherwise + # fall back to reading the chunk size back from the IOC, forcing a + # fresh-startup 0 to 1. + if self.flush_period is not None and period > 0: + frames_per_chunk = max(1, round(self.flush_period / period)) await self.writer.num_frames_chunks.set(frames_per_chunk) + else: + frames_per_chunk = await self.writer.num_frames_chunks.get_value() + if frames_per_chunk == 0: + frames_per_chunk = 1 + await self.writer.num_frames_chunks.set(frames_per_chunk) # Setup the HDF writer await asyncio.gather( self.writer.chunk_size_auto.set(True), @@ -251,7 +265,12 @@ class ADMultipartDataLogic(DetectorDataLogic): datakey_suffix: str = "" hinted: bool = True - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: + # A multipart writer writes one file per frame, so there is no chunk to + # size from the period. + del period # Work out where to write path_info = self.path_provider(datakey_name) # Setup the file writer @@ -373,6 +392,7 @@ def hdf( | Callable[[ADBaseIO], NDArrayDescription] | None = None, hinted: bool = True, + flush_period: float | None = None, ) -> "ADWriterFactory[NDFileHDF5IO]": """Create a factory for an HDF5 file writer. @@ -389,6 +409,10 @@ def hdf( when the shape/type comes from a plugin rather than the main driver (e.g. an ROI plugin). :param hinted: Include the image in hints. Set False for BestEffortCallback. + :param flush_period: Target seconds between HDF flushes. When set, the + chunk is sized from this and the frame period so the file flushes at + roughly this rate (#1309). Left as ``None`` (the default), the chunk + size is read back from the IOC as before. """ return ADWriterFactory( writer_cls=NDFileHDF5IO, @@ -404,6 +428,7 @@ def hdf( plugins=list(plugins), datakey_suffix=datakey_suffix, hinted=hinted, + flush_period=flush_period, ), ) diff --git a/src/ophyd_async/fastcs/odin/_data_logic.py b/src/ophyd_async/fastcs/odin/_data_logic.py index 282e707f6f..ef8e67363a 100644 --- a/src/ophyd_async/fastcs/odin/_data_logic.py +++ b/src/ophyd_async/fastcs/odin/_data_logic.py @@ -31,7 +31,11 @@ def __init__( self.detector_bit_depth = detector_bit_depth self.pixel_mask = pixel_mask - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: + # Odin sizes its own chunks, so the frame period is not used here yet. + del period # Work out where to write path_info = self.path_provider(datakey_name) # Get the current bit depth diff --git a/src/ophyd_async/fastcs/panda/_data_logic.py b/src/ophyd_async/fastcs/panda/_data_logic.py index ef8ec675e2..92f2630898 100644 --- a/src/ophyd_async/fastcs/panda/_data_logic.py +++ b/src/ophyd_async/fastcs/panda/_data_logic.py @@ -25,7 +25,12 @@ def __init__( self.path_provider = path_provider self.data_block = data_block - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: + # TODO: derive the PandA flush period / chunk size from `period` once the + # IOC exposes the chunk-size signal (see the chunk_shape TODO below). + del period # Work out where to write path_info = self.path_provider(datakey_name) # Set create dir depth first to guarantee that callback when setting diff --git a/src/ophyd_async/sim/_blob_data_logic.py b/src/ophyd_async/sim/_blob_data_logic.py index 01073f0f62..ab234d172b 100644 --- a/src/ophyd_async/sim/_blob_data_logic.py +++ b/src/ophyd_async/sim/_blob_data_logic.py @@ -25,7 +25,11 @@ def __init__( self.path_provider = path_provider self.pattern_generator = pattern_generator - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: + # The sim blob writer uses a fixed chunk shape, so the period is unused. + del period # Work out where to write path_info = self.path_provider(datakey_name) # Open the file diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 097f7dddd1..7dfa1a48c3 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -142,7 +142,9 @@ def __init__(self, tmp_path, datakey_suffix: str = ""): self.tmp_path = tmp_path self.datakey_suffix = datakey_suffix - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def prepare_unbounded( + self, datakey_name: str, period: float + ) -> StreamableDataProvider: resource = StreamResourceInfo( data_key=datakey_name, shape=(10, 15), @@ -1289,7 +1291,7 @@ async def test_data_logic_not_implemented_errors(): logic = DetectorDataLogic() with pytest.raises(NotImplementedError): - await logic.prepare_unbounded("test") + await logic.prepare_unbounded("test", 0.1) # stop() should not raise (has default implementation) await logic.stop() # Should pass diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 9e940744b3..41ababfb28 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -125,6 +125,47 @@ async def test_prepare_multipart( ) +@pytest.mark.parametrize( + "flush_period,livetime,deadtime,expected_frames_per_chunk", + [ + # 0.5s flush / 0.01s period -> 50 frames per chunk (#1309) + (0.5, 0.01, 0.0, 50), + # 0.5s flush / 0.05s period -> 10 frames per chunk + (0.5, 0.04, 0.01, 10), + # A sub-frame flush period floors to a single frame per chunk + (0.001, 0.05, 0.0, 1), + ], +) +async def test_hdf_chunk_sized_from_flush_period( + static_path_provider: StaticPathProvider, + flush_period: float, + livetime: float, + deadtime: float, + expected_frames_per_chunk: int, +): + """A configured flush_period sizes the HDF chunk from the frame period.""" + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADWriterFactory.hdf(static_path_provider, flush_period=flush_period), + ) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + set_mock_value(det.driver.data_type, adcore.ADBaseDataType.UINT16) + writer = det.get_plugin("hdf", adcore.NDPluginFileIO) + set_mock_value(writer.file_path_exists, True) + await det.prepare( + TriggerInfo(livetime=livetime, deadtime=deadtime, number_of_events=3) + ) + assert await writer.num_frames_chunks.get_value() == expected_frames_per_chunk + (sr, *_) = [doc async for doc in det.collect_asset_docs(3)] + assert sr[1]["parameters"]["chunk_shape"] == ( + expected_frames_per_chunk, + 768, + 1024, + ) + + @pytest.mark.parametrize( "factory_cls,is_hdf", [ diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index e0c9a7c04d..4ea02c3e80 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -656,7 +656,12 @@ async def test_step_scan_keep_numimages( }, ) - # Check we can prepare and change the exposure + # Check we can prepare and change the exposure. Changing the exposure changes + # the frame period, which (per ADR 0020) determines the HDF chunk shape, so a + # new StreamResource is opened rather than the old file being reused. Here the + # chunk shape is unchanged because this writer leaves flush_period unset and so + # reads the chunk size back from the IOC, but the period change still starts a + # fresh file. await det.prepare( TriggerInfo(collections_per_event=42, livetime=0.01, exposure_timeout=0.1) ) @@ -666,16 +671,20 @@ async def test_step_scan_keep_numimages( assert await det.driver.acquire_time.get_value() == 0.01 await det.trigger() readings = await det.read() + # The stats plugin is opted into read() by this test, so its signals are + # still reported alongside the file-backed data assert "det-stats-total" in readings - docs = [doc async for doc in det.collect_asset_docs()] - (doc,) = docs - assert doc == ( + new_sr, new_sd = [doc async for doc in det.collect_asset_docs()] + assert new_sr[0] == "stream_resource" + assert new_sr[1]["uid"] != sr[1]["uid"] + assert new_sr[1]["parameters"]["chunk_shape"] == (1, 768, 1024) + assert new_sd == ( "stream_datum", { "descriptor": "", - "indices": {"start": 1, "stop": 2}, + "indices": {"start": 0, "stop": 2}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": sr[1]["uid"], + "stream_resource": new_sr[1]["uid"], "uid": ANY, }, ) From e05a4a0ecf6cd13e732079b7592df3b4816816cf Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 20 Jul 2026 09:26:30 +0000 Subject: [PATCH 33/67] feat: add stats time-series bounded data logic for writer-less detectors Adds the areaDetector NDPluginStats time-series interface (NDStatsIO.ts_* signals plus NDStatsTSControl/NDStatsTSAcquireMode enums) and a bounded-tier StatsTimeSeriesDataLogic that exposes it as event pages, so a detector with no file writer can still record statistics. One plugin has one time-series control shared across many arrays, so one logic covers many arrays with the control embedded (defaulting to the Total series). prepare_bounded sizes ts_num_points, sets fixed-length mode and writes Erase/Start to arm and clear the buffer before the detector's frames feed it; the detector's own acquire logic drives the camera as usual. A file-writing detector should pull stats into the file as NDAttributes instead. The TS PV suffixes are derived from the ADCore templates and are exercised by mock unit tests here; they want verifying against a live IOC. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 32a60a6b933fea4efbca3049356b4eaa87e6b12f) --- src/ophyd_async/epics/adcore/__init__.py | 7 + src/ophyd_async/epics/adcore/_io.py | 42 +++++ .../epics/adcore/_stats_time_series.py | 143 ++++++++++++++++++ .../epics/adcore/test_stats_time_series.py | 137 +++++++++++++++++ 4 files changed, 329 insertions(+) create mode 100644 src/ophyd_async/epics/adcore/_stats_time_series.py create mode 100644 tests/unit_tests/epics/adcore/test_stats_time_series.py diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 9a69cd5fc1..689fd57248 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -30,6 +30,8 @@ NDROIStatIO, NDROIStatNIO, NDStatsIO, + NDStatsTSAcquireMode, + NDStatsTSControl, ) from ._ndattribute import ( NDAttributeDataType, @@ -39,6 +41,7 @@ ndattributes_to_xml, ) from ._plan_stubs import setup_ndattributes, setup_ndstats_sum +from ._stats_time_series import StatsTimeSeriesDataLogic, StatsTimeSeriesProvider from ._trigger_logic import ( ADContAcqTriggerLogic, prepare_exposures, @@ -56,6 +59,8 @@ "NDPluginBaseIO", "NDROIIO", "NDStatsIO", + "NDStatsTSControl", + "NDStatsTSAcquireMode", "NDROIStatNIO", "NDROIStatIO", "NDCBFlushOnSoftTrgMode", @@ -77,6 +82,8 @@ "ADHDFDataLogic", "ADMultipartDataLogic", "ADWriterFactory", + "StatsTimeSeriesDataLogic", + "StatsTimeSeriesProvider", # Detector "AreaDetector", "ContAcqDetector", diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 6740536258..6fef30a68f 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -1,7 +1,10 @@ from typing import Annotated as A from typing import TypeVar +import numpy as np + from ophyd_async.core import ( + Array1D, DeviceVector, EnableDisable, SignalR, @@ -193,6 +196,31 @@ class NDProcessIO(NDPluginBaseIO): scale: A[SignalRW[float], PvSuffix.rbv("Scale")] +class NDStatsTSControl(StrictEnum): + """Time-series control command for an NDPluginStats time series. + + Mirrors the ``TSControl`` record in ADCore/db/NDPluginTimeSeries.template. + Writing ``ERASE_START`` clears the buffer and resets the current point to 0 + before it starts filling again. + """ + + ERASE_START = "Erase/Start" + START = "Start" + STOP = "Stop" + READ = "Read" + + +class NDStatsTSAcquireMode(StrictEnum): + """Whether an NDPluginStats time series stops when full or wraps around. + + Mirrors the ``TSAcquireMode`` record in + ADCore/db/NDPluginTimeSeries.template. + """ + + FIXED_LENGTH = "Fixed length" + CIRCULAR_BUFFER = "Circular buffer" + + class NDStatsIO(StandardReadable, NDPluginBaseIO): """Plugin for computing statistics from an image or ROI within an image. @@ -221,6 +249,20 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] net: A[SignalR[float], PvSuffix("Net_RBV")] + # Time series. NDStats.template includes NDPluginTimeSeries.template with an + # inner "TS:" record prefix, so each statistic is also exposed as a fixed- + # length array. ts_num_points sizes the buffer, ts_control erases and starts + # it, ts_current_point reports progress and ts_total holds the Total series. + # These are not registered as readables: the scalars above serve read(), and + # the arrays are consumed by StatsTimeSeriesDataLogic as event pages. + # These suffixes are derived from the ADCore templates and want verifying + # against a live IOC. + ts_control: A[SignalRW[NDStatsTSControl], PvSuffix("TS:TSControl")] + ts_num_points: A[SignalRW[int], PvSuffix.rbv("TS:TSNumPoints")] + ts_current_point: A[SignalR[int], PvSuffix("TS:TSCurrentPoint")] + ts_acquiring: A[SignalR[bool], PvSuffix("TS:TSAcquiring")] + ts_acquire_mode: A[SignalRW[NDStatsTSAcquireMode], PvSuffix.rbv("TS:TSAcquireMode")] + ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TS:TSTotal")] # Centroid statistics compute_centroid: A[ SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py new file mode 100644 index 0000000000..68a76b592c --- /dev/null +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -0,0 +1,143 @@ +import asyncio +import time +from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass, field + +import numpy as np +from event_model import DataKey +from event_model.documents import PartialEventPage + +from ophyd_async.core import ( + Array1D, + DetectorDataLogic, + EnableDisable, + PageableDataProvider, + SignalR, +) + +from ._io import NDStatsIO, NDStatsTSAcquireMode, NDStatsTSControl + + +class StatsTimeSeriesProvider(PageableDataProvider): + """Emits an `NDPluginStats` time series as event pages. + + The plugin holds one fixed-length array per statistic, filled by NDArray + callbacks as the detector acquires. Progress is read from + ``ts_current_point``, and each configured array is sliced into + ``collections_per_event``-length chunks, one per event. + + :param arrays: datakey (already suffixed) to the array signal that backs it + :param current_point: the plugin's ``ts_current_point`` progress signal + """ + + def __init__( + self, + arrays: Mapping[str, SignalR[Array1D[np.float64]]], + current_point: SignalR[int], + ) -> None: + self.arrays = dict(arrays) + self.collections_written_signal = current_point + self.last_emitted = 0 + + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + return { + datakey: DataKey( + source=signal.source, + shape=[collections_per_event], + dtype="array", + dtype_numpy=" AsyncIterator[PartialEventPage]: + events = collections_written // collections_per_event + if events <= self.last_emitted: + return + new = range(self.last_emitted, events) + # A single timestamp per event: the plugin does not expose a per-point + # time, so use the read time. + now = time.time() + values = { + datakey: await signal.get_value() for datakey, signal in self.arrays.items() + } + page: PartialEventPage = { + "data": { + datakey: [ + list( + array[ + event * collections_per_event : (event + 1) + * collections_per_event + ] + ) + for event in new + ] + for datakey, array in values.items() + }, + "time": [now for _ in new], + "timestamps": {datakey: [now for _ in new] for datakey in values}, + } + self.last_emitted = events + yield page + + +@dataclass +class StatsTimeSeriesDataLogic(DetectorDataLogic): + """Bounded data logic for an `NDPluginStats` time series. + + For detectors that write no file: the stats plugin holds each statistic in a + fixed-length buffer that must be sized before acquisition, so this is a + bounded (`prepare_bounded`) data logic that emits event pages. One plugin has + one time-series control (`ts_control`, `ts_num_points`) shared across many + arrays, so one logic covers many arrays with the control embedded. + + The buffer is sized and erased in `prepare_bounded` by writing + ``Erase/Start`` to ``ts_control``; the detector's acquire logic then drives + the camera as usual and its frames feed the time series via NDArray + callbacks. A detector that writes a file should pull stats into the file as + NDAttributes instead (see `ADHDFDataLogic`). + + :param stats: the stats plugin whose time series to read + :param stat_signals: datakey suffix to the array signal for each statistic to + expose; defaults to the ``Total`` series under the bare datakey name + """ + + stats: NDStatsIO + stat_signals: Sequence[tuple[str, SignalR[Array1D[np.float64]]]] = field( + default_factory=list + ) + datakey_suffix: str = "" + + def _arrays(self, datakey_name: str) -> dict[str, SignalR[Array1D[np.float64]]]: + signals = self.stat_signals or [("", self.stats.ts_total)] + return {datakey_name + suffix: signal for suffix, signal in signals} + + async def prepare_bounded( + self, datakey_name: str, num_collections: int, period: float + ) -> PageableDataProvider: + # The buffer is sized by count, not rate, so the period is unused. + del period + # Size the buffer and put it in fixed-length mode before arming it. + await asyncio.gather( + self.stats.enable_callbacks.set(EnableDisable.ENABLE), + self.stats.compute_statistics.set(True), + self.stats.ts_num_points.set(num_collections), + self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), + ) + # Erase and start clears the arrays and resets ts_current_point to 0, so + # the buffer is armed and empty before the detector's frames arrive. This + # is the data logic performing the erase itself, which is why trigger()'s + # zero baseline is correct (see ADR 0020). + await self.stats.ts_control.set(NDStatsTSControl.ERASE_START) + return StatsTimeSeriesProvider( + self._arrays(datakey_name), self.stats.ts_current_point + ) + + async def stop(self) -> None: + await self.stats.ts_control.set(NDStatsTSControl.STOP) + + def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: + return list(self._arrays(datakey_name)) diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py new file mode 100644 index 0000000000..825d1adde1 --- /dev/null +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -0,0 +1,137 @@ +import numpy as np +import pytest + +from ophyd_async.core import ( + DetectorAcquireLogic, + DetectorTriggerLogic, + StandardDetector, + TriggerInfo, + init_devices, + set_mock_value, +) +from ophyd_async.epics import adcore +from ophyd_async.epics.adcore import ( + NDStatsTSAcquireMode, + NDStatsTSControl, + StatsTimeSeriesDataLogic, + StatsTimeSeriesProvider, +) + + +@pytest.fixture +async def stats() -> adcore.NDStatsIO: + async with init_devices(mock=True): + plugin = adcore.NDStatsIO("PREFIX:STATS:") + return plugin + + +async def test_prepare_bounded_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + provider = await logic.prepare_bounded("det-stats", num_collections=5, period=0.1) + + assert isinstance(provider, StatsTimeSeriesProvider) + assert await stats.ts_num_points.get_value() == 5 + assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH + # Erase/Start arms and clears the buffer + assert await stats.ts_control.get_value() == NDStatsTSControl.ERASE_START + # The datakey defaults to the Total series under the bare name + datakeys = await provider.make_datakeys(5) + assert list(datakeys) == ["det-stats"] + assert datakeys["det-stats"]["shape"] == [5] + assert logic.get_hinted_fields("det-stats") == ["det-stats"] + + +async def test_stop_stops_the_time_series(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + await logic.stop() + assert await stats.ts_control.get_value() == NDStatsTSControl.STOP + + +@pytest.mark.parametrize( + "collections_per_event,num_events,expected", + [ + # step scan: one event holds the whole 5-point buffer + (5, 1, [[10.0, 11.0, 12.0, 13.0, 14.0]]), + # fly scan: five events of one point each + (1, 5, [[10.0], [11.0], [12.0], [13.0], [14.0]]), + ], +) +async def test_provider_slices_array_into_events( + stats: adcore.NDStatsIO, + collections_per_event: int, + num_events: int, + expected: list[list[float]], +): + total = np.array([10.0, 11.0, 12.0, 13.0, 14.0]) + set_mock_value(stats.ts_total, total) + set_mock_value(stats.ts_current_point, 5) + provider = StatsTimeSeriesProvider( + {"det-stats": stats.ts_total}, stats.ts_current_point + ) + + pages = [ + page + async for page in provider.make_pages( + collections_written=5, collections_per_event=collections_per_event + ) + ] + (page,) = pages + assert page["data"]["det-stats"] == expected + assert len(page["time"]) == num_events + + +async def test_provider_make_readings_derives_single_reading(stats: adcore.NDStatsIO): + """A step-scan read derives one reading holding the whole buffer.""" + total = np.array([1.0, 2.0, 3.0]) + set_mock_value(stats.ts_total, total) + set_mock_value(stats.ts_current_point, 3) + provider = StatsTimeSeriesProvider( + {"det-stats": stats.ts_total}, stats.ts_current_point + ) + readings = await provider.make_readings(collections_per_event=3) + assert readings["det-stats"]["value"] == [1.0, 2.0, 3.0] + + +class _JustInternal(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): ... + + async def default_trigger_info(self) -> TriggerInfo: + return TriggerInfo() + + +class _FillStatsAcquireLogic(DetectorAcquireLogic): + """Fills the mock time series when acquisition starts.""" + + def __init__(self, stats: adcore.NDStatsIO, values: np.ndarray): + self.stats = stats + self.values = values + + async def start_acquiring(self): + set_mock_value(self.stats.ts_total, self.values) + set_mock_value(self.stats.ts_current_point, len(self.values)) + + async def wait_for_idle(self): ... + + async def ensure_stopped(self): ... + + +async def test_step_scan_collect_pages_end_to_end(stats: adcore.NDStatsIO): + """A writer-less detector with a stats time series emits event pages.""" + values = np.array([5.0, 6.0, 7.0, 8.0]) + det = StandardDetector(name="det") + det.add_detector_logics( + _JustInternal(), + _FillStatsAcquireLogic(stats, values), + StatsTimeSeriesDataLogic(stats), + ) + # A bounded logic exposes collect_pages, not collect_asset_docs + assert hasattr(det, "collect_pages") + assert not hasattr(det, "collect_asset_docs") + + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=4)) + await det.trigger() + pages = [page async for page in det.collect_pages()] + (page,) = pages + assert page["data"]["det"] == [[5.0, 6.0, 7.0, 8.0]] + await det.unstage() From 0bc0f38a9341db1c045d7604f062c3d7f12fed99 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 20 Jul 2026 09:31:35 +0000 Subject: [PATCH 34/67] docs: document the three data-logic tiers in the detector tutorial Explains how a DetectorDataLogic picks a tier by which prepare_* it implements (single / bounded / unbounded), what each returns, how the resolved frame period is threaded to the bounded and unbounded tiers, and the rules on mixing tiers and dropping a logic that cannot serve a scan. Ignores the event_model PartialEventPage nitpick, which has no intersphinx entry, so the docs build stays warning-clean. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit db00fc330481e32cba6b64de1ac8055c7c3761fc) --- docs/conf.py | 2 ++ docs/tutorials/implementing-detectors.md | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index a8efd17b34..88978522f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -193,6 +193,8 @@ def setup(app: application.Sphinx): # TypeVars documented on StandardFlyable, added in #1359 "PrepareT", "CtxT", + # event_model publishes no intersphinx entry for this TypedDict + "event_model.documents.PartialEventPage", ] nitpick_ignore = [] for var in obj_ignore: diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index 1a4310fe13..30a98894eb 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -237,6 +237,18 @@ The [](#StreamableDataProvider) returned from `prepare_unbounded()` contains: - `make_datakeys()` - creates DataKey descriptions for each dataset - `make_stream_docs()` - emits StreamResource and StreamDatum documents as frames are written +#### Choosing a data logic tier + +A [](#DetectorDataLogic) implements exactly one of three `prepare_*` methods, and the detector picks a logic by which one it implements — there is no precedence to configure. Which tier fits depends on how the underlying source produces its data: + +- `prepare_unbounded(datakey_name, period)` — for a source that works for any number of collections, like a file writer. It returns a [](#StreamableDataProvider) and the detector exposes `collect_asset_docs`. `BlobDataLogic` uses this tier. +- `prepare_bounded(datakey_name, num_collections, period)` — for a source that holds a *finite buffer* which must be sized before acquisition, like an areaDetector stats time series or a scaler whose array length is the frame count. It returns a [](#PageableDataProvider) that emits event pages, and the detector exposes `collect_pages` instead. See [](#StatsTimeSeriesDataLogic). +- `prepare_single(datakey_name)` — for a source that can only ever produce one collection, like reading a scalar PV. It returns a [](#ReadableDataProvider). `PluginSignalDataLogic` uses this tier. + +The `period` passed to the unbounded and bounded tiers is the frame period (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0. A logic can use it to size chunks or a flush rate; a single collection has no rate, so `prepare_single` is not given it. + +A detector may carry several data logics, but not both a bounded and an unbounded one — those produce mutually exclusive document kinds. A logic whose tier cannot serve the requested number of collections (a `prepare_single` logic in a multi-collection scan, say) is dropped with a warning rather than failing the scan. + ## Conclusion We have seen how to make a [](#StandardDetector) and how the [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) classes allow us to customise its behavior through composition. This separation of concerns makes it easy to: From e81e073486ff14401ee38e1e0a93678886150da0 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 22 Jul 2026 11:55:16 +0000 Subject: [PATCH 35/67] fix: bind collect_* as instance attrs for py3.12 protocol checks Python 3.12 changed runtime-checkable Protocol isinstance() to resolve members with inspect.getattr_static, which does not invoke __getattr__. The dynamic __getattr__ that exposed collect_asset_docs / collect_pages was therefore invisible to the bluesky bundler's WritesStreamAssets / EventPageCollectable checks on 3.12+, so stream_resource/stream_datum docs were never emitted (CI passed on 3.11, failed on 3.12/3.13/3.14). Bind the relevant method as a real instance attribute in add_detector_logics instead; getattr_static finds instance-dict entries. Both names are reserved by Device, so set them via object.__setattr__. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit df154492a8fac71926435f5f25fd2d4ecca99569) --- src/ophyd_async/core/_detector.py | 39 +++++++++++-------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index f64a888838..cbd53e13f7 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -445,9 +445,9 @@ class StandardDetector( `EventPageCollectable` (and its `collect_pages`) are *not* inherited: a detector writes stream assets or emits event pages depending on which data logics it carries, never both, and the bluesky bundler treats the two as - mutually exclusive. The relevant method is exposed via `__getattr__` only - when a data logic supporting it is present, so the bundler's structural - isinstance check sees exactly the one that applies. + mutually exclusive. The relevant method is bound as an instance attribute in + `add_detector_logics` only when a data logic supporting it is present, so the + bundler's structural isinstance check sees exactly the one that applies. """ # Logic for the detector @@ -545,28 +545,17 @@ def _describe(logics: Sequence[DetectorDataLogic]) -> str: f"({_describe(bounded)}) and unbounded data logics " f"({_describe(unbounded)}); these cannot be combined on one detector" ) - - def __getattr__(self, name: str): - # Expose collect_asset_docs / collect_pages only when a data logic that - # produces that kind of document is present, so the bluesky bundler's - # structural isinstance checks (WritesStreamAssets vs EventPageCollectable) - # match exactly the one that applies. __getattr__ runs only for attributes - # not found normally, and _data_logics has a class-level default, so this - # never recurses. - if name == "collect_asset_docs" and any( - _data_logic_supported(dl.prepare_unbounded) for dl in self._data_logics - ): - return self._collect_asset_docs - if name == "collect_pages" and any( - _data_logic_supported(dl.prepare_bounded) for dl in self._data_logics - ): - return self._collect_pages - # No base class defines __getattr__, so there is nothing to delegate to; - # raise the same AttributeError the default attribute lookup would, so - # hasattr()/getattr() and error messages behave as normal. - raise AttributeError( - f"{type(self).__name__!r} object has no attribute {name!r}" - ) + # Expose collect_asset_docs / collect_pages as real instance attributes so + # the bluesky bundler's runtime-checkable isinstance checks (WritesStreamAssets + # vs EventPageCollectable) match exactly the one that applies. Python 3.12+ + # resolves those checks with inspect.getattr_static, which does not invoke + # __getattr__, so a dynamic hook is invisible to them while a real instance + # attribute is not. Both names are bluesky protocol names reserved by Device, + # so bypass its reserved-name guard with object.__setattr__. + if unbounded: + object.__setattr__(self, "collect_asset_docs", self._collect_asset_docs) + if bounded: + object.__setattr__(self, "collect_pages", self._collect_pages) # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: From 5ad521decdddad74e0a408c7215b32945695fe58 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 24 Jul 2026 12:01:19 +0000 Subject: [PATCH 36/67] refactor: derive bounded readings in StandardDetector, error on ambiguity Move make_readings off PageableDataProvider and onto StandardDetector, since there is no case yet for a provider overriding it. A single-event step-scan read now raises if a page carries more than one value per key rather than silently keeping the last, where a single reading would be ambiguous. Addresses review on #1367. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 61ad2d2043bd768c88352445ae95c0fe1063f7b7) --- src/ophyd_async/core/_data_providers.py | 20 ++----------- src/ophyd_async/core/_detector.py | 39 ++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index ab54379d65..f005e0dcb7 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -2,7 +2,7 @@ from collections.abc import AsyncIterator, Sequence from typing import Any -from bluesky.protocols import Reading, StreamAsset +from bluesky.protocols import StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange from event_model.documents import PartialEventPage @@ -45,7 +45,7 @@ class PageableDataProvider: stream datums as it goes. A step-scan event has a single collection window, so `make_pages` yields - one page of one event which the base `make_readings` extracts to a single + one page of one event, which `StandardDetector.read` extracts to a single reading; the same `make_pages` serves fly-scan collection. """ @@ -70,22 +70,6 @@ def make_pages( :param collections_per_event: how many collections make up one event """ - async def make_readings(self, collections_per_event: int) -> dict[str, Reading]: - """Derive readings from the pages for the collections written so far. - - A step-scan prepare has a single event, so the page holds one event - whose per-key value is the `collections_per_event`-length array; this - extracts to a single reading per key. - """ - collections_written = await self.collections_written_signal.get_value() - readings: dict[str, Reading] = {} - async for page in self.make_pages(collections_written, collections_per_event): - times = page["time"] - for key, values in page["data"].items(): - for value, timestamp in zip(values, times, strict=True): - readings[key] = Reading(value=value, timestamp=timestamp) - return readings - class StreamResourceInfo(ConfinedModel): """A description of a single StreamResource that should be emitted.""" diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index cbd53e13f7..a745e5c1ed 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -17,6 +17,7 @@ Flyable, HasHints, Preparable, + Reading, Stageable, StreamAsset, Triggerable, @@ -968,11 +969,14 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] ctx = error_if_none(self._prepare_ctx, "Prepare not run") cpe = ctx.trigger_info.collections_per_event # Bounded providers hold a single-event page for this step-scan point, - # which read() extracts back to a reading. + # which _pageable_readings extracts back to a reading. That extraction + # lives here rather than on the provider so a provider cannot override + # it -- see the review note on #1367. for pdp in ctx.pageable_data_providers: - yield functools.partial( - pdp.make_datakeys if verb is _Verb.DESCRIBE else pdp.make_readings, cpe - ) + if verb is _Verb.DESCRIBE: + yield functools.partial(pdp.make_datakeys, cpe) + else: + yield functools.partial(self._pageable_readings, pdp, cpe) if verb is _Verb.DESCRIBE: # Streamable providers describe their shape for a step scan, but # produce their data through collect_asset_docs rather than read() @@ -994,6 +998,33 @@ def _extra_hint_sources(self) -> Iterator[HasHints]: if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): yield _HintedFields(fields) + async def _pageable_readings( + self, provider: PageableDataProvider, collections_per_event: int + ) -> dict[str, Reading]: + """Derive readings from a bounded provider's pages for a step-scan read. + + A step-scan prepare has a single event, so the page holds one event + whose per-key value is the `collections_per_event`-length array; this + extracts to a single reading per key. More than one value in a page + would mean `read()` was asked for a single reading from a multi-event + buffer, where the answer is ambiguous, so raise rather than silently + keep the last. + """ + collections_written = await provider.collections_written_signal.get_value() + readings: dict[str, Reading] = {} + async for page in provider.make_pages( + collections_written, collections_per_event + ): + times = page["time"] + for key, values in page["data"].items(): + if len(values) != 1: + raise ValueError( + f"read() expected a single event for {key!r}, " + f"got a page of {len(values)}" + ) + readings[key] = Reading(value=values[0], timestamp=times[0]) + return readings + async def _collect_asset_docs( self, index: int | None = None ) -> AsyncIterator[StreamAsset]: From 8b5e18443d5e83fff09b5d1abb81aac9e93b69eb Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 24 Jul 2026 12:01:33 +0000 Subject: [PATCH 37/67] fix: correct stats time-series PVs and read them in one batch The NDPluginStats time series is controlled by TS:TSAcquire (a bo written to 1 to erase+start, 0 to stop), not the TS:TSControl mbbo, which does not exist on this plugin; NDStatsTSControl is dropped with it. Always capture TS:TSTimestamp and use each event's last point as the reading timestamp instead of read time, and read every array plus the timestamps in one gather_dict batch. Addresses review on #1367. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 500565e696af1ea6d6b6d6305f311e935c67b2fd) --- src/ophyd_async/epics/adcore/__init__.py | 2 - src/ophyd_async/epics/adcore/_io.py | 29 ++----- .../epics/adcore/_stats_time_series.py | 67 ++++++++++------ .../epics/adcore/test_stats_time_series.py | 80 +++++++++++-------- 4 files changed, 97 insertions(+), 81 deletions(-) diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 689fd57248..ba4f2e4b54 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -31,7 +31,6 @@ NDROIStatNIO, NDStatsIO, NDStatsTSAcquireMode, - NDStatsTSControl, ) from ._ndattribute import ( NDAttributeDataType, @@ -59,7 +58,6 @@ "NDPluginBaseIO", "NDROIIO", "NDStatsIO", - "NDStatsTSControl", "NDStatsTSAcquireMode", "NDROIStatNIO", "NDROIStatIO", diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 6fef30a68f..b80bfb1e47 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -196,20 +196,6 @@ class NDProcessIO(NDPluginBaseIO): scale: A[SignalRW[float], PvSuffix.rbv("Scale")] -class NDStatsTSControl(StrictEnum): - """Time-series control command for an NDPluginStats time series. - - Mirrors the ``TSControl`` record in ADCore/db/NDPluginTimeSeries.template. - Writing ``ERASE_START`` clears the buffer and resets the current point to 0 - before it starts filling again. - """ - - ERASE_START = "Erase/Start" - START = "Start" - STOP = "Stop" - READ = "Read" - - class NDStatsTSAcquireMode(StrictEnum): """Whether an NDPluginStats time series stops when full or wraps around. @@ -249,20 +235,21 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] net: A[SignalR[float], PvSuffix("Net_RBV")] - # Time series. NDStats.template includes NDPluginTimeSeries.template with an - # inner "TS:" record prefix, so each statistic is also exposed as a fixed- - # length array. ts_num_points sizes the buffer, ts_control erases and starts - # it, ts_current_point reports progress and ts_total holds the Total series. + # Time series. NDStats delegates its time series to an embedded + # NDPluginTimeSeries instance under an inner "TS:" prefix, so each statistic + # is also exposed as a fixed-length array. ts_num_points sizes the buffer, + # ts_acquire erases and starts it (writing 1 clears the arrays and resets + # ts_current_point to 0), ts_current_point reports progress, ts_total holds + # the Total series and ts_timestamp holds the per-point acquisition times. # These are not registered as readables: the scalars above serve read(), and # the arrays are consumed by StatsTimeSeriesDataLogic as event pages. - # These suffixes are derived from the ADCore templates and want verifying - # against a live IOC. - ts_control: A[SignalRW[NDStatsTSControl], PvSuffix("TS:TSControl")] + ts_acquire: A[SignalRW[bool], PvSuffix("TS:TSAcquire")] ts_num_points: A[SignalRW[int], PvSuffix.rbv("TS:TSNumPoints")] ts_current_point: A[SignalR[int], PvSuffix("TS:TSCurrentPoint")] ts_acquiring: A[SignalR[bool], PvSuffix("TS:TSAcquiring")] ts_acquire_mode: A[SignalRW[NDStatsTSAcquireMode], PvSuffix.rbv("TS:TSAcquireMode")] ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TS:TSTotal")] + ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TS:TSTimestamp")] # Centroid statistics compute_centroid: A[ SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index 68a76b592c..bb66ba3dc4 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -1,5 +1,4 @@ import asyncio -import time from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass, field @@ -13,9 +12,10 @@ EnableDisable, PageableDataProvider, SignalR, + gather_dict, ) -from ._io import NDStatsIO, NDStatsTSAcquireMode, NDStatsTSControl +from ._io import NDStatsIO, NDStatsTSAcquireMode class StatsTimeSeriesProvider(PageableDataProvider): @@ -24,19 +24,24 @@ class StatsTimeSeriesProvider(PageableDataProvider): The plugin holds one fixed-length array per statistic, filled by NDArray callbacks as the detector acquires. Progress is read from ``ts_current_point``, and each configured array is sliced into - ``collections_per_event``-length chunks, one per event. + ``collections_per_event``-length chunks, one per event. The plugin's + ``ts_timestamp`` array gives the acquisition time of each point, so the last + point of an event supplies that event's timestamp. :param arrays: datakey (already suffixed) to the array signal that backs it :param current_point: the plugin's ``ts_current_point`` progress signal + :param timestamps: the plugin's ``ts_timestamp`` per-point time signal """ def __init__( self, arrays: Mapping[str, SignalR[Array1D[np.float64]]], current_point: SignalR[int], + timestamps: SignalR[Array1D[np.float64]], ) -> None: self.arrays = dict(arrays) self.collections_written_signal = current_point + self.timestamps = timestamps self.last_emitted = 0 async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: @@ -58,27 +63,35 @@ async def make_pages( if events <= self.last_emitted: return new = range(self.last_emitted, events) - # A single timestamp per event: the plugin does not expose a per-point - # time, so use the read time. - now = time.time() - values = { - datakey: await signal.get_value() for datakey, signal in self.arrays.items() - } + # Read every array and the per-point timestamps in one parallel batch. + read: dict[ + SignalR[Array1D[np.float64]], Array1D[np.float64] + ] = await gather_dict( + { + signal: signal.get_value() + for signal in (*self.arrays.values(), self.timestamps) + } + ) + stamps = read[self.timestamps] + # One timestamp per event: the acquisition time of that event's last point. + event_times = [ + float(stamps[(event + 1) * collections_per_event - 1]) for event in new + ] page: PartialEventPage = { "data": { datakey: [ list( - array[ + read[signal][ event * collections_per_event : (event + 1) * collections_per_event ] ) for event in new ] - for datakey, array in values.items() + for datakey, signal in self.arrays.items() }, - "time": [now for _ in new], - "timestamps": {datakey: [now for _ in new] for datakey in values}, + "time": event_times, + "timestamps": dict.fromkeys(self.arrays, event_times), } self.last_emitted = events yield page @@ -91,14 +104,14 @@ class StatsTimeSeriesDataLogic(DetectorDataLogic): For detectors that write no file: the stats plugin holds each statistic in a fixed-length buffer that must be sized before acquisition, so this is a bounded (`prepare_bounded`) data logic that emits event pages. One plugin has - one time-series control (`ts_control`, `ts_num_points`) shared across many + one time-series control (`ts_acquire`, `ts_num_points`) shared across many arrays, so one logic covers many arrays with the control embedded. - The buffer is sized and erased in `prepare_bounded` by writing - ``Erase/Start`` to ``ts_control``; the detector's acquire logic then drives - the camera as usual and its frames feed the time series via NDArray - callbacks. A detector that writes a file should pull stats into the file as - NDAttributes instead (see `ADHDFDataLogic`). + The buffer is sized and armed in `prepare_bounded` by writing 1 to + ``ts_acquire``, which erases the arrays and resets the current point before + the detector's acquire logic drives the camera; its frames then feed the time + series via NDArray callbacks. A detector that writes a file should pull stats + into the file as NDAttributes instead (see `ADHDFDataLogic`). :param stats: the stats plugin whose time series to read :param stat_signals: datakey suffix to the array signal for each statistic to @@ -127,17 +140,19 @@ async def prepare_bounded( self.stats.ts_num_points.set(num_collections), self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), ) - # Erase and start clears the arrays and resets ts_current_point to 0, so - # the buffer is armed and empty before the detector's frames arrive. This - # is the data logic performing the erase itself, which is why trigger()'s - # zero baseline is correct (see ADR 0020). - await self.stats.ts_control.set(NDStatsTSControl.ERASE_START) + # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to + # 0, so the buffer is armed and empty before the detector's frames arrive. + # This is the data logic performing the erase itself, which is why + # trigger()'s zero baseline is correct (see ADR 0020). + await self.stats.ts_acquire.set(True) return StatsTimeSeriesProvider( - self._arrays(datakey_name), self.stats.ts_current_point + self._arrays(datakey_name), + self.stats.ts_current_point, + self.stats.ts_timestamp, ) async def stop(self) -> None: - await self.stats.ts_control.set(NDStatsTSControl.STOP) + await self.stats.ts_acquire.set(False) def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: return list(self._arrays(datakey_name)) diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index 825d1adde1..9c23878a17 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -12,7 +12,6 @@ from ophyd_async.epics import adcore from ophyd_async.epics.adcore import ( NDStatsTSAcquireMode, - NDStatsTSControl, StatsTimeSeriesDataLogic, StatsTimeSeriesProvider, ) @@ -32,8 +31,8 @@ async def test_prepare_bounded_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO assert isinstance(provider, StatsTimeSeriesProvider) assert await stats.ts_num_points.get_value() == 5 assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH - # Erase/Start arms and clears the buffer - assert await stats.ts_control.get_value() == NDStatsTSControl.ERASE_START + # ts_acquire=1 arms and clears the buffer + assert await stats.ts_acquire.get_value() is True # The datakey defaults to the Total series under the bare name datakeys = await provider.make_datakeys(5) assert list(datakeys) == ["det-stats"] @@ -43,30 +42,36 @@ async def test_prepare_bounded_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO async def test_stop_stops_the_time_series(stats: adcore.NDStatsIO): logic = StatsTimeSeriesDataLogic(stats) + set_mock_value(stats.ts_acquire, True) await logic.stop() - assert await stats.ts_control.get_value() == NDStatsTSControl.STOP + assert await stats.ts_acquire.get_value() is False @pytest.mark.parametrize( - "collections_per_event,num_events,expected", + "collections_per_event,expected_data,expected_times", [ - # step scan: one event holds the whole 5-point buffer - (5, 1, [[10.0, 11.0, 12.0, 13.0, 14.0]]), - # fly scan: five events of one point each - (1, 5, [[10.0], [11.0], [12.0], [13.0], [14.0]]), + # step scan: one event holds the whole 5-point buffer, timed by its + # last point + (5, [[10.0, 11.0, 12.0, 13.0, 14.0]], [104.0]), + # fly scan: five events of one point each, timed point by point + ( + 1, + [[10.0], [11.0], [12.0], [13.0], [14.0]], + [100.0, 101.0, 102.0, 103.0, 104.0], + ), ], ) async def test_provider_slices_array_into_events( stats: adcore.NDStatsIO, collections_per_event: int, - num_events: int, - expected: list[list[float]], + expected_data: list[list[float]], + expected_times: list[float], ): - total = np.array([10.0, 11.0, 12.0, 13.0, 14.0]) - set_mock_value(stats.ts_total, total) + set_mock_value(stats.ts_total, np.array([10.0, 11.0, 12.0, 13.0, 14.0])) + set_mock_value(stats.ts_timestamp, np.array([100.0, 101.0, 102.0, 103.0, 104.0])) set_mock_value(stats.ts_current_point, 5) provider = StatsTimeSeriesProvider( - {"det-stats": stats.ts_total}, stats.ts_current_point + {"det-stats": stats.ts_total}, stats.ts_current_point, stats.ts_timestamp ) pages = [ @@ -76,20 +81,8 @@ async def test_provider_slices_array_into_events( ) ] (page,) = pages - assert page["data"]["det-stats"] == expected - assert len(page["time"]) == num_events - - -async def test_provider_make_readings_derives_single_reading(stats: adcore.NDStatsIO): - """A step-scan read derives one reading holding the whole buffer.""" - total = np.array([1.0, 2.0, 3.0]) - set_mock_value(stats.ts_total, total) - set_mock_value(stats.ts_current_point, 3) - provider = StatsTimeSeriesProvider( - {"det-stats": stats.ts_total}, stats.ts_current_point - ) - readings = await provider.make_readings(collections_per_event=3) - assert readings["det-stats"]["value"] == [1.0, 2.0, 3.0] + assert page["data"]["det-stats"] == expected_data + assert page["time"] == expected_times class _JustInternal(DetectorTriggerLogic): @@ -102,12 +95,14 @@ async def default_trigger_info(self) -> TriggerInfo: class _FillStatsAcquireLogic(DetectorAcquireLogic): """Fills the mock time series when acquisition starts.""" - def __init__(self, stats: adcore.NDStatsIO, values: np.ndarray): + def __init__(self, stats: adcore.NDStatsIO, values: np.ndarray, times: np.ndarray): self.stats = stats self.values = values + self.times = times async def start_acquiring(self): set_mock_value(self.stats.ts_total, self.values) + set_mock_value(self.stats.ts_timestamp, self.times) set_mock_value(self.stats.ts_current_point, len(self.values)) async def wait_for_idle(self): ... @@ -115,15 +110,23 @@ async def wait_for_idle(self): ... async def ensure_stopped(self): ... -async def test_step_scan_collect_pages_end_to_end(stats: adcore.NDStatsIO): - """A writer-less detector with a stats time series emits event pages.""" +@pytest.fixture +def stats_detector(stats: adcore.NDStatsIO) -> StandardDetector: + """A writer-less detector whose only data logic is a stats time series.""" values = np.array([5.0, 6.0, 7.0, 8.0]) + times = np.array([200.0, 201.0, 202.0, 203.0]) det = StandardDetector(name="det") det.add_detector_logics( _JustInternal(), - _FillStatsAcquireLogic(stats, values), + _FillStatsAcquireLogic(stats, values, times), StatsTimeSeriesDataLogic(stats), ) + return det + + +async def test_step_scan_collect_pages_end_to_end(stats_detector: StandardDetector): + """A writer-less detector with a stats time series emits event pages.""" + det = stats_detector # A bounded logic exposes collect_pages, not collect_asset_docs assert hasattr(det, "collect_pages") assert not hasattr(det, "collect_asset_docs") @@ -135,3 +138,16 @@ async def test_step_scan_collect_pages_end_to_end(stats: adcore.NDStatsIO): (page,) = pages assert page["data"]["det"] == [[5.0, 6.0, 7.0, 8.0]] await det.unstage() + + +async def test_step_scan_read_derives_single_reading(stats_detector: StandardDetector): + """A step-scan read derives one reading holding the whole buffer, timed by + the last point's ts_timestamp.""" + det = stats_detector + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=4)) + await det.trigger() + reading = await det.read() + assert reading["det"]["value"] == [5.0, 6.0, 7.0, 8.0] + assert reading["det"]["timestamp"] == 203.0 + await det.unstage() From 8259a07b7ef7c38f6a8c0cabf88f038018950f95 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 24 Jul 2026 12:01:48 +0000 Subject: [PATCH 38/67] feat: add ADWriterFactory.stats for the stats bounded data logic A fourth factory constructor alongside hdf/jpeg/tiff, building an NDStatsIO plugin and a StatsTimeSeriesDataLogic so a writer-less detector can expose its stats time series as event pages. The factory generic is widened from NDPluginFileIO to NDPluginBaseIO to admit the stats plugin. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 3d288283a0d1bb6572126bc9cd8c05516d2710f4) --- src/ophyd_async/epics/adcore/_data_logic.py | 68 ++++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index af1ce143bd..7a5182b162 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -2,12 +2,13 @@ from collections.abc import Callable, Sequence from dataclasses import dataclass from pathlib import PureWindowsPath -from typing import Any, Generic +from typing import Any, Generic, TypeVar from xml.etree import ElementTree as ET import numpy as np from ophyd_async.core import ( + Array1D, DetectorDataLogic, EnableDisable, PathInfo, @@ -29,9 +30,14 @@ NDFileHDF5IO, NDPluginBaseIO, NDPluginFileIO, - NDPluginFileIOT, + NDStatsIO, ) from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType +from ._stats_time_series import StatsTimeSeriesDataLogic + +#: A writer plugin built by an `ADWriterFactory`: a file writer (HDF, TIFF, JPEG) +#: or a stats plugin whose time series is read as event pages. +NDWriterPluginT = TypeVar("NDWriterPluginT", bound=NDPluginBaseIO) @dataclass @@ -309,17 +315,17 @@ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @dataclass -class ADWriterFactory(Generic[NDPluginFileIOT]): - """Factory that creates a file-writer plugin and its matching data logic. +class ADWriterFactory(Generic[NDWriterPluginT]): + """Factory that creates a writer plugin and its matching data logic. - Construct using the classmethods `hdf`, `jpeg`, or `tiff`, then pass one - or more instances to `AreaDetector` as positional `*writer_factories` + Construct using the classmethods `hdf`, `jpeg`, `tiff` or `stats`, then pass + one or more instances to `AreaDetector` as positional `*writer_factories` arguments. When the detector is initialised `__call__` is invoked with the detector's PV `prefix`, its `driver`, and the flat list of extra `plugins`; it returns the writer device and the corresponding `DetectorDataLogic`. - :param writer_cls: Concrete `NDPluginFileIO` subclass to instantiate. + :param writer_cls: Concrete `NDPluginBaseIO` subclass to instantiate. :param writer_suffix: PV suffix appended to *prefix* to form the writer's PV prefix. :param writer_name: Attribute name under which the writer device is stored on the @@ -338,7 +344,7 @@ class ADWriterFactory(Generic[NDPluginFileIOT]): that builds the data logic given the already-constructed writer. """ - writer_cls: type[NDPluginFileIOT] + writer_cls: type[NDWriterPluginT] writer_suffix: str writer_name: str datakey_suffix: str @@ -346,7 +352,7 @@ class ADWriterFactory(Generic[NDPluginFileIOT]): NDArrayDescription | Callable[[ADBaseIO], NDArrayDescription] | None ) data_logic_factory: Callable[ - [NDPluginFileIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], + [NDWriterPluginT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], DetectorDataLogic, ] @@ -355,7 +361,7 @@ def __call__( prefix: str, driver: ADBaseIO, plugins: Sequence[NDPluginBaseIO], - ) -> tuple[NDPluginFileIOT, DetectorDataLogic]: + ) -> tuple[NDWriterPluginT, DetectorDataLogic]: """Instantiate the writer plugin and build the data logic. :param prefix: EPICS PV prefix for the detector (same as `AreaDetector.prefix`). @@ -521,3 +527,45 @@ def tiff( ) ), ) + + @staticmethod + def stats( + writer_suffix: str = "STAT:", + writer_name: str = "stats", + datakey_suffix: str = "", + stat_signals: Callable[ + [NDStatsIO], Sequence[tuple[str, SignalR[Array1D[np.float64]]]] + ] + | None = None, + ) -> "ADWriterFactory[NDStatsIO]": + """Create a factory for an NDPluginStats time series. + + Unlike `hdf`, `jpeg` and `tiff`, this writes no file: the stats plugin's + time series is a fixed-length buffer read back as event pages, for + detectors that cannot write assets (see `StatsTimeSeriesDataLogic`). + + :param writer_suffix: PV suffix for the NDPluginStats plugin, defaults to + ``STAT:``. + :param writer_name: + Attribute name for the plugin on the detector, defaults to ``"stats"``. + :param datakey_suffix: Suffix appended to the datakey name, defaults to ``""``. + :param stat_signals: + Callable ``(stats) → [(suffix, array_signal), ...]`` choosing which + statistics to expose, given the constructed plugin. Left as ``None`` + (the default), only the ``Total`` series is exposed under the bare + datakey name. + """ + return ADWriterFactory( + writer_cls=NDStatsIO, + writer_suffix=writer_suffix, + writer_name=writer_name, + datakey_suffix=datakey_suffix, + array_description=None, + data_logic_factory=lambda writer, desc, driver, plugins: ( + StatsTimeSeriesDataLogic( + stats=writer, + stat_signals=list(stat_signals(writer)) if stat_signals else [], + datakey_suffix=datakey_suffix, + ) + ), + ) From eac1c078440cb6010704ba19c80231cfba250377 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 24 Jul 2026 12:01:49 +0000 Subject: [PATCH 39/67] test: exercise the stats time series against a real adsim IOC A container test driving a writer-less SimDetector whose only data logic is the STAT time series: prepare arms TS:TSNumPoints/TS:TSAcquire, the driver takes the frames, and read() slices TS:TSTotal into one 5-element reading. Proves the new TS:* PV names resolve and the bounded tier collects end to end. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit f43f56f26f7b6de13ec77a43aa58acc89c7751ce) --- tests/container_tests/test_adsim_system.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index 084ad1b55a..a9746150fd 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -111,6 +111,24 @@ def adsim(RE: RunEngine, shared_tmp_path: Path) -> AreaDetector: return adsim +@pytest.fixture +def adsimstat(RE: RunEngine) -> AreaDetector: + """A writer-less SimDetector whose only data logic is the STAT time series. + + It writes no file: the NDPluginStats time series is a fixed-length buffer read + back as event pages, so this exercises the bounded data-logic tier against a + real IOC (the `ADWriterFactory.stats` factory and NDStatsIO's `TS:*` signals). + """ + with init_devices(): + adsimstat = SimDetector( + "BL01T-DI-CAM-01:", + adcore.ADWriterFactory.stats(writer_suffix="STAT:"), + driver_suffix="DET:", + ) + + return adsimstat + + @pytest.fixture(autouse=True) def reset_adsim_to_baseline(RE: RunEngine, adsim: SimDetector) -> None: """Put the detector back to a known state before every test. @@ -352,6 +370,50 @@ def test_software_triggering( ] +@pytest.mark.timeout(TIMEOUT + 15.0) +def test_stats_time_series_step_scan( + RE: RunEngine, adsimstat: SimDetector, bl01t_di_cam_01: None +) -> None: + """A writer-less stats detector arms the TS buffer, acquires, and reads the + per-frame Total series back as a single event holding the whole array. + + This drives the real STAT plugin's ``TS:*`` records end to end: prepare sizes + and arms ``TS:TSNumPoints``/``TS:TSAcquire``, the sim driver takes the frames, + and read() slices ``TS:TSTotal`` (timestamped by ``TS:TSTimestamp``) into a + single 5-element reading. + """ + collections = 5 + + def prepare_then_count() -> MsgGenerator[None]: + yield from bps.prepare( + adsimstat, + TriggerInfo(collections_per_event=collections, livetime=0.1), + wait=True, + ) + yield from bp.count([adsimstat]) + + named_docs: list[tuple[str, DocumentType]] = [] + RE(prepare_then_count(), lambda name, doc: named_docs.append((name, doc))) + + # A bounded logic exposes collect_pages, never collect_asset_docs. + assert hasattr(adsimstat, "collect_pages") + assert not hasattr(adsimstat, "collect_asset_docs") + + descriptor = next(doc for name, doc in named_docs if name == "descriptor") + data_key = descriptor["data_keys"]["adsimstat"] + assert data_key["shape"] == [collections] + assert data_key["dtype"] == "array" + assert data_key["dtype_numpy"] == "= 0 + + def run_plan_and_get_documents( RE: RunEngine, plan: MsgGenerator[Any] ) -> list[DocumentType]: From 62e4bc2128c3533269e1fcbcd5718dd20ab448ac Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 25 Aug 2026 21:03:59 +0000 Subject: [PATCH 40/67] docs: register the NDWriterPluginT TypeVar with nitpick_ignore tox -e docs builds nitpicky with warnings-as-errors and TypeVars never resolve as references, so every one needs an entry. Without this the docs build fails on three 'reference target not found' warnings while every other env stays green. --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 88978522f2..9a6eb2d5ae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,6 +195,8 @@ def setup(app: application.Sphinx): "CtxT", # event_model publishes no intersphinx entry for this TypedDict "event_model.documents.PartialEventPage", + # TypeVar for the plugin an ADWriterFactory builds a data logic around + "ophyd_async.epics.adcore._data_logic.NDWriterPluginT", ] nitpick_ignore = [] for var in obj_ignore: From b6736e47ffedaf29cb1ff2d430304f7e837cfdf3 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 28 Aug 2026 12:01:54 +0000 Subject: [PATCH 41/67] docs: describe two data logic tiers, not three The single-event tier went with the previous branch, so ADR 0020 and the detector tutorial should never have described it: a scalar per event is a registered signal now. Also trims the comments that argued with themselves rather than saying what the code does. --- .../decisions/0020-bounded-data-logic-tier.md | 48 ++++++++----------- docs/tutorials/implementing-detectors.md | 9 ++-- src/ophyd_async/core/_detector.py | 44 ++++++----------- 3 files changed, 41 insertions(+), 60 deletions(-) diff --git a/docs/explanations/decisions/0020-bounded-data-logic-tier.md b/docs/explanations/decisions/0020-bounded-data-logic-tier.md index 9d4dd16c9b..9c4f149644 100644 --- a/docs/explanations/decisions/0020-bounded-data-logic-tier.md +++ b/docs/explanations/decisions/0020-bounded-data-logic-tier.md @@ -8,11 +8,10 @@ Accepted ## Context -ADR 0012 split data production out into `DetectorDataLogic`, with two tiers that a subclass -opts into by overriding the corresponding method: +ADR 0012 split data production out into `DetectorDataLogic`. After ADR 0021 removed the +single-event tier, one tier is left that a subclass opts into by overriding the +corresponding method: -- `prepare_single(datakey_name) -> ReadableDataProvider` — a single event only, feeding - `read()` and `describe()` - `prepare_unbounded(datakey_name) -> StreamableDataProvider` — any number of collections, feeding `collect_asset_docs()` and `describe_collect()` @@ -83,20 +82,16 @@ So one new tier serves both, and #1309's period rides along with it. ### The areaDetector stats duality An `NDPluginStats` exposes each statistic twice: as a scalar (`Total_RBV`) and as a time series -array (`TS:TSTotal`). Only the scalar is modelled today, by `PluginSignalDataLogic`. The scalar -can only ever serve a single collection; the time series can serve any number. Issue #1364 arises -directly from this: an AD detector with an HDF writer *and* a stats scalar cannot fly, because -the scalar is single-only. +array (`TS:TSTotal`). Only the scalar is modelled today, as a registered signal. A registered +signal is read once per event, so a detector with no file writer has no way to produce a value +per collection for a fly scan: the time series can, but nothing models it. ## Decision -### A third tier: `prepare_bounded` +### A second tier: `prepare_bounded` ```python class DetectorDataLogic: - async def prepare_single( - self, datakey_name: str - ) -> ReadableDataProvider: ... async def prepare_bounded( self, datakey_name: str, num_collections: int, period: float ) -> PageableDataProvider: ... @@ -105,8 +100,7 @@ class DetectorDataLogic: ) -> StreamableDataProvider: ... ``` -`prepare_bounded` and `prepare_unbounded` are told the frame period; `prepare_single` is not, -since a single collection has no rate to size against. +Both tiers are told the frame period, so either can size a chunk or a buffer against it. `num_collections` is always the **total** for the scan (`TriggerInfo.number_of_collections`), never a per-kickoff figure. A finite buffer is fed by data callbacks and does not care how many @@ -141,12 +135,10 @@ asks whether the tier a logic implements can serve the requested number of colle |---|---| | `prepare_unbounded` | always | | `prepare_bounded` | `n != 0` (finite) | -| `prepare_single` | `n == 1` | A logic whose tier cannot serve `n` is dropped from the prepare context with a warning, rather -than raising. This generalises the rule agreed in #1364 to all three tiers: it is what lets an -AD detector carry an HDF writer plus a `PluginSignalDataLogic` on some unrelated PV (a -temperature, a ring current) and still fly, with only the scalar dropped. +than raising, as agreed in #1364. That is what lets a detector carrying a bounded logic still be +prepared for an infinite fly scan, with only that logic dropped. ### Bounded providers are never reused @@ -209,11 +201,13 @@ without a word. ### `collect_asset_docs` and `collect_pages` are exposed dynamically -`StandardDetector` no longer inherits `WritesStreamAssets`. Instead `__getattr__` exposes -`collect_asset_docs` only when a data logic implementing `prepare_unbounded` is present, and -`collect_pages` only when one implementing `prepare_bounded` is present. Because bluesky's -protocols are `runtime_checkable`, an absent attribute is enough to make the isinstance check -fail, and the bundler then does the right thing. +`StandardDetector` no longer inherits `WritesStreamAssets`. Instead it binds +`collect_asset_docs` as an instance attribute only when a data logic implementing +`prepare_unbounded` is present, and `collect_pages` only when one implementing +`prepare_bounded` is present. Because bluesky's protocols are `runtime_checkable`, an absent +attribute is enough to make the isinstance check fail, and the bundler then does the right +thing. It has to be a real attribute rather than a `__getattr__` hook, because Python 3.12+ +resolves those checks with `inspect.getattr_static`, which never calls `__getattr__`. This keys off the data logics, which are fixed when the detector is constructed, rather than off the prepare context, which does not exist until `prepare()` runs. @@ -226,7 +220,7 @@ it needs no change to bluesky. `add_detector_logics()` raises if a detector is given both a bounded and an unbounded data logic, since that is the one combination where both `collect_asset_docs` and `collect_pages` would be -exposed and both would produce data in a fly scan. Single-tier logics may coexist with either. +exposed and both would produce data in a fly scan. ### Stats with a file writer go via NDAttributes @@ -235,9 +229,9 @@ not need to: `ADHDFDataLogic` already pulls NDAttribute datasets into the file, areaDetector's own mechanism for exactly this. The time series tier is for detectors with no writer at all. -`PluginSignalDataLogic` (the scalar) therefore remains the right choice for a detector that -writes files and wants a stats value in a step scan, and a stats time series data logic is the -right choice for one that writes no files. +Registering the scalar with `set_readable_format` therefore remains the right choice for a +detector that writes files and wants a stats value in a step scan, and a stats time series data +logic is the right choice for one that writes no files. ### One stats data logic for many arrays diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index 30a98894eb..ec1185cb93 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -239,15 +239,16 @@ The [](#StreamableDataProvider) returned from `prepare_unbounded()` contains: #### Choosing a data logic tier -A [](#DetectorDataLogic) implements exactly one of three `prepare_*` methods, and the detector picks a logic by which one it implements — there is no precedence to configure. Which tier fits depends on how the underlying source produces its data: +A [](#DetectorDataLogic) implements exactly one of two `prepare_*` methods, and the detector picks a logic by which one it implements — there is no precedence to configure. Which tier fits depends on how the underlying source produces its data: - `prepare_unbounded(datakey_name, period)` — for a source that works for any number of collections, like a file writer. It returns a [](#StreamableDataProvider) and the detector exposes `collect_asset_docs`. `BlobDataLogic` uses this tier. - `prepare_bounded(datakey_name, num_collections, period)` — for a source that holds a *finite buffer* which must be sized before acquisition, like an areaDetector stats time series or a scaler whose array length is the frame count. It returns a [](#PageableDataProvider) that emits event pages, and the detector exposes `collect_pages` instead. See [](#StatsTimeSeriesDataLogic). -- `prepare_single(datakey_name)` — for a source that can only ever produce one collection, like reading a scalar PV. It returns a [](#ReadableDataProvider). `PluginSignalDataLogic` uses this tier. -The `period` passed to the unbounded and bounded tiers is the frame period (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0. A logic can use it to size chunks or a flush rate; a single collection has no rate, so `prepare_single` is not given it. +A source that produces one value per event, like a plugin scalar, needs no data logic at all: a detector is a [](#StandardReadable), so register the signal with [](#StandardReadable.set_readable_format). -A detector may carry several data logics, but not both a bounded and an unbounded one — those produce mutually exclusive document kinds. A logic whose tier cannot serve the requested number of collections (a `prepare_single` logic in a multi-collection scan, say) is dropped with a warning rather than failing the scan. +The `period` given to both tiers is the frame period (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0, so a logic can use it to size chunks or a flush rate. + +A detector may carry several data logics, but not both a bounded and an unbounded one — those produce mutually exclusive document kinds. A logic whose tier cannot serve the requested number of collections (a bounded logic in an infinite fly scan, say) is dropped with a warning rather than failing the scan. ## Conclusion diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index a745e5c1ed..915cdf75d9 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -546,13 +546,11 @@ def _describe(logics: Sequence[DetectorDataLogic]) -> str: f"({_describe(bounded)}) and unbounded data logics " f"({_describe(unbounded)}); these cannot be combined on one detector" ) - # Expose collect_asset_docs / collect_pages as real instance attributes so - # the bluesky bundler's runtime-checkable isinstance checks (WritesStreamAssets - # vs EventPageCollectable) match exactly the one that applies. Python 3.12+ - # resolves those checks with inspect.getattr_static, which does not invoke - # __getattr__, so a dynamic hook is invisible to them while a real instance - # attribute is not. Both names are bluesky protocol names reserved by Device, - # so bypass its reserved-name guard with object.__setattr__. + # Bind whichever collect verb applies as a real instance attribute, so the + # bundler's isinstance checks (WritesStreamAssets vs EventPageCollectable) + # see exactly one of them. They resolve with inspect.getattr_static on + # Python 3.12+, which does not call __getattr__, so a dynamic hook would be + # invisible. Both names are reserved by Device, hence object.__setattr__. if unbounded: object.__setattr__(self, "collect_asset_docs", self._collect_asset_docs) if bounded: @@ -667,14 +665,11 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: self.name + dl.datakey_suffix, num_collections, ) - # Unbounded providers depend on collections_per_event (it sets the - # StreamResource shape) and on the period (it sets the chunk shape), so may be - # reused across prepares when both are unchanged (this avoids reopening files on - # every step-scan point). A period change invalidates reuse because the chunk - # shape is baked into the StreamResource at construction and, for areaDetector, - # num_frames_chunks can only be set before capture starts. Bounded providers are - # never reused: they hold a finite buffer that must be re-armed for each event, - # and re-calling prepare_bounded on every trigger() is what re-arms it. + # Unbounded providers are reused while collections_per_event and the period + # are unchanged, to avoid reopening a file on every step-scan point. Both are + # baked into the StreamResource at construction, so a change to either + # invalidates the reuse. Bounded providers are never reused: re-calling + # prepare_bounded is what re-arms their finite buffer. previous = self._prepare_ctx.trigger_info if self._prepare_ctx else None reusable = ( previous is not None @@ -951,18 +946,10 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] StandardReadable gathers these alongside the registered children, so the verb methods themselves need no overriding. - Raising when nothing has been prepared is deliberate, and predates - registered children being readable at all: a detector that has not been - prepared has no data keys, so a `describe()` that quietly succeeded - would emit a descriptor missing the detector's data. In practice - `trigger()` prepares implicitly, so `trigger_and_read` never reaches - this. `read_configuration()` and `describe_configuration()` are - unaffected, as this hook only feeds the data verbs. - - The cost is that the *registered children* are unreachable through - `read()`/`describe()` until then too, since this raises before the - registry is consulted. That is accepted rather than worked around: - reading an unprepared detector is already defined as an error. + Raises if nothing has been prepared, since an unprepared detector has no + data keys and would otherwise emit a descriptor missing its data. + `trigger()` prepares implicitly, so a step scan never sees this. + `read_configuration()` and `describe_configuration()` are unaffected. """ if verb not in (_Verb.DESCRIBE, _Verb.READ): return @@ -970,8 +957,7 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] cpe = ctx.trigger_info.collections_per_event # Bounded providers hold a single-event page for this step-scan point, # which _pageable_readings extracts back to a reading. That extraction - # lives here rather than on the provider so a provider cannot override - # it -- see the review note on #1367. + # lives here rather than on the provider so a provider cannot override it. for pdp in ctx.pageable_data_providers: if verb is _Verb.DESCRIBE: yield functools.partial(pdp.make_datakeys, cpe) From 2062bbda1745f1f07bf42483e4d36a07441855b3 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 28 Aug 2026 16:05:39 +0000 Subject: [PATCH 42/67] Ask a data logic what it would make before starting it DetectorDataLogic had one prepare_* method per tier, each of which both described the data and did the writes to make it happen. That forces the detector to arm hardware to find out what it would get: a provider it then decides not to use has to be stopped again, which is a wasted erase on a stats time series and impossible to justify once the choice depends on what the providers actually produce. Split them. make_data_provider() describes what this logic would produce and must not start anything; start() does the writes, and is called only for the providers the detector settles on. Which tier a logic is becomes the type of the provider it returns, so _DataLogicTier, the override detection and _tier_can_serve all go, and a logic sits a scan out by returning None -- a finite buffer asked for an unbounded scan, or a plugin that is switched off. That is silent rather than warned, since a detector may carry more logics than any one scan uses. Only knowable at prepare, so which collect verb the detector exposes moves there too, recomputed each time so the verb that no longer applies is removed. Mixing the two kinds still raises, but now after asking and before starting, so nothing is armed when it does. OdinDataLogic is the one that cannot describe without starting -- its frame shape is only readable once the file processor is writing -- so it keeps its writes in make_data_provider. Safe while it is never the logic a detector drops. ADHDFDataLogic and StatsTimeSeriesDataLogic gain an enable_callbacks flag, defaulting to today's force-on, which when False follows whatever the plugin is set to and produces nothing when it is off. That is the switch a detector needs to fall back from its file writer to its stats time series between runs. --- .../decisions/0020-bounded-data-logic-tier.md | 104 ++++-- docs/tutorials/implementing-detectors.md | 22 +- src/ophyd_async/core/_detector.py | 307 +++++++++--------- src/ophyd_async/epics/adcore/_data_logic.py | 135 +++++--- src/ophyd_async/epics/adcore/_io.py | 11 + .../epics/adcore/_stats_time_series.py | 50 ++- src/ophyd_async/fastcs/odin/_data_logic.py | 14 +- src/ophyd_async/fastcs/panda/_data_logic.py | 63 ++-- src/ophyd_async/sim/_blob_data_logic.py | 25 +- .../unit_tests/core/test_standard_detector.py | 69 ++-- .../epics/adcore/test_data_logic.py | 71 ++++ .../epics/adcore/test_stats_time_series.py | 53 ++- 12 files changed, 601 insertions(+), 323 deletions(-) diff --git a/docs/explanations/decisions/0020-bounded-data-logic-tier.md b/docs/explanations/decisions/0020-bounded-data-logic-tier.md index 9c4f149644..48d5df481e 100644 --- a/docs/explanations/decisions/0020-bounded-data-logic-tier.md +++ b/docs/explanations/decisions/0020-bounded-data-logic-tier.md @@ -13,7 +13,8 @@ single-event tier, one tier is left that a subclass opts into by overriding the corresponding method: - `prepare_unbounded(datakey_name) -> StreamableDataProvider` — any number of collections, - feeding `collect_asset_docs()` and `describe_collect()` + feeding `collect_asset_docs()` and `describe_collect()`, doing the writes to start the writer + as it goes Four open issues turn out to bottom out in the same two gaps. @@ -88,27 +89,44 @@ per collection for a fly scan: the time series can, but nothing models it. ## Decision -### A second tier: `prepare_bounded` +### One method that describes, and one that starts ```python class DetectorDataLogic: - async def prepare_bounded( + async def make_data_provider( self, datakey_name: str, num_collections: int, period: float - ) -> PageableDataProvider: ... - async def prepare_unbounded( - self, datakey_name: str, period: float - ) -> StreamableDataProvider: ... + ) -> StreamableDataProvider | PageableDataProvider | None: ... + async def start(self) -> None: ... ``` +`make_data_provider` says what this logic *would* produce for this scan without starting +anything; `start` does the writes that make it happen, and is called only for the providers the +detector will use. The detector therefore asks every data logic what it would make, decides which +ones it wants, and starts only those. + +The alternative — one `prepare_*` per tier, doing both jobs — was tried first. It forces the +detector to arm hardware in order to find out what it would get, so a provider it then decides +not to use has to be stopped again. That is wasteful with a `TSAcquire` erase and impossible to +justify once the choice depends on what the providers actually produce. + Both tiers are told the frame period, so either can size a chunk or a buffer against it. `num_collections` is always the **total** for the scan (`TriggerInfo.number_of_collections`), never a per-kickoff figure. A finite buffer is fed by data callbacks and does not care how many kickoffs span it, so per-kickoff sizing would mean resizing the buffer mid-scan. +### The tier is the type of the provider returned + +There is no declaration of which tier a logic implements and no override detection: a logic that +returns a [](#StreamableDataProvider) is unbounded for this scan and one that returns a +[](#PageableDataProvider) is bounded, so a logic may even return either depending on what it is +asked for. Returning `None` means "not this scan", which is how a finite buffer declines an +unbounded one. It is not an error and is not warned about, since the whole point is that a +detector may carry more logics than any one scan uses. + ### `PageableDataProvider`, with readings derived from pages -`prepare_bounded` returns a single provider type that emits pages. Readings are derived from +The bounded tier returns a single provider type that emits pages. Readings are derived from pages rather than being a separate code path: a step-scan prepare has `number_of_events == 1`, so its page contains exactly one event, which extracts to a single reading. `collections_per_event` appears in the datakey shape exactly as it does for `StreamableDataProvider`. @@ -127,17 +145,16 @@ see a period of 0, and the awkward case is handled once in core rather than by e ### Tier selection is by capability, with no precedence rule -Each data logic implements exactly one tier, so core does not choose between tiers — it only -asks whether the tier a logic implements can serve the requested number of collections -`n = TriggerInfo.number_of_collections`: +Core does not choose between tiers — it only asks each logic what it would make for the +requested number of collections `n = TriggerInfo.number_of_collections`: | tier | serves | |---|---| -| `prepare_unbounded` | always | -| `prepare_bounded` | `n != 0` (finite) | +| streaming | always | +| finite buffer | `n != 0` (finite) | -A logic whose tier cannot serve `n` is dropped from the prepare context with a warning, rather -than raising, as agreed in #1364. That is what lets a detector carrying a bounded logic still be +A logic that cannot serve `n` returns `None` and is left out of the prepare context, rather than +raising, as agreed in #1364. That is what lets a detector carrying a bounded logic still be prepared for an infinite fly scan, with only that logic dropped. ### Bounded providers are never reused @@ -166,7 +183,7 @@ Reuse consequently becomes a per-logic decision rather than an all-or-nothing br It is tempting to derive this instead of stating it, since `_update_prepare_context` refreshes `collections_written` after re-preparing, and an areaDetector stats time series is erased by the -`TSAcquire` write inside `prepare_bounded` — so the refreshed read already returns zero. That +`TSAcquire` write in its `start` — so the refreshed read already returns zero. That derivation only holds when the data logic performs the erase itself. Where the erase belongs to the acquire control, as with the CTR-08's `EraseStart`, it happens *after* `_update_prepare_context` has taken its reading, so the baseline would be the stale pre-erase value; the detector would then @@ -182,8 +199,8 @@ only one kind and the rule reduces to a single choice per call site. Where one control both starts acquisition and clears the data buffer, it belongs to the `DetectorAcquireLogic`, alongside the stop control and the idle status. Nothing needs to declare -the erase to the framework: `prepare_bounded` is always called before `start_acquiring()`, so the -buffer is sized before the control fires, and the baseline rule above means the detector does not +the erase to the framework: a data logic's `start` is always called before `start_acquiring()`, +so the buffer is sized before the control fires, and the baseline rule above means the detector does not care who erased it. The trigger/acquire/data split therefore holds for these devices, and the three logics share one @@ -192,7 +209,7 @@ not of hardware, so it survives the concerns landing on a single PV. ### Logic objects fill exactly one role -`add_detector_logics()` raises if an object satisfies more than one of the three logic protocols, +`add_detector_logics()` raises if an object satisfies more than one of the three logic roles, directing the author to pass separate objects. This is a hazard worth catching rather than a hypothetical: a device whose concerns all live on one control is a natural candidate for a single combined logic object, and the registration is a chain of `isinstance` tests, so such an object @@ -202,15 +219,22 @@ without a word. ### `collect_asset_docs` and `collect_pages` are exposed dynamically `StandardDetector` no longer inherits `WritesStreamAssets`. Instead it binds -`collect_asset_docs` as an instance attribute only when a data logic implementing -`prepare_unbounded` is present, and `collect_pages` only when one implementing -`prepare_bounded` is present. Because bluesky's protocols are `runtime_checkable`, an absent +`collect_asset_docs` as an instance attribute only when it has a streaming provider, and +`collect_pages` only when it has a paging one. Because bluesky's protocols are `runtime_checkable`, an absent attribute is enough to make the isinstance check fail, and the bundler then does the right thing. It has to be a real attribute rather than a `__getattr__` hook, because Python 3.12+ resolves those checks with `inspect.getattr_static`, which never calls `__getattr__`. -This keys off the data logics, which are fixed when the detector is constructed, rather than off -the prepare context, which does not exist until `prepare()` runs. +Which verb applies is only knowable once the logics have said what they would make, so it is +decided in `prepare()` and recomputed on each one — binding the verb that applies and removing +the one that does not. A detector therefore satisfies neither protocol until it has been +prepared, which is fine: everything in the RunEngine that dispatches on them runs after prepare. + +Because the decision is per-prepare rather than per-construction, which logic serves can change +between runs — a detector whose file writer is switched off can fall back to its stats time +series. That is a **between-runs** change, like a readable format: a run's descriptor is emitted +at its start, so switching part way through would leave `describe_collect()` and the documents +disagreeing. Documented rather than enforced, since a Device cannot see run boundaries. This keeps a single detector class able to be either kind depending on its `__init__` arguments — one `AravisDetector`, configured for files or for pages — which was the original requirement, and @@ -218,9 +242,10 @@ it needs no change to bluesky. ### Mixing bounded and unbounded logics raises -`add_detector_logics()` raises if a detector is given both a bounded and an unbounded data logic, -since that is the one combination where both `collect_asset_docs` and `collect_pages` would be -exposed and both would produce data in a fly scan. +`prepare()` raises if the providers it has been given include both kinds, since that is the one +combination where both `collect_asset_docs` and `collect_pages` would be exposed and both would +produce data in a fly scan. It raises after asking each logic what it would make and before +starting any of them, so nothing has been armed when it does. ### Stats with a file writer go via NDAttributes @@ -247,9 +272,13 @@ wanted becomes a constructor argument. ## Consequences -`DetectorDataLogic` implementations that override `prepare_unbounded` must take the new `period` -argument. This is a breaking change for out-of-tree data logics; the library is in alpha and no -compatibility shim is provided. In-tree, `ADHDFDataLogic` and `PandaHDFDataLogic` can then +`DetectorDataLogic` implementations must be rewritten onto `make_data_provider` and `start`, and +take the new `period` argument. This is a breaking change for out-of-tree data logics; the library +is in alpha and no compatibility shim is provided. Splitting them is usually mechanical — describe +in one, write in the other — but a logic that cannot describe its data without starting (in tree, +`OdinDataLogic`, whose frame shape is only readable once the file processor is writing) has to do +its writes in `make_data_provider` and leave `start` empty. That is safe only while such a logic is +never the one a detector drops. In-tree, `ADHDFDataLogic` and `PandaHDFDataLogic` can then compute chunk size and flush period from the rate, replacing the TODOs and hardcoded values that stand in for it today. @@ -257,12 +286,17 @@ stand in for it today. `livetime` and `deadtime` alongside the frame count. Those that do not will still work, but their detectors will fall back to today's behaviour of reading the chunk size back from hardware. -Detectors that cannot write files become expressible: a data logic implementing `prepare_bounded` -gives them `collect_pages()` and the whole of `StandardDetector` besides. This closes #1248 and -#888. +Detectors that cannot write files become expressible: a data logic returning a +`PageableDataProvider` gives them `collect_pages()` and the whole of `StandardDetector` besides. +This closes #1248 and #888. + +Fly scanning an areaDetector that carries an HDF writer plus a finite-buffer data logic starts +working, with the finite buffer sitting the scan out, closing #1364. -Fly scanning an areaDetector that carries an HDF writer plus a single-tier data logic starts -working, with the single-tier logic dropped and a warning, closing #1364. +A data logic may decline a scan for reasons of its own, not just because of the collection count: +`ADHDFDataLogic` and `StatsTimeSeriesDataLogic` take an `enable_callbacks` flag, defaulting to +today's behaviour of switching the plugin on, which when set `False` makes them follow whatever +the plugin is set to and produce nothing when it is off. Mixing bounded and unbounded data logics on one detector is not supported. If a use case appears that genuinely needs a file *and* pages in one fly scan, it needs the bluesky bundler to allow diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index ec1185cb93..e433eb831d 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -228,27 +228,33 @@ Then we have `BlobDataLogic`, a [](#DetectorDataLogic) subclass: ``` Its job is to manage the file writing and data streaming: -- `prepare_unbounded()` tells the detector to open a file, and returns a [](#StreamableDataProvider) that describes the datasets that will be written and tracks write progress +- `make_data_provider()` works out where the file will go and returns a [](#StreamableDataProvider) describing the datasets that will be written, without opening anything +- `start()` opens the file, so nothing is written until the detector has decided it wants this data - `get_hinted_fields()` returns the data keys that are interesting to plot - `stop()` tells the detector to close the file -The [](#StreamableDataProvider) returned from `prepare_unbounded()` contains: +The [](#StreamableDataProvider) returned from `make_data_provider()` contains: - `collections_written_signal` - a signal that tracks how many frames have been written - `make_datakeys()` - creates DataKey descriptions for each dataset - `make_stream_docs()` - emits StreamResource and StreamDatum documents as frames are written -#### Choosing a data logic tier +#### Which kind of data a logic produces -A [](#DetectorDataLogic) implements exactly one of two `prepare_*` methods, and the detector picks a logic by which one it implements — there is no precedence to configure. Which tier fits depends on how the underlying source produces its data: +A [](#DetectorDataLogic) is not declared as one kind or another: the detector asks each of its logics what it would make for this scan, and the type of provider that comes back says which kind it is. -- `prepare_unbounded(datakey_name, period)` — for a source that works for any number of collections, like a file writer. It returns a [](#StreamableDataProvider) and the detector exposes `collect_asset_docs`. `BlobDataLogic` uses this tier. -- `prepare_bounded(datakey_name, num_collections, period)` — for a source that holds a *finite buffer* which must be sized before acquisition, like an areaDetector stats time series or a scaler whose array length is the frame count. It returns a [](#PageableDataProvider) that emits event pages, and the detector exposes `collect_pages` instead. See [](#StatsTimeSeriesDataLogic). +- a [](#StreamableDataProvider) is for a source that works for any number of collections, like a file writer. The detector exposes `collect_asset_docs`, and `BlobDataLogic` is an example. +- a [](#PageableDataProvider) is for a source holding a *finite buffer* that must be sized before acquisition, like an areaDetector stats time series or a scaler whose array length is the frame count. It emits event pages, so the detector exposes `collect_pages` instead. See [](#StatsTimeSeriesDataLogic). +- `None` means "not this scan", for instance a finite buffer asked for an unbounded number of collections, or a plugin that is switched off. This is not an error: a detector may carry more data logics than any one scan uses. A source that produces one value per event, like a plugin scalar, needs no data logic at all: a detector is a [](#StandardReadable), so register the signal with [](#StandardReadable.set_readable_format). -The `period` given to both tiers is the frame period (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0, so a logic can use it to size chunks or a flush rate. +`make_data_provider` is given the total `num_collections` for the scan, 0 meaning unbounded, and the frame `period` (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0. It must not start anything: only the providers the detector settles on get their `start()` called. -A detector may carry several data logics, but not both a bounded and an unbounded one — those produce mutually exclusive document kinds. A logic whose tier cannot serve the requested number of collections (a bounded logic in an infinite fly scan, say) is dropped with a warning rather than failing the scan. +A detector may carry several data logics, but must not end up producing both kinds at once — those are mutually exclusive document kinds, and `prepare()` raises if it would. + +```{note} +Which logic serves is decided in `prepare()`, so it can change between runs — a detector whose file writer is switched off can fall back to its stats time series. Change it **between runs**, not inside one: a run's descriptor is emitted at its start, so switching part way through would leave the descriptor and the documents disagreeing. +``` ## Conclusion diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 915cdf75d9..8291cc2047 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -30,7 +30,6 @@ PageableDataProvider, StreamableDataProvider, ) -from ._log import logger from ._readable import ( StandardReadable, StandardReadableFormat, @@ -313,16 +312,22 @@ async def _get_collections_written( return 0 +#: A provider of either kind, as returned by +#: [](#DetectorDataLogic.make_data_provider). +_DataProvider = StreamableDataProvider | PageableDataProvider + + class DetectorDataLogic: """Abstract base class for detector data logic and handling. - Implementations must override exactly one of two tiers, according to how - many collections the data source can produce: + An implementation describes the data it would produce in + `make_data_provider`, and does the writes that make it happen in `start`. + The detector asks every data logic it has what it would make, decides which + providers it will use, and starts only those. - - `prepare_bounded` for sources that must be told how many collections to - expect because they hold a finite buffer (step scans and flyscans) - - `prepare_unbounded` for sources that work for any number of collections - (step scans and flyscans) + Whether the data is streamed or paged is the type of the provider that + `make_data_provider` returns, not a declaration on the logic, so one logic + may produce either depending on the scan. A source that produces a single value per event, like a plugin scalar, needs no data logic at all: register its signal with @@ -333,26 +338,36 @@ class DetectorDataLogic: #: different for each DetectorDataLogic added to a detector datakey_suffix: str = "" - async def prepare_bounded( + async def make_data_provider( self, datakey_name: str, num_collections: int, period: float - ) -> PageableDataProvider: - """Provider works for a known, finite number of collections. - - :param num_collections: total collections the buffer must be sized for - :param period: how long each collection takes, livetime + deadtime + ) -> StreamableDataProvider | PageableDataProvider | None: + """Describe the data this logic would produce for this scan. + + This must not start anything acquiring: the detector may discard the + provider without calling `start`, and only starts the ones it will use. + Return a [](#StreamableDataProvider) for a source that can produce any + number of collections, or a [](#PageableDataProvider) for one holding a + finite buffer sized to `num_collections`. + + Return `None` to sit this scan out, for instance a finite buffer asked + for an unbounded number of collections, or a plugin that is switched + off. This is not an error and is not warned about. + + :param datakey_name: the detector name plus this logic's datakey_suffix + :param num_collections: total collections for the scan, 0 meaning + unbounded + :param period: how long each collection takes, livetime + deadtime, so + the provider can size its chunks or its buffer. 0 means "use + whatever is currently set on the hardware". """ raise NotImplementedError(self) - async def prepare_unbounded( - self, datakey_name: str, period: float - ) -> StreamableDataProvider: - """Provider can work for an unbounded number of collections. + async def start(self) -> None: + """Make the provider just returned by `make_data_provider` take data. - :param period: how long each collection takes, livetime + deadtime, so - the provider can size its chunks or set a flush rate. 0 means "use - whatever is currently set on the hardware". + Called only for the providers the detector will actually use, so this is + where writes that arm hardware or open files belong. """ - raise NotImplementedError(self) def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: """Return the hinted streams.""" @@ -363,51 +378,21 @@ async def stop(self) -> None: return None -_data_logic_supported = functools.partial(_logic_supported, DetectorDataLogic) - - -class _DataLogicTier(Enum): - """Which prepare tier a data logic implements. - - Discovered by method-override detection (see `_data_logic_tier`); a data - logic must override exactly one of the two `prepare_*` methods. - """ - - UNBOUNDED = "unbounded" - BOUNDED = "bounded" - - -def _data_logic_tier(dl: DetectorDataLogic) -> _DataLogicTier: - """Return which prepare tier a data logic has overridden. - - Tiers are discovered by method-override detection, the same mechanism used - for trigger logic. A data logic must override exactly one. - """ - if _data_logic_supported(dl.prepare_unbounded): - return _DataLogicTier.UNBOUNDED - if _data_logic_supported(dl.prepare_bounded): - return _DataLogicTier.BOUNDED - raise RuntimeError(f"DataLogic hasn't overridden any prepare_* methods {dl}") - - -def _tier_can_serve(tier: _DataLogicTier, num_collections: int) -> bool: - """Whether a tier can serve the requested number of collections. - - - unbounded serves any number, including 0 (infinite) - - bounded serves any finite number, so not 0 - """ - if tier is _DataLogicTier.UNBOUNDED: - return True - return num_collections != 0 - - @dataclass class _PrepareCtx: trigger_info: TriggerInfo - streamable_data_providers: Sequence[StreamableDataProvider] - pageable_data_providers: Sequence[PageableDataProvider] + #: The data logics producing data for this scan, with what they made + serving: Sequence[tuple[DetectorDataLogic, _DataProvider]] collections_written: int + @property + def streamable_data_providers(self) -> list[StreamableDataProvider]: + return [dp for _, dp in self.serving if isinstance(dp, StreamableDataProvider)] + + @property + def pageable_data_providers(self) -> list[PageableDataProvider]: + return [dp for _, dp in self.serving if isinstance(dp, PageableDataProvider)] + @dataclass class _KickoffCtx: @@ -523,38 +508,30 @@ def add_detector_logics( self._data_logics = (*self._data_logics, logic) else: raise TypeError(f"Unknown logic type: {type(logic)}") - # A detector may not mix bounded and unbounded data logics: it would expose - # both collect_pages and collect_asset_docs and produce data from both in a - # fly scan, which the bluesky bundler treats as mutually exclusive. A file - # writer that also wants stats should carry them in the file (as NDAttributes) - # rather than as a separate bounded logic. - bounded = [ - dl for dl in self._data_logics if _data_logic_supported(dl.prepare_bounded) - ] - unbounded = [ - dl - for dl in self._data_logics - if _data_logic_supported(dl.prepare_unbounded) - ] - if bounded and unbounded: - def _describe(logics: Sequence[DetectorDataLogic]) -> str: - return ", ".join(type(dl).__name__ for dl in logics) + def _publish_collect_methods( + self, *, stream_assets: bool, event_pages: bool + ) -> None: + """Bind the collect verb that matches the providers we are using. - raise TypeError( - f"Detector {self.name} has both bounded data logics " - f"({_describe(bounded)}) and unbounded data logics " - f"({_describe(unbounded)}); these cannot be combined on one detector" - ) - # Bind whichever collect verb applies as a real instance attribute, so the - # bundler's isinstance checks (WritesStreamAssets vs EventPageCollectable) - # see exactly one of them. They resolve with inspect.getattr_static on - # Python 3.12+, which does not call __getattr__, so a dynamic hook would be - # invisible. Both names are reserved by Device, hence object.__setattr__. - if unbounded: - object.__setattr__(self, "collect_asset_docs", self._collect_asset_docs) - if bounded: - object.__setattr__(self, "collect_pages", self._collect_pages) + Whichever applies is bound as a real instance attribute, so the bluesky + bundler's isinstance checks (`WritesStreamAssets` vs + `EventPageCollectable`) see exactly one of them: they resolve with + `inspect.getattr_static` on Python 3.12+, which does not call + `__getattr__`, so a dynamic hook would be invisible. Both names are + reserved by `Device`, hence `object.__setattr__`. + + Recomputed on every prepare, so the verb that no longer applies is + removed rather than left behind from a previous scan. + """ + for name, method, wanted in ( + ("collect_asset_docs", self._collect_asset_docs, stream_assets), + ("collect_pages", self._collect_pages, event_pages), + ): + if wanted: + object.__setattr__(self, name, method) + elif name in self.__dict__: + object.__delattr__(self, name) # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: @@ -647,78 +624,100 @@ async def _resolve_period(self, value: TriggerInfo) -> TriggerInfo: async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: num_collections = trigger_info.number_of_collections period = trigger_info.livetime + trigger_info.deadtime - # Classify each data logic by the tier it implements, and drop any whose - # tier cannot serve the requested number of collections. This is the rule - # from #1364: rather than raising, a data logic that cannot serve this scan - # is left out with a warning, so a detector carrying a bounded logic can - # still fly for an unbounded number of events with only that logic dropped. - serving: list[tuple[DetectorDataLogic, _DataLogicTier]] = [] - for dl in self._data_logics: - tier = _data_logic_tier(dl) - if _tier_can_serve(tier, num_collections): - serving.append((dl, tier)) - else: - logger.warning( - "%s data logic %s cannot serve %d collections, " - "dropping it from this prepare", - tier.value, - self.name + dl.datakey_suffix, - num_collections, - ) - # Unbounded providers are reused while collections_per_event and the period - # are unchanged, to avoid reopening a file on every step-scan point. Both are - # baked into the StreamResource at construction, so a change to either - # invalidates the reuse. Bounded providers are never reused: re-calling - # prepare_bounded is what re-arms their finite buffer. - previous = self._prepare_ctx.trigger_info if self._prepare_ctx else None - reusable = ( - previous is not None - and previous.collections_per_event == trigger_info.collections_per_event - and previous.livetime + previous.deadtime == period + previous = self._prepare_ctx + # Providers are reused while the scan they were made for is unchanged, so + # that a step scan does not reopen its file on every point. Both parts of + # the reuse key are baked into a streaming provider when it is made: the + # shape from collections_per_event and the chunking from the period. A + # finite buffer is re-made either way, since re-making is what re-arms it. + reusable = previous is not None and ( + previous.trigger_info.collections_per_event + == trigger_info.collections_per_event + and previous.trigger_info.livetime + previous.trigger_info.deadtime + == period ) - if reusable and self._prepare_ctx is not None: - streamable_data_providers = self._prepare_ctx.streamable_data_providers + if reusable and previous is not None: + serving = await self._rearm_bounded(previous, num_collections, period) else: - # Stop the non-bounded logics we are replacing before making new ones - if self._prepare_ctx is not None: - await asyncio.gather( - *( - dl.stop() - for dl, tier in serving - if tier is not _DataLogicTier.BOUNDED - ) - ) - streamable_data_providers = await asyncio.gather( - *( - dl.prepare_unbounded(self.name + dl.datakey_suffix, period) - for dl, tier in serving - if tier is _DataLogicTier.UNBOUNDED - ) - ) - # Bounded providers are always (re)built, re-arming their buffers. - if self._prepare_ctx is not None: - await asyncio.gather( - *(dl.stop() for dl, tier in serving if tier is _DataLogicTier.BOUNDED) - ) - pageable_data_providers = await asyncio.gather( - *( - dl.prepare_bounded( - self.name + dl.datakey_suffix, num_collections, period - ) - for dl, tier in serving - if tier is _DataLogicTier.BOUNDED - ) + serving = await self._make_serving_providers(num_collections, period) + # Which collect verb the detector exposes follows what it will actually + # produce, so it is recomputed here rather than fixed at construction + self._publish_collect_methods( + stream_assets=any( + isinstance(dp, StreamableDataProvider) for _, dp in serving + ), + event_pages=any(isinstance(dp, PageableDataProvider) for _, dp in serving), ) - # Stash the prepare context so we can use it in trigger/kickoff self._prepare_ctx = _PrepareCtx( trigger_info=trigger_info, - streamable_data_providers=streamable_data_providers, - pageable_data_providers=pageable_data_providers, + serving=serving, collections_written=await _get_collections_written( - [*streamable_data_providers, *pageable_data_providers] + [dp for _, dp in serving] ), ) + async def _make_serving_providers( + self, num_collections: int, period: float + ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: + """Ask every data logic what it would make, and start the ones we use.""" + # Stop what is running before anything new is made, since a logic that + # cannot describe its data without opening its file does so here + if self._prepare_ctx is not None: + await asyncio.gather(*(dl.stop() for dl, _ in self._prepare_ctx.serving)) + made = await asyncio.gather( + *( + dl.make_data_provider( + self.name + dl.datakey_suffix, num_collections, period + ) + for dl in self._data_logics + ) + ) + # A logic returns None to sit this scan out, e.g. a finite buffer asked + # for an unbounded number of collections + serving = [ + (dl, dp) + for dl, dp in zip(self._data_logics, made, strict=True) + if dp is not None + ] + streamable = [dp for _, dp in serving if isinstance(dp, StreamableDataProvider)] + pageable = [dp for _, dp in serving if isinstance(dp, PageableDataProvider)] + # A detector may not produce both kinds at once: it would expose both + # collect_asset_docs and collect_pages and produce data from both in a fly + # scan, which the bluesky bundler treats as mutually exclusive. A file + # writer that also wants stats should carry them in the file (as + # NDAttributes) rather than as a separate bounded logic. + if streamable and pageable: + + def _describe(providers: Sequence[_DataProvider]) -> str: + return ", ".join(type(dp).__name__ for dp in providers) + + raise TypeError( + f"Detector {self.name} would produce both event pages " + f"({_describe(pageable)}) and stream assets ({_describe(streamable)}); " + "these cannot be combined on one detector" + ) + await asyncio.gather(*(dl.start() for dl, _ in serving)) + return serving + + async def _rearm_bounded( + self, previous: _PrepareCtx, num_collections: int, period: float + ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: + """Re-make and re-start the finite buffers, keeping everything else.""" + serving: list[tuple[DetectorDataLogic, _DataProvider]] = [] + for dl, dp in previous.serving: + if isinstance(dp, PageableDataProvider): + await dl.stop() + rearmed = await dl.make_data_provider( + self.name + dl.datakey_suffix, num_collections, period + ) + if rearmed is None: + continue + await dl.start() + serving.append((dl, rearmed)) + else: + serving.append((dl, dp)) + return serving + async def _wait_for_index( self, data_providers: Sequence[StreamableDataProvider | PageableDataProvider], diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 7a5182b162..b405d10f92 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -1,6 +1,6 @@ import asyncio -from collections.abc import Callable, Sequence -from dataclasses import dataclass +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, field from pathlib import PureWindowsPath from typing import Any, Generic, TypeVar from xml.etree import ElementTree as ET @@ -17,6 +17,7 @@ StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + error_if_none, set_and_wait_for_value, ) from ophyd_async.epics.core import stop_busy_record @@ -31,6 +32,7 @@ NDPluginBaseIO, NDPluginFileIO, NDStatsIO, + plugin_is_enabled, ) from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType from ._stats_time_series import StatsTimeSeriesDataLogic @@ -175,10 +177,24 @@ class ADHDFDataLogic(DetectorDataLogic): #: (#1309). Left as None, the chunk size is read back from the IOC, the #: behaviour before #1309. flush_period: float | None = None + #: Whether to switch the plugin on when starting. Left True, the plugin is + #: enabled as part of starting, which is the behaviour beamlines rely on. + #: Set False to follow whatever the plugin is set to instead: a disabled + #: plugin then makes no provider, so another data logic can serve the scan. + enable_callbacks: bool = True + #: What make_data_provider worked out for start to write + _to_start: "tuple[PathInfo, int | None] | None" = field( + default=None, init=False, repr=False + ) - async def prepare_unbounded( - self, datakey_name: str, period: float - ) -> StreamableDataProvider: + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float + ) -> StreamableDataProvider | None: + # A file writer captures for as long as it is told to, so it does not + # need to know how many collections the scan will take. + del num_collections + if not self.enable_callbacks and not await plugin_is_enabled(self.writer): + return None # Work out where to write path_info = self.path_provider(datakey_name) # Size the HDF chunk from the frame period and the target flush period: @@ -186,32 +202,15 @@ async def prepare_unbounded( # frames per chunk, so the file is flushed at ~2 Hz (#1309). Only when a # flush_period is configured and the period is known (non-zero); otherwise # fall back to reading the chunk size back from the IOC, forcing a - # fresh-startup 0 to 1. + # fresh-startup 0 to 1. Only a value we chose is written back, in start(). if self.flush_period is not None and period > 0: - frames_per_chunk = max(1, round(self.flush_period / period)) - await self.writer.num_frames_chunks.set(frames_per_chunk) + frames_per_chunk = to_write = max(1, round(self.flush_period / period)) else: frames_per_chunk = await self.writer.num_frames_chunks.get_value() + to_write = None if frames_per_chunk == 0: - frames_per_chunk = 1 - await self.writer.num_frames_chunks.set(frames_per_chunk) - # Setup the HDF writer - await asyncio.gather( - self.writer.chunk_size_auto.set(True), - self.writer.num_extra_dims.set(0), - self.writer.lazy_open.set(True), - self.writer.swmr_mode.set(True), - self.writer.xml_file_name.set(""), - self.writer.enable_callbacks.set(EnableDisable.ENABLE), - prepare_file_paths( - path_info=path_info, file_template="%s%s.h5", writer=self.writer - ), - ) - # Start capturing - await set_and_wait_for_value( - self.writer.capture, True, wait_for_set_completion=False - ) - # Return a provider that reflects what we have made + frames_per_chunk = to_write = 1 + # Describe what we would write main_dataset = await get_ndarray_resource_info( array_description=self.array_description, data_key=datakey_name, @@ -234,6 +233,7 @@ async def prepare_unbounded( ) for name, (dtype_numpy, source) in ndattribute_dtype_sources.items() ] + self._to_start = (path_info, to_write) return StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[main_dataset] + ndattribute_datasets, @@ -242,6 +242,31 @@ async def prepare_unbounded( flush_signal=self.writer.flush_now, ) + async def start(self) -> None: + path_info, frames_per_chunk = error_if_none( + self._to_start, "make_data_provider() has not been called" + ) + if frames_per_chunk is not None: + await self.writer.num_frames_chunks.set(frames_per_chunk) + # Setup the HDF writer + coros: list[Awaitable] = [ + self.writer.chunk_size_auto.set(True), + self.writer.num_extra_dims.set(0), + self.writer.lazy_open.set(True), + self.writer.swmr_mode.set(True), + self.writer.xml_file_name.set(""), + prepare_file_paths( + path_info=path_info, file_template="%s%s.h5", writer=self.writer + ), + ] + if self.enable_callbacks: + coros.append(self.writer.enable_callbacks.set(EnableDisable.ENABLE)) + await asyncio.gather(*coros) + # Start capturing + await set_and_wait_for_value( + self.writer.capture, True, wait_for_set_completion=False + ) + async def stop(self) -> None: await stop_busy_record(self.writer.capture) @@ -271,33 +296,24 @@ class ADMultipartDataLogic(DetectorDataLogic): datakey_suffix: str = "" hinted: bool = True - async def prepare_unbounded( - self, datakey_name: str, period: float + #: Where make_data_provider decided to write, for start to set up + _to_start: "PathInfo | None" = field(default=None, init=False, repr=False) + + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float ) -> StreamableDataProvider: # A multipart writer writes one file per frame, so there is no chunk to - # size from the period. - del period + # size from the period, and it writes for as long as it is told to. + del period, num_collections # Work out where to write path_info = self.path_provider(datakey_name) - # Setup the file writer - await asyncio.gather( - self.writer.lazy_open.set(True), - prepare_file_paths( - path_info=path_info, - file_template="%s%s_%6.6d" + self.extension, - writer=self.writer, - ), - ) - # Start capturing - await set_and_wait_for_value( - self.writer.capture, True, wait_for_set_completion=False - ) - # Return a provider that reflects what we have made + # Describe what we would write main_dataset = await get_ndarray_resource_info( array_description=self.array_description, data_key=datakey_name, parameters={"template": path_info.filename + "_{:06d}" + self.extension}, ) + self._to_start = path_info return StreamResourceDataProvider( # TODO: remove the type ignore after # https://github.com/bluesky/ophyd-async/issues/1186 @@ -307,6 +323,24 @@ async def prepare_unbounded( collections_written_signal=self.writer.num_captured, ) + async def start(self) -> None: + path_info = error_if_none( + self._to_start, "make_data_provider() has not been called" + ) + # Setup the file writer + await asyncio.gather( + self.writer.lazy_open.set(True), + prepare_file_paths( + path_info=path_info, + file_template="%s%s_%6.6d" + self.extension, + writer=self.writer, + ), + ) + # Start capturing + await set_and_wait_for_value( + self.writer.capture, True, wait_for_set_completion=False + ) + async def stop(self) -> None: await stop_busy_record(self.writer.capture) @@ -399,6 +433,7 @@ def hdf( | None = None, hinted: bool = True, flush_period: float | None = None, + enable_callbacks: bool = True, ) -> "ADWriterFactory[NDFileHDF5IO]": """Create a factory for an HDF5 file writer. @@ -419,6 +454,10 @@ def hdf( chunk is sized from this and the frame period so the file flushes at roughly this rate (#1309). Left as ``None`` (the default), the chunk size is read back from the IOC as before. + :param enable_callbacks: Whether to switch the plugin on when starting, + defaults to ``True``. Set ``False`` to follow whatever the plugin is + set to instead: a disabled plugin then writes nothing and another + data logic can serve the scan. """ return ADWriterFactory( writer_cls=NDFileHDF5IO, @@ -435,6 +474,7 @@ def hdf( datakey_suffix=datakey_suffix, hinted=hinted, flush_period=flush_period, + enable_callbacks=enable_callbacks, ), ) @@ -537,6 +577,7 @@ def stats( [NDStatsIO], Sequence[tuple[str, SignalR[Array1D[np.float64]]]] ] | None = None, + enable_callbacks: bool = True, ) -> "ADWriterFactory[NDStatsIO]": """Create a factory for an NDPluginStats time series. @@ -544,6 +585,9 @@ def stats( time series is a fixed-length buffer read back as event pages, for detectors that cannot write assets (see `StatsTimeSeriesDataLogic`). + :param enable_callbacks: Whether to switch the plugin on when starting, + defaults to ``True``. Set ``False`` to follow whatever the plugin is + set to instead: a disabled plugin then produces nothing. :param writer_suffix: PV suffix for the NDPluginStats plugin, defaults to ``STAT:``. :param writer_name: @@ -566,6 +610,7 @@ def stats( stats=writer, stat_signals=list(stat_signals(writer)) if stat_signals else [], datakey_suffix=datakey_suffix, + enable_callbacks=enable_callbacks, ) ), ) diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index b80bfb1e47..a4005788be 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -167,6 +167,17 @@ class NDPluginBaseIO(NDArrayBaseIO): queue_size: A[SignalRW[int], PvSuffix.rbv("QueueSize")] +async def plugin_is_enabled(plugin: NDPluginBaseIO) -> bool: + """Whether a plugin is switched on, so is receiving NDArrays. + + The nearest thing to "is this plugin in the chain": a plugin with callbacks + disabled certainly is not, though an enabled one may still have its + `nd_array_port` pointed somewhere else. Parsing the whole plugin graph was + rejected in ADR 0021 as too large. + """ + return await plugin.enable_callbacks.get_value() is EnableDisable.ENABLE + + class NDROIIO(StandardReadable, NDPluginBaseIO): """Plugin for taking a region of an NDArray. diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index bb66ba3dc4..6cb445ae97 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -1,5 +1,5 @@ import asyncio -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence from dataclasses import dataclass, field import numpy as np @@ -12,10 +12,11 @@ EnableDisable, PageableDataProvider, SignalR, + error_if_none, gather_dict, ) -from ._io import NDStatsIO, NDStatsTSAcquireMode +from ._io import NDStatsIO, NDStatsTSAcquireMode, plugin_is_enabled class StatsTimeSeriesProvider(PageableDataProvider): @@ -103,11 +104,11 @@ class StatsTimeSeriesDataLogic(DetectorDataLogic): For detectors that write no file: the stats plugin holds each statistic in a fixed-length buffer that must be sized before acquisition, so this is a - bounded (`prepare_bounded`) data logic that emits event pages. One plugin has + data logic that emits event pages. One plugin has one time-series control (`ts_acquire`, `ts_num_points`) shared across many arrays, so one logic covers many arrays with the control embedded. - The buffer is sized and armed in `prepare_bounded` by writing 1 to + The buffer is sized and armed in `start` by writing 1 to ``ts_acquire``, which erases the arrays and resets the current point before the detector's acquire logic drives the camera; its frames then feed the time series via NDArray callbacks. A detector that writes a file should pull stats @@ -123,33 +124,54 @@ class StatsTimeSeriesDataLogic(DetectorDataLogic): default_factory=list ) datakey_suffix: str = "" + #: Whether to switch the plugin on when starting. Left True, the plugin is + #: enabled as part of starting. Set False to follow whatever the plugin is + #: set to instead: a disabled plugin then makes no provider. + enable_callbacks: bool = True + #: What make_data_provider sized the buffer for, for start to write + _num_collections: int | None = field(default=None, init=False, repr=False) def _arrays(self, datakey_name: str) -> dict[str, SignalR[Array1D[np.float64]]]: signals = self.stat_signals or [("", self.stats.ts_total)] return {datakey_name + suffix: signal for suffix, signal in signals} - async def prepare_bounded( + async def make_data_provider( self, datakey_name: str, num_collections: int, period: float - ) -> PageableDataProvider: + ) -> PageableDataProvider | None: # The buffer is sized by count, not rate, so the period is unused. del period + if num_collections == 0: + # A finite buffer cannot serve an unbounded scan + return None + if not self.enable_callbacks and not await plugin_is_enabled(self.stats): + return None + # The datakeys are known from the arrays we were configured with, so + # nothing has to be armed to describe them + self._num_collections = num_collections + return StatsTimeSeriesProvider( + self._arrays(datakey_name), + self.stats.ts_current_point, + self.stats.ts_timestamp, + ) + + async def start(self) -> None: + num_collections = error_if_none( + self._num_collections, "make_data_provider() has not been called" + ) # Size the buffer and put it in fixed-length mode before arming it. - await asyncio.gather( - self.stats.enable_callbacks.set(EnableDisable.ENABLE), + coros: list[Awaitable] = [ self.stats.compute_statistics.set(True), self.stats.ts_num_points.set(num_collections), self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), - ) + ] + if self.enable_callbacks: + coros.append(self.stats.enable_callbacks.set(EnableDisable.ENABLE)) + await asyncio.gather(*coros) # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to # 0, so the buffer is armed and empty before the detector's frames arrive. # This is the data logic performing the erase itself, which is why # trigger()'s zero baseline is correct (see ADR 0020). await self.stats.ts_acquire.set(True) - return StatsTimeSeriesProvider( - self._arrays(datakey_name), - self.stats.ts_current_point, - self.stats.ts_timestamp, - ) async def stop(self) -> None: await self.stats.ts_acquire.set(False) diff --git a/src/ophyd_async/fastcs/odin/_data_logic.py b/src/ophyd_async/fastcs/odin/_data_logic.py index ef8e67363a..dcf0dc0ff3 100644 --- a/src/ophyd_async/fastcs/odin/_data_logic.py +++ b/src/ophyd_async/fastcs/odin/_data_logic.py @@ -31,11 +31,17 @@ def __init__( self.detector_bit_depth = detector_bit_depth self.pixel_mask = pixel_mask - async def prepare_unbounded( - self, datakey_name: str, period: float + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float ) -> StreamableDataProvider: - # Odin sizes its own chunks, so the frame period is not used here yet. - del period + # Odin sizes its own chunks and writes for as long as it is told to, so + # neither the frame period nor the count is used here yet. + del period, num_collections + # Unlike other data logics this one cannot describe its data without + # starting: the frame shape is only readable from the file processor + # once it is writing. So it does its own writes here and has nothing + # left for start(). That is safe while it is the only kind of provider a + # detector carrying it can produce, since it is then never discarded. # Work out where to write path_info = self.path_provider(datakey_name) # Get the current bit depth diff --git a/src/ophyd_async/fastcs/panda/_data_logic.py b/src/ophyd_async/fastcs/panda/_data_logic.py index 92f2630898..0a5e674e1a 100644 --- a/src/ophyd_async/fastcs/panda/_data_logic.py +++ b/src/ophyd_async/fastcs/panda/_data_logic.py @@ -5,10 +5,12 @@ from ophyd_async.core import ( DetectorDataLogic, + PathInfo, PathProvider, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + error_if_none, ) from ._block import DataBlock, PandaCaptureMode @@ -24,37 +26,20 @@ def __init__( ): self.path_provider = path_provider self.data_block = data_block + # Where make_data_provider decided to write, for start to set up + self._to_start: PathInfo | None = None - async def prepare_unbounded( - self, datakey_name: str, period: float + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float ) -> StreamableDataProvider: # TODO: derive the PandA flush period / chunk size from `period` once the - # IOC exposes the chunk-size signal (see the chunk_shape TODO below). - del period + # IOC exposes the chunk-size signal (see the chunk_shape TODO below). The + # PandA captures forever, so the number of collections is not needed. + del period, num_collections # Work out where to write path_info = self.path_provider(datakey_name) - # Set create dir depth first to guarantee that callback when setting - # directory path has correct value - await self.data_block.create_directory.set(path_info.create_dir_depth) - # Setup the HDF writer - await asyncio.gather( - self.data_block.flush_period.set(0), - self.data_block.hdf_directory.set(str(path_info.directory_path)), - self.data_block.hdf_file_name.set( - f"{path_info.filename}.h5", - ), - self.data_block.capture_mode.set(PandaCaptureMode.FOREVER), - ) - # Make sure that directory exists or has been created. - if not await self.data_block.directory_exists.get_value() == 1: - raise OSError( - f"Directory {path_info.directory_path} does not exist or " - "is not writable by the PandABlocks-ioc!" - ) - # Start capturing - await self.data_block.capture.set(True) - # Load data from the datasets PV on the panda, update internal - # representation of datasets that the panda will write. + # Load data from the datasets PV on the panda, which tells us which + # datasets it will write capture_table = await self.data_block.datasets.get_value() if len(capture_table) == 0: logger.warning( @@ -75,6 +60,7 @@ async def prepare_unbounded( ) for dataset_name in capture_table.name ] + self._to_start = path_info return StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=resources, @@ -82,5 +68,30 @@ async def prepare_unbounded( collections_written_signal=self.data_block.num_captured, ) + async def start(self) -> None: + path_info = error_if_none( + self._to_start, "make_data_provider() has not been called" + ) + # Set create dir depth first to guarantee that callback when setting + # directory path has correct value + await self.data_block.create_directory.set(path_info.create_dir_depth) + # Setup the HDF writer + await asyncio.gather( + self.data_block.flush_period.set(0), + self.data_block.hdf_directory.set(str(path_info.directory_path)), + self.data_block.hdf_file_name.set( + f"{path_info.filename}.h5", + ), + self.data_block.capture_mode.set(PandaCaptureMode.FOREVER), + ) + # Make sure that directory exists or has been created. + if not await self.data_block.directory_exists.get_value() == 1: + raise OSError( + f"Directory {path_info.directory_path} does not exist or " + "is not writable by the PandABlocks-ioc!" + ) + # Start capturing + await self.data_block.capture.set(True) + async def stop(self) -> None: await self.data_block.capture.set(False) diff --git a/src/ophyd_async/sim/_blob_data_logic.py b/src/ophyd_async/sim/_blob_data_logic.py index ab234d172b..d660c0248d 100644 --- a/src/ophyd_async/sim/_blob_data_logic.py +++ b/src/ophyd_async/sim/_blob_data_logic.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from pathlib import PurePath import numpy as np @@ -8,6 +9,7 @@ StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + error_if_none, ) from ._pattern_generator import DATA_PATH, SUM_PATH, PatternGenerator @@ -24,18 +26,19 @@ def __init__( ): self.path_provider = path_provider self.pattern_generator = pattern_generator + # Where make_data_provider decided to write, for start to open + self._to_open: PurePath | None = None - async def prepare_unbounded( - self, datakey_name: str, period: float + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float ) -> StreamableDataProvider: - # The sim blob writer uses a fixed chunk shape, so the period is unused. - del period + # The sim blob writer uses a fixed chunk shape and writes for as long as + # it is told to, so neither the period nor the count is needed. + del period, num_collections # Work out where to write path_info = self.path_provider(datakey_name) - # Open the file - write_path = path_info.directory_path / f"{path_info.filename}.h5" - self.pattern_generator.open_file(write_path, WIDTH, HEIGHT) - # Return a provider that reflects what we have made + self._to_open = path_info.directory_path / f"{path_info.filename}.h5" + # Describe what we would write data_resource = StreamResourceInfo( data_key=datakey_name, shape=(HEIGHT, WIDTH), @@ -61,6 +64,12 @@ async def prepare_unbounded( collections_written_signal=self.pattern_generator.images_written, ) + async def start(self) -> None: + write_path = error_if_none( + self._to_open, "make_data_provider() has not been called" + ) + self.pattern_generator.open_file(write_path, WIDTH, HEIGHT) + async def stop(self) -> None: self.pattern_generator.close_file() diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 7dfa1a48c3..582a1bd4ef 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -142,8 +142,8 @@ def __init__(self, tmp_path, datakey_suffix: str = ""): self.tmp_path = tmp_path self.datakey_suffix = datakey_suffix - async def prepare_unbounded( - self, datakey_name: str, period: float + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float ) -> StreamableDataProvider: resource = StreamResourceInfo( data_key=datakey_name, @@ -232,14 +232,22 @@ def __init__(self): self.collections_written = soft_signal_rw(int) self.prepare_calls: list[tuple[int, float]] = [] self.stop_count = 0 + self._to_start: tuple[int, float] | None = None - async def prepare_bounded( + async def make_data_provider( self, datakey_name: str, num_collections: int, period: float - ) -> PageableDataProvider: - self.prepare_calls.append((num_collections, period)) + ) -> PageableDataProvider | None: + if num_collections == 0: + # A finite buffer cannot serve an unbounded scan + return None + self._to_start = (num_collections, period) + return MockPageableProvider(datakey_name, self.collections_written) + + async def start(self) -> None: + assert self._to_start is not None + self.prepare_calls.append(self._to_start) # A real buffer clears its progress counter when armed await self.collections_written.set(0) - return MockPageableProvider(datakey_name, self.collections_written) async def stop(self) -> None: self.stop_count += 1 @@ -761,6 +769,7 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): # Yield so detector can get collections written, then set it so we complete await wait_for_pending_wakeups(raise_if_exceeded=False) await dl.collections_written.set(1) + await wait_for_pending_wakeups(raise_if_exceeded=False) assert status.done assert status.success docs = [doc async for doc in det.collect_asset_docs()] @@ -1052,22 +1061,23 @@ async def default_trigger_info(self) -> TriggerInfo: assert dl.prepare_calls == [(3, pytest.approx(expected_period))] -async def test_bounded_dropped_for_infinite_events(caplog): - """A bounded buffer cannot serve an infinite scan, so is dropped with a warning.""" +async def test_bounded_dropped_for_infinite_events(): + """A bounded buffer cannot serve an infinite scan, so it makes no provider.""" det = StandardDetector(name="foo") - det.add_detector_logics(JustInternalTriggerLogic(), BoundedOnlyDataLogic()) + dl = BoundedOnlyDataLogic() + det.add_detector_logics(JustInternalTriggerLogic(), dl) - with caplog.at_level("WARNING"): - await det.prepare(TriggerInfo(number_of_events=0)) + await det.prepare(TriggerInfo(number_of_events=0)) - assert "cannot serve 0 collections" in caplog.text ctx = det._prepare_ctx assert ctx is not None assert ctx.pageable_data_providers == [] + # Sitting the scan out is not an error, and nothing was armed + assert dl.prepare_calls == [] -async def test_data_logic_with_no_prepare_methods_raises(): - """Test error when DataLogic doesn't override any prepare methods.""" +async def test_data_logic_with_no_make_data_provider_raises(): + """Test error when a DataLogic doesn't say what it would make.""" class EmptyDataLogic(DetectorDataLogic): pass @@ -1075,7 +1085,7 @@ class EmptyDataLogic(DetectorDataLogic): det = StandardDetector() det.add_detector_logics(EmptyDataLogic()) - with pytest.raises(RuntimeError, match="hasn't overridden any prepare_\\* methods"): + with pytest.raises(NotImplementedError): await det.prepare(TriggerInfo()) @@ -1291,10 +1301,11 @@ async def test_data_logic_not_implemented_errors(): logic = DetectorDataLogic() with pytest.raises(NotImplementedError): - await logic.prepare_unbounded("test", 0.1) + await logic.make_data_provider("test", 1, 0.1) - # stop() should not raise (has default implementation) - await logic.stop() # Should pass + # start() and stop() should not raise (they have default implementations) + await logic.start() + await logic.stop() async def test_detector_readable_format_changes_at_runtime(): @@ -1337,6 +1348,11 @@ async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): det = StandardDetector() det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + # Which verb applies follows what the logics will produce, so it is not + # known until they have been asked + assert not isinstance(det, WritesStreamAssets) + await det.prepare(TriggerInfo()) + assert isinstance(det, WritesStreamAssets) assert not isinstance(det, EventPageCollectable) assert hasattr(det, "collect_asset_docs") @@ -1347,12 +1363,18 @@ async def test_bounded_logic_looks_like_event_page_collectable(): """A bounded logic exposes collect_pages, and only that.""" det = StandardDetector() det.add_detector_logics(BoundedOnlyDataLogic()) + await det.prepare(TriggerInfo()) assert isinstance(det, EventPageCollectable) assert not isinstance(det, WritesStreamAssets) assert hasattr(det, "collect_pages") assert not hasattr(det, "collect_asset_docs") + # An unbounded scan drops the finite buffer, so the verb goes with it + await det.prepare(TriggerInfo(number_of_events=0)) + assert not isinstance(det, EventPageCollectable) + assert not hasattr(det, "collect_pages") + async def test_no_data_logic_looks_like_neither(): """A detector with no data logic emits neither stream assets nor event pages.""" @@ -1378,13 +1400,14 @@ async def test_missing_attribute_raises_standard_attribute_error(): async def test_mixing_bounded_and_unbounded_logics_raises(tmp_path): """Bounded and unbounded logics are mutually exclusive on one detector.""" det = StandardDetector() + det.add_detector_logics(BoundedOnlyDataLogic(), StreamableOnlyDataLogic(tmp_path)) + + # Only knowable once the logics have said what they would make with pytest.raises( TypeError, match=( - r"has both bounded data logics \(BoundedOnlyDataLogic\) and unbounded " - r"data logics \(StreamableOnlyDataLogic\)" + r"would produce both event pages \(MockPageableProvider\) and stream " + r"assets \(StreamResourceDataProvider\)" ), ): - det.add_detector_logics( - BoundedOnlyDataLogic(), StreamableOnlyDataLogic(tmp_path) - ) + await det.prepare(TriggerInfo()) diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 41ababfb28..351f9c167e 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -14,6 +14,7 @@ set_mock_value, ) from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.epics.adcore import ADHDFDataLogic, NDArrayDescription from ophyd_async.testing import assert_has_calls @@ -33,6 +34,76 @@ async def hdf_det( return detector +async def test_make_data_provider_does_not_write( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, +): + """Describing the data must not open the file, since it may be discarded.""" + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + ) + + provider = await logic.make_data_provider("det", num_collections=5, period=0.1) + assert provider is not None + assert await writer.capture.get_value() is False + assert await writer.file_name.get_value() == "" + + await logic.start() + assert await writer.capture.get_value() is True + assert await writer.file_name.get_value() != "" + + +@pytest.mark.parametrize( + "enable_callbacks,plugin_enabled,makes_provider", + [ + # The default switches the plugin on, whatever it was set to + (True, EnableDisable.DISABLE, True), + # Following the plugin, a disabled one writes nothing... + (False, EnableDisable.DISABLE, False), + # ...and an enabled one still writes + (False, EnableDisable.ENABLE, True), + ], +) +async def test_hdf_follows_the_plugin_when_not_enabling_it( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, + enable_callbacks: bool, + plugin_enabled: EnableDisable, + makes_provider: bool, +): + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + set_mock_value(writer.enable_callbacks, plugin_enabled) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + enable_callbacks=enable_callbacks, + ) + + provider = await logic.make_data_provider("det", num_collections=5, period=0.1) + assert (provider is not None) is makes_provider + + if makes_provider: + await logic.start() + expected = EnableDisable.ENABLE if enable_callbacks else plugin_enabled + assert await writer.enable_callbacks.get_value() is expected + + async def test_hdf_writer_file_not_found(hdf_det: adcore.AreaDetector[adcore.ADBaseIO]): with pytest.raises( FileNotFoundError, match=r"Path .* doesn't exist or not writable!" diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index 9c23878a17..d96466a496 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -4,6 +4,7 @@ from ophyd_async.core import ( DetectorAcquireLogic, DetectorTriggerLogic, + EnableDisable, StandardDetector, TriggerInfo, init_devices, @@ -24,11 +25,16 @@ async def stats() -> adcore.NDStatsIO: return plugin -async def test_prepare_bounded_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): +async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): logic = StatsTimeSeriesDataLogic(stats) - provider = await logic.prepare_bounded("det-stats", num_collections=5, period=0.1) - + provider = await logic.make_data_provider( + "det-stats", num_collections=5, period=0.1 + ) assert isinstance(provider, StatsTimeSeriesProvider) + # Describing the buffer does not arm it + assert await stats.ts_num_points.get_value() == 0 + + await logic.start() assert await stats.ts_num_points.get_value() == 5 assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH # ts_acquire=1 arms and clears the buffer @@ -40,6 +46,42 @@ async def test_prepare_bounded_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO assert logic.get_hinted_fields("det-stats") == ["det-stats"] +async def test_unbounded_scan_makes_no_provider(stats: adcore.NDStatsIO): + """A finite buffer cannot serve an unbounded scan, so it sits it out.""" + logic = StatsTimeSeriesDataLogic(stats) + assert await logic.make_data_provider("det", num_collections=0, period=0.1) is None + + +@pytest.mark.parametrize( + "enable_callbacks,plugin_enabled,makes_provider", + [ + # The default switches the plugin on, whatever it was set to + (True, EnableDisable.DISABLE, True), + # Following the plugin, a disabled one produces nothing... + (False, EnableDisable.DISABLE, False), + # ...and an enabled one still works + (False, EnableDisable.ENABLE, True), + ], +) +async def test_follows_the_plugin_when_not_enabling_it( + stats: adcore.NDStatsIO, + enable_callbacks: bool, + plugin_enabled: EnableDisable, + makes_provider: bool, +): + set_mock_value(stats.enable_callbacks, plugin_enabled) + logic = StatsTimeSeriesDataLogic(stats, enable_callbacks=enable_callbacks) + + provider = await logic.make_data_provider("det", num_collections=5, period=0.1) + assert (provider is not None) is makes_provider + + if makes_provider: + await logic.start() + # Only the default reaches out and switches the plugin on + expected = EnableDisable.ENABLE if enable_callbacks else plugin_enabled + assert await stats.enable_callbacks.get_value() is expected + + async def test_stop_stops_the_time_series(stats: adcore.NDStatsIO): logic = StatsTimeSeriesDataLogic(stats) set_mock_value(stats.ts_acquire, True) @@ -127,12 +169,11 @@ def stats_detector(stats: adcore.NDStatsIO) -> StandardDetector: async def test_step_scan_collect_pages_end_to_end(stats_detector: StandardDetector): """A writer-less detector with a stats time series emits event pages.""" det = stats_detector + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=4)) # A bounded logic exposes collect_pages, not collect_asset_docs assert hasattr(det, "collect_pages") assert not hasattr(det, "collect_asset_docs") - - await det.stage() - await det.prepare(TriggerInfo(collections_per_event=4)) await det.trigger() pages = [page async for page in det.collect_pages()] (page,) = pages From e5ad453c79e302e3828e1a6597fa6887855d9c2c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 28 Aug 2026 18:23:49 +0000 Subject: [PATCH 43/67] Make StandardDetector a StandardFlyable The detector had its own prepare/kickoff/complete, threading its own state between them and guarding each stage on whether a context was None, so a verb called too early failed with an AttributeError on None. StandardFlyable already does all of that, for every other flyer in the tree. To satisfy it, the trigger, acquire and data logics become children of one DetectorLogic, which a detector builds in __init__ and returns from logic; the orchestration moves with them and the logic holds no back-reference, so ADR 0017 still holds unqualified. StandardDetector itself takes no __init__ args, since a Standard* class that did would interfere with multiple inheritance, and with_logics() builds both logic and Device for an ad-hoc detector. What the logic cannot get from its own fields is the datakey namespace, pushed in by set_name, and which collect verb the Device should expose, done with a required callback it is constructed with and called from prepare -- because only then is it known what the data logics will actually produce. That also lets carrying a file writer and a finite buffer stop being an error when the file covers every key the buffer would produce: the durable copy wins and the buffer sits the scan out, unarmed. Uncovered keys still raise, naming them. Progress reporting would otherwise have been lost, since complete() only produced WatcherUpdates for a MovableLogic; WatchableFlyableLogic lets a logic whose progress is neither a readback nor a setpoint yield its own. events_to_kickoff and multi-kickoff go with this: nothing drove them, the RunEngine completes what it kicked off, and every other flyer already worked that way. The providers stay on the logic rather than in the fly context, because a step scan reuses them across points and the RunEngine collects after complete() has returned; trigger() re-arms only what needs it, so a step scan does not repeat the trigger logic's puts. See ADR 0023. --- docs/conf.py | 4 + .../decisions/0020-bounded-data-logic-tier.md | 2 +- .../decisions/0023-detector-is-a-flyable.md | 139 +++ docs/how-to/implement-ad-detector.md | 4 +- docs/reference/environment-variables.md | 2 +- docs/tutorials/implementing-detectors.md | 2 +- src/ophyd_async/core/__init__.py | 10 +- src/ophyd_async/core/_detector.py | 958 +++++++++--------- src/ophyd_async/core/_flyable.py | 47 +- src/ophyd_async/epics/adcore/_detector.py | 19 +- src/ophyd_async/fastcs/eiger/_detector.py | 10 +- src/ophyd_async/fastcs/jungfrau/_detector.py | 16 +- src/ophyd_async/fastcs/panda/_detector.py | 11 +- src/ophyd_async/sim/_blob_detector.py | 19 +- .../unit_tests/core/test_standard_detector.py | 277 +++-- .../epics/adcore/test_acquire_logic.py | 5 +- .../epics/adcore/test_stats_time_series.py | 4 +- .../unit_tests/epics/adcore/test_use_cases.py | 6 +- tests/unit_tests/fastcs/odin/test_odin_io.py | 11 +- .../unit_tests/fastcs/panda/test_hdf_panda.py | 2 +- 20 files changed, 908 insertions(+), 640 deletions(-) create mode 100644 docs/explanations/decisions/0023-detector-is-a-flyable.md diff --git a/docs/conf.py b/docs/conf.py index 9a6eb2d5ae..2cbd14a731 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -197,6 +197,10 @@ def setup(app: application.Sphinx): "event_model.documents.PartialEventPage", # TypeVar for the plugin an ADWriterFactory builds a data logic around "ophyd_async.epics.adcore._data_logic.NDWriterPluginT", + # The context DetectorLogic threads prepare -> kickoff -> complete, private + # because nothing outside StandardDetector constructs or inspects one + "ophyd_async.core._detector._FlyCtx", + "ophyd_async.core._detector._DetectorData", ] nitpick_ignore = [] for var in obj_ignore: diff --git a/docs/explanations/decisions/0020-bounded-data-logic-tier.md b/docs/explanations/decisions/0020-bounded-data-logic-tier.md index 48d5df481e..780c199289 100644 --- a/docs/explanations/decisions/0020-bounded-data-logic-tier.md +++ b/docs/explanations/decisions/0020-bounded-data-logic-tier.md @@ -209,7 +209,7 @@ not of hardware, so it survives the concerns landing on a single PV. ### Logic objects fill exactly one role -`add_detector_logics()` raises if an object satisfies more than one of the three logic roles, +Building a `DetectorLogic` raises if an object satisfies more than one of the three logic roles, directing the author to pass separate objects. This is a hazard worth catching rather than a hypothetical: a device whose concerns all live on one control is a natural candidate for a single combined logic object, and the registration is a chain of `isinstance` tests, so such an object diff --git a/docs/explanations/decisions/0023-detector-is-a-flyable.md b/docs/explanations/decisions/0023-detector-is-a-flyable.md new file mode 100644 index 0000000000..807e77f171 --- /dev/null +++ b/docs/explanations/decisions/0023-detector-is-a-flyable.md @@ -0,0 +1,139 @@ +# 23. StandardDetector is a StandardFlyable + +Date: 2026-08-28 + +## Status + +Accepted + +## Context + +`StandardDetector` re-implemented `prepare`, `kickoff` and `complete` and threaded its own +state between them, while [](#StandardFlyable) already existed to do exactly that for every +other flyer in the tree (`Motor`, PMAC, the PandA trigger logics). Two implementations of +one lifecycle drift apart: the detector guarded its stages by checking whether a context +object was `None`, so calling a verb too early failed with `AttributeError: 'NoneType' +object has no attribute ...` rather than saying what was wrong. + +`StandardFlyable` requires a single [](#FlyableLogic) in a `logic` property (ADR 0022), and +a detector had three logic objects registered on the Device by `add_detector_logics()`. + +## Decision + +### The three logics become children of one `DetectorLogic` + +[](#DetectorLogic) holds the trigger, acquire and data logics, and is what `logic` returns. +The orchestration moves with them: preparing the trigger logic, making the data providers, +arming, and waiting for collections are all its methods now. It holds **no** back-reference +to its Device, so ADR 0017 continues to hold unqualified. + +A detector builds it in `__init__` and returns it from `logic`: + +```python +class MyDetector(StandardDetector): + def __init__(self, prefix: str, name: str = "") -> None: + self.driver = MyDriverIO(prefix) + self._logic = DetectorLogic( + MyTriggerLogic(self.driver), + MyAcquireLogic(self.driver), + publish_collect_methods=self._publish_collect_methods, + ) + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic +``` + +`add_detector_logics()` is removed rather than kept as a forwarder: a detector's logics are +fixed for its lifetime, so there is nothing a post-construction call can do that the +constructor cannot, and keeping it would leave two ways to wire the same Device. + +`StandardDetector` itself takes no `__init__` arguments, since a `Standard*` class that did +would interfere with multiple inheritance. The logic is supplied by overriding `logic`, +which `abstract_cached_property` makes a requirement checked when the Device is +instantiated. Assigning `self.logic` from `__init__` cannot serve instead: `ABCMeta` checks +`__abstractmethods__` in `__call__`, before `__init__` runs, so the class must override the +property to be concrete. For an ad-hoc detector in a plan or a test, +[](#StandardDetector.with_logics) builds both the logic and a Device around it. + +### Two things the logic cannot get from its own fields + +**The datakey namespace.** Data logics name their datakeys after the Device, whose name does +not exist until `init_devices` calls `set_name` on context exit. `StandardDetector.set_name` +pushes it into `DetectorLogic.datakey_prefix` — one string, set at the moment it exists, +rather than a reference to the whole Device. + +**Which collect verb to expose.** A detector exposes `collect_asset_docs` or `collect_pages` +depending on what its data logics produce, never both (ADR 0020). The Device passes +`_publish_collect_methods` into the constructor and the logic calls it from `on_prepare`, so +the binding happens on the Device without the logic holding one. It is a required argument +because the decision belongs to prepare, not to construction: which providers there are is +only known once the logics have been asked. + +### Mixing bounded and unbounded is an error unless the bounded keys are shadowed + +ADR 0020 made carrying both kinds of data logic an error, because the bundler treats stream +assets and event pages as mutually exclusive. Carrying both is still useful, though: the +same quantity can be written durably into a file *and* read from a plugin's buffer — an +areaDetector stats total, which the HDF writer pulls in as an NDAttribute. + +So the rule relaxes. Where **every** datakey a finite buffer would produce is also produced +by the stream assets, the durable copy wins and the buffer sits the scan out, unarmed. +Anything a buffer would produce that the file does not cover is still an error, naming the +uncovered keys. Nothing has been started when that is decided, because +`make_data_provider` describes without acquiring (ADR 0020). + +### A logic may report its own progress + +`StandardFlyable.complete()` could produce `WatcherUpdate`s only for a `MovableLogic`, by +observing its readback against its setpoint. A detector's progress is "collections written +out of collections requested", which is neither, so a detector inheriting the base +`complete()` would have silently lost its progress reporting. + +[](#WatchableFlyableLogic) adds `on_complete_updates(ctx)`, an async iterator of +`WatcherUpdate`; `complete()` yields from it when the logic implements it. The inherited +`on_complete` drains it, so callers that only want to block are unaffected. This is +symmetrical with the existing `MovableLogic` branch rather than a new mechanism. + +### One kickoff per prepare + +The detector used to be prepared for N events and kicked off a few at a time, with an +`events_to_kickoff` signal saying how many the next kickoff covered and an `is_last_kickoff` +flag deciding when to wait for idle. Nothing in tree drove it: the RunEngine kicks off once +and completes what it kicked off, and every other flyer already worked that way. + +Both go, and `kickoff()` requests the whole prepared scan. A second `kickoff()` without an +intervening `prepare()` now raises, from `StandardFlyable`, like any other flyer. + +### The data providers are the logic's state, not the fly context's + +`_FlyCtx` holds what `prepare()` was given — the `TriggerInfo` — plus the collections +baseline `kickoff()` took, and nothing else. The providers live on the logic instead, +because they outlive a single prepare → kickoff → complete cycle in two ways: a step scan +reuses an open file across its points, and `collect_while_completing` collects *after* +`complete()` has returned the flyer to IDLE. So the data verbs — `read`, `describe`, +`describe_collect`, `collect_asset_docs`, `collect_pages`, `get_index` — go through +`DetectorLogic.data`, which is cleared by `stage()`/`unstage()`, while the fly context +enforces the ordering of the fly verbs. + +`trigger()` therefore needs no context of its own. It prepares implicitly if nothing has +been, then asks the data state to re-arm what needs re-arming: a finite buffer holds one +event at a time so it is re-made per point, while a streaming provider carries on. The +trigger logic is not re-prepared per point, so a step scan does not repeat its puts. + +## Consequences + +- `StandardDetector` subclasses build a `DetectorLogic` in `__init__` and return it from + `logic`; `det.add_detector_logics(*logics)` is gone, and a logic conflict is reported when + the detector is constructed rather than at the `add` call. +- `kickoff()` → `kickoff()` without an intervening `complete()` is rejected, and so is a + second `kickoff()` after one. The old detector allowed both, because it only checked that + a prepare context existed. +- `events_to_kickoff` is removed from the public API. +- Calling a verb before `prepare()` says `: prepare() must be called first` instead of + `Prepare not run`. +- A detector satisfies neither `WritesStreamAssets` nor `EventPageCollectable` until it has + been prepared. Everything in the RunEngine that dispatches on them runs after prepare. +- `prepare`, `kickoff`, `complete`, `stage` and `unstage` are no longer implemented on + `StandardDetector` at all, so a change to the flyer lifecycle reaches detectors too. diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 2b3590d744..2a19dce6ca 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -91,7 +91,7 @@ For example, for ADAravis: The `AreaDetector` baseclass will: - Store the driver as `self.driver` -- Call `add_detector_logics()` to register your trigger and arm logic +- Build a [](#DetectorLogic) from your trigger and arm logic and return it from `logic` - Create and register a data logic for file writing for each factory in `writer_factories` - Add configuration signals (driver.acquire_time, driver.acquire_period, and any you specify) - Store any plugins as attributes on the detector @@ -113,7 +113,7 @@ For example, for ADAravis: ## Add multiple data streams (optional) -The composition-based architecture makes it possible to add multiple data outputs to a detector. After creating the detector, you can call `add_detector_logics()` to add additional data sources: +The composition-based architecture makes it possible to give a detector several data outputs: ### Reading stats plugins alongside file writing A plugin scalar needs no data logic: a detector is a [](#StandardReadable), so register diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 78afba8c51..f5d01a2099 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -12,7 +12,7 @@ effect on ophyd-async itself. Controls the implicit prepare performed by [`StandardDetector.trigger()`](#ophyd_async.core.StandardDetector.trigger) when -[`prepare()`](#ophyd_async.core.StandardDetector.prepare) has not been called since +[`prepare()`](#ophyd_async.core.StandardFlyable.prepare) has not been called since the last `stage()`. | Value | Behaviour | diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index e433eb831d..b43f58021d 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -179,7 +179,7 @@ It derives from [](#StandardDetector) which is a utility baseclass that implemen - [](#DetectorAcquireLogic) to start and stop acquisition and wait for it to complete - [](#DetectorDataLogic) to tell the detector to open a file, describe the datasets it will write, and emit StreamAsset documents as frames are written -In this case, we have three logic classes written just for this simulation, all taking a reference to the pattern generator that provides methods for both detector control and file writing. In other cases the detector control and filewriting may be handled by different sub-devices that talk to different parts of the control system. The job of the top level detector class is to take the arguments that the logic classes need, create the logic instances, and pass them to [](#StandardDetector.add_detector_logics). +In this case, we have three logic classes written just for this simulation, all taking a reference to the pattern generator that provides methods for both detector control and file writing. In other cases the detector control and filewriting may be handled by different sub-devices that talk to different parts of the control system. The job of the top level detector class is to take the arguments that the logic classes need, create the logic instances, and wire them into a [](#DetectorLogic), which it returns from `logic`. Now let's look at the underlying classes that define the detector behavior: diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 571552967e..fd0b8841ca 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -27,6 +27,7 @@ from ._detector import ( DetectorAcquireLogic, DetectorDataLogic, + DetectorLogic, DetectorTrigger, DetectorTriggerLogic, StandardDetector, @@ -51,7 +52,12 @@ OnOff, YesNo, ) -from ._flyable import FlyableLogic, FlyMotorInfo, StandardFlyable +from ._flyable import ( + FlyableLogic, + FlyMotorInfo, + StandardFlyable, + WatchableFlyableLogic, +) from ._log import config_ophyd_async_logging, logger, set_handler from ._mock_signal_backend import MockSignalBackend from ._mock_signal_utils import ( @@ -285,6 +291,7 @@ def __getattr__(name): "DetectorTriggerLogic", "DetectorAcquireLogic", "DetectorDataLogic", + "DetectorLogic", "StandardDetector", # Path "PathInfo", @@ -306,6 +313,7 @@ def __getattr__(name): "StandardFlyable", "FlyMotorInfo", "FlyableLogic", + "WatchableFlyableLogic", # Settings "Settings", "SettingsProvider", diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 8291cc2047..f7b68838a3 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -14,9 +14,7 @@ from bluesky.protocols import ( Collectable, - Flyable, HasHints, - Preparable, Reading, Stageable, StreamAsset, @@ -30,6 +28,7 @@ PageableDataProvider, StreamableDataProvider, ) +from ._flyable import StandardFlyable, WatchableFlyableLogic from ._readable import ( StandardReadable, StandardReadableFormat, @@ -38,18 +37,13 @@ _Verb, ) from ._settings import Settings -from ._signal import ( - SignalDict, - SignalR, - SignalRW, - observe_signals_value, - soft_signal_rw, -) -from ._status import AsyncStatus, WatchableAsyncStatus +from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value +from ._status import WatchableAsyncStatus from ._utils import ( DEFAULT_TIMEOUT, ConfinedModel, WatcherUpdate, + abstract_cached_property, error_if_none, merge_gathered_dicts, ) @@ -379,101 +373,73 @@ async def stop(self) -> None: @dataclass -class _PrepareCtx: - trigger_info: TriggerInfo - #: The data logics producing data for this scan, with what they made +class _DetectorData: + """The data providers a detector has prepared, and what they were made for. + + Outlives a single prepare -> kickoff -> complete cycle: a step scan reuses an + open file across its points, and the RunEngine collects from a flyer after + `complete()` has returned. Cleared by `stage()`/`unstage()`. + """ + serving: Sequence[tuple[DetectorDataLogic, _DataProvider]] - collections_written: int + #: What the providers were made for, and so when they can be reused + collections_per_event: int + period: float @property - def streamable_data_providers(self) -> list[StreamableDataProvider]: + def streamable(self) -> list[StreamableDataProvider]: return [dp for _, dp in self.serving if isinstance(dp, StreamableDataProvider)] @property - def pageable_data_providers(self) -> list[PageableDataProvider]: + def pageable(self) -> list[PageableDataProvider]: return [dp for _, dp in self.serving if isinstance(dp, PageableDataProvider)] + @property + def collectable(self) -> list[_DataProvider]: + """Every provider, whether it collects as stream datums or as pages.""" + return [dp for _, dp in self.serving] -@dataclass -class _KickoffCtx: - trigger_info: TriggerInfo - data_providers: Sequence[StreamableDataProvider | PageableDataProvider] - collections_written: int - collections_requested: int - is_last_kickoff: bool - - -class StandardDetector( - StandardReadable, - Stageable, - Triggerable, - Preparable, - Flyable, - Collectable, - HasHints, -): - """Detector base class for step and fly scanning detectors. - - Aggregates trigger, arm, reading or stream logic together. - - Signals read during a step scan are registered with - [](#StandardReadable.set_readable_format), exactly as on any other - `StandardReadable`; data produced by a `DetectorDataLogic` is added on top - of those in `read()` and `describe()`. - `read()` and `describe()` require `prepare()` to have run, and raise - otherwise, so that a descriptor can never be emitted without the detector's - data keys. `trigger()` prepares implicitly, so a step scan never has to do - it explicitly. `read_configuration()` and `describe_configuration()` have no - such requirement. +@dataclass +class _FlyCtx: + """What prepare() set up, threaded through kickoff() to complete().""" - `WritesStreamAssets` (and its `collect_asset_docs`) and - `EventPageCollectable` (and its `collect_pages`) are *not* inherited: a - detector writes stream assets or emits event pages depending on which data - logics it carries, never both, and the bluesky bundler treats the two as - mutually exclusive. The relevant method is bound as an instance attribute in - `add_detector_logics` only when a data logic supporting it is present, so the - bundler's structural isinstance check sees exactly the one that applies. + trigger_info: TriggerInfo + #: What the providers had written when kickoff() ran, None until then + kickoff_collections_written: int | None = None + + +class DetectorLogic(WatchableFlyableLogic[TriggerInfo, _FlyCtx]): + """Drive a detector through its trigger, acquire and data logics. + + :param logics: + At most one [](#DetectorTriggerLogic), at most one + [](#DetectorAcquireLogic), and any number of [](#DetectorDataLogic), in + any order. Each object must fill exactly one of those roles. + :param publish_collect_methods: + Called from `on_prepare` with which collect verbs the data logics turn + out to need, so that a [](#StandardDetector) can expose the matching + bluesky protocol. A detector passes its own + `_publish_collect_methods`; [](#StandardDetector.with_logics) does that + for an ad-hoc one. """ - # Logic for the detector - _trigger_logic: DetectorTriggerLogic | None = None - _acquire_logic: DetectorAcquireLogic | None = None - _data_logics: Sequence[DetectorDataLogic] = () - # Context produced by prepare, used by trigger and kickoff - _prepare_ctx: _PrepareCtx | None = None - # Context produced by kickoff, used by complete - _kickoff_ctx: _KickoffCtx | None = None - # The triggers that are supported by the trigger logic - _supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - # Contribute to _StandardBase's fan out rather than replacing stage(), - # so the readable children registered on this detector still get staged - self._stage_funcs += (self._stage_detector,) - self._unstage_funcs += (self._unstage_detector,) - - # Report the number of events for the next kickoff - @cached_property - def events_to_kickoff(self) -> SignalRW[int]: - # TODO: only allow this to be revised down when trigger_info.number_of_events >1 - # and we have a reusable data provider - # requries https://github.com/bluesky/ophyd-async/issues/1119 - signal = soft_signal_rw(int) - # Name and parent this manually as `Device` doesn't know how to deal with cached - # properties - signal.parent = self - signal.set_name(f"{self.name}-events_to_kickoff") - return signal - - def add_detector_logics( - self, *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic + def __init__( + self, + *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, + publish_collect_methods: Callable[..., None], ) -> None: - """Add acquire, trigger or data logic to the detector. - - :param logic: The logic to add - """ + self.logics = logics + self.publish_collect_methods = publish_collect_methods + self.trigger_logic: DetectorTriggerLogic | None = None + self.acquire_logic: DetectorAcquireLogic | None = None + self.data_logics: tuple[DetectorDataLogic, ...] = () + #: The trigger types the trigger logic implements + self.supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} + #: Prefix for the datakeys the data logics produce, set from `Device.name` + self.datakey_prefix = "" + #: The data providers currently prepared, or None if there are none + self.data: _DetectorData | None = None for logic in logics: # Each object must fill exactly one role. A single object that is both, # say, an AcquireLogic and a DataLogic would otherwise register as only @@ -495,110 +461,199 @@ def add_detector_logics( "for each logic role" ) if isinstance(logic, DetectorTriggerLogic): - if self._trigger_logic is not None: + if self.trigger_logic is not None: raise RuntimeError("Detector already has trigger logic") - self._trigger_logic = logic - # Store the triggers that are supported - self._supported_triggers = _get_supported_triggers(logic) + self.trigger_logic = logic + self.supported_triggers = _get_supported_triggers(logic) elif isinstance(logic, DetectorAcquireLogic): - if self._acquire_logic is not None: + if self.acquire_logic is not None: raise RuntimeError("Detector already has acquire logic") - self._acquire_logic = logic + self.acquire_logic = logic elif isinstance(logic, DetectorDataLogic): - self._data_logics = (*self._data_logics, logic) + self.data_logics = (*self.data_logics, logic) else: raise TypeError(f"Unknown logic type: {type(logic)}") + #: Whether the trigger logic can calculate a deadtime + self.supports_deadtime = self.trigger_logic is not None and ( + _trigger_logic_supported(self.trigger_logic.get_deadtime) + ) - def _publish_collect_methods( - self, *, stream_assets: bool, event_pages: bool - ) -> None: - """Bind the collect verb that matches the providers we are using. + @property + def prepared_data(self) -> _DetectorData: + """The prepared data providers, raising if prepare() has not run.""" + return error_if_none( + self.data, f"{self.datakey_prefix}: prepare() must be called first" + ) - Whichever applies is bound as a real instance attribute, so the bluesky - bundler's isinstance checks (`WritesStreamAssets` vs - `EventPageCollectable`) see exactly one of them: they resolve with - `inspect.getattr_static` on Python 3.12+, which does not call - `__getattr__`, so a dynamic hook would be invisible. Both names are - reserved by `Device`, hence `object.__setattr__`. + def get_deadtime(self, config_values: SignalDict) -> float: + """Return the deadtime the trigger logic calculates from `config_values`.""" + trigger_logic = error_if_none(self.trigger_logic, "No trigger logic") + return trigger_logic.get_deadtime(config_values) - Recomputed on every prepare, so the verb that no longer applies is - removed rather than left behind from a previous scan. + def get_hinted_fields(self) -> Iterator[Sequence[str]]: + """Yield the hinted fields of each data logic that is producing data. + + Before anything is prepared every data logic is asked, since `hints` is + read outside a scan too; once prepared, only the ones actually serving + it, so a logic that sat the scan out does not hint at data nobody will + produce. """ - for name, method, wanted in ( - ("collect_asset_docs", self._collect_asset_docs, stream_assets), - ("collect_pages", self._collect_pages, event_pages), - ): - if wanted: - object.__setattr__(self, name, method) - elif name in self.__dict__: - object.__delattr__(self, name) + logics = ( + [dl for dl, _ in self.data.serving] if self.data else list(self.data_logics) + ) + for dl in logics: + if fields := dl.get_hinted_fields(self._datakey_name(dl)): + yield fields + + async def on_stage(self) -> None: + """Stop data production and make sure the detector is idle.""" + self.data = None + coros: list[Awaitable] = [dl.stop() for dl in self.data_logics] + if self.acquire_logic: + coros.append(self.acquire_logic.ensure_ready()) + await asyncio.gather(*coros) - # Back compat - delete before 1.0 - def add_config_signals(self, *signals: SignalR) -> None: - """Add a signal to read_configuration(). + async def on_unstage(self) -> None: + """Stop data production and the detector at the end of a scan.""" + self.data = None + coros: list[Awaitable] = [dl.stop() for dl in self.data_logics] + if self.acquire_logic: + coros.append(self.acquire_logic.ensure_stopped()) + await asyncio.gather(*coros) - :param sig: The signal to add + async def on_prepare(self, value: TriggerInfo) -> _FlyCtx: + """Set the trigger logic up and make the data providers for this scan. + + :param value: TriggerInfo describing how to trigger the detector """ - warnings.warn( - DeprecationWarning( - "Use `set_readable_format(signal, StandardReadableFormat" - ".CONFIG_SIGNAL)` instead of `add_config_signals(signal)`" + await self._prepare_trigger_logic(value) + # This must come after preparing the trigger logic, as the period is read + # back from it and may determine the datatype of the streams + value = await self._resolve_period(value) + await self._prepare_data(value) + # External triggering can start acquiring now + if self.acquire_logic and value.trigger is not DetectorTrigger.INTERNAL: + await self.acquire_logic.start_acquiring() + return _FlyCtx(trigger_info=value) + + async def on_kickoff(self, ctx: _FlyCtx) -> _FlyCtx: + """Start the fly scan, noting where the providers had got to.""" + collectable = self.prepared_data.collectable + if not collectable: + raise ValueError( + f"Detector {self.datakey_prefix} has no collectable data, " + "so cannot kickoff" + ) + ctx.kickoff_collections_written = await _get_collections_written(collectable) + # External triggering has already started; internal starts now + if self.acquire_logic and ctx.trigger_info.trigger is DetectorTrigger.INTERNAL: + await self.acquire_logic.start_acquiring() + return ctx + + def on_complete_updates(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: + """Wait for the scan to finish, reporting collections written as progress.""" + return self._wait_for_collections( + trigger_info=ctx.trigger_info, + initial_collections_written=error_if_none( + ctx.kickoff_collections_written, "Kickoff not run" ), - stacklevel=2, + collections_requested=ctx.trigger_info.number_of_collections, + watcher_divisor=ctx.trigger_info.collections_per_event, ) - for signal in signals: - self.set_readable_format(signal, StandardReadableFormat.CONFIG_SIGNAL) - async def get_trigger_deadtime( - self, settings: Settings | None = None - ) -> tuple[set[DetectorTrigger], float | None]: - """Get supported trigger types and deadtime for the detector. + async def on_trigger(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: + """Take one event's worth of exposures and wait for them to be written.""" + if ctx.trigger_info.number_of_events != 1: + raise ValueError( + "trigger() is not supported for multiple events, the detector was " + f"prepared with number_of_events={ctx.trigger_info.number_of_events}." + ) + # A finite buffer holds one event at a time, so it is re-armed for each + # point of a step scan; a streaming provider carries on from where it was + data = await self._prepare_data(ctx.trigger_info) + # Take the baseline before acquisition starts, or frames written between + # the two would be counted towards this event. A re-armed buffer starts + # from zero, where a streaming provider continues from wherever it has got + # to -- which is not where prepare left it, since a step scan triggers many + # times against one prepare. A detector never mixes the two. + initial = ( + 0 if data.pageable else await _get_collections_written(data.collectable) + ) + if self.acquire_logic: + await self.acquire_logic.start_acquiring() + async for update in self._wait_for_collections( + trigger_info=ctx.trigger_info, + initial_collections_written=initial, + collections_requested=ctx.trigger_info.collections_per_event, + ): + yield update - The trigger logic is given the value of every signal this detector - reports as configuration, rather than nominating the ones it wants: - the values that determine deadtime are exactly the ones that ought to - be recorded for the scan anyway, so a second list to keep in step - earned nothing. + async def default_trigger_info(self) -> TriggerInfo: + """The TriggerInfo to prepare with when a plan calls trigger() directly.""" + # Opt-in: set OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES to have trigger() read + # back current hardware state (e.g. num_images) via default_trigger_info() + # instead of always falling back to TriggerInfo(). See ADR 0013. + # TODO: flip default to YES and remove this guard in a future PR once + # downstream code has had time to implement default_trigger_info(). + preserve_state = ( + os.environ.get("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "NO").upper() == "YES" + ) + if preserve_state and self.trigger_logic is not None: + if not _trigger_logic_supported(self.trigger_logic.default_trigger_info): + raise RuntimeError( + "OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but " + f"'{self.datakey_prefix}' has no default_trigger_info() - " + "implement default_trigger_info() on your DetectorTriggerLogic " + "subclass or unset the environment variable." + ) + return await self.trigger_logic.default_trigger_info() + return TriggerInfo() - :param settings: Optional settings to use when getting configuration values - :return: Tuple of supported trigger types and deadtime in seconds - """ - if self._trigger_logic and _trigger_logic_supported( - self._trigger_logic.get_deadtime + def _datakey_name(self, dl: DetectorDataLogic) -> str: + return self.datakey_prefix + dl.datakey_suffix + + async def _prepare_trigger_logic(self, value: TriggerInfo) -> None: + if self.trigger_logic and _trigger_logic_supported( + self.trigger_logic.prepare_exposures_per_collection ): - config_values = SignalDict() - to_read: list[SignalR] = [] - for sig in _config_signals(self): - if settings and sig in settings: - # Use value from settings if it is in there - # cast to a SignalRW because settings can only contain those - config_values[sig] = settings[cast(SignalRW, sig)] - else: - to_read.append(sig) - # Read live values concurrently: this is now every configuration - # signal rather than a handful the logic named, so doing it in - # series would be one round trip per signal - for sig, value in zip( - to_read, - await asyncio.gather(*(sig.get_value() for sig in to_read)), - strict=True, - ): - config_values[sig] = value - deadtime = self._trigger_logic.get_deadtime(config_values) - else: - deadtime = None - return self._supported_triggers, deadtime - - @AsyncStatus.wrap - async def _stage_detector(self) -> None: - """Make sure the detector is idle and ready to be used.""" - coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] - if self._acquire_logic: - coros.append(self._acquire_logic.ensure_ready()) - await asyncio.gather(*coros) - self._prepare_ctx = None - self._kickoff_ctx = None - await self.events_to_kickoff.set(0) + # If we can do multiple exposures per collection then set it up + # even if there was only 1 requested to clear previous settings + await self.trigger_logic.prepare_exposures_per_collection( + value.exposures_per_collection + ) + elif value.exposures_per_collection != 1: + raise ValueError( + "Multiple exposures per collection not supported by " + f"'{self.datakey_prefix}'" + ) + if value.trigger not in self.supported_triggers: + format_triggers = ", ".join(sorted(t.name for t in self.supported_triggers)) + raise ValueError( + f"Trigger type {value.trigger} not supported by " + f"'{self.datakey_prefix}', supported types are: [{format_triggers}]" + ) + if self.trigger_logic: + match value.trigger: + case DetectorTrigger.INTERNAL: + await self.trigger_logic.prepare_internal( + num=value.number_of_exposures, + livetime=value.livetime, + deadtime=value.deadtime, + ) + case DetectorTrigger.EXTERNAL_EDGE: + await self.trigger_logic.prepare_edge( + num=value.number_of_exposures, + livetime=value.livetime, + ) + case DetectorTrigger.EXTERNAL_LEVEL: + await self.trigger_logic.prepare_level( + num=value.number_of_exposures, + ) + elif value.livetime != 0.0 or value.deadtime != 0.0: + raise ValueError( + f"Detector {self.datakey_prefix} has no trigger logic, so cannot set " + "livetime or deadtime" + ) async def _resolve_period(self, value: TriggerInfo) -> TriggerInfo: # A livetime of 0 means "use whatever the detector currently has set". A @@ -608,10 +663,10 @@ async def _resolve_period(self, value: TriggerInfo) -> TriggerInfo: # prepared, so the hardware already holds the values we read. if ( value.livetime == 0.0 - and self._trigger_logic is not None - and _trigger_logic_supported(self._trigger_logic.default_trigger_info) + and self.trigger_logic is not None + and _trigger_logic_supported(self.trigger_logic.default_trigger_info) ): - current = await self._trigger_logic.default_trigger_info() + current = await self.trigger_logic.default_trigger_info() if current.livetime or current.deadtime: value = value.model_copy( update={ @@ -621,94 +676,115 @@ async def _resolve_period(self, value: TriggerInfo) -> TriggerInfo: ) return value - async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: - num_collections = trigger_info.number_of_collections + async def _prepare_data(self, trigger_info: TriggerInfo) -> _DetectorData: + """Make sure the data providers are ready for this scan.""" period = trigger_info.livetime + trigger_info.deadtime - previous = self._prepare_ctx # Providers are reused while the scan they were made for is unchanged, so # that a step scan does not reopen its file on every point. Both parts of - # the reuse key are baked into a streaming provider when it is made: the - # shape from collections_per_event and the chunking from the period. A - # finite buffer is re-made either way, since re-making is what re-arms it. - reusable = previous is not None and ( - previous.trigger_info.collections_per_event - == trigger_info.collections_per_event - and previous.trigger_info.livetime + previous.trigger_info.deadtime - == period - ) - if reusable and previous is not None: - serving = await self._rearm_bounded(previous, num_collections, period) + # the key are baked into a streaming provider when it is made: the shape + # from collections_per_event and the chunking from the period. A finite + # buffer is re-made either way, since re-making is what re-arms it. + previous = self.data + if ( + previous is not None + and previous.collections_per_event == trigger_info.collections_per_event + and previous.period == period + ): + serving = await self._rearm_bounded(previous, trigger_info) else: - serving = await self._make_serving_providers(num_collections, period) + serving = await self._make_serving(trigger_info) + self.data = _DetectorData( + serving=serving, + collections_per_event=trigger_info.collections_per_event, + period=period, + ) # Which collect verb the detector exposes follows what it will actually # produce, so it is recomputed here rather than fixed at construction - self._publish_collect_methods( - stream_assets=any( - isinstance(dp, StreamableDataProvider) for _, dp in serving - ), - event_pages=any(isinstance(dp, PageableDataProvider) for _, dp in serving), - ) - self._prepare_ctx = _PrepareCtx( - trigger_info=trigger_info, - serving=serving, - collections_written=await _get_collections_written( - [dp for _, dp in serving] - ), + self.publish_collect_methods( + stream_assets=bool(self.data.streamable), + event_pages=bool(self.data.pageable), ) + return self.data - async def _make_serving_providers( - self, num_collections: int, period: float + async def _make_serving( + self, trigger_info: TriggerInfo ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: """Ask every data logic what it would make, and start the ones we use.""" # Stop what is running before anything new is made, since a logic that # cannot describe its data without opening its file does so here - if self._prepare_ctx is not None: - await asyncio.gather(*(dl.stop() for dl, _ in self._prepare_ctx.serving)) + if self.data is not None: + await asyncio.gather(*(dl.stop() for dl, _ in self.data.serving)) + cpe = trigger_info.collections_per_event + period = trigger_info.livetime + trigger_info.deadtime made = await asyncio.gather( *( dl.make_data_provider( - self.name + dl.datakey_suffix, num_collections, period + self._datakey_name(dl), trigger_info.number_of_collections, period ) - for dl in self._data_logics + for dl in self.data_logics ) ) # A logic returns None to sit this scan out, e.g. a finite buffer asked - # for an unbounded number of collections + # for an unbounded number of collections, or a plugin that is switched off serving = [ (dl, dp) - for dl, dp in zip(self._data_logics, made, strict=True) + for dl, dp in zip(self.data_logics, made, strict=True) if dp is not None ] - streamable = [dp for _, dp in serving if isinstance(dp, StreamableDataProvider)] - pageable = [dp for _, dp in serving if isinstance(dp, PageableDataProvider)] - # A detector may not produce both kinds at once: it would expose both - # collect_asset_docs and collect_pages and produce data from both in a fly - # scan, which the bluesky bundler treats as mutually exclusive. A file - # writer that also wants stats should carry them in the file (as - # NDAttributes) rather than as a separate bounded logic. - if streamable and pageable: - - def _describe(providers: Sequence[_DataProvider]) -> str: - return ", ".join(type(dp).__name__ for dp in providers) - - raise TypeError( - f"Detector {self.name} would produce both event pages " - f"({_describe(pageable)}) and stream assets ({_describe(streamable)}); " - "these cannot be combined on one detector" - ) + serving = await self._drop_shadowed(serving, cpe) await asyncio.gather(*(dl.start() for dl, _ in serving)) return serving + async def _drop_shadowed( + self, + serving: Sequence[tuple[DetectorDataLogic, _DataProvider]], + collections_per_event: int, + ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: + """Drop finite buffers whose datakeys the stream assets already carry. + + A detector cannot produce both stream assets and event pages: the bundler + treats them as mutually exclusive. Carrying both logics is still useful, + because the same quantity can be written durably into the file *and* read + from a plugin's buffer -- an areaDetector stats total, say, which the HDF + writer pulls in as an NDAttribute. Where every key a finite buffer would + produce is also written to the file, the file wins and the buffer sits the + scan out. Anything else is a conflict, so it raises. + """ + pageable = [ + (dl, dp) for dl, dp in serving if isinstance(dp, PageableDataProvider) + ] + streamable = [dp for _, dp in serving if isinstance(dp, StreamableDataProvider)] + if not (pageable and streamable): + return serving + stream_keys: set[str] = set() + for dp in streamable: + stream_keys |= set(await dp.make_datakeys(collections_per_event)) + shadowed = [] + for dl, dp in pageable: + unshadowed = ( + set(await dp.make_datakeys(collections_per_event)) - stream_keys + ) + if unshadowed: + raise TypeError( + f"Detector {self.datakey_prefix} would produce " + f"{sorted(unshadowed)} as event pages and the rest of its data " + "as stream assets; these cannot be combined on one detector" + ) + shadowed.append(dl) + # Nothing has been started yet, so a shadowed logic needs no stopping + return [(dl, dp) for dl, dp in serving if dl not in shadowed] + async def _rearm_bounded( - self, previous: _PrepareCtx, num_collections: int, period: float + self, previous: _DetectorData, trigger_info: TriggerInfo ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: """Re-make and re-start the finite buffers, keeping everything else.""" + period = trigger_info.livetime + trigger_info.deadtime serving: list[tuple[DetectorDataLogic, _DataProvider]] = [] for dl, dp in previous.serving: if isinstance(dp, PageableDataProvider): await dl.stop() rearmed = await dl.make_data_provider( - self.name + dl.datakey_suffix, num_collections, period + self._datakey_name(dl), trigger_info.number_of_collections, period ) if rearmed is None: continue @@ -718,15 +794,14 @@ async def _rearm_bounded( serving.append((dl, dp)) return serving - async def _wait_for_index( + async def _wait_for_collections( self, - data_providers: Sequence[StreamableDataProvider | PageableDataProvider], trigger_info: TriggerInfo, initial_collections_written: int, collections_requested: int, - wait_for_idle: bool, watcher_divisor: int = 1, ) -> AsyncIterator[WatcherUpdate]: + data_providers = self.prepared_data.collectable start_time = time.monotonic() current_collections_written = { dp.collections_written_signal: initial_collections_written @@ -741,7 +816,7 @@ async def _wait_for_index( current_collections_written[sig] = value collections_written = min(current_collections_written.values()) yield WatcherUpdate( - name=self.name, + name=self.datakey_prefix, current=collections_written // watcher_divisor, initial=initial_collections_written // watcher_divisor, target=target_collections_written // watcher_divisor, @@ -751,192 +826,173 @@ async def _wait_for_index( ) if collections_written >= target_collections_written: break - if self._acquire_logic and wait_for_idle: - await self._acquire_logic.wait_for_idle() + if self.acquire_logic: + await self.acquire_logic.wait_for_idle() - @AsyncStatus.wrap - async def prepare(self, value: TriggerInfo) -> None: - """Prepare the detector for a number of triggers. - :param value: TriggerInfo describing how to trigger the detector - """ - if self._trigger_logic and _trigger_logic_supported( - self._trigger_logic.prepare_exposures_per_collection - ): - # If we can do multiple exposures per collection then set it up - # even if there was only 1 requested to clear previous settings - await self._trigger_logic.prepare_exposures_per_collection( - value.exposures_per_collection - ) - elif value.exposures_per_collection != 1: - raise ValueError( - f"Multiple exposures per collection not supported by {self}" - ) - # Setup the trigger logic for the right number of exposures - if value.trigger not in self._supported_triggers: - format_triggers = ", ".join( - sorted(t.name for t in self._supported_triggers) - ) - raise ValueError( - f"Trigger type {value.trigger} not supported by '{self.name}', " - f"supported types are: [{format_triggers}]" - ) - if self._trigger_logic: - match value.trigger: - case DetectorTrigger.INTERNAL: - await self._trigger_logic.prepare_internal( - num=value.number_of_exposures, - livetime=value.livetime, - deadtime=value.deadtime, - ) - case DetectorTrigger.EXTERNAL_EDGE: - await self._trigger_logic.prepare_edge( - num=value.number_of_exposures, - livetime=value.livetime, - ) - case DetectorTrigger.EXTERNAL_LEVEL: - await self._trigger_logic.prepare_level( - num=value.number_of_exposures, - ) - elif value.livetime != 0.0 or value.deadtime != 0.0: - raise ValueError( - f"Detector {self.name} has no trigger logic, so cannot set livetime or " - "deadtime" +class StandardDetector( + StandardReadable, + StandardFlyable[TriggerInfo, _FlyCtx], + Stageable, + Triggerable, + Collectable, + HasHints, +): + """Detector base class for step and fly scanning detectors. + + Aggregates trigger, acquire and data logic together in a + [](#DetectorLogic), which a subclass builds in its `__init__` and returns + from `logic`: + + ```python + class MyDetector(StandardDetector): + def __init__(self, prefix: str, name: str = "") -> None: + self.driver = MyDriverIO(prefix) + self._logic = DetectorLogic( + MyTriggerLogic(self.driver), + MyAcquireLogic(self.driver), + publish_collect_methods=self._publish_collect_methods, ) - # NOTE: this section must come after preparing the trigger logic as we may - # use parameters from it to determine datatype for the streams - value = await self._resolve_period(value) - await self._update_prepare_context(value) - # Tell people how many collections we will acquire for - await self.events_to_kickoff.set(value.number_of_events) - # External triggering can start acquiring now - if self._acquire_logic and value.trigger != DetectorTrigger.INTERNAL: - await self._acquire_logic.start_acquiring() + super().__init__(name=name) - @WatchableAsyncStatus.wrap - async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: - """Trigger a single exposure. + @cached_property + def logic(self) -> DetectorLogic: + return self._logic + ``` - If [`prepare()`](#StandardDetector.prepare) has not been called since - the last `stage()`, an implicit prepare is - performed. When [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` - [](#DetectorTriggerLogic.default_trigger_info) is called to read current - hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is - used. + For an ad-hoc detector, [](#StandardDetector.with_logics) does the same + thing without a subclass. + + Signals read during a step scan are registered with + [](#StandardReadable.set_readable_format), exactly as on any other + `StandardReadable`; data produced by a `DetectorDataLogic` is added on top + of those in `read()` and `describe()`. + + `read()` and `describe()` require `prepare()` to have run, and raise + otherwise, so that a descriptor can never be emitted without the detector's + data keys. `trigger()` prepares implicitly, so a step scan never has to do + it explicitly. `read_configuration()` and `describe_configuration()` have no + such requirement. + + `WritesStreamAssets` (and its `collect_asset_docs`) and + `EventPageCollectable` (and its `collect_pages`) are *not* inherited: which + of them applies depends on what the data logics produce for a given scan, so + whichever it is gets bound as an instance attribute by `prepare()`. + """ + + @abstract_cached_property + def logic(self) -> DetectorLogic: + """The logic that drives this detector, built in the subclass `__init__`.""" + raise NotImplementedError + + @classmethod + def with_logics( + cls, + *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, + name: str = "", + ) -> "StandardDetector": + """Make a detector from some logics, without writing a subclass. + + The logic object a `StandardDetector` needs is built with a callback into + the Device, so it cannot be constructed before the Device it belongs to. + This does both, for an ad-hoc detector in a plan or a test. """ - if self._prepare_ctx is None: - # Opt-in: set OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES to have - # trigger() read back current hardware state (e.g. num_images) via - # default_trigger_info() instead of always falling back to TriggerInfo(). - # See ADR 0013 for rationale. - # TODO: flip default to YES and remove this guard in a future PR once - # downstream code has had time to implement default_trigger_info(). - preserve_state = ( - os.environ.get("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "NO").upper() - == "YES" - ) - if preserve_state and self._trigger_logic is not None: - if not _trigger_logic_supported( - self._trigger_logic.default_trigger_info - ): - raise RuntimeError( - f"OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but " - f"'{self.name}' has no default_trigger_info() - implement " - "default_trigger_info() on your DetectorTriggerLogic subclass " - "or unset the environment variable." - ) - trigger_info = await self._trigger_logic.default_trigger_info() - else: - trigger_info = TriggerInfo() - await self.prepare(trigger_info) - else: - # Check the one that was provided is suitable for triggering - trigger_info = self._prepare_ctx.trigger_info - if trigger_info.number_of_events != 1: - msg = ( - "trigger() is not supported for multiple events, the detector was " - f"prepared with number_of_events={trigger_info.number_of_events}." - ) - raise ValueError(msg) - # Ensure the data provider is still usable - await self._update_prepare_context(trigger_info) - ctx = error_if_none(self._prepare_ctx, "Prepare should have been run") - # Start the detector acquiring and wait for it to finish. - if self._acquire_logic: - await self._acquire_logic.start_acquiring() - # A bounded provider has just been re-armed by the re-prepare above, so its - # buffer starts from zero; a streamable provider continues from wherever the - # prepared context left it. A detector never mixes the two. - collectable = [ - *ctx.streamable_data_providers, - *ctx.pageable_data_providers, - ] - initial = 0 if ctx.pageable_data_providers else ctx.collections_written - async for update in self._wait_for_index( - data_providers=collectable, - trigger_info=ctx.trigger_info, - initial_collections_written=initial, - collections_requested=ctx.trigger_info.collections_per_event, - watcher_divisor=1, - wait_for_idle=True, + return _LogicsDetector(*logics, name=name) + + def set_name(self, name: str, *, child_name_separator: str | None = None) -> None: + super().set_name(name, child_name_separator=child_name_separator) + # The data logics name their datakeys after the Device + self.logic.datakey_prefix = name + + def _publish_collect_methods( + self, *, stream_assets: bool, event_pages: bool + ) -> None: + """Bind the collect verb that matches what the data logics will produce. + + Whichever applies is bound as a real instance attribute, so the bluesky + bundler's isinstance checks (`WritesStreamAssets` vs + `EventPageCollectable`) see exactly one of them. They resolve with + `inspect.getattr_static` on Python 3.12+, which does not call + `__getattr__`, so a dynamic hook would be invisible. Both names are + reserved by `Device`, hence `object.__setattr__`. + + `DetectorLogic` calls this from every prepare, so the verb that no longer + applies is removed rather than left behind from a previous scan. + """ + for verb, method, wanted in ( + ("collect_asset_docs", self._collect_asset_docs, stream_assets), + ("collect_pages", self._collect_pages, event_pages), ): - yield update + if wanted: + object.__setattr__(self, verb, method) + elif verb in self.__dict__: + object.__delattr__(self, verb) - @AsyncStatus.wrap - async def kickoff(self): - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - # A fly scan collects from streamable providers (as stream datums) or from - # bounded providers (as event pages); either kind can be kicked off, but a - # detector with neither has nothing to collect. - collectable = [ - *ctx.streamable_data_providers, - *ctx.pageable_data_providers, - ] - if not collectable: - raise ValueError( - f"Detector {self.name} has no collectable data, so cannot kickoff" - ) - # Unlike trigger(), kickoff() does not re-arm a bounded buffer, so its - # progress is read live: a fly scan arms once and accumulates across kickoffs. - collections_written, events_to_kickoff = await asyncio.gather( - _get_collections_written(collectable), - self.events_to_kickoff.get_value(), - ) - collections_requested = ( - events_to_kickoff * ctx.trigger_info.collections_per_event - ) - last_requested_collection = collections_written + collections_requested - last_expected_collection = ( - ctx.collections_written + ctx.trigger_info.number_of_collections - ) - if last_requested_collection > last_expected_collection: - msg = ( - f"Kickoff requested {collections_written}:{last_requested_collection}, " - f"but detector was only prepared up to {last_expected_collection}" - ) - raise RuntimeError(msg) - self._kickoff_ctx = _KickoffCtx( - trigger_info=ctx.trigger_info, - data_providers=collectable, - collections_written=collections_written, - collections_requested=collections_requested, - is_last_kickoff=last_requested_collection == last_expected_collection, + # Back compat - delete before 1.0 + def add_config_signals(self, *signals: SignalR) -> None: + """Add a signal to read_configuration(). + + :param sig: The signal to add + """ + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(signal, StandardReadableFormat" + ".CONFIG_SIGNAL)` instead of `add_config_signals(signal)`" + ), + stacklevel=2, ) - # External triggering has already started; internal starts now - if self._acquire_logic and ctx.trigger_info.trigger == DetectorTrigger.INTERNAL: - await self._acquire_logic.start_acquiring() + for signal in signals: + self.set_readable_format(signal, StandardReadableFormat.CONFIG_SIGNAL) + + async def get_trigger_deadtime( + self, settings: Settings | None = None + ) -> tuple[set[DetectorTrigger], float | None]: + """Get supported trigger types and deadtime for the detector. + + The trigger logic is given the value of every signal this detector + reports as configuration, rather than nominating the ones it wants: + the values that determine deadtime are exactly the ones that ought to + be recorded for the scan anyway. + + :param settings: Optional settings to use when getting configuration values + :return: Tuple of supported trigger types and deadtime in seconds + """ + deadtime = None + if self.logic.supports_deadtime: + config_values = SignalDict() + to_read: list[SignalR] = [] + for sig in _config_signals(self): + if settings and sig in settings: + # Use value from settings if it is in there + # cast to a SignalRW because settings can only contain those + config_values[sig] = settings[cast(SignalRW, sig)] + else: + to_read.append(sig) + # Read live values concurrently: this is every configuration signal + # rather than a handful the logic named, so doing it in series would + # be one round trip per signal + for sig, value in zip( + to_read, + await asyncio.gather(*(sig.get_value() for sig in to_read)), + strict=True, + ): + config_values[sig] = value + deadtime = self.logic.get_deadtime(config_values) + return self.logic.supported_triggers, deadtime @WatchableAsyncStatus.wrap - async def complete(self): - ctx = error_if_none(self._kickoff_ctx, "Kickoff not called") - async for update in self._wait_for_index( - data_providers=ctx.data_providers, - trigger_info=ctx.trigger_info, - initial_collections_written=ctx.collections_written, - collections_requested=ctx.collections_requested, - wait_for_idle=ctx.is_last_kickoff, - watcher_divisor=ctx.trigger_info.collections_per_event, - ): + async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: + """Trigger a single exposure. + + If [`prepare()`](#StandardFlyable.prepare) has not been called since the + last `stage()`, an implicit prepare is performed. When + [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` + [](#DetectorTriggerLogic.default_trigger_info) is called to read current + hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is used. + """ + if self.logic.data is None: + await self.prepare(await self.logic.default_trigger_info()) + async for update in self.logic.on_trigger(self._prepared_fly_ctx): yield update def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: @@ -952,12 +1008,12 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] """ if verb not in (_Verb.DESCRIBE, _Verb.READ): return - ctx = error_if_none(self._prepare_ctx, "Prepare not run") - cpe = ctx.trigger_info.collections_per_event + data = self.logic.prepared_data + cpe = data.collections_per_event # Bounded providers hold a single-event page for this step-scan point, # which _pageable_readings extracts back to a reading. That extraction # lives here rather than on the provider so a provider cannot override it. - for pdp in ctx.pageable_data_providers: + for pdp in data.pageable: if verb is _Verb.DESCRIBE: yield functools.partial(pdp.make_datakeys, cpe) else: @@ -965,23 +1021,21 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] if verb is _Verb.DESCRIBE: # Streamable providers describe their shape for a step scan, but # produce their data through collect_asset_docs rather than read() - for sdp in ctx.streamable_data_providers: + for sdp in data.streamable: yield functools.partial(sdp.make_datakeys, cpe) async def describe_collect(self) -> dict[str, DataKey]: - ctx = error_if_none(self._prepare_ctx, "Prepare not run") + data = self.logic.prepared_data # Streamable providers collect stream datums, bounded providers collect pages - cpe = ctx.trigger_info.collections_per_event - coros = [dp.make_datakeys(cpe) for dp in ctx.streamable_data_providers] + [ - dp.make_datakeys(cpe) for dp in ctx.pageable_data_providers + coros = [ + dp.make_datakeys(data.collections_per_event) for dp in data.collectable ] return await merge_gathered_dicts(coros) def _extra_hint_sources(self) -> Iterator[HasHints]: """Contribute the data logics' hinted fields alongside the children's.""" - for dl in self._data_logics: - if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): - yield _HintedFields(fields) + for fields in self.logic.get_hinted_fields(): + yield _HintedFields(fields) async def _pageable_readings( self, provider: PageableDataProvider, collections_per_event: int @@ -1013,29 +1067,25 @@ async def _pageable_readings( async def _collect_asset_docs( self, index: int | None = None ) -> AsyncIterator[StreamAsset]: - # Collect stream datum documents for all indices written. Exposed as - # collect_asset_docs via __getattr__ only when an unbounded data logic is - # present, so the bluesky bundler dispatches it in place of collect_pages. - ctx = error_if_none(self._prepare_ctx, "Prepare not called") + # Bound as collect_asset_docs when there is a streaming provider + data = self.logic.prepared_data if index is None: # The index is optional, and provided for fly scans, if there is # more than one detector to make sure they collect in step index = await self.get_index() - for data_provider in ctx.streamable_data_providers: + for data_provider in data.streamable: async for doc in data_provider.make_stream_docs( - collections_written=index * ctx.trigger_info.collections_per_event, - collections_per_event=ctx.trigger_info.collections_per_event, + collections_written=index * data.collections_per_event, + collections_per_event=data.collections_per_event, ): yield doc async def _collect_pages(self) -> AsyncIterator[PartialEventPage]: - # Collect event pages for all indices written. Exposed as collect_pages via - # __getattr__ only when a bounded data logic is present, so the bluesky - # bundler dispatches it in place of collect_asset_docs. - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - cpe = ctx.trigger_info.collections_per_event + # Bound as collect_pages when there is a paging provider + data = self.logic.prepared_data + cpe = data.collections_per_event index = await self.get_index() - for data_provider in ctx.pageable_data_providers: + for data_provider in data.pageable: async for page in data_provider.make_pages( collections_written=index * cpe, collections_per_event=cpe, @@ -1043,20 +1093,26 @@ async def _collect_pages(self) -> AsyncIterator[PartialEventPage]: yield page async def get_index(self) -> int: - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - collectable = [ - *ctx.streamable_data_providers, - *ctx.pageable_data_providers, - ] + data = self.logic.prepared_data min_collections_written = await _get_collections_written( - collectable, reducer=min + data.collectable, reducer=min ) - return min_collections_written // ctx.trigger_info.collections_per_event - - @AsyncStatus.wrap - async def _unstage_detector(self) -> None: - """Stop the detector and file writing.""" - coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] - if self._acquire_logic: - coros.append(self._acquire_logic.ensure_stopped()) - await asyncio.gather(*coros) + return min_collections_written // data.collections_per_event + + +class _LogicsDetector(StandardDetector): + """A concrete `StandardDetector` built from logics, see `with_logics`.""" + + def __init__( + self, + *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, + name: str = "", + ) -> None: + self._logic = DetectorLogic( + *logics, publish_collect_methods=self._publish_collect_methods + ) + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index 548f9a8bd1..963ff659f0 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -88,6 +88,30 @@ def with_device(self, name: str = "") -> "StandardFlyable[PrepareT, CtxT]": return _EphemeralFlyable(self, name=name) +class WatchableFlyableLogic(FlyableLogic[PrepareT, CtxT]): + """A `FlyableLogic` that reports its own progress while completing. + + `StandardFlyable.complete()` reports progress for a `MovableLogic` by + observing its readback against its setpoint. A logic whose progress is + neither -- a detector's is "collections written out of collections + requested" -- yields its own updates instead. + + Implement `on_complete_updates` rather than `on_complete`; the inherited + `on_complete` drains the updates for callers that only want to block. + """ + + @abstractmethod + def on_complete_updates(self, ctx: CtxT) -> AsyncIterator[WatcherUpdate]: + """Block until the fly scan is done, yielding progress as it goes. + + :param ctx: the context returned by `on_kickoff`. + """ + + async def on_complete(self, ctx: CtxT) -> None: + async for _ in self.on_complete_updates(ctx): + pass + + class FlyMotorInfo(ConfinedModel): """Minimal set of information required to fly a motor.""" @@ -179,6 +203,19 @@ def logic(self) -> FlyableLogic[PrepareT, CtxT]: """ raise NotImplementedError + @property + def _prepared_fly_ctx(self) -> CtxT: + """The prepare context, for callers outside prepare -> kickoff -> complete. + + `_fly_ctx` is a `None` *typed as* `CtxT` until prepared, so a verb that + read it directly -- a detector's `describe_collect()` or `get_index()` + -- would fail with an `AttributeError` on `None` rather than saying what + was wrong. + """ + if self._fly_stage is _FlyStage.IDLE: + raise RuntimeError(f"{self.name}: prepare() must be called first") + return self._fly_ctx + def _reset_fly_state(self) -> None: self._fly_ctx = cast(CtxT, None) self._fly_stage = _FlyStage.IDLE @@ -215,8 +252,9 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: If the logic is also a `MovableLogic` (e.g. a flying `Motor`), report progress to watchers by observing its readback while the fly scan runs, - reusing the same watcher-update stream as `StandardMovable.set`. Flyers - whose logic is not movable simply block with no progress updates. + reusing the same watcher-update stream as `StandardMovable.set`. A + `WatchableFlyableLogic` reports its own progress instead. Any other + flyer simply blocks with no progress updates. """ if self._fly_stage is not _FlyStage.KICKED_OFF: raise RuntimeError( @@ -241,6 +279,11 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: unit=units, precision=precision, ) + elif isinstance(logic, WatchableFlyableLogic): + # Progress that is neither a readback nor a setpoint: the logic + # knows how to report it, so let it + async for update in logic.on_complete_updates(self._fly_ctx): + yield update else: await self.logic.on_complete(self._fly_ctx) self._fly_stage = _FlyStage.IDLE diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index c445904147..0ec86284bd 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -1,8 +1,11 @@ from collections.abc import Mapping, Sequence +from functools import cached_property from typing import Generic from ophyd_async.core import ( DetectorAcquireLogic, + DetectorDataLogic, + DetectorLogic, DetectorTriggerLogic, SignalR, StandardDetector, @@ -31,10 +34,13 @@ def __init__( if plugins is not None: for plugin_name, plugin in plugins.items(): setattr(self, plugin_name, plugin) + logics: list[ + DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic + ] = [] if trigger_logic: - self.add_detector_logics(trigger_logic) + logics.append(trigger_logic) if acquire_logic: - self.add_detector_logics(acquire_logic) + logics.append(acquire_logic) if writer_factories: if prefix is None: raise ValueError("prefix is required when writer_factories are given") @@ -48,7 +54,7 @@ def __init__( for factory in writer_factories: writer, data_logic = factory(prefix, driver, plugin_list) setattr(self, factory.writer_name, writer) - self.add_detector_logics(data_logic) + logics.append(data_logic) # The driver is always wired up, so register it whole and let ADBaseIO # declare which of its signals are configuration. Plugins are # deliberately *not* registered: a detector normally carries far more @@ -57,8 +63,15 @@ def __init__( self.set_readable_format(self.driver, Format.CHILD) for signal in config_sigs: self.set_readable_format(signal, Format.CONFIG_SIGNAL) + self._logic = DetectorLogic( + *logics, publish_collect_methods=self._publish_collect_methods + ) super().__init__(name=name) + @cached_property + def logic(self) -> DetectorLogic: + return self._logic + def get_plugin( self, name: str, plugin_type: type[NDPluginBaseIOT] = NDPluginBaseIO ) -> NDPluginBaseIOT: diff --git a/src/ophyd_async/fastcs/eiger/_detector.py b/src/ophyd_async/fastcs/eiger/_detector.py index 7257bde6a7..90cb7c4c36 100644 --- a/src/ophyd_async/fastcs/eiger/_detector.py +++ b/src/ophyd_async/fastcs/eiger/_detector.py @@ -1,4 +1,7 @@ +from functools import cached_property + from ophyd_async.core import ( + DetectorLogic, PathProvider, SignalR, StandardDetector, @@ -33,7 +36,7 @@ def __init__( # Need to do this first so the type hints are filled in connector = fastcs_connector(prefix, self) - self.add_detector_logics( + self._logic = DetectorLogic( EigerTriggerLogic(self.detector), EigerAcquireLogic(self.detector, self.arm_when_ready), odin.OdinDataLogic( @@ -42,8 +45,13 @@ def __init__( detector_bit_depth=self.detector.bit_depth_image, pixel_mask=self.pixel_mask, ), + publish_collect_methods=self._publish_collect_methods, ) self.add_config_signals(self.pixel_mask) super().__init__(name=name, connector=connector) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/fastcs/jungfrau/_detector.py b/src/ophyd_async/fastcs/jungfrau/_detector.py index 0adb10db3c..906569c9e7 100644 --- a/src/ophyd_async/fastcs/jungfrau/_detector.py +++ b/src/ophyd_async/fastcs/jungfrau/_detector.py @@ -1,4 +1,11 @@ -from ophyd_async.core import PathProvider, StandardDetector, soft_signal_rw +from functools import cached_property + +from ophyd_async.core import ( + DetectorLogic, + PathProvider, + StandardDetector, + soft_signal_rw, +) from ophyd_async.fastcs import odin from ophyd_async.fastcs.core import fastcs_connector @@ -29,7 +36,7 @@ def __init__( self.acquisition_type = soft_signal_rw( AcquisitionType, AcquisitionType.STANDARD ) - self.add_detector_logics( + self._logic = DetectorLogic( JungfrauTriggerLogic(self.detector, self.acquisition_type), JungfrauAcquireLogic(self.detector), odin.OdinDataLogic( @@ -37,5 +44,10 @@ def __init__( odin=self.odin, detector_bit_depth=self.detector.bit_depth, ), + publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/fastcs/panda/_detector.py b/src/ophyd_async/fastcs/panda/_detector.py index c365d56fa8..900d370a31 100644 --- a/src/ophyd_async/fastcs/panda/_detector.py +++ b/src/ophyd_async/fastcs/panda/_detector.py @@ -1,6 +1,8 @@ from __future__ import annotations -from ophyd_async.core import PathProvider, StandardDetector +from functools import cached_property + +from ophyd_async.core import DetectorLogic, PathProvider, StandardDetector from ophyd_async.fastcs.core import fastcs_connector from ._acquire_logic import PandaAcquireLogic @@ -23,9 +25,14 @@ def __init__( error_hint = f"Is PandABlocks-ioc at least version {MINIMUM_PANDA_IOC}?" # This has to be first so we make self.pcap connector = fastcs_connector(prefix, self, error_hint) - self.add_detector_logics( + self._logic = DetectorLogic( PandaTriggerLogic(self.pcap), PandaAcquireLogic(self.pcap), PandaHDFDataLogic(path_provider, self.data), + publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name, connector=connector) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/sim/_blob_detector.py b/src/ophyd_async/sim/_blob_detector.py index 6760e088a1..3ca2b40db8 100644 --- a/src/ophyd_async/sim/_blob_detector.py +++ b/src/ophyd_async/sim/_blob_detector.py @@ -1,6 +1,12 @@ from collections.abc import Sequence +from functools import cached_property -from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import ( + DetectorLogic, + PathProvider, + SignalR, + StandardDetector, +) from ophyd_async.core import StandardReadableFormat as Format from ._blob_acquire_logic import BlobAcquireLogic @@ -20,13 +26,18 @@ def __init__( name: str = "", ) -> None: self.pattern_generator = pattern_generator or PatternGenerator() - self.add_detector_logics( + for signal in config_sigs: + self.set_readable_format(signal, Format.CONFIG_SIGNAL) + self._logic = DetectorLogic( BlobTriggerLogic(pattern_generator=self.pattern_generator), BlobAcquireLogic(pattern_generator=self.pattern_generator), BlobDataLogic( path_provider=path_provider, pattern_generator=self.pattern_generator ), + publish_collect_methods=self._publish_collect_methods, ) - for signal in config_sigs: - self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 582a1bd4ef..350a9b009f 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -226,9 +226,10 @@ async def make_pages(self, collections_written: int, collections_per_event: int) class BoundedOnlyDataLogic(DetectorDataLogic): - """Produces bounded data held in a finite buffer, sized at prepare_bounded.""" + """Produces bounded data held in a finite buffer, sized when it is armed.""" - def __init__(self): + def __init__(self, datakey_suffix: str = ""): + self.datakey_suffix = datakey_suffix self.collections_written = soft_signal_rw(int) self.prepare_calls: list[tuple[int, float]] = [] self.stop_count = 0 @@ -284,9 +285,7 @@ async def test_get_trigger_deadtime( trigger_logic, expected_triggers, expected_deadtime ): """Test get_trigger_deadtime with various trigger logic implementations.""" - det = StandardDetector() - if trigger_logic: - det.add_detector_logics(trigger_logic) + det = StandardDetector.with_logics(*([trigger_logic] if trigger_logic else [])) if isinstance(trigger_logic, DeadtimeTriggerLogic): # A logic that needs a signal declares it as configuration; the logic # no longer nominates signals separately @@ -302,10 +301,8 @@ async def test_get_trigger_deadtime_with_settings(): deadtime_signal = soft_signal_rw(float, 0.02) # Create detector with DeadtimeTriggerLogic - det = StandardDetector() + det = StandardDetector.with_logics(DeadtimeTriggerLogic(deadtime_signal)) det.sig = deadtime_signal - tl = DeadtimeTriggerLogic(deadtime_signal) - det.add_detector_logics(tl) det.set_readable_format(deadtime_signal, Format.CONFIG_SIGNAL) # Verify initial deadtime from signal @@ -334,9 +331,8 @@ async def test_get_trigger_deadtime_with_settings(): ) async def test_prepare_trigger_types(trigger_type): """Test each trigger type is properly delegated to trigger logic.""" - det = StandardDetector() trigger_logic = AllTriggerTypesLogic() - det.add_detector_logics(trigger_logic) + det = StandardDetector.with_logics(trigger_logic) trigger_info = TriggerInfo( trigger=trigger_type, livetime=0.5, deadtime=0.1, number_of_events=10 @@ -350,8 +346,7 @@ async def test_prepare_trigger_types(trigger_type): async def test_prepare_unsupported_trigger_type(): """Test that preparing with unsupported trigger type raises error.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = StandardDetector.with_logics(JustInternalTriggerLogic()) with pytest.raises(ValueError, match="Trigger type.*EXTERNAL_EDGE not supported"): await det.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) @@ -388,9 +383,8 @@ async def test_trigger_info_calculations( @pytest.mark.parametrize("exposures_per_collection", [1, 2, 5, 10]) async def test_exposures_per_collection(exposures_per_collection): """Test exposure averaging configuration.""" - det = StandardDetector() tl = AveragingTriggerLogic() - det.add_detector_logics(tl) + det = StandardDetector.with_logics(tl) await det.prepare( TriggerInfo( @@ -405,8 +399,7 @@ async def test_exposures_per_collection(exposures_per_collection): async def test_exposures_per_collection_not_supported(): """Test that exposures_per_collection > 1 fails without supporting logic.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) # Doesn't support averaging + det = StandardDetector.with_logics(JustInternalTriggerLogic()) with pytest.raises( ValueError, match="Multiple exposures per collection not supported" @@ -424,11 +417,10 @@ async def test_exposures_per_collection_not_supported(): ) async def test_arm_timing(trigger_type, arm_timing, tmp_path): """Verify detector is armed at the correct time based on trigger type.""" - det = StandardDetector() tl = AllTriggerTypesLogic() al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, al, dl) + det = StandardDetector.with_logics(tl, al, dl) # Prepare the detector await det.prepare(TriggerInfo(trigger=trigger_type, number_of_events=2)) @@ -457,10 +449,9 @@ async def test_arm_timing(trigger_type, arm_timing, tmp_path): async def test_trigger_arms_detector(tmp_path): """Test that trigger() arms the detector when arm logic is present.""" - det = StandardDetector() al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), al, dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), al, dl) await det.prepare(TriggerInfo()) @@ -482,9 +473,8 @@ async def test_trigger_arms_detector(tmp_path): async def test_arm_logic_called_on_stage(): """Test that acquire logic is stopped on stage.""" - det = StandardDetector() al = MockAcquireLogic() - det.add_detector_logics(al) + det = StandardDetector.with_logics(al) al.armed = True # Simulate being armed await det.stage() @@ -495,26 +485,23 @@ async def test_arm_logic_called_on_stage(): async def test_describe_before_prepare_raises(tmp_path): """Test that describe() fails before prepare().""" - det = StandardDetector() - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path)) - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe() async def test_describe_collect_before_prepare_raises(tmp_path): """Test that describe_collect() fails before prepare().""" - det = StandardDetector() - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path)) - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe_collect() async def test_trigger_after_multi_event_prepare_raises(): """Test that trigger() after prepare with multiple events fails.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = StandardDetector.with_logics(JustInternalTriggerLogic()) await det.prepare(TriggerInfo(number_of_events=5)) @@ -527,9 +514,8 @@ async def test_preserve_detector_state_requires_default_trigger_info( ): """Test PRESERVE_DETECTOR_STATE=YES errors without default_trigger_info.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="mydet") + det = StandardDetector.with_logics(AllTriggerTypesLogic(), name="mydet") # AllTriggerTypesLogic intentionally does not implement default_trigger_info - det.add_detector_logics(AllTriggerTypesLogic()) await det.stage() with pytest.raises( @@ -547,12 +533,12 @@ async def test_preserve_detector_state_no_trigger_logic_falls_back( trigger() silently falls back to a bare TriggerInfo() rather than raising. A detector with no trigger logic has no hardware state to preserve.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="nodet") + det = StandardDetector.with_logics(name="nodet") await det.stage() # Should not raise — no trigger logic means nothing to preserve await det.trigger() - assert det._prepare_ctx is not None - assert det._prepare_ctx.trigger_info == TriggerInfo() + assert det.logic.data is not None + assert det.logic.data is not None async def test_preserve_detector_state_multi_collection_watcher_and_assets( @@ -571,9 +557,10 @@ async def default_trigger_info(self) -> TriggerInfo: return TriggerInfo(collections_per_event=5) monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="multidet") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(MultiCollectionTriggerLogic(), dl) + det = StandardDetector.with_logics( + MultiCollectionTriggerLogic(), dl, name="multidet" + ) await det.stage() # Collect watcher updates via watch() callback @@ -610,12 +597,11 @@ async def test_collect_asset_docs_uses_trigger_observed_event( It could be the case that ``dl.collections_written`` changes between ``trigger()`` and ``collect_asset_docs``.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) dl.collections_written, set_collections_written = soft_signal_r_and_setter( int, getter=lambda: 0 ) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="det") await det.prepare(TriggerInfo(collections_per_event=5, exposure_timeout=0.1)) status = det.trigger() @@ -628,57 +614,46 @@ async def test_collect_asset_docs_uses_trigger_observed_event( assert docs[1][1]["indices"] == {"start": 0, "stop": 1} -async def test_kickoff_respects_prepare_bounds(tmp_path): - """Test that multiple kickoff() calls respect prepared bounds.""" - det = StandardDetector() - tl = JustInternalTriggerLogic() +async def test_one_kickoff_per_prepare(tmp_path): + """A prepare serves one kickoff/complete cycle; the next needs a new prepare.""" dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") - # Prepare for 5 events await det.prepare(TriggerInfo(number_of_events=5)) - - # Update collections_written signal to simulate data being written - - # First kickoff for 3 events - await det.events_to_kickoff.set(3) await det.kickoff() - await dl.collections_written.set(3) - # Second kickoff for 2 events should work (total = 5) - await det.events_to_kickoff.set(2) - await det.kickoff() + # A second kickoff without an intervening complete is rejected + with pytest.raises(RuntimeError, match="prepare.* before kickoff"): + await det.kickoff() + + status = det.complete() await dl.collections_written.set(5) + await status - # Third kickoff should fail (would exceed 5) - await det.events_to_kickoff.set(1) - with pytest.raises( - RuntimeError, - match="Kickoff requested 5:6, but detector was only prepared up to 5", - ): + # ...and so is one after complete, until prepare() runs again + with pytest.raises(RuntimeError, match="prepare.* before kickoff"): await det.kickoff() async def test_stage_resets_state(): """Test that stage() resets detector state.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = StandardDetector.with_logics(JustInternalTriggerLogic()) await det.prepare(TriggerInfo(number_of_events=5)) - await det.events_to_kickoff.set(3) # Stage should reset everything await det.stage() - assert det._prepare_ctx is None - assert det._kickoff_ctx is None - assert await det.events_to_kickoff.get_value() == 0 + assert det.logic.data is None + with pytest.raises(RuntimeError, match="prepare.* must be called first"): + await det.describe() + with pytest.raises(RuntimeError, match="prepare.* before kickoff"): + await det.kickoff() async def test_hints_from_single_data_logic(tmp_path): """Test that hints come from data logic.""" - det = StandardDetector(name="bar") - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path), name="bar") await det.prepare(TriggerInfo()) @@ -687,10 +662,9 @@ async def test_hints_from_single_data_logic(tmp_path): async def test_hints_from_multiple_data_logics(tmp_path): """Test that hints are aggregated from multiple data logics.""" - det = StandardDetector(name="bar") dl1 = StreamableOnlyDataLogic(tmp_path) dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-extra") - det.add_detector_logics(dl1, dl2) + det = StandardDetector.with_logics(dl1, dl2, name="bar") await det.prepare(TriggerInfo()) @@ -712,7 +686,7 @@ async def test_config_signals_in_describe_configuration( signal_type, initial_value, expected_dtype, expected_dtype_numpy, expected_shape ): """Test that added config signals appear in describe_configuration.""" - det = StandardDetector() + det = StandardDetector.with_logics() signal = soft_signal_rw( signal_type, initial_value=initial_value, name="test-config" ) @@ -742,8 +716,7 @@ async def test_config_signals_in_describe_configuration( async def test_kickoff_without_streamable_data_raises(): """Test that kickoff() without streamable data fails.""" - det = StandardDetector(name="foo") - det.add_detector_logics(JustInternalTriggerLogic()) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), name="foo") # A detector with no data logic can still be triggered for a step scan await det.prepare(TriggerInfo()) @@ -759,10 +732,9 @@ async def test_kickoff_without_streamable_data_raises(): async def test_streamable_supports_both_step_and_fly(tmp_path): """Test that streamable data logic supports both step and fly scanning.""" - det = StandardDetector(name="foo") tl = JustInternalTriggerLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, dl) + det = StandardDetector.with_logics(tl, dl, name="foo") # Step scan should work status = det.trigger() @@ -838,7 +810,7 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): async def test_read_returns_correct_values(): """Test that read() returns the values of registered signals.""" - det = StandardDetector(name="det") + det = StandardDetector.with_logics(name="det") det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") det.set_readable_format(det.counts, Format.HINTED_SIGNAL) @@ -848,7 +820,7 @@ async def test_read_returns_correct_values(): async def test_detector_with_no_logics(): """Test that detector works with no logics for basic internal triggering.""" - det = StandardDetector() + det = StandardDetector.with_logics() # Should support only INTERNAL triggering triggers, deadtime = await det.get_trigger_deadtime() @@ -864,7 +836,7 @@ async def test_detector_with_no_logics(): async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_triggering(): """Test that detector without trigger logic cannot set livetime/deadtime/trigger.""" - det = StandardDetector(name="foo") + det = StandardDetector.with_logics(name="foo") msg = "Detector foo has no trigger logic, so " with pytest.raises(ValueError, match=msg + "cannot set livetime or deadtime"): @@ -874,44 +846,30 @@ async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_trigg await det.prepare(TriggerInfo(deadtime=0.1)) -async def test_cannot_add_two_trigger_logics(): - """Test that adding two trigger logics raises an error.""" - det = StandardDetector() - tl1 = JustInternalTriggerLogic() - tl2 = AllTriggerTypesLogic() - - det.add_detector_logics(tl1) - +async def test_cannot_have_two_trigger_logics(): + """Test that two trigger logics raises an error.""" with pytest.raises(RuntimeError, match="Detector already has trigger logic"): - det.add_detector_logics(tl2) - + StandardDetector.with_logics(JustInternalTriggerLogic(), AllTriggerTypesLogic()) -async def test_cannot_add_two_arm_logics(): - """Test that adding two acquire logics raises an error.""" - det = StandardDetector() - al1 = MockAcquireLogic() - al2 = MockAcquireLogic() - - det.add_detector_logics(al1) +async def test_cannot_have_two_arm_logics(): + """Test that two acquire logics raises an error.""" with pytest.raises(RuntimeError, match="Detector already has acquire logic"): - det.add_detector_logics(al2) + StandardDetector.with_logics(MockAcquireLogic(), MockAcquireLogic()) -async def test_add_unknown_logic_type_raises(): - """Test that adding an unknown logic type raises TypeError.""" - det = StandardDetector() +async def test_unknown_logic_type_raises(): + """Test that an unknown logic type raises TypeError.""" class UnknownLogic: pass with pytest.raises(TypeError, match="Unknown logic type"): - det.add_detector_logics(UnknownLogic()) + StandardDetector.with_logics(UnknownLogic()) # type: ignore[arg-type] -async def test_add_logic_filling_two_roles_raises(): +async def test_logic_filling_two_roles_raises(): """An object satisfying two logic protocols must not be silently half-registered.""" - det = StandardDetector() class BothAcquireAndData(DetectorAcquireLogic, DetectorDataLogic): async def start_acquiring(self): ... @@ -922,7 +880,7 @@ async def ensure_stopped(self): ... TypeError, match="is both DetectorAcquireLogic, DetectorDataLogic", ): - det.add_detector_logics(BothAcquireAndData()) + StandardDetector.with_logics(BothAcquireAndData()) @pytest.mark.parametrize("initial_shutter_closed", [True, False]) @@ -952,9 +910,8 @@ async def wait_for_idle(self): async def ensure_stopped(self): self.shutter_closed = True # close shutter at end of scan - det = StandardDetector() al = ShutterAcquireLogic(initial_shutter_closed) - det.add_detector_logics(al) + det = StandardDetector.with_logics(al) await det.stage() assert ( @@ -970,9 +927,8 @@ async def ensure_stopped(self): async def test_bounded_step_scan_reads_derived_from_page(): """A bounded buffer describes as an array and read() derives one reading.""" - det = StandardDetector(name="foo") dl = BoundedOnlyDataLogic() - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") ti = TriggerInfo( number_of_events=1, collections_per_event=5, livetime=0.1, deadtime=0.0 @@ -996,33 +952,27 @@ async def test_bounded_step_scan_reads_derived_from_page(): async def test_bounded_fly_scan_accumulates_across_kickoffs(): """A bounded buffer is armed once at prepare and not re-armed per kickoff.""" - det = StandardDetector(name="foo") dl = BoundedOnlyDataLogic() - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") await det.prepare( TriggerInfo(number_of_events=10, collections_per_event=1, livetime=0.1) ) assert dl.prepare_calls == [(10, pytest.approx(0.1))] - # First kickoff of 5 events, buffer fills 0 -> 5 - await det.events_to_kickoff.set(5) await det.kickoff() status = det.complete() + # The buffer fills as the scan runs, and pages come out as it goes await dl.collections_written.set(5) pages = [page async for page in det._collect_pages()] assert pages[0]["data"]["foo"] == [[0]] * 5 - assert status.done + assert not status.done - # Second kickoff of 5 more, buffer continues 5 -> 10 (monotonic, no reset) - await det.events_to_kickoff.set(5) - await det.kickoff() - status = det.complete() await dl.collections_written.set(10) pages = [page async for page in det._collect_pages()] assert pages[0]["data"]["foo"] == [[0]] * 5 - assert status.done - # kickoff() never re-arms: prepare_bounded was only ever called once + await status + # kickoff() never re-arms: the buffer was only ever armed once assert dl.prepare_calls == [(10, pytest.approx(0.1))] @@ -1049,9 +999,8 @@ async def prepare_internal(self, num: int, livetime: float, deadtime: float): async def default_trigger_info(self) -> TriggerInfo: return default - det = StandardDetector(name="foo") dl = BoundedOnlyDataLogic() - det.add_detector_logics(PeriodTriggerLogic(), dl) + det = StandardDetector.with_logics(PeriodTriggerLogic(), dl, name="foo") await det.prepare( TriggerInfo( @@ -1063,15 +1012,14 @@ async def default_trigger_info(self) -> TriggerInfo: async def test_bounded_dropped_for_infinite_events(): """A bounded buffer cannot serve an infinite scan, so it makes no provider.""" - det = StandardDetector(name="foo") dl = BoundedOnlyDataLogic() - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") await det.prepare(TriggerInfo(number_of_events=0)) - ctx = det._prepare_ctx + ctx = det.logic.data assert ctx is not None - assert ctx.pageable_data_providers == [] + assert ctx.pageable == [] # Sitting the scan out is not an error, and nothing was armed assert dl.prepare_calls == [] @@ -1082,8 +1030,7 @@ async def test_data_logic_with_no_make_data_provider_raises(): class EmptyDataLogic(DetectorDataLogic): pass - det = StandardDetector() - det.add_detector_logics(EmptyDataLogic()) + det = StandardDetector.with_logics(EmptyDataLogic()) with pytest.raises(NotImplementedError): await det.prepare(TriggerInfo()) @@ -1091,9 +1038,8 @@ class EmptyDataLogic(DetectorDataLogic): async def test_unstage_disarms_detector(): """Test that unstage() calls disarm on the detector.""" - det = StandardDetector() al = MockAcquireLogic() - det.add_detector_logics(al) + det = StandardDetector.with_logics(al) al.armed = True await det.unstage() @@ -1104,9 +1050,8 @@ async def test_unstage_disarms_detector(): async def test_prepare_stops_data_logic_when_recreating_providers(tmp_path): """Test that prepare() calls stop() on data logic when recreating providers.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="det") # First prepare with collections_per_event=2 await det.prepare(TriggerInfo(number_of_events=3, collections_per_event=2)) @@ -1119,10 +1064,9 @@ async def test_prepare_stops_data_logic_when_recreating_providers(tmp_path): async def test_different_collections_written_raises(tmp_path): """Test that different collections_written values from providers raises error.""" - det = StandardDetector(name="det") dl1 = StreamableOnlyDataLogic(tmp_path) dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl1, dl2) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl1, dl2, name="det") await det.prepare(TriggerInfo(number_of_events=5)) @@ -1142,12 +1086,11 @@ async def test_different_collections_written_raises(tmp_path): async def test_data_logic_and_registered_signal(tmp_path): """A registered signal and a data logic both describe(), only one collects.""" - det = StandardDetector(name="det") + det = StandardDetector.with_logics( + JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path), name="det" + ) det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") det.set_readable_format(det.counts, Format.HINTED_SIGNAL) - det.add_detector_logics( - JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path) - ) await det.prepare(TriggerInfo()) @@ -1186,9 +1129,8 @@ async def test_data_logic_and_registered_signal(tmp_path): async def test_collect_asset_docs_with_explicit_index(tmp_path): """Test collect_asset_docs() with explicitly provided index.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="det") await det.prepare(TriggerInfo(number_of_events=5, collections_per_event=2)) @@ -1229,7 +1171,7 @@ async def test_child_readable_config_signals_in_describe_configuration(): config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") child.add_readables([config_sig], Format.CONFIG_SIGNAL) - det = StandardDetector(name="det") + det = StandardDetector.with_logics(name="det") det.add_readables([child], Format.CHILD) await det.prepare(TriggerInfo()) @@ -1246,9 +1188,8 @@ async def test_child_readable_read_signals_in_read(tmp_path): read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") child.add_readables([read_sig], Format.HINTED_SIGNAL) - det = StandardDetector(name="det") + det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path), name="det") det.add_readables([child], Format.CHILD) - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) await det.prepare(TriggerInfo()) # The data logic describes its stream, the child describes its signal @@ -1267,9 +1208,8 @@ async def test_child_readable_hints_merged(tmp_path): hinted_sig = soft_signal_rw(float, name="child-intensity") child.add_readables([hinted_sig], Format.HINTED_SIGNAL) - det = StandardDetector(name="det") + det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path), name="det") det.add_readables([child], Format.CHILD) - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) assert "fields" in det.hints assert "child-intensity" in det.hints["fields"] @@ -1311,7 +1251,7 @@ async def test_data_logic_not_implemented_errors(): async def test_detector_readable_format_changes_at_runtime(): # A StandardDetector is a StandardReadable, so a plugin signal can be moved # between configuration and hinted reads without redefining the Device. - det = StandardDetector(name="det") + det = StandardDetector.with_logics(name="det") det.temperature, _ = soft_signal_r_and_setter(float, 20.0, name="temperature") name = det.temperature.name @@ -1330,7 +1270,7 @@ async def test_detector_readable_format_changes_at_runtime(): async def test_detector_add_config_signals_is_deprecated(): - det = StandardDetector(name="det") + det = StandardDetector.with_logics(name="det") signal, _ = soft_signal_r_and_setter(float, 1.0, name="sig") with pytest.deprecated_call(): det.add_config_signals(signal) @@ -1345,8 +1285,7 @@ async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): detector must look like exactly one of WritesStreamAssets / EventPageCollectable so the bundler routes it down a single path. """ - det = StandardDetector() - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path)) # Which verb applies follows what the logics will produce, so it is not # known until they have been asked @@ -1361,8 +1300,7 @@ async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): async def test_bounded_logic_looks_like_event_page_collectable(): """A bounded logic exposes collect_pages, and only that.""" - det = StandardDetector() - det.add_detector_logics(BoundedOnlyDataLogic()) + det = StandardDetector.with_logics(BoundedOnlyDataLogic()) await det.prepare(TriggerInfo()) assert isinstance(det, EventPageCollectable) @@ -1378,8 +1316,7 @@ async def test_bounded_logic_looks_like_event_page_collectable(): async def test_no_data_logic_looks_like_neither(): """A detector with no data logic emits neither stream assets nor event pages.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = StandardDetector.with_logics(JustInternalTriggerLogic()) assert not isinstance(det, WritesStreamAssets) assert not isinstance(det, EventPageCollectable) @@ -1389,25 +1326,47 @@ async def test_no_data_logic_looks_like_neither(): async def test_missing_attribute_raises_standard_attribute_error(): """__getattr__ falls through to a normal AttributeError for other names.""" - det = StandardDetector() + det = StandardDetector.with_logics() with pytest.raises( AttributeError, - match=r"'StandardDetector' object has no attribute 'does_not_exist'", + match=r"object has no attribute 'does_not_exist'", ): det.does_not_exist # noqa: B018 -async def test_mixing_bounded_and_unbounded_logics_raises(tmp_path): - """Bounded and unbounded logics are mutually exclusive on one detector.""" - det = StandardDetector() - det.add_detector_logics(BoundedOnlyDataLogic(), StreamableOnlyDataLogic(tmp_path)) +async def test_unshadowed_bounded_keys_raise(tmp_path): + """A finite buffer the stream assets do not cover cannot be combined with them.""" + det = StandardDetector.with_logics( + BoundedOnlyDataLogic(datakey_suffix="-stats"), + StreamableOnlyDataLogic(tmp_path), + name="det", + ) # Only knowable once the logics have said what they would make with pytest.raises( TypeError, match=( - r"would produce both event pages \(MockPageableProvider\) and stream " - r"assets \(StreamResourceDataProvider\)" + r"would produce \['det-stats'\] as event pages and the rest of its " + r"data as stream assets" ), ): await det.prepare(TriggerInfo()) + + +async def test_shadowed_bounded_logic_sits_the_scan_out(tmp_path): + """A finite buffer whose keys are all written durably gives way to the file.""" + bounded = BoundedOnlyDataLogic() + det = StandardDetector.with_logics( + bounded, StreamableOnlyDataLogic(tmp_path), name="det" + ) + + await det.prepare(TriggerInfo()) + + # Both would produce "det", so the durable copy wins and nothing was armed + assert bounded.prepare_calls == [] + assert det.logic.data is not None + assert det.logic.data.pageable == [] + assert isinstance(det, WritesStreamAssets) + assert not isinstance(det, EventPageCollectable) + # ...and it stops hinting at data nobody will produce + assert det.hints == {"fields": ["det"]} diff --git a/tests/unit_tests/epics/adcore/test_acquire_logic.py b/tests/unit_tests/epics/adcore/test_acquire_logic.py index 561c116b87..8f76f2fc2e 100644 --- a/tests/unit_tests/epics/adcore/test_acquire_logic.py +++ b/tests/unit_tests/epics/adcore/test_acquire_logic.py @@ -17,8 +17,9 @@ async def adbase_detector() -> adcore.AreaDetector[adcore.ADBaseIO]: driver = adcore.ADBaseIO("PREFIX:DRV:") async with init_devices(mock=True): - det = adcore.AreaDetector(driver=driver) - det.add_detector_logics(adcore.ADAcquireLogic(driver)) + det = adcore.AreaDetector( + driver=driver, acquire_logic=adcore.ADAcquireLogic(driver) + ) return det diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index d96466a496..2e92574d4d 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -157,11 +157,11 @@ def stats_detector(stats: adcore.NDStatsIO) -> StandardDetector: """A writer-less detector whose only data logic is a stats time series.""" values = np.array([5.0, 6.0, 7.0, 8.0]) times = np.array([200.0, 201.0, 202.0, 203.0]) - det = StandardDetector(name="det") - det.add_detector_logics( + det = StandardDetector.with_logics( _JustInternal(), _FillStatsAcquireLogic(stats, values, times), StatsTimeSeriesDataLogic(stats), + name="det", ) return det diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 4ea02c3e80..a2600c42ec 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -71,7 +71,7 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( assert config["det-driver-acquire_period"]["value"] == 0.1 assert config["det-driver-acquire_time"]["value"] == 0.05 # Check we need to call prepare or trigger before we can describe - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe() # Need to tell it the path is valid set_mock_value(writer.file_path_exists, True) @@ -385,7 +385,7 @@ async def test_flyscan_aravis_detector(static_path_provider: StaticPathProvider) # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, - match="Kickoff requested 15:30, but detector was only prepared up to 15", + match="prepare.* must be called before kickoff", ): await det.kickoff() # But preparing again is ok @@ -606,7 +606,7 @@ async def test_step_scan_keep_numimages( assert config["det-driver-acquire_period"]["value"] == 0.1 assert config["det-driver-acquire_time"]["value"] == 0.05 # Check we need to call prepare or trigger before we can describe - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe() # Need to tell it the path is valid set_mock_value(writer.file_path_exists, True) diff --git a/tests/unit_tests/fastcs/odin/test_odin_io.py b/tests/unit_tests/fastcs/odin/test_odin_io.py index aba2db062b..4ac2ca5d91 100644 --- a/tests/unit_tests/fastcs/odin/test_odin_io.py +++ b/tests/unit_tests/fastcs/odin/test_odin_io.py @@ -1,4 +1,5 @@ import asyncio +from functools import cached_property from pathlib import Path from unittest.mock import call @@ -6,6 +7,7 @@ from bluesky import RunEngine from ophyd_async.core import ( + DetectorLogic, StandardDetector, StaticFilenameProvider, StaticPathProvider, @@ -27,11 +29,16 @@ def __init__(self, tmp_path: Path, name="", connector=None): path_provider = StaticPathProvider(StaticFilenameProvider("filename"), tmp_path) self.odin = OdinIO(connector=fastcs_connector("PREFIX:")) self.bit_depth = soft_signal_rw(int, BIT_DEPTH) - self.add_detector_logics( - OdinDataLogic(path_provider, self.odin, self.bit_depth) + self._logic = DetectorLogic( + OdinDataLogic(path_provider, self.odin, self.bit_depth), + publish_collect_methods=self._publish_collect_methods, ) super().__init__(name, connector) + @cached_property + def logic(self) -> DetectorLogic: + return self._logic + @pytest.fixture def odin_det(RE: RunEngine, tmp_path) -> OdinDet: diff --git a/tests/unit_tests/fastcs/panda/test_hdf_panda.py b/tests/unit_tests/fastcs/panda/test_hdf_panda.py index 19a147a312..489a38d2a8 100644 --- a/tests/unit_tests/fastcs/panda/test_hdf_panda.py +++ b/tests/unit_tests/fastcs/panda/test_hdf_panda.py @@ -276,7 +276,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, - match="Kickoff requested 15:30, but detector was only prepared up to 15", + match="prepare.* must be called before kickoff", ): await hdf_panda.kickoff() # But preparing again is ok From c37f53c3eb3e00997b5200d2a72500aa1b9d811e Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 1 Sep 2026 10:20:03 +0000 Subject: [PATCH 44/67] refactor: let on_complete report its own progress, dropping WatchableFlyableLogic A whole extra base class to inherit was a lot of machinery for what is a property of one method's return value. on_complete now returns either an Awaitable[None] or an AsyncIterator[WatcherUpdate], and complete() looks at what it got: an iterator goes straight to the watchers, anything else is awaited. --- docs/how-to/write-a-flyer.md | 9 +++-- src/ophyd_async/core/__init__.py | 2 - src/ophyd_async/core/_detector.py | 6 +-- src/ophyd_async/core/_flyable.py | 67 +++++++++++++------------------ 4 files changed, 36 insertions(+), 48 deletions(-) diff --git a/docs/how-to/write-a-flyer.md b/docs/how-to/write-a-flyer.md index db33d32e89..da4d06f164 100644 --- a/docs/how-to/write-a-flyer.md +++ b/docs/how-to/write-a-flyer.md @@ -24,7 +24,7 @@ info passed to `prepare()`, and `CtxT`, the context threaded between the phases |--------|-------------------|--------------| | `on_prepare(value)` | *abstract* | move to the start, arm the hardware, and return a context | | `on_kickoff(ctx)` | *abstract* | start the scan and return the context for `on_complete` | -| `on_complete(ctx)` | *abstract* | block until the scan finishes | +| `on_complete(ctx)` | *abstract* | block until the scan finishes, optionally yielding progress | | `stop()` | no-op | disarm the hardware and wait for it to stop | | `on_stage()` | calls `stop()` | set the hardware up on `stage()` if it differs from stop | | `on_unstage()` | calls `stop()` | clean the hardware up on `unstage()` if it differs from stop | @@ -74,8 +74,11 @@ status on the context so `on_complete` can await it: Because the logic is also a `MovableLogic`, `complete()` reports progress to watchers (e.g. progress bars) automatically by observing the readback — reusing the same [](#WatcherUpdate) machinery as [](#StandardMovable.set). A flyer whose logic is *not* -movable (see the next section) just blocks with no progress updates. See -[](../explanations/when-to-extend-movable.md) for more on `MovableLogic`. +movable (see the next section) just blocks with no progress updates, unless it reports +its own: write `on_complete` as an async generator yielding [](#WatcherUpdate) and those +updates go straight to the watchers, as a detector's "collections written out of +collections requested" does. See [](../explanations/when-to-extend-movable.md) for more on +`MovableLogic`. Wire the logic into the Device with a `@cached_property` called `logic`. Each mix-in declares that same name with its own required type, so one logic object that diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index fd0b8841ca..b378668cdd 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -56,7 +56,6 @@ FlyableLogic, FlyMotorInfo, StandardFlyable, - WatchableFlyableLogic, ) from ._log import config_ophyd_async_logging, logger, set_handler from ._mock_signal_backend import MockSignalBackend @@ -313,7 +312,6 @@ def __getattr__(name): "StandardFlyable", "FlyMotorInfo", "FlyableLogic", - "WatchableFlyableLogic", # Settings "Settings", "SettingsProvider", diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index f7b68838a3..e0b1de235c 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -28,7 +28,7 @@ PageableDataProvider, StreamableDataProvider, ) -from ._flyable import StandardFlyable, WatchableFlyableLogic +from ._flyable import FlyableLogic, StandardFlyable from ._readable import ( StandardReadable, StandardReadableFormat, @@ -409,7 +409,7 @@ class _FlyCtx: kickoff_collections_written: int | None = None -class DetectorLogic(WatchableFlyableLogic[TriggerInfo, _FlyCtx]): +class DetectorLogic(FlyableLogic[TriggerInfo, _FlyCtx]): """Drive a detector through its trigger, acquire and data logics. :param logics: @@ -550,7 +550,7 @@ async def on_kickoff(self, ctx: _FlyCtx) -> _FlyCtx: await self.acquire_logic.start_acquiring() return ctx - def on_complete_updates(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: + def on_complete(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: """Wait for the scan to finish, reporting collections written as progress.""" return self._wait_for_collections( trigger_info=ctx.trigger_info, diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index 963ff659f0..8b79f4e4f5 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -1,6 +1,6 @@ import asyncio from abc import abstractmethod -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable from enum import Enum from functools import cached_property from typing import Generic, TypeVar, cast @@ -61,9 +61,12 @@ async def on_kickoff(self, ctx: CtxT) -> CtxT: """ @abstractmethod - async def on_complete(self, ctx: CtxT) -> None: + def on_complete(self, ctx: CtxT) -> Awaitable[None] | AsyncIterator[WatcherUpdate]: """Block until the fly scan is done. + Write it as an `async def` to just block, or as an async generator + yielding [](#WatcherUpdate) to report progress to watchers as it goes. + :param ctx: the context returned by `on_kickoff`. """ @@ -88,30 +91,6 @@ def with_device(self, name: str = "") -> "StandardFlyable[PrepareT, CtxT]": return _EphemeralFlyable(self, name=name) -class WatchableFlyableLogic(FlyableLogic[PrepareT, CtxT]): - """A `FlyableLogic` that reports its own progress while completing. - - `StandardFlyable.complete()` reports progress for a `MovableLogic` by - observing its readback against its setpoint. A logic whose progress is - neither -- a detector's is "collections written out of collections - requested" -- yields its own updates instead. - - Implement `on_complete_updates` rather than `on_complete`; the inherited - `on_complete` drains the updates for callers that only want to block. - """ - - @abstractmethod - def on_complete_updates(self, ctx: CtxT) -> AsyncIterator[WatcherUpdate]: - """Block until the fly scan is done, yielding progress as it goes. - - :param ctx: the context returned by `on_kickoff`. - """ - - async def on_complete(self, ctx: CtxT) -> None: - async for _ in self.on_complete_updates(ctx): - pass - - class FlyMotorInfo(ConfinedModel): """Minimal set of information required to fly a motor.""" @@ -153,6 +132,11 @@ def _ramp_distance(self, acceleration_time: float) -> float: ) +async def _awaited(awaitable: Awaitable[None]) -> None: + # AsyncStatus takes a coroutine, while on_complete may return any awaitable + await awaitable + + class _FlyStage(Enum): """Lifecycle stage of a `StandardFlyable`, used to enforce call ordering.""" @@ -250,26 +234,34 @@ async def kickoff(self) -> None: async def complete(self) -> AsyncIterator[WatcherUpdate]: """Block until the fly scan is done. - If the logic is also a `MovableLogic` (e.g. a flying `Motor`), report - progress to watchers by observing its readback while the fly scan runs, - reusing the same watcher-update stream as `StandardMovable.set`. A - `WatchableFlyableLogic` reports its own progress instead. Any other - flyer simply blocks with no progress updates. + A logic whose `on_complete` yields [](#WatcherUpdate) reports its own + progress, and those updates are passed straight on to watchers. One that + just blocks reports progress only if it is also a `MovableLogic` (e.g. a + flying `Motor`), by observing its readback while the fly scan runs, + reusing the same watcher-update stream as `StandardMovable.set`. Any + other flyer simply blocks with no progress updates. """ if self._fly_stage is not _FlyStage.KICKED_OFF: raise RuntimeError( f"{self.name}: kickoff() must be called before complete()" ) logic = self.logic - if isinstance(logic, MovableLogic): + completing = logic.on_complete(self._fly_ctx) + if isinstance(completing, AsyncIterator): + # Progress that is neither a readback nor a setpoint -- a detector's + # is "collections written out of collections requested" -- so the + # logic reports it itself + async for update in completing: + yield update + elif isinstance(logic, MovableLogic): initial, target, (units, precision) = await asyncio.gather( logic.readback.get_value(), logic.setpoint.get_value(), logic.get_units_precision(), ) - async with AsyncStatus(logic.on_complete(self._fly_ctx)) as completing: + async with AsyncStatus(_awaited(completing)) as completed: async for current_position in observe_value( - logic.readback, done_status=completing + logic.readback, done_status=completed ): yield WatcherUpdate( current=current_position, @@ -279,13 +271,8 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: unit=units, precision=precision, ) - elif isinstance(logic, WatchableFlyableLogic): - # Progress that is neither a readback nor a setpoint: the logic - # knows how to report it, so let it - async for update in logic.on_complete_updates(self._fly_ctx): - yield update else: - await self.logic.on_complete(self._fly_ctx) + await completing self._fly_stage = _FlyStage.IDLE From 963824b9d487b3a89102347802f2d331aa4e7e95 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 1 Sep 2026 10:21:11 +0000 Subject: [PATCH 45/67] refactor: publish the collect verb from the Device, not through the logic DetectorLogic no longer takes a publish_collect_methods callback: the Device binds whichever collect verb its data logics turn out to need, in prepare() and trigger(), reading it from the logic's prepared data. The logic therefore holds nothing that points back at its Device. That makes a DetectorLogic constructible on its own, so an ad-hoc detector is DetectorLogic(*logics).with_device(name) exactly as for any other FlyableLogic, and StandardDetector.with_logics goes. --- src/ophyd_async/core/_detector.py | 96 ++++++-------- src/ophyd_async/epics/adcore/_detector.py | 4 +- src/ophyd_async/fastcs/eiger/_detector.py | 1 - src/ophyd_async/fastcs/jungfrau/_detector.py | 1 - src/ophyd_async/fastcs/panda/_detector.py | 1 - src/ophyd_async/sim/_blob_detector.py | 1 - .../unit_tests/core/test_standard_detector.py | 120 +++++++++--------- .../epics/adcore/test_stats_time_series.py | 6 +- tests/unit_tests/fastcs/odin/test_odin_io.py | 1 - 9 files changed, 101 insertions(+), 130 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index e0b1de235c..7998ac939f 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -38,7 +38,7 @@ ) from ._settings import Settings from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value -from ._status import WatchableAsyncStatus +from ._status import AsyncStatus, WatchableAsyncStatus from ._utils import ( DEFAULT_TIMEOUT, ConfinedModel, @@ -188,14 +188,11 @@ async def default_trigger_info(self) -> TriggerInfo: raise NotImplementedError(self) -def _logic_supported(base_class, method) -> bool: +def _trigger_logic_supported(method) -> bool: # If the function that is bound in a subclass is the same as the function - # attached to the superclass, then the subclass has not overridden it, so - # this method is not supported by the subclass. - return method.__func__ is not getattr(base_class, method.__name__) - - -_trigger_logic_supported = functools.partial(_logic_supported, DetectorTriggerLogic) + # attached to DetectorTriggerLogic, then the subclass has not overridden it, + # so this method is not supported by the subclass. + return method.__func__ is not getattr(DetectorTriggerLogic, method.__name__) def _get_supported_triggers( @@ -416,21 +413,13 @@ class DetectorLogic(FlyableLogic[TriggerInfo, _FlyCtx]): At most one [](#DetectorTriggerLogic), at most one [](#DetectorAcquireLogic), and any number of [](#DetectorDataLogic), in any order. Each object must fill exactly one of those roles. - :param publish_collect_methods: - Called from `on_prepare` with which collect verbs the data logics turn - out to need, so that a [](#StandardDetector) can expose the matching - bluesky protocol. A detector passes its own - `_publish_collect_methods`; [](#StandardDetector.with_logics) does that - for an ad-hoc one. """ def __init__( self, *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, - publish_collect_methods: Callable[..., None], ) -> None: self.logics = logics - self.publish_collect_methods = publish_collect_methods self.trigger_logic: DetectorTriggerLogic | None = None self.acquire_logic: DetectorAcquireLogic | None = None self.data_logics: tuple[DetectorDataLogic, ...] = () @@ -485,6 +474,10 @@ def prepared_data(self) -> _DetectorData: self.data, f"{self.datakey_prefix}: prepare() must be called first" ) + def with_device(self, name: str = "") -> "StandardDetector": + """Wrap this logic in an ephemeral `StandardDetector` for use in a plan.""" + return _LogicDetector(self, name=name) + def get_deadtime(self, config_values: SignalDict) -> float: """Return the deadtime the trigger logic calculates from `config_values`.""" trigger_logic = error_if_none(self.trigger_logic, "No trigger logic") @@ -698,12 +691,6 @@ async def _prepare_data(self, trigger_info: TriggerInfo) -> _DetectorData: collections_per_event=trigger_info.collections_per_event, period=period, ) - # Which collect verb the detector exposes follows what it will actually - # produce, so it is recomputed here rather than fixed at construction - self.publish_collect_methods( - stream_assets=bool(self.data.streamable), - event_pages=bool(self.data.pageable), - ) return self.data async def _make_serving( @@ -851,7 +838,6 @@ def __init__(self, prefix: str, name: str = "") -> None: self._logic = DetectorLogic( MyTriggerLogic(self.driver), MyAcquireLogic(self.driver), - publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name) @@ -860,8 +846,8 @@ def logic(self) -> DetectorLogic: return self._logic ``` - For an ad-hoc detector, [](#StandardDetector.with_logics) does the same - thing without a subclass. + For an ad-hoc detector, [](#DetectorLogic.with_device) does the same thing + without a subclass. Signals read during a step scan are registered with [](#StandardReadable.set_readable_format), exactly as on any other @@ -885,28 +871,12 @@ def logic(self) -> DetectorLogic: """The logic that drives this detector, built in the subclass `__init__`.""" raise NotImplementedError - @classmethod - def with_logics( - cls, - *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, - name: str = "", - ) -> "StandardDetector": - """Make a detector from some logics, without writing a subclass. - - The logic object a `StandardDetector` needs is built with a callback into - the Device, so it cannot be constructed before the Device it belongs to. - This does both, for an ad-hoc detector in a plan or a test. - """ - return _LogicsDetector(*logics, name=name) - def set_name(self, name: str, *, child_name_separator: str | None = None) -> None: super().set_name(name, child_name_separator=child_name_separator) # The data logics name their datakeys after the Device self.logic.datakey_prefix = name - def _publish_collect_methods( - self, *, stream_assets: bool, event_pages: bool - ) -> None: + def _publish_collect_methods(self) -> None: """Bind the collect verb that matches what the data logics will produce. Whichever applies is bound as a real instance attribute, so the bluesky @@ -916,12 +886,18 @@ def _publish_collect_methods( `__getattr__`, so a dynamic hook would be invisible. Both names are reserved by `Device`, hence `object.__setattr__`. - `DetectorLogic` calls this from every prepare, so the verb that no longer + Called from every prepare and trigger, so the verb that no longer applies is removed rather than left behind from a previous scan. + + TODO: the bundler treats the two protocols as mutually exclusive from + the static type, so a detector cannot declare both and pick per scan. + Once bluesky decides from what a device actually produced, both verbs + become plain methods and this goes; see ADR 0023. """ + data = self.logic.data for verb, method, wanted in ( - ("collect_asset_docs", self._collect_asset_docs, stream_assets), - ("collect_pages", self._collect_pages, event_pages), + ("collect_asset_docs", self._collect_asset_docs, data and data.streamable), + ("collect_pages", self._collect_pages, data and data.pageable), ): if wanted: object.__setattr__(self, verb, method) @@ -980,11 +956,17 @@ async def get_trigger_deadtime( deadtime = self.logic.get_deadtime(config_values) return self.logic.supported_triggers, deadtime + @AsyncStatus.wrap + async def prepare(self, value: TriggerInfo) -> None: + """Set the detector up for a scan, as described by the TriggerInfo.""" + await super().prepare(value) + self._publish_collect_methods() + @WatchableAsyncStatus.wrap async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: """Trigger a single exposure. - If [`prepare()`](#StandardFlyable.prepare) has not been called since the + If [`prepare()`](#StandardDetector.prepare) has not been called since the last `stage()`, an implicit prepare is performed. When [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` [](#DetectorTriggerLogic.default_trigger_info) is called to read current @@ -994,6 +976,9 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: await self.prepare(await self.logic.default_trigger_info()) async for update in self.logic.on_trigger(self._prepared_fly_ctx): yield update + # A step scan re-arms its finite buffers on every trigger, and a logic + # may sit the rest of the scan out, so the verbs are recomputed + self._publish_collect_methods() def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: """Contribute what the data logics produce to read() and describe(). @@ -1100,17 +1085,16 @@ async def get_index(self) -> int: return min_collections_written // data.collections_per_event -class _LogicsDetector(StandardDetector): - """A concrete `StandardDetector` built from logics, see `with_logics`.""" +class _LogicDetector(StandardDetector): + """A concrete `StandardDetector` built around a logic object. - def __init__( - self, - *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, - name: str = "", - ) -> None: - self._logic = DetectorLogic( - *logics, publish_collect_methods=self._publish_collect_methods - ) + `StandardDetector.logic` is abstract, so the class itself cannot be + instantiated; `DetectorLogic.with_device` needs something concrete to wrap a + logic object in. + """ + + def __init__(self, logic: DetectorLogic, name: str = "") -> None: + self._logic = logic super().__init__(name=name) @cached_property diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index 0ec86284bd..198fa5dc7e 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -63,9 +63,7 @@ def __init__( self.set_readable_format(self.driver, Format.CHILD) for signal in config_sigs: self.set_readable_format(signal, Format.CONFIG_SIGNAL) - self._logic = DetectorLogic( - *logics, publish_collect_methods=self._publish_collect_methods - ) + self._logic = DetectorLogic(*logics) super().__init__(name=name) @cached_property diff --git a/src/ophyd_async/fastcs/eiger/_detector.py b/src/ophyd_async/fastcs/eiger/_detector.py index 90cb7c4c36..a60a802e81 100644 --- a/src/ophyd_async/fastcs/eiger/_detector.py +++ b/src/ophyd_async/fastcs/eiger/_detector.py @@ -45,7 +45,6 @@ def __init__( detector_bit_depth=self.detector.bit_depth_image, pixel_mask=self.pixel_mask, ), - publish_collect_methods=self._publish_collect_methods, ) self.add_config_signals(self.pixel_mask) diff --git a/src/ophyd_async/fastcs/jungfrau/_detector.py b/src/ophyd_async/fastcs/jungfrau/_detector.py index 906569c9e7..a1a0c9c832 100644 --- a/src/ophyd_async/fastcs/jungfrau/_detector.py +++ b/src/ophyd_async/fastcs/jungfrau/_detector.py @@ -44,7 +44,6 @@ def __init__( odin=self.odin, detector_bit_depth=self.detector.bit_depth, ), - publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name) diff --git a/src/ophyd_async/fastcs/panda/_detector.py b/src/ophyd_async/fastcs/panda/_detector.py index 900d370a31..d0197e7e1f 100644 --- a/src/ophyd_async/fastcs/panda/_detector.py +++ b/src/ophyd_async/fastcs/panda/_detector.py @@ -29,7 +29,6 @@ def __init__( PandaTriggerLogic(self.pcap), PandaAcquireLogic(self.pcap), PandaHDFDataLogic(path_provider, self.data), - publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name, connector=connector) diff --git a/src/ophyd_async/sim/_blob_detector.py b/src/ophyd_async/sim/_blob_detector.py index 3ca2b40db8..42e1be3cb0 100644 --- a/src/ophyd_async/sim/_blob_detector.py +++ b/src/ophyd_async/sim/_blob_detector.py @@ -34,7 +34,6 @@ def __init__( BlobDataLogic( path_provider=path_provider, pattern_generator=self.pattern_generator ), - publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name) diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 350a9b009f..b47fdd261f 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -13,13 +13,13 @@ Array1D, DetectorAcquireLogic, DetectorDataLogic, + DetectorLogic, DetectorTrigger, DetectorTriggerLogic, PageableDataProvider, Settings, SignalDict, SignalR, - StandardDetector, StandardReadable, StreamableDataProvider, StreamResourceDataProvider, @@ -285,7 +285,7 @@ async def test_get_trigger_deadtime( trigger_logic, expected_triggers, expected_deadtime ): """Test get_trigger_deadtime with various trigger logic implementations.""" - det = StandardDetector.with_logics(*([trigger_logic] if trigger_logic else [])) + det = DetectorLogic(*([trigger_logic] if trigger_logic else [])).with_device() if isinstance(trigger_logic, DeadtimeTriggerLogic): # A logic that needs a signal declares it as configuration; the logic # no longer nominates signals separately @@ -301,7 +301,7 @@ async def test_get_trigger_deadtime_with_settings(): deadtime_signal = soft_signal_rw(float, 0.02) # Create detector with DeadtimeTriggerLogic - det = StandardDetector.with_logics(DeadtimeTriggerLogic(deadtime_signal)) + det = DetectorLogic(DeadtimeTriggerLogic(deadtime_signal)).with_device() det.sig = deadtime_signal det.set_readable_format(deadtime_signal, Format.CONFIG_SIGNAL) @@ -332,7 +332,7 @@ async def test_get_trigger_deadtime_with_settings(): async def test_prepare_trigger_types(trigger_type): """Test each trigger type is properly delegated to trigger logic.""" trigger_logic = AllTriggerTypesLogic() - det = StandardDetector.with_logics(trigger_logic) + det = DetectorLogic(trigger_logic).with_device() trigger_info = TriggerInfo( trigger=trigger_type, livetime=0.5, deadtime=0.1, number_of_events=10 @@ -346,7 +346,7 @@ async def test_prepare_trigger_types(trigger_type): async def test_prepare_unsupported_trigger_type(): """Test that preparing with unsupported trigger type raises error.""" - det = StandardDetector.with_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() with pytest.raises(ValueError, match="Trigger type.*EXTERNAL_EDGE not supported"): await det.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) @@ -384,7 +384,7 @@ async def test_trigger_info_calculations( async def test_exposures_per_collection(exposures_per_collection): """Test exposure averaging configuration.""" tl = AveragingTriggerLogic() - det = StandardDetector.with_logics(tl) + det = DetectorLogic(tl).with_device() await det.prepare( TriggerInfo( @@ -399,7 +399,7 @@ async def test_exposures_per_collection(exposures_per_collection): async def test_exposures_per_collection_not_supported(): """Test that exposures_per_collection > 1 fails without supporting logic.""" - det = StandardDetector.with_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() with pytest.raises( ValueError, match="Multiple exposures per collection not supported" @@ -420,7 +420,7 @@ async def test_arm_timing(trigger_type, arm_timing, tmp_path): tl = AllTriggerTypesLogic() al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(tl, al, dl) + det = DetectorLogic(tl, al, dl).with_device() # Prepare the detector await det.prepare(TriggerInfo(trigger=trigger_type, number_of_events=2)) @@ -451,7 +451,7 @@ async def test_trigger_arms_detector(tmp_path): """Test that trigger() arms the detector when arm logic is present.""" al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(JustInternalTriggerLogic(), al, dl) + det = DetectorLogic(JustInternalTriggerLogic(), al, dl).with_device() await det.prepare(TriggerInfo()) @@ -474,7 +474,7 @@ async def test_trigger_arms_detector(tmp_path): async def test_arm_logic_called_on_stage(): """Test that acquire logic is stopped on stage.""" al = MockAcquireLogic() - det = StandardDetector.with_logics(al) + det = DetectorLogic(al).with_device() al.armed = True # Simulate being armed await det.stage() @@ -485,7 +485,7 @@ async def test_arm_logic_called_on_stage(): async def test_describe_before_prepare_raises(tmp_path): """Test that describe() fails before prepare().""" - det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device() with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe() @@ -493,7 +493,7 @@ async def test_describe_before_prepare_raises(tmp_path): async def test_describe_collect_before_prepare_raises(tmp_path): """Test that describe_collect() fails before prepare().""" - det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device() with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe_collect() @@ -501,7 +501,7 @@ async def test_describe_collect_before_prepare_raises(tmp_path): async def test_trigger_after_multi_event_prepare_raises(): """Test that trigger() after prepare with multiple events fails.""" - det = StandardDetector.with_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() await det.prepare(TriggerInfo(number_of_events=5)) @@ -514,7 +514,7 @@ async def test_preserve_detector_state_requires_default_trigger_info( ): """Test PRESERVE_DETECTOR_STATE=YES errors without default_trigger_info.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector.with_logics(AllTriggerTypesLogic(), name="mydet") + det = DetectorLogic(AllTriggerTypesLogic()).with_device("mydet") # AllTriggerTypesLogic intentionally does not implement default_trigger_info await det.stage() @@ -533,7 +533,7 @@ async def test_preserve_detector_state_no_trigger_logic_falls_back( trigger() silently falls back to a bare TriggerInfo() rather than raising. A detector with no trigger logic has no hardware state to preserve.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector.with_logics(name="nodet") + det = DetectorLogic().with_device("nodet") await det.stage() # Should not raise — no trigger logic means nothing to preserve await det.trigger() @@ -558,9 +558,7 @@ async def default_trigger_info(self) -> TriggerInfo: monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics( - MultiCollectionTriggerLogic(), dl, name="multidet" - ) + det = DetectorLogic(MultiCollectionTriggerLogic(), dl).with_device("multidet") await det.stage() # Collect watcher updates via watch() callback @@ -601,7 +599,7 @@ async def test_collect_asset_docs_uses_trigger_observed_event( dl.collections_written, set_collections_written = soft_signal_r_and_setter( int, getter=lambda: 0 ) - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="det") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("det") await det.prepare(TriggerInfo(collections_per_event=5, exposure_timeout=0.1)) status = det.trigger() @@ -617,7 +615,7 @@ async def test_collect_asset_docs_uses_trigger_observed_event( async def test_one_kickoff_per_prepare(tmp_path): """A prepare serves one kickoff/complete cycle; the next needs a new prepare.""" dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") await det.prepare(TriggerInfo(number_of_events=5)) await det.kickoff() @@ -637,7 +635,7 @@ async def test_one_kickoff_per_prepare(tmp_path): async def test_stage_resets_state(): """Test that stage() resets detector state.""" - det = StandardDetector.with_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() await det.prepare(TriggerInfo(number_of_events=5)) @@ -653,7 +651,7 @@ async def test_stage_resets_state(): async def test_hints_from_single_data_logic(tmp_path): """Test that hints come from data logic.""" - det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path), name="bar") + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device("bar") await det.prepare(TriggerInfo()) @@ -664,7 +662,7 @@ async def test_hints_from_multiple_data_logics(tmp_path): """Test that hints are aggregated from multiple data logics.""" dl1 = StreamableOnlyDataLogic(tmp_path) dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-extra") - det = StandardDetector.with_logics(dl1, dl2, name="bar") + det = DetectorLogic(dl1, dl2).with_device("bar") await det.prepare(TriggerInfo()) @@ -686,7 +684,7 @@ async def test_config_signals_in_describe_configuration( signal_type, initial_value, expected_dtype, expected_dtype_numpy, expected_shape ): """Test that added config signals appear in describe_configuration.""" - det = StandardDetector.with_logics() + det = DetectorLogic().with_device() signal = soft_signal_rw( signal_type, initial_value=initial_value, name="test-config" ) @@ -716,7 +714,7 @@ async def test_config_signals_in_describe_configuration( async def test_kickoff_without_streamable_data_raises(): """Test that kickoff() without streamable data fails.""" - det = StandardDetector.with_logics(JustInternalTriggerLogic(), name="foo") + det = DetectorLogic(JustInternalTriggerLogic()).with_device("foo") # A detector with no data logic can still be triggered for a step scan await det.prepare(TriggerInfo()) @@ -734,7 +732,7 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): """Test that streamable data logic supports both step and fly scanning.""" tl = JustInternalTriggerLogic() dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(tl, dl, name="foo") + det = DetectorLogic(tl, dl).with_device("foo") # Step scan should work status = det.trigger() @@ -810,7 +808,7 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): async def test_read_returns_correct_values(): """Test that read() returns the values of registered signals.""" - det = StandardDetector.with_logics(name="det") + det = DetectorLogic().with_device("det") det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") det.set_readable_format(det.counts, Format.HINTED_SIGNAL) @@ -820,7 +818,7 @@ async def test_read_returns_correct_values(): async def test_detector_with_no_logics(): """Test that detector works with no logics for basic internal triggering.""" - det = StandardDetector.with_logics() + det = DetectorLogic().with_device() # Should support only INTERNAL triggering triggers, deadtime = await det.get_trigger_deadtime() @@ -836,7 +834,7 @@ async def test_detector_with_no_logics(): async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_triggering(): """Test that detector without trigger logic cannot set livetime/deadtime/trigger.""" - det = StandardDetector.with_logics(name="foo") + det = DetectorLogic().with_device("foo") msg = "Detector foo has no trigger logic, so " with pytest.raises(ValueError, match=msg + "cannot set livetime or deadtime"): @@ -849,13 +847,13 @@ async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_trigg async def test_cannot_have_two_trigger_logics(): """Test that two trigger logics raises an error.""" with pytest.raises(RuntimeError, match="Detector already has trigger logic"): - StandardDetector.with_logics(JustInternalTriggerLogic(), AllTriggerTypesLogic()) + DetectorLogic(JustInternalTriggerLogic(), AllTriggerTypesLogic()).with_device() async def test_cannot_have_two_arm_logics(): """Test that two acquire logics raises an error.""" with pytest.raises(RuntimeError, match="Detector already has acquire logic"): - StandardDetector.with_logics(MockAcquireLogic(), MockAcquireLogic()) + DetectorLogic(MockAcquireLogic(), MockAcquireLogic()).with_device() async def test_unknown_logic_type_raises(): @@ -865,7 +863,7 @@ class UnknownLogic: pass with pytest.raises(TypeError, match="Unknown logic type"): - StandardDetector.with_logics(UnknownLogic()) # type: ignore[arg-type] + DetectorLogic(UnknownLogic()).with_device() # type: ignore[arg-type] async def test_logic_filling_two_roles_raises(): @@ -880,7 +878,7 @@ async def ensure_stopped(self): ... TypeError, match="is both DetectorAcquireLogic, DetectorDataLogic", ): - StandardDetector.with_logics(BothAcquireAndData()) + DetectorLogic(BothAcquireAndData()).with_device() @pytest.mark.parametrize("initial_shutter_closed", [True, False]) @@ -911,7 +909,7 @@ async def ensure_stopped(self): self.shutter_closed = True # close shutter at end of scan al = ShutterAcquireLogic(initial_shutter_closed) - det = StandardDetector.with_logics(al) + det = DetectorLogic(al).with_device() await det.stage() assert ( @@ -928,7 +926,7 @@ async def ensure_stopped(self): async def test_bounded_step_scan_reads_derived_from_page(): """A bounded buffer describes as an array and read() derives one reading.""" dl = BoundedOnlyDataLogic() - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") ti = TriggerInfo( number_of_events=1, collections_per_event=5, livetime=0.1, deadtime=0.0 @@ -953,7 +951,7 @@ async def test_bounded_step_scan_reads_derived_from_page(): async def test_bounded_fly_scan_accumulates_across_kickoffs(): """A bounded buffer is armed once at prepare and not re-armed per kickoff.""" dl = BoundedOnlyDataLogic() - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") await det.prepare( TriggerInfo(number_of_events=10, collections_per_event=1, livetime=0.1) @@ -1000,7 +998,7 @@ async def default_trigger_info(self) -> TriggerInfo: return default dl = BoundedOnlyDataLogic() - det = StandardDetector.with_logics(PeriodTriggerLogic(), dl, name="foo") + det = DetectorLogic(PeriodTriggerLogic(), dl).with_device("foo") await det.prepare( TriggerInfo( @@ -1013,7 +1011,7 @@ async def default_trigger_info(self) -> TriggerInfo: async def test_bounded_dropped_for_infinite_events(): """A bounded buffer cannot serve an infinite scan, so it makes no provider.""" dl = BoundedOnlyDataLogic() - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="foo") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") await det.prepare(TriggerInfo(number_of_events=0)) @@ -1030,7 +1028,7 @@ async def test_data_logic_with_no_make_data_provider_raises(): class EmptyDataLogic(DetectorDataLogic): pass - det = StandardDetector.with_logics(EmptyDataLogic()) + det = DetectorLogic(EmptyDataLogic()).with_device() with pytest.raises(NotImplementedError): await det.prepare(TriggerInfo()) @@ -1039,7 +1037,7 @@ class EmptyDataLogic(DetectorDataLogic): async def test_unstage_disarms_detector(): """Test that unstage() calls disarm on the detector.""" al = MockAcquireLogic() - det = StandardDetector.with_logics(al) + det = DetectorLogic(al).with_device() al.armed = True await det.unstage() @@ -1051,7 +1049,7 @@ async def test_unstage_disarms_detector(): async def test_prepare_stops_data_logic_when_recreating_providers(tmp_path): """Test that prepare() calls stop() on data logic when recreating providers.""" dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="det") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("det") # First prepare with collections_per_event=2 await det.prepare(TriggerInfo(number_of_events=3, collections_per_event=2)) @@ -1066,7 +1064,7 @@ async def test_different_collections_written_raises(tmp_path): """Test that different collections_written values from providers raises error.""" dl1 = StreamableOnlyDataLogic(tmp_path) dl2 = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl1, dl2, name="det") + det = DetectorLogic(JustInternalTriggerLogic(), dl1, dl2).with_device("det") await det.prepare(TriggerInfo(number_of_events=5)) @@ -1086,9 +1084,9 @@ async def test_different_collections_written_raises(tmp_path): async def test_data_logic_and_registered_signal(tmp_path): """A registered signal and a data logic both describe(), only one collects.""" - det = StandardDetector.with_logics( - JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path), name="det" - ) + det = DetectorLogic( + JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path) + ).with_device("det") det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") det.set_readable_format(det.counts, Format.HINTED_SIGNAL) @@ -1130,7 +1128,7 @@ async def test_data_logic_and_registered_signal(tmp_path): async def test_collect_asset_docs_with_explicit_index(tmp_path): """Test collect_asset_docs() with explicitly provided index.""" dl = StreamableOnlyDataLogic(tmp_path) - det = StandardDetector.with_logics(JustInternalTriggerLogic(), dl, name="det") + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("det") await det.prepare(TriggerInfo(number_of_events=5, collections_per_event=2)) @@ -1171,7 +1169,7 @@ async def test_child_readable_config_signals_in_describe_configuration(): config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") child.add_readables([config_sig], Format.CONFIG_SIGNAL) - det = StandardDetector.with_logics(name="det") + det = DetectorLogic().with_device("det") det.add_readables([child], Format.CHILD) await det.prepare(TriggerInfo()) @@ -1188,7 +1186,7 @@ async def test_child_readable_read_signals_in_read(tmp_path): read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") child.add_readables([read_sig], Format.HINTED_SIGNAL) - det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path), name="det") + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device("det") det.add_readables([child], Format.CHILD) await det.prepare(TriggerInfo()) @@ -1208,7 +1206,7 @@ async def test_child_readable_hints_merged(tmp_path): hinted_sig = soft_signal_rw(float, name="child-intensity") child.add_readables([hinted_sig], Format.HINTED_SIGNAL) - det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path), name="det") + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device("det") det.add_readables([child], Format.CHILD) assert "fields" in det.hints @@ -1251,7 +1249,7 @@ async def test_data_logic_not_implemented_errors(): async def test_detector_readable_format_changes_at_runtime(): # A StandardDetector is a StandardReadable, so a plugin signal can be moved # between configuration and hinted reads without redefining the Device. - det = StandardDetector.with_logics(name="det") + det = DetectorLogic().with_device("det") det.temperature, _ = soft_signal_r_and_setter(float, 20.0, name="temperature") name = det.temperature.name @@ -1270,7 +1268,7 @@ async def test_detector_readable_format_changes_at_runtime(): async def test_detector_add_config_signals_is_deprecated(): - det = StandardDetector.with_logics(name="det") + det = DetectorLogic().with_device("det") signal, _ = soft_signal_r_and_setter(float, 1.0, name="sig") with pytest.deprecated_call(): det.add_config_signals(signal) @@ -1285,7 +1283,7 @@ async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): detector must look like exactly one of WritesStreamAssets / EventPageCollectable so the bundler routes it down a single path. """ - det = StandardDetector.with_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device() # Which verb applies follows what the logics will produce, so it is not # known until they have been asked @@ -1300,7 +1298,7 @@ async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): async def test_bounded_logic_looks_like_event_page_collectable(): """A bounded logic exposes collect_pages, and only that.""" - det = StandardDetector.with_logics(BoundedOnlyDataLogic()) + det = DetectorLogic(BoundedOnlyDataLogic()).with_device() await det.prepare(TriggerInfo()) assert isinstance(det, EventPageCollectable) @@ -1316,7 +1314,7 @@ async def test_bounded_logic_looks_like_event_page_collectable(): async def test_no_data_logic_looks_like_neither(): """A detector with no data logic emits neither stream assets nor event pages.""" - det = StandardDetector.with_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() assert not isinstance(det, WritesStreamAssets) assert not isinstance(det, EventPageCollectable) @@ -1326,7 +1324,7 @@ async def test_no_data_logic_looks_like_neither(): async def test_missing_attribute_raises_standard_attribute_error(): """__getattr__ falls through to a normal AttributeError for other names.""" - det = StandardDetector.with_logics() + det = DetectorLogic().with_device() with pytest.raises( AttributeError, match=r"object has no attribute 'does_not_exist'", @@ -1336,11 +1334,9 @@ async def test_missing_attribute_raises_standard_attribute_error(): async def test_unshadowed_bounded_keys_raise(tmp_path): """A finite buffer the stream assets do not cover cannot be combined with them.""" - det = StandardDetector.with_logics( - BoundedOnlyDataLogic(datakey_suffix="-stats"), - StreamableOnlyDataLogic(tmp_path), - name="det", - ) + det = DetectorLogic( + BoundedOnlyDataLogic(datakey_suffix="-stats"), StreamableOnlyDataLogic(tmp_path) + ).with_device("det") # Only knowable once the logics have said what they would make with pytest.raises( @@ -1356,9 +1352,7 @@ async def test_unshadowed_bounded_keys_raise(tmp_path): async def test_shadowed_bounded_logic_sits_the_scan_out(tmp_path): """A finite buffer whose keys are all written durably gives way to the file.""" bounded = BoundedOnlyDataLogic() - det = StandardDetector.with_logics( - bounded, StreamableOnlyDataLogic(tmp_path), name="det" - ) + det = DetectorLogic(bounded, StreamableOnlyDataLogic(tmp_path)).with_device("det") await det.prepare(TriggerInfo()) diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index 2e92574d4d..e852dd5a5f 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -3,6 +3,7 @@ from ophyd_async.core import ( DetectorAcquireLogic, + DetectorLogic, DetectorTriggerLogic, EnableDisable, StandardDetector, @@ -157,12 +158,11 @@ def stats_detector(stats: adcore.NDStatsIO) -> StandardDetector: """A writer-less detector whose only data logic is a stats time series.""" values = np.array([5.0, 6.0, 7.0, 8.0]) times = np.array([200.0, 201.0, 202.0, 203.0]) - det = StandardDetector.with_logics( + det = DetectorLogic( _JustInternal(), _FillStatsAcquireLogic(stats, values, times), StatsTimeSeriesDataLogic(stats), - name="det", - ) + ).with_device("det") return det diff --git a/tests/unit_tests/fastcs/odin/test_odin_io.py b/tests/unit_tests/fastcs/odin/test_odin_io.py index 4ac2ca5d91..6fbf43fd8b 100644 --- a/tests/unit_tests/fastcs/odin/test_odin_io.py +++ b/tests/unit_tests/fastcs/odin/test_odin_io.py @@ -31,7 +31,6 @@ def __init__(self, tmp_path: Path, name="", connector=None): self.bit_depth = soft_signal_rw(int, BIT_DEPTH) self._logic = DetectorLogic( OdinDataLogic(path_provider, self.odin, self.bit_depth), - publish_collect_methods=self._publish_collect_methods, ) super().__init__(name, connector) From 9e3e9e516cea5fd351ef3c6607ca6f49290a4f09 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 1 Sep 2026 10:22:16 +0000 Subject: [PATCH 46/67] fix: emit no external key for a bounded provider's datakeys external="" fails event-model's descriptor schema (it must match '^[A-Z]+:?'), so any real scan reading a stats time series raised ValidationError when the descriptor was composed. Nothing caught it because the tests called the detector's verbs directly. Drive the two step/fly scan cases through the RunEngine instead, which is where a descriptor gets validated. That also pins the order these have to go in: stage() resets the detector, so a prepare() before it is discarded and the scan silently falls back to a default TriggerInfo. --- .../epics/adcore/_stats_time_series.py | 3 +- .../unit_tests/core/test_standard_detector.py | 1 - .../epics/adcore/test_stats_time_series.py | 65 +++++++++++++++---- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index 6cb445ae97..dc90c3cdc2 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -52,7 +52,6 @@ async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: shape=[collections_per_event], dtype="array", dtype_numpy=" None: # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to # 0, so the buffer is armed and empty before the detector's frames arrive. # This is the data logic performing the erase itself, which is why - # trigger()'s zero baseline is correct (see ADR 0020). + # trigger()'s zero baseline is correct (see ADR 0022). await self.stats.ts_acquire.set(True) async def stop(self) -> None: diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index b47fdd261f..ca967b5779 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -208,7 +208,6 @@ async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: shape=[collections_per_event], dtype="array", dtype_numpy=" StandardDetector: """A writer-less detector whose only data logic is a stats time series.""" values = np.array([5.0, 6.0, 7.0, 8.0]) times = np.array([200.0, 201.0, 202.0, 203.0]) - det = DetectorLogic( + return DetectorLogic( _JustInternal(), _FillStatsAcquireLogic(stats, values, times), StatsTimeSeriesDataLogic(stats), ).with_device("det") - return det -async def test_step_scan_collect_pages_end_to_end(stats_detector: StandardDetector): - """A writer-less detector with a stats time series emits event pages.""" +def test_step_scan_through_run_engine(RE: RunEngine, stats_detector: StandardDetector): + """A writer-less detector describes and reads as internal data. + + Driven through the RunEngine so the descriptor its datakeys make is the one + a real scan would emit, and so has to pass event-model validation. + """ det = stats_detector - await det.stage() - await det.prepare(TriggerInfo(collections_per_event=4)) - # A bounded logic exposes collect_pages, not collect_asset_docs - assert hasattr(det, "collect_pages") - assert not hasattr(det, "collect_asset_docs") - await det.trigger() - pages = [page async for page in det.collect_pages()] - (page,) = pages - assert page["data"]["det"] == [[5.0, 6.0, 7.0, 8.0]] - await det.unstage() + docs: dict[str, list] = defaultdict(list) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + # prepare() comes after stage(), which resets the detector + yield from bps.prepare(det, TriggerInfo(collections_per_event=4), wait=True) + yield from bps.trigger_and_read([det]) + + RE(plan(), lambda name, doc: docs[name].append(doc)) + + (descriptor,) = docs["descriptor"] + assert descriptor["data_keys"]["det"]["shape"] == [4] + assert "external" not in descriptor["data_keys"]["det"] + (event,) = docs["event"] + assert event["data"]["det"] == [5.0, 6.0, 7.0, 8.0] + assert event["timestamps"]["det"] == 203.0 + + +def test_fly_scan_emits_event_pages(RE: RunEngine, stats_detector: StandardDetector): + """A writer-less detector collects its buffer as event pages.""" + det = stats_detector + docs: dict[str, list] = defaultdict(list) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + yield from bps.prepare(det, TriggerInfo(number_of_events=4), wait=True) + yield from bps.declare_stream(det, name="primary") + yield from bps.kickoff(det, wait=True) + yield from bps.collect_while_completing( + flyers=[det], dets=[det], flush_period=0.05 + ) + + RE(plan(), lambda name, doc: docs[name].append(doc)) + + (page,) = docs["event_page"] + assert page["data"]["det"] == [[5.0], [6.0], [7.0], [8.0]] + assert docs["stop"][0]["num_events"] == {"primary": 4} async def test_step_scan_read_derives_single_reading(stats_detector: StandardDetector): From fe23b02a720988d22b1fcbd755c36bc04e50389d Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 1 Sep 2026 10:22:54 +0000 Subject: [PATCH 47/67] fix: name the NDStats time series records as the IOC has them The stats system test has never got past connect in CI. The IOC reports TSAcquireMode's choices as ('Fixed length', 'Circ. buffer'), and has no TS:TSNumPoints_RBV; the per-statistic arrays belong to NDStats itself, at its own prefix, rather than to the embedded time series plugin under TS:. Taken from the PVs in example-services' generated NDStats OPI, which is the IOC CI runs against - worth a check against a live IOC. The test's plan also prepared before bp.count staged the detector, which discards the prepare. --- src/ophyd_async/epics/adcore/_io.py | 25 +++++++++++----------- tests/container_tests/test_adsim_system.py | 6 +++++- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index a4005788be..91a8347883 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -173,7 +173,7 @@ async def plugin_is_enabled(plugin: NDPluginBaseIO) -> bool: The nearest thing to "is this plugin in the chain": a plugin with callbacks disabled certainly is not, though an enabled one may still have its `nd_array_port` pointed somewhere else. Parsing the whole plugin graph was - rejected in ADR 0021 as too large. + rejected in ADR 0020 as too large. """ return await plugin.enable_callbacks.get_value() is EnableDisable.ENABLE @@ -215,7 +215,7 @@ class NDStatsTSAcquireMode(StrictEnum): """ FIXED_LENGTH = "Fixed length" - CIRCULAR_BUFFER = "Circular buffer" + CIRCULAR_BUFFER = "Circ. buffer" class NDStatsIO(StandardReadable, NDPluginBaseIO): @@ -247,20 +247,21 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] net: A[SignalR[float], PvSuffix("Net_RBV")] # Time series. NDStats delegates its time series to an embedded - # NDPluginTimeSeries instance under an inner "TS:" prefix, so each statistic - # is also exposed as a fixed-length array. ts_num_points sizes the buffer, - # ts_acquire erases and starts it (writing 1 clears the arrays and resets - # ts_current_point to 0), ts_current_point reports progress, ts_total holds - # the Total series and ts_timestamp holds the per-point acquisition times. - # These are not registered as readables: the scalars above serve read(), and - # the arrays are consumed by StatsTimeSeriesDataLogic as event pages. + # NDPluginTimeSeries instance, so the controls live under an inner "TS:" + # prefix while the per-statistic arrays are named by NDStats itself at the + # plugin's own prefix. ts_num_points sizes the buffer, ts_acquire erases and + # starts it (writing 1 clears the arrays and resets ts_current_point to 0), + # ts_current_point reports progress, ts_total holds the Total series and + # ts_timestamp holds the per-point acquisition times. These are not + # registered as readables: the scalars above serve read(), and the arrays + # are consumed by StatsTimeSeriesDataLogic as event pages. ts_acquire: A[SignalRW[bool], PvSuffix("TS:TSAcquire")] - ts_num_points: A[SignalRW[int], PvSuffix.rbv("TS:TSNumPoints")] + ts_num_points: A[SignalRW[int], PvSuffix("TS:TSNumPoints")] ts_current_point: A[SignalR[int], PvSuffix("TS:TSCurrentPoint")] ts_acquiring: A[SignalR[bool], PvSuffix("TS:TSAcquiring")] ts_acquire_mode: A[SignalRW[NDStatsTSAcquireMode], PvSuffix.rbv("TS:TSAcquireMode")] - ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TS:TSTotal")] - ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TS:TSTimestamp")] + ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TSTotal")] + ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] # Centroid statistics compute_centroid: A[ SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index a9746150fd..a89c742052 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -34,6 +34,7 @@ import bluesky.plan_stubs as bps import bluesky.plans as bp +import bluesky.preprocessors as bpp import pytest from aioca import purge_channel_caches from bluesky.run_engine import RunEngine @@ -384,13 +385,16 @@ def test_stats_time_series_step_scan( """ collections = 5 + @bpp.stage_decorator([adsimstat]) + @bpp.run_decorator() def prepare_then_count() -> MsgGenerator[None]: + # prepare() must come after stage(), which resets the detector yield from bps.prepare( adsimstat, TriggerInfo(collections_per_event=collections, livetime=0.1), wait=True, ) - yield from bp.count([adsimstat]) + yield from bps.trigger_and_read([adsimstat]) named_docs: list[tuple[str, DocumentType]] = [] RE(prepare_then_count(), lambda name, doc: named_docs.append((name, doc))) From 4948f13167be07b04e6f4b5115e64346470de618 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 1 Sep 2026 10:24:24 +0000 Subject: [PATCH 48/67] docs: renumber the new ADRs into commit order and link them up The bounded-tier ADR was written first but lands third, so 0020-0022 read in an order no branch introduces them in. Number them the way the commits arrive - readable formats, one logic property, then the bounded tier - and turn the "ADR 00NN" prose references between them into links. The bounded-tier ADR is also rewritten: shorter, and without the per-kickoff sizing and _update_prepare_context internals that no longer exist. What it said about exposing the collect verbs now lives in 0023, next to the decision itself. --- .../decisions/0012-detector-rewrite.md | 2 +- .../decisions/0017-standard-movable.md | 2 +- .../decisions/0020-bounded-data-logic-tier.md | 311 ------------------ ...mat.md => 0020-runtime-readable-format.md} | 4 +- ...perty.md => 0021-single-logic-property.md} | 4 +- .../decisions/0022-bounded-data-logic-tier.md | 225 +++++++++++++ .../decisions/0023-detector-is-a-flyable.md | 64 ++-- docs/how-to/change-readable-format.md | 2 +- .../unit_tests/epics/adcore/test_use_cases.py | 2 +- 9 files changed, 272 insertions(+), 344 deletions(-) delete mode 100644 docs/explanations/decisions/0020-bounded-data-logic-tier.md rename docs/explanations/decisions/{0021-runtime-readable-format.md => 0020-runtime-readable-format.md} (98%) rename docs/explanations/decisions/{0022-single-logic-property.md => 0021-single-logic-property.md} (96%) create mode 100644 docs/explanations/decisions/0022-bounded-data-logic-tier.md diff --git a/docs/explanations/decisions/0012-detector-rewrite.md b/docs/explanations/decisions/0012-detector-rewrite.md index db9fdf2006..ee4448e5bc 100644 --- a/docs/explanations/decisions/0012-detector-rewrite.md +++ b/docs/explanations/decisions/0012-detector-rewrite.md @@ -5,7 +5,7 @@ Date: 2026-01-20 Accepted -Partly superseded by [](./0021-runtime-readable-format.md), which made `StandardDetector` a +Partly superseded by [](./0020-runtime-readable-format.md), which made `StandardDetector` a `StandardReadable` and removed the `prepare_single` tier along with `ReadableDataProvider`. ## Context diff --git a/docs/explanations/decisions/0017-standard-movable.md b/docs/explanations/decisions/0017-standard-movable.md index f6de85ce27..0b348e76ba 100644 --- a/docs/explanations/decisions/0017-standard-movable.md +++ b/docs/explanations/decisions/0017-standard-movable.md @@ -6,7 +6,7 @@ Date: 2026-03-10 Accepted -Partly superseded by [](./0022-single-logic-property.md), which replaced `movable_logic` +Partly superseded by [](./0021-single-logic-property.md), which replaced `movable_logic` with a single `logic` property. ## Context diff --git a/docs/explanations/decisions/0020-bounded-data-logic-tier.md b/docs/explanations/decisions/0020-bounded-data-logic-tier.md deleted file mode 100644 index 780c199289..0000000000 --- a/docs/explanations/decisions/0020-bounded-data-logic-tier.md +++ /dev/null @@ -1,311 +0,0 @@ -# 20. Add a bounded `DetectorDataLogic` tier and select tiers by capability - -Date: 2026-07-17 - -## Status - -Accepted - -## Context - -ADR 0012 split data production out into `DetectorDataLogic`. After ADR 0021 removed the -single-event tier, one tier is left that a subclass opts into by overriding the -corresponding method: - -- `prepare_unbounded(datakey_name) -> StreamableDataProvider` — any number of collections, - feeding `collect_asset_docs()` and `describe_collect()`, doing the writes to start the writer - as it goes - -Four open issues turn out to bottom out in the same two gaps. - -### The data logic is not told enough at prepare time - -Issue #1309 wants chunk size and flush rate configured from the scan parameters. An Xspress -running at 400 Hz with a 2 Hz flush rate wants 200 frames per chunk; the PandA wants a flush -period. Neither is expressible today, because a data logic is prepared for an *unbounded* number -of collections and so is told nothing about timing. Today's implementations paper over this: -`ADHDFDataLogic` reads `num_frames_chunks` back from the IOC and forces it to 1 if unset, the -PandA hardcodes `chunk_shape=(1024,)` behind a TODO, and Odin hardcodes `(1, *data_shape)`. - -The parameter that is actually needed is the frame **period** (`livetime + deadtime`), not the -number of frames — the requirement is a flush *rate*. This matters, because the period is known -for step scans as well as fly scans, whereas the total frame count is not. - -Two workarounds were considered and rejected. Subclassing `TriggerInfo` and `prepare()` to carry a -chunk size (as Malcolm did) puts a data-writing concern into the trigger path and defeats ADR -0012's rule that logic classes are handed only the fields they need. Computing the chunk size in -the `DetectorTriggerLogic` puts a data concern in the trigger logic. - -### `StandardDetector` unconditionally claims to write stream assets - -Issues #1248 and #888 both come from detectors that produce data but cannot write files: a Struck -SIS3820 scaler in a VME crate, a MeasComp CTR-08, a TetrAmm electrometer, an electron analyser. -These want the trigger, arm, prepare and stage machinery of `StandardDetector`, but need to emit -event pages rather than stream assets. - -These are worth describing concretely, because they are not areaDetectors and they put more -pressure on the trigger/acquire/data split than an areaDetector does. A MeasComp CTR-08 exposes: - -| PV | role | -|---|---| -| `MCS:EraseStart` | clears the arrays *and* starts counting — one control, no separate erase | -| `MCS:StopAll` | stop | -| `MCS:Acquiring` | idle status | -| `MCS:NuseAll` | number of channels to use — the buffer size, and the point at which it stops | -| `MCS:CurrentChannel` | how many points have been collected | -| `mca.VAL` | the growing waveform arrays that hold the data | - -The difference from an areaDetector is that there is no driver/plugin split in the hardware. An -areaDetector acquires with `driver.acquire` and captures with a *separate* `TSAcquire` or HDF -`capture`, so the acquire and data concerns land on different PVs and the split falls out of the -hardware. Here one control both gates acquisition and clears the data buffer, and one PV -(`NuseAll`) is both the exposure count and the buffer size. - -They cannot, because `StandardDetector` inherits `WritesStreamAssets` unconditionally, and the -bluesky bundler only calls `collect_pages()` on a device that is *not* a `WritesStreamAssets`. -The two protocols are mutually exclusive by design, to avoid the case where both produce blocks -of different sizes. - -#888 asks for the writer to be optional. The rewrite already made it optional in the literal -sense — `StandardDetector()` takes no writer at all. But the underlying request ("no HDF writer, -because I have PVs instead") is #1248, so #888 is a duplicate. - -### These are the same gap - -Both groups of devices must be told *how many* collections to expect at prepare time, because -they own a finite buffer that has to be sized before acquisition: the scaler's MCA array length -*is* the frame count, and the areaDetector stats plugin's `TSNumPoints` must be set before the -time series starts. A device with a finite buffer also naturally produces its data as pages at -the end of an event rather than as stream datums as it goes. - -So one new tier serves both, and #1309's period rides along with it. - -### The areaDetector stats duality - -An `NDPluginStats` exposes each statistic twice: as a scalar (`Total_RBV`) and as a time series -array (`TS:TSTotal`). Only the scalar is modelled today, as a registered signal. A registered -signal is read once per event, so a detector with no file writer has no way to produce a value -per collection for a fly scan: the time series can, but nothing models it. - -## Decision - -### One method that describes, and one that starts - -```python -class DetectorDataLogic: - async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider | PageableDataProvider | None: ... - async def start(self) -> None: ... -``` - -`make_data_provider` says what this logic *would* produce for this scan without starting -anything; `start` does the writes that make it happen, and is called only for the providers the -detector will use. The detector therefore asks every data logic what it would make, decides which -ones it wants, and starts only those. - -The alternative — one `prepare_*` per tier, doing both jobs — was tried first. It forces the -detector to arm hardware in order to find out what it would get, so a provider it then decides -not to use has to be stopped again. That is wasteful with a `TSAcquire` erase and impossible to -justify once the choice depends on what the providers actually produce. - -Both tiers are told the frame period, so either can size a chunk or a buffer against it. - -`num_collections` is always the **total** for the scan (`TriggerInfo.number_of_collections`), -never a per-kickoff figure. A finite buffer is fed by data callbacks and does not care how many -kickoffs span it, so per-kickoff sizing would mean resizing the buffer mid-scan. - -### The tier is the type of the provider returned - -There is no declaration of which tier a logic implements and no override detection: a logic that -returns a [](#StreamableDataProvider) is unbounded for this scan and one that returns a -[](#PageableDataProvider) is bounded, so a logic may even return either depending on what it is -asked for. Returning `None` means "not this scan", which is how a finite buffer declines an -unbounded one. It is not an error and is not warned about, since the whole point is that a -detector may carry more logics than any one scan uses. - -### `PageableDataProvider`, with readings derived from pages - -The bounded tier returns a single provider type that emits pages. Readings are derived from -pages rather than being a separate code path: a step-scan prepare has `number_of_events == 1`, so -its page contains exactly one event, which extracts to a single reading. `collections_per_event` -appears in the datakey shape exactly as it does for `StreamableDataProvider`. - -### The period is resolved in `prepare()` - -`TriggerInfo.livetime` of 0 means "whatever is currently set on the detector", so the period -cannot simply be forwarded from the `TriggerInfo` — an explicit -`prepare(TriggerInfo(number_of_events=10))` would hand the data logic a period of 0. - -Instead `prepare()` resolves it. `DetectorTriggerLogic.default_trigger_info()` is extended to -return the current `livetime` and `deadtime` as well as the frame count, and `prepare()` uses it -to fill in a zero `livetime` after the trigger logic has been prepared. ADR 0012 already requires -that ordering, so the hardware holds the real values by that point. Data logics therefore never -see a period of 0, and the awkward case is handled once in core rather than by every implementer. - -### Tier selection is by capability, with no precedence rule - -Core does not choose between tiers — it only asks each logic what it would make for the -requested number of collections `n = TriggerInfo.number_of_collections`: - -| tier | serves | -|---|---| -| streaming | always | -| finite buffer | `n != 0` (finite) | - -A logic that cannot serve `n` returns `None` and is left out of the prepare context, rather than -raising, as agreed in #1364. That is what lets a detector carrying a bounded logic still be -prepared for an infinite fly scan, with only that logic dropped. - -### Bounded providers are never reused - -`_update_prepare_context` currently reuses providers when `collections_per_event` is unchanged, -to avoid reopening files on every step-scan point. Two changes follow. - -The reuse check gains the period, since the period now determines chunk shape, and chunk shape is -baked into the `StreamResource` at provider construction. A period change therefore starts a new -file. This is inherent rather than incidental: areaDetector's `num_frames_chunks` can only be set -before `capture` starts. - -Bounded providers are excluded from reuse entirely, so `trigger()` re-prepares them on every -point. This is what re-arms a finite buffer for each event, and it needs no new API: `prepare()` -and `trigger()` both call `_update_prepare_context`, and `kickoff()` does not — which is exactly -the set of places a buffer should and should not be re-armed. - -Reuse consequently becomes a per-logic decision rather than an all-or-nothing branch, and -`_PrepareCtx` holds pageable providers in their own list — a re-armed bounded provider sitting at -0 alongside a monotonic HDF writer at 15 would otherwise trip the `_all_the_same` reducer. - -### A bounded provider's progress baseline is zero in `trigger()` - -`trigger()` takes a bounded provider's starting progress to be 0 rather than reading it back; -`kickoff()` continues to read it live. - -It is tempting to derive this instead of stating it, since `_update_prepare_context` refreshes -`collections_written` after re-preparing, and an areaDetector stats time series is erased by the -`TSAcquire` write in its `start` — so the refreshed read already returns zero. That -derivation only holds when the data logic performs the erase itself. Where the erase belongs to -the acquire control, as with the CTR-08's `EraseStart`, it happens *after* `_update_prepare_context` -has taken its reading, so the baseline would be the stale pre-erase value; the detector would then -wait for twice the buffer's length and hang. Stating the rule keeps the framework agnostic about -which component erases. - -`kickoff()` must not zero the baseline, because a fly scan erases once and accumulates across -kickoffs; zeroing there would make the second kickoff's target already satisfied and return -immediately. Since a detector may not mix bounded and unbounded logics, each call has providers of -only one kind and the rule reduces to a single choice per call site. - -### Erase and start controls belong to the acquire logic - -Where one control both starts acquisition and clears the data buffer, it belongs to the -`DetectorAcquireLogic`, alongside the stop control and the idle status. Nothing needs to declare -the erase to the framework: a data logic's `start` is always called before `start_acquiring()`, -so the buffer is sized before the control fires, and the baseline rule above means the detector does not -care who erased it. - -The trigger/acquire/data split therefore holds for these devices, and the three logics share one -IO device exactly as the areaDetector logics share a `driver`. The split is a split of concerns, -not of hardware, so it survives the concerns landing on a single PV. - -### Logic objects fill exactly one role - -Building a `DetectorLogic` raises if an object satisfies more than one of the three logic roles, -directing the author to pass separate objects. This is a hazard worth catching rather than a -hypothetical: a device whose concerns all live on one control is a natural candidate for a single -combined logic object, and the registration is a chain of `isinstance` tests, so such an object -would silently register as the first protocol it matched and its other roles would be dropped -without a word. - -### `collect_asset_docs` and `collect_pages` are exposed dynamically - -`StandardDetector` no longer inherits `WritesStreamAssets`. Instead it binds -`collect_asset_docs` as an instance attribute only when it has a streaming provider, and -`collect_pages` only when it has a paging one. Because bluesky's protocols are `runtime_checkable`, an absent -attribute is enough to make the isinstance check fail, and the bundler then does the right -thing. It has to be a real attribute rather than a `__getattr__` hook, because Python 3.12+ -resolves those checks with `inspect.getattr_static`, which never calls `__getattr__`. - -Which verb applies is only knowable once the logics have said what they would make, so it is -decided in `prepare()` and recomputed on each one — binding the verb that applies and removing -the one that does not. A detector therefore satisfies neither protocol until it has been -prepared, which is fine: everything in the RunEngine that dispatches on them runs after prepare. - -Because the decision is per-prepare rather than per-construction, which logic serves can change -between runs — a detector whose file writer is switched off can fall back to its stats time -series. That is a **between-runs** change, like a readable format: a run's descriptor is emitted -at its start, so switching part way through would leave `describe_collect()` and the documents -disagreeing. Documented rather than enforced, since a Device cannot see run boundaries. - -This keeps a single detector class able to be either kind depending on its `__init__` arguments — -one `AravisDetector`, configured for files or for pages — which was the original requirement, and -it needs no change to bluesky. - -### Mixing bounded and unbounded logics raises - -`prepare()` raises if the providers it has been given include both kinds, since that is the one -combination where both `collect_asset_docs` and `collect_pages` would be exposed and both would -produce data in a fly scan. It raises after asking each logic what it would make and before -starting any of them, so nothing has been armed when it does. - -### Stats with a file writer go via NDAttributes - -It follows that an areaDetector with an HDF writer does not use the stats time series. It does -not need to: `ADHDFDataLogic` already pulls NDAttribute datasets into the file, which is -areaDetector's own mechanism for exactly this. The time series tier is for detectors with no -writer at all. - -Registering the scalar with `set_readable_format` therefore remains the right choice for a -detector that writes files and wants a stats value in a step scan, and a stats time series data -logic is the right choice for one that writes no files. - -### One stats data logic for many arrays - -An `NDPluginStats` has one time series control (`TSControl`, `TSNumPoints`, `TSAcquire`) shared -across roughly twenty arrays. A stats data logic therefore covers *many* arrays with the control -embedded, rather than one logic per array with a shared control object. - -This follows the grain of the existing design — a provider already emits many datakeys, as -`ADHDFDataLogic` does for the main dataset plus its NDAttributes — and it avoids the coordination -problem, since the detector prepares each data logic independently and several logics contending -over one `TSAcquire` would need reference counting or an ordering guarantee. Which statistics are -wanted becomes a constructor argument. - -## Consequences - -`DetectorDataLogic` implementations must be rewritten onto `make_data_provider` and `start`, and -take the new `period` argument. This is a breaking change for out-of-tree data logics; the library -is in alpha and no compatibility shim is provided. Splitting them is usually mechanical — describe -in one, write in the other — but a logic that cannot describe its data without starting (in tree, -`OdinDataLogic`, whose frame shape is only readable once the file processor is writing) has to do -its writes in `make_data_provider` and leave `start` empty. That is safe only while such a logic is -never the one a detector drops. In-tree, `ADHDFDataLogic` and `PandaHDFDataLogic` can then -compute chunk size and flush period from the rate, replacing the TODOs and hardcoded values that -stand in for it today. - -`DetectorTriggerLogic.default_trigger_info()` implementations should return the current -`livetime` and `deadtime` alongside the frame count. Those that do not will still work, but their -detectors will fall back to today's behaviour of reading the chunk size back from hardware. - -Detectors that cannot write files become expressible: a data logic returning a -`PageableDataProvider` gives them `collect_pages()` and the whole of `StandardDetector` besides. -This closes #1248 and #888. - -Fly scanning an areaDetector that carries an HDF writer plus a finite-buffer data logic starts -working, with the finite buffer sitting the scan out, closing #1364. - -A data logic may decline a scan for reasons of its own, not just because of the collection count: -`ADHDFDataLogic` and `StatsTimeSeriesDataLogic` take an `enable_callbacks` flag, defaulting to -today's behaviour of switching the plugin on, which when set `False` makes them follow whatever -the plugin is set to and produce nothing when it is off. - -Mixing bounded and unbounded data logics on one detector is not supported. If a use case appears -that genuinely needs a file *and* pages in one fly scan, it needs the bluesky bundler to allow -both protocols and error only on a size mismatch — a larger change, deliberately deferred. - -A device whose exposure count and buffer size are the same PV will have it written twice: once by -the trigger logic as `number_of_exposures`, and once by the data logic as `num_collections`. The -two agree only when `exposures_per_collection` is 1. This is latent rather than live, because such -devices do not average exposures and so cannot ask for anything else, but a detector that could do -both would need its data logic to read the value back rather than set it. It is the same -one-PV-two-concerns shape as the combined erase-and-start control, and the same answer applies: -the split is of concerns, not of PVs. diff --git a/docs/explanations/decisions/0021-runtime-readable-format.md b/docs/explanations/decisions/0020-runtime-readable-format.md similarity index 98% rename from docs/explanations/decisions/0021-runtime-readable-format.md rename to docs/explanations/decisions/0020-runtime-readable-format.md index cb53bebdcf..430d8476fb 100644 --- a/docs/explanations/decisions/0021-runtime-readable-format.md +++ b/docs/explanations/decisions/0020-runtime-readable-format.md @@ -1,4 +1,4 @@ -# 21. Make readable format a runtime setting keyed by Device +# 20. Make readable format a runtime setting keyed by Device Date: 2026-08-18 @@ -156,7 +156,7 @@ at runtime. ### The `prepare_single` data logic tier goes -ADR 0012 gave `DetectorDataLogic` a `prepare_single` tier, returning a +[](./0012-detector-rewrite.md) gave `DetectorDataLogic` a `prepare_single` tier, returning a `ReadableDataProvider`, so that a source producing one value per event could reach `read()` and `describe()`. Registering a signal now does that, for any Device and without a data logic, so the tier has nothing left of its own: its only in-tree implementation was diff --git a/docs/explanations/decisions/0022-single-logic-property.md b/docs/explanations/decisions/0021-single-logic-property.md similarity index 96% rename from docs/explanations/decisions/0022-single-logic-property.md rename to docs/explanations/decisions/0021-single-logic-property.md index 77a5fcb6de..4a09838722 100644 --- a/docs/explanations/decisions/0022-single-logic-property.md +++ b/docs/explanations/decisions/0021-single-logic-property.md @@ -1,4 +1,4 @@ -# 22. One `logic` property per Device +# 21. One `logic` property per Device Date: 2026-08-18 @@ -19,7 +19,7 @@ The obstacle is that Python has no intersection type: there is no way to write " must be both a `MovableLogic[float]` and a `FlyableLogic[FlyMotorInfo, MotorFlyCtx]`" as a single annotation. -ADR 0017 also chose `@cached_property` stacked on `@abstractmethod` over an +[](./0017-standard-movable.md) also chose `@cached_property` stacked on `@abstractmethod` over an `add_movable_logic()` method, on the grounds that the property is "checkable by static analysis". That turned out not to hold. `functools.cached_property` does not forward `__isabstractmethod__` from the function it wraps, so `StandardMovable.__abstractmethods__` diff --git a/docs/explanations/decisions/0022-bounded-data-logic-tier.md b/docs/explanations/decisions/0022-bounded-data-logic-tier.md new file mode 100644 index 0000000000..7cade01cef --- /dev/null +++ b/docs/explanations/decisions/0022-bounded-data-logic-tier.md @@ -0,0 +1,225 @@ +# 22. Add a bounded `DetectorDataLogic` tier + +Date: 2026-07-17 + +## Status + +Accepted + +## Context + +[](./0012-detector-rewrite.md) split data production out into `DetectorDataLogic`. After +[](./0020-runtime-readable-format.md) removed the single-event tier, one tier was left: a +logic prepared for an unbounded number of collections, feeding `collect_asset_docs()` and +`describe_collect()` and doing its own writes as it went. + +Four open issues bottom out in two gaps. + +### The data logic is not told enough at prepare time + +Issue #1309 wants chunk size and flush rate configured from the scan parameters. An Xspress +running at 400 Hz with a 2 Hz flush rate wants 200 frames per chunk; the PandA wants a flush +period. Neither is expressible, because an unbounded logic is told nothing about timing: +`ADHDFDataLogic` reads `num_frames_chunks` back from the IOC and forces it to 1 if unset, the +PandA hardcodes `chunk_shape=(1024,)` behind a TODO, and Odin hardcodes `(1, *data_shape)`. + +The parameter actually needed is the frame **period** (`livetime + deadtime`), not the number +of frames, since the requirement is a flush *rate* — and the period is known for step scans as +well as fly scans, where the total frame count is not. + +Two workarounds were rejected: subclassing `TriggerInfo` and `prepare()` to carry a chunk size +(as Malcolm did) puts a data-writing concern in the trigger path, against +[](./0012-detector-rewrite.md)'s rule that a logic is handed only the fields it needs; +computing it in the `DetectorTriggerLogic` puts a data concern in the trigger logic. + +### Detectors that produce data but cannot write files + +Issues #1248 and #888 come from a Struck SIS3820 scaler in a VME crate, a MeasComp CTR-08, a +TetrAmm electrometer, an electron analyser. They want the trigger, arm, prepare and stage +machinery of `StandardDetector` but must emit event pages, and `StandardDetector` inherited +`WritesStreamAssets` unconditionally — the bundler calls `collect_pages()` only on a device +that is *not* one. (#888 asks for the writer to be optional, which the rewrite already did; +its real request is #1248, so it is a duplicate.) + +These are not areaDetectors, and they put more pressure on the trigger/acquire/data split. A +CTR-08 has no driver/plugin split in the hardware: `MCS:EraseStart` both clears the arrays and +starts counting, and `MCS:NuseAll` is both the exposure count and the buffer size, where an +areaDetector acquires with `driver.acquire` and captures with a separate `TSAcquire` or HDF +`capture`. + +### These are the same gap + +Both groups must be told *how many* collections to expect at prepare time, because they own a +finite buffer that has to be sized before acquisition: the scaler's MCA array length *is* the +frame count, and the stats plugin's `TSNumPoints` must be set before the time series starts. A +device with a finite buffer also naturally produces its data as pages at the end of an event +rather than as stream datums as it goes. + +So one new tier serves both, and #1309's period rides along with it. + +### The areaDetector stats duality + +An `NDPluginStats` exposes each statistic twice: as a scalar (`Total_RBV`) and as a time +series array (`TSTotal`). Only the scalar was modelled, as a registered signal read once per +event, so a detector with no file writer had no way to produce a value per collection for a +fly scan. + +## Decision + +### One method that describes, and one that starts + +```python +class DetectorDataLogic: + async def make_data_provider( + self, datakey_name: str, num_collections: int, period: float + ) -> StreamableDataProvider | PageableDataProvider | None: ... + async def start(self) -> None: ... +``` + +`make_data_provider` says what this logic *would* produce for this scan without starting +anything; `start` does the writes that make it happen, and is called only for the providers +the detector will use. So the detector asks every logic what it would make, decides which ones +it wants, and starts only those. + +One `prepare_*` per tier, doing both jobs, was tried first. It forces the detector to arm +hardware to find out what it would get, so a provider it then decides not to use has to be +stopped again — wasteful with a `TSAcquire` erase, and impossible to justify once the choice +depends on what the providers produce. + +`num_collections` is the **total** for the scan (`TriggerInfo.number_of_collections`), and +both tiers are told the period, so either can size a chunk or a buffer against it. + +### The tier is the type of the provider returned + +There is no declaration of which tier a logic implements and no override detection: a logic +returning a [](#StreamableDataProvider) is unbounded for this scan, one returning a +[](#PageableDataProvider) is bounded, and one may return either depending on what it is asked +for. Returning `None` means "not this scan" — how a finite buffer declines an unbounded one, +or a plugin that is switched off declines any. It is neither an error nor warned about, as +agreed in #1364: the point is that a detector may carry more logics than one scan uses. + +### `PageableDataProvider`, with readings derived from pages + +The bounded tier returns one provider type, which emits pages. Readings are derived from those +pages rather than being a separate code path: a step-scan prepare has `number_of_events == 1`, +so its page holds exactly one event, which extracts to a single reading. +`collections_per_event` appears in the datakey shape exactly as for a +`StreamableDataProvider`. + +### The period is resolved in `prepare()` + +`TriggerInfo.livetime` of 0 means "whatever is currently set on the detector", so the period +cannot simply be forwarded: an explicit `prepare(TriggerInfo(number_of_events=10))` would hand +the data logic a period of 0. Instead `DetectorTriggerLogic.default_trigger_info()` is extended +to return the current `livetime` and `deadtime`, and `prepare()` fills a zero `livetime` in +from it *after* preparing the trigger logic, which [](./0012-detector-rewrite.md) already +requires to come first, so the hardware holds the real values by then. Data logics therefore +never see a period of 0. + +### Streaming providers are reused, finite buffers are re-made + +Providers are reused while the scan they were made for is unchanged, so a step scan does not +reopen its file on every point. The reuse key is `collections_per_event` *and* the period, +because both are baked into a `StreamResource` when it is made — the shape from the first, the +chunking from the second — and areaDetector's `num_frames_chunks` can only be set before +`capture` starts. A period change therefore starts a new file. + +A finite buffer is re-made on every `prepare()` and `trigger()` regardless, because re-making +it is what re-arms it, and `kickoff()` never re-makes one: exactly the set of places a buffer +should and should not be re-armed. A re-armed buffer sitting at 0 alongside a monotonic HDF +writer at 15 would trip the `_all_the_same` reducer, so the two kinds are tracked separately. + +### A bounded provider's progress baseline is zero in `trigger()` + +`trigger()` takes a bounded provider's starting progress to be 0 rather than reading it back, +while `kickoff()` reads it live. + +It is tempting to derive this instead of stating it, since re-arming refreshes +`collections_written`, and a stats time series is erased by the `TSAcquire` write in its +`start` — so the refreshed read already returns zero. That only holds when the data logic +performs the erase. Where the erase belongs to the acquire control, as with the CTR-08's +`EraseStart`, it happens after that read, so the baseline would be the stale pre-erase value +and the detector would wait for twice the buffer's length and hang. Stating the rule keeps the +framework agnostic about which component erases. + +### Erase and start controls belong to the acquire logic + +Where one control both starts acquisition and clears the data buffer, it belongs to the +`DetectorAcquireLogic`, alongside the stop control and the idle status. Nothing needs to +declare the erase to the framework: a data logic's `start` always runs before +`start_acquiring()`, so the buffer is sized before the control fires, and the baseline rule +above means the detector does not care who erased it. + +The trigger/acquire/data split therefore survives these devices, with the three logics sharing +one IO device exactly as the areaDetector logics share a `driver`. The split is of concerns, +not of hardware. + +### Logic objects fill exactly one role + +Building a `DetectorLogic` raises if an object satisfies more than one of the three logic +roles, directing the author to pass separate objects. A device whose concerns all live on one +control is a natural candidate for a single combined object, and registration is a chain of +`isinstance` tests, so such an object would silently register as the first role it matched and +have the others dropped without a word. + +### A detector produces stream assets or event pages, never both + +The two bluesky protocols are mutually exclusive by design, to avoid a device producing blocks +of two different sizes, so a detector whose logics would produce both raises. How the +applicable verb is exposed, and the one case where the two may be carried together, are in +[](./0023-detector-is-a-flyable.md). + +Which kind a detector produces is decided per prepare, so a single detector class can be either +depending on its `__init__` arguments — one `AravisDetector`, configured for files or for pages +— which was the original requirement. That it can also change *between runs*, when a file +writer is switched off and a stats time series takes over, is documented rather than enforced: +a run's descriptor is emitted at its start, so switching part way through would leave +`describe_collect()` and the documents disagreeing, and a Device cannot see run boundaries. + +### Stats with a file writer go via NDAttributes + +An areaDetector with an HDF writer does not need the stats time series: `ADHDFDataLogic` +already pulls NDAttribute datasets into the file, which is areaDetector's own mechanism for +exactly this. So registering the scalar with `set_readable_format` remains right for a +detector that writes files and wants a stats value in a step scan, and a stats time series is +right for one that writes none. + +### One stats data logic for many arrays + +An `NDPluginStats` has one time series control (`TSNumPoints`, `TSAcquire`, `TSAcquireMode`) +shared across roughly twenty arrays, so one stats data logic covers *many* arrays with the +control embedded, rather than one logic per array with a shared control object. This follows +the existing grain — a provider already emits many datakeys, as `ADHDFDataLogic` does for the +main dataset plus its NDAttributes — and avoids the coordination problem of several logics +contending over one `TSAcquire`. Which statistics are wanted becomes a constructor argument. + +## Consequences + +`DetectorDataLogic` implementations must be rewritten onto `make_data_provider` and `start`, +and take the new `period` argument. This is a breaking change for out-of-tree data logics; the +library is in alpha and no shim is provided. Splitting them is usually mechanical — describe in +one, write in the other — but a logic that cannot describe its data without starting (in tree, +`OdinDataLogic`, whose frame shape is only readable once the file processor is writing) has to +do its writes in `make_data_provider` and leave `start` empty, which is safe only while such a +logic is never the one a detector drops. In-tree, `ADHDFDataLogic` and `PandaHDFDataLogic` can +compute chunk size and flush period from the rate, replacing the TODOs that stand in for it. + +`DetectorTriggerLogic.default_trigger_info()` implementations should return the current +`livetime` and `deadtime` alongside the frame count. Those that do not still work, but their +detectors fall back to reading the chunk size back from hardware. + +Detectors that cannot write files become expressible: a data logic returning a +`PageableDataProvider` gives them `collect_pages()` and the whole of `StandardDetector` +besides. This closes #1248 and #888. Fly scanning an areaDetector that carries an HDF writer +plus a finite-buffer data logic starts working, closing #1364. + +A data logic may decline a scan for reasons of its own: `ADHDFDataLogic` and +`StatsTimeSeriesDataLogic` take an `enable_callbacks` flag, defaulting to today's behaviour of +switching the plugin on, which when `False` makes them follow whatever the plugin is set to +and produce nothing when it is off. + +A device whose exposure count and buffer size are the same PV has it written twice: by the +trigger logic as `number_of_exposures`, and by the data logic as `num_collections`. The two +agree only when `exposures_per_collection` is 1. This is latent rather than live, because such +devices do not average exposures and so cannot ask for anything else, but a detector that could +do both would need its data logic to read the value back rather than set it. diff --git a/docs/explanations/decisions/0023-detector-is-a-flyable.md b/docs/explanations/decisions/0023-detector-is-a-flyable.md index 807e77f171..e292c811c2 100644 --- a/docs/explanations/decisions/0023-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0023-detector-is-a-flyable.md @@ -15,8 +15,9 @@ one lifecycle drift apart: the detector guarded its stages by checking whether a object was `None`, so calling a verb too early failed with `AttributeError: 'NoneType' object has no attribute ...` rather than saying what was wrong. -`StandardFlyable` requires a single [](#FlyableLogic) in a `logic` property (ADR 0022), and -a detector had three logic objects registered on the Device by `add_detector_logics()`. +`StandardFlyable` requires a single [](#FlyableLogic) in a `logic` property +([](./0021-single-logic-property.md)), and a detector had three logic objects registered on +the Device by `add_detector_logics()`. ## Decision @@ -25,7 +26,7 @@ a detector had three logic objects registered on the Device by `add_detector_log [](#DetectorLogic) holds the trigger, acquire and data logics, and is what `logic` returns. The orchestration moves with them: preparing the trigger logic, making the data providers, arming, and waiting for collections are all its methods now. It holds **no** back-reference -to its Device, so ADR 0017 continues to hold unqualified. +to its Device, so [](./0017-standard-movable.md) continues to hold unqualified. A detector builds it in `__init__` and returns it from `logic`: @@ -36,7 +37,6 @@ class MyDetector(StandardDetector): self._logic = DetectorLogic( MyTriggerLogic(self.driver), MyAcquireLogic(self.driver), - publish_collect_methods=self._publish_collect_methods, ) super().__init__(name=name) @@ -55,34 +55,47 @@ which `abstract_cached_property` makes a requirement checked when the Device is instantiated. Assigning `self.logic` from `__init__` cannot serve instead: `ABCMeta` checks `__abstractmethods__` in `__call__`, before `__init__` runs, so the class must override the property to be concrete. For an ad-hoc detector in a plan or a test, -[](#StandardDetector.with_logics) builds both the logic and a Device around it. +`DetectorLogic.with_device()` builds a Device around a logic, exactly as +`FlyableLogic.with_device()` does for any other flyer. -### Two things the logic cannot get from its own fields +### One thing the logic cannot get from its own fields -**The datakey namespace.** Data logics name their datakeys after the Device, whose name does -not exist until `init_devices` calls `set_name` on context exit. `StandardDetector.set_name` -pushes it into `DetectorLogic.datakey_prefix` — one string, set at the moment it exists, -rather than a reference to the whole Device. +Data logics name their datakeys after the Device, whose name does not exist until +`init_devices` calls `set_name` on context exit. `StandardDetector.set_name` pushes it into +`DetectorLogic.datakey_prefix` — one string, set at the moment it exists, rather than a +reference to the whole Device. -**Which collect verb to expose.** A detector exposes `collect_asset_docs` or `collect_pages` -depending on what its data logics produce, never both (ADR 0020). The Device passes -`_publish_collect_methods` into the constructor and the logic calls it from `on_prepare`, so -the binding happens on the Device without the logic holding one. It is a required argument -because the decision belongs to prepare, not to construction: which providers there are is -only known once the logics have been asked. +### Which collect verb to expose is the Device's business + +A detector produces stream assets or event pages depending on what its data logics make for +a given scan, never both (see [](./0022-bounded-data-logic-tier.md)). The bluesky bundler +decides which to ask for from the *static* type — `collect_pages()` is called only on a +device that is not a `WritesStreamAssets` — so a detector that declared both would have its +pages silently dropped, and one that declared neither could not fly at all. + +So `StandardDetector.prepare()` and `trigger()` bind whichever verb applies as a real +instance attribute, and remove the other. It is the Device that does this, from what +`DetectorLogic.data` holds after preparing; the logic reports what it made and is not told +where that goes, so it holds no back-reference. + +Choosing from the static type is a bundler limitation rather than a fact about detectors: a +device knows what it produced, and the bundler could decide from *that*, erroring only if +one device reported both ways in the same run. That change is a few lines in +`RunBundler.collect`, and with it both verbs become plain methods and the binding here goes +away. Until it is released, binding per prepare is what works against a released bluesky. ### Mixing bounded and unbounded is an error unless the bounded keys are shadowed -ADR 0020 made carrying both kinds of data logic an error, because the bundler treats stream -assets and event pages as mutually exclusive. Carrying both is still useful, though: the -same quantity can be written durably into a file *and* read from a plugin's buffer — an -areaDetector stats total, which the HDF writer pulls in as an NDAttribute. +[](./0022-bounded-data-logic-tier.md) made carrying both kinds of data logic an error, because +the bundler treats stream assets and event pages as mutually exclusive. Carrying both is still +useful, though: the same quantity can be written durably into a file *and* read from a plugin's +buffer — an areaDetector stats total, which the HDF writer pulls in as an NDAttribute. So the rule relaxes. Where **every** datakey a finite buffer would produce is also produced by the stream assets, the durable copy wins and the buffer sits the scan out, unarmed. Anything a buffer would produce that the file does not cover is still an error, naming the uncovered keys. Nothing has been started when that is decided, because -`make_data_provider` describes without acquiring (ADR 0020). +`make_data_provider` describes without acquiring (same ADR). ### A logic may report its own progress @@ -91,10 +104,11 @@ observing its readback against its setpoint. A detector's progress is "collectio out of collections requested", which is neither, so a detector inheriting the base `complete()` would have silently lost its progress reporting. -[](#WatchableFlyableLogic) adds `on_complete_updates(ctx)`, an async iterator of -`WatcherUpdate`; `complete()` yields from it when the logic implements it. The inherited -`on_complete` drains it, so callers that only want to block are unaffected. This is -symmetrical with the existing `MovableLogic` branch rather than a new mechanism. +So `FlyableLogic.on_complete` returns either an `Awaitable[None]` or an +`AsyncIterator[WatcherUpdate]`, and `complete()` looks at what it got back: an iterator is +yielded straight on to watchers, anything else is awaited. A logic that only blocks stays an +ordinary `async def`. A separate `WatchableFlyableLogic` base was tried first and dropped: +it made a second class to inherit for what is a property of one method's return. ### One kickoff per prepare diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index 839ee5242a..957ea2f0be 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -84,6 +84,6 @@ write a file *and* report a plugin scalar in a step scan. ```{seealso} [](./store-and-retrieve.md) for saving a set of formats alongside signal values and loading them back, and -[](../explanations/decisions/0021-runtime-readable-format.md) for why the registry is +[](../explanations/decisions/0020-runtime-readable-format.md) for why the registry is keyed by Device. ``` diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index a2600c42ec..e4139a7f1b 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -657,7 +657,7 @@ async def test_step_scan_keep_numimages( ) # Check we can prepare and change the exposure. Changing the exposure changes - # the frame period, which (per ADR 0020) determines the HDF chunk shape, so a + # the frame period, which (per ADR 0022) determines the HDF chunk shape, so a # new StreamResource is opened rather than the old file being reused. Here the # chunk shape is unchanged because this writer leaves flush_period unset and so # reads the chunk size back from the IOC, but the period change still starts a From 91d933a9ca6d10bb268fe985642b55bd4a09812b Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 11:42:14 +0000 Subject: [PATCH 49/67] refactor: type the stage hooks as awaitables, not statuses _stage_funcs/_unstage_funcs are gathered internally, so wrapping each one in an AsyncStatus only to reach for its .task is noise. Type them as Callable[[], Awaitable[None]] and let the registrants be plain coroutine functions. Type on_complete as returning a Coroutine rather than any Awaitable, which is what AsyncStatus wants, so the _awaited shim goes with it. Note on FlyableLogic.stop why it deliberately shares a name with MovableLogic.stop: a motor record is both moved and flown, and one stop serves both. Assisted-by: Claude Opus 5 --- src/ophyd_async/core/_flyable.py | 22 +++++++++++----------- src/ophyd_async/core/_readable.py | 3 --- src/ophyd_async/core/_standard_base.py | 12 ++++++------ 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index 8b79f4e4f5..7839778d82 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -1,9 +1,9 @@ import asyncio from abc import abstractmethod -from collections.abc import AsyncIterator, Awaitable +from collections.abc import AsyncIterator, Coroutine from enum import Enum from functools import cached_property -from typing import Generic, TypeVar, cast +from typing import Any, Generic, TypeVar, cast from bluesky.protocols import Flyable, Preparable from pydantic import Field @@ -61,7 +61,9 @@ async def on_kickoff(self, ctx: CtxT) -> CtxT: """ @abstractmethod - def on_complete(self, ctx: CtxT) -> Awaitable[None] | AsyncIterator[WatcherUpdate]: + def on_complete( + self, ctx: CtxT + ) -> Coroutine[Any, Any, None] | AsyncIterator[WatcherUpdate]: """Block until the fly scan is done. Write it as an `async def` to just block, or as an async generator @@ -75,6 +77,11 @@ async def stop(self) -> None: Called by `on_stage` and `on_unstage` by default; override those instead if stage and unstage need to differ. + + Deliberately the same name as `MovableLogic.stop`, so that a logic + implementing both -- a motor record's, which is moved *and* flown -- + writes one stop that serves as the move-stop verb and as fly-scan + teardown. The hardware action is the same in both cases. """ pass @@ -132,11 +139,6 @@ def _ramp_distance(self, acceleration_time: float) -> float: ) -async def _awaited(awaitable: Awaitable[None]) -> None: - # AsyncStatus takes a coroutine, while on_complete may return any awaitable - await awaitable - - class _FlyStage(Enum): """Lifecycle stage of a `StandardFlyable`, used to enforce call ordering.""" @@ -204,12 +206,10 @@ def _reset_fly_state(self) -> None: self._fly_ctx = cast(CtxT, None) self._fly_stage = _FlyStage.IDLE - @AsyncStatus.wrap async def _on_stage(self) -> None: await self.logic.on_stage() self._reset_fly_state() - @AsyncStatus.wrap async def _on_unstage(self) -> None: await self.logic.on_unstage() self._reset_fly_state() @@ -259,7 +259,7 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: logic.setpoint.get_value(), logic.get_units_precision(), ) - async with AsyncStatus(_awaited(completing)) as completed: + async with AsyncStatus(completing) as completed: async for current_position in observe_value( logic.readback, done_status=completed ): diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 138086a6c5..29bf0aba6a 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -12,7 +12,6 @@ from ._protocol import AsyncConfigurable, AsyncReadable, AsyncStageable from ._signal import SignalR, walk_devices from ._standard_base import _StandardBase -from ._status import AsyncStatus from ._utils import merge_gathered_dicts @@ -156,11 +155,9 @@ def __init__(self, *args, **kwargs) -> None: self._stage_funcs += (self._stage_readables,) self._unstage_funcs += (self._unstage_readables,) - @AsyncStatus.wrap async def _stage_readables(self) -> None: await asyncio.gather(*(sig.stage().task for sig in self._signals_to_stage())) - @AsyncStatus.wrap async def _unstage_readables(self) -> None: await asyncio.gather(*(sig.unstage().task for sig in self._signals_to_stage())) diff --git a/src/ophyd_async/core/_standard_base.py b/src/ophyd_async/core/_standard_base.py index 3c4d69a221..836f4b1b04 100644 --- a/src/ophyd_async/core/_standard_base.py +++ b/src/ophyd_async/core/_standard_base.py @@ -1,9 +1,9 @@ import asyncio -from collections.abc import Callable +from collections.abc import Awaitable, Callable from ._device import Device from ._protocol import AsyncStageable -from ._status import AsyncStatus, AsyncStatusBase +from ._status import AsyncStatus class _StandardBase(Device, AsyncStageable): @@ -17,13 +17,13 @@ class _StandardBase(Device, AsyncStageable): """ # Immutable defaults to avoid accidental sharing between instances - _stage_funcs: tuple[Callable[[], AsyncStatusBase], ...] = () - _unstage_funcs: tuple[Callable[[], AsyncStatusBase], ...] = () + _stage_funcs: tuple[Callable[[], Awaitable[None]], ...] = () + _unstage_funcs: tuple[Callable[[], Awaitable[None]], ...] = () @AsyncStatus.wrap async def stage(self) -> None: - await asyncio.gather(*(func().task for func in self._stage_funcs)) + await asyncio.gather(*(func() for func in self._stage_funcs)) @AsyncStatus.wrap async def unstage(self) -> None: - await asyncio.gather(*(func().task for func in self._unstage_funcs)) + await asyncio.gather(*(func() for func in self._unstage_funcs)) From 8c3e2b0adb9b321848f61d8e7231cc74dc57dbb4 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 11:42:16 +0000 Subject: [PATCH 50/67] test: drive the pmac trajectory abort through unstage() The two tests either patched the logic's stop() with an AsyncMock or called it directly. Fold them into one parametrized test that goes through the public unstage(), covering both the running and idle cases. Assisted-by: Claude Opus 5 --- .../epics/pmac/test_pmac_trajectory.py | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/tests/unit_tests/epics/pmac/test_pmac_trajectory.py b/tests/unit_tests/epics/pmac/test_pmac_trajectory.py index ec0b5ce857..2415d12bfe 100644 --- a/tests/unit_tests/epics/pmac/test_pmac_trajectory.py +++ b/tests/unit_tests/epics/pmac/test_pmac_trajectory.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock, call, patch +from unittest.mock import call, patch import numpy as np import pytest @@ -380,32 +380,26 @@ async def test_pmac_trajectory_stage(sim_motors: tuple[PmacIO, Motor, Motor]): ] -async def test_pmac_trajectory_unstage(sim_motors: tuple[PmacIO, Motor, Motor]): - pmac_io, _, _ = sim_motors - logic = PmacTrajectoryFlyableLogic(pmac_io) - logic.stop = AsyncMock() - flyer = logic.with_device() - await flyer.unstage() - logic.stop.assert_called_once() - - -async def test_trajectory_stop_if_running(sim_motors: tuple[PmacIO, Motor, Motor]): +@pytest.mark.parametrize( + "execute_state, aborts", + [(PmacExecuteState.DONE, False), (PmacExecuteState.EXECUTING, True)], +) +async def test_pmac_trajectory_unstage_aborts_only_if_running( + sim_motors: tuple[PmacIO, Motor, Motor], + execute_state: PmacExecuteState, + aborts: bool, +): pmac_io, _, _ = sim_motors - pmac_trajectory = PmacTrajectoryFlyableLogic(pmac_io) + flyer = PmacTrajectoryFlyableLogic(pmac_io).with_device() execute_mock = get_mock_execute(pmac_io.trajectory.abort_profile) + set_mock_value(pmac_io.trajectory.execute_state, execute_state) - # Method not called as no running trajectory - await pmac_trajectory.stop() - execute_mock.assert_not_awaited() - - # Mocking that trajectory is executing - set_mock_value( - pmac_trajectory.pmac.trajectory.execute_state, PmacExecuteState.EXECUTING - ) + await flyer.unstage() - # Method called as there is now a running trajectory - await pmac_trajectory.stop() - execute_mock.assert_awaited_once_with() + if aborts: + execute_mock.assert_awaited_once_with() + else: + execute_mock.assert_not_awaited() async def test_trajectory_raises_if_profile_status_not_in_good_state( From a71d6e015d8c796e8b0563711a93d9316a103b7e Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 11:42:47 +0000 Subject: [PATCH 51/67] refactor: stop DetectorLogic holding the logics it was given self.logics was never read after __init__ sorted the objects into their roles, so it was a second copy of state that could only go stale. Assisted-by: Claude Opus 5 --- src/ophyd_async/core/_detector.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 7998ac939f..2f701ee1f4 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -419,7 +419,6 @@ def __init__( self, *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, ) -> None: - self.logics = logics self.trigger_logic: DetectorTriggerLogic | None = None self.acquire_logic: DetectorAcquireLogic | None = None self.data_logics: tuple[DetectorDataLogic, ...] = () @@ -459,7 +458,7 @@ def __init__( raise RuntimeError("Detector already has acquire logic") self.acquire_logic = logic elif isinstance(logic, DetectorDataLogic): - self.data_logics = (*self.data_logics, logic) + self.data_logics += (logic,) else: raise TypeError(f"Unknown logic type: {type(logic)}") #: Whether the trigger logic can calculate a deadtime From 8f1ae6e60a3c4321a9b84f63a8a36c2f29ee045b Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 11:42:49 +0000 Subject: [PATCH 52/67] refactor: use the existing NDPluginBaseIOT for ADWriterFactory NDWriterPluginT was a second TypeVar with the same bound as the NDPluginBaseIOT already declared in _io.py, so it needed its own nitpick_ignore entry for nothing. Assisted-by: Claude Opus 5 --- docs/conf.py | 1 - src/ophyd_async/epics/adcore/_data_logic.py | 15 ++++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2cbd14a731..1c1effee5b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -196,7 +196,6 @@ def setup(app: application.Sphinx): # event_model publishes no intersphinx entry for this TypedDict "event_model.documents.PartialEventPage", # TypeVar for the plugin an ADWriterFactory builds a data logic around - "ophyd_async.epics.adcore._data_logic.NDWriterPluginT", # The context DetectorLogic threads prepare -> kickoff -> complete, private # because nothing outside StandardDetector constructs or inspects one "ophyd_async.core._detector._FlyCtx", diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index b405d10f92..d5e3f75c59 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -2,7 +2,7 @@ from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from pathlib import PureWindowsPath -from typing import Any, Generic, TypeVar +from typing import Any, Generic from xml.etree import ElementTree as ET import numpy as np @@ -30,6 +30,7 @@ NDArrayBaseIO, NDFileHDF5IO, NDPluginBaseIO, + NDPluginBaseIOT, NDPluginFileIO, NDStatsIO, plugin_is_enabled, @@ -37,10 +38,6 @@ from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType from ._stats_time_series import StatsTimeSeriesDataLogic -#: A writer plugin built by an `ADWriterFactory`: a file writer (HDF, TIFF, JPEG) -#: or a stats plugin whose time series is read as event pages. -NDWriterPluginT = TypeVar("NDWriterPluginT", bound=NDPluginBaseIO) - @dataclass class NDArrayDescription: @@ -349,7 +346,7 @@ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @dataclass -class ADWriterFactory(Generic[NDWriterPluginT]): +class ADWriterFactory(Generic[NDPluginBaseIOT]): """Factory that creates a writer plugin and its matching data logic. Construct using the classmethods `hdf`, `jpeg`, `tiff` or `stats`, then pass @@ -378,7 +375,7 @@ class ADWriterFactory(Generic[NDWriterPluginT]): that builds the data logic given the already-constructed writer. """ - writer_cls: type[NDWriterPluginT] + writer_cls: type[NDPluginBaseIOT] writer_suffix: str writer_name: str datakey_suffix: str @@ -386,7 +383,7 @@ class ADWriterFactory(Generic[NDWriterPluginT]): NDArrayDescription | Callable[[ADBaseIO], NDArrayDescription] | None ) data_logic_factory: Callable[ - [NDWriterPluginT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], + [NDPluginBaseIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], DetectorDataLogic, ] @@ -395,7 +392,7 @@ def __call__( prefix: str, driver: ADBaseIO, plugins: Sequence[NDPluginBaseIO], - ) -> tuple[NDWriterPluginT, DetectorDataLogic]: + ) -> tuple[NDPluginBaseIOT, DetectorDataLogic]: """Instantiate the writer plugin and build the data logic. :param prefix: EPICS PV prefix for the detector (same as `AreaDetector.prefix`). From f8d38af67eeae8ede944540da73e59eba72f1e22 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 11:55:34 +0000 Subject: [PATCH 53/67] refactor: thread a context from make_data_provider to start A DetectorDataLogic stashed what it worked out on itself, in a _to_start attribute, for start() to pick up. But the detector asks every logic what it would make and then starts only the ones it settles on, so a logic that is passed over is left holding a plan for a scan it never served. Give DetectorDataLogic a DataCtxT and have make_data_provider return the context alongside its provider, exactly as FlyableLogic threads its own between on_prepare, on_kickoff and on_complete. The detector hands it straight back to start(), so a logic keeps no state of its own. Two things ride on the same signature change: - flush_period moves from ADHDFDataLogic's __init__ to TriggerInfo, so the scan says how often it wants data rather than the device being built to a fixed rate. It sizes the HDF chunk as before, and now also drives PandaHDFDataLogic's flush period signal, which was hardcoded 0. - StatsTimeSeriesProvider becomes a concrete EventPageDataProvider in core: nothing in it was areaDetector-specific once the datakey dtype comes from the signal's datatype rather than a hardcoded StreamableDataProvider | PageableDataProvider | None: ... - async def start(self) -> None: ... + self, datakey_name: str, num_collections: int, + period: float, flush_period: float, + ) -> tuple[StreamableDataProvider | PageableDataProvider, DataCtxT] | None: ... + async def start(self, ctx: DataCtxT) -> None: ... ``` `make_data_provider` says what this logic *would* produce for this scan without starting @@ -81,6 +82,10 @@ anything; `start` does the writes that make it happen, and is called only for th the detector will use. So the detector asks every logic what it would make, decides which ones it wants, and starts only those. +What the first worked out and the second needs is threaded between them through an explicit +context object, the same way `FlyableLogic` threads its own, rather than stashed on the logic. +A logic that stored it would be left holding a plan for a scan it was never asked to serve. + One `prepare_*` per tier, doing both jobs, was tried first. It forces the detector to arm hardware to find out what it would get, so a provider it then decides not to use has to be stopped again — wasteful with a `TSAcquire` erase, and impossible to justify once the choice @@ -196,13 +201,14 @@ contending over one `TSAcquire`. Which statistics are wanted becomes a construct ## Consequences `DetectorDataLogic` implementations must be rewritten onto `make_data_provider` and `start`, -and take the new `period` argument. This is a breaking change for out-of-tree data logics; the +and take the new `period` and `flush_period` arguments. This is a breaking change for out-of-tree data logics; the library is in alpha and no shim is provided. Splitting them is usually mechanical — describe in one, write in the other — but a logic that cannot describe its data without starting (in tree, `OdinDataLogic`, whose frame shape is only readable once the file processor is writing) has to do its writes in `make_data_provider` and leave `start` empty, which is safe only while such a -logic is never the one a detector drops. In-tree, `ADHDFDataLogic` and `PandaHDFDataLogic` can -compute chunk size and flush period from the rate, replacing the TODOs that stand in for it. +logic is never the one a detector drops. In-tree, `ADHDFDataLogic` sizes its chunk from +`TriggerInfo.flush_period` and the rate, and `PandaHDFDataLogic` sets its flush period signal +from the same field, replacing the TODOs that stood in for both. `DetectorTriggerLogic.default_trigger_info()` implementations should return the current `livetime` and `deadtime` alongside the frame count. Those that do not still work, but their diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index b43f58021d..d628f53cfe 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -229,7 +229,7 @@ Then we have `BlobDataLogic`, a [](#DetectorDataLogic) subclass: Its job is to manage the file writing and data streaming: - `make_data_provider()` works out where the file will go and returns a [](#StreamableDataProvider) describing the datasets that will be written, without opening anything -- `start()` opens the file, so nothing is written until the detector has decided it wants this data +- `start()` opens the file at the path `make_data_provider()` handed back, so nothing is written until the detector has decided it wants this data - `get_hinted_fields()` returns the data keys that are interesting to plot - `stop()` tells the detector to close the file @@ -248,7 +248,7 @@ A [](#DetectorDataLogic) is not declared as one kind or another: the detector as A source that produces one value per event, like a plugin scalar, needs no data logic at all: a detector is a [](#StandardReadable), so register the signal with [](#StandardReadable.set_readable_format). -`make_data_provider` is given the total `num_collections` for the scan, 0 meaning unbounded, and the frame `period` (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0. It must not start anything: only the providers the detector settles on get their `start()` called. +`make_data_provider` is given the total `num_collections` for the scan, 0 meaning unbounded; the frame `period` (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0; and the `flush_period` the [](#TriggerInfo) asked for. It must not start anything: only the providers the detector settles on get their `start()` called. Whatever it worked out that `start()` will need comes back alongside the provider, and the detector hands it straight back — a logic keeps no state of its own between the two. A detector may carry several data logics, but must not end up producing both kinds at once — those are mutually exclusive document kinds, and `prepare()` raises if it would. diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index b378668cdd..2b3c19d7e6 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -11,6 +11,7 @@ soft_command, ) from ._data_providers import ( + EventPageDataProvider, PageableDataProvider, StreamableDataProvider, StreamResourceDataProvider, @@ -304,6 +305,7 @@ def __getattr__(name): "AutoMaxIncrementingPathProvider", "UUIDFilenameProvider", # Data Providers + "EventPageDataProvider", "PageableDataProvider", "StreamableDataProvider", "StreamResourceInfo", diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index f005e0dcb7..65ea079499 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -1,13 +1,15 @@ from abc import abstractmethod -from collections.abc import AsyncIterator, Sequence +from collections.abc import AsyncIterator, Mapping, Sequence from typing import Any +import numpy as np from bluesky.protocols import StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange from event_model.documents import PartialEventPage +from ._datatypes import Array1D from ._signal import SignalR, SignalW -from ._utils import ConfinedModel +from ._utils import ConfinedModel, error_if_none, gather_dict, get_dtype class StreamableDataProvider: @@ -71,6 +73,86 @@ def make_pages( """ +class EventPageDataProvider(PageableDataProvider): + """Emits a fixed-length array of collections as event pages. + + For a device that fills one array per field as it acquires, alongside an + array of per-collection timestamps and a signal counting how many + collections it has filled: an areaDetector stats time series, or a scaler's + MCA arrays. Each array is sliced into `collections_per_event`-length chunks, + one per event, and the last collection of an event supplies that event's + timestamp. + + :param data: datakey (already suffixed) to the array signal that backs it + :param collections_written_signal: counts the collections filled so far + :param timestamps: the acquisition time of each collection + """ + + def __init__( + self, + data: Mapping[str, SignalR[Array1D[Any]]], + collections_written_signal: SignalR[int], + timestamps: SignalR[Array1D[np.float64]], + ) -> None: + self.data = dict(data) + self.collections_written_signal = collections_written_signal + self.timestamps = timestamps + self.last_emitted = 0 + + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + return { + datakey: DataKey( + source=signal.source, + shape=[collections_per_event], + dtype="array", + dtype_numpy=get_dtype( + error_if_none(signal.datatype, f"{signal.source} has no datatype") + ).str, + ) + for datakey, signal in self.data.items() + } + + async def make_pages( + self, collections_written: int, collections_per_event: int + ) -> AsyncIterator[PartialEventPage]: + events = collections_written // collections_per_event + if events <= self.last_emitted: + return + new = range(self.last_emitted, events) + # Read every array and the per-collection timestamps in one parallel batch + read: dict[SignalR[Array1D[Any]], Array1D[Any]] = await gather_dict( + { + signal: signal.get_value() + for signal in (*self.data.values(), self.timestamps) + } + ) + stamps = read[self.timestamps] + # One timestamp per event: the acquisition time of that event's last + # collection. + event_times = [ + float(stamps[(event + 1) * collections_per_event - 1]) for event in new + ] + page: PartialEventPage = { + # TODO: numpy arrays are not in PartialEventPage's value type, but + # they round trip through the bundler fine; fix upstream in + # event-model rather than converting to lists here + "data": { # type: ignore[typeddict-item] + datakey: [ + read[signal][ + event * collections_per_event : (event + 1) + * collections_per_event + ] + for event in new + ] + for datakey, signal in self.data.items() + }, + "time": event_times, + "timestamps": dict.fromkeys(self.data, event_times), + } + self.last_emitted = events + yield page + + class StreamResourceInfo(ConfinedModel): """A description of a single StreamResource that should be emitted.""" diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 2f701ee1f4..b14cf61a78 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from enum import Enum from functools import cached_property -from typing import cast +from typing import Any, Generic, TypeVar, cast from bluesky.protocols import ( Collectable, @@ -104,6 +104,15 @@ class TriggerInfo(ConfinedModel): ) """What is the maximum timeout on waiting for an exposure""" + flush_period: float = Field(default=0.0, ge=0.0) + """How many seconds between flushes of the data being written. + + A data logic that buffers before it writes sizes that buffer from this and + the exposure period, so data appears at roughly this rate: an HDF chunk of + `flush_period / period` frames, say. Set it to the rate the plan collects at + so a fly scan sees each batch as it lands. 0 means leave whatever the + hardware is set to.""" + @computed_field @cached_property def number_of_collections(self) -> int: @@ -307,8 +316,12 @@ async def _get_collections_written( #: [](#DetectorDataLogic.make_data_provider). _DataProvider = StreamableDataProvider | PageableDataProvider +#: What a `DetectorDataLogic` works out in `make_data_provider` and needs back +#: in `start`. Use `None` for a logic whose `start` needs nothing. +DataCtxT = TypeVar("DataCtxT") + -class DetectorDataLogic: +class DetectorDataLogic(Generic[DataCtxT]): """Abstract base class for detector data logic and handling. An implementation describes the data it would produce in @@ -316,6 +329,13 @@ class DetectorDataLogic: The detector asks every data logic it has what it would make, decides which providers it will use, and starts only those. + What `make_data_provider` worked out and `start` needs is threaded between + them through an explicit context object rather than stored on the logic, as + `FlyableLogic` threads its own: the detector may make a provider and then + never start it, so a logic holding that state on itself would be left + carrying a plan that was abandoned. A logic whose `start` needs nothing from + `make_data_provider` uses `None` for the context. + Whether the data is streamed or paged is the type of the provider that `make_data_provider` returns, not a declaration on the logic, so one logic may produce either depending on the scan. @@ -330,15 +350,20 @@ class DetectorDataLogic: datakey_suffix: str = "" async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider | PageableDataProvider | None: + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider | PageableDataProvider, DataCtxT] | None: """Describe the data this logic would produce for this scan. This must not start anything acquiring: the detector may discard the provider without calling `start`, and only starts the ones it will use. Return a [](#StreamableDataProvider) for a source that can produce any number of collections, or a [](#PageableDataProvider) for one holding a - finite buffer sized to `num_collections`. + finite buffer sized to `num_collections`, together with the context to + hand back to `start`. Return `None` to sit this scan out, for instance a finite buffer asked for an unbounded number of collections, or a plugin that is switched @@ -350,14 +375,19 @@ async def make_data_provider( :param period: how long each collection takes, livetime + deadtime, so the provider can size its chunks or its buffer. 0 means "use whatever is currently set on the hardware". + :param flush_period: how many seconds between flushes of the data being + written, so a logic that buffers can size that buffer. 0 means "use + whatever is currently set on the hardware". """ raise NotImplementedError(self) - async def start(self) -> None: + async def start(self, ctx: DataCtxT) -> None: """Make the provider just returned by `make_data_provider` take data. Called only for the providers the detector will actually use, so this is where writes that arm hardware or open files belong. + + :param ctx: the context returned by `make_data_provider`. """ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @@ -369,6 +399,18 @@ async def stop(self) -> None: return None +@dataclass +class _Served: + """One data logic, the provider it made, and the context to start it with.""" + + logic: "DetectorDataLogic[Any]" + provider: _DataProvider + ctx: Any + + async def start(self) -> None: + await self.logic.start(self.ctx) + + @dataclass class _DetectorData: """The data providers a detector has prepared, and what they were made for. @@ -378,23 +420,31 @@ class _DetectorData: `complete()` has returned. Cleared by `stage()`/`unstage()`. """ - serving: Sequence[tuple[DetectorDataLogic, _DataProvider]] + serving: Sequence[_Served] #: What the providers were made for, and so when they can be reused collections_per_event: int period: float @property def streamable(self) -> list[StreamableDataProvider]: - return [dp for _, dp in self.serving if isinstance(dp, StreamableDataProvider)] + return [ + s.provider + for s in self.serving + if isinstance(s.provider, StreamableDataProvider) + ] @property def pageable(self) -> list[PageableDataProvider]: - return [dp for _, dp in self.serving if isinstance(dp, PageableDataProvider)] + return [ + s.provider + for s in self.serving + if isinstance(s.provider, PageableDataProvider) + ] @property def collectable(self) -> list[_DataProvider]: """Every provider, whether it collects as stream datums or as pages.""" - return [dp for _, dp in self.serving] + return [s.provider for s in self.serving] @dataclass @@ -417,11 +467,11 @@ class DetectorLogic(FlyableLogic[TriggerInfo, _FlyCtx]): def __init__( self, - *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic, + *logics: "DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic[Any]", ) -> None: self.trigger_logic: DetectorTriggerLogic | None = None self.acquire_logic: DetectorAcquireLogic | None = None - self.data_logics: tuple[DetectorDataLogic, ...] = () + self.data_logics: tuple[DetectorDataLogic[Any], ...] = () #: The trigger types the trigger logic implements self.supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} #: Prefix for the datakeys the data logics produce, set from `Device.name` @@ -491,7 +541,9 @@ def get_hinted_fields(self) -> Iterator[Sequence[str]]: produce. """ logics = ( - [dl for dl, _ in self.data.serving] if self.data else list(self.data_logics) + [s.logic for s in self.data.serving] + if self.data + else list(self.data_logics) ) for dl in logics: if fields := dl.get_hinted_fields(self._datakey_name(dl)): @@ -601,7 +653,7 @@ async def default_trigger_info(self) -> TriggerInfo: return await self.trigger_logic.default_trigger_info() return TriggerInfo() - def _datakey_name(self, dl: DetectorDataLogic) -> str: + def _datakey_name(self, dl: "DetectorDataLogic[Any]") -> str: return self.datakey_prefix + dl.datakey_suffix async def _prepare_trigger_logic(self, value: TriggerInfo) -> None: @@ -692,40 +744,42 @@ async def _prepare_data(self, trigger_info: TriggerInfo) -> _DetectorData: ) return self.data - async def _make_serving( - self, trigger_info: TriggerInfo - ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: + async def _make_serving(self, trigger_info: TriggerInfo) -> Sequence[_Served]: """Ask every data logic what it would make, and start the ones we use.""" # Stop what is running before anything new is made, since a logic that # cannot describe its data without opening its file does so here if self.data is not None: - await asyncio.gather(*(dl.stop() for dl, _ in self.data.serving)) + await asyncio.gather(*(s.logic.stop() for s in self.data.serving)) cpe = trigger_info.collections_per_event - period = trigger_info.livetime + trigger_info.deadtime made = await asyncio.gather( - *( - dl.make_data_provider( - self._datakey_name(dl), trigger_info.number_of_collections, period - ) - for dl in self.data_logics - ) + *(self._make_one(dl, trigger_info) for dl in self.data_logics) ) # A logic returns None to sit this scan out, e.g. a finite buffer asked # for an unbounded number of collections, or a plugin that is switched off - serving = [ - (dl, dp) - for dl, dp in zip(self.data_logics, made, strict=True) - if dp is not None - ] + serving = [s for s in made if s is not None] serving = await self._drop_shadowed(serving, cpe) - await asyncio.gather(*(dl.start() for dl, _ in serving)) + await asyncio.gather(*(s.start() for s in serving)) return serving + async def _make_one( + self, dl: "DetectorDataLogic[Any]", trigger_info: TriggerInfo + ) -> _Served | None: + made = await dl.make_data_provider( + self._datakey_name(dl), + trigger_info.number_of_collections, + trigger_info.livetime + trigger_info.deadtime, + trigger_info.flush_period, + ) + if made is None: + return None + provider, ctx = made + return _Served(logic=dl, provider=provider, ctx=ctx) + async def _drop_shadowed( self, - serving: Sequence[tuple[DetectorDataLogic, _DataProvider]], + serving: Sequence[_Served], collections_per_event: int, - ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: + ) -> Sequence[_Served]: """Drop finite buffers whose datakeys the stream assets already carry. A detector cannot produce both stream assets and event pages: the bundler @@ -736,19 +790,21 @@ async def _drop_shadowed( produce is also written to the file, the file wins and the buffer sits the scan out. Anything else is a conflict, so it raises. """ - pageable = [ - (dl, dp) for dl, dp in serving if isinstance(dp, PageableDataProvider) + pageable = [s for s in serving if isinstance(s.provider, PageableDataProvider)] + streamable = [ + s.provider + for s in serving + if isinstance(s.provider, StreamableDataProvider) ] - streamable = [dp for _, dp in serving if isinstance(dp, StreamableDataProvider)] if not (pageable and streamable): return serving stream_keys: set[str] = set() for dp in streamable: stream_keys |= set(await dp.make_datakeys(collections_per_event)) shadowed = [] - for dl, dp in pageable: + for s in pageable: unshadowed = ( - set(await dp.make_datakeys(collections_per_event)) - stream_keys + set(await s.provider.make_datakeys(collections_per_event)) - stream_keys ) if unshadowed: raise TypeError( @@ -756,28 +812,25 @@ async def _drop_shadowed( f"{sorted(unshadowed)} as event pages and the rest of its data " "as stream assets; these cannot be combined on one detector" ) - shadowed.append(dl) + shadowed.append(s.logic) # Nothing has been started yet, so a shadowed logic needs no stopping - return [(dl, dp) for dl, dp in serving if dl not in shadowed] + return [s for s in serving if s.logic not in shadowed] async def _rearm_bounded( self, previous: _DetectorData, trigger_info: TriggerInfo - ) -> Sequence[tuple[DetectorDataLogic, _DataProvider]]: + ) -> Sequence[_Served]: """Re-make and re-start the finite buffers, keeping everything else.""" - period = trigger_info.livetime + trigger_info.deadtime - serving: list[tuple[DetectorDataLogic, _DataProvider]] = [] - for dl, dp in previous.serving: - if isinstance(dp, PageableDataProvider): - await dl.stop() - rearmed = await dl.make_data_provider( - self._datakey_name(dl), trigger_info.number_of_collections, period - ) + serving: list[_Served] = [] + for s in previous.serving: + if isinstance(s.provider, PageableDataProvider): + await s.logic.stop() + rearmed = await self._make_one(s.logic, trigger_info) if rearmed is None: continue - await dl.start() - serving.append((dl, rearmed)) + await rearmed.start() + serving.append(rearmed) else: - serving.append((dl, dp)) + serving.append(s) return serving async def _wait_for_collections( diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index ba4f2e4b54..5614203953 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -5,6 +5,7 @@ from ._acquire_logic import ADAcquireLogic, ADContAcqAcquireLogic from ._data_logic import ( + ADHDFDataCtx, ADHDFDataLogic, ADMultipartDataLogic, ADWriterFactory, @@ -40,7 +41,7 @@ ndattributes_to_xml, ) from ._plan_stubs import setup_ndattributes, setup_ndstats_sum -from ._stats_time_series import StatsTimeSeriesDataLogic, StatsTimeSeriesProvider +from ._stats_time_series import StatsTimeSeriesDataLogic from ._trigger_logic import ( ADContAcqTriggerLogic, prepare_exposures, @@ -77,11 +78,11 @@ "ADContAcqAcquireLogic", # DataLogic "NDArrayDescription", + "ADHDFDataCtx", "ADHDFDataLogic", "ADMultipartDataLogic", "ADWriterFactory", "StatsTimeSeriesDataLogic", - "StatsTimeSeriesProvider", # Detector "AreaDetector", "ContAcqDetector", diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index d5e3f75c59..13cd087d8f 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Awaitable, Callable, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import PureWindowsPath from typing import Any, Generic from xml.etree import ElementTree as ET @@ -17,7 +17,6 @@ StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, - error_if_none, set_and_wait_for_value, ) from ophyd_async.epics.core import stop_busy_record @@ -150,7 +149,15 @@ async def prepare_file_paths( @dataclass -class ADHDFDataLogic(DetectorDataLogic): +class ADHDFDataCtx: + """What `ADHDFDataLogic.make_data_provider` worked out for `start` to write.""" + + path_info: PathInfo + frames_per_chunk: int + + +@dataclass +class ADHDFDataLogic(DetectorDataLogic[ADHDFDataCtx]): """Data logic for AreaDetector HDF5 writer plugin. :param array_description: Signals describing the NDArray shape and data type. @@ -169,44 +176,34 @@ class ADHDFDataLogic(DetectorDataLogic): plugins: Sequence[NDPluginBaseIO] = () datakey_suffix: str = "" hinted: bool = True - #: Target seconds between HDF flushes. When set, the chunk is sized from - #: this and the frame period so the file flushes at roughly this rate - #: (#1309). Left as None, the chunk size is read back from the IOC, the - #: behaviour before #1309. - flush_period: float | None = None #: Whether to switch the plugin on when starting. Left True, the plugin is #: enabled as part of starting, which is the behaviour beamlines rely on. #: Set False to follow whatever the plugin is set to instead: a disabled #: plugin then makes no provider, so another data logic can serve the scan. enable_callbacks: bool = True - #: What make_data_provider worked out for start to write - _to_start: "tuple[PathInfo, int | None] | None" = field( - default=None, init=False, repr=False - ) async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider | None: + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, ADHDFDataCtx] | None: # A file writer captures for as long as it is told to, so it does not # need to know how many collections the scan will take. - del num_collections if not self.enable_callbacks and not await plugin_is_enabled(self.writer): return None # Work out where to write path_info = self.path_provider(datakey_name) # Size the HDF chunk from the frame period and the target flush period: # e.g. 400 Hz frames (period 2.5 ms) with a 0.5 s flush period gives 200 - # frames per chunk, so the file is flushed at ~2 Hz (#1309). Only when a - # flush_period is configured and the period is known (non-zero); otherwise - # fall back to reading the chunk size back from the IOC, forcing a - # fresh-startup 0 to 1. Only a value we chose is written back, in start(). - if self.flush_period is not None and period > 0: - frames_per_chunk = to_write = max(1, round(self.flush_period / period)) + # frames per chunk, so the file is flushed at ~2 Hz (#1309). Only when + # both are known (non-zero); otherwise fall back to reading the chunk + # size back from the IOC, forcing a fresh-startup 0 to 1. + if flush_period > 0 and period > 0: + frames_per_chunk = max(1, round(flush_period / period)) else: - frames_per_chunk = await self.writer.num_frames_chunks.get_value() - to_write = None - if frames_per_chunk == 0: - frames_per_chunk = to_write = 1 + frames_per_chunk = await self.writer.num_frames_chunks.get_value() or 1 # Describe what we would write main_dataset = await get_ndarray_resource_info( array_description=self.array_description, @@ -230,21 +227,20 @@ async def make_data_provider( ) for name, (dtype_numpy, source) in ndattribute_dtype_sources.items() ] - self._to_start = (path_info, to_write) - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[main_dataset] + ndattribute_datasets, mimetype="application/x-hdf5", collections_written_signal=self.writer.num_captured, flush_signal=self.writer.flush_now, ) + return provider, ADHDFDataCtx(path_info, frames_per_chunk) - async def start(self) -> None: - path_info, frames_per_chunk = error_if_none( - self._to_start, "make_data_provider() has not been called" - ) - if frames_per_chunk is not None: - await self.writer.num_frames_chunks.set(frames_per_chunk) + async def start(self, ctx: ADHDFDataCtx) -> None: + path_info = ctx.path_info + # Write back the chunk size the provider was described with, so the file + # matches the chunk_shape in its stream resource either way + await self.writer.num_frames_chunks.set(ctx.frames_per_chunk) # Setup the HDF writer coros: list[Awaitable] = [ self.writer.chunk_size_auto.set(True), @@ -272,7 +268,7 @@ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @dataclass -class ADMultipartDataLogic(DetectorDataLogic): +class ADMultipartDataLogic(DetectorDataLogic[PathInfo]): """Data logic for multipart AreaDetector file writers (e.g. JPEG, TIFF). :param array_description: Signals describing the NDArray shape and data type. @@ -293,15 +289,15 @@ class ADMultipartDataLogic(DetectorDataLogic): datakey_suffix: str = "" hinted: bool = True - #: Where make_data_provider decided to write, for start to set up - _to_start: "PathInfo | None" = field(default=None, init=False, repr=False) - async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider: - # A multipart writer writes one file per frame, so there is no chunk to + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, PathInfo]: + # A multipart writer writes one file per frame, so there is nothing to # size from the period, and it writes for as long as it is told to. - del period, num_collections # Work out where to write path_info = self.path_provider(datakey_name) # Describe what we would write @@ -310,8 +306,7 @@ async def make_data_provider( data_key=datakey_name, parameters={"template": path_info.filename + "_{:06d}" + self.extension}, ) - self._to_start = path_info - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( # TODO: remove the type ignore after # https://github.com/bluesky/ophyd-async/issues/1186 uri=path_info.directory_uri, @@ -319,16 +314,14 @@ async def make_data_provider( mimetype=self.mimetype, collections_written_signal=self.writer.num_captured, ) + return provider, path_info - async def start(self) -> None: - path_info = error_if_none( - self._to_start, "make_data_provider() has not been called" - ) + async def start(self, ctx: PathInfo) -> None: # Setup the file writer await asyncio.gather( self.writer.lazy_open.set(True), prepare_file_paths( - path_info=path_info, + path_info=ctx, file_template="%s%s_%6.6d" + self.extension, writer=self.writer, ), @@ -429,7 +422,6 @@ def hdf( | Callable[[ADBaseIO], NDArrayDescription] | None = None, hinted: bool = True, - flush_period: float | None = None, enable_callbacks: bool = True, ) -> "ADWriterFactory[NDFileHDF5IO]": """Create a factory for an HDF5 file writer. @@ -447,10 +439,6 @@ def hdf( when the shape/type comes from a plugin rather than the main driver (e.g. an ROI plugin). :param hinted: Include the image in hints. Set False for BestEffortCallback. - :param flush_period: Target seconds between HDF flushes. When set, the - chunk is sized from this and the frame period so the file flushes at - roughly this rate (#1309). Left as ``None`` (the default), the chunk - size is read back from the IOC as before. :param enable_callbacks: Whether to switch the plugin on when starting, defaults to ``True``. Set ``False`` to follow whatever the plugin is set to instead: a disabled plugin then writes nothing and another @@ -470,7 +458,6 @@ def hdf( plugins=list(plugins), datakey_suffix=datakey_suffix, hinted=hinted, - flush_period=flush_period, enable_callbacks=enable_callbacks, ), ) diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index dc90c3cdc2..4e8a4c1cb1 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -1,104 +1,22 @@ import asyncio -from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence +from collections.abc import Awaitable, Sequence from dataclasses import dataclass, field import numpy as np -from event_model import DataKey -from event_model.documents import PartialEventPage from ophyd_async.core import ( Array1D, DetectorDataLogic, EnableDisable, - PageableDataProvider, + EventPageDataProvider, SignalR, - error_if_none, - gather_dict, ) from ._io import NDStatsIO, NDStatsTSAcquireMode, plugin_is_enabled -class StatsTimeSeriesProvider(PageableDataProvider): - """Emits an `NDPluginStats` time series as event pages. - - The plugin holds one fixed-length array per statistic, filled by NDArray - callbacks as the detector acquires. Progress is read from - ``ts_current_point``, and each configured array is sliced into - ``collections_per_event``-length chunks, one per event. The plugin's - ``ts_timestamp`` array gives the acquisition time of each point, so the last - point of an event supplies that event's timestamp. - - :param arrays: datakey (already suffixed) to the array signal that backs it - :param current_point: the plugin's ``ts_current_point`` progress signal - :param timestamps: the plugin's ``ts_timestamp`` per-point time signal - """ - - def __init__( - self, - arrays: Mapping[str, SignalR[Array1D[np.float64]]], - current_point: SignalR[int], - timestamps: SignalR[Array1D[np.float64]], - ) -> None: - self.arrays = dict(arrays) - self.collections_written_signal = current_point - self.timestamps = timestamps - self.last_emitted = 0 - - async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: - return { - datakey: DataKey( - source=signal.source, - shape=[collections_per_event], - dtype="array", - dtype_numpy=" AsyncIterator[PartialEventPage]: - events = collections_written // collections_per_event - if events <= self.last_emitted: - return - new = range(self.last_emitted, events) - # Read every array and the per-point timestamps in one parallel batch. - read: dict[ - SignalR[Array1D[np.float64]], Array1D[np.float64] - ] = await gather_dict( - { - signal: signal.get_value() - for signal in (*self.arrays.values(), self.timestamps) - } - ) - stamps = read[self.timestamps] - # One timestamp per event: the acquisition time of that event's last point. - event_times = [ - float(stamps[(event + 1) * collections_per_event - 1]) for event in new - ] - page: PartialEventPage = { - "data": { - datakey: [ - list( - read[signal][ - event * collections_per_event : (event + 1) - * collections_per_event - ] - ) - for event in new - ] - for datakey, signal in self.arrays.items() - }, - "time": event_times, - "timestamps": dict.fromkeys(self.arrays, event_times), - } - self.last_emitted = events - yield page - - @dataclass -class StatsTimeSeriesDataLogic(DetectorDataLogic): +class StatsTimeSeriesDataLogic(DetectorDataLogic[int]): """Bounded data logic for an `NDPluginStats` time series. For detectors that write no file: the stats plugin holds each statistic in a @@ -127,18 +45,19 @@ class StatsTimeSeriesDataLogic(DetectorDataLogic): #: enabled as part of starting. Set False to follow whatever the plugin is #: set to instead: a disabled plugin then makes no provider. enable_callbacks: bool = True - #: What make_data_provider sized the buffer for, for start to write - _num_collections: int | None = field(default=None, init=False, repr=False) def _arrays(self, datakey_name: str) -> dict[str, SignalR[Array1D[np.float64]]]: signals = self.stat_signals or [("", self.stats.ts_total)] return {datakey_name + suffix: signal for suffix, signal in signals} async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> PageableDataProvider | None: - # The buffer is sized by count, not rate, so the period is unused. - del period + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[EventPageDataProvider, int] | None: + # The buffer is sized by count, not rate, so neither period is used. if num_collections == 0: # A finite buffer cannot serve an unbounded scan return None @@ -146,21 +65,19 @@ async def make_data_provider( return None # The datakeys are known from the arrays we were configured with, so # nothing has to be armed to describe them - self._num_collections = num_collections - return StatsTimeSeriesProvider( + provider = EventPageDataProvider( self._arrays(datakey_name), self.stats.ts_current_point, self.stats.ts_timestamp, ) + return provider, num_collections - async def start(self) -> None: - num_collections = error_if_none( - self._num_collections, "make_data_provider() has not been called" - ) - # Size the buffer and put it in fixed-length mode before arming it. + async def start(self, ctx: int) -> None: + # Size the buffer to the collection count make_data_provider was given, + # and put it in fixed-length mode, before arming it. coros: list[Awaitable] = [ self.stats.compute_statistics.set(True), - self.stats.ts_num_points.set(num_collections), + self.stats.ts_num_points.set(ctx), self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), ] if self.enable_callbacks: diff --git a/src/ophyd_async/fastcs/odin/_data_logic.py b/src/ophyd_async/fastcs/odin/_data_logic.py index dcf0dc0ff3..c25b7134a1 100644 --- a/src/ophyd_async/fastcs/odin/_data_logic.py +++ b/src/ophyd_async/fastcs/odin/_data_logic.py @@ -18,7 +18,7 @@ from ._io import OdinIO -class OdinDataLogic(DetectorDataLogic): +class OdinDataLogic(DetectorDataLogic[None]): def __init__( self, path_provider: PathProvider, @@ -32,11 +32,14 @@ def __init__( self.pixel_mask = pixel_mask async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider: + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, None]: # Odin sizes its own chunks and writes for as long as it is told to, so - # neither the frame period nor the count is used here yet. - del period, num_collections + # neither the periods nor the count are used here yet. # Unlike other data logics this one cannot describe its data without # starting: the frame shape is only readable from the file processor # once it is writing. So it does its own writes here and has nothing @@ -83,13 +86,14 @@ async def make_data_provider( dtype_numpy=np.dtype(datatype).str, parameters={"dataset": "/data"}, ) - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( # Should be _vds instead of _000001, see https://github.com/bluesky/ophyd-async/issues/1272 uri=f"{path_info.directory_uri}{filename}_000001.h5", resources=[resource], mimetype="application/x-hdf5", collections_written_signal=self.odin.fp.frames_written, ) + return provider, None async def stop(self) -> None: await asyncio.gather( diff --git a/src/ophyd_async/fastcs/panda/_data_logic.py b/src/ophyd_async/fastcs/panda/_data_logic.py index 0a5e674e1a..ac304a4aa4 100644 --- a/src/ophyd_async/fastcs/panda/_data_logic.py +++ b/src/ophyd_async/fastcs/panda/_data_logic.py @@ -1,5 +1,6 @@ import asyncio import logging +from dataclasses import dataclass import numpy as np @@ -10,7 +11,6 @@ StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, - error_if_none, ) from ._block import DataBlock, PandaCaptureMode @@ -18,7 +18,15 @@ logger = logging.getLogger("ophyd_async") -class PandaHDFDataLogic(DetectorDataLogic): +@dataclass +class PandaHDFDataCtx: + """What `PandaHDFDataLogic.make_data_provider` worked out for `start`.""" + + path_info: PathInfo + flush_period: float + + +class PandaHDFDataLogic(DetectorDataLogic[PandaHDFDataCtx]): def __init__( self, path_provider: PathProvider, @@ -26,16 +34,17 @@ def __init__( ): self.path_provider = path_provider self.data_block = data_block - # Where make_data_provider decided to write, for start to set up - self._to_start: PathInfo | None = None async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider: - # TODO: derive the PandA flush period / chunk size from `period` once the - # IOC exposes the chunk-size signal (see the chunk_shape TODO below). The - # PandA captures forever, so the number of collections is not needed. - del period, num_collections + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, PandaHDFDataCtx]: + # TODO: size the chunk from `period` once the IOC exposes the chunk-size + # signal (see the chunk_shape TODO below). The PandA captures forever, so + # the number of collections is not needed. # Work out where to write path_info = self.path_provider(datakey_name) # Load data from the datasets PV on the panda, which tells us which @@ -60,24 +69,24 @@ async def make_data_provider( ) for dataset_name in capture_table.name ] - self._to_start = path_info - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=resources, mimetype="application/x-hdf5", collections_written_signal=self.data_block.num_captured, ) + return provider, PandaHDFDataCtx(path_info, flush_period) - async def start(self) -> None: - path_info = error_if_none( - self._to_start, "make_data_provider() has not been called" - ) + async def start(self, ctx: PandaHDFDataCtx) -> None: + path_info = ctx.path_info # Set create dir depth first to guarantee that callback when setting # directory path has correct value await self.data_block.create_directory.set(path_info.create_dir_depth) - # Setup the HDF writer + # Setup the HDF writer. The PandA flushes on a timer rather than by + # chunk size, so it takes the flush period directly; 0 means it flushes + # only when its buffer fills. await asyncio.gather( - self.data_block.flush_period.set(0), + self.data_block.flush_period.set(ctx.flush_period), self.data_block.hdf_directory.set(str(path_info.directory_path)), self.data_block.hdf_file_name.set( f"{path_info.filename}.h5", diff --git a/src/ophyd_async/sim/_blob_data_logic.py b/src/ophyd_async/sim/_blob_data_logic.py index d660c0248d..f927126e49 100644 --- a/src/ophyd_async/sim/_blob_data_logic.py +++ b/src/ophyd_async/sim/_blob_data_logic.py @@ -9,7 +9,6 @@ StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, - error_if_none, ) from ._pattern_generator import DATA_PATH, SUM_PATH, PatternGenerator @@ -18,7 +17,7 @@ HEIGHT = 240 -class BlobDataLogic(DetectorDataLogic): +class BlobDataLogic(DetectorDataLogic[PurePath]): def __init__( self, path_provider: PathProvider, @@ -26,18 +25,19 @@ def __init__( ): self.path_provider = path_provider self.pattern_generator = pattern_generator - # Where make_data_provider decided to write, for start to open - self._to_open: PurePath | None = None async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> StreamableDataProvider: + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, PurePath]: # The sim blob writer uses a fixed chunk shape and writes for as long as - # it is told to, so neither the period nor the count is needed. - del period, num_collections + # it is told to, so neither the periods nor the count are needed. # Work out where to write path_info = self.path_provider(datakey_name) - self._to_open = path_info.directory_path / f"{path_info.filename}.h5" + write_path = path_info.directory_path / f"{path_info.filename}.h5" # Describe what we would write data_resource = StreamResourceInfo( data_key=datakey_name, @@ -57,18 +57,16 @@ async def make_data_provider( dtype_numpy=np.dtype(np.int64).str, parameters={"dataset": SUM_PATH}, ) - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[data_resource, sum_resource], mimetype="application/x-hdf5", collections_written_signal=self.pattern_generator.images_written, ) + return provider, write_path - async def start(self) -> None: - write_path = error_if_none( - self._to_open, "make_data_provider() has not been called" - ) - self.pattern_generator.open_file(write_path, WIDTH, HEIGHT) + async def start(self, ctx: PurePath) -> None: + self.pattern_generator.open_file(ctx, WIDTH, HEIGHT) async def stop(self) -> None: self.pattern_generator.close_file() diff --git a/tests/unit_tests/core/test_data_providers.py b/tests/unit_tests/core/test_data_providers.py new file mode 100644 index 0000000000..6072bf45e5 --- /dev/null +++ b/tests/unit_tests/core/test_data_providers.py @@ -0,0 +1,72 @@ +import numpy as np +import pytest + +from ophyd_async.core import ( + Array1D, + EventPageDataProvider, + soft_signal_r_and_setter, +) + + +@pytest.fixture +def provider() -> EventPageDataProvider: + """A 5-collection buffer, filled, with one timestamp per collection.""" + data, _ = soft_signal_r_and_setter( + Array1D[np.float64], np.array([10.0, 11.0, 12.0, 13.0, 14.0]) + ) + timestamps, _ = soft_signal_r_and_setter( + Array1D[np.float64], np.array([100.0, 101.0, 102.0, 103.0, 104.0]) + ) + written, _ = soft_signal_r_and_setter(int, 5) + return EventPageDataProvider({"det-stats": data}, written, timestamps) + + +@pytest.mark.parametrize( + "collections_per_event,expected_data,expected_times", + [ + # step scan: one event holds the whole 5-collection buffer, timed by its + # last collection + (5, [[10.0, 11.0, 12.0, 13.0, 14.0]], [104.0]), + # fly scan: five events of one collection each, timed one by one + ( + 1, + [[10.0], [11.0], [12.0], [13.0], [14.0]], + [100.0, 101.0, 102.0, 103.0, 104.0], + ), + ], +) +async def test_slices_the_arrays_into_events( + provider: EventPageDataProvider, + collections_per_event: int, + expected_data: list[list[float]], + expected_times: list[float], +): + pages = [ + page + async for page in provider.make_pages( + collections_written=5, collections_per_event=collections_per_event + ) + ] + (page,) = pages + np.testing.assert_array_equal(page["data"]["det-stats"], expected_data) + assert page["time"] == expected_times + + +async def test_datakeys_take_dtype_from_the_signal(provider: EventPageDataProvider): + datakeys = await provider.make_datakeys(5) + assert list(datakeys) == ["det-stats"] + assert datakeys["det-stats"]["shape"] == [5] + assert datakeys["det-stats"]["dtype"] == "array" + assert datakeys["det-stats"]["dtype_numpy"] == " StreamableDataProvider: + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, None]: resource = StreamResourceInfo( data_key=datakey_name, shape=(10, 15), @@ -158,7 +162,7 @@ async def make_data_provider( mimetype="application/x-hdf5", collections_written_signal=self.collections_written, ) - return provider + return provider, None async def stop(self) -> None: self.stop_count += 1 @@ -224,7 +228,7 @@ async def make_pages(self, collections_written: int, collections_per_event: int) yield page -class BoundedOnlyDataLogic(DetectorDataLogic): +class BoundedOnlyDataLogic(DetectorDataLogic[tuple[int, float]]): """Produces bounded data held in a finite buffer, sized when it is armed.""" def __init__(self, datakey_suffix: str = ""): @@ -232,20 +236,22 @@ def __init__(self, datakey_suffix: str = ""): self.collections_written = soft_signal_rw(int) self.prepare_calls: list[tuple[int, float]] = [] self.stop_count = 0 - self._to_start: tuple[int, float] | None = None async def make_data_provider( - self, datakey_name: str, num_collections: int, period: float - ) -> PageableDataProvider | None: + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[PageableDataProvider, tuple[int, float]] | None: if num_collections == 0: # A finite buffer cannot serve an unbounded scan return None - self._to_start = (num_collections, period) - return MockPageableProvider(datakey_name, self.collections_written) + provider = MockPageableProvider(datakey_name, self.collections_written) + return provider, (num_collections, period) - async def start(self) -> None: - assert self._to_start is not None - self.prepare_calls.append(self._to_start) + async def start(self, ctx: tuple[int, float]) -> None: + self.prepare_calls.append(ctx) # A real buffer clears its progress counter when armed await self.collections_written.set(0) @@ -1235,13 +1241,13 @@ async def test_trigger_logic_not_implemented_errors(): async def test_data_logic_not_implemented_errors(): """Test NotImplementedError for unimplemented DetectorDataLogic methods.""" - logic = DetectorDataLogic() + logic = DetectorDataLogic[None]() with pytest.raises(NotImplementedError): - await logic.make_data_provider("test", 1, 0.1) + await logic.make_data_provider("test", 1, 0.1, 0.0) # start() and stop() should not raise (they have default implementations) - await logic.start() + await logic.start(None) await logic.stop() diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 351f9c167e..34bccecd40 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -52,12 +52,14 @@ async def test_make_data_provider_does_not_write( writer=writer, ) - provider = await logic.make_data_provider("det", num_collections=5, period=0.1) - assert provider is not None + made = await logic.make_data_provider( + "det", num_collections=5, period=0.1, flush_period=0.0 + ) + assert made is not None assert await writer.capture.get_value() is False assert await writer.file_name.get_value() == "" - await logic.start() + await logic.start(made[1]) assert await writer.capture.get_value() is True assert await writer.file_name.get_value() != "" @@ -95,11 +97,13 @@ async def test_hdf_follows_the_plugin_when_not_enabling_it( enable_callbacks=enable_callbacks, ) - provider = await logic.make_data_provider("det", num_collections=5, period=0.1) - assert (provider is not None) is makes_provider + made = await logic.make_data_provider( + "det", num_collections=5, period=0.1, flush_period=0.0 + ) + assert (made is not None) is makes_provider - if makes_provider: - await logic.start() + if made is not None: + await logic.start(made[1]) expected = EnableDisable.ENABLE if enable_callbacks else plugin_enabled assert await writer.enable_callbacks.get_value() is expected @@ -214,11 +218,11 @@ async def test_hdf_chunk_sized_from_flush_period( deadtime: float, expected_frames_per_chunk: int, ): - """A configured flush_period sizes the HDF chunk from the frame period.""" + """A TriggerInfo flush_period sizes the HDF chunk from the frame period.""" async with init_devices(mock=True): det = adsimdetector.SimDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider, flush_period=flush_period), + adcore.ADWriterFactory.hdf(static_path_provider), ) set_mock_value(det.driver.array_size_x, 1024) set_mock_value(det.driver.array_size_y, 768) @@ -226,7 +230,12 @@ async def test_hdf_chunk_sized_from_flush_period( writer = det.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) await det.prepare( - TriggerInfo(livetime=livetime, deadtime=deadtime, number_of_events=3) + TriggerInfo( + livetime=livetime, + deadtime=deadtime, + number_of_events=3, + flush_period=flush_period, + ) ) assert await writer.num_frames_chunks.get_value() == expected_frames_per_chunk (sr, *_) = [doc async for doc in det.collect_asset_docs(3)] diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index b99a277bc0..c1555dde3e 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -11,6 +11,7 @@ DetectorLogic, DetectorTriggerLogic, EnableDisable, + EventPageDataProvider, StandardDetector, TriggerInfo, init_devices, @@ -20,7 +21,6 @@ from ophyd_async.epics.adcore import ( NDStatsTSAcquireMode, StatsTimeSeriesDataLogic, - StatsTimeSeriesProvider, ) @@ -33,14 +33,16 @@ async def stats() -> adcore.NDStatsIO: async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): logic = StatsTimeSeriesDataLogic(stats) - provider = await logic.make_data_provider( - "det-stats", num_collections=5, period=0.1 + made = await logic.make_data_provider( + "det-stats", num_collections=5, period=0.1, flush_period=0.0 ) - assert isinstance(provider, StatsTimeSeriesProvider) + assert made is not None + provider, ctx = made + assert isinstance(provider, EventPageDataProvider) # Describing the buffer does not arm it assert await stats.ts_num_points.get_value() == 0 - await logic.start() + await logic.start(ctx) assert await stats.ts_num_points.get_value() == 5 assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH # ts_acquire=1 arms and clears the buffer @@ -55,7 +57,12 @@ async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): async def test_unbounded_scan_makes_no_provider(stats: adcore.NDStatsIO): """A finite buffer cannot serve an unbounded scan, so it sits it out.""" logic = StatsTimeSeriesDataLogic(stats) - assert await logic.make_data_provider("det", num_collections=0, period=0.1) is None + assert ( + await logic.make_data_provider( + "det", num_collections=0, period=0.1, flush_period=0.0 + ) + is None + ) @pytest.mark.parametrize( @@ -78,11 +85,13 @@ async def test_follows_the_plugin_when_not_enabling_it( set_mock_value(stats.enable_callbacks, plugin_enabled) logic = StatsTimeSeriesDataLogic(stats, enable_callbacks=enable_callbacks) - provider = await logic.make_data_provider("det", num_collections=5, period=0.1) - assert (provider is not None) is makes_provider + made = await logic.make_data_provider( + "det", num_collections=5, period=0.1, flush_period=0.0 + ) + assert (made is not None) is makes_provider - if makes_provider: - await logic.start() + if made is not None: + await logic.start(made[1]) # Only the default reaches out and switches the plugin on expected = EnableDisable.ENABLE if enable_callbacks else plugin_enabled assert await stats.enable_callbacks.get_value() is expected @@ -95,44 +104,6 @@ async def test_stop_stops_the_time_series(stats: adcore.NDStatsIO): assert await stats.ts_acquire.get_value() is False -@pytest.mark.parametrize( - "collections_per_event,expected_data,expected_times", - [ - # step scan: one event holds the whole 5-point buffer, timed by its - # last point - (5, [[10.0, 11.0, 12.0, 13.0, 14.0]], [104.0]), - # fly scan: five events of one point each, timed point by point - ( - 1, - [[10.0], [11.0], [12.0], [13.0], [14.0]], - [100.0, 101.0, 102.0, 103.0, 104.0], - ), - ], -) -async def test_provider_slices_array_into_events( - stats: adcore.NDStatsIO, - collections_per_event: int, - expected_data: list[list[float]], - expected_times: list[float], -): - set_mock_value(stats.ts_total, np.array([10.0, 11.0, 12.0, 13.0, 14.0])) - set_mock_value(stats.ts_timestamp, np.array([100.0, 101.0, 102.0, 103.0, 104.0])) - set_mock_value(stats.ts_current_point, 5) - provider = StatsTimeSeriesProvider( - {"det-stats": stats.ts_total}, stats.ts_current_point, stats.ts_timestamp - ) - - pages = [ - page - async for page in provider.make_pages( - collections_written=5, collections_per_event=collections_per_event - ) - ] - (page,) = pages - assert page["data"]["det-stats"] == expected_data - assert page["time"] == expected_times - - class _JustInternal(DetectorTriggerLogic): async def prepare_internal(self, num: int, livetime: float, deadtime: float): ... @@ -192,7 +163,7 @@ def plan(): assert descriptor["data_keys"]["det"]["shape"] == [4] assert "external" not in descriptor["data_keys"]["det"] (event,) = docs["event"] - assert event["data"]["det"] == [5.0, 6.0, 7.0, 8.0] + np.testing.assert_array_equal(event["data"]["det"], [5.0, 6.0, 7.0, 8.0]) assert event["timestamps"]["det"] == 203.0 @@ -214,7 +185,7 @@ def plan(): RE(plan(), lambda name, doc: docs[name].append(doc)) (page,) = docs["event_page"] - assert page["data"]["det"] == [[5.0], [6.0], [7.0], [8.0]] + np.testing.assert_array_equal(page["data"]["det"], [[5.0], [6.0], [7.0], [8.0]]) assert docs["stop"][0]["num_events"] == {"primary": 4} @@ -226,6 +197,6 @@ async def test_step_scan_read_derives_single_reading(stats_detector: StandardDet await det.prepare(TriggerInfo(collections_per_event=4)) await det.trigger() reading = await det.read() - assert reading["det"]["value"] == [5.0, 6.0, 7.0, 8.0] + np.testing.assert_array_equal(reading["det"]["value"], [5.0, 6.0, 7.0, 8.0]) assert reading["det"]["timestamp"] == 203.0 await det.unstage() From 4bbd68a5665a2c615bdf4dad5eab076c9097e317 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 14:16:03 +0000 Subject: [PATCH 54/67] refactor: gather each readable verb from a tuple, as stage() does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit read(), describe() and their configuration counterparts each derived their callables through a _funcs_for(verb) switch, and a subclass with data of its own contributed by overriding _extra_funcs_for(verb) — one hook serving four verbs, which every caller then had to filter. Give each verb its own tuple of contributors, exactly as stage() already gathers _stage_funcs. StandardReadable puts the registry's contribution in each, as _formatted_read, _formatted_describe and so on, which hold the format table their verb cares about and nothing else. StandardDetector appends what its data logics produce rather than overriding a hook. The _Verb enum, _funcs_for, both _extra_* hooks and _UncachedRead all go; everything in a tuple runs in parallel. add_readables is deprecated in favour of set_readable_format. Since ADR 0020 gave it "set" semantics it has not been adding anything, and its name says otherwise. Assisted-by: Claude Opus 5 --- .../decisions/0020-runtime-readable-format.md | 18 +- src/ophyd_async/core/_detector.py | 60 +++--- src/ophyd_async/core/_readable.py | 202 +++++++++--------- tests/unit_tests/core/test_readable.py | 49 +++-- .../unit_tests/core/test_standard_detector.py | 12 +- 5 files changed, 175 insertions(+), 166 deletions(-) diff --git a/docs/explanations/decisions/0020-runtime-readable-format.md b/docs/explanations/decisions/0020-runtime-readable-format.md index 430d8476fb..c080a17fe1 100644 --- a/docs/explanations/decisions/0020-runtime-readable-format.md +++ b/docs/explanations/decisions/0020-runtime-readable-format.md @@ -137,13 +137,13 @@ A detector's step-scan signals are registered exactly like any other Device's, a `read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. `add_config_signals` becomes a deprecated wrapper. -The detector does not *reimplement* any of those verbs to achieve it. `StandardReadable` -grows two hooks, `_extra_funcs_for(verb)` and `_extra_hint_sources()`, that contribute -from somewhere other than the registry, and `StandardDetector` overrides those to add what -its data logics produce. Likewise it registers its stage and unstage work into -`_stage_funcs`/`_unstage_funcs`, the way `StandardFlyable` already does, rather than -replacing `stage()`. Overriding the verbs meant re-deriving the registered children's -contributions by hand in each one, which is exactly what would drift. +The detector does not *reimplement* any of those verbs to achieve it. Each verb gathers a +tuple of callables — `_read_funcs`, `_describe_funcs` and friends — exactly as `stage()` +gathers `_stage_funcs`. `StandardReadable` puts the registry's contribution in each tuple, +and `StandardDetector` appends what its data logics produce, the way `StandardFlyable` +already appends to `_stage_funcs` rather than replacing `stage()`. Overriding the verbs +meant re-deriving the registered children's contributions by hand in each one, which is +exactly what would drift. Everything in a tuple runs in parallel. One consequence: a detector with no hinted fields now reports `hints` as `{}` rather than `{"fields": []}`, because it no longer special cases what `StandardReadable` already does. @@ -247,7 +247,9 @@ values, so a failed value write leaves the readable registry untouched. - A Device can be switched between techniques at runtime, and the switch can be saved and restored, closing #1394. - `add_readables` keeps working unchanged for all 85 in-tree call sites and for downstream - Devices, but gains "set" semantics. + Devices, but gains "set" semantics. Having gained them it is no longer adding anything, + so it has since been deprecated in favour of `set_readable_format`, which says what it + does; it still works, with a `DeprecationWarning`. - Tests that asserted on `_read_funcs`, `_has_hints` and friends had to be rewritten onto the public verbs. Those assertions were already against the testing conventions in `CLAUDE.md`, and the derived-on-demand design makes them impossible rather than merely diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index b14cf61a78..ca41d93aaf 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -1,7 +1,6 @@ """Module which defines abstract classes to work with detectors.""" import asyncio -import functools import os import time import warnings @@ -34,7 +33,6 @@ StandardReadableFormat, _config_signals, _HintedFields, - _Verb, ) from ._settings import Settings from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value @@ -918,6 +916,14 @@ def logic(self) -> DetectorLogic: whichever it is gets bound as an instance attribute by `prepare()`. """ + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # What the data logics produce, gathered alongside the registered + # children, so the verb methods themselves need no overriding + self._read_funcs += (self._data_read,) + self._describe_funcs += (self._data_describe,) + self._hint_sources += (self._data_hint_sources,) + @abstract_cached_property def logic(self) -> DetectorLogic: """The logic that drives this detector, built in the subclass `__init__`.""" @@ -1032,34 +1038,38 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: # may sit the rest of the scan out, so the verbs are recomputed self._publish_collect_methods() - def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: - """Contribute what the data logics produce to read() and describe(). - - StandardReadable gathers these alongside the registered children, so - the verb methods themselves need no overriding. + async def _data_read(self) -> dict[str, Reading]: + """What the data logics contribute to `read()`. Raises if nothing has been prepared, since an unprepared detector has no data keys and would otherwise emit a descriptor missing its data. `trigger()` prepares implicitly, so a step scan never sees this. - `read_configuration()` and `describe_configuration()` are unaffected. + `read_configuration()` is unaffected. + + Bounded providers hold a single-event page for this step-scan point, + which `_pageable_readings` extracts back to a reading. That extraction + lives here rather than on the provider so a provider cannot override it. + Streamable providers produce their data through `collect_asset_docs` + rather than `read()`, so they contribute nothing here. """ - if verb not in (_Verb.DESCRIBE, _Verb.READ): - return data = self.logic.prepared_data cpe = data.collections_per_event - # Bounded providers hold a single-event page for this step-scan point, - # which _pageable_readings extracts back to a reading. That extraction - # lives here rather than on the provider so a provider cannot override it. - for pdp in data.pageable: - if verb is _Verb.DESCRIBE: - yield functools.partial(pdp.make_datakeys, cpe) - else: - yield functools.partial(self._pageable_readings, pdp, cpe) - if verb is _Verb.DESCRIBE: - # Streamable providers describe their shape for a step scan, but - # produce their data through collect_asset_docs rather than read() - for sdp in data.streamable: - yield functools.partial(sdp.make_datakeys, cpe) + return await merge_gathered_dicts( + [self._pageable_readings(pdp, cpe) for pdp in data.pageable] + ) + + async def _data_describe(self) -> dict[str, DataKey]: + """What the data logics contribute to `describe()`. + + Every provider describes its shape for a step scan, whichever way it + goes on to produce the data. Raises if nothing has been prepared, as + `_data_read` does. + """ + data = self.logic.prepared_data + cpe = data.collections_per_event + return await merge_gathered_dicts( + [dp.make_datakeys(cpe) for dp in data.collectable] + ) async def describe_collect(self) -> dict[str, DataKey]: data = self.logic.prepared_data @@ -1069,8 +1079,8 @@ async def describe_collect(self) -> dict[str, DataKey]: ] return await merge_gathered_dicts(coros) - def _extra_hint_sources(self) -> Iterator[HasHints]: - """Contribute the data logics' hinted fields alongside the children's.""" + def _data_hint_sources(self) -> Iterator[HasHints]: + """The data logics' hinted fields, alongside the children's.""" for fields in self.logic.get_hinted_fields(): yield _HintedFields(fields) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 29bf0aba6a..c581a2223a 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -1,4 +1,5 @@ import asyncio +import itertools import warnings from collections.abc import Awaitable, Callable, Generator, Iterator, Sequence from contextlib import contextmanager @@ -15,19 +16,6 @@ from ._utils import merge_gathered_dicts -class _Verb(Enum): - """The four bluesky verbs a registered child can contribute to.""" - - DESCRIBE_CONFIG = "DESCRIBE_CONFIG" - READ_CONFIG = "READ_CONFIG" - DESCRIBE = "DESCRIBE" - READ = "READ" - - -_CONFIG_VERBS = frozenset({_Verb.DESCRIBE_CONFIG, _Verb.READ_CONFIG}) -_READ_VERBS = frozenset({_Verb.DESCRIBE, _Verb.READ}) - - def _as_signal_r(device: Device) -> SignalR: if not isinstance(device, SignalR): raise TypeError(f"{device} is not a SignalR") @@ -69,7 +57,7 @@ class StandardReadableFormat(Enum): def __call__(self, parent: Device, child: Device): if not isinstance(parent, StandardReadable): raise TypeError(f"Expected parent to be StandardReadable, got {parent}") - parent.add_readables([child], self) + parent.set_readable_format(child, self) #: The formats whose devices take part in `stage()`/`unstage()`, if they are @@ -148,12 +136,29 @@ def _readables(self) -> dict[Device, StandardReadableFormat]: # so the shared class level default is safe. _default_readables: dict[Device, StandardReadableFormat] = {} + # What contributes to each verb, gathered in parallel by the verb methods. + # Immutable defaults to avoid accidental sharing between instances. A + # subclass producing data from somewhere other than a registered child adds + # to these rather than overriding the verbs, so registered children keep + # working; `StandardDetector` does it for its data logics. + _read_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () + _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + _read_config_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () + _describe_config_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + _hint_sources: tuple[Callable[[], Iterator[HasHints]], ...] = () + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # Staging is derived from _readables on each call rather than registered - # up-front, so a format changed between runs is picked up on next stage(). + # Each of these derives from _readables on every call rather than being + # registered up-front, so a format changed between runs is picked up on + # the next stage() or the next descriptor. self._stage_funcs += (self._stage_readables,) self._unstage_funcs += (self._unstage_readables,) + self._read_funcs += (self._formatted_read,) + self._describe_funcs += (self._formatted_describe,) + self._read_config_funcs += (self._formatted_read_configuration,) + self._describe_config_funcs += (self._formatted_describe_configuration,) + self._hint_sources += (self._formatted_hint_sources,) async def _stage_readables(self) -> None: await asyncio.gather(*(sig.stage().task for sig in self._signals_to_stage())) @@ -169,81 +174,65 @@ def _signals_to_stage(self) -> Iterator[AsyncStageable]: if format in _STAGED_FORMATS and isinstance(device, AsyncStageable): yield device - def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: - """Contribute to a verb from something other than the registry. - - Subclasses that produce data from somewhere other than a registered - child override this rather than the verb methods themselves, so that - registered children keep working without being reimplemented. - `StandardDetector` uses it for the data its data logics produce. - """ - return iter(()) - - def _funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: - """Derive the callables contributing to one verb from the registry.""" - yield from self._extra_funcs_for(verb) + async def _formatted_read(self) -> dict[str, Reading]: + """What the registered children contribute to `read()`.""" + coros = [] for device, format in self._readables.items(): match format: - case StandardReadableFormat.CHILD: - if verb in _CONFIG_VERBS and isinstance(device, AsyncConfigurable): - yield ( - device.describe_configuration - if verb is _Verb.DESCRIBE_CONFIG - else device.read_configuration - ) - elif verb in _READ_VERBS and isinstance(device, AsyncReadable): - yield device.describe if verb is _Verb.DESCRIBE else device.read - case StandardReadableFormat.CONFIG_SIGNAL: - signal = _as_signal_r(device) - if verb is _Verb.DESCRIBE_CONFIG: - yield signal.describe - elif verb is _Verb.READ_CONFIG: - yield signal.read + case StandardReadableFormat.CHILD if isinstance(device, AsyncReadable): + coros.append(device.read()) case StandardReadableFormat.HINTED_SIGNAL: - signal = _as_signal_r(device) - if verb is _Verb.DESCRIBE: - yield signal.describe - elif verb is _Verb.READ: - yield signal.read + coros.append(_as_signal_r(device).read()) case ( StandardReadableFormat.UNCACHED_SIGNAL | StandardReadableFormat.HINTED_UNCACHED_SIGNAL ): - signal = _as_signal_r(device) - if verb is _Verb.DESCRIBE: - yield signal.describe - elif verb is _Verb.READ: - yield _UncachedRead(signal) - - async def describe_configuration(self) -> dict[str, DataKey]: - return await merge_gathered_dicts( - [func() for func in self._funcs_for(_Verb.DESCRIBE_CONFIG)] - ) + coros.append(_as_signal_r(device).read(cached=False)) + return await merge_gathered_dicts(coros) - async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts( - [func() for func in self._funcs_for(_Verb.READ_CONFIG)] - ) - - async def describe(self) -> dict[str, DataKey]: - return await merge_gathered_dicts( - [func() for func in self._funcs_for(_Verb.DESCRIBE)] - ) - - async def read(self) -> dict[str, Reading]: - return await merge_gathered_dicts( - [func() for func in self._funcs_for(_Verb.READ)] - ) + async def _formatted_describe(self) -> dict[str, DataKey]: + """What the registered children contribute to `describe()`.""" + coros = [] + for device, format in self._readables.items(): + match format: + case StandardReadableFormat.CHILD if isinstance(device, AsyncReadable): + coros.append(device.describe()) + case ( + StandardReadableFormat.HINTED_SIGNAL + | StandardReadableFormat.UNCACHED_SIGNAL + | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + ): + coros.append(_as_signal_r(device).describe()) + return await merge_gathered_dicts(coros) - def _extra_hint_sources(self) -> Iterator[HasHints]: - """Contribute hints from something other than the registry. + async def _formatted_read_configuration(self) -> dict[str, Reading]: + """What the registered children contribute to `read_configuration()`.""" + coros = [] + for device, format in self._readables.items(): + match format: + case StandardReadableFormat.CHILD if isinstance( + device, AsyncConfigurable + ): + coros.append(device.read_configuration()) + case StandardReadableFormat.CONFIG_SIGNAL: + coros.append(_as_signal_r(device).read()) + return await merge_gathered_dicts(coros) - The counterpart of [](#StandardReadable._extra_funcs_for) for `hints`. - """ - return iter(()) + async def _formatted_describe_configuration(self) -> dict[str, DataKey]: + """What the registered children contribute to `describe_configuration()`.""" + coros = [] + for device, format in self._readables.items(): + match format: + case StandardReadableFormat.CHILD if isinstance( + device, AsyncConfigurable + ): + coros.append(device.describe_configuration()) + case StandardReadableFormat.CONFIG_SIGNAL: + coros.append(_as_signal_r(device).describe()) + return await merge_gathered_dicts(coros) - def _hint_sources(self) -> Iterator[HasHints]: - yield from self._extra_hint_sources() + def _formatted_hint_sources(self) -> Iterator[HasHints]: + """What the registered children contribute to `hints`.""" for device, format in self._readables.items(): match format: case StandardReadableFormat.CHILD if isinstance(device, HasHints): @@ -254,10 +243,26 @@ def _hint_sources(self) -> Iterator[HasHints]: ): yield _HintsFromName(device) + async def describe_configuration(self) -> dict[str, DataKey]: + return await merge_gathered_dicts( + [func() for func in self._describe_config_funcs] + ) + + async def read_configuration(self) -> dict[str, Reading]: + return await merge_gathered_dicts([func() for func in self._read_config_funcs]) + + async def describe(self) -> dict[str, DataKey]: + return await merge_gathered_dicts([func() for func in self._describe_funcs]) + + async def read(self) -> dict[str, Reading]: + return await merge_gathered_dicts([func() for func in self._read_funcs]) + @property def hints(self) -> Hints: hints: Hints = {} - for new_hint in self._hint_sources(): + for new_hint in itertools.chain.from_iterable( + source() for source in self._hint_sources + ): # Merge the existing and new hints, based on the type of the value. # This avoids default dict merge behavior that overrides the values; # we want to combine them when they are Sequences, and ensure they are @@ -296,11 +301,11 @@ def add_children_as_readables( self, format: StandardReadableFormat = StandardReadableFormat.CHILD, ) -> Generator[None, None, None]: - """Context manager that calls [](#add_readables) on child Devices added within. + """Register child Devices added within with the given format. Scans `self.children()` on entry and exit to context manager, and calls - `add_readables()` on any that are added with the provided - `StandardReadableFormat`. + [](#StandardReadable.set_readable_format) on any that are added with the + provided `StandardReadableFormat`. """ dict_copy = dict(self.children()) @@ -331,8 +336,9 @@ def add_children_as_readables( else: flattened_values.append(value) - new_devices = list(filter(lambda x: isinstance(x, Device), flattened_values)) - self.add_readables(new_devices, format) + for device in flattened_values: + if isinstance(device, Device): + self.set_readable_format(device, format) def set_readable_format( self, device: Device, format: StandardReadableFormat | None @@ -406,6 +412,7 @@ def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: """ return dict(self._readables) + # Back compat - delete before 1.0 def add_readables( self, devices: Sequence[Device], @@ -413,19 +420,18 @@ def add_readables( ) -> None: """Add devices to contribute to various bluesky verbs. - Use output from the given devices to contribute to the verbs of the following - interfaces: - - - [](#bluesky.protocols.Readable) - - [](#bluesky.protocols.Configurable) - - [](#bluesky.protocols.Stageable) - - [](#bluesky.protocols.HasHints) - :param devices: The devices to be added :param format: Determines which of the devices functions are added to which verb as per the [](#StandardReadableFormat) documentation """ + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(device, format)` on each device " + "instead of `add_readables(devices, format)`" + ), + stacklevel=2, + ) for device in devices: self.set_readable_format(device, format) @@ -457,14 +463,6 @@ def _config_signals(device: Device) -> set[SignalR]: return signals -class _UncachedRead: - def __init__(self, signal: SignalR) -> None: - self.signal = signal - - async def __call__(self) -> dict[str, Reading]: - return await self.signal.read(cached=False) - - class _HintedFields(HasHints): """Present a fixed list of field names as `hints`.""" diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 4a3cd38f5e..0067353c23 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -19,7 +19,7 @@ def test_standard_readable_rejects_a_non_format(): sr = StandardReadable() with pytest.raises(TypeError, match="is not a StandardReadableFormat"): - sr.add_readables([soft_signal_rw(int)], "CONFIG_SIGNAL") # type: ignore + sr.set_readable_format(soft_signal_rw(int), "CONFIG_SIGNAL") # type: ignore def test_standard_readable_hints(): @@ -36,7 +36,9 @@ def test_standard_readable_hints(): hint3 = MagicMock(spec=HasHints) hint3.hints = {"fields": ["jkl"], "gridding": "rectilinear_nonsequential"} - sr.add_readables([hint1, hint2, hint3]) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) + sr.set_readable_format(hint3, Format.CHILD) assert sr.hints == { "fields": ["abc", "def", "ghi", "jkl"], @@ -54,7 +56,8 @@ def test_standard_readable_hints_raises_when_overriding_string_literal(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"gridding": "a different string"} - sr.add_readables([hint1, hint2]) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) with pytest.raises(RuntimeError, match=r"Hints key .* value may not be overridden"): sr.hints # noqa: B018 @@ -69,7 +72,8 @@ def test_standard_readable_hints_raises_when_overriding_sequence(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"fields": ["field2"]} - sr.add_readables([hint1, hint2]) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) with pytest.raises(RuntimeError, match=r"Hint fields .* overrides existing hint"): sr.hints # noqa: B018 @@ -82,7 +86,7 @@ def test_standard_readable_hints_invalid_types(invalid_type): hint1 = MagicMock(spec=HasHints) hint1.hints = {"test": invalid_type} - sr.add_readables([hint1]) + sr.set_readable_format(hint1, Format.CHILD) with pytest.raises(TypeError, match=r"Unknown type for value .* for key .*"): sr.hints # noqa: B018 @@ -90,17 +94,12 @@ def test_standard_readable_hints_invalid_types(invalid_type): def test_standard_readable_add_children_context_manager(): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock with sr.add_children_as_readables(): sr.a = MagicMock(spec=SignalR) sr.b = MagicMock(spec=SignalR) sr.c = MagicMock(spec=SignalR) - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == {sr.a, sr.b, sr.c} + assert sr.get_readable_formats() == dict.fromkeys((sr.a, sr.b, sr.c), Format.CHILD) @pytest.mark.parametrize( @@ -112,26 +111,19 @@ def test_standard_readable_add_children_context_manager(): ) def test_standard_readable_add_children_cm_device_with_mappings(device_type, keys): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock - # Create a mock for the DeviceVector/DeviceMap.children() call + # A DeviceVector/DeviceMap registers its values, not itself devices = [MagicMock(spec=SignalR) for _ in range(3)] device = device_type(dict(zip(keys, devices, strict=True))) with sr.add_children_as_readables(): sr.a = device - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == set(devices) + assert sr.get_readable_formats() == dict.fromkeys(devices, Format.CHILD) def test_standard_readable_add_children_cm_filters_non_devices(): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock with sr.add_children_as_readables(): sr.a = MagicMock(spec=SignalR) @@ -140,10 +132,8 @@ def test_standard_readable_add_children_cm_filters_non_devices(): sr.d = "abc" sr.e = MagicMock(spec=MockSignalBackend) - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == {sr.a, sr.b} + # Only the Devices are registered + assert sr.get_readable_formats() == dict.fromkeys((sr.a, sr.b), Format.CHILD) async def assert_contributes( @@ -227,7 +217,16 @@ def test_standard_readable_add_readables_raises_signalr_typeerror(format) -> Non # Ensure it raises TypeError with pytest.raises(TypeError, match=f"{mock_device} is not a SignalR"): - sr.add_readables([mock_device], format=format) + sr.set_readable_format(mock_device, format) + + +def test_add_readables_is_deprecated() -> None: + sr = StandardReadable() + sig = soft_signal_rw(int) + with pytest.warns(DeprecationWarning, match="set_readable_format"): + sr.add_readables([sig], Format.CONFIG_SIGNAL) + # Still does the job it always did + assert sr.get_readable_formats() == {sig: Format.CONFIG_SIGNAL} @pytest.mark.parametrize( diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index f6acf8829f..6b549ce204 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -1172,10 +1172,10 @@ async def test_child_readable_config_signals_in_describe_configuration(): """Child StandardReadable CONFIG_SIGNALs appear in describe_configuration.""" child = StandardReadable(name="child") config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") - child.add_readables([config_sig], Format.CONFIG_SIGNAL) + child.set_readable_format(config_sig, Format.CONFIG_SIGNAL) det = DetectorLogic().with_device("det") - det.add_readables([child], Format.CHILD) + det.set_readable_format(child, Format.CHILD) await det.prepare(TriggerInfo()) config = await det.describe_configuration() @@ -1189,10 +1189,10 @@ async def test_child_readable_read_signals_in_read(tmp_path): """Child StandardReadable HINTED_SIGNALs appear in read/describe.""" child = StandardReadable(name="child") read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") - child.add_readables([read_sig], Format.HINTED_SIGNAL) + child.set_readable_format(read_sig, Format.HINTED_SIGNAL) det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device("det") - det.add_readables([child], Format.CHILD) + det.set_readable_format(child, Format.CHILD) await det.prepare(TriggerInfo()) # The data logic describes its stream, the child describes its signal @@ -1209,10 +1209,10 @@ async def test_child_readable_hints_merged(tmp_path): """Child StandardReadable hints are merged with data logic hints.""" child = StandardReadable(name="child") hinted_sig = soft_signal_rw(float, name="child-intensity") - child.add_readables([hinted_sig], Format.HINTED_SIGNAL) + child.set_readable_format(hinted_sig, Format.HINTED_SIGNAL) det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device("det") - det.add_readables([child], Format.CHILD) + det.set_readable_format(child, Format.CHILD) assert "fields" in det.hints assert "child-intensity" in det.hints["fields"] From b10ac9ffea8f5b1cded15329cc02e1577223bf89 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 14:18:02 +0000 Subject: [PATCH 55/67] docs: record why FlyableLogic has one context type, not two Review asked whether a single CtxT is right or whether prepare and kickoff should have their own. Writing down the answer so it is not asked a third time. Assisted-by: Claude Opus 5 --- .../decisions/0023-detector-is-a-flyable.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/explanations/decisions/0023-detector-is-a-flyable.md b/docs/explanations/decisions/0023-detector-is-a-flyable.md index e292c811c2..7637e9fa54 100644 --- a/docs/explanations/decisions/0023-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0023-detector-is-a-flyable.md @@ -136,6 +136,23 @@ been, then asks the data state to re-arm what needs re-arming: a finite buffer h event at a time so it is re-made per point, while a streaming provider carries on. The trigger logic is not re-prepared per point, so a step scan does not repeat its puts. +### One context type, not one per stage + +`FlyableLogic` takes a single `CtxT` threaded through all three stages, so a field that +only exists after `kickoff()` is `| None` on the one dataclass — `_FlyCtx`'s +`kickoff_collections_written`, `MotorFlyCtx`'s `status`, `PmacFlyCtx`'s +`trajectory_status` — and `on_complete` unwraps it. + +Splitting into `PrepareCtxT` and `KickoffCtxT` would make those fields non-optional. It +was not worth it. Every flyer would need a second dataclass, or inheritance between two, +and `on_kickoff` would have to rebuild one from the other; every declaration site gains a +third parameter, including the three PandA logics that carry no state and would read +`FlyableLogic[SeqTableInfo, None, None]`. `StandardFlyable` gains nothing either, since +the context it holds changes type part way through the lifecycle and so cannot be typed +as one or the other regardless. Against that, three `error_if_none` calls in the whole +tree. The `| None` is doing useful work as documentation: it says *this appears at +kickoff*, which is exactly what a reader needs to know. + ## Consequences - `StandardDetector` subclasses build a `DetectorLogic` in `__init__` and return it from From 287ba5943f37569b54b5eeb96d79605a2cd5ca9e Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 14 Sep 2026 14:44:00 +0000 Subject: [PATCH 56/67] refactor: have the hint funcs yield Hints, not HasHints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _HintsFromName and _HintedFields existed only to wrap a list of field names in something with a .hints property, so the merge loop could read it back off again. Have the hint funcs yield Hints directly and both adapters go, along with the `name` neither of them had anything useful to put in — the merge error now names the Device doing the merging, which is the one the caller asked. Assisted-by: Claude Opus 5 --- src/ophyd_async/core/_detector.py | 8 +++--- src/ophyd_async/core/_readable.py | 48 +++++++------------------------ 2 files changed, 14 insertions(+), 42 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index ca41d93aaf..734530b4c0 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -14,6 +14,7 @@ from bluesky.protocols import ( Collectable, HasHints, + Hints, Reading, Stageable, StreamAsset, @@ -32,7 +33,6 @@ StandardReadable, StandardReadableFormat, _config_signals, - _HintedFields, ) from ._settings import Settings from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value @@ -922,7 +922,7 @@ def __init__(self, *args, **kwargs) -> None: # children, so the verb methods themselves need no overriding self._read_funcs += (self._data_read,) self._describe_funcs += (self._data_describe,) - self._hint_sources += (self._data_hint_sources,) + self._hint_funcs += (self._data_hints,) @abstract_cached_property def logic(self) -> DetectorLogic: @@ -1079,10 +1079,10 @@ async def describe_collect(self) -> dict[str, DataKey]: ] return await merge_gathered_dicts(coros) - def _data_hint_sources(self) -> Iterator[HasHints]: + def _data_hints(self) -> Iterator[Hints]: """The data logics' hinted fields, alongside the children's.""" for fields in self.logic.get_hinted_fields(): - yield _HintedFields(fields) + yield {"fields": list(fields)} async def _pageable_readings( self, provider: PageableDataProvider, collections_per_event: int diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index c581a2223a..8b59ab7f6c 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -145,7 +145,7 @@ def _readables(self) -> dict[Device, StandardReadableFormat]: _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () _read_config_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () _describe_config_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () - _hint_sources: tuple[Callable[[], Iterator[HasHints]], ...] = () + _hint_funcs: tuple[Callable[[], Iterator[Hints]], ...] = () def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -158,7 +158,7 @@ def __init__(self, *args, **kwargs) -> None: self._describe_funcs += (self._formatted_describe,) self._read_config_funcs += (self._formatted_read_configuration,) self._describe_config_funcs += (self._formatted_describe_configuration,) - self._hint_sources += (self._formatted_hint_sources,) + self._hint_funcs += (self._formatted_hints,) async def _stage_readables(self) -> None: await asyncio.gather(*(sig.stage().task for sig in self._signals_to_stage())) @@ -231,17 +231,19 @@ async def _formatted_describe_configuration(self) -> dict[str, DataKey]: coros.append(_as_signal_r(device).describe()) return await merge_gathered_dicts(coros) - def _formatted_hint_sources(self) -> Iterator[HasHints]: + def _formatted_hints(self) -> Iterator[Hints]: """What the registered children contribute to `hints`.""" for device, format in self._readables.items(): match format: case StandardReadableFormat.CHILD if isinstance(device, HasHints): - yield device + yield device.hints case ( StandardReadableFormat.HINTED_SIGNAL | StandardReadableFormat.HINTED_UNCACHED_SIGNAL ): - yield _HintsFromName(device) + # An unnamed signal has nothing to put in `fields`, but is + # still hinted, so it contributes the key and no name + yield {"fields": [device.name] if device.name else []} async def describe_configuration(self) -> dict[str, DataKey]: return await merge_gathered_dicts( @@ -261,13 +263,13 @@ async def read(self) -> dict[str, Reading]: def hints(self) -> Hints: hints: Hints = {} for new_hint in itertools.chain.from_iterable( - source() for source in self._hint_sources + func() for func in self._hint_funcs ): # Merge the existing and new hints, based on the type of the value. # This avoids default dict merge behavior that overrides the values; # we want to combine them when they are Sequences, and ensure they are # identical when string values. - for key, value in new_hint.hints.items(): + for key, value in new_hint.items(): # fail early for unkwon types if isinstance(value, str): if key in hints: @@ -289,8 +291,7 @@ def hints(self) -> Hints: hints[key] = value # type: ignore[literal-required] else: msg = ( - f"{new_hint.name}: Unknown type for value '{value}'" - f" for key '{key}'" + f"{self.name}: Unknown type for value '{value}' for key '{key}'" ) raise TypeError(msg) @@ -463,35 +464,6 @@ def _config_signals(device: Device) -> set[SignalR]: return signals -class _HintedFields(HasHints): - """Present a fixed list of field names as `hints`.""" - - def __init__(self, fields: Sequence[str]) -> None: - self._fields = list(fields) - - @property - def name(self) -> str: - return "" - - @property - def hints(self) -> Hints: - return {"fields": self._fields} - - -class _HintsFromName(HasHints): - def __init__(self, device: Device) -> None: - self.device = device - - @property - def name(self) -> str: - return self.device.name - - @property - def hints(self) -> Hints: - fields = [self.name] if self.name else [] - return {"fields": fields} - - #: Reserved key under which [](#store_settings) writes readable formats. Chosen #: so that it cannot be a dotted attribute path: `<` cannot start a Python #: identifier, so no attribute assignment can produce a colliding key, and From 63e13900cabfc713aa3da783aa0ae9327b3303c0 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 15 Sep 2026 15:05:44 +0000 Subject: [PATCH 57/67] refactor: bind the collect verb in prepare() only, inline The verb was recomputed after every trigger() on the grounds that a re-armed bounded provider might sit the rest of the scan out. It cannot: on_trigger re-prepares the data from the same TriggerInfo prepare() used, so both cache-key comparisons match and _rearm_bounded runs, which passes streamable providers through untouched and re-makes bounded ones from byte-identical arguments. Nor is there a consumer -- the bound verbs exist for the bundler's isinstance checks on the fly path, and a step scan reads through _data_read, which consults data.pageable directly. With one call site left, _publish_collect_methods is inlined into prepare(), and its docstring becomes a comment saying what the bundler change now in progress upstream will let us delete: the binding goes and _collect_asset_docs/_collect_pages become plain public methods. Assisted-by: Claude Opus 5 --- .../decisions/0023-detector-is-a-flyable.md | 10 ++-- src/ophyd_async/core/_detector.py | 51 +++++++------------ 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/docs/explanations/decisions/0023-detector-is-a-flyable.md b/docs/explanations/decisions/0023-detector-is-a-flyable.md index 7637e9fa54..47ea9debb0 100644 --- a/docs/explanations/decisions/0023-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0023-detector-is-a-flyable.md @@ -73,10 +73,12 @@ decides which to ask for from the *static* type — `collect_pages()` is called device that is not a `WritesStreamAssets` — so a detector that declared both would have its pages silently dropped, and one that declared neither could not fly at all. -So `StandardDetector.prepare()` and `trigger()` bind whichever verb applies as a real -instance attribute, and remove the other. It is the Device that does this, from what -`DetectorLogic.data` holds after preparing; the logic reports what it made and is not told -where that goes, so it holds no back-reference. +So `StandardDetector.prepare()` binds whichever verb applies as a real instance attribute, +and removes the other. It is the Device that does this, from what `DetectorLogic.data` holds +after preparing; the logic reports what it made and is not told where that goes, so it holds +no back-reference. Only `prepare()` does it: re-arming a bounded provider on each point of a +step scan cannot change which verb applies, because the providers are re-made from the same +`TriggerInfo` that `prepare()` used. Choosing from the static type is a bundler limitation rather than a fact about detectors: a device knows what it produced, and the bundler could decide from *that*, erroring only if diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 734530b4c0..3cca1db01c 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -934,34 +934,6 @@ def set_name(self, name: str, *, child_name_separator: str | None = None) -> Non # The data logics name their datakeys after the Device self.logic.datakey_prefix = name - def _publish_collect_methods(self) -> None: - """Bind the collect verb that matches what the data logics will produce. - - Whichever applies is bound as a real instance attribute, so the bluesky - bundler's isinstance checks (`WritesStreamAssets` vs - `EventPageCollectable`) see exactly one of them. They resolve with - `inspect.getattr_static` on Python 3.12+, which does not call - `__getattr__`, so a dynamic hook would be invisible. Both names are - reserved by `Device`, hence `object.__setattr__`. - - Called from every prepare and trigger, so the verb that no longer - applies is removed rather than left behind from a previous scan. - - TODO: the bundler treats the two protocols as mutually exclusive from - the static type, so a detector cannot declare both and pick per scan. - Once bluesky decides from what a device actually produced, both verbs - become plain methods and this goes; see ADR 0023. - """ - data = self.logic.data - for verb, method, wanted in ( - ("collect_asset_docs", self._collect_asset_docs, data and data.streamable), - ("collect_pages", self._collect_pages, data and data.pageable), - ): - if wanted: - object.__setattr__(self, verb, method) - elif verb in self.__dict__: - object.__delattr__(self, verb) - # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: """Add a signal to read_configuration(). @@ -1018,7 +990,25 @@ async def get_trigger_deadtime( async def prepare(self, value: TriggerInfo) -> None: """Set the detector up for a scan, as described by the TriggerInfo.""" await super().prepare(value) - self._publish_collect_methods() + # Bind the collect verb matching what the data logics will produce, as a + # real instance attribute so the bundler's isinstance checks + # (WritesStreamAssets vs EventPageCollectable) see exactly one of them. + # Those resolve with inspect.getattr_static on Python 3.12+, which does not + # call __getattr__, so a dynamic hook would be invisible; both names are + # reserved by Device, hence object.__setattr__. + # TODO: all of this goes once the run bundler picks the collect verb from + # what a device actually produced rather than from its static type - a + # change in progress upstream. _collect_asset_docs and _collect_pages then + # become plain public methods and a detector declares both. See ADR 0023. + data = self.logic.data + for verb, method, wanted in ( + ("collect_asset_docs", self._collect_asset_docs, data and data.streamable), + ("collect_pages", self._collect_pages, data and data.pageable), + ): + if wanted: + object.__setattr__(self, verb, method) + elif verb in self.__dict__: + object.__delattr__(self, verb) @WatchableAsyncStatus.wrap async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: @@ -1034,9 +1024,6 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: await self.prepare(await self.logic.default_trigger_info()) async for update in self.logic.on_trigger(self._prepared_fly_ctx): yield update - # A step scan re-arms its finite buffers on every trigger, and a logic - # may sit the rest of the scan out, so the verbs are recomputed - self._publish_collect_methods() async def _data_read(self) -> dict[str, Reading]: """What the data logics contribute to `read()`. From d6120384bf6683ec62e146a754186fb62e85d4ee Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 15 Sep 2026 15:37:00 +0000 Subject: [PATCH 58/67] fix: don't re-arm a bounded buffer on the first trigger after prepare on_trigger re-prepared the data unconditionally, so the first point of a step scan armed the buffer twice: prepare() made and started the provider, then on_trigger stopped it and made and started it again. For an NDStats time series that is TSAcquire 1, 0, 1 plus four duplicated config writes before a frame arrives -- harmless against an empty buffer, but a visible glitch on the arming control, and with external triggering prepare() has already started acquiring, so the gap can drop frames. The re-arm is still needed from the second point on, where the buffer holds the previous point's data. _FlyCtx carries a needs_rearm flag saying which case this is: prepare() leaves it False, and each trigger sets it. Assisted-by: Claude Opus 5 --- .../decisions/0022-bounded-data-logic-tier.md | 11 +++++++---- .../decisions/0023-detector-is-a-flyable.md | 10 ++++++---- src/ophyd_async/core/_detector.py | 11 +++++++++-- tests/unit_tests/core/test_standard_detector.py | 14 ++++++++++++-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/explanations/decisions/0022-bounded-data-logic-tier.md b/docs/explanations/decisions/0022-bounded-data-logic-tier.md index ca9402b85d..4bdfec60a9 100644 --- a/docs/explanations/decisions/0022-bounded-data-logic-tier.md +++ b/docs/explanations/decisions/0022-bounded-data-logic-tier.md @@ -129,10 +129,13 @@ because both are baked into a `StreamResource` when it is made — the shape fro chunking from the second — and areaDetector's `num_frames_chunks` can only be set before `capture` starts. A period change therefore starts a new file. -A finite buffer is re-made on every `prepare()` and `trigger()` regardless, because re-making -it is what re-arms it, and `kickoff()` never re-makes one: exactly the set of places a buffer -should and should not be re-armed. A re-armed buffer sitting at 0 alongside a monotonic HDF -writer at 15 would trip the `_all_the_same` reducer, so the two kinds are tracked separately. +A finite buffer is re-made on every `prepare()` regardless, and on every `trigger()` after the +first one since that `prepare()`, because re-making it is what re-arms it; `kickoff()` never +re-makes one. That is exactly the set of places a buffer should and should not be re-armed: the +first point of a step scan is already armed by the `prepare()` in front of it, and re-arming it +again would erase an empty buffer, putting a spurious stop/start on the arming control before a +frame had arrived. A re-armed buffer sitting at 0 alongside a monotonic HDF writer at 15 would +trip the `_all_the_same` reducer, so the two kinds are tracked separately. ### A bounded provider's progress baseline is zero in `trigger()` diff --git a/docs/explanations/decisions/0023-detector-is-a-flyable.md b/docs/explanations/decisions/0023-detector-is-a-flyable.md index 47ea9debb0..d81cb180e8 100644 --- a/docs/explanations/decisions/0023-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0023-detector-is-a-flyable.md @@ -133,10 +133,12 @@ reuses an open file across its points, and `collect_while_completing` collects * `DetectorLogic.data`, which is cleared by `stage()`/`unstage()`, while the fly context enforces the ordering of the fly verbs. -`trigger()` therefore needs no context of its own. It prepares implicitly if nothing has -been, then asks the data state to re-arm what needs re-arming: a finite buffer holds one -event at a time so it is re-made per point, while a streaming provider carries on. The -trigger logic is not re-prepared per point, so a step scan does not repeat its puts. +`trigger()` therefore needs no context of its own, beyond a `needs_rearm` flag on the fly +context. It prepares implicitly if nothing has been, then asks the data state to re-arm what +needs re-arming: a finite buffer holds one event at a time so it is re-made per point, while a +streaming provider carries on. The flag is what keeps the first point from being re-armed on +top of the `prepare()` that just armed it. The trigger logic is not re-prepared per point, so +a step scan does not repeat its puts. ### One context type, not one per stage diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 3cca1db01c..cf52d9266b 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -452,6 +452,8 @@ class _FlyCtx: trigger_info: TriggerInfo #: What the providers had written when kickoff() ran, None until then kickoff_collections_written: int | None = None + #: Whether a finite buffer holds a previous point's data, so needs re-arming + needs_rearm: bool = False class DetectorLogic(FlyableLogic[TriggerInfo, _FlyCtx]): @@ -611,8 +613,13 @@ async def on_trigger(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: f"prepared with number_of_events={ctx.trigger_info.number_of_events}." ) # A finite buffer holds one event at a time, so it is re-armed for each - # point of a step scan; a streaming provider carries on from where it was - data = await self._prepare_data(ctx.trigger_info) + # point of a step scan; a streaming provider carries on from where it was. + # prepare() armed it for the first point, so re-arming here as well would + # erase and re-erase it before a frame arrived. + if ctx.needs_rearm: + await self._prepare_data(ctx.trigger_info) + ctx.needs_rearm = True + data = self.prepared_data # Take the baseline before acquisition starts, or frames written between # the two would be counted towards this event. A re-armed buffer starts # from zero, where a streaming provider continues from wherever it has got diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 6b549ce204..684c4c0c1e 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -947,11 +947,21 @@ async def test_bounded_step_scan_reads_derived_from_page(): await dl.collections_written.set(5) assert status.done assert status.success - # trigger() re-armed the buffer, so prepare_bounded ran a second time - assert dl.prepare_calls == [(5, pytest.approx(0.1)), (5, pytest.approx(0.1))] + # prepare() armed the buffer for this point, so trigger() does not re-arm it + assert dl.prepare_calls == [(5, pytest.approx(0.1))] + assert dl.stop_count == 0 reading = await det.read() assert reading["foo"]["value"] == [0, 0, 0, 0, 0] + # The next point does re-arm, as the buffer holds the last point's data + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(5) + assert status.done + assert status.success + assert dl.prepare_calls == [(5, pytest.approx(0.1)), (5, pytest.approx(0.1))] + assert dl.stop_count == 1 + async def test_bounded_fly_scan_accumulates_across_kickoffs(): """A bounded buffer is armed once at prepare and not re-armed per kickoff.""" From cfa978e74e6f42580603d37edebac7f6c64d5d0c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 15 Sep 2026 15:59:36 +0000 Subject: [PATCH 59/67] refactor: keep the collections baseline with the data it describes _wait_for_collections was handed a single initial_collections_written and seeded every provider from it. That is only sound because _drop_shadowed bans one detector serving both a finite buffer and a file writer, which would be at different counts -- an invariant enforced three methods away from the code relying on it. Put the number on _DetectorData, next to the pageable/streamable split it depends on, and say there what would have to change if the ban were ever lifted. on_trigger and on_kickoff each record it under their own rule (a re-armed buffer starts at zero; a fly scan reads live because buffers accumulate across the kickoffs of one prepare) and _wait_for_collections reads it back, so it loses a parameter. _FlyCtx.kickoff_collections_written goes with it, along with its "Kickoff not run" guard: StandardFlyable.complete() already refuses to run before kickoff, with a better message. Assisted-by: Claude Opus 5 --- src/ophyd_async/core/_detector.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index cf52d9266b..c9a7ddddc0 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -422,6 +422,13 @@ class _DetectorData: #: What the providers were made for, and so when they can be reused collections_per_event: int period: float + #: Where every provider was when the current event started, set by the + #: trigger or kickoff that started it. One number covers all of them only + #: because `_drop_shadowed` bans a detector serving both kinds at once: a + #: finite buffer and a file writer would be at different counts, and there + #: is no use case for mixing them. Lift that ban and this becomes one + #: baseline per provider. + initial_collections_written: int = 0 @property def streamable(self) -> list[StreamableDataProvider]: @@ -450,8 +457,6 @@ class _FlyCtx: """What prepare() set up, threaded through kickoff() to complete().""" trigger_info: TriggerInfo - #: What the providers had written when kickoff() ran, None until then - kickoff_collections_written: int | None = None #: Whether a finite buffer holds a previous point's data, so needs re-arming needs_rearm: bool = False @@ -588,7 +593,11 @@ async def on_kickoff(self, ctx: _FlyCtx) -> _FlyCtx: f"Detector {self.datakey_prefix} has no collectable data, " "so cannot kickoff" ) - ctx.kickoff_collections_written = await _get_collections_written(collectable) + # Read live: a buffer accumulates across the kickoffs of one prepare, so + # it is not back at zero the way a re-armed one is in on_trigger() + self.prepared_data.initial_collections_written = await _get_collections_written( + collectable + ) # External triggering has already started; internal starts now if self.acquire_logic and ctx.trigger_info.trigger is DetectorTrigger.INTERNAL: await self.acquire_logic.start_acquiring() @@ -598,9 +607,6 @@ def on_complete(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: """Wait for the scan to finish, reporting collections written as progress.""" return self._wait_for_collections( trigger_info=ctx.trigger_info, - initial_collections_written=error_if_none( - ctx.kickoff_collections_written, "Kickoff not run" - ), collections_requested=ctx.trigger_info.number_of_collections, watcher_divisor=ctx.trigger_info.collections_per_event, ) @@ -624,15 +630,14 @@ async def on_trigger(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: # the two would be counted towards this event. A re-armed buffer starts # from zero, where a streaming provider continues from wherever it has got # to -- which is not where prepare left it, since a step scan triggers many - # times against one prepare. A detector never mixes the two. - initial = ( + # times against one prepare. + data.initial_collections_written = ( 0 if data.pageable else await _get_collections_written(data.collectable) ) if self.acquire_logic: await self.acquire_logic.start_acquiring() async for update in self._wait_for_collections( trigger_info=ctx.trigger_info, - initial_collections_written=initial, collections_requested=ctx.trigger_info.collections_per_event, ): yield update @@ -841,11 +846,14 @@ async def _rearm_bounded( async def _wait_for_collections( self, trigger_info: TriggerInfo, - initial_collections_written: int, collections_requested: int, watcher_divisor: int = 1, ) -> AsyncIterator[WatcherUpdate]: - data_providers = self.prepared_data.collectable + data = self.prepared_data + data_providers = data.collectable + # Where this event started from, recorded by the trigger or kickoff that + # started it, since by now the providers have moved on + initial_collections_written = data.initial_collections_written start_time = time.monotonic() current_collections_written = { dp.collections_written_signal: initial_collections_written From cdf34940b90c6d4bb68ce16bff75f16ec7279895 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 15 Sep 2026 16:00:59 +0000 Subject: [PATCH 60/67] fix: only arm an internally triggered detector in trigger() on_trigger armed unconditionally, so an externally triggered detector was armed once by prepare() and then again on every point of a step scan, restarting an acquisition that was already sitting waiting on its trigger source. on_prepare and on_kickoff have always split on the trigger type; on_trigger is now the same, and matches what DetectorAcquireLogic .start_acquiring's own docstring already claimed: "called from prepare() for external-trigger modes, and from kickoff() / trigger() for internal". Carried over from main, where trigger() armed unconditionally too. No test covered trigger() in an external mode; test_trigger_arms_detector becomes test_trigger_arms_only_an_internally_triggered_detector, parametrized over all three modes and asserting the detector ends up armed exactly once either way. Assisted-by: Claude Opus 5 --- src/ophyd_async/core/_detector.py | 5 +++- .../unit_tests/core/test_standard_detector.py | 30 +++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index c9a7ddddc0..64019e912b 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -634,7 +634,10 @@ async def on_trigger(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: data.initial_collections_written = ( 0 if data.pageable else await _get_collections_written(data.collectable) ) - if self.acquire_logic: + # External triggering started acquiring at prepare and is sitting waiting + # for its trigger source, so starting it again would restart it; internal + # starts here, as it does in on_kickoff() + if self.acquire_logic and ctx.trigger_info.trigger is DetectorTrigger.INTERNAL: await self.acquire_logic.start_acquiring() async for update in self._wait_for_collections( trigger_info=ctx.trigger_info, diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 684c4c0c1e..890a6501ed 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -452,22 +452,34 @@ async def test_arm_timing(trigger_type, arm_timing, tmp_path): assert al.armed is True -async def test_trigger_arms_detector(tmp_path): - """Test that trigger() arms the detector when arm logic is present.""" +@pytest.mark.parametrize( + "trigger_type,arms_at_trigger", + [ + (DetectorTrigger.INTERNAL, True), + (DetectorTrigger.EXTERNAL_EDGE, False), + (DetectorTrigger.EXTERNAL_LEVEL, False), + ], +) +async def test_trigger_arms_only_an_internally_triggered_detector( + trigger_type, arms_at_trigger, tmp_path +): + """trigger() arms the detector, unless prepare() already did. + + An externally triggered detector was armed by prepare() and is sitting + waiting on its trigger source, so arming it again would restart it. + """ al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det = DetectorLogic(JustInternalTriggerLogic(), al, dl).with_device() + det = DetectorLogic(AllTriggerTypesLogic(), al, dl).with_device() - await det.prepare(TriggerInfo()) - - # Should not be armed yet - assert al.armed is False - assert al.arm_count == 0 + await det.prepare(TriggerInfo(trigger=trigger_type)) + assert al.arm_count == (0 if arms_at_trigger else 1) + assert al.armed is not arms_at_trigger - # Trigger should arm it status = det.trigger() # Give it a moment to arm await wait_for_pending_wakeups(raise_if_exceeded=False) + # Armed either way by now, but only ever once assert al.armed is True assert al.arm_count == 1 From ae18243f4a43b3449c47a502c8f4cb9a84938da6 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 7 Oct 2026 16:38:06 +0000 Subject: [PATCH 61/67] dryrun: ruff format blank line left by the test_standard_detector conflict resolution Assisted-by: Claude Sonnet 5.5 --- tests/unit_tests/core/test_standard_detector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 890a6501ed..e0927c9d4b 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -197,6 +197,7 @@ async def test_stream_resource_data_provider_1d_single_collection_is_array(): } } + class MockPageableProvider(PageableDataProvider): """A finite buffer emitted as event pages, one value per collection.""" From 90c9dcd233d79d5684c133643392f2e48eaa8b63 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 7 Oct 2026 16:38:08 +0000 Subject: [PATCH 62/67] dryrun: adapt main-added eiger pixel_mask and adcore hints test to the stack APIs Eiger uses set_readable_format instead of the deprecated add_config_signals; test_writer_image_hints drops add_detector_logics and PluginSignalDataLogic, which the stack removes. Assisted-by: Claude Sonnet 5.5 --- src/ophyd_async/fastcs/eiger/_detector.py | 3 ++- tests/unit_tests/epics/adcore/test_detector.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ophyd_async/fastcs/eiger/_detector.py b/src/ophyd_async/fastcs/eiger/_detector.py index a60a802e81..3b51efe1c2 100644 --- a/src/ophyd_async/fastcs/eiger/_detector.py +++ b/src/ophyd_async/fastcs/eiger/_detector.py @@ -5,6 +5,7 @@ PathProvider, SignalR, StandardDetector, + StandardReadableFormat, TriggerableCommand, soft_signal_rw, ) @@ -47,7 +48,7 @@ def __init__( ), ) - self.add_config_signals(self.pixel_mask) + self.set_readable_format(self.pixel_mask, StandardReadableFormat.CONFIG_SIGNAL) super().__init__(name=name, connector=connector) diff --git a/tests/unit_tests/epics/adcore/test_detector.py b/tests/unit_tests/epics/adcore/test_detector.py index 9f15424db4..9ac07e021e 100644 --- a/tests/unit_tests/epics/adcore/test_detector.py +++ b/tests/unit_tests/epics/adcore/test_detector.py @@ -1,6 +1,7 @@ import pytest from ophyd_async.core import ( + StandardReadableFormat, StaticPathProvider, TriggerInfo, init_devices, @@ -28,10 +29,12 @@ def test_writer_image_hints(static_path_provider: StaticPathProvider, factory, h plugins={"stats": stats}, name="det", ) - assert det.hints == {"fields": ["det-image"] if hinted else []} - det.add_detector_logics(adcore.PluginSignalDataLogic(driver, stats.total)) + # No hinted fields means no hints at all + assert det.hints == ({"fields": ["det-image"]} if hinted else {}) + det.set_readable_format(stats.total, StandardReadableFormat.HINTED_SIGNAL) + # Readable-format signals are hinted ahead of the data logics' fields assert det.hints == { - "fields": (["det-image"] if hinted else []) + ["det-stats-total"] + "fields": ["det-stats-total"] + (["det-image"] if hinted else []) } From 6cc7d90049d9aaf87e7eeaf81e8c692e4163f324 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Thu, 8 Oct 2026 15:12:16 +0000 Subject: [PATCH 63/67] fix: warn on unknown reserved keys when retrieving settings They were dropped silently, so a typo such as lost the formats without any sign. Still not a NameError, as a newer version may write keys this one does not understand. Assisted-by: Claude Opus 5.5 --- src/ophyd_async/plan_stubs/_settings.py | 16 +++++++++++----- tests/unit_tests/plan_stubs/test_settings.py | 13 ++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 3a4b65e5d3..d6e5129559 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import warnings from collections.abc import Callable, Mapping from typing import Any @@ -112,13 +113,18 @@ def retrieve_settings( # Copy, as the provider may hand back a dict it holds on to and the reserved # keys are popped out of it below data = dict((yield from wait_for_awaitable(provider.retrieve(name)))) - # Reserved keys first, so what is left is signal paths and the unknown name - # check below still catches genuine typos. Any key is dropped, - # not just the one understood here: a path can never look like that, so a - # key written by a newer version is ignored rather than reported as a typo. + # Reserved keys first, so what is left is signal paths. Any other + # key is dropped with a warning: it is either a typo of a reserved key or + # written by a newer version, and a path can never look like that. stored_formats = data.pop(READABLE_FORMATS_KEY, {}) - for key in [k for k in data if k.startswith("<") and k.endswith(">")]: + unknown_reserved = sorted(k for k in data if k.startswith("<") and k.endswith(">")) + for key in unknown_reserved: del data[key] + if unknown_reserved: + warnings.warn( + f"Ignoring unknown reserved keys {unknown_reserved} in settings {name!r}", + stacklevel=2, + ) readable_formats: ReadableFormats = { owner: { # A null means "stop this child contributing", so it is not coerced diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index 72fb808ed8..04b233a02c 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -339,20 +339,19 @@ def my_plan(): RE(my_plan()) -async def test_retrieve_settings_ignores_unknown_reserved_keys( +async def test_retrieve_settings_warns_on_unknown_reserved_keys( RE, technique_device, tmp_path ): - """A key from a newer version is skipped, not reported as an unknown signal. - - No signal path can look like <...>, so there is nothing to reserve up front. - """ + """A typo'd or newer reserved key is skipped with a warning, not a NameError.""" provider = YamlSettingsProvider(tmp_path) (tmp_path / "newer.yaml").write_text( - "energy: 3.0\n:\n : mono\n: 1\n" + "energy: 3.0\n:\n : {}\n: 1\n" ) def my_plan(): - settings = yield from retrieve_settings(provider, "newer", technique_device) + match = r"unknown reserved keys \['', ''\]" + with pytest.warns(UserWarning, match=match): + settings = yield from retrieve_settings(provider, "newer", technique_device) assert dict(settings) == {technique_device.energy: 3.0} yield from apply_settings(settings) assert (yield from bps.rd(technique_device.energy)) == 3.0 From 0c4a7b449b3d34da39b821104b7b45b2bf40ce5c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Thu, 8 Oct 2026 15:20:56 +0000 Subject: [PATCH 64/67] feat: store readable format changes against class defaults, owner relative A settings file used to record every registered format, which pinned the class defaults of the day into it and, as applying merged, could not return from one technique to another. Store only the changes since construction, with a dropped child as null, and reset the tree to its class defaults before applying them. A file with no formats key still leaves formats alone; an empty one now restores the defaults. Child paths are now relative to their owner rather than to the root, so the owner is not repeated on every line. Registrations of objects outside their owner's subtree (dodal's Reference and __init__ argument cases) are outside the stored tree too, so are no less storable, and as __init__ registrations they are defaults that are never written. The fixtures go back to main's formatting plus the one new line. Assisted-by: Claude Opus 5.5 --- .../decisions/0020-runtime-readable-format.md | 68 +++--- docs/how-to/change-readable-format.md | 6 +- docs/how-to/store-and-retrieve.md | 27 +-- src/ophyd_async/core/_readable.py | 111 +++++---- src/ophyd_async/core/_settings.py | 49 ++-- src/ophyd_async/plan_stubs/_settings.py | 50 ++-- .../tango/core/test_yaml_save.yaml | 1 + tests/unit_tests/plan_stubs/test_settings.py | 87 +++++-- .../test_data/test_yaml_config_save.yaml | 77 +++--- .../unit_tests/test_data/test_yaml_save.yaml | 227 ++++++++---------- 10 files changed, 380 insertions(+), 323 deletions(-) diff --git a/docs/explanations/decisions/0020-runtime-readable-format.md b/docs/explanations/decisions/0020-runtime-readable-format.md index c080a17fe1..c7e68ec0c4 100644 --- a/docs/explanations/decisions/0020-runtime-readable-format.md +++ b/docs/explanations/decisions/0020-runtime-readable-format.md @@ -172,15 +172,15 @@ collection, while the other simply works. Values and formats are applied on the same cadence — a technique change wants both — so they belong in one file and one `apply_settings` call. `store_settings` keeps values flat -and adds one reserved key: +and adds one reserved key holding the format changes since construction: ```yaml energy: 7.0 temperature: 20.0 : : - energy: CONFIG_SIGNAL - temperature: CONFIG_SIGNAL + energy: HINTED_SIGNAL + temperature: null ``` `Settings` gains a `readable_formats` attribute, so one `retrieve_settings` returns the @@ -198,10 +198,10 @@ store-time guard is required. `<` is also not a YAML indicator, so unlike `*FORM reason and because an empty-string key makes PyYAML emit its hard-to-read explicit `? '' :` form. -Because no path can look like `<...>`, `retrieve_settings` drops *every* such key rather -than only the one it understands. So a key written by a newer version — device names, say -— is ignored by an older one instead of being reported as an unknown signal, and adding -one later needs no migration and no constant reserved up front. +Because no path can look like `<...>`, `retrieve_settings` drops every such key it does +not understand with a warning, rather than reporting it as an unknown signal. A key written +by a newer version is then ignored by an older one, and a typo of a reserved key is still +noticed. **The formats section is two levels while values are flat.** A value is intrinsic to one path; a format is a fact about an (owner, child) *pair*, and the same child can be @@ -212,34 +212,43 @@ direct child" rule, so `AreaDetector` can keep registering `driver.acquire_time` `DeviceVector` elements keep working, with no dependency on #1395 and no second migration later. -### Applying merges, and unregistering is explicit +The child path is relative to its owner, not to the root, so the owner is not repeated on +every line. A survey of dodal, haven and the NSLS-II repositories found that every +registration of something outside its owner's subtree is of an object outside the stored +Device altogether (an `__init__` argument or a `Reference`), which a root-relative path +cannot store either. -`apply_readable_formats` **merges** into what is already registered rather than replacing -it: a child the file does not mention keeps whatever format it has. +### Only changes are stored, and applying resets to the class defaults first -Replacement was tried first, because it makes technique switching automatic — loading a -profile drops what the previous one registered. It was rejected because it breaks the case -that matters more: store a file, add a signal to the Device in a later version, apply the -old file, and the new signal is silently unregistered. Signal *values* already behave the -other way round — applying a stored file leaves signals it has never heard of alone — and -formats living in the same file should not behave differently from the values beside them. +`walk_readable_formats` records only how each `StandardReadable` differs from what it +registered during construction, with a dropped child as `None`. `apply_readable_formats` +resets every `StandardReadable` in the tree to its class defaults, then applies those +changes. The key is written, even if empty, whenever the tree has a `StandardReadable`, as +an empty entry means "back to the defaults", where a missing key means "leave alone". -A child is dropped by giving it a format of `None`, a null in the stored file. Nothing -writes those nulls automatically, because storing a Device records only what it currently -registers and cannot know what some other profile registered; a profile that needs a child -dropped has to say so. The cost is that round-tripping A -> B -> A no longer drops -automatically, which is the price of not silently unregistering signals added since a file -was written. +Two alternatives were rejected: -It follows that a `StandardReadable` registering nothing gets no entry at all, since an -empty entry would be a no-op. `apply_readable_formats` resolves every path before changing -anything, so a bad path cannot leave the tree half applied. +- **Store everything, apply by replacing.** Store a file, add a signal to the Device in a + later version, apply the old file, and the new signal is silently unregistered. It also + pins the class defaults of the day into every file, so a release that changes a default + is undone by old files. +- **Store everything, apply by merging.** Fixes the new-signal problem but cannot return + from technique B to technique A: anything B changed that A does not mention stays as B + left it, and dropping a child needs a hand-written null. + +Storing changes against the class defaults avoids both: a new signal keeps its class +format, defaults stay current, and a dropped child is written as a null automatically. +Registrations made in `__init__` are part of the defaults, so one of an object outside the +stored tree is never written, and does not warn on every store. + +`apply_readable_formats` resolves every path before changing anything, so a bad path cannot +leave the tree half applied. `Settings.partition` copies the formats to **both** halves. A device-specific apply plan normally applies only the halves and never the original `Settings` — `apply_panda_settings` partitions and calls `apply_settings` twice — so putting formats on one half would let the -plan silently decide whether they were restored at all. Applying them more than once is -harmless: it touches no hardware and is idempotent. `apply_settings` applies formats after +plan silently decide whether they were restored at all. Applying them twice is harmless: it +touches no hardware and gives the same result. `apply_settings` applies formats after values, so a failed value write leaves the readable registry untouched. ## Consequences @@ -256,7 +265,8 @@ values, so a failed value write leaves the readable registry untouched. discouraged. - Deriving callables per call costs a pass over the registry on every verb. The registries are small (tens of entries), and it is what makes the format mutable. -- Registering a device that is not within the tree still works, but its format cannot be - stored, since there is no stable path for it. `walk_readable_formats` warns and skips. +- Registering a device that is not below its owner still works, but a runtime change to + its format cannot be stored, since there is no stable path for it. + `walk_readable_formats` warns and skips. - This ADR is numbered 21 rather than 20 to leave 20 for the unmerged bounded-data-logic ADR in #1367. diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md index 957ea2f0be..cf80f1816b 100644 --- a/docs/how-to/change-readable-format.md +++ b/docs/how-to/change-readable-format.md @@ -48,11 +48,9 @@ So a plan that retunes a Device can put it back without having to record what it changed. It resets that Device only, not its children — to do a whole tree: ```python -from ophyd_async.core import walk_devices +from ophyd_async.core import apply_readable_formats -for dev in walk_devices(detector).values(): - if isinstance(dev, StandardReadable): - dev.reset_readable_formats() +apply_readable_formats(detector, {}) ``` ```{note} diff --git a/docs/how-to/store-and-retrieve.md b/docs/how-to/store-and-retrieve.md index a10a890734..f4ec1931aa 100644 --- a/docs/how-to/store-and-retrieve.md +++ b/docs/how-to/store-and-retrieve.md @@ -48,34 +48,21 @@ def load_panda(panda1: HDFPanda): As well as signal values, [](#store_settings) records how each child contributes to the Device's bluesky verbs — which signals are hinted, which are configuration — and [](#apply_settings) restores that alongside the values. The two change on the same cadence, so they live in one file and are applied by one call. This means a stored file can switch a Device between techniques as well as putting its values back; see [](./change-readable-format.md) for changing a format directly. -Values stay flat in the yaml, and formats go under the reserved [](#READABLE_FORMATS_KEY), so the file stays readable and can be edited by hand: +Values stay flat in the yaml. Formats go under the reserved [](#READABLE_FORMATS_KEY), and only what has changed since the Device was constructed is recorded, so the file stays short and can be edited by hand: ```yaml energy: 7.0 temperature: 20.0 -: - : - energy: CONFIG_SIGNAL - temperature: CONFIG_SIGNAL - stats: - stats.total: HINTED_UNCACHED_SIGNAL -``` - -The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with `` ([](#ROOT_DEVICE_KEY)) standing for the Device you stored. It is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. - -```{note} -Applying **merges** into what is already registered: a child the file does not mention keeps whatever format it has. This matches how values behave — applying a stored file leaves signals it has never heard of alone — so a file stored against an older version of a Device does not silently unregister signals that have been added since. -``` - -To stop a child contributing, the file has to say so explicitly with a null: - -```yaml : : energy: HINTED_SIGNAL temperature: null + stats: + total: HINTED_UNCACHED_SIGNAL ``` -Nothing writes those nulls for you, because storing a Device records only what it currently registers and cannot know what some other profile registered. A profile that needs a child dropped has to spell it out. +The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with `` ([](#ROOT_DEVICE_KEY)) standing for the Device you stored. The inner key is the child's path from that owner. The owner is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. A null means the child has been dropped. + +Applying first resets every Device in the tree to the formats its class declares, then applies the changes in the file. So a file stored with nothing changed (`: {}`) puts the class defaults back, and a signal added to a Device class after the file was stored keeps the format the class gives it. -Both parts are optional. A file with no `` key changes no formats, so settings files saved before formats were storable keep working untouched. A file with *only* that key switches technique without writing a single value to hardware. +A file with no `` key changes no formats, so settings files saved before formats were storable keep working untouched. A file with *only* that key switches technique without writing a single value to hardware. diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 8b59ab7f6c..3c17e6f320 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -387,15 +387,8 @@ def reset_readable_formats(self) -> None: Intended for a Device that is retuned per technique, so a scan can put it back without knowing what it changed. - This resets **this Device only**, not its children; it is the - counterpart of `set_readable_format` rather than of - [](#apply_readable_formats). To reset a whole tree: - - ```python - for dev in walk_devices(detector).values(): - if isinstance(dev, StandardReadable): - dev.reset_readable_formats() - ``` + This resets **this Device only**, not its children. To reset a whole + tree use `apply_readable_formats(device, {})`. The baseline is the *class declaration*, not the last stored settings file, so this discards a technique loaded by [](#apply_settings) too. @@ -477,11 +470,13 @@ def _config_signals(device: Device) -> set[SignalR]: #: key syntax. ROOT_DEVICE_KEY = "" -#: The readable formats of a Device tree in a form that can be stored and -#: retrieved: `{path of the StandardReadable: {path of the child: format}}`, -#: where a format of `None` means "stop this child contributing". -#: Paths are dotted attribute paths from the root Device, as produced by -#: [](#walk_devices), with [](#ROOT_DEVICE_KEY) meaning the root Device itself. +#: The changes to the readable formats of a Device tree since construction, in +#: a form that can be stored and retrieved: +#: `{path of the StandardReadable: {path of the child: format}}`, where a format +#: of `None` means "stop this child contributing". +#: The owner path is a dotted attribute path from the root Device, as produced +#: by [](#walk_devices), with [](#ROOT_DEVICE_KEY) meaning the root Device +#: itself. The child path is a dotted attribute path from the owner. #: #: The outer key is needed because a format is a fact about an (owner, child) #: *pair*, not about a single Device: the same child can be registered on more @@ -493,37 +488,54 @@ def _paths_to_devices(device: Device) -> dict[str, Device]: return {ROOT_DEVICE_KEY: device, **walk_devices(device)} -def walk_readable_formats(device: Device) -> ReadableFormats: - """Retrieve the readable format of every registered child in a Device tree. +def _changed_formats( + device: StandardReadable, +) -> dict[Device, StandardReadableFormat | None]: + current = device.get_readable_formats() + default = device._default_readables # noqa: SLF001 + changed: dict[Device, StandardReadableFormat | None] = { + child: format + for child, format in current.items() + if default.get(child) is not format + } + changed.update({child: None for child in default if child not in current}) + return changed + + +def walk_readable_formats(device: Device) -> ReadableFormats | None: + """Retrieve how the readable formats in a Device tree differ from construction. Used as part of saving a Device, so that which signals are hinted or - config can be restored later. Only `StandardReadable`s that actually - register something get an entry: applying merges rather than replaces, so - an empty entry would do nothing. + config can be restored later. Only the differences from what each + `StandardReadable` registered during construction are returned, with a + child it has since dropped given a format of `None`. :param device: The root Device to walk. - :return: A [](#ReadableFormats) suitable for storing. + :return: + A [](#ReadableFormats) suitable for storing, or `None` if there is no + `StandardReadable` in the tree. """ paths_to_devices = _paths_to_devices(device) - device_to_path = {dev: path for path, dev in paths_to_devices.items()} + readables = { + path: dev + for path, dev in paths_to_devices.items() + if isinstance(dev, StandardReadable) + } + if not readables: + return None formats: ReadableFormats = {} - for path, dev in paths_to_devices.items(): - if not isinstance(dev, StandardReadable): - continue + for path, dev in readables.items(): + device_to_path = {child: p for p, child in walk_devices(dev).items()} entries: dict[str, StandardReadableFormat | None] = {} - for child, format in dev.get_readable_formats().items(): + for child, format in _changed_formats(dev).items(): child_path = device_to_path.get(child) if child_path is None: - # Registered something that is not in this tree, so it has no - # stable path to store it against. Its value is not stored - # either, as store_settings only walks the tree, so it will not - # survive a store/apply round trip at all. See - # https://github.com/bluesky/ophyd-async/issues/1402 + # Not below its owner, so it has no stable path to store it + # against. See https://github.com/bluesky/ophyd-async/issues/1402 warnings.warn( f"{dev.name or dev}: {child.name or child} is not within " - f"{device.name or device}, so neither its readable format " - "nor its value will be stored, and applying stored settings " - "will not restore it", + f"{dev.name or dev}, so the change to its readable format " + "will not be stored", stacklevel=2, ) else: @@ -534,16 +546,17 @@ def walk_readable_formats(device: Device) -> ReadableFormats: def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: - """Set the readable format of children in a Device tree. + """Reset the readable formats in a Device tree, then apply the given changes. - This **merges** into what is already registered rather than replacing it: a - child the formats do not mention keeps whatever format it has. That matches - how signal values behave, where applying a stored file leaves signals it has - never heard of alone -- so a file stored against an older version of a - Device does not silently unregister signals added since. + Every `StandardReadable` in the tree is first put back to what it + registered during construction, as + [](#StandardReadable.reset_readable_formats) does, then the changes in + `formats` are applied on top. So an empty `formats` restores the class + defaults, and a child added to a Device since `formats` was stored keeps + the format its class gives it. - To stop a child contributing, give it a format of `None` explicitly, which - in a stored file is a null: + To stop a child contributing, give it a format of `None`, which in a stored + file is a null: ```yaml : @@ -552,14 +565,10 @@ def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: temperature: null ``` - Nothing writes those nulls for you, because storing a Device records only - what it currently registers and cannot know what some other profile - registered. A profile that needs a child dropped has to say so. - - :param device: The root Device the paths in `formats` are relative to. + :param device: The root Device the owner paths in `formats` are relative to. :param formats: A [](#ReadableFormats), e.g. from [](#walk_readable_formats). :raises KeyError: If a path does not name a Device in the tree. - :raises TypeError: If a path does not name a `StandardReadable`. + :raises TypeError: If an owner path does not name a `StandardReadable`. """ devices = _paths_to_devices(device) resolved: list[ @@ -573,17 +582,21 @@ def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: raise KeyError(f"No Device at {owner_path!r} in {device.name or device}") if not isinstance(owner, StandardReadable): raise TypeError(f"{owner_path!r} is not a StandardReadable, got {owner}") + children = walk_devices(owner) readables: list[tuple[Device, StandardReadableFormat | None]] = [] for child_path, format in entries.items(): - child = devices.get(child_path) + child = children.get(child_path) if child is None: raise KeyError( - f"No Device at {child_path!r} in {device.name or device}" + f"No Device at {child_path!r} in {owner.name or owner_path}" ) readables.append( (child, None if format is None else StandardReadableFormat(format)) ) resolved.append((owner, readables)) + for dev in devices.values(): + if isinstance(dev, StandardReadable): + dev.reset_readable_formats() for owner, readables in resolved: for child, format in readables: owner.set_readable_format(child, format) diff --git a/src/ophyd_async/core/_settings.py b/src/ophyd_async/core/_settings.py index 83ae4a9aac..41ef871e54 100644 --- a/src/ophyd_async/core/_settings.py +++ b/src/ophyd_async/core/_settings.py @@ -42,11 +42,17 @@ def __init__( self.device = device self._settings = {} self.update(settings or {}) - self.readable_formats: ReadableFormats = dict(readable_formats or {}) + self.readable_formats: ReadableFormats | None = ( + None + if readable_formats is None + else {owner: dict(entries) for owner, entries in readable_formats.items()} + ) """How the Device's children contribute to its bluesky verbs. Stored and applied alongside the signal values, as the two change on - the same cadence. Applied by [](#apply_settings). + the same cadence. Applied by [](#apply_settings) with + [](#apply_readable_formats), so a dict (even an empty one) first resets + the tree to its class defaults, while `None` leaves formats alone. """ def __getitem__(self, key: SignalRW[SignalDatatypeT]) -> SignalDatatypeT: @@ -83,35 +89,33 @@ def __or__(self, other: MutableMapping[SignalRW, Any]) -> Settings[DeviceT]: """Create a new Settings that is the union of self overridden by other. Readable formats merge per (owner, child), the same granularity that - values merge at and that [](#apply_readable_formats) applies at, so - `other` only replaces the children it actually mentions. + values merge at, so `other` only replaces the children it mentions. :raises ValueError: If `other` is a `Settings` for a Device outside this one, or for a - Device below this one while carrying readable formats -- their paths - would be relative to a different root. + Device below this one while carrying readable formats. """ - formats = { - owner: dict(entries) for owner, entries in self.readable_formats.items() - } + formats = self.readable_formats if isinstance(other, Settings): if other.device is not self.device: if not self._is_in_device(other.device): msg = f"{other.device} is not a child of {self.device}" raise ValueError(msg) - if other.readable_formats: - # Paths in a ReadableFormats are relative to its own device, - # so and every dotted path would mean something - # different once merged. Re-rooting them is not implemented - # as nothing needs it yet; better to say so than to silently - # attach them to the wrong Device. + if other.readable_formats is not None: + # Re-rooting the owner paths is easy, but applying resets + # the whole tree, so a sub-device's formats would reset + # Devices it knows nothing about msg = ( f"Cannot merge readable formats from {other.device}, whose " f"paths are relative to it rather than to {self.device}" ) raise ValueError(msg) - for owner, entries in other.readable_formats.items(): - formats.setdefault(owner, {}).update(entries) + if other.readable_formats is not None: + formats = { + owner: dict(entries) for owner, entries in (formats or {}).items() + } + for owner, entries in other.readable_formats.items(): + formats.setdefault(owner, {}).update(entries) return Settings(self.device, self._settings | dict(other), formats) def partition( @@ -127,13 +131,10 @@ def partition( The first contains the signals for which the predicate returned True, and the second contains the signals for which the predicate returned False. - Both halves carry the *same* `readable_formats`. Formats are not a value - that can differ, and a device specific apply plan normally applies only - the halves and never the original Settings (see `apply_panda_settings`), - so putting them on one half would mean whichever half that plan applied - first silently decided whether formats were restored at all. Applying - them more than once is harmless: it writes to no hardware and is - idempotent. + Both halves carry the *same* `readable_formats`, as a device specific + apply plan normally applies only the halves (see + `apply_panda_settings`). Applying them twice is harmless: it writes to + no hardware and gives the same result. :example: ```python diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index d6e5129559..638d6a9536 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -65,9 +65,12 @@ def store_settings( Readable formats are stored in both cases, since they are what *determines* which signals are configuration. - Values are stored flat, keyed by dotted attribute path. Readable formats go - under the reserved [](#READABLE_FORMATS_KEY), which is not a valid Python - identifier and so cannot collide with a path. + Values are stored flat, keyed by dotted attribute path. The changes to + readable formats since construction go under the reserved + [](#READABLE_FORMATS_KEY), which is not a valid Python identifier and so + cannot collide with a path. It is written, even if empty, whenever the tree + contains a `StandardReadable`, as applying it resets formats to the class + defaults. :param provider: The provider to store the settings with. :param name: The name to store the settings under. @@ -80,9 +83,7 @@ def store_settings( signals = walk_rw_signals(device) data: dict[str, Any] = yield from _get_values_of_signals(signals) formats = walk_readable_formats(device) - if formats: - # Omitted entirely when there is nothing to say, so that a settings only - # file stays exactly as it was before formats were storable + if formats is not None: data[READABLE_FORMATS_KEY] = { owner: { child: None if format is None else format.value @@ -101,9 +102,8 @@ def retrieve_settings( If `only_config` is True, retrieve only configuration settings on `Configurable`. - A file stored before readable formats existed simply has no - [](#READABLE_FORMATS_KEY), which reads correctly as "do not change any - formats" -- so no migration or version marker is needed. + A file with no [](#READABLE_FORMATS_KEY), such as one stored before + readable formats existed, gives Settings that leave formats alone. :param provider: The provider to retrieve the settings from. :param name: The name of the settings to retrieve. @@ -116,7 +116,11 @@ def retrieve_settings( # Reserved keys first, so what is left is signal paths. Any other # key is dropped with a warning: it is either a typo of a reserved key or # written by a newer version, and a path can never look like that. - stored_formats = data.pop(READABLE_FORMATS_KEY, {}) + # Present but null, as a hand written ":" would be, still + # means reset to the class defaults + stored_formats = ( + data.pop(READABLE_FORMATS_KEY) or {} if READABLE_FORMATS_KEY in data else None + ) unknown_reserved = sorted(k for k in data if k.startswith("<") and k.endswith(">")) for key in unknown_reserved: del data[key] @@ -125,14 +129,18 @@ def retrieve_settings( f"Ignoring unknown reserved keys {unknown_reserved} in settings {name!r}", stacklevel=2, ) - readable_formats: ReadableFormats = { - owner: { - # A null means "stop this child contributing", so it is not coerced - child: None if format is None else StandardReadableFormat(format) - for child, format in entries.items() + readable_formats: ReadableFormats | None = ( + None + if stored_formats is None + else { + owner: { + # A null means "stop this child contributing", so is not coerced + child: None if format is None else StandardReadableFormat(format) + for child, format in entries.items() + } + for owner, entries in stored_formats.items() } - for owner, entries in stored_formats.items() - } + ) if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: @@ -148,9 +156,9 @@ def retrieve_settings( def apply_settings(settings: Settings) -> MsgGenerator[None]: """Set every SignalRW to the given value in Settings. If value is None ignore it. - Then apply any readable formats the Settings carries. Formats go last - because they touch no hardware and cannot fail, so a failed value write - leaves the Device's readable registry untouched. + Then apply any readable formats the Settings carries with + [](#apply_readable_formats). Formats go last because they touch no hardware + and cannot fail, so a failed value write leaves them untouched. """ signal_values = { signal: value for signal, value in settings.items() if value is not None @@ -159,7 +167,7 @@ def apply_settings(settings: Settings) -> MsgGenerator[None]: for signal, value in signal_values.items(): yield from bps.abs_set(signal, value, group="apply_settings") yield from bps.wait("apply_settings") - if settings.readable_formats: + if settings.readable_formats is not None: apply_readable_formats(settings.device, settings.readable_formats) diff --git a/tests/system_tests/tango/core/test_yaml_save.yaml b/tests/system_tests/tango/core/test_yaml_save.yaml index c7103cfd3b..a6b0ac7f08 100644 --- a/tests/system_tests/tango/core/test_yaml_save.yaml +++ b/tests/system_tests/tango/core/test_yaml_save.yaml @@ -1,3 +1,4 @@ +: {} a_bool: true a_str: test_string float64: 1.234 diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index 04b233a02c..2f056739c6 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -274,11 +274,7 @@ def my_plan(): assert ( technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL ) - # Merge, so temperature keeps the format the previous profile gave it - assert ( - technique_device.get_readable_formats().get(temperature) - is Format.CONFIG_SIGNAL - ) + assert technique_device.get_readable_formats().get(temperature) is None assert (yield from bps.rd(energy)) == 9.0 RE(my_plan()) @@ -294,7 +290,7 @@ async def test_settings_file_without_formats_leaves_them_alone( def my_plan(): settings = yield from retrieve_settings(provider, "old_style", technique_device) - assert settings.readable_formats == {} + assert settings.readable_formats is None yield from apply_settings(settings) # Values applied, formats untouched assert (yield from bps.rd(energy)) == 3.0 @@ -339,6 +335,61 @@ def my_plan(): RE(my_plan()) +class _Mono(StandardReadable): + def __init__(self, name=""): + with self.add_children_as_readables(Format.CONFIG_SIGNAL): + self.energy = soft_signal_rw(float, 7.0) + self.temperature = soft_signal_rw(float, 20.0) + super().__init__(name=name) + + +async def test_store_settings_writes_only_changes_from_class_defaults(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + mono = _Mono(name="mono") + + def stored_formats(name: str): + with open(tmp_path / f"{name}.yaml") as f: + return yaml.safe_load(f)[""] + + def my_plan(): + yield from store_settings(provider, "defaults", mono) + mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + mono.set_readable_format(mono.temperature, None) + yield from store_settings(provider, "scan", mono) + assert stored_formats("defaults") == {} + # A dropped default is written as a null + assert stored_formats("scan") == { + "": {"energy": "HINTED_SIGNAL", "temperature": None} + } + # The defaults file undoes changes it does not mention + defaults = yield from retrieve_settings(provider, "defaults", mono) + yield from apply_settings(defaults) + assert mono.get_readable_formats() == { + mono.energy: Format.CONFIG_SIGNAL, + mono.temperature: Format.CONFIG_SIGNAL, + } + scan = yield from retrieve_settings(provider, "scan", mono) + yield from apply_settings(scan) + assert mono.get_readable_formats() == {mono.energy: Format.HINTED_SIGNAL} + + RE(my_plan()) + + +async def test_null_formats_key_resets_to_class_defaults(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + mono = _Mono(name="mono") + mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + (tmp_path / "reset.yaml").write_text(":\n") + + def my_plan(): + settings = yield from retrieve_settings(provider, "reset", mono) + assert settings.readable_formats == {} + yield from apply_settings(settings) + assert mono.get_readable_formats().get(mono.energy) is Format.CONFIG_SIGNAL + + RE(my_plan()) + + async def test_retrieve_settings_warns_on_unknown_reserved_keys( RE, technique_device, tmp_path ): @@ -383,6 +434,8 @@ async def test_store_settings_walks_nested_devices(RE, tmp_path): inner.set_readable_format(inner.sig, Format.HINTED_SIGNAL) outer.set_readable_format(outer.top, Format.CONFIG_SIGNAL) outer.set_readable_format(inner, Format.CHILD) + # The same signal registered on a second owner, from deeper in its tree + outer.set_readable_format(inner.sig, Format.CONFIG_SIGNAL) def my_plan(): yield from store_settings(provider, "nested", outer) @@ -391,8 +444,13 @@ def my_plan(): "top": 2.0, "inner.sig": 1.0, "": { - "": {"top": "CONFIG_SIGNAL", "inner": "CHILD"}, - "inner": {"inner.sig": "HINTED_SIGNAL"}, + "": { + "top": "CONFIG_SIGNAL", + "inner": "CHILD", + "inner.sig": "CONFIG_SIGNAL", + }, + # Child paths are relative to their owner + "inner": {"sig": "HINTED_SIGNAL"}, }, } @@ -468,7 +526,7 @@ def test_settings_or_rejects_formats_from_a_sub_device(technique_device): with pytest.raises(ValueError, match="paths are relative to it"): base | sub # Without formats it is just a value merge, which is fine - assert (base | Settings(technique_device)).readable_formats == {} + assert (base | Settings(technique_device)).readable_formats is None def test_settings_partition_carries_formats_to_both_halves(technique_device): @@ -484,11 +542,12 @@ def test_settings_partition_carries_formats_to_both_halves(technique_device): assert b.readable_formats == settings.readable_formats -async def test_store_settings_omits_a_readable_that_registers_nothing(RE, tmp_path): - """An owner with no registered children is a no-op when applied. +async def test_store_settings_writes_empty_formats_for_an_unchanged_readable( + RE, tmp_path +): + """The key is still written, as applying it resets runtime changes. - This is the shape of TangoTestDevice, and is why its golden file needs no - formats key. + This is the shape of TangoTestDevice. """ provider = YamlSettingsProvider(tmp_path) device = StandardReadable(name="dev") @@ -498,7 +557,7 @@ async def test_store_settings_omits_a_readable_that_registers_nothing(RE, tmp_pa def my_plan(): yield from store_settings(provider, "empty", device) with open(tmp_path / "empty.yaml") as f: - assert yaml.safe_load(f) == {"sig": 1.0} + assert yaml.safe_load(f) == {"sig": 1.0, "": {}} RE(my_plan()) diff --git a/tests/unit_tests/test_data/test_yaml_config_save.yaml b/tests/unit_tests/test_data/test_yaml_config_save.yaml index f2245dda43..1c981f1ef5 100644 --- a/tests/unit_tests/test_data/test_yaml_config_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_config_save.yaml @@ -1,27 +1,4 @@ -: - : - a_bool: CONFIG_SIGNAL - a_enum: CONFIG_SIGNAL - a_float: CONFIG_SIGNAL - a_int: CONFIG_SIGNAL - a_str: CONFIG_SIGNAL - a_subset_enum: CONFIG_SIGNAL - a_superset_enum: CONFIG_SIGNAL - boola: CONFIG_SIGNAL - enuma: CONFIG_SIGNAL - float32a: CONFIG_SIGNAL - float64a: CONFIG_SIGNAL - int16a: CONFIG_SIGNAL - int32a: CONFIG_SIGNAL - int64a: CONFIG_SIGNAL - int8a: CONFIG_SIGNAL - ndarray: CONFIG_SIGNAL - stra: CONFIG_SIGNAL - table: CONFIG_SIGNAL - uint16a: CONFIG_SIGNAL - uint32a: CONFIG_SIGNAL - uint64a: CONFIG_SIGNAL - uint8a: CONFIG_SIGNAL +: {} a_bool: true a_enum: Bbb a_float: 1.234 @@ -31,35 +8,53 @@ a_subset_enum: Bbb a_superset_enum: Bbb boola: [false, false, true] enuma: -- Aaa -- Ccc -float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, - 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] -float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, - 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] + - Aaa + - Ccc +float32a: + [ + -3.4028234663852886e+38, + 3.4028234663852886e+38, + 1.1754943508222875e-38, + 1.401298464324817e-45, + 0.0, + 1.2339999675750732, + 234000.0, + 3.4499998946557753e-06, + ] +float64a: + [ + -1.7976931348623157e+308, + 1.7976931348623157e+308, + 2.2250738585072014e-308, + 5.0e-324, + 0.0, + 1.234, + 234000.0, + 3.45e-06, + ] int16a: [-32768, 32767, 0, 1, 2, 3, 4] int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] int8a: [-128, 127, 0, 1, 2, 3, 4] ndarray: [[1, 2, 3], [4, 5, 6]] stra: -- one -- two -- three + - one + - two + - three table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar uint16a: [0, 65535, 0, 1, 2, 3, 4] uint32a: [0, 4294967295, 0, 1, 2, 3, 4] uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] diff --git a/tests/unit_tests/test_data/test_yaml_save.yaml b/tests/unit_tests/test_data/test_yaml_save.yaml index af0e7ab207..1d2d7047bb 100644 --- a/tests/unit_tests/test_data/test_yaml_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_save.yaml @@ -1,73 +1,4 @@ -: - child: - child.a_bool: CONFIG_SIGNAL - child.a_enum: CONFIG_SIGNAL - child.a_float: CONFIG_SIGNAL - child.a_int: CONFIG_SIGNAL - child.a_str: CONFIG_SIGNAL - child.a_subset_enum: CONFIG_SIGNAL - child.a_superset_enum: CONFIG_SIGNAL - child.boola: CONFIG_SIGNAL - child.enuma: CONFIG_SIGNAL - child.float32a: CONFIG_SIGNAL - child.float64a: CONFIG_SIGNAL - child.int16a: CONFIG_SIGNAL - child.int32a: CONFIG_SIGNAL - child.int64a: CONFIG_SIGNAL - child.int8a: CONFIG_SIGNAL - child.ndarray: CONFIG_SIGNAL - child.stra: CONFIG_SIGNAL - child.table: CONFIG_SIGNAL - child.uint16a: CONFIG_SIGNAL - child.uint32a: CONFIG_SIGNAL - child.uint64a: CONFIG_SIGNAL - child.uint8a: CONFIG_SIGNAL - vector.1: - vector.1.a_bool: CONFIG_SIGNAL - vector.1.a_enum: CONFIG_SIGNAL - vector.1.a_float: CONFIG_SIGNAL - vector.1.a_int: CONFIG_SIGNAL - vector.1.a_str: CONFIG_SIGNAL - vector.1.a_subset_enum: CONFIG_SIGNAL - vector.1.a_superset_enum: CONFIG_SIGNAL - vector.1.boola: CONFIG_SIGNAL - vector.1.enuma: CONFIG_SIGNAL - vector.1.float32a: CONFIG_SIGNAL - vector.1.float64a: CONFIG_SIGNAL - vector.1.int16a: CONFIG_SIGNAL - vector.1.int32a: CONFIG_SIGNAL - vector.1.int64a: CONFIG_SIGNAL - vector.1.int8a: CONFIG_SIGNAL - vector.1.ndarray: CONFIG_SIGNAL - vector.1.stra: CONFIG_SIGNAL - vector.1.table: CONFIG_SIGNAL - vector.1.uint16a: CONFIG_SIGNAL - vector.1.uint32a: CONFIG_SIGNAL - vector.1.uint64a: CONFIG_SIGNAL - vector.1.uint8a: CONFIG_SIGNAL - vector.3: - vector.3.a_bool: CONFIG_SIGNAL - vector.3.a_enum: CONFIG_SIGNAL - vector.3.a_float: CONFIG_SIGNAL - vector.3.a_int: CONFIG_SIGNAL - vector.3.a_str: CONFIG_SIGNAL - vector.3.a_subset_enum: CONFIG_SIGNAL - vector.3.a_superset_enum: CONFIG_SIGNAL - vector.3.boola: CONFIG_SIGNAL - vector.3.enuma: CONFIG_SIGNAL - vector.3.float32a: CONFIG_SIGNAL - vector.3.float64a: CONFIG_SIGNAL - vector.3.int16a: CONFIG_SIGNAL - vector.3.int32a: CONFIG_SIGNAL - vector.3.int64a: CONFIG_SIGNAL - vector.3.int8a: CONFIG_SIGNAL - vector.3.ndarray: CONFIG_SIGNAL - vector.3.stra: CONFIG_SIGNAL - vector.3.table: CONFIG_SIGNAL - vector.3.uint16a: CONFIG_SIGNAL - vector.3.uint32a: CONFIG_SIGNAL - vector.3.uint64a: CONFIG_SIGNAL - vector.3.uint8a: CONFIG_SIGNAL +: {} _sig_rw: Top level private SignalRW child.a_bool: true child.a_enum: Bbb @@ -78,35 +9,53 @@ child.a_subset_enum: Bbb child.a_superset_enum: Bbb child.boola: [false, false, true] child.enuma: -- Aaa -- Ccc -child.float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, - 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] -child.float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, - 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] + - Aaa + - Ccc +child.float32a: + [ + -3.4028234663852886e+38, + 3.4028234663852886e+38, + 1.1754943508222875e-38, + 1.401298464324817e-45, + 0.0, + 1.2339999675750732, + 234000.0, + 3.4499998946557753e-06, + ] +child.float64a: + [ + -1.7976931348623157e+308, + 1.7976931348623158e+308, + 2.2250738585072014e-308, + 5.0e-324, + 0.0, + 1.234, + 234000.0, + 3.45e-06, + ] child.int16a: [-32768, 32767, 0, 1, 2, 3, 4] child.int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] child.int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] child.int8a: [-128, 127, 0, 1, 2, 3, 4] child.ndarray: [[1, 2, 3], [4, 5, 6]] child.stra: -- one -- two -- three + - one + - two + - three child.table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar child.uint16a: [0, 65535, 0, 1, 2, 3, 4] child.uint32a: [0, 4294967295, 0, 1, 2, 3, 4] child.uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] @@ -121,35 +70,53 @@ vector.1.a_subset_enum: Bbb vector.1.a_superset_enum: Bbb vector.1.boola: [false, false, true] vector.1.enuma: -- Aaa -- Ccc -vector.1.float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, - 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] -vector.1.float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, - 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] + - Aaa + - Ccc +vector.1.float32a: + [ + -3.4028234663852886e+38, + 3.4028234663852886e+38, + 1.1754943508222875e-38, + 1.401298464324817e-45, + 0.0, + 1.2339999675750732, + 234000.0, + 3.4499998946557753e-06, + ] +vector.1.float64a: + [ + -1.7976931348623157e+308, + 1.7976931348623157e+308, + 2.2250738585072014e-308, + 5.0e-324, + 0.0, + 1.234, + 234000.0, + 3.45e-06, + ] vector.1.int16a: [-32768, 32767, 0, 1, 2, 3, 4] vector.1.int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] vector.1.int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] vector.1.int8a: [-128, 127, 0, 1, 2, 3, 4] vector.1.ndarray: [[1, 2, 3], [4, 5, 6]] vector.1.stra: -- one -- two -- three + - one + - two + - three vector.1.table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar vector.1.uint16a: [0, 65535, 0, 1, 2, 3, 4] vector.1.uint32a: [0, 4294967295, 0, 1, 2, 3, 4] vector.1.uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] @@ -163,35 +130,53 @@ vector.3.a_subset_enum: Bbb vector.3.a_superset_enum: Bbb vector.3.boola: [false, false, true] vector.3.enuma: -- Aaa -- Ccc -vector.3.float32a: [-3.4028234663852886e+38, 3.4028234663852886e+38, 1.1754943508222875e-38, - 1.401298464324817e-45, 0.0, 1.2339999675750732, 234000.0, 3.4499998946557753e-06] -vector.3.float64a: [-1.7976931348623157e+308, 1.7976931348623157e+308, 2.2250738585072014e-308, - 5.0e-324, 0.0, 1.234, 234000.0, 3.45e-06] + - Aaa + - Ccc +vector.3.float32a: + [ + -3.4028234663852886e+38, + 3.4028234663852886e+38, + 1.1754943508222875e-38, + 1.401298464324817e-45, + 0.0, + 1.2339999675750732, + 234000.0, + 3.4499998946557753e-06, + ] +vector.3.float64a: + [ + -1.7976931348623157e+308, + 1.7976931348623157e+308, + 2.2250738585072014e-308, + 5.0e-324, + 0.0, + 1.234, + 234000.0, + 3.45e-06, + ] vector.3.int16a: [-32768, 32767, 0, 1, 2, 3, 4] vector.3.int32a: [-2147483648, 2147483647, 0, 1, 2, 3, 4] vector.3.int64a: [-9223372036854775808, 9223372036854775807, 0, 1, 2, 3, 4] vector.3.int8a: [-128, 127, 0, 1, 2, 3, 4] vector.3.ndarray: [[1, 2, 3], [4, 5, 6]] vector.3.stra: -- one -- two -- three + - one + - two + - three vector.3.table: a_bool: [false, false, true, true] a_enum: - - Aaa - - Bbb - - Aaa - - Ccc + - Aaa + - Bbb + - Aaa + - Ccc a_float: [1.8, 8.2, -6.0, 32.9887] a_int: [1, 8, -9, 32] a_str: - - Hello - - World - - Foo - - Bar + - Hello + - World + - Foo + - Bar vector.3.uint16a: [0, 65535, 0, 1, 2, 3, 4] vector.3.uint32a: [0, 4294967295, 0, 1, 2, 3, 4] vector.3.uint64a: [0, 18446744073709551615, 0, 1, 2, 3, 4] From 3f835446d10223bfc7a020c24186a14583f105f2 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 9 Oct 2026 12:43:56 +0000 Subject: [PATCH 65/67] refactor: inline the staged formats next to the comment that explains them Assisted-by: Claude Opus 5.5 --- src/ophyd_async/core/_readable.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 3c17e6f320..e1d52da4e3 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -60,13 +60,6 @@ def __call__(self, parent: Device, child: Device): parent.set_readable_format(child, self) -#: The formats whose devices take part in `stage()`/`unstage()`, if they are -#: `Stageable` at all. -_STAGED_FORMATS = frozenset( - {StandardReadableFormat.HINTED_SIGNAL, StandardReadableFormat.CHILD} -) - - class _SealDefaultFormatsMeta(type(_StandardBase)): # type: ignore[misc] """Record the formats a `StandardReadable` was constructed with. @@ -170,8 +163,12 @@ def _signals_to_stage(self) -> Iterator[AsyncStageable]: # HINTED_SIGNAL stages so that caching is set up and read() is fast; a # CHILD stages because it may have staging of its own. The uncached and # config formats deliberately do not. + staged_formats = ( + StandardReadableFormat.HINTED_SIGNAL, + StandardReadableFormat.CHILD, + ) for device, format in self._readables.items(): - if format in _STAGED_FORMATS and isinstance(device, AsyncStageable): + if format in staged_formats and isinstance(device, AsyncStageable): yield device async def _formatted_read(self) -> dict[str, Reading]: From 28da2d4df5635429b8e53b30382abf9be07206b4 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 12 Oct 2026 13:59:08 +0000 Subject: [PATCH 66/67] refactor: hold readable formats in a validating mapping, match exhaustively The readable registry becomes a MutableMapping[Device, Format] whose __setitem__ validates the entry, as Settings does, and whose entries() yields typed (device, format) tuples for the verbs to match on. Enable pyright's reportMatchNotExhaustive so a new format cannot be silently ignored by a match. _config_signals moves next to StandardDetector and uses the public get_readable_formats(). Assisted-by: Claude Opus 5.5 --- .../decisions/0020-runtime-readable-format.md | 11 +- pyproject.toml | 1 + src/ophyd_async/core/_detector.py | 50 +++- src/ophyd_async/core/_readable.py | 232 +++++++++++------- .../tango/core/_tango_transport.py | 4 + 5 files changed, 201 insertions(+), 97 deletions(-) diff --git a/docs/explanations/decisions/0020-runtime-readable-format.md b/docs/explanations/decisions/0020-runtime-readable-format.md index c7e68ec0c4..eff77143cb 100644 --- a/docs/explanations/decisions/0020-runtime-readable-format.md +++ b/docs/explanations/decisions/0020-runtime-readable-format.md @@ -59,10 +59,13 @@ a driver, a plugin list and a path provider. ### The registry is a `{device: format}` dict -`StandardReadable._readables` maps each registered child to its format, in registration -order, and the callables for each verb are derived on demand. `add_readables` becomes a -thin wrapper over a new `set_readable_format(device, format)`, and `format=None` removes -the device. `get_readable_formats()` returns a copy. +`StandardReadable._readables` is a `MutableMapping[Device, StandardReadableFormat]` that +maps each registered child to its format, in registration order, and the callables for +each verb are derived on demand. Its `__setitem__` validates, as `Settings` does for its +keys: a signal format is only accepted for a `SignalR`, so the verbs can match on +`(signal, format)` entries already typed. `add_readables` becomes a thin wrapper over a +new `set_readable_format(device, format)`, and `format=None` removes the device. +`get_readable_formats()` returns a copy. Keying by `Device` is identity keying, which is what a registry of children needs: `Device` does not override `__eq__`, and while `DeviceVector` and `DeviceMap` inherit a value based diff --git a/pyproject.toml b/pyproject.toml index b387728898..21836bc59b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ version_file = "src/ophyd_async/_version.py" [tool.pyright] typeCheckingMode = "standard" reportMissingImports = false # Ignore missing stubs in imported modules +reportMatchNotExhaustive = true [tool.pytest.ini_options] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 64019e912b..7ae51e8ad4 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -28,12 +28,9 @@ PageableDataProvider, StreamableDataProvider, ) +from ._device import Device from ._flyable import FlyableLogic, StandardFlyable -from ._readable import ( - StandardReadable, - StandardReadableFormat, - _config_signals, -) +from ._readable import StandardReadable, StandardReadableFormat from ._settings import Settings from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value from ._status import AsyncStatus, WatchableAsyncStatus @@ -885,6 +882,41 @@ async def _wait_for_collections( await self.acquire_logic.wait_for_idle() +def _config_signals(device: Device) -> set[SignalR]: + """Return every `Signal` contributing to a Device's `read_configuration()`. + + Recurses into children registered as [](#StandardReadableFormat.CHILD), so + a Device that registers a sub-Device whole picks up whatever that + sub-Device declares as configuration. A Device that is not a + `StandardReadable` contributes nothing. + + Unlike `walk_config_signals` this reads the registry rather than calling + `read_configuration()`, so it does no I/O and includes read-only `SignalR`s + as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is + exactly the signal an areaDetector trigger logic needs for deadtime. + + Deliberately private and unexported: #1367 restructures detector logic, so + what the public shape of this should be is not settled yet. + """ + signals: set[SignalR] = set() + if not isinstance(device, StandardReadable): + return signals + for child, format in device.get_readable_formats().items(): + match format: + case StandardReadableFormat.CONFIG_SIGNAL: + # set_readable_format only accepts a SignalR for a signal format + signals.add(cast(SignalR, child)) + case StandardReadableFormat.CHILD: + signals |= _config_signals(child) + case ( + StandardReadableFormat.HINTED_SIGNAL + | StandardReadableFormat.UNCACHED_SIGNAL + | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + ): + pass + return signals + + class StandardDetector( StandardReadable, StandardFlyable[TriggerInfo, _FlyCtx], @@ -1132,11 +1164,15 @@ async def _collect_asset_docs( ): yield doc - async def _collect_pages(self) -> AsyncIterator[PartialEventPage]: + async def _collect_pages( + self, index: int | None = None + ) -> AsyncIterator[PartialEventPage]: # Bound as collect_pages when there is a paging provider data = self.logic.prepared_data cpe = data.collections_per_event - index = await self.get_index() + if index is None: + # As in _collect_asset_docs, so detectors can collect in step + index = await self.get_index() for data_provider in data.pageable: async for page in data_provider.make_pages( collections_written=index * cpe, diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index e1d52da4e3..ab45536936 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -1,10 +1,18 @@ import asyncio import itertools import warnings -from collections.abc import Awaitable, Callable, Generator, Iterator, Sequence +from collections.abc import ( + Awaitable, + Callable, + Generator, + Iterator, + MutableMapping, + Sequence, +) from contextlib import contextmanager from enum import Enum from functools import cached_property +from typing import Literal, cast from bluesky.protocols import HasHints, Hints, Reading from event_model import DataKey @@ -16,12 +24,6 @@ from ._utils import merge_gathered_dicts -def _as_signal_r(device: Device) -> SignalR: - if not isinstance(device, SignalR): - raise TypeError(f"{device} is not a SignalR") - return device - - class StandardReadableFormat(Enum): """Declare how a `Device` should contribute to the `StandardReadable` verbs.""" @@ -60,6 +62,64 @@ def __call__(self, parent: Device, child: Device): parent.set_readable_format(child, self) +# Short alias for the case statements below +Format = StandardReadableFormat + +_SignalFormat = Literal[ + Format.CONFIG_SIGNAL, + Format.HINTED_SIGNAL, + Format.UNCACHED_SIGNAL, + Format.HINTED_UNCACHED_SIGNAL, +] +_ReadableEntry = tuple[Device, Literal[Format.CHILD]] | tuple[SignalR, _SignalFormat] + + +class _Readables(MutableMapping[Device, StandardReadableFormat]): + """The registered children and their formats, in registration order. + + Only a `SignalR` may be given a signal format, which is what lets + `entries()` hand each one to the verbs already typed. + """ + + def __init__(self) -> None: + self._formats: dict[Device, StandardReadableFormat] = {} + + def __getitem__(self, device: Device) -> StandardReadableFormat: + return self._formats[device] + + def __setitem__(self, device: Device, format: StandardReadableFormat) -> None: + if not isinstance(format, StandardReadableFormat): + raise TypeError(f"{format} is not a StandardReadableFormat") + match format: + case Format.CHILD: + pass + case ( + Format.CONFIG_SIGNAL + | Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL + ): + # Fails type checking unless these are exactly _SignalFormat, + # which is what makes the cast in entries() true + _: _SignalFormat = format + if not isinstance(device, SignalR): + raise TypeError(f"{device} is not a SignalR") + self._formats[device] = format + + def __delitem__(self, device: Device) -> None: + del self._formats[device] + + def __iter__(self) -> Iterator[Device]: + return iter(self._formats) + + def __len__(self) -> int: + return len(self._formats) + + def entries(self) -> list[_ReadableEntry]: + """Each child and its format, typed by what `__setitem__` allowed.""" + return cast(list[_ReadableEntry], list(self._formats.items())) + + class _SealDefaultFormatsMeta(type(_StandardBase)): # type: ignore[misc] """Record the formats a `StandardReadable` was constructed with. @@ -107,7 +167,7 @@ class StandardReadable( """ @cached_property - def _readables(self) -> dict[Device, StandardReadableFormat]: + def _readables(self) -> _Readables: """The registered children and their formats, in registration order. A `cached_property` rather than an attribute set in `__init__`, because @@ -122,7 +182,7 @@ def _readables(self) -> dict[Device, StandardReadableFormat]: `Mapping`, they hash by `id()`. That is injective over live objects, so two distinct devices never share a hash and `__eq__` is never reached. """ - return {} + return _Readables() # The formats declared during construction, sealed by the metaclass once # __init__ has fully returned. Only ever replaced, never mutated in place, @@ -163,84 +223,116 @@ def _signals_to_stage(self) -> Iterator[AsyncStageable]: # HINTED_SIGNAL stages so that caching is set up and read() is fast; a # CHILD stages because it may have staging of its own. The uncached and # config formats deliberately do not. - staged_formats = ( - StandardReadableFormat.HINTED_SIGNAL, - StandardReadableFormat.CHILD, - ) - for device, format in self._readables.items(): - if format in staged_formats and isinstance(device, AsyncStageable): - yield device + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncStageable): + yield device + case (signal, Format.HINTED_SIGNAL): + yield signal + case ( + _, + Format.CONFIG_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass async def _formatted_read(self) -> dict[str, Reading]: """What the registered children contribute to `read()`.""" coros = [] - for device, format in self._readables.items(): - match format: - case StandardReadableFormat.CHILD if isinstance(device, AsyncReadable): - coros.append(device.read()) - case StandardReadableFormat.HINTED_SIGNAL: - coros.append(_as_signal_r(device).read()) + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncReadable): + coros.append(device.read()) + case (signal, Format.HINTED_SIGNAL): + coros.append(signal.read()) case ( - StandardReadableFormat.UNCACHED_SIGNAL - | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + signal, + Format.UNCACHED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, ): - coros.append(_as_signal_r(device).read(cached=False)) + coros.append(signal.read(cached=False)) + case (_, Format.CONFIG_SIGNAL): + pass return await merge_gathered_dicts(coros) async def _formatted_describe(self) -> dict[str, DataKey]: """What the registered children contribute to `describe()`.""" coros = [] - for device, format in self._readables.items(): - match format: - case StandardReadableFormat.CHILD if isinstance(device, AsyncReadable): - coros.append(device.describe()) + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncReadable): + coros.append(device.describe()) case ( - StandardReadableFormat.HINTED_SIGNAL - | StandardReadableFormat.UNCACHED_SIGNAL - | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + signal, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, ): - coros.append(_as_signal_r(device).describe()) + coros.append(signal.describe()) + case (_, Format.CONFIG_SIGNAL): + pass return await merge_gathered_dicts(coros) async def _formatted_read_configuration(self) -> dict[str, Reading]: """What the registered children contribute to `read_configuration()`.""" coros = [] - for device, format in self._readables.items(): - match format: - case StandardReadableFormat.CHILD if isinstance( - device, AsyncConfigurable + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncConfigurable): + coros.append(device.read_configuration()) + case (signal, Format.CONFIG_SIGNAL): + coros.append(signal.read()) + case ( + _, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, ): - coros.append(device.read_configuration()) - case StandardReadableFormat.CONFIG_SIGNAL: - coros.append(_as_signal_r(device).read()) + pass return await merge_gathered_dicts(coros) async def _formatted_describe_configuration(self) -> dict[str, DataKey]: """What the registered children contribute to `describe_configuration()`.""" coros = [] - for device, format in self._readables.items(): - match format: - case StandardReadableFormat.CHILD if isinstance( - device, AsyncConfigurable + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncConfigurable): + coros.append(device.describe_configuration()) + case (signal, Format.CONFIG_SIGNAL): + coros.append(signal.describe()) + case ( + _, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, ): - coros.append(device.describe_configuration()) - case StandardReadableFormat.CONFIG_SIGNAL: - coros.append(_as_signal_r(device).describe()) + pass return await merge_gathered_dicts(coros) def _formatted_hints(self) -> Iterator[Hints]: """What the registered children contribute to `hints`.""" - for device, format in self._readables.items(): - match format: - case StandardReadableFormat.CHILD if isinstance(device, HasHints): - yield device.hints + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, HasHints): + yield device.hints case ( - StandardReadableFormat.HINTED_SIGNAL - | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + signal, + Format.HINTED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, ): # An unnamed signal has nothing to put in `fields`, but is # still hinted, so it contributes the key and no name - yield {"fields": [device.name] if device.name else []} + yield {"fields": [signal.name] if signal.name else []} + case ( + _, + Format.CONFIG_SIGNAL | Format.UNCACHED_SIGNAL, + ): + pass async def describe_configuration(self) -> dict[str, DataKey]: return await merge_gathered_dicts( @@ -359,11 +451,6 @@ def set_readable_format( second contribution. :raises TypeError: If a signal-only format is given a non-`SignalR` """ - if format is not None: - if not isinstance(format, StandardReadableFormat): - raise TypeError(f"{format} is not a StandardReadableFormat") - if format is not StandardReadableFormat.CHILD: - _as_signal_r(device) if format is None: self._readables.pop(device, None) else: @@ -427,33 +514,6 @@ def add_readables( self.set_readable_format(device, format) -def _config_signals(device: Device) -> set[SignalR]: - """Return every `Signal` contributing to a Device's `read_configuration()`. - - Recurses into children registered as [](#StandardReadableFormat.CHILD), so - a Device that registers a sub-Device whole picks up whatever that - sub-Device declares as configuration. A Device that is not a - `StandardReadable` contributes nothing. - - Unlike `walk_config_signals` this reads the registry rather than calling - `read_configuration()`, so it does no I/O and includes read-only `SignalR`s - as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is - exactly the signal an areaDetector trigger logic needs for deadtime. - - Deliberately private and unexported: #1367 restructures detector logic, so - what the public shape of this should be is not settled yet. - """ - signals: set[SignalR] = set() - if not isinstance(device, StandardReadable): - return signals - for child, format in device._readables.items(): # noqa: SLF001 - if format is StandardReadableFormat.CONFIG_SIGNAL: - signals.add(_as_signal_r(child)) - elif format is StandardReadableFormat.CHILD: - signals |= _config_signals(child) - return signals - - #: Reserved key under which [](#store_settings) writes readable formats. Chosen #: so that it cannot be a dotted attribute path: `<` cannot start a Python #: identifier, so no attribute assignment can produce a colliding key, and diff --git a/src/ophyd_async/tango/core/_tango_transport.py b/src/ophyd_async/tango/core/_tango_transport.py index c202f2c9f5..017fe3f364 100644 --- a/src/ophyd_async/tango/core/_tango_transport.py +++ b/src/ophyd_async/tango/core/_tango_transport.py @@ -749,6 +749,8 @@ def make_converter(info: AttributeConfig | CommandConfig, datatype) -> TangoConv AttrDataFormat.IMAGE, ]: return TangoDevStateArrayConverter() + case _: + pass else: # command info match info.in_type: case CmdArgType.DevState: @@ -765,6 +767,8 @@ def make_converter(info: AttributeConfig | CommandConfig, datatype) -> TangoConv return TangoLongStringTableConverter() case CmdArgType.DevVarDoubleStringArray: return TangoDoubleStringTableConverter() + case _: + pass # default case return trivial converter return TangoConverter() From e004992a970ca51f2ff176e4ed5b843bc64ba2cc Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Mon, 12 Oct 2026 14:44:09 +0000 Subject: [PATCH 67/67] refactor: add StandardReadable.get_readable_format_changes() Settings retrieval needs the formats that differ from construction. Expose that as a method rather than reaching into _default_readables from a module helper. Assisted-by: Claude Opus 5.5 --- src/ophyd_async/core/_readable.py | 33 ++++++++++++++------------ tests/unit_tests/core/test_readable.py | 6 +++++ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index ab45536936..8a2fe0e7db 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -490,6 +490,23 @@ def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: """ return dict(self._readables) + def get_readable_format_changes( + self, + ) -> dict[Device, StandardReadableFormat | None]: + """Return the formats that differ from those declared at construction. + + A child registered at construction but since dropped has a format of `None`. + """ + changes: dict[Device, StandardReadableFormat | None] = { + child: format + for child, format in self._readables.items() + if self._default_readables.get(child) is not format + } + for child in self._default_readables: + if child not in self._readables: + changes[child] = None + return changes + # Back compat - delete before 1.0 def add_readables( self, @@ -545,20 +562,6 @@ def _paths_to_devices(device: Device) -> dict[str, Device]: return {ROOT_DEVICE_KEY: device, **walk_devices(device)} -def _changed_formats( - device: StandardReadable, -) -> dict[Device, StandardReadableFormat | None]: - current = device.get_readable_formats() - default = device._default_readables # noqa: SLF001 - changed: dict[Device, StandardReadableFormat | None] = { - child: format - for child, format in current.items() - if default.get(child) is not format - } - changed.update({child: None for child in default if child not in current}) - return changed - - def walk_readable_formats(device: Device) -> ReadableFormats | None: """Retrieve how the readable formats in a Device tree differ from construction. @@ -584,7 +587,7 @@ def walk_readable_formats(device: Device) -> ReadableFormats | None: for path, dev in readables.items(): device_to_path = {child: p for p, child in walk_devices(dev).items()} entries: dict[str, StandardReadableFormat | None] = {} - for child, format in _changed_formats(dev).items(): + for child, format in dev.get_readable_format_changes().items(): child_path = device_to_path.get(child) if child_path is None: # Not below its owner, so it has no stable path to store it diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 0067353c23..469ff39d1f 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -354,9 +354,15 @@ def __init__(self, name=""): dev.energy: Format.HINTED_SIGNAL, dev.extra: Format.HINTED_SIGNAL, } + assert dev.get_readable_format_changes() == { + dev.energy: Format.HINTED_SIGNAL, + dev.extra: Format.HINTED_SIGNAL, + dev.temp: None, + } dev.reset_readable_formats() assert dev.get_readable_formats() == declared + assert dev.get_readable_format_changes() == {} # Repeatable, and the baseline is not consumed dev.set_readable_format(dev.energy, None) dev.reset_readable_formats()