From 0f8a029c58078ba4d741d3561b2f87899f9e19f1 Mon Sep 17 00:00:00 2001 From: "Tom C (DLS)" <101418278+coretl@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:55:36 +0000 Subject: [PATCH 01/25] Bump example-services to main and fail loudly when compose up fails (#1470) example-services 3c0274b includes services/epics-opis/compose.yml twice. Compose 5.x (Docker 29, ubuntu-26.04) rejects the merged duplicate security_opt, so `docker compose up` exits 1 and no container starts. The ca_gateway fixture read compose's output to EOF and yielded anyway, so the tests failed later with "Connection refused". Bump to 9e0faf7, which removes the duplicate include. docker_composer now raises with compose's output when it exits non-zero before printing the ready line. Assisted-by: Claude Sonnet 5.5 --- example-services | 2 +- tests/conftest.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/example-services b/example-services index 3c0274b244..9e0faf7cbe 160000 --- a/example-services +++ b/example-services @@ -1 +1 @@ -Subproject commit 3c0274b244d28bc6e197f9e3610355cc7c0cc5d3 +Subproject commit 9e0faf7cbea7e40ea63d857f046f61675b35b907 diff --git a/tests/conftest.py b/tests/conftest.py index 91a440abe1..b49c963d09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -337,8 +337,10 @@ def inner_docker_composer( if ready_log_line is not None: try: assert process.stdout is not None + output: list[str] = [] for line in process.stdout: print(line, end="") + output.append(line) if ready_log_line in line: break if ( @@ -348,6 +350,13 @@ def inner_docker_composer( raise TimeoutError( f"docker compose with args {docker_args} timed out" ) + else: + # Output ended without the ready line, so compose has exited + if process.wait() != 0: + raise RuntimeError( + f"docker compose with args {docker_args} exited " + f"with code {process.returncode}:\n{''.join(output)}" + ) except Exception: process.terminate() raise @@ -440,7 +449,7 @@ def bl01t_di_cam_01(ca_gateway, docker_composer): # ever on a cold start, where the gap is wide enough to lose the race. # `Acquire 1` is the *last* command in that block and the commands run # in order, so seeing it echoed means the whole block has run. That ties - # us to a pinned submodule's config (example-services, tag 2025.8.2): if + # us to a pinned submodule's config (example-services): if # a command is ever appended after it, this goes back to being too early. ready_log_line="dbpf BL01T-DI-CAM-01:DET:Acquire 1", ) From 06b3283eee858371579253b1446a4bf4d0f50eaf Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 30 Oct 2026 17:04:36 +0000 Subject: [PATCH 02/25] WIP: flyable/movable/status: single logic, fly generator, ensure_stopped Split cut F of detector-rederive (d0872e185) onto origin/main: the StandardMovable/StandardFlyable logic rework, the fly() generator, ensure_stopped(), StoppedError/TransitionError and the motor, pmac, panda and sim motor ports, with ADRs 0021 and 0022. The old detector keeps running on it unchanged. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- CHANGELOG.md | 51 + CLAUDE.md | 4 +- docs/conf.py | 6 +- .../decisions/0017-standard-movable.md | 4 + .../decisions/0021-single-logic-property.md | 78 ++ .../0022-fly-generator-and-ensure-stopped.md | 44 + .../explanations/declarative-vs-procedural.md | 4 +- docs/explanations/when-to-extend-movable.md | 7 +- docs/how-to/write-a-flyer.md | 148 +-- pyproject.toml | 3 +- src/ophyd_async/core/__init__.py | 9 +- src/ophyd_async/core/_device.py | 22 +- src/ophyd_async/core/_flyable.py | 440 +++++--- src/ophyd_async/core/_movable.py | 210 ++-- src/ophyd_async/core/_signal.py | 54 +- src/ophyd_async/core/_standard_base.py | 177 +++- src/ophyd_async/core/_status.py | 87 +- src/ophyd_async/core/_utils.py | 99 +- src/ophyd_async/epics/demo/_motor.py | 5 +- src/ophyd_async/epics/motor.py | 145 +-- .../epics/pmac/_pmac_trajectory.py | 221 ++-- src/ophyd_async/fastcs/panda/_fly_logic.py | 44 +- src/ophyd_async/sim/_blob_acquire_logic.py | 11 +- src/ophyd_async/sim/_motor.py | 148 ++- src/ophyd_async/tango/demo/_motor.py | 6 +- tests/unit_tests/core/test_cancel_swallow.py | 71 ++ tests/unit_tests/core/test_flyer.py | 993 +++++++++++++++--- tests/unit_tests/core/test_movable.py | 272 ++++- .../core/test_multi_derived_signal.py | 4 +- tests/unit_tests/core/test_readable.py | 11 + tests/unit_tests/core/test_signal.py | 34 + .../unit_tests/core/test_standard_detector.py | 2 +- tests/unit_tests/core/test_status.py | 32 + tests/unit_tests/core/test_utils.py | 178 +++- .../core/test_watchable_async_status.py | 211 ++++ .../unit_tests/epics/adcore/test_use_cases.py | 3 +- .../epics/pmac/test_pmac_trajectory.py | 66 +- tests/unit_tests/epics/test_motor.py | 196 +++- .../unit_tests/fastcs/panda/test_hdf_panda.py | 4 +- tests/unit_tests/fastcs/panda/test_trigger.py | 17 +- tests/unit_tests/sim/test_sim_motor.py | 201 +++- uv.lock | 78 +- 42 files changed, 3458 insertions(+), 942 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 docs/explanations/decisions/0021-single-logic-property.md create mode 100644 docs/explanations/decisions/0022-fly-generator-and-ensure-stopped.md create mode 100644 tests/unit_tests/core/test_cancel_swallow.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..a490d700bf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,51 @@ +# Changelog + +## Unreleased + +### Breaking changes + +- `StandardMovable.movable_logic` and `StandardFlyable.flyable_logic` are replaced by a single `logic` property, declared by each mix-in with its own type, so a Device that moves and flies implements it once with a logic inheriting both `MovableLogic` and `FlyableLogic` (see ADR 0021). Rename your `movable_logic`/`flyable_logic` `@cached_property` to `logic`; no deprecated aliases are left. Forgetting it is now a `TypeError` at instantiation rather than an error the first time it is used. `StandardFlyable` is now abstract, use `FlyableLogic.with_device()` for an ephemeral flyer. +- `FlyableLogic` hooks are renamed and reshaped; a subclass that still defines `on_prepare`, `on_kickoff`, `on_complete`, `on_stage`, `on_unstage` or (unless it is also a `MovableLogic`) `stop` raises `TypeError` naming the replacement: + - `on_prepare(value)` is now `prepare_to_fly(value)`, an async generator that yields the context first, then optionally `WatcherUpdate`s that report the slow part (such as moving to the start) to the watchers of `prepare()`. A logic with no context yields `None`. `prepare()` keeps the context only once the generator has finished; it raises `TypeError` if the first yield is a `WatcherUpdate` or a later one is not, and `RuntimeError` if it yields nothing. The generator is closed if `prepare()` fails or is cancelled, so it must own anything it starts (for example with `async with AsyncStatus(...)`). + - `on_kickoff(ctx)` and `on_complete(ctx)` are replaced by one `fly(ctx)`, an async generator that `StandardFlyable` runs in the task of the status `complete()` returns. It yields `KICKED_OFF` once the scan is up to speed, which finishes `kickoff()`, then optionally `WatcherUpdate`s, which go to the watchers of `complete()`; `complete()` finishes when it returns. A failure before `KICKED_OFF` fails `kickoff()`, one after it fails `complete()`. It is cancelled on `unstage()`, when `complete()`'s status is cancelled and when `kickoff()` is cancelled before `KICKED_OFF`, so `fly` must wait for everything it starts (for example with `async with AsyncStatus(...)`). + - `stop()` is now the abstract `ensure_stopped()`, which must wait until the hardware has stopped. `on_stage()` is now `ensure_ready()`, which defaults to `ensure_stopped()` and is called on `stage()`. `unstage()` cancels a `prepare()` or `fly` in progress, then always calls `ensure_stopped()`. +- "Device was stopped." is now raised as `StoppedError`, a `RuntimeError`, so `except RuntimeError` still catches it. It is raised by `StandardMovable` and `StandardFlyable` themselves, not by the logic: a `MovableLogic.move()` may end early without an error when stopped, as `Motor` and `SimMotor` now do. +- `StandardMovable.stop()` ends the `set()` in progress, then calls `ensure_stopped()` without waiting for it to finish failing. `stop(success=True)` lets it finish normally, `stop(success=False)` fails it with `StoppedError`, even if its move has not started yet, in which case the hardware is not touched. A second `stop()` only calls `ensure_stopped()` again. It does not end a `prepare()` or fly, only `unstage()` does, but `ensure_stopped()` still halts the hardware, so a `stop()` during the run-up of `prepare()` can leave it finishing short of the start; a future `pause()` hook is the place to fix this. `StandardMovable.set()` is now a plain method returning a `WatchableAsyncStatus`. +- A new `set()` on a `StandardMovable` ends the one in progress, which fails with `StoppedError` saying it was superseded; the hardware is not stopped, the new move retargets it. +- `StandardMovable` is now stageable, but a `StandardMovable` alone does nothing on `stage()` or `unstage()`. A Device that also flies runs the `FlyableLogic`'s `ensure_ready()` on `stage()` and `ensure_stopped()` on `unstage()`, so it stops a `set()` in progress at both; the RunEngine already stops everything it set when a run ends or pauses. +- `Motor` sends `.STOP` and waits for `.DMOV` every time `ensure_stopped()` is called, even if the motor looks stopped, as `.DMOV` can flicker at standstill on some motors; so every staged `Motor` gets one `.STOP` at `unstage()`. `Motor.stage()` stops a motion in progress rather than waiting for it, and `MotorFlyableMovableLogic.ready_timeout` is removed. +- `FlyableLogic.prepare_to_fly` and `FlyableLogic.fly` are annotated as returning `AsyncGenerator`; annotate overrides the same way. +- `complete()` fails with `TypeError` if `fly` yields `KICKED_OFF` more than once, as for any other yield that is not a `WatcherUpdate`. +- `MovableLogic.stop()` is now the abstract `ensure_stopped()`, which must wait until the motion has stopped, and `StandardMovable.stop()` returns once it has. A subclass that still defines `stop()` gets a `DeprecationWarning`, and its `stop()` is used as `ensure_stopped()`. `Motor` now waits for `.DMOV` after writing `.STOP`, allowing its acceleration time to decelerate, and `InstantMotorMock` finishes a move on `.STOP`. +- `ophyd_async` now needs `typing-extensions>=4.14`, for `KICKED_OFF`. Type-checking a `fly` annotated with `KICKED_OFF` needs pyright 1.1.410 or later. +- Renamed `MotorFlyCtx` to `MotorPrepareCtx`, `PmacFlyCtx` and `PmacPrepareContext` to `PmacPrepareCtx`, and `SimMotorMoveLogic` to `SimMotorLogic`. +- A flyer must now be staged before it is prepared: `prepare()` is only allowed after `stage()`, `kickoff()` after `prepare()` and `complete()` after `kickoff()`, and a verb in the wrong state raises `TransitionError` (a `RuntimeError`). Staging an already-staged flyer, even one with a verb running, used to silently reset it and now raises (see below). A fly that fails leaves the flyer staged. `complete()` returns the same status each time it is called after a `kickoff()`, until the next `prepare()` or `unstage()`. After `complete()` a flyer can be prepared again without restaging. Stock bluesky plans that stage their devices (`bp.count`, `bp.scan`) keep working; `bp.fly` neither stages nor prepares, so stage the flyer and prepare it before `bp.fly`, for example with `bpp.stage_wrapper`. +- Staging any already-staged Standard device (`StandardReadable`, `StandardMovable`, `StandardFlyable`, ...) now raises `TransitionError`, as ophyd raises `RedundantStaging`. For example, nesting `bpp.stage_wrapper` around a plan that also stages the same device now fails. +- `unstage()` now cancels the `prepare()` or fly that is running. `unstage()` waits for it before calling `ensure_stopped()`, then raises any error it raised while being cancelled. +- `MovableLogic` and `FlyableLogic` now share the base `Logic`, an `ABC`, so a subclass missing an abstract hook (`ensure_stopped`, `prepare_to_fly` or `fly`) raises `TypeError` when it is constructed. See ADR 0022 for the reasoning behind these hooks. +- `SimMotor` is now a `StandardFlyable`, so like any flyer it must be staged before `prepare()`. +- `SimMotor.stage()` now calls `SimMotorLogic.ensure_ready()`, which calls `ensure_stopped()` and so stops the simulated motion, waits for it to finish, and sets the setpoint to the current readback. `unstage()` cancels a fly in progress, then calls `ensure_stopped()`. +- The progress of `StandardFlyable.prepare()` now comes from the `WatcherUpdate`s the logic's `prepare_to_fly` yields, replacing the heuristic that watched setpoint writes (#1448). +- `WatchableAsyncStatus` closes the async generator it drives as soon as it is cancelled or fails, rather than when the generator is collected, and a watcher that raises is logged and ignored rather than failing the status. +- A `StandardFlyable` whose fly failed after `kickoff()` and was never completed logs a warning when it is unstaged or prepared again, instead of leaving the failure unretrieved. + +### Added + +- `KICKED_OFF`, the sentinel `FlyableLogic.fly` yields once the scan is up to speed. +- `watch_readback()` reports the progress of a move by watching a `MovableLogic` readback, for use from `FlyableLogic` hooks. +- `SimMotor.prepare()` and `SimMotor.complete()` report progress to watchers, as `Motor` does. +- `TransitionError`, a `RuntimeError` raised by Standard devices for a verb called in the wrong state. +- `StoppedError`, the `RuntimeError` raised when `stop()` or a new `set()` ends a move, prepare or fly early. +- `Logic`, the base of `MovableLogic` and `FlyableLogic`, with the abstract `ensure_stopped()`. `FlyableLogic` adds `ensure_ready()`, which defaults to it. +- An exception raised by a hook during `stage()`, `unstage()`, `prepare()` or a fly of a Standard device carries the note `raised by `. + +### Fixed + +- `complete()` finishing after `unstage()` no longer makes the device look prepared or flown to a new fly. +- `unstage()` now cancels a running fly, so the PMAC trajectory stops appending points and nothing is left running once the hardware is stopped. +- `unstage()` now ends a running `complete()`, such as for the PandA. +- A `prepare()` that fails or is cancelled now stops the move to the run-up start of a `Motor` or `SimMotor`. +- A failed `prepare()` no longer leaves the flyer prepared with the failed context, so there is nothing to kick off. +- `set_and_wait_for_other_value()` cancelled before the first value arrives, such as by `stop()` during a move, no longer leaves its value-watching task running. +- A signal `get_value()`, `set()` or `trigger()` cancelled just as it finishes no longer swallows the cancellation on Python 3.11. +- Leaving an `async with AsyncStatus(...)` block, `set_and_wait_for_other_value()` and the sim `BlobAcquireLogic.ensure_stopped()` no longer swallow a cancellation of the caller that arrives while they wait for the task they cancelled. diff --git a/CLAUDE.md b/CLAUDE.md index e1032b8b40..b4c8d61c40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,7 +51,9 @@ tox -p # all envs in parallel (CI equivalent) ## Code conventions - **No `ABC` base needed** — `@abstractmethod` without `ABC`/`ABCMeta` is intentional; pyright enforces implementation statically. -- **A new `TypeVar` needs an entry in `docs/conf.py`.** Docs build with `nitpicky = True` and warnings-as-errors, and TypeVars never resolve as references, so every one is listed by qualified name (`ophyd_async.core._utils.T`, …) in the `nitpick_ignore` list near the end of `conf.py`. Miss it and `tox -e docs` fails on `reference target not found` while every other env stays green. Prefer declaring TypeVars in the `_utils.py` of their package, next to the existing ones, so the list stays in one place. +- **A new `TypeVar` needs an entry in `docs/conf.py`.** Docs build with `nitpicky = True` and warnings-as-errors, and TypeVars never resolve as references, so every one is listed by qualified name (`ophyd_async.core._utils.T`, …) in the `nitpick_ignore` list near the end of `conf.py`. Miss it and `tox -e docs` fails on `reference target not found` while every other env stays green. Generic TypeVars (`T`, `V`, …) live in the `_utils.py` of their package; a TypeVar specific to one class is declared in that class's module, next to it. +- **Plain `Enum` members take their own name as their value** (`UNSTAGED = "UNSTAGED"`), never `auto()`. This is for enums that do not represent a control system value. `StrictEnum`, `SubsetEnum` and `SupersetEnum` are different: their values are the strings the control system uses (e.g. `ON = "On"`), so they keep those. +- **A lone marker value is a `typing_extensions.Sentinel`** (`KICKED_OFF = Sentinel("KICKED_OFF")`, usable as its own type in hints), not a one-member `Enum`, unless it crosses a serialisation boundary or is compared `==` to plain values: `CALCULATE_TIMEOUT` stays a string because pydantic fields and YAML/JSON plan arguments carry it. ## Updating this guide diff --git a/docs/conf.py b/docs/conf.py index fcd2422bff..695b7f8ea4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -171,9 +171,11 @@ def setup(app: application.Sphinx): "ophyd_async.epics.adcore._io.ADBaseIOT", "ophyd_async.epics.adcore._io.NDPluginBaseIOT", "ophyd_async.epics.adcore._io.NDPluginFileIOT", - "ophyd_async.epics.motor.MotorFlyCtx", + "ophyd_async.epics.motor.MotorPrepareCtx", "ophyd_async.epics.motor.MotorFlyableMovableLogic", - "ophyd_async.epics.pmac._pmac_trajectory.PmacFlyCtx", + "ophyd_async.epics.pmac._pmac_trajectory.PmacPrepareCtx", + "ophyd_async.sim._motor.SimMotorLogic", + "ophyd_async.sim._motor.SimMotorPrepareCtx", "ophyd_async.tango.core._base_device.T", "ophyd_async.tango.core._tango_transport.P", "ophyd_async.tango.core._tango_transport.R", diff --git a/docs/explanations/decisions/0017-standard-movable.md b/docs/explanations/decisions/0017-standard-movable.md index 3d70c39da9..4994dc3cfa 100644 --- a/docs/explanations/decisions/0017-standard-movable.md +++ b/docs/explanations/decisions/0017-standard-movable.md @@ -6,6 +6,10 @@ Date: 2026-03-10 Accepted +Partly superseded by [ADR 0021](0021-single-logic-property.md), which replaced `movable_logic` with a single `logic` property. + +The `stop()` hook below has since been replaced by an abstract `ensure_stopped()` that waits until the motion has stopped (see [ADR 0022](0022-fly-generator-and-ensure-stopped.md)). + ## Context Several device types share a common movement pattern: write a setpoint, then observe a diff --git a/docs/explanations/decisions/0021-single-logic-property.md b/docs/explanations/decisions/0021-single-logic-property.md new file mode 100644 index 0000000000..ac2dcab602 --- /dev/null +++ b/docs/explanations/decisions/0021-single-logic-property.md @@ -0,0 +1,78 @@ +# 21. One `logic` property per Device + +Date: 2026-08-18 + +## Status + +Accepted + +Supersedes part of ADR 0017, which introduced `movable_logic`. + +## Context + +A Device that both moves and flies had to expose the same logic object twice. `Motor` +built one `MotorFlyableMovableLogic` and returned it from a `movable_logic` property for +`StandardMovable` and a `flyable_logic` property for `StandardFlyable`, plus a +private `_logic` to avoid building it twice. Every future mix-in would add another name. + +The obstacle is that Python has no intersection type: there is no way to write "the logic +must be both a `MovableLogic[float]` and a `FlyableLogic[FlyMotorInfo, MotorFlyCtx]`" as a +single annotation. + +ADR 0017 also chose `@cached_property` stacked on `@abstractmethod` over an +`add_movable_logic()` method, on the grounds that the property is "checkable by static +analysis". That turned out not to hold. `functools.cached_property` does not forward +`__isabstractmethod__` from the function it wraps, so `StandardMovable.__abstractmethods__` +was empty, pyright reported nothing, and a Device that forgot its logic failed with +`AttributeError: 'NoneType' object has no attribute 'readback'` from inside +`Device.__init__`. + +## Decision + +Each mix-in declares the **same** attribute name, `logic`, with **its own** +required type: + +```python +class StandardMovable(Device, ..., Generic[SignalDatatypeT]): + @abstract_cached_property + def logic(self) -> MovableLogic[SignalDatatypeT]: ... + +class StandardFlyable(_StandardBase, ..., Generic[PrepareT, CtxT]): + @abstract_cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: ... +``` + +A Device inheriting both provides one implementation, and the type checker verifies it +against each base independently. The intersection is expressed by the logic class's own +MRO: `MotorFlyableMovableLogic` inherits both, so it satisfies both declarations, and the +concrete type is what callers see back. + +This catches, at type-check time, a logic that satisfies one mix-in but not the other, a +wrong datatype parameter, and a Device with two mix-ins and no implementation. A Device +with a *single* mix-in and no implementation is caught at instantiation instead, by +`abstract_cached_property`, a `cached_property` that sets `__isabstractmethod__` on the +descriptor so that `Device`'s metaclass collects it. It is declared as returning +`cached_property[T]` rather than as a subclass, because a subclass breaks the covariance +check on the override and would reject the correct case too. + +Because `StandardFlyable` is now genuinely abstract, `FlyableLogic.with_device` builds +a concrete `_EphemeralFlyable`, which takes the logic in its `__init__`. + +An alternative of a single `logics` collection searched by type was rejected: membership +cannot be proved statically, so it would reintroduce exactly the runtime-only failure +this change fixes. + +## Consequences + +- One property per Device however many mix-ins it combines; `Motor` loses two properties. +- Where the move and fly logic are genuinely unrelated implementations they must be + combined into one class, since one property returns one object. Nothing in-tree pays + this cost, as `MotorFlyableMovableLogic` was already that shape. +- Forgetting the logic now fails with `TypeError: Can't instantiate abstract class ... + with abstract method logic` rather than an `AttributeError` on `None`. +- `movable_logic` and `flyable_logic` are removed rather than deprecated. A shim would + only have helped code that *reads* the attribute: a subclass that still *defines* + `movable_logic` would override the shim and leave `logic` unimplemented, so the + Device would fail at instantiation anyway. +- The same mechanism is available to `StandardDetector` if it later inherits + `StandardFlyable`; its logic would need to satisfy `FlyableLogic` too. diff --git a/docs/explanations/decisions/0022-fly-generator-and-ensure-stopped.md b/docs/explanations/decisions/0022-fly-generator-and-ensure-stopped.md new file mode 100644 index 0000000000..064cda11e3 --- /dev/null +++ b/docs/explanations/decisions/0022-fly-generator-and-ensure-stopped.md @@ -0,0 +1,44 @@ +# 22. `fly` is a generator, and `ensure_stopped` waits + +Date: 2026-10-20 + +## Status + +Accepted + +Amends ADR 0017: `MovableLogic.stop()` is now `ensure_stopped()`, and the fly hooks are different from the ones that ADR describes. + +## Context + +`FlyableLogic` had separate `on_kickoff` and `on_complete` hooks, and `stop()` returned as soon as it had asked the hardware to stop. This allowed three kinds of bug. A scan started in `on_kickoff` could outlive `unstage()`, because nothing owned it. A new `stage()` or `set()` could start while the previous motion was still decelerating. A `set()` caught by `stop()` before its move had started was not ended, and ran on after it. + +## Decision + +**`prepare_to_fly()` and `fly()` are async generators.** `prepare_to_fly` yields the context first, then any progress; `fly` yields `KICKED_OFF` once flying, then any progress. Everything a scan starts lives inside the generator, so closing or cancelling it is enough to stop it. Writing it as `async with AsyncStatus(...)` plus `yield` makes the ownership visible in the code, and a failure before or after the first yield maps cleanly on to the verb that failed. Two hooks that shared state through the logic, or a hook that returned a `(context, progress)` tuple, could not guarantee this: the progress iterator was not closed if `prepare()` failed, and a context that was itself a tuple was ambiguous. `prepare()` keeps the context only once the generator has finished, so a half-failed prepare leaves nothing to kick off. The first yield is checked at runtime, as a `WatcherUpdate` yielded first is an easy mistake that the type checker cannot see. + +**The fly runs in the foreground of the status that `complete()` returns.** `kickoff()` makes a `WatchableAsyncStatus` that drives `fly`, and returns once it has yielded `KICKED_OFF`; `complete()` returns that same status. It already replays the latest update to a late watcher, and the state machine cancels it on `unstage()`, so no separate task, update forwarding or cleanup path is needed. `WatchableAsyncStatus` closes the generator it drives as soon as it is cancelled, rather than when it is collected. Cancelling `complete()`'s status therefore cancels the fly, and a fly cleanup that fails is reported by that status instead of its `CancelledError`. + +**`ensure_stopped()` waits until the motion has ended.** `unstage()` cancels `fly`, then calls `ensure_stopped()`, so when `unstage()` returns nothing is running and the hardware is still. Returning early would leave the next `stage()` or `set()` racing the deceleration of the last one. + +**One stop future ends a set.** Each `set()` passes a fresh future to the `stop` argument of its `WatchableAsyncStatus`, which races the operation against the future: when the future resolves first the operation is cancelled and the future's value becomes the outcome; statuses still have no public `stop()`, as bluesky treats a status as a task the device stops. The device keeps one future, the latest `set()`'s. `stop()` resolves it with `None` for `stop(success=True)`, so the `set()` finishes normally, or with `StoppedError("Device was stopped.")`, a `RuntimeError`, which it raises. `RunEngine.pause` calls `stop(success=True)` on everything it has set and expects that to finish without an error. A new `set()` resolves the previous future with a `StoppedError` saying it was superseded, so only one `set()` runs at a time; its move is cancelled but the hardware is not stopped, as the new move retargets it. The future is resolved with a value rather than an exception, so a future nobody awaits raises nothing. `stop()` ends only a `set()`, never a `prepare()` or fly: those are ended by `unstage()` through the state machine, so a `stop()` cannot reach a verb it was not meant for. A stop that arrives before the move starts ends the `set()` without touching the hardware, and a second `stop()` finds the future already resolved and only calls `ensure_stopped()` again. `stop()` does not wait for the `set()` to finish failing before `ensure_stopped()`: the hardware should stop as soon as possible. Logics no longer track whether they were stopped: a `move()` may end early without an error when stopped, as an EPICS motor's put completion does, and the stop future decides the outcome. + +**Only a flyer stages, and `unstage()` always stops.** A `StandardMovable` alone does nothing on `stage()` or `unstage()`: `stop()` is how it is stopped. `ensure_ready()` is a hook of `FlyableLogic` only, which defaults to `ensure_stopped()`, and only `StandardFlyable` calls it, on `stage()`; a device that moves and flies therefore stops a `set()` in progress at `stage()`, no worse than `caput MOTOR.STOP`. `StandardFlyable.unstage()` cancels a `prepare()` or fly, then always calls `ensure_stopped()`, whether or not the device was prepared. Sparing a `set()` there would only defer its stop: the bluesky RunEngine stops every movable it set when a run ends or pauses. The cost is that every staged `Motor` gets one `.STOP` at `unstage()`, because `Motor.ensure_stopped()` sends it unconditionally: `.DMOV` can flicker at standstill on some motors, so it is not read to decide, and the motor record does not write `.VAL` from `.RBV` on `.STOP`. + +The resulting model: + +| Verb | Mix-in | Does | +|------|--------|------| +| `stage()` | `StandardFlyable` | calls `logic.ensure_ready()` | +| `unstage()` | `StandardFlyable` | cancels a `prepare()` or fly in progress, then calls `logic.ensure_stopped()` | +| `stop()` | `StandardMovable` | ends the `set()` via the stop future, then calls `logic.ensure_stopped()` | +| `set()` | `StandardMovable` | supersedes the `set()` in progress | + +The logic hooks `prepare_to_fly` and `fly` are async generators with a first-yield marker: the context for `prepare_to_fly`, `KICKED_OFF` for `fly`. + +## Consequences + +- The stage and unstage of the previous plan are the only thing that keeps a motion from overlapping the next one. `stage()` can start `ensure_ready()` while the `ensure_stopped()` from the previous `unstage()` is still running, and we accept that. We assume every plan stages its devices with the stage/unstage decorator, as the stock plans do, and the tests run plans back to back and after an interruption to show that this behaves. A plan that does not stage gets none of this. +- A `FlyableLogic` that also moves gets `ensure_stopped()` called on `stage()` by default, which stops a `set()` in progress. Override `ensure_ready()` if stopping is wrong there. +- A plan that fails while a `set()` or `complete()` is in flight sees bluesky raise a `FailedStatus` for it from `unstage()`, where it would have seen the plan's own exception first. +- `StandardFlyable` has no `stop(success)` of its own, and `stop()` on a device that moves and flies does not end a `prepare()` or fly. `ensure_stopped` still halts the hardware, so a `stop()` during the run-up (for example `bps.stop`, or a `RunEngine` pause of a device that was also `set()`, with `rewindable=False`) can leave `prepare()` finishing short of the start. We do not check the position, as motors have a tolerance and none is baked in here. A future `pause()` hook is the better place to fix this. +- `complete()` returns the fly status even after the fly has failed, until the next `prepare()` or `unstage()`, so the failure is reported rather than a `TransitionError`. diff --git a/docs/explanations/declarative-vs-procedural.md b/docs/explanations/declarative-vs-procedural.md index a83025751b..f9481cce7b 100644 --- a/docs/explanations/declarative-vs-procedural.md +++ b/docs/explanations/declarative-vs-procedural.md @@ -6,8 +6,8 @@ Ophyd async has two styles of creating Devices, Declarative and Procedural. This The procedural style mirrors how you would create a traditional python class, you define an `__init__` method, add some class members, then call the superclass `__init__` method. In the case of ophyd async those class members are likely to be Signals and other Devices. For example, in the `ophyd_async.sim.SimMotor` we create its soft signal children in an `__init__` method: ```{literalinclude} ../../src/ophyd_async/sim/_motor.py -:start-after: class SimMotor( -:end-before: self._fly_info +:start-after: For usage when simulating a motor. +:end-before: super().__init__(name=name) ``` It is explicit and obvious, but verbose. It also allows you to embed arbitrary python logic in the creation of signals, so is required for making soft signals and DeviceVectors with contents based on an argument passed to `__init__`. It also allows you to use the [](#StandardReadable.add_children_as_readables) context manager which can save some typing. diff --git a/docs/explanations/when-to-extend-movable.md b/docs/explanations/when-to-extend-movable.md index 4862845c86..0f3178e46e 100644 --- a/docs/explanations/when-to-extend-movable.md +++ b/docs/explanations/when-to-extend-movable.md @@ -34,7 +34,8 @@ You only need to describe what is device-specific. ### Implementing MovableLogic Create a `@dataclass` subclass of [](#MovableLogic) that adds any extra signals your -device needs, and override the hook methods that differ from the defaults: +device needs, implement `ensure_stopped`, and override the hook methods that differ from +the defaults: ```{literalinclude} ../../src/ophyd_async/epics/demo/_motor.py :pyobject: DemoMotorMoveLogic @@ -44,9 +45,9 @@ The available hooks are: | Method | Default behaviour | Override to… | |--------|-------------------|--------------| -| `stop()` | no-op | trigger a stop signal or cancel an in-flight command | +| `ensure_stopped()` | *abstract* | stop the motion and wait until it has stopped | | `check_move(new)` | no-op | validate soft limits before the move begins | -| `calculate_timeout(old, new)` | `DEFAULT_TIMEOUT` | derive a velocity-based or distance-based timeout | +| `calculate_timeout(old, new)` | `None`, so no timeout | derive a velocity-based or distance-based timeout | | `get_units_precision()` | reads from `readback.describe()` | supply units and precision from dedicated signals | | `move(new_position, timeout)` | `set_and_wait_for_other_value` | write the setpoint and wait in a device-specific way | diff --git a/docs/how-to/write-a-flyer.md b/docs/how-to/write-a-flyer.md index 344e0f76c3..7731a58ba5 100644 --- a/docs/how-to/write-a-flyer.md +++ b/docs/how-to/write-a-flyer.md @@ -4,96 +4,128 @@ A *flyer* is a Device that hardware-triggers a scan: bluesky prepares it, kicks off, then waits for it to complete while the hardware runs autonomously. Rather than implement the `Preparable`, `Flyable`, and `Stageable` protocols by hand, compose a [](#FlyableLogic) instance into a [](#StandardFlyable). `StandardFlyable` drives the -three phases in order and threads a context object between them: - -- `prepare(value)` → `FlyableLogic.on_prepare(value)` moves to the start and arms the - hardware, returning a context object. -- `kickoff()` → `FlyableLogic.on_kickoff(ctx)` starts the scan, returning the - (possibly updated) context. -- `complete()` → `FlyableLogic.on_complete(ctx)` blocks until the scan is done. - -`stage()`/`unstage()` run the logic's `on_stage`/`on_unstage`, which default to `stop`. +phases in order and threads a context object between them: + +- `prepare(value)` → `FlyableLogic.prepare_to_fly(name, value, previous_ctx)`, an async generator that works + out the scan and yields a context object, then sets the hardware up and moves to the + start. It may report progress. +- `kickoff()` → starts `FlyableLogic.fly(ctx)`, an async generator that `StandardFlyable` + runs in the task of a status. `kickoff()` returns when it yields `KICKED_OFF`, once the + scan is up to speed. +- `complete()` returns that status, which finishes when `fly` does and reports the + progress it yields after `KICKED_OFF`. + +The flyer must be staged first: `prepare()` is only allowed after `stage()`, `kickoff()` +only after `prepare()`, and `complete()` only after `kickoff()`; a verb called out of +order raises `TransitionError`. After `complete()` the flyer can `prepare()` again for the +next point without restaging. `stage()` raises unless the flyer is unstaged, as does a +second `prepare()` before `complete()`. If `fly` fails before yielding `KICKED_OFF`, `kickoff()` +fails; after it, `complete()` fails. Either leaves the flyer staged. Cancelling +`complete()`'s status cancels `fly`, and so does cancelling `kickoff()` before +`KICKED_OFF`. + +`stage()` runs the logic's `ensure_ready`, which defaults to `ensure_stopped`. +`unstage()` works from any state: it cancels a `prepare()` or `fly` in flight (which sees +`CancelledError`) and waits for it, then always runs `ensure_stopped` to stop the +hardware. If the cancelled verb raised while being cancelled, `unstage()` raises it after `ensure_stopped`. On a device that also moves, +`stop()` ends only a `set()`: it does not end a `prepare()` or `fly`, but its +`ensure_stopped` halts the hardware, so a `stop()` during the run-up can leave +`prepare()` finishing short of the start. The hook names are deliberately not bluesky verbs, so a +logic passed to a plan stub does not look like a device. ## Implementing FlyableLogic Subclass [](#FlyableLogic) — it is generic over two types: `PrepareT`, the per-scan -info passed to `prepare()`, and `CtxT`, the context threaded between the phases (use -`None` if there is no state to carry). Fill in the hooks that your hardware needs: +info passed to `prepare()`; and `CtxT`, the context `prepare_to_fly` passes to `fly`. +`StandardFlyable` holds the context until `unstage()`, and passes it to the next +`prepare_to_fly` as `previous_ctx` (`None` for the first since `stage()`). It defaults +to `None`, so `FlyableLogic[PrepareT]` carries no state and its `prepare_to_fly` yields +`None` first. `prepare()` keeps the context only if `prepare_to_fly` finishes, so a +failed one leaves nothing to kick off. Fill in the hooks that your hardware needs: | Method | Default behaviour | Override to… | |--------|-------------------|--------------| -| `on_prepare(value)` | *abstract* | move to the start, arm the hardware, and return a context | -| `on_kickoff(ctx)` | *abstract* | start the scan and return the context for `on_complete` | -| `on_complete(ctx)` | *abstract* | block until the scan finishes | -| `stop()` | no-op | disarm the hardware and wait for it to stop | -| `on_stage()` | calls `stop()` | set the hardware up on `stage()` if it differs from stop | -| `on_unstage()` | calls `stop()` | clean the hardware up on `unstage()` if it differs from stop | +| `prepare_to_fly(name, value, previous_ctx)` | *abstract* | as an async generator: yield the context first, then set up the hardware, yielding any progress (see [](#flyer-progress)) | +| `fly(ctx)` | *abstract* | run the scan as an async generator: yield `KICKED_OFF` once up to speed, then any progress, and return when it is done | +| `ensure_stopped()` | *abstract* | stop the hardware and wait until it has stopped | +| `ensure_ready()` | calls `ensure_stopped()` | set the hardware up on `stage()` if it differs from stopping it | + +`prepare_to_fly` and `fly` must not leave anything running that they do not wait for, +because `StandardFlyable` cancels them on `unstage()` and expects that to stop +everything they started. Own a status with `async with AsyncStatus(...) as status:`, which cancels it on +the way out, and run concurrent work with [](#watch_readback) or an +`asyncio.TaskGroup`. There are two ways to get a Device out of a `FlyableLogic`: mix `StandardFlyable` into a Device class (for hardware that is always a flyer, like a motor), or call [](#FlyableLogic.with_device) to wrap a bare logic object in an ephemeral flyer inside a plan (for hardware configured per scan, like a PandA block). +(flyer-progress)= +### Reporting progress + +`prepare()` and `complete()` can report progress to watchers as [](#WatcherUpdate)s, +stamped with the flyer's name. + +- `prepare_to_fly` yields `WatcherUpdate`s after the context, for example the progress + of the move to the start. They go to the watchers of `prepare()`. +- `fly` yields `WatcherUpdate`s after `KICKED_OFF`. They go to the watchers of + `complete()`, which also gets the latest one yielded before it was called. + ## A movable device that also flies An EPICS [](#Motor) is the first case: it is a [](#StandardMovable) *and* a `StandardFlyable` (and a [](#StandardReadable)). A single logic object, `MotorFlyableMovableLogic`, implements both [](#MovableLogic) and `FlyableLogic`, so its fly hooks can reuse the same `move`/`check_move` methods as a normal `set()`. The -per-scan info is a [](#FlyMotorInfo) and the threaded context is a small dataclass that -carries the move status from `on_kickoff` to `on_complete`: +per-scan info is a [](#FlyMotorInfo) and the threaded context is a small frozen +dataclass of everything `prepare_to_fly` works out: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: MotorFlyCtx +:pyobject: MotorPrepareCtx ``` -`on_prepare` validates the requested velocity against the motor's limits, moves to the -run-up start position, and returns the context: +`prepare_to_fly` validates the requested velocity and the run-up and run-down positions +against the motor's limits and yields the context. It then owns the move to the run-up +start with `async with AsyncStatus(...)`, and yields its progress from +[](#watch_readback), which watches the readback of the logic until the move is done: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: MotorFlyableMovableLogic.on_prepare +:pyobject: MotorFlyableMovableLogic.prepare_to_fly ``` -`on_kickoff` starts the move to the far end of the run-down and stashes the resulting -status on the context so `on_complete` can await it: +`fly` starts the move to the far end of the run-down, waits out the run-up so the motor +is at constant velocity when `kickoff()` returns, then yields the progress of the rest of +the move: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: MotorFlyableMovableLogic.on_kickoff +:pyobject: MotorFlyableMovableLogic.fly ``` -`on_complete` simply blocks on that status: +`ensure_stopped` is shared with `MovableLogic`, so it serves both `stop()` and +`unstage()`. It sends `.STOP` every time and waits for the motor to be done moving: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: MotorFlyableMovableLogic.on_complete +:pyobject: MotorFlyableMovableLogic.ensure_stopped ``` -Because the logic is also a `MovableLogic`, `prepare()` and `complete()` report progress -to watchers (e.g. progress bars) automatically by observing the readback — reusing the same -[](#WatcherUpdate) machinery as [](#StandardMovable.set). A flyer whose logic is *not* -movable (see the next section) just blocks with no progress updates. See +Opting in to progress is up to each hook: the same [](#WatcherUpdate) machinery as +[](#StandardMovable.set) is used, but a flyer whose logic is *not* movable (see the next +section) just waits with no progress updates. See [](../explanations/when-to-extend-movable.md) for more on `MovableLogic`. Wire the logic into the Device with a `@cached_property`. `Motor` builds one shared -instance and returns it from both `movable_logic` and `flyable_logic` so the move and -fly paths act on the same signals: +instance and returns it as its single `logic`, which both the movable and flyable +mix-ins declare (see [ADR 0021](../explanations/decisions/0021-single-logic-property.md) for why), so the move and fly paths act +on the same signals: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: Motor._logic -``` - -```python -@cached_property -def movable_logic(self) -> MotorFlyableMovableLogic: - return self._logic - -@cached_property -def flyable_logic(self) -> MotorFlyableMovableLogic: - return self._logic +:pyobject: Motor.logic ``` ## An ephemeral flyer created in a plan @@ -101,37 +133,33 @@ def flyable_logic(self) -> MotorFlyableMovableLogic: Some hardware is only a flyer for the duration of a scan and is configured differently each time. The PandA position-compare block is an example: [](#StaticPcompFlyableLogic) wraps a single `PcompBlock` and carries no state between phases, so it is a -`FlyableLogic[PcompInfo, None]`. Its `on_prepare` disarms then programs the block from -the per-scan [](#PcompInfo): +`FlyableLogic[PcompInfo]`. Its `prepare_to_fly` yields its `None` context, then disarms +and programs the block from the per-scan [](#PcompInfo): ```{literalinclude} ../../src/ophyd_async/fastcs/panda/_fly_logic.py :language: python -:pyobject: StaticPcompFlyableLogic.on_prepare +:pyobject: StaticPcompFlyableLogic.prepare_to_fly ``` -`on_kickoff` enables the block and waits for it to go active, while `on_complete` waits -for it to go inactive again (the context is `None`, so it is ignored): - -```{literalinclude} ../../src/ophyd_async/fastcs/panda/_fly_logic.py -:language: python -:pyobject: StaticPcompFlyableLogic.on_kickoff -``` +`fly` enables the block, yields `KICKED_OFF` once it is active, then waits for it to go +inactive again. It reports no progress: ```{literalinclude} ../../src/ophyd_async/fastcs/panda/_fly_logic.py :language: python -:pyobject: StaticPcompFlyableLogic.on_complete +:pyobject: StaticPcompFlyableLogic.fly ``` -`stop()` disarms the block; it is called by the default `on_stage`/`on_unstage`: +`ensure_stopped` disarms the block and waits for it to go inactive; it is called on +`unstage()` and, through the default `ensure_ready`, on `stage()`: ```{literalinclude} ../../src/ophyd_async/fastcs/panda/_fly_logic.py :language: python -:pyobject: StaticPcompFlyableLogic.stop +:pyobject: StaticPcompFlyableLogic.ensure_stopped ``` Rather than declare this logic on a Device class, construct it in the plan from the -PandA block you want to drive and call [](#FlyableLogic.with_device) to get a ready -flyer to prepare, kickoff, and complete: +PandA block you want to drive and call [](#FlyableLogic.with_device) to get a flyer +to stage, prepare, kickoff, complete and unstage: ```python flyer = StaticPcompFlyableLogic(panda.pcomp[1]).with_device(name="pcomp") diff --git a/pyproject.toml b/pyproject.toml index b387728898..d9d7ca5652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "stamina>=23.1.0", "scanspec>=1.0.0", "velocity-profile", + "typing-extensions>=4.14", ] dynamic = ["version"] license.file = "LICENSE" @@ -59,7 +60,7 @@ dev = [ "pre-commit", "pydata-sphinx-theme>=0.12", "pyepics>=3.4.2", - "pyright", + "pyright>=1.1.410", "pytest", "pytest-asyncio", "pytest-cov", diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 2a94209fdd..0634602b62 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -52,7 +52,7 @@ OnOff, YesNo, ) -from ._flyable import FlyableLogic, FlyMotorInfo, StandardFlyable +from ._flyable import KICKED_OFF, FlyableLogic, FlyMotorInfo, StandardFlyable from ._log import config_ophyd_async_logging, logger, set_handler from ._mock_signal_backend import MockSignalBackend from ._mock_signal_utils import ( @@ -76,6 +76,7 @@ MovableLogic, StandardMovable, TimeoutCalculator, + watch_readback, ) from ._path_providers import ( AutoIncrementFilenameProvider, @@ -137,6 +138,7 @@ make_datakey, ) from ._soft_signal_backend import SoftSignalBackend +from ._standard_base import Logic, StoppedError, TransitionError from ._status import ( AsyncStatus, WatchableAsyncStatus, @@ -201,9 +203,12 @@ def __getattr__(name): "init_devices", # Movable "MovableLogic", + "StoppedError", + "Logic", "StandardMovable", "InstantMovableMock", "TimeoutCalculator", + "watch_readback", # Protocols "AsyncReadable", "AsyncConfigurable", @@ -236,6 +241,7 @@ def __getattr__(name): "SupersetEnum", "EnumTypes", "Table", + "TransitionError", "SignalMetadata", "Primitive", # Soft signal @@ -304,6 +310,7 @@ def __getattr__(name): "StandardFlyable", "FlyMotorInfo", "FlyableLogic", + "KICKED_OFF", # Settings "Settings", "SettingsProvider", diff --git a/src/ophyd_async/core/_device.py b/src/ophyd_async/core/_device.py index c44478fcba..5f10d9ae28 100644 --- a/src/ophyd_async/core/_device.py +++ b/src/ophyd_async/core/_device.py @@ -3,7 +3,15 @@ import asyncio import os import sys -from collections.abc import Awaitable, Callable, Iterator, Mapping, MutableMapping +from collections.abc import ( + Awaitable, + Callable, + Generator, + Iterator, + Mapping, + MutableMapping, +) +from contextlib import contextmanager from functools import cached_property from logging import LoggerAdapter, getLogger from typing import Any, Generic, TypeVar @@ -553,6 +561,18 @@ async def _on_exit(self) -> None: await self._process_devices(devices) +@contextmanager +def name_added_to_errors(device: HasName) -> Generator[None, None, None]: + """Note the device's name on any exception raised inside, once.""" + try: + yield + except Exception as e: + note = f"raised by {device.name}" + if note not in getattr(e, "__notes__", ()): + e.add_note(note) + raise + + def init_devices( set_name: bool = True, child_name_separator: str = "-", diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index 70fb85a2cb..ae5465ff00 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -1,92 +1,143 @@ import asyncio +import contextlib from abc import abstractmethod -from collections.abc import AsyncIterator -from enum import Enum +from collections.abc import AsyncGenerator, AsyncIterator +from dataclasses import replace from functools import cached_property -from typing import Generic, TypeVar, cast +from typing import Any, Generic, TypeVar, cast from bluesky.protocols import Flyable, Preparable from pydantic import Field +from typing_extensions import Sentinel +from typing_extensions import TypeVar as TypeVarWithDefault from ._movable import MovableLogic -from ._signal import observe_signals_value, observe_value -from ._standard_base import _StandardBase +from ._standard_base import ( + Logic, + TransitionError, + _StandardBase, + _State, + _StateMachine, +) from ._status import AsyncStatus, WatchableAsyncStatus from ._utils import ( CALCULATE_TIMEOUT, CalculatableTimeout, ConfinedModel, WatcherUpdate, + abstract_cached_property, ) -#: The per-scan info passed to `FlyableLogic.on_prepare` (often a +#: The per-scan info passed to `FlyableLogic.prepare_to_fly` (often a #: [`scanspec.core.Path`](inv:scanspec#scanspec.core.Path), but any type). PrepareT = TypeVar("PrepareT") -#: The context object threaded from `on_prepare` through `on_kickoff` to -#: `on_complete`. Use `None` for flyers that carry no state between stages. -CtxT = TypeVar("CtxT") +#: The context yielded by `FlyableLogic.prepare_to_fly` and passed on to `fly`. +#: Defaults to `None`, for flyers that carry no state from prepare to fly. +CtxT = TypeVarWithDefault("CtxT", default=None) + +#: Yielded by `FlyableLogic.fly` to mark the flyer as flying. +KICKED_OFF = Sentinel("KICKED_OFF") -class FlyableLogic(Generic[PrepareT, CtxT]): + +class FlyableLogic(Logic, Generic[PrepareT, CtxT]): """Minimum logic needed for controlling a `StandardFlyable`. Inherit and fill in the hooks for a particular flyable (e.g. a motion or trigger system). This base holds no state; concrete subclasses are typically `@dataclass`es that hold whatever signals or sub-devices they need as fields. - State that must be carried between stages is threaded through an explicit - context object rather than stored on the logic: `on_prepare` returns it, - `on_kickoff` receives and returns it, and `on_complete` receives it. - `StandardFlyable` owns that context and enforces the call ordering, so - subclasses do not need to guard against being called out of order. A flyer - with no cross-stage state uses `None` for the context (see the PandA trigger - logics). - """ + What a scan needs is carried by a context object rather than stored on the + logic: `prepare_to_fly` yields it, and it is passed to `fly` and to the next + `prepare_to_fly`. `StandardFlyable` holds it from a successful `prepare()` + until `unstage()` and enforces the call ordering, so subclasses do not need + to guard against being called out of order. - @abstractmethod - async def on_prepare(self, value: PrepareT) -> CtxT: - """Move to the start of the fly scan, set it up, and return its context. + The type parameters are the per-scan info `PrepareT` and the context `CtxT`, + which defaults to `None`, so `FlyableLogic[PrepareT]` is a logic without one. - :param value: the per-scan info for this fly scan. - """ - - @abstractmethod - async def on_kickoff(self, ctx: CtxT) -> CtxT: - """Start the fly scan. + The hook names are deliberately not bluesky verbs, so a logic passed to a + plan stub does not look like a device. + """ - :param ctx: the context returned by `on_prepare`. + def __init_subclass__(cls, **kwargs) -> None: + super().__init_subclass__(**kwargs) + renamed = { + "on_prepare": "prepare_to_fly", + "on_kickoff": "fly", + "on_complete": "fly", + "on_stage": "ensure_ready", + "on_unstage": "ensure_stopped", + # A logic that is also a MovableLogic gets its deprecation alias + **({} if issubclass(cls, MovableLogic) else {"stop": "ensure_stopped"}), + } + if stale := [name for name in renamed if name in cls.__dict__]: + raise TypeError( + f"{cls.__name__} defines {', '.join(stale)}, which FlyableLogic no " + "longer calls: " + + ", ".join(f"{name} -> {renamed[name]}" for name in stale) + ) - Return the (possibly updated) context to be passed to `on_complete`. - """ + async def ensure_ready(self) -> None: + """Get the device ready on `stage()`. Defaults to `ensure_stopped`.""" + await self.ensure_stopped() @abstractmethod - async def on_complete(self, ctx: CtxT) -> None: - """Block until the fly scan is done. - - :param ctx: the context returned by `on_kickoff`. + def prepare_to_fly( + self, name: str, value: PrepareT, previous_ctx: CtxT | None + ) -> AsyncGenerator[CtxT | WatcherUpdate, None]: + """Work out the scan, set up the hardware, and yield the scan's context. + + Write it as an async generator whose first yield is the context; any + later yields are `WatcherUpdate`s reporting the progress of the slow part, + e.g. moving to the start of the scan. `prepare()` finishes when the + generator does, and only then keeps the context. Own anything it starts + with `async with`, so it is stopped if `prepare()` fails or is cancelled: + + ```python + async def prepare_to_fly(self, name, value, previous_ctx): + ctx = MyCtx(start=value.start) + yield ctx + async with AsyncStatus(self.move(ctx.start, timeout)) as status: + async for update in watch_readback(self, status, target=ctx.start): + yield update + ``` + + :param name: the name of the device. + :param value: the per-scan info for this fly scan. + :param previous_ctx: the context of the last successful `prepare()` + since `stage()`, or `None` if there was none. """ - async def stop(self) -> None: - """Stop/disarm the flyer and wait for everything to be stopped. - - Called by `on_stage` and `on_unstage` by default; override those instead - if stage and unstage need to differ. + @abstractmethod + def fly(self, ctx: CtxT) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + """Run the fly scan: yield `KICKED_OFF` once flying, then its progress. + + Write it as an async generator. `kickoff()` finishes at the first yield, + which must be `KICKED_OFF`; later yields go to the watchers of + `complete()`, which finishes when the generator does. + + `StandardFlyable` runs it in the task of the status `complete()` returns, + and cancels that task on `unstage()`. Do not leave anything + running that this generator does not wait for: own a status with + `async with`, and run concurrent work through `watch_readback` or an + `asyncio.TaskGroup`. For example: + + ```python + async def fly(self, ctx): + async with AsyncStatus(self.move(ctx.end, timeout)) as status: + await asyncio.sleep(ctx.acceleration_time) + yield KICKED_OFF + async for update in watch_readback(self, status, target=ctx.end): + yield update + ``` + + :param ctx: the context yielded by `prepare_to_fly`. """ - pass - - async def on_stage(self) -> None: - """Set the flyer up on `stage()`. Defaults to `stop`.""" - await self.stop() - - async def on_unstage(self) -> None: - """Clean the flyer up on `unstage()`. Defaults to `stop`.""" - await self.stop() - def with_device(self, name: str = "") -> "StandardFlyable": + def with_device(self, name: str = "") -> "StandardFlyable[Any, Any]": """Wrap this logic in an ephemeral `StandardFlyable` for use in a plan.""" - flyer = StandardFlyable(name=name) - flyer.flyable_logic = self - return flyer + return _EphemeralFlyable(self, name=name) class FlyMotorInfo(ConfinedModel): @@ -130,14 +181,6 @@ def _ramp_distance(self, acceleration_time: float) -> float: ) -class _FlyStage(Enum): - """Lifecycle stage of a `StandardFlyable`, used to enforce call ordering.""" - - IDLE = "IDLE" - PREPARED = "PREPARED" - KICKED_OFF = "KICKED_OFF" - - class StandardFlyable( _StandardBase, Preparable, @@ -146,137 +189,200 @@ class StandardFlyable( ): """Device that provides standard logic for flying. - This class must be inherited and have a `flyable_logic` @cached_property. + This class must be inherited and have a `logic` @cached_property. For an ephemeral flyer in a plan, call `FlyableLogic.with_device` instead of - inheriting. It owns the context threaded between the logic's stages and - enforces prepare -> kickoff -> complete ordering. `stage()`/`unstage()` run - the logic's `on_stage`/`on_unstage` (which default to `stop`) and reset the - context. + inheriting. It must be staged before it is prepared. `prepare()`, `kickoff()` + and `complete()` must then be called in that order, otherwise they raise + `TransitionError`; it can be prepared again only once complete. It holds the + context yielded by the logic's `prepare_to_fly` from a successful + `prepare()` until `unstage()`. + + `kickoff()` starts the logic's `fly` generator in the task of a status and + finishes when it yields `KICKED_OFF`; `complete()` returns that status, which + finishes when the generator does and reports its later yields to watchers. + A failure before `KICKED_OFF` fails `kickoff()`, one after it fails + `complete()`; either leaves the device staged. Cancelling `complete()`'s + status cancels the fly, cancelling `kickoff()` before `KICKED_OFF` does too. + + `stage()` runs the logic's `ensure_ready`; it raises unless the device is + unstaged. On a device that is also a `StandardMovable` this stops a `set()` in + progress, as `ensure_ready` defaults to `ensure_stopped`. `unstage()` cancels a + running `prepare()` or fly, then always runs the logic's `ensure_stopped`, + which also stops a `set()` in progress. On a device that is also a + `StandardMovable`, `stop()` does not end a `prepare()` or fly, only + `unstage()` does, but it does halt the hardware, so it can leave `prepare()` + finishing short of the start. + `prepare()` and `complete()` report progress to watchers named after this + device; `kickoff()` reports none. """ + # The context of the last successful prepare(), until unstage() + _ctx: CtxT | None = None + # The fly started by kickoff(), returned by complete() until the next prepare() + _fly_status: WatchableAsyncStatus | None = None + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # Contribute the flyer's stage/unstage hooks (composes with any other - # _StandardBase mix-in, e.g. StandardReadable on a Motor). - self._stage_funcs += (self._on_stage,) - self._unstage_funcs += (self._on_unstage,) - # Context threaded prepare -> kickoff -> complete. Only meaningful once - # prepared; access is guarded by _fly_stage. - self._fly_ctx: CtxT = cast(CtxT, None) - self._fly_stage = _FlyStage.IDLE + # Composes with any other _StandardBase mix-in, e.g. StandardReadable + self._stage_funcs += (self.__on_stage,) + self._unstage_funcs += (self.__on_unstage,) @cached_property - def flyable_logic(self) -> FlyableLogic[PrepareT, CtxT]: + def _state(self) -> _StateMachine: + # super()._state caches the base machine in the instance first, then this + # property overwrites it with the combined one + return super()._state | { + _State.STAGED: {_State.PREPARING}, + _State.PREPARING: {_State.PREPARED, _State.STAGED}, + _State.PREPARED: {_State.FLYING}, + _State.FLYING: {_State.FLOWN, _State.STAGED}, + _State.FLOWN: {_State.PREPARING}, + } + + @abstract_cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: """The logic object that describes how this device flies. A static flyer (e.g. `Motor`) provides this as a `@cached_property` that builds a `FlyableLogic` from its signals. An ephemeral flyer created by - `FlyableLogic.with_device` has it set directly on the instance. + `FlyableLogic.with_device` returns the logic it was created with. + + A Device that is flyable *and* movable implements this once with a logic + object inheriting both `FlyableLogic` and `MovableLogic`; each mix-in + declares `logic` with its own required type, so the type + checker verifies the one implementation against both. """ - raise NotImplementedError( - f"{type(self).__name__} has no flyable_logic; override it as a " - "@cached_property or create the flyer via FlyableLogic.with_device()" + raise NotImplementedError + + async def __on_stage(self) -> None: + # Forget the last run here rather than in unstage, which RE.stop, abort + # or halt may have failed or cancelled + self._ctx = None + self._fly_status = None + await self.logic.ensure_ready() + + async def __on_unstage(self) -> None: + # Always stop: bluesky stops every movable it set when a run ends or + # pauses, so sparing a set() here would only defer its stop + await self.logic.ensure_stopped() + + def _prepared_ctx(self, verb: str) -> CtxT: + """The context of the last successful `prepare()`, for the called `verb`.""" + self._state.check_one_of( + _State.PREPARED, + _State.FLYING, + _State.FLOWN, + _State.TRIGGERING, + _State.TRIGGERED, + verb=verb, ) - - def _reset_fly_state(self) -> None: - self._fly_ctx = cast(CtxT, None) - self._fly_stage = _FlyStage.IDLE - - @AsyncStatus.wrap - async def _on_stage(self) -> None: - await self.flyable_logic.on_stage() - self._reset_fly_state() - - @AsyncStatus.wrap - async def _on_unstage(self) -> None: - await self.flyable_logic.on_unstage() - self._reset_fly_state() + return cast(CtxT, self._ctx) @WatchableAsyncStatus.wrap async def prepare(self, value: PrepareT) -> AsyncIterator[WatcherUpdate]: - """Move to the start and set up the fly scan. + """Set up the fly scan and move to its start. - If the logic is also a `MovableLogic` (e.g. a flying `Motor`), report - progress of the move to the start to watchers by observing its readback, - starting once `on_prepare` writes the setpoint it is moving to. + Reports the progress the logic's `prepare_to_fly` yields after the + context. """ - logic = self.flyable_logic - - async def on_prepare() -> None: - self._fly_ctx = await logic.on_prepare(value) - - if isinstance(logic, MovableLogic): - initial, (units, precision) = await asyncio.gather( - logic.readback.get_value(), logic.get_units_precision() - ) - current, target = initial, None - setpoint_seen = False - async with AsyncStatus(on_prepare()) as preparing: - async for signal, signal_value in observe_signals_value( - logic.readback, logic.setpoint, done_status=preparing - ): - if signal is logic.readback: - current = signal_value - elif setpoint_seen: - target = signal_value - else: - # The first setpoint value is the stale one from before the move - setpoint_seen = True - if target is not None: - yield WatcherUpdate( - current=current, - initial=initial, - target=target, - name=self.name, - unit=units, - precision=precision, + # A failed prepare must leave nothing to kick off + async with self._state.during( + _State.PREPARING, end=_State.PREPARED, on_error=_State.STAGED + ): + self._fly_status = None + hook = f"{type(self.logic).__name__}.prepare_to_fly()" + # A failed prepare leaves no context to carry over + previous_ctx, self._ctx = self._ctx, None + steps = self.logic.prepare_to_fly(self.name, value, previous_ctx) + async with contextlib.aclosing(steps): + try: + ctx = await anext(steps) + except StopAsyncIteration: + raise RuntimeError( + f"{hook} finished without yielding its context" + ) from None + if isinstance(ctx, WatcherUpdate): + raise TypeError(f"{hook} must yield its context first, not {ctx!r}") + async for update in steps: + if not isinstance(update, WatcherUpdate): + raise TypeError( + f"{hook} must yield WatcherUpdate after its context, " + f"not {type(update).__name__}: {update!r}" ) - else: - await on_prepare() - self._fly_stage = _FlyStage.PREPARED + yield replace(update, name=self.name) + self._ctx = ctx @AsyncStatus.wrap async def kickoff(self) -> None: - """Start the fly scan.""" - if self._fly_stage is not _FlyStage.PREPARED: - raise RuntimeError( - f"{self.name}: prepare() must be called before kickoff()" + """Start the logic's `fly`, returning once it has yielded `KICKED_OFF`.""" + kicked_off: asyncio.Future[None] = asyncio.get_running_loop().create_future() + status = WatchableAsyncStatus(self._fly(kicked_off), name=self.name) + try: + await asyncio.wait( + [kicked_off, status.task], return_when=asyncio.FIRST_COMPLETED ) - self._fly_ctx = await self.flyable_logic.on_kickoff(self._fly_ctx) - self._fly_stage = _FlyStage.KICKED_OFF + except BaseException: + # Cancelled, even just after KICKED_OFF: end the fly too, and report + # our cancel rather than any error the fly raised while ending + status.task.cancel() + await asyncio.wait([status.task]) + if not status.task.cancelled(): + status.task.exception() + raise + if not kicked_off.done(): + # The fly ended before KICKED_OFF, raise why + await status + self._fly_status = status + + async def _fly( + self, kicked_off: asyncio.Future[None] + ) -> AsyncIterator[WatcherUpdate]: + async with self._state.during( + _State.FLYING, end=_State.FLOWN, on_error=_State.STAGED + ): + hook = f"{type(self.logic).__name__}.fly()" + fly = self.logic.fly(self._prepared_ctx("kickoff")) + async with contextlib.aclosing(fly): + try: + first = await anext(fly) + except StopAsyncIteration: + raise RuntimeError( + f"{hook} finished without yielding KICKED_OFF" + ) from None + if first is not KICKED_OFF: + raise TypeError( + f"{hook} must yield KICKED_OFF first, not {first!r}" + ) + kicked_off.set_result(None) + async for update in fly: + if not isinstance(update, WatcherUpdate): + raise TypeError( + f"{hook} must yield WatcherUpdate after KICKED_OFF, " + f"not {type(update).__name__}: {update!r}" + ) + yield replace(update, name=self.name) - @WatchableAsyncStatus.wrap - async def complete(self) -> AsyncIterator[WatcherUpdate]: - """Block until the fly scan is done. + def complete(self) -> WatchableAsyncStatus[Any]: + """Return the status of the fly the last `kickoff()` started. - If the logic is also a `MovableLogic` (e.g. a flying `Motor`), report - progress to watchers by observing its readback while the fly scan runs, - reusing the same watcher-update stream as `StandardMovable.set`. Flyers - whose logic is not movable simply block with no progress updates. + It reports the progress `fly` yields after `KICKED_OFF`, starting with the + latest, and finishes when `fly` does. Cancelling it cancels the fly. """ - if self._fly_stage is not _FlyStage.KICKED_OFF: - raise RuntimeError( - f"{self.name}: kickoff() must be called before complete()" + if self._fly_status is None: + raise TransitionError( + f"{self.name}: cannot complete from {self._state.state.name}, " + "allowed only after kickoff()" ) - logic = self.flyable_logic - if isinstance(logic, MovableLogic): - initial, target, (units, precision) = await asyncio.gather( - logic.readback.get_value(), - logic.setpoint.get_value(), - logic.get_units_precision(), - ) - async with AsyncStatus(logic.on_complete(self._fly_ctx)) as completing: - async for current_position in observe_value( - logic.readback, done_status=completing - ): - yield WatcherUpdate( - current=current_position, - initial=initial, - target=target, - name=self.name, - unit=units, - precision=precision, - ) - else: - await self.flyable_logic.on_complete(self._fly_ctx) - self._fly_stage = _FlyStage.IDLE + return self._fly_status + + +class _EphemeralFlyable(StandardFlyable[PrepareT, CtxT]): + """A `StandardFlyable` with the logic it was created with.""" + + def __init__(self, logic: FlyableLogic[PrepareT, CtxT], name: str = "") -> None: + self._logic = logic + super().__init__(name=name) + + @cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: + return self._logic diff --git a/src/ophyd_async/core/_movable.py b/src/ophyd_async/core/_movable.py index beeb1a1ee8..d8329191b8 100644 --- a/src/ophyd_async/core/_movable.py +++ b/src/ophyd_async/core/_movable.py @@ -1,10 +1,8 @@ import asyncio import time -from abc import abstractmethod -from collections.abc import Callable -from dataclasses import dataclass, field -from enum import Enum -from functools import cached_property +import warnings +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass, field, replace from typing import Generic from bluesky.protocols import ( @@ -16,16 +14,18 @@ Subscribable, ) -from ._device import Device, DeviceMock, default_mock_class +from ._device import DeviceMock, default_mock_class from ._mock_signal_utils import callback_on_mock_put, set_mock_value from ._signal import SignalR, SignalRW, observe_value, set_and_wait_for_other_value from ._signal_backend import SignalDatatypeT +from ._standard_base import Logic, StoppedError, _StandardBase from ._status import AsyncStatus, WatchableAsyncStatus from ._utils import ( CALCULATE_TIMEOUT, CalculatableTimeout, Callback, WatcherUpdate, + abstract_cached_property, ) @@ -48,21 +48,28 @@ def __call__(self) -> float | None: @dataclass -class MovableLogic(Generic[SignalDatatypeT]): +class MovableLogic(Logic, Generic[SignalDatatypeT]): """Minimum logic needed for controlling a `StandardMovable`. - Can be inherited to add specialised logic for stop, checking if a move is valid, - calculate a valid timeout for a move, and add units and precision. Sub classes can - also override the `get_move_status` and `move` methods if more control is needed - to provide a custom AsyncStatus and to handle the move logic. + Inherit it to say how to stop the motion, and optionally to check a move is + valid, calculate a timeout for a move, and add units and precision. Subclasses + can also override `move` if more control is needed over the move itself. """ setpoint: SignalRW[SignalDatatypeT] readback: SignalR[SignalDatatypeT] - async def stop(self) -> None: - """Optional hook to add logic on how to stop the motion.""" - pass + def __init_subclass__(cls, **kwargs) -> None: + super().__init_subclass__(**kwargs) + if "stop" in cls.__dict__: + warnings.warn( + f"{cls.__name__}.stop() is deprecated: rename it to " + "ensure_stopped(), which must wait until the motion has stopped", + DeprecationWarning, + stacklevel=2, + ) + if "ensure_stopped" not in cls.__dict__: + cls.ensure_stopped = cls.__dict__["stop"] async def check_move(self, new_position: SignalDatatypeT) -> None: """Optional hook to validate the move. @@ -111,20 +118,47 @@ async def connect(self, device: "StandardMovable") -> None: """Mock signals to do an instant move on setpoint write.""" def _instant_move(value): - set_mock_value(device.movable_logic.readback, value) # Arrive instantly + set_mock_value(device.logic.readback, value) # Arrive instantly + + callback_on_mock_put(device.logic.setpoint, _instant_move) - callback_on_mock_put(device.movable_logic.setpoint, _instant_move) +async def watch_readback( + logic: MovableLogic[SignalDatatypeT], + status: AsyncStatus, + target: SignalDatatypeT, + initial: SignalDatatypeT | None = None, +) -> AsyncIterator[WatcherUpdate[SignalDatatypeT]]: + """Report the progress of a move by watching the readback of a logic. -class StopState(Enum): - NONE = "NONE" - USER_SUCCESS = "USER_SUCCESS" - USER_FAILURE = "USER_FAILURE" + Yields a `WatcherUpdate` for each readback value until the status is done, + then re-raises any error the status raised. Use it from a `FlyableLogic` + hook that moves the logic's device, returning it from the hook as its + progress. + + :param logic: the logic whose readback is watched, and which supplies the + units and precision. + :param status: the status of the move, which is left running. + :param target: the value the move is going to. + :param initial: where the move started from. Defaults to the readback now. + """ + if initial is None: + initial = await logic.readback.get_value() + units, precision = await logic.get_units_precision() + async for current in observe_value(logic.readback, done_status=status): + yield WatcherUpdate( + current=current, + initial=initial, + target=target, + name=logic.readback.name, + unit=units, + precision=precision, + ) @default_mock_class(InstantMovableMock) class StandardMovable( - Device, + _StandardBase, Locatable[SignalDatatypeT], Checkable[SignalDatatypeT], Stoppable, @@ -133,102 +167,104 @@ class StandardMovable( ): """Device that provides standard logic for moving. - This class must be inherited and have a `movable_logic` @cached_property. - If stop is called while moving, it will raise a RuntimeError. + This class must be inherited and have a `logic` @cached_property. + One `set()` runs at a time: a new one ends the one in progress, which raises + `StoppedError`. `stop()` ends the `set()` in progress, which raises + `StoppedError` unless it was `stop(success=True)`, then runs the logic's + `ensure_stopped` without waiting for the `set()` to finish. A + `StandardMovable` alone does nothing on `stage()` or `unstage()`; combined with + a `StandardFlyable`, `stage()` and `unstage()` run its `ensure_ready` and + `ensure_stopped`, which stop a `set()` in progress. + + `stop()` ends only a `set()`: it does not end a `prepare()` or fly of a + `StandardFlyable`, only `unstage()` does. `ensure_stopped` still halts the + hardware, so a `stop()` during the run-up of a `prepare()` (for example from + `bps.stop`, or a `RunEngine` pause of a device that was also `set()`, with + `rewindable=False`) can leave `prepare()` finishing short of the start. """ - # Whether set() should complete successfully or raise an error due to stop called or - # due to timeout. - _stop_state: Enum = StopState.NONE - _move_status: AsyncStatus | None = None + # Ends the latest set(): resolved by stop(), or by the next set() + _set_stop: asyncio.Future[Exception | None] | None = None - @cached_property - @abstractmethod - def movable_logic(self) -> MovableLogic: + @abstract_cached_property + def logic(self) -> MovableLogic[SignalDatatypeT]: """The logic object that describes how this device moves. This is intentionally public so that mock helpers (e.g. `InstantMovableMock`) and subclasses can access the `setpoint` and `readback` signals directly. Subclasses must implement this as a - `@cached_property` that returns a `MovableLogic` instance. + `@cached_property` returning a `MovableLogic`. + + A Device that is movable *and* flyable (e.g. `Motor`) implements this + once with a logic object that inherits both `MovableLogic` and + `FlyableLogic`; each mix-in declares `logic` with its own + required type, so the type checker verifies the one implementation + against both. """ + raise NotImplementedError async def check_value(self, value: SignalDatatypeT) -> None: """Check the move is valid before doing it.""" - await self.movable_logic.check_move(value) + await self.logic.check_move(value) - @WatchableAsyncStatus.wrap - async def set( + def set( self, new_position: SignalDatatypeT, timeout: CalculatableTimeout = CALCULATE_TIMEOUT, - ): - """Move to the given value.""" - self._stop_state = StopState.NONE - old_position, (units, precision) = await asyncio.gather( - self.movable_logic.readback.get_value(), - self.movable_logic.get_units_precision(), + ) -> WatchableAsyncStatus[SignalDatatypeT]: + """Move to the given value, ending any `set()` in progress.""" + self._end_set(StoppedError(f"Device {self.name}: superseded by a new set()")) + self._set_stop = asyncio.get_running_loop().create_future() + return WatchableAsyncStatus( + self._set(new_position, timeout), + name=self.name, + stop=self._set_stop, ) - await self.movable_logic.check_move(new_position) + async def _set( + self, new_position: SignalDatatypeT, timeout: CalculatableTimeout + ) -> AsyncIterator[WatcherUpdate[SignalDatatypeT]]: + old_position = await self.logic.readback.get_value() + await self.logic.check_move(new_position) if timeout == CALCULATE_TIMEOUT: - move_timeout = await self.movable_logic.calculate_timeout( + move_timeout = await self.logic.calculate_timeout( old_position, new_position ) else: move_timeout = timeout + move = self.logic.move( + new_position=new_position, timeout=MoveTimeout(move_timeout) + ) + async with AsyncStatus(move) as move_status: + async for update in watch_readback( + self.logic, move_status, target=new_position, initial=old_position + ): + yield replace(update, name=self.name) + + async def stop(self, success: bool = False) -> None: + """End any `set()` in progress, then run `ensure_stopped`. + + A `prepare()` or fly of a `StandardFlyable` is not ended, only halted. + """ + self._end_set( + None if success else StoppedError(f"Device {self.name} was stopped.") + ) + await self.logic.ensure_stopped() - try: - async with AsyncStatus( - self.movable_logic.move( - new_position=new_position, timeout=MoveTimeout(move_timeout) - ) - ) as self._move_status: - async for current_position in observe_value( - self.movable_logic.readback, - done_status=self._move_status, - ): - yield WatcherUpdate( - current=current_position, - initial=old_position, - target=new_position, - name=self.name, - unit=units, - precision=precision, - ) - # Convert cancellation into the appropriate stop result. - # If stop(success=True) was called, complete without raising. - # If stop(success=False) was called, raise a RuntimeError below. - # If not stopped but times out, show the CancelledError from the timeout. - except asyncio.CancelledError: - if self._stop_state == StopState.NONE: - raise - finally: - self._move_status = None - - # Raise error if needed, reset stop state. - stop_state = self._stop_state - self._stop_state = StopState.NONE - if stop_state == StopState.USER_FAILURE: - raise RuntimeError(f"Device {self.name} was stopped.") - - async def stop(self, success=False): - """Request to stop moving and return immediately.""" - self._stop_state = StopState.USER_SUCCESS if success else StopState.USER_FAILURE - await self.movable_logic.stop() - if self._move_status: - self._move_status.task.cancel() + def _end_set(self, error: StoppedError | None) -> None: + if self._set_stop is not None and not self._set_stop.done(): + self._set_stop.set_result(error) def set_name(self, name: str, *, child_name_separator: str | None = None) -> None: super().set_name(name, child_name_separator=child_name_separator) # Readback should be named the same as its parent in read() - self.movable_logic.readback.set_name(name) + self.logic.readback.set_name(name) async def locate(self) -> Location[SignalDatatypeT]: """Return the current setpoint and readback of the device.""" setpoint, readback = await asyncio.gather( - self.movable_logic.setpoint.get_value(), - self.movable_logic.readback.get_value(), + self.logic.setpoint.get_value(), + self.logic.readback.get_value(), ) return Location(setpoint=setpoint, readback=readback) @@ -236,7 +272,7 @@ def subscribe_reading( self, function: Callback[dict[str, Reading[SignalDatatypeT]]] ) -> None: """Subscribe to reading.""" - self.movable_logic.readback.subscribe_reading(function) + self.logic.readback.subscribe_reading(function) subscribe = subscribe_reading @@ -244,4 +280,4 @@ def clear_sub( self, function: Callback[dict[str, Reading[SignalDatatypeT]]] ) -> None: """Unsubscribe.""" - self.movable_logic.readback.clear_sub(function) + self.logic.readback.clear_sub(function) diff --git a/src/ophyd_async/core/_signal.py b/src/ophyd_async/core/_signal.py index 99ab1ce406..94796ec43d 100644 --- a/src/ophyd_async/core/_signal.py +++ b/src/ophyd_async/core/_signal.py @@ -1,12 +1,12 @@ from __future__ import annotations import asyncio -import contextlib import functools import inspect import time import warnings from collections.abc import AsyncGenerator, Callable +from contextlib import aclosing from typing import Any, Generic, TypeVar, cast from bluesky.protocols import ( @@ -477,13 +477,18 @@ async def observe_value( do_something_with(value) ``` """ - async for _, value in observe_signals_value( - signal, - timeout=timeout, - done_status=done_status, - done_timeout=done_timeout, - ): - yield value + # Closing this generator must close the inner one, so its unsubscribe runs + # then, not when the abandoned inner generator is finalised + async with aclosing( + observe_signals_value( + signal, + timeout=timeout, + done_status=done_status, + done_timeout=done_timeout, + ) + ) as updates: + async for _, value in updates: + yield value def _get_iteration_timeout( @@ -600,11 +605,14 @@ def __init__(self, match: SignalDatatypeT | Callable[[SignalDatatypeT], bool]): self._matcher_name = repr(match) async def _wait_for_value(self, signal: SignalR[SignalDatatypeT]): - async for value in observe_value(signal): - self._last_value = value - self.got_first_value.set() - if self._matcher(value): - return + # Closed on return, so the unsubscribe (which cancels any poll task) runs + # now, not when the abandoned generator is finalised after we have gone + async with aclosing(observe_value(signal)) as values: + async for value in values: + self._last_value = value + self.got_first_value.set() + if self._matcher(value): + return async def wait_for_value( self, signal: SignalR[SignalDatatypeT], timeout: float | None @@ -680,13 +688,11 @@ async def set_and_wait_for_other_value( """ # Start monitoring before the set to avoid a race condition checker = _ValueChecker(match_value) - wait_task = asyncio.create_task( + # The wait is cancelled when we exit, whether by error, cancel or success + async with AsyncStatus( checker.wait_for_value(match_signal, timeout=timeout) - ) - - # Put this in a try/except to ensure wait_task is cancelled when we exit - try: - # NOTE: this timeout races wait_task's own internal timeout (same + ) as wait_status: + # NOTE: this timeout races the wait's own internal timeout (same # duration, started moments earlier) - whichever elapses first wins. # That race is harmless because each path reports its own distinct, # helpful message. @@ -704,19 +710,13 @@ async def set_and_wait_for_other_value( if not wait_for_set_completion: # Just wait for the match_signal to be at value - await wait_task + await wait_status else: # Wait for both to complete, so if the set raises an exception it # is surfaced early. Wait for the status task so if either errors # the other is cancelled - await asyncio.gather(wait_task, status.task) + await asyncio.gather(wait_status.task, status.task) return status - except Exception: - # Make sure the wait is not left dangling if there was an error - wait_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await wait_task - raise async def set_and_wait_for_value( diff --git a/src/ophyd_async/core/_standard_base.py b/src/ophyd_async/core/_standard_base.py index 3c4d69a221..44f275b8c4 100644 --- a/src/ophyd_async/core/_standard_base.py +++ b/src/ophyd_async/core/_standard_base.py @@ -1,9 +1,145 @@ import asyncio -from collections.abc import Callable +import inspect +from abc import ABC, abstractmethod +from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping +from contextlib import asynccontextmanager +from enum import Enum +from functools import cached_property -from ._device import Device +from ._device import Device, name_added_to_errors from ._protocol import AsyncStageable -from ._status import AsyncStatus, AsyncStatusBase +from ._status import AsyncStatus + + +class _State(Enum): + """Every state of `_StandardBase`, `StandardFlyable` and `StandardDetector`. + + Each class in the stack adds the states it needs and the edges between them. + """ + + # _StandardBase + UNSTAGED = "UNSTAGED" + STAGED = "STAGED" + # StandardFlyable + PREPARING = "PREPARING" + PREPARED = "PREPARED" + FLYING = "FLYING" + FLOWN = "FLOWN" + # StandardDetector + TRIGGERING = "TRIGGERING" + TRIGGERED = "TRIGGERED" + + +#: States that mean a verb is running; leaving one is only allowed to the task +#: that entered it (or by `unstage()`, which cancels that task). +_IN_PROGRESS = frozenset({_State.PREPARING, _State.FLYING, _State.TRIGGERING}) + + +class TransitionError(RuntimeError): + """A verb was called in a state that does not allow it.""" + + +class _StateMachine: + """Current state of a device, the edges it may follow and the running verb.""" + + def __init__(self, device: Device, edges: Mapping[_State, set[_State]]) -> None: + self._device = device + self._edges = edges + self.state = _State.UNSTAGED + self._task: asyncio.Task | None = None + + def __or__(self, edges: Mapping[_State, set[_State]]) -> "_StateMachine": + """A new machine for the same device, with these edges added to ours.""" + return _StateMachine( + self._device, + {s: self._edges.get(s, set()) | edges.get(s, set()) for s in _State}, + ) + + def transition(self, to: _State) -> asyncio.Task | None: + """Move to `to`, or raise `TransitionError` if there is no such edge. + + Returns the verb's task if this cancelled it, so the caller can wait for it. + """ + # Unstage is the cleanup path: works from any state, cancels the verb + allowed = self._edges.get(self.state, set()) | {_State.UNSTAGED} + if to not in allowed: + raise TransitionError( + f"{self._device.name}: cannot go to {to.name} from " + f"{self.state.name}, allowed: " + f"{', '.join(sorted(s.name for s in allowed))}" + ) + cancelled = None + if self.state in _IN_PROGRESS: + if to is _State.UNSTAGED: + # Unstage is the only thing that ends a verb from outside + if self._task is not None and self._task is not asyncio.current_task(): + cancelled = self._task + cancelled.cancel() + elif self._task is not asyncio.current_task(): + raise TransitionError( + f"{self._device.name}: cannot go to {to.name} from " + f"{self.state.name}, a verb is running in another task and " + "only unstage can end it" + ) + self._task = None + if to in _IN_PROGRESS: + self._task = asyncio.current_task() + self.state = to + return cancelled + + def check_one_of(self, *states: _State, verb: str) -> None: + """Raise `TransitionError` naming `verb` unless the state is in `states`.""" + if self.state not in states: + raise TransitionError( + f"{self._device.name}: {verb} not allowed from {self.state.name}, " + f"allowed from: {', '.join(sorted(s.name for s in states))}" + ) + + @asynccontextmanager + async def during( + self, + state: _State, + *, + end: _State, + on_error: _State, + ) -> AsyncGenerator[None, None]: + """Run a verb in the in-progress `state`, then go to `end` or `on_error`. + + Notes the device's name on any exception the verb raises. + + Safe to wrap `yield`s of an async generator. If `unstage()` has moved the + machine on in the meantime, even into a new run of the same verb, this + verb no longer owns the state and is left alone to finish cancelling. + """ + self.transition(state) + try: + with name_added_to_errors(self._device): + yield + except BaseException: + # Can be false: unstage cancelled this verb and moved the state on, + # or an abandoned async generator was closed later from another task + if self.state is state and self._task is asyncio.current_task(): + self.transition(on_error) + raise + else: + if self.state is state and self._task is asyncio.current_task(): + self.transition(end) + + +class StoppedError(RuntimeError): + """A set, prepare or fly was ended early by `stop()` or a new `set()`.""" + + +class Logic(ABC): + """Base of `MovableLogic` and `FlyableLogic`: stopping.""" + + @abstractmethod + async def ensure_stopped(self) -> None: + """Stop the hardware and wait until it has stopped. + + Called by `StandardMovable.stop()`, which bluesky calls on pause and at + the end of a run, and by `StandardFlyable` on `unstage()`. + """ class _StandardBase(Device, AsyncStageable): @@ -13,17 +149,42 @@ class _StandardBase(Device, AsyncStageable): something on `stage()`/`unstage()`. Rather than each defining those methods and shadowing the others via the MRO when combined on one `Device` (e.g. a motor that is Readable + Movable + Flyable), they register callables into - `_stage_funcs`/`_unstage_funcs`, which this single base gathers. + `_stage_funcs`/`_unstage_funcs`, which this single base gathers. Each mix-in + names its hooks `__on_stage`/`__on_unstage`, name-mangled so they never + collide. """ # Immutable defaults to avoid accidental sharing between instances - _stage_funcs: tuple[Callable[[], AsyncStatusBase], ...] = () - _unstage_funcs: tuple[Callable[[], AsyncStatusBase], ...] = () + _stage_funcs: tuple[Callable[[], Awaitable[None]], ...] = () + _unstage_funcs: tuple[Callable[[], Awaitable[None]], ...] = () + + @cached_property + def _state(self) -> _StateMachine: + # cached_property as Device registration runs before super().__init__ + return _StateMachine(self, {_State.UNSTAGED: {_State.STAGED}}) @AsyncStatus.wrap async def stage(self) -> None: - await asyncio.gather(*(func().task for func in self._stage_funcs)) + self._state.transition(_State.STAGED) + with name_added_to_errors(self): + await asyncio.gather(*(func() for func in self._stage_funcs)) @AsyncStatus.wrap async def unstage(self) -> None: - await asyncio.gather(*(func().task for func in self._unstage_funcs)) + cancelled = self._state.transition(_State.UNSTAGED) + # Made now, so any state they take is taken before a restage, run below + unstaging = [func() for func in self._unstage_funcs] + try: + if cancelled is not None: + # Whatever the verb was doing must have stopped before the hardware + await asyncio.wait([cancelled]) + with name_added_to_errors(self): + await asyncio.gather(*unstaging) + finally: + for unstage in unstaging: + if inspect.iscoroutine(unstage): + unstage.close() + if cancelled is not None and not cancelled.cancelled(): + # The verb failed while being cancelled + if (error := cancelled.exception()) is not None: + raise error diff --git a/src/ophyd_async/core/_status.py b/src/ophyd_async/core/_status.py index 827df3f75e..f710d8c77b 100644 --- a/src/ophyd_async/core/_status.py +++ b/src/ophyd_async/core/_status.py @@ -3,11 +3,15 @@ from __future__ import annotations import asyncio -import contextlib import functools import time -from asyncio import CancelledError -from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine +from asyncio import FIRST_COMPLETED, CancelledError +from collections.abc import ( + AsyncIterator, + Awaitable, + Callable, + Coroutine, +) from dataclasses import asdict, replace from typing import Any, Generic @@ -15,7 +19,7 @@ from ._device import Device from ._protocol import Watcher -from ._utils import Callback, P, T, V, WatcherUpdate +from ._utils import Callback, P, T, V, WatcherUpdate, logger class AsyncStatusBase(Status, Awaitable[T]): @@ -132,10 +136,10 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): self.task.cancel() - # Need to await the task to suppress teardown warnings, but - # we know it will raise CancelledError as we just cancelled it - with contextlib.suppress(CancelledError): - await self.task + # An outside cancel while we wait propagates, it is not swallowed + await asyncio.wait([self.task]) + if not self.task.cancelled() and (error := self.task.exception()): + raise error # Raise any errors from the block, but not cancellation errors from the status return False @@ -225,12 +229,21 @@ class WatchableAsyncStatus(AsyncStatusBase[None], Generic[T]): """ def __init__( - self, iterator: AsyncIterator[WatcherUpdate[T]], name: str | None = None + self, + iterator: AsyncIterator[WatcherUpdate[T]], + name: str | None = None, + *, + stop: asyncio.Future[Exception | None] | None = None, ): self._watchers: list[Watcher[T]] = [] self._start = time.monotonic() self._last_update: WatcherUpdate[T] | None = None - super().__init__(self._notify_watchers_from(iterator), name) + super().__init__( + self._notify_watchers_from(iterator) + if stop is None + else self._notify_until_stopped(iterator, stop), + name, + ) async def _notify_watchers_from(self, iterator: AsyncIterator[WatcherUpdate[T]]): async for update in iterator: @@ -240,7 +253,59 @@ async def _notify_watchers_from(self, iterator: AsyncIterator[WatcherUpdate[T]]) else replace(update, time_elapsed=time.monotonic() - self._start) ) for watcher in self._watchers: - self._update_watcher(watcher, self._last_update) + # An observer must not be able to fail the operation it watches + try: + self._update_watcher(watcher, self._last_update) + except Exception: + logger.exception( + "%swatcher %r raised, ignoring it", + f"{self._name}: " if self._name else "", + watcher, + ) + + async def _notify_until_stopped( + self, + iterator: AsyncIterator[WatcherUpdate[T]], + stop: asyncio.Future[Exception | None], + ) -> None: + """Notify watchers until the iterator ends or `stop` resolves. + + `stop` resolves with `None` to finish normally or with the error to raise. + """ + # Stopped before we started, so leave the hardware alone + if not stop.done(): + # A task of its own, so the race below can cancel it without + # cancelling us, and a timeout spanning a yield still sees one task. + # Its only awaits are in the iterator, so ending the task ends that too + notify = asyncio.create_task(self._notify_watchers_from(iterator)) + again: asyncio.CancelledError | None = None + try: + await asyncio.wait({notify, stop}, return_when=FIRST_COMPLETED) + # If it finished, even in the same iteration as the stop, the move + # really ended that way, so its outcome wins over the stop + if notify.done(): + return await notify + finally: + # Nobody else holds the task, so we must end it however we leave, + # including on a cancel from outside + if not notify.done(): + notify.cancel() + while not notify.done(): + # Another cancel must not leave its cleanup running after + # we finish, so wait it out and raise the cancel after + try: + await asyncio.wait({notify}) + except asyncio.CancelledError as cancel: + again = cancel + if again is not None: + raise again + # Raise how it ended: an error raised while ending shows through, + # and on a cancel from outside its CancelledError says what it + # was waiting on. Only a cancel by our stop is not an error + if not (stop.done() and notify.cancelled()): + await notify + if (error := stop.result()) is not None: + raise error def _update_watcher(self, watcher: Watcher[T], update: WatcherUpdate[T]): vals = asdict( diff --git a/src/ophyd_async/core/_utils.py b/src/ophyd_async/core/_utils.py index 5de8b78c7c..4dd9e7a8ed 100644 --- a/src/ophyd_async/core/_utils.py +++ b/src/ophyd_async/core/_utils.py @@ -4,7 +4,7 @@ import logging import time from collections.abc import ( - AsyncIterator, + AsyncGenerator, Awaitable, Callable, Iterable, @@ -13,8 +13,7 @@ ) from dataclasses import dataclass from enum import Enum, EnumMeta, StrEnum -from functools import lru_cache -from inspect import isawaitable +from functools import cached_property, lru_cache from typing import ( Any, Generic, @@ -233,7 +232,7 @@ def cached_get_origin(tp: Any) -> Any: return get_origin(tp) -def get_dtype(datatype: type) -> np.dtype: +def get_dtype(datatype: Any) -> np.dtype: """Get the runtime dtype from a numpy ndarray type annotation. ```python @@ -308,9 +307,13 @@ def get_enum_cls(datatype: type | None) -> type[EnumTypes] | None: return None -def get_unique(values: dict[str, T], types: str) -> T: +def get_unique(values: Mapping[str, T] | Sequence[tuple[str, T]], types: str) -> T: """If all values are the same, return that value, otherwise raise TypeError. + :param values: the values by label, as a mapping or as `(label, value)` pairs + when labels may repeat + :param types: what the values are, for the error message + ```python >>> get_unique({"a": 1, "b": 1}, "integers") 1 @@ -321,9 +324,10 @@ def get_unique(values: dict[str, T], types: str) -> T: ``` """ - set_values = set(values.values()) + pairs = list(values.items()) if isinstance(values, Mapping) else values + set_values = {v for _, v in pairs} if len(set_values) != 1: - diffs = ", ".join(f"{k} has {v}" for k, v in values.items()) + diffs = ", ".join(f"{k} has {v}" for k, v in pairs) raise TypeError(f"Differing {types}: {diffs}") return set_values.pop() @@ -340,9 +344,20 @@ async def merge_gathered_dicts( combined_read = await merge_gathered_dicts(s.read() for s in signals) ``` """ + return merge_dicts(await asyncio.gather(*coros)) + + +def merge_dicts(dicts: Iterable[Mapping[str, T]]) -> dict[str, T]: + """Merge dictionaries, raising if any key appears in more than one. + + :example: + ```python + merged = merge_dicts([{"a": 1}, {"b": 2}]) + ``` + """ ret: dict[str, T] = {} duplicates: list[str] = [] - for result in await asyncio.gather(*coros): + for result in dicts: for key in result: if key in ret: duplicates.append(key) @@ -358,36 +373,10 @@ async def merge_gathered_dicts( return ret -def _partition_awaitable( - maybe_awaitables: Iterable[T | Awaitable[T]], -) -> tuple[dict[int, Awaitable[T]], dict[int, T]]: - awaitable: dict[int, Awaitable[T]] = {} - not_awaitable: dict[int, T] = {} - for i, x in enumerate(maybe_awaitables): - if isawaitable(x): - awaitable[i] = x - else: - not_awaitable[i] = x - return awaitable, not_awaitable - - -async def gather_dict(coros: dict[T | Awaitable[T], V | Awaitable[V]]) -> dict[T, V]: - """Await any coros in the keys or values of a dictionary.""" - k_awaitable, k_not_awaitable = _partition_awaitable(coros.keys()) - v_awaitable, v_not_awaitable = _partition_awaitable(coros.values()) - - # Await all awaitables in parallel - k_results, v_results = await asyncio.gather( - asyncio.gather(*k_awaitable.values()), - asyncio.gather(*v_awaitable.values()), - ) - - # Combine awaited and non-awaited values by index - k_map = k_not_awaitable | dict(zip(k_awaitable, k_results, strict=True)) - v_map = v_not_awaitable | dict(zip(v_awaitable, v_results, strict=True)) - - # Reconstruct dict in original index order - return {k_map[i]: v_map[i] for i in range(len(coros))} +async def gather_dict(coros: Mapping[T, Awaitable[V]]) -> dict[T, V]: + """Await the values of a dictionary in parallel, keeping the keys.""" + results = await asyncio.gather(*coros.values()) + return dict(zip(coros, results, strict=True)) def in_micros(t: float) -> int: @@ -464,11 +453,37 @@ def non_zero(value): return int(value) != 0 +def abstract_cached_property(func: Callable[[Any], T]) -> cached_property[T]: + """A `cached_property` that `ABCMeta` treats as abstract. + + `functools.cached_property` does not forward `__isabstractmethod__` from the + function it wraps, so `@cached_property` stacked on `@abstractmethod` + registers nothing: the class stays instantiable and a missing implementation + surfaces as an `AttributeError` on `None` from deep inside `__init__`. Setting + the flag on the descriptor itself makes `Device`'s metaclass collect it, so + the omission is a `TypeError` at instantiation instead. + + The declared return type is deliberately `cached_property[T]` rather than a + subclass, so that a subclass overriding it with a plain `@cached_property` + still gets checked for covariance by the type checker. + + :param func: The property body, which should raise `NotImplementedError`. + """ + prop = cached_property(func) + prop.__isabstractmethod__ = True # type: ignore[attr-defined] + return prop + + async def _wait_for(coro: Awaitable[T], timeout: float | None, source: str) -> T: + """Await `coro`, raising `TimeoutError(source)` if it takes over `timeout`.""" + cm = asyncio.timeout(timeout) try: - return await asyncio.wait_for(coro, timeout) - except TimeoutError as exc: - raise TimeoutError(source) from exc + async with cm: + return await coro + except TimeoutError: + if cm.expired(): + raise TimeoutError(source) from None + raise async def simulate_move( @@ -476,7 +491,7 @@ async def simulate_move( new_position: float, velocity: float, acceleration_time: float, -) -> AsyncIterator[float]: +) -> AsyncGenerator[float, None]: """Yield positions at 10Hz along a trapezoidal velocity profile. :param old_position: Where the move starts. diff --git a/src/ophyd_async/epics/demo/_motor.py b/src/ophyd_async/epics/demo/_motor.py index 308bc4f99a..8b6e57c880 100644 --- a/src/ophyd_async/epics/demo/_motor.py +++ b/src/ophyd_async/epics/demo/_motor.py @@ -24,7 +24,8 @@ class DemoMotorMoveLogic(MovableLogic[float]): velocity: SignalRW[float] stop_: TriggerableCommand - async def stop(self): + async def ensure_stopped(self) -> None: + """Stop the motor, which stops as soon as `stop_` has processed.""" await self.stop_.trigger() async def calculate_timeout( @@ -55,7 +56,7 @@ class DemoMotor(EpicsDevice, StandardReadable, StandardMovable[float]): stop_: A[TriggerableCommand, PvSuffix("Stop.PROC")] @cached_property - def movable_logic(self) -> MovableLogic: + def logic(self) -> MovableLogic: return DemoMotorMoveLogic( readback=self.readback, setpoint=self.setpoint, diff --git a/src/ophyd_async/epics/motor.py b/src/ophyd_async/epics/motor.py index 62dbfd0765..cb8c7fbae4 100644 --- a/src/ophyd_async/epics/motor.py +++ b/src/ophyd_async/epics/motor.py @@ -6,13 +6,16 @@ from __future__ import annotations import asyncio +from collections.abc import AsyncGenerator from dataclasses import dataclass from functools import cached_property from ophyd_async.core import ( CALCULATE_TIMEOUT, DEFAULT_TIMEOUT, + KICKED_OFF, AsyncStatus, + CalculatableTimeout, DeviceMock, FlyableLogic, FlyMotorInfo, @@ -25,11 +28,13 @@ StandardReadable, StrictEnum, TimeoutCalculator, + WatcherUpdate, callback_on_mock_put, default_mock_class, - error_if_none, set_mock_value, simulate_move, + wait_for_value, + watch_readback, ) from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.epics.core import epics_signal_r, epics_signal_rw, epics_signal_w @@ -92,26 +97,30 @@ class UseSetMode(StrictEnum): """Change offset (in record or in controller) when setpoint is written.""" -@dataclass -class MotorFlyCtx: - """State threaded through a motor fly scan (prepare -> kickoff -> complete).""" +@dataclass(frozen=True) +class MotorPrepareCtx: + """What a motor fly scan needs after `prepare_to_fly` has worked it out.""" - #: The info the scan was prepared with, produced by `on_prepare`. - fly_info: FlyMotorInfo - #: The move to the end position, produced by `on_kickoff`. - status: AsyncStatus | None = None + #: Where to start from: the start position with the run-up distance added. + start: float + #: Where to end: the end position with the run-down distance added. + end: float + #: How long the motor takes to get up to speed. + acceleration_time: float + #: Maximum time for the move from start to end. + timeout: CalculatableTimeout @dataclass class MotorFlyableMovableLogic( - MovableLogic[float], FlyableLogic[FlyMotorInfo, MotorFlyCtx] + MovableLogic[float], FlyableLogic[FlyMotorInfo, MotorPrepareCtx] ): """Combined move + fly logic for a motor record. - A single logic object backs a `Motor`'s `movable_logic` and `flyable_logic`: + A single logic object backs a `Motor`'s `logic`: it implements `MovableLogic` (the motor-record move, limit and timeout logic) and `FlyableLogic` (the fly hooks, which reuse `move`/`check_move` and carry - all per-scan state in a `MotorFlyCtx` rather than on the logic). + all per-scan state in a `MotorPrepareCtx` rather than on the logic). """ motor_stop: SignalW[int] @@ -125,16 +134,18 @@ class MotorFlyableMovableLogic( max_velocity: SignalR[float] motor_egu: SignalR[str] - async def stop(self): - """Request to stop moving, but only if the motor is currently moving. + async def ensure_stopped(self) -> None: + """Stop the motor and wait until it is done moving. + + This stops a fly too. The stop is sent even if the motor looks stopped, + as DMOV can flicker at standstill on some motors. - This makes stop idempotent when the motor is already stopped, so a motor - that is both moved and staged in a run (e.g. `bp.scan`) is not stopped - twice -- once by the RunEngine (it stops everything it set) and once by - `unstage`. + The wait allows the acceleration time to decelerate, plus `DEFAULT_TIMEOUT`. """ - if not await self.motor_done_move.get_value(): - await self.motor_stop.set(1) + await self.motor_stop.set(1) + # ACCL is the time to go between rest and VELO, so no stop takes longer + timeout = await self.acceleration_time.get_value() + DEFAULT_TIMEOUT + await wait_for_value(self.motor_done_move, 1, timeout=timeout) async def check_move(self, new_position: float): """Check the positions are within limits. @@ -200,8 +211,8 @@ async def calculate_timeout( async def move(self, new_position: float, timeout: TimeoutCalculator) -> None: """Move by setting the setpoint and waiting for put completion. - Raises `MotorAlarmError` if the readback has an alarm severity set once - the put has completed. + The put completes early if the motor is stopped. Raises `MotorAlarmError` + if the readback has an alarm severity set once the put has completed. """ await self.setpoint.set(new_position, timeout=timeout()) reading = await self.readback.read() @@ -212,8 +223,13 @@ async def move(self, new_position: float, timeout: TimeoutCalculator) -> None: f"after a move to {new_position}." ) - async def on_prepare(self, value: FlyMotorInfo) -> MotorFlyCtx: - """Move to the beginning of a run-up distance ready for a fly scan.""" + async def prepare_to_fly( + self, name: str, value: FlyMotorInfo, previous_ctx: MotorPrepareCtx | None + ) -> AsyncGenerator[MotorPrepareCtx | WatcherUpdate, None]: + """Check the scan is possible, then move to the beginning of the run-up. + + Progress of the move is reported. + """ # Velocity at which the motor travels from start to end, in motor egu/s. max_speed, egu = await asyncio.gather( self.max_velocity.get_value(), self.motor_egu.get_value() @@ -225,43 +241,50 @@ async def on_prepare(self, value: FlyMotorInfo) -> MotorFlyCtx: ) # Check the run-up and run-down positions are within limits acceleration_time = await self.acceleration_time.get_value() - ramp_up_start_pos = value.ramp_up_start_pos(acceleration_time) - ramp_down_end_pos = value.ramp_down_end_pos(acceleration_time) - await asyncio.gather( - self.check_move(ramp_up_start_pos), self.check_move(ramp_down_end_pos) + start = value.ramp_up_start_pos(acceleration_time) + end = value.ramp_down_end_pos(acceleration_time) + await asyncio.gather(self.check_move(start), self.check_move(end)) + yield MotorPrepareCtx( + start=start, + end=end, + acceleration_time=acceleration_time, + timeout=value.timeout, ) # Move to the run-up start at maximum velocity await self.velocity.set(abs(max_speed)) old_position = await self.readback.get_value() - timeout = await self.calculate_timeout(old_position, ramp_up_start_pos) - await self.move(ramp_up_start_pos, lambda: timeout) + timeout = await self.calculate_timeout(old_position, start) + async with AsyncStatus(self.move(start, lambda: timeout)) as status: + async for update in watch_readback( + self, status, target=start, initial=old_position + ): + yield update # Set the velocity we will use for the fly scan await self.velocity.set(value.speed) - return MotorFlyCtx(fly_info=value) - async def on_kickoff(self, ctx: MotorFlyCtx) -> MotorFlyCtx: - """Begin moving the motor from the prepared position to the final position.""" - acceleration_time = await self.acceleration_time.get_value() - target = ctx.fly_info.ramp_down_end_pos(acceleration_time) - if ctx.fly_info.timeout == CALCULATE_TIMEOUT: + async def fly( + self, ctx: MotorPrepareCtx + ) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + """Move from the run-up start to the run-down end, reporting progress. + + Kicked off once the run-up is done and the motor is at constant velocity. + """ + if ctx.timeout == CALCULATE_TIMEOUT: initial = await self.readback.get_value() - timeout = await self.calculate_timeout(initial, target) + timeout = await self.calculate_timeout(initial, ctx.end) else: - timeout = ctx.fly_info.timeout - ctx.status = AsyncStatus(self.move(target, lambda: timeout)) - # Wait out the run-up so the motor is at constant velocity before - # kickoff() returns. A plan can then kickoff the motor and afterwards - # kickoff internally-triggered detectors, which is more accurate for - # long acceleration times. - await asyncio.sleep(acceleration_time) - return ctx - - async def on_complete(self, ctx: MotorFlyCtx) -> None: - """Block until the motor reaches the fly-scan end position.""" - status = error_if_none( - ctx.status, f"kickoff for motor {self.readback.name} not called." - ) - await status + timeout = ctx.timeout + async with AsyncStatus(self.move(ctx.end, lambda: timeout)) as status: + # Wait out the run-up so the motor is at constant velocity before + # kickoff() returns. A plan can then kickoff the motor and afterwards + # kickoff internally-triggered detectors, which is more accurate for + # long acceleration times. + await asyncio.sleep(ctx.acceleration_time) + yield KICKED_OFF + async for update in watch_readback( + self, status, target=ctx.end, initial=ctx.start + ): + yield update class InstantMotorMock(DeviceMock["Motor"]): @@ -283,6 +306,10 @@ def _instant_move(value): set_mock_value(device.motor_done_move, 1) # Done callback_on_mock_put(device.user_setpoint, _instant_move) + # A STOP finishes any move the test has started + callback_on_mock_put( + device.motor_stop, lambda _: set_mock_value(device.motor_done_move, 1) + ) class VeloAndAcclRespectingMotorMock(DeviceMock["Motor"]): @@ -320,7 +347,9 @@ async def _move(value: float) -> None: @default_mock_class(InstantMotorMock) class Motor( - StandardMovable[float], StandardFlyable[FlyMotorInfo, MotorFlyCtx], StandardReadable + StandardMovable[float], + StandardFlyable[FlyMotorInfo, MotorPrepareCtx], + StandardReadable, ): """Device that moves a motor record.""" @@ -364,8 +393,8 @@ def __init__(self, prefix: str, name="") -> None: super().__init__(name) @cached_property - def _logic(self) -> MotorFlyableMovableLogic: - """The combined move + fly logic, shared by movable_logic and flyable_logic.""" + def logic(self) -> MotorFlyableMovableLogic: + """The combined move + fly logic, satisfying both mix-ins at once.""" return MotorFlyableMovableLogic( readback=self.user_readback, setpoint=self.user_setpoint, @@ -380,11 +409,3 @@ def _logic(self) -> MotorFlyableMovableLogic: max_velocity=self.max_velocity, motor_egu=self.motor_egu, ) - - @cached_property - def movable_logic(self) -> MotorFlyableMovableLogic: - return self._logic - - @cached_property - def flyable_logic(self) -> MotorFlyableMovableLogic: - return self._logic diff --git a/src/ophyd_async/epics/pmac/_pmac_trajectory.py b/src/ophyd_async/epics/pmac/_pmac_trajectory.py index 292367f612..c2e3cee0cd 100644 --- a/src/ophyd_async/epics/pmac/_pmac_trajectory.py +++ b/src/ophyd_async/epics/pmac/_pmac_trajectory.py @@ -1,5 +1,6 @@ import asyncio -from dataclasses import dataclass +from collections.abc import AsyncGenerator +from dataclasses import dataclass, field import numpy as np from scanspec.core import Path, Slice @@ -7,10 +8,9 @@ from ophyd_async.core import ( DEFAULT_TIMEOUT, - AsyncStatus, + KICKED_OFF, FlyableLogic, SignalR, - error_if_none, gather_dict, observe_value, set_and_wait_for_value, @@ -39,13 +39,6 @@ SLICE_SIZE = 4000 -@dataclass -class PmacPrepareContext: - path: Path - motor_info: _PmacMotorInfo - ramp_up_time: float - - @dataclass class PmacScanInfo: spec: Spec[Motor] @@ -54,82 +47,109 @@ class PmacScanInfo: @dataclass -class PmacFlyCtx: - """State threaded through a PMAC fly scan (prepare -> kickoff -> complete).""" +class _TrajectoryCursor: + """Where the running trajectory has got to, mutated as points are appended.""" + + #: The PVT the last built slice exited at, which the next slice starts from. + next_pvt: PVT | None = None + #: The number of points loaded into the PMAC so far. + loaded: int = 0 + - #: Trajectory + motion info built by `on_prepare`. - prepare: PmacPrepareContext - #: The running trajectory, produced by `on_kickoff`. - trajectory_status: AsyncStatus | None = None +@dataclass(frozen=True) +class PmacPrepareCtx: + """What a PMAC fly scan needs after `prepare_to_fly` has worked it out.""" + #: The path to fly, consumed a slice at a time as the trajectory runs. + path: Path + #: The first slice of the path, which is built into the trajectory to start with. + first_slice: Slice + #: How many points of the path are left after the first slice. + path_length: int + motor_info: _PmacMotorInfo + ramp_up_pos: dict[Motor, np.float64] + ramp_up_time: float + turnaround_time: float | None + cursor: _TrajectoryCursor = field(default_factory=_TrajectoryCursor) -class PmacTrajectoryFlyableLogic(FlyableLogic[PmacScanInfo, PmacFlyCtx]): + +class PmacTrajectoryFlyableLogic(FlyableLogic[PmacScanInfo, PmacPrepareCtx]): def __init__(self, pmac: PmacIO) -> None: self.pmac = pmac - # Working state mutated by the running trajectory task; it spans prepare - # and the executing trajectory, so it lives on the logic rather than in - # the cross-stage context. - self._next_pvt: PVT | None = None - self._loaded = 0 - self._turnaround_time: float | None = None - - async def on_prepare(self, value: PmacScanInfo) -> PmacFlyCtx: - spec = value.spec - self._turnaround_time = value.turnaround_time - self._next_pvt = None - self._loaded = 0 - path = Path(spec.calculate()) - slice = path.consume(SLICE_SIZE) - path_length = len(path) - motors = slice.axes() - motor_info = await _PmacMotorInfo.from_motors(self.pmac, motors) + + async def prepare_to_fly( + self, name: str, value: PmacScanInfo, previous_ctx: PmacPrepareCtx | None + ) -> AsyncGenerator[PmacPrepareCtx, None]: + path = Path(value.spec.calculate()) + first_slice = path.consume(SLICE_SIZE) + motor_info = await _PmacMotorInfo.from_motors(self.pmac, first_slice.axes()) ramp_up_pos, ramp_up_time = calculate_ramp_position_and_duration( - slice, motor_info, True, value.ramp_time + first_slice, motor_info, True, value.ramp_time ) + ctx = PmacPrepareCtx( + path=path, + first_slice=first_slice, + path_length=len(path), + motor_info=motor_info, + ramp_up_pos=ramp_up_pos, + ramp_up_time=ramp_up_time, + turnaround_time=value.turnaround_time, + ) + yield ctx await asyncio.gather( - self._build_trajectory(motor_info, slice, path_length, ramp_up_time), - self._move_to_start(motor_info, ramp_up_pos), + self._build_trajectory(ctx), + self._move_to_start(ctx.motor_info, ctx.ramp_up_pos), ) - return PmacFlyCtx( - prepare=PmacPrepareContext( - path=path, motor_info=motor_info, ramp_up_time=ramp_up_time + + async def fly(self, ctx: PmacPrepareCtx) -> AsyncGenerator[KICKED_OFF, None]: + """Execute the trajectory, appending the rest of the path as it runs.""" + trajectory = self.pmac.trajectory + async with trajectory.execute_profile.set(True, timeout=None) as execute: + # Wait for the ramp up to happen + await wait_for_value( + trajectory.total_points, + lambda v: v >= 1, + ctx.ramp_up_time + DEFAULT_TIMEOUT, ) + yield KICKED_OFF + # We consume SLICE_SIZE from self.path and parse a trajectory + # containing at least 2 * SLICE_SIZE, as a gapless trajectory + # will contain 2 points per slice frame. If gaps are present, + # additional points are inserted, overfilling the buffer. + min_buffer_size = SLICE_SIZE * 2 + async for current_point in observe_value( + trajectory.total_points, + done_status=execute, + # Limit on PMAC is 4 seconds between points + # Thus this timeout is fine. + timeout=DEFAULT_TIMEOUT, + ): + # Keep a minimum buffer size, if we have more points to append + if ( + len(ctx.path) != 0 + and ctx.cursor.loaded - current_point < min_buffer_size + ): + # Less than SLICE_SIZE * 2 points in the buffer, so refill + next_slice = ctx.path.consume(SLICE_SIZE) + await self._append_trajectory(ctx, next_slice, len(ctx.path)) + await self._check_profile_status( + trajectory.execute_status, trajectory.execute_message ) - async def on_kickoff(self, ctx: PmacFlyCtx) -> PmacFlyCtx: - prepare = ctx.prepare - ctx.trajectory_status = self._execute_trajectory( - prepare.path, prepare.motor_info - ) - # Wait for the ramp up to happen - await wait_for_value( - self.pmac.trajectory.total_points, - lambda v: v >= 1, - prepare.ramp_up_time + DEFAULT_TIMEOUT, - ) - return ctx + async def ensure_stopped(self) -> None: + """Abort the trajectory if one is running, and wait until it has stopped.""" + trajectory = self.pmac.trajectory + if await trajectory.execute_state.get_value() == PmacExecuteState.EXECUTING: + await trajectory.abort_profile.trigger() + await wait_for_value( + trajectory.execute_state, + lambda v: v != PmacExecuteState.EXECUTING, + timeout=DEFAULT_TIMEOUT, + ) - async def on_complete(self, ctx: PmacFlyCtx) -> None: - trajectory_status = error_if_none( - ctx.trajectory_status, "Cannot complete. Must call kickoff first." - ) - await trajectory_status - # Reset number of loaded points - self._loaded = 0 - - async def stop(self) -> None: - # Abort current trajectory, if one is running - if ( - await self.pmac.trajectory.execute_state.get_value() - == PmacExecuteState.EXECUTING - ): - await self.pmac.trajectory.abort_profile.trigger() - - async def on_stage(self) -> None: - # unstage()/on_unstage default to stop(), but stage additionally resets - await self.stop() - - # Run an empty fly scan to reset EQU on Panda Brick + async def ensure_ready(self) -> None: + """Stop, then run an empty fly scan to reset EQU on Panda Brick.""" + await self.ensure_stopped() for use_axis in self.pmac.trajectory.use_axis.values(): await use_axis.set(False) @@ -141,33 +161,6 @@ async def on_stage(self) -> None: await self.pmac.trajectory.build_profile.trigger() await self.pmac.trajectory.execute_profile.set(True) - @AsyncStatus.wrap - async def _execute_trajectory(self, path: Path, motor_info: _PmacMotorInfo): - execute_status = self.pmac.trajectory.execute_profile.set(True, timeout=None) - # We consume SLICE_SIZE from self.path and parse a trajectory - # containing at least 2 * SLICE_SIZE, as a gapless trajectory - # will contain 2 points per slice frame. If gaps are present, - # additional points are inserted, overfilling the buffer. - min_buffer_size = SLICE_SIZE * 2 - async for current_point in observe_value( - self.pmac.trajectory.total_points, - done_status=execute_status, - # Limit on PMAC is 4 seconds between points - # Thus this timeout is fine. - timeout=DEFAULT_TIMEOUT, - ): - # Ensure we maintain a minimum buffer size, if we have more points to append - if len(path) != 0 and self._loaded - current_point < min_buffer_size: - # We have less than SLICE_SIZE * 2 points in the buffer, so refill - next_slice = path.consume(SLICE_SIZE) - path_length = len(path) - await self._append_trajectory(next_slice, path_length, motor_info) - - await self._check_profile_status( - self.pmac.trajectory.execute_status, - self.pmac.trajectory.execute_message, - ) - async def _check_profile_status( self, status_signal: SignalR, @@ -193,10 +186,10 @@ async def _check_profile_status( ) from exc async def _append_trajectory( - self, slice: Slice, path_length: int, motor_info: _PmacMotorInfo + self, ctx: PmacPrepareCtx, slice: Slice, path_length: int ): - trajectory = await self._parse_trajectory(slice, path_length, motor_info, None) - await self._set_trajectory_arrays(trajectory, motor_info) + trajectory = await self._parse_trajectory(ctx, slice, path_length, None) + await self._set_trajectory_arrays(trajectory, ctx.motor_info) await self.pmac.trajectory.append_profile.trigger() await self._check_profile_status( @@ -204,15 +197,10 @@ async def _append_trajectory( self.pmac.trajectory.append_message, ) - async def _build_trajectory( - self, - motor_info: _PmacMotorInfo, - slice: Slice, - path_length: int, - ramp_up_time: float, - ): + async def _build_trajectory(self, ctx: PmacPrepareCtx): + motor_info = ctx.motor_info trajectory = await self._parse_trajectory( - slice, path_length, motor_info, ramp_up_time + ctx, ctx.first_slice, ctx.path_length, ctx.ramp_up_time ) use_axis = { i: (i in motor_info.motor_cs_index.values()) for i in CS_INDEX.values() @@ -236,17 +224,18 @@ async def _build_trajectory( async def _parse_trajectory( self, + ctx: PmacPrepareCtx, slice: Slice, path_length: int, - motor_info: _PmacMotorInfo, ramp_up_time: float | None = None, ) -> Trajectory: + motor_info = ctx.motor_info trajectory, exit_pvt = Trajectory.from_slice( slice, motor_info, - None if ramp_up_time else self._next_pvt, + None if ramp_up_time else ctx.cursor.next_pvt, ramp_up_time=ramp_up_time, - turnaround_time=self._turnaround_time, + turnaround_time=ctx.turnaround_time, ) if path_length == 0: @@ -256,8 +245,8 @@ async def _parse_trajectory( trajectory = trajectory.with_ramp_down( exit_pvt, ramp_down_pos, ramp_down_time, 0 ) - self._next_pvt = exit_pvt - self._loaded += len(trajectory) + ctx.cursor.next_pvt = exit_pvt + ctx.cursor.loaded += len(trajectory) return trajectory diff --git a/src/ophyd_async/fastcs/panda/_fly_logic.py b/src/ophyd_async/fastcs/panda/_fly_logic.py index a97dc18d41..0206855aae 100644 --- a/src/ophyd_async/fastcs/panda/_fly_logic.py +++ b/src/ophyd_async/fastcs/panda/_fly_logic.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +from collections.abc import AsyncGenerator from dataclasses import dataclass, field from typing import cast @@ -10,6 +11,7 @@ from scanspec.specs import Spec from ophyd_async.core import ( + KICKED_OFF, ConfinedModel, FlyableLogic, SignalRW, @@ -44,12 +46,15 @@ class ScanSpecInfo(ConfinedModel): @dataclass -class StaticSeqTableFlyableLogic(FlyableLogic[SeqTableInfo, None]): +class StaticSeqTableFlyableLogic(FlyableLogic[SeqTableInfo]): """For controlling the PandA `SeqTable` when fly scanning.""" seq: SeqBlock - async def on_prepare(self, value: SeqTableInfo): + async def prepare_to_fly( + self, name: str, value: SeqTableInfo, previous_ctx: None + ) -> AsyncGenerator[None, None]: + yield None await asyncio.gather( self.seq.prescale_units.set(PandaTimeUnits.US), self.seq.enable.set(PandaBitMux.ZERO), @@ -60,14 +65,13 @@ async def on_prepare(self, value: SeqTableInfo): self.seq.table.set(value.sequence_table), ) - async def on_kickoff(self, ctx: None) -> None: + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF, None]: await self.seq.enable.set(PandaBitMux.ONE) await wait_for_value(self.seq.active, True, timeout=1) - - async def on_complete(self, ctx: None) -> None: + yield KICKED_OFF await wait_for_value(self.seq.active, False, timeout=None) - async def stop(self): + async def ensure_stopped(self) -> None: await self.seq.enable.set(PandaBitMux.ZERO) await wait_for_value(self.seq.active, False, timeout=1) @@ -89,11 +93,14 @@ def from_inenc(cls, panda: CommonPandaBlocks, number: int) -> PosOutScaleOffset: @dataclass -class ScanSpecSeqTableFlyableLogic(FlyableLogic[ScanSpecInfo, None]): +class ScanSpecSeqTableFlyableLogic(FlyableLogic[ScanSpecInfo]): seq: SeqBlock motor_pos_outs: dict[Motor, PosOutScaleOffset] = field(default_factory=dict) - async def on_prepare(self, value: ScanSpecInfo): + async def prepare_to_fly( + self, name: str, value: ScanSpecInfo, previous_ctx: None + ) -> AsyncGenerator[None, None]: + yield None await self.seq.enable.set(PandaBitMux.ZERO) slice = Path(value.spec.calculate()).consume() slice_duration = error_if_none(slice.duration, "Slice must have duration") @@ -155,14 +162,13 @@ async def on_prepare(self, value: ScanSpecInfo): self.seq.table.set(rows), ) - async def on_kickoff(self, ctx: None) -> None: + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF, None]: await self.seq.enable.set(PandaBitMux.ONE) await wait_for_value(self.seq.active, True, timeout=1) - - async def on_complete(self, ctx: None) -> None: + yield KICKED_OFF await wait_for_value(self.seq.active, False, timeout=None) - async def stop(self): + async def ensure_stopped(self) -> None: await self.seq.enable.set(PandaBitMux.ZERO) await wait_for_value(self.seq.active, False, timeout=1) @@ -192,12 +198,15 @@ class PcompInfo(ConfinedModel): @dataclass -class StaticPcompFlyableLogic(FlyableLogic[PcompInfo, None]): +class StaticPcompFlyableLogic(FlyableLogic[PcompInfo]): """For controlling the PandA `PcompBlock` when fly scanning.""" pcomp: PcompBlock - async def on_prepare(self, value: PcompInfo): + async def prepare_to_fly( + self, name: str, value: PcompInfo, previous_ctx: None + ) -> AsyncGenerator[None, None]: + yield None await self.pcomp.enable.set(PandaBitMux.ZERO) await asyncio.gather( self.pcomp.start.set(value.start_position), @@ -207,13 +216,12 @@ async def on_prepare(self, value: PcompInfo): self.pcomp.dir.set(value.direction), ) - async def on_kickoff(self, ctx: None) -> None: + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF, None]: await self.pcomp.enable.set(PandaBitMux.ONE) await wait_for_value(self.pcomp.active, True, timeout=1) - - async def on_complete(self, ctx: None) -> None: + yield KICKED_OFF await wait_for_value(self.pcomp.active, False, timeout=None) - async def stop(self): + async def ensure_stopped(self) -> None: await self.pcomp.enable.set(PandaBitMux.ZERO) await wait_for_value(self.pcomp.active, False, timeout=1) diff --git a/src/ophyd_async/sim/_blob_acquire_logic.py b/src/ophyd_async/sim/_blob_acquire_logic.py index 2fa0faf74a..f880d1a3d7 100644 --- a/src/ophyd_async/sim/_blob_acquire_logic.py +++ b/src/ophyd_async/sim/_blob_acquire_logic.py @@ -1,5 +1,4 @@ import asyncio -from contextlib import suppress from ophyd_async.core import DetectorAcquireLogic, TriggerInfo @@ -24,7 +23,9 @@ async def wait_for_idle(self): async def ensure_stopped(self): # Stop the background task and wait for it to finish if self.task: - self.task.cancel() - with suppress(asyncio.CancelledError): - await self.task - self.task = None + task, self.task = self.task, None + task.cancel() + # An outside cancel while we wait propagates, it is not swallowed + await asyncio.wait([task]) + if not task.cancelled() and (error := task.exception()): + raise error diff --git a/src/ophyd_async/sim/_motor.py b/src/ophyd_async/sim/_motor.py index 41697fd2e1..98a5b7fec7 100644 --- a/src/ophyd_async/sim/_motor.py +++ b/src/ophyd_async/sim/_motor.py @@ -1,38 +1,65 @@ import asyncio -from collections.abc import Callable -from dataclasses import dataclass +from collections.abc import AsyncGenerator, Callable +from dataclasses import dataclass, field from functools import cached_property from ophyd_async.core import ( + KICKED_OFF, AsyncStatus, DeviceMock, + FlyableLogic, FlyMotorInfo, MovableLogic, SignalRW, + StandardFlyable, StandardMovable, StandardReadable, TimeoutCalculator, - WatchableAsyncStatus, + WatcherUpdate, default_mock_class, - error_if_none, simulate_move, soft_signal_r_and_setter, soft_signal_rw, + watch_readback, ) from ophyd_async.core import StandardReadableFormat as Format +@dataclass(frozen=True) +class SimMotorPrepareCtx: + """What a simulated motor fly scan needs after `prepare_to_fly`.""" + + #: Where to start from: the start position with the run-up distance added. + start: float + #: Where to end: the end position with the run-down distance added. + end: float + #: How long the motor takes to get up to speed. + acceleration_time: float + + @dataclass -class SimMotorMoveLogic(MovableLogic[float]): +class SimMotorLogic( + MovableLogic[float], FlyableLogic[FlyMotorInfo, SimMotorPrepareCtx] +): + """Combined move + fly logic for a simulated motor.""" + readback_set: Callable[[float], None] velocity: SignalRW[float] acceleration_time: SignalRW[float] + _motion: asyncio.Task | None = field(default=None, init=False, repr=False) - async def stop(self) -> None: - """Stop the motion.""" + async def ensure_stopped(self) -> None: + """Stop the simulated motion, a fly included, and wait for it to finish. + + Then set the setpoint to where it stopped. + """ + if (motion := self._motion) is not None: + motion.cancel() + await asyncio.wait([motion]) await self.setpoint.set(await self.readback.get_value()) async def move(self, new_position: float, timeout: TimeoutCalculator) -> None: + """Simulate the move, which ends early if `ensure_stopped` stops it.""" old_position, velocity, acceleration_time = await asyncio.gather( self.readback.get_value(), self.velocity.get_value(), @@ -40,16 +67,74 @@ async def move(self, new_position: float, timeout: TimeoutCalculator) -> None: ) await self.setpoint.set(new_position) - async for position in simulate_move( - old_position, new_position, velocity, acceleration_time - ): - self.readback_set(position) + + async def simulate() -> None: + moves = simulate_move( + old_position, new_position, velocity, acceleration_time + ) + async for position in moves: + self.readback_set(position) + + # In a task of its own, so that ensure_stopped can cancel it and wait + async with AsyncStatus(simulate()) as motion: + self._motion = motion.task + try: + await asyncio.wait([motion.task]) + finally: + if self._motion is motion.task: + self._motion = None + + async def prepare_to_fly( + self, name: str, value: FlyMotorInfo, previous_ctx: SimMotorPrepareCtx | None + ) -> AsyncGenerator[SimMotorPrepareCtx | WatcherUpdate, None]: + """Work out the run-up and run-down positions, then move to the run-up start. + + Progress of the move is reported. + """ + acceleration_time = await self.acceleration_time.get_value() + ctx = SimMotorPrepareCtx( + start=value.ramp_up_start_pos(acceleration_time), + end=value.ramp_down_end_pos(acceleration_time), + acceleration_time=acceleration_time, + ) + yield ctx + # Move to start as fast as we can + await self.velocity.set(0) + async with AsyncStatus(self.move(ctx.start, lambda: None)) as status: + async for update in watch_readback(self, status, target=ctx.start): + yield update + # Set the velocity for the actual move + await self.velocity.set(value.speed) + + async def fly( + self, ctx: SimMotorPrepareCtx + ) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + """Move from the run-up start to the run-down end, reporting progress. + + Kicked off once the run-up is done and the motor is at constant velocity. + """ + async with AsyncStatus(self.move(ctx.end, lambda: None)) as status: + await asyncio.sleep(ctx.acceleration_time) + yield KICKED_OFF + async for update in watch_readback( + self, status, target=ctx.end, initial=ctx.start + ): + yield update # Remove InstantMovableMock as SimMotor owns this logic, depends if instant=True/False @default_mock_class(DeviceMock) -class SimMotor(StandardReadable, StandardMovable[float]): - """For usage when simulating a motor.""" +class SimMotor( + StandardReadable, + StandardMovable[float], + StandardFlyable[FlyMotorInfo, SimMotorPrepareCtx], +): + """For usage when simulating a motor. + + `stage()` calls the logic's `ensure_ready`, which calls `ensure_stopped` and so + stops the simulated motion and sets the setpoint to the current readback. + `unstage()` cancels a fly in progress, then calls `ensure_stopped` too. + """ def __init__( self, @@ -75,47 +160,14 @@ def __init__( self.acceleration_time = soft_signal_rw(float, 0.5) self.user_setpoint = soft_signal_rw(float, initial_value, units=units) - # Stored in prepare - self._fly_info: FlyMotorInfo | None = None - # Set on kickoff(), complete when motor reaches end position - self._fly_status: WatchableAsyncStatus | None = None - super().__init__(name=name) @cached_property - def movable_logic(self): - return SimMotorMoveLogic( + def logic(self) -> SimMotorLogic: + return SimMotorLogic( readback=self.user_readback, readback_set=self._user_readback_set, setpoint=self.user_setpoint, velocity=self.velocity, acceleration_time=self.acceleration_time, ) - - @AsyncStatus.wrap - async def prepare(self, value: FlyMotorInfo): - """Calculate run-up and move there, setting fly velocity when there.""" - self._fly_info = value - # Move to start as fast as we can - await self.velocity.set(0) - await self.set( - value.ramp_up_start_pos(await self.acceleration_time.get_value()) - ) - # Set the velocity for the actual move - await self.velocity.set(value.speed) - - @AsyncStatus.wrap - async def kickoff(self): - """Begin moving motor from prepared position to final position.""" - fly_info = error_if_none( - self._fly_info, "Motor must be prepared before attempting to kickoff" - ) - acceleration_time = await self.acceleration_time.get_value() - self._fly_status = self.set(fly_info.ramp_down_end_pos(acceleration_time)) - # Wait for the acceleration time to ensure we are at velocity - await asyncio.sleep(acceleration_time) - - def complete(self) -> WatchableAsyncStatus: - """Mark as complete once motor reaches completed position.""" - fly_status = error_if_none(self._fly_status, "kickoff not called") - return fly_status diff --git a/src/ophyd_async/tango/demo/_motor.py b/src/ophyd_async/tango/demo/_motor.py index f1194763f0..c7df2676ed 100644 --- a/src/ophyd_async/tango/demo/_motor.py +++ b/src/ophyd_async/tango/demo/_motor.py @@ -23,8 +23,10 @@ class DemoMotorMoveLogic(MovableLogic[float]): stop_: TriggerableCommand state: SignalR[DevStateEnum] - async def stop(self): + async def ensure_stopped(self) -> None: + """Stop the motor and wait for its state to return to ON.""" await self.stop_.trigger() + await wait_for_value(self.state, DevStateEnum.ON, timeout=DEFAULT_TIMEOUT) async def calculate_timeout( self, old_position: float, new_position: float @@ -52,7 +54,7 @@ class DemoMotor(TangoDevice, StandardReadable, StandardMovable[float]): stop_: TriggerableCommand @cached_property - def movable_logic(self) -> MovableLogic: + def logic(self) -> MovableLogic: return DemoMotorMoveLogic( readback=self.readback, setpoint=self.setpoint, diff --git a/tests/unit_tests/core/test_cancel_swallow.py b/tests/unit_tests/core/test_cancel_swallow.py new file mode 100644 index 0000000000..36879efd8e --- /dev/null +++ b/tests/unit_tests/core/test_cancel_swallow.py @@ -0,0 +1,71 @@ +import asyncio +from collections.abc import Callable, Coroutine +from typing import Any +from unittest.mock import Mock + +import pytest + +from ophyd_async.core import AsyncStatus, soft_signal_rw + +# The waits these cleanups cancel are internal, with no public way to slow them down +from ophyd_async.core import _signal as signal_module # noqa: PLC2701 +from ophyd_async.sim._blob_acquire_logic import BlobAcquireLogic # noqa: PLC2701 + + +def _slow_to_cancel(cancelling: asyncio.Event): + """A coroutine that takes a moment to finish cancelling, like async cleanup.""" + + async def coro(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelling.set() + await asyncio.sleep(0.2) + raise + + return coro() + + +async def _status_aexit(cancelling: asyncio.Event, _: pytest.MonkeyPatch): + async with AsyncStatus(_slow_to_cancel(cancelling)): + await asyncio.sleep(0) # let the status task start + + +async def _blob_ensure_stopped(cancelling: asyncio.Event, _: pytest.MonkeyPatch): + pattern_generator = Mock() + pattern_generator.write_images_to_file = lambda: _slow_to_cancel(cancelling) + logic = BlobAcquireLogic(pattern_generator) + await logic.start_acquiring() + await asyncio.sleep(0) # let the task start + await logic.ensure_stopped() + + +async def _set_and_wait(cancelling: asyncio.Event, monkeypatch: pytest.MonkeyPatch): + async def wait_for_value(self, signal, timeout): + await _slow_to_cancel(cancelling) + + monkeypatch.setattr(signal_module._ValueChecker, "wait_for_value", wait_for_value) + sig = soft_signal_rw(int) + await sig.connect() + # No initial value ever arrives, so this times out and the error handler + # cancels the slow to cancel wait task + with pytest.raises(TimeoutError): + await signal_module.set_and_wait_for_value(sig, 1, timeout=0.1) + + +@pytest.mark.parametrize( + "cleanup", [_status_aexit, _blob_ensure_stopped, _set_and_wait] +) +async def test_cancelling_caller_during_cleanup_is_not_swallowed( + cleanup: Callable[[asyncio.Event, pytest.MonkeyPatch], Coroutine[Any, Any, None]], + monkeypatch: pytest.MonkeyPatch, +): + cancelling = asyncio.Event() + outer = asyncio.create_task(cleanup(cancelling, monkeypatch)) + # Cancel the caller while the cleanup is waiting for the inner task to finish + await cancelling.wait() + outer.cancel() + await asyncio.wait([outer]) + assert outer.cancelled() + # The slow cleanup carries on after the caller has gone, so let it finish + await asyncio.wait(asyncio.all_tasks() - {asyncio.current_task()}) diff --git a/tests/unit_tests/core/test_flyer.py b/tests/unit_tests/core/test_flyer.py index 70faeb4ce5..b57c9019b2 100644 --- a/tests/unit_tests/core/test_flyer.py +++ b/tests/unit_tests/core/test_flyer.py @@ -1,116 +1,192 @@ import asyncio +import sys +from collections.abc import AsyncGenerator, AsyncIterator from dataclasses import dataclass, field from functools import cached_property +from typing import Any, cast from unittest.mock import Mock import pytest from ophyd_async.core import ( + KICKED_OFF, + AsyncStatus, FlyableLogic, MovableLogic, StandardFlyable, StandardMovable, StandardReadable, StandardReadableFormat, + TransitionError, WatchableAsyncStatus, + WatcherUpdate, init_devices, set_mock_value, soft_signal_rw, + watch_readback, ) +from ophyd_async.testing import wait_for_pending_wakeups @dataclass -class RecordingFlyableLogic(FlyableLogic[int, None]): - """A `FlyableLogic` that records its calls. Not movable, so complete() reports - no progress to watchers.""" +class RecordingFlyableLogic(FlyableLogic[int]): + """A `FlyableLogic` that records its calls. Not movable, so prepare() and + complete() report no progress to watchers.""" calls: list[str] = field(default_factory=list) prepared_with: list[int] = field(default_factory=list) - async def on_prepare(self, value: int) -> None: - self.calls.append("on_prepare") + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[None, None]: + self.calls.append("prepare_to_fly") self.prepared_with.append(value) + yield None - async def on_kickoff(self, ctx: None) -> None: - self.calls.append("on_kickoff") + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + self.calls.append("kicking off") + yield KICKED_OFF + self.calls.append("completing") - async def on_complete(self, ctx: None) -> None: - self.calls.append("on_complete") + async def ensure_stopped(self) -> None: + self.calls.append("ensure_stopped") - async def stop(self) -> None: - self.calls.append("stop") + +@dataclass(frozen=True) +class Ctx: + value: int + # Anything that changes during the scan is an object the ctx holds + log: list[str] = field(default_factory=list) @dataclass -class CtxFlyableLogic(FlyableLogic[int, dict]): - """A `FlyableLogic` that threads a context dict between its stages.""" +class CtxFlyableLogic(FlyableLogic[int, Ctx]): + """A `FlyableLogic` that passes a ctx from prepare_to_fly to fly.""" - seen: list[tuple[str, dict]] = field(default_factory=list) + seen: list[Ctx] = field(default_factory=list) + made: list[Ctx] = field(default_factory=list) - async def on_prepare(self, value: int) -> dict: - return {"prepared": value} + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[Ctx, None]: + self.made.append(Ctx(value)) + yield self.made[0] - async def on_kickoff(self, ctx: dict) -> dict: - self.seen.append(("kickoff", dict(ctx))) - ctx["kicked"] = True - return ctx + async def fly(self, ctx: Ctx) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + self.seen.append(ctx) + yield KICKED_OFF - async def on_complete(self, ctx: dict) -> None: - self.seen.append(("complete", dict(ctx))) + async def ensure_stopped(self) -> None: ... @dataclass -class MovableFlyableLogic(MovableLogic[float], FlyableLogic[float, None]): +class MovableFlyableLogic(MovableLogic[float], FlyableLogic[float, float]): """Both movable and flyable, mirroring how `Motor` combines the two. - `prepare()` and `complete()` should report progress by observing the readback, - reusing the same watcher-update stream as `StandardMovable.set`. + `prepare()` and `complete()` should report progress by watching the readback + with `watch_readback`, reusing the same watcher updates as + `StandardMovable.set`. """ - async def on_prepare(self, value: float) -> None: - # Move to the fly start, writing the setpoint part-way in like Motor does - await asyncio.sleep(0) - set_mock_value(self.setpoint, value) - for position in (value / 2, value): - set_mock_value(self.readback, position) - await asyncio.sleep(0) - - async def on_kickoff(self, ctx: None) -> None: - pass - - async def on_complete(self, ctx: None) -> None: + async def _step_to(self, target: float) -> None: # Step the readback towards the target so watchers see progress. - target = await self.setpoint.get_value() for position in (target / 2, target): set_mock_value(self.readback, position) await asyncio.sleep(0) + async def prepare_to_fly( + self, name: str, value: float, previous_ctx: None + ) -> AsyncGenerator[float | WatcherUpdate, None]: + yield value + async with AsyncStatus(self._step_to(value)) as status: + async for update in watch_readback(self, status, target=value): + yield update + + async def fly(self, ctx: float) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + async with AsyncStatus(self._step_to(2 * ctx)) as status: + yield KICKED_OFF + async for update in watch_readback(self, status, 2 * ctx, initial=ctx): + yield update -class MovableFlyer(StandardMovable[float], StandardFlyable[float, None]): + async def ensure_stopped(self) -> None: ... + + +class MovableFlyer(StandardMovable[float], StandardFlyable[float, float]): def __init__(self, name: str = ""): self.readback = soft_signal_rw(float) self.setpoint = soft_signal_rw(float) super().__init__(name=name) @cached_property - def _logic(self) -> MovableFlyableLogic: + def logic(self) -> MovableFlyableLogic: return MovableFlyableLogic(setpoint=self.setpoint, readback=self.readback) - @cached_property - def movable_logic(self) -> MovableFlyableLogic: - return self._logic - @cached_property - def flyable_logic(self) -> MovableFlyableLogic: - return self._logic +@dataclass +class BlockingFlyableLogic(RecordingFlyableLogic): + """Blocks forever in the named verb's part of a hook, so it is left in flight.""" + + block: str = "" + + async def _maybe_block(self, verb: str) -> None: + self.calls.append(verb) + if self.block == verb: + await asyncio.Event().wait() + + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[None, None]: + await self._maybe_block("prepare") + yield None + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + await self._maybe_block("kickoff") + yield KICKED_OFF + await self._maybe_block("complete") + + +@dataclass +class FlyingLogic(FlyableLogic[int]): + """Kicks off a status that runs until cancelled; `fail` raises after kickoff + instead of waiting for it.""" + + fail: bool = False + fly_statuses: list[AsyncStatus] = field(default_factory=list) + running_when_stopped: list[bool] = field(default_factory=list) + + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[None, None]: + yield None + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + async with AsyncStatus(asyncio.Event().wait()) as status: + self.fly_statuses.append(status) + yield KICKED_OFF + await asyncio.sleep(0) + if self.fail: + raise ValueError("bad fly") + await status + + async def ensure_stopped(self) -> None: + self.running_when_stopped.append( + any(not status.done for status in self.fly_statuses) + ) + + +async def make_staged_flyer(logic: FlyableLogic[Any, Any]): + async with init_devices(mock=True): + flyer = logic.with_device(name="flyer") + await flyer.stage() + return flyer @pytest.fixture async def recording_flyer(): - """An ephemeral `StandardFlyable` wrapping a fresh `RecordingFlyableLogic`.""" + """A staged ephemeral `StandardFlyable` wrapping a `RecordingFlyableLogic`.""" logic = RecordingFlyableLogic() - async with init_devices(mock=True): - flyer = logic.with_device(name="flyer") + flyer = await make_staged_flyer(logic) + logic.calls.clear() # forget the ensure_stopped() from stage return logic, flyer @@ -120,148 +196,309 @@ async def test_ephemeral_flyable_drives_logic_hooks(recording_flyer): assert flyer.name == "flyer" assert isinstance(flyer, StandardFlyable) # The ephemeral device exposes the same logic instance - assert flyer.flyable_logic is logic + assert flyer.logic is logic - await flyer.prepare(5) - await flyer.kickoff() - await flyer.complete() + # Every point of a scan reuses the staged flyer: FLOWN can prepare again + for value in (5, 6): + await flyer.prepare(value) + await flyer.kickoff() + await flyer.complete() - assert logic.calls == ["on_prepare", "on_kickoff", "on_complete"] - assert logic.prepared_with == [5] + assert logic.calls == ["prepare_to_fly", "kicking off", "completing"] * 2 + assert logic.prepared_with == [5, 6] + await flyer.unstage() + assert logic.calls[-1] == "ensure_stopped" -async def test_flyable_threads_context_between_stages(): +async def test_flyable_passes_ctx_to_fly(): logic = CtxFlyableLogic() - async with init_devices(mock=True): - flyer = logic.with_device(name="flyer") + flyer = await make_staged_flyer(logic) await flyer.prepare(5) await flyer.kickoff() await flyer.complete() - # kickoff sees exactly what prepare produced; complete sees what kickoff added - assert logic.seen == [ - ("kickoff", {"prepared": 5}), - ("complete", {"prepared": 5, "kicked": True}), - ] - - -async def test_flyable_kickoff_before_prepare_raises(recording_flyer): - logic, flyer = recording_flyer + # fly sees the one ctx prepare_to_fly yielded + assert logic.seen == [logic.made[0]] + assert logic.seen[0] is logic.made[0] + + +@pytest.mark.parametrize( + "setup, verb, current", + [ + # Not staged + ([], "prepare", "UNSTAGED"), + # Out of order + (["stage"], "kickoff", "STAGED"), + (["stage"], "complete", "STAGED"), + (["stage", "prepare"], "complete", "PREPARED"), + # Prepared: kickoff before preparing again + (["stage", "prepare"], "prepare", "PREPARED"), + (["stage", "prepare", "kickoff"], "kickoff", "FLYING"), + # A running fly cannot be prepared over + (["stage", "prepare", "kickoff"], "prepare", "FLYING"), + # Flown: only a new prepare (or complete, stage, unstage) is allowed + (["stage", "prepare", "kickoff", "complete"], "kickoff", "FLOWN"), + # A new prepare forgets the last fly + ( + ["stage", "prepare", "kickoff", "complete", "prepare"], + "complete", + "PREPARED", + ), + ], +) +async def test_flyable_verb_in_the_wrong_state_raises(setup, verb, current): + # A fly that is kicked off keeps flying until completed + logic = BlockingFlyableLogic(block="complete" if setup[-1:] == ["kickoff"] else "") + async with init_devices(mock=True): + flyer = logic.with_device(name="flyer") + for step in setup: + await (getattr(flyer, step)(1) if step == "prepare" else getattr(flyer, step)()) + logic.calls.clear() - with pytest.raises(RuntimeError, match="prepare.* before kickoff"): - await flyer.kickoff() + with pytest.raises(TransitionError, match=rf"flyer.*from {current}.*allowed"): + await (getattr(flyer, verb)(1) if verb == "prepare" else getattr(flyer, verb)()) assert logic.calls == [] + await flyer.unstage() -async def test_flyable_complete_before_kickoff_raises(recording_flyer): - logic, flyer = recording_flyer +@pytest.mark.parametrize("verb", ["prepare", "kickoff", "complete"]) +async def test_stage_during_a_verb_raises_and_unstage_cancels_it(verb): + logic = BlockingFlyableLogic(block=verb) + flyer = await make_staged_flyer(logic) + verbs = { + "prepare": lambda: flyer.prepare(1), + "kickoff": flyer.kickoff, + "complete": flyer.complete, + } + running = None + for name, start in verbs.items(): + running = start() + if name == verb: + break + await running + await wait_for_pending_wakeups() + assert logic.calls[-1] == verb + + # Only unstage can end a verb that is in flight + with pytest.raises(TransitionError, match="cannot go to STAGED"): + await flyer.stage() + await flyer.unstage() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(cast(AsyncStatus, running), timeout=1) - with pytest.raises(RuntimeError, match="kickoff.* before complete"): - await flyer.complete() - assert logic.calls == [] + # The cancelled verb did not touch the state unstage left behind + logic.block = "" + await flyer.stage() + await flyer.prepare(2) -async def test_flyable_stage_resets_lifecycle(recording_flyer): - # After prepare+kickoff, staging should reset so a later complete errors again. +async def test_flyable_unstage_resets_lifecycle(recording_flyer): + # After prepare+kickoff, unstaging should reset so a later complete errors. logic, flyer = recording_flyer await flyer.prepare(5) await flyer.kickoff() + await flyer.unstage() await flyer.stage() - with pytest.raises(RuntimeError, match="kickoff.* before complete"): + with pytest.raises(TransitionError, match="from STAGED"): await flyer.complete() -async def test_flyable_complete_without_movable_logic_yields_no_updates( - recording_flyer, -): - # A flyer whose logic is not a MovableLogic has no readback to report, so - # complete() is still a WatchableAsyncStatus but calls no watchers. +async def test_flyable_stage_twice_raises(recording_flyer): logic, flyer = recording_flyer - await flyer.prepare(5) + with pytest.raises(TransitionError, match="cannot go to STAGED from STAGED"): + await flyer.stage() + # Nothing ran, and it is still staged + assert logic.calls == [] + await flyer.prepare(1) + + +async def test_flyable_without_progress_yields_no_updates(recording_flyer): + # A logic whose hooks yield no updates reports no progress, but prepare() + # and complete() are still watchable + logic, flyer = recording_flyer + + prepare_watcher, complete_watcher = Mock(), Mock() + status = flyer.prepare(5) + assert isinstance(status, WatchableAsyncStatus) + status.watch(prepare_watcher) + await status await flyer.kickoff() status = flyer.complete() assert isinstance(status, WatchableAsyncStatus) - updates: list[dict] = [] - status.watch(lambda **kwargs: updates.append(kwargs)) + status.watch(complete_watcher) await status - assert logic.calls == ["on_prepare", "on_kickoff", "on_complete"] - assert updates == [] + prepare_watcher.assert_not_called() + complete_watcher.assert_not_called() + assert logic.calls == ["prepare_to_fly", "kicking off", "completing"] -async def test_flyable_complete_watches_movable_logic(): +async def test_flyable_reports_progress_from_logic(): async with init_devices(mock=True): flyer = MovableFlyer(name="flyer") + await flyer.stage() - # Prepare at 0, then set the fly target on the setpoint without triggering the - # instant-move mock (so the readback stays at 0 until on_complete steps it). - await flyer.prepare(0.0) - set_mock_value(flyer.setpoint, 10.0) + prepare_watcher, complete_watcher = Mock(), Mock() + status = flyer.prepare(10.0) + assert isinstance(status, WatchableAsyncStatus) + status.watch(prepare_watcher) + await status await flyer.kickoff() status = flyer.complete() assert isinstance(status, WatchableAsyncStatus) - updates: list[dict] = [] - status.watch(lambda **kwargs: updates.append(kwargs)) + status.watch(complete_watcher) await status - # Progress was reported by observing the readback up to the target, with the - # target read from the setpoint (as StandardMovable.set reports a move). - assert updates, "expected at least one watcher update" - assert updates[-1]["current"] == 10.0 + # Progress is stamped with the flyer's name, and prepare() moves to its + # start before kickoff is allowed + updates = [c.kwargs for c in prepare_watcher.call_args_list] + assert [u["current"] for u in updates] == [0.0, 5.0, 10.0] assert {u["initial"] for u in updates} == {0.0} assert {u["target"] for u in updates} == {10.0} assert {u["name"] for u in updates} == {"flyer"} + updates = [c.kwargs for c in complete_watcher.call_args_list] + assert updates[-1]["current"] == 20.0 + assert {u["initial"] for u in updates} == {10.0} + assert {u["target"] for u in updates} == {20.0} + assert {u["name"] for u in updates} == {"flyer"} -async def test_flyable_prepare_watches_movable_logic(): +async def test_watch_readback_reraises_the_status_error(): async with init_devices(mock=True): flyer = MovableFlyer(name="flyer") - status = flyer.prepare(10.0) - assert isinstance(status, WatchableAsyncStatus) - watcher = Mock() - status.watch(watcher) - await status + async def fail(): + await asyncio.sleep(0) + raise ValueError("boom") - updates = [c.kwargs for c in watcher.call_args_list] - assert [u["current"] for u in updates] == [0.0, 5.0, 10.0] - assert {u["initial"] for u in updates} == {0.0} - # No updates before on_prepare writes the setpoint, so the target is never stale - assert {u["target"] for u in updates} == {10.0} - assert {u["name"] for u in updates} == {"flyer"} - # prepare() still completes the lifecycle stage, so kickoff is allowed - await flyer.kickoff() + updates = [] + with pytest.raises(ValueError, match="boom"): + async with AsyncStatus(fail()) as status: + async for update in watch_readback(flyer.logic, status, target=1.0): + updates.append(update) + # Initial defaults to the readback now + assert {u.initial for u in updates} == {0.0} -async def test_flyable_stage_unstage_default_to_stop(recording_flyer): +@pytest.mark.parametrize("prepare", [False, True]) +async def test_unstage_always_stops(recording_flyer, prepare): logic, flyer = recording_flyer + if prepare: + await flyer.prepare(1) + + await flyer.unstage() + assert "ensure_stopped" in logic.calls + # ensure_ready defaults to ensure_stopped + logic.calls.clear() await flyer.stage() + assert logic.calls == ["ensure_stopped"] await flyer.unstage() - # on_stage/on_unstage both default to stop() - assert logic.calls == ["stop", "stop"] + assert logic.calls == ["ensure_stopped", "ensure_stopped"] + + +@dataclass +class FailingFlyableLogic(RecordingFlyableLogic): + """Raises from the hook named by `failing`.""" + + failing: str = "" + + def _maybe_fail(self, hook: str) -> None: + if self.failing == hook: + raise ValueError(f"bad {hook}") + + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[None, None]: + self._maybe_fail("prepare_to_fly") + yield None + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + self._maybe_fail("kickoff") + yield KICKED_OFF + self._maybe_fail("complete") + + async def ensure_ready(self) -> None: + self._maybe_fail("ensure_ready") + + async def ensure_stopped(self) -> None: + self._maybe_fail("ensure_stopped") + + +@pytest.mark.parametrize( + "failing, verbs", + [ + ("ensure_ready", ["stage"]), + ("prepare_to_fly", ["stage", "prepare"]), + ("kickoff", ["stage", "prepare", "kickoff"]), + ("complete", ["stage", "prepare", "kickoff", "complete"]), + ("ensure_stopped", ["stage", "prepare", "unstage"]), + ], +) +async def test_an_error_from_a_hook_names_the_device_once(failing, verbs): + logic = FailingFlyableLogic(failing=failing) + async with init_devices(mock=True): + flyer = logic.with_device(name="flyer") + + def call(verb: str): + return flyer.prepare(1) if verb == "prepare" else getattr(flyer, verb)() + + *before, last = verbs + for verb in before: + await call(verb) + with pytest.raises(ValueError, match=f"bad {failing}") as error: + await call(last) + assert error.value.__notes__ == ["raised by flyer"] async def test_flyable_stage_unstage_can_differ(): # A flyer (e.g. PmacTrajectoryFlyableLogic) may need distinct stage/unstage. @dataclass class StageUnstageLogic(RecordingFlyableLogic): - async def on_stage(self) -> None: - self.calls.append("on_stage") - - async def on_unstage(self) -> None: - self.calls.append("on_unstage") + async def ensure_ready(self) -> None: + self.calls.append("ensure_ready") logic = StageUnstageLogic() async with init_devices(mock=True): flyer = logic.with_device(name="flyer") await flyer.stage() + assert logic.calls == ["ensure_ready"] + await flyer.prepare(1) + # prepare() does not get it ready again + assert logic.calls == ["ensure_ready", "prepare_to_fly"] await flyer.unstage() - assert logic.calls == ["on_stage", "on_unstage"] + assert logic.calls == ["ensure_ready", "prepare_to_fly", "ensure_stopped"] + + +@pytest.mark.parametrize( + "stale", + ["on_prepare", "on_kickoff", "on_complete", "on_stage", "on_unstage", "stop"], +) +def test_flyable_logic_with_a_removed_hook_raises(stale: str): + with pytest.raises(TypeError, match=rf"StaleLogic defines {stale}, which"): + type("StaleLogic", (RecordingFlyableLogic,), {stale: lambda self: None}) + + +@pytest.mark.parametrize( + "logic_class, kwargs, missing", + [ + (MovableLogic, {"setpoint": None, "readback": None}, "ensure_stopped"), + (FlyableLogic, {}, "ensure_stopped, fly, prepare_to_fly"), + ], +) +def test_logic_missing_an_abstract_hook_raises_at_construction( + logic_class: type, kwargs: dict, missing: str +): + incomplete = type("Incomplete", (logic_class,), {}) + with pytest.raises(TypeError, match=r"Incomplete.*abstract") as exc_info: + incomplete(**kwargs) + # The order of the missing methods in the message varies between Pythons + for name in missing.split(", "): + assert name in str(exc_info.value) async def test_flyable_composes_with_readable_staging(): @@ -276,7 +513,7 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def flyable_logic(self) -> FlyableLogic[int, None]: + def logic(self) -> FlyableLogic[int, None]: return logic async with init_devices(mock=True): @@ -287,10 +524,518 @@ def flyable_logic(self) -> FlyableLogic[int, None]: # "not being monitored" if StandardFlyable.stage had shadowed it). await device.sig.read(cached=True) # StandardFlyable contribution ran too. - assert logic.calls == ["stop"] + assert logic.calls == ["ensure_stopped"] await device.unstage() - assert logic.calls == ["stop", "stop"] + # Unstage stops even though it was not prepared + assert logic.calls == ["ensure_stopped", "ensure_stopped"] # The cache is torn down on unstage. with pytest.raises(RuntimeError, match="not being monitored"): await device.sig.read(cached=True) + + +# Bug: fly statuses were never cancelled +async def test_unstage_cancels_the_running_fly_before_stopping_the_hardware(): + logic = FlyingLogic() + flyer = await make_staged_flyer(logic) + await flyer.prepare(1) + await flyer.kickoff() + await asyncio.sleep(0) + assert not logic.fly_statuses[0].done + + await flyer.unstage() + + assert logic.fly_statuses[0].task.cancelled() + # Neither stage's nor unstage's ensure_stopped saw the fly running + assert logic.running_when_stopped == [False, False] + + +# Bug: an unstage() cancelled before it awaited the fly left it running unowned +async def test_cancelled_unstage_still_cancels_the_fly(): + logic = FlyingLogic() + flyer = await make_staged_flyer(logic) + await flyer.prepare(1) + await flyer.kickoff() + + unstaging = flyer.unstage() + await asyncio.sleep(0) + unstaging.task.cancel() + with pytest.raises(asyncio.CancelledError): + await unstaging + await wait_for_pending_wakeups() + + assert logic.fly_statuses[0].task.cancelled() + + +# Bug: an unstage() that failed or was cancelled while stopping left its ctx +# behind as the next prepare's previous_ctx +@pytest.mark.parametrize("stop", ["fail", "hang"]) +async def test_restage_after_an_interrupted_unstage_starts_afresh(stop: str): + @dataclass + class StopLogic(FlyableLogic[int, int]): + stopping: str = "" + previous_ctxs: list[int | None] = field(default_factory=list) + + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: int | None + ) -> AsyncGenerator[int, None]: + self.previous_ctxs.append(previous_ctx) + yield value + + async def fly(self, ctx: int) -> AsyncGenerator[KICKED_OFF, None]: + yield KICKED_OFF + + async def ensure_stopped(self) -> None: + if self.stopping == "fail": + raise ValueError("stop failed") + elif self.stopping == "hang": + await asyncio.Event().wait() + + logic = StopLogic() + flyer = await make_staged_flyer(logic) + await flyer.prepare(1) + logic.stopping = stop + unstaging = flyer.unstage() + await asyncio.sleep(0) + # RE.stop/abort/halt cancel a hanging unstage + unstaging.task.cancel() + with pytest.raises((ValueError, asyncio.CancelledError)): + await unstaging + + logic.stopping = "" + await flyer.stage() + await flyer.prepare(2) + await flyer.kickoff() + await flyer.complete() + await flyer.unstage() + assert logic.previous_ctxs == [None, None] + + +# Bug: a failed re-prepare left the flyer prepared with the failed prepare's ctx +async def test_failed_reprepare_leaves_nothing_to_kick_off(recording_flyer): + logic, flyer = recording_flyer + await flyer.prepare(1) + await flyer.kickoff() + await flyer.complete() + logic.calls.clear() + + async def fail(name: str, value: int, previous_ctx: None) -> AsyncIterator[None]: + raise ValueError("bad prepare") + yield None + + logic.prepare_to_fly = fail # type: ignore + with pytest.raises(ValueError, match="bad prepare"): + await flyer.prepare(2) + + with pytest.raises(TransitionError, match="cannot go to FLYING"): + await flyer.kickoff() + assert logic.calls == [] + + +async def _raise_before_kicked_off(ctx: None) -> AsyncIterator[Any]: + raise ValueError("bad kickoff") + yield KICKED_OFF + + +async def _yield_an_update_first(ctx: None) -> AsyncIterator[Any]: + yield WatcherUpdate(current=0, initial=0, target=1) + + +async def _finish_without_kicking_off(ctx: None) -> AsyncIterator[Any]: + return + yield KICKED_OFF + + +async def _block_until_cancelled(ctx: None) -> AsyncIterator[Any]: + await asyncio.Event().wait() + yield KICKED_OFF + + +# A kickoff that does not reach KICKED_OFF leaves the flyer staged, so it has to +# be prepared again, and leaves nothing running +@pytest.mark.parametrize( + "fly, error, match", + [ + (_raise_before_kicked_off, ValueError, "bad kickoff"), + (_yield_an_update_first, TypeError, "must yield KICKED_OFF first"), + (_finish_without_kicking_off, RuntimeError, "finished without yielding"), + (_block_until_cancelled, asyncio.CancelledError, None), + ], +) +async def test_kickoff_that_does_not_kick_off_goes_back_to_staged( + recording_flyer, fly, error, match +): + logic, flyer = recording_flyer + await flyer.prepare(1) + + logic.fly = fly + kickoff = flyer.kickoff() + if error is asyncio.CancelledError: + # Cancelled directly, not by unstage + await asyncio.sleep(0) + kickoff.task.cancel() + with pytest.raises(error, match=match): + await kickoff + + with pytest.raises(TransitionError, match="from STAGED"): + await flyer.kickoff() + await flyer.prepare(2) + + +async def test_kickoff_cancelled_just_after_kicked_off_ends_the_fly(recording_flyer): + logic, flyer = recording_flyer + await flyer.prepare(1) + kickoff = flyer.kickoff() + ended = asyncio.Event() + + async def fly(ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + try: + yield KICKED_OFF + # An outside cancel (e.g. a timeout) that lands before kickoff resumes + kickoff.task.cancel() + await asyncio.Event().wait() + finally: + ended.set() + + logic.fly = fly + with pytest.raises(asyncio.CancelledError): + await kickoff + assert ended.is_set() + with pytest.raises(TransitionError, match="from STAGED"): + await flyer.kickoff() + await flyer.prepare(2) + + +@pytest.mark.parametrize( + "bad_yield, described_as", [(42, "int: 42"), (KICKED_OFF, r"\w+: ?")] +) +async def test_a_non_update_yielded_after_kicked_off_fails_complete( + recording_flyer, bad_yield, described_as +): + logic, flyer = recording_flyer + + async def fly(ctx: None) -> AsyncIterator[Any]: + yield KICKED_OFF + yield bad_yield + + logic.fly = fly + await flyer.prepare(1) + await flyer.kickoff() + with pytest.raises( + TypeError, + match=rf"RecordingFlyableLogic\.fly\(\) must yield WatcherUpdate after " + rf"KICKED_OFF, not {described_as}", + ): + await flyer.complete() + + +# A complete() that does not succeed ends the fly and leaves the flyer staged, +# whether it is cancelled directly (not via unstage) or fly raises after kickoff +@pytest.mark.parametrize("fail", [False, True]) +async def test_unsuccessful_complete_cancels_the_fly_and_goes_to_staged(fail): + logic = FlyingLogic(fail=fail) + flyer = await make_staged_flyer(logic) + await flyer.prepare(1) + await flyer.kickoff() + completing = flyer.complete() + + if fail: + with pytest.raises(ValueError, match="bad fly"): + await completing + else: + await asyncio.sleep(0) + assert not logic.fly_statuses[0].done + completing.task.cancel() + with pytest.raises(asyncio.CancelledError): + await completing + + assert logic.fly_statuses[0].task.cancelled() + # complete() still returns the failed fly, but it cannot be kicked off again + assert flyer.complete() is completing + with pytest.raises(TransitionError, match="cannot go to FLYING from STAGED"): + await flyer.kickoff() + # Prepared again, it can fly again + logic.fail = False + await flyer.prepare(2) + await flyer.kickoff() + await flyer.unstage() + + +def test_logic_that_is_also_movable_may_define_stop_with_a_deprecation(): + with pytest.deprecated_call(match=r"CombinedLogic.stop\(\) is deprecated"): + + class CombinedLogic(MovableLogic[float], FlyableLogic[None]): + async def stop(self) -> None: ... + + assert CombinedLogic.ensure_stopped is CombinedLogic.stop + + +@dataclass +class FailingCleanupLogic(RecordingFlyableLogic): + """Its fly raises while it is being cancelled, `block_kickoff` or not.""" + + block_kickoff: bool = False + #: Fail on its own just after kicking off, rather than when cancelled + raise_after_kickoff: bool = False + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + try: + if self.block_kickoff: + await asyncio.Event().wait() + yield KICKED_OFF + if not self.raise_after_kickoff: + await asyncio.Event().wait() + finally: + raise ValueError("cleanup failed") # noqa: B012 + + +# A cancelled kickoff() reports its cancel; complete() is the fly, so reports +# the failure of the fly's cleanup +@pytest.mark.parametrize( + "phase, error", + [ + ("kickoff", asyncio.CancelledError), + ("complete", ValueError), + ], +) +async def test_failing_fly_cleanup_when_kickoff_or_complete_is_cancelled( + phase: str, error: type[BaseException] +): + logic = FailingCleanupLogic(block_kickoff=phase == "kickoff") + flyer = await make_staged_flyer(logic) + await flyer.prepare(1) + status = flyer.kickoff() + if phase == "complete": + await status + status = flyer.complete() + await asyncio.sleep(0) + status.task.cancel() + + with pytest.raises(error): + await status + # Staged again, ready to be prepared + await flyer.prepare(2) + + +async def test_unstage_stops_the_hardware_then_raises_the_fly_failure(): + # The fly fails while being cancelled + logic = FailingCleanupLogic() + flyer = await make_staged_flyer(logic) + await flyer.prepare(1) + await flyer.kickoff() + await asyncio.sleep(0) + logic.calls.clear() + + with pytest.raises(ValueError, match="cleanup failed"): + await flyer.unstage() + + assert logic.calls == ["ensure_stopped"] + # Unstaged all the same + await flyer.stage() + + +@dataclass +class SlowToCancelLogic(RecordingFlyableLogic): + """Its verbs take a moment to finish being cancelled.""" + + block: str = "" + running: set[str] = field(default_factory=set) + running_when_stopped: list[set[str]] = field(default_factory=list) + + async def _block(self, verb: str) -> None: + if self.block != verb: + return + self.running.add(verb) + try: + await asyncio.Event().wait() + finally: + await asyncio.sleep(0.05) + self.running.discard(verb) + + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[None, None]: + await self._block("prepare") + yield None + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + await self._block("kickoff") + yield KICKED_OFF + + async def ensure_stopped(self) -> None: + self.running_when_stopped.append(set(self.running)) + + +@pytest.mark.parametrize("verb", ["prepare", "kickoff"]) +async def test_unstage_waits_for_the_cancelled_verb_before_stopping(verb: str): + logic = SlowToCancelLogic(block=verb) + flyer = await make_staged_flyer(logic) + logic.running_when_stopped.clear() + if verb == "prepare": + running = flyer.prepare(1) + else: + await flyer.prepare(1) + running = flyer.kickoff() + await wait_for_pending_wakeups() + assert logic.running == {verb} + + await flyer.unstage() + + assert logic.running == set() + assert logic.running_when_stopped == [set()] + with pytest.raises(asyncio.CancelledError): + await running + + +@pytest.mark.parametrize( + "eager", + [ + False, + pytest.param( + True, + marks=pytest.mark.skipif( + sys.version_info < (3, 12), reason="eager_task_factory is 3.12+" + ), + ), + ], +) +async def test_complete_starts_with_the_latest_update_fly_yielded_before_it( + eager: bool, +): + if eager: + loop = asyncio.get_running_loop() + loop.set_task_factory(asyncio.eager_task_factory) # pyright: ignore + release = asyncio.Event() + + class UpdatingLogic(RecordingFlyableLogic): + async def fly( + self, ctx: None + ) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + yield KICKED_OFF + yield WatcherUpdate(current=1, initial=0, target=3) + yield WatcherUpdate(current=2, initial=0, target=3) + await release.wait() + yield WatcherUpdate(current=3, initial=0, target=3) + + flyer = await make_staged_flyer(UpdatingLogic()) + await flyer.prepare(1) + await flyer.kickoff() + await asyncio.sleep(0.01) + + watcher = Mock() + status = flyer.complete() + status.watch(watcher) + await asyncio.sleep(0.01) + release.set() + await status + + assert [c.kwargs["current"] for c in watcher.call_args_list] == [2, 3] + # complete() is the fly's own status, however often it is called + assert flyer.complete() is status + + +@dataclass +class RunUpLogic(RecordingFlyableLogic): + """Its prepare_to_fly runs up until cancelled, or fails part way if `fail`.""" + + fail: bool = False + run_ups: list[AsyncStatus] = field(default_factory=list) + closed: bool = False + + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncGenerator[Any, None]: + yield value + try: + async with AsyncStatus(asyncio.Event().wait()) as run_up: + self.run_ups.append(run_up) + yield WatcherUpdate(current=0, initial=0, target=1) + if self.fail: + raise ValueError("bad run-up") + await run_up + finally: + self.closed = True + + +@pytest.mark.parametrize("fail", [False, True]) +async def test_prepare_that_fails_or_is_cancelled_stops_the_run_up(fail: bool): + logic = RunUpLogic(fail=fail) + flyer = await make_staged_flyer(logic) + preparing = flyer.prepare(1) + await wait_for_pending_wakeups() + + if fail: + with pytest.raises(ValueError, match="bad run-up"): + await preparing + else: + assert not logic.closed + preparing.task.cancel() + with pytest.raises(asyncio.CancelledError): + await preparing + + # The generator was closed and the run-up stopped before prepare() finished + assert logic.closed + assert logic.run_ups[0].task.cancelled() + # The half-failed prepare left no ctx to fly with + with pytest.raises(TransitionError, match="cannot go to FLYING from STAGED"): + await flyer.kickoff() + + +async def _update_first( + name: str, value: int, previous_ctx: None +) -> AsyncIterator[Any]: + yield WatcherUpdate(current=0, initial=0, target=1) + + +async def _no_ctx(name: str, value: int, previous_ctx: None) -> AsyncIterator[Any]: + return + yield + + +async def _ctx_then_not_an_update( + name: str, value: int, previous_ctx: None +) -> AsyncIterator[Any]: + yield value + yield 42 + + +@pytest.mark.parametrize( + "prepare_to_fly, error, match", + [ + ( + _update_first, + TypeError, + r"\.prepare_to_fly\(\) must yield its context first", + ), + (_no_ctx, RuntimeError, "finished without yielding its context"), + ( + _ctx_then_not_an_update, + TypeError, + "must yield WatcherUpdate after its context, not int: 42", + ), + ], +) +async def test_prepare_to_fly_that_does_not_yield_its_context_first_raises( + recording_flyer, prepare_to_fly, error, match +): + logic, flyer = recording_flyer + logic.prepare_to_fly = prepare_to_fly + + with pytest.raises(error, match=match): + await flyer.prepare(1) + with pytest.raises(TransitionError, match="cannot go to FLYING from STAGED"): + await flyer.kickoff() + + +async def test_a_fly_that_fails_and_is_completed_is_not_logged( + caplog: pytest.LogCaptureFixture, +): + flyer = await make_staged_flyer(FlyingLogic(fail=True)) + await flyer.prepare(1) + await flyer.kickoff() + with pytest.raises(ValueError, match="bad fly"): + await flyer.complete() + + await flyer.unstage() + + assert not caplog.records diff --git a/tests/unit_tests/core/test_movable.py b/tests/unit_tests/core/test_movable.py index c3df8fd096..1e9c843b14 100644 --- a/tests/unit_tests/core/test_movable.py +++ b/tests/unit_tests/core/test_movable.py @@ -1,12 +1,20 @@ import asyncio +from collections.abc import AsyncGenerator +from dataclasses import dataclass, field from functools import cached_property from unittest.mock import AsyncMock, Mock, patch import pytest from ophyd_async.core import ( + KICKED_OFF, + FlyableLogic, MovableLogic, + StandardFlyable, StandardMovable, + StandardReadable, + StoppedError, + WatcherUpdate, callback_on_mock_put, init_devices, mock_puts_blocked, @@ -43,6 +51,11 @@ async def test_move_timeout_with_timeout_none_returns_none(): assert move_timeout() is None +class InstantStopMovableLogic(MovableLogic[float]): + async def ensure_stopped(self) -> None: + pass + + class StandardMovableImpl(StandardMovable[float]): def __init__(self, name: str = ""): self.readback, _ = soft_signal_r_and_setter(float) @@ -50,8 +63,8 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def movable_logic(self) -> MovableLogic[float]: - return MovableLogic(setpoint=self.setpoint, readback=self.readback) + def logic(self) -> MovableLogic[float]: + return InstantStopMovableLogic(setpoint=self.setpoint, readback=self.readback) @pytest.fixture @@ -61,17 +74,17 @@ async def movable() -> StandardMovableImpl: return movable -def test_movable_logic_is_cached(movable: StandardMovableImpl): - logic = movable.movable_logic - logic2 = movable.movable_logic +def test_logic_is_cached(movable: StandardMovableImpl): + logic = movable.logic + logic2 = movable.logic assert logic == logic2 async def test_movable_check_value(movable: StandardMovableImpl): - movable.movable_logic.check_move = AsyncMock() + movable.logic.check_move = AsyncMock() await movable.check_value(5) - movable.movable_logic.check_move.assert_awaited_once_with(5) + movable.logic.check_move.assert_awaited_once_with(5) async def test_locatable(movable: StandardMovableImpl) -> None: @@ -130,15 +143,15 @@ async def test_movable_moving_stopped(movable: StandardMovableImpl): await move_status -async def test_movable_set_calls_movable_logic_check_move_and_calculate_timeout( +async def test_movable_set_calls_logic_check_move_and_calculate_timeout( movable: StandardMovableImpl, ): - mock_check_move = movable.movable_logic.check_move = AsyncMock() + mock_check_move = movable.logic.check_move = AsyncMock() timeout = 5 - mock_calculate_timeout = movable.movable_logic.calculate_timeout = AsyncMock( + mock_calculate_timeout = movable.logic.calculate_timeout = AsyncMock( return_value=timeout ) - mock_move = movable.movable_logic.move = AsyncMock() + mock_move = movable.logic.move = AsyncMock() with patch("ophyd_async.core._movable.MoveTimeout") as move_timeout: pos = 10 @@ -173,3 +186,240 @@ async def test_motor_set_with_instant_mock( await status assert status.success assert await movable.readback.get_value() == -50.0 + + +async def test_movable_logic_stop_is_deprecated_but_still_called(): + with pytest.deprecated_call(match=r"OldStopLogic.stop\(\) is deprecated"): + + class OldStopLogic(MovableLogic[float]): + async def stop(self) -> None: + stopped.append(True) + + stopped: list[bool] = [] + + class OldStopMovable(StandardMovable[float]): + def __init__(self, name: str = ""): + self.readback, _ = soft_signal_r_and_setter(float) + self.setpoint = soft_signal_rw(float) + super().__init__(name=name) + + @cached_property + def logic(self) -> MovableLogic[float]: + # The deprecated stop() is turned into ensure_stopped() when the class + # is made, which pyright can not see + return OldStopLogic( # pyright: ignore[reportAbstractUsage] + setpoint=self.setpoint, readback=self.readback + ) + + async with init_devices(mock=True): + movable = OldStopMovable() + await movable.stop() + assert stopped == [True] + + +@dataclass +class HeldMoveLogic(MovableLogic[float], FlyableLogic[None, None]): + """Moves until released or cancelled; records when it was stopped.""" + + moving: asyncio.Event = field(default_factory=asyncio.Event) + release: asyncio.Event = field(default_factory=asyncio.Event) + calls: list[str] = field(default_factory=list) + + async def move(self, new_position: float, timeout) -> None: + self.calls.append(f"move {new_position}") + self.moving.set() + await self.release.wait() + set_mock_value(self.setpoint, new_position) + + async def ensure_stopped(self) -> None: + self.calls.append("ensure_stopped") + + async def prepare_to_fly( + self, name: str, value: None, previous_ctx: None + ) -> AsyncGenerator[None, None]: + yield None + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + yield KICKED_OFF + + +class StageableMovable(StandardReadable, StandardMovable[float]): + def __init__(self, name: str = ""): + self.readback, _ = soft_signal_r_and_setter(float) + self.setpoint = soft_signal_rw(float) + super().__init__(name=name) + + @cached_property + def logic(self) -> HeldMoveLogic: + return HeldMoveLogic(setpoint=self.setpoint, readback=self.readback) + + +class FlyableMovable(StageableMovable, StandardFlyable[None, None]): + pass + + +@pytest.fixture +async def held() -> FlyableMovable: + async with init_devices(mock=True): + held = FlyableMovable(name="movable") + return held + + +@pytest.mark.parametrize("success", [False, True]) +async def test_stop_ends_a_set_with_or_without_error( + held: FlyableMovable, success: bool +): + move_status = held.set(1.0) + await held.logic.moving.wait() + + await held.stop(success=success) + + if success: + await move_status + else: + with pytest.raises(StoppedError, match="Device movable was stopped"): + await move_status + assert held.logic.calls == ["move 1.0", "ensure_stopped"] + + +async def test_stop_before_any_set_and_twice_is_harmless(held: FlyableMovable): + await held.stop() + move_status = held.set(1.0) + await held.logic.moving.wait() + await held.stop() + await held.stop() + with pytest.raises(StoppedError, match="was stopped"): + await move_status + # A later set is not affected by the earlier stops + held.logic.release.set() + await held.set(2.0) + assert held.logic.calls == [ + "ensure_stopped", + "move 1.0", + "ensure_stopped", + "ensure_stopped", + "move 2.0", + ] + + +async def test_stop_before_the_set_starts_leaves_the_hardware_alone( + held: FlyableMovable, +): + move_status = held.set(1.0) + await held.stop() + with pytest.raises(StoppedError, match="was stopped"): + await move_status + assert held.logic.calls == ["ensure_stopped"] + + +async def test_second_set_supersedes_the_first(held: FlyableMovable): + first = held.set(1.0) + await held.logic.moving.wait() + second = held.set(2.0) + + with pytest.raises(StoppedError, match=r"movable: superseded by a new set\(\)"): + await first + held.logic.release.set() + await second + assert await held.setpoint.get_value() == 2.0 + # Neither set stopped the hardware: the second move retargets it + assert held.logic.calls == ["move 1.0", "move 2.0"] + + +# A timeout of the logic's own is not mistaken for a stop +async def test_a_timeout_inside_move_raises_timeout_error( + movable: StandardMovableImpl, +): + async def move(new_position: float, timeout) -> None: + async with asyncio.timeout(0.01): + await asyncio.Event().wait() + + movable.logic.move = move + with pytest.raises(TimeoutError): + await movable.set(1.0) + + +@pytest.mark.parametrize("flies", [False, True]) +async def test_only_a_flyer_stops_a_set_on_stage_and_unstage(flies: bool): + async with init_devices(mock=True): + movable = FlyableMovable() if flies else StageableMovable() + move_status = movable.set(1.0) + await movable.logic.moving.wait() + + await movable.stage() + await movable.unstage() + + # A pure movable stages nothing; a flyer gets ready once on stage and stops on + # unstage, which also halt a set() in progress without ending it + assert movable.logic.calls == ( + ["move 1.0", "ensure_stopped", "ensure_stopped"] if flies else ["move 1.0"] + ) + assert not move_status.done + movable.logic.release.set() + await move_status + + +async def test_unstage_stops_the_hardware_of_a_set_in_progress(held: FlyableMovable): + await held.stage() + move_status = held.set(1.0) + await held.logic.moving.wait() + + await held.unstage() + + # Never prepared, but stopped anyway; only stop() ends the set() itself + assert not move_status.done + held.logic.release.set() + await move_status + assert held.logic.calls == ["ensure_stopped", "move 1.0", "ensure_stopped"] + + +@dataclass +class HeldFlyLogic(HeldMoveLogic): + """Its prepare and fly run until released.""" + + async def prepare_to_fly( + self, name: str, value: None, previous_ctx: None + ) -> AsyncGenerator[None, None]: + yield None + await self.release.wait() + + async def fly(self, ctx: None) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + yield KICKED_OFF + await self.release.wait() + + +class HeldFlyer(FlyableMovable): + @cached_property + def logic(self) -> HeldFlyLogic: + return HeldFlyLogic(setpoint=self.setpoint, readback=self.readback) + + +@pytest.mark.parametrize("success", [False, True]) +async def test_stop_halts_the_hardware_but_does_not_end_a_prepare_or_fly( + success: bool, +): + async with init_devices(mock=True): + flyer = HeldFlyer(name="flyer") + await flyer.stage() + preparing = flyer.prepare(None) + await wait_for_pending_wakeups() + + await flyer.stop(success=success) + + # Only the hardware was halted: unstage() and a new prepare() end a prepare() + assert not preparing.done + flyer.logic.release.set() + await preparing + flyer.logic.release.clear() + await flyer.kickoff() + flying = flyer.complete() + await wait_for_pending_wakeups() + + await flyer.stop(success=success) + + assert not flying.done + flyer.logic.release.set() + await flying + # stage() got it ready once, then the two stop()s + assert flyer.logic.calls.count("ensure_stopped") == 3 + await flyer.unstage() diff --git a/tests/unit_tests/core/test_multi_derived_signal.py b/tests/unit_tests/core/test_multi_derived_signal.py index dd252af6fa..3f8b038301 100644 --- a/tests/unit_tests/core/test_multi_derived_signal.py +++ b/tests/unit_tests/core/test_multi_derived_signal.py @@ -298,8 +298,8 @@ async def _put(val: float) -> None: derived_sig = derived_signal_rw(_get, _put, m1=m1, m2=m2) await derived_sig.set(6) - assert await m1.movable_logic.readback.get_value() == 3 - assert await m2.movable_logic.readback.get_value() == 3 + assert await m1.logic.readback.get_value() == 3 + assert await m2.logic.readback.get_value() == 3 assert await derived_sig.get_value() == 6 diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index c0d682bc1d..f6b0aafa54 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -15,6 +15,7 @@ MockSignalBackend, SignalR, StandardReadable, + TransitionError, soft_signal_r_and_setter, soft_signal_rw, ) @@ -348,3 +349,13 @@ def __init__(self, name): with pytest.raises(KeyError): DummyDerivedDevice("test_duplicates") + + +async def test_standard_readable_stage_twice_raises(): + sr = StandardReadable(name="sr") + await sr.stage() + # Like ophyd's RedundantStaging + with pytest.raises(TransitionError, match="cannot go to STAGED from STAGED"): + await sr.stage() + await sr.unstage() + await sr.stage() diff --git a/tests/unit_tests/core/test_signal.py b/tests/unit_tests/core/test_signal.py index e9a7b7a973..7792315f3c 100644 --- a/tests/unit_tests/core/test_signal.py +++ b/tests/unit_tests/core/test_signal.py @@ -64,6 +64,7 @@ assert_reading, assert_value, partial_reading, + wait_for_pending_wakeups, ) _array_vals = { @@ -1344,3 +1345,36 @@ async def test_soft_signal_r_and_setter_with_poll_period(): async def test_soft_signal_r_and_setter_poll_period_without_getter_raises(): with pytest.raises(ValueError, match="poll_period requires a getter"): soft_signal_r_and_setter(float, poll_period=0.1) + + +async def test_set_and_wait_for_other_value_cancelled_before_first_value(): + set_signal = epics_signal_rw(int, "pva://signal", name="s") + match_signal = epics_signal_rw(int, "pva://match_signal", name="m") + await set_signal.connect(mock=True) + await match_signal.connect(mock=True) + + async def never_yields(signal): + await asyncio.sleep(1000) + yield # pragma: no cover - unreachable, just makes this a generator + + with patch("ophyd_async.core._signal.observe_value", never_yields): + task = asyncio.create_task( + set_and_wait_for_other_value(set_signal, 1, match_signal, 20, timeout=None) + ) + await wait_for_pending_wakeups() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + # The match wait it started must not outlive it + assert asyncio.all_tasks() == {asyncio.current_task()} + + +async def test_wait_for_value_unsubscribes_before_it_returns(): + signal = soft_signal_rw(int, initial_value=1) + await signal.connect() + with patch.object( + SignalR, "clear_sub", autospec=True, side_effect=SignalR.clear_sub + ) as clear_sub: + await wait_for_value(signal, 1, timeout=1) + # Not left for the abandoned generators to do when they are finalised + clear_sub.assert_called_once() diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 02e8be2ac2..6741959336 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -710,7 +710,7 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): # Yield so detector can get collections written, then set it so we complete await wait_for_pending_wakeups(raise_if_exceeded=False) await dl.collections_written.set(1) - assert status.done + await status assert status.success docs = [doc async for doc in det.collect_asset_docs()] assert docs == [ diff --git a/tests/unit_tests/core/test_status.py b/tests/unit_tests/core/test_status.py index 767b51ba23..107e379a4b 100644 --- a/tests/unit_tests/core/test_status.py +++ b/tests/unit_tests/core/test_status.py @@ -413,3 +413,35 @@ async def test_forward_watcher_updates(moves: dict[str, list], combine: bool): # Updates from different children interleave, but each child's stay in order for name, values in moves.items(): assert [u["current"] for u in updates if u["name"] == name] == values + + +async def test_a_watcher_that_raises_is_logged_and_does_not_fail_the_status( + caplog: pytest.LogCaptureFixture, +): + closed = False + + async def updates(): + nonlocal closed + try: + for value in range(3): + yield WatcherUpdate(current=value, initial=0, target=2, name="x") + await asyncio.sleep(0) + finally: + closed = True + + status = WatchableAsyncStatus(updates()) + bad = Mock(side_effect=ValueError("bad watcher")) + good = Mock() + status.watch(bad) + status.watch(good) + + await status + + assert status.success + assert closed + # Every update still reached both, and each failure was logged + assert bad.call_count == good.call_count == 3 + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 3 + assert errors[0].exc_info is not None + assert str(errors[0].exc_info[1]) == "bad watcher" diff --git a/tests/unit_tests/core/test_utils.py b/tests/unit_tests/core/test_utils.py index 165b03c759..759a45f21a 100644 --- a/tests/unit_tests/core/test_utils.py +++ b/tests/unit_tests/core/test_utils.py @@ -1,8 +1,11 @@ import asyncio +import gc import re import sys from asyncio import CancelledError +from typing import TypeVar, assert_type +import numpy as np import pytest from ophyd_async.core import ( @@ -21,10 +24,14 @@ simulate_move, soft_signal_rw, ) +from ophyd_async.core._utils import _wait_for from ophyd_async.epics.core import epics_signal_rw from ophyd_async.epics.motor import Motor +T = TypeVar("T") + + class ValueErrorBackend(SoftSignalBackend): def __init__(self, exc_text=""): self.exc_text = exc_text @@ -338,7 +345,7 @@ def test_core_notconnected_emits_deprecation_warning(): from ophyd_async.core import NotConnected # noqa: F401 -async def _async_value(val): +async def _async_value(val: T) -> T: """Helper function to create an awaitable that returns a value.""" return val @@ -349,32 +356,26 @@ async def test_gather_dict_empty(): assert result == {} -async def test_gather_dict_duplicate_resolved_keys(): - """Test gather_dict when different awaitable keys resolve to the same value.""" - # Two coroutine objects that resolve to the same key - second should win +async def test_gather_dict_awaits_values_and_infers_types(): + """Test gather_dict keeps keys and order, and types the result without hints.""" result = await gather_dict( - { - _async_value("key"): "first", - _async_value("key"): "second", - } + {"a": _async_value(1), "b": _async_value(2), "c": _async_value(3)} ) - assert result == {"key": "second"} + # pyright fails on this line in CI if the awaited types are not inferred + assert_type(result, dict[str, int]) + assert result == {"a": 1, "b": 2, "c": 3} + assert list(result) == ["a", "b", "c"] + +async def test_gather_dict_infers_array_values(): + """Test gather_dict types ndarray results, which the old signature got wrong.""" -@pytest.mark.parametrize("keys_awaitable", ("", "b", "abc")) -@pytest.mark.parametrize("values_awaitable", ([], [2], [1, 2, 3])) -async def test_gather_dict(keys_awaitable, values_awaitable): - """Test gather_dict with plain values, awaitable keys, and awaitable values.""" - expected = {"a": 1, "b": 2, "c": 3} + async def get_array(n: int) -> np.ndarray: + return np.zeros(n) - input_dict = {} - for k, v in expected.items(): - key = _async_value(k) if k in keys_awaitable else k - value = _async_value(v) if v in values_awaitable else v - input_dict[key] = value - result = await gather_dict(input_dict) - assert result == expected - assert list(result.keys()) == list(expected.keys()) + result = await gather_dict({k: get_array(n) for k, n in (("a", 1), ("b", 2))}) + assert_type(result, dict[str, np.ndarray]) + assert list(result) == ["a", "b"] # Cancellation propagation is broken in asyncio.gather on Python 3.10, so the exception @@ -484,3 +485,136 @@ async def test_simulate_move_ends_at_target(acceleration_time: float): positions = [p async for p in simulate_move(0, 1.5, 10, acceleration_time)] assert positions[-1] == 1.5 assert positions == sorted(positions) + + +class StuckPutBackend(SoftSignalBackend): + """A put that never finishes, and may fail while being cancelled.""" + + def __init__(self, cleanup_error: Exception | None): + self.cleanup_error = cleanup_error + super().__init__(datatype=int) + + async def put(self, value: int | None) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + if self.cleanup_error is not None: + raise self.cleanup_error from None + raise + + +@pytest.mark.parametrize( + "cleanup_error, expected", + [(None, TimeoutError), (ValueError("cleanup failed"), ValueError)], +) +async def test_a_timed_out_put_raises_the_error_its_cancellation_raised( + cleanup_error: Exception | None, expected: type[Exception] +): + loop = asyncio.get_running_loop() + unhandled = [] + loop.set_exception_handler(lambda _, context: unhandled.append(context)) + signal = SignalRW(StuckPutBackend(cleanup_error), name="sig") + await signal.connect() + + with pytest.raises(expected): + await signal.set(1, timeout=0.01) + + # The error was retrieved, not left for the loop to log + gc.collect() + await asyncio.sleep(0) + loop.set_exception_handler(None) + assert unhandled == [] + + +async def test_a_cancelled_wait_for_leaves_no_unretrieved_error(): + loop = asyncio.get_running_loop() + unhandled = [] + loop.set_exception_handler(lambda _, context: unhandled.append(context)) + + async def fails_when_cancelled(): + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + raise ValueError("cleanup failed") from None + + outer = asyncio.create_task(_wait_for(fails_when_cancelled(), 10, "source")) + await asyncio.sleep(0.01) + outer.cancel() + # The error raised while ending the cancelled work shows through + with pytest.raises(ValueError, match="cleanup failed"): + await outer + + gc.collect() + await asyncio.sleep(0) + loop.set_exception_handler(None) + assert unhandled == [] + + +async def test_wait_for_returns_the_result_with_and_without_a_timeout(): + async def value(): + await asyncio.sleep(0) + return 42 + + assert await _wait_for(value(), 5, "source") == 42 + assert await _wait_for(value(), None, "source") == 42 + + +async def test_wait_for_expiry_raises_timeout_error_naming_the_source(): + with pytest.raises(TimeoutError, match="^source$"): + await _wait_for(asyncio.sleep(1), 0.01, "source") + + +async def test_wait_for_does_not_relabel_a_timeout_error_from_the_coroutine(): + async def inner(): + raise TimeoutError("inner") + + with pytest.raises(TimeoutError, match="^inner$"): + await _wait_for(inner(), 5, "source") + + +async def test_wait_for_does_not_swallow_a_cancel_arriving_as_the_work_finishes(): + finished = asyncio.get_running_loop().create_future() + + async def inner(): + return await finished + + outer = asyncio.create_task(_wait_for(inner(), 5, "source")) + await asyncio.sleep(0) + finished.set_result(1) + outer.cancel() + with pytest.raises(asyncio.CancelledError): + await outer + + +async def test_wait_for_shows_an_error_raised_while_being_cancelled(): + async def inner(): + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + raise ValueError("cleanup") from None + + with pytest.raises(ValueError, match="cleanup"): + await _wait_for(inner(), 0.01, "source") + + +async def test_wait_for_expiry_and_an_outside_cancel_is_a_cancel(): + outer = asyncio.create_task(_wait_for(asyncio.sleep(1), 0.05, "source")) + await asyncio.sleep(0.05) # About when the timeout expires + outer.cancel() + with pytest.raises(asyncio.CancelledError): + await outer + + +async def test_wait_for_expiry_after_a_swallowed_cancel_is_still_a_timeout_error(): + async def body(): + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + pass # Swallowed without uncancel() + await _wait_for(asyncio.sleep(1), 0.01, "source") + + task = asyncio.create_task(body()) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(TimeoutError, match="^source$"): + await task diff --git a/tests/unit_tests/core/test_watchable_async_status.py b/tests/unit_tests/core/test_watchable_async_status.py index cabdc18b44..f86afa6e34 100644 --- a/tests/unit_tests/core/test_watchable_async_status.py +++ b/tests/unit_tests/core/test_watchable_async_status.py @@ -216,3 +216,214 @@ async def set(self, value) -> AsyncIterator: # undecorated methods will not print the device name status: AsyncStatus = ctx.value.args[0] assert f"device: {device_name}" in repr(status) + + +class StopError(Exception): + pass + + +class Ticks: + """Yields an update then waits forever, or until `finish` is resolved.""" + + def __init__(self, finish: asyncio.Future | None = None): + self.finish = finish + self.started = False + + def __aiter__(self): + return self + + async def __anext__(self) -> WatcherUpdate[int]: + if self.started: + await (self.finish or asyncio.sleep(10)) + raise StopAsyncIteration + self.started = True + return WatcherUpdate(current=1, initial=0, target=2) + + +@pytest.mark.parametrize("error", [None, StopError("stopped")]) +@pytest.mark.parametrize("before_start", [True, False]) +async def test_stop_future_ends_a_watchable_status(error, before_start): + ticks = Ticks() + stop = asyncio.get_running_loop().create_future() + updates: list[dict] = [] + if before_start: + stop.set_result(error) + status = WatchableAsyncStatus(ticks, stop=stop) + status.watch(lambda **kw: updates.append(kw)) + if not before_start: + await asyncio.sleep(0.01) + assert not status.done + stop.set_result(error) + if error is None: + await status + assert status.success + else: + with pytest.raises(StopError): + await status + # Stopped before it started, so the iterator body never ran + assert ticks.started is not before_start + assert len(updates) == (0 if before_start else 1) + + +async def test_no_stop_future_adds_nothing(): + tasks = asyncio.all_tasks() + ticks = Ticks(finish=asyncio.get_running_loop().create_future()) + status = WatchableAsyncStatus(ticks) + assert asyncio.all_tasks() - tasks == {status.task} + await asyncio.sleep(0.01) + assert asyncio.all_tasks() - tasks == {status.task} + assert ticks.finish + ticks.finish.set_result(None) + await status + assert status.success + + +@pytest.mark.parametrize("stop_first", [True, False]) +async def test_outside_cancel_with_stop_in_the_same_iteration_cancels(stop_first): + stop = asyncio.get_running_loop().create_future() + status = WatchableAsyncStatus(Ticks(), stop=stop) + await asyncio.sleep(0.01) + if stop_first: + stop.set_result(None) + status.task.cancel() + else: + status.task.cancel() + stop.set_result(None) + with pytest.raises(asyncio.CancelledError): + await status + assert status.task.cancelled() + + +@pytest.mark.parametrize("yields", range(5)) +async def test_outside_cancel_while_the_iterator_is_finishing_is_not_lost(yields): + loop = asyncio.get_running_loop() + finish, stop = loop.create_future(), loop.create_future() + ticks = Ticks(finish) + status = WatchableAsyncStatus(ticks, stop=stop) + await asyncio.sleep(0.01) + finish.set_result(None) + for _ in range(yields): + await asyncio.sleep(0) + if status.task.cancel(): + with pytest.raises(asyncio.CancelledError): + await status + assert status.task.cancelled() + else: + await status + + +async def test_timeout_inside_a_stoppable_watchable_status_raises_timeout_error(): + async def iterator(): + async with asyncio.timeout(0.01): + yield WatcherUpdate(current=1, initial=0, target=2) + await asyncio.sleep(10) + + stop = asyncio.get_running_loop().create_future() + status = WatchableAsyncStatus(iterator(), stop=stop) + with pytest.raises(TimeoutError): + await status + assert not stop.done() + + +async def test_stop_future_decides_even_if_the_iterator_absorbs_the_cancel(): + async def iterator(): + yield WatcherUpdate(current=1, initial=0, target=2) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + pass + yield WatcherUpdate(current=2, initial=0, target=2) + + stop = asyncio.get_running_loop().create_future() + status = WatchableAsyncStatus(iterator(), stop=stop) + await asyncio.sleep(0.01) + stop.set_result(StopError("stopped")) + with pytest.raises(StopError): + await status + + +class MoveError(Exception): + pass + + +@pytest.mark.parametrize("error", [None, MoveError("failed")]) +async def test_an_iterator_ending_in_the_same_iteration_as_the_stop_wins(error): + loop = asyncio.get_running_loop() + finish, stop = loop.create_future(), loop.create_future() + + async def iterator(): + yield WatcherUpdate(current=1, initial=0, target=2) + await finish + if error: + raise error + + status = WatchableAsyncStatus(iterator(), stop=stop) + await asyncio.sleep(0.01) + finish.set_result(None) + # The iterator resumes before the stop resolves, so both are done by the + # time the race looks + await asyncio.sleep(0) + stop.set_result(StopError("stopped")) + if error: + with pytest.raises(MoveError): + await status + else: + await status + assert status.success + + +async def test_an_error_raised_while_stopping_shows_through(): + async def iterator(): + yield WatcherUpdate(current=1, initial=0, target=2) + try: + await asyncio.sleep(10) + except asyncio.CancelledError as e: + raise MoveError("failed to stop") from e + + stop = asyncio.get_running_loop().create_future() + status = WatchableAsyncStatus(iterator(), stop=stop) + await asyncio.sleep(0.01) + stop.set_result(StopError("stopped")) + with pytest.raises(MoveError): + await status + + +async def test_a_second_outside_cancel_waits_for_the_iterator_cleanup(): + cleaned_up = asyncio.Event() + + async def iterator(): + try: + yield WatcherUpdate(current=1, initial=0, target=2) + await asyncio.sleep(10) + finally: + # Slow cleanup, which a second cancel must not abandon + await asyncio.sleep(0.05) + cleaned_up.set() + + stop = asyncio.get_running_loop().create_future() + status = WatchableAsyncStatus(iterator(), stop=stop) + await asyncio.sleep(0.01) + status.task.cancel() + await asyncio.sleep(0.01) + status.task.cancel() + with pytest.raises(asyncio.CancelledError): + await status + assert cleaned_up.is_set() + + +@pytest.mark.parametrize("name, prefix", [("dev", "dev: "), (None, "")]) +async def test_a_failing_watcher_is_logged_and_ignored(name, prefix, caplog): + async def iterator(): + yield WatcherUpdate(current=1, initial=0, target=2) + + status = WatchableAsyncStatus(iterator(), name=name) + + def bad_watcher(**kw): + raise ZeroDivisionError + + status.watch(bad_watcher) + await status + assert status.success + assert [r.getMessage() for r in caplog.records if r.levelname == "ERROR"] == [ + f"{prefix}watcher {bad_watcher!r} raised, ignoring it" + ] diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index dbcf1aacbe..2452074cec 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -373,7 +373,7 @@ async def test_flyscan_aravis_detector(static_path_provider: StaticPathProvider) }, ), ] - assert status.done + await status # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, @@ -399,6 +399,7 @@ async def test_flyscan_aravis_detector(static_path_provider: StaticPathProvider) }, ), ] + await status async def test_2_rois_with_hdf(tmp_path): diff --git a/tests/unit_tests/epics/pmac/test_pmac_trajectory.py b/tests/unit_tests/epics/pmac/test_pmac_trajectory.py index f2ee9bf8dc..c5ed3cb17f 100644 --- a/tests/unit_tests/epics/pmac/test_pmac_trajectory.py +++ b/tests/unit_tests/epics/pmac/test_pmac_trajectory.py @@ -1,3 +1,4 @@ +import asyncio from unittest.mock import AsyncMock, call, patch import numpy as np @@ -8,6 +9,7 @@ callback_on_mock_execute, get_mock, get_mock_execute, + mock_puts_blocked, set_and_wait_for_value, set_callback_filter, set_mock_value, @@ -39,6 +41,7 @@ async def test_pmac_prepare(sim_motors: tuple[PmacIO, Motor, Motor]): spec = Fly(2.0 @ Line(sim_x_motor, 1, 5, 2)) value = PmacScanInfo(spec=spec, ramp_time=None, turnaround_time=None) flyer = PmacTrajectoryFlyableLogic(pmac_io).with_device() + await flyer.stage() await flyer.prepare(value) assert await pmac_io.coord[1].cs_axis_setpoint[7].get_value() == -1.2 @@ -65,6 +68,7 @@ async def test_pmac_prepare_with_configured_ramp( spec = Fly(2.0 @ Line(sim_x_motor, 1, 5, 2)) value = PmacScanInfo(spec=spec, ramp_time=2, turnaround_time=None) flyer = PmacTrajectoryFlyableLogic(pmac_io).with_device() + await flyer.stage() await flyer.prepare(value) assert await pmac_io.coord[1].cs_axis_setpoint[7].get_value() == -3.0 @@ -91,6 +95,7 @@ async def test_pmac_prepare_with_configured_ramp_and_turnaround( spec = Fly(2.0 @ (2 * ~Line(sim_x_motor, 1, 5, 2))) value = PmacScanInfo(spec=spec, ramp_time=2, turnaround_time=3) flyer = PmacTrajectoryFlyableLogic(pmac_io).with_device() + await flyer.stage() await flyer.prepare(value) assert await pmac_io.coord[1].cs_axis_setpoint[7].get_value() == -3.0 @@ -224,6 +229,7 @@ async def test_pmac_trajectory_kickoff( value = PmacScanInfo(spec=spec, ramp_time=None, turnaround_time=None) with patch("ophyd_async.epics.pmac._pmac_trajectory.SLICE_SIZE", 2): # This will prepare the buffer with 2 frames of info + await flyer.stage() await flyer.prepare(value) # This will consume another 2 frames set_mock_value( @@ -332,21 +338,38 @@ async def test_pmac_trajectory_kickoff( ) -async def test_pmac_trajectory_kickoff_trajectory_raises_exception_if_no_prepare( - sim_motors: tuple[PmacIO, Motor, Motor], +@pytest.mark.parametrize("run_up_ends_by", ["timeout", "unstage"]) +async def test_pmac_kickoff_that_fails_during_run_up_stops_the_trajectory( + sim_motors: tuple[PmacIO, Motor, Motor], run_up_ends_by: str ): - # Ordering is enforced by StandardFlyable, so drive through the device. - pmac_io, _, _ = sim_motors - flyer = PmacTrajectoryFlyableLogic(pmac_io).with_device() - with pytest.raises(RuntimeError, match="prepare.* before kickoff"): - await flyer.kickoff() - - -async def test_pmac_trajectory_complete(sim_motors: tuple[PmacIO, Motor, Motor]): - pmac_io, _, _ = sim_motors + pmac_io, sim_x_motor, sim_y_motor = sim_motors flyer = PmacTrajectoryFlyableLogic(pmac_io).with_device() - with pytest.raises(RuntimeError, match="kickoff.* before complete"): - await flyer.complete() + spec = Fly(2.0 @ (Line(sim_y_motor, 1, 5, 2) * ~Line(sim_x_motor, 1, 5, 2))) + value = PmacScanInfo(spec=spec, ramp_time=None, turnaround_time=None) + positions = get_mock(pmac_io.trajectory.positions[7]) + with patch("ophyd_async.epics.pmac._pmac_trajectory.SLICE_SIZE", 2): + await flyer.stage() + await flyer.prepare(value) + appends = len(positions.mock_calls) + # Keep the trajectory executing so it would go on appending + with mock_puts_blocked(pmac_io.trajectory.execute_profile): + if run_up_ends_by == "timeout": + with patch( + "ophyd_async.epics.pmac._pmac_trajectory.wait_for_value", + AsyncMock(side_effect=TimeoutError), + ): + with pytest.raises(TimeoutError): + await flyer.kickoff() + else: + status = flyer.kickoff() + await asyncio.sleep(0.01) + await flyer.unstage() + with pytest.raises(asyncio.CancelledError): + await status + # A trajectory still running would refill the buffer as it empties + set_mock_value(pmac_io.trajectory.total_points, 2) + await asyncio.sleep(0.01) + assert len(positions.mock_calls) == appends async def test_pmac_trajectory_stage(sim_motors: tuple[PmacIO, Motor, Motor]): @@ -377,19 +400,25 @@ async def test_pmac_trajectory_stage(sim_motors: tuple[PmacIO, Motor, Motor]): async def test_pmac_trajectory_unstage(sim_motors: tuple[PmacIO, Motor, Motor]): pmac_io, _, _ = sim_motors logic = PmacTrajectoryFlyableLogic(pmac_io) - logic.stop = AsyncMock() + logic.ensure_stopped = AsyncMock() flyer = logic.with_device() await flyer.unstage() - logic.stop.assert_called_once() + # Unstage stops even though it was not prepared since stage + logic.ensure_stopped.assert_awaited_once() async def test_trajectory_stop_if_running(sim_motors: tuple[PmacIO, Motor, Motor]): pmac_io, _, _ = sim_motors pmac_trajectory = PmacTrajectoryFlyableLogic(pmac_io) execute_mock = get_mock_execute(pmac_io.trajectory.abort_profile) + # Aborting stops the trajectory + callback_on_mock_execute( + pmac_io.trajectory.abort_profile, + lambda: set_mock_value(pmac_io.trajectory.execute_state, PmacExecuteState.DONE), + ) # Method not called as no running trajectory - await pmac_trajectory.stop() + await pmac_trajectory.ensure_stopped() execute_mock.assert_not_awaited() # Mocking that trajectory is executing @@ -397,9 +426,10 @@ async def test_trajectory_stop_if_running(sim_motors: tuple[PmacIO, Motor, Motor pmac_trajectory.pmac.trajectory.execute_state, PmacExecuteState.EXECUTING ) - # Method called as there is now a running trajectory - await pmac_trajectory.stop() + # Method called as there is now a running trajectory, and waits for it to stop + await pmac_trajectory.ensure_stopped() execute_mock.assert_awaited_once_with() + assert await pmac_io.trajectory.execute_state.get_value() == PmacExecuteState.DONE async def test_trajectory_raises_if_profile_status_not_in_good_state( diff --git a/tests/unit_tests/epics/test_motor.py b/tests/unit_tests/epics/test_motor.py index eeed03faf0..f8634312ec 100644 --- a/tests/unit_tests/epics/test_motor.py +++ b/tests/unit_tests/epics/test_motor.py @@ -1,14 +1,17 @@ import asyncio -from unittest.mock import Mock +import contextlib +from unittest.mock import Mock, patch import pytest from bluesky.protocols import Reading from ophyd_async.core import ( CALCULATE_TIMEOUT, + DEFAULT_TIMEOUT, Device, DeviceMock, FlyMotorInfo, + StoppedError, callback_on_mock_put, default_mock_class, get_mock_put, @@ -47,6 +50,11 @@ async def motor(): set_mock_value(motor.high_limit_travel, 21) set_mock_value(motor.dial_low_limit_travel, -11) set_mock_value(motor.dial_high_limit_travel, 21) + # Not moving, and a STOP stops it + set_mock_value(motor.motor_done_move, 1) + callback_on_mock_put( + motor.motor_stop, lambda _: set_mock_value(motor.motor_done_move, 1) + ) yield motor @@ -157,6 +165,15 @@ async def test_motor_moving_stopped(motor: Motor): assert move_status.done +async def test_stage_and_unstage_each_send_stop_even_when_not_moving(motor: Motor): + set_mock_value(motor.motor_done_move, 1) + await motor.stage() + get_mock_put(motor.motor_stop).assert_called_once_with(1) + + await motor.unstage() + assert get_mock_put(motor.motor_stop).call_count == 2 + + async def test_read_motor(motor: Motor): await motor.stage() assert (await motor.read())["motor"]["value"] == 0.0 @@ -244,6 +261,7 @@ async def test_set(motor: Motor, setpoint, velocity, timeout) -> None: async def test_prepare_velocity_limit_error(motor: Motor): + await motor.stage() set_mock_value(motor.max_velocity, 10) with pytest.raises(MotorLimitsError): fly_info = FlyMotorInfo(start_position=-10, end_position=0, time_for_move=0.9) @@ -251,6 +269,7 @@ async def test_prepare_velocity_limit_error(motor: Motor): async def test_valid_prepare_velocity(motor: Motor): + await motor.stage() # Widen user limits to accommodate ramp distances (trajectory goes to -10.5) set_mock_value(motor.low_limit_travel, -11) set_mock_value(motor.high_limit_travel, 21) @@ -279,6 +298,7 @@ async def test_prepare_motor_limits_error( upper_limit, lower_limit, ): + await motor.stage() set_mock_value(motor.acceleration_time, acceleration_time) set_mock_value(motor.low_limit_travel, lower_limit) set_mock_value(motor.high_limit_travel, upper_limit) @@ -292,28 +312,33 @@ async def test_prepare_motor_limits_error( await motor.prepare(fly_info) -async def test_prepare_valid_limits(motor: Motor): +@pytest.mark.parametrize( + "start_position, end_position, expected_setpoint, expected_end", + [(0, 10, -5, 15), (12, 2, 17, -3)], +) +async def test_prepare_valid_limits( + motor: Motor, + start_position: float, + end_position: float, + expected_setpoint: float, + expected_end: float, +): + await motor.stage() set_mock_value(motor.acceleration_time, 1) set_mock_value(motor.low_limit_travel, -10.01) set_mock_value(motor.high_limit_travel, 20.01) set_mock_value(motor.max_velocity, 10) fly_info = FlyMotorInfo( - start_position=0, - end_position=10, + start_position=start_position, + end_position=end_position, time_for_move=1, ) await motor.prepare(fly_info) - assert await motor.user_setpoint.get_value() == -5 - - assert fly_info.ramp_down_end_pos(await motor.acceleration_time.get_value()) == 15 - fly_info = FlyMotorInfo( - start_position=12, - end_position=2, - time_for_move=1, + assert await motor.user_setpoint.get_value() == expected_setpoint + assert ( + fly_info.ramp_down_end_pos(await motor.acceleration_time.get_value()) + == expected_end ) - await motor.prepare(fly_info) - assert await motor.user_setpoint.get_value() == 17 - assert fly_info.ramp_down_end_pos(await motor.acceleration_time.get_value()) == -3 @pytest.mark.parametrize( @@ -331,6 +356,7 @@ async def test_fly_scan( expected_end: float, ): """Happy-path prepare -> kickoff -> complete through the public interface.""" + await motor.stage() # A small acceleration time keeps the up-to-speed wait in kickoff() negligible set_mock_value(motor.acceleration_time, 0.2) set_mock_value(motor.low_limit_travel, -15) @@ -354,30 +380,53 @@ async def test_fly_scan( assert await motor.user_setpoint.get_value() == pytest.approx(expected_end) -async def test_kickoff_before_prepare_raises(motor: Motor): - with pytest.raises(RuntimeError, match="prepare.* before kickoff"): - await motor.kickoff() +async def test_stop_during_a_fly_halts_the_motor_but_does_not_end_complete( + velo_and_accl_respecting_motor: Motor, +): + motor = velo_and_accl_respecting_motor + await motor.stage() + await motor.prepare(FlyMotorInfo(start_position=0, end_position=2, time_for_move=1)) + await motor.kickoff() + status = motor.complete() + await asyncio.sleep(0.2) + await motor.stop() -async def test_complete_before_kickoff_raises(motor: Motor): - with pytest.raises(RuntimeError, match="kickoff.* before complete"): - await motor.complete() + # stop() halts the hardware, but only unstage() ends the fly + assert await motor.motor_done_move.get_value() == 1 + await motor.unstage() + with contextlib.suppress(asyncio.CancelledError): + await status -@pytest.mark.parametrize( - "done_move, expect_stop", - [ - (0, True), # moving -> stop is issued - (1, False), # already stopped -> no redundant STOP write - ], -) -async def test_stop_only_when_moving(motor: Motor, done_move: int, expect_stop: bool): +async def test_unstage_during_kickoff_run_up_cancels_the_move(motor: Motor): + await motor.stage() + get_mock_put(motor.motor_stop).reset_mock() + set_mock_value(motor.acceleration_time, 10) + set_mock_value(motor.max_velocity, 10) + await motor.prepare(FlyMotorInfo(start_position=0, end_position=1, time_for_move=1)) + # The fly move never finishes, and the motor is moving + set_mock_put_proceeds(motor.user_setpoint, False) + kickoff = motor.kickoff() + await wait_for_pending_wakeups() + set_mock_value(motor.motor_done_move, 0) + assert not kickoff.done + + await motor.unstage() + with pytest.raises(asyncio.CancelledError): + await kickoff + get_mock_put(motor.motor_stop).assert_called_once_with(1) + assert await motor.motor_done_move.get_value() == 1 + + +@pytest.mark.parametrize("done_move", [0, 1]) +async def test_stop_is_sent_whether_or_not_the_motor_looks_moving( + motor: Motor, done_move: int +): + # DMOV can flicker at standstill on some motors, so it does not decide set_mock_value(motor.motor_done_move, done_move) await motor.stop() - if expect_stop: - get_mock_put(motor.motor_stop).assert_called_once_with(1) - else: - get_mock_put(motor.motor_stop).assert_not_called() + get_mock_put(motor.motor_stop).assert_called_once_with(1) def test_core_notconnected_emits_deprecation_warning(): @@ -617,6 +666,8 @@ async def test_velo_and_accl_respecting_motor_mock_behavior( [ # RunEngine path: Motor.stop() puts to STOP, then cancels the set() status ("motor_stop", RuntimeError), + # RunEngine pause path: stop(success=True) lets the set() finish normally + ("motor_stop_success", None), # Someone else writes to STOP; the motor record finishes the move normally ("stop_signal", None), # set() timeout cancels the simulated move without any stop @@ -629,9 +680,9 @@ async def test_velo_and_accl_respecting_motor_mock_interrupted_move( motor = velo_and_accl_respecting_motor # velocity 10 over 10 units, so the full move takes about 1.1s status = motor.set(10.0, timeout=0.3 if interrupt == "timeout" else 5) - if interrupt == "motor_stop": + if interrupt in ("motor_stop", "motor_stop_success"): await wait_for_value(motor.user_readback, lambda v: v > 0, timeout=1) - await motor.stop() + await motor.stop(success=interrupt == "motor_stop_success") elif interrupt == "stop_signal": await wait_for_value(motor.user_readback, lambda v: v > 0, timeout=1) await motor.motor_stop.set(1) @@ -644,3 +695,80 @@ async def test_velo_and_accl_respecting_motor_mock_interrupted_move( # the target means the move ended early rather than finishing assert await motor.motor_done_move.get_value() == 1 assert await motor.user_readback.get_value() < 10.0 + + +@pytest.mark.parametrize("moving", [False, True], ids=["before_motion", "moving"]) +async def test_a_set_stopped_before_or_during_motion_fails_and_stays_still( + velo_and_accl_respecting_motor: Motor, moving: bool +): + motor = velo_and_accl_respecting_motor + await motor.stage() + status = motor.set(10.0) + if moving: + await wait_for_value(motor.user_readback, lambda v: v > 0, timeout=1) + + await motor.stop() + + # stop() does not wait for the set to finish failing + with pytest.raises(StoppedError, match=f"Device {motor.name} was stopped"): + await status + assert await motor.motor_done_move.get_value() == 1 + stopped_at = await motor.user_readback.get_value() + assert (0 < stopped_at < 10) if moving else stopped_at == 0 + await asyncio.sleep(0.2) + assert await motor.user_readback.get_value() == stopped_at + + +async def test_stage_stops_a_move_in_progress( + velo_and_accl_respecting_motor: Motor, +): + motor = velo_and_accl_respecting_motor + status = motor.set(0.5) + await wait_for_value(motor.user_readback, lambda v: v > 0, timeout=1) + + await motor.stage() + + get_mock_put(motor.motor_stop).assert_called_once_with(1) + stopped_at = await motor.user_readback.get_value() + assert 0 < stopped_at < 0.5 + await status + assert await motor.user_readback.get_value() == stopped_at + await motor.unstage() + + +# Stopping the motor during the run-up does not end prepare, which finishes short +async def test_stop_during_prepare_run_up_halts_the_motor_and_prepare_finishes_short( + velo_and_accl_respecting_motor: Motor, +): + motor = velo_and_accl_respecting_motor + await motor.stage() + preparing = motor.prepare( + FlyMotorInfo(start_position=100, end_position=200, time_for_move=1) + ) + await wait_for_value(motor.user_readback, lambda v: v > 1, timeout=1) + await motor.stop() + await asyncio.wait_for(preparing, timeout=1) + assert await motor.user_readback.get_value() < 100 + await motor.unstage() + + +async def test_default_mock_motor_stop_finishes_a_move_in_progress(): + async with init_devices(mock=True): + motor = Motor("BLxxI-MO-TABLE-01:X", name="motor") + # A test simulating a move that has not finished + set_mock_value(motor.motor_done_move, 0) + + await asyncio.wait_for(motor.stop(), timeout=1) + + get_mock_put(motor.motor_stop).assert_called_once_with(1) + assert await motor.motor_done_move.get_value() == 1 + + +async def test_motor_stop_allows_the_acceleration_time_to_decelerate(motor: Motor): + await motor.acceleration_time.set(30) + set_mock_value(motor.motor_done_move, 0) + + with patch("ophyd_async.epics.motor.wait_for_value") as wait_for_value: + await motor.stop() + + assert wait_for_value.call_args.kwargs["timeout"] == 30 + DEFAULT_TIMEOUT diff --git a/tests/unit_tests/fastcs/panda/test_hdf_panda.py b/tests/unit_tests/fastcs/panda/test_hdf_panda.py index 7b9e87b4e0..6b5e5599b1 100644 --- a/tests/unit_tests/fastcs/panda/test_hdf_panda.py +++ b/tests/unit_tests/fastcs/panda/test_hdf_panda.py @@ -270,7 +270,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): }, ), ] - assert status.done + await status # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, @@ -313,7 +313,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): }, ), ] - assert status.done + await status async def test_get_indices_written(hdf_panda: HDFPanda): diff --git a/tests/unit_tests/fastcs/panda/test_trigger.py b/tests/unit_tests/fastcs/panda/test_trigger.py index 345a967a30..31732cf01b 100644 --- a/tests/unit_tests/fastcs/panda/test_trigger.py +++ b/tests/unit_tests/fastcs/panda/test_trigger.py @@ -66,6 +66,7 @@ async def set_active(value: bool): set_mock_value(mock_panda.seq[1].active, value) flyer = trigger_logic.with_device() + await flyer.stage() await flyer.prepare(seq_table_info) await asyncio.gather(flyer.kickoff(), set_active(True)) await asyncio.gather(flyer.complete(), set_active(False)) @@ -105,7 +106,8 @@ async def test_seq_scanspec_trigger_logic(mock_panda, sim_x_motor, sim_y_motor) ), # type: ignore }, ) - await trigger_logic.on_prepare(info) + async for _ in trigger_logic.prepare_to_fly("", info, None): + pass out = await trigger_logic.seq.table.get_value() assert out.repeats == pytest.approx([1, 1, 1, 5, 1, 1, 1, 5, 1, 1, 1, 5]) assert out.trigger == [ @@ -146,7 +148,8 @@ async def test_seq_scanspec_trigger_logic_no_gaps( ) }, ) - await trigger_logic.on_prepare(info) + async for _ in trigger_logic.prepare_to_fly("", info, None): + pass out = await trigger_logic.seq.table.get_value() assert out.repeats == pytest.approx([1, 1, 1, 3]) assert out.trigger == [ @@ -181,7 +184,8 @@ async def test_seq_scanspec_trigger_logic_duration_error( }, ) with pytest.raises(RuntimeError, match="Slice must have duration"): - await trigger_logic.on_prepare(info) + async for _ in trigger_logic.prepare_to_fly("", info, None): + pass async def test_seq_scanspec_trigger_logic_motor_not_passed( @@ -199,7 +203,8 @@ async def test_seq_scanspec_trigger_logic_motor_not_passed( ) }, ) - await trigger_logic.on_prepare(info) + async for _ in trigger_logic.prepare_to_fly("", info, None): + pass out = await trigger_logic.seq.table.get_value() assert out.repeats == pytest.approx([1, 1, 3]) assert out.trigger == [ @@ -227,7 +232,8 @@ async def test_seq_scanspec_trigger_logic_equal( ) }, ) - await trigger_logic.on_prepare(info) + async for _ in trigger_logic.prepare_to_fly("", info, None): + pass out = await trigger_logic.seq.table.get_value() assert out.repeats == pytest.approx([1, 1, 1, 1, 1, 1, 1, 1, 1]) assert out.trigger == [ @@ -261,6 +267,7 @@ async def set_active(value: bool): set_mock_value(mock_panda.pcomp[1].active, value) flyer = trigger_logic.with_device() + await flyer.stage() await flyer.prepare(pcomp_info) await asyncio.gather(flyer.kickoff(), set_active(True)) await asyncio.gather(flyer.complete(), set_active(False)) diff --git a/tests/unit_tests/sim/test_sim_motor.py b/tests/unit_tests/sim/test_sim_motor.py index 6d2d9f061f..3deeb137b9 100644 --- a/tests/unit_tests/sim/test_sim_motor.py +++ b/tests/unit_tests/sim/test_sim_motor.py @@ -1,20 +1,25 @@ import asyncio +import contextlib import sys +import threading from unittest.mock import Mock, call, patch +import bluesky.plan_stubs as bps +import bluesky.preprocessors as bpp import pytest from bluesky.plans import spiral_square from bluesky.protocols import Reading from bluesky.run_engine import RunEngine +from bluesky.utils import FailedStatus, RunEngineInterrupted from ophyd_async.core import ( Device, FlyMotorInfo, + StoppedError, WatchableAsyncStatus, forward_watcher_updates, ) from ophyd_async.sim import SimMotor -from ophyd_async.testing import StatusWatcher class TwoMotorStage(Device): @@ -96,19 +101,86 @@ async def test_short_move_is_exactly_move_time(m2: SimMotor): @pytest.mark.timeout(3) -async def test_stop(m2: SimMotor): +@pytest.mark.parametrize("moving", [False, True], ids=["before_motion", "moving"]) +@pytest.mark.parametrize("success", [False, True]) +async def test_stop(m2: SimMotor, success: bool, moving: bool): # this move should take 10 seconds but we will stop it partway through. # The sim motor updates its readback at 10Hz, so 0.2s is two update ticks # in - enough for the readback to have moved off 0 without waiting longer. move_status = m2.set(10) - await asyncio.sleep(0.2) - await m2.stop(success=False) + if moving: + await asyncio.sleep(0.2) + await m2.stop(success=success) new_pos = await m2.user_readback.get_value() - assert 0 < new_pos < 10 + assert (0 < new_pos < 10) if moving else new_pos == 0 - assert not move_status.success - with pytest.raises(RuntimeError, match=f"Device {m2.name} was stopped"): + if success: await move_status + assert move_status.success + else: + assert not move_status.success + with pytest.raises(StoppedError, match=f"Device {m2.name} was stopped"): + await move_status + # Nothing is left moving + await asyncio.sleep(0.3) + assert await m2.user_readback.get_value() == new_pos + + +@pytest.mark.timeout(3) +async def test_second_set_supersedes_the_first(m2: SimMotor): + first = m2.set(10) + await asyncio.sleep(0.2) + second = m2.set(0.5) + + with pytest.raises(StoppedError, match=r"M2: superseded by a new set\(\)"): + await first + await second + # Only the second move is left moving the motor, and it has finished + assert await m2.user_readback.get_value() == 0.5 + await asyncio.sleep(0.3) + assert await m2.user_readback.get_value() == 0.5 + + +@pytest.mark.timeout(3) +async def test_unstage_stops_a_fly_and_waits_for_it(m2: SimMotor): + await asyncio.gather(m2.velocity.set(10), m2.acceleration_time.set(0.05)) + await m2.stage() + await m2.prepare(FlyMotorInfo(start_position=0, end_position=5, time_for_move=1)) + await m2.kickoff() + status = m2.complete() + await asyncio.sleep(0.2) + + await m2.unstage() + + # Already stopped when unstage returns, not stopping soon after + with pytest.raises(asyncio.CancelledError): + await status + stopped_at = await m2.user_readback.get_value() + assert 0 < stopped_at < 5 + assert await m2.user_setpoint.get_value() == stopped_at + await asyncio.sleep(0.3) + assert await m2.user_readback.get_value() == stopped_at + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("success", [False, True]) +async def test_stop_during_a_fly_halts_the_motor_but_does_not_end_complete( + m2: SimMotor, success: bool +): + await asyncio.gather(m2.velocity.set(10), m2.acceleration_time.set(0.05)) + await m2.stage() + await m2.prepare(FlyMotorInfo(start_position=0, end_position=2, time_for_move=1)) + await m2.kickoff() + status = m2.complete() + await asyncio.sleep(0.2) + + await m2.stop(success=success) + + # stop() halts the motor but it is unstage() that ends the fly + assert 0 < await m2.user_readback.get_value() < 2 + await m2.unstage() + with contextlib.suppress(asyncio.CancelledError): + await status @pytest.mark.skipif("win" in sys.platform, reason="windows CI runners too weedy") @@ -116,26 +188,43 @@ async def test_fly(m1: SimMotor): await m1.acceleration_time.set(0.1) info = FlyMotorInfo(start_position=0, end_position=1, time_for_move=0.2) fly_start, fly_end, velocity = -0.25, 1.25, 5 + await m1.stage() await m1.prepare(info) assert await m1.user_readback.get_value() == fly_start assert await m1.velocity.get_value() == velocity await m1.kickoff() status = m1.complete() - watcher = StatusWatcher(status) - for i, v in enumerate([-0.25, 0, 0.5, 1.0, 1.25]): - await watcher.wait_for_call( - current=pytest.approx(v), - initial=fly_start, - name="M1", - target=fly_end, - time_elapsed=pytest.approx(i * 0.1, abs=0.1), - unit="mm", - ) + watcher = Mock() + status.watch(watcher) await status - watcher.mock.assert_not_called() + # Progress is reported from where the motor is when complete is called + # (it has been running up for the acceleration time) to the end + updates = [c.kwargs for c in watcher.call_args_list] + currents = [u["current"] for u in updates] + assert currents == sorted(currents) + assert currents[-1] == pytest.approx(fly_end) + assert {(u["initial"], u["target"], u["name"], u["unit"]) for u in updates} == { + (fly_start, fly_end, "M1", "mm") + } assert await m1.user_readback.get_value() == fly_end +@pytest.mark.timeout(3) +async def test_unstage_during_kickoff_run_up_leaves_no_move_running(m2: SimMotor): + await m2.acceleration_time.set(0.5) + await m2.stage() + await m2.prepare(FlyMotorInfo(start_position=0, end_position=5, time_for_move=1)) + kickoff_status = m2.kickoff() + await asyncio.sleep(0.2) + await m2.unstage() + with pytest.raises(asyncio.CancelledError): + await kickoff_status + # The readback updates at 10Hz, so a move still running would change it + stopped_at = await m2.user_readback.get_value() + await asyncio.sleep(0.3) + assert await m2.user_readback.get_value() == stopped_at + + async def test_sim_motor_can_be_set_to_its_current_position(m1: SimMotor): await m1.set(0) @@ -221,3 +310,79 @@ def on_readback(value: dict[str, Reading[float]]) -> None: previous <= current for previous, current in zip(readbacks, readbacks[1:], strict=False) ) + + +FLY_INFO = FlyMotorInfo(start_position=0, end_position=1, time_for_move=0.3) +FLY_END = FLY_INFO.ramp_down_end_pos(0.05) + + +async def make_fast_motor() -> SimMotor: + motor = SimMotor("M", instant=False) + await asyncio.gather(motor.velocity.set(20), motor.acceleration_time.set(0.05)) + return motor + + +def move_and_fly_plan( + motor: SimMotor, move_to: float, hang: str | None = None, fail: bool = False +): + """Move then fly the motor in a run, hanging in `hang` ("move" or "fly") if given. + + The hang ends in a failure if `fail`, otherwise it waits to be interrupted. + """ + + def hang_then_fail(): + yield from bps.sleep(0.3) + if fail: + raise ValueError("plan failed") + yield from bps.sleep(5) + + @bpp.run_decorator() + @bpp.stage_decorator([motor]) + def plan(): + if hang == "move": + yield from bps.abs_set(motor, 1000, group="move") + yield from hang_then_fail() + yield from bps.mv(motor, move_to) + yield from bps.prepare(motor, FLY_INFO, wait=True) + yield from bps.kickoff(motor, wait=True) + if hang == "fly": + yield from bps.complete(motor, group="fly") + yield from hang_then_fail() + yield from bps.complete(motor, wait=True) + + return plan() + + +async def test_staged_plans_back_to_back_move_and_fly_the_same_motor(): + RE = RunEngine() + motor = await make_fast_motor() + for move_to in (3, -2): + RE(move_and_fly_plan(motor, move_to)) + assert await motor.user_readback.get_value() == pytest.approx(FLY_END) + assert await motor.user_setpoint.get_value() == pytest.approx(FLY_END) + + +@pytest.mark.timeout(20) +@pytest.mark.parametrize("hang", ["move", "fly"]) +@pytest.mark.parametrize("interrupt", ["fail", "pause_then_abort"]) +async def test_a_plan_interrupted_mid_move_or_fly_leaves_the_motor_ready_for_the_next( + hang: str, interrupt: str +): + RE = RunEngine() + motor = await make_fast_motor() + if interrupt == "fail": + # unstage ends the move or fly, which bluesky reports as a FailedStatus + with pytest.raises((ValueError, FailedStatus)): + RE(move_and_fly_plan(motor, 3, hang=hang, fail=True)) + else: + threading.Timer(0.5, RE.request_pause).start() + with pytest.raises(RunEngineInterrupted): + RE(move_and_fly_plan(motor, 3, hang=hang)) + RE.abort() + # Nothing is left moving the motor + stopped_at = await motor.user_readback.get_value() + await asyncio.sleep(0.3) + assert await motor.user_readback.get_value() == stopped_at + + RE(move_and_fly_plan(motor, 3)) + assert await motor.user_readback.get_value() == pytest.approx(FLY_END) diff --git a/uv.lock b/uv.lock index 5fdfad44f5..e36da88e3b 100644 --- a/uv.lock +++ b/uv.lock @@ -1580,6 +1580,7 @@ dependencies = [ { name = "pyyaml" }, { name = "scanspec" }, { name = "stamina" }, + { name = "typing-extensions" }, { name = "velocity-profile" }, ] @@ -1654,6 +1655,7 @@ requires-dist = [ { name = "pyyaml" }, { name = "scanspec", specifier = ">=1.0.0" }, { name = "stamina", specifier = ">=23.1.0" }, + { name = "typing-extensions", specifier = ">=4.14" }, { name = "velocity-profile" }, ] provides-extras = ["sim", "ca", "pva", "tango", "demo"] @@ -1675,7 +1677,7 @@ dev = [ { name = "pre-commit" }, { name = "pydata-sphinx-theme", specifier = ">=0.12" }, { name = "pyepics", specifier = ">=3.4.2" }, - { name = "pyright" }, + { name = "pyright", specifier = ">=1.1.410" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -2267,15 +2269,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.409" +version = "1.1.414" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/4e/3aa27f74211522dba7e9cbc3e74de779c6d4b654c54e50a4840623be8014/pyright-1.1.409.tar.gz", hash = "sha256:986ee05beca9e077c165758ad123667c679e050059a2546aa02473930394bc93", size = 4430434, upload-time = "2026-04-23T11:02:03.799Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/16/6b/330d8ebae582b30c2959a1ef4c3bc344ebde48c2ff0c3f113c4710735e11/pyright-1.1.409-py3-none-any.whl", hash = "sha256:aa3ea228cab90c845c7a60d28db7a844c04315356392aa09fafcee98c8c22fb3", size = 6438161, upload-time = "2026-04-23T11:02:01.309Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, ] [[package]] @@ -2816,23 +2818,23 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version < '3.12'" }, - { name = "babel", marker = "python_full_version < '3.12'" }, - { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.12'" }, - { name = "imagesize", marker = "python_full_version < '3.12'" }, - { name = "jinja2", marker = "python_full_version < '3.12'" }, - { name = "packaging", marker = "python_full_version < '3.12'" }, - { name = "pygments", marker = "python_full_version < '3.12'" }, - { name = "requests", marker = "python_full_version < '3.12'" }, - { name = "roman-numerals", marker = "python_full_version < '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -2847,23 +2849,23 @@ resolution-markers = [ "python_full_version >= '3.12'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version >= '3.12'" }, - { name = "babel", marker = "python_full_version >= '3.12'" }, - { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.12'" }, - { name = "imagesize", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pygments", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ From 3e50dccbdf5c559d5d7250acaad16f7bc684dad8 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 30 Oct 2026 17:07:03 +0000 Subject: [PATCH 03/25] WIP: readable formats keyed by device, changeable at runtime, stored in settings Split cut G1 (#1400) of detector-rederive (d0872e185) on split/F: StandardReadable's per-device format registry and per-verb hook registries, set/reset/get_readable_format(s), and storing the format changes with store_settings/apply_settings, with ADR 0020. The shadowing hook used only by the detector is left to the detector PR. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- CHANGELOG.md | 6 + .../decisions/0025-runtime-readable-format.md | 273 ++++++++ docs/how-to/change-readable-format.md | 87 +++ docs/how-to/store-and-retrieve.md | 23 + pyproject.toml | 1 + src/ophyd_async/core/__init__.py | 12 +- src/ophyd_async/core/_readable.py | 617 ++++++++++++++---- src/ophyd_async/core/_settings.py | 61 +- src/ophyd_async/plan_stubs/_settings.py | 82 ++- .../tango/core/_tango_transport.py | 4 + .../tango/core/test_yaml_save.yaml | 1 + tests/unit_tests/core/test_readable.py | 421 ++++++++---- tests/unit_tests/epics/pvi/test_pvi.py | 2 +- tests/unit_tests/plan_stubs/test_settings.py | 442 ++++++++++++- .../test_data/test_yaml_config_save.yaml | 1 + .../unit_tests/test_data/test_yaml_save.yaml | 1 + 16 files changed, 1772 insertions(+), 262 deletions(-) create mode 100644 docs/explanations/decisions/0025-runtime-readable-format.md create mode 100644 docs/how-to/change-readable-format.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a490d700bf..c86ec7b930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ - The progress of `StandardFlyable.prepare()` now comes from the `WatcherUpdate`s the logic's `prepare_to_fly` yields, replacing the heuristic that watched setpoint writes (#1448). - `WatchableAsyncStatus` closes the async generator it drives as soon as it is cancelled or fails, rather than when the generator is collected, and a watcher that raises is logged and ignored rather than failing the status. - A `StandardFlyable` whose fly failed after `kickoff()` and was never completed logs a warning when it is unstaged or prepared again, instead of leaving the failure unretrieved. +- `StandardReadable` now keeps one format per registered child, keyed by the child, so registering a child again replaces its format instead of adding a second contribution, and the order of `hints` is the order children were first registered. The private `_describe_config_funcs`, `_read_config_funcs` and `_has_hints` tuples are gone: a subclass that added to them now adds a hook to `_describe_configuration_funcs`, `_read_configuration_funcs` or `_hints_funcs` (see ADR 0025). +- The deprecated `ConfigSignal` and `HintedSignal` aliases are removed from `ophyd_async.core`; use `StandardReadableFormat.CONFIG_SIGNAL`, `HINTED_SIGNAL` or `HINTED_UNCACHED_SIGNAL`. `StandardReadable.add_readables()` is deprecated in favour of `set_readable_format()` on each device. +- `store_settings()` writes the readable format changes of the Device tree under the reserved key ``, and `apply_settings()` resets the tree to its class formats before applying them. Settings files stored before this have no such key and leave formats alone. `retrieve_settings()` warns about and drops any other `<...>` key. ### Added @@ -38,6 +41,9 @@ - `StoppedError`, the `RuntimeError` raised when `stop()` or a new `set()` ends a move, prepare or fly early. - `Logic`, the base of `MovableLogic` and `FlyableLogic`, with the abstract `ensure_stopped()`. `FlyableLogic` adds `ensure_ready()`, which defaults to it. - An exception raised by a hook during `stage()`, `unstage()`, `prepare()` or a fly of a Standard device carries the note `raised by `. +- `StandardReadable.set_readable_format()`, `reset_readable_formats()`, `get_readable_formats()` and `get_readable_format_changes()`, to change how a child contributes to `read()`, `read_configuration()` and `hints` at runtime, the equivalent of `Kind` in ophyd v1 (see "How to change what a Device reports at runtime"). +- `ReadableFormats`, `walk_readable_formats()`, `apply_readable_formats()` and `READABLE_FORMATS_KEY` in `ophyd_async.core`, and a `readable_formats` field on `Settings`, so formats are stored and retrieved with the signal values. +- pyright now runs with `reportMatchNotExhaustive`. ### Fixed diff --git a/docs/explanations/decisions/0025-runtime-readable-format.md b/docs/explanations/decisions/0025-runtime-readable-format.md new file mode 100644 index 0000000000..036dfe8f0e --- /dev/null +++ b/docs/explanations/decisions/0025-runtime-readable-format.md @@ -0,0 +1,273 @@ +# 25. Make readable format a runtime setting keyed by Device + +Date: 2026-08-18 + +## Status + +Accepted + +## Context + +In ophyd v1 a `Component`'s `kind` attribute could be changed at runtime, so a beamline +could reuse one Device across techniques: a monochromator's energy signal is +configuration for a fixed-energy technique and a hinted read for one that scans it. + +`StandardReadable` had no equivalent. `add_readables(devices, format)` appended bound +methods into four tuples: + +```python +case StandardReadableFormat.CONFIG_SIGNAL: + self._describe_config_funcs += (signal.describe,) + self._read_config_funcs += (signal.read,) +``` + +Once appended, the tuples no longer recorded which device a contribution came from, so +nothing could be found, removed or re-formatted afterwards. The only ways to change what +a device reported were to edit its class definition or to keep two near-identical Device +classes differing in one signal's format. This is issue #1394. + +The same gap showed up from the other direction in #1395. `StandardDetector` did not +inherit `StandardReadable` and had its own `add_config_signals` instead, so detectors and +every other Device disagreed about how a signal is nominated for `read_configuration()`. +Reviewers of that PR were unwilling to have areaDetector plugins registered automatically, +because a detector typically carries far more plugins (16 ROIs, 16 stats) than are wired +into its chain, and reporting all of them gives no way to tell which values are +meaningful. Per-plugin opt-in needs a per-device call that can be made after construction +— which is the same primitive #1394 asks for. + +Three designs were considered. + +**Chain parsing.** Inspect the areaDetector plugin graph and register what is actually +wired up. Rejected as too large, and it breaks down once gather/scatter and PVA transfer +plugins are used between IOCs, since it requires every plugin to be exposed to ophyd. + +**A single pluggable registry** holding both formats and logic objects, so that +`StandardReadable`, `StandardMovable`, `StandardFlyable` and `StandardDetector` all +register their behaviour the same way. Rejected: a format entry is keyed by child, +unbounded in number and merged by union, whereas a logic object is keyed by role, at most +one per role, and conflicts are an error. A single registry would have to branch on entry +kind at every insert. Runtime mutation also means different things for the two — changing +a format alters the contents of `describe()`, while changing a data logic alters which +bluesky protocols the Device satisfies. Finally the declarative annotation form only fits +formats: `A[SignalRW[float], PvSuffix.rbv("AcquireTime"), Format.CONFIG_SIGNAL]` works +because a format needs exactly `(parent, child)`, whereas `ADHDFDataLogic` needs a prefix, +a driver, a plugin list and a path provider. + +**Key the format registry by device**, and leave logic objects alone. Chosen. + +## Decision + +### The registry is a `{device: format}` dict + +`StandardReadable._readables` is a `MutableMapping[Device, StandardReadableFormat]` that +maps each registered child to its format, in registration order, and the callables for +each verb are derived on demand. Its `__setitem__` validates, as `Settings` does for its +keys: a signal format is only accepted for a `SignalR`, so the verbs can match on +`(signal, format)` entries already typed. `add_readables` becomes a thin wrapper over a +new `set_readable_format(device, format)`, and `format=None` removes the device. +`get_readable_formats()` returns a copy. + +Keying by `Device` is identity keying, which is what a registry of children needs: `Device` +does not override `__eq__`, and while `DeviceVector` and `DeviceMap` inherit a value based +`__eq__` from `Mapping`, they hash by `id()` — injective over live objects, so two distinct +devices never share a hash and `__eq__` is never reached. + +A tuple of `(device, format)` pairs was tried first, for immutability. It made +`set_readable_format` rebuild and rescan the whole tuple, so registering *n* children was +O(n²) — 2000 signals cost 77 ms at construction, against 13 ms for the dict. The registry +is a `@cached_property`, matching `Device._child_devices`, because `AreaDetector` and +`add_children_as_readables` register children *before* calling `super().__init__()`, so +anything assigned in `__init__` would discard them. + +Deriving on demand also fixes staging: `stage()` walks the current registry rather than a +tuple captured at registration, so a format changed between runs takes effect on the next +`stage()`. + +Registration is "set" rather than "add": registering the same device twice replaces its +format rather than contributing it twice. This is a behaviour change, and the previous +behaviour was not deliberate. + +The deprecated `ConfigSignal` and `HintedSignal` markers are removed rather than carried +across. They worked by comparing equal to their target, which relied on the `match` +statement running during `add_readables`; deriving the callables on demand moves that +comparison to the first `read()` instead. Rather than keep a shim alive to preserve where +the warning fires, they go — they have been deprecated since well before this change. + +### Runtime changes can be undone, and the baseline is sealed by a metaclass + +@oliwenmandiamond raised that a class declaring `sig_a` as hinted, then having that +changed at runtime, makes the class definition misleading. Two safeguards were proposed: +making class-declared formats immutable unless opted out, or a separate +`StandardDynamicReadable`. + +Both were rejected, because static-by-default blocks the case this ADR exists for: #1394's +motivating example is a monochromator whose `energy` is *declared in the class* as +configuration and which one technique wants hinted. Locking it precisely because it was +declared would mean a beamline had to get the Device class changed to switch technique, and +the "allow dynamic" decision would be made by whoever is furthest from the technique. +@jwlodek made the same call from the other side: ophyd v1 has always been mutable, v1 +codebases at NSLS-II make extensive use of changing `kind` at runtime, and behaviour +differences between the two libraries are a cost they do not want. + +What was adopted instead is @oliwenmandiamond's and @jwlodek's counter-proposal: +[](#StandardReadable.reset_readable_formats) puts a Device back to what its class declared, +so a plan that retunes a Device can undo it without recording what it changed. + +**The baseline is sealed by a metaclass**, not at the end of `StandardReadable.__init__`. +Formats can be declared in three places: annotations, applied inside `Device.__init__`; +registration a subclass does *before* its `super().__init__()`, which is what all 24 +in-tree Devices do; and registration it does *after*, which nothing in tree does but which +is legal. `type.__call__` runs the whole `__init__` chain before returning, and is the only +point after all three. Snapshotting at the end of `StandardReadable.__init__` was measured +to miss the third, and the symptom would be `reset_readable_formats` silently dropping a +child the class had declared. `Device` already has a metaclass (`_ProtocolMeta`, from the +`HasName` Protocol), so this adds no new machinery to the hierarchy. + +The reset is per-Device rather than per-tree, pairing with `set_readable_format` rather +than with `apply_readable_formats`, and its baseline is the class declaration rather than +the last applied settings file — so it discards a stored technique too. + +### Format changes apply between runs, not within one + +A run's descriptor is emitted at its start, and `HINTED_SIGNAL` sets up monitoring in +`stage()`. Changing a format part way through a run would make `describe()` and `read()` +disagree. This is documented rather than enforced: the check would have to know about run +boundaries, which a Device does not. + +### `StandardDetector` inherits `StandardReadable` + +A detector's step-scan signals are registered exactly like any other Device's, and +`read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. +`add_config_signals` becomes a deprecated wrapper. + +The detector does not *reimplement* any of those verbs to achieve it. Each verb gathers a +tuple of callables — `_read_funcs`, `_describe_funcs` and friends — exactly as `stage()` +gathers `_stage_funcs`. `StandardReadable` puts the registry's contribution in each tuple, +and `StandardDetector` appends what its data logics produce, the way `StandardFlyable` +already appends to `_stage_funcs` rather than replacing `stage()`. Overriding the verbs +meant re-deriving the registered children's contributions by hand in each one, which is +exactly what would drift. Everything in a tuple runs in parallel. + +One consequence: a detector with no hinted fields now reports `hints` as `{}` rather than +`{"fields": []}`, because it no longer special cases what `StandardReadable` already does. + +Plugins are **not** registered automatically. `AreaDetector` registers only the driver's +`acquire_time` and `acquire_period`, plus whatever the caller passes as `config_sigs`. +Registering a plugin is a `set_readable_format()` call at the call site, which is the +explicit opt-in the #1395 reviewers asked for, now expressible per plugin and reversible +at runtime. + +### The `prepare_single` data logic tier goes + +[](./0012-detector-rewrite.md) gave `DetectorDataLogic` a `prepare_single` tier, returning a +`ReadableDataProvider`, so that a source producing one value per event could reach +`read()` and `describe()`. Registering a signal now does that, for any Device and without +a data logic, so the tier has nothing left of its own: its only in-tree implementation was +`PluginSignalDataLogic`, wrapping a driver and one `SignalR`, and that is a +`set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL)` call. + +`prepare_single`, `ReadableDataProvider` and `SignalDataProvider` are therefore deleted +rather than kept alongside. Keeping them would leave two ways to get a scalar into an +event, one of which drops itself with a warning as soon as the scan asks for more than one +collection, while the other simply works. + +### Formats are stored in the same file as settings, under a reserved key + +Values and formats are applied on the same cadence — a technique change wants both — so +they belong in one file and one `apply_settings` call. `store_settings` keeps values flat +and adds one reserved key holding the format changes since construction: + +```yaml +energy: 7.0 +temperature: 20.0 +: + : + energy: HINTED_SIGNAL + temperature: null +``` + +`Settings` gains a `readable_formats` attribute, so one `retrieve_settings` returns the +whole stored state. + +**No version marker is needed.** A file written before formats existed simply lacks the +reserved key, which means "change no formats". Each part being independently optional also +makes a hand-written formats-only profile work, changing technique without writing a value +to hardware. + +**The reserved keys cannot collide.** `` starts with `<`, which cannot +begin a Python identifier, so no attribute assignment can produce a colliding path — no +store-time guard is required. `<` is also not a YAML indicator, so unlike `*FORMATS*` or +`%FORMATS%` the key needs no quoting. `` replaces `""` as the root owner for the same +reason and because an empty-string key makes PyYAML emit its hard-to-read explicit +`? '' :` form. + +Because no path can look like `<...>`, `retrieve_settings` drops every such key it does +not understand with a warning, rather than reporting it as an unknown signal. A key written +by a newer version is then ignored by an older one, and a typo of a reserved key is still +noticed. + +**The formats section is two levels while values are flat.** A value is intrinsic to one +path; a format is a fact about an (owner, child) *pair*, and the same child can be +registered on two owners with different formats — which #1395 creates directly, since +`NDStatsIO` will declare `total` on itself while a detector may also register `stats.total`. +Recording the owner rather than inferring it also means the layout needs no "must be a +direct child" rule, so `AreaDetector` can keep registering `driver.acquire_time` and +`DeviceVector` elements keep working, with no dependency on #1395 and no second migration +later. + +The child path is relative to its owner, not to the root, so the owner is not repeated on +every line. A survey of dodal, haven and the NSLS-II repositories found that every +registration of something outside its owner's subtree is of an object outside the stored +Device altogether (an `__init__` argument or a `Reference`), which a root-relative path +cannot store either. + +### Only changes are stored, and applying resets to the class defaults first + +`walk_readable_formats` records only how each `StandardReadable` differs from what it +registered during construction, with a dropped child as `None`. `apply_readable_formats` +resets every `StandardReadable` in the tree to its class defaults, then applies those +changes. The key is written, even if empty, whenever the tree has a `StandardReadable`, as +an empty entry means "back to the defaults", where a missing key means "leave alone". + +Two alternatives were rejected: + +- **Store everything, apply by replacing.** Store a file, add a signal to the Device in a + later version, apply the old file, and the new signal is silently unregistered. It also + pins the class defaults of the day into every file, so a release that changes a default + is undone by old files. +- **Store everything, apply by merging.** Fixes the new-signal problem but cannot return + from technique B to technique A: anything B changed that A does not mention stays as B + left it, and dropping a child needs a hand-written null. + +Storing changes against the class defaults avoids both: a new signal keeps its class +format, defaults stay current, and a dropped child is written as a null automatically. +Registrations made in `__init__` are part of the defaults, so one of an object outside the +stored tree is never written, and does not warn on every store. + +`apply_readable_formats` resolves every path before changing anything, so a bad path cannot +leave the tree half applied. + +`Settings.partition` copies the formats to **both** halves. A device-specific apply plan +normally applies only the halves and never the original `Settings` — `apply_panda_settings` +partitions and calls `apply_settings` twice — so putting formats on one half would let the +plan silently decide whether they were restored at all. Applying them twice is harmless: it +touches no hardware and gives the same result. `apply_settings` applies formats after +values, so a failed value write leaves the readable registry untouched. + +## Consequences + +- A Device can be switched between techniques at runtime, and the switch can be saved and + restored, closing #1394. +- `add_readables` keeps working unchanged for all 85 in-tree call sites and for downstream + Devices, but gains "set" semantics. Having gained them it is no longer adding anything, + so it has since been deprecated in favour of `set_readable_format`, which says what it + does; it still works, with a `DeprecationWarning`. +- Tests that asserted on `_read_funcs`, `_has_hints` and friends had to be rewritten onto + the public verbs. Those assertions were already against the testing conventions in + `CLAUDE.md`, and the derived-on-demand design makes them impossible rather than merely + discouraged. +- Deriving callables per call costs a pass over the registry on every verb. The registries + are small (tens of entries), and it is what makes the format mutable. +- Registering a device that is not below its owner still works, but a runtime change to + its format cannot be stored, since there is no stable path for it. + `walk_readable_formats` warns and skips. diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md new file mode 100644 index 0000000000..3420830eed --- /dev/null +++ b/docs/how-to/change-readable-format.md @@ -0,0 +1,87 @@ +# How to change what a Device reports at runtime + +A [](#StandardReadable) decides what appears in `read()`, `read_configuration()` and +`hints` from the [](#StandardReadableFormat) each of its children is registered with. +Normally that is fixed when the Device class is written, but it can also be changed on a +live Device with [](#StandardReadable.set_readable_format). + +This is the equivalent of changing a `Component`'s `kind` in ophyd v1. + +## Changing one signal + +A beamline with two techniques might scan a monochromator's energy in one and hold it +fixed in the other. The same Device serves both: + +```python +from ophyd_async.core import StandardReadableFormat as Format + +# Scanning technique: energy is a hinted read +mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + +# Fixed technique: energy is configuration, recorded once per run +mono.set_readable_format(mono.energy, Format.CONFIG_SIGNAL) + +# Not interesting for this technique at all +mono.set_readable_format(mono.energy, None) +``` + +[](#StandardReadable.get_readable_formats) reports what is currently registered, keyed by +child Device, so a child that does not contribute is simply absent: + +```python +mono.get_readable_formats().get(mono.energy) +``` + +## Putting it back + +[](#StandardReadable.reset_readable_formats) undoes every runtime change, back to +how the class declared things — annotations, `add_children_as_readables`, and +anything the Device's own `__init__` registered: + +```python +mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) # retune for a technique +... +mono.reset_readable_formats() # back to the class defaults +``` + +So a plan that retunes a Device can put it back without having to record what it +changed. It resets that Device only, not its children — to do a whole tree: + +```python +from ophyd_async.core import apply_readable_formats + +apply_readable_formats(detector, {}) +``` + +```{note} +The baseline is what the Device class declares, so this also discards formats that came +from a stored settings file — see [](./store-and-retrieve.md). +``` + +```{note} +Change formats **between runs**, not inside one. A run's descriptor is emitted when the +run starts, and [](#StandardReadableFormat.HINTED_SIGNAL) begins monitoring in `stage()`, +so a format changed part way through a run would leave `describe()` and `read()` +disagreeing. +``` + +## Opting a detector plugin in + +A [](#StandardDetector) is a [](#StandardReadable), so the same call works on one. This +matters most for areaDetector, where a detector often carries many more plugins than are +wired into its chain — so plugins are never registered automatically, and you opt in to +the ones that are actually producing meaningful values: + +```python +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) +``` + +Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can +write a file *and* report a plugin scalar in a step scan. + +```{seealso} +[](./store-and-retrieve.md) for saving a set of formats alongside signal values and +loading them back, and +[](../explanations/decisions/0025-runtime-readable-format.md) for why the registry is +keyed by Device. +``` diff --git a/docs/how-to/store-and-retrieve.md b/docs/how-to/store-and-retrieve.md index e91ec08dd5..53f080f314 100644 --- a/docs/how-to/store-and-retrieve.md +++ b/docs/how-to/store-and-retrieve.md @@ -43,3 +43,26 @@ def load_panda(panda1: HDFPanda): settings = yield from retrieve_settings(provider, "yaml_file_name", panda1) yield from apply_settings_if_different(settings, apply_panda_settings) ``` + +## Storing readable formats too + +As well as signal values, [](#store_settings) records how each child contributes to the Device's bluesky verbs — which signals are hinted, which are configuration — and [](#apply_settings) restores that alongside the values. The two change on the same cadence, so they live in one file and are applied by one call. This means a stored file can switch a Device between techniques as well as putting its values back; see [](./change-readable-format.md) for changing a format directly. + +Values stay flat in the yaml. Formats go under the reserved [](#READABLE_FORMATS_KEY), and only what has changed since the Device was constructed is recorded, so the file stays short and can be edited by hand: + +```yaml +energy: 7.0 +temperature: 20.0 +: + : + energy: HINTED_SIGNAL + temperature: null + stats: + total: HINTED_UNCACHED_SIGNAL +``` + +The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with `` standing for the Device you stored. The inner key is the child's path from that owner. The owner is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. A null means the child has been dropped. + +Applying first resets every Device in the tree to the formats its class declares, then applies the changes in the file. So a file stored with nothing changed (`: {}`) puts the class defaults back, and a signal added to a Device class after the file was stored keeps the format the class gives it. + +A file with no `` key changes no formats, so settings files saved before formats were storable keep working untouched. A file with *only* that key switches technique without writing a single value to hardware. diff --git a/pyproject.toml b/pyproject.toml index d9d7ca5652..849b961408 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ version_file = "src/ophyd_async/_version.py" [tool.pyright] typeCheckingMode = "standard" reportMissingImports = false # Ignore missing stubs in imported modules +reportMatchNotExhaustive = true [tool.pytest.ini_options] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index 0634602b62..e3a62e5a51 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -102,10 +102,12 @@ Watcher, ) from ._readable import ( - ConfigSignal, - HintedSignal, + READABLE_FORMATS_KEY, + ReadableFormats, StandardReadable, StandardReadableFormat, + apply_readable_formats, + walk_readable_formats, ) from ._settings import Settings, SettingsProvider from ._signal import ( @@ -282,6 +284,10 @@ def __getattr__(name): # Readable "StandardReadable", "StandardReadableFormat", + "ReadableFormats", + "walk_readable_formats", + "apply_readable_formats", + "READABLE_FORMATS_KEY", # Detector "DetectorTrigger", "TriggerInfo", @@ -346,8 +352,6 @@ def __getattr__(name): "DerivedSignalFactory", "merge_gathered_dicts", # Back compat - delete before 1.0 - "ConfigSignal", - "HintedSignal", # Standard enums "EnabledDisabled", "EnableDisable", diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 8b6d33eda4..0a862c76f5 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -1,15 +1,25 @@ +import asyncio +import itertools import warnings -from collections.abc import Awaitable, Callable, Generator, Sequence +from collections.abc import ( + Awaitable, + Callable, + Generator, + Iterator, + MutableMapping, + Sequence, +) from contextlib import contextmanager from enum import Enum -from typing import Any, cast +from functools import cached_property +from typing import Literal, cast from bluesky.protocols import HasHints, Hints, Reading from event_model import DataKey from ._device import Device, DeviceMap, DeviceVector from ._protocol import AsyncConfigurable, AsyncReadable, AsyncStageable -from ._signal import SignalR +from ._signal import SignalR, walk_devices from ._standard_base import _StandardBase from ._utils import merge_gathered_dicts @@ -49,38 +59,101 @@ class StandardReadableFormat(Enum): def __call__(self, parent: Device, child: Device): if not isinstance(parent, StandardReadable): raise TypeError(f"Expected parent to be StandardReadable, got {parent}") - parent.add_readables([child], self) + parent.set_readable_format(child, self) -# Back compat -class _WarningMatcher: - def __init__(self, name: str, target: StandardReadableFormat): - self._name = name - self._target = target +# Short alias for the case statements below +Format = StandardReadableFormat - def __eq__(self, value: object) -> bool: - warnings.warn( - DeprecationWarning( - f"Use `StandardReadableFormat.{self._target.name}` " - f"instead of `{self._name}`" - ), - stacklevel=2, - ) - return value == self._target +_SignalFormat = Literal[ + Format.CONFIG_SIGNAL, + Format.HINTED_SIGNAL, + Format.UNCACHED_SIGNAL, + Format.HINTED_UNCACHED_SIGNAL, +] +_ReadableEntry = tuple[Device, Literal[Format.CHILD]] | tuple[SignalR, _SignalFormat] -def _compat_format(name: str, target: StandardReadableFormat) -> StandardReadableFormat: - return cast(StandardReadableFormat, _WarningMatcher(name, target)) +class _Readables(MutableMapping[Device, StandardReadableFormat]): + """The registered children and their formats, in registration order. + Only a `SignalR` may be given a signal format, which is what lets + `entries()` hand each one to the verbs already typed. + """ -ConfigSignal = _compat_format("ConfigSignal", StandardReadableFormat.CONFIG_SIGNAL) -HintedSignal: Any = _compat_format("HintedSignal", StandardReadableFormat.HINTED_SIGNAL) -HintedSignal.uncached = _compat_format( - "HintedSignal.uncached", StandardReadableFormat.HINTED_UNCACHED_SIGNAL -) + def __init__(self) -> None: + self._formats: dict[Device, StandardReadableFormat] = {} + + def __getitem__(self, device: Device) -> StandardReadableFormat: + return self._formats[device] + + def __setitem__(self, device: Device, format: StandardReadableFormat) -> None: + if not isinstance(format, StandardReadableFormat): + raise TypeError(f"{format} is not a StandardReadableFormat") + match format: + case Format.CHILD: + pass + case ( + Format.CONFIG_SIGNAL + | Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL + ): + # Fails type checking unless these are exactly _SignalFormat, + # which is what makes the cast in entries() true + _: _SignalFormat = format + if not isinstance(device, SignalR): + raise TypeError(f"{device} is not a SignalR") + self._formats[device] = format + + def __delitem__(self, device: Device) -> None: + del self._formats[device] + + def __iter__(self) -> Iterator[Device]: + return iter(self._formats) + + def __len__(self) -> int: + return len(self._formats) + + def entries(self) -> list[_ReadableEntry]: + """Each child and its format, typed by what `__setitem__` allowed.""" + return cast(list[_ReadableEntry], list(self._formats.items())) + + +class _SealDefaultFormatsMeta(type(_StandardBase)): # type: ignore[misc] + """Record the formats a `StandardReadable` was constructed with. + + `type.__call__` runs `__new__` and then the *whole* `__init__` chain before + returning, which is the only point that is after every place a format can + be declared: + + - annotations, applied by `create_children_from_annotations` inside + `Device.__init__` + - registration a subclass does *before* calling `super().__init__()`, which + is what all 24 in-tree Devices do + - registration a subclass does *after* it, which nothing in tree does but + which is perfectly legal + + Snapshotting at the end of `StandardReadable.__init__` would silently miss + the third, and the symptom would be + [](#StandardReadable.reset_readable_formats) dropping a child the class had + declared. Devices are already built on a metaclass (`_ProtocolMeta`, from + the `HasName` Protocol), so this adds no new machinery to the hierarchy. + """ + + def __call__(cls, *args, **kwargs): + self = super().__call__(*args, **kwargs) + self._default_readables = dict(self._readables) + return self -class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints): +class StandardReadable( + _StandardBase, + AsyncReadable, + AsyncConfigurable, + HasHints, + metaclass=_SealDefaultFormatsMeta, +): """Device that provides selected child Device values in `read()`. Provides the ability for children to be registered to: @@ -89,24 +162,193 @@ class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints - Provide their value in `read_configuration()` and `describe_configuration() - Select a value to appear in `hints` - The behavior is customized with a [](#StandardReadableFormat) + The behavior is customized with a [](#StandardReadableFormat), which can be + changed at runtime with [](#StandardReadable.set_readable_format). """ - # These must be immutable types to avoid accidental sharing between - # different instances of the class - _describe_config_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () - _read_config_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () - _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + @cached_property + def _readables(self) -> _Readables: + """The registered children and their formats, in registration order. + + A `cached_property` rather than an attribute set in `__init__`, because + `add_children_as_readables` and `set_readable_format` are routinely + called *before* `super().__init__()` (e.g. `AreaDetector`), so anything + `__init__` assigned would discard them. It also keeps each instance's + registry its own, where a mutable class attribute would be shared. + + Keying by Device is identity keying, which is what a registry of + children needs: `Device` does not override `__eq__`, and while + `DeviceVector` and `DeviceMap` inherit a value based `__eq__` from + `Mapping`, they hash by `id()`. That is injective over live objects, so + two distinct devices never share a hash and `__eq__` is never reached. + """ + return _Readables() + + # The formats declared during construction, sealed by the metaclass once + # __init__ has fully returned. Only ever replaced, never mutated in place, + # so the shared class level default is safe. + _default_readables: dict[Device, StandardReadableFormat] = {} + + # What contributes to each verb, gathered in parallel by the verb methods. + # Immutable defaults to avoid accidental sharing between instances. A + # subclass producing data from somewhere other than a registered child adds + # to these rather than overriding the verbs, so registered children keep + # working; `StandardDetector` does it for its data logics. Each registry + # is named after the method that gathers it, and each hook added to it is + # `__on_`, name-mangled so mix-ins in one MRO never collide. _read_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () - _has_hints: tuple[HasHints, ...] = () + _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + _read_configuration_funcs: tuple[ + Callable[[], Awaitable[dict[str, Reading]]], ... + ] = () + _describe_configuration_funcs: tuple[ + Callable[[], Awaitable[dict[str, DataKey]]], ... + ] = () + _hints_funcs: tuple[Callable[[], Iterator[Hints]], ...] = () + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Each of these derives from _readables on every call rather than being + # registered up-front, so a format changed between runs is picked up on + # the next stage() or the next descriptor. + self._stage_funcs += (self.__on_stage,) + self._unstage_funcs += (self.__on_unstage,) + self._read_funcs += (self.__on_read,) + self._describe_funcs += (self.__on_describe,) + self._read_configuration_funcs += (self.__on_read_configuration,) + self._describe_configuration_funcs += (self.__on_describe_configuration,) + self._hints_funcs += (self.__on_hints,) + + def _signals_to_stage(self) -> Iterator[AsyncStageable]: + # HINTED_SIGNAL stages so that caching is set up and read() is fast; a + # CHILD stages because it may have staging of its own. The uncached and + # config formats deliberately do not. + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncStageable): + yield device + case (signal, Format.HINTED_SIGNAL): + yield signal + case ( + _, + Format.CONFIG_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass + + async def __on_stage(self) -> None: + await asyncio.gather(*(sig.stage().task for sig in self._signals_to_stage())) + + async def __on_unstage(self) -> None: + await asyncio.gather(*(sig.unstage().task for sig in self._signals_to_stage())) + + async def __on_describe_configuration(self) -> dict[str, DataKey]: + """What the registered children contribute to `describe_configuration()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncConfigurable): + coros.append(device.describe_configuration()) + case (signal, Format.CONFIG_SIGNAL): + coros.append(signal.describe()) + case ( + _, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass + return await merge_gathered_dicts(coros) + + async def __on_read_configuration(self) -> dict[str, Reading]: + """What the registered children contribute to `read_configuration()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncConfigurable): + coros.append(device.read_configuration()) + case (signal, Format.CONFIG_SIGNAL): + coros.append(signal.read()) + case ( + _, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass + return await merge_gathered_dicts(coros) + + async def __on_describe(self) -> dict[str, DataKey]: + """What the registered children contribute to `describe()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncReadable): + coros.append(device.describe()) + case ( + signal, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + coros.append(signal.describe()) + case (_, Format.CONFIG_SIGNAL): + pass + return await merge_gathered_dicts(coros) + + async def __on_read(self) -> dict[str, Reading]: + """What the registered children contribute to `read()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncReadable): + coros.append(device.read()) + case (signal, Format.HINTED_SIGNAL): + coros.append(signal.read()) + case ( + signal, + Format.UNCACHED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, + ): + coros.append(signal.read(cached=False)) + case (_, Format.CONFIG_SIGNAL): + pass + return await merge_gathered_dicts(coros) + + def __on_hints(self) -> Iterator[Hints]: + """What the registered children contribute to `hints`.""" + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, HasHints): + yield device.hints + case ( + signal, + Format.HINTED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, + ): + # An unnamed signal has nothing to put in `fields`, but is + # still hinted, so it contributes the key and no name + yield {"fields": [signal.name] if signal.name else []} + case ( + _, + Format.CONFIG_SIGNAL | Format.UNCACHED_SIGNAL, + ): + pass async def describe_configuration(self) -> dict[str, DataKey]: return await merge_gathered_dicts( - [func() for func in self._describe_config_funcs] + [func() for func in self._describe_configuration_funcs] ) async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts([func() for func in self._read_config_funcs]) + return await merge_gathered_dicts( + [func() for func in self._read_configuration_funcs] + ) async def describe(self) -> dict[str, DataKey]: return await merge_gathered_dicts([func() for func in self._describe_funcs]) @@ -122,14 +364,16 @@ def hints(self) -> Hints: flattened_dimensions = [] hints: Hints = {} - for new_hint in self._has_hints: + for new_hint in itertools.chain.from_iterable( + func() for func in self._hints_funcs + ): # If a hints key in a subdevice isn't given in bluesky protocols, then it # doesn't carry over to the device holding it. - for key, value in new_hint.hints.items(): + for key, value in new_hint.items(): if key in ("fields", "dimensions"): if isinstance(value, str) or not isinstance(value, Sequence): raise RuntimeError( - f"{new_hint.name}: '{key}' key must be a Sequence, " + f"{self.name}: '{key}' key must be a Sequence, " f"received {type(value)}" ) already_parsed = ( @@ -158,11 +402,11 @@ def add_children_as_readables( self, format: StandardReadableFormat = StandardReadableFormat.CHILD, ) -> Generator[None, None, None]: - """Context manager that calls [](#add_readables) on child Devices added within. + """Register child Devices added within with the given format. Scans `self.children()` on entry and exit to context manager, and calls - `add_readables()` on any that are added with the provided - `StandardReadableFormat`. + [](#StandardReadable.set_readable_format) on any that are added with the + provided `StandardReadableFormat`. """ dict_copy = dict(self.children()) @@ -193,9 +437,88 @@ def add_children_as_readables( else: flattened_values.append(value) - new_devices = list(filter(lambda x: isinstance(x, Device), flattened_values)) - self.add_readables(new_devices, format) + for device in flattened_values: + if isinstance(device, Device): + self.set_readable_format(device, format) + + def set_readable_format( + self, device: Device, format: StandardReadableFormat | None + ) -> None: + """Set how a child Device contributes to this Device's bluesky verbs. + + This is the runtime equivalent of the `Kind` attribute in ophyd v1: it + can be called after construction to move a child between configuration, + hinted and uncached reads, or to drop it from the verbs entirely. + + Call it **between runs**. The descriptor for a run is emitted at its + start, and [](#StandardReadableFormat.HINTED_SIGNAL) sets up caching in + `stage()`, so changing a format part way through a run would make + `describe()` and `read()` disagree. + + :param device: The Device to set the format of, normally a child of this one + :param format: + The format to give it, or `None` to stop it contributing at all. + Replaces any format the device already has, rather than adding a + second contribution. + :raises TypeError: If a signal-only format is given a non-`SignalR` + """ + if format is None: + self._readables.pop(device, None) + else: + # Re-formatting an already registered child keeps its position, so + # changing a format does not reorder `hints` + self._readables[device] = format + + def reset_readable_formats(self) -> None: + """Undo every runtime format change, back to how the class declared them. + + The baseline is everything registered by the time construction + finished: annotations, `add_children_as_readables`, and any + [](#StandardReadable.set_readable_format) an `__init__` made. Anything + done after that -- switching a child between config and hinted, + dropping one with a format of `None`, registering a new one -- is + undone. + + Intended for a Device that is retuned per technique, so a scan can put + it back without knowing what it changed. + + This resets **this Device only**, not its children. To reset a whole + tree use `apply_readable_formats(device, {})`. + + The baseline is the *class declaration*, not the last stored settings + file, so this discards a technique loaded by [](#apply_settings) too. + """ + # Mutated rather than replaced, so the cached_property stays the same + # object and nothing holding a reference sees a stale registry + self._readables.clear() + self._readables.update(self._default_readables) + + def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: + """Return the registered children and their formats, in registration order. + + A Device that does not contribute is absent, so + `get_readable_formats().get(child)` gives its format or `None`. + """ + return dict(self._readables) + + def get_readable_format_changes( + self, + ) -> dict[Device, StandardReadableFormat | None]: + """Return the formats that differ from those declared at construction. + A child registered at construction but since dropped has a format of `None`. + """ + changes: dict[Device, StandardReadableFormat | None] = { + child: format + for child, format in self._readables.items() + if self._default_readables.get(child) is not format + } + for child in self._default_readables: + if child not in self._readables: + changes[child] = None + return changes + + # Back compat - delete before 1.0 def add_readables( self, devices: Sequence[Device], @@ -203,78 +526,148 @@ def add_readables( ) -> None: """Add devices to contribute to various bluesky verbs. - Use output from the given devices to contribute to the verbs of the following - interfaces: - - - [](#bluesky.protocols.Readable) - - [](#bluesky.protocols.Configurable) - - [](#bluesky.protocols.Stageable) - - [](#bluesky.protocols.HasHints) - :param devices: The devices to be added :param format: Determines which of the devices functions are added to which verb as per the [](#StandardReadableFormat) documentation """ - - def assert_device_is_signalr(device: Device) -> SignalR: - if not isinstance(device, SignalR): - raise TypeError(f"{device} is not a SignalR") - return device - + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(device, format)` on each device " + "instead of `add_readables(devices, format)`" + ), + stacklevel=2, + ) for device in devices: - match format: - case StandardReadableFormat.CHILD: - if isinstance(device, AsyncConfigurable): - self._describe_config_funcs += (device.describe_configuration,) - self._read_config_funcs += (device.read_configuration,) - if isinstance(device, AsyncReadable): - self._describe_funcs += (device.describe,) - self._read_funcs += (device.read,) - if isinstance(device, AsyncStageable): - self._stage_funcs += (device.stage,) - self._unstage_funcs += (device.unstage,) - if isinstance(device, HasHints): - self._has_hints += (device,) - case StandardReadableFormat.CONFIG_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_config_funcs += (signalr_device.describe,) - self._read_config_funcs += (signalr_device.read,) - case StandardReadableFormat.HINTED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (signalr_device.read,) - self._stage_funcs += (signalr_device.stage,) - self._unstage_funcs += (signalr_device.unstage,) - self._has_hints += (_HintsFromName(signalr_device),) - case StandardReadableFormat.UNCACHED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (_UncachedRead(signalr_device),) - case StandardReadableFormat.HINTED_UNCACHED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (_UncachedRead(signalr_device),) - self._has_hints += (_HintsFromName(signalr_device),) - - -class _UncachedRead: - def __init__(self, signal: SignalR) -> None: - self.signal = signal - - async def __call__(self) -> dict[str, Reading]: - return await self.signal.read(cached=False) - - -class _HintsFromName(HasHints): - def __init__(self, device: Device) -> None: - self.device = device - - @property - def name(self) -> str: - return self.device.name - - @property - def hints(self) -> Hints: - fields = [self.name] if self.name else [] - return {"fields": fields} + self.set_readable_format(device, format) + + +#: Reserved key under which [](#store_settings) writes readable formats. Chosen +#: so that it cannot be a dotted attribute path: `<` cannot start a Python +#: identifier, so no attribute assignment can produce a colliding key, and +#: unlike `*FORMATS*` it needs no quoting in yaml. +READABLE_FORMATS_KEY = "" + +#: The key standing in for the root Device itself in a [](#ReadableFormats), +#: where every other key is a dotted attribute path. Not a valid Python +#: identifier, so no attribute assignment can produce a colliding path, and +#: unlike `""` it does not make yaml fall back to its hard-to-read explicit +#: key syntax. +ROOT_DEVICE_KEY = "" + +#: The changes to the readable formats of a Device tree since construction, in +#: a form that can be stored and retrieved: +#: `{path of the StandardReadable: {path of the child: format}}`, where a format +#: of `None` means "stop this child contributing". +#: The owner path is a dotted attribute path from the root Device, as produced +#: by [](#walk_devices), with `ROOT_DEVICE_KEY` meaning the root Device +#: itself. The child path is a dotted attribute path from the owner. +#: +#: The outer key is needed because a format is a fact about an (owner, child) +#: *pair*, not about a single Device: the same child can be registered on more +#: than one `StandardReadable` with a different format each time. +ReadableFormats = dict[str, dict[str, StandardReadableFormat | None]] + + +def _paths_to_devices(device: Device) -> dict[str, Device]: + return {ROOT_DEVICE_KEY: device, **walk_devices(device)} + + +def walk_readable_formats(device: Device) -> ReadableFormats | None: + """Retrieve how the readable formats in a Device tree differ from construction. + + Used as part of saving a Device, so that which signals are hinted or + config can be restored later. Only the differences from what each + `StandardReadable` registered during construction are returned, with a + child it has since dropped given a format of `None`. + + :param device: The root Device to walk. + :return: + A [](#ReadableFormats) suitable for storing, or `None` if there is no + `StandardReadable` in the tree. + """ + paths_to_devices = _paths_to_devices(device) + readables = { + path: dev + for path, dev in paths_to_devices.items() + if isinstance(dev, StandardReadable) + } + if not readables: + return None + formats: ReadableFormats = {} + for path, dev in readables.items(): + device_to_path = {child: p for p, child in walk_devices(dev).items()} + entries: dict[str, StandardReadableFormat | None] = {} + for child, format in dev.get_readable_format_changes().items(): + child_path = device_to_path.get(child) + if child_path is None: + # Not below its owner, so it has no stable path to store it + # against. See https://github.com/bluesky/ophyd-async/issues/1402 + warnings.warn( + f"{dev.name or dev}: {child.name or child} is not within " + f"{dev.name or dev}, so the change to its readable format " + "will not be stored", + stacklevel=2, + ) + else: + entries[child_path] = format + if entries: + formats[path] = entries + return formats + + +def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: + """Reset the readable formats in a Device tree, then apply the given changes. + + Every `StandardReadable` in the tree is first put back to what it + registered during construction, as + [](#StandardReadable.reset_readable_formats) does, then the changes in + `formats` are applied on top. So an empty `formats` restores the class + defaults, and a child added to a Device since `formats` was stored keeps + the format its class gives it. + + To stop a child contributing, give it a format of `None`, which in a stored + file is a null: + + ```yaml + : + : + energy: HINTED_SIGNAL + temperature: null + ``` + + :param device: The root Device the owner paths in `formats` are relative to. + :param formats: A [](#ReadableFormats), e.g. from [](#walk_readable_formats). + :raises KeyError: If a path does not name a Device in the tree. + :raises TypeError: If an owner path does not name a `StandardReadable`. + """ + devices = _paths_to_devices(device) + resolved: list[ + tuple[StandardReadable, list[tuple[Device, StandardReadableFormat | None]]] + ] = [] + # Resolve everything before changing anything, so a bad path cannot leave + # the tree half applied + for owner_path, entries in formats.items(): + owner = devices.get(owner_path) + if owner is None: + raise KeyError(f"No Device at {owner_path!r} in {device.name or device}") + if not isinstance(owner, StandardReadable): + raise TypeError(f"{owner_path!r} is not a StandardReadable, got {owner}") + children = walk_devices(owner) + readables: list[tuple[Device, StandardReadableFormat | None]] = [] + for child_path, format in entries.items(): + child = children.get(child_path) + if child is None: + raise KeyError( + f"No Device at {child_path!r} in {owner.name or owner_path}" + ) + readables.append( + (child, None if format is None else StandardReadableFormat(format)) + ) + resolved.append((owner, readables)) + for dev in devices.values(): + if isinstance(dev, StandardReadable): + dev.reset_readable_formats() + for owner, readables in resolved: + for child, format in readables: + owner.set_readable_format(child, format) diff --git a/src/ophyd_async/core/_settings.py b/src/ophyd_async/core/_settings.py index 4edc87b2e5..41ef871e54 100644 --- a/src/ophyd_async/core/_settings.py +++ b/src/ophyd_async/core/_settings.py @@ -5,6 +5,7 @@ from typing import Any, Generic from ._device import Device, DeviceT +from ._readable import ReadableFormats from ._signal import SignalRW from ._signal_backend import SignalDatatypeT @@ -33,11 +34,26 @@ class Settings(MutableMapping[SignalRW[Any], Any], Generic[DeviceT]): """ def __init__( - self, device: DeviceT, settings: MutableMapping[SignalRW, Any] | None = None + self, + device: DeviceT, + settings: MutableMapping[SignalRW, Any] | None = None, + readable_formats: ReadableFormats | None = None, ): self.device = device self._settings = {} self.update(settings or {}) + self.readable_formats: ReadableFormats | None = ( + None + if readable_formats is None + else {owner: dict(entries) for owner, entries in readable_formats.items()} + ) + """How the Device's children contribute to its bluesky verbs. + + Stored and applied alongside the signal values, as the two change on + the same cadence. Applied by [](#apply_settings) with + [](#apply_readable_formats), so a dict (even an empty one) first resets + the tree to its class defaults, while `None` leaves formats alone. + """ def __getitem__(self, key: SignalRW[SignalDatatypeT]) -> SignalDatatypeT: return self._settings[key] @@ -70,10 +86,37 @@ def __len__(self) -> int: return len(self._settings) def __or__(self, other: MutableMapping[SignalRW, Any]) -> Settings[DeviceT]: - """Create a new Settings that is the union of self overridden by other.""" - if isinstance(other, Settings) and not self._is_in_device(other.device): - raise ValueError(f"{other.device} is not a child of {self.device}") - return Settings(self.device, self._settings | dict(other)) + """Create a new Settings that is the union of self overridden by other. + + Readable formats merge per (owner, child), the same granularity that + values merge at, so `other` only replaces the children it mentions. + + :raises ValueError: + If `other` is a `Settings` for a Device outside this one, or for a + Device below this one while carrying readable formats. + """ + formats = self.readable_formats + if isinstance(other, Settings): + if other.device is not self.device: + if not self._is_in_device(other.device): + msg = f"{other.device} is not a child of {self.device}" + raise ValueError(msg) + if other.readable_formats is not None: + # Re-rooting the owner paths is easy, but applying resets + # the whole tree, so a sub-device's formats would reset + # Devices it knows nothing about + msg = ( + f"Cannot merge readable formats from {other.device}, whose " + f"paths are relative to it rather than to {self.device}" + ) + raise ValueError(msg) + if other.readable_formats is not None: + formats = { + owner: dict(entries) for owner, entries in (formats or {}).items() + } + for owner, entries in other.readable_formats.items(): + formats.setdefault(owner, {}).update(entries) + return Settings(self.device, self._settings | dict(other), formats) def partition( self, predicate: Callable[[SignalRW], bool] @@ -88,13 +131,19 @@ def partition( The first contains the signals for which the predicate returned True, and the second contains the signals for which the predicate returned False. + Both halves carry the *same* `readable_formats`, as a device specific + apply plan normally applies only the halves (see + `apply_panda_settings`). Applying them twice is harmless: it writes to + no hardware and gives the same result. + :example: ```python settings = Settings(device, {device.special: 1, device.sig: 2}) specials, others = settings.partition(lambda sig: "special" in sig.name) ``` """ - where_true, where_false = Settings(self.device), Settings(self.device) + where_true = Settings(self.device, readable_formats=self.readable_formats) + where_false = Settings(self.device, readable_formats=self.readable_formats) for signal, value in self.items(): dest = where_true if predicate(signal) else where_false dest[signal] = value diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 5e75c6eb83..638d6a9536 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import warnings from collections.abc import Callable, Mapping from typing import Any @@ -9,12 +10,17 @@ from bluesky.utils import MsgGenerator, plan from ophyd_async.core import ( + READABLE_FORMATS_KEY, Device, + ReadableFormats, Settings, SettingsProvider, SignalRW, + StandardReadableFormat, Table, + apply_readable_formats, walk_config_signals, + walk_readable_formats, walk_rw_signals, ) @@ -53,21 +59,39 @@ def get_current_settings( def store_settings( provider: SettingsProvider, name: str, device: Device, only_config: bool = False ) -> MsgGenerator[None]: - """Walk a Device for SignalRWs and store their values. + """Walk a Device for SignalRWs and store their values and readable formats. If `only_config` is True, store only configuration settings on `Configurable`. + Readable formats are stored in both cases, since they are what *determines* + which signals are configuration. + + Values are stored flat, keyed by dotted attribute path. The changes to + readable formats since construction go under the reserved + [](#READABLE_FORMATS_KEY), which is not a valid Python identifier and so + cannot collide with a path. It is written, even if empty, whenever the tree + contains a `StandardReadable`, as applying it resets formats to the class + defaults. :param provider: The provider to store the settings with. :param name: The name to store the settings under. :param device: The Device to walk for SignalRWs. - :param only_config: If True, store only configuration settings. + :param only_config: If True, store only configuration signal values. """ if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: signals = walk_rw_signals(device) - named_values = yield from _get_values_of_signals(signals) - yield from wait_for_awaitable(provider.store(name, named_values)) + data: dict[str, Any] = yield from _get_values_of_signals(signals) + formats = walk_readable_formats(device) + if formats is not None: + data[READABLE_FORMATS_KEY] = { + owner: { + child: None if format is None else format.value + for child, format in entries.items() + } + for owner, entries in formats.items() + } + yield from wait_for_awaitable(provider.store(name, data)) @plan @@ -78,26 +102,64 @@ def retrieve_settings( If `only_config` is True, retrieve only configuration settings on `Configurable`. + A file with no [](#READABLE_FORMATS_KEY), such as one stored before + readable formats existed, gives Settings that leave formats alone. + :param provider: The provider to retrieve the settings from. :param name: The name of the settings to retrieve. :param device: The Device to retrieve the settings for. :param only_config: If True, retrieve only configuration settings. """ - named_values = yield from wait_for_awaitable(provider.retrieve(name)) + # Copy, as the provider may hand back a dict it holds on to and the reserved + # keys are popped out of it below + data = dict((yield from wait_for_awaitable(provider.retrieve(name)))) + # Reserved keys first, so what is left is signal paths. Any other + # key is dropped with a warning: it is either a typo of a reserved key or + # written by a newer version, and a path can never look like that. + # Present but null, as a hand written ":" would be, still + # means reset to the class defaults + stored_formats = ( + data.pop(READABLE_FORMATS_KEY) or {} if READABLE_FORMATS_KEY in data else None + ) + unknown_reserved = sorted(k for k in data if k.startswith("<") and k.endswith(">")) + for key in unknown_reserved: + del data[key] + if unknown_reserved: + warnings.warn( + f"Ignoring unknown reserved keys {unknown_reserved} in settings {name!r}", + stacklevel=2, + ) + readable_formats: ReadableFormats | None = ( + None + if stored_formats is None + else { + owner: { + # A null means "stop this child contributing", so is not coerced + child: None if format is None else StandardReadableFormat(format) + for child, format in entries.items() + } + for owner, entries in stored_formats.items() + } + ) if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: signals = walk_rw_signals(device) - unknown_names = set(named_values) - set(signals) + unknown_names = set(data) - set(signals) if unknown_names: raise NameError(f"Unknown signal names {sorted(unknown_names)}") - signal_values = {signals[name]: value for name, value in named_values.items()} - return Settings(device, signal_values) + signal_values = {signals[name]: value for name, value in data.items()} + return Settings(device, signal_values, readable_formats) @plan def apply_settings(settings: Settings) -> MsgGenerator[None]: - """Set every SignalRW to the given value in Settings. If value is None ignore it.""" + """Set every SignalRW to the given value in Settings. If value is None ignore it. + + Then apply any readable formats the Settings carries with + [](#apply_readable_formats). Formats go last because they touch no hardware + and cannot fail, so a failed value write leaves them untouched. + """ signal_values = { signal: value for signal, value in settings.items() if value is not None } @@ -105,6 +167,8 @@ def apply_settings(settings: Settings) -> MsgGenerator[None]: for signal, value in signal_values.items(): yield from bps.abs_set(signal, value, group="apply_settings") yield from bps.wait("apply_settings") + if settings.readable_formats is not None: + apply_readable_formats(settings.device, settings.readable_formats) @plan diff --git a/src/ophyd_async/tango/core/_tango_transport.py b/src/ophyd_async/tango/core/_tango_transport.py index 3585337884..d063458cce 100644 --- a/src/ophyd_async/tango/core/_tango_transport.py +++ b/src/ophyd_async/tango/core/_tango_transport.py @@ -752,6 +752,8 @@ def make_converter(info: AttributeConfig | CommandConfig, datatype) -> TangoConv AttrDataFormat.IMAGE, ]: return TangoDevStateArrayConverter() + case _: + pass else: # command info match info.in_type: case CmdArgType.DevState: @@ -768,6 +770,8 @@ def make_converter(info: AttributeConfig | CommandConfig, datatype) -> TangoConv return TangoLongStringTableConverter() case CmdArgType.DevVarDoubleStringArray: return TangoDoubleStringTableConverter() + case _: + pass # default case return trivial converter return TangoConverter() diff --git a/tests/system_tests/tango/core/test_yaml_save.yaml b/tests/system_tests/tango/core/test_yaml_save.yaml index c7103cfd3b..a6b0ac7f08 100644 --- a/tests/system_tests/tango/core/test_yaml_save.yaml +++ b/tests/system_tests/tango/core/test_yaml_save.yaml @@ -1,3 +1,4 @@ +: {} a_bool: true a_str: test_string float64: 1.234 diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index f6b0aafa54..19880ca953 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -4,14 +4,9 @@ from bluesky.protocols import HasHints from ophyd_async.core import ( - AsyncConfigurable, - AsyncReadable, - AsyncStageable, - ConfigSignal, Device, DeviceMap, DeviceVector, - HintedSignal, MockSignalBackend, SignalR, StandardReadable, @@ -22,11 +17,10 @@ from ophyd_async.core import StandardReadableFormat as Format -@pytest.mark.parametrize("wrapper", [HintedSignal, HintedSignal.uncached, ConfigSignal]) -def test_standard_readable_wrappers_raise_deprecation_warning(wrapper): +def test_standard_readable_rejects_a_non_format(): sr = StandardReadable() - with pytest.deprecated_call(): - sr.add_readables([soft_signal_rw(int)], wrapper) + with pytest.raises(TypeError, match="is not a StandardReadableFormat"): + sr.set_readable_format(soft_signal_rw(int), "CONFIG_SIGNAL") # type: ignore def test_standard_readable_hints(): @@ -43,7 +37,9 @@ def test_standard_readable_hints(): hint3 = MagicMock(spec=HasHints) hint3.hints = {"fields": ["jkl"], "gridding": "rectilinear_nonsequential"} - sr.add_readables([hint1, hint2, hint3]) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) + sr.set_readable_format(hint3, Format.CHILD) assert sr.hints == { "fields": ["abc", "def", "ghi", "jkl"], @@ -60,10 +56,8 @@ def test_standard_readable_hints_raises_when_overriding_string_literal(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"gridding": "a different string"} - sr._has_hints = ( - hint1, - hint2, - ) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) with pytest.raises(RuntimeError, match=r"Hint key .* may not be overridden"): sr.hints # noqa: B018 @@ -88,7 +82,7 @@ def test_incorrect_fields_and_dimensions_in_hints(sub_hints: dict, error: str): sr = StandardReadable() has_hints = MagicMock(spec=HasHints) has_hints.hints = sub_hints - sr._has_hints = (has_hints,) + sr.set_readable_format(has_hints, Format.CHILD) with pytest.raises(RuntimeError, match=error): sr.hints # noqa: B018 @@ -115,10 +109,8 @@ def test_standard_readable_arbitrary_hints(): "gridding": "rectilinear_nonsequential", } - sr._has_hints = ( - hint1, - hint2, - ) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) assert sr.hints == { "gridding": "rectilinear_nonsequential", "fields": ["field1", "field2"], @@ -135,10 +127,8 @@ def test_standard_readable_hints_raises_when_overriding_sequence(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"fields": ["field2"]} - sr._has_hints = ( - hint1, - hint2, - ) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) with pytest.raises(RuntimeError, match=r"Hint 'fields' .* override existing hint"): sr.hints # noqa: B018 @@ -146,17 +136,12 @@ def test_standard_readable_hints_raises_when_overriding_sequence(): def test_standard_readable_add_children_context_manager(): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock with sr.add_children_as_readables(): sr.a = MagicMock(spec=SignalR) sr.b = MagicMock(spec=SignalR) sr.c = MagicMock(spec=SignalR) - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == {sr.a, sr.b, sr.c} + assert sr.get_readable_formats() == dict.fromkeys((sr.a, sr.b, sr.c), Format.CHILD) @pytest.mark.parametrize( @@ -168,26 +153,19 @@ def test_standard_readable_add_children_context_manager(): ) def test_standard_readable_add_children_cm_device_with_mappings(device_type, keys): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock - # Create a mock for the DeviceVector/DeviceMap.children() call + # A DeviceVector/DeviceMap registers its values, not itself devices = [MagicMock(spec=SignalR) for _ in range(3)] device = device_type(dict(zip(keys, devices, strict=True))) with sr.add_children_as_readables(): sr.a = device - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == set(devices) + assert sr.get_readable_formats() == dict.fromkeys(devices, Format.CHILD) def test_standard_readable_add_children_cm_filters_non_devices(): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock with sr.add_children_as_readables(): sr.a = MagicMock(spec=SignalR) @@ -196,76 +174,73 @@ def test_standard_readable_add_children_cm_filters_non_devices(): sr.d = "abc" sr.e = MagicMock(spec=MockSignalBackend) - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == {sr.a, sr.b} - - -def assert_sr_has_attrs(sr: StandardReadable, expected_attrs: dict[str, tuple]): - attrs_to_check = ( - "_describe_config_funcs", - "_read_config_funcs", - "_describe_funcs", - "_read_funcs", - "_stage_funcs", - "_unstage_funcs", - "_has_hints", - ) - actual = {attr: getattr(sr, attr) for attr in attrs_to_check} - expected = {attr: expected_attrs.get(attr, ()) for attr in attrs_to_check} - assert actual == expected + # Only the Devices are registered + assert sr.get_readable_formats() == dict.fromkeys((sr.a, sr.b), Format.CHILD) -signal_r = MagicMock(spec=SignalR) -async_readable = MagicMock(spec=AsyncReadable) -async_configurable = MagicMock(spec=AsyncConfigurable) -async_stageable = MagicMock(spec=AsyncStageable) -has_hints = MagicMock(spec=HasHints) +async def assert_contributes( + sr: StandardReadable, + *, + read: set[str] = frozenset(), + config: set[str] = frozenset(), + hints: set[str] = frozenset(), +) -> None: + """Assert which keys a StandardReadable contributes to each verb.""" + assert set(await sr.read()) == set(read) + assert set(await sr.describe()) == set(read) + assert set(await sr.read_configuration()) == set(config) + assert set(await sr.describe_configuration()) == set(config) + assert set(sr.hints.get("fields", [])) == set(hints) @pytest.mark.parametrize( - "readable, expected_attrs", + "format, in_read, in_config, in_hints", [ - ( - signal_r, - { - "_read_funcs": (signal_r.read,), - "_describe_funcs": (signal_r.describe,), - "_stage_funcs": (signal_r.stage,), - "_unstage_funcs": (signal_r.unstage,), - }, - ), - ( - async_readable, - { - "_read_funcs": (async_readable.read,), - "_describe_funcs": (async_readable.describe,), - }, - ), - ( - async_configurable, - { - "_read_config_funcs": (async_configurable.read_configuration,), - "_describe_config_funcs": (async_configurable.describe_configuration,), - }, - ), - ( - async_stageable, - { - "_stage_funcs": (async_stageable.stage,), - "_unstage_funcs": (async_stageable.unstage,), - }, - ), - (has_hints, {"_has_hints": (has_hints,)}), + (Format.CONFIG_SIGNAL, False, True, False), + (Format.HINTED_SIGNAL, True, False, True), + (Format.UNCACHED_SIGNAL, True, False, False), + (Format.HINTED_UNCACHED_SIGNAL, True, False, True), ], ) -def test_standard_readable_add_readables_adds_to_expected_attrs( - readable, expected_attrs: dict[str, tuple] +async def test_standard_readable_signal_formats_route_to_expected_verbs( + format, in_read, in_config, in_hints ): - sr = StandardReadable() - sr.add_readables([readable]) - assert_sr_has_attrs(sr, expected_attrs) + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + sr.set_readable_format(sr.sig, format) + name = {sr.sig.name} + await assert_contributes( + sr, + read=name if in_read else set(), + config=name if in_config else set(), + hints=name if in_hints else set(), + ) + + +async def test_standard_readable_child_format_uses_child_verbs(): + inner = StandardReadable(name="inner") + inner.conf, _ = soft_signal_r_and_setter(int, 0, name="conf") + inner.val, _ = soft_signal_r_and_setter(int, 0, name="val") + inner.set_readable_format(inner.conf, Format.CONFIG_SIGNAL) + inner.set_readable_format(inner.val, Format.HINTED_SIGNAL) + + outer = StandardReadable(name="outer") + outer.inner = inner + outer.set_readable_format(inner, Format.CHILD) + + await assert_contributes( + outer, + read={inner.val.name}, + config={inner.conf.name}, + hints={inner.val.name}, + ) + + +async def test_standard_readable_child_format_ignores_plain_device(): + sr = StandardReadable(name="sr") + sr.child = Device(name="child") + sr.set_readable_format(sr.child, Format.CHILD) + await assert_contributes(sr) @pytest.mark.parametrize( @@ -284,43 +259,48 @@ def test_standard_readable_add_readables_raises_signalr_typeerror(format) -> Non # Ensure it raises TypeError with pytest.raises(TypeError, match=f"{mock_device} is not a SignalR"): - sr.add_readables([mock_device], format=format) + sr.set_readable_format(mock_device, format) -def test_standard_readable_config_signal(): - signal_r = MagicMock(spec=SignalR) +def test_add_readables_is_deprecated() -> None: sr = StandardReadable() - sr.add_readables([signal_r], Format.CONFIG_SIGNAL) - assert sr._describe_config_funcs == (signal_r.describe,) - assert sr._read_config_funcs == (signal_r.read,) + sig = soft_signal_rw(int) + with pytest.warns(DeprecationWarning, match="set_readable_format"): + sr.add_readables([sig], Format.CONFIG_SIGNAL) + # Still does the job it always did + assert sr.get_readable_formats() == {sig: Format.CONFIG_SIGNAL} -def test_standard_readable_hinted_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.HINTED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs == (signal_r.read,) - assert sr._stage_funcs == (signal_r.stage,) - assert sr._unstage_funcs == (signal_r.unstage,) - assert sr._has_hints[0].device == signal_r - - -def test_standard_readable_uncached_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.UNCACHED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs[0].signal == signal_r +@pytest.mark.parametrize( + "format, staged", + [ + (Format.HINTED_SIGNAL, True), + (Format.CHILD, True), + (Format.CONFIG_SIGNAL, False), + (Format.UNCACHED_SIGNAL, False), + (Format.HINTED_UNCACHED_SIGNAL, False), + ], +) +async def test_standard_readable_stage_starts_caching_only_for_cached_formats( + format, staged +): + # stage() exists to start monitoring, and reading cached raises if it isn't + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + await sr.sig.connect(mock=False) + sr.set_readable_format(sr.sig, format) + await sr.stage() + if staged: + assert await sr.sig.read(cached=True) + else: + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) -def test_standard_readable_hinted_uncached_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.HINTED_UNCACHED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs[0].signal == signal_r - assert sr._has_hints[0].device == signal_r + # unstage tears the monitor back down + await sr.unstage() + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) def test_standard_readable_add_children_multi_nested(): @@ -359,3 +339,186 @@ async def test_standard_readable_stage_twice_raises(): await sr.stage() await sr.unstage() await sr.stage() + + +async def test_set_readable_format_changes_verb_at_runtime(): + # The ophyd v1 `kind` use case from #1394: a signal that is config for one + # technique and hinted for another, without redefining the Device. + sr = StandardReadable(name="sr") + sr.energy, _ = soft_signal_r_and_setter(float, 0.0, name="energy") + + sr.set_readable_format(sr.energy, Format.CONFIG_SIGNAL) + await assert_contributes(sr, config={sr.energy.name}) + + sr.set_readable_format(sr.energy, Format.HINTED_SIGNAL) + await assert_contributes(sr, read={sr.energy.name}, hints={sr.energy.name}) + + sr.set_readable_format(sr.energy, None) + await assert_contributes(sr) + + +async def test_set_readable_format_replaces_rather_than_duplicates(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + + assert sr.hints == {"fields": [sr.sig.name]} + assert sr.get_readable_formats() == {sr.sig: Format.HINTED_SIGNAL} + + +async def test_get_readable_formats_omits_unregistered_children(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + sr.other, _ = soft_signal_r_and_setter(int, 0, name="other") + assert sr.get_readable_formats() == {} + sr.set_readable_format(sr.sig, Format.CONFIG_SIGNAL) + sr.set_readable_format(sr.other, Format.HINTED_SIGNAL) + # Registration order is preserved, and an unregistered child is absent + assert sr.get_readable_formats() == { + sr.sig: Format.CONFIG_SIGNAL, + sr.other: Format.HINTED_SIGNAL, + } + assert list(sr.get_readable_formats()) == [sr.sig, sr.other] + sr.set_readable_format(sr.sig, None) + assert sr.get_readable_formats() == {sr.other: Format.HINTED_SIGNAL} + + +async def test_reset_readable_formats_undoes_runtime_changes(): + class Dev(StandardReadable): + def __init__(self, name=""): + self.energy, _ = soft_signal_r_and_setter(float, 0.0, name="energy") + self.temp, _ = soft_signal_r_and_setter(float, 0.0, name="temp") + self.set_readable_format(self.energy, Format.CONFIG_SIGNAL) + self.set_readable_format(self.temp, Format.CONFIG_SIGNAL) + super().__init__(name=name) + + dev = Dev(name="dev") + dev.extra, _ = soft_signal_r_and_setter(float, 0.0, name="extra") + declared = dev.get_readable_formats() + + # Change one, drop one, add one that was never declared + dev.set_readable_format(dev.energy, Format.HINTED_SIGNAL) + dev.set_readable_format(dev.temp, None) + dev.set_readable_format(dev.extra, Format.HINTED_SIGNAL) + assert dev.get_readable_formats() == { + dev.energy: Format.HINTED_SIGNAL, + dev.extra: Format.HINTED_SIGNAL, + } + assert dev.get_readable_format_changes() == { + dev.energy: Format.HINTED_SIGNAL, + dev.extra: Format.HINTED_SIGNAL, + dev.temp: None, + } + + dev.reset_readable_formats() + assert dev.get_readable_formats() == declared + assert dev.get_readable_format_changes() == {} + # Repeatable, and the baseline is not consumed + dev.set_readable_format(dev.energy, None) + dev.reset_readable_formats() + assert dev.get_readable_formats() == declared + + +async def test_reset_readable_formats_keeps_registrations_after_super_init(): + # The reason the baseline is sealed by the metaclass rather than at the end + # of StandardReadable.__init__: this registration happens after super() and + # is still part of what the class declares + class Dev(StandardReadable): + def __init__(self, name=""): + self.before, _ = soft_signal_r_and_setter(int, 0, name="before") + self.set_readable_format(self.before, Format.CONFIG_SIGNAL) + super().__init__(name=name) + self.after, _ = soft_signal_r_and_setter(int, 0, name="after") + self.set_readable_format(self.after, Format.HINTED_SIGNAL) + + dev = Dev(name="dev") + expected = {dev.before: Format.CONFIG_SIGNAL, dev.after: Format.HINTED_SIGNAL} + assert dev.get_readable_formats() == expected + + dev.set_readable_format(dev.after, None) + dev.reset_readable_formats() + assert dev.get_readable_formats() == expected + + +async def test_reset_readable_formats_restores_annotated_declarations(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + # Nothing declared, so a runtime registration resets away entirely + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.reset_readable_formats() + assert sr.get_readable_formats() == {} + + +async def test_default_readable_formats_are_not_shared_between_instances(): + class Dev(StandardReadable): + def __init__(self, name=""): + self.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + self.set_readable_format(self.sig, Format.CONFIG_SIGNAL) + super().__init__(name=name) + + a, b = Dev(name="a"), Dev(name="b") + a.set_readable_format(a.sig, None) + a.reset_readable_formats() + b.reset_readable_formats() + assert a.get_readable_formats() == {a.sig: Format.CONFIG_SIGNAL} + assert b.get_readable_formats() == {b.sig: Format.CONFIG_SIGNAL} + assert StandardReadable._default_readables == {} + + +async def test_readables_registered_before_super_init_survive(): + # AreaDetector registers config signals before calling super().__init__(), + # so the registry must not be created there. It must also be per instance. + class Dev(StandardReadable): + def __init__(self, name=""): + self.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + self.set_readable_format(self.sig, Format.HINTED_SIGNAL) + super().__init__(name=name) + + a, b = Dev(name="a"), Dev(name="b") + assert a.get_readable_formats() == {a.sig: Format.HINTED_SIGNAL} + assert b.get_readable_formats() == {b.sig: Format.HINTED_SIGNAL} + assert a.get_readable_formats() != b.get_readable_formats() + + b.set_readable_format(b.sig, None) + assert a.get_readable_formats() == {a.sig: Format.HINTED_SIGNAL} + assert b.get_readable_formats() == {} + + +async def test_set_readable_format_keeps_position_when_reformatting(): + # Changing a format must not reorder hints + sr = StandardReadable(name="sr") + sr.one, _ = soft_signal_r_and_setter(int, 0, name="one") + sr.two, _ = soft_signal_r_and_setter(int, 0, name="two") + sr.set_readable_format(sr.one, Format.HINTED_SIGNAL) + sr.set_readable_format(sr.two, Format.HINTED_SIGNAL) + assert sr.hints == {"fields": [sr.one.name, sr.two.name]} + + sr.set_readable_format(sr.one, Format.HINTED_UNCACHED_SIGNAL) + assert sr.hints == {"fields": [sr.one.name, sr.two.name]} + + +async def test_set_readable_format_affects_staging_of_later_runs(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + await sr.sig.connect(mock=False) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + + await sr.stage() + assert await sr.sig.read(cached=True) + await sr.unstage() + + # Dropping the signal means the next run does not monitor it + sr.set_readable_format(sr.sig, None) + await sr.stage() + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) + await sr.unstage() + + +def test_set_readable_format_rejects_non_signal_for_signal_format(): + sr = StandardReadable(name="sr") + sr.child = Device(name="child") + with pytest.raises(TypeError, match="is not a SignalR"): + sr.set_readable_format(sr.child, Format.CONFIG_SIGNAL) diff --git a/tests/unit_tests/epics/pvi/test_pvi.py b/tests/unit_tests/epics/pvi/test_pvi.py index 7dcafdd3e4..f6bdb29007 100644 --- a/tests/unit_tests/epics/pvi/test_pvi.py +++ b/tests/unit_tests/epics/pvi/test_pvi.py @@ -155,7 +155,7 @@ async def test_device_create_children_from_annotations_with_device_vectors(): await device.connect(mock=True) block_1_device = device.device - assert block_1_device in device._has_hints + assert device.get_readable_formats().get(block_1_device) is Format.CHILD block_2_device_vector = device.device_vector assert device.device_vector[1].name == "test_device-device_vector-1" diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index c64f2a36c3..2f056739c6 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -6,7 +6,17 @@ import pytest import yaml -from ophyd_async.core import Settings, YamlSettingsProvider, get_mock +from ophyd_async.core import ( + Device, + Settings, + StandardReadable, + YamlSettingsProvider, + apply_readable_formats, + get_mock, + soft_signal_rw, + walk_readable_formats, +) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.plan_stubs import ( apply_settings, apply_settings_if_different, @@ -179,3 +189,433 @@ def my_plan(): assert m.mock_calls == [call.sig_rw.put("foo")] RE(my_plan()) + + +@pytest.fixture +def technique_device() -> StandardReadable: + """A Device whose energy signal is config for one technique, hinted for another.""" + device = StandardReadable(name="mono") + device.energy = soft_signal_rw(float, 7.0) + device.temperature = soft_signal_rw(float, 20.0) + device.set_name("mono") + device.set_readable_format(device.energy, Format.CONFIG_SIGNAL) + device.set_readable_format(device.temperature, Format.CONFIG_SIGNAL) + return device + + +async def test_store_settings_writes_formats_under_reserved_key( + RE, technique_device, tmp_path +): + provider = YamlSettingsProvider(tmp_path) + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + with open(tmp_path / "fixed_energy.yaml") as f: + stored = yaml.safe_load(f) + # Values stay flat, formats live under a key that cannot be a path + assert stored == { + "energy": 7.0, + "temperature": 20.0, + "": { + "": { + "energy": "CONFIG_SIGNAL", + "temperature": "CONFIG_SIGNAL", + } + }, + } + + RE(my_plan()) + + +async def test_reserved_keys_need_no_quoting_in_yaml(RE, technique_device, tmp_path): + provider = YamlSettingsProvider(tmp_path) + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + text = (tmp_path / "fixed_energy.yaml").read_text() + assert ":" in text + assert ":" in text + # An empty string root would make yaml fall back to "? ''" explicit keys + assert "?" not in text + + RE(my_plan()) + + +async def test_settings_round_trip_switches_technique(RE, technique_device, tmp_path): + provider = YamlSettingsProvider(tmp_path) + energy, temperature = technique_device.energy, technique_device.temperature + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + + # Switch to a technique that scans energy and drops temperature entirely + technique_device.set_readable_format(energy, Format.HINTED_SIGNAL) + technique_device.set_readable_format(temperature, None) + yield from bps.abs_set(energy, 9.0, wait=True) + yield from store_settings(provider, "scan_energy", technique_device) + + # Going back restores both the value and the formats in one apply + fixed = yield from retrieve_settings(provider, "fixed_energy", technique_device) + yield from apply_settings(fixed) + assert ( + technique_device.get_readable_formats().get(energy) is Format.CONFIG_SIGNAL + ) + assert ( + technique_device.get_readable_formats().get(temperature) + is Format.CONFIG_SIGNAL + ) + assert (yield from bps.rd(energy)) == 7.0 + + # And forward again drops temperature rather than merging the two + scanning = yield from retrieve_settings( + provider, "scan_energy", technique_device + ) + yield from apply_settings(scanning) + assert ( + technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL + ) + assert technique_device.get_readable_formats().get(temperature) is None + assert (yield from bps.rd(energy)) == 9.0 + + RE(my_plan()) + + +async def test_settings_file_without_formats_leaves_them_alone( + RE, technique_device, tmp_path +): + """A file stored before formats existed must not clear them.""" + provider = YamlSettingsProvider(tmp_path) + energy = technique_device.energy + (tmp_path / "old_style.yaml").write_text("energy: 3.0\ntemperature: 4.0\n") + + def my_plan(): + settings = yield from retrieve_settings(provider, "old_style", technique_device) + assert settings.readable_formats is None + yield from apply_settings(settings) + # Values applied, formats untouched + assert (yield from bps.rd(energy)) == 3.0 + assert ( + technique_device.get_readable_formats().get(energy) is Format.CONFIG_SIGNAL + ) + + RE(my_plan()) + + +async def test_formats_only_file_applies_without_touching_values( + RE, technique_device, tmp_path +): + """A hand written formats only profile changes no hardware. + + Also covers the explicit null, which is the only way to unregister a child, + since applying otherwise merges. + """ + provider = YamlSettingsProvider(tmp_path) + energy = technique_device.energy + (tmp_path / "hinted.yaml").write_text( + ":\n" + " :\n" + " energy: HINTED_SIGNAL\n" + " temperature: null\n" + ) + + def my_plan(): + settings = yield from retrieve_settings(provider, "hinted", technique_device) + assert dict(settings) == {} + yield from apply_settings(settings) + assert ( + technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL + ) + # An explicit null drops it, where omitting it would have left it alone + assert ( + technique_device.get_readable_formats().get(technique_device.temperature) + is None + ) + assert (yield from bps.rd(energy)) == 7.0 + + RE(my_plan()) + + +class _Mono(StandardReadable): + def __init__(self, name=""): + with self.add_children_as_readables(Format.CONFIG_SIGNAL): + self.energy = soft_signal_rw(float, 7.0) + self.temperature = soft_signal_rw(float, 20.0) + super().__init__(name=name) + + +async def test_store_settings_writes_only_changes_from_class_defaults(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + mono = _Mono(name="mono") + + def stored_formats(name: str): + with open(tmp_path / f"{name}.yaml") as f: + return yaml.safe_load(f)[""] + + def my_plan(): + yield from store_settings(provider, "defaults", mono) + mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + mono.set_readable_format(mono.temperature, None) + yield from store_settings(provider, "scan", mono) + assert stored_formats("defaults") == {} + # A dropped default is written as a null + assert stored_formats("scan") == { + "": {"energy": "HINTED_SIGNAL", "temperature": None} + } + # The defaults file undoes changes it does not mention + defaults = yield from retrieve_settings(provider, "defaults", mono) + yield from apply_settings(defaults) + assert mono.get_readable_formats() == { + mono.energy: Format.CONFIG_SIGNAL, + mono.temperature: Format.CONFIG_SIGNAL, + } + scan = yield from retrieve_settings(provider, "scan", mono) + yield from apply_settings(scan) + assert mono.get_readable_formats() == {mono.energy: Format.HINTED_SIGNAL} + + RE(my_plan()) + + +async def test_null_formats_key_resets_to_class_defaults(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + mono = _Mono(name="mono") + mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + (tmp_path / "reset.yaml").write_text(":\n") + + def my_plan(): + settings = yield from retrieve_settings(provider, "reset", mono) + assert settings.readable_formats == {} + yield from apply_settings(settings) + assert mono.get_readable_formats().get(mono.energy) is Format.CONFIG_SIGNAL + + RE(my_plan()) + + +async def test_retrieve_settings_warns_on_unknown_reserved_keys( + RE, technique_device, tmp_path +): + """A typo'd or newer reserved key is skipped with a warning, not a NameError.""" + provider = YamlSettingsProvider(tmp_path) + (tmp_path / "newer.yaml").write_text( + "energy: 3.0\n:\n : {}\n: 1\n" + ) + + def my_plan(): + match = r"unknown reserved keys \['', ''\]" + with pytest.warns(UserWarning, match=match): + settings = yield from retrieve_settings(provider, "newer", technique_device) + assert dict(settings) == {technique_device.energy: 3.0} + yield from apply_settings(settings) + assert (yield from bps.rd(technique_device.energy)) == 3.0 + + RE(my_plan()) + + +async def test_retrieve_settings_still_rejects_a_genuine_typo( + RE, technique_device, tmp_path +): + provider = YamlSettingsProvider(tmp_path) + (tmp_path / "typo.yaml").write_text("energyy: 3.0\n") + + def my_plan(): + with pytest.raises(NameError, match=r"Unknown signal names \['energyy'\]"): + yield from retrieve_settings(provider, "typo", technique_device) + + RE(my_plan()) + + +async def test_store_settings_walks_nested_devices(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + inner = StandardReadable(name="inner") + inner.sig = soft_signal_rw(float, 1.0) + outer = StandardReadable(name="outer") + outer.inner = inner + outer.top = soft_signal_rw(float, 2.0) + outer.set_name("outer") + inner.set_readable_format(inner.sig, Format.HINTED_SIGNAL) + outer.set_readable_format(outer.top, Format.CONFIG_SIGNAL) + outer.set_readable_format(inner, Format.CHILD) + # The same signal registered on a second owner, from deeper in its tree + outer.set_readable_format(inner.sig, Format.CONFIG_SIGNAL) + + def my_plan(): + yield from store_settings(provider, "nested", outer) + with open(tmp_path / "nested.yaml") as f: + assert yaml.safe_load(f) == { + "top": 2.0, + "inner.sig": 1.0, + "": { + "": { + "top": "CONFIG_SIGNAL", + "inner": "CHILD", + "inner.sig": "CONFIG_SIGNAL", + }, + # Child paths are relative to their owner + "inner": {"sig": "HINTED_SIGNAL"}, + }, + } + + RE(my_plan()) + + +def test_apply_readable_formats_rejects_unknown_path(technique_device): + with pytest.raises(KeyError, match="No Device at 'nope'"): + apply_readable_formats(technique_device, {"nope": {}}) + + +def test_apply_readable_formats_rejects_non_readable(technique_device): + with pytest.raises(TypeError, match="is not a StandardReadable"): + apply_readable_formats(technique_device, {"energy": {}}) + + +def test_settings_or_merges_formats_per_child(technique_device): + # Values merge per signal and apply_readable_formats merges per child, so + # `other` must only replace the children it mentions -- overriding a whole + # owner would drop children it has nothing to say about + base = Settings( + technique_device, + {technique_device.energy: 1.0}, + { + "": { + "energy": Format.CONFIG_SIGNAL, + "temperature": Format.CHILD, + } + }, + ) + override = Settings( + technique_device, + {technique_device.temperature: 2.0}, + {"": {"energy": Format.HINTED_SIGNAL}}, + ) + merged = base | override + assert merged.readable_formats == { + "": { + "energy": Format.HINTED_SIGNAL, # overridden + "temperature": Format.CHILD, # untouched, not dropped + } + } + assert dict(merged) == { + technique_device.energy: 1.0, + technique_device.temperature: 2.0, + } + # Neither operand is mutated + assert base.readable_formats[""]["energy"] is Format.CONFIG_SIGNAL + assert override.readable_formats[""] == { + "energy": Format.HINTED_SIGNAL + } + + +def test_settings_or_keeps_owners_the_other_does_not_mention(technique_device): + base = Settings(technique_device, {}, {"a": {"x": Format.CONFIG_SIGNAL}}) + override = Settings(technique_device, {}, {"b": {"y": Format.HINTED_SIGNAL}}) + assert (base | override).readable_formats == { + "a": {"x": Format.CONFIG_SIGNAL}, + "b": {"y": Format.HINTED_SIGNAL}, + } + + +def test_settings_or_rejects_formats_from_a_sub_device(technique_device): + # A sub-device's paths are relative to itself, so would silently come + # to mean the parent + parent = StandardReadable(name="parent") + parent.child = technique_device + parent.set_name("parent") + base = Settings(parent) + sub = Settings( + technique_device, {}, {"": {"energy": Format.HINTED_SIGNAL}} + ) + with pytest.raises(ValueError, match="paths are relative to it"): + base | sub + # Without formats it is just a value merge, which is fine + assert (base | Settings(technique_device)).readable_formats is None + + +def test_settings_partition_carries_formats_to_both_halves(technique_device): + # apply_panda_settings applies only the halves, never the original, so + # formats must survive on whichever half is applied + settings = Settings( + technique_device, + {technique_device.energy: 1.0, technique_device.temperature: 2.0}, + {"": {"energy": Format.CONFIG_SIGNAL}}, + ) + a, b = settings.partition(lambda sig: "energy" in sig.name) + assert a.readable_formats == settings.readable_formats + assert b.readable_formats == settings.readable_formats + + +async def test_store_settings_writes_empty_formats_for_an_unchanged_readable( + RE, tmp_path +): + """The key is still written, as applying it resets runtime changes. + + This is the shape of TangoTestDevice. + """ + provider = YamlSettingsProvider(tmp_path) + device = StandardReadable(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + + def my_plan(): + yield from store_settings(provider, "empty", device) + with open(tmp_path / "empty.yaml") as f: + assert yaml.safe_load(f) == {"sig": 1.0, "": {}} + + RE(my_plan()) + + +async def test_store_settings_warns_and_skips_a_readable_outside_the_tree(RE, tmp_path): + """A child registered from outside the tree has no path to store it against. + + Its value is not stored either, since store_settings only walks the tree, so + it cannot survive a round trip at all. See + https://github.com/bluesky/ophyd-async/issues/1402. + """ + provider = YamlSettingsProvider(tmp_path) + device = StandardReadable(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + outsider = soft_signal_rw(float, 2.0, name="outsider") + device.set_readable_format(device.sig, Format.CONFIG_SIGNAL) + device.set_readable_format(outsider, Format.CONFIG_SIGNAL) + + match = "dev: outsider is not within dev" + with pytest.warns(UserWarning, match=match): + assert walk_readable_formats(device) == { + "": {"sig": Format.CONFIG_SIGNAL} + } + + def my_plan(): + with pytest.warns(UserWarning, match=match): + yield from store_settings(provider, "outsider", device) + with open(tmp_path / "outsider.yaml") as f: + # Neither the outsider's format nor its value is stored + assert yaml.safe_load(f) == { + "sig": 1.0, + "": {"": {"sig": "CONFIG_SIGNAL"}}, + } + + RE(my_plan()) + + +async def test_store_settings_omits_the_key_for_a_device_with_no_readables( + RE, tmp_path +): + """A Device with no StandardReadable in its tree has nothing to clear. + + This is the shape of EpicsTestCaDevice, and is why the EPICS golden files + needed no regeneration. + """ + provider = YamlSettingsProvider(tmp_path) + + class Plain(Device): + pass + + device = Plain(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + + def my_plan(): + yield from store_settings(provider, "plain", device) + with open(tmp_path / "plain.yaml") as f: + assert yaml.safe_load(f) == {"sig": 1.0} + + RE(my_plan()) diff --git a/tests/unit_tests/test_data/test_yaml_config_save.yaml b/tests/unit_tests/test_data/test_yaml_config_save.yaml index 99fb4ac43c..1c981f1ef5 100644 --- a/tests/unit_tests/test_data/test_yaml_config_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_config_save.yaml @@ -1,3 +1,4 @@ +: {} a_bool: true a_enum: Bbb a_float: 1.234 diff --git a/tests/unit_tests/test_data/test_yaml_save.yaml b/tests/unit_tests/test_data/test_yaml_save.yaml index 900837389f..1d2d7047bb 100644 --- a/tests/unit_tests/test_data/test_yaml_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_save.yaml @@ -1,3 +1,4 @@ +: {} _sig_rw: Top level private SignalRW child.a_bool: true child.a_enum: Bbb From b9bf5f9a6af0334ad285bc3204d6a16365dea20f Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 25 Aug 2026 20:36:55 +0000 Subject: [PATCH 04/25] Make StandardDetector a StandardReadable So that set_readable_format() works on a detector at all. read(), describe() and hints merge the registered children on top of whatever the data logics produce, and add_config_signals becomes a deprecated wrapper. The detector overrides _extra_funcs_for()/_extra_hint_sources() rather than reimplementing the verbs, so a detector with no hinted fields now reports hints as {} rather than {"fields": []}, matching StandardReadable. Split out of the runtime-readable-format work so that PR stays core-only. Co-Authored-By: Claude Opus 5 --- src/ophyd_async/core/_detector.py | 118 +++++++++++------- src/ophyd_async/epics/adcore/_detector.py | 14 ++- src/ophyd_async/sim/_blob_detector.py | 4 +- .../unit_tests/core/test_standard_detector.py | 32 ++++- .../unit_tests/fastcs/panda/test_hdf_panda.py | 4 +- 5 files changed, 124 insertions(+), 48 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index e56308cfa2..0d23b2aedc 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -4,8 +4,9 @@ import functools import os import time +import warnings from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence from dataclasses import dataclass from enum import Enum from functools import cached_property @@ -15,9 +16,7 @@ Collectable, Flyable, HasHints, - Hints, Preparable, - Reading, Stageable, StreamAsset, Triggerable, @@ -29,8 +28,12 @@ from ophyd_async.core._log import logger from ._data_providers import ReadableDataProvider, StreamableDataProvider -from ._device import Device -from ._protocol import AsyncConfigurable, AsyncReadable +from ._readable import ( + StandardReadable, + StandardReadableFormat, + _HintedFields, + _Verb, +) from ._settings import Settings from ._signal import ( SignalDict, @@ -353,28 +356,34 @@ class _KickoffCtx: class StandardDetector( - Device, + StandardReadable, Stageable, - AsyncConfigurable, - AsyncReadable, Triggerable, Preparable, Flyable, Collectable, WritesStreamAssets, - HasHints, ): """Detector base class for step and fly scanning detectors. Aggregates trigger, arm, reading or stream logic together. + + Signals read during a step scan are registered with + [](#StandardReadable.set_readable_format), exactly as on any other + `StandardReadable`; data produced by a `DetectorDataLogic` is added on top + of those in `read()` and `describe()`. + + `read()` and `describe()` require `prepare()` to have run, and raise + otherwise, so that a descriptor can never be emitted without the detector's + data keys. `trigger()` prepares implicitly, so a step scan never has to do + it explicitly. `read_configuration()` and `describe_configuration()` have no + such requirement. """ # Logic for the detector _trigger_logic: DetectorTriggerLogic | None = None _acquire_logic: DetectorAcquireLogic | None = None _data_logics: Sequence[DetectorDataLogic] = () - # Signals to include in read_configuration - _config_signals: Sequence[SignalR] = () # Context produced by prepare, used by trigger and kickoff _prepare_ctx: _PrepareCtx | None = None # Context produced by kickoff, used by complete @@ -382,6 +391,13 @@ class StandardDetector( # The triggers that are supported by the trigger logic _supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Contribute to _StandardBase's fan out rather than replacing stage(), + # so the readable children registered on this detector still get staged + self._stage_funcs += (self._stage_detector,) + self._unstage_funcs += (self._unstage_detector,) + # Report the number of events for the next kickoff @cached_property def events_to_kickoff(self) -> SignalRW[int]: @@ -410,7 +426,8 @@ def add_detector_logics( # Store the triggers that are supported self._supported_triggers = _get_supported_triggers(logic) # Add the config signals it needs - self.add_config_signals(*logic.config_sigs()) + for sig in logic.config_sigs(): + self.set_readable_format(sig, StandardReadableFormat.CONFIG_SIGNAL) elif isinstance(logic, DetectorAcquireLogic): if self._acquire_logic is not None: raise RuntimeError("Detector already has acquire logic") @@ -420,12 +437,21 @@ def add_detector_logics( else: raise TypeError(f"Unknown logic type: {type(logic)}") + # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: """Add a signal to read_configuration(). :param sig: The signal to add """ - self._config_signals = (*self._config_signals, *signals) + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(signal, StandardReadableFormat" + ".CONFIG_SIGNAL)` instead of `add_config_signals(signal)`" + ), + stacklevel=2, + ) + for signal in signals: + self.set_readable_format(signal, StandardReadableFormat.CONFIG_SIGNAL) async def get_trigger_deadtime( self, settings: Settings | None = None @@ -453,9 +479,9 @@ async def get_trigger_deadtime( return self._supported_triggers, deadtime @AsyncStatus.wrap - async def stage(self) -> None: + async def _stage_detector(self) -> None: """Make sure the detector is idle and ready to be used.""" - coros = [data_logic.stop() for data_logic in self._data_logics] + coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] if self._acquire_logic: coros.append(self._acquire_logic.ensure_ready()) await asyncio.gather(*coros) @@ -617,7 +643,7 @@ async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: """Trigger a single exposure. If [`prepare()`](#StandardDetector.prepare) has not been called since - the last [`stage()`](#StandardDetector.stage), an implicit prepare is + the last `stage()`, an implicit prepare is performed. When [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` [](#DetectorTriggerLogic.default_trigger_info) is called to read current hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is @@ -721,22 +747,37 @@ async def complete(self): ): yield update - async def describe_configuration(self) -> dict[str, DataKey]: - return await merge_gathered_dicts( - sig.describe() for sig in self._config_signals - ) + def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: + """Contribute what the data logics produce to read() and describe(). + + StandardReadable gathers these alongside the registered children, so + the verb methods themselves need no overriding. - async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts(sig.read() for sig in self._config_signals) + Raising when nothing has been prepared is deliberate, and predates + registered children being readable at all: a detector that has not been + prepared has no data keys, so a `describe()` that quietly succeeded + would emit a descriptor missing the detector's data. In practice + `trigger()` prepares implicitly, so `trigger_and_read` never reaches + this. `read_configuration()` and `describe_configuration()` are + unaffected, as this hook only feeds the data verbs. - async def describe(self) -> dict[str, DataKey]: + The cost is that the *registered children* are unreachable through + `read()`/`describe()` until then too, since this raises before the + registry is consulted. That is accepted rather than worked around: + reading an unprepared detector is already defined as an error. + """ + if verb not in (_Verb.DESCRIBE, _Verb.READ): + return ctx = error_if_none(self._prepare_ctx, "Prepare not run") - # Readable and Streamable data providers produce data during read - coros = [dp.make_datakeys() for dp in ctx.readable_data_providers] + [ - dp.make_datakeys(ctx.trigger_info.collections_per_event) - for dp in ctx.streamable_data_providers - ] - return await merge_gathered_dicts(coros) + for dp in ctx.readable_data_providers: + yield dp.make_datakeys if verb is _Verb.DESCRIBE else dp.make_readings + if verb is _Verb.DESCRIBE: + # Streamable providers describe their shape for a step scan, but + # produce their data through collect_asset_docs rather than read() + for sdp in ctx.streamable_data_providers: + yield functools.partial( + sdp.make_datakeys, ctx.trigger_info.collections_per_event + ) async def describe_collect(self) -> dict[str, DataKey]: ctx = error_if_none(self._prepare_ctx, "Prepare not run") @@ -747,18 +788,11 @@ async def describe_collect(self) -> dict[str, DataKey]: ] return await merge_gathered_dicts(coros) - @property - def hints(self) -> Hints: - fields: list[str] = [] + def _extra_hint_sources(self) -> Iterator[HasHints]: + """Contribute the data logics' hinted fields alongside the children's.""" for dl in self._data_logics: - fields.extend(dl.get_hinted_fields(self.name + dl.datakey_suffix)) - return Hints(fields=fields) - - async def read(self) -> dict[str, Reading]: - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - return await merge_gathered_dicts( - dp.make_readings() for dp in ctx.readable_data_providers - ) + if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): + yield _HintedFields(fields) async def collect_asset_docs( self, index: int | None = None @@ -784,9 +818,9 @@ async def get_index(self) -> int: return min_collections_written // ctx.trigger_info.collections_per_event @AsyncStatus.wrap - async def unstage(self) -> None: + async def _unstage_detector(self) -> None: """Stop the detector and file writing.""" - coros = [data_logic.stop() for data_logic in self._data_logics] + coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] if self._acquire_logic: coros.append(self._acquire_logic.ensure_stopped()) await asyncio.gather(*coros) diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index 8b773b7973..cc8bef3e75 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -7,6 +7,7 @@ SignalR, StandardDetector, ) +from ophyd_async.core import StandardReadableFormat as Format from ._acquire_logic import ADContAcqAcquireLogic from ._data_logic import ADWriterFactory @@ -55,9 +56,16 @@ def __init__( writer, data_logic = factory(prefix, driver, plugin_list) setattr(self, factory.writer_name, writer) self.add_detector_logics(data_logic) - self.add_config_signals( - self.driver.acquire_period, self.driver.acquire_time, *config_sigs - ) + # Register the driver's timing signals, plus anything the caller asked + # for, as configuration. Plugins are deliberately not registered here: + # a detector normally carries far more plugins than are wired into the + # chain, so opt in per plugin with set_readable_format() instead. + for signal in ( + self.driver.acquire_period, + self.driver.acquire_time, + *config_sigs, + ): + self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) def get_plugin( diff --git a/src/ophyd_async/sim/_blob_detector.py b/src/ophyd_async/sim/_blob_detector.py index ed21968784..6760e088a1 100644 --- a/src/ophyd_async/sim/_blob_detector.py +++ b/src/ophyd_async/sim/_blob_detector.py @@ -1,6 +1,7 @@ from collections.abc import Sequence from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import StandardReadableFormat as Format from ._blob_acquire_logic import BlobAcquireLogic from ._blob_data_logic import BlobDataLogic @@ -26,5 +27,6 @@ def __init__( path_provider=path_provider, pattern_generator=self.pattern_generator ), ) - self.add_config_signals(*config_sigs) + for signal in config_sigs: + self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 6741959336..15a5756c72 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -25,6 +25,7 @@ soft_signal_r_and_setter, soft_signal_rw, ) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.testing import ( assert_configuration, assert_reading, @@ -657,7 +658,7 @@ async def test_config_signals_in_describe_configuration( signal = soft_signal_rw( signal_type, initial_value=initial_value, name="test-config" ) - det.add_config_signals(signal) + det.set_readable_format(signal, Format.CONFIG_SIGNAL) await det.stage() @@ -1082,3 +1083,32 @@ async def test_data_logic_not_implemented_errors(): # stop() should not raise (has default implementation) await logic.stop() # Should pass + + +async def test_detector_readable_format_changes_at_runtime(): + # A StandardDetector is a StandardReadable, so a plugin signal can be moved + # between configuration and hinted reads without redefining the Device. + det = StandardDetector(name="det") + det.temperature, _ = soft_signal_r_and_setter(float, 20.0, name="temperature") + + name = det.temperature.name + det.set_readable_format(det.temperature, Format.CONFIG_SIGNAL) + assert set(await det.read_configuration()) == {name} + # Empty hints are {} rather than {"fields": []}, as for any StandardReadable + assert det.hints == {} + + det.set_readable_format(det.temperature, Format.HINTED_UNCACHED_SIGNAL) + assert set(await det.read_configuration()) == set() + assert det.hints == {"fields": [name]} + + det.set_readable_format(det.temperature, None) + assert set(await det.read_configuration()) == set() + assert det.hints == {} + + +async def test_detector_add_config_signals_is_deprecated(): + det = StandardDetector(name="det") + signal, _ = soft_signal_r_and_setter(float, 1.0, name="sig") + with pytest.deprecated_call(): + det.add_config_signals(signal) + assert set(await det.read_configuration()) == {signal.name} diff --git a/tests/unit_tests/fastcs/panda/test_hdf_panda.py b/tests/unit_tests/fastcs/panda/test_hdf_panda.py index 6b5e5599b1..075b314219 100644 --- a/tests/unit_tests/fastcs/panda/test_hdf_panda.py +++ b/tests/unit_tests/fastcs/panda/test_hdf_panda.py @@ -187,7 +187,9 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): "source": uri, }, } - assert hdf_panda.hints == {"fields": []} + # No hinted fields is {} rather than {"fields": []}, as for any + # StandardReadable -- the detector no longer special cases this + assert hdf_panda.hints == {} # Kick it off and start it completing await hdf_panda.kickoff() status = hdf_panda.complete() From 1c785730ba2a11e37e41f9a13cf5ae90500276ea Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 11:49:27 -0400 Subject: [PATCH 05/25] Make standard detector extend from standard readable. Allow for pulling read/config/describe from children automatically (cherry picked from commit 32193035d7412c81bf42b930a2e275f6f7781df3) --- src/ophyd_async/core/_detector.py | 13 ++-- .../unit_tests/core/test_standard_detector.py | 59 ++++++++++++++++++- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 0d23b2aedc..a75f228234 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -136,18 +136,18 @@ class DetectorTriggerLogic: Subclasses must implement the appropriate `prepare_*` method for any trigger mode the detector supports, `get_deadtime` if it supports external - triggering, and `config_sigs` if the deadtime would vary according to + triggering, and `deadtime_sigs` if the deadtime would vary according to detector parameters. """ - def config_sigs(self) -> set[SignalR]: - """Return the signals that should appear in read_configuration.""" + def deadtime_sigs(self) -> set[SignalR]: + """Return the signals whose values are needed to compute deadtime.""" return set() def get_deadtime(self, config_values: SignalDict) -> float: """Return the deadtime in seconds for the detector. - :param config_values: the value of each signal in `config_sigs` + :param config_values: the value of each signal in `deadtime_sigs` """ raise NotImplementedError(self) @@ -425,9 +425,6 @@ def add_detector_logics( self._trigger_logic = logic # Store the triggers that are supported self._supported_triggers = _get_supported_triggers(logic) - # Add the config signals it needs - for sig in logic.config_sigs(): - self.set_readable_format(sig, StandardReadableFormat.CONFIG_SIGNAL) elif isinstance(logic, DetectorAcquireLogic): if self._acquire_logic is not None: raise RuntimeError("Detector already has acquire logic") @@ -465,7 +462,7 @@ async def get_trigger_deadtime( self._trigger_logic.get_deadtime ): config_values = SignalDict() - for sig in self._trigger_logic.config_sigs(): + for sig in self._trigger_logic.deadtime_sigs(): if settings and sig in settings: # Use value from settings if it is in there # cast to a SignalRW because settings can only contain those diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 15a5756c72..d8f0724d4f 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -18,6 +18,7 @@ SignalDict, SignalR, StandardDetector, + StandardReadable, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -98,8 +99,7 @@ class DeadtimeTriggerLogic(DetectorTriggerLogic): def __init__(self, deadtime_signal: SignalR[float]): self.deadtime_signal = deadtime_signal - def config_sigs(self) -> set[SignalR]: - """Return the deadtime signal as a config signal.""" + def deadtime_sigs(self) -> set[SignalR]: return {self.deadtime_signal} def get_deadtime(self, config_values: SignalDict) -> float: @@ -1051,6 +1051,61 @@ async def test_collect_asset_docs_with_explicit_index(tmp_path): ] +async def test_child_readable_config_signals_in_describe_configuration(): + """Child StandardReadable CONFIG_SIGNALs appear in describe_configuration.""" + child = StandardReadable(name="child") + config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") + child.add_readables([config_sig], Format.CONFIG_SIGNAL) + + det = StandardDetector(name="det") + det.add_readables([child], Format.CHILD) + det.add_detector_logics(ReadableOnlyDataLogic()) + await det.prepare(TriggerInfo()) + + config = await det.describe_configuration() + assert "child-exposure" in config + reading = await det.read_configuration() + assert "child-exposure" in reading + assert reading["child-exposure"]["value"] == 1.5 + + +async def test_child_readable_read_signals_in_read(): + """Child StandardReadable HINTED_SIGNALs appear in read/describe.""" + child = StandardReadable(name="child") + read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") + child.add_readables([read_sig], Format.HINTED_SIGNAL) + + det = StandardDetector(name="det") + det.add_readables([child], Format.CHILD) + det.add_detector_logics(ReadableOnlyDataLogic()) + await det.prepare(TriggerInfo()) + + desc = await det.describe() + assert "child-counts" in desc + assert "foo-value" in desc + + reading = await det.read() + assert "child-counts" in reading + assert reading["child-counts"]["value"] == 99 + # Data provider signal is also present + assert "foo-value" in reading + + +async def test_child_readable_hints_merged(): + """Child StandardReadable hints are merged with data logic hints.""" + child = StandardReadable(name="child") + hinted_sig = soft_signal_rw(float, name="child-intensity") + child.add_readables([hinted_sig], Format.HINTED_SIGNAL) + + det = StandardDetector(name="det") + det.add_readables([child], Format.CHILD) + det.add_detector_logics(ReadableOnlyDataLogic()) + + assert "fields" in det.hints + assert "child-intensity" in det.hints["fields"] + assert "foo-value" in det.hints["fields"] + + async def test_trigger_logic_not_implemented_errors(): """Test NotImplementedError for unimplemented DetectorTriggerLogic methods.""" logic = DetectorTriggerLogic() From 7ad7949723615a10ba997e5d5454fa37ac93089d Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 11:50:14 -0400 Subject: [PATCH 06/25] Update EPICS areaDetector classes to use standard readable child components for config/read (cherry picked from commit 64f4cd32b07ae6a7d6ee23e4c494ac56f8d812f8) --- src/ophyd_async/epics/adcore/__init__.py | 2 - src/ophyd_async/epics/adcore/_data_logic.py | 17 --- src/ophyd_async/epics/adcore/_detector.py | 16 ++- src/ophyd_async/epics/adcore/_io.py | 117 ++++++++++++------ .../epics/adcore/test_data_logic.py | 28 ++--- .../unit_tests/epics/adcore/test_use_cases.py | 108 ++++++++-------- 6 files changed, 155 insertions(+), 133 deletions(-) diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 08dbd0836d..80e87ecfee 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -9,7 +9,6 @@ ADMultipartDataLogic, ADWriterFactory, NDArrayDescription, - PluginSignalDataLogic, ) from ._detector import AreaDetector, ContAcqDetector from ._io import ( @@ -95,7 +94,6 @@ "ADContAcqAcquireLogic", # DataLogic "NDArrayDescription", - "PluginSignalDataLogic", "ADHDFDataLogic", "ADMultipartDataLogic", "ADWriterFactory", diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 9beb01237d..2542ed1838 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -12,7 +12,6 @@ EnableDisable, PathInfo, PathProvider, - SignalDataProvider, SignalR, StreamableDataProvider, StreamResourceDataProvider, @@ -35,22 +34,6 @@ from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType -@dataclass -class PluginSignalDataLogic(DetectorDataLogic): - driver: ADBaseIO - signal: SignalR - hinted: bool = True - - async def prepare_single(self, datakey_name: str) -> SignalDataProvider: - # Need to wait for all the plugins to have finished before we can read - # the plugin signal - await self.driver.wait_for_plugins.set(True) - return SignalDataProvider(self.signal) - - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return [self.signal.name] if self.hinted else [] - - @dataclass class NDArrayDescription: """Signals that describe the shape and data type of an NDArray frame. diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index cc8bef3e75..6e395254f7 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -56,15 +56,13 @@ def __init__( writer, data_logic = factory(prefix, driver, plugin_list) setattr(self, factory.writer_name, writer) self.add_detector_logics(data_logic) - # Register the driver's timing signals, plus anything the caller asked - # for, as configuration. Plugins are deliberately not registered here: - # a detector normally carries far more plugins than are wired into the - # chain, so opt in per plugin with set_readable_format() instead. - for signal in ( - self.driver.acquire_period, - self.driver.acquire_time, - *config_sigs, - ): + # The driver is always wired up, so register it whole and let ADBaseIO + # declare which of its signals are configuration. Plugins are + # deliberately *not* registered: a detector normally carries far more + # plugins than are wired into the chain, so opt in per plugin with + # `set_readable_format(det.stats, Format.CHILD)` instead. + self.set_readable_format(self.driver, Format.CHILD) + for signal in config_sigs: self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 0c6b086193..54157116a6 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -6,17 +6,15 @@ EnableDisable, SignalR, SignalRW, + StandardReadable, StrictEnum, SubsetEnum, SupersetEnum, TriggerableCommand, non_zero, ) -from ophyd_async.epics.core import ( - EpicsDevice, - EpicsOptions, - PvSuffix, -) +from ophyd_async.core import StandardReadableFormat as Format +from ophyd_async.epics.core import EpicsDevice, EpicsOptions, PvSuffix # Common classes for drivers and plugins @@ -116,7 +114,7 @@ class ADState(StrictEnum): ADBaseIOT = TypeVar("ADBaseIOT", bound="ADBaseIO") -class ADBaseIO(NDArrayBaseIO): +class ADBaseIO(StandardReadable, NDArrayBaseIO): """Base class from which areaDetector drivers are derived. This mirrors the interface provided by ADCore/db/ADBase.template. @@ -127,10 +125,14 @@ class ADBaseIO(NDArrayBaseIO): compatibility with changes made in https://github.com/areaDetector/ADCore/blob/master/RELEASE.md#asynndarraydriver-addriver """ - acquire_time: A[SignalRW[float], PvSuffix.rbv("AcquireTime")] - acquire_period: A[SignalRW[float], PvSuffix.rbv("AcquirePeriod")] - num_images: A[SignalRW[int], PvSuffix.rbv("NumImages")] - image_mode: A[SignalRW[ADImageMode], PvSuffix.rbv("ImageMode")] + acquire_time: A[SignalRW[float], PvSuffix.rbv("AcquireTime"), Format.CONFIG_SIGNAL] + acquire_period: A[ + SignalRW[float], PvSuffix.rbv("AcquirePeriod"), Format.CONFIG_SIGNAL + ] + num_images: A[SignalRW[int], PvSuffix.rbv("NumImages"), Format.CONFIG_SIGNAL] + image_mode: A[ + SignalRW[ADImageMode], PvSuffix.rbv("ImageMode"), Format.CONFIG_SIGNAL + ] detector_state: A[SignalR[ADState], PvSuffix("DetectorState_RBV")] # The following signals have been moved from NDArrayBaseIO for backwards @@ -140,11 +142,13 @@ class ADBaseIO(NDArrayBaseIO): # There is no _RBV for this one wait_for_plugins: A[SignalRW[bool], PvSuffix("WaitForPlugins")] - manufacturer: A[SignalR[str], PvSuffix("Manufacturer_RBV")] - model: A[SignalR[str], PvSuffix("Model_RBV")] - serial_number: A[SignalR[str], PvSuffix("SerialNumber_RBV")] - sdk_version: A[SignalR[str], PvSuffix("SDKVersion_RBV")] - firmware_version: A[SignalR[str], PvSuffix("FirmwareVersion_RBV")] + manufacturer: A[SignalR[str], PvSuffix("Manufacturer_RBV"), Format.CONFIG_SIGNAL] + model: A[SignalR[str], PvSuffix("Model_RBV"), Format.CONFIG_SIGNAL] + serial_number: A[SignalR[str], PvSuffix("SerialNumber_RBV"), Format.CONFIG_SIGNAL] + sdk_version: A[SignalR[str], PvSuffix("SDKVersion_RBV"), Format.CONFIG_SIGNAL] + firmware_version: A[ + SignalR[str], PvSuffix("FirmwareVersion_RBV"), Format.CONFIG_SIGNAL + ] # Classes for plugins @@ -165,13 +169,15 @@ class NDPluginBaseIO(NDArrayBaseIO): queue_size: A[SignalRW[int], PvSuffix.rbv("QueueSize")] -class NDROIIO(NDPluginBaseIO): +class NDROIIO(StandardReadable, NDPluginBaseIO): """Plugin for taking a region of an NDArray. This mirrors the interface provided by ADCore/db/NDROI.template. See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROI.html """ + min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] size_z: A[SignalRW[int], PvSuffix.rbv("SizeZ")] @@ -180,8 +186,14 @@ class NDROIIO(NDPluginBaseIO): bin_y: A[SignalRW[int], PvSuffix.rbv("BinY")] bin_z: A[SignalRW[int], PvSuffix.rbv("BinZ")] + def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): + super().__init__(prefix, with_pvi, name) + self.add_readables( + [self.min_x, self.min_y, self.size_x, self.size_y], Format.CONFIG_SIGNAL + ) + -class NDStatsIO(NDPluginBaseIO): +class NDStatsIO(StandardReadable, NDPluginBaseIO): """Plugin for computing statistics from an image or ROI within an image. This mirrors the interface provided by ADCore/db/NDStats.template. @@ -191,11 +203,24 @@ class NDStatsIO(NDPluginBaseIO): # Basic statistics compute_statistics: A[SignalRW[bool], PvSuffix.rbv("ComputeStatistics")] bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth")] - total: A[SignalR[float], PvSuffix("Total_RBV")] + min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] + min_x: A[SignalR[float], PvSuffix("MinX_RBV")] + min_y: A[SignalR[float], PvSuffix("MinY_RBV")] + max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] + max_x: A[SignalR[float], PvSuffix("MaxX_RBV")] + max_y: A[SignalR[float], PvSuffix("MaxY_RBV")] mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] + sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] + total: A[SignalR[float], PvSuffix("Total_RBV")] + net: A[SignalR[float], PvSuffix("Net_RBV")] # Centroid statistics compute_centroid: A[SignalRW[bool], PvSuffix.rbv("ComputeCentroid")] centroid_threshold: A[SignalRW[float], PvSuffix.rbv("CentroidThreshold")] + centroid_x: A[SignalR[float], PvSuffix("CentroidX_RBV")] + centroid_y: A[SignalR[float], PvSuffix("CentroidY_RBV")] + sigma_x: A[SignalR[float], PvSuffix("SigmaX_RBV")] + sigma_y: A[SignalR[float], PvSuffix("SigmaY_RBV")] + sigma_xy: A[SignalR[float], PvSuffix("SigmaXY_RBV")] # X and Y Profiles compute_profiles: A[SignalRW[bool], PvSuffix.rbv("ComputeProfiles")] profile_size_x: A[SignalR[int], PvSuffix("ProfileSizeX_RBV")] @@ -208,27 +233,39 @@ class NDStatsIO(NDPluginBaseIO): hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin")] hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax")] + def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): + super().__init__(prefix, with_pvi, name) + self.add_readables( + [ + self.compute_statistics, + self.bgd_width, + self.compute_centroid, + self.centroid_threshold, + self.compute_profiles, + self.cursor_x, + self.cursor_y, + self.compute_histogram, + self.hist_size, + self.hist_min, + self.hist_max, + ], + Format.CONFIG_SIGNAL, + ) + self.add_readables( + [self.total, self.mean_value, self.min_value, self.max_value], + Format.HINTED_UNCACHED_SIGNAL, + ) + self.add_readables( + [self.min_x, self.min_y, self.max_x, self.max_y], + Format.UNCACHED_SIGNAL, + ) -class NDROIStatNIO(EpicsDevice): - """Defines the parameters for a single ROI used for statistics calculation. - Each instance represents a single ROI, with attributes for its position - (min_x, min_y) and size (size_x, size_y), as well as a name and use status. +class NDROIStatNIO(StandardReadable, EpicsDevice): + """Defines the parameters for a single ROI used for statistics calculation. This mirrors the interface provided by ADCore/db/NDROIStatN.template. See definition in ADApp/pluginSrc/NDPluginROIStat.h in https://github.com/areaDetector/ADCore. - - Attributes: - name: The name of the ROI. - use: Flag indicating whether the ROI is used. - min_x: The start X-coordinate of the ROI. - min_y: The start Y-coordinate of the ROI. - size_x: The width of the ROI. - size_y: The height of the ROI. - min_value: Minimum count value in the ROI. - max_value: Maximum count value in the ROI. - mean_value: Mean counts value in the ROI. - total: Total counts in the ROI. """ name_: A[SignalRW[str], PvSuffix("Name")] @@ -237,12 +274,22 @@ class NDROIStatNIO(EpicsDevice): min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] - # stats min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] total: A[SignalR[float], PvSuffix("Total_RBV")] + def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): + super().__init__(prefix, with_pvi, name) + self.add_readables( + [self.use, self.min_x, self.min_y, self.size_x, self.size_y], + Format.CONFIG_SIGNAL, + ) + self.add_readables( + [self.min_value, self.max_value, self.mean_value, self.total], + Format.HINTED_UNCACHED_SIGNAL, + ) + class NDROIStatIO(NDPluginBaseIO): """Plugin for calculating basic statistics for multiple ROIs. diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 4696f9ee98..9e940744b3 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -282,14 +282,12 @@ async def test_describe_different_color_modes( # Expected to give the right shape in the descriptor await hdf_det.prepare(TriggerInfo()) describe = await hdf_det.describe() - assert describe == { - "detector": { - "dtype": "array", - "dtype_numpy": " Date: Wed, 12 Aug 2026 11:50:39 -0400 Subject: [PATCH 07/25] Update implementations to move config_sigs -> deadtime_sigs for trigger logics (cherry picked from commit 404930bbdaa8648f63cdb7224ac120b6e85fa4cc) --- src/ophyd_async/epics/adaravis.py | 2 +- src/ophyd_async/epics/advimba.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index 695c78e886..fb979f4f03 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -62,7 +62,7 @@ class AravisTriggerLogic(DetectorTriggerLogic): process_plugin: NDProcessIO | None override_deadtime: float | None = None - def config_sigs(self) -> set[SignalR]: + def deadtime_sigs(self) -> set[SignalR]: return {self.driver.model} def get_deadtime(self, config_values: SignalDict) -> float: diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index f6afd41ba9..efece768da 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -99,7 +99,7 @@ class VimbaTriggerLogic(DetectorTriggerLogic): process_plugin: NDProcessIO | None override_deadtime: float | None = None - def config_sigs(self) -> set[SignalR]: + def deadtime_sigs(self) -> set[SignalR]: return {self.driver.model} def get_deadtime(self, config_values: SignalDict) -> float: From 7712bfcd7735bdd0c12e4553b56e79c24018eecb Mon Sep 17 00:00:00 2001 From: Jakub Wlodek Date: Wed, 12 Aug 2026 12:18:11 -0400 Subject: [PATCH 08/25] Fix container test (cherry picked from commit dd36b7a64825d848f8ccf7792c3ba9d3c486399b) --- tests/container_tests/test_adsim_system.py | 58 ++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index f1791639f0..b9a3c3f426 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -198,10 +198,24 @@ def test_software_triggering( "data": { "adsim-driver-acquire_period": 0.005, "adsim-driver-acquire_time": 0.1, + "adsim-driver-num_images": 1, + "adsim-driver-image_mode": "Multiple", + "adsim-driver-manufacturer": "Simulated detector", + "adsim-driver-model": "Basic simulator", + "adsim-driver-serial_number": "No serial number", + "adsim-driver-sdk_version": ANY, + "adsim-driver-firmware_version": "No firmware", }, "timestamps": { "adsim-driver-acquire_period": ANY, "adsim-driver-acquire_time": ANY, + "adsim-driver-num_images": ANY, + "adsim-driver-image_mode": ANY, + "adsim-driver-manufacturer": ANY, + "adsim-driver-model": ANY, + "adsim-driver-serial_number": ANY, + "adsim-driver-sdk_version": ANY, + "adsim-driver-firmware_version": ANY, }, "data_keys": { "adsim-driver-acquire_period": { @@ -220,6 +234,50 @@ def test_software_triggering( "units": "", "precision": 3, }, + "adsim-driver-num_images": { + "dtype": "integer", + "shape": [], + "dtype_numpy": " Date: Wed, 19 Aug 2026 12:31:29 +0000 Subject: [PATCH 09/25] Opt plugins into read()/describe() explicitly in the AD use case tests #1395 registered every StandardReadable plugin as a CHILD readable in AreaDetector.__init__. That is dropped in the rebase onto #1400, for the reason @jwlodek gave on the PR himself: "what if we didn't add plugins as readables automatically when passed in at __init__, and instead explicitly required an add_readables call to include them?" and the reason @coretl gave for asking: a detector typically carries far more plugins (16 ROIs, 16 stats) than are wired into its chain, so reporting all of them gives no way to tell which values are meaningful. So the three tests that expected stats readings now say so, with set_readable_format(stat, Format.CHILD). NDStatsIO declares its own config and hinted signals, so opting the plugin in as a CHILD picks up exactly what these tests already assert. The wait_for_plugins assertions go with the AreaDetector override that set it, which both reviewers agreed was too hidden and belongs in an explicit data logic instead. Co-Authored-By: Claude Opus 5 (cherry picked from commit f56c52368a961fdb1053e888811d44a33190782d) --- tests/unit_tests/epics/adcore/test_use_cases.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index df09f29cbf..1c287a30f9 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -2,6 +2,7 @@ import pytest +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.core import ( StaticPathProvider, TriggerInfo, @@ -24,6 +25,8 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( plugins={"stats": stat}, ) temp = epics_signal_r(float, "SAMP:TEMP:RBV") + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) ndattributes = [ adcore.NDAttributeParam( name="det-sum", @@ -46,7 +49,6 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( set_mock_value(det.driver.array_size_y, 768) await det.stage() - assert await det.driver.wait_for_plugins.get_value() is False config_description = await det.describe_configuration() assert { "det-driver-acquire_period": { @@ -538,11 +540,11 @@ async def test_simdetector_with_stats_signal(): stat = adcore.NDStatsIO("PREFIX:STAT:") async with init_devices(mock=True): det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}) + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) set_mock_value(stat.total, 1.8) await det.stage() - assert await det.driver.wait_for_plugins.get_value() is False await det.trigger() - assert await det.driver.wait_for_plugins.get_value() is True description = await det.describe() assert "det-stats-total" in description assert description["det-stats-total"] == { @@ -569,6 +571,8 @@ async def test_step_scan_keep_numimages( adcore.ADWriterFactory.hdf(static_path_provider), plugins={"stats": stat}, ) + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) set_mock_value(det.driver.acquire_period, 0.1) set_mock_value(det.driver.acquire_time, 0.05) From 9dc430c719af2c75a46635f534f5a238b85d8bd8 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 13:46:55 +0000 Subject: [PATCH 10/25] Set WaitForPlugins from ADAcquireLogic rather than hiding it in AreaDetector _update_prepare_context override, conditional on number_of_events == 1. Both reviewers called that too hidden, and the condition was unnecessary: WaitForPlugins gates the driver's AcquireBusy going to 0 when acquisition *completes*, so it never throttles frames. A flyscan does exactly one wait_for_idle, at the end, which is precisely when the plugins should have drained. It belongs on the acquire logic because that is what owns "poke this, then check acquire is done": wait_for_idle already blocks on the acquire put callback, which is the AcquireBusy that WaitForPlugins gates, so the existing wait needs no change at all. Defaults to on, since racing is silent and the wait costs nothing when a file writer is the slowest thing in the chain anyway. Off is a constructor argument rather than a subclass, matching how driver_armed_signal already parametrises this class; a detector needing genuinely custom arm/idle logic still subclasses instead. Set from ensure_ready, not start_acquiring: the PV is latched and has no _RBV, so it is written once per scan while the driver is known idle rather than on the latency path before every arm. Stop first, then always write it -- never left at whatever the IOC booted with, so a scan cannot depend on who touched the PV last. ADContAcqAcquireLogic deliberately gets no equivalent: the driver is always acquiring there, so there is no AcquireBusy transition to gate. Said so on the class so it does not look like an oversight. Co-Authored-By: Claude Opus 5 (cherry picked from commit fe787ce0cbf7ce97584c429265e39ea4b4f84231) --- .../epics/adcore/_acquire_logic.py | 38 +++++++++++++++++++ .../unit_tests/epics/adcore/test_use_cases.py | 21 ++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/ophyd_async/epics/adcore/_acquire_logic.py b/src/ophyd_async/epics/adcore/_acquire_logic.py index 4ba9591c27..2ae1a7a587 100644 --- a/src/ophyd_async/epics/adcore/_acquire_logic.py +++ b/src/ophyd_async/epics/adcore/_acquire_logic.py @@ -15,10 +15,29 @@ class ADAcquireLogic(DetectorAcquireLogic): + """Arm and disarm an areaDetector driver. + + :param driver: The areaDetector driver to arm. + :param driver_armed_signal: + The signal to watch for the driver having armed, if it is not `acquire`. + :param wait_for_plugins: + Whether the detector should wait for its plugins to drain before it + reports acquisition as complete. `WaitForPlugins` makes the driver's + `AcquireBusy` wait for the number of queued arrays to reach zero, and + `wait_for_idle` blocks on the `acquire` put callback, so turning it on + is what makes a step scan read plugin scalars for the frame it just + took rather than the one before. Defaults to on, since racing is + silent and the wait costs nothing when a file writer is the slowest + thing in the chain anyway. Turn it off for a detector whose plugin + chain must not gate acquisition. + :param timeout: How long to wait for the driver to arm and to return to idle. + """ + def __init__( self, driver: ADBaseIO, driver_armed_signal: SignalR[bool] | None = None, + wait_for_plugins: bool = True, timeout: float = DEFAULT_TIMEOUT, ): self.driver = driver @@ -26,9 +45,19 @@ def __init__( self.driver_armed_signal = driver_armed_signal else: self.driver_armed_signal = driver.acquire + self.wait_for_plugins = wait_for_plugins self.timeout = timeout self.acquire_status: AsyncStatus | None = None + async def ensure_ready(self): + # Stop first: WaitForPlugins is latched and has no _RBV, so set it once + # per scan while the driver is known to be idle rather than on the + # latency path before every arm. Always written, never left at whatever + # the IOC happened to boot with, so a scan does not depend on who + # touched the PV last. + await self.ensure_stopped() + await self.driver.wait_for_plugins.set(self.wait_for_plugins) + async def start_acquiring(self): self.acquire_status = await set_and_wait_for_other_value( set_signal=self.driver.acquire, @@ -53,6 +82,15 @@ async def ensure_stopped(self): class ADContAcqAcquireLogic(DetectorAcquireLogic): + """Start and stop capture on a circular buffer, leaving the driver acquiring. + + Deliberately has no `wait_for_plugins` equivalent to [](#ADAcquireLogic). + That setting gates the driver's `AcquireBusy` going to 0 at the end of an + acquisition, and here the driver is always acquiring -- there is no such + transition to wait on. `wait_for_idle` waits on the circular buffer's + capture instead. + """ + def __init__(self, driver: ADBaseIO, cb_plugin: NDCircularBuffIO): self.driver = driver self.cb_plugin = cb_plugin diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 1c287a30f9..02485a3428 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -49,6 +49,8 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( set_mock_value(det.driver.array_size_y, 768) await det.stage() + # ensure_ready sets WaitForPlugins so read() sees this frame's scalars + assert await det.driver.wait_for_plugins.get_value() is True config_description = await det.describe_configuration() assert { "det-driver-acquire_period": { @@ -544,6 +546,7 @@ async def test_simdetector_with_stats_signal(): det.set_readable_format(stat, Format.CHILD) set_mock_value(stat.total, 1.8) await det.stage() + assert await det.driver.wait_for_plugins.get_value() is True await det.trigger() description = await det.describe() assert "det-stats-total" in description @@ -559,6 +562,24 @@ async def test_simdetector_with_stats_signal(): assert "det-stats-mean_value" in det.hints["fields"] +@pytest.mark.parametrize("wait_for_plugins", [True, False]) +async def test_acquire_logic_writes_wait_for_plugins_on_stage(wait_for_plugins: bool): + # Always written, never left at whatever the IOC booted with, so a scan + # cannot depend on who touched the PV last + driver = adcore.ADBaseIO("PREFIX:DET:") + async with init_devices(mock=True): + det = adcore.AreaDetector( + driver, + acquire_logic=adcore.ADAcquireLogic( + driver, wait_for_plugins=wait_for_plugins + ), + name="det", + ) + set_mock_value(det.driver.wait_for_plugins, not wait_for_plugins) + await det.stage() + assert await det.driver.wait_for_plugins.get_value() is wait_for_plugins + + async def test_step_scan_keep_numimages( static_path_provider: StaticPathProvider, monkeypatch: pytest.MonkeyPatch, From 6ee08450b7b86e3a76a4efbc105ba0c4fa96d172 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 14:20:09 +0000 Subject: [PATCH 11/25] Feed get_deadtime from the readable registry, dropping deadtime_sigs DetectorTriggerLogic no longer nominates the signals its deadtime depends on. get_trigger_deadtime passes the value of every signal the detector reports as configuration instead, so there is no second list to keep in step with the first: the values that determine deadtime are exactly the ones that ought to be recorded for the scan anyway. StandardReadable gains get_config_signals(), which walks the registry and recurses through CHILD-registered children. That recursion is what makes it work for areaDetector: AreaDetector registers the driver whole, and ADBaseIO declares model as CONFIG_SIGNAL, so the signal AravisTriggerLogic and VimbaTriggerLogic need arrives without either class saying anything. Both deadtime_sigs overrides are deleted. Unlike walk_config_signals it reads the registry rather than calling read_configuration(), so it does no I/O and includes read-only SignalRs -- driver.model is a SignalR, so the existing helper would have dropped precisely the signal that matters. Two consequences, both real: - A logic needing a signal must ensure it is declared as configuration. If it is not, get_deadtime gets a KeyError rather than a silently fetched value. - get_trigger_deadtime now reads every configuration signal, where a logic could previously declare none and pay nothing. Reads are gathered so it stays one round trip, but it does mean the call now needs a connected device even when deadtime is a constant -- test_adpilatus::test_deadtime had to start connecting its detector. Also moves every add_readables call in adcore/_io.py into the signal annotations, which removes three __init__ methods that existed only to declare formats. The remaining registration calls elsewhere cannot follow: they take caller-supplied config_sigs or a driver passed in as a constructor argument, neither of which is declarable. Co-Authored-By: Claude Opus 5 (cherry picked from commit c239654d0491b350bca0aaba2ed62db184129f84) --- docs/how-to/implement-ad-detector.md | 9 +- docs/tutorials/implementing-detectors.md | 6 +- src/ophyd_async/core/_detector.py | 40 ++++-- src/ophyd_async/core/_readable.py | 22 +++ src/ophyd_async/epics/adaravis.py | 3 - src/ophyd_async/epics/adcore/_io.py | 130 ++++++++---------- src/ophyd_async/epics/advimba.py | 3 - .../unit_tests/core/test_standard_detector.py | 14 +- tests/unit_tests/epics/test_adpilatus.py | 11 +- 9 files changed, 133 insertions(+), 105 deletions(-) diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 54a0c6074b..4611e0323e 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -28,11 +28,14 @@ Implement methods for each trigger mode your detector supports: Only implement the prepare methods for trigger modes your detector actually supports. The detector will automatically report which trigger types are available based on which methods are implemented. -If the detector has configuration values that should be captured in the scan then implement: -- `config_sigs()` - Return the set of signals that should appear in read_configuration() +Configuration values that should be captured in the scan are declared on the IO class, +with [](#StandardReadableFormat.CONFIG_SIGNAL), exactly as on any other +[](#StandardReadable) — the trigger logic does not nominate them. If you support external triggering you should also implement: -- `get_deadtime(config_values)` - Calculate the minimum time between exposures based on configuration values +- `get_deadtime(config_values)` - Calculate the minimum time between exposures. It is + given the value of *every* signal the detector reports as configuration, so a signal + it needs must be declared with [](#StandardReadableFormat.CONFIG_SIGNAL). To preserve hardware state in plans like [`bp.count`](#bluesky.plans.count) and [`bps.trigger_and_read`](#bluesky.plan_stubs.trigger_and_read) that call `trigger()` diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index 4fa2fe93ea..1a4310fe13 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -199,8 +199,10 @@ If we wanted to support external triggering, we would also implement: - `prepare_level()` for external level/gate triggering (high level duration determines exposure) We could also implement: -- `get_deadtime()` to calculate the minimum time between exposures based on configuration -- `config_sigs()` to return signals that should appear in read_configuration() +- `get_deadtime()` to calculate the minimum time between exposures. It is given the value + of every signal the detector reports as configuration, so a signal it needs must be + declared with [](#StandardReadableFormat.CONFIG_SIGNAL) like on any other + [](#StandardReadable) — the trigger logic does not nominate signals itself - `default_trigger_info()` to return the [](#TriggerInfo) to use when `trigger()` is called without a preceding `prepare()` (governed by [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE)). ### `BlobAcquireLogic` diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index a75f228234..49c3c42933 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -135,19 +135,22 @@ class DetectorTriggerLogic: - Multi-exposure collection batching Subclasses must implement the appropriate `prepare_*` method for any trigger - mode the detector supports, `get_deadtime` if it supports external - triggering, and `deadtime_sigs` if the deadtime would vary according to - detector parameters. + mode the detector supports, and `get_deadtime` if it supports external + triggering. """ - def deadtime_sigs(self) -> set[SignalR]: - """Return the signals whose values are needed to compute deadtime.""" - return set() - def get_deadtime(self, config_values: SignalDict) -> float: """Return the deadtime in seconds for the detector. - :param config_values: the value of each signal in `deadtime_sigs` + :param config_values: + The value of every `Signal` the detector reports as configuration, + i.e. everything registered with + [](#StandardReadableFormat.CONFIG_SIGNAL), including those declared + on children registered as [](#StandardReadableFormat.CHILD). A + logic that needs a signal must make sure it is declared as + configuration, rather than nominating it separately: the values + that determine deadtime are exactly the ones that ought to be + recorded for the scan anyway. """ raise NotImplementedError(self) @@ -455,6 +458,12 @@ async def get_trigger_deadtime( ) -> tuple[set[DetectorTrigger], float | None]: """Get supported trigger types and deadtime for the detector. + The trigger logic is given the value of every signal this detector + reports as configuration, rather than nominating the ones it wants: + the values that determine deadtime are exactly the ones that ought to + be recorded for the scan anyway, so a second list to keep in step + earned nothing. + :param settings: Optional settings to use when getting configuration values :return: Tuple of supported trigger types and deadtime in seconds """ @@ -462,14 +471,23 @@ async def get_trigger_deadtime( self._trigger_logic.get_deadtime ): config_values = SignalDict() - for sig in self._trigger_logic.deadtime_sigs(): + to_read: list[SignalR] = [] + for sig in self.get_config_signals(): if settings and sig in settings: # Use value from settings if it is in there # cast to a SignalRW because settings can only contain those config_values[sig] = settings[cast(SignalRW, sig)] else: - # Get the value live - config_values[sig] = await sig.get_value() + to_read.append(sig) + # Read live values concurrently: this is now every configuration + # signal rather than a handful the logic named, so doing it in + # series would be one round trip per signal + for sig, value in zip( + to_read, + await asyncio.gather(*(sig.get_value() for sig in to_read)), + strict=True, + ): + config_values[sig] = value deadtime = self._trigger_logic.get_deadtime(config_values) else: deadtime = None diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 0a862c76f5..577e8bfb90 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -469,6 +469,28 @@ def set_readable_format( # changing a format does not reorder `hints` self._readables[device] = format + def get_config_signals(self) -> set[SignalR]: + """Return every `Signal` contributing to `read_configuration()`. + + Recurses into children registered as + [](#StandardReadableFormat.CHILD), so a Device that registers a + sub-Device whole picks up whatever that sub-Device declares as + configuration. + + Unlike [](#walk_config_signals) this reads the registry rather than + calling `read_configuration()`, so it does no I/O and includes + read-only `SignalR`s as well as `SignalRW`s. + """ + signals: set[SignalR] = set() + for device, format in self._readables.items(): + if format is StandardReadableFormat.CONFIG_SIGNAL: + signals.add(_as_signal_r(device)) + elif format is StandardReadableFormat.CHILD and isinstance( + device, StandardReadable + ): + signals |= device.get_config_signals() + return signals + def reset_readable_formats(self) -> None: """Undo every runtime format change, back to how the class declared them. diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index fb979f4f03..936f2951ca 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -62,9 +62,6 @@ class AravisTriggerLogic(DetectorTriggerLogic): process_plugin: NDProcessIO | None override_deadtime: float | None = None - def deadtime_sigs(self) -> set[SignalR]: - return {self.driver.model} - def get_deadtime(self, config_values: SignalDict) -> float: return get_camera_deadtime( model=config_values[self.driver.model], diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 54157116a6..08fb680375 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -176,22 +176,16 @@ class NDROIIO(StandardReadable, NDPluginBaseIO): See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROI.html """ - min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] - min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] - size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] - size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] + min_x: A[SignalRW[int], PvSuffix.rbv("MinX"), Format.CONFIG_SIGNAL] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY"), Format.CONFIG_SIGNAL] + size_x: A[SignalRW[int], PvSuffix.rbv("SizeX"), Format.CONFIG_SIGNAL] + size_y: A[SignalRW[int], PvSuffix.rbv("SizeY"), Format.CONFIG_SIGNAL] size_z: A[SignalRW[int], PvSuffix.rbv("SizeZ")] bin_x: A[SignalRW[int], PvSuffix.rbv("BinX")] bin_y: A[SignalRW[int], PvSuffix.rbv("BinY")] bin_z: A[SignalRW[int], PvSuffix.rbv("BinZ")] - def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): - super().__init__(prefix, with_pvi, name) - self.add_readables( - [self.min_x, self.min_y, self.size_x, self.size_y], Format.CONFIG_SIGNAL - ) - class NDStatsIO(StandardReadable, NDPluginBaseIO): """Plugin for computing statistics from an image or ROI within an image. @@ -201,64 +195,53 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): """ # Basic statistics - compute_statistics: A[SignalRW[bool], PvSuffix.rbv("ComputeStatistics")] - bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth")] - min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] - min_x: A[SignalR[float], PvSuffix("MinX_RBV")] - min_y: A[SignalR[float], PvSuffix("MinY_RBV")] - max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] - max_x: A[SignalR[float], PvSuffix("MaxX_RBV")] - max_y: A[SignalR[float], PvSuffix("MaxY_RBV")] - mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] + compute_statistics: A[ + SignalRW[bool], PvSuffix.rbv("ComputeStatistics"), Format.CONFIG_SIGNAL + ] + bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth"), Format.CONFIG_SIGNAL] + min_value: A[ + SignalR[float], PvSuffix("MinValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + min_x: A[SignalR[float], PvSuffix("MinX_RBV"), Format.UNCACHED_SIGNAL] + min_y: A[SignalR[float], PvSuffix("MinY_RBV"), Format.UNCACHED_SIGNAL] + max_value: A[ + SignalR[float], PvSuffix("MaxValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + max_x: A[SignalR[float], PvSuffix("MaxX_RBV"), Format.UNCACHED_SIGNAL] + max_y: A[SignalR[float], PvSuffix("MaxY_RBV"), Format.UNCACHED_SIGNAL] + mean_value: A[ + SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] - total: A[SignalR[float], PvSuffix("Total_RBV")] + total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] net: A[SignalR[float], PvSuffix("Net_RBV")] # Centroid statistics - compute_centroid: A[SignalRW[bool], PvSuffix.rbv("ComputeCentroid")] - centroid_threshold: A[SignalRW[float], PvSuffix.rbv("CentroidThreshold")] + compute_centroid: A[ + SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL + ] + centroid_threshold: A[ + SignalRW[float], PvSuffix.rbv("CentroidThreshold"), Format.CONFIG_SIGNAL + ] centroid_x: A[SignalR[float], PvSuffix("CentroidX_RBV")] centroid_y: A[SignalR[float], PvSuffix("CentroidY_RBV")] sigma_x: A[SignalR[float], PvSuffix("SigmaX_RBV")] sigma_y: A[SignalR[float], PvSuffix("SigmaY_RBV")] sigma_xy: A[SignalR[float], PvSuffix("SigmaXY_RBV")] # X and Y Profiles - compute_profiles: A[SignalRW[bool], PvSuffix.rbv("ComputeProfiles")] + compute_profiles: A[ + SignalRW[bool], PvSuffix.rbv("ComputeProfiles"), Format.CONFIG_SIGNAL + ] profile_size_x: A[SignalR[int], PvSuffix("ProfileSizeX_RBV")] profile_size_y: A[SignalR[int], PvSuffix("ProfileSizeY_RBV")] - cursor_x: A[SignalRW[int], PvSuffix.rbv("CursorX")] - cursor_y: A[SignalRW[int], PvSuffix.rbv("CursorY")] + cursor_x: A[SignalRW[int], PvSuffix.rbv("CursorX"), Format.CONFIG_SIGNAL] + cursor_y: A[SignalRW[int], PvSuffix.rbv("CursorY"), Format.CONFIG_SIGNAL] # Array Histogram - compute_histogram: A[SignalRW[bool], PvSuffix.rbv("ComputeHistogram")] - hist_size: A[SignalRW[int], PvSuffix.rbv("HistSize")] - hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin")] - hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax")] - - def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): - super().__init__(prefix, with_pvi, name) - self.add_readables( - [ - self.compute_statistics, - self.bgd_width, - self.compute_centroid, - self.centroid_threshold, - self.compute_profiles, - self.cursor_x, - self.cursor_y, - self.compute_histogram, - self.hist_size, - self.hist_min, - self.hist_max, - ], - Format.CONFIG_SIGNAL, - ) - self.add_readables( - [self.total, self.mean_value, self.min_value, self.max_value], - Format.HINTED_UNCACHED_SIGNAL, - ) - self.add_readables( - [self.min_x, self.min_y, self.max_x, self.max_y], - Format.UNCACHED_SIGNAL, - ) + compute_histogram: A[ + SignalRW[bool], PvSuffix.rbv("ComputeHistogram"), Format.CONFIG_SIGNAL + ] + hist_size: A[SignalRW[int], PvSuffix.rbv("HistSize"), Format.CONFIG_SIGNAL] + hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin"), Format.CONFIG_SIGNAL] + hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax"), Format.CONFIG_SIGNAL] class NDROIStatNIO(StandardReadable, EpicsDevice): @@ -269,26 +252,21 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): """ name_: A[SignalRW[str], PvSuffix("Name")] - use: A[SignalRW[bool], PvSuffix.rbv("Use")] - min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] - min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] - size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] - size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] - min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] - max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] - mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] - total: A[SignalR[float], PvSuffix("Total_RBV")] - - def __init__(self, prefix: str = "", with_pvi: bool = False, name: str = ""): - super().__init__(prefix, with_pvi, name) - self.add_readables( - [self.use, self.min_x, self.min_y, self.size_x, self.size_y], - Format.CONFIG_SIGNAL, - ) - self.add_readables( - [self.min_value, self.max_value, self.mean_value, self.total], - Format.HINTED_UNCACHED_SIGNAL, - ) + use: A[SignalRW[bool], PvSuffix.rbv("Use"), Format.CONFIG_SIGNAL] + min_x: A[SignalRW[int], PvSuffix.rbv("MinX"), Format.CONFIG_SIGNAL] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY"), Format.CONFIG_SIGNAL] + size_x: A[SignalRW[int], PvSuffix.rbv("SizeX"), Format.CONFIG_SIGNAL] + size_y: A[SignalRW[int], PvSuffix.rbv("SizeY"), Format.CONFIG_SIGNAL] + min_value: A[ + SignalR[float], PvSuffix("MinValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + max_value: A[ + SignalR[float], PvSuffix("MaxValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + mean_value: A[ + SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] class NDROIStatIO(NDPluginBaseIO): diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index efece768da..9c3bd14472 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -99,9 +99,6 @@ class VimbaTriggerLogic(DetectorTriggerLogic): process_plugin: NDProcessIO | None override_deadtime: float | None = None - def deadtime_sigs(self) -> set[SignalR]: - return {self.driver.model} - def get_deadtime(self, config_values: SignalDict) -> float: return get_camera_deadtime( model=config_values[self.driver.model], diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index d8f0724d4f..1637e6a08f 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -99,11 +99,12 @@ class DeadtimeTriggerLogic(DetectorTriggerLogic): def __init__(self, deadtime_signal: SignalR[float]): self.deadtime_signal = deadtime_signal - def deadtime_sigs(self) -> set[SignalR]: - return {self.deadtime_signal} - def get_deadtime(self, config_values: SignalDict) -> float: - """Return the deadtime from the signal value.""" + """Return the deadtime from the signal value. + + The signal reaches us because the detector registers it as + CONFIG_SIGNAL, not because this logic nominates it. + """ return config_values[self.deadtime_signal] async def prepare_internal(self, num: int, livetime: float, deadtime: float): @@ -232,6 +233,10 @@ async def test_get_trigger_deadtime( det = StandardDetector() if trigger_logic: det.add_detector_logics(trigger_logic) + if isinstance(trigger_logic, DeadtimeTriggerLogic): + # A logic that needs a signal declares it as configuration; the logic + # no longer nominates signals separately + det.set_readable_format(trigger_logic.deadtime_signal, Format.CONFIG_SIGNAL) triggers, deadtime = await det.get_trigger_deadtime() assert triggers == expected_triggers assert deadtime == expected_deadtime @@ -247,6 +252,7 @@ async def test_get_trigger_deadtime_with_settings(): det.sig = deadtime_signal tl = DeadtimeTriggerLogic(deadtime_signal) det.add_detector_logics(tl) + det.set_readable_format(deadtime_signal, Format.CONFIG_SIGNAL) # Verify initial deadtime from signal triggers, deadtime = await det.get_trigger_deadtime() diff --git a/tests/unit_tests/epics/test_adpilatus.py b/tests/unit_tests/epics/test_adpilatus.py index 69d0f25020..70641c51db 100644 --- a/tests/unit_tests/epics/test_adpilatus.py +++ b/tests/unit_tests/epics/test_adpilatus.py @@ -61,9 +61,14 @@ async def test_threshold_apply_triggers_command( ) async def test_deadtime(readout_time: adpilatus.PilatusReadoutTime, tmp_path): path_provider = StaticPathProvider(StaticFilenameProvider("data"), tmp_path) - pilatus = adpilatus.PilatusDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(path_provider), readout_time=readout_time - ) + # Connected because get_trigger_deadtime() now reads every configuration + # signal, even for a logic like this one whose deadtime is a constant + async with init_devices(mock=True): + pilatus = adpilatus.PilatusDetector( + "PREFIX:", + adcore.ADWriterFactory.hdf(path_provider), + readout_time=readout_time, + ) trigger_modes, deadtime = await pilatus.get_trigger_deadtime() assert trigger_modes == { DetectorTrigger.INTERNAL, From 4c8c0bd1d0633ff720e24a8b98a05c5e498b2c61 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 19 Aug 2026 14:26:04 +0000 Subject: [PATCH 12/25] Make the config signal walk a private helper rather than public API get_config_signals was added as a public method on StandardReadable in the previous commit. #1367 restructures detector logic, and whether this should be a method, a function, or something else falls out of that, so it stays a module private _config_signals() in _readable.py until then. Nothing is added to core.__all__. Taking a Device rather than a StandardReadable also simplifies the recursion: a CHILD that is not a StandardReadable contributes nothing without the caller having to check. Co-Authored-By: Claude Opus 5 (cherry picked from commit ca5fcb7680eae179c98b0419eaa0f9653b0ddebb) --- src/ophyd_async/core/_detector.py | 3 +- src/ophyd_async/core/_readable.py | 52 ++++++++++++++++++------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 49c3c42933..1303060e70 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -31,6 +31,7 @@ from ._readable import ( StandardReadable, StandardReadableFormat, + _config_signals, _HintedFields, _Verb, ) @@ -472,7 +473,7 @@ async def get_trigger_deadtime( ): config_values = SignalDict() to_read: list[SignalR] = [] - for sig in self.get_config_signals(): + for sig in _config_signals(self): if settings and sig in settings: # Use value from settings if it is in there # cast to a SignalRW because settings can only contain those diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index 577e8bfb90..f851d97485 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -469,28 +469,6 @@ def set_readable_format( # changing a format does not reorder `hints` self._readables[device] = format - def get_config_signals(self) -> set[SignalR]: - """Return every `Signal` contributing to `read_configuration()`. - - Recurses into children registered as - [](#StandardReadableFormat.CHILD), so a Device that registers a - sub-Device whole picks up whatever that sub-Device declares as - configuration. - - Unlike [](#walk_config_signals) this reads the registry rather than - calling `read_configuration()`, so it does no I/O and includes - read-only `SignalR`s as well as `SignalRW`s. - """ - signals: set[SignalR] = set() - for device, format in self._readables.items(): - if format is StandardReadableFormat.CONFIG_SIGNAL: - signals.add(_as_signal_r(device)) - elif format is StandardReadableFormat.CHILD and isinstance( - device, StandardReadable - ): - signals |= device.get_config_signals() - return signals - def reset_readable_formats(self) -> None: """Undo every runtime format change, back to how the class declared them. @@ -564,6 +542,36 @@ def add_readables( self.set_readable_format(device, format) +def _config_signals(device: Device) -> set[SignalR]: + """Return every `Signal` contributing to a Device's `read_configuration()`. + + Recurses into children registered as [](#StandardReadableFormat.CHILD), so + a Device that registers a sub-Device whole picks up whatever that + sub-Device declares as configuration. A Device that is not a + `StandardReadable` contributes nothing. + + Unlike `walk_config_signals` this reads the registry rather than calling + `read_configuration()`, so it does no I/O and includes read-only `SignalR`s + as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is + exactly the signal an areaDetector trigger logic needs for deadtime. + + Deliberately private and unexported: #1367 restructures detector logic, so + what the public shape of this should be is not settled yet. + """ + signals: set[SignalR] = set() + if not isinstance(device, StandardReadable): + return signals + for entry in device._readables.entries(): # noqa: SLF001 + match entry: + case (signal, Format.CONFIG_SIGNAL): + signals.add(signal) + case (child, Format.CHILD): + signals |= _config_signals(child) + case _: + pass + return signals + + #: Reserved key under which [](#store_settings) writes readable formats. Chosen #: so that it cannot be a dotted attribute path: `<` cannot start a Python #: identifier, so no attribute assignment can produce a colliding key, and From ac98fee3864a5187bb4bed0f7cfecce5eb8525c6 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 28 Aug 2026 11:52:37 +0000 Subject: [PATCH 13/25] Delete the prepare_single data logic tier A StandardDetector is a StandardReadable now, so a source that produces one value per event is a registered signal, not a data logic: set_readable_format puts it in read() and describe() for any number of collections, where a prepare_single logic dropped itself with a warning as soon as a scan asked for more than one. That leaves the tier with nothing of its own -- its only implementation, PluginSignalDataLogic, has already gone the same way -- so prepare_single, ReadableDataProvider and SignalDataProvider go rather than staying as a second way to get a scalar into an event. Also restores the two doc sections held back from the previous branch, since this is the change that makes them true, and points ADR 0012 at 0021. --- .../decisions/0012-detector-rewrite.md | 3 + docs/how-to/implement-ad-detector.md | 11 +- src/ophyd_async/core/__init__.py | 4 - src/ophyd_async/core/_data_providers.py | 27 +---- src/ophyd_async/core/_detector.py | 46 ++------ .../unit_tests/core/test_standard_detector.py | 102 ++++++------------ 6 files changed, 56 insertions(+), 137 deletions(-) diff --git a/docs/explanations/decisions/0012-detector-rewrite.md b/docs/explanations/decisions/0012-detector-rewrite.md index 3e1145b66b..db9fdf2006 100644 --- a/docs/explanations/decisions/0012-detector-rewrite.md +++ b/docs/explanations/decisions/0012-detector-rewrite.md @@ -5,6 +5,9 @@ Date: 2026-01-20 Accepted +Partly superseded by [](./0021-runtime-readable-format.md), which made `StandardDetector` a +`StandardReadable` and removed the `prepare_single` tier along with `ReadableDataProvider`. + ## Context The original `StandardDetector` implementation had several architectural issues: diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 4611e0323e..d0704ba4e4 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -118,12 +118,15 @@ For example, for ADAravis: The composition-based architecture makes it possible to add multiple data outputs to a detector. After creating the detector, you can call `add_detector_logics()` to add additional data sources: ### Reading stats plugins alongside file writing +A plugin scalar needs no data logic: a detector is a [](#StandardReadable), so register +the signal and it appears in `read()` alongside whatever the file writer produces. + ```python -from ophyd_async.epics.adcore import PluginSignalDataLogic +from ophyd_async.core import StandardReadableFormat as Format -det = adaravis.AravisDetector(prefix, path_provider) -# Add stats total as a readable signal in events -det.add_detector_logics(adcore.PluginSignalDataLogic(det.driver, det.stats.total)) +det = adaravis.AravisDetector(prefix, path_provider, plugins={"stats": stats}) +# Add stats total as a hinted signal in events +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) ``` ### Multiple HDF writers for different ROIs diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index e3a62e5a51..c239816695 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -11,8 +11,6 @@ soft_command, ) from ._data_providers import ( - ReadableDataProvider, - SignalDataProvider, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -307,9 +305,7 @@ def __getattr__(name): "AutoMaxIncrementingPathProvider", "UUIDFilenameProvider", # Data Providers - "ReadableDataProvider", "StreamableDataProvider", - "SignalDataProvider", "StreamResourceInfo", "StreamResourceDataProvider", # Flyer diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index ad2983cb59..8faa7ba069 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -1,39 +1,14 @@ from abc import abstractmethod from collections.abc import AsyncIterator, Sequence -from dataclasses import dataclass from typing import Any -from bluesky.protocols import Reading, StreamAsset +from bluesky.protocols import StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange from ._signal import SignalR, SignalW from ._utils import ConfinedModel -class ReadableDataProvider: - @abstractmethod - async def make_datakeys(self) -> dict[str, DataKey]: - """Return a DataKey for each Readable that produces a Reading. - - Called before the first exposure is taken. - """ - - @abstractmethod - async def make_readings(self) -> dict[str, Reading]: - """Read the Signals and return their values.""" - - -@dataclass -class SignalDataProvider(ReadableDataProvider): - signal: SignalR - - async def make_datakeys(self) -> dict[str, DataKey]: - return await self.signal.describe() - - async def make_readings(self) -> dict[str, Reading]: - return await self.signal.read(cached=False) - - class StreamableDataProvider: collections_written_signal: SignalR[int] diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 1303060e70..9b49029c6d 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -25,9 +25,7 @@ from event_model import DataKey from pydantic import Field, NonNegativeInt, PositiveInt, computed_field -from ophyd_async.core._log import logger - -from ._data_providers import ReadableDataProvider, StreamableDataProvider +from ._data_providers import StreamableDataProvider from ._readable import ( StandardReadable, StandardReadableFormat, @@ -313,19 +311,16 @@ async def _get_collections_written( class DetectorDataLogic: """Abstract base class for detector data logic and handling. - Implementations must implement either prepare_unbounded for data sources - that work with step scans as well as flyscans, or prepare_single for those - that only work with step scans. + Implementations must override `prepare_unbounded`. A source that produces a + single value per event, like a plugin scalar, does not need a data logic at + all: register its signal with + [](#StandardReadable.set_readable_format) instead. """ #: Add this suffix to the detector name to specify the datakey. These need to be #: different for each DetectorDataLogic added to a detector datakey_suffix: str = "" - async def prepare_single(self, datakey_name: str) -> ReadableDataProvider: - """Provider can only work for a single event.""" - raise NotImplementedError(self) - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: """Provider can work for an unbounded number of collections.""" raise NotImplementedError(self) @@ -345,7 +340,6 @@ async def stop(self) -> None: @dataclass class _PrepareCtx: trigger_info: TriggerInfo - readable_data_providers: Sequence[ReadableDataProvider] streamable_data_providers: Sequence[StreamableDataProvider] collections_written: int @@ -516,7 +510,6 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: == trigger_info.collections_per_event ): # Reuse the existing data providers - readable_data_providers = self._prepare_ctx.readable_data_providers streamable_data_providers = self._prepare_ctx.streamable_data_providers else: # Stop the existing providers if there is a context and make new ones @@ -525,36 +518,19 @@ async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: await data_logic.stop() # Setup the data logic for the right number of collections streamable_coros: list[Awaitable[StreamableDataProvider]] = [] - readable_coros: list[Awaitable[ReadableDataProvider]] = [] for dl in self._data_logics: - if _data_logic_supported(dl.prepare_unbounded): - streamable_coros.append( - dl.prepare_unbounded(self.name + dl.datakey_suffix) - ) - elif _data_logic_supported(dl.prepare_single): - if trigger_info.number_of_collections == 1: - readable_coros.append( - dl.prepare_single(self.name + dl.datakey_suffix) - ) - else: - logger.warning( - f"DataLogic {dl} only supports a single collection, but " - "the detector was prepared for " - f"{trigger_info.number_of_collections} collections." - ) - else: + if not _data_logic_supported(dl.prepare_unbounded): msg = f"DataLogic hasn't overridden any prepare_* methods {dl}" raise RuntimeError(msg) - streamable_data_providers, readable_data_providers = await asyncio.gather( - asyncio.gather(*streamable_coros), - asyncio.gather(*readable_coros), - ) + streamable_coros.append( + dl.prepare_unbounded(self.name + dl.datakey_suffix) + ) + streamable_data_providers = await asyncio.gather(*streamable_coros) # Stash the prepare context so we can use it in trigger/kickoff self._prepare_ctx = _PrepareCtx( trigger_info=trigger_info, streamable_data_providers=streamable_data_providers, - readable_data_providers=readable_data_providers, collections_written=await _get_collections_written( streamable_data_providers ), @@ -785,8 +761,6 @@ def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict] if verb not in (_Verb.DESCRIBE, _Verb.READ): return ctx = error_if_none(self._prepare_ctx, "Prepare not run") - for dp in ctx.readable_data_providers: - yield dp.make_datakeys if verb is _Verb.DESCRIBE else dp.make_readings if verb is _Verb.DESCRIBE: # Streamable providers describe their shape for a step scan, but # produce their data through collect_asset_docs rather than read() diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 1637e6a08f..59b754aa30 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -12,9 +12,7 @@ DetectorDataLogic, DetectorTrigger, DetectorTriggerLogic, - ReadableDataProvider, Settings, - SignalDataProvider, SignalDict, SignalR, StandardDetector, @@ -131,26 +129,14 @@ async def ensure_stopped(self): self.disarm_count += 1 -class ReadableOnlyDataLogic(DetectorDataLogic): - """Produces only readable (non-streaming) data.""" - - def __init__(self): - self.signal = soft_signal_rw(int, initial_value=42, name="foo-value") - - async def prepare_single(self, datakey_name: str) -> ReadableDataProvider: - return SignalDataProvider(self.signal) - - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return ["foo-value"] - - class StreamableOnlyDataLogic(DetectorDataLogic): """Produces only streamable (file-based) data.""" - def __init__(self, tmp_path): + def __init__(self, tmp_path, datakey_suffix: str = ""): self.collections_written = soft_signal_rw(int) self.stop_count = 0 self.tmp_path = tmp_path + self.datakey_suffix = datakey_suffix async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: resource = StreamResourceInfo( @@ -439,10 +425,10 @@ async def test_arm_logic_called_on_stage(): assert al.armed is False -async def test_describe_before_prepare_raises(): +async def test_describe_before_prepare_raises(tmp_path): """Test that describe() fails before prepare().""" det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) + det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) with pytest.raises(RuntimeError, match="Prepare not run"): await det.describe() @@ -621,23 +607,21 @@ async def test_stage_resets_state(): assert await det.events_to_kickoff.get_value() == 0 -async def test_hints_from_single_data_logic(): +async def test_hints_from_single_data_logic(tmp_path): """Test that hints come from data logic.""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) + det = StandardDetector(name="bar") + det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) await det.prepare(TriggerInfo()) - assert det.hints == { - "fields": ["foo-value"] - } # ReadableOnlyDataLogic uses foo-value + assert det.hints == {"fields": ["bar"]} async def test_hints_from_multiple_data_logics(tmp_path): """Test that hints are aggregated from multiple data logics.""" det = StandardDetector(name="bar") - dl1 = ReadableOnlyDataLogic() - dl2 = StreamableOnlyDataLogic(tmp_path) + dl1 = StreamableOnlyDataLogic(tmp_path) + dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-extra") det.add_detector_logics(dl1, dl2) await det.prepare(TriggerInfo()) @@ -645,7 +629,7 @@ async def test_hints_from_multiple_data_logics(tmp_path): # Should include hints from both logics hints = det.hints assert "fields" in hints - assert hints["fields"] == ["foo-value", "bar"] + assert hints["fields"] == ["bar", "bar-extra"] @pytest.mark.parametrize( @@ -691,13 +675,13 @@ async def test_config_signals_in_describe_configuration( async def test_kickoff_without_streamable_data_raises(): """Test that kickoff() without streamable data fails.""" det = StandardDetector(name="foo") - det.add_detector_logics(JustInternalTriggerLogic(), ReadableOnlyDataLogic()) + det.add_detector_logics(JustInternalTriggerLogic()) - # Single event prepare for readable-only logic + # A detector with no data logic can still be triggered for a step scan await det.prepare(TriggerInfo()) await det.trigger() # This works - # Readable-only logic doesn't support kickoff + # ...but there is nothing to collect, so it cannot be flown await det.prepare(TriggerInfo(number_of_events=5)) with pytest.raises( ValueError, match="Detector foo is not streamable, so cannot kickoff" @@ -784,13 +768,13 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): async def test_read_returns_correct_values(): - """Test that read() returns values from readable providers.""" + """Test that read() returns the values of registered signals.""" det = StandardDetector(name="det") - dl = ReadableOnlyDataLogic() - det.add_detector_logics(dl) + det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") + det.set_readable_format(det.counts, Format.HINTED_SIGNAL) await det.trigger() - await assert_reading(det, {"foo-value": {"value": 42}}) + await assert_reading(det, {"det-counts": {"value": 42}}) async def test_detector_with_no_logics(): @@ -899,19 +883,6 @@ async def ensure_stopped(self): assert al.shutter_closed is True # unstage() must close the shutter -async def test_multiple_collections_with_single_only_logic_warns(caplog): - """Test that requesting multiple collections warns, leads to empty prepare ctx.""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) - - await det.prepare(TriggerInfo(number_of_events=5)) - - assert "only supports a single collection" in caplog.text - assert det._prepare_ctx is not None - assert det._prepare_ctx.readable_data_providers == [] - assert det._prepare_ctx.streamable_data_providers == [] - - async def test_data_logic_with_no_prepare_methods_raises(): """Test error when DataLogic doesn't override any prepare methods.""" @@ -976,16 +947,18 @@ async def test_different_collections_written_raises(tmp_path): await det.kickoff() -async def test_multiple_data_logics(tmp_path): - """Test detector with multiple data logics.""" +async def test_data_logic_and_registered_signal(tmp_path): + """A registered signal and a data logic both describe(), only one collects.""" det = StandardDetector(name="det") - dl1 = ReadableOnlyDataLogic() - dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl1, dl2) + det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") + det.set_readable_format(det.counts, Format.HINTED_SIGNAL) + det.add_detector_logics( + JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path) + ) await det.prepare(TriggerInfo()) - # Should have data from both logics + # Should have data from the data logic and the registered signal description = await det.describe() assert description == { "det": { @@ -995,11 +968,11 @@ async def test_multiple_data_logics(tmp_path): "shape": [1, 10, 15], "source": f"file://localhost/{tmp_path.as_posix().lstrip('/')}/test.h5", }, - "foo-value": { + "det-counts": { "dtype": "integer", "dtype_numpy": " Date: Fri, 30 Oct 2026 17:14:10 +0000 Subject: [PATCH 14/25] WIP: port onto the per-verb StandardReadable registries The replayed #1403 commits target the old StandardReadable hooks (_Verb, _extra_funcs_for, _extra_hint_sources, _HintedFields), which #1400 replaced with per-verb registries. StandardDetector now adds __on_read/__on_describe/__on_hints to _read_funcs/_describe_funcs/ _hints_funcs, appended after the registered children as on the tip. _config_signals walks the typed registry entries. Tests and the eiger detector move off the deprecated add_readables/add_config_signals, the stats hints test registers stats.total as a hinted signal now that PluginSignalDataLogic is gone, and two doc links are fixed. Drafts the CHANGELOG entries for this PR. Assisted-by: Claude Opus 5.5 --- CHANGELOG.md | 5 ++ .../decisions/0012-detector-rewrite.md | 2 +- docs/reference/environment-variables.md | 2 +- src/ophyd_async/core/_detector.py | 59 +++++++++---------- src/ophyd_async/fastcs/eiger/_detector.py | 3 +- .../unit_tests/core/test_standard_detector.py | 12 ++-- .../unit_tests/epics/adcore/test_detector.py | 9 ++- .../unit_tests/epics/adcore/test_use_cases.py | 1 + 8 files changed, 51 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c86ec7b930..0facdf1bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,11 @@ - `StandardReadable` now keeps one format per registered child, keyed by the child, so registering a child again replaces its format instead of adding a second contribution, and the order of `hints` is the order children were first registered. The private `_describe_config_funcs`, `_read_config_funcs` and `_has_hints` tuples are gone: a subclass that added to them now adds a hook to `_describe_configuration_funcs`, `_read_configuration_funcs` or `_hints_funcs` (see ADR 0025). - The deprecated `ConfigSignal` and `HintedSignal` aliases are removed from `ophyd_async.core`; use `StandardReadableFormat.CONFIG_SIGNAL`, `HINTED_SIGNAL` or `HINTED_UNCACHED_SIGNAL`. `StandardReadable.add_readables()` is deprecated in favour of `set_readable_format()` on each device. - `store_settings()` writes the readable format changes of the Device tree under the reserved key ``, and `apply_settings()` resets the tree to its class formats before applying them. Settings files stored before this have no such key and leave formats alone. `retrieve_settings()` warns about and drops any other `<...>` key. +- `StandardDetector` is now a `StandardReadable`: signals read in a step scan or reported as configuration are registered with `set_readable_format()`, as on any other `StandardReadable`, and what the data logics produce is added to `read()`, `describe()` and `hints` alongside them. `add_config_signals()` is deprecated in favour of `set_readable_format(signal, StandardReadableFormat.CONFIG_SIGNAL)`. A detector with no hinted fields now reports `hints` as `{}` rather than `{"fields": []}`. +- `DetectorTriggerLogic.config_sigs()` is removed: `get_deadtime()` is given the value of every signal the detector reports as configuration, including those declared on children registered as `CHILD`, so a signal that determines deadtime must be registered as `CONFIG_SIGNAL`. +- The `prepare_single` data logic tier is removed, with `ReadableDataProvider`, `SignalDataProvider` and `adcore.PluginSignalDataLogic`. Register a plugin scalar such as `NDStatsIO.total` with `set_readable_format(signal, StandardReadableFormat.HINTED_SIGNAL)` instead. +- The areaDetector driver and plugin classes (`ADBaseIO`, `NDROIIO`, `NDStatsIO` and `NDROIStatNIO`) are now `StandardReadable`s that declare their own configuration and read signals, so registering a plugin as `CHILD` of an `AreaDetector` puts them in `read()` and `read_configuration()`. +- `ADAcquireLogic` sets `WaitForPlugins` on `stage()`, from its new `wait_for_plugins` argument (default on), rather than `AreaDetector` hiding it, so a step scan reads plugin scalars for the frame it has just taken. ### Added diff --git a/docs/explanations/decisions/0012-detector-rewrite.md b/docs/explanations/decisions/0012-detector-rewrite.md index db9fdf2006..2e059e6571 100644 --- a/docs/explanations/decisions/0012-detector-rewrite.md +++ b/docs/explanations/decisions/0012-detector-rewrite.md @@ -5,7 +5,7 @@ Date: 2026-01-20 Accepted -Partly superseded by [](./0021-runtime-readable-format.md), which made `StandardDetector` a +Partly superseded by [](./0025-runtime-readable-format.md), which made `StandardDetector` a `StandardReadable` and removed the `prepare_single` tier along with `ReadableDataProvider`. ## Context diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 8ab91db8d8..78afba8c51 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -13,7 +13,7 @@ effect on ophyd-async itself. Controls the implicit prepare performed by [`StandardDetector.trigger()`](#ophyd_async.core.StandardDetector.trigger) when [`prepare()`](#ophyd_async.core.StandardDetector.prepare) has not been called since -the last [`stage()`](#ophyd_async.core.StandardDetector.stage). +the last `stage()`. | Value | Behaviour | |-------|-----------| diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index 9b49029c6d..fd8b36abba 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -15,8 +15,9 @@ from bluesky.protocols import ( Collectable, Flyable, - HasHints, + Hints, Preparable, + Reading, Stageable, StreamAsset, Triggerable, @@ -30,8 +31,6 @@ StandardReadable, StandardReadableFormat, _config_signals, - _HintedFields, - _Verb, ) from ._settings import Settings from ._signal import ( @@ -395,6 +394,11 @@ def __init__(self, *args, **kwargs) -> None: # so the readable children registered on this detector still get staged self._stage_funcs += (self._stage_detector,) self._unstage_funcs += (self._unstage_detector,) + # What the data logics produce, gathered alongside the registered + # children, so the verb methods themselves need no overriding + self._read_funcs += (self.__on_read,) + self._describe_funcs += (self.__on_describe,) + self._hints_funcs += (self.__on_hints,) # Report the number of events for the next kickoff @cached_property @@ -739,35 +743,30 @@ async def complete(self): ): yield update - def _extra_funcs_for(self, verb: _Verb) -> Iterator[Callable[[], Awaitable[dict]]]: - """Contribute what the data logics produce to read() and describe(). - - StandardReadable gathers these alongside the registered children, so - the verb methods themselves need no overriding. + async def __on_read(self) -> dict[str, Reading]: + """What the data logics contribute to `read()`: nothing, but checks prepare. Raising when nothing has been prepared is deliberate, and predates registered children being readable at all: a detector that has not been - prepared has no data keys, so a `describe()` that quietly succeeded - would emit a descriptor missing the detector's data. In practice - `trigger()` prepares implicitly, so `trigger_and_read` never reaches - this. `read_configuration()` and `describe_configuration()` are - unaffected, as this hook only feeds the data verbs. - - The cost is that the *registered children* are unreachable through - `read()`/`describe()` until then too, since this raises before the - registry is consulted. That is accepted rather than worked around: - reading an unprepared detector is already defined as an error. + prepared has no data keys. In practice `trigger()` prepares implicitly, + so `trigger_and_read` never reaches this. + """ + error_if_none(self._prepare_ctx, "Prepare not run") + return {} + + async def __on_describe(self) -> dict[str, DataKey]: + """What the data logics contribute to `describe()`. + + Raises when nothing has been prepared, as a `describe()` that quietly + succeeded would emit a descriptor missing the detector's data. + Streamable providers describe their shape for a step scan, but produce + their data through collect_asset_docs rather than read(). """ - if verb not in (_Verb.DESCRIBE, _Verb.READ): - return ctx = error_if_none(self._prepare_ctx, "Prepare not run") - if verb is _Verb.DESCRIBE: - # Streamable providers describe their shape for a step scan, but - # produce their data through collect_asset_docs rather than read() - for sdp in ctx.streamable_data_providers: - yield functools.partial( - sdp.make_datakeys, ctx.trigger_info.collections_per_event - ) + return await merge_gathered_dicts( + sdp.make_datakeys(ctx.trigger_info.collections_per_event) + for sdp in ctx.streamable_data_providers + ) async def describe_collect(self) -> dict[str, DataKey]: ctx = error_if_none(self._prepare_ctx, "Prepare not run") @@ -778,11 +777,11 @@ async def describe_collect(self) -> dict[str, DataKey]: ] return await merge_gathered_dicts(coros) - def _extra_hint_sources(self) -> Iterator[HasHints]: - """Contribute the data logics' hinted fields alongside the children's.""" + def __on_hints(self) -> Iterator[Hints]: + """The data logics' hinted fields, alongside the children's.""" for dl in self._data_logics: if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): - yield _HintedFields(fields) + yield {"fields": list(fields)} async def collect_asset_docs( self, index: int | None = None diff --git a/src/ophyd_async/fastcs/eiger/_detector.py b/src/ophyd_async/fastcs/eiger/_detector.py index 7257bde6a7..f993114c40 100644 --- a/src/ophyd_async/fastcs/eiger/_detector.py +++ b/src/ophyd_async/fastcs/eiger/_detector.py @@ -2,6 +2,7 @@ PathProvider, SignalR, StandardDetector, + StandardReadableFormat, TriggerableCommand, soft_signal_rw, ) @@ -44,6 +45,6 @@ def __init__( ), ) - self.add_config_signals(self.pixel_mask) + self.set_readable_format(self.pixel_mask, StandardReadableFormat.CONFIG_SIGNAL) super().__init__(name=name, connector=connector) diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 59b754aa30..79df2c9091 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -1034,10 +1034,10 @@ async def test_child_readable_config_signals_in_describe_configuration(): """Child StandardReadable CONFIG_SIGNALs appear in describe_configuration.""" child = StandardReadable(name="child") config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") - child.add_readables([config_sig], Format.CONFIG_SIGNAL) + child.set_readable_format(config_sig, Format.CONFIG_SIGNAL) det = StandardDetector(name="det") - det.add_readables([child], Format.CHILD) + det.set_readable_format(child, Format.CHILD) await det.prepare(TriggerInfo()) config = await det.describe_configuration() @@ -1051,10 +1051,10 @@ async def test_child_readable_read_signals_in_read(tmp_path): """Child StandardReadable HINTED_SIGNALs appear in read/describe.""" child = StandardReadable(name="child") read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") - child.add_readables([read_sig], Format.HINTED_SIGNAL) + child.set_readable_format(read_sig, Format.HINTED_SIGNAL) det = StandardDetector(name="det") - det.add_readables([child], Format.CHILD) + det.set_readable_format(child, Format.CHILD) det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) await det.prepare(TriggerInfo()) @@ -1072,10 +1072,10 @@ async def test_child_readable_hints_merged(tmp_path): """Child StandardReadable hints are merged with data logic hints.""" child = StandardReadable(name="child") hinted_sig = soft_signal_rw(float, name="child-intensity") - child.add_readables([hinted_sig], Format.HINTED_SIGNAL) + child.set_readable_format(hinted_sig, Format.HINTED_SIGNAL) det = StandardDetector(name="det") - det.add_readables([child], Format.CHILD) + det.set_readable_format(child, Format.CHILD) det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) assert "fields" in det.hints diff --git a/tests/unit_tests/epics/adcore/test_detector.py b/tests/unit_tests/epics/adcore/test_detector.py index 6b9edb2633..5e2bbd0c1a 100644 --- a/tests/unit_tests/epics/adcore/test_detector.py +++ b/tests/unit_tests/epics/adcore/test_detector.py @@ -1,6 +1,7 @@ import pytest from ophyd_async.core import ( + StandardReadableFormat, StaticPathProvider, TriggerInfo, init_devices, @@ -28,10 +29,12 @@ def test_writer_image_hints(static_path_provider: StaticPathProvider, factory, h plugins={"stats": stats}, name="det", ) - assert det.hints == {"fields": ["det-image"] if hinted else []} - det.add_detector_logics(adcore.PluginSignalDataLogic(driver, stats.total)) + # No hinted fields means no hints at all + assert det.hints == ({"fields": ["det-image"]} if hinted else {}) + det.set_readable_format(stats.total, StandardReadableFormat.HINTED_SIGNAL) + # Readable-format signals are hinted ahead of the data logics' fields assert det.hints == { - "fields": (["det-image"] if hinted else []) + ["det-stats-total"] + "fields": ["det-stats-total"] + (["det-image"] if hinted else []) } diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 02485a3428..a7ffbc7cee 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -12,6 +12,7 @@ ) from ophyd_async.epics import adaravis, adcore, adsimdetector from ophyd_async.epics.core import epics_signal_r +from ophyd_async.testing import assert_reading async def test_step_scan_hdf_detector_with_stats_and_temp( From bec7db1ce97cd286aafc8dd4268b28f8271e57a0 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 30 Oct 2026 17:18:20 +0000 Subject: [PATCH 15/25] WIP: detector core rewrite: DetectorLogic, bounded/unbounded data logics Split cut G3 (#1408) of detector-rederive (d0872e185): every file at its tip content (merged with origin/main), including the readable suppression hook moved here from #1400, minus the AD stats time series, which is the next PR. Deletes the unreleased ADWriterFactory.stats and ADWriterFactory.hdf(enable_callbacks=) shims rather than splitting them. ADRs 0023 and 0024. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- CHANGELOG.md | 19 +- docs/conf.py | 11 + .../decisions/0012-detector-rewrite.md | 2 +- .../decisions/0023-bounded-data-logic-tier.md | 274 +++ .../decisions/0024-detector-is-a-flyable.md | 215 +++ docs/explanations/plan-stubs.md | 27 +- docs/how-to/choose-right-baseclass.md | 4 +- docs/how-to/implement-ad-detector.md | 30 +- docs/reference/environment-variables.md | 14 +- docs/tutorials/implementing-detectors.md | 36 +- src/ophyd_async/core/__init__.py | 19 +- src/ophyd_async/core/_data_providers.py | 207 ++- src/ophyd_async/core/_detector.py | 1562 ++++++++++------ src/ophyd_async/core/_readable.py | 114 +- src/ophyd_async/epics/adandor.py | 13 +- src/ophyd_async/epics/adaravis.py | 8 +- src/ophyd_async/epics/adcore/__init__.py | 4 + .../epics/adcore/_acquire_logic.py | 27 +- src/ophyd_async/epics/adcore/_data_logic.py | 254 ++- src/ophyd_async/epics/adcore/_detector.py | 46 +- src/ophyd_async/epics/adcore/_io.py | 11 + .../epics/adcore/_trigger_logic.py | 74 +- src/ophyd_async/epics/adkinetix.py | 8 +- src/ophyd_async/epics/admerlin.py | 8 +- src/ophyd_async/epics/adpilatus.py | 16 +- src/ophyd_async/epics/adsimdetector.py | 8 +- src/ophyd_async/epics/advimba.py | 8 +- .../fastcs/eiger/_acquire_logic.py | 7 +- src/ophyd_async/fastcs/eiger/_detector.py | 30 +- .../fastcs/jungfrau/_acquire_logic.py | 7 +- src/ophyd_async/fastcs/jungfrau/_detector.py | 33 +- src/ophyd_async/fastcs/odin/_data_logic.py | 31 +- .../fastcs/panda/_acquire_logic.py | 10 +- src/ophyd_async/fastcs/panda/_data_logic.py | 39 +- src/ophyd_async/fastcs/panda/_detector.py | 18 +- src/ophyd_async/sim/_blob_acquire_logic.py | 28 +- src/ophyd_async/sim/_blob_data_logic.py | 38 +- src/ophyd_async/sim/_blob_detector.py | 29 +- tests/container_tests/test_adsim_system.py | 12 +- tests/unit_tests/core/test_cancel_swallow.py | 12 +- tests/unit_tests/core/test_data_providers.py | 63 + .../core/test_detector_use_cases.py | 1618 +++++++++++++++++ tests/unit_tests/core/test_readable.py | 80 + .../unit_tests/core/test_standard_detector.py | 838 ++++++--- tests/unit_tests/core/test_trigger_info.py | 4 +- .../epics/adcore/test_acquire_logic.py | 34 +- .../epics/adcore/test_cont_acq_detector.py | 79 +- .../epics/adcore/test_data_logic.py | 217 ++- .../unit_tests/epics/adcore/test_detector.py | 60 +- .../unit_tests/epics/adcore/test_use_cases.py | 154 +- tests/unit_tests/epics/test_adandor.py | 20 +- tests/unit_tests/epics/test_adaravis.py | 19 +- tests/unit_tests/epics/test_adkinetix.py | 12 +- tests/unit_tests/epics/test_admerlin.py | 6 +- tests/unit_tests/epics/test_adpilatus.py | 33 +- tests/unit_tests/epics/test_adsimdetector.py | 6 +- tests/unit_tests/epics/test_advimba.py | 12 +- .../epics/test_default_trigger_info.py | 2 +- .../fastcs/eiger/test_eiger_detector.py | 4 + .../fastcs/jungfrau/test_controller.py | 18 +- tests/unit_tests/fastcs/odin/test_odin_io.py | 19 +- .../unit_tests/fastcs/panda/test_hdf_panda.py | 121 +- .../plan_stubs/test_ndattributes.py | 2 +- 63 files changed, 5450 insertions(+), 1284 deletions(-) create mode 100644 docs/explanations/decisions/0023-bounded-data-logic-tier.md create mode 100644 docs/explanations/decisions/0024-detector-is-a-flyable.md create mode 100644 tests/unit_tests/core/test_data_providers.py create mode 100644 tests/unit_tests/core/test_detector_use_cases.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0facdf1bad..f8e6503d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,15 +19,22 @@ - `MovableLogic.stop()` is now the abstract `ensure_stopped()`, which must wait until the motion has stopped, and `StandardMovable.stop()` returns once it has. A subclass that still defines `stop()` gets a `DeprecationWarning`, and its `stop()` is used as `ensure_stopped()`. `Motor` now waits for `.DMOV` after writing `.STOP`, allowing its acceleration time to decelerate, and `InstantMotorMock` finishes a move on `.STOP`. - `ophyd_async` now needs `typing-extensions>=4.14`, for `KICKED_OFF`. Type-checking a `fly` annotated with `KICKED_OFF` needs pyright 1.1.410 or later. - Renamed `MotorFlyCtx` to `MotorPrepareCtx`, `PmacFlyCtx` and `PmacPrepareContext` to `PmacPrepareCtx`, and `SimMotorMoveLogic` to `SimMotorLogic`. -- A flyer must now be staged before it is prepared: `prepare()` is only allowed after `stage()`, `kickoff()` after `prepare()` and `complete()` after `kickoff()`, and a verb in the wrong state raises `TransitionError` (a `RuntimeError`). Staging an already-staged flyer, even one with a verb running, used to silently reset it and now raises (see below). A fly that fails leaves the flyer staged. `complete()` returns the same status each time it is called after a `kickoff()`, until the next `prepare()` or `unstage()`. After `complete()` a flyer can be prepared again without restaging. Stock bluesky plans that stage their devices (`bp.count`, `bp.scan`) keep working; `bp.fly` neither stages nor prepares, so stage the flyer and prepare it before `bp.fly`, for example with `bpp.stage_wrapper`. +- A flyer must now be staged before it is prepared: `prepare()` is only allowed after `stage()`, `kickoff()` after `prepare()` and `complete()` after `kickoff()`, and a verb in the wrong state raises `TransitionError` (a `RuntimeError`). Staging an already-staged flyer, even one with a verb running, used to silently reset it and now raises (see below). A fly that fails leaves the flyer staged. `complete()` returns the same status each time it is called after a `kickoff()`, until the next `prepare()` or `unstage()`. After `complete()` a flyer can be prepared again without restaging, but a prepared flyer cannot be prepared again until its fly has completed (a detector also after a `trigger()`). Stock bluesky plans that stage their devices (`bp.count`, `bp.scan`) keep working; `bp.fly` neither stages nor prepares, so stage the flyer and prepare it before `bp.fly`, for example with `bpp.stage_wrapper`. - Staging any already-staged Standard device (`StandardReadable`, `StandardMovable`, `StandardFlyable`, ...) now raises `TransitionError`, as ophyd raises `RedundantStaging`. For example, nesting `bpp.stage_wrapper` around a plan that also stages the same device now fails. - `unstage()` now cancels the `prepare()` or fly that is running. `unstage()` waits for it before calling `ensure_stopped()`, then raises any error it raised while being cancelled. - `MovableLogic` and `FlyableLogic` now share the base `Logic`, an `ABC`, so a subclass missing an abstract hook (`ensure_stopped`, `prepare_to_fly` or `fly`) raises `TypeError` when it is constructed. See ADR 0022 for the reasoning behind these hooks. +- `DetectorAcquireLogic.start_acquiring()` and `wait_for_idle()` are replaced by one `acquire()`, an async generator that starts acquiring, yields once armed, then waits until idle. `StandardDetector` enters it in `prepare()` for external triggering and in `kickoff()` or `trigger()` for internal triggering, runs it to the end once the data is written, and closes it at its `yield` if the scan is cancelled, so it must own what it starts (`async with` a status, or cancel a task in `finally`). An `acquire()` that finishes without yielding, or yields more than once, raises `RuntimeError` naming the logic class. `ADAcquireLogic` and `ADContAcqAcquireLogic` no longer have an `acquire_status` attribute, nor `BlobAcquireLogic` a `task`. +- `unstage()` now cancels a running `StandardDetector.trigger()`, and `kickoff()` raises `TransitionError` while one is running. A failed `trigger()` leaves the detector staged, so the next `trigger()` prepares again. +- A `StandardDetector` with no data logic can now be kicked off: its fly has nothing to collect and completes once the acquisition has. `kickoff()` after `trigger()` raises the state machine's `TransitionError` (`cannot go to FLYING from TRIGGERED`). +- A `BoundedDataLogic` that served the first point of a step scan and returns `None` at a later one raises `RuntimeError`. A data provider carried over to the next `prepare()` publishes from that prepare's baseline, so its first event page or stream datum no longer includes collections written before it. +- `StreamableDataProvider.make_stream_docs(collections_written, collections_per_event)` is now `make_stream_docs(collections_from, collections_to, collections_per_event)`, the range of collections to publish, as for `PageableDataProvider.make_page`; the detector keeps the count of what it has published. `StreamResourceDataProvider.last_emitted` is removed, and an override of `make_stream_docs` must take the new arguments and keep no count of its own. +- A failed `StandardDetector.prepare()` stops every data logic but leaves the collect verb of the last `prepare()` bound; calling it raises until the next `prepare()`. - `SimMotor` is now a `StandardFlyable`, so like any flyer it must be staged before `prepare()`. - `SimMotor.stage()` now calls `SimMotorLogic.ensure_ready()`, which calls `ensure_stopped()` and so stops the simulated motion, waits for it to finish, and sets the setpoint to the current readback. `unstage()` cancels a fly in progress, then calls `ensure_stopped()`. - The progress of `StandardFlyable.prepare()` now comes from the `WatcherUpdate`s the logic's `prepare_to_fly` yields, replacing the heuristic that watched setpoint writes (#1448). - `WatchableAsyncStatus` closes the async generator it drives as soon as it is cancelled or fails, rather than when the generator is collected, and a watcher that raises is logged and ignored rather than failing the status. - A `StandardFlyable` whose fly failed after `kickoff()` and was never completed logs a warning when it is unstaged or prepared again, instead of leaving the failure unretrieved. +- A `StandardReadable` that a detector suppresses with its data keeps the hints of the signals it drops from `read()` and `describe()`, as the data has the same name. A data logic that hints such a name now makes `hints` raise. - `StandardReadable` now keeps one format per registered child, keyed by the child, so registering a child again replaces its format instead of adding a second contribution, and the order of `hints` is the order children were first registered. The private `_describe_config_funcs`, `_read_config_funcs` and `_has_hints` tuples are gone: a subclass that added to them now adds a hook to `_describe_configuration_funcs`, `_read_configuration_funcs` or `_hints_funcs` (see ADR 0025). - The deprecated `ConfigSignal` and `HintedSignal` aliases are removed from `ophyd_async.core`; use `StandardReadableFormat.CONFIG_SIGNAL`, `HINTED_SIGNAL` or `HINTED_UNCACHED_SIGNAL`. `StandardReadable.add_readables()` is deprecated in favour of `set_readable_format()` on each device. - `store_settings()` writes the readable format changes of the Device tree under the reserved key ``, and `apply_settings()` resets the tree to its class formats before applying them. Settings files stored before this have no such key and leave formats alone. `retrieve_settings()` warns about and drops any other `<...>` key. @@ -36,6 +43,12 @@ - The `prepare_single` data logic tier is removed, with `ReadableDataProvider`, `SignalDataProvider` and `adcore.PluginSignalDataLogic`. Register a plugin scalar such as `NDStatsIO.total` with `set_readable_format(signal, StandardReadableFormat.HINTED_SIGNAL)` instead. - The areaDetector driver and plugin classes (`ADBaseIO`, `NDROIIO`, `NDStatsIO` and `NDROIStatNIO`) are now `StandardReadable`s that declare their own configuration and read signals, so registering a plugin as `CHILD` of an `AreaDetector` puts them in `read()` and `read_configuration()`. - `ADAcquireLogic` sets `WaitForPlugins` on `stage()`, from its new `wait_for_plugins` argument (default on), rather than `AreaDetector` hiding it, so a step scan reads plugin scalars for the frame it has just taken. +- `StandardDetector` is now a `StandardFlyable` whose `logic` is a `DetectorLogic` holding the trigger, acquire and data logics (see ADR 0024). A detector builds it in a `logic` `@cached_property` rather than passing the logics to `StandardDetector.__init__()`, which takes no arguments, and `add_detector_logics()` is removed. Use `DetectorLogic.with_device()` for an ad-hoc detector. Like any flyer it follows the `stage()`, `prepare()`, `kickoff()`, `complete()` ordering and raises `TransitionError` out of order. +- Data logics are now either a `BoundedDataLogic`, told the number of collections in the scan, or an `UnboundedDataLogic`, and implement one `start_provider()` that sets up and arms the writer and returns a `StreamableDataProvider`, a `PageableDataProvider` or `None` to sit the stage out (see ADR 0023). `prepare_unbounded()` is replaced by it. Which logics serve is decided at the first `prepare()` after `stage()` and holds until `unstage()`, and no two serving providers may make the same datakey. +- `StandardDetector` exposes `collect_asset_docs()` or `collect_pages()`, whichever the serving providers produce, as an instance attribute bound by `prepare()`, so the bluesky run bundler asks for the right one. +- A registered signal with the same name as a datakey a serving provider makes is left out of `read()`, `describe()` and `hints`, so the provider's data replaces it; `describe()` raises `ValueError` if the two differ in dtype or shape beyond one leading dimension. +- `TriggerInfo.number_of_events` must now be positive. +- `adcore.ADWriterFactory` is renamed `ADDataFactory`; the old name still works with a `DeprecationWarning`. Its factories take `always_enable` (default `True`): `False` follows the plugin's `EnableCallbacks` PV and sits the plugin out if it is off. ### Added @@ -49,6 +62,7 @@ - `StandardReadable.set_readable_format()`, `reset_readable_formats()`, `get_readable_formats()` and `get_readable_format_changes()`, to change how a child contributes to `read()`, `read_configuration()` and `hints` at runtime, the equivalent of `Kind` in ophyd v1 (see "How to change what a Device reports at runtime"). - `ReadableFormats`, `walk_readable_formats()`, `apply_readable_formats()` and `READABLE_FORMATS_KEY` in `ophyd_async.core`, and a `readable_formats` field on `Settings`, so formats are stored and retrieved with the signal values. - pyright now runs with `reportMatchNotExhaustive`. +- `BoundedDataLogic`, `UnboundedDataLogic`, `DetectorLogic`, `DataProvider`, `PageableDataProvider` and `EventPageDataProvider` in `ophyd_async.core`, and `ADDataFactory` and `exposures_per_collection_from_filter` in `ophyd_async.epics.adcore`. ### Fixed @@ -57,6 +71,7 @@ - `unstage()` now ends a running `complete()`, such as for the PandA. - A `prepare()` that fails or is cancelled now stops the move to the run-up start of a `Motor` or `SimMotor`. - A failed `prepare()` no longer leaves the flyer prepared with the failed context, so there is nothing to kick off. +- `unstage()` closes an acquisition that an externally triggered `StandardDetector.prepare()` armed and nothing ran, rather than leaving it to be closed when garbage collected. - `set_and_wait_for_other_value()` cancelled before the first value arrives, such as by `stop()` during a move, no longer leaves its value-watching task running. - A signal `get_value()`, `set()` or `trigger()` cancelled just as it finishes no longer swallows the cancellation on Python 3.11. -- Leaving an `async with AsyncStatus(...)` block, `set_and_wait_for_other_value()` and the sim `BlobAcquireLogic.ensure_stopped()` no longer swallow a cancellation of the caller that arrives while they wait for the task they cancelled. +- Leaving an `async with AsyncStatus(...)` block, `set_and_wait_for_other_value()` and closing the sim `BlobAcquireLogic.acquire()` no longer swallow a cancellation of the caller that arrives while they wait for the task they cancelled. diff --git a/docs/conf.py b/docs/conf.py index 695b7f8ea4..e68dd74102 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -157,6 +157,7 @@ def setup(app: application.Sphinx): "ophyd_async.core._standard_base._StandardBase", "ophyd_async.core._flyable.PrepareT", "ophyd_async.core._flyable.CtxT", + "ophyd_async.core._detector.ProviderT", "ophyd_async.core._utils.P", "ophyd_async.core._utils.P.args", "ophyd_async.core._utils.P.kwargs", @@ -193,6 +194,16 @@ def setup(app: application.Sphinx): "0.001", "1.0", "bluesky.protocols.T_co", + # TypeVars documented on StandardFlyable, added in #1359 + "PrepareT", + "CtxT", + # event_model publishes no intersphinx entry for this TypedDict + "event_model.documents.PartialEventPage", + # TypeVar for the plugin an ADDataFactory builds a data logic around + # The context DetectorLogic threads from prepare to unstage, private + # because nothing outside StandardDetector constructs or inspects one + "ophyd_async.core._detector.DetectorCtx", + "ophyd_async.core._detector._DetectorData", ] nitpick_ignore = [] for var in obj_ignore: diff --git a/docs/explanations/decisions/0012-detector-rewrite.md b/docs/explanations/decisions/0012-detector-rewrite.md index 2e059e6571..93ee54f95d 100644 --- a/docs/explanations/decisions/0012-detector-rewrite.md +++ b/docs/explanations/decisions/0012-detector-rewrite.md @@ -55,7 +55,7 @@ We will restructure `StandardDetector` to use composition with three separate lo - **`StreamableDataProvider`** - For streaming data (appears in StreamDatum documents) - `make_datakeys(collections_per_event)` - Generate DataKey descriptions - - `make_stream_docs(collections_written, collections_per_event)` - Emit StreamAsset documents + - `make_stream_docs(collections_from, collections_to, collections_per_event)` - Emit StreamAsset documents - `collections_written_signal` - Signal tracking write progress ### Detector Changes diff --git a/docs/explanations/decisions/0023-bounded-data-logic-tier.md b/docs/explanations/decisions/0023-bounded-data-logic-tier.md new file mode 100644 index 0000000000..dc90eaf120 --- /dev/null +++ b/docs/explanations/decisions/0023-bounded-data-logic-tier.md @@ -0,0 +1,274 @@ +# 23. Add a bounded `DetectorDataLogic` tier + +Date: 2026-07-17 + +## Status + +Accepted + +## Context + +[](./0012-detector-rewrite.md) split data production out into `DetectorDataLogic`. After +[](./0025-runtime-readable-format.md) removed the single-event tier, one tier was left: a +logic prepared for an unbounded number of collections, feeding `collect_asset_docs()` and +`describe_collect()` and doing its own writes as it went. + +Four open issues bottom out in two gaps. + +### The data logic is not told enough at prepare time + +Issue #1309 wants chunk size and flush rate configured from the scan parameters. An Xspress +running at 400 Hz with a 2 Hz flush rate wants 200 frames per chunk; the PandA wants a flush +period. Neither is expressible, because an unbounded logic is told nothing about timing: +`ADHDFDataLogic` reads `num_frames_chunks` back from the IOC and forces it to 1 if unset, the +PandA hardcodes `chunk_shape=(1024,)` behind a TODO, and Odin hardcodes `(1, *data_shape)`. + +The parameter actually needed is the frame **period** (`livetime + deadtime`), not the number +of frames, since the requirement is a flush *rate* — and the period is known for step scans as +well as fly scans, where the total frame count is not. + +Two workarounds were rejected: subclassing `TriggerInfo` and `prepare()` to carry a chunk size +(as Malcolm did) puts a data-writing concern in the trigger path, against +[](./0012-detector-rewrite.md)'s rule that a logic is handed only the fields it needs; +computing it in the `DetectorTriggerLogic` puts a data concern in the trigger logic. + +### Detectors that produce data but cannot write files + +Issues #1248 and #888 come from a Struck SIS3820 scaler in a VME crate, a MeasComp CTR-08, a +TetrAmm electrometer, an electron analyser. They want the trigger, arm, prepare and stage +machinery of `StandardDetector` but must emit event pages, and `StandardDetector` inherited +`WritesStreamAssets` unconditionally — the bundler calls `collect_pages()` only on a device +that is *not* one. (#888 asks for the writer to be optional, which the rewrite already did; +its real request is #1248, so it is a duplicate.) + +These are not areaDetectors, and they put more pressure on the trigger/acquire/data split. A +CTR-08 has no driver/plugin split in the hardware: `MCS:EraseStart` both clears the arrays and +starts counting, and `MCS:NuseAll` is both the exposure count and the buffer size, where an +areaDetector acquires with `driver.acquire` and captures with a separate `TSAcquire` or HDF +`capture`. + +### These are the same gap + +Both groups must be told *how many* collections to expect at prepare time, because they own a +finite buffer that has to be sized before acquisition: the scaler's MCA array length *is* the +frame count, and the stats plugin's `TSNumPoints` must be set before the time series starts. A +device with a finite buffer also naturally produces its data as pages at the end of an event +rather than as stream datums as it goes. + +So one new tier serves both, and #1309's period rides along with it. + +### The areaDetector stats duality + +An `NDPluginStats` exposes each statistic twice: as a scalar (`Total_RBV`) and as a time +series array (`TSTotal`). Only the scalar was modelled, as a registered signal read once per +event, so a detector with no file writer had no way to produce a value per collection for a +fly scan. + +## Decision + +### One method that starts and describes + +```python +ProviderT = TypeVar("ProviderT", bound=DataProvider) + +class DetectorDataLogic(Generic[ProviderT]): + def get_hinted_fields(self, name: str) -> Sequence[str]: ... + +class UnboundedDataLogic(DetectorDataLogic[ProviderT]): + async def start_provider( + self, name: str, collections_per_event: int, + collection_period: float, flush_period: float, + ) -> ProviderT | None: ... + +class BoundedDataLogic(DetectorDataLogic[ProviderT]): + async def start_provider( + self, name: str, existing_datakeys: Sequence[str], num_collections: int, + collections_per_event: int, collection_period: float, flush_period: float, + ) -> ProviderT | None: ... +``` + +`name` is the detector name: a logic that needs a suffix on its datakeys, such as +`ADHDFDataLogic`'s `datakey_suffix`, applies it itself, as it does in `get_hinted_fields(name)`. + +Whether a source must be told how many collections to take is a property of the logic's +*type*, so a logic inherits from exactly one of `BoundedDataLogic` and `UnboundedDataLogic` +and implements the one abstract method of that class. + +`start_provider` does all the work: it works out the path, sets up and arms the writer, and +returns a provider describing what is being written. Returning `None` sits the logic out, and +it decides that without writing anything. At the first `prepare()` of a stage the detector +starts every unbounded logic, then every bounded one, handing each bounded logic the datakeys +the unbounded providers make. The bounded provider must make none of them: it publishes the +rest, or the logic returns `None` if nothing is left. The logics that return a provider serve +the stage until `unstage()`, and no two providers may make the same datakey. + +Describing and starting were split at first, into a `make_*_provider` that described without +writing and a `start` handed the provider back, so the detector could choose between providers +before anything was written. That made the provider carry what `start` needed (a `PathInfo`, a +collection count) and was a poor fit for the logics that cannot describe their data until they +are writing: Odin, whose frame shape is only readable once the file processor runs, and the +PandA on PandABlocks-ioc, whose `DATASETS` table is empty until capture starts. Telling a bounded +logic what the unbounded providers already make removes the choice, so nothing needs to be +described before it is started. A single `prepare_*` per tier was tried before that, and +discarded for the same reason: it armed hardware to find out what it would get. + +`num_collections` is the **total** for the scan (`TriggerInfo.number_of_collections`), and +both kinds are told the collection period, `exposures_per_collection * (livetime + deadtime)`, +so either can size a chunk or a buffer against it: a writer after a processing chain that +averages exposures sees one frame per collection. + +### The tier is the type of the provider returned + +There is no declaration of which tier a logic implements and no override detection: a logic +returning a [](#StreamableDataProvider) publishes stream assets for this scan, one returning a +[](#PageableDataProvider) publishes event pages, and one may return either depending on what it +is asked for. Whether a logic needs a fixed number of collections is its type, bounded or +unbounded, not a publication tier. Returning `None` means "not this stage", for example a plugin +that is switched off. It is neither an error nor warned about, as +agreed in #1364: the point is that a detector may carry more logics than one scan uses. + +### `PageableDataProvider`, with readings derived from pages + +The pageable tier returns one provider type, which emits pages. Its +`make_page(collections_from, collections_to, collections_per_event)` is stateless: the detector +tracks what it has published. Readings are derived from pages rather than being a separate +code path: `read()` asks each provider for the current step point's event only, which extracts +to a single reading, so a second `read()` gives the same answer. +`collections_per_event` appears in the datakey shape exactly as for a +`StreamableDataProvider`. + +### The period is resolved in `prepare()` + +`TriggerInfo.livetime` of 0 means "whatever is currently set on the detector", so the period +cannot simply be forwarded: an explicit `prepare(TriggerInfo(number_of_events=10))` would hand +the data logic a period of 0. Instead `DetectorTriggerLogic.default_trigger_info()` is extended +to return the current `livetime` and `deadtime`, and `prepare()` fills a zero `livetime` in +from it *after* preparing the trigger logic, which [](./0012-detector-rewrite.md) already +requires to come first, so the hardware holds the real values by then. This reads hardware +state, so like the implicit prepare in `trigger()` it happens only when +`OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES`; otherwise a zero `livetime` stays 0 and data logics +get a period of 0, meaning "use what is on the hardware". + +### Unbounded providers carry over, bounded providers are re-made + +Within one stage, an unbounded provider that is serving is carried over to the next +`prepare()` without its logic being asked again, so a step scan does not reopen its file on +every point and a source that keeps counting is not restarted. It does start a new +`StreamResource` in the same file, its indices carrying on, as a run cannot refer to the +resource of an earlier one. It carries over while the +datakey name, `collections_per_event`, period and flush period are unchanged, because they are +baked into a `StreamResource` when it is made (the shape, the chunking) and areaDetector's +`num_frames_chunks` can only be set before `capture` starts. When any of them changes, the old +provider is stopped *before* its logic is started again, so a logic's `start_provider` never +writes while its old provider runs. A logic that returned `None` at the first `prepare()` of the +stage is not asked again until the next stage: it would count from 0 while a carried provider +counts on. So whether a logic serves is decided once, at the first `prepare()`; the file +writers with `always_enable=False` read `EnableCallbacks` there. The detector counts the events published since a scan started, once for all its +providers, so a carried provider's first page or stream datum of the next `prepare()` starts at +that scan's baseline, not at what the last one left unpublished. + +`unstage()` stops everything, so a change to Settings (a plugin's `EnableCallbacks` switched +off, say) takes effect at the next `stage()`, not between the `prepare()` calls of one stage. +A carried writer whose plugin is switched off by hand mid-stage is not switched back on, as +Settings changes belong before `stage()`. + +A bounded logic is started again on every `prepare()`, and on every `trigger()` after the first +one since that `prepare()` it is stopped and started again: a finite buffer is re-armed that +way, and an unbounded source keeps counting. One that served the first +point and returns `None` at a later one raises, as the scan's datakeys are already fixed. `kickoff()` never does +this. That is exactly the set of places a buffer should and should not be re-armed: the first +point of a step scan is already armed by the `prepare()` in front of it, and re-arming it +again would erase an empty buffer, putting a spurious stop/start on the arming control before +a frame had arrived. + +A bounded streamable logic therefore gives one `StreamResource` per arm, which is one per step +point. A logic that wants one resource per scan should be an `UnboundedDataLogic`. + +### The progress baseline is read live + +`trigger()` reads every provider's starting progress after re-arming the bounded logics, and +`kickoff()` reads it too, so a source that keeps counting is measured from where it is. There is +one baseline per bound group: bounded providers are in step with each other (they start at 0), +and unbounded providers are in step with each other (they start wherever their source is), and +progress is reported in one frame. A stats time series is erased by the `TSAcquire` write in its +`start_provider`, which waits for `TSCurrentPoint` to read 0, so its baseline is 0. Whoever +resets a provider's collections-written counter does it in its logic's `start_provider`, and +waits until it reads 0 before returning. A CTR-08-style logic therefore does `EraseStart` in its +data provider, and its acquire logic does not erase; the baseline is then never stale. + +### Erase controls belong to the data provider + +Where one control both starts acquisition and clears the data buffer, the data logic fires it +in `start_provider`, because the erase is what resets the counter the detector reads. The +`DetectorAcquireLogic` keeps the stop control and the idle status, and does not erase. A data +logic's `start_provider` runs before `acquire()` arms, so the buffer is sized before the +control fires. + +The trigger/acquire/data split therefore survives these devices, with the three logics sharing +one IO device exactly as the areaDetector logics share a `driver`. The split is of concerns, +not of hardware. + +### Logic objects fill exactly one role + +`DetectorLogic(trigger=..., acquire=..., data=(...))` takes each role by keyword, so an object +fills the role it is passed as and no other. + +### A detector produces stream assets or event pages, never both + +The bundler collects stream assets or event pages from a device, not both, so a detector +whose logics would produce both raises. How the +applicable verb is exposed, and the one case where the two may be carried together, are in +[](./0024-detector-is-a-flyable.md). + +Which kind a detector produces is decided per prepare, so a single detector class can be either +depending on its `__init__` arguments — one `AravisDetector`, configured for files or for pages +— which was the original requirement. That it can also change *between runs*, when a file +writer is switched off and a stats time series takes over, is documented rather than enforced: +a run's descriptor is emitted at its start, so switching part way through would leave +`describe_collect()` and the documents disagreeing, and a Device cannot see run boundaries. + +### Stats with a file writer go via NDAttributes + +An areaDetector with an HDF writer does not need the stats time series: `ADHDFDataLogic` +already pulls NDAttribute datasets into the file, which is areaDetector's own mechanism for +exactly this. So registering the scalar with `set_readable_format` remains right for a +detector that writes files and wants a stats value in a step scan, and a stats time series is +right for one that writes none. + +### One stats data logic for many arrays + +An `NDPluginStats` has one time series control (`TSNumPoints`, `TSAcquire`, `TSAcquireMode`) +shared across roughly twenty arrays, so one stats data logic covers *many* arrays with the +control embedded, rather than one logic per array with a shared control object. This follows +the existing grain — a provider already emits many datakeys, as `ADHDFDataLogic` does for the +main dataset plus its NDAttributes — and avoids the coordination problem of several logics +contending over one `TSAcquire`. Which statistics are wanted becomes a constructor argument. + +## Consequences + +`DetectorDataLogic` implementations must be rewritten onto `BoundedDataLogic` or +`UnboundedDataLogic` and their `start_provider`, and take the new `collection_period` and +`flush_period` arguments. This is a breaking change for out-of-tree data logics; the library +is in alpha and no shim is provided. In-tree, `ADHDFDataLogic` sizes its chunk from +`TriggerInfo.flush_period` and the rate, and `PandaHDFDataLogic` sets its flush period signal +from the same field, replacing the TODOs that stood in for both. + +`DetectorTriggerLogic.default_trigger_info()` implementations should return the current +`livetime` and `deadtime` alongside the frame count. Those that do not still work, but their +detectors fall back to reading the chunk size back from hardware. + +Detectors that cannot write files become expressible: a data logic returning a +`PageableDataProvider` gives them `collect_pages()` and the whole of `StandardDetector` +besides. This closes #1248 and #888. Fly scanning an areaDetector that carries an HDF writer +plus a finite-buffer data logic starts working, closing #1364. + +A data logic may decline a scan for reasons of its own: `ADHDFDataLogic` and +`StatsTimeSeriesDataLogic` take an `always_enable` flag, defaulting to today's behaviour of +switching the plugin on, which when `False` makes them follow whatever the plugin is set to +(at the first `prepare()` of a stage) and produce nothing when it is off. + +A device whose exposure count and buffer size are the same PV has it written twice: by the +trigger logic as `number_of_exposures`, and by the data logic as `num_collections`. The two +agree only when `exposures_per_collection` is 1. This is latent rather than live, because such +devices do not average exposures and so cannot ask for anything else, but a detector that could +do both would need its data logic to read the value back rather than set it. diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md new file mode 100644 index 0000000000..df84b3b7df --- /dev/null +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -0,0 +1,215 @@ +# 24. StandardDetector is a StandardFlyable + +Date: 2026-08-28 + +## Status + +Accepted + +## Context + +`StandardDetector` re-implemented `prepare`, `kickoff` and `complete` and threaded its own +state between them, while [](#StandardFlyable) already existed to do exactly that for every +other flyer in the tree (`Motor`, PMAC, the PandA trigger logics). Two implementations of +one lifecycle drift apart: the detector guarded its stages by checking whether a context +object was `None`, so calling a verb too early failed with `AttributeError: 'NoneType' +object has no attribute ...` rather than saying what was wrong. + +`StandardFlyable` requires a single [](#FlyableLogic) in a `logic` property +([](./0021-single-logic-property.md)), and a detector had three logic objects registered on +the Device by `add_detector_logics()`. + +## Decision + +### The three logics become children of one `DetectorLogic` + +[](#DetectorLogic) holds the trigger, acquire and data logics, and is what `logic` returns. +The orchestration moves with them: preparing the trigger logic, making the data providers, +arming, and waiting for collections are all its methods now. It holds **no** back-reference +to its Device, so [](./0017-standard-movable.md) continues to hold unqualified. + +A detector builds it in `logic`: + +```python +class MyDetector(StandardDetector): + def __init__(self, prefix: str, name: str = "") -> None: + self.driver = MyDriverIO(prefix) + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=MyTriggerLogic(self.driver), + acquire=MyAcquireLogic(self.driver), + ) +``` + +`add_detector_logics()` is removed rather than kept as a forwarder: a detector's logics are +fixed for its lifetime, so there is nothing a post-construction call can do that the +constructor cannot, and keeping it would leave two ways to wire the same Device. + +`StandardDetector` itself takes no `__init__` arguments, since a `Standard*` class that did +would interfere with multiple inheritance. The logic is supplied by overriding `logic`, +which `abstract_cached_property` makes a requirement checked when the Device is +instantiated. Assigning `self.logic` from `__init__` cannot serve instead: `ABCMeta` checks +`__abstractmethods__` in `__call__`, before `__init__` runs, so the class must override the +property to be concrete. For an ad-hoc detector in a plan or a test, +`DetectorLogic.with_device()` builds a Device around a logic, exactly as +`FlyableLogic.with_device()` does for any other flyer. + +### One thing the logic cannot get from its own fields + +Data logics name their datakeys after the Device, whose name does not exist until +`init_devices` calls `set_name` on context exit. `StandardFlyable` runs hooks a subclass +registers with the context as soon as `prepare_to_fly` yields it. `StandardDetector` +registers one, which writes its name and the description of its registered signals into the +context, then calls `DetectorLogic.serve_data` with it to choose and start the data +providers. It passes its name to `get_hinted_fields()` too. The logic holds no reference to +the Device: `StandardFlyable` calls `prepare_to_fly` with the `TriggerInfo` alone, so the +context is where the Device leaves what serving needs. + +### Which collect verb to expose is the Device's business + +A detector produces stream assets or event pages depending on what its data logics make for +a given scan, never both (see [](./0023-bounded-data-logic-tier.md)). The bluesky bundler +decides which to ask for from the *static* type — `collect_pages()` is called only on a +device that is not a `WritesStreamAssets` — so a detector that declared both would have its +pages silently dropped, and one that declared neither could not fly at all. + +So the hook `StandardDetector` registers for `prepare()` binds whichever verb applies as a +real instance attribute once the data is served, and removes the other. It is the Device that +does this, from what `DetectorLogic.produces` says the serving providers make; the logic is +not told where that goes, so it holds no back-reference. Only a `prepare()` that serves binds +or unbinds: nothing asks for a collect verb before a `prepare()`, and the step-boundary re-arm +on each point of a step scan cannot change which verb applies, because it changes no +provider. A failed `prepare()`, `kickoff()` or `complete()` leaves the verb bound until the +next `prepare()`, and calling it then raises. + +Choosing from the static type is a bundler limitation rather than a fact about detectors: a +device knows what it produced, and the bundler could decide from *that*, erroring only if +one device reported both ways in the same run. That change is a few lines in +`RunBundler.collect`, and with it both verbs become plain methods and the binding here goes +away. Until it is released, binding per prepare is what works against a released bluesky. + +### Mixing pageable and streamable is an error unless the pageable keys are suppressed + +[](./0023-bounded-data-logic-tier.md) made carrying both kinds of data logic an error, because +the bundler treats stream assets and event pages as mutually exclusive. Carrying both is still +useful, though: the same quantity can be written durably into a file *and* read from a plugin's +buffer — an areaDetector stats total, which the HDF writer pulls in as an NDAttribute. + +So the rule relaxes. A bounded logic is started after the unbounded ones and handed the +datakeys they make, which its provider leaves out. Where the stream assets make **every** +datakey a finite buffer would produce, the durable copy wins and the buffer sits the scan out, +unarmed. A registered signal is suppressed the same way by a provider key, and is +dropped from `read()` and `describe()` while the detector is prepared. Its hint stays, as the +provider key has its name. +A registered signal gives one value per event, so a provider key of its dtype and of its shape, +or of that shape with one extra leading dimension (`[collections_per_event, *shape]`), suppresses +it. Any other dtype or shape is a `ValueError` at `describe()`: two different things share a +name, so one must be renamed. A registered signal that nothing suppresses is read once per event. +Nothing has been started when that is decided, because a logic's `start_provider` returns +`None` without writing (same ADR). + +Suppression is one case of a general `StandardReadable` hook: a mapping from datakey name to +either `None` or the datakey that replaces it, matched by name after the registered children +have been read, so a signal is found wherever it is registered. `None` drops the name from +`read()`, `describe()` and `hints`: it is for a signal that is switched off, which has no +replacement. A datakey drops it from `read()` and `describe()` after checking the datakey can +replace it, and leaves it in `hints`, because the replacement has the same name. A data logic +that hints the name of a signal it replaces makes `hints` raise, as two hints of one field +always do. + +### The detector flies with one `fly()` generator + +`FlyableLogic.fly` is a single async generator ([](./0022-fly-generator-and-ensure-stopped.md)): +it yields `KICKED_OFF` once flying, then `WatcherUpdate`s, and `complete()` finishes when it +returns. A detector's progress is "collections written out of collections requested", so +`DetectorLogic.fly` notes where the data providers have got to, arms an internally triggered +acquisition, yields `KICKED_OFF`, then yields an update each time the collections written move +on, and ends once the acquisition has returned to idle. `StandardFlyable` runs it in the task +of the status `complete()` returns and cancels it on `unstage()`, so the detector needs no stop +logic beyond its `ensure_stopped`, which `unstage()` always calls, like any flyer. + +### An acquisition is a generator too + +`DetectorAcquireLogic` started acquiring and waited for idle in two hooks, `start_acquiring` and +`wait_for_idle`, so whatever the first started -- the put-callback status of `Acquire`, the sim +detector's writing task -- had to be held on the logic between them. Cancelling the fly between +the two left it running until `ensure_stopped`, against the rule that a hook never leaves +anything running that it does not wait for. + +So they become one async generator, `acquire()`, in the same idiom as `fly`: start, yield once +armed, wait for idle. It owns what it starts (`async with` a status, or cancelling a task in a +`finally`), so closing it at its `yield` ends it. It takes no trigger information: the same +generator serves both kinds of triggering, and only where it is entered differs. An externally +triggered detector arms in `prepare()`, so that it is waiting on its trigger source by the time +`kickoff()` returns; the armed acquisition is held on the logic until `fly` (or `trigger()`) +takes it and runs it to the end, closing it if cancelled. An internally triggered one is +entered by `fly` or `trigger()` itself. One still armed at `unstage()` is closed by +`ensure_stopped`, so its cleanup runs then, not when it is garbage collected after a restage. +A prepared detector cannot be prepared again before it is kicked off or triggered, so no +`prepare()` replaces an armed acquisition. + +### One kickoff per prepare + +The detector used to be prepared for N events and kicked off a few at a time, with an +`events_to_kickoff` signal saying how many the next kickoff covered and an `is_last_kickoff` +flag deciding when to wait for idle. Nothing in tree drove it: the RunEngine kicks off once +and completes what it kicked off, and every other flyer already worked that way. + +Both go, and `kickoff()` requests the whole prepared scan. A second `kickoff()` without an +intervening `prepare()` now raises, from `StandardFlyable`, like any other flyer. A detector +with no data logic flies too, with nothing to collect: it is a valid flyer for its triggering +alone. + +### Unbounded providers are the stage's, bounded providers the prepare's + +`_FlyCtx` holds the `TriggerInfo` that `prepare()` resolved, the serving bounded providers, +the datakeys and suppressed signals they and the unbounded ones make, the progress baselines and +a count of the `trigger()`s since, and of the events published since the scan started, one +count for all providers. The serving unbounded providers and an armed acquisition +live on the logic for the stage instead, because they outlive a single prepare: a step scan +reuses an open file across its points. A provider carried to the next `prepare()` publishes +from that prepare's baseline. The data verbs — `read`, `describe`, `describe_collect`, +`collect_asset_docs`, `collect_pages`, `get_index` — call the logic with the context, which +`StandardFlyable` keeps after `complete()`, so `collect_while_completing` collects after +`complete()` has finished. + +`trigger()` therefore needs no context of its own, beyond a count of triggers on the fly +context. It is part of the state machine all the same: it runs in a `TRIGGERING` state entered +from `PREPARED` and ends in `TRIGGERED`, so `unstage()` cancels it, `kickoff()` cannot overlap +it, and a `kickoff()` after it needs a new `prepare()`. A failed `trigger()` leaves the detector +staged, as a failed `kickoff()` or `complete()` does. It prepares implicitly if nothing has been, then from the second point on calls each +serving bounded logic: it is stopped, asked for a new provider and started, so a finite +buffer holding one event at a time is re-armed, while an unbounded source carries on. The count is what keeps the first point from +being re-armed on top of the `prepare()` that just armed it. The trigger logic is not re-prepared per point, so +a step scan does not repeat its puts. + +### One context type, from `prepare_to_fly` to `fly` + +`FlyableLogic` threads a single `CtxT` from `prepare_to_fly` to `fly`. What only exists once +flying lives in `fly` itself, since one generator spans `kickoff()` and `complete()`, so no +field of the context waits for kickoff as `| None`. The context may be mutable: the detector +counts its `trigger()`s and notes its baselines on it. + +## Consequences + +- `StandardDetector` subclasses build a `DetectorLogic` in `logic`; + `det.add_detector_logics(*logics)` is gone. +- `kickoff()` → `kickoff()` without an intervening `complete()` is rejected, and so is a + second `kickoff()` after one. The old detector allowed both, because it only checked that + a prepare context existed. +- `events_to_kickoff` is removed from the public API. +- Calling a verb before `prepare()` says `: prepare() must be called first` instead of + `Prepare not run`. Before `stage()`, and after `unstage()`, every verb but `stage()` raises, + for every `StandardFlyable`. +- A detector satisfies neither `WritesStreamAssets` nor `EventPageCollectable` until it has + been prepared. Everything in the RunEngine that dispatches on them runs after prepare. +- `complete`, `kickoff`, `prepare`, `stage` and `unstage` are no longer implemented on + `StandardDetector`, nor are the private steps of `StandardFlyable` and `StandardReadable`, so + a change to the flyer lifecycle reaches detectors too. It adds to them only through the hooks + they run: one to serve the data and bind the collect verbs when `prepare_to_fly` yields the + context, and one giving the datakeys that replace registered signals. +- `trigger()` is cancelled by `unstage()`, and `kickoff()` raises while one runs. +- `DetectorAcquireLogic.start_acquiring()` and `wait_for_idle()` are replaced by `acquire()`. diff --git a/docs/explanations/plan-stubs.md b/docs/explanations/plan-stubs.md index 91010b5c8c..ebc2ef38dd 100644 --- a/docs/explanations/plan-stubs.md +++ b/docs/explanations/plan-stubs.md @@ -200,7 +200,7 @@ This code is to aid understanding rather than for copying. This will be document ### Fly scan nested within a step scan -After `stage` and `open_run` the detectors are `prepare`d for the entire scan. The inner loop `set`s the slow (software) motors to each point, followed by the hardware driven section of the scan. Just like the [](#pure-fly-scan), this consists of `prepare`, then `kickoff`, then `complete` of the fast (hardware) motors, `collect`ing from the detectors repeatedly until they are finished. Finally `close_run` and `unstage` do the cleanup. +After `stage` and `open_run` the inner loop `set`s the slow (software) motors to each point, followed by the hardware driven section of the scan. Just like the [](#pure-fly-scan), this consists of `prepare`, then `kickoff`, then `complete` of the detectors and fast (hardware) motors, `collect`ing from the detectors repeatedly until they are finished. A detector accepts one `kickoff` per `prepare`, so it is `prepare`d again for each hardware driven section. Finally `close_run` and `unstage` do the cleanup. ::::{tab-set} :sync-group: diagram-code @@ -213,9 +213,9 @@ After `stage` and `open_run` the detectors are `prepare`d for the entire scan. T :align: center flowchart TD stage["stage dets, motors"] --> open_run - open_run --> prepare_d["prepare dets"] - prepare_d --> set["set slow motors"] - set --> prepare_m["prepare fast motors"] + open_run --> set["set slow motors"] + set --> prepare_d["prepare dets"] + prepare_d --> prepare_m["prepare fast motors"] prepare_m --> kickoff["kickoff dets, fast motors"] kickoff --> complete["complete dets, fast motors"] complete --> collect["collect dets"] @@ -240,16 +240,17 @@ def fly_inside_step( exposure_time: float, readout_time: float, ) -> MsgGenerator: - yield from bps.prepare( - detector, - TriggerInfo( - livetime=exposure_time, - deadtime=readout_time, - number_of_events=exposures_per_row * len(columns), - ), - ) - for column in columns: + for column in columns: yield from bps.abs_set(column_motor, column) + # One kickoff per prepare, so the detector is prepared for each row + yield from bps.prepare( + detector, + TriggerInfo( + livetime=exposure_time, + deadtime=readout_time, + number_of_events=exposures_per_row, + ), + ) yield from bps.prepare( row_motor, FlyMotorInfo( diff --git a/docs/how-to/choose-right-baseclass.md b/docs/how-to/choose-right-baseclass.md index bbea3b6df6..7fa89dd5e0 100644 --- a/docs/how-to/choose-right-baseclass.md +++ b/docs/how-to/choose-right-baseclass.md @@ -10,9 +10,9 @@ When writing a new Device there are several base classes to choose from that wil There are some utility baseclasses that allow you to create a Device pre-populated with the right verbs to work in bluesky plans: -- [](#StandardReadable) allows you to compose the values of child Signals and Devices together so that you can `read()` the Device during a step scan. +- [](#StandardReadable) allows you to compose the values of child Signals and Devices together so that you can `read()` the Device during a step scan. What each child contributes can also be changed on a live Device, see [](./change-readable-format.md). - [](#StandardMovable) allows devices that move to a target value and wait for a readback to confirm arrival (motors, temperature controllers, piezo stages, and similar) to be implemented by composing a [](#MovableLogic) instance, rather than directly implementing the `Locatable`, `Stoppable`, and `Subscribable` protocols. See [](../explanations/when-to-extend-movable.md) for more details. -- [](#StandardDetector) allows file-writing detectors to be used within both step and fly scans, reporting periodic references to the data that has been written so far. Instances of [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) are composed together to provide this functionality. This separation of concerns makes it easy to mix and match different trigger modes, acquisition strategies, and data outputs. +- [](#StandardDetector) allows detectors to be used within both step and fly scans, reporting data as it is acquired: references to file-written data, or the values themselves. Instances of [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) are composed together to provide this functionality. This separation of concerns makes it easy to mix and match different trigger modes, acquisition strategies, and data outputs. - [](#StandardFlyable) allows actuators (like a motor controller) to be used within a fly scan, by composing a [](#FlyableLogic) instance rather than directly implementing the `Preparable`, `Flyable`, and `Stageable` protocols. Use it as a mix-in on a Device, or call [](#FlyableLogic.with_device) to wrap a bare logic object in an ephemeral flyer within a plan. ## Adding verbs via protocols diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index d0704ba4e4..798503b558 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -41,9 +41,11 @@ To preserve hardware state in plans like [`bp.count`](#bluesky.plans.count) and [`bps.trigger_and_read`](#bluesky.plan_stubs.trigger_and_read) that call `trigger()` without a preceding `prepare()`, implement: - `default_trigger_info()` - Return the [](#TriggerInfo) to use for the implicit - prepare. For AD detectors call `await trigger_info_from_driver(self.driver.num_images)` to - read back the current `num_images` from the driver rather than resetting it to 1. You should also pass a process - plugin to `trigger_info_from_driver()` if your detector uses one to produce multiple exposures per collection. + prepare. For AD detectors call `await trigger_info_from_driver(self.driver)` to + read back the current `num_images`, `acquire_time` and `acquire_period` from the + driver rather than resetting them. Also pass the process plugin, + `trigger_info_from_driver(self.driver, self.process_plugin)`, if your detector uses + one to produce multiple exposures per collection. @@ -64,8 +66,7 @@ For example, for ADAravis: ## Use ADAcquireLogic or create custom Acquire Logic Most areaDetectors can use the standard [](#adcore.ADAcquireLogic) which handles starting and stopping acquisition via the driver's `acquire` signal. If your detector requires custom acquisition behaviour (e.g., waiting for a specific ready signal), create a [](#DetectorAcquireLogic) subclass with: -- `start_acquiring()` - Start acquisition -- `wait_for_idle()` - Wait until acquisition is complete +- `acquire()` - An async generator that starts acquisition, yields once armed, then waits until acquisition is complete - `ensure_stopped()` - Stop acquisition - `ensure_ready()` - (optional) Override if stage-time reset differs from `ensure_stopped` @@ -78,7 +79,7 @@ Now you need to make an [](#adcore.AreaDetector) subclass that uses your IO and The constructor parameters should include: - `prefix`: The PV prefix for the driver and plugins -- `*writer_factories`: Zero or more [](#adcore.ADWriterFactory) instances (e.g. `ADWriterFactory.hdf(path_provider)`); omit entirely to skip file writing +- `*writer_factories`: Zero or more [](#adcore.ADDataFactory) instances (e.g. `ADDataFactory.hdf(path_provider)`); omit entirely to skip file writing - `driver_suffix`: A PV suffix for the driver, defaulting to `"cam1:"` - `plugins`: An optional mapping of {`name`: [](#adcore.NDPluginBaseIO)} for additional plugins - `config_sigs`: Additional signals to report in configuration (beyond the standard acquire_time and acquire_period) @@ -93,7 +94,7 @@ For example, for ADAravis: The `AreaDetector` baseclass will: - Store the driver as `self.driver` -- Call `add_detector_logics()` to register your trigger and arm logic +- Build a [](#DetectorLogic) from your trigger and arm logic and return it from `logic` - Create and register a data logic for file writing for each factory in `writer_factories` - Add configuration signals (driver.acquire_time, driver.acquire_period, and any you specify) - Store any plugins as attributes on the detector @@ -115,7 +116,7 @@ For example, for ADAravis: ## Add multiple data streams (optional) -The composition-based architecture makes it possible to add multiple data outputs to a detector. After creating the detector, you can call `add_detector_logics()` to add additional data sources: +The composition-based architecture makes it possible to give a detector several data outputs. Every [](#adcore.ADDataFactory) factory takes `always_enable`: `True` (the default) switches the plugin on at the start of a scan, `False` follows the plugin's `EnableCallbacks` PV, so a data output whose plugin is off sits out and another can serve the scan. ### Reading stats plugins alongside file writing A plugin scalar needs no data logic: a detector is a [](#StandardReadable), so register @@ -124,7 +125,9 @@ the signal and it appears in `read()` alongside whatever the file writer produce ```python from ophyd_async.core import StandardReadableFormat as Format -det = adaravis.AravisDetector(prefix, path_provider, plugins={"stats": stats}) +det = adaravis.AravisDetector( + prefix, adcore.ADDataFactory.hdf(path_provider), plugins={"stats": stats} +) # Add stats total as a hinted signal in events det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) ``` @@ -135,7 +138,7 @@ roi1 = adcore.NDROIIO("PREFIX:ROI1:") roi2 = adcore.NDROIIO("PREFIX:ROI2:") det = adaravis.AravisDetector( "PREFIX:", - adcore.ADWriterFactory.hdf( + adcore.ADDataFactory.hdf( path_provider, writer_suffix="HDF1:", writer_name="hdf1", @@ -146,7 +149,7 @@ det = adaravis.AravisDetector( color_mode_signal=driver.color_mode, ), ), - adcore.ADWriterFactory.hdf( + adcore.ADDataFactory.hdf( path_provider, writer_suffix="HDF2:", writer_name="hdf2", @@ -181,10 +184,11 @@ Example implementation: driver = adcore.ADBaseIO("PREFIX:DRV:") cb_plugin = adcore.NDCircularBuffIO("PREFIX:CB1:") det = adcore.AreaDetector( - driver=driver, + driver, + "PREFIX:", + adcore.ADDataFactory.hdf(path_provider), acquire_logic=adcore.ADContAcqAcquireLogic(driver, cb_plugin), trigger_logic=adcore.ADContAcqTriggerLogic(driver, cb_plugin), - path_provider=path_provider, plugins={"cb1": cb_plugin}, ) ``` diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 78afba8c51..48cf30ad97 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -10,15 +10,19 @@ effect on ophyd-async itself. ## `OPHYD_ASYNC_PRESERVE_DETECTOR_STATE` -Controls the implicit prepare performed by +Controls whether a [`StandardDetector`](#ophyd_async.core.StandardDetector) reads +hardware state back with +[`DetectorTriggerLogic.default_trigger_info()`](#ophyd_async.core.DetectorTriggerLogic.default_trigger_info). +It does so in two places: the implicit prepare performed by [`StandardDetector.trigger()`](#ophyd_async.core.StandardDetector.trigger) when -[`prepare()`](#ophyd_async.core.StandardDetector.prepare) has not been called since -the last `stage()`. +[`prepare()`](#ophyd_async.core.StandardFlyable.prepare) has not been called since +the last `stage()`, and `prepare()` with a `livetime` of 0, which reads back the +current livetime and deadtime to give the data logics the exposure period. | Value | Behaviour | |-------|-----------| -| `YES` | Calls [`DetectorTriggerLogic.default_trigger_info()`](#ophyd_async.core.DetectorTriggerLogic.default_trigger_info) to read current hardware state before the implicit prepare. Raises `RuntimeError` if `default_trigger_info()` is not implemented on the trigger logic. | -| anything else (default) | Uses a bare [`TriggerInfo()`](#ophyd_async.core.TriggerInfo), resetting detector state to defaults. | +| `YES` | Calls `default_trigger_info()` in both places. The implicit prepare raises `RuntimeError` if it is not implemented on the trigger logic. | +| anything else (default) | The implicit prepare uses a bare [`TriggerInfo()`](#ophyd_async.core.TriggerInfo), resetting detector state to defaults, and a `livetime` of 0 gives the data logics a period of 0. | See [ADR 0013](../explanations/decisions/0013-preserve-hardware-state-in-step-scan.md) for rationale. diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index 1a4310fe13..f00fec2532 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -179,7 +179,7 @@ It derives from [](#StandardDetector) which is a utility baseclass that implemen - [](#DetectorAcquireLogic) to start and stop acquisition and wait for it to complete - [](#DetectorDataLogic) to tell the detector to open a file, describe the datasets it will write, and emit StreamAsset documents as frames are written -In this case, we have three logic classes written just for this simulation, all taking a reference to the pattern generator that provides methods for both detector control and file writing. In other cases the detector control and filewriting may be handled by different sub-devices that talk to different parts of the control system. The job of the top level detector class is to take the arguments that the logic classes need, create the logic instances, and pass them to [](#StandardDetector.add_detector_logics). +In this case, we have three logic classes written just for this simulation, all taking a reference to the pattern generator that provides methods for both detector control and file writing. In other cases the detector control and filewriting may be handled by different sub-devices that talk to different parts of the control system. The job of the top level detector class is to take the arguments that the logic classes need and, in `logic`, create the logic instances and wire them into a [](#DetectorLogic) by role: `DetectorLogic(trigger=..., acquire=..., data=[...])`. Now let's look at the underlying classes that define the detector behavior: @@ -215,28 +215,46 @@ Next we have `BlobAcquireLogic`, a [](#DetectorAcquireLogic) subclass: Its job is to control the acquisition process on the detector, starting and stopping the collection of data: - `ensure_ready()` (inherited default) delegates to `ensure_stopped`, putting the detector into a known state at the start of each scan. -- `start_acquiring()` starts the acquisition process that has been prepared. In this case we create a background task that will write our simulation images to file. -- `wait_for_idle()` waits for that acquisition process to be complete. -- `ensure_stopped()` interrupts the acquisition process and waits for it to complete. Called from `unstage()` at the end of a scan. +- `acquire()` is an async generator for one acquisition. It starts the acquisition process that has been prepared, in this case a background task that will write our simulation images to file, yields once it is running, then waits for it to be complete. If the scan is cancelled the detector closes it at the `yield`, so the `finally` cancels the task. +- `ensure_stopped()` stops the detector at the end of a scan, called from `unstage()`. Here `acquire()` owns the only thing running, so there is nothing to do. ### `BlobDataLogic` -Then we have `BlobDataLogic`, a [](#DetectorDataLogic) subclass: +Then we have `BlobDataLogic`, an [](#UnboundedDataLogic) subclass: ```{literalinclude} ../../src/ophyd_async/sim/_blob_data_logic.py :language: python ``` Its job is to manage the file writing and data streaming: -- `prepare_unbounded()` tells the detector to open a file, and returns a [](#StreamableDataProvider) that describes the datasets that will be written and tracks write progress +- `start_provider()` works out where the file will go, opens it, and returns a [](#StreamableDataProvider) describing the datasets being written - `get_hinted_fields()` returns the data keys that are interesting to plot -- `stop()` tells the detector to close the file +- `ensure_stopped()` tells the detector to close the file -The [](#StreamableDataProvider) returned from `prepare_unbounded()` contains: -- `collections_written_signal` - a signal that tracks how many frames have been written +The logic also has `collections_written`, a signal that tracks how many frames have been written. The [](#StreamableDataProvider) returned from `start_provider()` is made for the scan's collections per event, and contains: - `make_datakeys()` - creates DataKey descriptions for each dataset - `make_stream_docs()` - emits StreamResource and StreamDatum documents as frames are written +#### Which kind of data a logic produces + +The detector starts each of its logics for the scan, and the type of provider that comes back says which kind of data it is. + +- a [](#StreamableDataProvider) publishes stream assets, like a file writer. The detector exposes `collect_asset_docs`, and `BlobDataLogic` is an example. +- a [](#PageableDataProvider) publishes event pages, like an areaDetector stats time series or a scaler's arrays, so the detector exposes `collect_pages` instead. Its `make_page()` carries on from where its last page got to; a step scan's `read()` takes the last event of the page made up to the end of the point. +- `None` means "not this stage", for instance a plugin that is switched off. The logic decides this without writing anything. This is not an error: a detector may carry more data logics than any one scan uses. + +Separately from that, a data logic is either a [](#BoundedDataLogic) or an [](#UnboundedDataLogic), whichever it inherits from (not both). An unbounded logic, like a file writer, keeps counting for as long as it is told to; its collections-written signal is read when each `prepare()` finishes. A bounded logic needs to be told how many collections to take, like a finite buffer: it is started after the unbounded ones and handed the datakeys their providers already make, and its provider must make none of them. It returns `None` if nothing is left to make, and raises for a scan with no fixed number of collections (`num_collections` of 0) if something is. At each point of a step scan after the first it is stopped and started again, so the buffer is cleared. Either kind may return either kind of provider. + +A source that produces one value per event, like a plugin scalar, needs no data logic at all: a detector is a [](#StandardReadable), so register the signal with [](#StandardReadable.set_readable_format). + +Both kinds of `start_provider()` are given the detector `name`, which a logic uses for its datakeys (adding its own suffix if it has one), and the fields they need from the scan's [](#TriggerInfo): `collections_per_event`, a `collection_period` and a `flush_period`, and the bounded one also `existing_datakeys` and `num_collections`. The `collection_period` is `exposures_per_collection * (livetime + deadtime)`, where `prepare()` reads the livetime and deadtime back from the trigger logic when the plan's [](#TriggerInfo) leaves `livetime` at 0 and [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES`; otherwise it is 0 for "use what is on the hardware". + +No two providers may make the same datakey: `prepare()` raises if they do. A datakey a provider makes replaces a registered signal of that name. + +```{note} +Which logics serve is decided at the first `prepare()` after `stage()` and holds until `unstage()`, so it can change from one stage to the next — a detector whose file writer is switched off can serve its stats time series instead. Change it **between stages**, not inside one: a run's descriptor is emitted at its start, so switching part way through would leave the descriptor and the documents disagreeing. +``` + ## Conclusion We have seen how to make a [](#StandardDetector) and how the [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) classes allow us to customise its behavior through composition. This separation of concerns makes it easy to: diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index c239816695..81363cf736 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -11,6 +11,9 @@ soft_command, ) from ._data_providers import ( + DataProvider, + EventPageDataProvider, + PageableDataProvider, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -24,12 +27,15 @@ ) from ._derived_signal_backend import Transform, merge_gathered_dicts from ._detector import ( + BoundedDataLogic, DetectorAcquireLogic, DetectorDataLogic, + DetectorLogic, DetectorTrigger, DetectorTriggerLogic, StandardDetector, TriggerInfo, + UnboundedDataLogic, ) from ._device import ( Device, @@ -50,7 +56,12 @@ OnOff, YesNo, ) -from ._flyable import KICKED_OFF, FlyableLogic, FlyMotorInfo, StandardFlyable +from ._flyable import ( + KICKED_OFF, + FlyableLogic, + FlyMotorInfo, + StandardFlyable, +) from ._log import config_ophyd_async_logging, logger, set_handler from ._mock_signal_backend import MockSignalBackend from ._mock_signal_utils import ( @@ -292,6 +303,9 @@ def __getattr__(name): "DetectorTriggerLogic", "DetectorAcquireLogic", "DetectorDataLogic", + "BoundedDataLogic", + "UnboundedDataLogic", + "DetectorLogic", "StandardDetector", # Path "PathInfo", @@ -305,6 +319,9 @@ def __getattr__(name): "AutoMaxIncrementingPathProvider", "UUIDFilenameProvider", # Data Providers + "DataProvider", + "EventPageDataProvider", + "PageableDataProvider", "StreamableDataProvider", "StreamResourceInfo", "StreamResourceDataProvider", diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index 8faa7ba069..0c86a070c8 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -1,36 +1,148 @@ -from abc import abstractmethod -from collections.abc import AsyncIterator, Sequence +import asyncio +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator, Mapping, Sequence from typing import Any +import numpy as np from bluesky.protocols import StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange +from event_model.documents import PartialEventPage +from ._datatypes import Array1D from ._signal import SignalR, SignalW -from ._utils import ConfinedModel +from ._utils import ConfinedModel, error_if_none, gather_dict, get_dtype -class StreamableDataProvider: - collections_written_signal: SignalR[int] +class DataProvider(ABC): + """What a [](#DetectorDataLogic) returns to publish the data it is writing. + + The two kinds, [](#StreamableDataProvider) and [](#PageableDataProvider), + differ in how the data is published; what they share is here. A provider + counts its events from an origin: collection 0 of its data logic's + `collections_written` unless `start_new_resources` says otherwise. + + :param collections_per_event: how many collections make up one event + """ + + def __init__(self, collections_per_event: int) -> None: + self.collections_per_event = collections_per_event @abstractmethod - async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: - """Return a DataKey for each Signal that produces a Reading. + async def make_datakeys(self) -> dict[str, DataKey]: + """Return a DataKey for each field, of shape `[collections_per_event, ...]`. Called before the first exposure is taken. + """ + + @abstractmethod + def start_new_resources(self, origin: int) -> None: + """Count events from collection `origin`, with nothing yet published. + + Called on an unbounded provider at each `prepare()`, so each run has its + own stream resources. + """ - :param collections_per_event: this should appear in the shape of each DataKey + +class StreamableDataProvider(DataProvider): + """Publishes its data as stream assets: a StreamResource and StreamDatums.""" + + @abstractmethod + def make_stream_docs(self, index: int) -> AsyncIterator[StreamAsset]: + """Make the StreamAsset documents for the events up to `index`. + + Emits the StreamResources the first time, then StreamDatums from where + the last call got to. + + :param index: the event to publish up to, counted from the origin """ - async def make_stream_docs( - self, collections_written: int, collections_per_event: int - ) -> AsyncIterator[StreamAsset]: - """Make StreamAsset documents up to the given index. - Default implementation is a no-op. Subclasses should override this - to emit actual StreamAsset documents. +class PageableDataProvider(DataProvider): + """Publishes its data as event pages. + + For a source read back as arrays rather than written to a file: an + areaDetector stats time series, or a scaler's MCA arrays. + """ + + @abstractmethod + async def make_page(self, index: int) -> PartialEventPage | None: + """Return one page of the events from where the last call got to. + + :param index: the event to publish up to + :return: the page, or `None` if there are no new events """ - while False: - yield + + +class EventPageDataProvider(PageableDataProvider): + """Emits a fixed-length array of collections as event pages. + + For a device that fills one array per field as it acquires, alongside an + array of per-collection timestamps: an areaDetector stats time series, or a + scaler's MCA arrays. Each array is sliced into `collections_per_event`-length + chunks, one per event, and the last collection of an event supplies that + event's timestamp. + + :param data: datakey (already suffixed) to the array signal that backs it + :param timestamps: the acquisition time of each collection + :param collections_per_event: how many collections make up one event + """ + + def __init__( + self, + data: Mapping[str, SignalR[Array1D[Any]]], + timestamps: SignalR[Array1D[np.float64]], + collections_per_event: int, + ) -> None: + super().__init__(collections_per_event) + self.data = dict(data) + self.timestamps = timestamps + self.start_new_resources(0) + + def start_new_resources(self, origin: int) -> None: + self._origin = origin + # The events published by make_page + self._published = 0 + + async def make_datakeys(self) -> dict[str, DataKey]: + return { + datakey: DataKey( + source=signal.source, + shape=[self.collections_per_event], + dtype="array", + dtype_numpy=get_dtype( + error_if_none(signal.datatype, f"{signal.source} has no datatype") + ).str, + ) + for datakey, signal in self.data.items() + } + + async def make_page(self, index: int) -> PartialEventPage | None: + if index <= self._published: + return None + cpe = self.collections_per_event + # Read every array and the per-collection timestamps in one parallel batch + arrays, stamps = await asyncio.gather( + gather_dict( + {datakey: signal.get_value() for datakey, signal in self.data.items()} + ), + self.timestamps.get_value(), + ) + starts = [self._origin + event * cpe for event in range(self._published, index)] + self._published = index + # One timestamp per event: the acquisition time of its last collection + event_times = [float(stamps[start + cpe - 1]) for start in starts] + data = { + datakey: [array[start : start + cpe] for start in starts] + for datakey, array in arrays.items() + } + return { + # TODO: numpy arrays are not in PartialEventPage's value type, but + # they round trip through the bundler fine; fix upstream in + # event-model rather than converting to lists here + "data": data, # type: ignore[typeddict-item] + "time": event_times, + "timestamps": dict.fromkeys(data, event_times), + } class StreamResourceInfo(ConfinedModel): @@ -63,8 +175,11 @@ class StreamResourceInfo(ConfinedModel): class StreamResourceDataProvider(StreamableDataProvider): """A helper class to make stream resource and datums for HDF datasets. - :param full_file_name: Absolute path to the file that has been written - :param datasets: Descriptions of each of the datasets that will appear in the file + :param uri: the file that is being written + :param resources: descriptions of each of the datasets that will appear in it + :param mimetype: the mimetype of the file + :param collections_per_event: how many collections make up one event + :param flush_signal: set before making stream docs, to flush the file """ def __init__( @@ -72,19 +187,22 @@ def __init__( uri: str, resources: Sequence[StreamResourceInfo], mimetype: str, - collections_written_signal: SignalR[int], + collections_per_event: int, flush_signal: SignalW[bool] | None = None, ) -> None: + super().__init__(collections_per_event) self.uri = uri self.resources = list(resources) - self.collections_written_signal = collections_written_signal + self.mimetype = mimetype self.flush_signal = flush_signal - self.last_emitted = 0 + self.start_new_resources(0) + + def start_new_resources(self, origin: int) -> None: bundler_composer = ComposeStreamResource() self.bundles = [ bundler_composer( - mimetype=mimetype, - uri=uri, + mimetype=self.mimetype, + uri=self.uri, data_key=resource.data_key, parameters={ "chunk_shape": resource.chunk_shape, @@ -95,38 +213,37 @@ def __init__( ) for resource in self.resources ] - - async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: - describe = { + self._resources_emitted = False + self._origin = origin + # The collections in the file that StreamDatums have been made for + self._published = origin + + async def make_datakeys(self) -> dict[str, DataKey]: + cpe = self.collections_per_event + return { resource.data_key: DataKey( source=resource.source or self.uri, - shape=[collections_per_event, *resource.shape], - dtype="array" - if collections_per_event > 1 or len(resource.shape) > 0 - else "number", + shape=[cpe, *resource.shape], + dtype="array" if cpe > 1 or len(resource.shape) > 0 else "number", dtype_numpy=resource.dtype_numpy, external="STREAM:", ) for resource in self.resources } - return describe - async def make_stream_docs( - self, collections_written: int, collections_per_event: int - ) -> AsyncIterator[StreamAsset]: + async def make_stream_docs(self, index: int) -> AsyncIterator[StreamAsset]: if self.flush_signal: await self.flush_signal.set(True) # TODO: fail if we get dropped frames - indices_written = collections_written // collections_per_event - if indices_written and not self.last_emitted: - for bundle in self.bundles: - yield "stream_resource", bundle.stream_resource_doc - - if indices_written > self.last_emitted: - indices: StreamRange = { - "start": self.last_emitted, - "stop": indices_written, - } - self.last_emitted = indices_written + cpe = self.collections_per_event + written = self._origin + index * cpe + # StreamDatum indices are events in the file, which may hold earlier runs + indices: StreamRange = {"start": self._published // cpe, "stop": written // cpe} + if indices["stop"] > indices["start"]: + if not self._resources_emitted: + self._resources_emitted = True + for bundle in self.bundles: + yield "stream_resource", bundle.stream_resource_doc for bundle in self.bundles: yield "stream_datum", bundle.compose_stream_datum(indices) + self._published = written diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index fd8b36abba..af85fe5e1c 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -1,51 +1,59 @@ """Module which defines abstract classes to work with detectors.""" import asyncio -import functools import os import time import warnings -from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence -from dataclasses import dataclass +from abc import abstractmethod +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Collection, + Iterator, + Mapping, + Sequence, +) +from contextlib import aclosing, asynccontextmanager +from dataclasses import dataclass, field from enum import Enum from functools import cached_property -from typing import cast +from typing import Any, Generic, TypeVar, cast from bluesky.protocols import ( Collectable, - Flyable, + HasHints, Hints, - Preparable, Reading, Stageable, + Status, StreamAsset, Triggerable, - WritesStreamAssets, ) from event_model import DataKey -from pydantic import Field, NonNegativeInt, PositiveInt, computed_field +from event_model.documents import PartialEventPage +from pydantic import Field, PositiveInt, computed_field -from ._data_providers import StreamableDataProvider -from ._readable import ( - StandardReadable, - StandardReadableFormat, - _config_signals, +from ._data_providers import ( + DataProvider, + PageableDataProvider, + StreamableDataProvider, ) +from ._device import Device, name_added_to_errors +from ._flyable import KICKED_OFF, FlyableLogic, StandardFlyable +from ._readable import StandardReadable, StandardReadableFormat from ._settings import Settings -from ._signal import ( - SignalDict, - SignalR, - SignalRW, - observe_signals_value, - soft_signal_rw, -) -from ._status import AsyncStatus, WatchableAsyncStatus +from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value +from ._standard_base import _State, _StateMachine +from ._status import WatchableAsyncStatus from ._utils import ( DEFAULT_TIMEOUT, ConfinedModel, WatcherUpdate, + abstract_cached_property, error_if_none, + gather_dict, + get_unique, + merge_dicts, merge_gathered_dicts, ) @@ -96,14 +104,25 @@ class TriggerInfo(ConfinedModel): w) for each. """ - number_of_events: NonNegativeInt = Field(default=1) - """Number of bluesky events that will be emitted, (0 means infinite).""" + number_of_events: PositiveInt = Field(default=1) + """Number of bluesky events that will be emitted.""" exposure_timeout: float = Field( default_factory=lambda d: d["livetime"] + d["deadtime"] + DEFAULT_TIMEOUT, gt=0, ) - """What is the maximum timeout on waiting for an exposure""" + """The longest wait for progress before giving up: the most time allowed + between one collection being written and the next, not a total for the + scan.""" + + flush_period: float = Field(default=0.0, ge=0.0) + """How many seconds between flushes of the data being written. + + A data logic that buffers before it writes sizes that buffer from this and + the exposure period, so data appears at roughly this rate: an HDF chunk of + `flush_period / period` frames, say. Set it to the rate the plan collects at + so a fly scan sees each batch as it lands. 0 means leave whatever the + hardware is set to.""" @computed_field @cached_property @@ -189,14 +208,11 @@ async def default_trigger_info(self) -> TriggerInfo: raise NotImplementedError(self) -def _logic_supported(base_class, method) -> bool: +def _trigger_logic_supported(method) -> bool: # If the function that is bound in a subclass is the same as the function - # attached to the superclass, then the subclass has not overridden it, so - # this method is not supported by the subclass. - return method.__func__ is not getattr(base_class, method.__name__) - - -_trigger_logic_supported = functools.partial(_logic_supported, DetectorTriggerLogic) + # attached to DetectorTriggerLogic, then the subclass has not overridden it, + # so this method is not supported by the subclass. + return method.__func__ is not getattr(DetectorTriggerLogic, method.__name__) def _get_supported_triggers( @@ -212,52 +228,51 @@ def _get_supported_triggers( return supported_triggers -class DetectorAcquireLogic(ABC): - """Abstract base class for detector acquisition lifecycle hooks. +class DetectorAcquireLogic: + """Base class for detector acquisition lifecycle hooks. - Subclasses implement four hooks that are called at defined points in the - `[](#StandardDetector)` lifecycle: + Subclasses implement these hooks, called at defined points in the + [](#StandardDetector) lifecycle: - `ensure_ready` — called from `stage()` to put the detector into a known - idle state before a scan begins. - - `start_acquiring` — called from `prepare()`, `kickoff()`, or `trigger()` - to start the detector acquiring. - - `wait_for_idle` — called after the final collection completes to confirm - the detector has returned to idle. + idle state before a scan begins. Defaults to `ensure_stopped`. + - `acquire` — an async generator for one acquisition: start the detector + acquiring, yield once it is armed, then wait for it to return to idle. - `ensure_stopped` — called from `unstage()` to stop the detector and perform any end-of-scan cleanup. - - The default `ensure_ready` delegates to `ensure_stopped`, which is correct - for detectors where stage-time reset and scan-end teardown are identical. - Override `ensure_ready` when the two phases require different behaviour - (e.g. arming the detector once at stage time while keeping it armed across - multiple kickoff/complete cycles). """ async def ensure_ready(self): """Ensure the detector is idle before a scan. - Called from `stage()`. The default implementation delegates to - `ensure_stopped`, which is sufficient for detectors that perform the - same reset at stage time as at unstage time. Override this method when - a different action is required at stage time (for example, arming the - detector once so it is ready for multiple kickoff/complete cycles - without re-arming). + Called from `stage()`. Defaults to `ensure_stopped`; override it when + stage needs a different action, such as arming once for many + kickoff/complete cycles. """ await self.ensure_stopped() @abstractmethod - async def start_acquiring(self): - """Start the detector acquiring. - - Called from `prepare()` for external-trigger modes, and from - `kickoff()` / `trigger()` for internal-trigger modes. + def acquire(self) -> AsyncGenerator[None, None]: + """Start acquiring, yield once armed, then wait until idle. + + Write it as an async generator that yields once. It is entered in + `prepare()` for external triggering, and in `kickoff()` or `trigger()` + for internal triggering, then run to the end once the data is written. + Acquisition must stop by itself once the data is written: a free-running + detector is not supported. + If the scan is cancelled it is closed at its `yield`, so own whatever it + starts: hold a status with `async with`, or cancel a task in `finally`. + For example: + + ```python + async def acquire(self): + async with self.driver.acquire.set(True) as status: + await wait_for_value(self.driver.armed, True, timeout=10) + yield # armed + await status + ``` """ - @abstractmethod - async def wait_for_idle(self): - """Wait for the detector to return to idle after the final collection.""" - @abstractmethod async def ensure_stopped(self): """Stop the detector and perform end-of-scan cleanup. @@ -266,175 +281,1007 @@ async def ensure_stopped(self): """ -def _all_the_same(collections_written: set[int]) -> int: - """Ensure all collection counts are the same, raising an error if they differ. +#: The provider a data logic returns from `start_provider` +ProviderT = TypeVar("ProviderT", bound=DataProvider) - :param collections_written: Set of collection counts from different providers - :return: The single collection count value - :raises RuntimeError: If the set contains more than one distinct value - """ - if len(collections_written) != 1: - msg = ( - "Detectors have written different numbers of collections: " - + f"{collections_written}" - ) - raise RuntimeError(msg) - return collections_written.pop() +class DetectorDataLogic(Generic[ProviderT]): + """Base class for detector data logic: do not inherit from it directly. -async def _get_collections_written( - data_providers: Sequence[StreamableDataProvider], - reducer: Callable[[set[int]], int] = _all_the_same, -) -> int: - """Return a single collections_written value for the given providers. - - By default this function ensures all providers agree and returns that - single value. If `reducer` is provided it will be called with the set of - observed values and should return a single int to use. - """ - # Work out where all the streamable data providers are up to - collections_written = set( - await asyncio.gather( - *[sdp.collections_written_signal.get_value() for sdp in data_providers] - ) - ) - if collections_written: - # Let our reducer decide how to return a single int - return reducer(collections_written) - else: - # There are none, this is valid as we then don't use the value anywhere - # so just return 0 - return 0 + Inherit from [](#BoundedDataLogic) or [](#UnboundedDataLogic), which say + whether the data source needs to know how many collections to take. + At the first `prepare()` after `stage()` the detector starts every + unbounded data logic, then every bounded one, handing each bounded logic the + datakeys the unbounded providers already make. The logics that return a + provider serve the stage, and are started again at each later `prepare()`. + The provider describes and publishes what is being written. -class DetectorDataLogic: - """Abstract base class for detector data logic and handling. + Whether the data is published as stream assets or event pages is the type + of the provider that is returned, not a declaration on the logic, so one + logic may produce either depending on the scan. - Implementations must override `prepare_unbounded`. A source that produces a - single value per event, like a plugin scalar, does not need a data logic at - all: register its signal with + A source that produces a single value per event, like a plugin scalar, needs + no data logic at all: register its signal with [](#StandardReadable.set_readable_format) instead. """ - #: Add this suffix to the detector name to specify the datakey. These need to be - #: different for each DetectorDataLogic added to a detector - datakey_suffix: str = "" + def __init_subclass__(cls, **kwargs) -> None: + super().__init_subclass__(**kwargs) + if "stop" in cls.__dict__: + raise TypeError( + f"{cls.__name__} defines stop, which DetectorDataLogic no longer " + "calls: stop -> ensure_stopped" + ) - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - """Provider can work for an unbounded number of collections.""" - raise NotImplementedError(self) + #: Counts the collections written so far. `start_provider` resets it, and + #: waits until it reads 0 before returning + collections_written: SignalR[int] - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - """Return the hinted streams.""" + async def ensure_ready(self) -> None: + """Ensure the data logic is idle before a scan. + + Called from `stage()`. Defaults to `ensure_stopped`. + """ + await self.ensure_stopped() + + def get_hinted_fields(self, name: str) -> Sequence[str]: + """Return the datakeys to hint, of those this logic may produce. + + The detector hints only those its providers actually make. + + :param name: the detector name, which prefixes every datakey + """ return [] - async def stop(self) -> None: + async def ensure_stopped(self) -> None: """Stop taking data.""" return None -_data_logic_supported = functools.partial(_logic_supported, DetectorDataLogic) +class BoundedDataLogic(DetectorDataLogic[ProviderT]): + """A data logic whose source must be told how many collections to take. + For a finite buffer, like an areaDetector stats time series. It is started + after the unbounded logics, so it can leave out datakeys they already make. + At each step point after the first the detector stops it and starts a new + provider, so a buffer holding the last point's data is cleared. + """ + + @abstractmethod + async def start_provider( + self, + name: str, + existing_datakeys: Sequence[str], + num_collections: int, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> ProviderT | None: + """Start writing and return a new provider describing what is written. + + Return a [](#StreamableDataProvider) to publish stream assets, or a + [](#PageableDataProvider) to publish event pages, made for + `collections_per_event`. The provider must make none of + `existing_datakeys`. Return `None`, without writing, if there is nothing + left to make: this is not an error and is not warned about. + + :param name: the detector name, which prefixes every datakey + :param existing_datakeys: the datakeys the unbounded providers make + :param num_collections: how many collections the scan takes + :param collections_per_event: how many collections make up one event + :param collection_period: seconds per collection, + `exposures_per_collection * (livetime + deadtime)`, for sizing + chunks (0 means "use whatever is currently set on the hardware") + :param flush_period: how often to flush + """ + raise NotImplementedError(self) + + +class UnboundedDataLogic(DetectorDataLogic[ProviderT]): + """A data logic whose source takes data for as long as it is told to. + + For a file writer, or anything else that does not need to know how many + collections to take. Its provider carries over to the next `prepare()` of + the same stage if that is for the same collections per event, collection period + and flush period, so it may already have written collections when a scan + starts. + """ + + @abstractmethod + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> ProviderT | None: + """Start writing and return a new provider describing what is written. + + Return a [](#StreamableDataProvider) to publish stream assets, or a + [](#PageableDataProvider) to publish event pages, made for + `collections_per_event`. Return `None`, without writing, to sit out the + scans of this stage, for instance a plugin that is switched off: this is + not an error and is not warned about. + + :param name: the detector name, which prefixes every datakey + :param collections_per_event: how many collections make up one event + :param collection_period: seconds per collection, + `exposures_per_collection * (livetime + deadtime)`, for sizing + chunks (0 means "use whatever is currently set on the hardware") + :param flush_period: how often to flush + """ + raise NotImplementedError(self) -@dataclass -class _PrepareCtx: - trigger_info: TriggerInfo - streamable_data_providers: Sequence[StreamableDataProvider] - collections_written: int + +def _merge_pages( + pages: Sequence[tuple[PageableDataProvider, PartialEventPage | None]], +) -> PartialEventPage | None: + """Merge one page per pageable provider into a single page. + + `None` if every provider returned `None`. Raises if only some did, if the + pages hold different numbers of events, or if a key appears twice. + """ + if all(page is None for _, page in pages): + return None + # A None page has length None, so a mix of None and real pages differs too + lengths = [ + (type(p).__name__, len(page["time"]) if page else None) for p, page in pages + ] + get_unique(lengths, "numbers of events") + made = [page for _, page in pages if page] + # Each event happened when the last of its providers' data arrived + times = zip(*(page["time"] for page in made), strict=True) + return { + "data": merge_dicts(page["data"] for page in made), + "timestamps": merge_dicts(page["timestamps"] for page in made), + "time": [max(t) for t in times], + } @dataclass -class _KickoffCtx: +class DetectorCtx: + """What `prepare()` set up, held until `unstage()`.""" + + #: The plan's TriggerInfo trigger_info: TriggerInfo - data_providers: Sequence[StreamableDataProvider] - collections_written: int - collections_requested: int - is_last_kickoff: bool + #: The detector name, which prefixes every datakey + name: str + #: Seconds per collection: `exposures_per_collection` times the livetime + #: plus deadtime, or what the hardware has set when the livetime is 0 and + #: its state is preserved + collection_period: float + #: The bounded data logics of this stage: those that returned a provider + #: when last started + bounded_logics: Sequence[BoundedDataLogic[Any]] = () + #: The unbounded data logics of this stage: those that returned a provider + #: when last started + unbounded_logics: Sequence[UnboundedDataLogic[Any]] = () + #: The providers `bounded_logics` started for this prepare, one per logic. + #: Restarted at each step point after the first + bounded_providers: Sequence[DataProvider] = () + #: The providers `unbounded_logics` started, one per logic. Carried over from + #: the previous prepare if made for the same collections per event, + #: collection period and flush period + unbounded_providers: Sequence[DataProvider] = () + #: The datakeys of every provider, the same at every prepare of the stage + datakeys: dict[str, DataKey] = field(default_factory=dict) + #: The collections every unbounded data logic had written at the end of + #: prepare() + unbounded_start_collections: int = 0 + #: How many trigger()s this prepare has served + triggers: int = 0 + #: The readings of `DetectorLogic.get_readings` at the current step point, + #: `None` until read + readings: dict[str, Reading] | None = None + #: The acquire logic's `acquire()`, run to its yield by prepare() for + #: external triggering + external_acquire_gen: AsyncGenerator[None, None] | None = None + + @property + def providers(self) -> list[DataProvider]: + """Every serving provider, the unbounded ones first.""" + return [*self.unbounded_providers, *self.bounded_providers] + + +async def _datakeys(providers: Sequence[DataProvider]) -> dict[str, DataKey]: + """The datakeys of every provider, raising on one made by more than one.""" + return await merge_gathered_dicts(p.make_datakeys() for p in providers) + + +async def _stop_logics(logics: Sequence[DetectorDataLogic[Any]]) -> None: + """Stop every logic in `logics`.""" + await asyncio.gather(*(dl.ensure_stopped() for dl in logics)) + + +async def _start_unbounded(ctx: DetectorCtx) -> None: + """Start `ctx.unbounded_logics`, keeping those that return a provider.""" + info = ctx.trigger_info + providers = await asyncio.gather( + *( + dl.start_provider( + ctx.name, + info.collections_per_event, + ctx.collection_period, + info.flush_period, + ) + for dl in ctx.unbounded_logics + ) + ) + started = list(zip(ctx.unbounded_logics, providers, strict=True)) + ctx.unbounded_logics = [dl for dl, p in started if p] + ctx.unbounded_providers = [p for _, p in started if p] + + +async def _start_bounded(ctx: DetectorCtx) -> None: + """Start `ctx.bounded_logics`, keeping those that return a provider. + + Their providers leave out the datakeys the unbounded providers make. + """ + info = ctx.trigger_info + existing_datakeys = list(await _datakeys(ctx.unbounded_providers)) + providers = await asyncio.gather( + *( + dl.start_provider( + ctx.name, + existing_datakeys, + info.number_of_collections, + info.collections_per_event, + ctx.collection_period, + info.flush_period, + ) + for dl in ctx.bounded_logics + ) + ) + started = list(zip(ctx.bounded_logics, providers, strict=True)) + ctx.bounded_logics = [dl for dl, p in started if p] + ctx.bounded_providers = [p for _, p in started if p] + + +async def _restart_bounded(ctx: DetectorCtx, when: str) -> None: + """Stop and start `ctx.bounded_logics`, then check the datakeys are unchanged. + + :param when: when this happens, for the error message + """ + await _stop_logics(ctx.bounded_logics) + await _start_bounded(ctx) + before = ctx.datakeys + ctx.datakeys = await _datakeys(ctx.providers) + _check_datakeys(ctx, before, when) + + +def _check_datakeys(ctx: DetectorCtx, before: Collection[str], when: str) -> None: + """Raise unless the serving providers make the datakeys `before`.""" + if missing_or_extra := sorted(set(before) ^ ctx.datakeys.keys()): + raise RuntimeError( + f"The data logics made {sorted(ctx.datakeys)} at {when}, but " + f"{sorted(before)} before it, which differ in {missing_or_extra}: " + "a data logic that serves a stage must make the same datakeys for " + "all of it, so unstage() and stage() to change what it makes" + ) + + +def _made_for(ctx: DetectorCtx) -> tuple[str, int, float, float]: + """What an unbounded provider was made for: it carries over if unchanged.""" + info = ctx.trigger_info + return ( + ctx.name, + info.collections_per_event, + ctx.collection_period, + info.flush_period, + ) + + +def _bounded_origin(ctx: DetectorCtx) -> int: + """The events written before the bounded providers were last started. + + The step points before the current one, as each point restarts them, and 0 + in a fly scan. + """ + return max(ctx.triggers - 1, 0) + + +def _collections_offset(ctx: DetectorCtx) -> dict[SignalR[int], int]: + """The offset from each data logic's `collections_written` to the scan count. + + Added to `collections_written` it gives the collections since `prepare()`: + positive for a bounded data logic, which only counts from the current step + point, and negative for an unbounded one, which counts from before + `prepare()`. + """ + cpe = ctx.trigger_info.collections_per_event + bounded = _bounded_origin(ctx) * cpe + unbounded = -ctx.unbounded_start_collections + return { + **{dl.collections_written: bounded for dl in ctx.bounded_logics}, + **{dl.collections_written: unbounded for dl in ctx.unbounded_logics}, + } + + +def _provider_ends(ctx: DetectorCtx, index: int) -> Iterator[tuple[DataProvider, int]]: + """Each serving provider, with event `index` counted from its origin.""" + for provider in ctx.bounded_providers: + yield provider, index - _bounded_origin(ctx) + for provider in ctx.unbounded_providers: + yield provider, index + + +async def _no_acquisition() -> AsyncGenerator[None, None]: + """The acquisition of a detector with no acquire logic.""" + yield + + +def _preserve_detector_state() -> bool: + # Opt-in: set OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES to have the detector + # read back current hardware state via default_trigger_info(). See ADR 0013. + # TODO: flip default to YES and remove this guard in a future PR once + # downstream code has had time to implement default_trigger_info(). + return os.environ.get("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "NO").upper() == "YES" + + +@dataclass(frozen=True, kw_only=True, eq=False) +class DetectorLogic(FlyableLogic[TriggerInfo, DetectorCtx]): + """Drive a detector through its trigger, acquire and data logics. + + The fields are configuration and never change; what a scan sets up is held + in the context, so `dataclasses.replace` makes an independent copy. + + :param trigger: sets up exposures; without one only internal triggering + with the hardware's current timing is supported + :param acquire: starts and stops acquisition + :param data: the data logics, each publishing one part of the data + """ + + trigger: DetectorTriggerLogic | None = None + acquire: DetectorAcquireLogic | None = None + data: Sequence[BoundedDataLogic[Any] | UnboundedDataLogic[Any]] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "data", tuple(self.data)) + for dl in self.data: + if isinstance(dl, BoundedDataLogic) == isinstance(dl, UnboundedDataLogic): + raise TypeError( + f"{dl!r} must inherit from exactly one of BoundedDataLogic and " + "UnboundedDataLogic" + ) + + @cached_property + def supported_triggers(self) -> set[DetectorTrigger]: + """The trigger types the trigger logic implements.""" + if self.trigger is None: + return {DetectorTrigger.INTERNAL} + return _get_supported_triggers(self.trigger) + + @cached_property + def supports_deadtime(self) -> bool: + """Whether the trigger logic can calculate a deadtime.""" + return self.trigger is not None and _trigger_logic_supported( + self.trigger.get_deadtime + ) + + def with_device(self, name: str = "") -> "StandardDetector": + """Wrap this logic in an ephemeral `StandardDetector` for use in a plan.""" + return _LogicDetector(self, name=name) + + def get_deadtime(self, config_values: SignalDict) -> float: + """Return the deadtime the trigger logic calculates from `config_values`.""" + trigger = error_if_none(self.trigger, "No trigger logic") + return trigger.get_deadtime(config_values) + + async def default_trigger_info(self) -> TriggerInfo: + """The TriggerInfo to prepare with when a plan calls trigger() directly. + + The trigger logic's `default_trigger_info()` if + [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES`, otherwise a bare + [`TriggerInfo()`](#TriggerInfo). + """ + info = TriggerInfo() + if _preserve_detector_state() and self.trigger is not None: + if not _trigger_logic_supported(self.trigger.default_trigger_info): + raise RuntimeError( + "OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but the " + "trigger logic has no default_trigger_info(): implement it on " + f"{type(self.trigger).__name__} or unset the environment " + "variable" + ) + info = await self.trigger.default_trigger_info() + return info + + async def ensure_ready(self) -> None: + """Make every data logic and the detector ready, in parallel.""" + await asyncio.gather( + *(dl.ensure_ready() for dl in self.data), + *([self.acquire.ensure_ready()] if self.acquire else []), + ) + + async def prepare_to_fly( + self, name: str, value: TriggerInfo, previous_ctx: DetectorCtx | None + ) -> AsyncGenerator[DetectorCtx | WatcherUpdate, None]: + """Set the trigger logic up, yield the context, then start the data. + + The context holds `value` and the collection period, which is read back + from the trigger logic when `value` leaves the livetime at 0. An + externally triggered detector is armed last, after the data logics are + started so their buffers are sized. + """ + await self._prepare_trigger_logic(value) + # After preparing the trigger logic, as the hardware then holds the + # values read back + collection_period = await self._resolve_collection_period(value) + ctx = DetectorCtx( + trigger_info=value, name=name, collection_period=collection_period + ) + yield ctx + await self._start_providers(ctx, previous_ctx) + if value.trigger is not DetectorTrigger.INTERNAL: + ctx.external_acquire_gen = await self._arm() + + async def _start_providers( + self, ctx: DetectorCtx, previous_ctx: DetectorCtx | None + ) -> None: + """Start the data providers for this scan. + + At the first `prepare()` of a stage every unbounded data logic is + started, then every bounded one, leaving out the datakeys the unbounded + providers make. The logics that return a provider serve the rest of the + stage. At a later `prepare()` the unbounded providers of `previous_ctx` + carry over if they were made for the same name, collections per event, + collection period and flush period, and are restarted otherwise; the + bounded ones are always restarted. Every unbounded provider then starts + new stream resources at `ctx.unbounded_start_collections`, so each run + has its own. + """ + if previous_ctx is None: + ctx.unbounded_logics = [ + dl for dl in self.data if isinstance(dl, UnboundedDataLogic) + ] + ctx.bounded_logics = [ + dl for dl in self.data if isinstance(dl, BoundedDataLogic) + ] + await _start_unbounded(ctx) + await _start_bounded(ctx) + ctx.datakeys = await _datakeys(ctx.providers) + else: + ctx.unbounded_logics = previous_ctx.unbounded_logics + ctx.bounded_logics = previous_ctx.bounded_logics + ctx.datakeys = previous_ctx.datakeys + if _made_for(previous_ctx) == _made_for(ctx): + ctx.unbounded_providers = previous_ctx.unbounded_providers + else: + await _stop_logics(ctx.unbounded_logics) + await _start_unbounded(ctx) + await _restart_bounded(ctx, "this prepare()") + # TODO: delete this check if the run bundler learns to collect both from + # one device (coretl/bluesky#1), with the prepare() override. See ADR 0024. + if all(self.produces(ctx)): + pageable = await _datakeys( + [p for p in ctx.providers if isinstance(p, PageableDataProvider)] + ) + raise TypeError( + f"The detector would produce {sorted(pageable)} as event pages and " + "the rest of its data as stream assets; bluesky collects only one " + "of them from a detector, so its stream assets must make these " + "datakeys too, or a data logic must not serve this scan" + ) + # Every unbounded data logic has written the same: they all started + # together, after stage() stopped them all + written = await asyncio.gather( + *(dl.collections_written.get_value() for dl in ctx.unbounded_logics) + ) + counts = [ + (type(dl).__name__, n) + for dl, n in zip(ctx.unbounded_logics, written, strict=True) + ] + ctx.unbounded_start_collections = ( + get_unique(counts, "numbers of collections written") if counts else 0 + ) + for provider in ctx.unbounded_providers: + provider.start_new_resources(ctx.unbounded_start_collections) + + async def fly( + self, ctx: DetectorCtx + ) -> AsyncGenerator[KICKED_OFF | WatcherUpdate, None]: + """Start the fly scan, then report events written until it finishes.""" + info = ctx.trigger_info + events = info.number_of_events + async with self._acquiring(ctx): + yield KICKED_OFF + async for update in self._wait_for( + ctx, 0, events, divisor=info.collections_per_event + ): + yield update + + async def step(self, ctx: DetectorCtx) -> AsyncGenerator[WatcherUpdate, None]: + """Take one step point's exposures and wait for them to be written. + + At each point after the first the bounded providers are restarted, so a + buffer holding the last point's data is cleared. Unbounded providers + keep counting. + """ + info = ctx.trigger_info + if info.number_of_events != 1: + raise ValueError( + "trigger() is not supported for multiple events, the detector was " + f"prepared with number_of_events={info.number_of_events}: prepare " + "with number_of_events=1 for a step scan" + ) + if ctx.triggers and info.trigger is not DetectorTrigger.INTERNAL: + raise RuntimeError( + "An externally triggered detector must be prepared again before " + "each trigger()" + ) + # prepare() started the bounded providers for the first point + if ctx.triggers: + await _restart_bounded(ctx, f"step point {ctx.triggers + 1}") + ctx.triggers += 1 + ctx.readings = None + async with self._acquiring(ctx): + async for update in self._wait_for( + ctx, ctx.triggers - 1, ctx.triggers, divisor=1 + ): + yield update + + async def ensure_stopped(self) -> None: + """Stop every data logic and the detector. + + An acquisition armed by `prepare()` and never taken is dropped. + """ + await asyncio.gather( + *(dl.ensure_stopped() for dl in self.data), + *([self.acquire.ensure_stopped()] if self.acquire else []), + ) + + def get_hinted_fields(self, name: str, ctx: DetectorCtx | None = None) -> list[str]: + """The hinted fields of the data logics, each once. + + :param name: the detector name, which prefixes every datakey, used if + `ctx` is `None` + :param ctx: the prepared context, whose serving logics are asked for + `ctx.name` and whose datakeys filter their fields; every data logic + is asked if `None` + """ + if ctx is None: + return [field for dl in self.data for field in dl.get_hinted_fields(name)] + fields = ( + field + for dl in (*ctx.unbounded_logics, *ctx.bounded_logics) + for field in dl.get_hinted_fields(ctx.name) + if field in ctx.datakeys + ) + # A bounded logic may hint a datakey an unbounded provider makes + return list(dict.fromkeys(fields)) + + def produces(self, ctx: DetectorCtx) -> tuple[bool, bool]: + """Whether the serving providers make (stream assets, event pages).""" + return ( + any(isinstance(p, StreamableDataProvider) for p in ctx.providers), + any(isinstance(p, PageableDataProvider) for p in ctx.providers), + ) + + async def get_readings(self, ctx: DetectorCtx) -> dict[str, Reading]: + """Read the current step point's event from each pageable provider. + + Makes the page up to the end of the point and reads its last event. The + readings are kept in `ctx`, so a repeat read at the same point returns + them again. + """ + if ctx.readings is None: + page = await self._make_page(ctx, ctx.triggers) + ctx.readings = ( + { + key: Reading( + value=values[-1], timestamp=page["timestamps"][key][-1] + ) + for key, values in page["data"].items() + } + if page + else {} + ) + return ctx.readings + + async def get_written_index(self, ctx: DetectorCtx) -> int: + """The events every serving data logic has written since `prepare()`. + + In a step scan this counts every point, though the bounded data logics + only hold the current one. + """ + offsets = _collections_offset(ctx) + written = await gather_dict({s: s.get_value() for s in offsets}) + collections = min((written[s] + o for s, o in offsets.items()), default=0) + return collections // ctx.trigger_info.collections_per_event + + async def _collect_end(self, ctx: DetectorCtx, index: int | None) -> int: + """The event a collect publishes up to, counted since `prepare()`. + + `index` if given, else in a step scan the step point `trigger()` waited + for, so collect publishes exactly what trigger saw written and never + reads the counter a second time, or `get_written_index` in a fly scan. + """ + if index is None: + index = ctx.triggers or await self.get_written_index(ctx) + return index + + async def _make_page(self, ctx: DetectorCtx, index: int) -> PartialEventPage | None: + """One page merging each pageable provider's events up to event `index`. + + `None` if no provider has new events. + """ + providers = [ + (p, end) + for p, end in _provider_ends(ctx, index) + if isinstance(p, PageableDataProvider) + ] + pages = await asyncio.gather(*(p.make_page(end) for p, end in providers)) + return _merge_pages( + [(p, page) for (p, _), page in zip(providers, pages, strict=True)] + ) + + async def asset_docs( + self, ctx: DetectorCtx, index: int | None + ) -> AsyncIterator[StreamAsset]: + """The stream assets of each streamable provider up to event `index`. + + `None` collects a step scan to the end of its point, and a fly scan up to + `get_written_index`. + """ + for p, end in _provider_ends(ctx, await self._collect_end(ctx, index)): + if isinstance(p, StreamableDataProvider): + async for doc in p.make_stream_docs(end): + yield doc + + async def event_pages( + self, ctx: DetectorCtx, index: int | None + ) -> AsyncIterator[PartialEventPage]: + """One page merging each pageable provider's events up to event `index`. + + `None` collects a step scan to the end of its point, and a fly scan up to + `get_written_index`. + """ + if page := await self._make_page(ctx, await self._collect_end(ctx, index)): + yield page + + async def _wait_for( + self, ctx: DetectorCtx, start: int, end: int, divisor: int + ) -> AsyncIterator[WatcherUpdate[int]]: + """Wait until every serving data logic has written up to event `end`. + + Yields the collections written since event `start`, divided by + `divisor`. Events are counted since `prepare()`. + """ + offsets = _collections_offset(ctx) + # A detector with no data logic has nothing to wait for + if not offsets: + return + cpe = ctx.trigger_info.collections_per_event + target = (end - start) * cpe + # Until each signal reports, count it as at `start` + collections = dict.fromkeys(offsets, start * cpe) + start_time = time.monotonic() + async for signal, value in observe_signals_value( + *offsets, timeout=ctx.trigger_info.exposure_timeout + ): + collections[signal] = value + offsets[signal] + done = min(collections.values()) - start * cpe + yield WatcherUpdate( + name=ctx.name, + current=done // divisor, + initial=0, + target=target // divisor, + unit="", + precision=0, + time_elapsed=time.monotonic() - start_time, + ) + if done >= target: + break + + async def _arm(self) -> AsyncGenerator[None, None]: + """Run the acquire logic's `acquire()` to its yield, when it is armed.""" + acquisition = self.acquire.acquire() if self.acquire else _no_acquisition() + try: + await anext(acquisition) + except StopAsyncIteration: + raise RuntimeError( + f"{type(self.acquire).__name__}.acquire() finished without " + "yielding: it must yield exactly once, when armed" + ) from None + return acquisition + + @asynccontextmanager + async def _acquiring(self, ctx: DetectorCtx) -> AsyncGenerator[None, None]: + """Take the acquisition while the data is waited for. + + Takes the one armed by `prepare()` for external triggering, else arms + one now. Runs it to the end after a clean exit, and closes it otherwise. + """ + acquisition = ctx.external_acquire_gen or await self._arm() + # Taken once: a later step point arms afresh + ctx.external_acquire_gen = None + async with aclosing(acquisition): + yield + async for _ in acquisition: + raise RuntimeError( + f"{type(self.acquire).__name__}.acquire() yielded more than " + "once: it must yield exactly once, when armed" + ) + + async def _prepare_trigger_logic(self, value: TriggerInfo) -> None: + """Set the trigger logic up for `value`, raising if it cannot do it.""" + if value.trigger not in self.supported_triggers: + supported = ", ".join(sorted(t.name for t in self.supported_triggers)) + raise ValueError( + f"Trigger type {value.trigger} not supported, supported types are: " + f"[{supported}]" + ) + if not self.trigger: + if value.livetime or value.deadtime or value.exposures_per_collection != 1: + raise ValueError( + "No trigger logic, so cannot set livetime or deadtime " + "or exposures_per_collection" + ) + return + if _trigger_logic_supported(self.trigger.prepare_exposures_per_collection): + # Set up even for 1, to clear a previous setting + await self.trigger.prepare_exposures_per_collection( + value.exposures_per_collection + ) + elif value.exposures_per_collection != 1: + raise ValueError( + "Multiple exposures per collection not supported: the trigger " + "logic must implement prepare_exposures_per_collection()" + ) + match value.trigger: + case DetectorTrigger.INTERNAL: + await self.trigger.prepare_internal( + num=value.number_of_exposures, + livetime=value.livetime, + deadtime=value.deadtime, + ) + case DetectorTrigger.EXTERNAL_EDGE: + await self.trigger.prepare_edge( + num=value.number_of_exposures, livetime=value.livetime + ) + case DetectorTrigger.EXTERNAL_LEVEL: + await self.trigger.prepare_level(num=value.number_of_exposures) + + async def _resolve_collection_period(self, value: TriggerInfo) -> float: + # A livetime of 0 means "use whatever the detector currently has set", + # which a data logic sizing chunks or a buffer by the period needs to + # know. Reading it back is reading hardware state, so is opt-in. + period = value.livetime + value.deadtime + if ( + value.livetime == 0.0 + and _preserve_detector_state() + and self.trigger is not None + and _trigger_logic_supported(self.trigger.default_trigger_info) + ): + current = await self.trigger.default_trigger_info() + if current.livetime or current.deadtime: + period = current.livetime + current.deadtime + return value.exposures_per_collection * period + + +def _config_signals(device: Device) -> set[SignalR]: + """Return every `Signal` contributing to a Device's `read_configuration()`. + + Recurses into children registered as [](#StandardReadableFormat.CHILD), so + a Device that registers a sub-Device whole picks up whatever that + sub-Device declares as configuration. A Device that is not a + `StandardReadable` contributes nothing. + + A format changed at runtime cannot remove a signal: the formats the class + declared count as well as the current ones, so dropping `driver.model` from + configuration does not stop a trigger logic reading it for deadtime. + + Unlike `walk_config_signals` this reads the registry rather than calling + `read_configuration()`, so it does no I/O and includes read-only `SignalR`s + as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is + exactly the signal an areaDetector trigger logic needs for deadtime. + + Deliberately private and unexported: #1367 restructures detector logic, so + what the public shape of this should be is not settled yet. + """ + signals: set[SignalR] = set() + if not isinstance(device, StandardReadable): + return signals + current = device.get_readable_formats() + declared = device._default_readables # noqa: SLF001 + for child, format in [*declared.items(), *current.items()]: + match format: + case StandardReadableFormat.CONFIG_SIGNAL: + # set_readable_format only accepts a SignalR for a signal format + signals.add(cast(SignalR, child)) + case StandardReadableFormat.CHILD: + signals |= _config_signals(child) + case ( + StandardReadableFormat.HINTED_SIGNAL + | StandardReadableFormat.UNCACHED_SIGNAL + | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + ): + pass + return signals class StandardDetector( StandardReadable, + StandardFlyable[TriggerInfo, DetectorCtx], Stageable, Triggerable, - Preparable, - Flyable, Collectable, - WritesStreamAssets, + HasHints, ): """Detector base class for step and fly scanning detectors. - Aggregates trigger, arm, reading or stream logic together. + Aggregates trigger, acquire and data logic together in a + [](#DetectorLogic), which a subclass builds in `logic`: + + ```python + class MyDetector(StandardDetector): + def __init__(self, prefix: str, name: str = "") -> None: + self.driver = MyDriverIO(prefix) + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=MyTriggerLogic(self.driver), + acquire=MyAcquireLogic(self.driver), + ) + ``` + + For an ad-hoc detector, [](#DetectorLogic.with_device) does the same thing + without a subclass. Signals read during a step scan are registered with [](#StandardReadable.set_readable_format), exactly as on any other `StandardReadable`; data produced by a `DetectorDataLogic` is added on top - of those in `read()` and `describe()`. - - `read()` and `describe()` require `prepare()` to have run, and raise - otherwise, so that a descriptor can never be emitted without the detector's - data keys. `trigger()` prepares implicitly, so a step scan never has to do - it explicitly. `read_configuration()` and `describe_configuration()` have no - such requirement. + of those in `read()` and `describe()`, replacing a registered signal of the + same name, shape and dtype. + + Every verb but `stage()`, `read_configuration()` and + `describe_configuration()` is for after `stage()`. A step scan triggers and + reads, and `trigger()` prepares implicitly if needed; a fly scan prepares, + kicks off, completes and collects. `read()` is for after a `trigger()`, the + collect verbs for after a `kickoff()`, and `describe()`, + `describe_collect()` and `get_index()` for after a `prepare()`. An + externally triggered detector is prepared again before each `trigger()`. + After any verb raises, `unstage()` and `stage()` it before using it again. + + `WritesStreamAssets` (and its `collect_asset_docs`) and + `EventPageCollectable` (and its `collect_pages`) are *not* inherited: which + of them applies depends on what the data logics produce for a given scan, so + whichever it is gets bound as an instance attribute by `prepare()`. """ - # Logic for the detector - _trigger_logic: DetectorTriggerLogic | None = None - _acquire_logic: DetectorAcquireLogic | None = None - _data_logics: Sequence[DetectorDataLogic] = () - # Context produced by prepare, used by trigger and kickoff - _prepare_ctx: _PrepareCtx | None = None - # Context produced by kickoff, used by complete - _kickoff_ctx: _KickoffCtx | None = None - # The triggers that are supported by the trigger logic - _supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} - def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # Contribute to _StandardBase's fan out rather than replacing stage(), - # so the readable children registered on this detector still get staged - self._stage_funcs += (self._stage_detector,) - self._unstage_funcs += (self._unstage_detector,) # What the data logics produce, gathered alongside the registered # children, so the verb methods themselves need no overriding self._read_funcs += (self.__on_read,) self._describe_funcs += (self.__on_describe,) self._hints_funcs += (self.__on_hints,) + self._suppressed_datakeys_funcs += (self.__on_suppressed_datakeys,) - # Report the number of events for the next kickoff @cached_property - def events_to_kickoff(self) -> SignalRW[int]: - # TODO: only allow this to be revised down when trigger_info.number_of_events >1 - # and we have a reusable data provider - # requries https://github.com/bluesky/ophyd-async/issues/1119 - signal = soft_signal_rw(int) - # Name and parent this manually as `Device` doesn't know how to deal with cached - # properties - signal.parent = self - signal.set_name(f"{self.name}-events_to_kickoff") - return signal - - def add_detector_logics( - self, *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic - ) -> None: - """Add acquire, trigger or data logic to the detector. - - :param logic: The logic to add - """ - for logic in logics: - if isinstance(logic, DetectorTriggerLogic): - if self._trigger_logic is not None: - raise RuntimeError("Detector already has trigger logic") - self._trigger_logic = logic - # Store the triggers that are supported - self._supported_triggers = _get_supported_triggers(logic) - elif isinstance(logic, DetectorAcquireLogic): - if self._acquire_logic is not None: - raise RuntimeError("Detector already has acquire logic") - self._acquire_logic = logic - elif isinstance(logic, DetectorDataLogic): - self._data_logics = (*self._data_logics, logic) - else: - raise TypeError(f"Unknown logic type: {type(logic)}") + def _state(self) -> _StateMachine: + return super()._state | { + _State.PREPARED: {_State.TRIGGERING}, + _State.TRIGGERING: {_State.TRIGGERED, _State.STAGED}, + _State.TRIGGERED: {_State.TRIGGERING, _State.PREPARING}, + } + + @abstract_cached_property + def logic(self) -> DetectorLogic: + """The logic that drives this detector.""" + raise NotImplementedError + + def __on_suppressed_datakeys(self) -> Mapping[str, DataKey | None]: + """The datakeys of the serving providers, which replace registered signals.""" + return {} if self._ctx is None else self._ctx.datakeys + + async def __on_describe(self) -> dict[str, DataKey]: + """What the data logics contribute to `describe()`. + + Every serving provider describes its shape for a step scan, whichever + way it goes on to produce the data. + """ + return self._prepared_ctx("describe").datakeys + + async def __on_read(self) -> dict[str, Reading]: + """What the data logics contribute to `read()`: pageable data only. + + Streamable providers produce their data through `collect_asset_docs`. + """ + self._state.check_one_of(_State.TRIGGERED, verb="read") + with name_added_to_errors(self): + return await self.logic.get_readings(self._prepared_ctx("read")) + + def __on_hints(self) -> Iterator[Hints]: + """The data logics' hinted fields, alongside the children's.""" + if fields := self.logic.get_hinted_fields(self.name, self._ctx): + yield {"fields": fields} + + def prepare(self, value: TriggerInfo) -> WatchableAsyncStatus[Any]: + """Set up the scan as `StandardFlyable.prepare`, then bind the collect verb. + + The collect verb matching what the serving providers produce is bound as + a real instance attribute, so the bundler's isinstance checks + (`WritesStreamAssets` vs `EventPageCollectable`) see exactly one of them. + """ + # TODO: delete this override once the run bundler picks the collect verb + # from what a device produced (coretl/bluesky#1). See ADR 0024. + status = super().prepare(value) + # Added first, so it runs before whoever waits on the status + status.add_callback(self._bind_collect_verb) + return status + + def _bind_collect_verb(self, status: Status) -> None: + if not status.success: + return + # Those checks resolve with inspect.getattr_static on Python 3.12+, which + # does not call __getattr__, so a dynamic hook would be invisible; both + # names are reserved by Device, hence object.__setattr__. + streamable, pageable = self.logic.produces(self._prepared_ctx("prepare")) + for verb, method, wanted in ( + ("collect_asset_docs", self._collect_asset_docs, streamable), + ("collect_pages", self._collect_pages, pageable), + ): + if wanted: + object.__setattr__(self, verb, method) + elif verb in self.__dict__: + object.__delattr__(self, verb) + + @WatchableAsyncStatus.wrap + async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: + """Trigger a single exposure. + + If [`prepare()`](#StandardFlyable.prepare) has not been called since the + last `stage()`, an implicit prepare is performed. When + [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` + [](#DetectorTriggerLogic.default_trigger_info) is called to read current + hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is used. + `unstage()` cancels a running trigger, and a failed one leaves the detector + staged. + """ + if self._state.state is _State.STAGED: + with name_added_to_errors(self): + info = await self.logic.default_trigger_info() + await self.prepare(info) + async with self._state.during( + _State.TRIGGERING, end=_State.TRIGGERED, on_error=_State.STAGED + ): + with name_added_to_errors(self): + async for update in self.logic.step(self._prepared_ctx("trigger")): + yield update + + async def describe_collect(self) -> dict[str, DataKey]: + return self._prepared_ctx("describe_collect").datakeys + + async def _collect_asset_docs( + self, index: int | None = None + ) -> AsyncIterator[StreamAsset]: + # Bound as collect_asset_docs when there is a streamable provider + self._state.check_one_of( + _State.TRIGGERED, _State.FLYING, _State.FLOWN, verb="collect_asset_docs" + ) + with name_added_to_errors(self): + async for doc in self.logic.asset_docs( + self._prepared_ctx("collect_asset_docs"), index + ): + yield doc + + async def _collect_pages( + self, index: int | None = None + ) -> AsyncIterator[PartialEventPage]: + # Bound as collect_pages when there is a pageable provider + self._state.check_one_of(_State.FLYING, _State.FLOWN, verb="collect_pages") + with name_added_to_errors(self): + async for page in self.logic.event_pages( + self._prepared_ctx("collect_pages"), index + ): + yield page + + async def get_index(self) -> int: + return await self.logic.get_written_index(self._prepared_ctx("get_index")) # Back compat - delete before 1.0 def add_config_signals(self, *signals: SignalR) -> None: @@ -460,15 +1307,13 @@ async def get_trigger_deadtime( The trigger logic is given the value of every signal this detector reports as configuration, rather than nominating the ones it wants: the values that determine deadtime are exactly the ones that ought to - be recorded for the scan anyway, so a second list to keep in step - earned nothing. + be recorded for the scan anyway. :param settings: Optional settings to use when getting configuration values :return: Tuple of supported trigger types and deadtime in seconds """ - if self._trigger_logic and _trigger_logic_supported( - self._trigger_logic.get_deadtime - ): + deadtime = None + if self.logic.supports_deadtime: config_values = SignalDict() to_read: list[SignalR] = [] for sig in _config_signals(self): @@ -478,338 +1323,31 @@ async def get_trigger_deadtime( config_values[sig] = settings[cast(SignalRW, sig)] else: to_read.append(sig) - # Read live values concurrently: this is now every configuration - # signal rather than a handful the logic named, so doing it in - # series would be one round trip per signal + # Read live values concurrently: this is every configuration signal + # rather than a handful the logic named, so doing it in series would + # be one round trip per signal for sig, value in zip( to_read, await asyncio.gather(*(sig.get_value() for sig in to_read)), strict=True, ): config_values[sig] = value - deadtime = self._trigger_logic.get_deadtime(config_values) - else: - deadtime = None - return self._supported_triggers, deadtime - - @AsyncStatus.wrap - async def _stage_detector(self) -> None: - """Make sure the detector is idle and ready to be used.""" - coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] - if self._acquire_logic: - coros.append(self._acquire_logic.ensure_ready()) - await asyncio.gather(*coros) - self._prepare_ctx = None - self._kickoff_ctx = None - await self.events_to_kickoff.set(0) - - async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: - # The only thing that would stop us being able to reuse a provider is - # if the collections_per_event changes, as that would change the - # StreamResource shape. All other TriggerInfo parameters (exposures, livetime, - # etc.) don't affect the data provider configuration. - if ( - self._prepare_ctx - and self._prepare_ctx.trigger_info.collections_per_event - == trigger_info.collections_per_event - ): - # Reuse the existing data providers - streamable_data_providers = self._prepare_ctx.streamable_data_providers - else: - # Stop the existing providers if there is a context and make new ones - if self._prepare_ctx: - for data_logic in self._data_logics: - await data_logic.stop() - # Setup the data logic for the right number of collections - streamable_coros: list[Awaitable[StreamableDataProvider]] = [] - for dl in self._data_logics: - if not _data_logic_supported(dl.prepare_unbounded): - msg = f"DataLogic hasn't overridden any prepare_* methods {dl}" - raise RuntimeError(msg) - streamable_coros.append( - dl.prepare_unbounded(self.name + dl.datakey_suffix) - ) - streamable_data_providers = await asyncio.gather(*streamable_coros) - - # Stash the prepare context so we can use it in trigger/kickoff - self._prepare_ctx = _PrepareCtx( - trigger_info=trigger_info, - streamable_data_providers=streamable_data_providers, - collections_written=await _get_collections_written( - streamable_data_providers - ), - ) + deadtime = self.logic.get_deadtime(config_values) + return self.logic.supported_triggers, deadtime - async def _wait_for_index( - self, - data_providers: Sequence[StreamableDataProvider], - trigger_info: TriggerInfo, - initial_collections_written: int, - collections_requested: int, - wait_for_idle: bool, - watcher_divisor: int = 1, - ) -> AsyncIterator[WatcherUpdate]: - start_time = time.monotonic() - current_collections_written = { - dp.collections_written_signal: initial_collections_written - for dp in data_providers - } - target_collections_written = initial_collections_written + collections_requested - if data_providers: - async for sig, value in observe_signals_value( - *current_collections_written.keys(), - timeout=trigger_info.exposure_timeout, - ): - current_collections_written[sig] = value - collections_written = min(current_collections_written.values()) - yield WatcherUpdate( - name=self.name, - current=collections_written // watcher_divisor, - initial=initial_collections_written // watcher_divisor, - target=target_collections_written // watcher_divisor, - unit="", - precision=0, - time_elapsed=time.monotonic() - start_time, - ) - if collections_written >= target_collections_written: - break - if self._acquire_logic and wait_for_idle: - await self._acquire_logic.wait_for_idle() - @AsyncStatus.wrap - async def prepare(self, value: TriggerInfo) -> None: - """Prepare the detector for a number of triggers. +class _LogicDetector(StandardDetector): + """A concrete `StandardDetector` built around a logic object. - :param value: TriggerInfo describing how to trigger the detector - """ - if self._trigger_logic and _trigger_logic_supported( - self._trigger_logic.prepare_exposures_per_collection - ): - # If we can do multiple exposures per collection then set it up - # even if there was only 1 requested to clear previous settings - await self._trigger_logic.prepare_exposures_per_collection( - value.exposures_per_collection - ) - elif value.exposures_per_collection != 1: - raise ValueError( - f"Multiple exposures per collection not supported by {self}" - ) - # Setup the trigger logic for the right number of exposures - if value.trigger not in self._supported_triggers: - format_triggers = ", ".join( - sorted(t.name for t in self._supported_triggers) - ) - raise ValueError( - f"Trigger type {value.trigger} not supported by '{self.name}', " - f"supported types are: [{format_triggers}]" - ) - if self._trigger_logic: - match value.trigger: - case DetectorTrigger.INTERNAL: - await self._trigger_logic.prepare_internal( - num=value.number_of_exposures, - livetime=value.livetime, - deadtime=value.deadtime, - ) - case DetectorTrigger.EXTERNAL_EDGE: - await self._trigger_logic.prepare_edge( - num=value.number_of_exposures, - livetime=value.livetime, - ) - case DetectorTrigger.EXTERNAL_LEVEL: - await self._trigger_logic.prepare_level( - num=value.number_of_exposures, - ) - elif value.livetime != 0.0 or value.deadtime != 0.0: - raise ValueError( - f"Detector {self.name} has no trigger logic, so cannot set livetime or " - "deadtime" - ) - # NOTE: this section must come after preparing the trigger logic as we may - # use parameters from it to determine datatype for the streams - await self._update_prepare_context(value) - # Tell people how many collections we will acquire for - await self.events_to_kickoff.set(value.number_of_events) - # External triggering can start acquiring now - if self._acquire_logic and value.trigger != DetectorTrigger.INTERNAL: - await self._acquire_logic.start_acquiring() - - @WatchableAsyncStatus.wrap - async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: - """Trigger a single exposure. - - If [`prepare()`](#StandardDetector.prepare) has not been called since - the last `stage()`, an implicit prepare is - performed. When [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` - [](#DetectorTriggerLogic.default_trigger_info) is called to read current - hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is - used. - """ - if self._prepare_ctx is None: - # Opt-in: set OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES to have - # trigger() read back current hardware state (e.g. num_images) via - # default_trigger_info() instead of always falling back to TriggerInfo(). - # See ADR 0013 for rationale. - # TODO: flip default to YES and remove this guard in a future PR once - # downstream code has had time to implement default_trigger_info(). - preserve_state = ( - os.environ.get("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "NO").upper() - == "YES" - ) - if preserve_state and self._trigger_logic is not None: - if not _trigger_logic_supported( - self._trigger_logic.default_trigger_info - ): - raise RuntimeError( - f"OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but " - f"'{self.name}' has no default_trigger_info() - implement " - "default_trigger_info() on your DetectorTriggerLogic subclass " - "or unset the environment variable." - ) - trigger_info = await self._trigger_logic.default_trigger_info() - else: - trigger_info = TriggerInfo() - await self.prepare(trigger_info) - else: - # Check the one that was provided is suitable for triggering - trigger_info = self._prepare_ctx.trigger_info - if trigger_info.number_of_events != 1: - msg = ( - "trigger() is not supported for multiple events, the detector was " - f"prepared with number_of_events={trigger_info.number_of_events}." - ) - raise ValueError(msg) - # Ensure the data provider is still usable - await self._update_prepare_context(trigger_info) - ctx = error_if_none(self._prepare_ctx, "Prepare should have been run") - # Start the detector acquiring and wait for it to finish. - if self._acquire_logic: - await self._acquire_logic.start_acquiring() - async for update in self._wait_for_index( - data_providers=ctx.streamable_data_providers, - trigger_info=ctx.trigger_info, - initial_collections_written=ctx.collections_written, - collections_requested=ctx.trigger_info.collections_per_event, - watcher_divisor=1, - wait_for_idle=True, - ): - yield update - - @AsyncStatus.wrap - async def kickoff(self): - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - if not ctx.streamable_data_providers: - raise ValueError( - f"Detector {self.name} is not streamable, so cannot kickoff" - ) - collections_written, events_to_kickoff = await asyncio.gather( - _get_collections_written(ctx.streamable_data_providers), - self.events_to_kickoff.get_value(), - ) - collections_requested = ( - events_to_kickoff * ctx.trigger_info.collections_per_event - ) - last_requested_collection = collections_written + collections_requested - last_expected_collection = ( - ctx.collections_written + ctx.trigger_info.number_of_collections - ) - if last_requested_collection > last_expected_collection: - msg = ( - f"Kickoff requested {collections_written}:{last_requested_collection}, " - f"but detector was only prepared up to {last_expected_collection}" - ) - raise RuntimeError(msg) - self._kickoff_ctx = _KickoffCtx( - trigger_info=ctx.trigger_info, - data_providers=ctx.streamable_data_providers, - collections_written=collections_written, - collections_requested=collections_requested, - is_last_kickoff=last_requested_collection == last_expected_collection, - ) - # External triggering has already started; internal starts now - if self._acquire_logic and ctx.trigger_info.trigger == DetectorTrigger.INTERNAL: - await self._acquire_logic.start_acquiring() - - @WatchableAsyncStatus.wrap - async def complete(self): - ctx = error_if_none(self._kickoff_ctx, "Kickoff not called") - async for update in self._wait_for_index( - data_providers=ctx.data_providers, - trigger_info=ctx.trigger_info, - initial_collections_written=ctx.collections_written, - collections_requested=ctx.collections_requested, - wait_for_idle=ctx.is_last_kickoff, - watcher_divisor=ctx.trigger_info.collections_per_event, - ): - yield update - - async def __on_read(self) -> dict[str, Reading]: - """What the data logics contribute to `read()`: nothing, but checks prepare. - - Raising when nothing has been prepared is deliberate, and predates - registered children being readable at all: a detector that has not been - prepared has no data keys. In practice `trigger()` prepares implicitly, - so `trigger_and_read` never reaches this. - """ - error_if_none(self._prepare_ctx, "Prepare not run") - return {} - - async def __on_describe(self) -> dict[str, DataKey]: - """What the data logics contribute to `describe()`. - - Raises when nothing has been prepared, as a `describe()` that quietly - succeeded would emit a descriptor missing the detector's data. - Streamable providers describe their shape for a step scan, but produce - their data through collect_asset_docs rather than read(). - """ - ctx = error_if_none(self._prepare_ctx, "Prepare not run") - return await merge_gathered_dicts( - sdp.make_datakeys(ctx.trigger_info.collections_per_event) - for sdp in ctx.streamable_data_providers - ) - - async def describe_collect(self) -> dict[str, DataKey]: - ctx = error_if_none(self._prepare_ctx, "Prepare not run") - # Only streamable data providers produce data during collect - coros = [ - dp.make_datakeys(ctx.trigger_info.collections_per_event) - for dp in ctx.streamable_data_providers - ] - return await merge_gathered_dicts(coros) - - def __on_hints(self) -> Iterator[Hints]: - """The data logics' hinted fields, alongside the children's.""" - for dl in self._data_logics: - if fields := dl.get_hinted_fields(self.name + dl.datakey_suffix): - yield {"fields": list(fields)} + `StandardDetector.logic` is abstract, so the class itself cannot be + instantiated; `DetectorLogic.with_device` needs something concrete to wrap a + logic object in. + """ - async def collect_asset_docs( - self, index: int | None = None - ) -> AsyncIterator[StreamAsset]: - # Collect stream datum documents for all indices written. - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - if index is None: - # The index is optional, and provided for fly scans, if there is - # more than one detector to make sure they collect in step - index = await self.get_index() - for data_provider in ctx.streamable_data_providers: - async for doc in data_provider.make_stream_docs( - collections_written=index * ctx.trigger_info.collections_per_event, - collections_per_event=ctx.trigger_info.collections_per_event, - ): - yield doc + def __init__(self, logic: DetectorLogic, name: str = "") -> None: + self._logic = logic + super().__init__(name=name) - async def get_index(self) -> int: - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - min_collections_written = await _get_collections_written( - ctx.streamable_data_providers, reducer=min - ) - return min_collections_written // ctx.trigger_info.collections_per_event - - @AsyncStatus.wrap - async def _unstage_detector(self) -> None: - """Stop the detector and file writing.""" - coros: list[Awaitable] = [data_logic.stop() for data_logic in self._data_logics] - if self._acquire_logic: - coros.append(self._acquire_logic.ensure_stopped()) - await asyncio.gather(*coros) + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index f851d97485..2949dbd3df 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -4,8 +4,10 @@ from collections.abc import ( Awaitable, Callable, + Collection, Generator, Iterator, + Mapping, MutableMapping, Sequence, ) @@ -21,7 +23,7 @@ from ._protocol import AsyncConfigurable, AsyncReadable, AsyncStageable from ._signal import SignalR, walk_devices from ._standard_base import _StandardBase -from ._utils import merge_gathered_dicts +from ._utils import merge_dicts, merge_gathered_dicts class StandardReadableFormat(Enum): @@ -115,9 +117,30 @@ def __iter__(self) -> Iterator[Device]: def __len__(self) -> int: return len(self._formats) - def entries(self) -> list[_ReadableEntry]: - """Each child and its format, typed by what `__setitem__` allowed.""" - return cast(list[_ReadableEntry], list(self._formats.items())) + def entries(self, without: Collection[str] = ()) -> list[_ReadableEntry]: + """Each child and its format, typed by what `__setitem__` allowed. + + :param without: names of signals to leave out + """ + return [ + entry + for entry in cast(list[_ReadableEntry], list(self._formats.items())) + if entry[1] is Format.CHILD or entry[0].name not in without + ] + + +def _switched_off_names(suppressed: Mapping[str, DataKey | None]) -> set[str]: + """The names that are switched off, which have no replacement.""" + return {name for name, datakey in suppressed.items() if datakey is None} + + +def _without_fields(hints: Hints, names: Collection[str]) -> Hints: + """`hints` with `names` left out of its fields.""" + fields = hints.get("fields") + if isinstance(fields, str) or not isinstance(fields, Sequence): + # Absent or malformed: hints reports the malformed ones + return hints + return cast(Hints, {**hints, "fields": [n for n in fields if n not in names]}) class _SealDefaultFormatsMeta(type(_StandardBase)): # type: ignore[misc] @@ -205,6 +228,14 @@ def _readables(self) -> _Readables: Callable[[], Awaitable[dict[str, DataKey]]], ... ] = () _hints_funcs: tuple[Callable[[], Iterator[Hints]], ...] = () + # The names of registered children that are not read for now, by name so a + # child's own children are covered. A DataKey means something else provides + # that name instead: the child is dropped from read() and describe() after + # checking it against the DataKey, and kept in hints. None means the name is + # switched off: it is dropped from read(), describe() and hints. + _suppressed_datakeys_funcs: tuple[ + Callable[[], Mapping[str, DataKey | None]], ... + ] = () def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -282,10 +313,21 @@ async def __on_read_configuration(self) -> dict[str, Reading]: pass return await merge_gathered_dicts(coros) + def _suppressed_datakeys(self) -> dict[str, DataKey | None]: + return merge_dicts(func() for func in self._suppressed_datakeys_funcs) + async def __on_describe(self) -> dict[str, DataKey]: - """What the registered children contribute to `describe()`.""" + """What the registered children contribute to `describe()`. + + Every signal with a replacement is described too, to check the + replacement can stand in for it. + """ coros = [] - for entry in self._readables.entries(): + suppressed = self._suppressed_datakeys() + replacements = { + name: datakey for name, datakey in suppressed.items() if datakey is not None + } + for entry in self._readables.entries(without=_switched_off_names(suppressed)): match entry: case (device, Format.CHILD): if isinstance(device, AsyncReadable): @@ -299,12 +341,29 @@ async def __on_describe(self) -> dict[str, DataKey]: coros.append(signal.describe()) case (_, Format.CONFIG_SIGNAL): pass - return await merge_gathered_dicts(coros) + datakeys = await merge_gathered_dicts(coros) + # A replacement needs the signal's dtype, and its shape or that shape with + # one extra leading dimension, e.g. one per collection of an event + mismatched = [] + for key in sorted(datakeys.keys() & replacements.keys()): + signal, data = datakeys[key], replacements[key] + same_dtype = signal.get("dtype_numpy") == data.get("dtype_numpy") + same_shape = signal["shape"] in (data["shape"], data["shape"][1:]) + if not (same_dtype and same_shape): + mismatched.append(key) + if mismatched: + raise ValueError( + f"{self.name}: duplicate datakeys {mismatched}: registered signals " + "with a different shape or dtype from the data that would replace " + "them, so rename one of them" + ) + return {k: v for k, v in datakeys.items() if k not in suppressed} async def __on_read(self) -> dict[str, Reading]: """What the registered children contribute to `read()`.""" coros = [] - for entry in self._readables.entries(): + suppressed = self._suppressed_datakeys() + for entry in self._readables.entries(without=suppressed): match entry: case (device, Format.CHILD): if isinstance(device, AsyncReadable): @@ -318,15 +377,18 @@ async def __on_read(self) -> dict[str, Reading]: coros.append(signal.read(cached=False)) case (_, Format.CONFIG_SIGNAL): pass - return await merge_gathered_dicts(coros) + readings = await merge_gathered_dicts(coros) + # entries() leaves out suppressed signals, but not those read by a child + return {k: v for k, v in readings.items() if k not in suppressed} def __on_hints(self) -> Iterator[Hints]: """What the registered children contribute to `hints`.""" - for entry in self._readables.entries(): + switched_off = _switched_off_names(self._suppressed_datakeys()) + for entry in self._readables.entries(without=switched_off): match entry: case (device, Format.CHILD): if isinstance(device, HasHints): - yield device.hints + yield _without_fields(device.hints, switched_off) case ( signal, Format.HINTED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, @@ -542,36 +604,6 @@ def add_readables( self.set_readable_format(device, format) -def _config_signals(device: Device) -> set[SignalR]: - """Return every `Signal` contributing to a Device's `read_configuration()`. - - Recurses into children registered as [](#StandardReadableFormat.CHILD), so - a Device that registers a sub-Device whole picks up whatever that - sub-Device declares as configuration. A Device that is not a - `StandardReadable` contributes nothing. - - Unlike `walk_config_signals` this reads the registry rather than calling - `read_configuration()`, so it does no I/O and includes read-only `SignalR`s - as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is - exactly the signal an areaDetector trigger logic needs for deadtime. - - Deliberately private and unexported: #1367 restructures detector logic, so - what the public shape of this should be is not settled yet. - """ - signals: set[SignalR] = set() - if not isinstance(device, StandardReadable): - return signals - for entry in device._readables.entries(): # noqa: SLF001 - match entry: - case (signal, Format.CONFIG_SIGNAL): - signals.add(signal) - case (child, Format.CHILD): - signals |= _config_signals(child) - case _: - pass - return signals - - #: Reserved key under which [](#store_settings) writes readable formats. Chosen #: so that it cannot be a dotted attribute path: `<` cannot start a Python #: identifier, so no attribute assignment can produce a colliding key, and diff --git a/src/ophyd_async/epics/adandor.py b/src/ophyd_async/epics/adandor.py index 5e1feef409..885aaf3c43 100644 --- a/src/ophyd_async/epics/adandor.py +++ b/src/ophyd_async/epics/adandor.py @@ -17,7 +17,7 @@ from ophyd_async.epics.adcore import ( ADAcquireLogic, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, NDProcessIO, @@ -35,7 +35,6 @@ ] _MIN_DEAD_TIME = 0.1 -_MAX_NUM_IMAGE = 999_999 class Andor2TriggerMode(StrictEnum): @@ -75,19 +74,17 @@ def get_deadtime(self, config_values: SignalDict) -> float: async def prepare_internal(self, num: int, livetime: float, deadtime: float): await self.driver.trigger_mode.set(Andor2TriggerMode.INTERNAL) - await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE, livetime, deadtime) + await prepare_exposures(self.driver, num, livetime, deadtime) async def prepare_edge(self, num: int, livetime: float): await self.driver.trigger_mode.set(Andor2TriggerMode.EXT_TRIGGER) - await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE, livetime) + await prepare_exposures(self.driver, num, livetime) async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class AndorDetector(AreaDetector[Andor2DriverIO]): @@ -105,7 +102,7 @@ class AndorDetector(AreaDetector[Andor2DriverIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_suffix="cam1:", proc_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index 936f2951ca..b4e109f28f 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -19,7 +19,7 @@ from .adcore import ( ADAcquireLogic, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, NDProcessIO, @@ -86,9 +86,7 @@ async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class AravisDetector(AreaDetector[AravisDriverIO]): @@ -109,7 +107,7 @@ class AravisDetector(AreaDetector[AravisDriverIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_suffix: str = "cam1:", proc_suffix: str | None = None, override_deadtime: float | None = None, diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 80e87ecfee..df4e943dc8 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -5,6 +5,7 @@ from ._acquire_logic import ADAcquireLogic, ADContAcqAcquireLogic from ._data_logic import ( + ADDataFactory, ADHDFDataLogic, ADMultipartDataLogic, ADWriterFactory, @@ -50,6 +51,7 @@ from ._plan_stubs import setup_ndattributes, setup_ndstats_sum from ._trigger_logic import ( ADContAcqTriggerLogic, + exposures_per_collection_from_filter, prepare_exposures, prepare_filter, trigger_info_from_driver, @@ -89,6 +91,7 @@ "prepare_filter", "ADContAcqTriggerLogic", "trigger_info_from_driver", + "exposures_per_collection_from_filter", # AcquireLogic "ADAcquireLogic", "ADContAcqAcquireLogic", @@ -96,6 +99,7 @@ "NDArrayDescription", "ADHDFDataLogic", "ADMultipartDataLogic", + "ADDataFactory", "ADWriterFactory", # Detector "AreaDetector", diff --git a/src/ophyd_async/epics/adcore/_acquire_logic.py b/src/ophyd_async/epics/adcore/_acquire_logic.py index 2ae1a7a587..037ce3e049 100644 --- a/src/ophyd_async/epics/adcore/_acquire_logic.py +++ b/src/ophyd_async/epics/adcore/_acquire_logic.py @@ -1,8 +1,8 @@ import asyncio +from collections.abc import AsyncGenerator from ophyd_async.core import ( DEFAULT_TIMEOUT, - AsyncStatus, DetectorAcquireLogic, SignalR, set_and_wait_for_other_value, @@ -24,7 +24,7 @@ class ADAcquireLogic(DetectorAcquireLogic): Whether the detector should wait for its plugins to drain before it reports acquisition as complete. `WaitForPlugins` makes the driver's `AcquireBusy` wait for the number of queued arrays to reach zero, and - `wait_for_idle` blocks on the `acquire` put callback, so turning it on + `acquire` waits for the `acquire` put callback, so turning it on is what makes a step scan read plugin scalars for the frame it just took rather than the one before. Defaults to on, since racing is silent and the wait costs nothing when a file writer is the slowest @@ -47,7 +47,6 @@ def __init__( self.driver_armed_signal = driver.acquire self.wait_for_plugins = wait_for_plugins self.timeout = timeout - self.acquire_status: AsyncStatus | None = None async def ensure_ready(self): # Stop first: WaitForPlugins is latched and has no _RBV, so set it once @@ -58,19 +57,17 @@ async def ensure_ready(self): await self.ensure_stopped() await self.driver.wait_for_plugins.set(self.wait_for_plugins) - async def start_acquiring(self): - self.acquire_status = await set_and_wait_for_other_value( + async def acquire(self) -> AsyncGenerator[None, None]: + async with await set_and_wait_for_other_value( set_signal=self.driver.acquire, set_value=True, match_signal=self.driver_armed_signal, match_value=True, wait_for_set_completion=False, timeout=self.timeout, - ) - - async def wait_for_idle(self): - if self.acquire_status: - await self.acquire_status + ) as status: + yield + await status await wait_for_good_state( self.driver.detector_state, {ADState.IDLE, ADState.ABORTED}, @@ -87,14 +84,13 @@ class ADContAcqAcquireLogic(DetectorAcquireLogic): Deliberately has no `wait_for_plugins` equivalent to [](#ADAcquireLogic). That setting gates the driver's `AcquireBusy` going to 0 at the end of an acquisition, and here the driver is always acquiring -- there is no such - transition to wait on. `wait_for_idle` waits on the circular buffer's - capture instead. + transition to wait on. `acquire` waits on the circular buffer's trigger + instead. """ def __init__(self, driver: ADBaseIO, cb_plugin: NDCircularBuffIO): self.driver = driver self.cb_plugin = cb_plugin - self.acquire_status: AsyncStatus | None = None async def ensure_ready(self): await set_and_wait_for_value( @@ -104,10 +100,9 @@ async def ensure_ready(self): timeout=DEFAULT_TIMEOUT, ) - async def start_acquiring(self): + async def acquire(self) -> AsyncGenerator[None, None]: await self.cb_plugin.trigger_.set(True) - - async def wait_for_idle(self): + yield num_images, acquire_time = await asyncio.gather( self.cb_plugin.post_count.get_value(), self.driver.acquire_time.get_value(), diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 2542ed1838..1ef6b4de8a 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -1,5 +1,6 @@ import asyncio -from collections.abc import Callable, Sequence +import warnings +from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass from pathlib import PureWindowsPath from typing import Any, Generic @@ -8,15 +9,17 @@ import numpy as np from ophyd_async.core import ( - DetectorDataLogic, + DEFAULT_TIMEOUT, + BoundedDataLogic, EnableDisable, PathInfo, PathProvider, SignalR, - StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + UnboundedDataLogic, set_and_wait_for_value, + wait_for_value, ) from ophyd_async.epics.core import stop_busy_record @@ -28,8 +31,9 @@ NDArrayBaseIO, NDFileHDF5IO, NDPluginBaseIO, + NDPluginBaseIOT, NDPluginFileIO, - NDPluginFileIOT, + plugin_is_enabled, ) from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType @@ -144,8 +148,14 @@ async def prepare_file_paths( await writer.num_capture.set(0) +async def _start_capture(writer: NDPluginFileIO) -> None: + await set_and_wait_for_value(writer.capture, True, wait_for_set_completion=False) + # Capturing resets NumCaptured, which the detector reads as its baseline + await wait_for_value(writer.num_captured, 0, timeout=DEFAULT_TIMEOUT) + + @dataclass -class ADHDFDataLogic(DetectorDataLogic): +class ADHDFDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): """Data logic for AreaDetector HDF5 writer plugin. :param array_description: Signals describing the NDArray shape and data type. @@ -164,73 +174,91 @@ class ADHDFDataLogic(DetectorDataLogic): plugins: Sequence[NDPluginBaseIO] = () datakey_suffix: str = "" hinted: bool = True + #: True serves every scan. False sits out the scans of a stage if the + #: plugin's EnableCallbacks PV is off at the first prepare. Starting switches + #: it on. + always_enable: bool = True - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - # Work out where to write + def __post_init__(self) -> None: + self.collections_written = self.writer.num_captured + + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> StreamResourceDataProvider | None: + if not self.always_enable and not await plugin_is_enabled(self.writer): + return None + datakey_name = name + self.datakey_suffix path_info = self.path_provider(datakey_name) - # Determine number of frames that will be saved per HDF chunk. - # On a fresh IOC startup, this is set to zero until the first capture, - # so if it is zero, set it to 1. - frames_per_chunk = await self.writer.num_frames_chunks.get_value() - if frames_per_chunk == 0: - frames_per_chunk = 1 - await self.writer.num_frames_chunks.set(frames_per_chunk) - # Setup the HDF writer - await asyncio.gather( - self.writer.chunk_size_auto.set(True), - self.writer.num_extra_dims.set(0), - self.writer.lazy_open.set(True), - self.writer.swmr_mode.set(True), - self.writer.xml_file_name.set(""), - self.writer.enable_callbacks.set(EnableDisable.ENABLE), - prepare_file_paths( - path_info=path_info, file_template="%s%s.h5", writer=self.writer + # Size the HDF chunk to hold a flush period of collections, e.g. 400 Hz + # (2.5 ms) with a 0.5 s flush period gives 200 frames per chunk (#1309). + # Otherwise use the chunk size set on the IOC, forcing a fresh-startup 0 + # to 1. + if flush_period > 0 and collection_period > 0: + frames_per_chunk = max(1, round(flush_period / collection_period)) + else: + frames_per_chunk = await self.writer.num_frames_chunks.get_value() or 1 + main_dataset, ndattribute_dtype_sources = await asyncio.gather( + get_ndarray_resource_info( + array_description=self.array_description, + data_key=datakey_name, + parameters={"dataset": "/entry/data/data"}, + frames_per_chunk=frames_per_chunk, + ), + get_ndattribute_dtype_source( + (self.driver, *self.plugins) if self.driver else (*self.plugins,) ), - ) - # Start capturing - await set_and_wait_for_value( - self.writer.capture, True, wait_for_set_completion=False - ) - # Return a provider that reflects what we have made - main_dataset = await get_ndarray_resource_info( - array_description=self.array_description, - data_key=datakey_name, - parameters={"dataset": "/entry/data/data"}, - frames_per_chunk=frames_per_chunk, - ) - ndattribute_dtype_sources = await get_ndattribute_dtype_source( - (self.driver, *self.plugins) if self.driver else (*self.plugins,) ) ndattribute_datasets = [ StreamResourceInfo( - data_key=name, + data_key=attr_name, shape=(), # NDAttributes appear to always be configured with # this chunk size chunk_shape=(16384,), dtype_numpy=dtype_numpy, source=source, - parameters={"dataset": f"/entry/instrument/NDAttributes/{name}"}, + parameters={"dataset": f"/entry/instrument/NDAttributes/{attr_name}"}, ) - for name, (dtype_numpy, source) in ndattribute_dtype_sources.items() + for attr_name, (dtype_numpy, source) in ndattribute_dtype_sources.items() ] + # Write back the chunk size the main dataset is described with, so the + # file matches the chunk_shape in its stream resource either way + await self.writer.num_frames_chunks.set(frames_per_chunk) + # Setup the HDF writer + coros: list[Awaitable] = [ + self.writer.chunk_size_auto.set(True), + self.writer.num_extra_dims.set(0), + self.writer.lazy_open.set(True), + self.writer.swmr_mode.set(True), + self.writer.xml_file_name.set(""), + prepare_file_paths( + path_info=path_info, file_template="%s%s.h5", writer=self.writer + ), + self.writer.enable_callbacks.set(EnableDisable.ENABLE), + ] + await asyncio.gather(*coros) + await _start_capture(self.writer) return StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[main_dataset] + ndattribute_datasets, mimetype="application/x-hdf5", - collections_written_signal=self.writer.num_captured, + collections_per_event=collections_per_event, flush_signal=self.writer.flush_now, ) - async def stop(self) -> None: + async def ensure_stopped(self) -> None: await stop_busy_record(self.writer.capture) - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return [datakey_name] if self.hinted else [] + def get_hinted_fields(self, name: str) -> Sequence[str]: + return [name + self.datakey_suffix] if self.hinted else [] @dataclass -class ADMultipartDataLogic(DetectorDataLogic): +class ADMultipartDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): """Data logic for multipart AreaDetector file writers (e.g. JPEG, TIFF). :param array_description: Signals describing the NDArray shape and data type. @@ -241,6 +269,10 @@ class ADMultipartDataLogic(DetectorDataLogic): Mimetype for the written files (e.g. "multipart/related;type=image/jpeg"). :param datakey_suffix: Suffix to append to the data key for the main dataset :param hinted: Include the image in hints. Set False for BestEffortCallback. + :param always_enable: + True serves every scan. False sits out the scans of a stage if the + plugin's EnableCallbacks PV is off at the first prepare. Starting + switches it on. """ array_description: NDArrayDescription @@ -250,10 +282,29 @@ class ADMultipartDataLogic(DetectorDataLogic): mimetype: str datakey_suffix: str = "" hinted: bool = True + always_enable: bool = True + + def __post_init__(self) -> None: + self.collections_written = self.writer.num_captured - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - # Work out where to write + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> StreamResourceDataProvider | None: + # A multipart writer writes one file per frame, so there is nothing to + # size from the periods + if not self.always_enable and not await plugin_is_enabled(self.writer): + return None + datakey_name = name + self.datakey_suffix path_info = self.path_provider(datakey_name) + main_dataset = await get_ndarray_resource_info( + array_description=self.array_description, + data_key=datakey_name, + parameters={"template": path_info.filename + "_{:06d}" + self.extension}, + ) # Setup the file writer await asyncio.gather( self.writer.lazy_open.set(True), @@ -262,45 +313,37 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: file_template="%s%s_%6.6d" + self.extension, writer=self.writer, ), + self.writer.enable_callbacks.set(EnableDisable.ENABLE), ) - # Start capturing - await set_and_wait_for_value( - self.writer.capture, True, wait_for_set_completion=False - ) - # Return a provider that reflects what we have made - main_dataset = await get_ndarray_resource_info( - array_description=self.array_description, - data_key=datakey_name, - parameters={"template": path_info.filename + "_{:06d}" + self.extension}, - ) + await _start_capture(self.writer) return StreamResourceDataProvider( # TODO: remove the type ignore after # https://github.com/bluesky/ophyd-async/issues/1186 uri=path_info.directory_uri, resources=[main_dataset], mimetype=self.mimetype, - collections_written_signal=self.writer.num_captured, + collections_per_event=collections_per_event, ) - async def stop(self) -> None: + async def ensure_stopped(self) -> None: await stop_busy_record(self.writer.capture) - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return [datakey_name] if self.hinted else [] + def get_hinted_fields(self, name: str) -> Sequence[str]: + return [name + self.datakey_suffix] if self.hinted else [] @dataclass -class ADWriterFactory(Generic[NDPluginFileIOT]): - """Factory that creates a file-writer plugin and its matching data logic. +class ADDataFactory(Generic[NDPluginBaseIOT]): + """Factory that creates a plugin and its matching data logic. - Construct using the classmethods `hdf`, `jpeg`, or `tiff`, then pass one + Construct using the static methods `hdf`, `jpeg` or `tiff`, then pass one or more instances to `AreaDetector` as positional `*writer_factories` - arguments. When the detector is initialised `__call__` is invoked with - the detector's PV `prefix`, its `driver`, and the flat list of extra - `plugins`; it returns the writer device and the corresponding - `DetectorDataLogic`. + arguments. When the detector is initialised + `__call__` is invoked with the detector's PV `prefix`, its `driver`, and the + flat list of extra `plugins`; it returns the writer device and the + corresponding data logic. - :param writer_cls: Concrete `NDPluginFileIO` subclass to instantiate. + :param writer_cls: Concrete `NDPluginBaseIO` subclass to instantiate. :param writer_suffix: PV suffix appended to *prefix* to form the writer's PV prefix. :param writer_name: Attribute name under which the writer device is stored on the @@ -315,11 +358,11 @@ class ADWriterFactory(Generic[NDPluginFileIOT]): depends on signals that are only available once the driver has been constructed (e.g. inside a detector subclass that creates its own driver). :param data_logic_factory: - Callable ``(writer, array_description, driver, plugins) → DetectorDataLogic`` + Callable ``(writer, array_description, driver, plugins) → data logic`` that builds the data logic given the already-constructed writer. """ - writer_cls: type[NDPluginFileIOT] + writer_cls: type[NDPluginBaseIOT] writer_suffix: str writer_name: str datakey_suffix: str @@ -327,8 +370,8 @@ class ADWriterFactory(Generic[NDPluginFileIOT]): NDArrayDescription | Callable[[ADBaseIO], NDArrayDescription] | None ) data_logic_factory: Callable[ - [NDPluginFileIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], - DetectorDataLogic, + [NDPluginBaseIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], + BoundedDataLogic | UnboundedDataLogic, ] def __call__( @@ -336,7 +379,7 @@ def __call__( prefix: str, driver: ADBaseIO, plugins: Sequence[NDPluginBaseIO], - ) -> tuple[NDPluginFileIOT, DetectorDataLogic]: + ) -> tuple[NDPluginBaseIOT, BoundedDataLogic | UnboundedDataLogic]: """Instantiate the writer plugin and build the data logic. :param prefix: EPICS PV prefix for the detector (same as `AreaDetector.prefix`). @@ -373,7 +416,8 @@ def hdf( | Callable[[ADBaseIO], NDArrayDescription] | None = None, hinted: bool = True, - ) -> "ADWriterFactory[NDFileHDF5IO]": + always_enable: bool = True, + ) -> "ADDataFactory[NDFileHDF5IO]": """Create a factory for an HDF5 file writer. :param path_provider: Provides file path information for each acquisition. @@ -389,8 +433,12 @@ def hdf( when the shape/type comes from a plugin rather than the main driver (e.g. an ROI plugin). :param hinted: Include the image in hints. Set False for BestEffortCallback. + :param always_enable: True (the default) serves every scan. False sits + out the scans of a stage if the plugin's EnableCallbacks PV is off at + the first prepare, so another data logic can serve them. Starting + switches the plugin on either way. """ - return ADWriterFactory( + return ADDataFactory( writer_cls=NDFileHDF5IO, writer_suffix=writer_suffix, writer_name=writer_name, @@ -404,6 +452,7 @@ def hdf( plugins=list(plugins), datakey_suffix=datakey_suffix, hinted=hinted, + always_enable=always_enable, ), ) @@ -417,7 +466,8 @@ def jpeg( | Callable[[ADBaseIO], NDArrayDescription] | None = None, hinted: bool = True, - ) -> "ADWriterFactory[NDPluginFileIO]": + always_enable: bool = True, + ) -> "ADDataFactory[NDPluginFileIO]": """Create a factory for a JPEG file writer. :param path_provider: Provides file path information for each acquisition. @@ -432,8 +482,12 @@ def jpeg( Pass an `NDArrayDescription` or a callable ``(driver) → NDArrayDescription`` when the shape/type comes from a plugin. :param hinted: Include the image in hints. Set False for BestEffortCallback. + :param always_enable: True (the default) serves every scan. False sits + out the scans of a stage if the plugin's EnableCallbacks PV is off at + the first prepare, so another data logic can serve them. Starting + switches the plugin on either way. """ - return ADWriterFactory( + return ADDataFactory( writer_cls=NDPluginFileIO, writer_suffix=writer_suffix, writer_name=writer_name, @@ -448,6 +502,7 @@ def jpeg( mimetype="multipart/related;type=image/jpeg", datakey_suffix=datakey_suffix, hinted=hinted, + always_enable=always_enable, ) ), ) @@ -462,7 +517,8 @@ def tiff( | Callable[[ADBaseIO], NDArrayDescription] | None = None, hinted: bool = True, - ) -> "ADWriterFactory[NDPluginFileIO]": + always_enable: bool = True, + ) -> "ADDataFactory[NDPluginFileIO]": """Create a factory for a TIFF file writer. :param path_provider: Provides file path information for each acquisition. @@ -477,8 +533,12 @@ def tiff( Pass an `NDArrayDescription` or a callable ``(driver) → NDArrayDescription`` when the shape/type comes from a plugin. :param hinted: Include the image in hints. Set False for BestEffortCallback. + :param always_enable: True (the default) serves every scan. False sits + out the scans of a stage if the plugin's EnableCallbacks PV is off at + the first prepare, so another data logic can serve them. Starting + switches the plugin on either way. """ - return ADWriterFactory( + return ADDataFactory( writer_cls=NDPluginFileIO, writer_suffix=writer_suffix, writer_name=writer_name, @@ -493,6 +553,38 @@ def tiff( mimetype="multipart/related;type=image/tiff", datakey_suffix=datakey_suffix, hinted=hinted, + always_enable=always_enable, ) ), ) + + +class ADWriterFactory(ADDataFactory[NDPluginBaseIOT]): + """Deprecated name of `ADDataFactory`.""" + + def __post_init__(self) -> None: + _warn_writer_factory() + + @staticmethod + def hdf(*args, **kwargs) -> ADDataFactory[NDFileHDF5IO]: + _warn_writer_factory() + return ADDataFactory.hdf(*args, **kwargs) + + @staticmethod + def jpeg(*args, **kwargs) -> ADDataFactory[NDPluginFileIO]: + _warn_writer_factory() + return ADDataFactory.jpeg(*args, **kwargs) + + @staticmethod + def tiff(*args, **kwargs) -> ADDataFactory[NDPluginFileIO]: + _warn_writer_factory() + return ADDataFactory.tiff(*args, **kwargs) + + +def _warn_writer_factory() -> None: + warnings.warn( + DeprecationWarning( + "'ADWriterFactory' is deprecated, use 'ADDataFactory' instead" + ), + stacklevel=3, + ) diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index 6e395254f7..7a0e1abec5 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -1,16 +1,20 @@ from collections.abc import Mapping, Sequence +from functools import cached_property from typing import Generic from ophyd_async.core import ( + BoundedDataLogic, DetectorAcquireLogic, + DetectorLogic, DetectorTriggerLogic, SignalR, StandardDetector, + UnboundedDataLogic, ) from ophyd_async.core import StandardReadableFormat as Format from ._acquire_logic import ADContAcqAcquireLogic -from ._data_logic import ADWriterFactory +from ._data_logic import ADDataFactory from ._io import ( ADBaseIO, ADBaseIOT, @@ -27,7 +31,7 @@ def __init__( self, driver: ADBaseIOT, prefix: str | None = None, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, acquire_logic: DetectorAcquireLogic | None = None, trigger_logic: DetectorTriggerLogic | None = None, plugins: Mapping[str, NDPluginBaseIO] | None = None, @@ -37,11 +41,10 @@ def __init__( self.driver = driver if plugins is not None: for plugin_name, plugin in plugins.items(): - setattr(self, plugin_name, plugin) - if trigger_logic: - self.add_detector_logics(trigger_logic) - if acquire_logic: - self.add_detector_logics(acquire_logic) + self._set_unique(plugin_name, plugin) + self._trigger_logic = trigger_logic + self._acquire_logic = acquire_logic + data_logics: list[BoundedDataLogic | UnboundedDataLogic] = [] if writer_factories: if prefix is None: raise ValueError("prefix is required when writer_factories are given") @@ -54,8 +57,9 @@ def __init__( plugin_list = list(plugins.values()) if plugins else [] for factory in writer_factories: writer, data_logic = factory(prefix, driver, plugin_list) - setattr(self, factory.writer_name, writer) - self.add_detector_logics(data_logic) + self._set_unique(factory.writer_name, writer) + data_logics.append(data_logic) + self._data_logics = tuple(data_logics) # The driver is always wired up, so register it whole and let ADBaseIO # declare which of its signals are configuration. Plugins are # deliberately *not* registered: a detector normally carries far more @@ -66,6 +70,22 @@ def __init__( self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) + def _set_unique(self, attr: str, value: NDPluginBaseIO) -> None: + if attr in self.__dict__ or hasattr(type(self), attr): + raise ValueError( + f"Cannot add {type(value).__name__} as attribute '{attr}': " + f"{type(self).__name__} already has an attribute with that name" + ) + setattr(self, attr, value) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=self._trigger_logic, + acquire=self._acquire_logic, + data=self._data_logics, + ) + def get_plugin( self, name: str, plugin_type: type[NDPluginBaseIOT] = NDPluginBaseIO ) -> NDPluginBaseIOT: @@ -96,6 +116,7 @@ class ContAcqDetector(AreaDetector[ADBaseIOT]): :param driver_cls: IO class for the driver, defaults to ADBaseIO :param driver_suffix: Suffix for the driver PV, defaults to "cam1:" :param cb_suffix: Suffix for the circular buffer plugin PV, defaults to "CB1:" + :param cb_plugin_name: Attribute name of the circular buffer plugin :param proc_suffix: Suffix for the NDProcessIO plugin PV, if any, defaults to None :param plugins: Additional areaDetector plugins to include :param config_sigs: Additional signals to include in configuration @@ -105,17 +126,18 @@ class ContAcqDetector(AreaDetector[ADBaseIOT]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_cls: type[ADBaseIOT] = ADBaseIO, driver_suffix: str = "cam1:", cb_suffix: str = "CB1:", + cb_plugin_name: str = "cb", proc_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), name: str = "", ) -> None: driver = driver_cls(prefix + driver_suffix) - cb_plugin = NDCircularBuffIO(prefix + cb_suffix) + cb_plugin = NDCircularBuffIO(prefix + cb_suffix, name=cb_plugin_name) proc_plugin = NDProcessIO(prefix + proc_suffix) if proc_suffix else None super().__init__( driver, @@ -124,7 +146,7 @@ def __init__( acquire_logic=ADContAcqAcquireLogic(driver, cb_plugin), trigger_logic=ADContAcqTriggerLogic(driver, cb_plugin, proc_plugin), plugins=(plugins or {}) - | {"cb": cb_plugin} + | {cb_plugin.name: cb_plugin} | ({"proc": proc_plugin} if proc_plugin else {}), config_sigs=config_sigs, name=name, diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 08fb680375..6463b5fbb3 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -169,6 +169,17 @@ class NDPluginBaseIO(NDArrayBaseIO): queue_size: A[SignalRW[int], PvSuffix.rbv("QueueSize")] +async def plugin_is_enabled(plugin: NDPluginBaseIO) -> bool: + """Whether a plugin is switched on, so is receiving NDArrays. + + The nearest thing to "is this plugin in the chain": a plugin with callbacks + disabled certainly is not, though an enabled one may still have its + `nd_array_port` pointed somewhere else. Parsing the whole plugin graph was + rejected in ADR 0025 as too large. + """ + return await plugin.enable_callbacks.get_value() is EnableDisable.ENABLE + + class NDROIIO(StandardReadable, NDPluginBaseIO): """Plugin for taking a region of an NDArray. diff --git a/src/ophyd_async/epics/adcore/_trigger_logic.py b/src/ophyd_async/epics/adcore/_trigger_logic.py index 6fccdc3efa..777ac243c3 100644 --- a/src/ophyd_async/epics/adcore/_trigger_logic.py +++ b/src/ophyd_async/epics/adcore/_trigger_logic.py @@ -2,13 +2,11 @@ from dataclasses import dataclass from ophyd_async.core import ( - DetectorTrigger, - EnableDisable, - SignalR, - TriggerInfo, + DetectorTriggerLogic as _DetectorTriggerLogic, ) from ophyd_async.core import ( - DetectorTriggerLogic as _DetectorTriggerLogic, + EnableDisable, + TriggerInfo, ) from ._io import ( @@ -83,30 +81,47 @@ async def prepare_filter( ) -async def trigger_info_from_driver( - num_images_signal: SignalR[int], - process_plugin: NDProcessIO | None = None, -) -> TriggerInfo: - """Default TriggerInfo for AD detectors, reading num_images from the driver. +async def exposures_per_collection_from_filter( + process_plugin: NDProcessIO | None, +) -> int: + """Read the exposures per collection from the process plugin's filter. - :param num_images_signal: SignalR for the number of images to acquire - :param process_plugin: Optional NDProcessIO plugin for exposures per collection + :param process_plugin: The NDProcessIO plugin instance, if any + :return: `num_filter` if the filter is enabled, otherwise 1 + :raises ValueError: If the filter is enabled with `num_filter` below 1 """ - num_images = await num_images_signal.get_value() - exposures_per_collection = 1 + if process_plugin is None or not await process_plugin.enable_filter.get_value(): + return 1 + num_filter = await process_plugin.num_filter.get_value() + if num_filter < 1: + raise ValueError( + f"{process_plugin.name}: filter enabled with " + f"num_filter={num_filter}, set it to 1 or more" + ) + return num_filter - if process_plugin is not None and await process_plugin.enable_filter.get_value(): - exposures_per_collection = await process_plugin.num_filter.get_value() - if exposures_per_collection < 1: - raise ValueError( - f"{process_plugin.name}: filter enabled with ", - f"num_filter={exposures_per_collection}, set it to 1 or more", - ) +async def trigger_info_from_driver( + driver: ADBaseIO, process_plugin: NDProcessIO | None = None +) -> TriggerInfo: + """Read the default TriggerInfo for AD detectors back from the driver. + + :param driver: The driver to read num_images, acquire_time and acquire_period from + :param process_plugin: The NDProcessIO plugin filtering exposures, if any + """ + num_images, livetime, period = await asyncio.gather( + driver.num_images.get_value(), + driver.acquire_time.get_value(), + driver.acquire_period.get_value(), + ) + exposures_per_collection = await exposures_per_collection_from_filter( + process_plugin + ) return TriggerInfo( collections_per_event=max(1, num_images // exposures_per_collection), exposures_per_collection=exposures_per_collection, - trigger=DetectorTrigger.INTERNAL, + livetime=livetime, + deadtime=max(0.0, period - livetime), ) @@ -158,6 +173,17 @@ async def default_trigger_info(self) -> TriggerInfo: # Read post_count (not driver.num_images) because the CB plugin's # post_count is what governs how many frames are buffered per trigger # in continuous-acquisition mode, not the driver's num_images. - return await trigger_info_from_driver( - self.cb_plugin.post_count, self.process_plugin + post_count, livetime, period = await asyncio.gather( + self.cb_plugin.post_count.get_value(), + self.driver.acquire_time.get_value(), + self.driver.acquire_period.get_value(), + ) + exposures_per_collection = await exposures_per_collection_from_filter( + self.process_plugin + ) + return TriggerInfo( + collections_per_event=max(1, post_count // exposures_per_collection), + exposures_per_collection=exposures_per_collection, + livetime=livetime, + deadtime=max(0.0, period - livetime), ) diff --git a/src/ophyd_async/epics/adkinetix.py b/src/ophyd_async/epics/adkinetix.py index ca855bb581..c6eee8b663 100644 --- a/src/ophyd_async/epics/adkinetix.py +++ b/src/ophyd_async/epics/adkinetix.py @@ -19,7 +19,7 @@ from .adcore import ( ADAcquireLogic, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, prepare_exposures, @@ -87,9 +87,7 @@ async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, num_filter=exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class KinetixDetector(AreaDetector[KinetixDriverIO]): @@ -107,7 +105,7 @@ class KinetixDetector(AreaDetector[KinetixDriverIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_suffix: str = "cam1:", proc_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, diff --git a/src/ophyd_async/epics/admerlin.py b/src/ophyd_async/epics/admerlin.py index 0ace7baa64..c0ddb93b59 100644 --- a/src/ophyd_async/epics/admerlin.py +++ b/src/ophyd_async/epics/admerlin.py @@ -22,7 +22,7 @@ ADAcquireLogic, ADBaseDataType, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, NDProcessIO, @@ -112,9 +112,7 @@ async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class MerlinDetector(AreaDetector[MerlinDriverIO]): @@ -132,7 +130,7 @@ class MerlinDetector(AreaDetector[MerlinDriverIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_suffix="cam1:", proc_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, diff --git a/src/ophyd_async/epics/adpilatus.py b/src/ophyd_async/epics/adpilatus.py index a5209a0276..cc3bf189fe 100644 --- a/src/ophyd_async/epics/adpilatus.py +++ b/src/ophyd_async/epics/adpilatus.py @@ -21,7 +21,7 @@ from .adcore import ( ADAcquireLogic, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, NDProcessIO, @@ -40,8 +40,6 @@ "PilatusReadoutTime", ] -_MAX_NUM_IMAGE = 999_999 - class PilatusTriggerMode(StrictEnum): """Trigger modes for ADPilatus detector.""" @@ -114,23 +112,21 @@ def get_deadtime(self, config_values: SignalDict) -> float: async def prepare_internal(self, num: int, livetime: float, deadtime: float): await self.driver.trigger_mode.set(PilatusTriggerMode.INTERNAL) - await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE, livetime, deadtime) + await prepare_exposures(self.driver, num, livetime, deadtime) async def prepare_edge(self, num: int, livetime: float): await self.driver.trigger_mode.set(self.edge_trigger_mode) - await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE, livetime) + await prepare_exposures(self.driver, num, livetime) async def prepare_level(self, num: int): await self.driver.trigger_mode.set(PilatusTriggerMode.EXT_ENABLE) - await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE) + await prepare_exposures(self.driver, num) async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class PilatusDetector(AreaDetector[PilatusDriverIO]): @@ -153,7 +149,7 @@ class PilatusDetector(AreaDetector[PilatusDriverIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, readout_time: PilatusReadoutTime = PilatusReadoutTime.PILATUS3, edge_trigger_mode: PilatusTriggerMode = PilatusTriggerMode.EXT_TRIGGER, driver_suffix="cam1:", diff --git a/src/ophyd_async/epics/adsimdetector.py b/src/ophyd_async/epics/adsimdetector.py index c73b6fd2ce..84965e5fd3 100644 --- a/src/ophyd_async/epics/adsimdetector.py +++ b/src/ophyd_async/epics/adsimdetector.py @@ -14,7 +14,7 @@ from .adcore import ( ADAcquireLogic, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, NDProcessIO, @@ -43,9 +43,7 @@ async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class SimDetector(AreaDetector[ADBaseIO]): @@ -63,7 +61,7 @@ class SimDetector(AreaDetector[ADBaseIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_suffix="cam1:", proc_suffix: str | None = None, plugins: dict[str, NDPluginBaseIO] | None = None, diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index 9c3bd14472..8f37a2982c 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -22,7 +22,7 @@ from .adcore import ( ADAcquireLogic, ADBaseIO, - ADWriterFactory, + ADDataFactory, AreaDetector, NDPluginBaseIO, NDProcessIO, @@ -133,9 +133,7 @@ async def prepare_exposures_per_collection(self, exposures_per_collection: int): await prepare_filter(self.process_plugin, exposures_per_collection) async def default_trigger_info(self): - return await trigger_info_from_driver( - self.driver.num_images, self.process_plugin - ) + return await trigger_info_from_driver(self.driver, self.process_plugin) class VimbaDetector(AreaDetector[VimbaDriverIO]): @@ -156,7 +154,7 @@ class VimbaDetector(AreaDetector[VimbaDriverIO]): def __init__( self, prefix: str, - *writer_factories: ADWriterFactory, + *writer_factories: ADDataFactory, driver_suffix="cam1:", override_deadtime: float | None = None, proc_suffix: str | None = None, diff --git a/src/ophyd_async/fastcs/eiger/_acquire_logic.py b/src/ophyd_async/fastcs/eiger/_acquire_logic.py index b3f84af3c9..f5a4c6c66d 100644 --- a/src/ophyd_async/fastcs/eiger/_acquire_logic.py +++ b/src/ophyd_async/fastcs/eiger/_acquire_logic.py @@ -1,3 +1,5 @@ +from collections.abc import AsyncGenerator + from ophyd_async.core import ( DEFAULT_TIMEOUT, DetectorAcquireLogic, @@ -15,10 +17,9 @@ def __init__( self.detector = detector self.arm_when_ready = arm_when_ready - async def start_acquiring(self): + async def acquire(self) -> AsyncGenerator[None, None]: await self.arm_when_ready.trigger() - - async def wait_for_idle(self): + yield await wait_for_value(self.detector.state, "idle", timeout=DEFAULT_TIMEOUT) async def ensure_stopped(self): diff --git a/src/ophyd_async/fastcs/eiger/_detector.py b/src/ophyd_async/fastcs/eiger/_detector.py index f993114c40..f7718f2d78 100644 --- a/src/ophyd_async/fastcs/eiger/_detector.py +++ b/src/ophyd_async/fastcs/eiger/_detector.py @@ -1,4 +1,7 @@ +from functools import cached_property + from ophyd_async.core import ( + DetectorLogic, PathProvider, SignalR, StandardDetector, @@ -34,17 +37,22 @@ def __init__( # Need to do this first so the type hints are filled in connector = fastcs_connector(prefix, self) - self.add_detector_logics( - EigerTriggerLogic(self.detector), - EigerAcquireLogic(self.detector, self.arm_when_ready), - odin.OdinDataLogic( - path_provider=path_provider, - odin=self.od, - detector_bit_depth=self.detector.bit_depth_image, - pixel_mask=self.pixel_mask, - ), - ) - + self._path_provider = path_provider self.set_readable_format(self.pixel_mask, StandardReadableFormat.CONFIG_SIGNAL) super().__init__(name=name, connector=connector) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=EigerTriggerLogic(self.detector), + acquire=EigerAcquireLogic(self.detector, self.arm_when_ready), + data=[ + odin.OdinDataLogic( + path_provider=self._path_provider, + odin=self.od, + detector_bit_depth=self.detector.bit_depth_image, + pixel_mask=self.pixel_mask, + ) + ], + ) diff --git a/src/ophyd_async/fastcs/jungfrau/_acquire_logic.py b/src/ophyd_async/fastcs/jungfrau/_acquire_logic.py index 28ff8cd031..f34767fda4 100644 --- a/src/ophyd_async/fastcs/jungfrau/_acquire_logic.py +++ b/src/ophyd_async/fastcs/jungfrau/_acquire_logic.py @@ -1,3 +1,5 @@ +from collections.abc import AsyncGenerator + from ophyd_async.core import DEFAULT_TIMEOUT, DetectorAcquireLogic, wait_for_value from ._io import DetectorStatus, JungfrauDriverIO, PedestalMode @@ -7,10 +9,9 @@ class JungfrauAcquireLogic(DetectorAcquireLogic): def __init__(self, detector: JungfrauDriverIO) -> None: self.detector = detector - async def start_acquiring(self): + async def acquire(self) -> AsyncGenerator[None, None]: await self.detector.acquisition_start.trigger() - - async def wait_for_idle(self): + yield await wait_for_value( self.detector.detector_status, DetectorStatus.IDLE, timeout=DEFAULT_TIMEOUT ) diff --git a/src/ophyd_async/fastcs/jungfrau/_detector.py b/src/ophyd_async/fastcs/jungfrau/_detector.py index 0adb10db3c..d05fbbd58e 100644 --- a/src/ophyd_async/fastcs/jungfrau/_detector.py +++ b/src/ophyd_async/fastcs/jungfrau/_detector.py @@ -1,4 +1,11 @@ -from ophyd_async.core import PathProvider, StandardDetector, soft_signal_rw +from functools import cached_property + +from ophyd_async.core import ( + DetectorLogic, + PathProvider, + StandardDetector, + soft_signal_rw, +) from ophyd_async.fastcs import odin from ophyd_async.fastcs.core import fastcs_connector @@ -29,13 +36,19 @@ def __init__( self.acquisition_type = soft_signal_rw( AcquisitionType, AcquisitionType.STANDARD ) - self.add_detector_logics( - JungfrauTriggerLogic(self.detector, self.acquisition_type), - JungfrauAcquireLogic(self.detector), - odin.OdinDataLogic( - path_provider=path_provider, - odin=self.odin, - detector_bit_depth=self.detector.bit_depth, - ), - ) + self._path_provider = path_provider super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=JungfrauTriggerLogic(self.detector, self.acquisition_type), + acquire=JungfrauAcquireLogic(self.detector), + data=[ + odin.OdinDataLogic( + path_provider=self._path_provider, + odin=self.odin, + detector_bit_depth=self.detector.bit_depth, + ) + ], + ) diff --git a/src/ophyd_async/fastcs/odin/_data_logic.py b/src/ophyd_async/fastcs/odin/_data_logic.py index 282e707f6f..233facd16d 100644 --- a/src/ophyd_async/fastcs/odin/_data_logic.py +++ b/src/ophyd_async/fastcs/odin/_data_logic.py @@ -5,20 +5,19 @@ from ophyd_async.core import ( DEFAULT_TIMEOUT, - DetectorDataLogic, PathProvider, SignalR, SignalRW, - StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + UnboundedDataLogic, wait_for_value, ) from ._io import OdinIO -class OdinDataLogic(DetectorDataLogic): +class OdinDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): def __init__( self, path_provider: PathProvider, @@ -30,10 +29,19 @@ def __init__( self.odin = odin self.detector_bit_depth = detector_bit_depth self.pixel_mask = pixel_mask + self.collections_written = self.odin.fp.frames_written - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> StreamResourceDataProvider: + # Odin sizes its own chunks and writes for as long as it is told to, so + # the periods are not used here yet. # Work out where to write - path_info = self.path_provider(datakey_name) + path_info = self.path_provider(name) # Get the current bit depth datatype = f"uint{await self.detector_bit_depth.get_value()}" # Setup the HDF writer @@ -62,12 +70,13 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: # Must ensure frames_written reset # See issue: https://github.com/DiamondLightSource/fastcs-odin/issues/107 await wait_for_value(self.odin.fp.frames_written, 0, timeout=DEFAULT_TIMEOUT) - # Return a provider that reflects what we have made + # The frame shape is only readable from the file processor once it is + # writing data_shape = await asyncio.gather( self.odin.fp.data_dims_0.get_value(), self.odin.fp.data_dims_1.get_value() ) resource = StreamResourceInfo( - data_key=datakey_name, + data_key=name, shape=data_shape, chunk_shape=(1, *data_shape), dtype_numpy=np.dtype(datatype).str, @@ -78,15 +87,15 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: uri=f"{path_info.directory_uri}{filename}_000001.h5", resources=[resource], mimetype="application/x-hdf5", - collections_written_signal=self.odin.fp.frames_written, + collections_per_event=collections_per_event, ) - async def stop(self) -> None: + async def ensure_stopped(self) -> None: await asyncio.gather( self.odin.fp.stop_writing.trigger(), self.odin.mw.stop_.trigger(), ) - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: + def get_hinted_fields(self, name: str) -> Sequence[str]: # The main dataset is always hinted - return [datakey_name] + return [name] diff --git a/src/ophyd_async/fastcs/panda/_acquire_logic.py b/src/ophyd_async/fastcs/panda/_acquire_logic.py index b594b2c1a7..726888c56c 100644 --- a/src/ophyd_async/fastcs/panda/_acquire_logic.py +++ b/src/ophyd_async/fastcs/panda/_acquire_logic.py @@ -1,3 +1,5 @@ +from collections.abc import AsyncGenerator + from ophyd_async.core import ( DetectorAcquireLogic, set_and_wait_for_other_value, @@ -12,12 +14,10 @@ class PandaAcquireLogic(DetectorAcquireLogic): def __init__(self, pcap: PcapBlock) -> None: self.pcap = pcap - async def start_acquiring(self): + async def acquire(self) -> AsyncGenerator[None, None]: await set_and_wait_for_other_value(self.pcap.arm, True, self.pcap.active, True) - - async def wait_for_idle(self): - # TODO: https://github.com/PandABlocks/PandABlocks-FPGA/issues/262 - pass + # TODO: wait for idle, https://github.com/PandABlocks/PandABlocks-FPGA/issues/262 + yield async def ensure_stopped(self): await set_and_wait_for_other_value( diff --git a/src/ophyd_async/fastcs/panda/_data_logic.py b/src/ophyd_async/fastcs/panda/_data_logic.py index ef8ec675e2..89602de2f8 100644 --- a/src/ophyd_async/fastcs/panda/_data_logic.py +++ b/src/ophyd_async/fastcs/panda/_data_logic.py @@ -4,11 +4,12 @@ import numpy as np from ophyd_async.core import ( - DetectorDataLogic, + DEFAULT_TIMEOUT, PathProvider, - StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + UnboundedDataLogic, + wait_for_value, ) from ._block import DataBlock, PandaCaptureMode @@ -16,7 +17,7 @@ logger = logging.getLogger("ophyd_async") -class PandaHDFDataLogic(DetectorDataLogic): +class PandaHDFDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): def __init__( self, path_provider: PathProvider, @@ -24,16 +25,26 @@ def __init__( ): self.path_provider = path_provider self.data_block = data_block + self.collections_written = self.data_block.num_captured - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - # Work out where to write - path_info = self.path_provider(datakey_name) + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> StreamResourceDataProvider: + # TODO: size the chunk from `collection_period` once the IOC exposes the + # chunk-size signal (see the chunk_shape TODO below). + path_info = self.path_provider(name) # Set create dir depth first to guarantee that callback when setting # directory path has correct value await self.data_block.create_directory.set(path_info.create_dir_depth) - # Setup the HDF writer + # Setup the HDF writer. The PandA flushes on a timer rather than by + # chunk size, so it takes the flush period directly; 0 means it flushes + # only when its buffer fills. await asyncio.gather( - self.data_block.flush_period.set(0), + self.data_block.flush_period.set(flush_period), self.data_block.hdf_directory.set(str(path_info.directory_path)), self.data_block.hdf_file_name.set( f"{path_info.filename}.h5", @@ -48,12 +59,14 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: ) # Start capturing await self.data_block.capture.set(True) - # Load data from the datasets PV on the panda, update internal - # representation of datasets that the panda will write. + # Capturing resets NumCaptured, which the detector reads as its baseline + await wait_for_value(self.data_block.num_captured, 0, timeout=DEFAULT_TIMEOUT) + # The datasets the PandA writes: PandABlocks-ioc only fills the table + # once capturing has started capture_table = await self.data_block.datasets.get_value() if len(capture_table) == 0: logger.warning( - f"PandA {datakey_name} DATASETS table is empty! " + f"PandA {name} DATASETS table is empty! " "No stream resource docs will be generated. " "Make sure captured positions have their corresponding " "*:DATASET PV set to a scientifically relevant name." @@ -74,8 +87,8 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=resources, mimetype="application/x-hdf5", - collections_written_signal=self.data_block.num_captured, + collections_per_event=collections_per_event, ) - async def stop(self) -> None: + async def ensure_stopped(self) -> None: await self.data_block.capture.set(False) diff --git a/src/ophyd_async/fastcs/panda/_detector.py b/src/ophyd_async/fastcs/panda/_detector.py index c365d56fa8..f9e90207ea 100644 --- a/src/ophyd_async/fastcs/panda/_detector.py +++ b/src/ophyd_async/fastcs/panda/_detector.py @@ -1,6 +1,8 @@ from __future__ import annotations -from ophyd_async.core import PathProvider, StandardDetector +from functools import cached_property + +from ophyd_async.core import DetectorLogic, PathProvider, StandardDetector from ophyd_async.fastcs.core import fastcs_connector from ._acquire_logic import PandaAcquireLogic @@ -23,9 +25,13 @@ def __init__( error_hint = f"Is PandABlocks-ioc at least version {MINIMUM_PANDA_IOC}?" # This has to be first so we make self.pcap connector = fastcs_connector(prefix, self, error_hint) - self.add_detector_logics( - PandaTriggerLogic(self.pcap), - PandaAcquireLogic(self.pcap), - PandaHDFDataLogic(path_provider, self.data), - ) + self._path_provider = path_provider super().__init__(name=name, connector=connector) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=PandaTriggerLogic(self.pcap), + acquire=PandaAcquireLogic(self.pcap), + data=[PandaHDFDataLogic(self._path_provider, self.data)], + ) diff --git a/src/ophyd_async/sim/_blob_acquire_logic.py b/src/ophyd_async/sim/_blob_acquire_logic.py index f880d1a3d7..02a84ad432 100644 --- a/src/ophyd_async/sim/_blob_acquire_logic.py +++ b/src/ophyd_async/sim/_blob_acquire_logic.py @@ -1,6 +1,6 @@ -import asyncio +from collections.abc import AsyncGenerator -from ophyd_async.core import DetectorAcquireLogic, TriggerInfo +from ophyd_async.core import AsyncStatus, DetectorAcquireLogic, TriggerInfo from ._pattern_generator import PatternGenerator @@ -9,23 +9,13 @@ class BlobAcquireLogic(DetectorAcquireLogic): def __init__(self, pattern_generator: PatternGenerator): self.pattern_generator = pattern_generator self.trigger_info: TriggerInfo | None = None - self.task: asyncio.Task | None = None - async def start_acquiring(self): - # Start a background process off writing the images to file - self.task = asyncio.create_task(self.pattern_generator.write_images_to_file()) - - async def wait_for_idle(self): - # Wait for the background task to complete - if self.task: - await self.task + async def acquire(self) -> AsyncGenerator[None, None]: + # Write the images to file in the background, until done or closed + async with AsyncStatus(self.pattern_generator.write_images_to_file()) as status: + yield + await status async def ensure_stopped(self): - # Stop the background task and wait for it to finish - if self.task: - task, self.task = self.task, None - task.cancel() - # An outside cancel while we wait propagates, it is not swallowed - await asyncio.wait([task]) - if not task.cancelled() and (error := task.exception()): - raise error + # acquire() owns the writing task, so there is nothing else to stop + pass diff --git a/src/ophyd_async/sim/_blob_data_logic.py b/src/ophyd_async/sim/_blob_data_logic.py index 01073f0f62..6b39a9178f 100644 --- a/src/ophyd_async/sim/_blob_data_logic.py +++ b/src/ophyd_async/sim/_blob_data_logic.py @@ -3,11 +3,10 @@ import numpy as np from ophyd_async.core import ( - DetectorDataLogic, PathProvider, - StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, + UnboundedDataLogic, ) from ._pattern_generator import DATA_PATH, SUM_PATH, PatternGenerator @@ -16,7 +15,7 @@ HEIGHT = 240 -class BlobDataLogic(DetectorDataLogic): +class BlobDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): def __init__( self, path_provider: PathProvider, @@ -24,16 +23,23 @@ def __init__( ): self.path_provider = path_provider self.pattern_generator = pattern_generator + self.collections_written = self.pattern_generator.images_written - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - # Work out where to write - path_info = self.path_provider(datakey_name) - # Open the file - write_path = path_info.directory_path / f"{path_info.filename}.h5" - self.pattern_generator.open_file(write_path, WIDTH, HEIGHT) - # Return a provider that reflects what we have made + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> StreamResourceDataProvider: + # The sim blob writer uses a fixed chunk shape and writes for as long as + # it is told to, so neither period is needed + path_info = self.path_provider(name) + self.pattern_generator.open_file( + path_info.directory_path / f"{path_info.filename}.h5", WIDTH, HEIGHT + ) data_resource = StreamResourceInfo( - data_key=datakey_name, + data_key=name, shape=(HEIGHT, WIDTH), # NDAttributes appear to always be configured with # this chunk size @@ -42,7 +48,7 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: parameters={"dataset": DATA_PATH}, ) sum_resource = StreamResourceInfo( - data_key=f"{datakey_name}-sum", + data_key=f"{name}-sum", shape=(), # NDAttributes appear to always be configured with # this chunk size @@ -54,12 +60,12 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[data_resource, sum_resource], mimetype="application/x-hdf5", - collections_written_signal=self.pattern_generator.images_written, + collections_per_event=collections_per_event, ) - async def stop(self) -> None: + async def ensure_stopped(self) -> None: self.pattern_generator.close_file() - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: + def get_hinted_fields(self, name: str) -> Sequence[str]: # The main dataset is always hinted - return [datakey_name] + return [name] diff --git a/src/ophyd_async/sim/_blob_detector.py b/src/ophyd_async/sim/_blob_detector.py index 6760e088a1..932b6ba5e0 100644 --- a/src/ophyd_async/sim/_blob_detector.py +++ b/src/ophyd_async/sim/_blob_detector.py @@ -1,6 +1,12 @@ from collections.abc import Sequence +from functools import cached_property -from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import ( + DetectorLogic, + PathProvider, + SignalR, + StandardDetector, +) from ophyd_async.core import StandardReadableFormat as Format from ._blob_acquire_logic import BlobAcquireLogic @@ -20,13 +26,20 @@ def __init__( name: str = "", ) -> None: self.pattern_generator = pattern_generator or PatternGenerator() - self.add_detector_logics( - BlobTriggerLogic(pattern_generator=self.pattern_generator), - BlobAcquireLogic(pattern_generator=self.pattern_generator), - BlobDataLogic( - path_provider=path_provider, pattern_generator=self.pattern_generator - ), - ) for signal in config_sigs: self.set_readable_format(signal, Format.CONFIG_SIGNAL) + self._path_provider = path_provider super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + trigger=BlobTriggerLogic(pattern_generator=self.pattern_generator), + acquire=BlobAcquireLogic(pattern_generator=self.pattern_generator), + data=[ + BlobDataLogic( + path_provider=self._path_provider, + pattern_generator=self.pattern_generator, + ) + ], + ) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index b9a3c3f426..4035bd0f90 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -34,6 +34,7 @@ import bluesky.plan_stubs as bps import bluesky.plans as bp +import bluesky.preprocessors as bpp import pytest from aioca import purge_channel_caches from bluesky.run_engine import RunEngine @@ -105,7 +106,7 @@ def adsim(RE: RunEngine, shared_tmp_path: Path) -> AreaDetector: process_plugin = NDProcessIO(f"{prefix}-DI-CAM-01:PROC:") adsim = SimDetector( f"{prefix}-DI-CAM-01:", - adcore.ADWriterFactory.hdf(provider, writer_suffix="HDF5:"), + adcore.ADDataFactory.hdf(provider, writer_suffix="HDF5:"), driver_suffix="DET:", plugins={"proc1": process_plugin}, ) @@ -145,15 +146,18 @@ def apply_baseline_settings(adsim: SimDetector) -> MsgGenerator[None]: def test_prepare_is_idempotent_and_sets_exposure_time( RE: RunEngine, adsim: SimDetector, bl01t_di_cam_01: None ) -> None: - def prepare_then_count() -> MsgGenerator[None]: + @bpp.stage_decorator([adsim]) + @bpp.run_decorator() + def prepare_then_trigger() -> MsgGenerator[None]: + # prepare() must come after stage() yield from bps.prepare( adsim, TriggerInfo(livetime=0.2), wait=True, ) - yield from bp.count([adsim]) + yield from bps.trigger_and_read([adsim]) - RE(prepare_then_count()) + RE(prepare_then_trigger()) actual_exposure_time: float = RE(bps.rd(adsim.driver.acquire_time)).plan_result assert actual_exposure_time == 0.2 diff --git a/tests/unit_tests/core/test_cancel_swallow.py b/tests/unit_tests/core/test_cancel_swallow.py index 36879efd8e..358e18e5ff 100644 --- a/tests/unit_tests/core/test_cancel_swallow.py +++ b/tests/unit_tests/core/test_cancel_swallow.py @@ -31,13 +31,13 @@ async def _status_aexit(cancelling: asyncio.Event, _: pytest.MonkeyPatch): await asyncio.sleep(0) # let the status task start -async def _blob_ensure_stopped(cancelling: asyncio.Event, _: pytest.MonkeyPatch): +async def _blob_acquire_close(cancelling: asyncio.Event, _: pytest.MonkeyPatch): pattern_generator = Mock() pattern_generator.write_images_to_file = lambda: _slow_to_cancel(cancelling) - logic = BlobAcquireLogic(pattern_generator) - await logic.start_acquiring() + acquisition = BlobAcquireLogic(pattern_generator).acquire() + await anext(acquisition) await asyncio.sleep(0) # let the task start - await logic.ensure_stopped() + await acquisition.aclose() async def _set_and_wait(cancelling: asyncio.Event, monkeypatch: pytest.MonkeyPatch): @@ -53,9 +53,7 @@ async def wait_for_value(self, signal, timeout): await signal_module.set_and_wait_for_value(sig, 1, timeout=0.1) -@pytest.mark.parametrize( - "cleanup", [_status_aexit, _blob_ensure_stopped, _set_and_wait] -) +@pytest.mark.parametrize("cleanup", [_status_aexit, _blob_acquire_close, _set_and_wait]) async def test_cancelling_caller_during_cleanup_is_not_swallowed( cleanup: Callable[[asyncio.Event, pytest.MonkeyPatch], Coroutine[Any, Any, None]], monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit_tests/core/test_data_providers.py b/tests/unit_tests/core/test_data_providers.py new file mode 100644 index 0000000000..bc3b8a0609 --- /dev/null +++ b/tests/unit_tests/core/test_data_providers.py @@ -0,0 +1,63 @@ +import numpy as np +import pytest + +from ophyd_async.core import ( + Array1D, + EventPageDataProvider, + soft_signal_r_and_setter, +) + + +def make_provider(collections_per_event: int) -> EventPageDataProvider: + """A 5-collection buffer, filled, with one timestamp per collection.""" + data, _ = soft_signal_r_and_setter( + Array1D[np.float64], np.array([10.0, 11.0, 12.0, 13.0, 14.0]) + ) + timestamps, _ = soft_signal_r_and_setter( + Array1D[np.float64], np.array([100.0, 101.0, 102.0, 103.0, 104.0]) + ) + return EventPageDataProvider({"det-stats": data}, timestamps, collections_per_event) + + +@pytest.mark.parametrize( + "collections_per_event,expected_data,expected_times", + [ + # step scan: one event holds the whole 5-collection buffer, timed by its + # last collection + (5, [[10.0, 11.0, 12.0, 13.0, 14.0]], [104.0]), + # fly scan: five events of one collection each, timed one by one + ( + 1, + [[10.0], [11.0], [12.0], [13.0], [14.0]], + [100.0, 101.0, 102.0, 103.0, 104.0], + ), + ], +) +async def test_slices_the_arrays_into_events( + collections_per_event: int, + expected_data: list[list[float]], + expected_times: list[float], +): + provider = make_provider(collections_per_event) + page = await provider.make_page(5 // collections_per_event) + assert page is not None + np.testing.assert_array_equal(page["data"]["det-stats"], expected_data) + assert page["time"] == expected_times + + +async def test_datakeys_take_dtype_from_the_signal(): + datakeys = await make_provider(5).make_datakeys() + assert list(datakeys) == ["det-stats"] + assert datakeys["det-stats"]["shape"] == [5] + assert datakeys["det-stats"]["dtype"] == "array" + assert datakeys["det-stats"]["dtype_numpy"] == " None: + self.written = soft_signal_rw(int) + self.data: list[int] = [] + self._next = 0 + + async def produce(self, collections: int) -> None: + for _ in range(collections): + self.data.append(self._next) + self._next += 1 + await self.written.set(len(self.data)) + + async def rearm(self) -> None: + self.data.clear() + await self.written.set(0) + + +class InternalTrigger(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): ... + + async def default_trigger_info(self) -> TriggerInfo: + return TriggerInfo() + + +class ExternalTrigger(InternalTrigger): + async def prepare_edge(self, num: int, livetime: float): ... + + +class FillingAcquire(DetectorAcquireLogic): + """Every acquire() produces `per_start` collections in each source.""" + + def __init__(self, *sources: Source, per_start: int = 1) -> None: + self.sources = sources + self.per_start = per_start + self.idle_waits = 0 + + async def acquire(self): + for source in self.sources: + await source.produce(self.per_start) + yield + self.idle_waits += 1 + + async def ensure_stopped(self): ... + + +class BoomError(Exception): + """What a fake raises when a test tells it to.""" + + +class FakeStream(StreamResourceDataProvider): + def __init__(self, *args: Any, owner: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.owner = owner + + async def make_stream_docs(self, index): + if self.owner.fail: + raise BoomError("make_stream_docs") + async for doc in super().make_stream_docs(index): + yield doc + + +class FakePageable(PageableDataProvider): + """Event pages straight from a source, one value per collection. + + Every datakey holds the same values. `make_page` carries on from the last + page. + """ + + def __init__( + self, + datakey_names: Sequence[str], + source: Source, + collections_per_event: int, + shape_tail: tuple[int, ...] = (), + dtype_numpy: str = " None: + super().__init__(collections_per_event) + self.datakey_names = datakey_names + self.source = source + self.owner = owner + self.shape_tail = shape_tail + self.dtype_numpy = dtype_numpy + self.start_new_resources(0) + + def start_new_resources(self, origin: int) -> None: + self.origin = origin + self.published = 0 + + async def make_datakeys(self) -> dict[str, DataKey]: + return { + datakey_name: DataKey( + source="fake", + shape=[self.collections_per_event, *self.shape_tail], + dtype="array", + dtype_numpy=self.dtype_numpy, + ) + for datakey_name in self.datakey_names + } + + def _page(self, first: int, last: int) -> PartialEventPage: + cpe = self.collections_per_event + events = range(first, last) + starts = [self.origin + e * cpe for e in events] + values = [self.source.data[start : start + cpe] for start in starts] + times = [float(e) for e in events] + return { + "data": dict.fromkeys(self.datakey_names, values), + "time": times, + "timestamps": dict.fromkeys(self.datakey_names, times), + } + + async def make_page(self, index: int) -> PartialEventPage | None: + if self.owner and self.owner.fail: + raise BoomError("make_page") + if index <= self.published: + return None + page = self._page(self.published, index) + self.published = index + return page + + +class _FakeData: + """Publishes a `Source` as event pages or stream assets. + + :param bounded: a finite buffer: it needs a finite N (raises for N=0 if it + has datakeys left to make) and is cleared at each start + :param suffix: the suffix of the hinted datakey + :param also: the suffixes of further datakeys holding the same data + """ + + bounded: bool + + def __init__( + self, + source: Source, + *, + publication: str, + suffix: str = "", + also: Sequence[str] = (), + shape_tail: tuple[int, ...] = (), + dtype_numpy: str = " None: + self.source = source + self.publication = publication + self.datakey_suffix = suffix + self.also = also + self.shape_tail = shape_tail + self.dtype_numpy = dtype_numpy + self.tmp_path = tmp_path + self.hints = hints + self.periods: list[float] = [] + self.starts = 0 + self.stops = 0 + self.fail = False + self.fail_start = False + self.sit_out = False + #: What was done to this logic, in order + self.log: list[str] = [] + + @property + def collections_written(self) -> SignalR[int]: + return self.source.written + + async def _start( + self, + name: str, + existing_datakeys: Sequence[str], + num_collections: int, + collections_per_event: int, + collection_period: float, + ) -> Any: + self.periods.append(collection_period) + datakey_names = [ + datakey_name + for datakey_name in ( + name + suffix for suffix in (self.datakey_suffix, *self.also) + ) + if datakey_name not in existing_datakeys + ] + if self.sit_out or not datakey_names: + return None + self.starts += 1 + self.log.append("start") + if self.fail_start: + raise BoomError("start") + if self.bounded: + # A finite buffer is armed empty + await self.source.rearm() + if self.publication == "stream": + return FakeStream( + uri=f"file://localhost/{self.tmp_path.as_posix().lstrip('/')}/f.h5", + resources=[ + StreamResourceInfo( + data_key=datakey_name, + shape=self.shape_tail, + chunk_shape=(1, *self.shape_tail), + dtype_numpy=self.dtype_numpy, + parameters={"dataset": "/data"}, + ) + for datakey_name in datakey_names + ], + mimetype="application/x-hdf5", + collections_per_event=collections_per_event, + owner=self, + ) + return FakePageable( + datakey_names, + self.source, + collections_per_event, + shape_tail=self.shape_tail, + dtype_numpy=self.dtype_numpy, + owner=self, + ) + + async def ensure_stopped(self) -> None: + self.stops += 1 + self.log.append("stop") + + def get_hinted_fields(self, name: str) -> Sequence[str]: + datakey_name = name + self.datakey_suffix + return self.hints(datakey_name) if self.hints else [datakey_name] + + +class FakeBoundedData(_FakeData, BoundedDataLogic[Any]): + bounded = True + + async def start_provider( + self, + name: str, + existing_datakeys: Sequence[str], + num_collections: int, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> Any: + return await self._start( + name, + existing_datakeys, + num_collections, + collections_per_event, + collection_period, + ) + + +class FakeUnboundedData(_FakeData, UnboundedDataLogic[Any]): + bounded = False + + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> Any: + # An unbounded source takes data for as long as it is told to + return await self._start(name, (), 1, collections_per_event, collection_period) + + +def FakeData( # noqa: N802 + source: Source, *, bounded: bool, **kwargs: Any +) -> FakeBoundedData | FakeUnboundedData: + return (FakeBoundedData if bounded else FakeUnboundedData)(source, **kwargs) + + +def make_detector( + name: str = "det", + *data: _FakeData, + acquire: DetectorAcquireLogic | None = None, + trigger: DetectorTriggerLogic | None = None, + readable: SignalR | None = None, +) -> StandardDetector: + det = DetectorLogic( + trigger=trigger or InternalTrigger(), acquire=acquire, data=data + ).with_device(name) + if readable is not None: + det.counts = readable + det.set_readable_format(readable, Format.HINTED_SIGNAL) + return det + + +def run_plan(RE: RunEngine, plan) -> dict[str, list[dict]]: + docs: dict[str, list[dict]] = defaultdict(list) + RE(plan, lambda name, doc: docs[name].append(doc)) + return docs + + +def step_scan(det: StandardDetector, points: int, info: TriggerInfo | None = None): + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + if info is not None: + # prepare() comes after stage(), which resets the detector + yield from bps.prepare(det, info, wait=True) + for _ in range(points): + yield from bps.trigger_and_read([det]) + + return plan() + + +def fly_scan(det: StandardDetector, info: TriggerInfo): + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + yield from bps.prepare(det, info, wait=True) + yield from bps.declare_stream(det, name="primary", collect=True) + yield from bps.kickoff(det, wait=True) + yield from bps.collect_while_completing( + flyers=[det], dets=[det], flush_period=0.05 + ) + + return plan() + + +# --------------------------------------------------------------------------- +# Use cases 1, 2, 3, 5, 11 and the four quadrants of D2, through the RunEngine +# --------------------------------------------------------------------------- + + +def test_uc1_readables_only_step_scan_needs_no_prepare(RE: RunEngine): + """Use case 1: no data logic, trigger() prepares implicitly.""" + counts = soft_signal_rw(int, 42, name="det-counts") + acquire = FillingAcquire() + det = make_detector("det", acquire=acquire, readable=counts) + + docs = run_plan(RE, step_scan(det, points=2)) + + assert [e["data"] for e in docs["event"]] == [{"det-counts": 42}] * 2 + assert set(docs["descriptor"][0]["data_keys"]) == {"det-counts"} + assert docs["stream_resource"] == [] + assert acquire.idle_waits == 2 + + +def test_uc2_streamable_step_scan_with_readables(RE: RunEngine, tmp_path): + """Use case 2: one StreamResource, one StreamDatum per point, Events too.""" + source = Source() + counts = soft_signal_rw(int, 7, name="det-counts") + det = make_detector( + "det", + FakeData(source, publication="stream", bounded=False, tmp_path=tmp_path), + acquire=FillingAcquire(source), + readable=counts, + ) + + docs = run_plan(RE, step_scan(det, points=3)) + + (resource,) = docs["stream_resource"] + assert resource["data_key"] == "det" + assert [ + (d["indices"]["start"], d["indices"]["stop"]) for d in docs["stream_datum"] + ] == [ + (0, 1), + (1, 2), + (2, 3), + ] + assert {d["stream_resource"] for d in docs["stream_datum"]} == {resource["uid"]} + assert [e["data"] for e in docs["event"]] == [{"det-counts": 7}] * 3 + keys = docs["descriptor"][0]["data_keys"] + assert keys["det"]["external"] == "STREAM:" + assert "external" not in keys["det-counts"] + + +def test_uc3_streamable_fly_scan_makes_assets_and_no_events(RE: RunEngine, tmp_path): + """Use case 3: stream assets only, covering all N events.""" + source = Source() + det = make_detector( + "det", + FakeData(source, publication="stream", bounded=False, tmp_path=tmp_path), + acquire=FillingAcquire(source, per_start=6), + ) + + docs = run_plan( + RE, fly_scan(det, TriggerInfo(number_of_events=3, collections_per_event=2)) + ) + + assert docs["event"] == [] and docs["event_page"] == [] + (resource,) = docs["stream_resource"] + assert resource["data_key"] == "det" + stops = [d["indices"]["stop"] for d in docs["stream_datum"]] + assert stops[-1] == 3 + assert docs["stop"][0]["num_events"] == {"primary": 3} + + +def test_two_runs_in_one_stage_each_have_their_own_stream_resource( + RE: RunEngine, tmp_path +): + source = Source() + data = FakeData(source, publication="stream", bounded=False, tmp_path=tmp_path) + det = make_detector("det", data, acquire=FillingAcquire(source)) + + @bpp.stage_decorator([det]) + def plan(): + for _ in range(2): + yield from bpp.run_wrapper( + bpp.pchain( + bps.prepare(det, TriggerInfo(), wait=True), + bps.trigger_and_read([det]), + ) + ) + + runs: list[dict[str, list[dict]]] = [] + + def collect(name: str, doc: dict) -> None: + if name == "start": + runs.append(defaultdict(list)) + runs[-1][name].append(doc) + + RE(plan(), collect) + + # The provider carries over, so the same file with the indices carrying on + assert data.log.count("start") == 1 + for run, index in zip(runs, (0, 1), strict=True): + (resource,) = run["stream_resource"] + (datum,) = run["stream_datum"] + assert datum["stream_resource"] == resource["uid"] + assert datum["indices"] == {"start": index, "stop": index + 1} + assert runs[0]["stream_resource"][0]["uid"] != runs[1]["stream_resource"][0]["uid"] + + +def test_uc5_pageable_step_scan_with_several_collections_per_event(RE: RunEngine): + """Use case 5 (and 4's buffer): one array reading of shape [N] per point. + + The finite buffer is cleared between points, so each point holds its own N. + """ + source = Source() + det = make_detector( + "det", + FakeData(source, publication="page", bounded=True), + acquire=FillingAcquire(source, per_start=3), + ) + + docs = run_plan(RE, step_scan(det, 2, TriggerInfo(collections_per_event=3))) + + assert docs["descriptor"][0]["data_keys"]["det"]["shape"] == [3] + assert [e["data"]["det"] for e in docs["event"]] == [[0, 1, 2], [3, 4, 5]] + + +def test_uc4_pageable_fly_scan_collects_event_pages(RE: RunEngine): + """Use case 4: N events come out of collect_pages as event pages.""" + source = Source() + det = make_detector( + "det", + FakeData(source, publication="page", bounded=True), + acquire=FillingAcquire(source, per_start=4), + ) + + docs = run_plan(RE, fly_scan(det, TriggerInfo(number_of_events=4))) + + (page,) = docs["event_page"] + assert page["data"]["det"] == [[0], [1], [2], [3]] + assert docs["stop"][0]["num_events"] == {"primary": 4} + + +@pytest.mark.parametrize( + "publication,bounded", + [ + ("stream", False), + ("stream", True), + ("page", True), + ("page", False), + ], +) +def test_d2_all_four_bound_by_publication_quadrants_step_scan( + RE: RunEngine, tmp_path, publication: str, bounded: bool +): + """Bound and publication are separate axes (D2): every quadrant step-scans. + + An unbounded pageable source (a monitor-fed accumulator) is never re-armed + and its count keeps rising, so the progress baseline of each point must be + read live; a baseline of 0 reads stale data. + """ + source = Source() + det = make_detector( + "det", + FakeData(source, publication=publication, bounded=bounded, tmp_path=tmp_path), + acquire=FillingAcquire(source, per_start=1), + ) + + docs = run_plan(RE, step_scan(det, 3)) + + if publication == "page": + assert [e["data"]["det"] for e in docs["event"]] == [[0], [1], [2]] + else: + # A bounded provider is re-made at each point, so it starts a new stream + # resource each time; an unbounded one carries on in the same one + stops = [1, 1, 1] if bounded else [1, 2, 3] + assert [d["indices"]["stop"] for d in docs["stream_datum"]] == stops + assert len(docs["stream_resource"]) == (3 if bounded else 1) + assert docs["event"][0]["data"] == {} + + +@pytest.mark.parametrize("publication", ["stream", "page"]) +async def test_d2_progress_baseline_is_read_live_for_unbounded_sources( + publication: str, tmp_path +): + """A source that already holds collections (e.g. left running) is not stale.""" + source = Source() + await source.produce(5) + det = make_detector( + "det", + FakeData(source, publication=publication, bounded=False, tmp_path=tmp_path), + acquire=FillingAcquire(source, per_start=1), + ) + await det.stage() + await det.prepare(TriggerInfo()) + await det.trigger() + await det.trigger() + # Events since prepare(), from the 5 already there + assert await det.get_index() == 2 + + +# --------------------------------------------------------------------------- +# Bounded logics are re-armed at each step point after the first (D2, D3) +# --------------------------------------------------------------------------- + + +async def test_d3_bounded_logic_is_restarted_at_each_point_after_the_first(): + source = Source() + data = FakeData(source, publication="page", bounded=True) + det = make_detector("det", data, acquire=FillingAcquire(source)) + await det.stage() + await det.prepare(TriggerInfo()) + + await det.trigger() + assert data.starts == 1 + await det.trigger() + await det.trigger() + assert data.starts == 3 + + +async def test_d3_step_scan_index_counts_since_prepare_across_points(tmp_path): + """get_index counts every point; a bounded provider only holds its own.""" + bounded_source, open_source = Source(), Source() + det = make_detector( + "det", + FakeData( + bounded_source, + publication="stream", + bounded=True, + suffix="-b", + tmp_path=tmp_path, + ), + FakeData(open_source, publication="stream", bounded=False, tmp_path=tmp_path), + acquire=FillingAcquire(bounded_source, open_source), + ) + await det.stage() + await det.prepare(TriggerInfo()) + points = [] + for _ in range(3): + await det.trigger() + index = await det.get_index() + docs = [d async for d in det.collect_asset_docs(index)] # type: ignore[attr-defined] + points.append( + ( + index, + [d["data_key"] for name, d in docs if name == "stream_resource"], + [ + (d["indices"]["start"], d["indices"]["stop"]) + for name, d in docs + if name == "stream_datum" + ], + ) + ) + # The bounded provider is restarted at each point, so has a new resource + # holding one event, while the unbounded one carries on in its first + assert points == [ + (1, ["det-b", "det"], [(0, 1), (0, 1)]), + (2, ["det-b"], [(0, 1), (1, 2)]), + (3, ["det-b"], [(0, 1), (2, 3)]), + ] + + +async def test_d3_unbounded_logic_is_never_restarted_and_fly_never_rearms(): + source = Source() + bounded = FakeData(source, publication="page", bounded=True) + other = Source() + unbounded = FakeData(other, publication="page", bounded=False, suffix="-u") + det = make_detector( + "det", bounded, unbounded, acquire=FillingAcquire(source, other, per_start=2) + ) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=2)) + await det.kickoff() + await det.complete() + assert (bounded.starts, unbounded.starts) == (1, 1) + await det.prepare(TriggerInfo()) + await det.trigger() + await det.trigger() + # The unbounded provider carried over to the second prepare + assert (bounded.starts, unbounded.starts) == (3, 1) + + +async def test_d3_mixed_bounded_and_unbounded_pageable_steps_repeatedly(): + """One baseline per bound group: the bounded one resets, the other counts on.""" + bounded_source, open_source = Source(), Source() + await open_source.produce(3) + bounded = FakeData(bounded_source, publication="page", bounded=True, suffix="-b") + unbounded = FakeData(open_source, publication="page", bounded=False, suffix="-u") + det = make_detector( + "det", + bounded, + unbounded, + acquire=FillingAcquire(bounded_source, open_source), + ) + await det.stage() + await det.prepare(TriggerInfo()) + values = [] + for _ in range(3): + await det.trigger() + reading = await det.read() + values.append((reading["det-b"]["value"], reading["det-u"]["value"])) + assert values == [([0], [3]), ([1], [4]), ([2], [5])] + + +async def test_d3_fly_scan_with_an_unbounded_source_non_zero_at_kickoff(): + bounded_source, open_source = Source(), Source() + await open_source.produce(4) + det = make_detector( + "det", + FakeData(bounded_source, publication="page", bounded=True, suffix="-b"), + FakeData(open_source, publication="page", bounded=False, suffix="-u"), + acquire=FillingAcquire(bounded_source, open_source, per_start=3), + ) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=3)) + await det.kickoff() + await det.complete() + # Events since prepare() + assert await det.get_index() == 3 + (page,) = [p async for p in det.collect_pages()] # type: ignore[attr-defined] + assert page["data"]["det-b"] == [[0], [1], [2]] + assert page["data"]["det-u"] == [[4], [5], [6]] + + +class _BoundedAndUnbounded(BoundedDataLogic[Any], UnboundedDataLogic[Any]): ... + + +@pytest.mark.parametrize("dl", [DetectorDataLogic(), _BoundedAndUnbounded()]) +def test_d3_data_must_be_a_bounded_or_an_unbounded_logic(dl): + with pytest.raises(TypeError, match="must inherit from exactly one of"): + DetectorLogic(data=[dl]) + + +# --------------------------------------------------------------------------- +# Lifecycle state machine (spec §3), every verb from every state +# --------------------------------------------------------------------------- + +STATES = ["U", "S", "P", "T", "K", "C"] + + +class GatedAcquire(FillingAcquire): + """Runs acquire() to its end only once `done` is set, so a fly stays FLYING.""" + + def __init__(self, *sources: Source) -> None: + super().__init__(*sources) + self.done = asyncio.Event() + + async def acquire(self): + async for _ in super().acquire(): + yield + await self.done.wait() + + +async def reach(state: str, tmp_path) -> StandardDetector: + source = Source() + acquire = GatedAcquire(source) + det = make_detector( + "det", + FakeData(source, publication="stream", bounded=False, tmp_path=tmp_path), + acquire=acquire, + ) + # "K" is kicked off with the fly still running + if state != "K": + acquire.done.set() + if state == "U": + return det + await det.stage() + if state == "S": + return det + await det.prepare(TriggerInfo()) + if state == "T": + await det.trigger() + elif state in "KC": + await det.kickoff() + if state == "C": + await det.complete() + return det + + +async def call(det: StandardDetector, verb: str) -> None: + if verb == "prepare": + await det.prepare(TriggerInfo()) + elif verb == "complete": + # Let a running fly finish + cast(GatedAcquire, det.logic.acquire).done.set() + await asyncio.wait_for(det.complete(), 2) + elif verb == "collect_asset_docs": + _ = [doc async for doc in det.collect_asset_docs()] # type: ignore[attr-defined] + else: + await asyncio.wait_for(getattr(det, verb)(), 2) + + +# (state, verb) -> does the verb succeed. From the §3 table. +TABLE: dict[tuple[str, str], bool] = {} +for _verb, _row in { + "prepare": "FTFTFT", + "trigger": "FTTTFF", + "kickoff": "FFTFFF", + "complete": "FFFFTT", + "read": "FFFTFF", + "describe": "FFTTTT", + "describe_collect": "FFTTTT", + "get_index": "FFTTTT", +}.items(): + for _state, _ok in zip(STATES, _row, strict=True): + TABLE[(_state, _verb)] = _ok == "T" +for _state in "PTKC": + # Allowed after a trigger() too: bluesky's read() collects stream assets + TABLE[(_state, "collect_asset_docs")] = _state in "TKC" + + +#: The states in which a verb may read the prepared context +_PREPARED = "FLOWN, FLYING, PREPARED, TRIGGERED, TRIGGERING" + +# What each verb that fails from a state says, as the start of the message +MESSAGES: dict[tuple[str, str], str] = { + ("U", "prepare"): "cannot go to PREPARING from UNSTAGED", + ("P", "prepare"): "cannot go to PREPARING from PREPARED", + ("K", "prepare"): "cannot go to PREPARING from FLYING", + ("U", "trigger"): "cannot go to TRIGGERING from UNSTAGED", + ("K", "trigger"): "cannot go to TRIGGERING from FLYING", + ("C", "trigger"): "cannot go to TRIGGERING from FLOWN", + ("U", "kickoff"): "cannot go to FLYING from UNSTAGED", + ("S", "kickoff"): "cannot go to FLYING from STAGED", + ("T", "kickoff"): "cannot go to FLYING from TRIGGERED", + ("K", "kickoff"): "cannot go to FLYING from FLYING", + ("C", "kickoff"): "cannot go to FLYING from FLOWN", + ("U", "complete"): "cannot complete from UNSTAGED, allowed only after kickoff()", + ("S", "complete"): "cannot complete from STAGED, allowed only after kickoff()", + ("P", "complete"): "cannot complete from PREPARED, allowed only after kickoff()", + ("T", "complete"): "cannot complete from TRIGGERED, allowed only after kickoff()", + ("U", "read"): "det: read not allowed from UNSTAGED, allowed from: TRIGGERED", + ("S", "read"): "det: read not allowed from STAGED, allowed from: TRIGGERED", + ("P", "read"): "det: read not allowed from PREPARED, allowed from: TRIGGERED", + ("K", "read"): "det: read not allowed from FLYING, allowed from: TRIGGERED", + ("C", "read"): "det: read not allowed from FLOWN, allowed from: TRIGGERED", + ( + "U", + "describe", + ): f"det: describe not allowed from UNSTAGED, allowed from: {_PREPARED}", + ( + "S", + "describe", + ): f"det: describe not allowed from STAGED, allowed from: {_PREPARED}", + ( + "U", + "describe_collect", + ): f"det: describe_collect not allowed from UNSTAGED, allowed from: {_PREPARED}", + ( + "S", + "describe_collect", + ): f"det: describe_collect not allowed from STAGED, allowed from: {_PREPARED}", + ( + "U", + "get_index", + ): f"det: get_index not allowed from UNSTAGED, allowed from: {_PREPARED}", + ( + "S", + "get_index", + ): f"det: get_index not allowed from STAGED, allowed from: {_PREPARED}", + ("P", "collect_asset_docs"): ( + "det: collect_asset_docs not allowed from PREPARED, allowed from: " + "FLOWN, FLYING, TRIGGERED" + ), +} + + +def _rows(): + for (state, verb), ok in TABLE.items(): + id = f"{state}-{verb}" + yield pytest.param(state, verb, ok, id=id) + + +@pytest.mark.parametrize("state,verb,ok", _rows()) +async def test_lifecycle_verb_from_state(state, verb, ok, tmp_path): + det = await reach(state, tmp_path) + if ok: + await call(det, verb) + else: + with pytest.raises(RuntimeError, match=re.escape(MESSAGES[(state, verb)])): + await call(det, verb) + # Ends a fly still running in "K" + await det.unstage() + + +async def test_a_prepare_refused_mid_fly_leaves_the_collect_verb_bound(tmp_path): + det = await reach("K", tmp_path) + with pytest.raises(RuntimeError, match="cannot go to PREPARING from FLYING"): + await det.prepare(TriggerInfo()) + cast(GatedAcquire, det.logic.acquire).done.set() + await det.complete() + assert [doc async for doc in det.collect_asset_docs()] # type: ignore[attr-defined] + await det.unstage() + + +@pytest.mark.parametrize("state", STATES) +async def test_restage_returns_to_staged_from_any_state(state, tmp_path): + det = await reach(state, tmp_path) + await det.unstage() + await det.stage() + with pytest.raises(TransitionError, match="describe not allowed from STAGED"): + await det.describe() + await det.prepare(TriggerInfo()) + await det.describe() + + +@pytest.mark.parametrize("state", STATES) +async def test_unstage_from_any_state_is_never_an_error(state, tmp_path): + det = await reach(state, tmp_path) + await det.unstage() + await det.unstage() + + +@pytest.mark.parametrize("state", ["S", "P", "T", "K", "C"]) +async def test_unstage_leads_to_unstaged(state, tmp_path): + det = await reach(state, tmp_path) + await det.unstage() + with pytest.raises(RuntimeError, match="from UNSTAGED"): + await det.prepare(TriggerInfo()) + + +class _Flyer(StandardFlyable[int, None]): + @cached_property + def logic(self) -> FlyableLogic[int, None]: + return _NoopFlyableLogic() + + +class _NoopFlyableLogic(FlyableLogic[int, None]): + async def prepare_to_fly( + self, name: str, value: int, previous_ctx: None + ) -> AsyncIterator[None]: + yield None + + async def fly(self, ctx: None) -> AsyncIterator[KICKED_OFF]: + yield KICKED_OFF + + async def ensure_stopped(self) -> None: ... + + +@pytest.mark.parametrize("verb", ["prepare", "kickoff", "complete"]) +async def test_r10_standard_flyable_verbs_raise_before_stage(verb: str): + flyer = _Flyer(name="flyer") + with pytest.raises(RuntimeError, match="from UNSTAGED"): + await (flyer.prepare(1) if verb == "prepare" else getattr(flyer, verb)()) + await flyer.stage() + await flyer.prepare(1) + await flyer.unstage() + await flyer.unstage() # a no-op + with pytest.raises(RuntimeError, match="from UNSTAGED"): + await flyer.prepare(1) + + +async def test_trigger_after_complete_says_to_prepare_again(tmp_path): + det = await reach("C", tmp_path) + with pytest.raises( + TransitionError, + match="cannot go to TRIGGERING from FLOWN, allowed: PREPARING", + ): + await det.trigger() + + +async def test_external_trigger_second_trigger_without_prepare_raises(): + source = Source() + det = make_detector( + "det", trigger=ExternalTrigger(), acquire=FillingAcquire(source) + ) + await det.stage() + await det.prepare( + TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE, exposure_timeout=0.1) + ) + await det.trigger() + with pytest.raises(RuntimeError, match="prepare"): + await det.trigger() + + +async def test_internal_trigger_repeats_on_one_prepare(): + acquire = FillingAcquire() + det = make_detector("det", acquire=acquire) + await det.stage() + await det.prepare(TriggerInfo()) + for _ in range(5): + await det.trigger() + assert acquire.idle_waits == 5 + + +class HeldAcquire(DetectorAcquireLogic): + """Arms with a status that only ensure_stopped() finishes, like a driver.""" + + def __init__(self) -> None: + self.statuses: list[AsyncStatus] = [] + self.closed = 0 + self.armed = asyncio.Event() + self._stopped = asyncio.Event() + + async def ensure_ready(self): + self._stopped.clear() + + async def acquire(self): + try: + async with AsyncStatus(self._stopped.wait()) as status: + self.statuses.append(status) + self.armed.set() + yield + await status + finally: + self.closed += 1 + + async def ensure_stopped(self): + self._stopped.set() + + +@pytest.mark.xfail( + reason="an armed acquisition never taken is dropped, not closed (Tom 2026-10-22)", + strict=True, +) +async def test_unstage_closes_the_acquisition_an_external_prepare_armed(): + acquire = HeldAcquire() + det = make_detector("det", trigger=ExternalTrigger(), acquire=acquire) + await det.stage() + await det.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) + assert acquire.armed.is_set() + assert acquire.closed == 0 + + await det.unstage() + assert acquire.closed == 1 + assert acquire.statuses[0].done + + +async def test_unstage_cancels_a_running_trigger_that_kickoff_cannot_overlap(): + acquire = HeldAcquire() + det = make_detector( + "det", FakeData(Source(), publication="page", bounded=False), acquire=acquire + ) + await det.stage() + # Prepares implicitly, arms, then waits for data that never comes + status = det.trigger() + await acquire.armed.wait() + with pytest.raises(RuntimeError, match="cannot go to FLYING from TRIGGERING"): + await det.kickoff() + await det.unstage() + with pytest.raises(asyncio.CancelledError): + await status + assert acquire.closed == 1 + assert acquire.statuses[0].done + + +# --------------------------------------------------------------------------- +# Use cases 10 and 11 +# --------------------------------------------------------------------------- + + +async def test_a_bounded_logic_publishes_only_the_keys_unbounded_do_not_make(): + unbounded_source, bounded_source = Source(), Source() + det = make_detector( + "det", + FakeData(unbounded_source, publication="page", bounded=False, suffix="-u"), + FakeData( + bounded_source, publication="page", bounded=True, suffix="-u", also=["-b"] + ), + acquire=FillingAcquire(unbounded_source, bounded_source, per_start=2), + ) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=2)) + await det.kickoff() + await det.complete() + + (page,) = [p async for p in det.collect_pages()] # type: ignore[attr-defined] + assert page["data"] == {"det-u": [[0], [1]], "det-b": [[0], [1]]} + # Hinted fields are those the providers make + assert det.hints == {"fields": ["det-u"]} + + +@pytest.mark.parametrize( + "bounded, again", + [ + # A bounded logic is restarted at the next step point + (True, lambda det: det.trigger()), + # An unbounded one at a prepare() for a different scan + (False, lambda det: det.prepare(TriggerInfo(collections_per_event=2))), + ], +) +async def test_a_data_logic_that_served_the_stage_must_serve_all_of_it( + bounded: bool, again +): + source = Source() + data = FakeData(source, publication="page", bounded=bounded) + det = make_detector("det", data, acquire=FillingAcquire(source)) + await det.stage() + await det.trigger() + + data.sit_out = True + with pytest.raises(RuntimeError, match=r"made \[\] at .*, but \['det'\] before it"): + await again(det) + + +async def test_uc11_no_data_no_readables_steps_and_flies(): + acquire = FillingAcquire() + det = make_detector("det", acquire=acquire) + await det.stage() + + await det.trigger() + await det.trigger() + assert acquire.idle_waits == 2 + assert await det.read() == {} + + # A fly has nothing to collect, so completes once acquisition has run + await det.prepare(TriggerInfo(number_of_events=3)) + await det.kickoff() + await det.complete() + assert acquire.idle_waits == 3 + assert not hasattr(det, "collect_asset_docs") + assert not hasattr(det, "collect_pages") + + +# --------------------------------------------------------------------------- +# Data-set rules (§5) +# --------------------------------------------------------------------------- + + +async def test_u5_read_is_idempotent_for_pageable_data(): + source = Source() + det = make_detector( + "det", + FakeData(source, publication="page", bounded=True), + acquire=FillingAcquire(source, per_start=2), + ) + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=2)) + await det.trigger() + + first = await det.read() + second = await det.read() + + assert first["det"]["value"] == [0, 1] + assert second == first + + +async def test_u5_collect_pages_takes_an_index(): + source = Source() + det = make_detector( + "det", + FakeData(source, publication="page", bounded=True), + acquire=FillingAcquire(source, per_start=5), + ) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=5)) + await det.kickoff() + + first = [p async for p in det.collect_pages(index=2)] # type: ignore[attr-defined] + rest = [p async for p in det.collect_pages(index=5)] # type: ignore[attr-defined] + + assert [p["data"]["det"] for p in first] == [[[0], [1]]] + assert [p["data"]["det"] for p in rest] == [[[2], [3], [4]]] + + +async def test_uc12b_multiple_pageable_on_one_detector_merge_into_one_page(): + """Use case 12(b), written as if the bundler limits were fixed.""" + a, b = Source(), Source() + det = make_detector( + "det", + FakeData(a, publication="page", bounded=True, suffix="-a"), + FakeData(b, publication="page", bounded=True, suffix="-b"), + acquire=FillingAcquire(a, b, per_start=3), + ) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=3)) + await det.kickoff() + await det.complete() + + pages = [p async for p in det.collect_pages()] # type: ignore[attr-defined] + + (page,) = pages + assert page["data"] == {"det-a": [[0], [1], [2]], "det-b": [[0], [1], [2]]} + assert set(page["timestamps"]) == {"det-a", "det-b"} + + +@pytest.mark.xfail( + strict=True, + reason="bluesky's RunBundler collects stream assets XOR event pages per " + "device, so prepare() raises on the mix; passes once that goes", +) +async def test_uc12a_mixed_pageable_and_streamable_on_one_detector(tmp_path): + """Use case 12(a), written as if the bundler limits were fixed.""" + stream_source, page_source = Source(), Source() + det = make_detector( + "det", + FakeData(stream_source, publication="stream", bounded=False, tmp_path=tmp_path), + FakeData(page_source, publication="page", bounded=True, suffix="-stats"), + acquire=FillingAcquire(stream_source, page_source, per_start=2), + ) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=2)) + assert set(await det.describe_collect()) == {"det", "det-stats"} + await det.kickoff() + await det.complete() + + docs = [d async for d in det.collect_asset_docs()] # type: ignore[attr-defined] + pages = [p async for p in det.collect_pages()] # type: ignore[attr-defined] + + assert [name for name, _ in docs] == ["stream_resource", "stream_datum"] + (page,) = pages + assert page["data"] == {"det-stats": [[0], [1]]} + + +class _BreachingData(FakeUnboundedData): + """Its provider's pages drop their last event, are `None`, or page `det` too.""" + + breach = "" + + async def start_provider(self, *args: Any) -> Any: + provider = await super().start_provider(*args) + make_page = provider.make_page + + async def breaching_make_page(index: int) -> PartialEventPage | None: + page = await make_page(index) + if page and self.breach == "short": + page = { + "data": {k: v[:-1] for k, v in page["data"].items()}, + "timestamps": {k: v[:-1] for k, v in page["timestamps"].items()}, + "time": page["time"][:-1], + } + elif self.breach == "none": + page = None + elif page and self.breach == "extra_key": + page["data"]["det"] = page["data"]["det-b"] + return page + + provider.make_page = breaching_make_page + return provider + + +@pytest.mark.parametrize( + "breach, error, match", + [ + ( + "short", + TypeError, + "Differing numbers of events: FakePageable has 1, FakePageable has 0", + ), + ( + "none", + TypeError, + "Differing numbers of events: FakePageable has 1, FakePageable has None", + ), + ( + "extra_key", + ValueError, + "Duplicate keys found while merging dictionaries: det", + ), + ], +) +async def test_pageable_providers_must_page_each_event_of_their_own_keys( + breach: str, error: type[Exception], match: str +): + source = Source() + breaching = _BreachingData(source, publication="page", suffix="-b") + breaching.breach = breach + det = make_detector( + "det", + FakeData(source, publication="page", bounded=False), + breaching, + acquire=FillingAcquire(source), + ) + await det.stage() + await det.trigger() + with pytest.raises(error, match=match): + await det.read() + + +def _reading_detector(suppressor: _FakeData, source: Source, value: Any): + reading = soft_signal_rw(Array1D[np.int64], value, name="det-reading") + return make_detector( + "det", suppressor, acquire=FillingAcquire(source, per_start=3), readable=None + ), reading + + +@pytest.mark.parametrize("publication", ["page", "stream"]) +async def test_readable_is_suppressed_by_pageable_and_streamable_keys( + publication, tmp_path +): + source = Source() + data = FakeData( + source, + publication=publication, + bounded=False, + suffix="-reading", + shape_tail=(), + dtype_numpy=" TriggerInfo: + return TriggerInfo(livetime=0.1, deadtime=0.02) + + +@pytest.mark.parametrize( + "env,expected_period", + [ + ("YES", pytest.approx(0.12)), + (None, 0.0), + ], +) +async def test_u12_zero_livetime_readback_is_gated_by_the_env_var( + monkeypatch: pytest.MonkeyPatch, env, expected_period +): + if env is None: + monkeypatch.delenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", raising=False) + else: + monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", env) + data = FakeData(Source(), publication="page", bounded=False) + det = make_detector( + "det", data, trigger=ReadbackTrigger(), acquire=FillingAcquire() + ) + await det.stage() + + await det.prepare(TriggerInfo(number_of_events=1)) + + assert data.periods == [expected_period] + + +# --------------------------------------------------------------------------- +# collect_pages from each state (U1): only after kickoff() +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "state,error", + [ + ( + "P", + "det: collect_pages not allowed from PREPARED, allowed from: FLOWN, FLYING", + ), + ( + "T", + "det: collect_pages not allowed from TRIGGERED, " + "allowed from: FLOWN, FLYING", + ), + ("K", None), + ("C", None), + ], +) +async def test_collect_pages_from_each_state(state: str, error: str | None): + source = Source() + det = make_detector( + "det", + FakeData(source, publication="page", bounded=False), + acquire=FillingAcquire(source), + ) + await det.stage() + await det.prepare(TriggerInfo()) + if state == "T": + await det.trigger() + elif state in "KC": + await det.kickoff() + if state == "C": + await det.complete() + + collect = det.collect_pages() # type: ignore[attr-defined] + if error is None: + assert [page async for page in collect] != [] + else: + with pytest.raises(RuntimeError, match=re.escape(error)): + _ = [page async for page in collect] + + +# --------------------------------------------------------------------------- +# Verifier findings 2 and 4 to 6: re-asking, notes, failed prepare and start +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("publication", ["stream", "page"]) +async def test_an_unbounded_provider_carries_over_without_its_logic_being_asked( + publication: str, tmp_path +): + source = Source() + unbounded = FakeData( + source, publication=publication, bounded=False, tmp_path=tmp_path + ) + det = make_detector("det", unbounded, acquire=FillingAcquire(source)) + await det.stage() + unbounded.log.clear() + info = TriggerInfo(livetime=0.1, deadtime=0.01) + await det.prepare(info) + await det.trigger() + await det.prepare(info) + await det.trigger() + await det.prepare(info) + + assert unbounded.log == ["start"] + assert set(await det.describe_collect()) == {"det"} + + +async def test_a_carried_pageable_provider_first_page_starts_at_the_new_baseline(): + source = Source() + det = make_detector( + "det", + FakeData(source, publication="page", bounded=False), + acquire=FillingAcquire(source), + ) + await det.stage() + await det.prepare(TriggerInfo()) + await det.trigger() + await det.prepare(TriggerInfo()) + await det.kickoff() + await det.complete() + + # The first run's collection is not repeated in the second run's page + pages = [page async for page in det.collect_pages()] # type: ignore[attr-defined] + assert [page["data"] for page in pages] == [{"det": [[1]]}] + + +async def test_a_changed_scan_stops_the_old_provider_before_making_the_new(tmp_path): + sources = Source(), Source() + unbounded = FakeData( + sources[0], publication="stream", bounded=False, tmp_path=tmp_path + ) + steady = FakeData( + sources[1], publication="stream", bounded=False, tmp_path=tmp_path, suffix="-s" + ) + det = make_detector( + "det", unbounded, steady, acquire=FillingAcquire(*sources, per_start=2) + ) + await det.stage() + for data in ( + unbounded, + steady, + ): + data.log.clear() + await det.prepare(TriggerInfo(livetime=0.1)) + assert unbounded.log == steady.log == ["start"] + + # Each of what a provider is made for, in turn + for changed in ( + TriggerInfo(livetime=0.2), + TriggerInfo(livetime=0.2, flush_period=2.0), + TriggerInfo(livetime=0.2, flush_period=2.0, collections_per_event=2), + ): + await det.trigger() + unbounded.log.clear() + steady.log.clear() + await det.prepare(changed) + assert unbounded.log == ["stop", "start"] + assert steady.log == ["stop", "start"] + + +async def test_a_settings_change_takes_effect_at_the_next_stage_not_prepare(tmp_path): + # Stands for a plugin's EnableCallbacks being switched off in the Settings + source = Source() + plugin = FakeData(source, publication="stream", bounded=False, tmp_path=tmp_path) + det = make_detector("det", plugin, acquire=FillingAcquire(source)) + await det.stage() + info = TriggerInfo() + await det.prepare(info) + await det.trigger() + + plugin.sit_out = True + await det.prepare(info) + assert set(await det.describe_collect()) == {"det"} + + await det.unstage() + assert plugin.log[-1] == "stop" + await det.stage() + await det.prepare(info) + assert await det.describe_collect() == {} + + +@pytest.mark.parametrize( + "verb", + [ + "stage", + "unstage", + "trigger", + "kickoff", + "complete", + "read", + "collect_pages", + "collect_asset_docs", + ], +) +async def test_f4_exceptions_from_logic_carry_the_detectors_name(verb: str, tmp_path): + class Failing(FillingAcquire): + failing = "" + + async def ensure_ready(self): + if self.failing == "stage": + raise BoomError(self.failing) + + async def ensure_stopped(self): + if self.failing == "unstage": + raise BoomError(self.failing) + + async def acquire(self): + if self.failing in ("trigger", "kickoff"): + raise BoomError(self.failing) + async for _ in super().acquire(): + yield + if self.failing == "complete": + raise BoomError(self.failing) + + source = Source() + acquire = Failing(source) + publication = "stream" if verb == "collect_asset_docs" else "page" + data = FakeData(source, publication=publication, bounded=False, tmp_path=tmp_path) + det = make_detector("det", data, acquire=acquire) + # Before kickoff, as fly() runs on to the end of acquire() as soon as it can + acquire.failing = verb + if verb != "stage": + await det.stage() + if verb not in ("stage", "unstage"): + await det.prepare(TriggerInfo(number_of_events=1)) + if verb == "read": + await det.trigger() + if verb in ("complete", "collect_pages", "collect_asset_docs"): + await det.kickoff() + data.fail = verb in ("read", "collect_pages", "collect_asset_docs") + + with pytest.raises(BoomError) as error: + if verb == "collect_pages": + _ = [page async for page in det.collect_pages()] # type: ignore[attr-defined] + elif verb == "collect_asset_docs": + _ = [doc async for doc in det.collect_asset_docs()] # type: ignore[attr-defined] + else: + await getattr(det, verb)() + assert error.value.__notes__ == ["raised by det"] + + +@pytest.mark.parametrize("publication", ["page", "stream"]) +@pytest.mark.parametrize("cpe", [1, 3]) +async def test_scalar_readable_is_suppressed_by_a_key_of_any_collections_per_event( + publication: str, cpe: int, tmp_path +): + """A readable gives one value per event, so [cpe] suppresses it, not just [].""" + source = Source() + data = FakeData( + source, + publication=publication, + bounded=False, + suffix="-reading", + tmp_path=tmp_path, + ) + det = make_detector("det", data, acquire=FillingAcquire(source, per_start=cpe)) + det.reading = soft_signal_rw(int, 9, name="det-reading") + det.set_readable_format(det.reading, Format.HINTED_SIGNAL) + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=cpe)) + await det.trigger() + + description = await det.describe() + + assert description["det-reading"]["shape"] == [cpe] + reading = await det.read() + if publication == "page": + assert reading["det-reading"]["value"] == list(range(cpe)) + else: + assert reading == {} + + +def test_data_logic_with_the_removed_stop_raises(): + with pytest.raises(TypeError, match=r"StaleData defines stop, which .* -> ensure"): + type("StaleData", (BoundedDataLogic,), {"stop": lambda self: None}) diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 19880ca953..9dad576b5c 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -2,6 +2,7 @@ import pytest from bluesky.protocols import HasHints +from event_model import DataKey from ophyd_async.core import ( Device, @@ -11,6 +12,7 @@ SignalR, StandardReadable, TransitionError, + set_mock_attr, soft_signal_r_and_setter, soft_signal_rw, ) @@ -522,3 +524,81 @@ def test_set_readable_format_rejects_non_signal_for_signal_format(): sr.child = Device(name="child") with pytest.raises(TypeError, match="is not a SignalR"): sr.set_readable_format(sr.child, Format.CONFIG_SIGNAL) + + +class _Suppressing(StandardReadable): + """A readable that is told which names to suppress, as a detector would.""" + + def __init__(self, name: str = ""): + self.suppressed: dict[str, DataKey | None] = {} + super().__init__(name) + self._suppressed_datakeys_funcs += (lambda: self.suppressed,) + + +class _Pair(StandardReadable): + """A child that reads two hinted signals, to be suppressed from its parent.""" + + def __init__(self, name: str = ""): + with self.add_children_as_readables(Format.HINTED_SIGNAL): + self.one = soft_signal_rw(int, 1) + self.two = soft_signal_rw(int, 2) + super().__init__(name) + + +async def _suppressing(suppressed: dict[str, DataKey | None]) -> _Suppressing: + sr = _Suppressing("sr") + sr.pair = _Pair() + sr.sig = soft_signal_rw(int, 3) + sr.set_readable_format(sr.pair, Format.CHILD) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.set_name("sr") + await sr.connect(mock=True) + sr.suppressed = suppressed + return sr + + +async def _datakeys() -> dict[str, DataKey]: + return await (await _suppressing({})).describe() + + +@pytest.mark.parametrize("suppressed_name", ["sr-sig", "sr-pair-two"]) +async def test_a_switched_off_name_is_dropped_from_read_describe_and_hints( + suppressed_name: str, +): + sr = await _suppressing({suppressed_name: None}) + + expected = {"sr-pair-one", "sr-pair-two", "sr-sig"} - {suppressed_name} + assert set(await sr.read()) == expected + assert set(await sr.describe()) == expected + assert set(sr.hints["fields"]) == expected + + +@pytest.mark.parametrize("suppressed_name", ["sr-sig", "sr-pair-two"]) +async def test_a_replaced_name_is_dropped_from_read_and_describe_but_kept_in_hints( + suppressed_name: str, +): + replacement = (await _datakeys())[suppressed_name] + sr = await _suppressing({suppressed_name: replacement}) + + expected = {"sr-pair-one", "sr-pair-two", "sr-sig"} - {suppressed_name} + assert set(await sr.read()) == expected + assert set(await sr.describe()) == expected + assert set(sr.hints["fields"]) == {"sr-pair-one", "sr-pair-two", "sr-sig"} + + +async def test_a_replacement_of_a_different_dtype_is_a_duplicate_datakey(): + replacement = (await _datakeys())["sr-sig"] + replacement["dtype_numpy"] = " StreamableDataProvider: + self.shape = shape + self.dtype_numpy = dtype_numpy + + async def start_provider( + self, + name: str, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> StreamResourceDataProvider: resource = StreamResourceInfo( - data_key=datakey_name, - shape=(10, 15), - chunk_shape=(1, 10, 15), - dtype_numpy="|u1", + data_key=name + self.datakey_suffix, + shape=self.shape, + chunk_shape=(1, *self.shape), + dtype_numpy=self.dtype_numpy, parameters={"dataset": "/data"}, ) - provider = StreamResourceDataProvider( + return StreamResourceDataProvider( uri=f"file://localhost/{self.tmp_path.as_posix().lstrip('/')}/test.h5", resources=[resource], mimetype="application/x-hdf5", - collections_written_signal=self.collections_written, + collections_per_event=collections_per_event, ) - return provider - async def stop(self) -> None: + async def ensure_stopped(self) -> None: self.stop_count += 1 - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return [datakey_name] + def get_hinted_fields(self, name: str) -> Sequence[str]: + return [name + self.datakey_suffix] async def test_stream_resource_data_provider_1d_single_collection_is_array(): @@ -174,10 +193,10 @@ async def test_stream_resource_data_provider_1d_single_collection_is_array(): ) ], mimetype="application/x-hdf5", - collections_written_signal=soft_signal_rw(int), + collections_per_event=1, ) - assert await provider.make_datakeys(collections_per_event=1) == { + assert await provider.make_datakeys() == { "det-spectrum": { "dtype": "array", "dtype_numpy": " None: + self.published = 0 + + async def make_datakeys(self) -> dict[str, DataKey]: + return { + self.datakey_name: DataKey( + source="mock", + shape=[self.collections_per_event], + dtype="array", + dtype_numpy=" PartialEventPage: + new = range(events) + return { + "data": { + self.datakey_name: [[0] * self.collections_per_event for _ in new] + }, + "time": [0.0 for _ in new], + "timestamps": {self.datakey_name: [0.0 for _ in new]}, + } + + async def make_page(self, index: int) -> PartialEventPage | None: + if index <= self.published: + return None + page = self._page(index - self.published) + self.published = index + return page + + +class BoundedOnlyDataLogic(BoundedDataLogic[MockPageableProvider]): + """Produces bounded data held in a finite buffer, sized when it is armed.""" + + def __init__(self, datakey_suffix: str = ""): + self.datakey_suffix = datakey_suffix + self.collections_written = soft_signal_rw(int) + self.prepare_calls: list[tuple[int, float]] = [] + self.stop_count = 0 + self.providers: list[MockPageableProvider] = [] + + async def start_provider( + self, + name: str, + existing_datakeys: Sequence[str], + num_collections: int, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> MockPageableProvider | None: + datakey_name = name + self.datakey_suffix + if datakey_name in existing_datakeys: + return None + self.prepare_calls.append((num_collections, collection_period)) + # A real buffer clears its progress counter when armed + await self.collections_written.set(0) + provider = MockPageableProvider(datakey_name, collections_per_event) + self.providers.append(provider) + return provider + + async def ensure_stopped(self) -> None: + self.stop_count += 1 + + def get_hinted_fields(self, name: str) -> Sequence[str]: + return [name + self.datakey_suffix] + + # Parameterized Tests for Logic Combinations @@ -216,9 +314,8 @@ async def test_get_trigger_deadtime( trigger_logic, expected_triggers, expected_deadtime ): """Test get_trigger_deadtime with various trigger logic implementations.""" - det = StandardDetector() - if trigger_logic: - det.add_detector_logics(trigger_logic) + det = DetectorLogic(trigger=trigger_logic).with_device() + await det.stage() if isinstance(trigger_logic, DeadtimeTriggerLogic): # A logic that needs a signal declares it as configuration; the logic # no longer nominates signals separately @@ -234,10 +331,9 @@ async def test_get_trigger_deadtime_with_settings(): deadtime_signal = soft_signal_rw(float, 0.02) # Create detector with DeadtimeTriggerLogic - det = StandardDetector() + det = DetectorLogic(trigger=DeadtimeTriggerLogic(deadtime_signal)).with_device() + await det.stage() det.sig = deadtime_signal - tl = DeadtimeTriggerLogic(deadtime_signal) - det.add_detector_logics(tl) det.set_readable_format(deadtime_signal, Format.CONFIG_SIGNAL) # Verify initial deadtime from signal @@ -266,9 +362,9 @@ async def test_get_trigger_deadtime_with_settings(): ) async def test_prepare_trigger_types(trigger_type): """Test each trigger type is properly delegated to trigger logic.""" - det = StandardDetector() trigger_logic = AllTriggerTypesLogic() - det.add_detector_logics(trigger_logic) + det = DetectorLogic(trigger=trigger_logic).with_device() + await det.stage() trigger_info = TriggerInfo( trigger=trigger_type, livetime=0.5, deadtime=0.1, number_of_events=10 @@ -282,8 +378,8 @@ async def test_prepare_trigger_types(trigger_type): async def test_prepare_unsupported_trigger_type(): """Test that preparing with unsupported trigger type raises error.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = DetectorLogic(trigger=JustInternalTriggerLogic()).with_device() + await det.stage() with pytest.raises(ValueError, match="Trigger type.*EXTERNAL_EDGE not supported"): await det.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) @@ -320,9 +416,9 @@ async def test_trigger_info_calculations( @pytest.mark.parametrize("exposures_per_collection", [1, 2, 5, 10]) async def test_exposures_per_collection(exposures_per_collection): """Test exposure averaging configuration.""" - det = StandardDetector() tl = AveragingTriggerLogic() - det.add_detector_logics(tl) + det = DetectorLogic(trigger=tl).with_device() + await det.stage() await det.prepare( TriggerInfo( @@ -337,8 +433,8 @@ async def test_exposures_per_collection(exposures_per_collection): async def test_exposures_per_collection_not_supported(): """Test that exposures_per_collection > 1 fails without supporting logic.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) # Doesn't support averaging + det = DetectorLogic(trigger=JustInternalTriggerLogic()).with_device() + await det.stage() with pytest.raises( ValueError, match="Multiple exposures per collection not supported" @@ -356,11 +452,11 @@ async def test_exposures_per_collection_not_supported(): ) async def test_arm_timing(trigger_type, arm_timing, tmp_path): """Verify detector is armed at the correct time based on trigger type.""" - det = StandardDetector() tl = AllTriggerTypesLogic() al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, al, dl) + det = DetectorLogic(trigger=tl, acquire=al, data=[dl]).with_device() + await det.stage() # Prepare the detector await det.prepare(TriggerInfo(trigger=trigger_type, number_of_events=2)) @@ -385,25 +481,40 @@ async def test_arm_timing(trigger_type, arm_timing, tmp_path): # External triggers should still be armed from prepare assert al.arm_count == 1 assert al.armed is True + await det.unstage() -async def test_trigger_arms_detector(tmp_path): - """Test that trigger() arms the detector when arm logic is present.""" - det = StandardDetector() +@pytest.mark.parametrize( + "trigger_type,arms_at_trigger", + [ + (DetectorTrigger.INTERNAL, True), + (DetectorTrigger.EXTERNAL_EDGE, False), + (DetectorTrigger.EXTERNAL_LEVEL, False), + ], +) +async def test_trigger_arms_only_an_internally_triggered_detector( + trigger_type, arms_at_trigger, tmp_path +): + """trigger() arms the detector, unless prepare() already did. + + An externally triggered detector was armed by prepare() and is sitting + waiting on its trigger source, so arming it again would restart it. + """ al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), al, dl) - - await det.prepare(TriggerInfo()) + det = DetectorLogic( + trigger=AllTriggerTypesLogic(), acquire=al, data=[dl] + ).with_device() + await det.stage() - # Should not be armed yet - assert al.armed is False - assert al.arm_count == 0 + await det.prepare(TriggerInfo(trigger=trigger_type)) + assert al.arm_count == (0 if arms_at_trigger else 1) + assert al.armed is not arms_at_trigger - # Trigger should arm it status = det.trigger() # Give it a moment to arm await wait_for_pending_wakeups(raise_if_exceeded=False) + # Armed either way by now, but only ever once assert al.armed is True assert al.arm_count == 1 @@ -412,11 +523,53 @@ async def test_trigger_arms_detector(tmp_path): await status +class NoYieldAcquireLogic(MockAcquireLogic): + async def acquire(self): + return + yield + + +class TwoYieldAcquireLogic(MockAcquireLogic): + async def acquire(self): + yield + yield + + +@pytest.mark.parametrize( + "logic_cls, message", + [ + (NoYieldAcquireLogic, "NoYieldAcquireLogic.acquire.. finished without yield"), + (TwoYieldAcquireLogic, "TwoYieldAcquireLogic.acquire.. yielded more than once"), + ], +) +@pytest.mark.parametrize("entry", ["trigger", "fly", "external_fly"]) +async def test_acquire_must_yield_exactly_once(tmp_path, logic_cls, message, entry): + dl = StreamableOnlyDataLogic(tmp_path) + det = DetectorLogic( + acquire=logic_cls(), data=[dl], trigger=AllTriggerTypesLogic() + ).with_device() + await det.stage() + external = entry == "external_fly" + info = TriggerInfo( + trigger=DetectorTrigger.EXTERNAL_EDGE if external else DetectorTrigger.INTERNAL, + livetime=0.1, + ) + with pytest.raises(RuntimeError, match=message): + await det.prepare(info) + if entry == "trigger": + status = det.trigger() + else: + await det.kickoff() + status = det.complete() + await wait_for_pending_wakeups() + await dl.collections_written.set(1) + await status + + async def test_arm_logic_called_on_stage(): """Test that acquire logic is stopped on stage.""" - det = StandardDetector() al = MockAcquireLogic() - det.add_detector_logics(al) + det = DetectorLogic(acquire=al).with_device() al.armed = True # Simulate being armed await det.stage() @@ -427,26 +580,26 @@ async def test_arm_logic_called_on_stage(): async def test_describe_before_prepare_raises(tmp_path): """Test that describe() fails before prepare().""" - det = StandardDetector() - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(data=[StreamableOnlyDataLogic(tmp_path)]).with_device() + await det.stage() - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(TransitionError, match="not allowed from STAGED"): await det.describe() async def test_describe_collect_before_prepare_raises(tmp_path): """Test that describe_collect() fails before prepare().""" - det = StandardDetector() - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(data=[StreamableOnlyDataLogic(tmp_path)]).with_device() + await det.stage() - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(TransitionError, match="not allowed from STAGED"): await det.describe_collect() async def test_trigger_after_multi_event_prepare_raises(): """Test that trigger() after prepare with multiple events fails.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = DetectorLogic(trigger=JustInternalTriggerLogic()).with_device() + await det.stage() await det.prepare(TriggerInfo(number_of_events=5)) @@ -459,15 +612,14 @@ async def test_preserve_detector_state_requires_default_trigger_info( ): """Test PRESERVE_DETECTOR_STATE=YES errors without default_trigger_info.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="mydet") + det = DetectorLogic(trigger=AllTriggerTypesLogic()).with_device("mydet") # AllTriggerTypesLogic intentionally does not implement default_trigger_info - det.add_detector_logics(AllTriggerTypesLogic()) await det.stage() with pytest.raises( RuntimeError, - match="OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but 'mydet' has no " - "default_trigger_info\\(\\)", + match="OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but the " + "trigger logic has no default_trigger_info\\(\\)", ): await det.trigger() @@ -479,12 +631,12 @@ async def test_preserve_detector_state_no_trigger_logic_falls_back( trigger() silently falls back to a bare TriggerInfo() rather than raising. A detector with no trigger logic has no hardware state to preserve.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="nodet") + det = DetectorLogic().with_device("nodet") await det.stage() # Should not raise — no trigger logic means nothing to preserve await det.trigger() - assert det._prepare_ctx is not None - assert det._prepare_ctx.trigger_info == TriggerInfo() + # ...and it was prepared, implicitly + assert await det.describe() == {} async def test_preserve_detector_state_multi_collection_watcher_and_assets( @@ -503,9 +655,10 @@ async def default_trigger_info(self) -> TriggerInfo: return TriggerInfo(collections_per_event=5) monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="multidet") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(MultiCollectionTriggerLogic(), dl) + det = DetectorLogic(trigger=MultiCollectionTriggerLogic(), data=[dl]).with_device( + "multidet" + ) await det.stage() # Collect watcher updates via watch() callback @@ -529,11 +682,6 @@ async def default_trigger_info(self) -> TriggerInfo: assert all(update["target"] == 5 for update in updates) -@pytest.mark.xfail( - reason="collect_asset_docs() checks ``dl.collections_written`` again after " - "``trigger()``", - strict=True, -) async def test_collect_asset_docs_uses_trigger_observed_event( tmp_path, ): @@ -542,12 +690,14 @@ async def test_collect_asset_docs_uses_trigger_observed_event( It could be the case that ``dl.collections_written`` changes between ``trigger()`` and ``collect_asset_docs``.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) dl.collections_written, set_collections_written = soft_signal_r_and_setter( int, getter=lambda: 0 ) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = DetectorLogic(trigger=JustInternalTriggerLogic(), data=[dl]).with_device( + "det" + ) + await det.stage() await det.prepare(TriggerInfo(collections_per_event=5, exposure_timeout=0.1)) status = det.trigger() @@ -560,57 +710,51 @@ async def test_collect_asset_docs_uses_trigger_observed_event( assert docs[1][1]["indices"] == {"start": 0, "stop": 1} -async def test_kickoff_respects_prepare_bounds(tmp_path): - """Test that multiple kickoff() calls respect prepared bounds.""" - det = StandardDetector() - tl = JustInternalTriggerLogic() +async def test_one_kickoff_per_prepare(tmp_path): + """A prepare serves one kickoff/complete cycle; the next needs a new prepare.""" dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, dl) + det = DetectorLogic(trigger=JustInternalTriggerLogic(), data=[dl]).with_device( + "foo" + ) + await det.stage() - # Prepare for 5 events await det.prepare(TriggerInfo(number_of_events=5)) - - # Update collections_written signal to simulate data being written - - # First kickoff for 3 events - await det.events_to_kickoff.set(3) await det.kickoff() - await dl.collections_written.set(3) - # Second kickoff for 2 events should work (total = 5) - await det.events_to_kickoff.set(2) - await det.kickoff() + # A second kickoff without an intervening complete is rejected + with pytest.raises(RuntimeError, match="cannot go to FLYING from FLYING"): + await det.kickoff() + + status = det.complete() await dl.collections_written.set(5) + await status - # Third kickoff should fail (would exceed 5) - await det.events_to_kickoff.set(1) - with pytest.raises( - RuntimeError, - match="Kickoff requested 5:6, but detector was only prepared up to 5", - ): + # ...and so is one after complete, until prepare() runs again + with pytest.raises(RuntimeError, match="cannot go to FLYING from FLOWN"): await det.kickoff() -async def test_stage_resets_state(): - """Test that stage() resets detector state.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) +async def test_restage_resets_state(): + """Test that unstage() then stage() resets detector state.""" + det = DetectorLogic(trigger=JustInternalTriggerLogic()).with_device() + await det.stage() await det.prepare(TriggerInfo(number_of_events=5)) - await det.events_to_kickoff.set(3) - # Stage should reset everything + # Restaging should reset everything + await det.unstage() await det.stage() - assert det._prepare_ctx is None - assert det._kickoff_ctx is None - assert await det.events_to_kickoff.get_value() == 0 + with pytest.raises(TransitionError, match="describe not allowed from STAGED"): + await det.describe() + with pytest.raises(TransitionError, match="cannot go to FLYING from STAGED"): + await det.kickoff() async def test_hints_from_single_data_logic(tmp_path): """Test that hints come from data logic.""" - det = StandardDetector(name="bar") - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(data=[StreamableOnlyDataLogic(tmp_path)]).with_device("bar") + await det.stage() await det.prepare(TriggerInfo()) @@ -619,10 +763,10 @@ async def test_hints_from_single_data_logic(tmp_path): async def test_hints_from_multiple_data_logics(tmp_path): """Test that hints are aggregated from multiple data logics.""" - det = StandardDetector(name="bar") dl1 = StreamableOnlyDataLogic(tmp_path) dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-extra") - det.add_detector_logics(dl1, dl2) + det = DetectorLogic(data=[dl1, dl2]).with_device("bar") + await det.stage() await det.prepare(TriggerInfo()) @@ -644,14 +788,13 @@ async def test_config_signals_in_describe_configuration( signal_type, initial_value, expected_dtype, expected_dtype_numpy, expected_shape ): """Test that added config signals appear in describe_configuration.""" - det = StandardDetector() + det = DetectorLogic().with_device() + await det.stage() signal = soft_signal_rw( signal_type, initial_value=initial_value, name="test-config" ) det.set_readable_format(signal, Format.CONFIG_SIGNAL) - await det.stage() - # Check describe_configuration config_desc = await det.describe_configuration() assert config_desc == { @@ -672,38 +815,23 @@ async def test_config_signals_in_describe_configuration( ) -async def test_kickoff_without_streamable_data_raises(): - """Test that kickoff() without streamable data fails.""" - det = StandardDetector(name="foo") - det.add_detector_logics(JustInternalTriggerLogic()) - - # A detector with no data logic can still be triggered for a step scan - await det.prepare(TriggerInfo()) - await det.trigger() # This works - - # ...but there is nothing to collect, so it cannot be flown - await det.prepare(TriggerInfo(number_of_events=5)) - with pytest.raises( - ValueError, match="Detector foo is not streamable, so cannot kickoff" - ): - await det.kickoff() - - async def test_streamable_supports_both_step_and_fly(tmp_path): """Test that streamable data logic supports both step and fly scanning.""" - det = StandardDetector(name="foo") tl = JustInternalTriggerLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, dl) + det = DetectorLogic(trigger=tl, data=[dl]).with_device("foo") + await det.stage() # Step scan should work status = det.trigger() # Yield so detector can get collections written, then set it so we complete await wait_for_pending_wakeups(raise_if_exceeded=False) await dl.collections_written.set(1) - await status + await wait_for_pending_wakeups(raise_if_exceeded=False) + assert status.done assert status.success docs = [doc async for doc in det.collect_asset_docs()] + sr = docs[:1] assert docs == [ ( "stream_resource", @@ -733,17 +861,21 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): await det.prepare(TriggerInfo(number_of_events=5)) await det.kickoff() status = det.complete() - # Check that setting collections written will only give the first 4 + # Check that setting collections written will only give the first 4, in a + # new resource for the new prepare await dl.collections_written.set(5) docs = [doc async for doc in det.collect_asset_docs()] - assert docs == [ + new_sr = docs[:1] + assert new_sr == [(name, {**doc, "uid": ANY}) for name, doc in sr] + assert new_sr[0][1]["uid"] != sr[0][1]["uid"] + assert docs[1:] == [ ( "stream_datum", { "descriptor": "", "indices": {"start": 1, "stop": 5}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": ANY, + "stream_resource": new_sr[0][1]["uid"], "uid": ANY, }, ), @@ -769,7 +901,8 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): async def test_read_returns_correct_values(): """Test that read() returns the values of registered signals.""" - det = StandardDetector(name="det") + det = DetectorLogic().with_device("det") + await det.stage() det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") det.set_readable_format(det.counts, Format.HINTED_SIGNAL) @@ -779,7 +912,8 @@ async def test_read_returns_correct_values(): async def test_detector_with_no_logics(): """Test that detector works with no logics for basic internal triggering.""" - det = StandardDetector() + det = DetectorLogic().with_device() + await det.stage() # Should support only INTERNAL triggering triggers, deadtime = await det.get_trigger_deadtime() @@ -788,56 +922,59 @@ async def test_detector_with_no_logics(): # Can prepare with INTERNAL trigger but not external await det.prepare(TriggerInfo()) - - with pytest.raises(ValueError, match="Trigger type.*not supported"): + await det.trigger() + with pytest.raises(ValueError, match="Trigger type.*EXTERNAL_EDGE not supported"): await det.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) -async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_triggering(): - """Test that detector without trigger logic cannot set livetime/deadtime/trigger.""" - det = StandardDetector(name="foo") - msg = "Detector foo has no trigger logic, so " - - with pytest.raises(ValueError, match=msg + "cannot set livetime or deadtime"): - await det.prepare(TriggerInfo(livetime=0.5)) - - with pytest.raises(ValueError, match=msg + "cannot set livetime or deadtime"): - await det.prepare(TriggerInfo(deadtime=0.1)) - - -async def test_cannot_add_two_trigger_logics(): - """Test that adding two trigger logics raises an error.""" - det = StandardDetector() - tl1 = JustInternalTriggerLogic() - tl2 = AllTriggerTypesLogic() - - det.add_detector_logics(tl1) - - with pytest.raises(RuntimeError, match="Detector already has trigger logic"): - det.add_detector_logics(tl2) +@pytest.mark.parametrize( + "info", + [ + TriggerInfo(livetime=0.5), + TriggerInfo(deadtime=0.1), + TriggerInfo(exposures_per_collection=2), + ], +) +async def test_detector_without_trigger_logic_cannot_set_timing(info: TriggerInfo): + det = DetectorLogic().with_device("foo") + await det.stage() + with pytest.raises(ValueError, match="No trigger logic, so cannot set livetime"): + await det.prepare(info) -async def test_cannot_add_two_arm_logics(): - """Test that adding two acquire logics raises an error.""" - det = StandardDetector() - al1 = MockAcquireLogic() - al2 = MockAcquireLogic() +async def test_errors_from_logic_calls_keep_their_message_and_name_the_detector(): + det = DetectorLogic().with_device("foo") + await det.stage() + with pytest.raises(ValueError) as exc_info: + await det.prepare(TriggerInfo(livetime=0.5)) + assert exc_info.value.args == ( + "No trigger logic, so cannot set livetime or deadtime " + "or exposures_per_collection", + ) + assert exc_info.value.__notes__ == ["raised by foo"] - det.add_detector_logics(al1) - with pytest.raises(RuntimeError, match="Detector already has acquire logic"): - det.add_detector_logics(al2) +async def test_detector_logic_takes_keywords_only(): + """Each logic's role is named, so none can be given positionally.""" + with pytest.raises(TypeError, match="positional"): + DetectorLogic(JustInternalTriggerLogic()) # type: ignore[misc] -async def test_add_unknown_logic_type_raises(): - """Test that adding an unknown logic type raises TypeError.""" - det = StandardDetector() +async def test_replaced_detector_logic_does_not_share_prepared_data(tmp_path): + """A copy made with dataclasses.replace has its own prepared state.""" + logic = DetectorLogic( + trigger=JustInternalTriggerLogic(), data=[StreamableOnlyDataLogic(tmp_path)] + ) + det = logic.with_device("det") + copy = dataclasses.replace(logic).with_device("copy") + await det.stage() + await copy.stage() - class UnknownLogic: - pass + await det.prepare(TriggerInfo()) - with pytest.raises(TypeError, match="Unknown logic type"): - det.add_detector_logics(UnknownLogic()) + assert list(await det.describe()) == ["det"] + with pytest.raises(TransitionError, match="describe not allowed from STAGED"): + await copy.describe() @pytest.mark.parametrize("initial_shutter_closed", [True, False]) @@ -858,18 +995,15 @@ def __init__(self, shutter_closed: bool): async def ensure_ready(self): pass # don't touch the shutter at stage time - async def start_acquiring(self): + async def acquire(self): self.shutter_closed = False # open shutter when acquiring - - async def wait_for_idle(self): - pass + yield async def ensure_stopped(self): self.shutter_closed = True # close shutter at end of scan - det = StandardDetector() al = ShutterAcquireLogic(initial_shutter_closed) - det.add_detector_logics(al) + det = DetectorLogic(acquire=al).with_device() await det.stage() assert ( @@ -877,86 +1011,207 @@ async def ensure_stopped(self): ) # stage() must not touch the shutter await det.trigger() - assert al.shutter_closed is False # start_acquiring() opens the shutter + assert al.shutter_closed is False # acquire() opens the shutter await det.unstage() assert al.shutter_closed is True # unstage() must close the shutter -async def test_data_logic_with_no_prepare_methods_raises(): - """Test error when DataLogic doesn't override any prepare methods.""" +async def test_bounded_step_scan_reads_derived_from_page(): + """A bounded buffer describes as an array and read() derives one reading.""" + dl = BoundedOnlyDataLogic() + det = DetectorLogic(trigger=JustInternalTriggerLogic(), data=[dl]).with_device( + "foo" + ) + await det.stage() + + ti = TriggerInfo( + number_of_events=1, collections_per_event=5, livetime=0.1, deadtime=0.0 + ) + await det.prepare(ti) + desc = await det.describe() + assert desc["foo"]["shape"] == [5] + # prepare sized the buffer for 5 collections at the 0.1s period + assert dl.prepare_calls == [(5, pytest.approx(0.1))] + + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(5) + await status + assert status.success + # prepare() armed the buffer for this point, so trigger() does not re-arm it + assert len(dl.providers) == 1 + assert dl.stop_count == 1 + reading = await det.read() + assert reading["foo"]["value"] == [0, 0, 0, 0, 0] + + # The next point stops, re-makes and re-starts the logic, as the buffer + # holds the last point's data, and counts from the re-armed buffer's 0 + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + assert len(dl.providers) == 2 + assert dl.stop_count == 2 + assert not status.done + await dl.collections_written.set(5) + await status + assert status.success + assert dl.prepare_calls == [(5, pytest.approx(0.1))] * 2 - class EmptyDataLogic(DetectorDataLogic): - pass - det = StandardDetector() - det.add_detector_logics(EmptyDataLogic()) +async def test_bounded_fly_scan_accumulates_across_kickoffs(): + """A bounded buffer is armed once at prepare and not re-armed per kickoff.""" + dl = BoundedOnlyDataLogic() + det = DetectorLogic(trigger=JustInternalTriggerLogic(), data=[dl]).with_device( + "foo" + ) + await det.stage() - with pytest.raises(RuntimeError, match="hasn't overridden any prepare_\\* methods"): + await det.prepare( + TriggerInfo(number_of_events=10, collections_per_event=1, livetime=0.1) + ) + assert dl.prepare_calls == [(10, pytest.approx(0.1))] + + await det.kickoff() + status = det.complete() + # The buffer fills as the scan runs, and pages come out as it goes + await dl.collections_written.set(5) + pages = [page async for page in det.collect_pages()] # type: ignore[attr-defined] + assert pages[0]["data"]["foo"] == [[0]] * 5 + assert not status.done + + await dl.collections_written.set(10) + pages = [page async for page in det.collect_pages()] # type: ignore[attr-defined] + assert pages[0]["data"]["foo"] == [[0]] * 5 + await status + # kickoff() never re-arms: the buffer was only ever armed once + assert dl.prepare_calls == [(10, pytest.approx(0.1))] + + +@pytest.mark.parametrize( + "requested_livetime,default,expected_period", + [ + # livetime unset -> resolved from the trigger logic's current state + (0.0, TriggerInfo(livetime=0.1, deadtime=0.02), 0.12), + # livetime given -> used as-is, no readback substitution + (0.05, TriggerInfo(livetime=0.1, deadtime=0.02), 0.05), + # livetime unset but trigger logic has nothing set -> stays 0 + (0.0, TriggerInfo(), 0.0), + ], +) +async def test_prepare_resolves_zero_livetime_for_bounded_period( + requested_livetime, default, expected_period, monkeypatch: pytest.MonkeyPatch +): + """A livetime of 0 is filled in from the trigger logic before sizing a buffer. + + Only when hardware state is preserved: test_u12 in test_detector_use_cases + covers the variable being unset. + """ + monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") + + class PeriodTriggerLogic(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): + pass + + async def default_trigger_info(self) -> TriggerInfo: + return default + + dl = BoundedOnlyDataLogic() + det = DetectorLogic(trigger=PeriodTriggerLogic(), data=[dl]).with_device("foo") + await det.stage() + + await det.prepare( + TriggerInfo( + number_of_events=1, collections_per_event=3, livetime=requested_livetime + ) + ) + assert dl.prepare_calls == [(3, pytest.approx(expected_period))] + + +async def test_data_logic_with_no_start_provider_raises(): + """Test error when a DataLogic does not implement start_provider.""" + + class EmptyDataLogic(UnboundedDataLogic): + async def start_provider(self, *args): + return await super().start_provider(*args) # type: ignore[safe-super] + + det = DetectorLogic(data=[EmptyDataLogic()]).with_device() + await det.stage() + + with pytest.raises(NotImplementedError): await det.prepare(TriggerInfo()) async def test_unstage_disarms_detector(): """Test that unstage() calls disarm on the detector.""" - det = StandardDetector() al = MockAcquireLogic() - det.add_detector_logics(al) + det = DetectorLogic(acquire=al).with_device() + await det.stage() al.armed = True await det.unstage() - assert al.disarm_count == 1 + # Once on stage(), once on unstage() + assert al.disarm_count == 2 assert al.armed is False -async def test_prepare_stops_data_logic_when_recreating_providers(tmp_path): - """Test that prepare() calls stop() on data logic when recreating providers.""" - det = StandardDetector(name="det") +@pytest.mark.parametrize( + "second_collections_per_event,stops", [(2, 0), (3, 1)], ids=["kept", "remade"] +) +async def test_prepare_stops_data_logic_when_recreating_providers( + tmp_path, second_collections_per_event: int, stops: int +): + """A streamable provider is kept while it would be made the same, else remade.""" dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = DetectorLogic(trigger=JustInternalTriggerLogic(), data=[dl]).with_device( + "det" + ) + await det.stage() + staged_stops = dl.stop_count - # First prepare with collections_per_event=2 - await det.prepare(TriggerInfo(number_of_events=3, collections_per_event=2)) - assert dl.stop_count == 0 # No stop on first prepare + await det.prepare(TriggerInfo(collections_per_event=2)) + assert dl.stop_count == staged_stops # No stop on first prepare + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(2) + await status - # Second prepare with different collections_per_event triggers recreation - await det.prepare(TriggerInfo(number_of_events=3, collections_per_event=3)) - assert dl.stop_count == 1 # stop() should have been called + await det.prepare(TriggerInfo(collections_per_event=second_collections_per_event)) + assert dl.stop_count == staged_stops + stops async def test_different_collections_written_raises(tmp_path): """Test that different collections_written values from providers raises error.""" - det = StandardDetector(name="det") dl1 = StreamableOnlyDataLogic(tmp_path) - dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl1, dl2) - - await det.prepare(TriggerInfo(number_of_events=5)) - - # Set different collection counts for each data logic + dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-2") + det = DetectorLogic( + trigger=JustInternalTriggerLogic(), data=[dl1, dl2] + ).with_device("det") + await det.stage() + # Neither start_provider() resets its count await dl1.collections_written.set(3) await dl2.collections_written.set(5) - # Should raise RuntimeError when collect_asset_docs tries to validate with pytest.raises( - RuntimeError, + TypeError, match=re.escape( - "Detectors have written different numbers of collections: {3, 5}" + "Differing numbers of collections written: " + "StreamableOnlyDataLogic has 3, StreamableOnlyDataLogic has 5" ), ): - await det.kickoff() + await det.prepare(TriggerInfo(number_of_events=5)) async def test_data_logic_and_registered_signal(tmp_path): """A registered signal and a data logic both describe(), only one collects.""" - det = StandardDetector(name="det") + det = DetectorLogic( + trigger=JustInternalTriggerLogic(), data=[StreamableOnlyDataLogic(tmp_path)] + ).with_device("det") + await det.stage() det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") det.set_readable_format(det.counts, Format.HINTED_SIGNAL) - det.add_detector_logics( - JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path) - ) - await det.prepare(TriggerInfo()) + await det.prepare(TriggerInfo(number_of_events=3)) # Should have data from the data logic and the registered signal description = await det.describe() @@ -987,20 +1242,24 @@ async def test_data_logic_and_registered_signal(tmp_path): }, } # Should be able to do fly scanning (has streamable logic) - await det.prepare(TriggerInfo(number_of_events=3)) await det.kickoff() + await det.unstage() async def test_collect_asset_docs_with_explicit_index(tmp_path): """Test collect_asset_docs() with explicitly provided index.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = DetectorLogic(trigger=JustInternalTriggerLogic(), data=[dl]).with_device( + "det" + ) + await det.stage() await det.prepare(TriggerInfo(number_of_events=5, collections_per_event=2)) + await det.kickoff() # Collect with explicit index (not relying on get_index) docs = [doc async for doc in det.collect_asset_docs(index=3)] + await det.unstage() # Should emit docs for 3 events assert docs == [ @@ -1036,7 +1295,8 @@ async def test_child_readable_config_signals_in_describe_configuration(): config_sig = soft_signal_rw(float, initial_value=1.5, name="child-exposure") child.set_readable_format(config_sig, Format.CONFIG_SIGNAL) - det = StandardDetector(name="det") + det = DetectorLogic().with_device("det") + await det.stage() det.set_readable_format(child, Format.CHILD) await det.prepare(TriggerInfo()) @@ -1053,10 +1313,15 @@ async def test_child_readable_read_signals_in_read(tmp_path): read_sig = soft_signal_rw(int, initial_value=99, name="child-counts") child.set_readable_format(read_sig, Format.HINTED_SIGNAL) - det = StandardDetector(name="det") + dl = StreamableOnlyDataLogic(tmp_path) + det = DetectorLogic(data=[dl]).with_device("det") + await det.stage() det.set_readable_format(child, Format.CHILD) - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) await det.prepare(TriggerInfo()) + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(1) + await status # The data logic describes its stream, the child describes its signal desc = await det.describe() @@ -1074,9 +1339,9 @@ async def test_child_readable_hints_merged(tmp_path): hinted_sig = soft_signal_rw(float, name="child-intensity") child.set_readable_format(hinted_sig, Format.HINTED_SIGNAL) - det = StandardDetector(name="det") + det = DetectorLogic(data=[StreamableOnlyDataLogic(tmp_path)]).with_device("det") + await det.stage() det.set_readable_format(child, Format.CHILD) - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) assert "fields" in det.hints assert "child-intensity" in det.hints["fields"] @@ -1105,19 +1370,32 @@ async def test_trigger_logic_not_implemented_errors(): async def test_data_logic_not_implemented_errors(): """Test NotImplementedError for unimplemented DetectorDataLogic methods.""" - logic = DetectorDataLogic() + class Bounded(BoundedDataLogic[DataProvider]): + async def start_provider(self, *args): + return await super().start_provider(*args) # type: ignore[safe-super] + + class Unbounded(UnboundedDataLogic[DataProvider]): + async def start_provider(self, *args): + return await super().start_provider(*args) # type: ignore[safe-super] + + with pytest.raises(NotImplementedError): + await Bounded().start_provider("test", [], 1, 1, 0, 0) with pytest.raises(NotImplementedError): - await logic.prepare_unbounded("test") + await Unbounded().start_provider("test", 1, 0, 0) + logic = Bounded() - # stop() should not raise (has default implementation) - await logic.stop() # Should pass + # These have default implementations, so should not raise + assert logic.get_hinted_fields("test") == [] + await logic.ensure_ready() + await logic.ensure_stopped() async def test_detector_readable_format_changes_at_runtime(): # A StandardDetector is a StandardReadable, so a plugin signal can be moved # between configuration and hinted reads without redefining the Device. - det = StandardDetector(name="det") + det = DetectorLogic().with_device("det") + await det.stage() det.temperature, _ = soft_signal_r_and_setter(float, 20.0, name="temperature") name = det.temperature.name @@ -1136,8 +1414,104 @@ async def test_detector_readable_format_changes_at_runtime(): async def test_detector_add_config_signals_is_deprecated(): - det = StandardDetector(name="det") + det = DetectorLogic().with_device("det") + await det.stage() signal, _ = soft_signal_r_and_setter(float, 1.0, name="sig") with pytest.deprecated_call(): det.add_config_signals(signal) assert set(await det.read_configuration()) == {signal.name} + + +async def test_streamable_logic_looks_like_writes_stream_assets(tmp_path): + """An unbounded logic exposes collect_asset_docs, and only that. + + The bluesky bundler picks up stream assets via a structural isinstance + against WritesStreamAssets (a hasattr check for collect_asset_docs). A + detector must look like exactly one of WritesStreamAssets / + EventPageCollectable so the bundler routes it down a single path. + """ + det = DetectorLogic(data=[StreamableOnlyDataLogic(tmp_path)]).with_device() + await det.stage() + + # Which verb applies follows what the logics will produce, so it is not + # known until they have been asked + assert not isinstance(det, WritesStreamAssets) + await det.prepare(TriggerInfo()) + + assert isinstance(det, WritesStreamAssets) + assert not isinstance(det, EventPageCollectable) + assert hasattr(det, "collect_asset_docs") + assert not hasattr(det, "collect_pages") + + +async def test_bounded_logic_looks_like_event_page_collectable(): + """A bounded logic exposes collect_pages, and only that.""" + det = DetectorLogic(data=[BoundedOnlyDataLogic()]).with_device() + await det.stage() + await det.prepare(TriggerInfo()) + + assert isinstance(det, EventPageCollectable) + assert not isinstance(det, WritesStreamAssets) + assert hasattr(det, "collect_pages") + assert not hasattr(det, "collect_asset_docs") + + +async def test_no_data_logic_looks_like_neither(): + """A detector with no data logic emits neither stream assets nor event pages.""" + det = DetectorLogic(trigger=JustInternalTriggerLogic()).with_device() + await det.stage() + + assert not isinstance(det, WritesStreamAssets) + assert not isinstance(det, EventPageCollectable) + assert not hasattr(det, "collect_asset_docs") + assert not hasattr(det, "collect_pages") + + +async def test_missing_attribute_raises_standard_attribute_error(): + """__getattr__ falls through to a normal AttributeError for other names.""" + det = DetectorLogic().with_device() + await det.stage() + with pytest.raises( + AttributeError, + match=r"object has no attribute 'does_not_exist'", + ): + det.does_not_exist # noqa: B018 + + +async def test_unsuppressed_bounded_keys_raise(tmp_path): + """A finite buffer the stream assets do not cover cannot be combined with them.""" + det = DetectorLogic( + data=[ + BoundedOnlyDataLogic(datakey_suffix="-stats"), + StreamableOnlyDataLogic(tmp_path), + ] + ).with_device("det") + await det.stage() + + # Only knowable once the logics have said what they would make + with pytest.raises( + TypeError, + match=( + r"would produce \['det-stats'\] as event pages and the rest of its " + r"data as stream assets" + ), + ): + await det.prepare(TriggerInfo()) + + +async def test_suppressed_bounded_logic_sits_the_scan_out(tmp_path): + """A finite buffer whose keys are all written durably gives way to the file.""" + bounded = BoundedOnlyDataLogic() + # The same quantity as the buffer: one integer per collection + streamable = StreamableOnlyDataLogic(tmp_path, shape=(), dtype_numpy=" adcore.AreaDetector[adcore.ADBaseIO]: driver = adcore.ADBaseIO("PREFIX:DRV:") async with init_devices(mock=True): - det = adcore.AreaDetector(driver=driver) - det.add_detector_logics(adcore.ADAcquireLogic(driver)) + det = adcore.AreaDetector( + driver=driver, acquire_logic=adcore.ADAcquireLogic(driver) + ) return det async def test_acquire_logic_trigger_internal_calls_acquire( adbase_detector: adcore.AreaDetector[adcore.ADBaseIO], ): + await adbase_detector.stage() + # Only the calls after stage() are of interest + get_mock(adbase_detector).reset_mock() await adbase_detector.trigger() assert_has_calls( adbase_detector.driver, @@ -38,14 +43,16 @@ async def test_acquire_logic_trigger_internal_calls_acquire( async def test_acquire_logic_when_arming_times_out(): driver = adcore.ADBaseIO("PREFIX:DRV:") async with init_devices(mock=True): - det = adcore.AreaDetector(driver=driver) - det.add_detector_logics(adcore.ADAcquireLogic(driver, timeout=0.02)) + det = adcore.AreaDetector( + driver=driver, acquire_logic=adcore.ADAcquireLogic(driver, timeout=0.02) + ) async def sleep_for_a_bit(value): await asyncio.sleep(0.02) callback_on_mock_put(det.driver.acquire, sleep_for_a_bit) + await det.stage() with pytest.raises( TimeoutError, match=re.escape( @@ -60,11 +67,13 @@ async def sleep_for_a_bit(value): async def test_acquire_logic_wait_for_idle_in_bad_state(): driver = adcore.ADBaseIO("PREFIX:DRV:") async with init_devices(mock=True): - det = adcore.AreaDetector(driver=driver) - det.add_detector_logics(adcore.ADAcquireLogic(driver, timeout=0.05)) + det = adcore.AreaDetector( + driver=driver, acquire_logic=adcore.ADAcquireLogic(driver, timeout=0.05) + ) set_mock_value(driver.detector_state, adcore.ADState.ERROR) + await det.stage() with pytest.raises(ValueError) as exc_info: await det.trigger() @@ -74,7 +83,7 @@ async def test_acquire_logic_wait_for_idle_in_bad_state(): assert "ADState.ABORTED" in error_msg -async def test_start_acquiring_driver_and_ensure_status_timing( +async def test_acquire_driver_and_ensure_status_timing( adbase_detector: adcore.AreaDetector[adcore.ADBaseIO], ): """This test ensures the camera has time to return to a good state. @@ -97,12 +106,15 @@ async def complete_acquire() -> None: adcore.ADState.IDLE, ) + await adbase_detector.stage() await asyncio.gather(adbase_detector.trigger(), complete_acquire()) async def test_acquire_logic_disarm( adbase_detector: adcore.AreaDetector[adcore.ADBaseIO], ): + await adbase_detector.stage() + get_mock(adbase_detector).reset_mock() await adbase_detector.unstage() assert_has_calls( adbase_detector.driver, @@ -112,18 +124,20 @@ async def test_acquire_logic_disarm( ) -async def test_start_acquiring_driver_and_ensure_status_disconnected(): +async def test_acquire_driver_and_ensure_status_disconnected(): """This test ensures the function behaves gracefully if no detector states are available. """ driver = adcore.ADBaseIO("PREFIX:DRV:") async with init_devices(mock=True): - det = adcore.AreaDetector(driver=driver) - det.add_detector_logics(adcore.ADAcquireLogic(driver, timeout=0.1)) + det = adcore.AreaDetector( + driver=driver, acquire_logic=adcore.ADAcquireLogic(driver, timeout=0.1) + ) set_callback_filter(driver.detector_state, lambda v: None) + await det.stage() with pytest.raises(asyncio.TimeoutError) as exc: await det.trigger() assert ( diff --git a/tests/unit_tests/epics/adcore/test_cont_acq_detector.py b/tests/unit_tests/epics/adcore/test_cont_acq_detector.py index 6cb5f06a3c..a8472fb9de 100644 --- a/tests/unit_tests/epics/adcore/test_cont_acq_detector.py +++ b/tests/unit_tests/epics/adcore/test_cont_acq_detector.py @@ -6,6 +6,7 @@ from ophyd_async.core import ( DetectorTrigger, EnableDisable, + StaticPathProvider, TriggerInfo, callback_on_mock_put, init_devices, @@ -37,10 +38,11 @@ async def test_cont_acq_controller_invalid_trigger_mode( cont_acq_detector: adcore.AreaDetector[adcore.ADBaseIO], ): trigger_info = TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE) + await cont_acq_detector.stage() with pytest.raises( ValueError, match=re.escape( - "Trigger type DetectorTrigger.EXTERNAL_EDGE not supported by 'det', " + "Trigger type DetectorTrigger.EXTERNAL_EDGE not supported, " "supported types are: [INTERNAL]" ), ): @@ -56,6 +58,7 @@ async def test_cont_acq_controller_not_in_continuous_mode( cont_acq_detector: adcore.AreaDetector[adcore.ADBaseIO], ): set_mock_value(cont_acq_detector.driver.image_mode, adcore.ADImageMode.SINGLE) + await cont_acq_detector.stage() with pytest.raises(RuntimeError, match=ERROR_MESSAGE): await cont_acq_detector.trigger() @@ -64,6 +67,7 @@ async def test_cont_acq_controller_not_acquiring( cont_acq_detector: adcore.AreaDetector[adcore.ADBaseIO], ): set_mock_value(cont_acq_detector.driver.acquire, False) + await cont_acq_detector.stage() with pytest.raises(RuntimeError, match=ERROR_MESSAGE): await cont_acq_detector.trigger() @@ -71,6 +75,7 @@ async def test_cont_acq_controller_not_acquiring( async def test_cont_acq_controller_invalid_exposure_time( cont_acq_detector: adcore.AreaDetector[adcore.ADBaseIO], ): + await cont_acq_detector.stage() with pytest.raises( ValueError, match=re.escape( @@ -151,3 +156,75 @@ async def test_cont_acq_controller_success_with_process_plugin( call.cb.trigger_.put(True), ], ) + + +@pytest.fixture +async def cont_acq_hdf_detector( + static_path_provider: StaticPathProvider, +) -> adcore.AreaDetector[adcore.ADBaseIO]: + async with init_devices(mock=True): + det = adcore.ContAcqDetector( + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) + ) + set_mock_value(det.driver.image_mode, adcore.ADImageMode.CONTINUOUS) + set_mock_value(det.driver.acquire_time, 0.8) + set_mock_value(det.driver.acquire_period, 1.0) + set_mock_value(det.driver.acquire, True) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + set_mock_value(det.get_plugin("hdf", adcore.NDFileHDF5IO).file_path_exists, True) + return det + + +async def test_uc9_continuous_acquisition_step_then_fly_scan( + cont_acq_hdf_detector: adcore.AreaDetector[adcore.ADBaseIO], +): + """Use case 9: the driver stays acquiring, the buffer captures N per trigger. + + Streamable and bounded: stream assets for step and fly, the buffer sized by + the prepare that precedes each. + """ + det = cont_acq_hdf_detector + cb = det.get_plugin("cb", adcore.NDCircularBuffIO) + hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) + captured = 0 + + frames_per_capture = 0 + + def soft_trigger(value: bool) -> bool | None: + # Each soft trigger of the circular buffer writes this many frames, + # then the busy record resets + nonlocal captured + if value: + captured += frames_per_capture + set_mock_value(hdf.num_captured, captured) + return False + return None + + callback_on_mock_put(cb.trigger_, soft_trigger) + + # Step scan: two points of 2 frames each, one StreamResource + await det.stage() + frames_per_capture = 2 + await det.prepare(TriggerInfo(collections_per_event=2)) + await det.trigger() + await det.trigger() + assert await cb.post_count.get_value() == 2 + docs = [doc async for doc in det.collect_asset_docs()] # pyright: ignore[reportAttributeAccessIssue] + assert [name for name, _ in docs] == [ + "stream_resource", + "stream_datum", + ] + assert docs[1][1]["indices"] == {"start": 0, "stop": 2} + + # Fly scan: 3 events of 2 frames, captured in one go + frames_per_capture = 6 + await det.prepare(TriggerInfo(number_of_events=3, collections_per_event=2)) + assert await cb.post_count.get_value() == 6 + await det.kickoff() + await det.complete() + docs = [doc async for doc in det.collect_asset_docs()] # pyright: ignore[reportAttributeAccessIssue] + assert [d["indices"] for n, d in docs if n == "stream_datum"] == [ + {"start": 2, "stop": 5} + ] + await det.unstage() diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 9e940744b3..7679e29082 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -1,3 +1,4 @@ +import asyncio import os from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath from unittest.mock import ANY, call @@ -10,10 +11,16 @@ StaticFilenameProvider, StaticPathProvider, TriggerInfo, + get_mock, init_devices, set_mock_value, ) from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.epics.adcore import ( + ADHDFDataLogic, + ADMultipartDataLogic, + NDArrayDescription, +) from ophyd_async.testing import assert_has_calls @@ -24,7 +31,7 @@ async def hdf_det( async with init_devices(mock=True): detector = adsimdetector.SimDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), plugins={"stats": adcore.NDStatsIO("PREFIX:STATS:")}, ) set_mock_value(detector.driver.array_size_x, 1024) @@ -33,7 +40,75 @@ async def hdf_det( return detector +async def test_start_provider_opens_the_file_and_starts_capture( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, +): + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + ) + + assert await writer.capture.get_value() is False + + made = await logic.start_provider("det", 1, 0.1, 0) + assert made is not None + assert await writer.capture.get_value() is True + assert await writer.file_name.get_value() != "" + + +@pytest.mark.parametrize( + "always_enable,plugin_enabled,makes_provider", + [ + # The default serves, whatever the plugin was set to + (True, EnableDisable.DISABLE, True), + # Following the plugin, a disabled one writes nothing... + (False, EnableDisable.DISABLE, False), + # ...and an enabled one still writes + (False, EnableDisable.ENABLE, True), + ], +) +async def test_hdf_follows_the_plugin_when_not_enabling_it( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, + always_enable: bool, + plugin_enabled: EnableDisable, + makes_provider: bool, +): + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + set_mock_value(writer.enable_callbacks, plugin_enabled) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + always_enable=always_enable, + ) + + made = await logic.start_provider("det", 1, 0.1, 0) + assert (made is not None) is makes_provider + # Starting switches the plugin on either way, and sitting out writes nothing + assert await writer.capture.get_value() is makes_provider + assert await writer.enable_callbacks.get_value() is ( + EnableDisable.ENABLE if makes_provider else EnableDisable.DISABLE + ) + + async def test_hdf_writer_file_not_found(hdf_det: adcore.AreaDetector[adcore.ADBaseIO]): + await hdf_det.stage() with pytest.raises( FileNotFoundError, match=r"Path .* doesn't exist or not writable!" ): @@ -46,7 +121,7 @@ async def test_hdf_writer_passes_parent_name_to_path_provider(tmp_path: Path): ) async with init_devices(mock=True): det = adsimdetector.SimDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(pp), name="sim_detector" + "PREFIX:", adcore.ADDataFactory.hdf(pp), name="sim_detector" ) writer = det.get_plugin("hdf", adcore.NDPluginFileIO) @@ -65,6 +140,9 @@ async def test_prepare_hdf( ): writer = hdf_det.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) + await hdf_det.stage() + # Only the calls after stage() are of interest + get_mock(hdf_det).reset_mock() await hdf_det.prepare(TriggerInfo(number_of_events=3)) assert_has_calls( hdf_det, @@ -95,8 +173,8 @@ async def test_prepare_hdf( @pytest.mark.parametrize( "factory_cls,extension", [ - (adcore.ADWriterFactory.jpeg, ".jpg"), - (adcore.ADWriterFactory.tiff, ".tiff"), + (adcore.ADDataFactory.jpeg, ".jpg"), + (adcore.ADDataFactory.tiff, ".tiff"), ], ) async def test_prepare_multipart( @@ -107,11 +185,15 @@ async def test_prepare_multipart( name = factory_cls.__name__ writer = det.get_plugin(name, adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) + await det.stage() + # Only the calls after stage() are of interest + get_mock(det).reset_mock() await det.prepare(TriggerInfo(number_of_events=3)) assert_has_calls( writer, [ call.lazy_open.put(True), + call.enable_callbacks.put(EnableDisable.ENABLE), call.create_directory.put(0), call.file_path.put(f"{static_path_provider().directory_path}{os.sep}"), call.file_name.put("ophyd_async_tests"), @@ -125,12 +207,67 @@ async def test_prepare_multipart( ) +@pytest.mark.parametrize( + "flush_period,livetime,deadtime,exposures_per_collection,expected_frames_per_chunk", + [ + # 0.5s flush / 0.01s period -> 50 frames per chunk (#1309) + (0.5, 0.01, 0.0, 1, 50), + # 0.5s flush / 0.05s period -> 10 frames per chunk + (0.5, 0.04, 0.01, 1, 10), + # A sub-frame flush period floors to a single frame per chunk + (0.001, 0.05, 0.0, 1, 1), + # 5 exposures of 0.01s make each frame written, so 0.5s holds 10 + (0.5, 0.01, 0.0, 5, 10), + ], +) +async def test_hdf_chunk_sized_from_flush_period( + static_path_provider: StaticPathProvider, + flush_period: float, + livetime: float, + deadtime: float, + exposures_per_collection: int, + expected_frames_per_chunk: int, +): + """A TriggerInfo flush_period sizes the HDF chunk from the collection period.""" + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADDataFactory.hdf(static_path_provider), + # Averages the exposures of each collection into one frame + proc_suffix="PROC:", + ) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + set_mock_value(det.driver.data_type, adcore.ADBaseDataType.UINT16) + writer = det.get_plugin("hdf", adcore.NDPluginFileIO) + set_mock_value(writer.file_path_exists, True) + await det.stage() + await det.prepare( + TriggerInfo( + livetime=livetime, + deadtime=deadtime, + exposures_per_collection=exposures_per_collection, + number_of_events=3, + flush_period=flush_period, + ) + ) + assert await writer.num_frames_chunks.get_value() == expected_frames_per_chunk + await det.kickoff() + (sr, *_) = [doc async for doc in det.collect_asset_docs(3)] + await det.unstage() + assert sr[1]["parameters"]["chunk_shape"] == ( + expected_frames_per_chunk, + 768, + 1024, + ) + + @pytest.mark.parametrize( "factory_cls,is_hdf", [ - (adcore.ADWriterFactory.hdf, True), - (adcore.ADWriterFactory.jpeg, False), - (adcore.ADWriterFactory.tiff, False), + (adcore.ADDataFactory.hdf, True), + (adcore.ADDataFactory.jpeg, False), + (adcore.ADDataFactory.tiff, False), ], ) async def test_can_specify_different_uri_and_path( @@ -151,7 +288,9 @@ async def test_can_specify_different_uri_and_path( set_mock_value(writer.file_path_exists, True) await det.stage() await det.prepare(TriggerInfo()) + await det.kickoff() docs = [doc async for doc in det.collect_asset_docs(1)] + await det.unstage() # Make sure we set the write path to the directory_path attr from our path info assert await writer.file_path.get_value() == str(tmp_path) + os.sep @@ -185,9 +324,9 @@ async def test_can_specify_different_uri_and_path( @pytest.mark.parametrize( "factory_cls,is_hdf", [ - (adcore.ADWriterFactory.hdf, True), - (adcore.ADWriterFactory.jpeg, False), - (adcore.ADWriterFactory.tiff, False), + (adcore.ADDataFactory.hdf, True), + (adcore.ADDataFactory.jpeg, False), + (adcore.ADDataFactory.tiff, False), ], ) async def test_can_override_uri_with_different_path_semantics( @@ -208,7 +347,9 @@ async def test_can_override_uri_with_different_path_semantics( set_mock_value(writer.file_path_exists, True) await det.stage() await det.prepare(TriggerInfo()) + await det.kickoff() docs = [doc async for doc in det.collect_asset_docs(1)] + await det.unstage() assert await writer.file_path.get_value() == f"{write_path}{expected_separator}" assert len(docs) == 2 @@ -241,6 +382,7 @@ async def test_stats_describe_raises_error_with_dbr_native( """, ) + await hdf_det.stage() with pytest.raises( RuntimeError, match="NDAttribute mydetector-Temperature has dbrtype DBR_NATIVE," @@ -270,6 +412,7 @@ async def test_describe_different_color_modes( writer = hdf_det.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) set_mock_value(hdf_det.driver.color_mode, color_mode) + await hdf_det.stage() if shape is RuntimeError: # Expected to fail for this mode with pytest.raises( @@ -295,6 +438,7 @@ async def test_3d_dataset_shape(hdf_det: adcore.AreaDetector[adcore.ADBaseIO]): writer = hdf_det.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) set_mock_value(hdf_det.driver.array_size_z, 10) + await hdf_det.stage() await hdf_det.prepare(TriggerInfo()) describe = await hdf_det.describe() assert describe["detector"] == { @@ -304,3 +448,56 @@ async def test_3d_dataset_shape(hdf_det: adcore.AreaDetector[adcore.ADBaseIO]): "shape": [1, 10, 768, 1024], "source": ANY, } + + +async def _start_waits_for_a_zero_counter(logic, counter, previous: int = 5) -> None: + set_mock_value(counter, previous) + starting = asyncio.create_task(logic.start_provider("det", 1, 0.1, 0)) + await asyncio.sleep(0.1) + # The IOC has not yet reset NumCaptured, so the detector's baseline would + # be the previous scan's count + assert not starting.done() + set_mock_value(counter, 0) + assert await asyncio.wait_for(starting, timeout=1) is not None + + +async def test_hdf_start_waits_for_num_captured_to_read_zero( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, +): + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + ) + await _start_waits_for_a_zero_counter(logic, writer.num_captured) + + +async def test_multipart_start_waits_for_num_captured_to_read_zero( + static_path_provider: StaticPathProvider, +): + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", adcore.ADDataFactory.tiff(static_path_provider) + ) + writer = det.get_plugin("tiff", adcore.NDPluginFileIO) + set_mock_value(writer.file_path_exists, True) + logic = ADMultipartDataLogic( + array_description=NDArrayDescription( + shape_signals=[det.driver.array_size_y, det.driver.array_size_x], + data_type_signal=det.driver.data_type, + color_mode_signal=det.driver.color_mode, + ), + path_provider=static_path_provider, + writer=writer, + extension=".tiff", + mimetype="multipart/related;type=image/tiff", + ) + await _start_waits_for_a_zero_counter(logic, writer.num_captured) diff --git a/tests/unit_tests/epics/adcore/test_detector.py b/tests/unit_tests/epics/adcore/test_detector.py index 5e2bbd0c1a..123c1ceff4 100644 --- a/tests/unit_tests/epics/adcore/test_detector.py +++ b/tests/unit_tests/epics/adcore/test_detector.py @@ -13,9 +13,9 @@ @pytest.mark.parametrize( "factory", [ - adcore.ADWriterFactory.hdf, - adcore.ADWriterFactory.jpeg, - adcore.ADWriterFactory.tiff, + adcore.ADDataFactory.hdf, + adcore.ADDataFactory.jpeg, + adcore.ADDataFactory.tiff, ], ) @pytest.mark.parametrize("hinted", [True, False]) @@ -41,9 +41,9 @@ def test_writer_image_hints(static_path_provider: StaticPathProvider, factory, h @pytest.mark.parametrize( "factory", [ - adcore.ADWriterFactory.hdf, - adcore.ADWriterFactory.jpeg, - adcore.ADWriterFactory.tiff, + adcore.ADDataFactory.hdf, + adcore.ADDataFactory.jpeg, + adcore.ADDataFactory.tiff, ], ) def test_writer_image_hints_enabled_by_default( @@ -74,7 +74,7 @@ def test_area_detector_requires_prefix_when_factories_given( adcore.AreaDetector( driver, None, - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), name="det", ) @@ -91,12 +91,44 @@ def test_area_detector_rejects_duplicate_writer_names( adcore.AreaDetector( driver, "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider, writer_name="hdf"), - adcore.ADWriterFactory.hdf(static_path_provider, writer_name="hdf"), + adcore.ADDataFactory.hdf(static_path_provider, writer_name="hdf"), + adcore.ADDataFactory.hdf(static_path_provider, writer_name="hdf"), name="det", ) +def test_plugin_named_like_a_factory_attribute_raises( + static_path_provider: StaticPathProvider, +): + driver = adcore.ADBaseIO("PREFIX:DRV:") + + with pytest.raises(ValueError, match=r"attribute 'stats'"): + adcore.AreaDetector( + driver, + "PREFIX:", + adcore.ADDataFactory.hdf(static_path_provider, writer_name="stats"), + plugins={"stats": adcore.NDStatsIO("PREFIX:STATS:")}, + name="det", + ) + + +def test_plugin_named_like_the_driver_raises(): + driver = adcore.ADBaseIO("PREFIX:DRV:") + + with pytest.raises(ValueError, match=r"attribute 'driver'"): + adcore.AreaDetector( + driver, plugins={"driver": adcore.NDStatsIO("PREFIX:S:")}, name="det" + ) + + +def test_deprecated_writer_factory_still_builds_the_new_factories( + static_path_provider: StaticPathProvider, +): + with pytest.warns(DeprecationWarning, match="ADWriterFactory"): + hdf = adcore.ADWriterFactory.hdf(static_path_provider) + assert isinstance(hdf, adcore.ADDataFactory) + + # --------------------------------------------------------------------------- # AreaDetector.get_plugin guards # --------------------------------------------------------------------------- @@ -151,14 +183,15 @@ async def test_get_ndarray_resource_info_undefined_datatype( det = adcore.AreaDetector( adcore.ADBaseIO("PREFIX:DRV:"), "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), name="det", ) set_mock_value(det.driver.data_type, adcore.ADBaseDataType.UNDEFINED) set_mock_value(det.get_plugin("hdf", adcore.NDFileHDF5IO).file_path_exists, True) + await det.stage() with pytest.raises( ValueError, - match=r"^mock\+ca://PREFIX:DRV:DataType_RBV is blank, this is not supported$", + match=r"^mock\+ca://PREFIX:DRV:DataType_RBV is blank, this is not supported", ): await det.prepare(TriggerInfo()) @@ -170,13 +203,14 @@ async def test_get_ndarray_resource_info_unsupported_color_mode( det = adcore.AreaDetector( adcore.ADBaseIO("PREFIX:DRV:"), "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), name="det", ) set_mock_value(det.driver.color_mode, adcore.ADBaseColorMode.BAYER) set_mock_value(det.get_plugin("hdf", adcore.NDFileHDF5IO).file_path_exists, True) + await det.stage() with pytest.raises( RuntimeError, - match=r"^Unsupported ColorMode Bayer! Only Mono and RGB1 are supported\.$", + match=r"^Unsupported ColorMode Bayer! Only Mono and RGB1 are supported\.", ): await det.prepare(TriggerInfo()) diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index a7ffbc7cee..d9d1e970b0 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -2,14 +2,16 @@ import pytest -from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.core import ( + EnableDisable, StaticPathProvider, + TransitionError, TriggerInfo, callback_on_mock_put, init_devices, set_mock_value, ) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.epics import adaravis, adcore, adsimdetector from ophyd_async.epics.core import epics_signal_r from ophyd_async.testing import assert_reading @@ -22,7 +24,7 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( async with init_devices(mock=True): det = adsimdetector.SimDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), plugins={"stats": stat}, ) temp = epics_signal_r(float, "SAMP:TEMP:RBV") @@ -71,8 +73,7 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( assert {"det-driver-acquire_period", "det-driver-acquire_time"} <= config.keys() assert config["det-driver-acquire_period"]["value"] == 0.1 assert config["det-driver-acquire_time"]["value"] == 0.05 - # Check we need to call prepare or trigger before we can describe - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(TransitionError, match="describe not allowed from STAGED"): await det.describe() # Need to tell it the path is valid set_mock_value(writer.file_path_exists, True) @@ -188,14 +189,18 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( "det-stats-max_value", } <= readings.keys() docs = [doc async for doc in det.collect_asset_docs()] - assert docs == [ + # Each prepare starts new resources in the same file, the indices carrying on + new_sr, sd = docs[:3], docs[3:] + assert new_sr == [(name, {**doc, "uid": ANY}) for name, doc in sr] + assert {doc["uid"] for _, doc in new_sr}.isdisjoint(doc["uid"] for _, doc in sr) + assert sd == [ ( "stream_datum", { "descriptor": "", "indices": {"start": 1, "stop": 2}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": sr[i][1]["uid"], + "stream_resource": new_sr[i][1]["uid"], "uid": ANY, }, ) @@ -213,7 +218,7 @@ async def test_step_scan_tiff_detector( ): async with init_devices(mock=True): det = adsimdetector.SimDetector( - "PREFIX:", adcore.ADWriterFactory.tiff(static_path_provider) + "PREFIX:", adcore.ADDataFactory.tiff(static_path_provider) ) writer = det.get_plugin("tiff", adcore.NDPluginFileIO) @@ -279,7 +284,7 @@ async def test_step_scan_tiff_detector( async def test_flyscan_aravis_detector(static_path_provider: StaticPathProvider): async with init_devices(mock=True): det = adaravis.AravisDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = det.get_plugin("hdf", adcore.NDPluginFileIO) @@ -341,6 +346,7 @@ async def test_flyscan_aravis_detector(static_path_provider: StaticPathProvider) # Make 2 and 1/3 events and check they are produced set_mock_value(writer.num_captured, 7) docs = [doc async for doc in det.collect_asset_docs()] + sr = docs[:1] assert docs == [ ( "stream_resource", @@ -386,24 +392,27 @@ async def test_flyscan_aravis_detector(static_path_provider: StaticPathProvider) # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, - match="Kickoff requested 15:30, but detector was only prepared up to 15", + match="cannot go to FLYING from FLOWN", ): await det.kickoff() # But preparing again is ok await det.prepare(TriggerInfo(collections_per_event=3, number_of_events=3)) await det.kickoff() - # Completing adds indexes to the same resource + # Completing starts a new resource in the same file, the indices carrying on status = det.complete() set_mock_value(writer.num_captured, 24) docs = [doc async for doc in det.collect_asset_docs()] - assert docs == [ + new_sr = docs[:1] + assert new_sr == [(name, {**doc, "uid": ANY}) for name, doc in sr] + assert new_sr[0][1]["uid"] != sr[0][1]["uid"] + assert docs[1:] == [ ( "stream_datum", { "descriptor": "", "indices": {"start": 5, "stop": 8}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": ANY, + "stream_resource": new_sr[0][1]["uid"], "uid": ANY, }, ), @@ -420,7 +429,7 @@ async def test_2_rois_with_hdf(tmp_path): async with init_devices(mock=True): det = adaravis.AravisDetector( "PREFIX:", - adcore.ADWriterFactory.hdf( + adcore.ADDataFactory.hdf( path_provider, writer_suffix="HDF1:", writer_name="hdf1", @@ -431,7 +440,7 @@ async def test_2_rois_with_hdf(tmp_path): color_mode_signal=driver.color_mode, ), ), - adcore.ADWriterFactory.hdf( + adcore.ADDataFactory.hdf( path_provider, writer_suffix="HDF2:", writer_name="hdf2", @@ -590,11 +599,11 @@ async def test_step_scan_keep_numimages( async with init_devices(mock=True): det = adsimdetector.SimDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), plugins={"stats": stat}, ) - # Plugins are not registered automatically; opt this one in - det.set_readable_format(stat, Format.CHILD) + # Plugins are not registered automatically, and the stats readables give one + # value per event so cannot be used with 42 collections per event set_mock_value(det.driver.acquire_period, 0.1) set_mock_value(det.driver.acquire_time, 0.05) @@ -607,8 +616,8 @@ async def test_step_scan_keep_numimages( assert {"det-driver-acquire_period", "det-driver-acquire_time"} <= config.keys() assert config["det-driver-acquire_period"]["value"] == 0.1 assert config["det-driver-acquire_time"]["value"] == 0.05 - # Check we need to call prepare or trigger before we can describe - with pytest.raises(RuntimeError, match="Prepare not run"): + await det.stage() + with pytest.raises(TransitionError, match="describe not allowed from STAGED"): await det.describe() # Need to tell it the path is valid set_mock_value(writer.file_path_exists, True) @@ -630,8 +639,7 @@ async def test_step_scan_keep_numimages( "source": uri, } assert "det" in det.hints["fields"] - reading = await det.read() - assert "det-stats-total" in reading + assert await det.read() == {} sr, sd = [doc async for doc in det.collect_asset_docs()] assert sr == ( "stream_resource", @@ -658,26 +666,37 @@ async def test_step_scan_keep_numimages( }, ) - # Check we can prepare and change the exposure + # Check we can prepare and change the exposure. Changing the exposure changes + # the frame period, which (per ADR 0023) determines the HDF chunk shape, so a + # new StreamResource is opened rather than the old file being reused. Here the + # chunk shape is unchanged because this writer leaves flush_period unset and so + # reads the chunk size back from the IOC, but the period change still starts a + # fresh file. + # Starting a capture resets the IOC's NumCaptured, which start() waits for + callback_on_mock_put( + writer.capture, lambda v: set_mock_value(writer.num_captured, 0) if v else None + ) await det.prepare( TriggerInfo(collections_per_event=42, livetime=0.01, exposure_timeout=0.1) ) callback_on_mock_put( - det.driver.acquire, lambda v: set_mock_value(writer.num_captured, 84) + det.driver.acquire, lambda v: set_mock_value(writer.num_captured, 42) ) assert await det.driver.acquire_time.get_value() == 0.01 await det.trigger() readings = await det.read() - assert "det-stats-total" in readings - docs = [doc async for doc in det.collect_asset_docs()] - (doc,) = docs - assert doc == ( + assert "det-stats-total" not in readings + new_sr, new_sd = [doc async for doc in det.collect_asset_docs()] + assert new_sr[0] == "stream_resource" + assert new_sr[1]["uid"] != sr[1]["uid"] + assert new_sr[1]["parameters"]["chunk_shape"] == (1, 768, 1024) + assert new_sd == ( "stream_datum", { "descriptor": "", - "indices": {"start": 1, "stop": 2}, + "indices": {"start": 0, "stop": 1}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": sr[1]["uid"], + "stream_resource": new_sr[1]["uid"], "uid": ANY, }, ) @@ -697,7 +716,7 @@ async def test_kinetix_step_scan_with_averaging_filter( det = adkinetix.KinetixDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), proc_suffix="PROC:", ) @@ -788,3 +807,78 @@ async def test_kinetix_step_scan_with_averaging_filter( }, ), ] + + +# Ruling 1: a once-per-event readable is shadowed by a same-named higher tier at +# any collections_per_event, and read once per event when nothing shadows it + + +@pytest.mark.parametrize("cpe", [1, 2]) +async def test_readable_with_no_data_logic_is_read_once_per_event(cpe: int): + stat = adcore.NDStatsIO("PREFIX:STATS:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}, name="det") + det.set_readable_format(stat.total, Format.UNCACHED_SIGNAL) + await det.stage() + await det.prepare(TriggerInfo(number_of_events=1, collections_per_event=cpe)) + + assert (await det.describe())["det-stats-total"]["shape"] == [] + + +@pytest.mark.parametrize("on_at_stage", [True, False]) +async def test_hdf_by_settings_follows_its_plugin_at_stage_for_the_whole_stage( + tmp_path, on_at_stage: bool +): + """Switching a writer's plugin mid-stage changes nothing until restaged.""" + path_provider = StaticPathProvider( + lambda datakey_name=None: str(datakey_name), tmp_path + ) + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + *( + adcore.ADDataFactory.hdf( + path_provider, + writer_suffix=f"HDF{i}:", + writer_name=f"hdf{i}", + datakey_suffix=f"-roi{i}", + always_enable=False, + ) + for i in (1, 2) + ), + ) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + hdf1, hdf2 = (det.get_plugin(f"hdf{i}", adcore.NDFileHDF5IO) for i in (1, 2)) + for hdf in (hdf1, hdf2): + set_mock_value(hdf.file_path_exists, True) + set_mock_value(hdf1.enable_callbacks, EnableDisable.ENABLE) + frames = 0 + + def acquire(value: bool) -> None: + nonlocal frames + if value: + frames += 3 + for hdf in (hdf1, hdf2): + set_mock_value(hdf.num_captured, frames) + + callback_on_mock_put(det.driver.acquire, acquire) + served = {"det-roi1", "det-roi2"} if on_at_stage else {"det-roi1"} + info = TriggerInfo(number_of_events=3) + + def enable_hdf2(enabled: bool) -> None: + value = EnableDisable.ENABLE if enabled else EnableDisable.DISABLE + set_mock_value(hdf2.enable_callbacks, value) + + enable_hdf2(on_at_stage) + await det.stage() + for flip in (False, True): + if flip: + enable_hdf2(not on_at_stage) + await det.prepare(info) + await det.kickoff() + await det.complete() + assert set(await det.describe_collect()) == served + # Events since prepare() + assert await det.get_index() == info.number_of_events + await det.unstage() diff --git a/tests/unit_tests/epics/test_adandor.py b/tests/unit_tests/epics/test_adandor.py index 36784c8289..3c5d83eeeb 100644 --- a/tests/unit_tests/epics/test_adandor.py +++ b/tests/unit_tests/epics/test_adandor.py @@ -7,6 +7,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -20,7 +21,7 @@ async def test_adandor( ) -> adandor.AndorDetector: async with init_devices(mock=True): detector = adandor.AndorDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) @@ -46,6 +47,9 @@ async def test_deadtime( async def test_prepare_external_edge( test_adandor: adandor.AndorDetector, ): + await test_adandor.stage() + # Only the calls after stage() are of interest + get_mock(test_adandor).reset_mock() await test_adandor.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, @@ -65,20 +69,6 @@ async def test_prepare_external_edge( ) -async def test_prepare_forever( - test_adandor: adandor.AndorDetector, -): - await test_adandor.prepare(TriggerInfo(number_of_events=0)) - assert_has_calls( - test_adandor.driver, - [ - call.trigger_mode.put(adandor.Andor2TriggerMode.INTERNAL), - call.image_mode.put(adcore.ADImageMode.MULTIPLE), - call.num_images.put(999_999), - ], - ) - - @pytest.mark.parametrize("num_images,expected_first_dim", [(3, 3), (0, 1)]) async def test_trigger_uses_num_images( test_adandor: adandor.AndorDetector, diff --git a/tests/unit_tests/epics/test_adaravis.py b/tests/unit_tests/epics/test_adaravis.py index 551e76cb6a..83c9210f9c 100644 --- a/tests/unit_tests/epics/test_adaravis.py +++ b/tests/unit_tests/epics/test_adaravis.py @@ -8,6 +8,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -21,7 +22,7 @@ async def test_adaravis( ) -> adaravis.AravisDetector: async with init_devices(mock=True): detector = adaravis.AravisDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) @@ -51,9 +52,22 @@ async def test_deadtime( assert deadtime == actual_deadtime +async def test_deadtime_inputs_survive_dropping_them_from_configuration( + test_adaravis: adaravis.AravisDetector, +): + set_mock_value(test_adaravis.driver.model, "Mako G-125") + test_adaravis.driver.set_readable_format(test_adaravis.driver.model, None) + test_adaravis.set_readable_format(test_adaravis.driver, None) + _, deadtime = await test_adaravis.get_trigger_deadtime() + assert deadtime == 70e-6 + + async def test_prepare_external_edge( test_adaravis: adaravis.AravisDetector, ): + await test_adaravis.stage() + # Only the calls after stage() are of interest + get_mock(test_adaravis).reset_mock() await test_adaravis.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, @@ -77,6 +91,9 @@ async def test_prepare_external_edge( async def test_prepare_internal( test_adaravis: adaravis.AravisDetector, ): + await test_adaravis.stage() + # Only the calls after stage() are of interest + get_mock(test_adaravis).reset_mock() await test_adaravis.prepare(TriggerInfo(number_of_events=11)) assert_has_calls( test_adaravis.driver, diff --git a/tests/unit_tests/epics/test_adkinetix.py b/tests/unit_tests/epics/test_adkinetix.py index 2260f33715..a502148dea 100644 --- a/tests/unit_tests/epics/test_adkinetix.py +++ b/tests/unit_tests/epics/test_adkinetix.py @@ -7,6 +7,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -20,7 +21,7 @@ async def test_adkinetix( ) -> adkinetix.KinetixDetector: async with init_devices(mock=True): detector = adkinetix.KinetixDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) @@ -50,6 +51,9 @@ async def test_deadtime( async def test_prepare_external_edge( test_adkinetix: adkinetix.KinetixDetector, ): + await test_adkinetix.stage() + # Only the calls after stage() are of interest + get_mock(test_adkinetix).reset_mock() await test_adkinetix.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, @@ -72,6 +76,9 @@ async def test_prepare_external_edge( async def test_prepare_external_level( test_adkinetix: adkinetix.KinetixDetector, ): + await test_adkinetix.stage() + # Only the calls after stage() are of interest + get_mock(test_adkinetix).reset_mock() await test_adkinetix.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_LEVEL, @@ -92,6 +99,9 @@ async def test_prepare_external_level( async def test_prepare_internal( test_adkinetix: adkinetix.KinetixDetector, ): + await test_adkinetix.stage() + # Only the calls after stage() are of interest + get_mock(test_adkinetix).reset_mock() await test_adkinetix.prepare(TriggerInfo(number_of_events=2, livetime=0.3)) assert_has_calls( test_adkinetix.driver, diff --git a/tests/unit_tests/epics/test_admerlin.py b/tests/unit_tests/epics/test_admerlin.py index 34d1286398..cc77d2968d 100644 --- a/tests/unit_tests/epics/test_admerlin.py +++ b/tests/unit_tests/epics/test_admerlin.py @@ -7,6 +7,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -20,7 +21,7 @@ async def test_admerlin( ) -> admerlin.MerlinDetector: async with init_devices(mock=True): detector = admerlin.MerlinDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) @@ -62,6 +63,9 @@ async def test_data_type_inferred_from_counter_depth( async def test_prepare_external_edge( test_admerlin: admerlin.MerlinDetector, ): + await test_admerlin.stage() + # Only the calls after stage() are of interest + get_mock(test_admerlin).reset_mock() await test_admerlin.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, diff --git a/tests/unit_tests/epics/test_adpilatus.py b/tests/unit_tests/epics/test_adpilatus.py index 70641c51db..28d2e62c97 100644 --- a/tests/unit_tests/epics/test_adpilatus.py +++ b/tests/unit_tests/epics/test_adpilatus.py @@ -8,6 +8,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -21,7 +22,7 @@ async def test_adpilatus( ) -> adpilatus.PilatusDetector: async with init_devices(mock=True): detector = adpilatus.PilatusDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) set_mock_value(detector.driver.armed, True) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) @@ -66,7 +67,7 @@ async def test_deadtime(readout_time: adpilatus.PilatusReadoutTime, tmp_path): async with init_devices(mock=True): pilatus = adpilatus.PilatusDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(path_provider), + adcore.ADDataFactory.hdf(path_provider), readout_time=readout_time, ) trigger_modes, deadtime = await pilatus.get_trigger_deadtime() @@ -87,6 +88,7 @@ async def test_times_out_if_not_armed( assert isinstance(acquire_logic, adcore.ADAcquireLogic) acquire_logic.timeout = 0.01 + await test_adpilatus.stage() with pytest.raises(TimeoutError): await test_adpilatus.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) @@ -94,6 +96,9 @@ async def test_times_out_if_not_armed( async def test_prepare_external_edge( test_adpilatus: adpilatus.PilatusDetector, ): + await test_adpilatus.stage() + # Only the calls after stage() are of interest + get_mock(test_adpilatus).reset_mock() await test_adpilatus.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, @@ -119,12 +124,15 @@ async def test_prepare_external_edge_with_mult_trigger_mode( async with init_devices(mock=True): detector = adpilatus.PilatusDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider), edge_trigger_mode=adpilatus.PilatusTriggerMode.MULT_TRIGGER, ) set_mock_value(detector.driver.armed, True) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) + await detector.stage() + # Only the calls after stage() are of interest + get_mock(detector).reset_mock() await detector.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, @@ -165,7 +173,7 @@ def test_edge_trigger_mode_rejects_non_edge_modes( ): adpilatus.PilatusDetector( "PREFIX:", - adcore.ADWriterFactory.hdf(path_provider), + adcore.ADDataFactory.hdf(path_provider), edge_trigger_mode=invalid_mode, ) @@ -173,6 +181,9 @@ def test_edge_trigger_mode_rejects_non_edge_modes( async def test_prepare_external_level( test_adpilatus: adpilatus.PilatusDetector, ): + await test_adpilatus.stage() + # Only the calls after stage() are of interest + get_mock(test_adpilatus).reset_mock() await test_adpilatus.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_LEVEL, @@ -190,20 +201,6 @@ async def test_prepare_external_level( ) -async def test_prepare_forever( - test_adpilatus: adpilatus.PilatusDetector, -): - await test_adpilatus.prepare(TriggerInfo(number_of_events=0)) - assert_has_calls( - test_adpilatus.driver, - [ - call.trigger_mode.put(adpilatus.PilatusTriggerMode.INTERNAL), - call.image_mode.put(adcore.ADImageMode.MULTIPLE), - call.num_images.put(999_999), - ], - ) - - @pytest.mark.parametrize("num_images,expected_first_dim", [(7, 7), (0, 1)]) async def test_trigger_uses_num_images( test_adpilatus: adpilatus.PilatusDetector, diff --git a/tests/unit_tests/epics/test_adsimdetector.py b/tests/unit_tests/epics/test_adsimdetector.py index 4c6ff7cc48..fb8b59fd96 100644 --- a/tests/unit_tests/epics/test_adsimdetector.py +++ b/tests/unit_tests/epics/test_adsimdetector.py @@ -6,6 +6,7 @@ DetectorTrigger, StaticPathProvider, TriggerInfo, + get_mock, init_devices, set_mock_value, ) @@ -19,7 +20,7 @@ async def test_adsimdetector( ) -> adsimdetector.SimDetector: async with init_devices(mock=True): detector = adsimdetector.SimDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) @@ -43,6 +44,9 @@ async def test_deadtime( async def test_prepare_internal( test_adsimdetector: adsimdetector.SimDetector, ): + await test_adsimdetector.stage() + # Only the calls after stage() are of interest + get_mock(test_adsimdetector).reset_mock() await test_adsimdetector.prepare(TriggerInfo(number_of_events=11)) assert_has_calls( test_adsimdetector.driver, diff --git a/tests/unit_tests/epics/test_advimba.py b/tests/unit_tests/epics/test_advimba.py index 610579365b..e44ed1b435 100644 --- a/tests/unit_tests/epics/test_advimba.py +++ b/tests/unit_tests/epics/test_advimba.py @@ -8,6 +8,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -21,7 +22,7 @@ async def test_advimba( ) -> advimba.VimbaDetector: async with init_devices(mock=True): detector = advimba.VimbaDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) writer = detector.get_plugin("hdf", adcore.NDPluginFileIO) set_mock_value(writer.file_path_exists, True) @@ -58,6 +59,9 @@ async def test_deadtime( async def test_prepare_external_level( test_advimba: advimba.VimbaDetector, ): + await test_advimba.stage() + # Only the calls after stage() are of interest + get_mock(test_advimba).reset_mock() await test_advimba.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_LEVEL, @@ -80,6 +84,9 @@ async def test_prepare_external_level( async def test_prepare_external_edge( test_advimba: advimba.VimbaDetector, ): + await test_advimba.stage() + # Only the calls after stage() are of interest + get_mock(test_advimba).reset_mock() await test_advimba.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_EDGE, @@ -104,6 +111,9 @@ async def test_prepare_external_edge( async def test_prepare_internal( test_advimba: advimba.VimbaDetector, ): + await test_advimba.stage() + # Only the calls after stage() are of interest + get_mock(test_advimba).reset_mock() await test_advimba.prepare(TriggerInfo(number_of_events=11)) assert_has_calls( test_advimba.driver, diff --git a/tests/unit_tests/epics/test_default_trigger_info.py b/tests/unit_tests/epics/test_default_trigger_info.py index 8904c99918..10b09c814d 100644 --- a/tests/unit_tests/epics/test_default_trigger_info.py +++ b/tests/unit_tests/epics/test_default_trigger_info.py @@ -50,7 +50,7 @@ async def _make_detector( async with init_devices(mock=True): det = detector_cls( "PREFIX:", - adcore.ADWriterFactory.hdf(path_provider), + adcore.ADDataFactory.hdf(path_provider), proc_suffix=proc_suffix, ) set_mock_value(det.get_plugin("hdf", adcore.NDFileHDF5IO).file_path_exists, True) diff --git a/tests/unit_tests/fastcs/eiger/test_eiger_detector.py b/tests/unit_tests/fastcs/eiger/test_eiger_detector.py index 2570f72f28..6eb3914db1 100644 --- a/tests/unit_tests/fastcs/eiger/test_eiger_detector.py +++ b/tests/unit_tests/fastcs/eiger/test_eiger_detector.py @@ -7,6 +7,7 @@ StaticFilenameProvider, StaticPathProvider, TriggerInfo, + get_mock, init_devices, set_mock_value, ) @@ -29,6 +30,9 @@ def detector(RE, tmp_path): async def test_prepare_internal_calls_correct_parameters( detector: EigerDetector, tmp_path ): + await detector.stage() + # Only the calls after stage() are of interest + get_mock(detector).reset_mock() await detector.prepare( TriggerInfo( number_of_events=10, diff --git a/tests/unit_tests/fastcs/jungfrau/test_controller.py b/tests/unit_tests/fastcs/jungfrau/test_controller.py index 2505ea80ad..39da8f837e 100644 --- a/tests/unit_tests/fastcs/jungfrau/test_controller.py +++ b/tests/unit_tests/fastcs/jungfrau/test_controller.py @@ -41,7 +41,7 @@ def jungfrau(RE: RunEngine, tmp_path): [ ( TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL), - "Trigger type DetectorTrigger.EXTERNAL_LEVEL not supported by 'jungfrau', " + "Trigger type DetectorTrigger.EXTERNAL_LEVEL not supported, " "supported types are: [EXTERNAL_EDGE, INTERNAL]", ), ( @@ -54,6 +54,7 @@ def jungfrau(RE: RunEngine, tmp_path): async def test_prepare_val_error_on_bad_trigger_info( trigger_info: TriggerInfo, match: str, jungfrau: JungfrauDetector ): + await jungfrau.stage() with pytest.raises( ValueError, match=re.escape(match), @@ -63,6 +64,7 @@ async def test_prepare_val_error_on_bad_trigger_info( async def test_prepare_warn_on_small_exposure(jungfrau: JungfrauDetector, caplog): bad_trigger_info = TriggerInfo(livetime=1e-6) + await jungfrau.stage() with caplog.at_level(logging.WARNING): await jungfrau.prepare(bad_trigger_info) assert "Exposure time shorter than 2μs is not recommended" in caplog.messages @@ -70,6 +72,7 @@ async def test_prepare_warn_on_small_exposure(jungfrau: JungfrauDetector, caplog async def test_arm(jungfrau: JungfrauDetector): mock = get_mock(jungfrau.detector) + await jungfrau.stage() await jungfrau.prepare(TriggerInfo()) mock.reset_mock() callback_on_mock_execute( @@ -81,6 +84,8 @@ async def test_arm(jungfrau: JungfrauDetector): async def test_disarm(jungfrau: JungfrauDetector): + await jungfrau.stage() + get_mock(jungfrau).reset_mock() await jungfrau.unstage() assert_has_calls( jungfrau.detector, @@ -103,6 +108,9 @@ async def test_signals_set_in_pedestal_mode(jungfrau: JungfrauDetector): ) mock = get_mock(jungfrau.detector) mock.reset_mock() + await jungfrau.stage() + # Only the calls after stage() are of interest + get_mock(jungfrau).reset_mock() await jungfrau.prepare(good_trigger_info) assert_has_calls( jungfrau.detector, @@ -121,6 +129,9 @@ async def test_signals_set_in_standard_internal_mode(jungfrau: JungfrauDetector) collections_per_event=10, trigger=DetectorTrigger.INTERNAL, ) + await jungfrau.stage() + # Only the calls after stage() are of interest + get_mock(jungfrau).reset_mock() await jungfrau.prepare(good_trigger_info) assert_has_calls( jungfrau.detector, @@ -140,6 +151,9 @@ async def test_signals_set_in_standard_external_mode(jungfrau: JungfrauDetector) trigger=DetectorTrigger.EXTERNAL_EDGE, deadtime=1, ) + await jungfrau.stage() + # Only the calls after stage() are of interest + get_mock(jungfrau).reset_mock() await jungfrau.prepare(good_trigger_info) assert_has_calls( jungfrau.detector, @@ -155,6 +169,7 @@ async def test_signals_set_in_standard_external_mode(jungfrau: JungfrauDetector) async def test_error_in_pedestal_and_external_modes(jungfrau: JungfrauDetector): await jungfrau.acquisition_type.set(AcquisitionType.PEDESTAL) + await jungfrau.stage() with pytest.raises( ValueError, match="Jungfrau must be triggered internally while in pedestal mode.", @@ -176,6 +191,7 @@ async def test_prepare_val_error_if_pedestal_mode_and_odd_number_of_events( trigger_info = TriggerInfo(livetime=1e-3, deadtime=1, collections_per_event=15) + await jungfrau.stage() with pytest.raises( ValueError, match=re.escape( diff --git a/tests/unit_tests/fastcs/odin/test_odin_io.py b/tests/unit_tests/fastcs/odin/test_odin_io.py index aba2db062b..bc4db9c0bc 100644 --- a/tests/unit_tests/fastcs/odin/test_odin_io.py +++ b/tests/unit_tests/fastcs/odin/test_odin_io.py @@ -1,4 +1,5 @@ import asyncio +from functools import cached_property from pathlib import Path from unittest.mock import call @@ -6,11 +7,13 @@ from bluesky import RunEngine from ophyd_async.core import ( + DetectorLogic, StandardDetector, StaticFilenameProvider, StaticPathProvider, TriggerInfo, callback_on_mock_execute, + get_mock, init_devices, set_mock_value, soft_signal_rw, @@ -27,11 +30,15 @@ def __init__(self, tmp_path: Path, name="", connector=None): path_provider = StaticPathProvider(StaticFilenameProvider("filename"), tmp_path) self.odin = OdinIO(connector=fastcs_connector("PREFIX:")) self.bit_depth = soft_signal_rw(int, BIT_DEPTH) - self.add_detector_logics( - OdinDataLogic(path_provider, self.odin, self.bit_depth) - ) + self._path_provider = path_provider super().__init__(name, connector) + @cached_property + def logic(self) -> DetectorLogic: + return DetectorLogic( + data=[OdinDataLogic(self._path_provider, self.odin, self.bit_depth)] + ) + @pytest.fixture def odin_det(RE: RunEngine, tmp_path) -> OdinDet: @@ -44,6 +51,7 @@ async def test_describe_gives_detector_shape(odin_det: OdinDet, tmp_path): set_mock_value(odin_det.odin.writing, True) set_mock_value(odin_det.odin.fp.data_dims_1, 1024) set_mock_value(odin_det.odin.fp.data_dims_0, 768) + await odin_det.stage() await odin_det.prepare(TriggerInfo()) description = await odin_det.describe() assert description == { @@ -62,6 +70,8 @@ async def test_describe_gives_detector_shape(odin_det: OdinDet, tmp_path): async def test_when_closed_then_data_capture_turned_off(odin_det: OdinDet): + await odin_det.stage() + get_mock(odin_det).reset_mock() await odin_det.unstage() assert_has_calls( odin_det, @@ -78,6 +88,8 @@ async def test_wait_for_active_and_file_names_before_capture_then_wait_for_writi odin: OdinIO = odin_det.odin ev = asyncio.Event() callback_on_mock_execute(odin_det.odin.fp.start_writing, lambda: ev.set()) + await odin_det.stage() + get_mock(odin_det).reset_mock() # Start it preparing status = odin_det.prepare(TriggerInfo(number_of_events=15)) # Wait for start_writing to be called @@ -110,5 +122,6 @@ async def test_wait_for_active_and_file_names_before_capture_then_wait_for_writi @pytest.mark.timeout(15) async def test_hinted_fields(odin_det: OdinDet): set_mock_value(odin_det.odin.writing, True) + await odin_det.stage() await odin_det.prepare(TriggerInfo()) assert odin_det.hints == {"fields": ["det"]} diff --git a/tests/unit_tests/fastcs/panda/test_hdf_panda.py b/tests/unit_tests/fastcs/panda/test_hdf_panda.py index 075b314219..96ec681dfb 100644 --- a/tests/unit_tests/fastcs/panda/test_hdf_panda.py +++ b/tests/unit_tests/fastcs/panda/test_hdf_panda.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import logging from unittest.mock import ANY, call @@ -11,6 +12,7 @@ StaticPathProvider, TriggerInfo, callback_on_mock_put, + get_mock, init_devices, set_mock_value, ) @@ -20,7 +22,11 @@ PandaCaptureMode, PandaHdf5DatasetType, ) -from ophyd_async.testing import assert_configuration, assert_has_calls +from ophyd_async.testing import ( + assert_configuration, + assert_has_calls, + wait_for_pending_wakeups, +) PANDA_DETECTOR_NAME = "mock_panda" @@ -78,6 +84,7 @@ async def test_open_returns_correct_descriptors_and_resources( hdf_panda: HDFPanda, table: DatasetTable, caplog, tmp_path ): set_mock_value(hdf_panda.data.datasets, table) + await hdf_panda.stage() with caplog.at_level(logging.WARNING): await hdf_panda.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL)) description = await hdf_panda.describe() @@ -99,6 +106,7 @@ async def test_open_returns_correct_descriptors_and_resources( "external": "STREAM:", } + await hdf_panda.kickoff() set_mock_value(hdf_panda.data.num_captured, 7) docs = [doc async for doc in hdf_panda.collect_asset_docs()] sr = docs[: len(table)] @@ -134,6 +142,9 @@ async def test_open_returns_correct_descriptors_and_resources( async def test_open_sets_correct_signals(hdf_panda: HDFPanda, tmp_path): + await hdf_panda.stage() + # Only the calls after stage() are of interest + get_mock(hdf_panda).reset_mock() await hdf_panda.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL)) assert_has_calls( hdf_panda, @@ -150,6 +161,8 @@ async def test_open_sets_correct_signals(hdf_panda: HDFPanda, tmp_path): async def test_close_sets_correct_signals(hdf_panda: HDFPanda): + await hdf_panda.stage() + get_mock(hdf_panda).reset_mock() await hdf_panda.unstage() assert_has_calls(hdf_panda.pcap, [call.arm.put(False)]) assert_has_calls(hdf_panda.data, [call.capture.put(False)]) @@ -161,6 +174,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): assert config_description == {} await assert_configuration(hdf_panda, {}) # Prepare for N frames + await hdf_panda.stage() await hdf_panda.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_LEVEL, @@ -276,7 +290,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, - match="Kickoff requested 15:30, but detector was only prepared up to 15", + match="cannot go to FLYING from FLOWN", ): await hdf_panda.kickoff() # But preparing again is ok @@ -288,19 +302,22 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): ) ) await hdf_panda.kickoff() - # Completing adds indexes to the same resource + # Completing starts new resources in the same file, the indices carrying on status = hdf_panda.complete() set_mock_value(hdf_panda.data.num_captured, 24) set_mock_value(hdf_panda.pcap.active, False) docs = [doc async for doc in hdf_panda.collect_asset_docs()] - assert docs == [ + new_sr = docs[:2] + assert new_sr == [(name, {**doc, "uid": ANY}) for name, doc in sr] + assert {doc["uid"] for _, doc in new_sr}.isdisjoint(doc["uid"] for _, doc in sr) + assert docs[2:] == [ ( "stream_datum", { "descriptor": "", "indices": {"start": 5, "stop": 8}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": sr[0][1]["uid"], + "stream_resource": new_sr[0][1]["uid"], "uid": ANY, }, ), @@ -310,7 +327,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): "descriptor": "", "indices": {"start": 5, "stop": 8}, "seq_nums": {"start": 0, "stop": 0}, - "stream_resource": sr[1][1]["uid"], + "stream_resource": new_sr[1][1]["uid"], "uid": ANY, }, ), @@ -319,6 +336,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): async def test_get_indices_written(hdf_panda: HDFPanda): + await hdf_panda.stage() await hdf_panda.prepare( TriggerInfo( trigger=DetectorTrigger.EXTERNAL_LEVEL, @@ -334,6 +352,7 @@ async def test_get_indices_written(hdf_panda: HDFPanda): async def test_oserror_when_hdf_dir_does_not_exist(hdf_panda: HDFPanda): set_mock_value(hdf_panda.data.directory_exists, False) + await hdf_panda.stage() with pytest.raises(OSError): await hdf_panda.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL)) @@ -342,3 +361,93 @@ async def test_deadtime(hdf_panda: HDFPanda): supported_triggers, deadtime = await hdf_panda.get_trigger_deadtime() assert deadtime == 8e-9 assert supported_triggers == {DetectorTrigger.EXTERNAL_LEVEL} + + +async def test_uc8_step_scan_stages_first_and_emits_stream_assets( + hdf_panda: HDFPanda, tmp_path +): + """Use case 8, step: a trigger() gives one StreamDatum per dataset. + + Written for the rewrite's lifecycle: stage() comes first. The PandA is + externally triggered, so prepare() arms it and trigger() waits for the point. + """ + await hdf_panda.stage() + + await hdf_panda.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL)) + status = hdf_panda.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + assert not status.done + # The external trigger fires and the PandA captures the point + set_mock_value(hdf_panda.data.num_captured, 1) + await status + docs = [doc async for doc in hdf_panda.collect_asset_docs()] # pyright: ignore[reportAttributeAccessIssue] + assert [(n, d.get("data_key")) for n, d in docs[:2]] == [ + ("stream_resource", "x"), + ("stream_resource", "y"), + ] + assert [d["indices"] for n, d in docs[2:]] == [{"start": 0, "stop": 1}] * 2 + assert await hdf_panda.read() == {} + await hdf_panda.unstage() + + +def test_uc8_fly_scan_through_the_run_engine(RE, hdf_panda: HDFPanda): + """Use case 8, fly: stream assets covering N events, no Events.""" + import bluesky.plan_stubs as bps # noqa: PLC0415 + import bluesky.preprocessors as bpp # noqa: PLC0415 + + @bpp.stage_decorator([hdf_panda]) + @bpp.run_decorator() + def plan(): + yield from bps.prepare( + hdf_panda, + TriggerInfo( + trigger=DetectorTrigger.EXTERNAL_LEVEL, + collections_per_event=3, + number_of_events=2, + ), + wait=True, + ) + yield from bps.declare_stream(hdf_panda, name="primary", collect=True) + yield from bps.kickoff(hdf_panda, wait=True) + # The external triggers fire and the PandA captures everything + set_mock_value(hdf_panda.data.num_captured, 6) + set_mock_value(hdf_panda.pcap.active, False) + yield from bps.collect_while_completing( + flyers=[hdf_panda], dets=[hdf_panda], flush_period=0.05 + ) + + docs: dict[str, list] = {} + RE(plan(), lambda n, d: docs.setdefault(n, []).append(d)) + + assert [r["data_key"] for r in docs["stream_resource"]] == ["x", "y"] + assert "event" not in docs + assert [d["indices"]["stop"] for d in docs["stream_datum"]][-2:] == [2, 2] + + +async def test_prepare_waits_for_num_captured_to_read_zero(hdf_panda: HDFPanda): + await hdf_panda.stage() + set_mock_value(hdf_panda.data.num_captured, 5) + + preparing = hdf_panda.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL)) + await asyncio.sleep(0.1) + # NumCaptured has not been reset by the capture yet + assert not preparing.done + set_mock_value(hdf_panda.data.num_captured, 0) + await asyncio.wait_for(preparing, timeout=1) + + +async def test_datasets_filled_only_once_capture_starts_are_described( + hdf_panda: HDFPanda, +): + # PandABlocks-ioc fills DATASETS when capture starts, not before + set_mock_value(hdf_panda.data.datasets, TABLES[0]) + callback_on_mock_put( + hdf_panda.data.capture, + lambda value, wait=True: ( + set_mock_value(hdf_panda.data.datasets, TABLES[1]) if value else None + ), + ) + await hdf_panda.stage() + await hdf_panda.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_LEVEL)) + + assert list(await hdf_panda.describe()) == ["x"] diff --git a/tests/unit_tests/plan_stubs/test_ndattributes.py b/tests/unit_tests/plan_stubs/test_ndattributes.py index 8246e48c10..8695c67415 100644 --- a/tests/unit_tests/plan_stubs/test_ndattributes.py +++ b/tests/unit_tests/plan_stubs/test_ndattributes.py @@ -10,7 +10,7 @@ def test_setup_ndstats_raises_type_error(RE, static_path_provider: StaticPathProvider): with init_devices(mock=True): det = adsimdetector.SimDetector( - "PREFIX:", adcore.ADWriterFactory.hdf(static_path_provider) + "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) ) with pytest.raises( AttributeError, From 28845477157c201544c791b3901b11caa26e2600 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Fri, 30 Oct 2026 17:20:28 +0000 Subject: [PATCH 16/25] WIP: AD stats time series as a bounded data logic Split cut G4 (#1367) of detector-rederive (d0872e185): the StatsTimeSeriesDataLogic, the ADDataFactory.stats_time_series factory and their tests (UC6, UC7, HDF plus time series, two stats plugins, the adsim container tests). Together with the PRs below it, the tree is the tip merged with origin/main, less the deleted ADWriterFactory shims. Round 3 of the stats naming review folded in: the plugin snapshot is taken once at stage and also records NDArrayPort and ROIStat channel names, the plugins a trigger logic holds are not recorded, plugins= are registered as readables, data factories take plugins by reference, NDROIStatIO is a StandardReadable, and the writers no longer write EnableCallbacks at prepare. Assisted-by: Claude Opus 5.5 Assisted-by: Claude Sonnet 5.5 --- CHANGELOG.md | 14 +- .../decisions/0023-bounded-data-logic-tier.md | 5 +- .../decisions/0024-detector-is-a-flyable.md | 74 ++ docs/how-to/implement-ad-detector.md | 36 +- docs/tutorials/implementing-detectors.md | 2 +- src/ophyd_async/epics/adcore/__init__.py | 11 +- src/ophyd_async/epics/adcore/_data_logic.py | 232 +++-- src/ophyd_async/epics/adcore/_detector.py | 151 +++- src/ophyd_async/epics/adcore/_io.py | 300 +++++- src/ophyd_async/epics/adcore/_ndattribute.py | 18 +- src/ophyd_async/epics/adcore/_plan_stubs.py | 23 +- .../epics/adcore/_stats_time_series.py | 94 ++ tests/container_tests/test_adsim_system.py | 80 ++ .../epics/adcore/test_cont_acq_detector.py | 16 + .../epics/adcore/test_data_logic.py | 51 +- .../unit_tests/epics/adcore/test_detector.py | 115 ++- .../epics/adcore/test_stats_time_series.py | 319 +++++++ .../unit_tests/epics/adcore/test_use_cases.py | 854 ++++++++++++++++-- .../plan_stubs/test_ndattributes.py | 16 +- 19 files changed, 2209 insertions(+), 202 deletions(-) create mode 100644 src/ophyd_async/epics/adcore/_stats_time_series.py create mode 100644 tests/unit_tests/epics/adcore/test_stats_time_series.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f8e6503d10..838c87efc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,12 @@ - The progress of `StandardFlyable.prepare()` now comes from the `WatcherUpdate`s the logic's `prepare_to_fly` yields, replacing the heuristic that watched setpoint writes (#1448). - `WatchableAsyncStatus` closes the async generator it drives as soon as it is cancelled or fails, rather than when the generator is collected, and a watcher that raises is logged and ignored rather than failing the status. - A `StandardFlyable` whose fly failed after `kickoff()` and was never completed logs a warning when it is unstaged or prepared again, instead of leaving the failure unretrieved. +- Each statistic of an `NDStatsIO` has one name, that of its scalar signal (e.g. `det-stats-total`), as a readable, as a stats time series and as an HDF NDAttribute, so the highest of these serving a scan suppresses the others, and the hint of the scalar stays. The plugin's `EnableCallbacks` and `compute_statistics`/`compute_centroid` PVs at `stage()` decide which statistics are used: the scalars of those switched off are left out of `read()`, `describe()` and `hints`, `AreaDetector.prepare()` raises, naming the PVs, if any of these switches, or the `NDArrayPort` of any plugin it uses, has changed since `stage()`, and a stats time series publishes every statistic switched on that has a read format on the plugin. +- `StatsTimeSeriesDataLogic` and `ADDataFactory.stats_time_series` lose `stat_signals`, `datakey_suffix` and `always_enable`, and no longer write `EnableCallbacks` or `ComputeStatistics`: switch the plugin on with its PVs. `StatsTimeSeriesDataLogic` no longer hints the statistics it publishes: the stats plugin registered as a readable does. +- `ADDataFactory.stats_time_series(stats)` takes the `NDStatsIO` to read instead of making one (so no `writer_suffix` or `writer_name`), which must be in the detector's device tree, e.g. in `plugins`, or `AreaDetector` raises. Only the factories of file writers make a plugin. +- `AreaDetector` registers every plugin in `plugins` as a `CHILD` readable, so the signals a plugin declares as read, configuration or hinted appear in the detector's `read()`, `read_configuration()` and `hints`. Opt a plugin out with `set_readable_format(plugin, None)`. +- `setup_ndstats_sum` is removed: list the stats plugin in `ADDataFactory.hdf(ndattribute_sources=[stats])` and the HDF writer writes its NDAttributes, named after their scalars, so `det-sum` is now `det-stats-total`. A listed plugin's `nd_attributes_file` is overwritten when the HDF writer serves a scan; `setup_ndattributes` on another plugin or the driver works as before. +- `ADHDFDataLogic` and `ADMultipartDataLogic` with `always_enable=True` switch their plugin on at `stage()`, and no longer write `EnableCallbacks` at each `prepare()`. - A `StandardReadable` that a detector suppresses with its data keeps the hints of the signals it drops from `read()` and `describe()`, as the data has the same name. A data logic that hints such a name now makes `hints` raise. - `StandardReadable` now keeps one format per registered child, keyed by the child, so registering a child again replaces its format instead of adding a second contribution, and the order of `hints` is the order children were first registered. The private `_describe_config_funcs`, `_read_config_funcs` and `_has_hints` tuples are gone: a subclass that added to them now adds a hook to `_describe_configuration_funcs`, `_read_configuration_funcs` or `_hints_funcs` (see ADR 0025). - The deprecated `ConfigSignal` and `HintedSignal` aliases are removed from `ophyd_async.core`; use `StandardReadableFormat.CONFIG_SIGNAL`, `HINTED_SIGNAL` or `HINTED_UNCACHED_SIGNAL`. `StandardReadable.add_readables()` is deprecated in favour of `set_readable_format()` on each device. @@ -52,6 +58,11 @@ ### Added +- `NDStatistic` and `NDStatsIO.statistics`, each statistic with its scalar, asyn parameter, time series and compute switch, and the `ts_*` array signals of every statistic on `NDStatsIO`. +- `NDAttributeSource`, a protocol for a plugin that returns the NDAttributes it stamps; `NDStatsIO` and `NDROIStatIO` implement it. +- `ADDataFactory.hdf(ndattribute_sources=)`, the plugins whose NDAttributes the HDF writer writes. A source listed by two writers, or not in the detector's device tree, raises when the `AreaDetector` is built. The NDAttributes of other plugins and the driver are parsed from the XML the user set, with the names in it. +- `NDPluginBaseIO.snapshot()`, `get_snapshot_signals()`, `get_changed_signals()`, `forget_snapshot()`, `switch_is_on()` and `get_switched_off_readables()`, which `AreaDetector` calls on its plugins and every plugin below them from `stage()` to `unstage()`. The snapshot is taken once, at `stage()`, and records the switches and the `NDArrayPort`. The plugins a trigger logic holds, such as its process plugin or circular buffer, are left out as the logic switches them on itself. +- `NDROIStatIO` stamps the `min_value`, `max_value`, `mean_value` and `total` of each channel that is used and has a read format, named after the scalar. At `stage()` an `AreaDetector` names each channel `-`, with characters other than letters, digits and underscores replaced by underscores and the channel number used for an empty name, and raises if two would be the same. The `use` of each channel is a switch, so a channel that is not used leaves `read()`, `describe()` and `hints`, and `prepare()` raises if the `Name` of a channel has changed since `stage()`. `NDROIStatIO` is a `StandardReadable` whose channels are registered as `CHILD` readables, so there is no need to register each channel. - `KICKED_OFF`, the sentinel `FlyableLogic.fly` yields once the scan is up to speed. - `watch_readback()` reports the progress of a move by watching a `MovableLogic` readback, for use from `FlyableLogic` hooks. - `SimMotor.prepare()` and `SimMotor.complete()` report progress to watchers, as `Motor` does. @@ -62,7 +73,8 @@ - `StandardReadable.set_readable_format()`, `reset_readable_formats()`, `get_readable_formats()` and `get_readable_format_changes()`, to change how a child contributes to `read()`, `read_configuration()` and `hints` at runtime, the equivalent of `Kind` in ophyd v1 (see "How to change what a Device reports at runtime"). - `ReadableFormats`, `walk_readable_formats()`, `apply_readable_formats()` and `READABLE_FORMATS_KEY` in `ophyd_async.core`, and a `readable_formats` field on `Settings`, so formats are stored and retrieved with the signal values. - pyright now runs with `reportMatchNotExhaustive`. -- `BoundedDataLogic`, `UnboundedDataLogic`, `DetectorLogic`, `DataProvider`, `PageableDataProvider` and `EventPageDataProvider` in `ophyd_async.core`, and `ADDataFactory` and `exposures_per_collection_from_filter` in `ophyd_async.epics.adcore`. +- `BoundedDataLogic`, `UnboundedDataLogic`, `DetectorLogic`, `DataProvider`, `PageableDataProvider` and `EventPageDataProvider` in `ophyd_async.core`, and `ADDataFactory`, `NDStatsTSAcquireMode` and `exposures_per_collection_from_filter` in `ophyd_async.epics.adcore`. +- `adcore.StatsTimeSeriesDataLogic` and the `ADDataFactory.stats_time_series(stats)` factory, a `BoundedDataLogic` that publishes an `NDPluginStats` time series as event pages, for an areaDetector that writes no file. It needs a fixed number of collections, as the buffer is sized before acquisition, and sits out when the plugin is switched off at `stage()`, so a detector can switch between its file writer and its stats time series from one stage to the next. ### Fixed diff --git a/docs/explanations/decisions/0023-bounded-data-logic-tier.md b/docs/explanations/decisions/0023-bounded-data-logic-tier.md index dc90eaf120..8facc40e0b 100644 --- a/docs/explanations/decisions/0023-bounded-data-logic-tier.md +++ b/docs/explanations/decisions/0023-bounded-data-logic-tier.md @@ -243,6 +243,8 @@ control embedded, rather than one logic per array with a shared control object. the existing grain — a provider already emits many datakeys, as `ADHDFDataLogic` does for the main dataset plus its NDAttributes — and avoids the coordination problem of several logics contending over one `TSAcquire`. Which statistics are wanted becomes a constructor argument. +(Superseded in [](./0024-detector-is-a-flyable.md): the plugin's switches decide, and each +series is named after its scalar.) ## Consequences @@ -265,7 +267,8 @@ plus a finite-buffer data logic starts working, closing #1364. A data logic may decline a scan for reasons of its own: `ADHDFDataLogic` and `StatsTimeSeriesDataLogic` take an `always_enable` flag, defaulting to today's behaviour of switching the plugin on, which when `False` makes them follow whatever the plugin is set to -(at the first `prepare()` of a stage) and produce nothing when it is off. +(at the first `prepare()` of a stage) and produce nothing when it is off. (The stats time +series lost the flag again in [](./0024-detector-is-a-flyable.md).) A device whose exposure count and buffer size are the same PV has it written twice: by the trigger logic as `number_of_exposures`, and by the data logic as `num_collections`. The two diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md index df84b3b7df..d277f6d89e 100644 --- a/docs/explanations/decisions/0024-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -120,6 +120,80 @@ replace it, and leaves it in `hints`, because the replacement has the same name. that hints the name of a signal it replaces makes `hints` raise, as two hints of one field always do. +### One name per statistic, and the plugin's switches decide which are used + +Suppression only works if every tier calls a quantity the same thing. An areaDetector statistic +reaches the documents in up to three ways: the scalar read once per event, its stats time +series, and an NDAttribute in the HDF file. All three are named after the scalar signal +(`det-stats-total`), because that is the name a user already gets by registering it, and so +the highest tier serving a scan suppresses the others without any configuration. The time +series' `stat_signals` and `datakey_suffix`, and `setup_ndstats_sum`, which named the same +total `det-sum`, are removed. + +Which statistics are used is decided by the plugin's own PVs, not by arguments to the data +logics: a statistic is used if the plugin's `EnableCallbacks` and the compute switch of its +group (`ComputeStatistics`, `ComputeCentroid`) are on, and it has a read format on the plugin. +The PVs are what make the IOC produce the data, they are already captured by settings, and a +second set of switches in Python could only disagree with them. So the stats time series has +no `always_enable` and never writes these PVs, and a scalar whose statistic is switched off is +dropped from `read()`, `describe()` and `hints` rather than read as a stale value. The HDF +writer keeps `always_enable`, defaulting to `True`, for compatibility with `ADWriterFactory`: it switches its plugin on at `stage()`, before the recording below is taken, and +writes nothing to `EnableCallbacks` after that. + +`AreaDetector` records the plugins it uses once, at `stage()`, after the data logics have had +their say, because plugins are not usually staged themselves. Those are its `plugins`, the +writers of its factories, the NDAttribute sources of its writers and the plugins its factories +take by reference, and every plugin below them, so a stats plugin that is a child of an ROI +plugin is covered. For each it records the switches (`EnableCallbacks`, the compute switches, +the `Use` of each ROI stat channel), the `NDArrayPort`, and the `Name` of each ROI stat channel. +`describe()` and `read()` follow the recording, so they agree for the whole stage. Each +`prepare()` raises if a recorded PV has a different value now, naming the PVs, so a run cannot +be split by someone changing a PV part way through, and nothing is written to a PV the user +owns. The recording is never retaken: staging again records the new values, and `unstage()` +forgets them. + +The exception is a switch the detector's own logic writes. A trigger logic switches on the +process plugin it filters with, and the continuous acquisition one its circular buffer, at every +`prepare()`, so recording them would report the detector's own write as a change by someone +else. The plugins a trigger logic holds are therefore left out of the recording. Ownership is by +logic, not by plugin type: a process plugin that only appears in `plugins` is the user's, and is +recorded like any other. + +Only a plugin's own `NDArrayPort` is recorded, so a rewired port is caught, but the chain of +plugins upstream of it is not walked to see whether it still reaches the driver. A chain can +cross IOCs over PVA, so its reachability is not something one process can establish, and the +chain as wired at `stage()` is the user's responsibility. + +A statistic that is switched off is such a `None`. Neither the stats time series nor the HDF writer hints +a statistic, so the plugin registered as a readable is what hints it, whichever tier serves it. +`AreaDetector` registers every plugin in its `plugins` as a readable, so that is always the case +for a plugin the user gave it, and a user who does not want a plugin read opts it out with +`set_readable_format(plugin, None)`. Data factories take such a plugin by reference, and only +the factories of file writers make a device: `ADDataFactory.stats_time_series(stats)` raises +in `AreaDetector.__init__` if `stats` is not in the detector's device tree, like an NDAttribute +source. + +The HDF writer stays free of stats knowledge. `ADDataFactory.hdf(ndattribute_sources=)` lists +the plugins, as objects, whose NDAttributes it stamps: a plugin that implements the +`NDAttributeSource` protocol returns its NDAttributes, which the writer writes to the plugin's +`nd_attributes_file` and turns into datasets directly, each named after the scalar signal it +publishes. Nothing is stamped that is not listed, so two HDF writers stamp the statistics of +their own ROIs and no others. `AreaDetector` raises if a source is listed by two writers or is +not in its device tree. Any other plugin's, and the driver's, NDAttributes are still parsed from +the XML the user set, and keep the names in it, as they may be about something outside the +detector. + +An ROI stat plugin is a source too. Each channel whose `Use` is on and which has a read format +on a scalar stamps that scalar, named after it, from the channel's asyn address. Its switches +are `EnableCallbacks` and the `Use` of every channel, and a channel that is not used has its +scalars dropped like a switched-off statistic. The channels are named when the detector is +staged, as `-`, with characters other than letters, digits and underscores +replaced by underscores and the channel number used if the PV is empty, so the scalars have +names the documents accept and say what the channel measures. Two channels that would be named +the same raise, and a `Name` PV that changes after that raises at `prepare()` like a switch. The plugin is a +readable whose channels are its readable children, so a user does not register each channel. There is no ROI stat time series: a fly scan that needs the channels while the +HDF writer sits out raises at `prepare()`, as a stats plugin only registered as a readable does. + ### The detector flies with one `fly()` generator `FlyableLogic.fly` is a single async generator ([](./0022-fly-generator-and-ensure-stopped.md)): diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 798503b558..2b489b6e86 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -97,7 +97,7 @@ The `AreaDetector` baseclass will: - Build a [](#DetectorLogic) from your trigger and arm logic and return it from `logic` - Create and register a data logic for file writing for each factory in `writer_factories` - Add configuration signals (driver.acquire_time, driver.acquire_period, and any you specify) -- Store any plugins as attributes on the detector +- Store any plugins as attributes on the detector, and register each as a readable, so the signals it declares appear in `read()`, `read_configuration()` and `hints` (opt one out with `det.set_readable_format(plugin, None)`) ## Declare the public interface @@ -116,22 +116,46 @@ For example, for ADAravis: ## Add multiple data streams (optional) -The composition-based architecture makes it possible to give a detector several data outputs. Every [](#adcore.ADDataFactory) factory takes `always_enable`: `True` (the default) switches the plugin on at the start of a scan, `False` follows the plugin's `EnableCallbacks` PV, so a data output whose plugin is off sits out and another can serve the scan. +The composition-based architecture makes it possible to give a detector several data outputs. The file writer factories ([](#adcore.ADDataFactory.hdf), `jpeg` and `tiff`) take `always_enable`: `True` (the default) switches the plugin on at `stage()`, `False` follows the plugin's `EnableCallbacks` PV, so a writer whose plugin is off sits out and another output can serve the scan. A stats time series has no `always_enable`: its plugin's switches decide, as below. A writer you leave `always_enable=True` is switched on once, at `stage()`; the factory does not write `EnableCallbacks` again at `prepare()`. ### Reading stats plugins alongside file writing -A plugin scalar needs no data logic: a detector is a [](#StandardReadable), so register -the signal and it appears in `read()` alongside whatever the file writer produces. +A plugin in `plugins` is registered as a readable, so a detector with a stats plugin needs no +data logic to read it: the plugin declares which of its signals are read and hinted, and they +appear in `read()` alongside whatever the file writer produces. ```python from ophyd_async.core import StandardReadableFormat as Format +stats = adcore.NDStatsIO("PREFIX:STATS:") det = adaravis.AravisDetector( prefix, adcore.ADDataFactory.hdf(path_provider), plugins={"stats": stats} ) -# Add stats total as a hinted signal in events -det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) +# Read a statistic the plugin does not read by default +det.set_readable_format(stats.sigma, Format.UNCACHED_SIGNAL) ``` +Each statistic of an [](#adcore.NDStatsIO) has one name, that of its scalar signal (`det-stats-total` here), however it is published: + +- as the scalar, read once per event, when nothing else serves it +- as a stats time series ([](#adcore.ADDataFactory.stats_time_series)), for a detector that writes no file +- as an NDAttribute in the HDF file, for each plugin listed in the `ndattribute_sources` of [](#adcore.ADDataFactory.hdf) + +The time series and the HDF writer take the plugin by reference, so make it first and put it in `plugins`. Only the factories of file writers make a plugin of their own. + +```python +stats = adcore.NDStatsIO("PREFIX:STATS:") +det = adaravis.AravisDetector( + prefix, + adcore.ADDataFactory.hdf(path_provider, ndattribute_sources=[stats]), + adcore.ADDataFactory.stats_time_series(stats), + plugins={"stats": stats}, +) +``` + +The highest of these that serves a scan replaces the others, and the plugin's hints stay. Which statistics are used is set on the plugin, not in Python: a statistic is used if the plugin's `EnableCallbacks` and its group's compute switch (`compute_statistics` or `compute_centroid`) are on at `stage()`, and it has a read format on the plugin. A scalar whose statistic is off is left out of `read()`, `describe()` and `hints`. + +`stage()` records these switches and each plugin's `nd_array_port`, once, and `prepare()` raises if any of them has changed since, so set them before staging, for example in a settings file. The chain of plugins that feeds a plugin is not checked: how it is wired at `stage()` is up to you. The process plugin of a trigger logic and the circular buffer of a continuously acquiring detector are switched on by their logic, so they are not recorded. An [](#adcore.NDROIStatIO) works the same way, with the `use` and `Name` of each channel recorded too: its channels are readable children of the plugin, named `-` at `stage()`, so a channel called Blue has the total `det-roistat-Blue-total`. + ### Multiple HDF writers for different ROIs ```python roi1 = adcore.NDROIIO("PREFIX:ROI1:") diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index f00fec2532..99cc1e9286 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -240,7 +240,7 @@ The logic also has `collections_written`, a signal that tracks how many frames h The detector starts each of its logics for the scan, and the type of provider that comes back says which kind of data it is. - a [](#StreamableDataProvider) publishes stream assets, like a file writer. The detector exposes `collect_asset_docs`, and `BlobDataLogic` is an example. -- a [](#PageableDataProvider) publishes event pages, like an areaDetector stats time series or a scaler's arrays, so the detector exposes `collect_pages` instead. Its `make_page()` carries on from where its last page got to; a step scan's `read()` takes the last event of the page made up to the end of the point. +- a [](#PageableDataProvider) publishes event pages, like an areaDetector stats time series or a scaler's arrays, so the detector exposes `collect_pages` instead. Its `make_page()` carries on from where its last page got to; a step scan's `read()` takes the last event of the page made up to the end of the point. See [](#StatsTimeSeriesDataLogic). - `None` means "not this stage", for instance a plugin that is switched off. The logic decides this without writing anything. This is not an error: a detector may carry more data logics than any one scan uses. Separately from that, a data logic is either a [](#BoundedDataLogic) or an [](#UnboundedDataLogic), whichever it inherits from (not both). An unbounded logic, like a file writer, keeps counting for as long as it is told to; its collections-written signal is read when each `prepare()` finishes. A bounded logic needs to be told how many collections to take, like a finite buffer: it is started after the unbounded ones and handed the datakeys their providers already make, and its provider must make none of them. It returns `None` if nothing is left to make, and raises for a scan with no fixed number of collections (`num_collections` of 0) if something is. At each point of a step scan after the first it is stopped and started again, so the buffer is cleared. Either kind may return either kind of provider. diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index df4e943dc8..2775847983 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -39,16 +39,20 @@ NDProcessIO, NDROIStatIO, NDROIStatNIO, + NDStatistic, NDStatsIO, + NDStatsTSAcquireMode, ) from ._ndattribute import ( NDAttributeDataType, NDAttributeParam, NDAttributePv, NDAttributePvDbrType, + NDAttributeSource, ndattributes_to_xml, ) -from ._plan_stubs import setup_ndattributes, setup_ndstats_sum +from ._plan_stubs import setup_ndattributes +from ._stats_time_series import StatsTimeSeriesDataLogic from ._trigger_logic import ( ADContAcqTriggerLogic, exposures_per_collection_from_filter, @@ -67,7 +71,9 @@ "ADBaseIO", "NDPluginBaseIO", "NDROIIO", + "NDStatistic", "NDStatsIO", + "NDStatsTSAcquireMode", "NDROIStatNIO", "NDROIStatIO", "NDCBFlushOnSoftTrgMode", @@ -101,6 +107,7 @@ "ADMultipartDataLogic", "ADDataFactory", "ADWriterFactory", + "StatsTimeSeriesDataLogic", # Detector "AreaDetector", "ContAcqDetector", @@ -109,7 +116,7 @@ "NDAttributePvDbrType", "NDAttributePv", "NDAttributeParam", + "NDAttributeSource", "ndattributes_to_xml", "setup_ndattributes", - "setup_ndstats_sum", ] diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 1ef6b4de8a..d324491287 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -33,9 +33,16 @@ NDPluginBaseIO, NDPluginBaseIOT, NDPluginFileIO, + NDStatsIO, plugin_is_enabled, ) -from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType +from ._ndattribute import ( + NDAttributeDataType, + NDAttributePvDbrType, + NDAttributeSource, + ndattributes_to_xml, +) +from ._stats_time_series import StatsTimeSeriesDataLogic @dataclass @@ -90,31 +97,55 @@ async def get_ndarray_resource_info( async def get_ndattribute_dtype_source( elements: Sequence[NDArrayBaseIO], + sources: Sequence[NDAttributeSource] = (), ) -> dict[str, tuple[str, str]]: - nd_attribute_xmls = await asyncio.gather( - *[x.nd_attributes_file.get_value() for x in elements] + """The dtype and source of each NDAttribute stamped on a frame. + + :param elements: parse the NDAttributes in the `nd_attributes_file` of each + :param sources: write the NDAttributes of each to its `nd_attributes_file` + """ + per_element = await asyncio.gather( + *(_parse_ndattribute_dtype_source(element) for element in elements), + *(_write_ndattribute_dtype_source(source) for source in sources), ) + return {name: value for found in per_element for name, value in found.items()} + + +async def _write_ndattribute_dtype_source( + source: NDAttributeSource, +) -> dict[str, tuple[str, str]]: + ndattributes = source.get_ndattributes() + await source.nd_attributes_file.set(ndattributes_to_xml(ndattributes)) + return { + ndattribute.name: (ndattribute.datatype.value, "") + for ndattribute in ndattributes + } + + +async def _parse_ndattribute_dtype_source( + element: NDArrayBaseIO, +) -> dict[str, tuple[str, str]]: + maybe_xml = await element.nd_attributes_file.get_value() ndattribute_dtypes: dict[str, tuple[str, str]] = {} - for maybe_xml in nd_attribute_xmls: - # This is the check that ADCore does to see if it is an XML string - # rather than a filename to parse - if "" in maybe_xml: - root = ET.fromstring(maybe_xml) - for child in root: - if child.attrib.get("type", "EPICS_PV") == "EPICS_PV": - dbrtype = child.attrib.get("dbrtype", "DBR_NATIVE") - if dbrtype == "DBR_NATIVE": - raise RuntimeError( - f"NDAttribute {child.attrib['name']} has dbrtype " - "DBR_NATIVE, which is not supported" - ) - dtype_numpy = NDAttributePvDbrType[dbrtype].value - source = "ca://" + child.attrib["source"] - else: - datatype = child.attrib.get("datatype", "INT") - dtype_numpy = NDAttributeDataType[datatype].value - source = "" - ndattribute_dtypes[child.attrib["name"]] = (dtype_numpy, source) + # This is the check that ADCore does to see if it is an XML string + # rather than a filename to parse + if "" in maybe_xml: + root = ET.fromstring(maybe_xml) + for child in root: + if child.attrib.get("type", "EPICS_PV") == "EPICS_PV": + dbrtype = child.attrib.get("dbrtype", "DBR_NATIVE") + if dbrtype == "DBR_NATIVE": + raise RuntimeError( + f"NDAttribute {child.attrib['name']} has dbrtype " + "DBR_NATIVE, which is not supported" + ) + dtype_numpy = NDAttributePvDbrType[dbrtype].value + source = "ca://" + child.attrib["source"] + else: + datatype = child.attrib.get("datatype", "INT") + dtype_numpy = NDAttributeDataType[datatype].value + source = "" + ndattribute_dtypes[child.attrib["name"]] = (dtype_numpy, source) return ndattribute_dtypes @@ -148,6 +179,16 @@ async def prepare_file_paths( await writer.num_capture.set(0) +async def _ensure_writer_ready(writer: NDPluginFileIO, always_enable: bool) -> None: + """Stop the writer, and switch it on if `always_enable`. + + Called at stage, before `AreaDetector` records the plugins' switches. + """ + await stop_busy_record(writer.capture) + if always_enable: + await writer.enable_callbacks.set(EnableDisable.ENABLE) + + async def _start_capture(writer: NDPluginFileIO) -> None: await set_and_wait_for_value(writer.capture, True, wait_for_set_completion=False) # Capturing resets NumCaptured, which the detector reads as its baseline @@ -162,7 +203,11 @@ class ADHDFDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): :param path_provider: Callable that provides path information for file writing. :param writer: The NDFileHDFIO plugin instance. :param driver: The AreaDetector driver instance to extract NDAttributes from. - :param plugins: Additional NDPluginBaseIO instances to extract NDAttributes from. + :param plugins: Additional NDPluginBaseIO instances to parse NDAttributes + from, in the XML the user set on each. + :param ndattribute_sources: Plugins to write the NDAttributes of, named after + what they publish. A plugin in `plugins` that is also listed here is + written, not parsed. :param datakey_suffix: Suffix to append to the data key for the main dataset :param hinted: Include the image in hints. Set False for BestEffortCallback. """ @@ -172,16 +217,19 @@ class ADHDFDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): writer: NDFileHDF5IO driver: NDArrayBaseIO | None = None plugins: Sequence[NDPluginBaseIO] = () + ndattribute_sources: Sequence[NDAttributeSource] = () datakey_suffix: str = "" hinted: bool = True - #: True serves every scan. False sits out the scans of a stage if the - #: plugin's EnableCallbacks PV is off at the first prepare. Starting switches - #: it on. + #: True switches the plugin on (EnableCallbacks) at stage, so it serves every + #: scan. False sits out the scans of a stage if the plugin is off at stage. always_enable: bool = True def __post_init__(self) -> None: self.collections_written = self.writer.num_captured + async def ensure_ready(self) -> None: + await _ensure_writer_ready(self.writer, self.always_enable) + async def start_provider( self, name: str, @@ -201,6 +249,12 @@ async def start_provider( frames_per_chunk = max(1, round(flush_period / collection_period)) else: frames_per_chunk = await self.writer.num_frames_chunks.get_value() or 1 + elements = (self.driver, *self.plugins) if self.driver else (*self.plugins,) + parsed = [ + element + for element in elements + if not any(element is source for source in self.ndattribute_sources) + ] main_dataset, ndattribute_dtype_sources = await asyncio.gather( get_ndarray_resource_info( array_description=self.array_description, @@ -208,9 +262,7 @@ async def start_provider( parameters={"dataset": "/entry/data/data"}, frames_per_chunk=frames_per_chunk, ), - get_ndattribute_dtype_source( - (self.driver, *self.plugins) if self.driver else (*self.plugins,) - ), + get_ndattribute_dtype_source(parsed, self.ndattribute_sources), ) ndattribute_datasets = [ StreamResourceInfo( @@ -238,7 +290,6 @@ async def start_provider( prepare_file_paths( path_info=path_info, file_template="%s%s.h5", writer=self.writer ), - self.writer.enable_callbacks.set(EnableDisable.ENABLE), ] await asyncio.gather(*coros) await _start_capture(self.writer) @@ -270,9 +321,9 @@ class ADMultipartDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): :param datakey_suffix: Suffix to append to the data key for the main dataset :param hinted: Include the image in hints. Set False for BestEffortCallback. :param always_enable: - True serves every scan. False sits out the scans of a stage if the - plugin's EnableCallbacks PV is off at the first prepare. Starting - switches it on. + True switches the plugin on (EnableCallbacks) at stage, so it serves + every scan. False sits out the scans of a stage if the plugin is off at + stage. """ array_description: NDArrayDescription @@ -287,6 +338,9 @@ class ADMultipartDataLogic(UnboundedDataLogic[StreamResourceDataProvider]): def __post_init__(self) -> None: self.collections_written = self.writer.num_captured + async def ensure_ready(self) -> None: + await _ensure_writer_ready(self.writer, self.always_enable) + async def start_provider( self, name: str, @@ -313,7 +367,6 @@ async def start_provider( file_template="%s%s_%6.6d" + self.extension, writer=self.writer, ), - self.writer.enable_callbacks.set(EnableDisable.ENABLE), ) await _start_capture(self.writer) return StreamResourceDataProvider( @@ -334,22 +387,28 @@ def get_hinted_fields(self, name: str) -> Sequence[str]: @dataclass class ADDataFactory(Generic[NDPluginBaseIOT]): - """Factory that creates a plugin and its matching data logic. + """Factory that builds a data logic, and the writer plugin it needs if any. - Construct using the static methods `hdf`, `jpeg` or `tiff`, then pass one - or more instances to `AreaDetector` as positional `*writer_factories` - arguments. When the detector is initialised + Construct using the static methods `hdf`, `jpeg`, `tiff` or + `stats_time_series`, then pass one or more instances to `AreaDetector` as + positional `*writer_factories` arguments. When the detector is initialised `__call__` is invoked with the detector's PV `prefix`, its `driver`, and the - flat list of extra `plugins`; it returns the writer device and the - corresponding data logic. + flat list of extra `plugins`; it returns the writer device, or `None` if the + factory makes none, and the corresponding data logic. + + Only the factories of file writers make a device. The others take the + plugins they use by reference. - :param writer_cls: Concrete `NDPluginBaseIO` subclass to instantiate. + :param writer_cls: + Concrete `NDPluginBaseIO` subclass to instantiate, or `None` if the + factory makes no plugin. :param writer_suffix: PV suffix appended to *prefix* to form the writer's PV prefix. :param writer_name: Attribute name under which the writer device is stored on the `AreaDetector` instance. Each static method defaults to its own name (``"hdf"``, ``"jpeg"``, ``"tiff"``); override when passing multiple - factories so each writer gets a distinct name. + factories so each writer gets a distinct name. Unused without a + `writer_cls`. :param datakey_suffix: Suffix appended to the datakey name in stream resources. :param array_description: Override the array shape/type description built from the driver. @@ -360,9 +419,16 @@ class ADDataFactory(Generic[NDPluginBaseIOT]): :param data_logic_factory: Callable ``(writer, array_description, driver, plugins) → data logic`` that builds the data logic given the already-constructed writer. + :param ndattribute_sources: + The plugins whose NDAttributes the data logic writes. `AreaDetector` + checks that they are in its device tree and that no two factories list + the same one. + :param plugin: + The plugin the data logic reads, when the factory makes none and the + logic is built from it. `AreaDetector` checks it is in its device tree. """ - writer_cls: type[NDPluginBaseIOT] + writer_cls: type[NDPluginBaseIOT] | None writer_suffix: str writer_name: str datakey_suffix: str @@ -373,22 +439,30 @@ class ADDataFactory(Generic[NDPluginBaseIOT]): [NDPluginBaseIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], BoundedDataLogic | UnboundedDataLogic, ] + ndattribute_sources: Sequence[NDAttributeSource] = () + plugin: NDPluginBaseIOT | None = None def __call__( self, prefix: str, driver: ADBaseIO, plugins: Sequence[NDPluginBaseIO], - ) -> tuple[NDPluginBaseIOT, BoundedDataLogic | UnboundedDataLogic]: - """Instantiate the writer plugin and build the data logic. + ) -> tuple[NDPluginBaseIOT | None, BoundedDataLogic | UnboundedDataLogic]: + """Instantiate the writer plugin, if any, and build the data logic. :param prefix: EPICS PV prefix for the detector (same as `AreaDetector.prefix`). :param driver: The detector driver, used to read array shape/type metadata. :param plugins: Additional plugins whose NDAttribute XML files should be included in HDF5 metadata. - :return: ``(writer, data_logic)`` tuple ready to attach to the detector. + :return: ``(writer, data_logic)`` tuple ready to attach to the detector, + where ``writer`` is `None` if the factory makes no plugin. """ - writer = self.writer_cls(prefix + self.writer_suffix) + writer = ( + self.writer_cls(prefix + self.writer_suffix) if self.writer_cls else None + ) + used = self.plugin if writer is None else writer + if used is None: + raise ValueError("A data factory needs a writer_cls or a plugin") if callable(self.array_description): array_description = self.array_description(driver) elif self.array_description is not None: @@ -403,7 +477,7 @@ def __call__( data_type_signal=driver.data_type, color_mode_signal=driver.color_mode, ) - data_logic = self.data_logic_factory(writer, array_description, driver, plugins) + data_logic = self.data_logic_factory(used, array_description, driver, plugins) return writer, data_logic @staticmethod @@ -417,6 +491,7 @@ def hdf( | None = None, hinted: bool = True, always_enable: bool = True, + ndattribute_sources: Sequence[NDAttributeSource] = (), ) -> "ADDataFactory[NDFileHDF5IO]": """Create a factory for an HDF5 file writer. @@ -433,10 +508,16 @@ def hdf( when the shape/type comes from a plugin rather than the main driver (e.g. an ROI plugin). :param hinted: Include the image in hints. Set False for BestEffortCallback. - :param always_enable: True (the default) serves every scan. False sits - out the scans of a stage if the plugin's EnableCallbacks PV is off at - the first prepare, so another data logic can serve them. Starting - switches the plugin on either way. + :param always_enable: True (the default) switches the plugin on at + stage, so it serves every scan. False sits out the scans of a stage + if the plugin's EnableCallbacks PV is off at stage, so another data + logic can serve them. + :param ndattribute_sources: The plugins whose NDAttributes this writer + writes into the file, as datasets named after what each publishes, + e.g. ``det-stats-total``. Each must be in the detector's device tree + and be listed by one writer only. The NDAttributes of the driver and + of the detector's `plugins` that are not listed are parsed from + their ``nd_attributes_file`` as the user set it. """ return ADDataFactory( writer_cls=NDFileHDF5IO, @@ -450,10 +531,12 @@ def hdf( writer=writer, driver=driver, plugins=list(plugins), + ndattribute_sources=ndattribute_sources, datakey_suffix=datakey_suffix, hinted=hinted, always_enable=always_enable, ), + ndattribute_sources=ndattribute_sources, ) @staticmethod @@ -482,10 +565,10 @@ def jpeg( Pass an `NDArrayDescription` or a callable ``(driver) → NDArrayDescription`` when the shape/type comes from a plugin. :param hinted: Include the image in hints. Set False for BestEffortCallback. - :param always_enable: True (the default) serves every scan. False sits - out the scans of a stage if the plugin's EnableCallbacks PV is off at - the first prepare, so another data logic can serve them. Starting - switches the plugin on either way. + :param always_enable: True (the default) switches the plugin on at + stage, so it serves every scan. False sits out the scans of a stage + if the plugin's EnableCallbacks PV is off at stage, so another data + logic can serve them. """ return ADDataFactory( writer_cls=NDPluginFileIO, @@ -533,10 +616,10 @@ def tiff( Pass an `NDArrayDescription` or a callable ``(driver) → NDArrayDescription`` when the shape/type comes from a plugin. :param hinted: Include the image in hints. Set False for BestEffortCallback. - :param always_enable: True (the default) serves every scan. False sits - out the scans of a stage if the plugin's EnableCallbacks PV is off at - the first prepare, so another data logic can serve them. Starting - switches the plugin on either way. + :param always_enable: True (the default) switches the plugin on at + stage, so it serves every scan. False sits out the scans of a stage + if the plugin's EnableCallbacks PV is off at stage, so another data + logic can serve them. """ return ADDataFactory( writer_cls=NDPluginFileIO, @@ -558,6 +641,33 @@ def tiff( ), ) + @staticmethod + def stats_time_series(stats: NDStatsIO) -> "ADDataFactory[NDStatsIO]": + """Create a factory for an NDPluginStats time series. + + Unlike `hdf`, `jpeg` and `tiff`, this makes no plugin and writes no file: + it reads the time series of a stats plugin you give it, a fixed-length + buffer read back as event pages, for detectors that cannot write assets + (see `StatsTimeSeriesDataLogic`). Each series is named after its scalar + signal, e.g. ``det-stats-total``, and the plugin's EnableCallbacks and + compute switches decide which are published. + + :param stats: The stats plugin, which must be in the detector's device + tree, e.g. in its `plugins`. `AreaDetector` raises if it is not. + The same plugin can be listed in ``hdf(ndattribute_sources=)``. + """ + return ADDataFactory( + writer_cls=None, + writer_suffix="", + writer_name="", + datakey_suffix="", + array_description=None, + data_logic_factory=lambda used, desc, driver, plugins: ( + StatsTimeSeriesDataLogic(stats=used) + ), + plugin=stats, + ) + class ADWriterFactory(ADDataFactory[NDPluginBaseIOT]): """Deprecated name of `ADDataFactory`.""" diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index 7a0e1abec5..5f6605d7d1 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -1,4 +1,7 @@ -from collections.abc import Mapping, Sequence +import asyncio +import contextlib +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import cached_property from typing import Generic @@ -7,6 +10,7 @@ DetectorAcquireLogic, DetectorLogic, DetectorTriggerLogic, + Device, SignalR, StandardDetector, UnboundedDataLogic, @@ -23,9 +27,67 @@ NDPluginBaseIOT, NDProcessIO, ) +from ._ndattribute import NDAttributeSource from ._trigger_logic import ADContAcqTriggerLogic +def _descendants(device: Device) -> Iterator[Device]: + """The device, then each of its children and theirs.""" + yield device + for _, child in device.children(): + yield from _descendants(child) + + +def _plugins_held_by(logic: object | None) -> list[NDPluginBaseIO]: + """The plugins a logic holds, which it switches on itself when it prepares.""" + held = vars(logic).values() if logic is not None else () + return [value for value in held if isinstance(value, NDPluginBaseIO)] + + +@dataclass(frozen=True, kw_only=True, eq=False) +class _ADDetectorLogic(DetectorLogic): + """A `DetectorLogic` that holds its plugins' recording from stage to unstage. + + :param plugins: the plugins to record at stage, check at each prepare and + forget at unstage. The recording is taken once, so a plugin the + detector's own logics switch on at prepare is not one of these. + """ + + plugins: Sequence[NDPluginBaseIO] = () + + async def ensure_ready(self) -> None: + # After the data logics, which may switch their plugins on + await super().ensure_ready() + await asyncio.gather(*(plugin.snapshot() for plugin in self.plugins)) + + # Parameter and return types are those of DetectorLogic.prepare_to_fly, + # whose context type is not public + async def prepare_to_fly(self, name, value, previous_ctx): + await self._raise_if_recording_changed(name) + steps = super().prepare_to_fly(name, value, previous_ctx) + async with contextlib.aclosing(steps): + async for step in steps: + yield step + + async def _raise_if_recording_changed(self, name: str) -> None: + changed_per_plugin = await asyncio.gather( + *(plugin.get_changed_signals() for plugin in self.plugins) + ) + sources = sorted( + signal.source for changed in changed_per_plugin for signal in changed + ) + if sources: + raise RuntimeError( + f"{name}: plugin switches changed since stage(): {sources}. " + "unstage() and stage() to use the new values." + ) + + async def ensure_stopped(self) -> None: + await super().ensure_stopped() + for plugin in self.plugins: + plugin.forget_snapshot() + + class AreaDetector(StandardDetector, Generic[ADBaseIOT]): def __init__( self, @@ -44,31 +106,101 @@ def __init__( self._set_unique(plugin_name, plugin) self._trigger_logic = trigger_logic self._acquire_logic = acquire_logic + plugin_list = list(plugins.values()) if plugins else [] + writers: list[NDPluginBaseIO] = [] data_logics: list[BoundedDataLogic | UnboundedDataLogic] = [] if writer_factories: if prefix is None: raise ValueError("prefix is required when writer_factories are given") - names = [f.writer_name for f in writer_factories] + names = [f.writer_name for f in writer_factories if f.writer_cls] if len(names) != len(set(names)): duplicates = sorted({n for n in names if names.count(n) > 1}) raise ValueError( f"Duplicate writer_name(s) in writer_factories: {duplicates}" ) - plugin_list = list(plugins.values()) if plugins else [] for factory in writer_factories: writer, data_logic = factory(prefix, driver, plugin_list) - self._set_unique(factory.writer_name, writer) + if writer is not None: + self._set_unique(factory.writer_name, writer) + writers.append(writer) data_logics.append(data_logic) self._data_logics = tuple(data_logics) + tree = list(_descendants(self)) + referenced = self._check_referenced_plugins(writer_factories, tree) + sources = self._check_ndattribute_sources(writer_factories, tree) + # The plugins the trigger logic holds are switched on by it at prepare, so + # they are not the user's switches to record + held = _plugins_held_by(trigger_logic) + self._used_plugins = tuple( + plugin + for plugin in self._find_plugins( + [*plugin_list, *writers, *sources, *referenced] + ) + if not any(plugin is owned for owned in held) + ) # The driver is always wired up, so register it whole and let ADBaseIO - # declare which of its signals are configuration. Plugins are - # deliberately *not* registered: a detector normally carries far more - # plugins than are wired into the chain, so opt in per plugin with - # `set_readable_format(det.stats, Format.CHILD)` instead. + # declare which of its signals are configuration. So is each of the + # plugins, whose own formats declare what they read. self.set_readable_format(self.driver, Format.CHILD) + for plugin in plugin_list: + self.set_readable_format(plugin, Format.CHILD) for signal in config_sigs: self.set_readable_format(signal, Format.CONFIG_SIGNAL) super().__init__(name=name) + self._suppressed_datakeys_funcs += (self.__on_suppressed_datakeys,) + + def __on_suppressed_datakeys(self) -> Mapping[str, None]: + """The names of the readables the plugins' switches turned off at stage.""" + return { + signal.name: None + for plugin in self._used_plugins + for signal in plugin.get_switched_off_readables() + } + + def _check_referenced_plugins( + self, writer_factories: Sequence[ADDataFactory], tree: Sequence[Device] + ) -> list[NDPluginBaseIO]: + """The plugins the factories take by reference, raising if one is missing.""" + referenced = [f.plugin for f in writer_factories if f.plugin is not None] + for plugin in referenced: + if not any(plugin is device for device in tree): + raise ValueError( + f"{plugin!r} is used by a data factory but is not in the " + "device tree of the detector: pass it, or the plugin it " + "is a child of, in plugins=" + ) + return referenced + + def _check_ndattribute_sources( + self, writer_factories: Sequence[ADDataFactory], tree: Sequence[Device] + ) -> list[NDAttributeSource]: + """The NDAttribute sources of the factories, raising if one is misplaced.""" + sources: list[NDAttributeSource] = [] + for factory in writer_factories: + for source in factory.ndattribute_sources: + if any(source is listed for listed in sources): + raise ValueError( + f"{source!r} is listed more than once in ndattribute_sources" + ) + if not any(source is device for device in tree): + raise ValueError( + f"{source!r} is in ndattribute_sources but is not in the " + "device tree of the detector: pass it, or the plugin it " + "is a child of, in plugins=" + ) + sources.append(source) + return sources + + def _find_plugins(self, roots: Sequence[object]) -> tuple[NDPluginBaseIO, ...]: + """Each plugin that is a root or a descendant of one, once.""" + found = { + device: None + for root in roots + if isinstance(root, Device) + for device in _descendants(root) + if isinstance(device, NDPluginBaseIO) + } + return tuple(found) def _set_unique(self, attr: str, value: NDPluginBaseIO) -> None: if attr in self.__dict__ or hasattr(type(self), attr): @@ -80,10 +212,11 @@ def _set_unique(self, attr: str, value: NDPluginBaseIO) -> None: @cached_property def logic(self) -> DetectorLogic: - return DetectorLogic( + return _ADDetectorLogic( trigger=self._trigger_logic, acquire=self._acquire_logic, data=self._data_logics, + plugins=self._used_plugins, ) def get_plugin( diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 6463b5fbb3..bd51e2405c 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -1,7 +1,16 @@ +import asyncio +import re +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from functools import cached_property from typing import Annotated as A -from typing import TypeVar +from typing import Any, TypeVar + +import numpy as np from ophyd_async.core import ( + Array1D, DeviceVector, EnableDisable, SignalR, @@ -16,6 +25,8 @@ from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.epics.core import EpicsDevice, EpicsOptions, PvSuffix +from ._ndattribute import NDAttributeDataType, NDAttributeParam + # Common classes for drivers and plugins @@ -168,6 +179,60 @@ class NDPluginBaseIO(NDArrayBaseIO): nd_array_address: A[SignalRW[int], PvSuffix.rbv("NDArrayAddress")] queue_size: A[SignalRW[int], PvSuffix.rbv("QueueSize")] + @cached_property + def _snapshot(self) -> dict[SignalRW[Any], Any]: + return {} + + def get_switches(self) -> Sequence[SignalRW[Any]]: + """The PVs that switch what this plugin produces on and off.""" + return [self.enable_callbacks] + + def get_snapshot_signals(self) -> Sequence[SignalRW[Any]]: + """The PVs `snapshot()` records: the switches and the NDArrayPort. + + Only this plugin's own NDArrayPort is recorded. The chain of plugins that + feeds it, which may cross IOCs, is not walked: the chain as wired at + `stage()` is the user's responsibility. + """ + return [*self.get_switches(), self.nd_array_port] + + async def snapshot(self) -> None: + """Record the value of each of `get_snapshot_signals()`. + + `AreaDetector` calls this on `stage()`, once. The values stay until + `forget_snapshot()`. + """ + signals = self.get_snapshot_signals() + values = await asyncio.gather(*(signal.get_value() for signal in signals)) + self._snapshot.clear() + self._snapshot.update(zip(signals, values, strict=True)) + + async def get_changed_signals(self) -> list[SignalRW[Any]]: + """The recorded signals whose PV has a different value now. + + `AreaDetector` calls this on each `prepare()`. + """ + recorded = list(self._snapshot.items()) + current = await asyncio.gather(*(signal.get_value() for signal, _ in recorded)) + return [ + signal + for (signal, value), now in zip(recorded, current, strict=True) + if now != value + ] + + def forget_snapshot(self) -> None: + """Drop the recorded values. `AreaDetector` calls this on `unstage()`.""" + self._snapshot.clear() + + def switch_is_on(self, switch: SignalRW[Any]) -> bool: + """Whether `switch` was on when recorded, True if it was not recorded.""" + value = self._snapshot.get(switch, True) + return value is True or value is EnableDisable.ENABLE + + def get_switched_off_readables(self) -> set[SignalR]: + """The readable signals the recorded switches turn off.""" + return set() + async def plugin_is_enabled(plugin: NDPluginBaseIO) -> bool: """Whether a plugin is switched on, so is receiving NDArrays. @@ -198,6 +263,41 @@ class NDROIIO(StandardReadable, NDPluginBaseIO): bin_z: A[SignalRW[int], PvSuffix.rbv("BinZ")] +class NDStatsTSAcquireMode(StrictEnum): + """Whether an NDPluginStats time series stops when full or wraps around. + + Mirrors the ``TSAcquireMode`` record in + ADCore/db/NDPluginTimeSeries.template. + """ + + FIXED_LENGTH = "Fixed length" + CIRCULAR_BUFFER = "Circ. buffer" + + +# The formats that put a signal in read(), rather than in configuration +_READ_FORMATS = ( + Format.HINTED_SIGNAL, + Format.UNCACHED_SIGNAL, + Format.HINTED_UNCACHED_SIGNAL, +) + + +@dataclass(frozen=True) +class NDStatistic: + """One statistic of an `NDStatsIO`, in each form the plugin produces it. + + :param scalar: the value for the last frame + :param param: the asyn parameter, the source of an NDAttribute + :param time_series: the value for each frame of the time series + :param compute: the switch for the group of statistics this belongs to + """ + + scalar: SignalR[float] + param: str + time_series: SignalR[Array1D[np.float64]] + compute: SignalRW[bool] + + class NDStatsIO(StandardReadable, NDPluginBaseIO): """Plugin for computing statistics from an image or ROI within an image. @@ -226,6 +326,36 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] net: A[SignalR[float], PvSuffix("Net_RBV")] + # Time series. NDStats delegates its time series to an embedded + # NDPluginTimeSeries instance, so the controls live under an inner "TS:" + # prefix while the per-statistic arrays are named by NDStats itself at the + # plugin's own prefix. ts_num_points sizes the buffer, ts_acquire erases and + # starts it (writing 1 clears the arrays and resets ts_current_point to 0), + # ts_current_point reports progress, each ts_ holds the series of + # that statistic and ts_timestamp holds the per-point acquisition times. + # These are not registered as readables: the scalars serve read(), and the + # arrays are consumed by StatsTimeSeriesDataLogic as event pages. + ts_acquire: A[SignalRW[bool], PvSuffix("TS:TSAcquire"), EpicsOptions(wait=non_zero)] + ts_num_points: A[SignalRW[int], PvSuffix("TS:TSNumPoints")] + ts_current_point: A[SignalR[int], PvSuffix("TS:TSCurrentPoint")] + ts_acquiring: A[SignalR[bool], PvSuffix("TS:TSAcquiring")] + ts_acquire_mode: A[SignalRW[NDStatsTSAcquireMode], PvSuffix.rbv("TS:TSAcquireMode")] + ts_min_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMinValue")] + ts_min_x: A[SignalR[Array1D[np.float64]], PvSuffix("TSMinX")] + ts_min_y: A[SignalR[Array1D[np.float64]], PvSuffix("TSMinY")] + ts_max_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMaxValue")] + ts_max_x: A[SignalR[Array1D[np.float64]], PvSuffix("TSMaxX")] + ts_max_y: A[SignalR[Array1D[np.float64]], PvSuffix("TSMaxY")] + ts_mean_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMeanValue")] + ts_sigma: A[SignalR[Array1D[np.float64]], PvSuffix("TSSigma")] + ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TSTotal")] + ts_net: A[SignalR[Array1D[np.float64]], PvSuffix("TSNet")] + ts_centroid_x: A[SignalR[Array1D[np.float64]], PvSuffix("TSCentroidX")] + ts_centroid_y: A[SignalR[Array1D[np.float64]], PvSuffix("TSCentroidY")] + ts_sigma_x: A[SignalR[Array1D[np.float64]], PvSuffix("TSSigmaX")] + ts_sigma_y: A[SignalR[Array1D[np.float64]], PvSuffix("TSSigmaY")] + ts_sigma_xy: A[SignalR[Array1D[np.float64]], PvSuffix("TSSigmaXY")] + ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] # Centroid statistics compute_centroid: A[ SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL @@ -254,6 +384,71 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin"), Format.CONFIG_SIGNAL] hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax"), Format.CONFIG_SIGNAL] + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._suppressed_datakeys_funcs += (self._get_switched_off_datakeys,) + + @cached_property + def statistics(self) -> tuple[NDStatistic, ...]: + """Each statistic, as ADCore's NDStats.template defines it.""" + basic = self.compute_statistics + centroid = self.compute_centroid + return ( + NDStatistic(self.min_value, "MIN_VALUE", self.ts_min_value, basic), + NDStatistic(self.min_x, "MIN_X", self.ts_min_x, basic), + NDStatistic(self.min_y, "MIN_Y", self.ts_min_y, basic), + NDStatistic(self.max_value, "MAX_VALUE", self.ts_max_value, basic), + NDStatistic(self.max_x, "MAX_X", self.ts_max_x, basic), + NDStatistic(self.max_y, "MAX_Y", self.ts_max_y, basic), + NDStatistic(self.mean_value, "MEAN_VALUE", self.ts_mean_value, basic), + NDStatistic(self.sigma, "SIGMA_VALUE", self.ts_sigma, basic), + NDStatistic(self.total, "TOTAL", self.ts_total, basic), + NDStatistic(self.net, "NET", self.ts_net, basic), + NDStatistic( + self.centroid_x, "CENTROIDX_VALUE", self.ts_centroid_x, centroid + ), + NDStatistic( + self.centroid_y, "CENTROIDY_VALUE", self.ts_centroid_y, centroid + ), + NDStatistic(self.sigma_x, "SIGMAX_VALUE", self.ts_sigma_x, centroid), + NDStatistic(self.sigma_y, "SIGMAY_VALUE", self.ts_sigma_y, centroid), + NDStatistic(self.sigma_xy, "SIGMAXY_VALUE", self.ts_sigma_xy, centroid), + ) + + def get_switches(self) -> Sequence[SignalRW[Any]]: + return [*super().get_switches(), self.compute_statistics, self.compute_centroid] + + def _is_computed(self, statistic: NDStatistic) -> bool: + return self.switch_is_on(self.enable_callbacks) and self.switch_is_on( + statistic.compute + ) + + def get_switched_off_readables(self) -> set[SignalR]: + return {stat.scalar for stat in self.statistics if not self._is_computed(stat)} + + def _get_switched_off_datakeys(self) -> dict[str, None]: + return {signal.name: None for signal in self.get_switched_off_readables()} + + def get_published_statistics(self) -> list[NDStatistic]: + """The statistics that are computed and have a read format on this plugin.""" + formats = self.get_readable_formats() + return [ + stat + for stat in self.statistics + if self._is_computed(stat) and formats.get(stat.scalar) in _READ_FORMATS + ] + + def get_ndattributes(self) -> list[NDAttributeParam]: + """Each published statistic, named after its scalar signal.""" + return [ + NDAttributeParam( + name=stat.scalar.name, + param=stat.param, + datatype=NDAttributeDataType.DOUBLE, + ) + for stat in self.get_published_statistics() + ] + class NDROIStatNIO(StandardReadable, EpicsDevice): """Defines the parameters for a single ROI used for statistics calculation. @@ -279,23 +474,114 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): ] total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] - -class NDROIStatIO(NDPluginBaseIO): + def get_ndattributes(self, addr: int) -> list[NDAttributeParam]: + """Each scalar with a read format, named after it, at asyn address `addr`.""" + formats = self.get_readable_formats() + return [ + NDAttributeParam( + name=scalar.name, + param=param, + datatype=NDAttributeDataType.DOUBLE, + addr=addr, + ) + for scalar, param in self.params.items() + if formats.get(scalar) in _READ_FORMATS + ] + + @cached_property + def params(self) -> dict[SignalR[float], str]: + """The asyn parameter each scalar signal of this ROI is read from.""" + return { + self.min_value: "ROISTAT_MIN_VALUE", + self.max_value: "ROISTAT_MAX_VALUE", + self.mean_value: "ROISTAT_MEAN_VALUE", + self.total: "ROISTAT_TOTAL", + } + + +class NDROIStatIO(StandardReadable, NDPluginBaseIO): """Plugin for calculating basic statistics for multiple ROIs. Each ROI is implemented as an instance of NDROIStatNIO, - and the collection of ROIs is held as a DeviceVector. + and the collection of ROIs is held as a DeviceVector. Each channel is + registered as a [](#StandardReadableFormat.CHILD) readable, so it reads the + scalars of its own formats. + + When an `AreaDetector` stages, each channel is renamed to + `-`, with characters other than letters, digits and + underscores in the PV replaced by underscores, or the channel number if the + PV is empty. A `Name` PV that changes after that makes `prepare()` raise. This mirrors the interface provided by ADCore/db/NDROIStat.template. See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROIStat.html """ def __init__(self, prefix, num_channels=8, with_pvi=False, name=""): - self.channels = DeviceVector( - {i: NDROIStatNIO(f"{prefix}{i}:") for i in range(1, num_channels + 1)} - ) + with self.add_children_as_readables(): + self.channels = DeviceVector( + {i: NDROIStatNIO(f"{prefix}{i}:") for i in range(1, num_channels + 1)} + ) super().__init__(prefix, with_pvi, name) + def get_switches(self) -> Sequence[SignalRW[Any]]: + channel_switches = [channel.use for channel in self.channels.values()] + return [*super().get_switches(), *channel_switches] + + def get_snapshot_signals(self) -> Sequence[SignalRW[Any]]: + channel_names = [channel.name_ for channel in self.channels.values()] + return [*super().get_snapshot_signals(), *channel_names] + + async def snapshot(self) -> None: + """Record the signals, and name each channel after its Name PV.""" + await super().snapshot() + self._name_channels() + + def _name_channels(self) -> None: + """Name each channel after its recorded Name PV, raising on a clash.""" + names = { + number: self._channel_name(number, channel) + for number, channel in self.channels.items() + } + counts = Counter(names.values()) + clashes = sorted(name for name, count in counts.items() if count > 1) + if clashes: + raise ValueError( + f"{self.name}: more than one channel is named {clashes}, " + "so change the Name PV of all but one" + ) + if self.name: + for number, name in names.items(): + self.channels[number].set_name( + f"{self.name}{self._child_name_separator}{name}", + child_name_separator=self._child_name_separator, + ) + + def _channel_name(self, number: int, channel: NDROIStatNIO) -> str: + pv_name = self._snapshot[channel.name_] + return re.sub(r"[^A-Za-z0-9_]", "_", pv_name) if pv_name else str(number) + + def _is_computed(self, channel: NDROIStatNIO) -> bool: + return self.switch_is_on(self.enable_callbacks) and self.switch_is_on( + channel.use + ) + + def get_switched_off_readables(self) -> set[SignalR]: + return { + scalar + for channel in self.channels.values() + if not self._is_computed(channel) + for scalar in channel.params + } + + def get_ndattributes(self) -> list[NDAttributeParam]: + """The NDAttributes of each channel that is computed, by asyn address.""" + return [ + ndattribute + for number, channel in self.channels.items() + if self._is_computed(channel) + for ndattribute in channel.get_ndattributes(addr=number - 1) + ] + class NDCBFlushOnSoftTrgMode(StrictEnum): ON_NEW_IMAGE = "OnNewImage" diff --git a/src/ophyd_async/epics/adcore/_ndattribute.py b/src/ophyd_async/epics/adcore/_ndattribute.py index 571196eabe..2e65b0a455 100644 --- a/src/ophyd_async/epics/adcore/_ndattribute.py +++ b/src/ophyd_async/epics/adcore/_ndattribute.py @@ -1,11 +1,12 @@ from collections.abc import Sequence from dataclasses import dataclass from enum import Enum +from typing import Protocol, runtime_checkable from xml.etree import ElementTree as ET import numpy as np -from ophyd_async.core import SignalR +from ophyd_async.core import SignalR, SignalRW class NDAttributeDataType(Enum): @@ -45,6 +46,21 @@ class NDAttributeParam: description: str = "" # A description that appears in the HDF file as an attribute +@runtime_checkable +class NDAttributeSource(Protocol): + """A plugin that says which NDAttributes it stamps on each frame. + + An HDF writer that lists it in `ADDataFactory.hdf(ndattribute_sources=)` + writes them to the plugin's `nd_attributes_file` and makes a dataset of each. + """ + + nd_attributes_file: SignalRW[str] + + def get_ndattributes(self) -> Sequence[NDAttributeParam]: + """The NDAttributes to stamp on each frame of this scan.""" + ... + + def ndattributes_to_xml( ndattributes: Sequence[NDAttributeParam | NDAttributePv], ) -> str: diff --git a/src/ophyd_async/epics/adcore/_plan_stubs.py b/src/ophyd_async/epics/adcore/_plan_stubs.py index 82dab00448..a23a0dc511 100644 --- a/src/ophyd_async/epics/adcore/_plan_stubs.py +++ b/src/ophyd_async/epics/adcore/_plan_stubs.py @@ -3,10 +3,8 @@ import bluesky.plan_stubs as bps from bluesky.utils import plan -from ._detector import AreaDetector -from ._io import NDArrayBaseIO, NDStatsIO +from ._io import NDArrayBaseIO from ._ndattribute import ( - NDAttributeDataType, NDAttributeParam, NDAttributePv, ndattributes_to_xml, @@ -24,22 +22,3 @@ def setup_ndattributes( xml, wait=True, ) - - -@plan -def setup_ndstats_sum(detector: AreaDetector, stats_name: str = "stats"): - """Set up nd stats sum nd attribute for a detector.""" - stats = detector.get_plugin(stats_name, NDStatsIO) - yield from ( - setup_ndattributes( - stats, - [ - NDAttributeParam( - name=f"{detector.name}-sum", - param="TOTAL", - datatype=NDAttributeDataType.DOUBLE, - description="Sum of the array", - ) - ], - ) - ) diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py new file mode 100644 index 0000000000..d37b2a67a3 --- /dev/null +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -0,0 +1,94 @@ +import asyncio +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np + +from ophyd_async.core import ( + DEFAULT_TIMEOUT, + Array1D, + BoundedDataLogic, + EventPageDataProvider, + SignalR, + set_and_wait_for_value, + wait_for_value, +) +from ophyd_async.epics.core import stop_busy_record + +from ._io import NDStatsIO, NDStatsTSAcquireMode + + +@dataclass +class StatsTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): + """Data logic publishing an `NDPluginStats` time series as event pages. + + For detectors that write no file: the stats plugin holds each statistic in a + fixed-length buffer that must be sized before acquisition, so this logic + raises for a scan with no fixed number of collections, unless the unbounded + data logics already make all its datakeys. One plugin has one time-series + control (`ts_acquire`, `ts_num_points`) shared across many arrays, so one + logic covers many arrays with the control embedded. + + It publishes the time series of each of `NDStatsIO.get_published_statistics` + under the name of its scalar signal, so the plugin's EnableCallbacks and + compute switches decide what is published, and this logic never writes them. + It hints nothing: a hinted scalar of a plugin registered as a readable stays + in the hints while its series replaces it. + + The buffer is sized and armed in `start_provider`, which the detector calls + again at each step point after the first, by writing 1 to ``ts_acquire``, + which erases the arrays and resets the current point before the detector's + acquire logic drives the camera; its frames then feed the time series via + NDArray callbacks. A detector that writes a file should pull stats into the + file as NDAttributes instead (see `ADHDFDataLogic`). + + :param stats: the stats plugin whose time series to read + """ + + stats: NDStatsIO + + def _arrays(self) -> dict[str, SignalR[Array1D[np.float64]]]: + return { + stat.scalar.name: stat.time_series + for stat in self.stats.get_published_statistics() + } + + def __post_init__(self) -> None: + self.collections_written = self.stats.ts_current_point + + async def start_provider( + self, + name: str, + existing_datakeys: Sequence[str], + num_collections: int, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> EventPageDataProvider | None: + # The buffer is sized by count, not rate, so neither period is used + arrays = { + datakey: signal + for datakey, signal in self._arrays().items() + if datakey not in existing_datakeys + } + if not arrays: + return None + # Size the buffer and put it in fixed-length mode before arming it + await asyncio.gather( + self.stats.ts_num_points.set(num_collections), + self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), + ) + # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to + # 0. The put completes when the buffer is full, so do not wait for it. + await set_and_wait_for_value( + self.stats.ts_acquire, True, wait_for_set_completion=False + ) + # The IOC resets the count within the put, but its monitor may arrive + # after the put's readback, and the detector reads the count next + await wait_for_value(self.stats.ts_current_point, 0, timeout=DEFAULT_TIMEOUT) + return EventPageDataProvider( + arrays, self.stats.ts_timestamp, collections_per_event + ) + + async def ensure_stopped(self) -> None: + await stop_busy_record(self.stats.ts_acquire) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index 4035bd0f90..4e2fa0b8ec 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -50,6 +50,7 @@ ) from ophyd_async.core import ( + EnableDisable, StaticFilenameProvider, StaticPathProvider, TriggerInfo, @@ -114,6 +115,28 @@ def adsim(RE: RunEngine, shared_tmp_path: Path) -> AreaDetector: return adsim +@pytest.fixture +def adsimstat(RE: RunEngine) -> AreaDetector: + """A writer-less SimDetector whose only data logic is the STAT time series. + + The stats plugin is given to the factory by reference, and listed in `plugins` + so it is in the detector's tree. It writes no file: the NDPluginStats time + series is a fixed-length buffer read back as event pages, so this exercises + the bounded data-logic tier against a real IOC (the + `ADDataFactory.stats_time_series` factory and NDStatsIO's `TS:*` signals). + """ + stats = adcore.NDStatsIO("BL01T-DI-CAM-01:STAT:") + with init_devices(): + adsimstat = SimDetector( + "BL01T-DI-CAM-01:", + adcore.ADDataFactory.stats_time_series(stats), + driver_suffix="DET:", + plugins={"stats": stats}, + ) + + return adsimstat + + @pytest.fixture(autouse=True) def reset_adsim_to_baseline(RE: RunEngine, adsim: SimDetector) -> None: """Put the detector back to a known state before every test. @@ -358,6 +381,63 @@ def test_software_triggering( ] +@pytest.mark.timeout(TIMEOUT + 15.0) +def test_stats_time_series_step_scan( + RE: RunEngine, adsimstat: SimDetector, bl01t_di_cam_01: None +) -> None: + """A writer-less stats detector arms the TS buffer, acquires, and reads the + per-frame Total series back as a single event holding the whole array. + + This drives the real STAT plugin's ``TS:*`` records end to end: prepare sizes + and arms ``TS:TSNumPoints``/``TS:TSAcquire``, the sim driver takes the frames, + and read() slices ``TSTotal`` (timestamped by ``TSTimestamp``) into a + single 5-element reading named after the ``Total_RBV`` scalar. + """ + collections = 5 + stats = adsimstat.get_plugin("stats", adcore.NDStatsIO) + # The plugin's switches at stage decide which statistics are published + RE( + bps.mv( + stats.enable_callbacks, + EnableDisable.ENABLE, + stats.compute_statistics, + True, + ) + ) + + @bpp.stage_decorator([adsimstat]) + @bpp.run_decorator() + def prepare_then_count() -> MsgGenerator[None]: + # prepare() must come after stage(), which resets the detector + yield from bps.prepare( + adsimstat, + TriggerInfo(collections_per_event=collections, livetime=0.1), + wait=True, + ) + yield from bps.trigger_and_read([adsimstat]) + + named_docs: list[tuple[str, DocumentType]] = [] + RE(prepare_then_count(), lambda name, doc: named_docs.append((name, doc))) + + # A bounded logic exposes collect_pages, never collect_asset_docs. + assert hasattr(adsimstat, "collect_pages") + assert not hasattr(adsimstat, "collect_asset_docs") + + descriptor = next(doc for name, doc in named_docs if name == "descriptor") + data_key = descriptor["data_keys"]["adsimstat-stats-total"] + assert data_key["shape"] == [collections] + assert data_key["dtype"] == "array" + assert data_key["dtype_numpy"] == "= 0 + + def run_plan_and_get_documents( RE: RunEngine, plan: MsgGenerator[Any] ) -> list[DocumentType]: diff --git a/tests/unit_tests/epics/adcore/test_cont_acq_detector.py b/tests/unit_tests/epics/adcore/test_cont_acq_detector.py index a8472fb9de..7e845bdf28 100644 --- a/tests/unit_tests/epics/adcore/test_cont_acq_detector.py +++ b/tests/unit_tests/epics/adcore/test_cont_acq_detector.py @@ -228,3 +228,19 @@ def soft_trigger(value: bool) -> bool | None: {"start": 2, "stop": 5} ] await det.unstage() + + +async def test_cont_acq_prepare_again_after_it_switched_the_circular_buffer_on( + cont_acq_detector: adcore.AreaDetector[adcore.ADBaseIO], +): + set_mock_value(cont_acq_detector.cb.enable_callbacks, EnableDisable.DISABLE) + await cont_acq_detector.stage() + # The first trigger prepares, switching the circular buffer on + await cont_acq_detector.trigger() + assert ( + await cont_acq_detector.cb.enable_callbacks.get_value() is EnableDisable.ENABLE + ) + + # That is not a switch changed by someone else since stage() + await cont_acq_detector.prepare(TriggerInfo()) + await cont_acq_detector.trigger() diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 7679e29082..aec9cae2c0 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -98,9 +98,10 @@ async def test_hdf_follows_the_plugin_when_not_enabling_it( always_enable=always_enable, ) + await logic.ensure_ready() made = await logic.start_provider("det", 1, 0.1, 0) assert (made is not None) is makes_provider - # Starting switches the plugin on either way, and sitting out writes nothing + # Staging switches the plugin on only if told to, and sitting out writes nothing assert await writer.capture.get_value() is makes_provider assert await writer.enable_callbacks.get_value() is ( EnableDisable.ENABLE if makes_provider else EnableDisable.DISABLE @@ -156,7 +157,6 @@ async def test_prepare_hdf( call.hdf.lazy_open.put(True), call.hdf.swmr_mode.put(True), call.hdf.xml_file_name.put(""), - call.hdf.enable_callbacks.put(EnableDisable.ENABLE), call.hdf.create_directory.put(0), call.hdf.file_path.put(f"{static_path_provider().directory_path}{os.sep}"), call.hdf.file_name.put("ophyd_async_tests"), @@ -193,7 +193,6 @@ async def test_prepare_multipart( writer, [ call.lazy_open.put(True), - call.enable_callbacks.put(EnableDisable.ENABLE), call.create_directory.put(0), call.file_path.put(f"{static_path_provider().directory_path}{os.sep}"), call.file_name.put("ophyd_async_tests"), @@ -501,3 +500,49 @@ async def test_multipart_start_waits_for_num_captured_to_read_zero( mimetype="multipart/related;type=image/tiff", ) await _start_waits_for_a_zero_counter(logic, writer.num_captured) + + +async def test_hdf_writes_the_ndattributes_of_its_sources_and_parses_the_others( + static_path_provider: StaticPathProvider, +): + listed = adcore.NDStatsIO("PREFIX:LISTED:") + unlisted = adcore.NDStatsIO("PREFIX:UNLISTED:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADDataFactory.hdf( + static_path_provider, ndattribute_sources=[listed] + ), + plugins={"listed": listed, "unlisted": unlisted}, + name="det", + ) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + set_mock_value(det.get_plugin("hdf", adcore.NDFileHDF5IO).file_path_exists, True) + # Opt the unlisted plugin out of being read, so only the file could name it + det.set_readable_format(unlisted, None) + for stats in (listed, unlisted): + set_mock_value(stats.enable_callbacks, EnableDisable.ENABLE) + set_mock_value(stats.compute_statistics, True) + # Names that the unlisted plugin was told to stamp are kept as they are + unlisted_attributes = [ + adcore.NDAttributeParam( + name="free-form", param="TOTAL", datatype=adcore.NDAttributeDataType.DOUBLE + ) + ] + set_mock_value( + unlisted.nd_attributes_file, adcore.ndattributes_to_xml(unlisted_attributes) + ) + await det.stage() + await det.prepare(TriggerInfo()) + + datakeys = set(await det.describe()) + + assert "det-listed-total" in datakeys + assert "free-form" in datakeys + assert not [key for key in datakeys if key.startswith("det-unlisted")] + # The listed plugin's file was written, the unlisted plugin's left as set + assert "det-listed-total" in await listed.nd_attributes_file.get_value() + assert await unlisted.nd_attributes_file.get_value() == ( + adcore.ndattributes_to_xml(unlisted_attributes) + ) diff --git a/tests/unit_tests/epics/adcore/test_detector.py b/tests/unit_tests/epics/adcore/test_detector.py index 123c1ceff4..a1e269c3fb 100644 --- a/tests/unit_tests/epics/adcore/test_detector.py +++ b/tests/unit_tests/epics/adcore/test_detector.py @@ -1,7 +1,6 @@ import pytest from ophyd_async.core import ( - StandardReadableFormat, StaticPathProvider, TriggerInfo, init_devices, @@ -20,21 +19,31 @@ ) @pytest.mark.parametrize("hinted", [True, False]) def test_writer_image_hints(static_path_provider: StaticPathProvider, factory, hinted): - driver = adcore.ADBaseIO("PREFIX:DRV:") stats = adcore.NDStatsIO("PREFIX:STAT:") - det = adcore.AreaDetector( - driver, + det_with_stats = adcore.AreaDetector( + adcore.ADBaseIO("PREFIX:DRV:"), "PREFIX:", factory(static_path_provider, datakey_suffix="-image", hinted=hinted), plugins={"stats": stats}, name="det", ) + det = adcore.AreaDetector( + adcore.ADBaseIO("PREFIX:DRV2:"), + "PREFIX:", + factory(static_path_provider, datakey_suffix="-image", hinted=hinted), + name="det2", + ) # No hinted fields means no hints at all - assert det.hints == ({"fields": ["det-image"]} if hinted else {}) - det.set_readable_format(stats.total, StandardReadableFormat.HINTED_SIGNAL) - # Readable-format signals are hinted ahead of the data logics' fields - assert det.hints == { - "fields": ["det-stats-total"] + (["det-image"] if hinted else []) + assert det.hints == ({"fields": ["det2-image"]} if hinted else {}) + # A plugin in plugins= is a readable, hinted ahead of the data logics' fields + assert det_with_stats.hints == { + "fields": [ + "det-stats-min_value", + "det-stats-max_value", + "det-stats-mean_value", + "det-stats-total", + *(["det-image"] if hinted else []), + ] } @@ -121,6 +130,94 @@ def test_plugin_named_like_the_driver_raises(): ) +def test_area_detector_rejects_an_ndattribute_source_listed_twice( + static_path_provider: StaticPathProvider, +): + stats = adcore.NDStatsIO("PREFIX:STATS:") + + with pytest.raises( + ValueError, match="listed more than once in ndattribute_sources" + ): + adcore.AreaDetector( + adcore.ADBaseIO("PREFIX:DRV:"), + "PREFIX:", + adcore.ADDataFactory.hdf( + static_path_provider, writer_name="hdf1", ndattribute_sources=[stats] + ), + adcore.ADDataFactory.hdf( + static_path_provider, writer_name="hdf2", ndattribute_sources=[stats] + ), + plugins={"stats": stats}, + name="det", + ) + + +def test_area_detector_rejects_an_ndattribute_source_outside_its_device_tree( + static_path_provider: StaticPathProvider, +): + stats = adcore.NDStatsIO("PREFIX:STATS:") + + with pytest.raises(ValueError, match="not in the device tree of the detector"): + adcore.AreaDetector( + adcore.ADBaseIO("PREFIX:DRV:"), + "PREFIX:", + adcore.ADDataFactory.hdf(static_path_provider, ndattribute_sources=[stats]), + name="det", + ) + + +@pytest.mark.parametrize("in_tree", [False, True]) +def test_stats_time_series_takes_a_plugin_that_is_in_the_device_tree(in_tree: bool): + stats = adcore.NDStatsIO("PREFIX:STATS:") + plugins = {"stats": stats} if in_tree else {} + + def make_detector(): + return adcore.AreaDetector( + adcore.ADBaseIO("PREFIX:DRV:"), + "PREFIX:", + adcore.ADDataFactory.stats_time_series(stats), + plugins=plugins, + name="det", + ) + + if in_tree: + det = make_detector() + # It made no plugin of its own + assert [name for name, _ in det.children()] == ["driver", "stats"] + else: + with pytest.raises(ValueError, match="not in the device tree of the detector"): + make_detector() + + +def test_a_data_factory_needs_a_writer_class_or_a_plugin(): + factory = adcore.ADDataFactory( + writer_cls=None, + writer_suffix="", + writer_name="", + datakey_suffix="", + array_description=None, + data_logic_factory=lambda used, desc, driver, plugins: None, # type: ignore + ) + + with pytest.raises(ValueError, match="needs a writer_cls or a plugin"): + factory("PREFIX:", adcore.ADBaseIO("PREFIX:DRV:"), []) + + +def test_area_detector_accepts_an_ndattribute_source_below_a_plugin( + static_path_provider: StaticPathProvider, +): + roi = adcore.NDROIIO("PREFIX:ROI:") + roi.stat = adcore.NDStatsIO("PREFIX:STATS:") + + adcore.AreaDetector( + adcore.ADBaseIO("PREFIX:DRV:"), + "PREFIX:", + adcore.ADDataFactory.hdf(static_path_provider, ndattribute_sources=[roi.stat]), + plugins={"roi": roi}, + name="det", + ) + + def test_deprecated_writer_factory_still_builds_the_new_factories( static_path_provider: StaticPathProvider, ): diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py new file mode 100644 index 0000000000..af81f8b46c --- /dev/null +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -0,0 +1,319 @@ +import asyncio +from collections import defaultdict +from typing import get_type_hints +from unittest.mock import call + +import bluesky.plan_stubs as bps +import bluesky.preprocessors as bpp +import numpy as np +import pytest +from bluesky import RunEngine + +from ophyd_async.core import ( + Array1D, + DetectorAcquireLogic, + DetectorLogic, + DetectorTriggerLogic, + EnableDisable, + EventPageDataProvider, + SignalR, + StandardDetector, + TriggerInfo, + callback_on_mock_put, + get_mock_put, + init_devices, + non_zero, + set_mock_put_proceeds, + set_mock_value, +) +from ophyd_async.core import StandardReadableFormat as Format +from ophyd_async.epics import adcore +from ophyd_async.epics.adcore import ( + NDStatsTSAcquireMode, + StatsTimeSeriesDataLogic, +) +from ophyd_async.epics.core import EpicsOptions +from ophyd_async.testing import assert_has_calls + + +@pytest.fixture +async def stats() -> adcore.NDStatsIO: + async with init_devices(mock=True): + plugin = adcore.NDStatsIO("PREFIX:STATS:", name="det-stats") + return plugin + + +# The statistics NDStatsIO gives a read format, so publishes with switches on +PUBLISHED = [ + "det-stats-min_value", + "det-stats-min_x", + "det-stats-min_y", + "det-stats-max_value", + "det-stats-max_x", + "det-stats-max_y", + "det-stats-mean_value", + "det-stats-total", +] + + +async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + provider = await logic.start_provider("det-stats", [], 5, 5, 0.1, 0) + assert isinstance(provider, EventPageDataProvider) + assert await stats.ts_num_points.get_value() == 5 + assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH + # ts_acquire=1 arms and clears the buffer + assert await stats.ts_acquire.get_value() is True + # Each series is named after its scalar + datakeys = await provider.make_datakeys() + assert list(datakeys) == PUBLISHED + assert datakeys["det-stats-total"]["shape"] == [5] + + +def test_every_time_series_is_a_statistic(stats: adcore.NDStatsIO): + """Every ts_* array but the timestamps is in `statistics`, so can be published.""" + hints = get_type_hints(adcore.NDStatsIO) + arrays = { + signal + for attr, signal in stats.children() + if attr.startswith("ts_") + and attr != "ts_timestamp" + and hints[attr] == SignalR[Array1D[np.float64]] + } + assert arrays == {stat.time_series for stat in stats.statistics} + + +async def test_start_leaves_the_progress_counter_at_zero(stats: adcore.NDStatsIO): + """The count a step point is measured from is 0 once start_provider returns. + + A buffer that still holds a previous scan's points reports them until it is + armed; the IOC resets TSCurrentPoint inside the TSAcquire write (ADCore + NDPluginTimeSeries::writeInt32 -> acquireReset), which is what the fake does. + """ + set_mock_value(stats.ts_current_point, 5) + callback_on_mock_put( + stats.ts_acquire, + lambda value: set_mock_value(stats.ts_current_point, 0) if value else None, + ) + logic = StatsTimeSeriesDataLogic(stats) + + assert await logic.start_provider("det", [], 5, 1, 0.1, 0) is not None + + assert await stats.ts_current_point.get_value() == 0 + + +def test_ts_acquire_is_declared_as_a_busy_record(): + """TSAcquire completes its put only when the buffer is full, so only a put + of 1 may wait for completion, as for the other areaDetector busy records.""" + (options,) = [ + meta + for meta in get_type_hints(adcore.NDStatsIO, include_extras=True)[ + "ts_acquire" + ].__metadata__ + if isinstance(meta, EpicsOptions) + ] + assert options.wait is non_zero + + +async def test_start_does_not_wait_for_the_buffer_to_fill(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + # The put of 1 to TSAcquire does not complete until the buffer is full + set_mock_put_proceeds(stats.ts_acquire, False) + + await asyncio.wait_for(logic.start_provider("det", [], 5, 1, 0.1, 0), timeout=1) + + assert await stats.ts_acquire.get_value() is True + set_mock_put_proceeds(stats.ts_acquire, True) + await logic.ensure_stopped() + assert_has_calls(stats.ts_acquire, [call.put(True), call.put(False)]) + assert await stats.ts_acquire.get_value() is False + + +@pytest.mark.parametrize( + "plugin_enabled,compute_statistics,compute_centroid,expected_datakeys", + [ + # Switched on, it publishes every statistic with a read format... + (EnableDisable.ENABLE, True, False, PUBLISHED), + # ...including a centroid given one, when its group is computed + (EnableDisable.ENABLE, True, True, [*PUBLISHED, "det-stats-centroid_x"]), + (EnableDisable.ENABLE, False, True, ["det-stats-centroid_x"]), + # Nothing is computed, so it sits out + (EnableDisable.ENABLE, False, False, None), + # A disabled plugin computes nothing, so it sits out + (EnableDisable.DISABLE, True, True, None), + ], +) +async def test_publishes_what_the_switches_recorded_at_stage_leave_on( + stats: adcore.NDStatsIO, + plugin_enabled: EnableDisable, + compute_statistics: bool, + compute_centroid: bool, + expected_datakeys: list[str] | None, +): + stats.set_readable_format(stats.centroid_x, Format.UNCACHED_SIGNAL) + set_mock_value(stats.enable_callbacks, plugin_enabled) + set_mock_value(stats.compute_statistics, compute_statistics) + set_mock_value(stats.compute_centroid, compute_centroid) + await stats.snapshot() + # Switched later, which the recorded switches ignore + set_mock_value(stats.compute_statistics, not compute_statistics) + logic = StatsTimeSeriesDataLogic(stats) + + made = await logic.start_provider("det", [], 5, 1, 0.1, 0) + + if expected_datakeys is None: + assert made is None + # Sitting out arms nothing + assert await stats.ts_acquire.get_value() is False + else: + assert made is not None + assert list(await made.make_datakeys()) == expected_datakeys + # The switches are the plugin's to decide, never written by this logic + switches = [ + stats.enable_callbacks, + stats.compute_statistics, + stats.compute_centroid, + ] + for switch in switches: + get_mock_put(switch).assert_not_called() + + +@pytest.mark.parametrize( + "existing_datakeys,expected_datakeys", + [ + # Nothing made elsewhere, so it publishes every array + ([], PUBLISHED), + # Only the arrays the unbounded providers do not make + ([*PUBLISHED[1:], "det"], PUBLISHED[:1]), + # Nothing left to make, so it sits out + (PUBLISHED, None), + ], +) +async def test_publishes_only_the_arrays_not_already_made( + stats: adcore.NDStatsIO, + existing_datakeys: list[str], + expected_datakeys: list[str] | None, +): + logic = StatsTimeSeriesDataLogic(stats) + + made = await logic.start_provider("det", existing_datakeys, 5, 1, 0.1, 0) + + if expected_datakeys is None: + assert made is None + # Sitting out arms nothing + assert await stats.ts_acquire.get_value() is False + else: + assert made is not None + assert list(await made.make_datakeys()) == expected_datakeys + + +async def test_ensure_stopped_stops_the_time_series(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + set_mock_value(stats.ts_acquire, True) + await logic.ensure_stopped() + assert await stats.ts_acquire.get_value() is False + + +class _JustInternal(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): ... + + async def default_trigger_info(self) -> TriggerInfo: + return TriggerInfo() + + +class _FillStatsAcquireLogic(DetectorAcquireLogic): + """Fills the mock time series when acquisition starts.""" + + def __init__(self, stats: adcore.NDStatsIO, values: np.ndarray, times: np.ndarray): + self.stats = stats + self.values = values + self.times = times + + async def acquire(self): + for stat in self.stats.statistics: + set_mock_value(stat.time_series, self.values) + set_mock_value(self.stats.ts_timestamp, self.times) + set_mock_value(self.stats.ts_current_point, len(self.values)) + yield + + async def ensure_stopped(self): ... + + +@pytest.fixture +def stats_detector(stats: adcore.NDStatsIO) -> StandardDetector: + """A writer-less detector whose only data logic is a stats time series.""" + values = np.array([5.0, 6.0, 7.0, 8.0]) + times = np.array([200.0, 201.0, 202.0, 203.0]) + return DetectorLogic( + trigger=_JustInternal(), + acquire=_FillStatsAcquireLogic(stats, values, times), + data=[StatsTimeSeriesDataLogic(stats)], + ).with_device("det") + + +def test_step_scan_through_run_engine(RE: RunEngine, stats_detector: StandardDetector): + """A writer-less detector describes and reads as internal data. + + Driven through the RunEngine so the descriptor its datakeys make is the one + a real scan would emit, and so has to pass event-model validation. + """ + det = stats_detector + docs: dict[str, list] = defaultdict(list) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + # prepare() comes after stage(), which resets the detector + yield from bps.prepare(det, TriggerInfo(collections_per_event=4), wait=True) + yield from bps.trigger_and_read([det]) + + RE(plan(), lambda name, doc: docs[name].append(doc)) + + (descriptor,) = docs["descriptor"] + assert list(descriptor["data_keys"]) == PUBLISHED + total = descriptor["data_keys"]["det-stats-total"] + assert total["shape"] == [4] + assert "external" not in total + (event,) = docs["event"] + np.testing.assert_array_equal( + event["data"]["det-stats-total"], [5.0, 6.0, 7.0, 8.0] + ) + assert event["timestamps"]["det-stats-total"] == 203.0 + + +def test_fly_scan_emits_event_pages(RE: RunEngine, stats_detector: StandardDetector): + """A writer-less detector collects its buffer as event pages.""" + det = stats_detector + docs: dict[str, list] = defaultdict(list) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + yield from bps.prepare(det, TriggerInfo(number_of_events=4), wait=True) + yield from bps.declare_stream(det, name="primary") + yield from bps.kickoff(det, wait=True) + yield from bps.collect_while_completing( + flyers=[det], dets=[det], flush_period=0.05 + ) + + RE(plan(), lambda name, doc: docs[name].append(doc)) + + (page,) = docs["event_page"] + np.testing.assert_array_equal( + page["data"]["det-stats-total"], [[5.0], [6.0], [7.0], [8.0]] + ) + assert docs["stop"][0]["num_events"] == {"primary": 4} + + +async def test_step_scan_read_derives_single_reading(stats_detector: StandardDetector): + """A step-scan read derives one reading holding the whole buffer, timed by + the last point's ts_timestamp.""" + det = stats_detector + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=4)) + await det.trigger() + reading = await det.read() + total = reading["det-stats-total"] + np.testing.assert_array_equal(total["value"], [5.0, 6.0, 7.0, 8.0]) + assert total["timestamp"] == 203.0 + await det.unstage() diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index d9d1e970b0..1ff3d98381 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -1,6 +1,12 @@ +from collections import defaultdict from unittest.mock import ANY +from xml.etree import ElementTree as ET +import bluesky.plan_stubs as bps +import bluesky.preprocessors as bpp +import numpy as np import pytest +from bluesky import RunEngine from ophyd_async.core import ( EnableDisable, @@ -8,6 +14,7 @@ TransitionError, TriggerInfo, callback_on_mock_put, + get_mock_put, init_devices, set_mock_value, ) @@ -24,19 +31,14 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( async with init_devices(mock=True): det = adsimdetector.SimDetector( "PREFIX:", - adcore.ADDataFactory.hdf(static_path_provider), + adcore.ADDataFactory.hdf(static_path_provider, ndattribute_sources=[stat]), plugins={"stats": stat}, ) temp = epics_signal_r(float, "SAMP:TEMP:RBV") # Plugins are not registered automatically; opt this one in det.set_readable_format(stat, Format.CHILD) + # The HDF writer stamps the stats plugin's NDAttributes; the driver stamps these ndattributes = [ - adcore.NDAttributeParam( - name="det-sum", - param="TOTAL", - datatype=adcore.NDAttributeDataType.DOUBLE, - description="Sum of the array", - ), adcore.NDAttributePv( name="sample-temp", signal=temp, @@ -44,7 +46,11 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( description="Temperature of the sample", ), ] - set_mock_value(stat.nd_attributes_file, adcore.ndattributes_to_xml(ndattributes)) + set_mock_value( + det.driver.nd_attributes_file, adcore.ndattributes_to_xml(ndattributes) + ) + set_mock_value(stat.enable_callbacks, EnableDisable.ENABLE) + set_mock_value(stat.compute_statistics, True) writer = det.get_plugin("hdf", adcore.NDFileHDF5IO) set_mock_value(det.driver.acquire_period, 0.1) set_mock_value(det.driver.acquire_time, 0.05) @@ -94,7 +100,8 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( "shape": [1, 768, 1024], "source": uri, }, - "det-sum": { + # The HDF file serves the stats, under the names of their scalars + "det-stats-total": { "dtype": "number", "dtype_numpy": " None: + """Set the PVs that switch the basic statistics of a stats plugin.""" + set_mock_value( + stats.enable_callbacks, EnableDisable.ENABLE if on else EnableDisable.DISABLE + ) + set_mock_value(stats.compute_statistics, on) + + +def _fill_time_series( + det, plugins: list[adcore.NDStatsIO], points: int, also=lambda: None +): + """Behave like the IOC: arming a time series erases it, acquiring fills it.""" + + def erase(value): + if value: + for plugin in plugins: + set_mock_value(plugin.ts_current_point, 0) + + def fill(value): + if value: + for plugin in plugins: + for stat in plugin.statistics: + set_mock_value( + stat.time_series, # pyright: ignore[reportArgumentType] + np.arange(points, dtype=np.float64), + ) + set_mock_value( + plugin.ts_timestamp, # pyright: ignore[reportArgumentType] + 100.0 + np.arange(points), + ) + set_mock_value(plugin.ts_current_point, points) + also() + + for plugin in plugins: + callback_on_mock_put(plugin.ts_acquire, erase) + callback_on_mock_put(det.driver.acquire, fill) + + +def _fly_plan(det, info: TriggerInfo): + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + yield from bps.prepare(det, info, wait=True) + yield from bps.declare_stream(det, name="primary", collect=True) + yield from bps.kickoff(det, wait=True) + yield from bps.collect_while_completing( + flyers=[det], dets=[det], flush_period=0.05 + ) + + return plan() + + +def _fly( + RE: RunEngine, + det, + series: list[adcore.NDStatsIO], + points: int, + hdf: adcore.NDFileHDF5IO | None = None, +) -> dict[str, list]: + also = lambda: None # noqa: E731 + if hdf is not None: + also = lambda: set_mock_value(hdf.num_captured, points) # noqa: E731 + _fill_time_series(det, series, points=points, also=also) + docs: dict[str, list] = defaultdict(list) + RE( + _fly_plan(det, TriggerInfo(number_of_events=points)), + lambda n, d: docs[n].append(d), + ) + return docs + + +async def _superset_detector( + path_provider: StaticPathProvider, always_enable: bool = False +): + """Use case 6: the readables, time series and HDF NDAttributes of one stats + plugin, with its basic statistics switched on.""" + stats = adcore.NDStatsIO("PREFIX:STAT:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADDataFactory.hdf( + path_provider, always_enable=always_enable, ndattribute_sources=[stats] + ), + adcore.ADDataFactory.stats_time_series(stats), + plugins={"stats": stats}, + ) + _switch_stats(stats, on=True) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + set_mock_value(det.get_plugin("hdf", adcore.NDFileHDF5IO).file_path_exists, True) + return det + + +# Use cases A and B: a detector with a stats plugin in plugins= only, then with its +# time series. A plugin in plugins= is a readable, so its hints come with it. + + +async def test_uc_a_detector_with_readable_stats(RE: RunEngine): + stats = adcore.NDStatsIO("PREFIX:STAT:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stats}, name="det") + _switch_stats(stats, on=True) + set_mock_value(stats.total, 12.0) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def step_plan(): + yield from bps.trigger_and_read([det]) + + docs: dict[str, list] = defaultdict(list) + RE(step_plan(), lambda name, doc: docs[name].append(doc)) + + (descriptor,) = docs["descriptor"] + assert set(descriptor["data_keys"]) == set(STATS_KEYS) + assert descriptor["hints"]["det"]["fields"] == STATS_HINTS + (event,) = docs["event"] + assert event["data"]["det-stats-total"] == 12.0 + + +async def test_uc_a_a_plugin_opts_out_of_being_read(): + stats = adcore.NDStatsIO("PREFIX:STAT:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stats}, name="det") + _switch_stats(stats, on=True) + det.set_readable_format(stats, None) + + await det.stage() + await det.prepare(TriggerInfo()) + + assert not [key for key in await det.describe() if "stats" in key] + assert det.hints == {} + + +def test_uc_b_detector_with_a_stats_time_series(RE: RunEngine): + stats = adcore.NDStatsIO("PREFIX:STAT:") + with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADDataFactory.stats_time_series(stats), + plugins={"stats": stats}, + name="det", + ) + _switch_stats(stats, on=True) + + docs = _fly(RE, det, [stats], points=3) + + (descriptor,) = docs["descriptor"] + assert set(descriptor["data_keys"]) == set(STATS_KEYS) + # The time series replaces the readable, whose hints stay + assert descriptor["hints"]["det"]["fields"] == STATS_HINTS + (page,) = docs["event_page"] + assert page["data"]["det-stats-total"] == [[0.0], [1.0], [2.0]] + # It made no plugin of its own + assert [name for name, _ in det.children() if "stat" in name] == ["stats"] + + +@pytest.mark.parametrize( + "always_enable,hdf_enabled,hdf_serves", + [ + # By its PV, HDF on: the file serves, every time series suppressed + (False, EnableDisable.ENABLE, True), + # By its PV, HDF off: the time series serves + (False, EnableDisable.DISABLE, False), + # Always enabled, HDF serves whatever its PV was + (True, EnableDisable.DISABLE, True), + ], +) +async def test_uc6_the_hdf_switch_picks_the_file_or_the_time_series( + RE: RunEngine, + static_path_provider: StaticPathProvider, + always_enable: bool, + hdf_enabled: EnableDisable, + hdf_serves: bool, +): + det = await _superset_detector(static_path_provider, always_enable) + hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(hdf.enable_callbacks, hdf_enabled) + + docs = _fly(RE, det, [det.get_plugin("stats", adcore.NDStatsIO)], 3, hdf=hdf) + + # The registered stats plugin hints the statistics, whoever serves them + (descriptor,) = docs["descriptor"] + assert descriptor["hints"]["det"]["fields"] == [ + *STATS_HINTS, + *(["det"] if hdf_serves else []), + ] + if hdf_serves: + assert docs["event_page"] == [] + keys = [doc["data_key"] for doc in docs["stream_resource"]] + assert keys == ["det", *STATS_KEYS] + else: + assert docs["stream_resource"] == [] + (page,) = docs["event_page"] + assert list(page["data"]) == STATS_KEYS + assert page["data"]["det-stats-total"] == [[0.0], [1.0], [2.0]] + + +async def test_uc6_hdf_enabled_the_file_serves_every_stats_quantity( + static_path_provider: StaticPathProvider, +): + det = await _superset_detector(static_path_provider) + hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(hdf.enable_callbacks, EnableDisable.ENABLE) + await det.stage() + callback_on_mock_put( + det.driver.acquire, lambda v: set_mock_value(hdf.num_captured, 1) + ) + await det.trigger() + + description = await det.describe() + + assert description["det-stats-total"]["external"] == "STREAM:" + assert "det-stats-total" not in await det.read() + + +@pytest.fixture +async def two_roi_detector(tmp_path): + """Use case 7: two HDF writers and two time series, one per ROI.""" + path_provider = StaticPathProvider( + lambda datakey_name=None: str(datakey_name), tmp_path + ) + stats1 = adcore.NDStatsIO("PREFIX:STAT1:") + stats2 = adcore.NDStatsIO("PREFIX:STAT2:") + async with init_devices(mock=True): + det = adaravis.AravisDetector( + "PREFIX:", + adcore.ADDataFactory.hdf( + path_provider, + writer_suffix="HDF1:", + writer_name="hdf1", + datakey_suffix="-roi1", + always_enable=False, + ), + adcore.ADDataFactory.hdf( + path_provider, + writer_suffix="HDF2:", + writer_name="hdf2", + datakey_suffix="-roi2", + always_enable=False, + ), + adcore.ADDataFactory.stats_time_series(stats1), + adcore.ADDataFactory.stats_time_series(stats2), + plugins={"stats1": stats1, "stats2": stats2}, + ) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + for name in ("hdf1", "hdf2"): + plugin = det.get_plugin(name, adcore.NDFileHDF5IO) + set_mock_value(plugin.enable_callbacks, EnableDisable.DISABLE) + set_mock_value(plugin.file_path_exists, True) + return det + + +@pytest.mark.parametrize( + "stats1_on,stats2_on", + [(True, True), (False, True)], +) +def test_uc7_hdf_disabled_the_time_series_switched_on_serve_in_one_page( + RE: RunEngine, two_roi_detector, stats1_on: bool, stats2_on: bool +): + det = two_roi_detector + series = [det.get_plugin(n, adcore.NDStatsIO) for n in ("stats1", "stats2")] + _switch_stats(series[0], stats1_on) + _switch_stats(series[1], stats2_on) + + docs = _fly(RE, det, series, points=2) + + assert docs["stream_resource"] == [] + (page,) = docs["event_page"] + totals = [ + f"det-{name}-total" + for name, on in (("stats1", stats1_on), ("stats2", stats2_on)) + if on + ] + assert sorted(key for key in page["data"] if key.endswith("-total")) == totals + for total in totals: + assert page["data"][total] == [[0.0], [1.0]] + + +# Ruling 1: a once-per-event readable is suppressed by a same-named higher tier at +# any collections_per_event, and read once per event when nothing suppresses it + + +@pytest.mark.parametrize("cpe", [1, 2]) +@pytest.mark.parametrize("hdf_enabled", [False, True]) +@pytest.mark.parametrize("bounded", [False, True]) +async def test_readable_stats_total_is_suppressed_by_any_higher_tier( + static_path_provider: StaticPathProvider, + bounded: bool, + hdf_enabled: bool, + cpe: int, +): + """readable total < time series total (bounded) < HDF NDAttribute (unbounded).""" + stats = adcore.NDStatsIO("PREFIX:STAT:") + factories: list[adcore.ADDataFactory] = [ + adcore.ADDataFactory.hdf( + static_path_provider, always_enable=False, ndattribute_sources=[stats] + ) + ] + if bounded: + factories.append(adcore.ADDataFactory.stats_time_series(stats)) + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", *factories, plugins={"stats": stats}, name="det" + ) + _switch_stats(stats, on=True) + hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value( + hdf.enable_callbacks, + EnableDisable.ENABLE if hdf_enabled else EnableDisable.DISABLE, + ) + set_mock_value(hdf.file_path_exists, True) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + await det.stage() + info = TriggerInfo(number_of_events=1, collections_per_event=cpe) + + await det.prepare(info) + total = (await det.describe())["det-stats-total"] + if bounded or hdf_enabled: + # Suppressed: the key now describes a [cpe] collections, not one value + assert total["shape"] == [cpe] + else: + assert total["shape"] == [] @pytest.mark.parametrize("cpe", [1, 2]) @@ -818,18 +1165,130 @@ async def test_readable_with_no_data_logic_is_read_once_per_event(cpe: int): stat = adcore.NDStatsIO("PREFIX:STATS:") async with init_devices(mock=True): det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}, name="det") - det.set_readable_format(stat.total, Format.UNCACHED_SIGNAL) + _switch_stats(stat, on=True) await det.stage() await det.prepare(TriggerInfo(number_of_events=1, collections_per_event=cpe)) assert (await det.describe())["det-stats-total"]["shape"] == [] -@pytest.mark.parametrize("on_at_stage", [True, False]) -async def test_hdf_by_settings_follows_its_plugin_at_stage_for_the_whole_stage( - tmp_path, on_at_stage: bool +@pytest.mark.parametrize( + "enabled,computed,published", + [ + (EnableDisable.ENABLE, True, True), + # The plugin computes nothing while disabled + (EnableDisable.DISABLE, True, False), + (EnableDisable.ENABLE, False, False), + ], +) +async def test_stats_scalar_follows_its_switches_at_stage( + enabled: EnableDisable, computed: bool, published: bool ): - """Switching a writer's plugin mid-stage changes nothing until restaged.""" + """The switches at stage decide which scalars are read.""" + stat = adcore.NDStatsIO("PREFIX:STATS:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}, name="det") + set_mock_value(stat.enable_callbacks, enabled) + set_mock_value(stat.compute_statistics, computed) + + await det.stage() + await det.prepare(TriggerInfo()) + await det.trigger() + + assert ("det-stats-total" in await det.describe()) is published + assert ("det-stats-total" in await det.read()) is published + assert ("det-stats-total" in det.hints.get("fields", [])) is published + await det.unstage() + + +async def test_prepare_raises_for_a_switch_changed_since_stage(): + stat = adcore.NDStatsIO("PREFIX:STATS:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}, name="det") + _switch_stats(stat, on=True) + await det.stage() + set_mock_value(stat.compute_statistics, False) + + with pytest.raises(RuntimeError, match=r"changed since stage.*ComputeStatistics"): + await det.prepare(TriggerInfo()) + + # Staging again takes the new value, and nothing was written to the PV + await det.unstage() + await det.stage() + await det.prepare(TriggerInfo()) + await det.trigger() + assert "det-stats-total" not in await det.read() + get_mock_put(stat.compute_statistics).assert_not_called() + + +@pytest.mark.parametrize( + "changed", + [ + # A switch of the plugin, or the port it takes its NDArrays from + lambda stat: set_mock_value(stat.enable_callbacks, EnableDisable.DISABLE), + lambda stat: set_mock_value(stat.nd_array_port, "ROI1"), + ], + ids=["switch", "port"], +) +async def test_prepare_raises_for_a_switch_or_port_changed_since_stage(changed): + stat = adcore.NDStatsIO("PREFIX:STATS:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}, name="det") + set_mock_value(stat.nd_array_port, "DET") + _switch_stats(stat, on=True) + await det.stage() + changed(stat) + + with pytest.raises(RuntimeError, match=r"changed since stage.*(Callbacks|Port)"): + await det.prepare(TriggerInfo()) + + +async def test_prepare_takes_the_recording_of_stage_not_of_the_last_prepare(): + stat = adcore.NDStatsIO("PREFIX:STATS:") + async with init_devices(mock=True): + det = adsimdetector.SimDetector("PREFIX:", plugins={"stats": stat}, name="det") + _switch_stats(stat, on=True) + await det.stage() + await det.prepare(TriggerInfo()) + await det.trigger() + set_mock_value(stat.compute_statistics, False) + + with pytest.raises(RuntimeError, match=r"changed since stage"): + await det.prepare(TriggerInfo()) + # Still against stage(): turning it back on is not a change + set_mock_value(stat.compute_statistics, True) + await det.prepare(TriggerInfo()) + + +@pytest.mark.parametrize("owned", [True, False]) +async def test_a_process_plugin_is_recorded_unless_the_trigger_logic_owns_it( + owned: bool, +): + """Ownership is by logic: only the plugin the trigger logic switches is free.""" + async with init_devices(mock=True): + if owned: + det = adsimdetector.SimDetector("PREFIX:", proc_suffix="PROC:", name="det") + proc = det.get_plugin("proc", adcore.NDProcessIO) + else: + proc = adcore.NDProcessIO("PREFIX:PROC:") + det = adsimdetector.SimDetector("PREFIX:", plugins={"proc": proc}) + set_mock_value(proc.enable_callbacks, EnableDisable.DISABLE) + await det.stage() + # Filtering switches its own process plugin on at prepare + info = TriggerInfo(exposures_per_collection=2) + if owned: + await det.prepare(info) + await det.trigger() + await det.prepare(info) + else: + set_mock_value(proc.enable_callbacks, EnableDisable.ENABLE) + with pytest.raises(RuntimeError, match=r"changed since stage.*PROC:Enable"): + await det.prepare(info) + + +@pytest.mark.parametrize("on_at_stage", [True, False]) +async def test_hdf_by_settings_follows_its_plugin_at_stage(tmp_path, on_at_stage: bool): + """Switching a writer's plugin mid-stage is caught by the next prepare.""" path_provider = StaticPathProvider( lambda datakey_name=None: str(datakey_name), tmp_path ) @@ -872,13 +1331,278 @@ def enable_hdf2(enabled: bool) -> None: enable_hdf2(on_at_stage) await det.stage() - for flip in (False, True): - if flip: - enable_hdf2(not on_at_stage) + await det.prepare(info) + await det.kickoff() + await det.complete() + assert set(await det.describe_collect()) == served + # Events since prepare() + assert await det.get_index() == info.number_of_events + + enable_hdf2(not on_at_stage) + with pytest.raises( + RuntimeError, match=r"changed since stage.*HDF2:EnableCallbacks" + ): await det.prepare(info) - await det.kickoff() - await det.complete() - assert set(await det.describe_collect()) == served - # Events since prepare() - assert await det.get_index() == info.number_of_events await det.unstage() + + +# Use case 8: a stats plugin per ROI, each stamped into the file of its own HDF +# writer. A writer stamps only the NDAttribute sources listed for it. + + +def _publish_only_total(stats: adcore.NDStatsIO) -> None: + """Leave the total as the only statistic with a read format.""" + for statistic in stats.statistics: + if statistic.scalar is not stats.total: + stats.set_readable_format(statistic.scalar, None) + + +def _roi_array_description(roi: adcore.NDROIIO): + return lambda driver: adcore.NDArrayDescription( + shape_signals=(roi.size_y, roi.size_x), + data_type_signal=driver.data_type, + color_mode_signal=driver.color_mode, + ) + + +async def test_uc8_2_rois_each_with_stats_and_hdf(RE: RunEngine, tmp_path): + path_provider = StaticPathProvider( + lambda datakey_name=None: str(datakey_name), tmp_path + ) + rois = [adcore.NDROIIO(f"PREFIX:ROI{i}:") for i in (1, 2)] + for i, roi in enumerate(rois, start=1): + roi.stat = adcore.NDStatsIO(f"PREFIX:STAT{i}:") + _publish_only_total(roi.stat) + roi1, roi2 = rois + async with init_devices(mock=True): + det = adaravis.AravisDetector( + "PREFIX:", + *( + adcore.ADDataFactory.hdf( + path_provider, + writer_suffix=f"HDF{i}:", + writer_name=f"hdf{i}", + datakey_suffix=f"-roi{i}", + array_description=_roi_array_description(roi), + ndattribute_sources=[roi.stat], + ) + for i, roi in enumerate(rois, start=1) + ), + plugins={"roi1": roi1, "roi2": roi2}, + ) + hdfs = [det.get_plugin(f"hdf{i}", adcore.NDFileHDF5IO) for i in (1, 2)] + for roi in rois: + det.set_readable_format(roi.stat, Format.CHILD) + _switch_stats(roi.stat, on=True) + set_mock_value(roi.size_x, 40) + set_mock_value(roi.size_y, 30) + for hdf in hdfs: + set_mock_value(hdf.file_path_exists, True) + + def capture_three_frames(value: bool) -> None: + if value: + for hdf in hdfs: + set_mock_value(hdf.num_captured, 3) + + callback_on_mock_put(det.driver.acquire, capture_three_frames) + docs: dict[str, list] = defaultdict(list) + + RE( + _fly_plan(det, TriggerInfo(number_of_events=3)), + lambda name, doc: docs[name].append(doc), + ) + + # Each writer's file has its own ROI and the totals of its own stats plugin + uris = {doc["data_key"]: doc["uri"] for doc in docs["stream_resource"]} + assert uris == { + "det-roi1": ANY, + "det-roi1-stat-total": uris["det-roi1"], + "det-roi2": ANY, + "det-roi2-stat-total": uris["det-roi2"], + } + assert uris["det-roi1"] != uris["det-roi2"] + (descriptor,) = docs["descriptor"] + assert set(descriptor["data_keys"]) == set(uris) + # The totals stay hinted by the stats plugins that the files replace + assert sorted(descriptor["hints"]["det"]["fields"]) == sorted(uris) + + +# Use case 9: an ROIStat plugin, modelled on the test-rig spectroscopy plans +# (DRV -> ROISTAT -> HDF, three channels named Blue, Green and Red, only their +# totals published). The HDF writer lists the plugin as an NDAttribute source, +# so when it serves the totals are in its file, and when it sits out they are +# read from the channels. + +CHANNEL_NAMES = {1: "Blue", 2: "Green", 3: "Red"} + + +async def _roistat_detector( + tmp_path, hdf_on: bool, channel_names: dict[int, str] = CHANNEL_NAMES +): + path_provider = StaticPathProvider( + lambda datakey_name=None: str(datakey_name), tmp_path + ) + roistat = adcore.NDROIStatIO("PREFIX:ROISTAT:", num_channels=len(channel_names)) + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADDataFactory.hdf( + path_provider, always_enable=False, ndattribute_sources=[roistat] + ), + plugins={"roistat": roistat}, + name="det", + ) + set_mock_value(roistat.enable_callbacks, EnableDisable.ENABLE) + for number, name in channel_names.items(): + channel = roistat.channels[number] + set_mock_value(channel.name_, name) + set_mock_value(channel.use, True) + set_mock_value(channel.total, 10.0 * number) + for scalar in (channel.min_value, channel.max_value, channel.mean_value): + channel.set_readable_format(scalar, None) + hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value( + hdf.enable_callbacks, EnableDisable.ENABLE if hdf_on else EnableDisable.DISABLE + ) + set_mock_value(hdf.file_path_exists, True) + set_mock_value(det.driver.array_size_x, 64) + set_mock_value(det.driver.array_size_y, 48) + return det, roistat, hdf + + +def _scan_roistat_detector(RE: RunEngine, det, hdf: adcore.NDFileHDF5IO, hdf_on: bool): + """Run a fly scan if the HDF writer serves it, otherwise a step scan.""" + docs: dict[str, list] = defaultdict(list) + if hdf_on: + callback_on_mock_put( + det.driver.acquire, + lambda value: set_mock_value(hdf.num_captured, 3) if value else None, + ) + plan = _fly_plan(det, TriggerInfo(number_of_events=3)) + else: + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def step_plan(): + yield from bps.trigger_and_read([det]) + + plan = step_plan() + RE(plan, lambda name, doc: docs[name].append(doc)) + return docs + + +@pytest.mark.parametrize("hdf_on", [True, False]) +async def test_uc9_roistat_totals_are_in_the_file_or_read_from_the_channels( + RE: RunEngine, tmp_path, hdf_on: bool +): + det, roistat, hdf = await _roistat_detector(tmp_path, hdf_on) + # Before stage the channels have their default names + assert roistat.channels[1].total.name == "det-roistat-channels-1-total" + + docs = _scan_roistat_detector(RE, det, hdf, hdf_on) + + totals = {f"det-roistat-{name}-total" for name in CHANNEL_NAMES.values()} + (descriptor,) = docs["descriptor"] + assert set(descriptor["data_keys"]) >= totals + # Hinted whether they are in the file or read, along with the image if filed + assert set(descriptor["hints"]["det"]["fields"]) == ( + {"det", *totals} if hdf_on else totals + ) + if hdf_on: + assert {doc["data_key"] for doc in docs["stream_resource"]} == {"det", *totals} + assert all(key in descriptor["data_keys"] for key in totals) + assert ( + descriptor["data_keys"]["det-roistat-Blue-total"]["external"] == "STREAM:" + ) + # The plugin was told to stamp each total from its own channel's address + xml = await roistat.nd_attributes_file.get_value() + stamped = [ + ( + a.attrib["name"], + a.attrib["source"], + a.attrib["addr"], + a.attrib["datatype"], + ) + for a in ET.fromstring(xml) + ] + assert stamped == [ + ("det-roistat-Blue-total", "ROISTAT_TOTAL", "0", "DOUBLE"), + ("det-roistat-Green-total", "ROISTAT_TOTAL", "1", "DOUBLE"), + ("det-roistat-Red-total", "ROISTAT_TOTAL", "2", "DOUBLE"), + ] + else: + assert docs["stream_resource"] == [] + (event,) = docs["event"] + assert {key: event["data"][key] for key in totals} == { + "det-roistat-Blue-total": 10.0, + "det-roistat-Green-total": 20.0, + "det-roistat-Red-total": 30.0, + } + + +@pytest.mark.parametrize("hdf_on", [True, False]) +async def test_uc9_a_roistat_channel_that_is_not_used_disappears( + RE: RunEngine, tmp_path, hdf_on: bool +): + det, roistat, hdf = await _roistat_detector(tmp_path, hdf_on) + set_mock_value(roistat.channels[2].use, False) + + docs = _scan_roistat_detector(RE, det, hdf, hdf_on) + + (descriptor,) = docs["descriptor"] + assert "det-roistat-Blue-total" in descriptor["data_keys"] + assert "det-roistat-Red-total" in descriptor["data_keys"] + assert not [key for key in descriptor["data_keys"] if "Green" in key] + assert not [key for key in descriptor["hints"]["det"]["fields"] if "Green" in key] + if hdf_on: + assert not [ + doc for doc in docs["stream_resource"] if "Green" in doc["data_key"] + ] + else: + (event,) = docs["event"] + assert not [key for key in event["data"] if "Green" in key] + # Its name was still taken from the PV + assert roistat.channels[2].total.name == "det-roistat-Green-total" + + +async def test_uc9_roistat_channel_names_are_sanitised_at_stage(tmp_path): + names = {1: "Ice Cream", 2: "", 3: "r.o/i-3", 4: "keep_Case9"} + det, roistat, _ = await _roistat_detector(tmp_path, False, names) + + await det.stage() + + assert [roistat.channels[number].name for number in names] == [ + "det-roistat-Ice_Cream", + "det-roistat-2", + "det-roistat-r_o_i_3", + "det-roistat-keep_Case9", + ] + assert roistat.channels[1].total.name == "det-roistat-Ice_Cream-total" + + +async def test_uc9_roistat_channels_with_the_same_sanitised_name_raise_at_stage( + tmp_path, +): + det, _, _ = await _roistat_detector(tmp_path, False, {1: "A B", 2: "A_B", 3: "C"}) + + with pytest.raises(ValueError, match=r"more than one channel is named \['A_B'\]"): + await det.stage() + + +async def test_uc9_prepare_raises_for_a_roistat_channel_renamed_since_stage(tmp_path): + det, roistat, _ = await _roistat_detector(tmp_path, False) + await det.stage() + set_mock_value(roistat.channels[3].name_, "Infrared") + + with pytest.raises(RuntimeError, match=r"changed since stage.*ROISTAT:3:Name"): + await det.prepare(TriggerInfo()) + + +async def test_uc9_prepare_raises_for_a_roistat_channel_used_since_stage(tmp_path): + det, roistat, _ = await _roistat_detector(tmp_path, False) + set_mock_value(roistat.channels[3].use, False) + await det.stage() + set_mock_value(roistat.channels[3].use, True) + + with pytest.raises(RuntimeError, match=r"changed since stage.*ROISTAT:3:Use_RBV"): + await det.prepare(TriggerInfo()) diff --git a/tests/unit_tests/plan_stubs/test_ndattributes.py b/tests/unit_tests/plan_stubs/test_ndattributes.py index 8695c67415..09d3a69479 100644 --- a/tests/unit_tests/plan_stubs/test_ndattributes.py +++ b/tests/unit_tests/plan_stubs/test_ndattributes.py @@ -2,23 +2,11 @@ import pytest -from ophyd_async.core import StaticPathProvider, init_devices -from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.core import init_devices +from ophyd_async.epics import adcore from ophyd_async.epics.core import epics_signal_r -def test_setup_ndstats_raises_type_error(RE, static_path_provider: StaticPathProvider): - with init_devices(mock=True): - det = adsimdetector.SimDetector( - "PREFIX:", adcore.ADDataFactory.hdf(static_path_provider) - ) - with pytest.raises( - AttributeError, - match="det has no plugin named 'stats'", - ): - RE(adcore.setup_ndstats_sum(det)) - - async def test_nd_attributes_plan_stub(RE): async with init_devices(mock=True): stat = adcore.NDStatsIO("PREFIX:STATS:") From 0144206dc4d75c846e677f218af3db670c76b4bd Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 3 Nov 2026 13:46:44 +0000 Subject: [PATCH 17/25] WIP: ADR 0024 fixes; unstage closes an armed acquisition nothing took Assisted-by: Claude Opus 5.5 --- .../decisions/0024-detector-is-a-flyable.md | 28 +++++++++---------- src/ophyd_async/core/_detector.py | 14 +++++++--- .../epics/adcore/_stats_time_series.py | 3 +- .../core/test_detector_use_cases.py | 4 --- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md index d277f6d89e..1e507faf1c 100644 --- a/docs/explanations/decisions/0024-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -60,13 +60,10 @@ property to be concrete. For an ad-hoc detector in a plan or a test, ### One thing the logic cannot get from its own fields Data logics name their datakeys after the Device, whose name does not exist until -`init_devices` calls `set_name` on context exit. `StandardFlyable` runs hooks a subclass -registers with the context as soon as `prepare_to_fly` yields it. `StandardDetector` -registers one, which writes its name and the description of its registered signals into the -context, then calls `DetectorLogic.serve_data` with it to choose and start the data -providers. It passes its name to `get_hinted_fields()` too. The logic holds no reference to -the Device: `StandardFlyable` calls `prepare_to_fly` with the `TriggerInfo` alone, so the -context is where the Device leaves what serving needs. +`init_devices` calls `set_name` on context exit, so the logic cannot be given it when it is +made. `StandardFlyable` passes the Device's name to `prepare_to_fly` instead; the detector +keeps it on the context, for the data providers it starts after yielding, and +`get_hinted_fields()` takes it the same way. The logic holds no reference to the Device. ### Which collect verb to expose is the Device's business @@ -190,9 +187,11 @@ scalars dropped like a switched-off statistic. The channels are named when the d staged, as `-`, with characters other than letters, digits and underscores replaced by underscores and the channel number used if the PV is empty, so the scalars have names the documents accept and say what the channel measures. Two channels that would be named -the same raise, and a `Name` PV that changes after that raises at `prepare()` like a switch. The plugin is a -readable whose channels are its readable children, so a user does not register each channel. There is no ROI stat time series: a fly scan that needs the channels while the -HDF writer sits out raises at `prepare()`, as a stats plugin only registered as a readable does. +the same raise, and a `Name` PV that changes after that raises at `prepare()` like a switch. +The plugin is a readable whose channels are its readable children, so a user does not register +each channel. ophyd-async does not model the plugin's time series yet, so a fly scan collects +nothing from it: it collects only what the data logics make, and a registered readable is read +only by `trigger()`. ### The detector flies with one `fly()` generator @@ -218,10 +217,11 @@ armed, wait for idle. It owns what it starts (`async with` a status, or cancelli `finally`), so closing it at its `yield` ends it. It takes no trigger information: the same generator serves both kinds of triggering, and only where it is entered differs. An externally triggered detector arms in `prepare()`, so that it is waiting on its trigger source by the time -`kickoff()` returns; the armed acquisition is held on the logic until `fly` (or `trigger()`) -takes it and runs it to the end, closing it if cancelled. An internally triggered one is -entered by `fly` or `trigger()` itself. One still armed at `unstage()` is closed by -`ensure_stopped`, so its cleanup runs then, not when it is garbage collected after a restage. +`kickoff()` returns; the armed acquisition is held on the context until `fly` (or +`trigger()`) takes it and runs it to the end, closing it if cancelled. An internally +triggered one is entered by `fly` or `trigger()` itself. One still armed at `unstage()` is +closed then, so its cleanup runs then, not when it is garbage collected after a restage; +stopping the detector stays the job of `ensure_stopped`. A prepared detector cannot be prepared again before it is kicked off or triggered, so no `prepare()` replaces an armed acquisition. diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index af85fe5e1c..6cd9f44f0b 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -822,10 +822,7 @@ async def step(self, ctx: DetectorCtx) -> AsyncGenerator[WatcherUpdate, None]: yield update async def ensure_stopped(self) -> None: - """Stop every data logic and the detector. - - An acquisition armed by `prepare()` and never taken is dropped. - """ + """Stop every data logic and the detector.""" await asyncio.gather( *(dl.ensure_stopped() for dl in self.data), *([self.acquire.ensure_stopped()] if self.acquire else []), @@ -1160,6 +1157,7 @@ def __init__(self, *args, **kwargs) -> None: self._describe_funcs += (self.__on_describe,) self._hints_funcs += (self.__on_hints,) self._suppressed_datakeys_funcs += (self.__on_suppressed_datakeys,) + self._unstage_funcs += (self.__on_unstage,) @cached_property def _state(self) -> _StateMachine: @@ -1174,6 +1172,14 @@ def logic(self) -> DetectorLogic: """The logic that drives this detector.""" raise NotImplementedError + async def __on_unstage(self) -> None: + # Close an acquisition prepare() armed and nothing took. This does not + # stop the detector, the acquire logic's ensure_stopped() does that: it + # only ends the generator's tasks now, rather than when it is garbage + # collected, which asyncio warns about + if self._ctx and self._ctx.external_acquire_gen: + await self._ctx.external_acquire_gen.aclose() + def __on_suppressed_datakeys(self) -> Mapping[str, DataKey | None]: """The datakeys of the serving providers, which replace registered signals.""" return {} if self._ctx is None else self._ctx.datakeys diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index d37b2a67a3..a05eeecfb3 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -24,8 +24,7 @@ class StatsTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): For detectors that write no file: the stats plugin holds each statistic in a fixed-length buffer that must be sized before acquisition, so this logic - raises for a scan with no fixed number of collections, unless the unbounded - data logics already make all its datakeys. One plugin has one time-series + sizes it from the scan's number of collections. One plugin has one time-series control (`ts_acquire`, `ts_num_points`) shared across many arrays, so one logic covers many arrays with the control embedded. diff --git a/tests/unit_tests/core/test_detector_use_cases.py b/tests/unit_tests/core/test_detector_use_cases.py index 216c895a34..098ba9fa69 100644 --- a/tests/unit_tests/core/test_detector_use_cases.py +++ b/tests/unit_tests/core/test_detector_use_cases.py @@ -981,10 +981,6 @@ async def ensure_stopped(self): self._stopped.set() -@pytest.mark.xfail( - reason="an armed acquisition never taken is dropped, not closed (Tom 2026-10-22)", - strict=True, -) async def test_unstage_closes_the_acquisition_an_external_prepare_armed(): acquire = HeldAcquire() det = make_detector("det", trigger=ExternalTrigger(), acquire=acquire) From 3786d3531687f039aeed0d4c59bf58623e7225d4 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 3 Nov 2026 13:55:09 +0000 Subject: [PATCH 18/25] WIP: ROIStat time series as a bounded data logic ADDataFactory.roistat_time_series(roistat) publishes one array per used channel per scalar with a read format, named after that scalar, as event pages, mirroring stats_time_series. Assisted-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + .../decisions/0024-detector-is-a-flyable.md | 11 +- docs/how-to/implement-ad-detector.md | 11 +- src/ophyd_async/epics/adcore/__init__.py | 5 +- src/ophyd_async/epics/adcore/_data_logic.py | 31 ++- src/ophyd_async/epics/adcore/_io.py | 58 +++++- .../epics/adcore/_stats_time_series.py | 63 +++++- .../unit_tests/epics/adcore/test_use_cases.py | 193 ++++++++++++++++++ 8 files changed, 362 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 838c87efc7..50facf5fb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,7 @@ - pyright now runs with `reportMatchNotExhaustive`. - `BoundedDataLogic`, `UnboundedDataLogic`, `DetectorLogic`, `DataProvider`, `PageableDataProvider` and `EventPageDataProvider` in `ophyd_async.core`, and `ADDataFactory`, `NDStatsTSAcquireMode` and `exposures_per_collection_from_filter` in `ophyd_async.epics.adcore`. - `adcore.StatsTimeSeriesDataLogic` and the `ADDataFactory.stats_time_series(stats)` factory, a `BoundedDataLogic` that publishes an `NDPluginStats` time series as event pages, for an areaDetector that writes no file. It needs a fixed number of collections, as the buffer is sized before acquisition, and sits out when the plugin is switched off at `stage()`, so a detector can switch between its file writer and its stats time series from one stage to the next. +- `adcore.ROIStatTimeSeriesDataLogic` and the `ADDataFactory.roistat_time_series(roistat)` factory, which publish the time series of an `NDROIStatIO` as event pages in the same way: one array per used channel per scalar with a read format, named after the scalar (e.g. `det-roistat-Blue-total`), which it suppresses. With it come `NDROIStatTSControl`, the `ts_control`, `ts_num_points`, `ts_current_point` and `ts_acquiring` signals of `NDROIStatIO`, the `ts_*` arrays of `NDROIStatNIO`, and `NDROIStatNIO.time_series`, `NDROIStatNIO.get_published_scalars()` and `NDROIStatIO.get_computed_channels()`. ### Fixed diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md index 1e507faf1c..5042a854f0 100644 --- a/docs/explanations/decisions/0024-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -189,9 +189,14 @@ replaced by underscores and the channel number used if the PV is empty, so the s names the documents accept and say what the channel measures. Two channels that would be named the same raise, and a `Name` PV that changes after that raises at `prepare()` like a switch. The plugin is a readable whose channels are its readable children, so a user does not register -each channel. ophyd-async does not model the plugin's time series yet, so a fly scan collects -nothing from it: it collects only what the data logics make, and a registered readable is read -only by `trigger()`. +each channel. Its time series is published by `ADDataFactory.roistat_time_series(roistat)` +like the stats time series, one array per used channel per scalar with a read format, named +after that scalar, so it suppresses the scalar. It differs only where the plugin does: a +`TSControl` record erases and starts the buffer rather than a busy record, the buffer has no +circular mode, so it is always fixed length, and every channel has its own timestamp array. +All used channels are stamped from the same frame, so the first used channel's timestamps +serve them all. The plugin's `TSNet` array is not published, because `NDROIStatNIO` has no +`net` scalar to name it after. ### The detector flies with one `fly()` generator diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 2b489b6e86..39af2006d8 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -154,7 +154,16 @@ det = adaravis.AravisDetector( The highest of these that serves a scan replaces the others, and the plugin's hints stay. Which statistics are used is set on the plugin, not in Python: a statistic is used if the plugin's `EnableCallbacks` and its group's compute switch (`compute_statistics` or `compute_centroid`) are on at `stage()`, and it has a read format on the plugin. A scalar whose statistic is off is left out of `read()`, `describe()` and `hints`. -`stage()` records these switches and each plugin's `nd_array_port`, once, and `prepare()` raises if any of them has changed since, so set them before staging, for example in a settings file. The chain of plugins that feeds a plugin is not checked: how it is wired at `stage()` is up to you. The process plugin of a trigger logic and the circular buffer of a continuously acquiring detector are switched on by their logic, so they are not recorded. An [](#adcore.NDROIStatIO) works the same way, with the `use` and `Name` of each channel recorded too: its channels are readable children of the plugin, named `-` at `stage()`, so a channel called Blue has the total `det-roistat-Blue-total`. +`stage()` records these switches and each plugin's `nd_array_port`, once, and `prepare()` raises if any of them has changed since, so set them before staging, for example in a settings file. The chain of plugins that feeds a plugin is not checked: how it is wired at `stage()` is up to you. The process plugin of a trigger logic and the circular buffer of a continuously acquiring detector are switched on by their logic, so they are not recorded. An [](#adcore.NDROIStatIO) works the same way, with the `use` and `Name` of each channel recorded too: its channels are readable children of the plugin, named `-` at `stage()`, so a channel called Blue has the total `det-roistat-Blue-total`. Its time series ([](#adcore.ADDataFactory.roistat_time_series)) works like the stats time series, publishing each used channel's statistics under the same names. + +```python +roistat = adcore.NDROIStatIO("PREFIX:ROISTAT:", num_channels=3) +det = adaravis.AravisDetector( + prefix, + adcore.ADDataFactory.roistat_time_series(roistat), + plugins={"roistat": roistat}, +) +``` ### Multiple HDF writers for different ROIs ```python diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index 2775847983..dc469be02a 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -39,6 +39,7 @@ NDProcessIO, NDROIStatIO, NDROIStatNIO, + NDROIStatTSControl, NDStatistic, NDStatsIO, NDStatsTSAcquireMode, @@ -52,7 +53,7 @@ ndattributes_to_xml, ) from ._plan_stubs import setup_ndattributes -from ._stats_time_series import StatsTimeSeriesDataLogic +from ._stats_time_series import ROIStatTimeSeriesDataLogic, StatsTimeSeriesDataLogic from ._trigger_logic import ( ADContAcqTriggerLogic, exposures_per_collection_from_filter, @@ -76,6 +77,7 @@ "NDStatsTSAcquireMode", "NDROIStatNIO", "NDROIStatIO", + "NDROIStatTSControl", "NDCBFlushOnSoftTrgMode", "NDCircularBuffIO", "ADFileWriteMode", @@ -108,6 +110,7 @@ "ADDataFactory", "ADWriterFactory", "StatsTimeSeriesDataLogic", + "ROIStatTimeSeriesDataLogic", # Detector "AreaDetector", "ContAcqDetector", diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index d324491287..4591985582 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -33,6 +33,7 @@ NDPluginBaseIO, NDPluginBaseIOT, NDPluginFileIO, + NDROIStatIO, NDStatsIO, plugin_is_enabled, ) @@ -42,7 +43,10 @@ NDAttributeSource, ndattributes_to_xml, ) -from ._stats_time_series import StatsTimeSeriesDataLogic +from ._stats_time_series import ( + ROIStatTimeSeriesDataLogic, + StatsTimeSeriesDataLogic, +) @dataclass @@ -668,6 +672,31 @@ def stats_time_series(stats: NDStatsIO) -> "ADDataFactory[NDStatsIO]": plugin=stats, ) + @staticmethod + def roistat_time_series(roistat: NDROIStatIO) -> "ADDataFactory[NDROIStatIO]": + """Create a factory for an NDPluginROIStat time series. + + Like `stats_time_series`, but for an ROI stat plugin (see + `ROIStatTimeSeriesDataLogic`). Each series is named after its scalar + signal, e.g. ``det-roistat-Blue-total``, and the plugin's EnableCallbacks + and each channel's Use decide which are published. + + :param roistat: The ROI stat plugin, which must be in the detector's + device tree, e.g. in its `plugins`. `AreaDetector` raises if it is + not. The same plugin can be listed in ``hdf(ndattribute_sources=)``. + """ + return ADDataFactory( + writer_cls=None, + writer_suffix="", + writer_name="", + datakey_suffix="", + array_description=None, + data_logic_factory=lambda used, desc, driver, plugins: ( + ROIStatTimeSeriesDataLogic(roistat=used) + ), + plugin=roistat, + ) + class ADWriterFactory(ADDataFactory[NDPluginBaseIOT]): """Deprecated name of `ADDataFactory`.""" diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index bd51e2405c..45c361c0cc 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -473,19 +473,32 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL ] total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] + # Time series of this ROI, filled while the plugin's time series acquires. + # These are not registered as readables: the scalars serve read(), and the + # arrays are consumed by ROIStatTimeSeriesDataLogic as event pages. + ts_min_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMinValue")] + ts_max_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMaxValue")] + ts_mean_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMeanValue")] + ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TSTotal")] + ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] + + def get_published_scalars(self) -> list[SignalR[float]]: + """The scalars of this ROI that have a read format.""" + formats = self.get_readable_formats() + return [ + scalar for scalar in self.params if formats.get(scalar) in _READ_FORMATS + ] def get_ndattributes(self, addr: int) -> list[NDAttributeParam]: """Each scalar with a read format, named after it, at asyn address `addr`.""" - formats = self.get_readable_formats() return [ NDAttributeParam( name=scalar.name, - param=param, + param=self.params[scalar], datatype=NDAttributeDataType.DOUBLE, addr=addr, ) - for scalar, param in self.params.items() - if formats.get(scalar) in _READ_FORMATS + for scalar in self.get_published_scalars() ] @cached_property @@ -498,6 +511,29 @@ def params(self) -> dict[SignalR[float], str]: self.total: "ROISTAT_TOTAL", } + @cached_property + def time_series(self) -> dict[SignalR[float], SignalR[Array1D[np.float64]]]: + """The time series of each scalar signal of this ROI.""" + return { + self.min_value: self.ts_min_value, + self.max_value: self.ts_max_value, + self.mean_value: self.ts_mean_value, + self.total: self.ts_total, + } + + +class NDROIStatTSControl(StrictEnum): + """What a write to the time series control of an `NDROIStatIO` does. + + Mirrors the ``TSControl`` record in ADCore/db/NDROIStat.template. + """ + + ERASE_START = "Erase/Start" + START = "Start" + STOP = "Stop" + READ = "Read" + ERASE = "Erase" + class NDROIStatIO(StandardReadable, NDPluginBaseIO): """Plugin for calculating basic statistics for multiple ROIs. @@ -516,6 +552,14 @@ class NDROIStatIO(StandardReadable, NDPluginBaseIO): See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROIStat.html """ + # Time series control, shared by every channel. Writing ERASE_START to + # ts_control erases the arrays of every channel, resets ts_current_point to 0 + # and starts acquiring, which stops by itself after ts_num_points frames. + ts_control: A[SignalRW[NDROIStatTSControl], PvSuffix("TSControl")] + ts_num_points: A[SignalRW[int], PvSuffix("TSNumPoints")] + ts_current_point: A[SignalR[int], PvSuffix("TSCurrentPoint")] + ts_acquiring: A[SignalR[bool], PvSuffix("TSAcquiring")] + def __init__(self, prefix, num_channels=8, with_pvi=False, name=""): with self.add_children_as_readables(): self.channels = DeviceVector( @@ -573,6 +617,12 @@ def get_switched_off_readables(self) -> set[SignalR]: for scalar in channel.params } + def get_computed_channels(self) -> list[NDROIStatNIO]: + """The channels whose switches were on when recorded.""" + return [ + channel for channel in self.channels.values() if self._is_computed(channel) + ] + def get_ndattributes(self) -> list[NDAttributeParam]: """The NDAttributes of each channel that is computed, by asyn address.""" return [ diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index a05eeecfb3..02a23c0724 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -15,7 +15,7 @@ ) from ophyd_async.epics.core import stop_busy_record -from ._io import NDStatsIO, NDStatsTSAcquireMode +from ._io import NDROIStatIO, NDROIStatTSControl, NDStatsIO, NDStatsTSAcquireMode @dataclass @@ -91,3 +91,64 @@ async def start_provider( async def ensure_stopped(self) -> None: await stop_busy_record(self.stats.ts_acquire) + + +@dataclass +class ROIStatTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): + """Data logic publishing an `NDPluginROIStat` time series as event pages. + + The ROI stat counterpart of `StatsTimeSeriesDataLogic`, for detectors that + write no file. The plugin has one time-series control (`ts_control`, + `ts_num_points`) shared by every channel, and each channel has its own arrays, + so one logic covers every channel. + + It publishes the time series of each scalar of `NDROIStatNIO.get_published_scalars` + of each of `NDROIStatIO.get_computed_channels` under the name of the scalar, + so the plugin's EnableCallbacks and each channel's Use decide what is + published, and this logic never writes them. It hints nothing. Each channel + has its own timestamp array, and the first computed channel's is used. + + The buffer is sized and armed in `start_provider` by writing ERASE_START to + ``ts_control``, which erases the arrays and resets the current point; the + plugin stops acquiring by itself when the buffer is full. + + :param roistat: the ROI stat plugin whose time series to read + """ + + roistat: NDROIStatIO + + def __post_init__(self) -> None: + self.collections_written = self.roistat.ts_current_point + + async def start_provider( + self, + name: str, + existing_datakeys: Sequence[str], + num_collections: int, + collections_per_event: int, + collection_period: float, + flush_period: float, + ) -> EventPageDataProvider | None: + # The buffer is sized by count, not rate, so neither period is used + channels = self.roistat.get_computed_channels() + arrays = { + scalar.name: channel.time_series[scalar] + for channel in channels + for scalar in channel.get_published_scalars() + if scalar.name not in existing_datakeys + } + if not arrays: + return None + # The plugin keeps the current point when resized, so size it first and + # then erase and start, which resets the current point to 0 + await self.roistat.ts_num_points.set(num_collections) + await self.roistat.ts_control.set(NDROIStatTSControl.ERASE_START) + # The IOC resets the count within the put, but its monitor may arrive + # after the put's readback, and the detector reads the count next + await wait_for_value(self.roistat.ts_current_point, 0, timeout=DEFAULT_TIMEOUT) + timestamps = channels[0].ts_timestamp + return EventPageDataProvider(arrays, timestamps, collections_per_event) + + async def ensure_stopped(self) -> None: + await self.roistat.ts_control.set(NDROIStatTSControl.STOP) + await wait_for_value(self.roistat.ts_acquiring, False, timeout=DEFAULT_TIMEOUT) diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 1ff3d98381..b2609a5c6c 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -1606,3 +1606,196 @@ async def test_uc9_prepare_raises_for_a_roistat_channel_used_since_stage(tmp_pat with pytest.raises(RuntimeError, match=r"changed since stage.*ROISTAT:3:Use_RBV"): await det.prepare(TriggerInfo()) + + +# Use case 10: the time series of an ROIStat plugin, for a detector that writes no +# file. Each used channel publishes one array per statistic with a read format, +# named after the scalar its channel already reads, which it suppresses. + + +async def _roistat_time_series_detector( + path_provider: StaticPathProvider | None = None, +): + """Three channels named Blue, Green and Red, all used; with an HDF writer + stamping the plugin's NDAttributes if given a path provider.""" + roistat = adcore.NDROIStatIO("PREFIX:ROISTAT:", num_channels=len(CHANNEL_NAMES)) + hdf_factories = ( + [ + adcore.ADDataFactory.hdf( + path_provider, always_enable=False, ndattribute_sources=[roistat] + ) + ] + if path_provider + else [] + ) + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + *hdf_factories, + adcore.ADDataFactory.roistat_time_series(roistat), + plugins={"roistat": roistat}, + name="det", + ) + set_mock_value(roistat.enable_callbacks, EnableDisable.ENABLE) + for number, name in CHANNEL_NAMES.items(): + set_mock_value(roistat.channels[number].name_, name) + set_mock_value(roistat.channels[number].use, True) + return det, roistat + + +def _fly_roistat_time_series( + RE: RunEngine, det, roistat: adcore.NDROIStatIO, points: int, also=lambda: None +) -> dict[str, list]: + """Behave like the IOC: ERASE_START resets the count, acquiring fills each + channel's arrays with 10 * channel number + frame number. The timestamps + differ per channel, which the IOC's do not, to show which channel's are used.""" + + def erase(value): + if value == adcore.NDROIStatTSControl.ERASE_START: + set_mock_value(roistat.ts_current_point, 0) + + def fill(value): + if value: + for number, channel in roistat.channels.items(): + for array in channel.time_series.values(): + set_mock_value( + array, # pyright: ignore[reportArgumentType] + 10.0 * number + np.arange(points), + ) + set_mock_value( + channel.ts_timestamp, # pyright: ignore[reportArgumentType] + 100.0 * number + np.arange(points), + ) + set_mock_value(roistat.ts_current_point, points) + also() + + callback_on_mock_put(roistat.ts_control, erase) + callback_on_mock_put(det.driver.acquire, fill) + docs: dict[str, list] = defaultdict(list) + RE( + _fly_plan(det, TriggerInfo(number_of_events=points)), + lambda name, doc: docs[name].append(doc), + ) + return docs + + +async def test_uc10_roistat_time_series_publishes_each_used_channel(RE: RunEngine): + det, roistat = await _roistat_time_series_detector() + # Green is not used, and Red publishes only its total + set_mock_value(roistat.channels[2].use, False) + red = roistat.channels[3] + for scalar in (red.min_value, red.max_value, red.mean_value): + red.set_readable_format(scalar, None) + + docs = _fly_roistat_time_series(RE, det, roistat, points=3) + + blue_keys = [ + f"det-roistat-Blue-{stat}" + for stat in ("min_value", "max_value", "mean_value", "total") + ] + keys = [*blue_keys, "det-roistat-Red-total"] + (descriptor,) = docs["descriptor"] + # The arrays replace the scalars of the same name, and the unused channel is gone + assert sorted(descriptor["data_keys"]) == sorted(keys) + total = descriptor["data_keys"]["det-roistat-Blue-total"] + assert total["source"] == "mock+ca://PREFIX:ROISTAT:1:TSTotal" + assert total["shape"] == [1] + # The channels hint their scalars, whoever serves them + assert descriptor["hints"]["det"]["fields"] == keys + assert docs["event"] == [] + (page,) = docs["event_page"] + assert sorted(page["data"]) == sorted(keys) + assert page["data"]["det-roistat-Blue-total"] == [[10.0], [11.0], [12.0]] + assert page["data"]["det-roistat-Red-total"] == [[30.0], [31.0], [32.0]] + # Every array is timed by the first used channel's timestamps + assert page["timestamps"]["det-roistat-Red-total"] == [100.0, 101.0, 102.0] + # The buffer was sized to the scan, then erased and started, and stopped on + # unstage + assert get_mock_put(roistat.ts_num_points).call_args_list[-1].args == (3,) + assert [ + call.args[0] for call in get_mock_put(roistat.ts_control).call_args_list + ] == [ + adcore.NDROIStatTSControl.STOP, + adcore.NDROIStatTSControl.ERASE_START, + adcore.NDROIStatTSControl.STOP, + ] + + +@pytest.mark.parametrize( + "enable_callbacks,used,serves", + [ + (EnableDisable.ENABLE, True, True), + # Switched off at stage: the time series sits out, and the scalars go + (EnableDisable.DISABLE, True, False), + (EnableDisable.ENABLE, False, False), + ], +) +async def test_uc10_roistat_time_series_follows_the_switches_at_stage( + enable_callbacks: EnableDisable, used: bool, serves: bool +): + det, roistat = await _roistat_time_series_detector() + set_mock_value(roistat.enable_callbacks, enable_callbacks) + for channel in roistat.channels.values(): + set_mock_value(channel.use, used) + callback_on_mock_put( + roistat.ts_control, + lambda value: set_mock_value(roistat.ts_current_point, 0), + ) + + await det.stage() + await det.prepare(TriggerInfo(number_of_events=2)) + + shapes = { + key: datakey["shape"] + for key, datakey in (await det.describe()).items() + if "roistat" in key + } + assert shapes == ( + { + f"det-roistat-{name}-{stat}": [1] + for name in CHANNEL_NAMES.values() + for stat in ("min_value", "max_value", "mean_value", "total") + } + if serves + else {} + ) + erase_starts = [ + call + for call in get_mock_put(roistat.ts_control).call_args_list + if call.args[0] == adcore.NDROIStatTSControl.ERASE_START + ] + assert len(erase_starts) == (1 if serves else 0) + + +@pytest.mark.parametrize("hdf_enabled", [True, False]) +async def test_uc10_the_hdf_switch_picks_the_file_or_the_roistat_time_series( + RE: RunEngine, + static_path_provider: StaticPathProvider, + hdf_enabled: bool, +): + det, roistat = await _roistat_time_series_detector(static_path_provider) + hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(hdf.file_path_exists, True) + set_mock_value(det.driver.array_size_x, 64) + set_mock_value(det.driver.array_size_y, 48) + set_mock_value( + hdf.enable_callbacks, + EnableDisable.ENABLE if hdf_enabled else EnableDisable.DISABLE, + ) + for channel in roistat.channels.values(): + for scalar in (channel.min_value, channel.max_value, channel.mean_value): + channel.set_readable_format(scalar, None) + + docs = _fly_roistat_time_series( + RE, det, roistat, points=2, also=lambda: set_mock_value(hdf.num_captured, 2) + ) + + totals = [f"det-roistat-{name}-total" for name in CHANNEL_NAMES.values()] + if hdf_enabled: + assert docs["event_page"] == [] + keys = [doc["data_key"] for doc in docs["stream_resource"]] + assert keys == ["det", *totals] + else: + assert docs["stream_resource"] == [] + (page,) = docs["event_page"] + assert list(page["data"]) == totals From 817f58a514277fb0361ff2dde030373c80c2a36b Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 3 Nov 2026 14:26:36 +0000 Subject: [PATCH 19/25] WIP: time series net, refresh before each page, buffer size check NDROIStatNIO gains net (Net_RBV, no read format, as on NDStatsIO) and ts_net, so TSNet is published when net has a read format. EventPageDataProvider takes an optional refresh, awaited before each page's arrays are read. The stats time series writes TS:TSRead and the ROIStat one writes READ to TSControl, so the plugin posts its arrays up to the current point instead of only about once a second. Both logics read TSNumPoints back after sizing it and raise if the IOC clamped it below the scan's number of collections. Assisted-by: Claude Opus 5.5 --- CHANGELOG.md | 4 +- .../decisions/0024-detector-is-a-flyable.md | 17 ++- src/ophyd_async/core/_data_providers.py | 10 +- src/ophyd_async/epics/adcore/_io.py | 11 +- .../epics/adcore/_stats_time_series.py | 40 +++++-- .../epics/adcore/test_stats_time_series.py | 19 +++ .../unit_tests/epics/adcore/test_use_cases.py | 110 ++++++++++++------ 7 files changed, 163 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50facf5fb3..23f5df7140 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,7 @@ - `NDAttributeSource`, a protocol for a plugin that returns the NDAttributes it stamps; `NDStatsIO` and `NDROIStatIO` implement it. - `ADDataFactory.hdf(ndattribute_sources=)`, the plugins whose NDAttributes the HDF writer writes. A source listed by two writers, or not in the detector's device tree, raises when the `AreaDetector` is built. The NDAttributes of other plugins and the driver are parsed from the XML the user set, with the names in it. - `NDPluginBaseIO.snapshot()`, `get_snapshot_signals()`, `get_changed_signals()`, `forget_snapshot()`, `switch_is_on()` and `get_switched_off_readables()`, which `AreaDetector` calls on its plugins and every plugin below them from `stage()` to `unstage()`. The snapshot is taken once, at `stage()`, and records the switches and the `NDArrayPort`. The plugins a trigger logic holds, such as its process plugin or circular buffer, are left out as the logic switches them on itself. -- `NDROIStatIO` stamps the `min_value`, `max_value`, `mean_value` and `total` of each channel that is used and has a read format, named after the scalar. At `stage()` an `AreaDetector` names each channel `-`, with characters other than letters, digits and underscores replaced by underscores and the channel number used for an empty name, and raises if two would be the same. The `use` of each channel is a switch, so a channel that is not used leaves `read()`, `describe()` and `hints`, and `prepare()` raises if the `Name` of a channel has changed since `stage()`. `NDROIStatIO` is a `StandardReadable` whose channels are registered as `CHILD` readables, so there is no need to register each channel. +- `NDROIStatIO` stamps the `min_value`, `max_value`, `mean_value`, `total` and `net` of each channel that is used and has a read format, named after the scalar. At `stage()` an `AreaDetector` names each channel `-`, with characters other than letters, digits and underscores replaced by underscores and the channel number used for an empty name, and raises if two would be the same. The `use` of each channel is a switch, so a channel that is not used leaves `read()`, `describe()` and `hints`, and `prepare()` raises if the `Name` of a channel has changed since `stage()`. `NDROIStatIO` is a `StandardReadable` whose channels are registered as `CHILD` readables, so there is no need to register each channel. - `KICKED_OFF`, the sentinel `FlyableLogic.fly` yields once the scan is up to speed. - `watch_readback()` reports the progress of a move by watching a `MovableLogic` readback, for use from `FlyableLogic` hooks. - `SimMotor.prepare()` and `SimMotor.complete()` report progress to watchers, as `Motor` does. @@ -76,6 +76,8 @@ - `BoundedDataLogic`, `UnboundedDataLogic`, `DetectorLogic`, `DataProvider`, `PageableDataProvider` and `EventPageDataProvider` in `ophyd_async.core`, and `ADDataFactory`, `NDStatsTSAcquireMode` and `exposures_per_collection_from_filter` in `ophyd_async.epics.adcore`. - `adcore.StatsTimeSeriesDataLogic` and the `ADDataFactory.stats_time_series(stats)` factory, a `BoundedDataLogic` that publishes an `NDPluginStats` time series as event pages, for an areaDetector that writes no file. It needs a fixed number of collections, as the buffer is sized before acquisition, and sits out when the plugin is switched off at `stage()`, so a detector can switch between its file writer and its stats time series from one stage to the next. - `adcore.ROIStatTimeSeriesDataLogic` and the `ADDataFactory.roistat_time_series(roistat)` factory, which publish the time series of an `NDROIStatIO` as event pages in the same way: one array per used channel per scalar with a read format, named after the scalar (e.g. `det-roistat-Blue-total`), which it suppresses. With it come `NDROIStatTSControl`, the `ts_control`, `ts_num_points`, `ts_current_point` and `ts_acquiring` signals of `NDROIStatIO`, the `ts_*` arrays of `NDROIStatNIO`, and `NDROIStatNIO.time_series`, `NDROIStatNIO.get_published_scalars()` and `NDROIStatIO.get_computed_channels()`. +- `NDROIStatNIO.net` and `NDROIStatNIO.ts_net`: `net` has no read format, as on `NDStatsIO`, so give it one to read, stamp or publish the `TSNet` time series of a channel. +- `EventPageDataProvider(refresh=)`, an awaitable callable awaited before each page's arrays are read. The stats and ROI stat time series use it to ask the plugin to post its arrays (`NDStatsIO.ts_read`, or `NDROIStatTSControl.READ`), which it otherwise does only about once a second while acquiring, so a page collected mid-scan no longer reads arrays shorter than the count. Both time series raise at `prepare()` if the plugin holds fewer points than the scan needs. ### Fixed diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md index 5042a854f0..ff63b77e58 100644 --- a/docs/explanations/decisions/0024-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -195,8 +195,21 @@ after that scalar, so it suppresses the scalar. It differs only where the plugin `TSControl` record erases and starts the buffer rather than a busy record, the buffer has no circular mode, so it is always fixed length, and every channel has its own timestamp array. All used channels are stamped from the same frame, so the first used channel's timestamps -serve them all. The plugin's `TSNet` array is not published, because `NDROIStatNIO` has no -`net` scalar to name it after. +serve them all. Like `NDStatsIO`, `NDROIStatNIO` has a `net` scalar with no read format, so +its `TSNet` array is published only when a user gives `net` one. + +Both plugins count every frame in `TSCurrentPoint` but post their arrays only when the buffer +is full, on Stop or Read, or from a record that asks for a Read about once a second while +acquiring. A page collected between those posts would read arrays shorter than the count, so +each logic gives its `EventPageDataProvider` a refresh that asks for a Read before the arrays +are read: `TSRead` of the stats plugin's embedded time series, or Read on the ROI stat +plugin's `TSControl`. In both plugins a Read only posts the arrays up to the current point, +and does not stop or erase an acquiring buffer. The refresh is a callable on the provider +rather than a step of the data logic, because the provider is what decides when a page is +read. The buffer is sized before acquiring, and the IOC may clamp the size to the record's +`DRVH` rather than fail the put (the ROI stat plugin's is its `NCHANS`), which would leave a +longer scan waiting for a buffer that never fills, so each logic reads the size back and +raises at `prepare()` if it is smaller than the scan. ### The detector flies with one `fly()` generator diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index 0c86a070c8..eb9b4d0ad1 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -1,6 +1,6 @@ import asyncio from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from typing import Any import numpy as np @@ -85,6 +85,10 @@ class EventPageDataProvider(PageableDataProvider): :param data: datakey (already suffixed) to the array signal that backs it :param timestamps: the acquisition time of each collection :param collections_per_event: how many collections make up one event + :param refresh: + awaited before each page's arrays are read, for a device that posts its + arrays less often than it counts collections: it asks the device to post + them, so they hold every collection counted """ def __init__( @@ -92,10 +96,12 @@ def __init__( data: Mapping[str, SignalR[Array1D[Any]]], timestamps: SignalR[Array1D[np.float64]], collections_per_event: int, + refresh: Callable[[], Awaitable[None]] | None = None, ) -> None: super().__init__(collections_per_event) self.data = dict(data) self.timestamps = timestamps + self.refresh = refresh self.start_new_resources(0) def start_new_resources(self, origin: int) -> None: @@ -119,6 +125,8 @@ async def make_datakeys(self) -> dict[str, DataKey]: async def make_page(self, index: int) -> PartialEventPage | None: if index <= self._published: return None + if self.refresh: + await self.refresh() cpe = self.collections_per_event # Read every array and the per-collection timestamps in one parallel batch arrays, stamps = await asyncio.gather( diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index 45c361c0cc..b297ad316d 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -15,6 +15,7 @@ EnableDisable, SignalR, SignalRW, + SignalW, StandardReadable, StrictEnum, SubsetEnum, @@ -331,11 +332,13 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): # prefix while the per-statistic arrays are named by NDStats itself at the # plugin's own prefix. ts_num_points sizes the buffer, ts_acquire erases and # starts it (writing 1 clears the arrays and resets ts_current_point to 0), - # ts_current_point reports progress, each ts_ holds the series of - # that statistic and ts_timestamp holds the per-point acquisition times. + # ts_current_point reports progress, writing 1 to ts_read posts the arrays + # while acquiring, each ts_ holds the series of that statistic and + # ts_timestamp holds the per-point acquisition times. # These are not registered as readables: the scalars serve read(), and the # arrays are consumed by StatsTimeSeriesDataLogic as event pages. ts_acquire: A[SignalRW[bool], PvSuffix("TS:TSAcquire"), EpicsOptions(wait=non_zero)] + ts_read: A[SignalW[bool], PvSuffix("TS:TSRead")] ts_num_points: A[SignalRW[int], PvSuffix("TS:TSNumPoints")] ts_current_point: A[SignalR[int], PvSuffix("TS:TSCurrentPoint")] ts_acquiring: A[SignalR[bool], PvSuffix("TS:TSAcquiring")] @@ -473,6 +476,7 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL ] total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] + net: A[SignalR[float], PvSuffix("Net_RBV")] # Time series of this ROI, filled while the plugin's time series acquires. # These are not registered as readables: the scalars serve read(), and the # arrays are consumed by ROIStatTimeSeriesDataLogic as event pages. @@ -480,6 +484,7 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): ts_max_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMaxValue")] ts_mean_value: A[SignalR[Array1D[np.float64]], PvSuffix("TSMeanValue")] ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TSTotal")] + ts_net: A[SignalR[Array1D[np.float64]], PvSuffix("TSNet")] ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] def get_published_scalars(self) -> list[SignalR[float]]: @@ -509,6 +514,7 @@ def params(self) -> dict[SignalR[float], str]: self.max_value: "ROISTAT_MAX_VALUE", self.mean_value: "ROISTAT_MEAN_VALUE", self.total: "ROISTAT_TOTAL", + self.net: "ROISTAT_NET", } @cached_property @@ -519,6 +525,7 @@ def time_series(self) -> dict[SignalR[float], SignalR[Array1D[np.float64]]]: self.max_value: self.ts_max_value, self.mean_value: self.ts_mean_value, self.total: self.ts_total, + self.net: self.ts_net, } diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index 02a23c0724..c4960ec61f 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -1,6 +1,7 @@ import asyncio from collections.abc import Sequence from dataclasses import dataclass +from functools import partial import numpy as np @@ -10,6 +11,7 @@ BoundedDataLogic, EventPageDataProvider, SignalR, + SignalRW, set_and_wait_for_value, wait_for_value, ) @@ -18,6 +20,18 @@ from ._io import NDROIStatIO, NDROIStatTSControl, NDStatsIO, NDStatsTSAcquireMode +async def _set_buffer_size(num_points: SignalRW[int], num_collections: int) -> None: + """Size a time series buffer, raising if the IOC holds fewer points.""" + await num_points.set(num_collections) + # The record clamps a size above its DRVH rather than failing the put + size = await num_points.get_value() + if size < num_collections: + raise ValueError( + f"{num_points.source}: the scan needs {num_collections} points, " + f"but the plugin holds at most {size}" + ) + + @dataclass class StatsTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): """Data logic publishing an `NDPluginStats` time series as event pages. @@ -38,8 +52,11 @@ class StatsTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): again at each step point after the first, by writing 1 to ``ts_acquire``, which erases the arrays and resets the current point before the detector's acquire logic drives the camera; its frames then feed the time series via - NDArray callbacks. A detector that writes a file should pull stats into the - file as NDAttributes instead (see `ADHDFDataLogic`). + NDArray callbacks. `start_provider` raises if the plugin holds fewer points + than the scan needs. Before each page it writes 1 to ``ts_read``, so the + plugin posts its arrays up to the current point. A detector that writes a + file should pull stats into the file as NDAttributes instead (see + `ADHDFDataLogic`). :param stats: the stats plugin whose time series to read """ @@ -74,7 +91,7 @@ async def start_provider( return None # Size the buffer and put it in fixed-length mode before arming it await asyncio.gather( - self.stats.ts_num_points.set(num_collections), + _set_buffer_size(self.stats.ts_num_points, num_collections), self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), ) # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to @@ -85,8 +102,11 @@ async def start_provider( # The IOC resets the count within the put, but its monitor may arrive # after the put's readback, and the detector reads the count next await wait_for_value(self.stats.ts_current_point, 0, timeout=DEFAULT_TIMEOUT) + # The plugin posts its arrays about once a second while acquiring, so + # ask it to post them before each page + refresh = partial(self.stats.ts_read.set, True) return EventPageDataProvider( - arrays, self.stats.ts_timestamp, collections_per_event + arrays, self.stats.ts_timestamp, collections_per_event, refresh ) async def ensure_stopped(self) -> None: @@ -110,7 +130,10 @@ class ROIStatTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): The buffer is sized and armed in `start_provider` by writing ERASE_START to ``ts_control``, which erases the arrays and resets the current point; the - plugin stops acquiring by itself when the buffer is full. + plugin stops acquiring by itself when the buffer is full. `start_provider` + raises if the plugin holds fewer points than the scan needs. Before each page + it writes READ to ``ts_control``, so the plugin posts its arrays up to the + current point. :param roistat: the ROI stat plugin whose time series to read """ @@ -141,13 +164,16 @@ async def start_provider( return None # The plugin keeps the current point when resized, so size it first and # then erase and start, which resets the current point to 0 - await self.roistat.ts_num_points.set(num_collections) + await _set_buffer_size(self.roistat.ts_num_points, num_collections) await self.roistat.ts_control.set(NDROIStatTSControl.ERASE_START) # The IOC resets the count within the put, but its monitor may arrive # after the put's readback, and the detector reads the count next await wait_for_value(self.roistat.ts_current_point, 0, timeout=DEFAULT_TIMEOUT) timestamps = channels[0].ts_timestamp - return EventPageDataProvider(arrays, timestamps, collections_per_event) + # The plugin posts its arrays about once a second while acquiring, so + # ask it to post them before each page + refresh = partial(self.roistat.ts_control.set, NDROIStatTSControl.READ) + return EventPageDataProvider(arrays, timestamps, collections_per_event, refresh) async def ensure_stopped(self) -> None: await self.roistat.ts_control.set(NDROIStatTSControl.STOP) diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index af81f8b46c..f3783d572c 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -70,6 +70,25 @@ async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): assert datakeys["det-stats-total"]["shape"] == [5] +@pytest.mark.parametrize("points,raises", [(2048, False), (2049, True)]) +async def test_start_raises_for_a_scan_longer_than_the_buffer( + stats: adcore.NDStatsIO, points: int, raises: bool +): + # An IOC whose TSNumPoints has a DRVH clamps the size to it + callback_on_mock_put(stats.ts_num_points, lambda value: min(value, 2048)) + logic = StatsTimeSeriesDataLogic(stats) + + start = logic.start_provider("det", [], points, 1, 0.1, 0) + + if raises: + with pytest.raises( + ValueError, match="needs 2049 points, but the plugin holds at most 2048" + ): + await start + else: + assert await start is not None + + def test_every_time_series_is_a_statistic(stats: adcore.NDStatsIO): """Every ts_* array but the timestamps is in `statistics`, so can be published.""" hints = get_type_hints(adcore.NDStatsIO) diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index b2609a5c6c..97b6e6d12b 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -845,31 +845,36 @@ def _switch_stats(stats: adcore.NDStatsIO, on: bool) -> None: def _fill_time_series( det, plugins: list[adcore.NDStatsIO], points: int, also=lambda: None ): - """Behave like the IOC: arming a time series erases it, acquiring fills it.""" + """Behave like the IOC: arming a time series erases it, acquiring counts the + points, and the arrays are posted only when TSRead asks for them, as for a + page collected between the plugin's once-a-second posts.""" def erase(value): if value: for plugin in plugins: set_mock_value(plugin.ts_current_point, 0) - def fill(value): + def count(value): if value: for plugin in plugins: - for stat in plugin.statistics: - set_mock_value( - stat.time_series, # pyright: ignore[reportArgumentType] - np.arange(points, dtype=np.float64), - ) - set_mock_value( - plugin.ts_timestamp, # pyright: ignore[reportArgumentType] - 100.0 + np.arange(points), - ) set_mock_value(plugin.ts_current_point, points) also() + def post(plugin: adcore.NDStatsIO): + for stat in plugin.statistics: + set_mock_value( + stat.time_series, # pyright: ignore[reportArgumentType] + np.arange(points, dtype=np.float64), + ) + set_mock_value( + plugin.ts_timestamp, # pyright: ignore[reportArgumentType] + 100.0 + np.arange(points), + ) + for plugin in plugins: callback_on_mock_put(plugin.ts_acquire, erase) - callback_on_mock_put(det.driver.acquire, fill) + callback_on_mock_put(plugin.ts_read, lambda _, plugin=plugin: post(plugin)) + callback_on_mock_put(det.driver.acquire, count) def _fly_plan(det, info: TriggerInfo): @@ -986,6 +991,8 @@ def test_uc_b_detector_with_a_stats_time_series(RE: RunEngine): assert descriptor["hints"]["det"]["fields"] == STATS_HINTS (page,) = docs["event_page"] assert page["data"]["det-stats-total"] == [[0.0], [1.0], [2.0]] + # The arrays were posted for the page + get_mock_put(stats.ts_read).assert_called_once_with(True) # It made no plugin of its own assert [name for name, _ in det.children() if "stat" in name] == ["stats"] @@ -1646,31 +1653,37 @@ async def _roistat_time_series_detector( def _fly_roistat_time_series( RE: RunEngine, det, roistat: adcore.NDROIStatIO, points: int, also=lambda: None ) -> dict[str, list]: - """Behave like the IOC: ERASE_START resets the count, acquiring fills each - channel's arrays with 10 * channel number + frame number. The timestamps - differ per channel, which the IOC's do not, to show which channel's are used.""" + """Behave like the IOC: ERASE_START resets the count, acquiring counts the + points, and READ posts each channel's arrays, filled with + 10 * channel number + frame number, as for a page collected between the + plugin's once-a-second posts. The timestamps differ per channel, which the + IOC's do not, to show which channel's are used.""" + + def post(): + for number, channel in roistat.channels.items(): + for array in channel.time_series.values(): + set_mock_value( + array, # pyright: ignore[reportArgumentType] + 10.0 * number + np.arange(points), + ) + set_mock_value( + channel.ts_timestamp, # pyright: ignore[reportArgumentType] + 100.0 * number + np.arange(points), + ) - def erase(value): + def control(value): if value == adcore.NDROIStatTSControl.ERASE_START: set_mock_value(roistat.ts_current_point, 0) + elif value == adcore.NDROIStatTSControl.READ: + post() - def fill(value): + def count(value): if value: - for number, channel in roistat.channels.items(): - for array in channel.time_series.values(): - set_mock_value( - array, # pyright: ignore[reportArgumentType] - 10.0 * number + np.arange(points), - ) - set_mock_value( - channel.ts_timestamp, # pyright: ignore[reportArgumentType] - 100.0 * number + np.arange(points), - ) set_mock_value(roistat.ts_current_point, points) also() - callback_on_mock_put(roistat.ts_control, erase) - callback_on_mock_put(det.driver.acquire, fill) + callback_on_mock_put(roistat.ts_control, control) + callback_on_mock_put(det.driver.acquire, count) docs: dict[str, list] = defaultdict(list) RE( _fly_plan(det, TriggerInfo(number_of_events=points)), @@ -1681,17 +1694,19 @@ def fill(value): async def test_uc10_roistat_time_series_publishes_each_used_channel(RE: RunEngine): det, roistat = await _roistat_time_series_detector() - # Green is not used, and Red publishes only its total + # Green is not used, Red publishes only its total, and Blue its net too set_mock_value(roistat.channels[2].use, False) red = roistat.channels[3] for scalar in (red.min_value, red.max_value, red.mean_value): red.set_readable_format(scalar, None) + blue = roistat.channels[1] + blue.set_readable_format(blue.net, Format.UNCACHED_SIGNAL) docs = _fly_roistat_time_series(RE, det, roistat, points=3) blue_keys = [ f"det-roistat-Blue-{stat}" - for stat in ("min_value", "max_value", "mean_value", "total") + for stat in ("min_value", "max_value", "mean_value", "total", "net") ] keys = [*blue_keys, "det-roistat-Red-total"] (descriptor,) = docs["descriptor"] @@ -1700,27 +1715,52 @@ async def test_uc10_roistat_time_series_publishes_each_used_channel(RE: RunEngin total = descriptor["data_keys"]["det-roistat-Blue-total"] assert total["source"] == "mock+ca://PREFIX:ROISTAT:1:TSTotal" assert total["shape"] == [1] - # The channels hint their scalars, whoever serves them - assert descriptor["hints"]["det"]["fields"] == keys + net = descriptor["data_keys"]["det-roistat-Blue-net"] + assert net["source"] == "mock+ca://PREFIX:ROISTAT:1:TSNet" + # The channels hint their scalars, whoever serves them, and net is not hinted + hinted = [key for key in keys if key != "det-roistat-Blue-net"] + assert descriptor["hints"]["det"]["fields"] == hinted assert docs["event"] == [] (page,) = docs["event_page"] assert sorted(page["data"]) == sorted(keys) assert page["data"]["det-roistat-Blue-total"] == [[10.0], [11.0], [12.0]] + assert page["data"]["det-roistat-Blue-net"] == [[10.0], [11.0], [12.0]] assert page["data"]["det-roistat-Red-total"] == [[30.0], [31.0], [32.0]] # Every array is timed by the first used channel's timestamps assert page["timestamps"]["det-roistat-Red-total"] == [100.0, 101.0, 102.0] - # The buffer was sized to the scan, then erased and started, and stopped on - # unstage + # The buffer was sized to the scan, then erased and started, its arrays + # posted for the page, and stopped on unstage assert get_mock_put(roistat.ts_num_points).call_args_list[-1].args == (3,) assert [ call.args[0] for call in get_mock_put(roistat.ts_control).call_args_list ] == [ adcore.NDROIStatTSControl.STOP, adcore.NDROIStatTSControl.ERASE_START, + adcore.NDROIStatTSControl.READ, adcore.NDROIStatTSControl.STOP, ] +@pytest.mark.parametrize("points,raises", [(2048, False), (2049, True)]) +async def test_uc10_prepare_raises_for_a_scan_longer_than_the_roistat_buffer( + points: int, raises: bool +): + det, roistat = await _roistat_time_series_detector() + # The IOC clamps the size to DRVH, which is NCHANS + callback_on_mock_put(roistat.ts_num_points, lambda value: min(value, 2048)) + await det.stage() + + prepare = det.prepare(TriggerInfo(number_of_events=points)) + + if raises: + with pytest.raises( + ValueError, match="needs 2049 points, but the plugin holds at most 2048" + ): + await prepare + else: + await prepare + + @pytest.mark.parametrize( "enable_callbacks,used,serves", [ From e4ac87f77031b0272bd6c864e449347f7ab7c137 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 3 Nov 2026 14:48:57 +0000 Subject: [PATCH 20/25] WIP: time series pages wait for arrays to hold the count; check NELM at start Assisted-by: Claude Sonnet 5.5 --- CHANGELOG.md | 2 +- .../decisions/0024-detector-is-a-flyable.md | 15 +++++-- src/ophyd_async/core/_data_providers.py | 40 ++++++++++++++--- src/ophyd_async/epics/adcore/_io.py | 4 ++ .../epics/adcore/_stats_time_series.py | 39 ++++++++-------- tests/unit_tests/core/test_data_providers.py | 45 +++++++++++++++++++ .../epics/adcore/test_stats_time_series.py | 6 +-- .../unit_tests/epics/adcore/test_use_cases.py | 9 ++-- .../test_areadetector_subclass_naming.py | 3 +- 9 files changed, 126 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f5df7140..bdcfe4382c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,7 +77,7 @@ - `adcore.StatsTimeSeriesDataLogic` and the `ADDataFactory.stats_time_series(stats)` factory, a `BoundedDataLogic` that publishes an `NDPluginStats` time series as event pages, for an areaDetector that writes no file. It needs a fixed number of collections, as the buffer is sized before acquisition, and sits out when the plugin is switched off at `stage()`, so a detector can switch between its file writer and its stats time series from one stage to the next. - `adcore.ROIStatTimeSeriesDataLogic` and the `ADDataFactory.roistat_time_series(roistat)` factory, which publish the time series of an `NDROIStatIO` as event pages in the same way: one array per used channel per scalar with a read format, named after the scalar (e.g. `det-roistat-Blue-total`), which it suppresses. With it come `NDROIStatTSControl`, the `ts_control`, `ts_num_points`, `ts_current_point` and `ts_acquiring` signals of `NDROIStatIO`, the `ts_*` arrays of `NDROIStatNIO`, and `NDROIStatNIO.time_series`, `NDROIStatNIO.get_published_scalars()` and `NDROIStatIO.get_computed_channels()`. - `NDROIStatNIO.net` and `NDROIStatNIO.ts_net`: `net` has no read format, as on `NDStatsIO`, so give it one to read, stamp or publish the `TSNet` time series of a channel. -- `EventPageDataProvider(refresh=)`, an awaitable callable awaited before each page's arrays are read. The stats and ROI stat time series use it to ask the plugin to post its arrays (`NDStatsIO.ts_read`, or `NDROIStatTSControl.READ`), which it otherwise does only about once a second while acquiring, so a page collected mid-scan no longer reads arrays shorter than the count. Both time series raise at `prepare()` if the plugin holds fewer points than the scan needs. +- `EventPageDataProvider(refresh=)`, an awaitable callable awaited before each page's arrays are read. The stats and ROI stat time series use it to ask the plugin to post its arrays (`NDStatsIO.ts_read`, or `NDROIStatTSControl.READ`), which it otherwise does only about once a second while acquiring, so a page collected mid-scan no longer reads arrays shorter than the count. `EventPageDataProvider` now waits, up to `ARRAY_WAIT_TIMEOUT` (2 s), for each array and the timestamps to hold every collection of the page before slicing them, and raises `TimeoutError` naming the signal if one does not, so a page never slices arrays shorter than the count. Both time series raise `ValueError` at `prepare()` if the plugin's arrays (`ts_timestamp_nelm`, the waveforms' `NELM`) hold fewer points than the scan needs. ### Fixed diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md index ff63b77e58..515ff3d516 100644 --- a/docs/explanations/decisions/0024-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -206,10 +206,17 @@ are read: `TSRead` of the stats plugin's embedded time series, or Read on the RO plugin's `TSControl`. In both plugins a Read only posts the arrays up to the current point, and does not stop or erase an acquiring buffer. The refresh is a callable on the provider rather than a step of the data logic, because the provider is what decides when a page is -read. The buffer is sized before acquiring, and the IOC may clamp the size to the record's -`DRVH` rather than fail the put (the ROI stat plugin's is its `NCHANS`), which would leave a -longer scan waiting for a buffer that never fills, so each logic reads the size back and -raises at `prepare()` if it is smaller than the scan. +read. A Read put can complete before the plugin's I/O Intr waveform records have processed, +so the refresh alone does not guarantee that the arrays are up to date. The provider +therefore observes each array and the timestamps until each holds as many collections as the +page needs, for at most `ARRAY_WAIT_TIMEOUT` (longer than the one second between the IOC's +posts), and raises a `TimeoutError` naming the signal if one does not. It slices the values +it observed, never a later get, so a page never has an array shorter than the count it +indexes by. The arrays are waveforms of `NCHANS` elements in both plugins, and a post is cut +to that length, so a scan longer than that could never be served. Each logic reads the +timestamp waveform's `NELM` in `start_provider` and raises at `prepare()` if it is smaller +than the scan. The `TSNumPoints` record is not a reliable check: only the ROI stat plugin's +clamps it to `NCHANS`, the stats plugin's `DRVH` is far larger. ### The detector flies with one `fly()` generator diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index eb9b4d0ad1..9301a7cbab 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -9,9 +9,32 @@ from event_model.documents import PartialEventPage from ._datatypes import Array1D -from ._signal import SignalR, SignalW +from ._signal import SignalR, SignalW, observe_value from ._utils import ConfinedModel, error_if_none, gather_dict, get_dtype +ARRAY_WAIT_TIMEOUT = 2.0 +"""How long `EventPageDataProvider` waits for an array to hold the collections +counted, in seconds. Longer than the one second between the posts of an +areaDetector time series.""" + + +async def _wait_for_length(signal: SignalR[Array1D[Any]], needed: int) -> Array1D[Any]: + """Return the first value of `signal` holding at least `needed` collections.""" + held = 0 + kept = None + try: + async for value in observe_value(signal, done_timeout=ARRAY_WAIT_TIMEOUT): + held = len(value) + if held >= needed: + kept = value + break + except TimeoutError: + raise TimeoutError( + f"{signal.source} held {held} collections after {ARRAY_WAIT_TIMEOUT}s, " + f"but {needed} were needed" + ) from None + return error_if_none(kept, f"{signal.source} was not observed") + class DataProvider(ABC): """What a [](#DetectorDataLogic) returns to publish the data it is writing. @@ -80,7 +103,9 @@ class EventPageDataProvider(PageableDataProvider): array of per-collection timestamps: an areaDetector stats time series, or a scaler's MCA arrays. Each array is sliced into `collections_per_event`-length chunks, one per event, and the last collection of an event supplies that - event's timestamp. + event's timestamp. A page waits up to `ARRAY_WAIT_TIMEOUT` for each array to + hold every collection of its events, and raises `TimeoutError` naming the + signal if one does not. :param data: datakey (already suffixed) to the array signal that backs it :param timestamps: the acquisition time of each collection @@ -128,12 +153,17 @@ async def make_page(self, index: int) -> PartialEventPage | None: if self.refresh: await self.refresh() cpe = self.collections_per_event - # Read every array and the per-collection timestamps in one parallel batch + needed = self._origin + index * cpe + # Wait for every array and the per-collection timestamps to hold the + # collections counted, all at once arrays, stamps = await asyncio.gather( gather_dict( - {datakey: signal.get_value() for datakey, signal in self.data.items()} + { + datakey: _wait_for_length(signal, needed) + for datakey, signal in self.data.items() + } ), - self.timestamps.get_value(), + _wait_for_length(self.timestamps, needed), ) starts = [self._origin + event * cpe for event in range(self._published, index)] self._published = index diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index b297ad316d..d6fd4271c0 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -359,6 +359,8 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): ts_sigma_y: A[SignalR[Array1D[np.float64]], PvSuffix("TSSigmaY")] ts_sigma_xy: A[SignalR[Array1D[np.float64]], PvSuffix("TSSigmaXY")] ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] + # How many points the time series arrays hold (the waveforms' NELM) + ts_timestamp_nelm: A[SignalR[int], PvSuffix("TSTimestamp.NELM")] # Centroid statistics compute_centroid: A[ SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL @@ -486,6 +488,8 @@ class NDROIStatNIO(StandardReadable, EpicsDevice): ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TSTotal")] ts_net: A[SignalR[Array1D[np.float64]], PvSuffix("TSNet")] ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] + # How many points the time series arrays hold (the waveforms' NELM) + ts_timestamp_nelm: A[SignalR[int], PvSuffix("TSTimestamp.NELM")] def get_published_scalars(self) -> list[SignalR[float]]: """The scalars of this ROI that have a read format.""" diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index c4960ec61f..110b041f8a 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -11,7 +11,6 @@ BoundedDataLogic, EventPageDataProvider, SignalR, - SignalRW, set_and_wait_for_value, wait_for_value, ) @@ -20,15 +19,13 @@ from ._io import NDROIStatIO, NDROIStatTSControl, NDStatsIO, NDStatsTSAcquireMode -async def _set_buffer_size(num_points: SignalRW[int], num_collections: int) -> None: - """Size a time series buffer, raising if the IOC holds fewer points.""" - await num_points.set(num_collections) - # The record clamps a size above its DRVH rather than failing the put - size = await num_points.get_value() - if size < num_collections: +async def _check_capacity(capacity: SignalR[int], num_collections: int) -> None: + """Raise if the time series arrays hold fewer points than the scan needs.""" + points = await capacity.get_value() + if points < num_collections: raise ValueError( - f"{num_points.source}: the scan needs {num_collections} points, " - f"but the plugin holds at most {size}" + f"{capacity.source}: the scan needs {num_collections} points, " + f"but the plugin's arrays hold {points}" ) @@ -52,11 +49,11 @@ class StatsTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): again at each step point after the first, by writing 1 to ``ts_acquire``, which erases the arrays and resets the current point before the detector's acquire logic drives the camera; its frames then feed the time series via - NDArray callbacks. `start_provider` raises if the plugin holds fewer points - than the scan needs. Before each page it writes 1 to ``ts_read``, so the - plugin posts its arrays up to the current point. A detector that writes a - file should pull stats into the file as NDAttributes instead (see - `ADHDFDataLogic`). + NDArray callbacks. `start_provider` raises `ValueError` if the plugin's + arrays hold fewer points than the scan needs. Before each page it writes 1 to + ``ts_read``, so the plugin posts its arrays up to the current point. A detector + that writes a file should pull stats into the file as NDAttributes instead + (see `ADHDFDataLogic`). :param stats: the stats plugin whose time series to read """ @@ -90,8 +87,9 @@ async def start_provider( if not arrays: return None # Size the buffer and put it in fixed-length mode before arming it + await _check_capacity(self.stats.ts_timestamp_nelm, num_collections) await asyncio.gather( - _set_buffer_size(self.stats.ts_num_points, num_collections), + self.stats.ts_num_points.set(num_collections), self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), ) # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to @@ -131,9 +129,9 @@ class ROIStatTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): The buffer is sized and armed in `start_provider` by writing ERASE_START to ``ts_control``, which erases the arrays and resets the current point; the plugin stops acquiring by itself when the buffer is full. `start_provider` - raises if the plugin holds fewer points than the scan needs. Before each page - it writes READ to ``ts_control``, so the plugin posts its arrays up to the - current point. + raises `ValueError` if the plugin's arrays hold fewer points than the scan + needs. Before each page it writes READ to ``ts_control``, so the plugin posts + its arrays up to the current point. :param roistat: the ROI stat plugin whose time series to read """ @@ -164,14 +162,15 @@ async def start_provider( return None # The plugin keeps the current point when resized, so size it first and # then erase and start, which resets the current point to 0 - await _set_buffer_size(self.roistat.ts_num_points, num_collections) + await _check_capacity(channels[0].ts_timestamp_nelm, num_collections) + await self.roistat.ts_num_points.set(num_collections) await self.roistat.ts_control.set(NDROIStatTSControl.ERASE_START) # The IOC resets the count within the put, but its monitor may arrive # after the put's readback, and the detector reads the count next await wait_for_value(self.roistat.ts_current_point, 0, timeout=DEFAULT_TIMEOUT) - timestamps = channels[0].ts_timestamp # The plugin posts its arrays about once a second while acquiring, so # ask it to post them before each page + timestamps = channels[0].ts_timestamp refresh = partial(self.roistat.ts_control.set, NDROIStatTSControl.READ) return EventPageDataProvider(arrays, timestamps, collections_per_event, refresh) diff --git a/tests/unit_tests/core/test_data_providers.py b/tests/unit_tests/core/test_data_providers.py index bc3b8a0609..4f8a2d0017 100644 --- a/tests/unit_tests/core/test_data_providers.py +++ b/tests/unit_tests/core/test_data_providers.py @@ -1,3 +1,6 @@ +import asyncio +from collections.abc import Callable + import numpy as np import pytest @@ -61,3 +64,45 @@ async def test_pages_carry_on_from_the_last(): [102.0, 103.0], None, ] + + +def make_lagging_provider( + held: int, +) -> tuple[EventPageDataProvider, Callable[[int], None]]: + """Arrays holding the first `held` of 5 collections, and a way to post the rest.""" + values = np.array([10.0, 11.0, 12.0, 13.0, 14.0]) + times = np.array([100.0, 101.0, 102.0, 103.0, 104.0]) + data, set_data = soft_signal_r_and_setter(Array1D[np.float64], values[:held]) + timestamps, set_times = soft_signal_r_and_setter(Array1D[np.float64], times[:held]) + + def post(count: int) -> None: + set_data(values[:count]) + set_times(times[:count]) + + return EventPageDataProvider({"det-stats": data}, timestamps, 1), post + + +async def test_page_waits_for_arrays_shorter_than_the_count(): + provider, post = make_lagging_provider(held=2) + + async def post_later(): + await asyncio.sleep(0.1) + post(5) + + posting = asyncio.create_task(post_later()) + page = await provider.make_page(5) + await posting + + assert page is not None + np.testing.assert_array_equal( + page["data"]["det-stats"], [[10.0], [11.0], [12.0], [13.0], [14.0]] + ) + assert page["time"] == [100.0, 101.0, 102.0, 103.0, 104.0] + + +async def test_page_raises_if_the_arrays_never_catch_up(monkeypatch): + monkeypatch.setattr("ophyd_async.core._data_providers.ARRAY_WAIT_TIMEOUT", 0.1) + provider, _ = make_lagging_provider(held=2) + + with pytest.raises(TimeoutError, match="held 2 collections .* 5 were needed"): + await provider.make_page(5) diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index f3783d572c..bf8c2d7e40 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -40,6 +40,8 @@ async def stats() -> adcore.NDStatsIO: async with init_devices(mock=True): plugin = adcore.NDStatsIO("PREFIX:STATS:", name="det-stats") + # The IOC's arrays hold NCHANS points, 2048 by default + set_mock_value(plugin.ts_timestamp_nelm, 2048) return plugin @@ -74,15 +76,13 @@ async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): async def test_start_raises_for_a_scan_longer_than_the_buffer( stats: adcore.NDStatsIO, points: int, raises: bool ): - # An IOC whose TSNumPoints has a DRVH clamps the size to it - callback_on_mock_put(stats.ts_num_points, lambda value: min(value, 2048)) logic = StatsTimeSeriesDataLogic(stats) start = logic.start_provider("det", [], points, 1, 0.1, 0) if raises: with pytest.raises( - ValueError, match="needs 2049 points, but the plugin holds at most 2048" + ValueError, match="needs 2049 points, but the plugin's arrays hold 2048" ): await start else: diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 97b6e6d12b..4088b07c11 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -872,6 +872,7 @@ def post(plugin: adcore.NDStatsIO): ) for plugin in plugins: + set_mock_value(plugin.ts_timestamp_nelm, 2048) callback_on_mock_put(plugin.ts_acquire, erase) callback_on_mock_put(plugin.ts_read, lambda _, plugin=plugin: post(plugin)) callback_on_mock_put(det.driver.acquire, count) @@ -1147,6 +1148,7 @@ async def test_readable_stats_total_is_suppressed_by_any_higher_tier( "PREFIX:", *factories, plugins={"stats": stats}, name="det" ) _switch_stats(stats, on=True) + set_mock_value(stats.ts_timestamp_nelm, 2048) hdf = det.get_plugin("hdf", adcore.NDFileHDF5IO) set_mock_value( hdf.enable_callbacks, @@ -1645,6 +1647,7 @@ async def _roistat_time_series_detector( ) set_mock_value(roistat.enable_callbacks, EnableDisable.ENABLE) for number, name in CHANNEL_NAMES.items(): + set_mock_value(roistat.channels[number].ts_timestamp_nelm, 2048) set_mock_value(roistat.channels[number].name_, name) set_mock_value(roistat.channels[number].use, True) return det, roistat @@ -1746,15 +1749,15 @@ async def test_uc10_prepare_raises_for_a_scan_longer_than_the_roistat_buffer( points: int, raises: bool ): det, roistat = await _roistat_time_series_detector() - # The IOC clamps the size to DRVH, which is NCHANS - callback_on_mock_put(roistat.ts_num_points, lambda value: min(value, 2048)) + # The arrays hold NCHANS points + set_mock_value(roistat.channels[1].ts_timestamp_nelm, 2048) await det.stage() prepare = det.prepare(TriggerInfo(number_of_events=points)) if raises: with pytest.raises( - ValueError, match="needs 2049 points, but the plugin holds at most 2048" + ValueError, match="needs 2049 points, but the plugin's arrays hold 2048" ): await prepare else: diff --git a/tests/unit_tests/epics/test_areadetector_subclass_naming.py b/tests/unit_tests/epics/test_areadetector_subclass_naming.py index 496d71cb4f..576bb98a6c 100644 --- a/tests/unit_tests/epics/test_areadetector_subclass_naming.py +++ b/tests/unit_tests/epics/test_areadetector_subclass_naming.py @@ -42,7 +42,8 @@ def check_name(name: str, device: Device): # remove trailing underscore from name, # used to resolve clashes with Bluesky terms name = name[:-1] if name.endswith("_") else name - assert inflection.underscore(pv) == name + # A field of a record, such as TSTimestamp.NELM, is named record_field + assert inflection.underscore(pv.replace(".", "_")) == name elif isinstance(device, DeviceVector): return else: From 8fbf486b5b6b538910c35193a82261ed3eb364bf Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 3 Nov 2026 15:21:42 +0000 Subject: [PATCH 21/25] WIP: container test for ROIStat served by HDF file or time series Assisted-by: Claude Sonnet 5.5 --- tests/container_tests/test_adsim_system.py | 152 +++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index 4e2fa0b8ec..317cfffa80 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -28,13 +28,17 @@ import os import sys +from collections import defaultdict from pathlib import Path from typing import Any from unittest.mock import ANY, patch +from urllib.parse import urlparse import bluesky.plan_stubs as bps import bluesky.plans as bp import bluesky.preprocessors as bpp +import h5py +import numpy as np import pytest from aioca import purge_channel_caches from bluesky.run_engine import RunEngine @@ -69,6 +73,9 @@ TIMEOUT = 60.0 # allow extra time for docker compose +# The ROIStat plugin is on its own prefix in example-services' bl01t-di-cam-01 IOC +ROISTAT_PREFIX = "BL01T-EA-TST-02:" + # Applies to every test here, not just the one that used to carry it: they all # need the IOC, and the IOC needs services that are not set up on Windows. pytestmark = pytest.mark.skipif( @@ -137,6 +144,36 @@ def adsimstat(RE: RunEngine) -> AreaDetector: return adsimstat +@pytest.fixture +def adsimroi(RE: RunEngine, shared_tmp_path: Path) -> AreaDetector: + """A SimDetector with an HDF writer and the ROIStat time series. + + The ROIStat plugin is in `plugins` so it is in the detector's tree, and is + given to both factories by reference: the writer stamps its NDAttributes into + the file as `ndattribute_sources`, and the time series reads its `TS*` arrays. + Which of them serves the statistics is decided by the HDF plugin's + EnableCallbacks at stage (`always_enable=False`). Note its prefix is not the + camera's: example-services puts the ROIStat plugin on BL01T-EA-TST-02. + """ + provider = StaticPathProvider(StaticFilenameProvider("adsimroi"), shared_tmp_path) + roistat = adcore.NDROIStatIO(f"{ROISTAT_PREFIX}ROISTAT:", num_channels=4) + with init_devices(): + adsimroi = SimDetector( + "BL01T-DI-CAM-01:", + adcore.ADDataFactory.hdf( + provider, + writer_suffix="HDF5:", + always_enable=False, + ndattribute_sources=[roistat], + ), + adcore.ADDataFactory.roistat_time_series(roistat), + driver_suffix="DET:", + plugins={"roistat": roistat}, + ) + + return adsimroi + + @pytest.fixture(autouse=True) def reset_adsim_to_baseline(RE: RunEngine, adsim: SimDetector) -> None: """Put the detector back to a known state before every test. @@ -438,6 +475,121 @@ def prepare_then_count() -> MsgGenerator[None]: assert event["timestamps"]["adsimstat-stats-total"] >= 0 +@pytest.mark.timeout(TIMEOUT + 30.0) +@pytest.mark.parametrize("hdf_enabled", [True, False]) +def test_roistat_is_served_by_the_hdf_file_or_the_time_series( + RE: RunEngine, adsimroi: SimDetector, bl01t_di_cam_01: None, hdf_enabled: bool +) -> None: + """The HDF plugin's EnableCallbacks picks who serves the ROIStat statistics. + + On, the HDF writer stamps each used channel's NDAttributes into the file and + the time series sits out: the documents are stream resources and datums, with + no event pages. Off, the time series serves them as event pages and there is + no file. Either way a key is `--` for each used + channel, and an unused channel publishes nothing. + + Relies on ADCore/db/NDROIStat.template and NDROIStatN.template (read, not run + here): `Name`, `Use`, `MinX`.. are per channel; `TSControl` Erase/Start sizes + and arms a fixed-length buffer of `TSNumPoints` that the plugin fills one + point per array it processes; `TSControl` Read posts the arrays so far. + """ + frames = 5 + roistat = adsimroi.get_plugin("roistat", adcore.NDROIStatIO) + hdf = adsimroi.get_plugin("hdf", adcore.NDFileHDF5IO) + # The switches at stage decide who serves, and the Name PVs name the keys, so + # all of it is settled before the stage that records it. Channel 3 is unused. + left, centre, unused = (roistat.channels[number] for number in (1, 2, 3)) + RE( + bps.mv( + roistat.enable_callbacks, + EnableDisable.ENABLE, + hdf.enable_callbacks, + EnableDisable.ENABLE if hdf_enabled else EnableDisable.DISABLE, + left.name_, + "Left", + left.use, + True, + left.min_x, + 0, + left.min_y, + 0, + left.size_x, + 512, + left.size_y, + 1024, + centre.name_, + "Centre", + centre.use, + True, + centre.min_x, + 256, + centre.min_y, + 256, + centre.size_x, + 512, + centre.size_y, + 512, + unused.use, + False, + roistat.channels[4].use, + False, + ) + ) + pixels = {"Left": 512 * 1024, "Centre": 512 * 512} + stats = ("min_value", "max_value", "mean_value", "total") + keys = [f"adsimroi-roistat-{name}-{stat}" for name in pixels for stat in stats] + + @bpp.stage_decorator([adsimroi]) + @bpp.run_decorator() + def fly() -> MsgGenerator[None]: + yield from bps.prepare( + adsimroi, TriggerInfo(number_of_events=frames, livetime=0.1), wait=True + ) + yield from bps.declare_stream(adsimroi, name="primary", collect=True) + yield from bps.kickoff(adsimroi, wait=True) + yield from bps.collect_while_completing( + flyers=[adsimroi], dets=[adsimroi], flush_period=0.5 + ) + + docs: dict[str, list[Any]] = defaultdict(list) + RE(fly(), lambda name, doc: docs[name].append(doc)) + + assert docs["stop"][0]["exit_status"] == "success" + (descriptor,) = docs["descriptor"] + if hdf_enabled: + # The file serves them: stream resources, in the file, one value a frame + assert docs["event_page"] == [] + resources = {doc["data_key"]: doc for doc in docs["stream_resource"]} + assert sorted(resources) == sorted(["adsimroi", *keys]) + frames_indexed = max(doc["indices"]["stop"] for doc in docs["stream_datum"]) + assert frames_indexed == frames + (path,) = {urlparse(doc["uri"]).path for doc in resources.values()} + with h5py.File(path) as hdf_file: + for key in keys: + dataset = hdf_file[resources[key]["parameters"]["dataset"]] + assert isinstance(dataset, h5py.Dataset) + values = np.asarray(dataset[:]) + assert values.shape == (frames,) + assert np.isfinite(values).all() + else: + # The time series serves them: pages of one reading per frame, no file + assert docs["stream_resource"] == [] + assert sorted(descriptor["data_keys"]) == sorted(keys) + assert docs["event"] == [] + pages = docs["event_page"] + assert sum(len(page["seq_num"]) for page in pages) == frames + series = { + key: np.concatenate([np.ravel(page["data"][key]) for page in pages]) + for key in keys + } + for name, size in pixels.items(): + prefix = f"adsimroi-roistat-{name}-" + smallest, largest, mean, total = (series[prefix + stat] for stat in stats) + assert (smallest <= mean).all() + assert (mean <= largest).all() + assert total == pytest.approx(mean * size, rel=1e-3) + + def run_plan_and_get_documents( RE: RunEngine, plan: MsgGenerator[Any] ) -> list[DocumentType]: From ea7b8bace7b610db5875a84e562bcca36158dcb7 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Tue, 3 Nov 2026 15:34:12 +0000 Subject: [PATCH 22/25] WIP: halve ADR 0024 and fold in the audit's stale-passage fixes Assisted-by: Claude Opus 5.5 --- .../decisions/0024-detector-is-a-flyable.md | 422 +++++++----------- 1 file changed, 155 insertions(+), 267 deletions(-) diff --git a/docs/explanations/decisions/0024-detector-is-a-flyable.md b/docs/explanations/decisions/0024-detector-is-a-flyable.md index 515ff3d516..e5c0019ebe 100644 --- a/docs/explanations/decisions/0024-detector-is-a-flyable.md +++ b/docs/explanations/decisions/0024-detector-is-a-flyable.md @@ -9,26 +9,21 @@ Accepted ## Context `StandardDetector` re-implemented `prepare`, `kickoff` and `complete` and threaded its own -state between them, while [](#StandardFlyable) already existed to do exactly that for every -other flyer in the tree (`Motor`, PMAC, the PandA trigger logics). Two implementations of -one lifecycle drift apart: the detector guarded its stages by checking whether a context -object was `None`, so calling a verb too early failed with `AttributeError: 'NoneType' -object has no attribute ...` rather than saying what was wrong. +state between them, while [](#StandardFlyable) already did that for every other flyer. The +two drifted apart: the detector guarded its stages by checking a context for `None`, so a +verb called too early failed with `AttributeError: 'NoneType' object has no attribute ...`. `StandardFlyable` requires a single [](#FlyableLogic) in a `logic` property -([](./0021-single-logic-property.md)), and a detector had three logic objects registered on -the Device by `add_detector_logics()`. +([](./0021-single-logic-property.md)), and a detector had three logic objects registered by +`add_detector_logics()`. ## Decision -### The three logics become children of one `DetectorLogic` +### One `DetectorLogic` holds the trigger, acquire and data logics -[](#DetectorLogic) holds the trigger, acquire and data logics, and is what `logic` returns. -The orchestration moves with them: preparing the trigger logic, making the data providers, -arming, and waiting for collections are all its methods now. It holds **no** back-reference -to its Device, so [](./0017-standard-movable.md) continues to hold unqualified. - -A detector builds it in `logic`: +[](#DetectorLogic) is what `logic` returns, and the orchestration (preparing the trigger +logic, making the data providers, arming, waiting) moves into it. It is a frozen dataclass +with no back-reference to its Device, so [](./0017-standard-movable.md) holds unqualified. ```python class MyDetector(StandardDetector): @@ -44,271 +39,164 @@ class MyDetector(StandardDetector): ) ``` -`add_detector_logics()` is removed rather than kept as a forwarder: a detector's logics are -fixed for its lifetime, so there is nothing a post-construction call can do that the -constructor cannot, and keeping it would leave two ways to wire the same Device. - -`StandardDetector` itself takes no `__init__` arguments, since a `Standard*` class that did -would interfere with multiple inheritance. The logic is supplied by overriding `logic`, -which `abstract_cached_property` makes a requirement checked when the Device is -instantiated. Assigning `self.logic` from `__init__` cannot serve instead: `ABCMeta` checks -`__abstractmethods__` in `__call__`, before `__init__` runs, so the class must override the -property to be concrete. For an ad-hoc detector in a plan or a test, -`DetectorLogic.with_device()` builds a Device around a logic, exactly as -`FlyableLogic.with_device()` does for any other flyer. - -### One thing the logic cannot get from its own fields - -Data logics name their datakeys after the Device, whose name does not exist until -`init_devices` calls `set_name` on context exit, so the logic cannot be given it when it is -made. `StandardFlyable` passes the Device's name to `prepare_to_fly` instead; the detector -keeps it on the context, for the data providers it starts after yielding, and -`get_hinted_fields()` takes it the same way. The logic holds no reference to the Device. - -### Which collect verb to expose is the Device's business - -A detector produces stream assets or event pages depending on what its data logics make for -a given scan, never both (see [](./0023-bounded-data-logic-tier.md)). The bluesky bundler -decides which to ask for from the *static* type — `collect_pages()` is called only on a -device that is not a `WritesStreamAssets` — so a detector that declared both would have its -pages silently dropped, and one that declared neither could not fly at all. - -So the hook `StandardDetector` registers for `prepare()` binds whichever verb applies as a -real instance attribute once the data is served, and removes the other. It is the Device that -does this, from what `DetectorLogic.produces` says the serving providers make; the logic is -not told where that goes, so it holds no back-reference. Only a `prepare()` that serves binds -or unbinds: nothing asks for a collect verb before a `prepare()`, and the step-boundary re-arm -on each point of a step scan cannot change which verb applies, because it changes no -provider. A failed `prepare()`, `kickoff()` or `complete()` leaves the verb bound until the -next `prepare()`, and calling it then raises. - -Choosing from the static type is a bundler limitation rather than a fact about detectors: a -device knows what it produced, and the bundler could decide from *that*, erroring only if -one device reported both ways in the same run. That change is a few lines in -`RunBundler.collect`, and with it both verbs become plain methods and the binding here goes -away. Until it is released, binding per prepare is what works against a released bluesky. - -### Mixing pageable and streamable is an error unless the pageable keys are suppressed - -[](./0023-bounded-data-logic-tier.md) made carrying both kinds of data logic an error, because -the bundler treats stream assets and event pages as mutually exclusive. Carrying both is still -useful, though: the same quantity can be written durably into a file *and* read from a plugin's -buffer — an areaDetector stats total, which the HDF writer pulls in as an NDAttribute. - -So the rule relaxes. A bounded logic is started after the unbounded ones and handed the -datakeys they make, which its provider leaves out. Where the stream assets make **every** -datakey a finite buffer would produce, the durable copy wins and the buffer sits the scan out, -unarmed. A registered signal is suppressed the same way by a provider key, and is -dropped from `read()` and `describe()` while the detector is prepared. Its hint stays, as the -provider key has its name. -A registered signal gives one value per event, so a provider key of its dtype and of its shape, -or of that shape with one extra leading dimension (`[collections_per_event, *shape]`), suppresses -it. Any other dtype or shape is a `ValueError` at `describe()`: two different things share a -name, so one must be renamed. A registered signal that nothing suppresses is read once per event. -Nothing has been started when that is decided, because a logic's `start_provider` returns -`None` without writing (same ADR). - -Suppression is one case of a general `StandardReadable` hook: a mapping from datakey name to -either `None` or the datakey that replaces it, matched by name after the registered children -have been read, so a signal is found wherever it is registered. `None` drops the name from -`read()`, `describe()` and `hints`: it is for a signal that is switched off, which has no -replacement. A datakey drops it from `read()` and `describe()` after checking the datakey can -replace it, and leaves it in `hints`, because the replacement has the same name. A data logic -that hints the name of a signal it replaces makes `hints` raise, as two hints of one field -always do. +- `add_detector_logics()` is removed, not kept as a forwarder: a detector's logics are fixed + for its lifetime, and two ways to wire one Device is one too many. +- `StandardDetector` takes no `__init__` arguments, as they would interfere with multiple + inheritance. `logic` is an `abstract_cached_property`; assigning `self.logic` in + `__init__` cannot replace it, because `ABCMeta` checks abstract methods before `__init__` + runs. `DetectorLogic.with_device()` builds an ad-hoc detector, like any flyer's. +- Datakeys are named after the Device, whose name only exists once `init_devices` calls + `set_name`, so `StandardFlyable` passes the name to `prepare_to_fly`. + +### The Device binds the collect verb at `prepare()` + +A detector makes stream assets or event pages, never both +([](./0023-bounded-data-logic-tier.md)). The bluesky bundler picks which to call from the +*static* type, so declaring both would silently drop the pages, and declaring neither could +not fly. So `StandardDetector.prepare()` overrides the flyer's and, when it succeeds, binds +the verb `DetectorLogic.produces` names as an instance attribute and removes the other. A +step scan restarts its bounded providers at each point, but they must make the same +datakeys, so it keeps the verb it was prepared with. + +This is a bundler limitation: a device knows what it produced, and deciding from that is a +few lines in `RunBundler.collect`. Once released, both verbs become plain methods and the +override goes. + +### Pageable and streamable data mix when the stream assets cover them + +[](./0023-bounded-data-logic-tier.md) made carrying both kinds of data logic an error, but +it is useful: an areaDetector stats total can go into the HDF file as an NDAttribute *and* +be read from the plugin's buffer. So: + +- A bounded logic is started after the unbounded ones and leaves out the datakeys they make. + If they make all of its datakeys, it sits the scan out, unarmed. +- A registered signal whose name a provider key has is dropped from `read()` and + `describe()` while prepared, and keeps its hint, since the replacement has its name. +- The provider key must have the signal's dtype and shape, optionally with a leading + `collections_per_event` dimension. Anything else is a `ValueError` at `describe()`: two + different things share a name, and one must be renamed. +- A registered signal that nothing suppresses is read once per event. + +This is a general `StandardReadable` hook, `_suppressed_datakeys_funcs`: datakey name to +`None` or a replacing `DataKey`, matched by name, so a signal is found wherever it is +registered. `None` also drops the hint: it is for a signal that is switched off. ### One name per statistic, and the plugin's switches decide which are used -Suppression only works if every tier calls a quantity the same thing. An areaDetector statistic -reaches the documents in up to three ways: the scalar read once per event, its stats time -series, and an NDAttribute in the HDF file. All three are named after the scalar signal -(`det-stats-total`), because that is the name a user already gets by registering it, and so -the highest tier serving a scan suppresses the others without any configuration. The time -series' `stat_signals` and `datakey_suffix`, and `setup_ndstats_sum`, which named the same -total `det-sum`, are removed. - -Which statistics are used is decided by the plugin's own PVs, not by arguments to the data -logics: a statistic is used if the plugin's `EnableCallbacks` and the compute switch of its -group (`ComputeStatistics`, `ComputeCentroid`) are on, and it has a read format on the plugin. -The PVs are what make the IOC produce the data, they are already captured by settings, and a -second set of switches in Python could only disagree with them. So the stats time series has -no `always_enable` and never writes these PVs, and a scalar whose statistic is switched off is -dropped from `read()`, `describe()` and `hints` rather than read as a stale value. The HDF -writer keeps `always_enable`, defaulting to `True`, for compatibility with `ADWriterFactory`: it switches its plugin on at `stage()`, before the recording below is taken, and -writes nothing to `EnableCallbacks` after that. - -`AreaDetector` records the plugins it uses once, at `stage()`, after the data logics have had -their say, because plugins are not usually staged themselves. Those are its `plugins`, the -writers of its factories, the NDAttribute sources of its writers and the plugins its factories -take by reference, and every plugin below them, so a stats plugin that is a child of an ROI -plugin is covered. For each it records the switches (`EnableCallbacks`, the compute switches, -the `Use` of each ROI stat channel), the `NDArrayPort`, and the `Name` of each ROI stat channel. -`describe()` and `read()` follow the recording, so they agree for the whole stage. Each -`prepare()` raises if a recorded PV has a different value now, naming the PVs, so a run cannot -be split by someone changing a PV part way through, and nothing is written to a PV the user -owns. The recording is never retaken: staging again records the new values, and `unstage()` -forgets them. - -The exception is a switch the detector's own logic writes. A trigger logic switches on the -process plugin it filters with, and the continuous acquisition one its circular buffer, at every -`prepare()`, so recording them would report the detector's own write as a change by someone -else. The plugins a trigger logic holds are therefore left out of the recording. Ownership is by -logic, not by plugin type: a process plugin that only appears in `plugins` is the user's, and is -recorded like any other. - -Only a plugin's own `NDArrayPort` is recorded, so a rewired port is caught, but the chain of -plugins upstream of it is not walked to see whether it still reaches the driver. A chain can -cross IOCs over PVA, so its reachability is not something one process can establish, and the -chain as wired at `stage()` is the user's responsibility. - -A statistic that is switched off is such a `None`. Neither the stats time series nor the HDF writer hints -a statistic, so the plugin registered as a readable is what hints it, whichever tier serves it. -`AreaDetector` registers every plugin in its `plugins` as a readable, so that is always the case -for a plugin the user gave it, and a user who does not want a plugin read opts it out with -`set_readable_format(plugin, None)`. Data factories take such a plugin by reference, and only -the factories of file writers make a device: `ADDataFactory.stats_time_series(stats)` raises -in `AreaDetector.__init__` if `stats` is not in the detector's device tree, like an NDAttribute -source. - -The HDF writer stays free of stats knowledge. `ADDataFactory.hdf(ndattribute_sources=)` lists -the plugins, as objects, whose NDAttributes it stamps: a plugin that implements the -`NDAttributeSource` protocol returns its NDAttributes, which the writer writes to the plugin's -`nd_attributes_file` and turns into datasets directly, each named after the scalar signal it -publishes. Nothing is stamped that is not listed, so two HDF writers stamp the statistics of -their own ROIs and no others. `AreaDetector` raises if a source is listed by two writers or is -not in its device tree. Any other plugin's, and the driver's, NDAttributes are still parsed from -the XML the user set, and keep the names in it, as they may be about something outside the -detector. - -An ROI stat plugin is a source too. Each channel whose `Use` is on and which has a read format -on a scalar stamps that scalar, named after it, from the channel's asyn address. Its switches -are `EnableCallbacks` and the `Use` of every channel, and a channel that is not used has its -scalars dropped like a switched-off statistic. The channels are named when the detector is -staged, as `-`, with characters other than letters, digits and underscores -replaced by underscores and the channel number used if the PV is empty, so the scalars have -names the documents accept and say what the channel measures. Two channels that would be named -the same raise, and a `Name` PV that changes after that raises at `prepare()` like a switch. -The plugin is a readable whose channels are its readable children, so a user does not register -each channel. Its time series is published by `ADDataFactory.roistat_time_series(roistat)` -like the stats time series, one array per used channel per scalar with a read format, named -after that scalar, so it suppresses the scalar. It differs only where the plugin does: a -`TSControl` record erases and starts the buffer rather than a busy record, the buffer has no -circular mode, so it is always fixed length, and every channel has its own timestamp array. -All used channels are stamped from the same frame, so the first used channel's timestamps -serve them all. Like `NDStatsIO`, `NDROIStatNIO` has a `net` scalar with no read format, so -its `TSNet` array is published only when a user gives `net` one. - -Both plugins count every frame in `TSCurrentPoint` but post their arrays only when the buffer -is full, on Stop or Read, or from a record that asks for a Read about once a second while -acquiring. A page collected between those posts would read arrays shorter than the count, so -each logic gives its `EventPageDataProvider` a refresh that asks for a Read before the arrays -are read: `TSRead` of the stats plugin's embedded time series, or Read on the ROI stat -plugin's `TSControl`. In both plugins a Read only posts the arrays up to the current point, -and does not stop or erase an acquiring buffer. The refresh is a callable on the provider -rather than a step of the data logic, because the provider is what decides when a page is -read. A Read put can complete before the plugin's I/O Intr waveform records have processed, -so the refresh alone does not guarantee that the arrays are up to date. The provider -therefore observes each array and the timestamps until each holds as many collections as the -page needs, for at most `ARRAY_WAIT_TIMEOUT` (longer than the one second between the IOC's -posts), and raises a `TimeoutError` naming the signal if one does not. It slices the values -it observed, never a later get, so a page never has an array shorter than the count it -indexes by. The arrays are waveforms of `NCHANS` elements in both plugins, and a post is cut -to that length, so a scan longer than that could never be served. Each logic reads the -timestamp waveform's `NELM` in `start_provider` and raises at `prepare()` if it is smaller -than the scan. The `TSNumPoints` record is not a reliable check: only the ROI stat plugin's -clamps it to `NCHANS`, the stats plugin's `DRVH` is far larger. +Suppression only works if every tier names a quantity the same. An areaDetector statistic +can be a scalar read per event, a stats time series, or an HDF NDAttribute, and all three +are named after the scalar signal (`det-stats-total`), the name a user already gets by +registering it, so the highest tier serving a scan suppresses the others unconfigured. The +time series' `stat_signals` and `datakey_suffix`, and `setup_ndstats_sum`, go. + +Which statistics are used is decided by the plugin's PVs: they make the IOC produce the +data and are captured by settings, and switches in Python could only disagree with them. + +- A statistic is used if `EnableCallbacks` and its group's compute switch are on and it has + a read format. A switched-off one is suppressed with `None`, not read as a stale value. +- The time series never write these PVs. The HDF writer keeps `always_enable` (default + `True`) for `ADWriterFactory` compatibility, enabling its plugin at `stage()`. +- At `stage()` `AreaDetector` records the switches, `NDArrayPort` and ROI channel `Name` of + every plugin it uses, down the tree, as plugins are not staged themselves. `describe()` + and `read()` follow the recording, and `prepare()` raises naming any recorded PV that has + changed, so a run is never split part way and no PV the user owns is written. +- Plugins a trigger logic holds are not recorded: it switches them on at every `prepare()`. + Ownership is by logic, not type, so a process plugin only in `plugins` is recorded. +- The chain upstream of each `NDArrayPort` is not walked: it can cross IOCs over PVA, so one + process cannot establish it, and the chain wired at `stage()` is the user's business. + +Only the plugin registered as a readable hints a statistic, whichever tier serves it. +`AreaDetector` registers every plugin in `plugins` as a readable; +`set_readable_format(plugin, None)` opts one out. Data factories take plugins by reference +and raise in `AreaDetector.__init__` if one is not in the detector's tree. + +The HDF writer stays free of stats knowledge: `ADDataFactory.hdf(ndattribute_sources=)` +lists `NDAttributeSource` plugins, each of which returns its NDAttributes, named after the +scalars they publish. Only listed plugins are stamped, so two writers stamp only their own +ROIs. NDAttributes from the user's XML keep their names, as they may be about something +outside the detector. + +An ROI stat plugin is a source and a readable whose used channels are its children. Each +channel is named at `stage()` as `-`, sanitised, or by number if +`Name` is empty, so the names say what it measures; a clash raises, and a changed `Name` +raises at `prepare()` like a switch. `ADDataFactory.roistat_time_series(roistat)` +publishes one array per used channel per scalar, like the stats time series. + +### Time series pages are read only once the arrays hold them + +Both plugins post their arrays only when full, on Stop or Read, or about once a second, so a +page read between posts would see arrays shorter than its count: + +- The provider is given a refresh that requests a Read before each page, posting the arrays + so far without stopping the buffer. It lives on the provider, which decides when a page is + read. +- A Read can complete before the waveforms process, so the provider `observe_value`s each + array until it holds the page, for at most `ARRAY_WAIT_TIMEOUT`, and slices what it + observed. +- The arrays hold `NCHANS` elements, so `start_provider` checks the timestamp waveform's + `NELM` and raises at `prepare()` for a longer scan. `TSNumPoints` cannot serve: only the + ROI stat plugin clamps it to `NCHANS`. ### The detector flies with one `fly()` generator -`FlyableLogic.fly` is a single async generator ([](./0022-fly-generator-and-ensure-stopped.md)): -it yields `KICKED_OFF` once flying, then `WatcherUpdate`s, and `complete()` finishes when it -returns. A detector's progress is "collections written out of collections requested", so -`DetectorLogic.fly` notes where the data providers have got to, arms an internally triggered -acquisition, yields `KICKED_OFF`, then yields an update each time the collections written move -on, and ends once the acquisition has returned to idle. `StandardFlyable` runs it in the task -of the status `complete()` returns and cancels it on `unstage()`, so the detector needs no stop -logic beyond its `ensure_stopped`, which `unstage()` always calls, like any flyer. +`FlyableLogic.fly` is one async generator ([](./0022-fly-generator-and-ensure-stopped.md)). +A detector's progress is collections written out of collections requested, so +`prepare_to_fly` notes what the unbounded providers have written +(`ctx.unbounded_start_collections`), and `fly` arms an internally triggered acquisition, +yields `KICKED_OFF`, yields an update as the count moves on, and returns once idle. +`unstage()` cancels it, so no stop logic is needed beyond `ensure_stopped`. + +A fly scan collects only what the data logics make: registered readables are read only by +`trigger()`. ### An acquisition is a generator too -`DetectorAcquireLogic` started acquiring and waited for idle in two hooks, `start_acquiring` and -`wait_for_idle`, so whatever the first started -- the put-callback status of `Acquire`, the sim -detector's writing task -- had to be held on the logic between them. Cancelling the fly between -the two left it running until `ensure_stopped`, against the rule that a hook never leaves -anything running that it does not wait for. - -So they become one async generator, `acquire()`, in the same idiom as `fly`: start, yield once -armed, wait for idle. It owns what it starts (`async with` a status, or cancelling a task in a -`finally`), so closing it at its `yield` ends it. It takes no trigger information: the same -generator serves both kinds of triggering, and only where it is entered differs. An externally -triggered detector arms in `prepare()`, so that it is waiting on its trigger source by the time -`kickoff()` returns; the armed acquisition is held on the context until `fly` (or -`trigger()`) takes it and runs it to the end, closing it if cancelled. An internally -triggered one is entered by `fly` or `trigger()` itself. One still armed at `unstage()` is -closed then, so its cleanup runs then, not when it is garbage collected after a restage; -stopping the detector stays the job of `ensure_stopped`. -A prepared detector cannot be prepared again before it is kicked off or triggered, so no -`prepare()` replaces an armed acquisition. +`DetectorAcquireLogic.start_acquiring` and `wait_for_idle` had to hold what the first +started on the logic, and cancelling between them left it running. They become one async +generator, `acquire()`: start, yield once armed, wait for idle. It owns what it starts, so +closing it at its `yield` ends it, and it needs no trigger information: + +- An externally triggered detector arms in `prepare()`, so it is waiting on its trigger + source when `kickoff()` returns, and the context holds it until `fly` or `trigger()` + takes it. It cannot be prepared again before then, so nothing replaces it. +- An internally triggered one is entered by `fly` or `trigger()` itself. +- One still armed at `unstage()` is closed then, not at garbage collection; stopping the + detector stays the job of `ensure_stopped`. ### One kickoff per prepare -The detector used to be prepared for N events and kicked off a few at a time, with an -`events_to_kickoff` signal saying how many the next kickoff covered and an `is_last_kickoff` -flag deciding when to wait for idle. Nothing in tree drove it: the RunEngine kicks off once -and completes what it kicked off, and every other flyer already worked that way. - -Both go, and `kickoff()` requests the whole prepared scan. A second `kickoff()` without an -intervening `prepare()` now raises, from `StandardFlyable`, like any other flyer. A detector -with no data logic flies too, with nothing to collect: it is a valid flyer for its triggering -alone. - -### Unbounded providers are the stage's, bounded providers the prepare's - -`_FlyCtx` holds the `TriggerInfo` that `prepare()` resolved, the serving bounded providers, -the datakeys and suppressed signals they and the unbounded ones make, the progress baselines and -a count of the `trigger()`s since, and of the events published since the scan started, one -count for all providers. The serving unbounded providers and an armed acquisition -live on the logic for the stage instead, because they outlive a single prepare: a step scan -reuses an open file across its points. A provider carried to the next `prepare()` publishes -from that prepare's baseline. The data verbs — `read`, `describe`, `describe_collect`, -`collect_asset_docs`, `collect_pages`, `get_index` — call the logic with the context, which -`StandardFlyable` keeps after `complete()`, so `collect_while_completing` collects after -`complete()` has finished. - -`trigger()` therefore needs no context of its own, beyond a count of triggers on the fly -context. It is part of the state machine all the same: it runs in a `TRIGGERING` state entered -from `PREPARED` and ends in `TRIGGERED`, so `unstage()` cancels it, `kickoff()` cannot overlap -it, and a `kickoff()` after it needs a new `prepare()`. A failed `trigger()` leaves the detector -staged, as a failed `kickoff()` or `complete()` does. It prepares implicitly if nothing has been, then from the second point on calls each -serving bounded logic: it is stopped, asked for a new provider and started, so a finite -buffer holding one event at a time is re-armed, while an unbounded source carries on. The count is what keeps the first point from -being re-armed on top of the `prepare()` that just armed it. The trigger logic is not re-prepared per point, so -a step scan does not repeat its puts. - -### One context type, from `prepare_to_fly` to `fly` - -`FlyableLogic` threads a single `CtxT` from `prepare_to_fly` to `fly`. What only exists once -flying lives in `fly` itself, since one generator spans `kickoff()` and `complete()`, so no -field of the context waits for kickoff as `| None`. The context may be mutable: the detector -counts its `trigger()`s and notes its baselines on it. +The detector was prepared for N events and kicked off a few at a time (`events_to_kickoff`, +`is_last_kickoff`). Nothing drove that: the RunEngine kicks off once, as for every other +flyer. Both go; `kickoff()` requests the whole scan, and a second needs a new `prepare()`. A +detector with no data logic is still a valid flyer for its triggering. + +### Unbounded providers carry over between prepares, bounded ones are restarted + +One `DetectorCtx` threads from `prepare_to_fly` to `fly`; what exists only once flying lives +in `fly`, so no field waits for kickoff as `| None`. The logic holds no state, and the next +`prepare()` gets the last context as `previous_ctx`: + +- Unbounded providers carry over if made for the same name, collections per event, period + and flush period, because a step scan reuses an open file across its points. +- Bounded providers are restarted at every `prepare()` and every step point after the + first, so a buffer holding one event is re-armed. The trigger logic is not re-prepared + per point. +- `StandardFlyable` keeps the context after `complete()`, so `collect_while_completing` + collects after `complete()` has finished. + +`trigger()` runs in a `TRIGGERING` state, so `unstage()` cancels it, `kickoff()` cannot +overlap it, and a failed one leaves the detector staged. It prepares implicitly if nothing +has been. ## Consequences -- `StandardDetector` subclasses build a `DetectorLogic` in `logic`; - `det.add_detector_logics(*logics)` is gone. -- `kickoff()` → `kickoff()` without an intervening `complete()` is rejected, and so is a - second `kickoff()` after one. The old detector allowed both, because it only checked that - a prepare context existed. -- `events_to_kickoff` is removed from the public API. -- Calling a verb before `prepare()` says `: prepare() must be called first` instead of - `Prepare not run`. Before `stage()`, and after `unstage()`, every verb but `stage()` raises, - for every `StandardFlyable`. -- A detector satisfies neither `WritesStreamAssets` nor `EventPageCollectable` until it has - been prepared. Everything in the RunEngine that dispatches on them runs after prepare. -- `complete`, `kickoff`, `prepare`, `stage` and `unstage` are no longer implemented on - `StandardDetector`, nor are the private steps of `StandardFlyable` and `StandardReadable`, so - a change to the flyer lifecycle reaches detectors too. It adds to them only through the hooks - they run: one to serve the data and bind the collect verbs when `prepare_to_fly` yields the - context, and one giving the datakeys that replace registered signals. -- `trigger()` is cancelled by `unstage()`, and `kickoff()` raises while one runs. -- `DetectorAcquireLogic.start_acquiring()` and `wait_for_idle()` are replaced by `acquire()`. +- A verb called out of order raises a `TransitionError` naming it and the state + (`: describe not allowed from STAGED, allowed from: ...`), not `Prepare not run`, + and a second `kickoff()` without a `prepare()` is rejected. +- A detector satisfies neither `WritesStreamAssets` nor `EventPageCollectable` until + prepared; the RunEngine only dispatches on them after prepare. +- `StandardDetector` no longer implements `complete`, `kickoff`, `stage` or `unstage`, so + lifecycle changes reach detectors too. It adds `__on_*` hooks for `read()`, `describe()`, + `hints`, suppression and `unstage()`, and overrides `prepare()` to bind the collect verb. +- `add_detector_logics()`, `events_to_kickoff`, `start_acquiring()` and `wait_for_idle()` + are gone. From e4b1f086053782d78f08768c5d3d744d8bb5ef0f Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 4 Nov 2026 09:27:45 +0000 Subject: [PATCH 23/25] WIP: fix three CI failures on the ROIStat and stats time series - Enable the NDPluginTimeSeries that NDPluginStats embeds when arming the stats time series; the IOC starts it disabled, so TSCurrentPoint never moved. - Leave an empty description out of the NDAttributes XML: eight ROIStat attributes were 1093 characters, over the plugin's 1024 element waveform (CAException 176, invalid element count). - Cancel and close the sibling array waits when one times out in make_page, which leaked a gather_dict task. Assisted-by: Claude Sonnet 5.5 --- src/ophyd_async/core/_data_providers.py | 32 +++++++++++----- src/ophyd_async/epics/adcore/__init__.py | 2 + src/ophyd_async/epics/adcore/_io.py | 14 ++++++- src/ophyd_async/epics/adcore/_ndattribute.py | 37 +++++++++---------- .../epics/adcore/_stats_time_series.py | 12 ++++-- .../epics/adcore/test_stats_time_series.py | 2 + .../plan_stubs/test_ndattributes.py | 8 ++++ 7 files changed, 74 insertions(+), 33 deletions(-) diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index 9301a7cbab..b17181a4e3 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -10,7 +10,7 @@ from ._datatypes import Array1D from ._signal import SignalR, SignalW, observe_value -from ._utils import ConfinedModel, error_if_none, gather_dict, get_dtype +from ._utils import ConfinedModel, error_if_none, get_dtype ARRAY_WAIT_TIMEOUT = 2.0 """How long `EventPageDataProvider` waits for an array to hold the collections @@ -36,6 +36,22 @@ async def _wait_for_length(signal: SignalR[Array1D[Any]], needed: int) -> Array1 return error_if_none(kept, f"{signal.source} was not observed") +async def _gather_or_cancel_all(waits: Sequence[Awaitable[Any]]) -> list[Any]: + """Await `waits` in parallel. + + If one raises, or this is cancelled, the rest are cancelled and closed before + the error propagates, so no wait outlives the call. + """ + tasks = [asyncio.ensure_future(wait) for wait in waits] + try: + values = await asyncio.gather(*tasks) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + return values + + class DataProvider(ABC): """What a [](#DetectorDataLogic) returns to publish the data it is writing. @@ -156,15 +172,13 @@ async def make_page(self, index: int) -> PartialEventPage | None: needed = self._origin + index * cpe # Wait for every array and the per-collection timestamps to hold the # collections counted, all at once - arrays, stamps = await asyncio.gather( - gather_dict( - { - datakey: _wait_for_length(signal, needed) - for datakey, signal in self.data.items() - } - ), - _wait_for_length(self.timestamps, needed), + *arrays_held, stamps = await _gather_or_cancel_all( + [ + *(_wait_for_length(signal, needed) for signal in self.data.values()), + _wait_for_length(self.timestamps, needed), + ] ) + arrays = dict(zip(self.data, arrays_held, strict=True)) starts = [self._origin + event * cpe for event in range(self._published, index)] self._published = index # One timestamp per event: the acquisition time of its last collection diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index dc469be02a..754903524b 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -42,6 +42,7 @@ NDROIStatTSControl, NDStatistic, NDStatsIO, + NDStatsTimeSeriesIO, NDStatsTSAcquireMode, ) from ._ndattribute import ( @@ -74,6 +75,7 @@ "NDROIIO", "NDStatistic", "NDStatsIO", + "NDStatsTimeSeriesIO", "NDStatsTSAcquireMode", "NDROIStatNIO", "NDROIStatIO", diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index d6fd4271c0..4106bac35d 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -264,6 +264,15 @@ class NDROIIO(StandardReadable, NDPluginBaseIO): bin_z: A[SignalRW[int], PvSuffix.rbv("BinZ")] +class NDStatsTimeSeriesIO(EpicsDevice): + """The NDPluginTimeSeries that NDPluginStats embeds, at the `TS:` sub-prefix. + + Its `enable_callbacks` is separate from the stats plugin's own. + """ + + enable_callbacks: A[SignalRW[EnableDisable], PvSuffix.rbv("EnableCallbacks")] + + class NDStatsTSAcquireMode(StrictEnum): """Whether an NDPluginStats time series stops when full or wraps around. @@ -334,9 +343,12 @@ class NDStatsIO(StandardReadable, NDPluginBaseIO): # starts it (writing 1 clears the arrays and resets ts_current_point to 0), # ts_current_point reports progress, writing 1 to ts_read posts the arrays # while acquiring, each ts_ holds the series of that statistic and - # ts_timestamp holds the per-point acquisition times. + # ts_timestamp holds the per-point acquisition times. The embedded plugin + # takes no points while ts_plugin.enable_callbacks is Disable, which is the + # IOC's default. # These are not registered as readables: the scalars serve read(), and the # arrays are consumed by StatsTimeSeriesDataLogic as event pages. + ts_plugin: A[NDStatsTimeSeriesIO, PvSuffix("TS:")] ts_acquire: A[SignalRW[bool], PvSuffix("TS:TSAcquire"), EpicsOptions(wait=non_zero)] ts_read: A[SignalW[bool], PvSuffix("TS:TSRead")] ts_num_points: A[SignalRW[int], PvSuffix("TS:TSNumPoints")] diff --git a/src/ophyd_async/epics/adcore/_ndattribute.py b/src/ophyd_async/epics/adcore/_ndattribute.py index 2e65b0a455..b38380d3be 100644 --- a/src/ophyd_async/epics/adcore/_ndattribute.py +++ b/src/ophyd_async/epics/adcore/_ndattribute.py @@ -68,31 +68,30 @@ def ndattributes_to_xml( root = ET.Element("Attributes") for ndattribute in ndattributes: if isinstance(ndattribute, NDAttributeParam): - ET.SubElement( - root, - "Attribute", - name=ndattribute.name, - type="PARAM", - source=ndattribute.param, - addr=str(ndattribute.addr), - datatype=ndattribute.datatype.name, - description=ndattribute.description, - ) + fields = { + "name": ndattribute.name, + "type": "PARAM", + "source": ndattribute.param, + "addr": str(ndattribute.addr), + "datatype": ndattribute.datatype.name, + } elif isinstance(ndattribute, NDAttributePv): - ET.SubElement( - root, - "Attribute", - name=ndattribute.name, - type="EPICS_PV", + fields = { + "name": ndattribute.name, + "type": "EPICS_PV", # Strip protocol prefix if present - source=ndattribute.signal.source.split("://")[-1], - dbrtype=ndattribute.dbrtype.name, - description=ndattribute.description, - ) + "source": ndattribute.signal.source.split("://")[-1], + "dbrtype": ndattribute.dbrtype.name, + } else: raise ValueError( f"Invalid type for ndattributes: {type(ndattribute)}. " "Expected NDAttributePv or NDAttributeParam." ) + # An empty description is left out: the plugin's XML waveform is only + # 256 to 1024 characters, and each one costs 15 of them + if ndattribute.description: + fields["description"] = ndattribute.description + ET.SubElement(root, "Attribute", fields) xml_text = ET.tostring(root, encoding="unicode") return xml_text diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py index 110b041f8a..fe6fec19c3 100644 --- a/src/ophyd_async/epics/adcore/_stats_time_series.py +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -9,6 +9,7 @@ DEFAULT_TIMEOUT, Array1D, BoundedDataLogic, + EnableDisable, EventPageDataProvider, SignalR, set_and_wait_for_value, @@ -50,10 +51,11 @@ class StatsTimeSeriesDataLogic(BoundedDataLogic[EventPageDataProvider]): which erases the arrays and resets the current point before the detector's acquire logic drives the camera; its frames then feed the time series via NDArray callbacks. `start_provider` raises `ValueError` if the plugin's - arrays hold fewer points than the scan needs. Before each page it writes 1 to - ``ts_read``, so the plugin posts its arrays up to the current point. A detector - that writes a file should pull stats into the file as NDAttributes instead - (see `ADHDFDataLogic`). + arrays hold fewer points than the scan needs. It also enables the embedded + time series plugin, which the IOC starts disabled. Before each page it writes + 1 to ``ts_read``, so the plugin posts its arrays up to the current point. A + detector that writes a file should pull stats into the file as NDAttributes + instead (see `ADHDFDataLogic`). :param stats: the stats plugin whose time series to read """ @@ -88,7 +90,9 @@ async def start_provider( return None # Size the buffer and put it in fixed-length mode before arming it await _check_capacity(self.stats.ts_timestamp_nelm, num_collections) + # The embedded time series plugin ignores the stats until it is enabled await asyncio.gather( + self.stats.ts_plugin.enable_callbacks.set(EnableDisable.ENABLE), self.stats.ts_num_points.set(num_collections), self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), ) diff --git a/tests/unit_tests/epics/adcore/test_stats_time_series.py b/tests/unit_tests/epics/adcore/test_stats_time_series.py index bf8c2d7e40..13ae001e0d 100644 --- a/tests/unit_tests/epics/adcore/test_stats_time_series.py +++ b/tests/unit_tests/epics/adcore/test_stats_time_series.py @@ -66,6 +66,8 @@ async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH # ts_acquire=1 arms and clears the buffer assert await stats.ts_acquire.get_value() is True + # The embedded time series plugin takes no points while it is disabled + assert await stats.ts_plugin.enable_callbacks.get_value() == EnableDisable.ENABLE # Each series is named after its scalar datakeys = await provider.make_datakeys() assert list(datakeys) == PUBLISHED diff --git a/tests/unit_tests/plan_stubs/test_ndattributes.py b/tests/unit_tests/plan_stubs/test_ndattributes.py index 09d3a69479..329ee55f2a 100644 --- a/tests/unit_tests/plan_stubs/test_ndattributes.py +++ b/tests/unit_tests/plan_stubs/test_ndattributes.py @@ -55,3 +55,11 @@ async def test_nd_attributes_plan_stub_gives_correct_error(RE, arg): + "Expected NDAttributePv or NDAttributeParam.", ): RE(adcore.setup_ndattributes(stat, [arg])) + + +def test_empty_description_is_left_out_of_the_xml(): + param = adcore.NDAttributeParam( + name="det-sum", param="sum", datatype=adcore.NDAttributeDataType.DOUBLE + ) + xml = ET.fromstring(adcore.ndattributes_to_xml([param])) + assert "description" not in xml[0].attrib From 32ef4236274e9eae0ed83f873d318f5f2bb0eff6 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 4 Nov 2026 09:32:37 +0000 Subject: [PATCH 24/25] WIP: HDF takes the ROIStat container test's frames from the ROIStat port Assisted-by: Claude Sonnet 5.5 --- tests/container_tests/test_adsim_system.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index 317cfffa80..675345caad 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -505,6 +505,11 @@ def test_roistat_is_served_by_the_hdf_file_or_the_time_series( EnableDisable.ENABLE, hdf.enable_callbacks, EnableDisable.ENABLE if hdf_enabled else EnableDisable.DISABLE, + # The ROIStat plugin stamps its NDAttributes on the arrays it passes + # on, so the file only receives them downstream of it. The baseline + # puts the port back before the next test. + hdf.nd_array_port, + "DET.ROISTAT", left.name_, "Left", left.use, From 0e5d9d5698607b8741823face9843569dba77f3f Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 4 Nov 2026 13:05:51 +0000 Subject: [PATCH 25/25] WIP: gate the set in test_status_of_set_and_wait_for_value on an Event, not a 1 ms sleep Assisted-by: Claude Sonnet 5.5 --- tests/unit_tests/core/test_signal.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/core/test_signal.py b/tests/unit_tests/core/test_signal.py index 7792315f3c..fef1b82ee9 100644 --- a/tests/unit_tests/core/test_signal.py +++ b/tests/unit_tests/core/test_signal.py @@ -321,10 +321,15 @@ async def test_status_of_set_and_wait_for_value(): set_signal = epics_signal_rw(int, "pva://signal") match_signal = epics_signal_rw(int, "pva://match_signal") + # Open by default; closed around one call to hold its set running after the + # match, so "status not done yet" does not depend on loop timing + set_may_finish = asyncio.Event() + set_may_finish.set() + async def set_match_signal_after_delay(value: Any, **kwargs): await asyncio.sleep(0.2) await match_signal.set(value / 2) - await asyncio.sleep(0.001) + await set_may_finish.wait() await set_signal.connect(mock=True) await match_signal.connect(mock=True) @@ -345,10 +350,12 @@ async def set_match_signal_after_delay(value: Any, **kwargs): status = await set_and_wait_for_other_value(set_signal, 32, match_signal, 16) assert status.done + set_may_finish.clear() status = await set_and_wait_for_other_value( set_signal, 30, match_signal, 15, wait_for_set_completion=False ) assert not status.done + set_may_finish.set() await status with pytest.raises(asyncio.TimeoutError):