diff --git a/docs/conf.py b/docs/conf.py index 9b3c96a9ed..5a469aa279 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -156,6 +156,7 @@ def setup(app: application.Sphinx): "ophyd_async.core._standard_base._StandardBase", "ophyd_async.core._flyable.PrepareT", "ophyd_async.core._flyable.CtxT", + "ophyd_async.core._detector.DataCtxT", "ophyd_async.core._utils.P", "ophyd_async.core._utils.P.args", "ophyd_async.core._utils.P.kwargs", @@ -190,6 +191,16 @@ def setup(app: application.Sphinx): "0.001", "1.0", "bluesky.protocols.T_co", + # TypeVars documented on StandardFlyable, added in #1359 + "PrepareT", + "CtxT", + # event_model publishes no intersphinx entry for this TypedDict + "event_model.documents.PartialEventPage", + # TypeVar for the plugin an ADWriterFactory builds a data logic around + # The context DetectorLogic threads prepare -> kickoff -> complete, private + # because nothing outside StandardDetector constructs or inspects one + "ophyd_async.core._detector._FlyCtx", + "ophyd_async.core._detector._DetectorData", ] nitpick_ignore = [] for var in obj_ignore: diff --git a/docs/explanations/decisions/0012-detector-rewrite.md b/docs/explanations/decisions/0012-detector-rewrite.md index 3e1145b66b..ee4448e5bc 100644 --- a/docs/explanations/decisions/0012-detector-rewrite.md +++ b/docs/explanations/decisions/0012-detector-rewrite.md @@ -5,6 +5,9 @@ Date: 2026-01-20 Accepted +Partly superseded by [](./0020-runtime-readable-format.md), which made `StandardDetector` a +`StandardReadable` and removed the `prepare_single` tier along with `ReadableDataProvider`. + ## Context The original `StandardDetector` implementation had several architectural issues: diff --git a/docs/explanations/decisions/0017-standard-movable.md b/docs/explanations/decisions/0017-standard-movable.md index 3d70c39da9..0b348e76ba 100644 --- a/docs/explanations/decisions/0017-standard-movable.md +++ b/docs/explanations/decisions/0017-standard-movable.md @@ -6,6 +6,9 @@ Date: 2026-03-10 Accepted +Partly superseded by [](./0021-single-logic-property.md), which replaced `movable_logic` +with a single `logic` property. + ## Context Several device types share a common movement pattern: write a setpoint, then observe a diff --git a/docs/explanations/decisions/0020-runtime-readable-format.md b/docs/explanations/decisions/0020-runtime-readable-format.md new file mode 100644 index 0000000000..eff77143cb --- /dev/null +++ b/docs/explanations/decisions/0020-runtime-readable-format.md @@ -0,0 +1,275 @@ +# 20. Make readable format a runtime setting keyed by Device + +Date: 2026-08-18 + +## Status + +Accepted + +## Context + +In ophyd v1 a `Component`'s `kind` attribute could be changed at runtime, so a beamline +could reuse one Device across techniques: a monochromator's energy signal is +configuration for a fixed-energy technique and a hinted read for one that scans it. + +`StandardReadable` had no equivalent. `add_readables(devices, format)` appended bound +methods into four tuples: + +```python +case StandardReadableFormat.CONFIG_SIGNAL: + self._describe_config_funcs += (signal.describe,) + self._read_config_funcs += (signal.read,) +``` + +Once appended, the tuples no longer recorded which device a contribution came from, so +nothing could be found, removed or re-formatted afterwards. The only ways to change what +a device reported were to edit its class definition or to keep two near-identical Device +classes differing in one signal's format. This is issue #1394. + +The same gap showed up from the other direction in #1395. `StandardDetector` did not +inherit `StandardReadable` and had its own `add_config_signals` instead, so detectors and +every other Device disagreed about how a signal is nominated for `read_configuration()`. +Reviewers of that PR were unwilling to have areaDetector plugins registered automatically, +because a detector typically carries far more plugins (16 ROIs, 16 stats) than are wired +into its chain, and reporting all of them gives no way to tell which values are +meaningful. Per-plugin opt-in needs a per-device call that can be made after construction +— which is the same primitive #1394 asks for. + +Three designs were considered. + +**Chain parsing.** Inspect the areaDetector plugin graph and register what is actually +wired up. Rejected as too large, and it breaks down once gather/scatter and PVA transfer +plugins are used between IOCs, since it requires every plugin to be exposed to ophyd. + +**A single pluggable registry** holding both formats and logic objects, so that +`StandardReadable`, `StandardMovable`, `StandardFlyable` and `StandardDetector` all +register their behaviour the same way. Rejected: a format entry is keyed by child, +unbounded in number and merged by union, whereas a logic object is keyed by role, at most +one per role, and conflicts are an error. A single registry would have to branch on entry +kind at every insert. Runtime mutation also means different things for the two — changing +a format alters the contents of `describe()`, while changing a data logic alters which +bluesky protocols the Device satisfies. Finally the declarative annotation form only fits +formats: `A[SignalRW[float], PvSuffix.rbv("AcquireTime"), Format.CONFIG_SIGNAL]` works +because a format needs exactly `(parent, child)`, whereas `ADHDFDataLogic` needs a prefix, +a driver, a plugin list and a path provider. + +**Key the format registry by device**, and leave logic objects alone. Chosen. + +## Decision + +### The registry is a `{device: format}` dict + +`StandardReadable._readables` is a `MutableMapping[Device, StandardReadableFormat]` that +maps each registered child to its format, in registration order, and the callables for +each verb are derived on demand. Its `__setitem__` validates, as `Settings` does for its +keys: a signal format is only accepted for a `SignalR`, so the verbs can match on +`(signal, format)` entries already typed. `add_readables` becomes a thin wrapper over a +new `set_readable_format(device, format)`, and `format=None` removes the device. +`get_readable_formats()` returns a copy. + +Keying by `Device` is identity keying, which is what a registry of children needs: `Device` +does not override `__eq__`, and while `DeviceVector` and `DeviceMap` inherit a value based +`__eq__` from `Mapping`, they hash by `id()` — injective over live objects, so two distinct +devices never share a hash and `__eq__` is never reached. + +A tuple of `(device, format)` pairs was tried first, for immutability. It made +`set_readable_format` rebuild and rescan the whole tuple, so registering *n* children was +O(n²) — 2000 signals cost 77 ms at construction, against 13 ms for the dict. The registry +is a `@cached_property`, matching `Device._child_devices`, because `AreaDetector` and +`add_children_as_readables` register children *before* calling `super().__init__()`, so +anything assigned in `__init__` would discard them. + +Deriving on demand also fixes staging: `stage()` walks the current registry rather than a +tuple captured at registration, so a format changed between runs takes effect on the next +`stage()`. + +Registration is "set" rather than "add": registering the same device twice replaces its +format rather than contributing it twice. This is a behaviour change, and the previous +behaviour was not deliberate. + +The deprecated `ConfigSignal` and `HintedSignal` markers are removed rather than carried +across. They worked by comparing equal to their target, which relied on the `match` +statement running during `add_readables`; deriving the callables on demand moves that +comparison to the first `read()` instead. Rather than keep a shim alive to preserve where +the warning fires, they go — they have been deprecated since well before this change. + +### Runtime changes can be undone, and the baseline is sealed by a metaclass + +@oliwenmandiamond raised that a class declaring `sig_a` as hinted, then having that +changed at runtime, makes the class definition misleading. Two safeguards were proposed: +making class-declared formats immutable unless opted out, or a separate +`StandardDynamicReadable`. + +Both were rejected, because static-by-default blocks the case this ADR exists for: #1394's +motivating example is a monochromator whose `energy` is *declared in the class* as +configuration and which one technique wants hinted. Locking it precisely because it was +declared would mean a beamline had to get the Device class changed to switch technique, and +the "allow dynamic" decision would be made by whoever is furthest from the technique. +@jwlodek made the same call from the other side: ophyd v1 has always been mutable, v1 +codebases at NSLS-II make extensive use of changing `kind` at runtime, and behaviour +differences between the two libraries are a cost they do not want. + +What was adopted instead is @oliwenmandiamond's and @jwlodek's counter-proposal: +[](#StandardReadable.reset_readable_formats) puts a Device back to what its class declared, +so a plan that retunes a Device can undo it without recording what it changed. + +**The baseline is sealed by a metaclass**, not at the end of `StandardReadable.__init__`. +Formats can be declared in three places: annotations, applied inside `Device.__init__`; +registration a subclass does *before* its `super().__init__()`, which is what all 24 +in-tree Devices do; and registration it does *after*, which nothing in tree does but which +is legal. `type.__call__` runs the whole `__init__` chain before returning, and is the only +point after all three. Snapshotting at the end of `StandardReadable.__init__` was measured +to miss the third, and the symptom would be `reset_readable_formats` silently dropping a +child the class had declared. `Device` already has a metaclass (`_ProtocolMeta`, from the +`HasName` Protocol), so this adds no new machinery to the hierarchy. + +The reset is per-Device rather than per-tree, pairing with `set_readable_format` rather +than with `apply_readable_formats`, and its baseline is the class declaration rather than +the last applied settings file — so it discards a stored technique too. + +### Format changes apply between runs, not within one + +A run's descriptor is emitted at its start, and `HINTED_SIGNAL` sets up monitoring in +`stage()`. Changing a format part way through a run would make `describe()` and `read()` +disagree. This is documented rather than enforced: the check would have to know about run +boundaries, which a Device does not. + +### `StandardDetector` inherits `StandardReadable` + +A detector's step-scan signals are registered exactly like any other Device's, and +`read()`, `describe()` and `hints` merge them on top of whatever the data logics produce. +`add_config_signals` becomes a deprecated wrapper. + +The detector does not *reimplement* any of those verbs to achieve it. Each verb gathers a +tuple of callables — `_read_funcs`, `_describe_funcs` and friends — exactly as `stage()` +gathers `_stage_funcs`. `StandardReadable` puts the registry's contribution in each tuple, +and `StandardDetector` appends what its data logics produce, the way `StandardFlyable` +already appends to `_stage_funcs` rather than replacing `stage()`. Overriding the verbs +meant re-deriving the registered children's contributions by hand in each one, which is +exactly what would drift. Everything in a tuple runs in parallel. + +One consequence: a detector with no hinted fields now reports `hints` as `{}` rather than +`{"fields": []}`, because it no longer special cases what `StandardReadable` already does. + +Plugins are **not** registered automatically. `AreaDetector` registers only the driver's +`acquire_time` and `acquire_period`, plus whatever the caller passes as `config_sigs`. +Registering a plugin is a `set_readable_format()` call at the call site, which is the +explicit opt-in the #1395 reviewers asked for, now expressible per plugin and reversible +at runtime. + +### The `prepare_single` data logic tier goes + +[](./0012-detector-rewrite.md) gave `DetectorDataLogic` a `prepare_single` tier, returning a +`ReadableDataProvider`, so that a source producing one value per event could reach +`read()` and `describe()`. Registering a signal now does that, for any Device and without +a data logic, so the tier has nothing left of its own: its only in-tree implementation was +`PluginSignalDataLogic`, wrapping a driver and one `SignalR`, and that is a +`set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL)` call. + +`prepare_single`, `ReadableDataProvider` and `SignalDataProvider` are therefore deleted +rather than kept alongside. Keeping them would leave two ways to get a scalar into an +event, one of which drops itself with a warning as soon as the scan asks for more than one +collection, while the other simply works. + +### Formats are stored in the same file as settings, under a reserved key + +Values and formats are applied on the same cadence — a technique change wants both — so +they belong in one file and one `apply_settings` call. `store_settings` keeps values flat +and adds one reserved key holding the format changes since construction: + +```yaml +energy: 7.0 +temperature: 20.0 +: + : + energy: HINTED_SIGNAL + temperature: null +``` + +`Settings` gains a `readable_formats` attribute, so one `retrieve_settings` returns the +whole stored state. + +**No version marker is needed.** A file written before formats existed simply lacks the +reserved key, which means "change no formats". Each part being independently optional also +makes a hand-written formats-only profile work, changing technique without writing a value +to hardware. + +**The reserved keys cannot collide.** `` starts with `<`, which cannot +begin a Python identifier, so no attribute assignment can produce a colliding path — no +store-time guard is required. `<` is also not a YAML indicator, so unlike `*FORMATS*` or +`%FORMATS%` the key needs no quoting. `` replaces `""` as the root owner for the same +reason and because an empty-string key makes PyYAML emit its hard-to-read explicit +`? '' :` form. + +Because no path can look like `<...>`, `retrieve_settings` drops every such key it does +not understand with a warning, rather than reporting it as an unknown signal. A key written +by a newer version is then ignored by an older one, and a typo of a reserved key is still +noticed. + +**The formats section is two levels while values are flat.** A value is intrinsic to one +path; a format is a fact about an (owner, child) *pair*, and the same child can be +registered on two owners with different formats — which #1395 creates directly, since +`NDStatsIO` will declare `total` on itself while a detector may also register `stats.total`. +Recording the owner rather than inferring it also means the layout needs no "must be a +direct child" rule, so `AreaDetector` can keep registering `driver.acquire_time` and +`DeviceVector` elements keep working, with no dependency on #1395 and no second migration +later. + +The child path is relative to its owner, not to the root, so the owner is not repeated on +every line. A survey of dodal, haven and the NSLS-II repositories found that every +registration of something outside its owner's subtree is of an object outside the stored +Device altogether (an `__init__` argument or a `Reference`), which a root-relative path +cannot store either. + +### Only changes are stored, and applying resets to the class defaults first + +`walk_readable_formats` records only how each `StandardReadable` differs from what it +registered during construction, with a dropped child as `None`. `apply_readable_formats` +resets every `StandardReadable` in the tree to its class defaults, then applies those +changes. The key is written, even if empty, whenever the tree has a `StandardReadable`, as +an empty entry means "back to the defaults", where a missing key means "leave alone". + +Two alternatives were rejected: + +- **Store everything, apply by replacing.** Store a file, add a signal to the Device in a + later version, apply the old file, and the new signal is silently unregistered. It also + pins the class defaults of the day into every file, so a release that changes a default + is undone by old files. +- **Store everything, apply by merging.** Fixes the new-signal problem but cannot return + from technique B to technique A: anything B changed that A does not mention stays as B + left it, and dropping a child needs a hand-written null. + +Storing changes against the class defaults avoids both: a new signal keeps its class +format, defaults stay current, and a dropped child is written as a null automatically. +Registrations made in `__init__` are part of the defaults, so one of an object outside the +stored tree is never written, and does not warn on every store. + +`apply_readable_formats` resolves every path before changing anything, so a bad path cannot +leave the tree half applied. + +`Settings.partition` copies the formats to **both** halves. A device-specific apply plan +normally applies only the halves and never the original `Settings` — `apply_panda_settings` +partitions and calls `apply_settings` twice — so putting formats on one half would let the +plan silently decide whether they were restored at all. Applying them twice is harmless: it +touches no hardware and gives the same result. `apply_settings` applies formats after +values, so a failed value write leaves the readable registry untouched. + +## Consequences + +- A Device can be switched between techniques at runtime, and the switch can be saved and + restored, closing #1394. +- `add_readables` keeps working unchanged for all 85 in-tree call sites and for downstream + Devices, but gains "set" semantics. Having gained them it is no longer adding anything, + so it has since been deprecated in favour of `set_readable_format`, which says what it + does; it still works, with a `DeprecationWarning`. +- Tests that asserted on `_read_funcs`, `_has_hints` and friends had to be rewritten onto + the public verbs. Those assertions were already against the testing conventions in + `CLAUDE.md`, and the derived-on-demand design makes them impossible rather than merely + discouraged. +- Deriving callables per call costs a pass over the registry on every verb. The registries + are small (tens of entries), and it is what makes the format mutable. +- Registering a device that is not below its owner still works, but a runtime change to + its format cannot be stored, since there is no stable path for it. + `walk_readable_formats` warns and skips. +- This ADR is numbered 21 rather than 20 to leave 20 for the unmerged bounded-data-logic + ADR in #1367. diff --git a/docs/explanations/decisions/0021-single-logic-property.md b/docs/explanations/decisions/0021-single-logic-property.md new file mode 100644 index 0000000000..4a09838722 --- /dev/null +++ b/docs/explanations/decisions/0021-single-logic-property.md @@ -0,0 +1,78 @@ +# 21. One `logic` property per Device + +Date: 2026-08-18 + +## Status + +Accepted + +Supersedes part of [](./0017-standard-movable.md), which introduced `movable_logic`. + +## Context + +A Device that both moves and flies had to expose the same logic object twice. `Motor` +built one `MotorFlyableMovableLogic` and returned it from a `movable_logic` property for +[](#StandardMovable) and a `flyable_logic` property for [](#StandardFlyable), plus a +private `_logic` to avoid building it twice. Every future mix-in would add another name. + +The obstacle is that Python has no intersection type: there is no way to write "the logic +must be both a `MovableLogic[float]` and a `FlyableLogic[FlyMotorInfo, MotorFlyCtx]`" as a +single annotation. + +[](./0017-standard-movable.md) also chose `@cached_property` stacked on `@abstractmethod` over an +`add_movable_logic()` method, on the grounds that the property is "checkable by static +analysis". That turned out not to hold. `functools.cached_property` does not forward +`__isabstractmethod__` from the function it wraps, so `StandardMovable.__abstractmethods__` +was empty, pyright reported nothing, and a Device that forgot its logic failed with +`AttributeError: 'NoneType' object has no attribute 'readback'` from inside +`Device.__init__`. + +## Decision + +Each mix-in declares the **same** attribute name, `logic`, with **its own** +required type: + +```python +class StandardMovable(Device, ..., Generic[SignalDatatypeT]): + @abstract_cached_property + def logic(self) -> MovableLogic[SignalDatatypeT]: ... + +class StandardFlyable(_StandardBase, ..., Generic[PrepareT, CtxT]): + @abstract_cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: ... +``` + +A Device inheriting both provides one implementation, and the type checker verifies it +against each base independently. The intersection is expressed by the logic class's own +MRO: `MotorFlyableMovableLogic` inherits both, so it satisfies both declarations, and the +concrete type is what callers see back. + +This catches, at type-check time, a logic that satisfies one mix-in but not the other, a +wrong datatype parameter, and a Device with two mix-ins and no implementation. A Device +with a *single* mix-in and no implementation is caught at instantiation instead, by +`abstract_cached_property`, a `cached_property` that sets `__isabstractmethod__` on the +descriptor so that `Device`'s metaclass collects it. It is declared as returning +`cached_property[T]` rather than as a subclass, because a subclass breaks the covariance +check on the override and would reject the correct case too. + +Because [](#StandardFlyable) is now genuinely abstract, `FlyableLogic.with_device` builds +a concrete `_EphemeralFlyable` to attach the logic to. + +An alternative of a single `logics` collection searched by type was rejected: membership +cannot be proved statically, so it would reintroduce exactly the runtime-only failure +this change fixes. + +## Consequences + +- One property per Device however many mix-ins it combines; `Motor` loses two properties. +- Where the move and fly logic are genuinely unrelated implementations they must be + combined into one class, since one property returns one object. Nothing in-tree pays + this cost, as `MotorFlyableMovableLogic` was already that shape. +- Forgetting the logic now fails with `TypeError: Can't instantiate abstract class ... + with abstract method logic` rather than an `AttributeError` on `None`. +- `movable_logic` and `flyable_logic` are removed rather than deprecated. A shim would + only have helped code that *reads* the attribute: a subclass that still *defines* + `movable_logic` would override the shim and leave `logic` unimplemented, so the + Device would fail at instantiation anyway. +- The same mechanism is available to `StandardDetector` if it later inherits + [](#StandardFlyable); its logic would need to satisfy `FlyableLogic` too. diff --git a/docs/explanations/decisions/0022-bounded-data-logic-tier.md b/docs/explanations/decisions/0022-bounded-data-logic-tier.md new file mode 100644 index 0000000000..4bdfec60a9 --- /dev/null +++ b/docs/explanations/decisions/0022-bounded-data-logic-tier.md @@ -0,0 +1,234 @@ +# 22. Add a bounded `DetectorDataLogic` tier + +Date: 2026-07-17 + +## Status + +Accepted + +## Context + +[](./0012-detector-rewrite.md) split data production out into `DetectorDataLogic`. After +[](./0020-runtime-readable-format.md) removed the single-event tier, one tier was left: a +logic prepared for an unbounded number of collections, feeding `collect_asset_docs()` and +`describe_collect()` and doing its own writes as it went. + +Four open issues bottom out in two gaps. + +### The data logic is not told enough at prepare time + +Issue #1309 wants chunk size and flush rate configured from the scan parameters. An Xspress +running at 400 Hz with a 2 Hz flush rate wants 200 frames per chunk; the PandA wants a flush +period. Neither is expressible, because an unbounded logic is told nothing about timing: +`ADHDFDataLogic` reads `num_frames_chunks` back from the IOC and forces it to 1 if unset, the +PandA hardcodes `chunk_shape=(1024,)` behind a TODO, and Odin hardcodes `(1, *data_shape)`. + +The parameter actually needed is the frame **period** (`livetime + deadtime`), not the number +of frames, since the requirement is a flush *rate* — and the period is known for step scans as +well as fly scans, where the total frame count is not. + +Two workarounds were rejected: subclassing `TriggerInfo` and `prepare()` to carry a chunk size +(as Malcolm did) puts a data-writing concern in the trigger path, against +[](./0012-detector-rewrite.md)'s rule that a logic is handed only the fields it needs; +computing it in the `DetectorTriggerLogic` puts a data concern in the trigger logic. + +### Detectors that produce data but cannot write files + +Issues #1248 and #888 come from a Struck SIS3820 scaler in a VME crate, a MeasComp CTR-08, a +TetrAmm electrometer, an electron analyser. They want the trigger, arm, prepare and stage +machinery of `StandardDetector` but must emit event pages, and `StandardDetector` inherited +`WritesStreamAssets` unconditionally — the bundler calls `collect_pages()` only on a device +that is *not* one. (#888 asks for the writer to be optional, which the rewrite already did; +its real request is #1248, so it is a duplicate.) + +These are not areaDetectors, and they put more pressure on the trigger/acquire/data split. A +CTR-08 has no driver/plugin split in the hardware: `MCS:EraseStart` both clears the arrays and +starts counting, and `MCS:NuseAll` is both the exposure count and the buffer size, where an +areaDetector acquires with `driver.acquire` and captures with a separate `TSAcquire` or HDF +`capture`. + +### These are the same gap + +Both groups must be told *how many* collections to expect at prepare time, because they own a +finite buffer that has to be sized before acquisition: the scaler's MCA array length *is* the +frame count, and the stats plugin's `TSNumPoints` must be set before the time series starts. A +device with a finite buffer also naturally produces its data as pages at the end of an event +rather than as stream datums as it goes. + +So one new tier serves both, and #1309's period rides along with it. + +### The areaDetector stats duality + +An `NDPluginStats` exposes each statistic twice: as a scalar (`Total_RBV`) and as a time +series array (`TSTotal`). Only the scalar was modelled, as a registered signal read once per +event, so a detector with no file writer had no way to produce a value per collection for a +fly scan. + +## Decision + +### One method that describes, and one that starts + +```python +class DetectorDataLogic(Generic[DataCtxT]): + async def make_data_provider( + self, datakey_name: str, num_collections: int, + period: float, flush_period: float, + ) -> tuple[StreamableDataProvider | PageableDataProvider, DataCtxT] | None: ... + async def start(self, ctx: DataCtxT) -> None: ... +``` + +`make_data_provider` says what this logic *would* produce for this scan without starting +anything; `start` does the writes that make it happen, and is called only for the providers +the detector will use. So the detector asks every logic what it would make, decides which ones +it wants, and starts only those. + +What the first worked out and the second needs is threaded between them through an explicit +context object, the same way `FlyableLogic` threads its own, rather than stashed on the logic. +A logic that stored it would be left holding a plan for a scan it was never asked to serve. + +One `prepare_*` per tier, doing both jobs, was tried first. It forces the detector to arm +hardware to find out what it would get, so a provider it then decides not to use has to be +stopped again — wasteful with a `TSAcquire` erase, and impossible to justify once the choice +depends on what the providers produce. + +`num_collections` is the **total** for the scan (`TriggerInfo.number_of_collections`), and +both tiers are told the period, so either can size a chunk or a buffer against it. + +### The tier is the type of the provider returned + +There is no declaration of which tier a logic implements and no override detection: a logic +returning a [](#StreamableDataProvider) is unbounded for this scan, one returning a +[](#PageableDataProvider) is bounded, and one may return either depending on what it is asked +for. Returning `None` means "not this scan" — how a finite buffer declines an unbounded one, +or a plugin that is switched off declines any. It is neither an error nor warned about, as +agreed in #1364: the point is that a detector may carry more logics than one scan uses. + +### `PageableDataProvider`, with readings derived from pages + +The bounded tier returns one provider type, which emits pages. Readings are derived from those +pages rather than being a separate code path: a step-scan prepare has `number_of_events == 1`, +so its page holds exactly one event, which extracts to a single reading. +`collections_per_event` appears in the datakey shape exactly as for a +`StreamableDataProvider`. + +### The period is resolved in `prepare()` + +`TriggerInfo.livetime` of 0 means "whatever is currently set on the detector", so the period +cannot simply be forwarded: an explicit `prepare(TriggerInfo(number_of_events=10))` would hand +the data logic a period of 0. Instead `DetectorTriggerLogic.default_trigger_info()` is extended +to return the current `livetime` and `deadtime`, and `prepare()` fills a zero `livetime` in +from it *after* preparing the trigger logic, which [](./0012-detector-rewrite.md) already +requires to come first, so the hardware holds the real values by then. Data logics therefore +never see a period of 0. + +### Streaming providers are reused, finite buffers are re-made + +Providers are reused while the scan they were made for is unchanged, so a step scan does not +reopen its file on every point. The reuse key is `collections_per_event` *and* the period, +because both are baked into a `StreamResource` when it is made — the shape from the first, the +chunking from the second — and areaDetector's `num_frames_chunks` can only be set before +`capture` starts. A period change therefore starts a new file. + +A finite buffer is re-made on every `prepare()` regardless, and on every `trigger()` after the +first one since that `prepare()`, because re-making it is what re-arms it; `kickoff()` never +re-makes one. That is exactly the set of places a buffer should and should not be re-armed: the +first point of a step scan is already armed by the `prepare()` in front of it, and re-arming it +again would erase an empty buffer, putting a spurious stop/start on the arming control before a +frame had arrived. A re-armed buffer sitting at 0 alongside a monotonic HDF writer at 15 would +trip the `_all_the_same` reducer, so the two kinds are tracked separately. + +### A bounded provider's progress baseline is zero in `trigger()` + +`trigger()` takes a bounded provider's starting progress to be 0 rather than reading it back, +while `kickoff()` reads it live. + +It is tempting to derive this instead of stating it, since re-arming refreshes +`collections_written`, and a stats time series is erased by the `TSAcquire` write in its +`start` — so the refreshed read already returns zero. That only holds when the data logic +performs the erase. Where the erase belongs to the acquire control, as with the CTR-08's +`EraseStart`, it happens after that read, so the baseline would be the stale pre-erase value +and the detector would wait for twice the buffer's length and hang. Stating the rule keeps the +framework agnostic about which component erases. + +### Erase and start controls belong to the acquire logic + +Where one control both starts acquisition and clears the data buffer, it belongs to the +`DetectorAcquireLogic`, alongside the stop control and the idle status. Nothing needs to +declare the erase to the framework: a data logic's `start` always runs before +`start_acquiring()`, so the buffer is sized before the control fires, and the baseline rule +above means the detector does not care who erased it. + +The trigger/acquire/data split therefore survives these devices, with the three logics sharing +one IO device exactly as the areaDetector logics share a `driver`. The split is of concerns, +not of hardware. + +### Logic objects fill exactly one role + +Building a `DetectorLogic` raises if an object satisfies more than one of the three logic +roles, directing the author to pass separate objects. A device whose concerns all live on one +control is a natural candidate for a single combined object, and registration is a chain of +`isinstance` tests, so such an object would silently register as the first role it matched and +have the others dropped without a word. + +### A detector produces stream assets or event pages, never both + +The two bluesky protocols are mutually exclusive by design, to avoid a device producing blocks +of two different sizes, so a detector whose logics would produce both raises. How the +applicable verb is exposed, and the one case where the two may be carried together, are in +[](./0023-detector-is-a-flyable.md). + +Which kind a detector produces is decided per prepare, so a single detector class can be either +depending on its `__init__` arguments — one `AravisDetector`, configured for files or for pages +— which was the original requirement. That it can also change *between runs*, when a file +writer is switched off and a stats time series takes over, is documented rather than enforced: +a run's descriptor is emitted at its start, so switching part way through would leave +`describe_collect()` and the documents disagreeing, and a Device cannot see run boundaries. + +### Stats with a file writer go via NDAttributes + +An areaDetector with an HDF writer does not need the stats time series: `ADHDFDataLogic` +already pulls NDAttribute datasets into the file, which is areaDetector's own mechanism for +exactly this. So registering the scalar with `set_readable_format` remains right for a +detector that writes files and wants a stats value in a step scan, and a stats time series is +right for one that writes none. + +### One stats data logic for many arrays + +An `NDPluginStats` has one time series control (`TSNumPoints`, `TSAcquire`, `TSAcquireMode`) +shared across roughly twenty arrays, so one stats data logic covers *many* arrays with the +control embedded, rather than one logic per array with a shared control object. This follows +the existing grain — a provider already emits many datakeys, as `ADHDFDataLogic` does for the +main dataset plus its NDAttributes — and avoids the coordination problem of several logics +contending over one `TSAcquire`. Which statistics are wanted becomes a constructor argument. + +## Consequences + +`DetectorDataLogic` implementations must be rewritten onto `make_data_provider` and `start`, +and take the new `period` and `flush_period` arguments. This is a breaking change for out-of-tree data logics; the +library is in alpha and no shim is provided. Splitting them is usually mechanical — describe in +one, write in the other — but a logic that cannot describe its data without starting (in tree, +`OdinDataLogic`, whose frame shape is only readable once the file processor is writing) has to +do its writes in `make_data_provider` and leave `start` empty, which is safe only while such a +logic is never the one a detector drops. In-tree, `ADHDFDataLogic` sizes its chunk from +`TriggerInfo.flush_period` and the rate, and `PandaHDFDataLogic` sets its flush period signal +from the same field, replacing the TODOs that stood in for both. + +`DetectorTriggerLogic.default_trigger_info()` implementations should return the current +`livetime` and `deadtime` alongside the frame count. Those that do not still work, but their +detectors fall back to reading the chunk size back from hardware. + +Detectors that cannot write files become expressible: a data logic returning a +`PageableDataProvider` gives them `collect_pages()` and the whole of `StandardDetector` +besides. This closes #1248 and #888. Fly scanning an areaDetector that carries an HDF writer +plus a finite-buffer data logic starts working, closing #1364. + +A data logic may decline a scan for reasons of its own: `ADHDFDataLogic` and +`StatsTimeSeriesDataLogic` take an `enable_callbacks` flag, defaulting to today's behaviour of +switching the plugin on, which when `False` makes them follow whatever the plugin is set to +and produce nothing when it is off. + +A device whose exposure count and buffer size are the same PV has it written twice: by the +trigger logic as `number_of_exposures`, and by the data logic as `num_collections`. The two +agree only when `exposures_per_collection` is 1. This is latent rather than live, because such +devices do not average exposures and so cannot ask for anything else, but a detector that could +do both would need its data logic to read the value back rather than set it. diff --git a/docs/explanations/decisions/0023-detector-is-a-flyable.md b/docs/explanations/decisions/0023-detector-is-a-flyable.md new file mode 100644 index 0000000000..d81cb180e8 --- /dev/null +++ b/docs/explanations/decisions/0023-detector-is-a-flyable.md @@ -0,0 +1,174 @@ +# 23. StandardDetector is a StandardFlyable + +Date: 2026-08-28 + +## Status + +Accepted + +## Context + +`StandardDetector` re-implemented `prepare`, `kickoff` and `complete` and threaded its own +state between them, while [](#StandardFlyable) already existed to do exactly that for every +other flyer in the tree (`Motor`, PMAC, the PandA trigger logics). Two implementations of +one lifecycle drift apart: the detector guarded its stages by checking whether a context +object was `None`, so calling a verb too early failed with `AttributeError: 'NoneType' +object has no attribute ...` rather than saying what was wrong. + +`StandardFlyable` requires a single [](#FlyableLogic) in a `logic` property +([](./0021-single-logic-property.md)), and a detector had three logic objects registered on +the Device by `add_detector_logics()`. + +## Decision + +### The three logics become children of one `DetectorLogic` + +[](#DetectorLogic) holds the trigger, acquire and data logics, and is what `logic` returns. +The orchestration moves with them: preparing the trigger logic, making the data providers, +arming, and waiting for collections are all its methods now. It holds **no** back-reference +to its Device, so [](./0017-standard-movable.md) continues to hold unqualified. + +A detector builds it in `__init__` and returns it from `logic`: + +```python +class MyDetector(StandardDetector): + def __init__(self, prefix: str, name: str = "") -> None: + self.driver = MyDriverIO(prefix) + self._logic = DetectorLogic( + MyTriggerLogic(self.driver), + MyAcquireLogic(self.driver), + ) + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic +``` + +`add_detector_logics()` is removed rather than kept as a forwarder: a detector's logics are +fixed for its lifetime, so there is nothing a post-construction call can do that the +constructor cannot, and keeping it would leave two ways to wire the same Device. + +`StandardDetector` itself takes no `__init__` arguments, since a `Standard*` class that did +would interfere with multiple inheritance. The logic is supplied by overriding `logic`, +which `abstract_cached_property` makes a requirement checked when the Device is +instantiated. Assigning `self.logic` from `__init__` cannot serve instead: `ABCMeta` checks +`__abstractmethods__` in `__call__`, before `__init__` runs, so the class must override the +property to be concrete. For an ad-hoc detector in a plan or a test, +`DetectorLogic.with_device()` builds a Device around a logic, exactly as +`FlyableLogic.with_device()` does for any other flyer. + +### One thing the logic cannot get from its own fields + +Data logics name their datakeys after the Device, whose name does not exist until +`init_devices` calls `set_name` on context exit. `StandardDetector.set_name` pushes it into +`DetectorLogic.datakey_prefix` — one string, set at the moment it exists, rather than a +reference to the whole Device. + +### Which collect verb to expose is the Device's business + +A detector produces stream assets or event pages depending on what its data logics make for +a given scan, never both (see [](./0022-bounded-data-logic-tier.md)). The bluesky bundler +decides which to ask for from the *static* type — `collect_pages()` is called only on a +device that is not a `WritesStreamAssets` — so a detector that declared both would have its +pages silently dropped, and one that declared neither could not fly at all. + +So `StandardDetector.prepare()` binds whichever verb applies as a real instance attribute, +and removes the other. It is the Device that does this, from what `DetectorLogic.data` holds +after preparing; the logic reports what it made and is not told where that goes, so it holds +no back-reference. Only `prepare()` does it: re-arming a bounded provider on each point of a +step scan cannot change which verb applies, because the providers are re-made from the same +`TriggerInfo` that `prepare()` used. + +Choosing from the static type is a bundler limitation rather than a fact about detectors: a +device knows what it produced, and the bundler could decide from *that*, erroring only if +one device reported both ways in the same run. That change is a few lines in +`RunBundler.collect`, and with it both verbs become plain methods and the binding here goes +away. Until it is released, binding per prepare is what works against a released bluesky. + +### Mixing bounded and unbounded is an error unless the bounded keys are shadowed + +[](./0022-bounded-data-logic-tier.md) made carrying both kinds of data logic an error, because +the bundler treats stream assets and event pages as mutually exclusive. Carrying both is still +useful, though: the same quantity can be written durably into a file *and* read from a plugin's +buffer — an areaDetector stats total, which the HDF writer pulls in as an NDAttribute. + +So the rule relaxes. Where **every** datakey a finite buffer would produce is also produced +by the stream assets, the durable copy wins and the buffer sits the scan out, unarmed. +Anything a buffer would produce that the file does not cover is still an error, naming the +uncovered keys. Nothing has been started when that is decided, because +`make_data_provider` describes without acquiring (same ADR). + +### A logic may report its own progress + +`StandardFlyable.complete()` could produce `WatcherUpdate`s only for a `MovableLogic`, by +observing its readback against its setpoint. A detector's progress is "collections written +out of collections requested", which is neither, so a detector inheriting the base +`complete()` would have silently lost its progress reporting. + +So `FlyableLogic.on_complete` returns either an `Awaitable[None]` or an +`AsyncIterator[WatcherUpdate]`, and `complete()` looks at what it got back: an iterator is +yielded straight on to watchers, anything else is awaited. A logic that only blocks stays an +ordinary `async def`. A separate `WatchableFlyableLogic` base was tried first and dropped: +it made a second class to inherit for what is a property of one method's return. + +### One kickoff per prepare + +The detector used to be prepared for N events and kicked off a few at a time, with an +`events_to_kickoff` signal saying how many the next kickoff covered and an `is_last_kickoff` +flag deciding when to wait for idle. Nothing in tree drove it: the RunEngine kicks off once +and completes what it kicked off, and every other flyer already worked that way. + +Both go, and `kickoff()` requests the whole prepared scan. A second `kickoff()` without an +intervening `prepare()` now raises, from `StandardFlyable`, like any other flyer. + +### The data providers are the logic's state, not the fly context's + +`_FlyCtx` holds what `prepare()` was given — the `TriggerInfo` — plus the collections +baseline `kickoff()` took, and nothing else. The providers live on the logic instead, +because they outlive a single prepare → kickoff → complete cycle in two ways: a step scan +reuses an open file across its points, and `collect_while_completing` collects *after* +`complete()` has returned the flyer to IDLE. So the data verbs — `read`, `describe`, +`describe_collect`, `collect_asset_docs`, `collect_pages`, `get_index` — go through +`DetectorLogic.data`, which is cleared by `stage()`/`unstage()`, while the fly context +enforces the ordering of the fly verbs. + +`trigger()` therefore needs no context of its own, beyond a `needs_rearm` flag on the fly +context. It prepares implicitly if nothing has been, then asks the data state to re-arm what +needs re-arming: a finite buffer holds one event at a time so it is re-made per point, while a +streaming provider carries on. The flag is what keeps the first point from being re-armed on +top of the `prepare()` that just armed it. The trigger logic is not re-prepared per point, so +a step scan does not repeat its puts. + +### One context type, not one per stage + +`FlyableLogic` takes a single `CtxT` threaded through all three stages, so a field that +only exists after `kickoff()` is `| None` on the one dataclass — `_FlyCtx`'s +`kickoff_collections_written`, `MotorFlyCtx`'s `status`, `PmacFlyCtx`'s +`trajectory_status` — and `on_complete` unwraps it. + +Splitting into `PrepareCtxT` and `KickoffCtxT` would make those fields non-optional. It +was not worth it. Every flyer would need a second dataclass, or inheritance between two, +and `on_kickoff` would have to rebuild one from the other; every declaration site gains a +third parameter, including the three PandA logics that carry no state and would read +`FlyableLogic[SeqTableInfo, None, None]`. `StandardFlyable` gains nothing either, since +the context it holds changes type part way through the lifecycle and so cannot be typed +as one or the other regardless. Against that, three `error_if_none` calls in the whole +tree. The `| None` is doing useful work as documentation: it says *this appears at +kickoff*, which is exactly what a reader needs to know. + +## Consequences + +- `StandardDetector` subclasses build a `DetectorLogic` in `__init__` and return it from + `logic`; `det.add_detector_logics(*logics)` is gone, and a logic conflict is reported when + the detector is constructed rather than at the `add` call. +- `kickoff()` → `kickoff()` without an intervening `complete()` is rejected, and so is a + second `kickoff()` after one. The old detector allowed both, because it only checked that + a prepare context existed. +- `events_to_kickoff` is removed from the public API. +- Calling a verb before `prepare()` says `: prepare() must be called first` instead of + `Prepare not run`. +- A detector satisfies neither `WritesStreamAssets` nor `EventPageCollectable` until it has + been prepared. Everything in the RunEngine that dispatches on them runs after prepare. +- `prepare`, `kickoff`, `complete`, `stage` and `unstage` are no longer implemented on + `StandardDetector` at all, so a change to the flyer lifecycle reaches detectors too. diff --git a/docs/how-to/change-readable-format.md b/docs/how-to/change-readable-format.md new file mode 100644 index 0000000000..cf80f1816b --- /dev/null +++ b/docs/how-to/change-readable-format.md @@ -0,0 +1,87 @@ +# How to change what a Device reports at runtime + +A [](#StandardReadable) decides what appears in `read()`, `read_configuration()` and +`hints` from the [](#StandardReadableFormat) each of its children is registered with. +Normally that is fixed when the Device class is written, but it can also be changed on a +live Device with [](#StandardReadable.set_readable_format). + +This is the equivalent of changing a `Component`'s `kind` in ophyd v1. + +## Changing one signal + +A beamline with two techniques might scan a monochromator's energy in one and hold it +fixed in the other. The same Device serves both: + +```python +from ophyd_async.core import StandardReadableFormat as Format + +# Scanning technique: energy is a hinted read +mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + +# Fixed technique: energy is configuration, recorded once per run +mono.set_readable_format(mono.energy, Format.CONFIG_SIGNAL) + +# Not interesting for this technique at all +mono.set_readable_format(mono.energy, None) +``` + +[](#StandardReadable.get_readable_formats) reports what is currently registered, keyed by +child Device, so a child that does not contribute is simply absent: + +```python +mono.get_readable_formats().get(mono.energy) +``` + +## Putting it back + +[](#StandardReadable.reset_readable_formats) undoes every runtime change, back to +how the class declared things — annotations, `add_children_as_readables`, and +anything the Device's own `__init__` registered: + +```python +mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) # retune for a technique +... +mono.reset_readable_formats() # back to the class defaults +``` + +So a plan that retunes a Device can put it back without having to record what it +changed. It resets that Device only, not its children — to do a whole tree: + +```python +from ophyd_async.core import apply_readable_formats + +apply_readable_formats(detector, {}) +``` + +```{note} +The baseline is what the Device class declares, so this also discards formats that came +from a stored settings file — see [](./store-and-retrieve.md). +``` + +```{note} +Change formats **between runs**, not inside one. A run's descriptor is emitted when the +run starts, and [](#StandardReadableFormat.HINTED_SIGNAL) begins monitoring in `stage()`, +so a format changed part way through a run would leave `describe()` and `read()` +disagreeing. +``` + +## Opting a detector plugin in + +A [](#StandardDetector) is a [](#StandardReadable), so the same call works on one. This +matters most for areaDetector, where a detector often carries many more plugins than are +wired into its chain — so plugins are never registered automatically, and you opt in to +the ones that are actually producing meaningful values: + +```python +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) +``` + +Data produced by a [](#DetectorDataLogic) is added on top of these, so a detector can +write a file *and* report a plugin scalar in a step scan. + +```{seealso} +[](./store-and-retrieve.md) for saving a set of formats alongside signal values and +loading them back, and +[](../explanations/decisions/0020-runtime-readable-format.md) for why the registry is +keyed by Device. +``` diff --git a/docs/how-to/choose-right-baseclass.md b/docs/how-to/choose-right-baseclass.md index bbea3b6df6..83b7224a95 100644 --- a/docs/how-to/choose-right-baseclass.md +++ b/docs/how-to/choose-right-baseclass.md @@ -10,7 +10,7 @@ When writing a new Device there are several base classes to choose from that wil There are some utility baseclasses that allow you to create a Device pre-populated with the right verbs to work in bluesky plans: -- [](#StandardReadable) allows you to compose the values of child Signals and Devices together so that you can `read()` the Device during a step scan. +- [](#StandardReadable) allows you to compose the values of child Signals and Devices together so that you can `read()` the Device during a step scan. What each child contributes can also be changed on a live Device, see [](./change-readable-format.md). - [](#StandardMovable) allows devices that move to a target value and wait for a readback to confirm arrival (motors, temperature controllers, piezo stages, and similar) to be implemented by composing a [](#MovableLogic) instance, rather than directly implementing the `Locatable`, `Stoppable`, and `Subscribable` protocols. See [](../explanations/when-to-extend-movable.md) for more details. - [](#StandardDetector) allows file-writing detectors to be used within both step and fly scans, reporting periodic references to the data that has been written so far. Instances of [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) are composed together to provide this functionality. This separation of concerns makes it easy to mix and match different trigger modes, acquisition strategies, and data outputs. - [](#StandardFlyable) allows actuators (like a motor controller) to be used within a fly scan, by composing a [](#FlyableLogic) instance rather than directly implementing the `Preparable`, `Flyable`, and `Stageable` protocols. Use it as a mix-in on a Device, or call [](#FlyableLogic.with_device) to wrap a bare logic object in an ephemeral flyer within a plan. diff --git a/docs/how-to/implement-ad-detector.md b/docs/how-to/implement-ad-detector.md index 287b30e414..2a19dce6ca 100644 --- a/docs/how-to/implement-ad-detector.md +++ b/docs/how-to/implement-ad-detector.md @@ -26,18 +26,22 @@ Implement methods for each trigger mode your detector supports: - `prepare_level(num)` - Setup for external level/gate triggering (high level duration determines exposure time) Only implement the prepare methods for trigger modes your detector actually supports. The detector will automatically report which trigger types are available based on which methods are implemented. -If the detector has configuration values that should be captured in the scan then implement: -- `config_sigs()` - Return the set of signals that should appear in read_configuration() +Configuration values that should be captured in the scan are declared on the IO class, +with [](#StandardReadableFormat.CONFIG_SIGNAL), exactly as on any other +[](#StandardReadable) — the trigger logic does not nominate them. If you support external triggering you should also implement: -- `get_deadtime(config_values)` - Calculate the minimum time between exposures based on configuration values +- `get_deadtime(config_values)` - Calculate the minimum time between exposures. It is + given the value of *every* signal the detector reports as configuration, so a signal + it needs must be declared with [](#StandardReadableFormat.CONFIG_SIGNAL). To preserve hardware state in plans like [`bp.count`](#bluesky.plans.count) and [`bps.trigger_and_read`](#bluesky.plan_stubs.trigger_and_read) that call `trigger()` without a preceding `prepare()`, implement: - `default_trigger_info()` - Return the [](#TriggerInfo) to use for the implicit - prepare. For AD detectors call `await trigger_info_from_num_images(self.driver)` to - read back the current `num_images` from the driver rather than resetting it to 1. + prepare. For AD detectors call `await trigger_info_from_driver(self.driver)` to + read back the current `num_images`, `acquire_time` and `acquire_period` from the + driver rather than resetting them. @@ -87,7 +91,7 @@ For example, for ADAravis: The `AreaDetector` baseclass will: - Store the driver as `self.driver` -- Call `add_detector_logics()` to register your trigger and arm logic +- Build a [](#DetectorLogic) from your trigger and arm logic and return it from `logic` - Create and register a data logic for file writing for each factory in `writer_factories` - Add configuration signals (driver.acquire_time, driver.acquire_period, and any you specify) - Store any plugins as attributes on the detector @@ -109,15 +113,18 @@ For example, for ADAravis: ## Add multiple data streams (optional) -The composition-based architecture makes it possible to add multiple data outputs to a detector. After creating the detector, you can call `add_detector_logics()` to add additional data sources: +The composition-based architecture makes it possible to give a detector several data outputs: ### Reading stats plugins alongside file writing +A plugin scalar needs no data logic: a detector is a [](#StandardReadable), so register +the signal and it appears in `read()` alongside whatever the file writer produces. + ```python -from ophyd_async.epics.adcore import PluginSignalDataLogic +from ophyd_async.core import StandardReadableFormat as Format -det = adaravis.AravisDetector(prefix, path_provider) -# Add stats total as a readable signal in events -det.add_detector_logics(adcore.PluginSignalDataLogic(det.driver, det.stats.total)) +det = adaravis.AravisDetector(prefix, path_provider, plugins={"stats": stats}) +# Add stats total as a hinted signal in events +det.set_readable_format(det.stats.total, Format.HINTED_UNCACHED_SIGNAL) ``` ### Multiple HDF writers for different ROIs diff --git a/docs/how-to/store-and-retrieve.md b/docs/how-to/store-and-retrieve.md index e91ec08dd5..f4ec1931aa 100644 --- a/docs/how-to/store-and-retrieve.md +++ b/docs/how-to/store-and-retrieve.md @@ -43,3 +43,26 @@ def load_panda(panda1: HDFPanda): settings = yield from retrieve_settings(provider, "yaml_file_name", panda1) yield from apply_settings_if_different(settings, apply_panda_settings) ``` + +## Storing readable formats too + +As well as signal values, [](#store_settings) records how each child contributes to the Device's bluesky verbs — which signals are hinted, which are configuration — and [](#apply_settings) restores that alongside the values. The two change on the same cadence, so they live in one file and are applied by one call. This means a stored file can switch a Device between techniques as well as putting its values back; see [](./change-readable-format.md) for changing a format directly. + +Values stay flat in the yaml. Formats go under the reserved [](#READABLE_FORMATS_KEY), and only what has changed since the Device was constructed is recorded, so the file stays short and can be edited by hand: + +```yaml +energy: 7.0 +temperature: 20.0 +: + : + energy: HINTED_SIGNAL + temperature: null + stats: + total: HINTED_UNCACHED_SIGNAL +``` + +The outer key of each formats entry is the [](#StandardReadable) the child is registered on, with `` ([](#ROOT_DEVICE_KEY)) standing for the Device you stored. The inner key is the child's path from that owner. The owner is recorded rather than worked out from the child's path, because the same child can be registered on more than one Device with a different format each time. A null means the child has been dropped. + +Applying first resets every Device in the tree to the formats its class declares, then applies the changes in the file. So a file stored with nothing changed (`: {}`) puts the class defaults back, and a signal added to a Device class after the file was stored keeps the format the class gives it. + +A file with no `` key changes no formats, so settings files saved before formats were storable keep working untouched. A file with *only* that key switches technique without writing a single value to hardware. diff --git a/docs/how-to/write-a-flyer.md b/docs/how-to/write-a-flyer.md index a794ab47f4..da4d06f164 100644 --- a/docs/how-to/write-a-flyer.md +++ b/docs/how-to/write-a-flyer.md @@ -24,7 +24,7 @@ info passed to `prepare()`, and `CtxT`, the context threaded between the phases |--------|-------------------|--------------| | `on_prepare(value)` | *abstract* | move to the start, arm the hardware, and return a context | | `on_kickoff(ctx)` | *abstract* | start the scan and return the context for `on_complete` | -| `on_complete(ctx)` | *abstract* | block until the scan finishes | +| `on_complete(ctx)` | *abstract* | block until the scan finishes, optionally yielding progress | | `stop()` | no-op | disarm the hardware and wait for it to stop | | `on_stage()` | calls `stop()` | set the hardware up on `stage()` if it differs from stop | | `on_unstage()` | calls `stop()` | clean the hardware up on `unstage()` if it differs from stop | @@ -74,26 +74,20 @@ status on the context so `on_complete` can await it: Because the logic is also a `MovableLogic`, `complete()` reports progress to watchers (e.g. progress bars) automatically by observing the readback — reusing the same [](#WatcherUpdate) machinery as [](#StandardMovable.set). A flyer whose logic is *not* -movable (see the next section) just blocks with no progress updates. See -[](../explanations/when-to-extend-movable.md) for more on `MovableLogic`. +movable (see the next section) just blocks with no progress updates, unless it reports +its own: write `on_complete` as an async generator yielding [](#WatcherUpdate) and those +updates go straight to the watchers, as a detector's "collections written out of +collections requested" does. See [](../explanations/when-to-extend-movable.md) for more on +`MovableLogic`. -Wire the logic into the Device with a `@cached_property`. `Motor` builds one shared -instance and returns it from both `movable_logic` and `flyable_logic` so the move and -fly paths act on the same signals: +Wire the logic into the Device with a `@cached_property` called `logic`. Each +mix-in declares that same name with its own required type, so one logic object that +inherits both `MovableLogic` and `FlyableLogic` satisfies both, and the move and fly +paths act on the same signals: ```{literalinclude} ../../src/ophyd_async/epics/motor.py :language: python -:pyobject: Motor._logic -``` - -```python -@cached_property -def movable_logic(self) -> MotorFlyableMovableLogic: - return self._logic - -@cached_property -def flyable_logic(self) -> MotorFlyableMovableLogic: - return self._logic +:pyobject: Motor.logic ``` ## An ephemeral flyer created in a plan diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 8ab91db8d8..f5d01a2099 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -12,8 +12,8 @@ effect on ophyd-async itself. Controls the implicit prepare performed by [`StandardDetector.trigger()`](#ophyd_async.core.StandardDetector.trigger) when -[`prepare()`](#ophyd_async.core.StandardDetector.prepare) has not been called since -the last [`stage()`](#ophyd_async.core.StandardDetector.stage). +[`prepare()`](#ophyd_async.core.StandardFlyable.prepare) has not been called since +the last `stage()`. | Value | Behaviour | |-------|-----------| diff --git a/docs/tutorials/implementing-detectors.md b/docs/tutorials/implementing-detectors.md index 4fa2fe93ea..d628f53cfe 100644 --- a/docs/tutorials/implementing-detectors.md +++ b/docs/tutorials/implementing-detectors.md @@ -179,7 +179,7 @@ It derives from [](#StandardDetector) which is a utility baseclass that implemen - [](#DetectorAcquireLogic) to start and stop acquisition and wait for it to complete - [](#DetectorDataLogic) to tell the detector to open a file, describe the datasets it will write, and emit StreamAsset documents as frames are written -In this case, we have three logic classes written just for this simulation, all taking a reference to the pattern generator that provides methods for both detector control and file writing. In other cases the detector control and filewriting may be handled by different sub-devices that talk to different parts of the control system. The job of the top level detector class is to take the arguments that the logic classes need, create the logic instances, and pass them to [](#StandardDetector.add_detector_logics). +In this case, we have three logic classes written just for this simulation, all taking a reference to the pattern generator that provides methods for both detector control and file writing. In other cases the detector control and filewriting may be handled by different sub-devices that talk to different parts of the control system. The job of the top level detector class is to take the arguments that the logic classes need, create the logic instances, and wire them into a [](#DetectorLogic), which it returns from `logic`. Now let's look at the underlying classes that define the detector behavior: @@ -199,8 +199,10 @@ If we wanted to support external triggering, we would also implement: - `prepare_level()` for external level/gate triggering (high level duration determines exposure) We could also implement: -- `get_deadtime()` to calculate the minimum time between exposures based on configuration -- `config_sigs()` to return signals that should appear in read_configuration() +- `get_deadtime()` to calculate the minimum time between exposures. It is given the value + of every signal the detector reports as configuration, so a signal it needs must be + declared with [](#StandardReadableFormat.CONFIG_SIGNAL) like on any other + [](#StandardReadable) — the trigger logic does not nominate signals itself - `default_trigger_info()` to return the [](#TriggerInfo) to use when `trigger()` is called without a preceding `prepare()` (governed by [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE)). ### `BlobAcquireLogic` @@ -226,15 +228,34 @@ Then we have `BlobDataLogic`, a [](#DetectorDataLogic) subclass: ``` Its job is to manage the file writing and data streaming: -- `prepare_unbounded()` tells the detector to open a file, and returns a [](#StreamableDataProvider) that describes the datasets that will be written and tracks write progress +- `make_data_provider()` works out where the file will go and returns a [](#StreamableDataProvider) describing the datasets that will be written, without opening anything +- `start()` opens the file at the path `make_data_provider()` handed back, so nothing is written until the detector has decided it wants this data - `get_hinted_fields()` returns the data keys that are interesting to plot - `stop()` tells the detector to close the file -The [](#StreamableDataProvider) returned from `prepare_unbounded()` contains: +The [](#StreamableDataProvider) returned from `make_data_provider()` contains: - `collections_written_signal` - a signal that tracks how many frames have been written - `make_datakeys()` - creates DataKey descriptions for each dataset - `make_stream_docs()` - emits StreamResource and StreamDatum documents as frames are written +#### Which kind of data a logic produces + +A [](#DetectorDataLogic) is not declared as one kind or another: the detector asks each of its logics what it would make for this scan, and the type of provider that comes back says which kind it is. + +- a [](#StreamableDataProvider) is for a source that works for any number of collections, like a file writer. The detector exposes `collect_asset_docs`, and `BlobDataLogic` is an example. +- a [](#PageableDataProvider) is for a source holding a *finite buffer* that must be sized before acquisition, like an areaDetector stats time series or a scaler whose array length is the frame count. It emits event pages, so the detector exposes `collect_pages` instead. See [](#StatsTimeSeriesDataLogic). +- `None` means "not this scan", for instance a finite buffer asked for an unbounded number of collections, or a plugin that is switched off. This is not an error: a detector may carry more data logics than any one scan uses. + +A source that produces one value per event, like a plugin scalar, needs no data logic at all: a detector is a [](#StandardReadable), so register the signal with [](#StandardReadable.set_readable_format). + +`make_data_provider` is given the total `num_collections` for the scan, 0 meaning unbounded; the frame `period` (livetime + deadtime), resolved by `prepare()` from the trigger logic even when the [](#TriggerInfo) leaves `livetime` at 0; and the `flush_period` the [](#TriggerInfo) asked for. It must not start anything: only the providers the detector settles on get their `start()` called. Whatever it worked out that `start()` will need comes back alongside the provider, and the detector hands it straight back — a logic keeps no state of its own between the two. + +A detector may carry several data logics, but must not end up producing both kinds at once — those are mutually exclusive document kinds, and `prepare()` raises if it would. + +```{note} +Which logic serves is decided in `prepare()`, so it can change between runs — a detector whose file writer is switched off can fall back to its stats time series. Change it **between runs**, not inside one: a run's descriptor is emitted at its start, so switching part way through would leave the descriptor and the documents disagreeing. +``` + ## Conclusion We have seen how to make a [](#StandardDetector) and how the [](#DetectorTriggerLogic), [](#DetectorAcquireLogic), and [](#DetectorDataLogic) classes allow us to customise its behavior through composition. This separation of concerns makes it easy to: diff --git a/pyproject.toml b/pyproject.toml index b387728898..21836bc59b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ version_file = "src/ophyd_async/_version.py" [tool.pyright] typeCheckingMode = "standard" reportMissingImports = false # Ignore missing stubs in imported modules +reportMatchNotExhaustive = true [tool.pytest.ini_options] # Run pytest with all our checkers, and don't spam us with massive tracebacks on error diff --git a/src/ophyd_async/core/__init__.py b/src/ophyd_async/core/__init__.py index b9770c804b..2b3c19d7e6 100644 --- a/src/ophyd_async/core/__init__.py +++ b/src/ophyd_async/core/__init__.py @@ -11,8 +11,8 @@ soft_command, ) from ._data_providers import ( - ReadableDataProvider, - SignalDataProvider, + EventPageDataProvider, + PageableDataProvider, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -28,6 +28,7 @@ from ._detector import ( DetectorAcquireLogic, DetectorDataLogic, + DetectorLogic, DetectorTrigger, DetectorTriggerLogic, StandardDetector, @@ -52,7 +53,11 @@ OnOff, YesNo, ) -from ._flyable import FlyableLogic, FlyMotorInfo, StandardFlyable +from ._flyable import ( + FlyableLogic, + FlyMotorInfo, + StandardFlyable, +) from ._log import config_ophyd_async_logging, logger, set_handler from ._mock_signal_backend import MockSignalBackend from ._mock_signal_utils import ( @@ -99,10 +104,13 @@ Watcher, ) from ._readable import ( - ConfigSignal, - HintedSignal, + READABLE_FORMATS_KEY, + ROOT_DEVICE_KEY, + ReadableFormats, StandardReadable, StandardReadableFormat, + apply_readable_formats, + walk_readable_formats, ) from ._settings import Settings, SettingsProvider from ._signal import ( @@ -154,6 +162,7 @@ SubsetEnum, SupersetEnum, WatcherUpdate, + abstract_cached_property, error_if_none, gather_dict, get_dtype, @@ -271,12 +280,18 @@ def __getattr__(name): # Readable "StandardReadable", "StandardReadableFormat", + "ReadableFormats", + "walk_readable_formats", + "apply_readable_formats", + "READABLE_FORMATS_KEY", + "ROOT_DEVICE_KEY", # Detector "DetectorTrigger", "TriggerInfo", "DetectorTriggerLogic", "DetectorAcquireLogic", "DetectorDataLogic", + "DetectorLogic", "StandardDetector", # Path "PathInfo", @@ -290,9 +305,9 @@ def __getattr__(name): "AutoMaxIncrementingPathProvider", "UUIDFilenameProvider", # Data Providers - "ReadableDataProvider", + "EventPageDataProvider", + "PageableDataProvider", "StreamableDataProvider", - "SignalDataProvider", "StreamResourceInfo", "StreamResourceDataProvider", # Flyer @@ -323,6 +338,7 @@ def __getattr__(name): "make_datakey", "wait_for_connection", "Ignore", + "abstract_cached_property", "non_zero", "simulate_move", # Derived signal @@ -333,8 +349,6 @@ def __getattr__(name): "DerivedSignalFactory", "merge_gathered_dicts", # Back compat - delete before 1.0 - "ConfigSignal", - "HintedSignal", # Standard enums "EnabledDisabled", "EnableDisable", diff --git a/src/ophyd_async/core/_data_providers.py b/src/ophyd_async/core/_data_providers.py index ad2983cb59..65ea079499 100644 --- a/src/ophyd_async/core/_data_providers.py +++ b/src/ophyd_async/core/_data_providers.py @@ -1,61 +1,156 @@ from abc import abstractmethod -from collections.abc import AsyncIterator, Sequence -from dataclasses import dataclass +from collections.abc import AsyncIterator, Mapping, Sequence from typing import Any -from bluesky.protocols import Reading, StreamAsset +import numpy as np +from bluesky.protocols import StreamAsset from event_model import ComposeStreamResource, DataKey, StreamRange +from event_model.documents import PartialEventPage +from ._datatypes import Array1D from ._signal import SignalR, SignalW -from ._utils import ConfinedModel +from ._utils import ConfinedModel, error_if_none, gather_dict, get_dtype -class ReadableDataProvider: +class StreamableDataProvider: + collections_written_signal: SignalR[int] + @abstractmethod - async def make_datakeys(self) -> dict[str, DataKey]: - """Return a DataKey for each Readable that produces a Reading. + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + """Return a DataKey for each Signal that produces a Reading. Called before the first exposure is taken. + + :param collections_per_event: this should appear in the shape of each DataKey """ - @abstractmethod - async def make_readings(self) -> dict[str, Reading]: - """Read the Signals and return their values.""" + async def make_stream_docs( + self, collections_written: int, collections_per_event: int + ) -> AsyncIterator[StreamAsset]: + """Make StreamAsset documents up to the given index. + Default implementation is a no-op. Subclasses should override this + to emit actual StreamAsset documents. + """ + while False: + yield -@dataclass -class SignalDataProvider(ReadableDataProvider): - signal: SignalR - async def make_datakeys(self) -> dict[str, DataKey]: - return await self.signal.describe() +class PageableDataProvider: + """For bounded data held in a finite buffer, emitted as event pages. - async def make_readings(self) -> dict[str, Reading]: - return await self.signal.read(cached=False) + Used by data logics whose device must be told how many collections to + expect before it starts acquiring, because it holds a finite buffer that + has to be sized upfront: an areaDetector stats time series (`TSNumPoints`), + or a scaler whose MCA array length is the frame count. Such a device + produces its data as event pages at the end of an event rather than as + stream datums as it goes. + A step-scan event has a single collection window, so `make_pages` yields + one page of one event, which `StandardDetector.read` extracts to a single + reading; the same `make_pages` serves fly-scan collection. + """ -class StreamableDataProvider: collections_written_signal: SignalR[int] @abstractmethod async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: - """Return a DataKey for each Signal that produces a Reading. + """Return a DataKey for each field this provider produces. Called before the first exposure is taken. :param collections_per_event: this should appear in the shape of each DataKey """ - async def make_stream_docs( + @abstractmethod + def make_pages( self, collections_written: int, collections_per_event: int - ) -> AsyncIterator[StreamAsset]: - """Make StreamAsset documents up to the given index. + ) -> AsyncIterator[PartialEventPage]: + """Emit event pages for collections written since the last call. - Default implementation is a no-op. Subclasses should override this - to emit actual StreamAsset documents. + :param collections_written: how many collections have been written so far + :param collections_per_event: how many collections make up one event """ - while False: - yield + + +class EventPageDataProvider(PageableDataProvider): + """Emits a fixed-length array of collections as event pages. + + For a device that fills one array per field as it acquires, alongside an + array of per-collection timestamps and a signal counting how many + collections it has filled: an areaDetector stats time series, or a scaler's + MCA arrays. Each array is sliced into `collections_per_event`-length chunks, + one per event, and the last collection of an event supplies that event's + timestamp. + + :param data: datakey (already suffixed) to the array signal that backs it + :param collections_written_signal: counts the collections filled so far + :param timestamps: the acquisition time of each collection + """ + + def __init__( + self, + data: Mapping[str, SignalR[Array1D[Any]]], + collections_written_signal: SignalR[int], + timestamps: SignalR[Array1D[np.float64]], + ) -> None: + self.data = dict(data) + self.collections_written_signal = collections_written_signal + self.timestamps = timestamps + self.last_emitted = 0 + + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + return { + datakey: DataKey( + source=signal.source, + shape=[collections_per_event], + dtype="array", + dtype_numpy=get_dtype( + error_if_none(signal.datatype, f"{signal.source} has no datatype") + ).str, + ) + for datakey, signal in self.data.items() + } + + async def make_pages( + self, collections_written: int, collections_per_event: int + ) -> AsyncIterator[PartialEventPage]: + events = collections_written // collections_per_event + if events <= self.last_emitted: + return + new = range(self.last_emitted, events) + # Read every array and the per-collection timestamps in one parallel batch + read: dict[SignalR[Array1D[Any]], Array1D[Any]] = await gather_dict( + { + signal: signal.get_value() + for signal in (*self.data.values(), self.timestamps) + } + ) + stamps = read[self.timestamps] + # One timestamp per event: the acquisition time of that event's last + # collection. + event_times = [ + float(stamps[(event + 1) * collections_per_event - 1]) for event in new + ] + page: PartialEventPage = { + # TODO: numpy arrays are not in PartialEventPage's value type, but + # they round trip through the bundler fine; fix upstream in + # event-model rather than converting to lists here + "data": { # type: ignore[typeddict-item] + datakey: [ + read[signal][ + event * collections_per_event : (event + 1) + * collections_per_event + ] + for event in new + ] + for datakey, signal in self.data.items() + }, + "time": event_times, + "timestamps": dict.fromkeys(self.data, event_times), + } + self.last_emitted = events + yield page class StreamResourceInfo(ConfinedModel): diff --git a/src/ophyd_async/core/_detector.py b/src/ophyd_async/core/_detector.py index e56308cfa2..7ae51e8ad4 100644 --- a/src/ophyd_async/core/_detector.py +++ b/src/ophyd_async/core/_detector.py @@ -1,49 +1,44 @@ """Module which defines abstract classes to work with detectors.""" import asyncio -import functools import os import time +import warnings from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Sequence from dataclasses import dataclass from enum import Enum from functools import cached_property -from typing import cast +from typing import Any, Generic, TypeVar, cast from bluesky.protocols import ( Collectable, - Flyable, HasHints, Hints, - Preparable, Reading, Stageable, StreamAsset, Triggerable, - WritesStreamAssets, ) from event_model import DataKey +from event_model.documents import PartialEventPage from pydantic import Field, NonNegativeInt, PositiveInt, computed_field -from ophyd_async.core._log import logger - -from ._data_providers import ReadableDataProvider, StreamableDataProvider +from ._data_providers import ( + PageableDataProvider, + StreamableDataProvider, +) from ._device import Device -from ._protocol import AsyncConfigurable, AsyncReadable +from ._flyable import FlyableLogic, StandardFlyable +from ._readable import StandardReadable, StandardReadableFormat from ._settings import Settings -from ._signal import ( - SignalDict, - SignalR, - SignalRW, - observe_signals_value, - soft_signal_rw, -) +from ._signal import SignalDict, SignalR, SignalRW, observe_signals_value from ._status import AsyncStatus, WatchableAsyncStatus from ._utils import ( DEFAULT_TIMEOUT, ConfinedModel, WatcherUpdate, + abstract_cached_property, error_if_none, merge_gathered_dicts, ) @@ -104,6 +99,15 @@ class TriggerInfo(ConfinedModel): ) """What is the maximum timeout on waiting for an exposure""" + flush_period: float = Field(default=0.0, ge=0.0) + """How many seconds between flushes of the data being written. + + A data logic that buffers before it writes sizes that buffer from this and + the exposure period, so data appears at roughly this rate: an HDF chunk of + `flush_period / period` frames, say. Set it to the rate the plan collects at + so a fly scan sees each batch as it lands. 0 means leave whatever the + hardware is set to.""" + @computed_field @cached_property def number_of_collections(self) -> int: @@ -132,19 +136,22 @@ class DetectorTriggerLogic: - Multi-exposure collection batching Subclasses must implement the appropriate `prepare_*` method for any trigger - mode the detector supports, `get_deadtime` if it supports external - triggering, and `config_sigs` if the deadtime would vary according to - detector parameters. + mode the detector supports, and `get_deadtime` if it supports external + triggering. """ - def config_sigs(self) -> set[SignalR]: - """Return the signals that should appear in read_configuration.""" - return set() - def get_deadtime(self, config_values: SignalDict) -> float: """Return the deadtime in seconds for the detector. - :param config_values: the value of each signal in `config_sigs` + :param config_values: + The value of every `Signal` the detector reports as configuration, + i.e. everything registered with + [](#StandardReadableFormat.CONFIG_SIGNAL), including those declared + on children registered as [](#StandardReadableFormat.CHILD). A + logic that needs a signal must make sure it is declared as + configuration, rather than nominating it separately: the values + that determine deadtime are exactly the ones that ought to be + recorded for the scan anyway. """ raise NotImplementedError(self) @@ -185,14 +192,11 @@ async def default_trigger_info(self) -> TriggerInfo: raise NotImplementedError(self) -def _logic_supported(base_class, method) -> bool: +def _trigger_logic_supported(method) -> bool: # If the function that is bound in a subclass is the same as the function - # attached to the superclass, then the subclass has not overridden it, so - # this method is not supported by the subclass. - return method.__func__ is not getattr(base_class, method.__name__) - - -_trigger_logic_supported = functools.partial(_logic_supported, DetectorTriggerLogic) + # attached to DetectorTriggerLogic, then the subclass has not overridden it, + # so this method is not supported by the subclass. + return method.__func__ is not getattr(DetectorTriggerLogic, method.__name__) def _get_supported_triggers( @@ -279,7 +283,7 @@ def _all_the_same(collections_written: set[int]) -> int: async def _get_collections_written( - data_providers: Sequence[StreamableDataProvider], + data_providers: Sequence[StreamableDataProvider | PageableDataProvider], reducer: Callable[[set[int]], int] = _all_the_same, ) -> int: """Return a single collections_written value for the given providers. @@ -303,25 +307,83 @@ async def _get_collections_written( return 0 -class DetectorDataLogic: +#: A provider of either kind, as returned by +#: [](#DetectorDataLogic.make_data_provider). +_DataProvider = StreamableDataProvider | PageableDataProvider + +#: What a `DetectorDataLogic` works out in `make_data_provider` and needs back +#: in `start`. Use `None` for a logic whose `start` needs nothing. +DataCtxT = TypeVar("DataCtxT") + + +class DetectorDataLogic(Generic[DataCtxT]): """Abstract base class for detector data logic and handling. - Implementations must implement either prepare_unbounded for data sources - that work with step scans as well as flyscans, or prepare_single for those - that only work with step scans. + An implementation describes the data it would produce in + `make_data_provider`, and does the writes that make it happen in `start`. + The detector asks every data logic it has what it would make, decides which + providers it will use, and starts only those. + + What `make_data_provider` worked out and `start` needs is threaded between + them through an explicit context object rather than stored on the logic, as + `FlyableLogic` threads its own: the detector may make a provider and then + never start it, so a logic holding that state on itself would be left + carrying a plan that was abandoned. A logic whose `start` needs nothing from + `make_data_provider` uses `None` for the context. + + Whether the data is streamed or paged is the type of the provider that + `make_data_provider` returns, not a declaration on the logic, so one logic + may produce either depending on the scan. + + A source that produces a single value per event, like a plugin scalar, needs + no data logic at all: register its signal with + [](#StandardReadable.set_readable_format) instead. """ #: Add this suffix to the detector name to specify the datakey. These need to be #: different for each DetectorDataLogic added to a detector datakey_suffix: str = "" - async def prepare_single(self, datakey_name: str) -> ReadableDataProvider: - """Provider can only work for a single event.""" + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider | PageableDataProvider, DataCtxT] | None: + """Describe the data this logic would produce for this scan. + + This must not start anything acquiring: the detector may discard the + provider without calling `start`, and only starts the ones it will use. + Return a [](#StreamableDataProvider) for a source that can produce any + number of collections, or a [](#PageableDataProvider) for one holding a + finite buffer sized to `num_collections`, together with the context to + hand back to `start`. + + Return `None` to sit this scan out, for instance a finite buffer asked + for an unbounded number of collections, or a plugin that is switched + off. This is not an error and is not warned about. + + :param datakey_name: the detector name plus this logic's datakey_suffix + :param num_collections: total collections for the scan, 0 meaning + unbounded + :param period: how long each collection takes, livetime + deadtime, so + the provider can size its chunks or its buffer. 0 means "use + whatever is currently set on the hardware". + :param flush_period: how many seconds between flushes of the data being + written, so a logic that buffers can size that buffer. 0 means "use + whatever is currently set on the hardware". + """ raise NotImplementedError(self) - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: - """Provider can work for an unbounded number of collections.""" - raise NotImplementedError(self) + async def start(self, ctx: DataCtxT) -> None: + """Make the provider just returned by `make_data_provider` take data. + + Called only for the providers the detector will actually use, so this is + where writes that arm hardware or open files belong. + + :param ctx: the context returned by `make_data_provider`. + """ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: """Return the hinted streams.""" @@ -332,201 +394,466 @@ async def stop(self) -> None: return None -_data_logic_supported = functools.partial(_logic_supported, DetectorDataLogic) +@dataclass +class _Served: + """One data logic, the provider it made, and the context to start it with.""" + + logic: "DetectorDataLogic[Any]" + provider: _DataProvider + ctx: Any + + async def start(self) -> None: + await self.logic.start(self.ctx) @dataclass -class _PrepareCtx: - trigger_info: TriggerInfo - readable_data_providers: Sequence[ReadableDataProvider] - streamable_data_providers: Sequence[StreamableDataProvider] - collections_written: int +class _DetectorData: + """The data providers a detector has prepared, and what they were made for. + + Outlives a single prepare -> kickoff -> complete cycle: a step scan reuses an + open file across its points, and the RunEngine collects from a flyer after + `complete()` has returned. Cleared by `stage()`/`unstage()`. + """ + + serving: Sequence[_Served] + #: What the providers were made for, and so when they can be reused + collections_per_event: int + period: float + #: Where every provider was when the current event started, set by the + #: trigger or kickoff that started it. One number covers all of them only + #: because `_drop_shadowed` bans a detector serving both kinds at once: a + #: finite buffer and a file writer would be at different counts, and there + #: is no use case for mixing them. Lift that ban and this becomes one + #: baseline per provider. + initial_collections_written: int = 0 + + @property + def streamable(self) -> list[StreamableDataProvider]: + return [ + s.provider + for s in self.serving + if isinstance(s.provider, StreamableDataProvider) + ] + + @property + def pageable(self) -> list[PageableDataProvider]: + return [ + s.provider + for s in self.serving + if isinstance(s.provider, PageableDataProvider) + ] + + @property + def collectable(self) -> list[_DataProvider]: + """Every provider, whether it collects as stream datums or as pages.""" + return [s.provider for s in self.serving] @dataclass -class _KickoffCtx: +class _FlyCtx: + """What prepare() set up, threaded through kickoff() to complete().""" + trigger_info: TriggerInfo - data_providers: Sequence[StreamableDataProvider] - collections_written: int - collections_requested: int - is_last_kickoff: bool + #: Whether a finite buffer holds a previous point's data, so needs re-arming + needs_rearm: bool = False -class StandardDetector( - Device, - Stageable, - AsyncConfigurable, - AsyncReadable, - Triggerable, - Preparable, - Flyable, - Collectable, - WritesStreamAssets, - HasHints, -): - """Detector base class for step and fly scanning detectors. +class DetectorLogic(FlyableLogic[TriggerInfo, _FlyCtx]): + """Drive a detector through its trigger, acquire and data logics. - Aggregates trigger, arm, reading or stream logic together. + :param logics: + At most one [](#DetectorTriggerLogic), at most one + [](#DetectorAcquireLogic), and any number of [](#DetectorDataLogic), in + any order. Each object must fill exactly one of those roles. """ - # Logic for the detector - _trigger_logic: DetectorTriggerLogic | None = None - _acquire_logic: DetectorAcquireLogic | None = None - _data_logics: Sequence[DetectorDataLogic] = () - # Signals to include in read_configuration - _config_signals: Sequence[SignalR] = () - # Context produced by prepare, used by trigger and kickoff - _prepare_ctx: _PrepareCtx | None = None - # Context produced by kickoff, used by complete - _kickoff_ctx: _KickoffCtx | None = None - # The triggers that are supported by the trigger logic - _supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} - - # Report the number of events for the next kickoff - @cached_property - def events_to_kickoff(self) -> SignalRW[int]: - # TODO: only allow this to be revised down when trigger_info.number_of_events >1 - # and we have a reusable data provider - # requries https://github.com/bluesky/ophyd-async/issues/1119 - signal = soft_signal_rw(int) - # Name and parent this manually as `Device` doesn't know how to deal with cached - # properties - signal.parent = self - signal.set_name(f"{self.name}-events_to_kickoff") - return signal - - def add_detector_logics( - self, *logics: DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic + def __init__( + self, + *logics: "DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic[Any]", ) -> None: - """Add acquire, trigger or data logic to the detector. - - :param logic: The logic to add - """ + self.trigger_logic: DetectorTriggerLogic | None = None + self.acquire_logic: DetectorAcquireLogic | None = None + self.data_logics: tuple[DetectorDataLogic[Any], ...] = () + #: The trigger types the trigger logic implements + self.supported_triggers: set[DetectorTrigger] = {DetectorTrigger.INTERNAL} + #: Prefix for the datakeys the data logics produce, set from `Device.name` + self.datakey_prefix = "" + #: The data providers currently prepared, or None if there are none + self.data: _DetectorData | None = None for logic in logics: + # Each object must fill exactly one role. A single object that is both, + # say, an AcquireLogic and a DataLogic would otherwise register as only + # the first match and have its other role silently dropped, so require + # separate objects even when one device's concerns live on one control. + roles = [ + base + for base in ( + DetectorTriggerLogic, + DetectorAcquireLogic, + DetectorDataLogic, + ) + if isinstance(logic, base) + ] + if len(roles) > 1: + names = ", ".join(base.__name__ for base in roles) + raise TypeError( + f"{type(logic).__name__} is both {names}; pass a separate object " + "for each logic role" + ) if isinstance(logic, DetectorTriggerLogic): - if self._trigger_logic is not None: + if self.trigger_logic is not None: raise RuntimeError("Detector already has trigger logic") - self._trigger_logic = logic - # Store the triggers that are supported - self._supported_triggers = _get_supported_triggers(logic) - # Add the config signals it needs - self.add_config_signals(*logic.config_sigs()) + self.trigger_logic = logic + self.supported_triggers = _get_supported_triggers(logic) elif isinstance(logic, DetectorAcquireLogic): - if self._acquire_logic is not None: + if self.acquire_logic is not None: raise RuntimeError("Detector already has acquire logic") - self._acquire_logic = logic + self.acquire_logic = logic elif isinstance(logic, DetectorDataLogic): - self._data_logics = (*self._data_logics, logic) + self.data_logics += (logic,) else: raise TypeError(f"Unknown logic type: {type(logic)}") + #: Whether the trigger logic can calculate a deadtime + self.supports_deadtime = self.trigger_logic is not None and ( + _trigger_logic_supported(self.trigger_logic.get_deadtime) + ) - def add_config_signals(self, *signals: SignalR) -> None: - """Add a signal to read_configuration(). + @property + def prepared_data(self) -> _DetectorData: + """The prepared data providers, raising if prepare() has not run.""" + return error_if_none( + self.data, f"{self.datakey_prefix}: prepare() must be called first" + ) - :param sig: The signal to add + def with_device(self, name: str = "") -> "StandardDetector": + """Wrap this logic in an ephemeral `StandardDetector` for use in a plan.""" + return _LogicDetector(self, name=name) + + def get_deadtime(self, config_values: SignalDict) -> float: + """Return the deadtime the trigger logic calculates from `config_values`.""" + trigger_logic = error_if_none(self.trigger_logic, "No trigger logic") + return trigger_logic.get_deadtime(config_values) + + def get_hinted_fields(self) -> Iterator[Sequence[str]]: + """Yield the hinted fields of each data logic that is producing data. + + Before anything is prepared every data logic is asked, since `hints` is + read outside a scan too; once prepared, only the ones actually serving + it, so a logic that sat the scan out does not hint at data nobody will + produce. """ - self._config_signals = (*self._config_signals, *signals) + logics = ( + [s.logic for s in self.data.serving] + if self.data + else list(self.data_logics) + ) + for dl in logics: + if fields := dl.get_hinted_fields(self._datakey_name(dl)): + yield fields + + async def on_stage(self) -> None: + """Stop data production and make sure the detector is idle.""" + self.data = None + coros: list[Awaitable] = [dl.stop() for dl in self.data_logics] + if self.acquire_logic: + coros.append(self.acquire_logic.ensure_ready()) + await asyncio.gather(*coros) - async def get_trigger_deadtime( - self, settings: Settings | None = None - ) -> tuple[set[DetectorTrigger], float | None]: - """Get supported trigger types and deadtime for the detector. + async def on_unstage(self) -> None: + """Stop data production and the detector at the end of a scan.""" + self.data = None + coros: list[Awaitable] = [dl.stop() for dl in self.data_logics] + if self.acquire_logic: + coros.append(self.acquire_logic.ensure_stopped()) + await asyncio.gather(*coros) - :param settings: Optional settings to use when getting configuration values - :return: Tuple of supported trigger types and deadtime in seconds + async def on_prepare(self, value: TriggerInfo) -> _FlyCtx: + """Set the trigger logic up and make the data providers for this scan. + + :param value: TriggerInfo describing how to trigger the detector """ - if self._trigger_logic and _trigger_logic_supported( - self._trigger_logic.get_deadtime + await self._prepare_trigger_logic(value) + # This must come after preparing the trigger logic, as the period is read + # back from it and may determine the datatype of the streams + value = await self._resolve_period(value) + await self._prepare_data(value) + # External triggering can start acquiring now + if self.acquire_logic and value.trigger is not DetectorTrigger.INTERNAL: + await self.acquire_logic.start_acquiring() + return _FlyCtx(trigger_info=value) + + async def on_kickoff(self, ctx: _FlyCtx) -> _FlyCtx: + """Start the fly scan, noting where the providers had got to.""" + collectable = self.prepared_data.collectable + if not collectable: + raise ValueError( + f"Detector {self.datakey_prefix} has no collectable data, " + "so cannot kickoff" + ) + # Read live: a buffer accumulates across the kickoffs of one prepare, so + # it is not back at zero the way a re-armed one is in on_trigger() + self.prepared_data.initial_collections_written = await _get_collections_written( + collectable + ) + # External triggering has already started; internal starts now + if self.acquire_logic and ctx.trigger_info.trigger is DetectorTrigger.INTERNAL: + await self.acquire_logic.start_acquiring() + return ctx + + def on_complete(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: + """Wait for the scan to finish, reporting collections written as progress.""" + return self._wait_for_collections( + trigger_info=ctx.trigger_info, + collections_requested=ctx.trigger_info.number_of_collections, + watcher_divisor=ctx.trigger_info.collections_per_event, + ) + + async def on_trigger(self, ctx: _FlyCtx) -> AsyncIterator[WatcherUpdate]: + """Take one event's worth of exposures and wait for them to be written.""" + if ctx.trigger_info.number_of_events != 1: + raise ValueError( + "trigger() is not supported for multiple events, the detector was " + f"prepared with number_of_events={ctx.trigger_info.number_of_events}." + ) + # A finite buffer holds one event at a time, so it is re-armed for each + # point of a step scan; a streaming provider carries on from where it was. + # prepare() armed it for the first point, so re-arming here as well would + # erase and re-erase it before a frame arrived. + if ctx.needs_rearm: + await self._prepare_data(ctx.trigger_info) + ctx.needs_rearm = True + data = self.prepared_data + # Take the baseline before acquisition starts, or frames written between + # the two would be counted towards this event. A re-armed buffer starts + # from zero, where a streaming provider continues from wherever it has got + # to -- which is not where prepare left it, since a step scan triggers many + # times against one prepare. + data.initial_collections_written = ( + 0 if data.pageable else await _get_collections_written(data.collectable) + ) + # External triggering started acquiring at prepare and is sitting waiting + # for its trigger source, so starting it again would restart it; internal + # starts here, as it does in on_kickoff() + if self.acquire_logic and ctx.trigger_info.trigger is DetectorTrigger.INTERNAL: + await self.acquire_logic.start_acquiring() + async for update in self._wait_for_collections( + trigger_info=ctx.trigger_info, + collections_requested=ctx.trigger_info.collections_per_event, ): - config_values = SignalDict() - for sig in self._trigger_logic.config_sigs(): - if settings and sig in settings: - # Use value from settings if it is in there - # cast to a SignalRW because settings can only contain those - config_values[sig] = settings[cast(SignalRW, sig)] - else: - # Get the value live - config_values[sig] = await sig.get_value() - deadtime = self._trigger_logic.get_deadtime(config_values) - else: - deadtime = None - return self._supported_triggers, deadtime + yield update - @AsyncStatus.wrap - async def stage(self) -> None: - """Make sure the detector is idle and ready to be used.""" - coros = [data_logic.stop() for data_logic in self._data_logics] - if self._acquire_logic: - coros.append(self._acquire_logic.ensure_ready()) - await asyncio.gather(*coros) - self._prepare_ctx = None - self._kickoff_ctx = None - await self.events_to_kickoff.set(0) - - async def _update_prepare_context(self, trigger_info: TriggerInfo) -> None: - # The only thing that would stop us being able to reuse a provider is - # if the collections_per_event changes, as that would change the - # StreamResource shape. All other TriggerInfo parameters (exposures, livetime, - # etc.) don't affect the data provider configuration. - if ( - self._prepare_ctx - and self._prepare_ctx.trigger_info.collections_per_event - == trigger_info.collections_per_event + async def default_trigger_info(self) -> TriggerInfo: + """The TriggerInfo to prepare with when a plan calls trigger() directly.""" + # Opt-in: set OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES to have trigger() read + # back current hardware state (e.g. num_images) via default_trigger_info() + # instead of always falling back to TriggerInfo(). See ADR 0013. + # TODO: flip default to YES and remove this guard in a future PR once + # downstream code has had time to implement default_trigger_info(). + preserve_state = ( + os.environ.get("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "NO").upper() == "YES" + ) + if preserve_state and self.trigger_logic is not None: + if not _trigger_logic_supported(self.trigger_logic.default_trigger_info): + raise RuntimeError( + "OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but " + f"'{self.datakey_prefix}' has no default_trigger_info() - " + "implement default_trigger_info() on your DetectorTriggerLogic " + "subclass or unset the environment variable." + ) + return await self.trigger_logic.default_trigger_info() + return TriggerInfo() + + def _datakey_name(self, dl: "DetectorDataLogic[Any]") -> str: + return self.datakey_prefix + dl.datakey_suffix + + async def _prepare_trigger_logic(self, value: TriggerInfo) -> None: + if self.trigger_logic and _trigger_logic_supported( + self.trigger_logic.prepare_exposures_per_collection ): - # Reuse the existing data providers - readable_data_providers = self._prepare_ctx.readable_data_providers - streamable_data_providers = self._prepare_ctx.streamable_data_providers - else: - # Stop the existing providers if there is a context and make new ones - if self._prepare_ctx: - for data_logic in self._data_logics: - await data_logic.stop() - # Setup the data logic for the right number of collections - streamable_coros: list[Awaitable[StreamableDataProvider]] = [] - readable_coros: list[Awaitable[ReadableDataProvider]] = [] - for dl in self._data_logics: - if _data_logic_supported(dl.prepare_unbounded): - streamable_coros.append( - dl.prepare_unbounded(self.name + dl.datakey_suffix) + # If we can do multiple exposures per collection then set it up + # even if there was only 1 requested to clear previous settings + await self.trigger_logic.prepare_exposures_per_collection( + value.exposures_per_collection + ) + elif value.exposures_per_collection != 1: + raise ValueError( + "Multiple exposures per collection not supported by " + f"'{self.datakey_prefix}'" + ) + if value.trigger not in self.supported_triggers: + format_triggers = ", ".join(sorted(t.name for t in self.supported_triggers)) + raise ValueError( + f"Trigger type {value.trigger} not supported by " + f"'{self.datakey_prefix}', supported types are: [{format_triggers}]" + ) + if self.trigger_logic: + match value.trigger: + case DetectorTrigger.INTERNAL: + await self.trigger_logic.prepare_internal( + num=value.number_of_exposures, + livetime=value.livetime, + deadtime=value.deadtime, ) - elif _data_logic_supported(dl.prepare_single): - if trigger_info.number_of_collections == 1: - readable_coros.append( - dl.prepare_single(self.name + dl.datakey_suffix) - ) - else: - logger.warning( - f"DataLogic {dl} only supports a single collection, but " - "the detector was prepared for " - f"{trigger_info.number_of_collections} collections." - ) - else: - msg = f"DataLogic hasn't overridden any prepare_* methods {dl}" - raise RuntimeError(msg) - streamable_data_providers, readable_data_providers = await asyncio.gather( - asyncio.gather(*streamable_coros), - asyncio.gather(*readable_coros), + case DetectorTrigger.EXTERNAL_EDGE: + await self.trigger_logic.prepare_edge( + num=value.number_of_exposures, + livetime=value.livetime, + ) + case DetectorTrigger.EXTERNAL_LEVEL: + await self.trigger_logic.prepare_level( + num=value.number_of_exposures, + ) + elif value.livetime != 0.0 or value.deadtime != 0.0: + raise ValueError( + f"Detector {self.datakey_prefix} has no trigger logic, so cannot set " + "livetime or deadtime" ) - # Stash the prepare context so we can use it in trigger/kickoff - self._prepare_ctx = _PrepareCtx( - trigger_info=trigger_info, - streamable_data_providers=streamable_data_providers, - readable_data_providers=readable_data_providers, - collections_written=await _get_collections_written( - streamable_data_providers - ), + async def _resolve_period(self, value: TriggerInfo) -> TriggerInfo: + # A livetime of 0 means "use whatever the detector currently has set". A + # data logic that sizes chunks or a buffer by the exposure period needs a + # real value, so read the current livetime and deadtime back from the + # trigger logic and fill them in. This runs after the trigger logic has been + # prepared, so the hardware already holds the values we read. + if ( + value.livetime == 0.0 + and self.trigger_logic is not None + and _trigger_logic_supported(self.trigger_logic.default_trigger_info) + ): + current = await self.trigger_logic.default_trigger_info() + if current.livetime or current.deadtime: + value = value.model_copy( + update={ + "livetime": current.livetime, + "deadtime": current.deadtime, + } + ) + return value + + async def _prepare_data(self, trigger_info: TriggerInfo) -> _DetectorData: + """Make sure the data providers are ready for this scan.""" + period = trigger_info.livetime + trigger_info.deadtime + # Providers are reused while the scan they were made for is unchanged, so + # that a step scan does not reopen its file on every point. Both parts of + # the key are baked into a streaming provider when it is made: the shape + # from collections_per_event and the chunking from the period. A finite + # buffer is re-made either way, since re-making is what re-arms it. + previous = self.data + if ( + previous is not None + and previous.collections_per_event == trigger_info.collections_per_event + and previous.period == period + ): + serving = await self._rearm_bounded(previous, trigger_info) + else: + serving = await self._make_serving(trigger_info) + self.data = _DetectorData( + serving=serving, + collections_per_event=trigger_info.collections_per_event, + period=period, + ) + return self.data + + async def _make_serving(self, trigger_info: TriggerInfo) -> Sequence[_Served]: + """Ask every data logic what it would make, and start the ones we use.""" + # Stop what is running before anything new is made, since a logic that + # cannot describe its data without opening its file does so here + if self.data is not None: + await asyncio.gather(*(s.logic.stop() for s in self.data.serving)) + cpe = trigger_info.collections_per_event + made = await asyncio.gather( + *(self._make_one(dl, trigger_info) for dl in self.data_logics) ) + # A logic returns None to sit this scan out, e.g. a finite buffer asked + # for an unbounded number of collections, or a plugin that is switched off + serving = [s for s in made if s is not None] + serving = await self._drop_shadowed(serving, cpe) + await asyncio.gather(*(s.start() for s in serving)) + return serving + + async def _make_one( + self, dl: "DetectorDataLogic[Any]", trigger_info: TriggerInfo + ) -> _Served | None: + made = await dl.make_data_provider( + self._datakey_name(dl), + trigger_info.number_of_collections, + trigger_info.livetime + trigger_info.deadtime, + trigger_info.flush_period, + ) + if made is None: + return None + provider, ctx = made + return _Served(logic=dl, provider=provider, ctx=ctx) - async def _wait_for_index( + async def _drop_shadowed( + self, + serving: Sequence[_Served], + collections_per_event: int, + ) -> Sequence[_Served]: + """Drop finite buffers whose datakeys the stream assets already carry. + + A detector cannot produce both stream assets and event pages: the bundler + treats them as mutually exclusive. Carrying both logics is still useful, + because the same quantity can be written durably into the file *and* read + from a plugin's buffer -- an areaDetector stats total, say, which the HDF + writer pulls in as an NDAttribute. Where every key a finite buffer would + produce is also written to the file, the file wins and the buffer sits the + scan out. Anything else is a conflict, so it raises. + """ + pageable = [s for s in serving if isinstance(s.provider, PageableDataProvider)] + streamable = [ + s.provider + for s in serving + if isinstance(s.provider, StreamableDataProvider) + ] + if not (pageable and streamable): + return serving + stream_keys: set[str] = set() + for dp in streamable: + stream_keys |= set(await dp.make_datakeys(collections_per_event)) + shadowed = [] + for s in pageable: + unshadowed = ( + set(await s.provider.make_datakeys(collections_per_event)) - stream_keys + ) + if unshadowed: + raise TypeError( + f"Detector {self.datakey_prefix} would produce " + f"{sorted(unshadowed)} as event pages and the rest of its data " + "as stream assets; these cannot be combined on one detector" + ) + shadowed.append(s.logic) + # Nothing has been started yet, so a shadowed logic needs no stopping + return [s for s in serving if s.logic not in shadowed] + + async def _rearm_bounded( + self, previous: _DetectorData, trigger_info: TriggerInfo + ) -> Sequence[_Served]: + """Re-make and re-start the finite buffers, keeping everything else.""" + serving: list[_Served] = [] + for s in previous.serving: + if isinstance(s.provider, PageableDataProvider): + await s.logic.stop() + rearmed = await self._make_one(s.logic, trigger_info) + if rearmed is None: + continue + await rearmed.start() + serving.append(rearmed) + else: + serving.append(s) + return serving + + async def _wait_for_collections( self, - data_providers: Sequence[StreamableDataProvider], trigger_info: TriggerInfo, - initial_collections_written: int, collections_requested: int, - wait_for_idle: bool, watcher_divisor: int = 1, ) -> AsyncIterator[WatcherUpdate]: + data = self.prepared_data + data_providers = data.collectable + # Where this event started from, recorded by the trigger or kickoff that + # started it, since by now the providers have moved on + initial_collections_written = data.initial_collections_written start_time = time.monotonic() current_collections_written = { dp.collections_written_signal: initial_collections_written @@ -541,7 +868,7 @@ async def _wait_for_index( current_collections_written[sig] = value collections_written = min(current_collections_written.values()) yield WatcherUpdate( - name=self.name, + name=self.datakey_prefix, current=collections_written // watcher_divisor, initial=initial_collections_written // watcher_divisor, target=target_collections_written // watcher_divisor, @@ -551,242 +878,328 @@ async def _wait_for_index( ) if collections_written >= target_collections_written: break - if self._acquire_logic and wait_for_idle: - await self._acquire_logic.wait_for_idle() + if self.acquire_logic: + await self.acquire_logic.wait_for_idle() - @AsyncStatus.wrap - async def prepare(self, value: TriggerInfo) -> None: - """Prepare the detector for a number of triggers. - :param value: TriggerInfo describing how to trigger the detector +def _config_signals(device: Device) -> set[SignalR]: + """Return every `Signal` contributing to a Device's `read_configuration()`. + + Recurses into children registered as [](#StandardReadableFormat.CHILD), so + a Device that registers a sub-Device whole picks up whatever that + sub-Device declares as configuration. A Device that is not a + `StandardReadable` contributes nothing. + + Unlike `walk_config_signals` this reads the registry rather than calling + `read_configuration()`, so it does no I/O and includes read-only `SignalR`s + as well as `SignalRW`s -- `ADBaseIO.model` is a `SignalR`, and it is + exactly the signal an areaDetector trigger logic needs for deadtime. + + Deliberately private and unexported: #1367 restructures detector logic, so + what the public shape of this should be is not settled yet. + """ + signals: set[SignalR] = set() + if not isinstance(device, StandardReadable): + return signals + for child, format in device.get_readable_formats().items(): + match format: + case StandardReadableFormat.CONFIG_SIGNAL: + # set_readable_format only accepts a SignalR for a signal format + signals.add(cast(SignalR, child)) + case StandardReadableFormat.CHILD: + signals |= _config_signals(child) + case ( + StandardReadableFormat.HINTED_SIGNAL + | StandardReadableFormat.UNCACHED_SIGNAL + | StandardReadableFormat.HINTED_UNCACHED_SIGNAL + ): + pass + return signals + + +class StandardDetector( + StandardReadable, + StandardFlyable[TriggerInfo, _FlyCtx], + Stageable, + Triggerable, + Collectable, + HasHints, +): + """Detector base class for step and fly scanning detectors. + + Aggregates trigger, acquire and data logic together in a + [](#DetectorLogic), which a subclass builds in its `__init__` and returns + from `logic`: + + ```python + class MyDetector(StandardDetector): + def __init__(self, prefix: str, name: str = "") -> None: + self.driver = MyDriverIO(prefix) + self._logic = DetectorLogic( + MyTriggerLogic(self.driver), + MyAcquireLogic(self.driver), + ) + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic + ``` + + For an ad-hoc detector, [](#DetectorLogic.with_device) does the same thing + without a subclass. + + Signals read during a step scan are registered with + [](#StandardReadable.set_readable_format), exactly as on any other + `StandardReadable`; data produced by a `DetectorDataLogic` is added on top + of those in `read()` and `describe()`. + + `read()` and `describe()` require `prepare()` to have run, and raise + otherwise, so that a descriptor can never be emitted without the detector's + data keys. `trigger()` prepares implicitly, so a step scan never has to do + it explicitly. `read_configuration()` and `describe_configuration()` have no + such requirement. + + `WritesStreamAssets` (and its `collect_asset_docs`) and + `EventPageCollectable` (and its `collect_pages`) are *not* inherited: which + of them applies depends on what the data logics produce for a given scan, so + whichever it is gets bound as an instance attribute by `prepare()`. + """ + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # What the data logics produce, gathered alongside the registered + # children, so the verb methods themselves need no overriding + self._read_funcs += (self._data_read,) + self._describe_funcs += (self._data_describe,) + self._hint_funcs += (self._data_hints,) + + @abstract_cached_property + def logic(self) -> DetectorLogic: + """The logic that drives this detector, built in the subclass `__init__`.""" + raise NotImplementedError + + def set_name(self, name: str, *, child_name_separator: str | None = None) -> None: + super().set_name(name, child_name_separator=child_name_separator) + # The data logics name their datakeys after the Device + self.logic.datakey_prefix = name + + # Back compat - delete before 1.0 + def add_config_signals(self, *signals: SignalR) -> None: + """Add a signal to read_configuration(). + + :param sig: The signal to add """ - if self._trigger_logic and _trigger_logic_supported( - self._trigger_logic.prepare_exposures_per_collection + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(signal, StandardReadableFormat" + ".CONFIG_SIGNAL)` instead of `add_config_signals(signal)`" + ), + stacklevel=2, + ) + for signal in signals: + self.set_readable_format(signal, StandardReadableFormat.CONFIG_SIGNAL) + + async def get_trigger_deadtime( + self, settings: Settings | None = None + ) -> tuple[set[DetectorTrigger], float | None]: + """Get supported trigger types and deadtime for the detector. + + The trigger logic is given the value of every signal this detector + reports as configuration, rather than nominating the ones it wants: + the values that determine deadtime are exactly the ones that ought to + be recorded for the scan anyway. + + :param settings: Optional settings to use when getting configuration values + :return: Tuple of supported trigger types and deadtime in seconds + """ + deadtime = None + if self.logic.supports_deadtime: + config_values = SignalDict() + to_read: list[SignalR] = [] + for sig in _config_signals(self): + if settings and sig in settings: + # Use value from settings if it is in there + # cast to a SignalRW because settings can only contain those + config_values[sig] = settings[cast(SignalRW, sig)] + else: + to_read.append(sig) + # Read live values concurrently: this is every configuration signal + # rather than a handful the logic named, so doing it in series would + # be one round trip per signal + for sig, value in zip( + to_read, + await asyncio.gather(*(sig.get_value() for sig in to_read)), + strict=True, + ): + config_values[sig] = value + deadtime = self.logic.get_deadtime(config_values) + return self.logic.supported_triggers, deadtime + + @AsyncStatus.wrap + async def prepare(self, value: TriggerInfo) -> None: + """Set the detector up for a scan, as described by the TriggerInfo.""" + await super().prepare(value) + # Bind the collect verb matching what the data logics will produce, as a + # real instance attribute so the bundler's isinstance checks + # (WritesStreamAssets vs EventPageCollectable) see exactly one of them. + # Those resolve with inspect.getattr_static on Python 3.12+, which does not + # call __getattr__, so a dynamic hook would be invisible; both names are + # reserved by Device, hence object.__setattr__. + # TODO: all of this goes once the run bundler picks the collect verb from + # what a device actually produced rather than from its static type - a + # change in progress upstream. _collect_asset_docs and _collect_pages then + # become plain public methods and a detector declares both. See ADR 0023. + data = self.logic.data + for verb, method, wanted in ( + ("collect_asset_docs", self._collect_asset_docs, data and data.streamable), + ("collect_pages", self._collect_pages, data and data.pageable), ): - # If we can do multiple exposures per collection then set it up - # even if there was only 1 requested to clear previous settings - await self._trigger_logic.prepare_exposures_per_collection( - value.exposures_per_collection - ) - elif value.exposures_per_collection != 1: - raise ValueError( - f"Multiple exposures per collection not supported by {self}" - ) - # Setup the trigger logic for the right number of exposures - if value.trigger not in self._supported_triggers: - format_triggers = ", ".join( - sorted(t.name for t in self._supported_triggers) - ) - raise ValueError( - f"Trigger type {value.trigger} not supported by '{self.name}', " - f"supported types are: [{format_triggers}]" - ) - if self._trigger_logic: - match value.trigger: - case DetectorTrigger.INTERNAL: - await self._trigger_logic.prepare_internal( - num=value.number_of_exposures, - livetime=value.livetime, - deadtime=value.deadtime, - ) - case DetectorTrigger.EXTERNAL_EDGE: - await self._trigger_logic.prepare_edge( - num=value.number_of_exposures, - livetime=value.livetime, - ) - case DetectorTrigger.EXTERNAL_LEVEL: - await self._trigger_logic.prepare_level( - num=value.number_of_exposures, - ) - elif value.livetime != 0.0 or value.deadtime != 0.0: - raise ValueError( - f"Detector {self.name} has no trigger logic, so cannot set livetime or " - "deadtime" - ) - # NOTE: this section must come after preparing the trigger logic as we may - # use parameters from it to determine datatype for the streams - await self._update_prepare_context(value) - # Tell people how many collections we will acquire for - await self.events_to_kickoff.set(value.number_of_events) - # External triggering can start acquiring now - if self._acquire_logic and value.trigger != DetectorTrigger.INTERNAL: - await self._acquire_logic.start_acquiring() + if wanted: + object.__setattr__(self, verb, method) + elif verb in self.__dict__: + object.__delattr__(self, verb) @WatchableAsyncStatus.wrap async def trigger(self) -> AsyncIterator[WatcherUpdate[int]]: """Trigger a single exposure. - If [`prepare()`](#StandardDetector.prepare) has not been called since - the last [`stage()`](#StandardDetector.stage), an implicit prepare is - performed. When [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` + If [`prepare()`](#StandardDetector.prepare) has not been called since the + last `stage()`, an implicit prepare is performed. When + [](#OPHYD_ASYNC_PRESERVE_DETECTOR_STATE) is `YES` [](#DetectorTriggerLogic.default_trigger_info) is called to read current - hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is - used. + hardware state; otherwise a bare [`TriggerInfo()`](#TriggerInfo) is used. """ - if self._prepare_ctx is None: - # Opt-in: set OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES to have - # trigger() read back current hardware state (e.g. num_images) via - # default_trigger_info() instead of always falling back to TriggerInfo(). - # See ADR 0013 for rationale. - # TODO: flip default to YES and remove this guard in a future PR once - # downstream code has had time to implement default_trigger_info(). - preserve_state = ( - os.environ.get("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "NO").upper() - == "YES" - ) - if preserve_state and self._trigger_logic is not None: - if not _trigger_logic_supported( - self._trigger_logic.default_trigger_info - ): - raise RuntimeError( - f"OPHYD_ASYNC_PRESERVE_DETECTOR_STATE=YES is set but " - f"'{self.name}' has no default_trigger_info() - implement " - "default_trigger_info() on your DetectorTriggerLogic subclass " - "or unset the environment variable." - ) - trigger_info = await self._trigger_logic.default_trigger_info() - else: - trigger_info = TriggerInfo() - await self.prepare(trigger_info) - else: - # Check the one that was provided is suitable for triggering - trigger_info = self._prepare_ctx.trigger_info - if trigger_info.number_of_events != 1: - msg = ( - "trigger() is not supported for multiple events, the detector was " - f"prepared with number_of_events={trigger_info.number_of_events}." - ) - raise ValueError(msg) - # Ensure the data provider is still usable - await self._update_prepare_context(trigger_info) - ctx = error_if_none(self._prepare_ctx, "Prepare should have been run") - # Start the detector acquiring and wait for it to finish. - if self._acquire_logic: - await self._acquire_logic.start_acquiring() - async for update in self._wait_for_index( - data_providers=ctx.streamable_data_providers, - trigger_info=ctx.trigger_info, - initial_collections_written=ctx.collections_written, - collections_requested=ctx.trigger_info.collections_per_event, - watcher_divisor=1, - wait_for_idle=True, - ): + if self.logic.data is None: + await self.prepare(await self.logic.default_trigger_info()) + async for update in self.logic.on_trigger(self._prepared_fly_ctx): yield update - @AsyncStatus.wrap - async def kickoff(self): - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - if not ctx.streamable_data_providers: - raise ValueError( - f"Detector {self.name} is not streamable, so cannot kickoff" - ) - collections_written, events_to_kickoff = await asyncio.gather( - _get_collections_written(ctx.streamable_data_providers), - self.events_to_kickoff.get_value(), - ) - collections_requested = ( - events_to_kickoff * ctx.trigger_info.collections_per_event - ) - last_requested_collection = collections_written + collections_requested - last_expected_collection = ( - ctx.collections_written + ctx.trigger_info.number_of_collections - ) - if last_requested_collection > last_expected_collection: - msg = ( - f"Kickoff requested {collections_written}:{last_requested_collection}, " - f"but detector was only prepared up to {last_expected_collection}" - ) - raise RuntimeError(msg) - self._kickoff_ctx = _KickoffCtx( - trigger_info=ctx.trigger_info, - data_providers=ctx.streamable_data_providers, - collections_written=collections_written, - collections_requested=collections_requested, - is_last_kickoff=last_requested_collection == last_expected_collection, - ) - # External triggering has already started; internal starts now - if self._acquire_logic and ctx.trigger_info.trigger == DetectorTrigger.INTERNAL: - await self._acquire_logic.start_acquiring() + async def _data_read(self) -> dict[str, Reading]: + """What the data logics contribute to `read()`. - @WatchableAsyncStatus.wrap - async def complete(self): - ctx = error_if_none(self._kickoff_ctx, "Kickoff not called") - async for update in self._wait_for_index( - data_providers=ctx.data_providers, - trigger_info=ctx.trigger_info, - initial_collections_written=ctx.collections_written, - collections_requested=ctx.collections_requested, - wait_for_idle=ctx.is_last_kickoff, - watcher_divisor=ctx.trigger_info.collections_per_event, - ): - yield update + Raises if nothing has been prepared, since an unprepared detector has no + data keys and would otherwise emit a descriptor missing its data. + `trigger()` prepares implicitly, so a step scan never sees this. + `read_configuration()` is unaffected. - async def describe_configuration(self) -> dict[str, DataKey]: + Bounded providers hold a single-event page for this step-scan point, + which `_pageable_readings` extracts back to a reading. That extraction + lives here rather than on the provider so a provider cannot override it. + Streamable providers produce their data through `collect_asset_docs` + rather than `read()`, so they contribute nothing here. + """ + data = self.logic.prepared_data + cpe = data.collections_per_event return await merge_gathered_dicts( - sig.describe() for sig in self._config_signals + [self._pageable_readings(pdp, cpe) for pdp in data.pageable] ) - async def read_configuration(self) -> dict[str, Reading]: - return await merge_gathered_dicts(sig.read() for sig in self._config_signals) + async def _data_describe(self) -> dict[str, DataKey]: + """What the data logics contribute to `describe()`. - async def describe(self) -> dict[str, DataKey]: - ctx = error_if_none(self._prepare_ctx, "Prepare not run") - # Readable and Streamable data providers produce data during read - coros = [dp.make_datakeys() for dp in ctx.readable_data_providers] + [ - dp.make_datakeys(ctx.trigger_info.collections_per_event) - for dp in ctx.streamable_data_providers - ] - return await merge_gathered_dicts(coros) + Every provider describes its shape for a step scan, whichever way it + goes on to produce the data. Raises if nothing has been prepared, as + `_data_read` does. + """ + data = self.logic.prepared_data + cpe = data.collections_per_event + return await merge_gathered_dicts( + [dp.make_datakeys(cpe) for dp in data.collectable] + ) async def describe_collect(self) -> dict[str, DataKey]: - ctx = error_if_none(self._prepare_ctx, "Prepare not run") - # Only streamable data providers produce data during collect + data = self.logic.prepared_data + # Streamable providers collect stream datums, bounded providers collect pages coros = [ - dp.make_datakeys(ctx.trigger_info.collections_per_event) - for dp in ctx.streamable_data_providers + dp.make_datakeys(data.collections_per_event) for dp in data.collectable ] return await merge_gathered_dicts(coros) - @property - def hints(self) -> Hints: - fields: list[str] = [] - for dl in self._data_logics: - fields.extend(dl.get_hinted_fields(self.name + dl.datakey_suffix)) - return Hints(fields=fields) - - async def read(self) -> dict[str, Reading]: - ctx = error_if_none(self._prepare_ctx, "Prepare not called") - return await merge_gathered_dicts( - dp.make_readings() for dp in ctx.readable_data_providers - ) + def _data_hints(self) -> Iterator[Hints]: + """The data logics' hinted fields, alongside the children's.""" + for fields in self.logic.get_hinted_fields(): + yield {"fields": list(fields)} + + async def _pageable_readings( + self, provider: PageableDataProvider, collections_per_event: int + ) -> dict[str, Reading]: + """Derive readings from a bounded provider's pages for a step-scan read. + + A step-scan prepare has a single event, so the page holds one event + whose per-key value is the `collections_per_event`-length array; this + extracts to a single reading per key. More than one value in a page + would mean `read()` was asked for a single reading from a multi-event + buffer, where the answer is ambiguous, so raise rather than silently + keep the last. + """ + collections_written = await provider.collections_written_signal.get_value() + readings: dict[str, Reading] = {} + async for page in provider.make_pages( + collections_written, collections_per_event + ): + times = page["time"] + for key, values in page["data"].items(): + if len(values) != 1: + raise ValueError( + f"read() expected a single event for {key!r}, " + f"got a page of {len(values)}" + ) + readings[key] = Reading(value=values[0], timestamp=times[0]) + return readings - async def collect_asset_docs( + async def _collect_asset_docs( self, index: int | None = None ) -> AsyncIterator[StreamAsset]: - # Collect stream datum documents for all indices written. - ctx = error_if_none(self._prepare_ctx, "Prepare not called") + # Bound as collect_asset_docs when there is a streaming provider + data = self.logic.prepared_data if index is None: # The index is optional, and provided for fly scans, if there is # more than one detector to make sure they collect in step index = await self.get_index() - for data_provider in ctx.streamable_data_providers: + for data_provider in data.streamable: async for doc in data_provider.make_stream_docs( - collections_written=index * ctx.trigger_info.collections_per_event, - collections_per_event=ctx.trigger_info.collections_per_event, + collections_written=index * data.collections_per_event, + collections_per_event=data.collections_per_event, ): yield doc + async def _collect_pages( + self, index: int | None = None + ) -> AsyncIterator[PartialEventPage]: + # Bound as collect_pages when there is a paging provider + data = self.logic.prepared_data + cpe = data.collections_per_event + if index is None: + # As in _collect_asset_docs, so detectors can collect in step + index = await self.get_index() + for data_provider in data.pageable: + async for page in data_provider.make_pages( + collections_written=index * cpe, + collections_per_event=cpe, + ): + yield page + async def get_index(self) -> int: - ctx = error_if_none(self._prepare_ctx, "Prepare not called") + data = self.logic.prepared_data min_collections_written = await _get_collections_written( - ctx.streamable_data_providers, reducer=min + data.collectable, reducer=min ) - return min_collections_written // ctx.trigger_info.collections_per_event + return min_collections_written // data.collections_per_event - @AsyncStatus.wrap - async def unstage(self) -> None: - """Stop the detector and file writing.""" - coros = [data_logic.stop() for data_logic in self._data_logics] - if self._acquire_logic: - coros.append(self._acquire_logic.ensure_stopped()) - await asyncio.gather(*coros) + +class _LogicDetector(StandardDetector): + """A concrete `StandardDetector` built around a logic object. + + `StandardDetector.logic` is abstract, so the class itself cannot be + instantiated; `DetectorLogic.with_device` needs something concrete to wrap a + logic object in. + """ + + def __init__(self, logic: DetectorLogic, name: str = "") -> None: + self._logic = logic + super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/core/_flyable.py b/src/ophyd_async/core/_flyable.py index 3eb68c076a..7839778d82 100644 --- a/src/ophyd_async/core/_flyable.py +++ b/src/ophyd_async/core/_flyable.py @@ -1,9 +1,9 @@ import asyncio from abc import abstractmethod -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Coroutine from enum import Enum from functools import cached_property -from typing import Generic, TypeVar, cast +from typing import Any, Generic, TypeVar, cast from bluesky.protocols import Flyable, Preparable from pydantic import Field @@ -17,6 +17,7 @@ CalculatableTimeout, ConfinedModel, WatcherUpdate, + abstract_cached_property, ) #: The per-scan info passed to `FlyableLogic.on_prepare` (often a @@ -60,9 +61,14 @@ async def on_kickoff(self, ctx: CtxT) -> CtxT: """ @abstractmethod - async def on_complete(self, ctx: CtxT) -> None: + def on_complete( + self, ctx: CtxT + ) -> Coroutine[Any, Any, None] | AsyncIterator[WatcherUpdate]: """Block until the fly scan is done. + Write it as an `async def` to just block, or as an async generator + yielding [](#WatcherUpdate) to report progress to watchers as it goes. + :param ctx: the context returned by `on_kickoff`. """ @@ -71,6 +77,11 @@ async def stop(self) -> None: Called by `on_stage` and `on_unstage` by default; override those instead if stage and unstage need to differ. + + Deliberately the same name as `MovableLogic.stop`, so that a logic + implementing both -- a motor record's, which is moved *and* flown -- + writes one stop that serves as the move-stop verb and as fly-scan + teardown. The hardware action is the same in both cases. """ pass @@ -82,11 +93,9 @@ async def on_unstage(self) -> None: """Clean the flyer up on `unstage()`. Defaults to `stop`.""" await self.stop() - def with_device(self, name: str = "") -> "StandardFlyable": + def with_device(self, name: str = "") -> "StandardFlyable[PrepareT, CtxT]": """Wrap this logic in an ephemeral `StandardFlyable` for use in a plan.""" - flyer = StandardFlyable(name=name) - flyer.flyable_logic = self - return flyer + return _EphemeralFlyable(self, name=name) class FlyMotorInfo(ConfinedModel): @@ -146,7 +155,7 @@ class StandardFlyable( ): """Device that provides standard logic for flying. - This class must be inherited and have a `flyable_logic` @cached_property. + This class must be inherited and have a `logic` @cached_property. For an ephemeral flyer in a plan, call `FlyableLogic.with_device` instead of inheriting. It owns the context threaded between the logic's stages and enforces prepare -> kickoff -> complete ordering. `stage()`/`unstage()` run @@ -165,37 +174,50 @@ def __init__(self, *args, **kwargs) -> None: self._fly_ctx: CtxT = cast(CtxT, None) self._fly_stage = _FlyStage.IDLE - @cached_property - def flyable_logic(self) -> FlyableLogic[PrepareT, CtxT]: + @abstract_cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: """The logic object that describes how this device flies. A static flyer (e.g. `Motor`) provides this as a `@cached_property` that builds a `FlyableLogic` from its signals. An ephemeral flyer created by `FlyableLogic.with_device` has it set directly on the instance. + + A Device that is flyable *and* movable implements this once with a logic + object inheriting both `FlyableLogic` and `MovableLogic`; each mix-in + declares `logic` with its own required type, so the type + checker verifies the one implementation against both. """ - raise NotImplementedError( - f"{type(self).__name__} has no flyable_logic; override it as a " - "@cached_property or create the flyer via FlyableLogic.with_device()" - ) + raise NotImplementedError + + @property + def _prepared_fly_ctx(self) -> CtxT: + """The prepare context, for callers outside prepare -> kickoff -> complete. + + `_fly_ctx` is a `None` *typed as* `CtxT` until prepared, so a verb that + read it directly -- a detector's `describe_collect()` or `get_index()` + -- would fail with an `AttributeError` on `None` rather than saying what + was wrong. + """ + if self._fly_stage is _FlyStage.IDLE: + raise RuntimeError(f"{self.name}: prepare() must be called first") + return self._fly_ctx def _reset_fly_state(self) -> None: self._fly_ctx = cast(CtxT, None) self._fly_stage = _FlyStage.IDLE - @AsyncStatus.wrap async def _on_stage(self) -> None: - await self.flyable_logic.on_stage() + await self.logic.on_stage() self._reset_fly_state() - @AsyncStatus.wrap async def _on_unstage(self) -> None: - await self.flyable_logic.on_unstage() + await self.logic.on_unstage() self._reset_fly_state() @AsyncStatus.wrap async def prepare(self, value: PrepareT) -> None: """Move to the start and set up the fly scan.""" - self._fly_ctx = await self.flyable_logic.on_prepare(value) + self._fly_ctx = await self.logic.on_prepare(value) self._fly_stage = _FlyStage.PREPARED @AsyncStatus.wrap @@ -205,32 +227,41 @@ async def kickoff(self) -> None: raise RuntimeError( f"{self.name}: prepare() must be called before kickoff()" ) - self._fly_ctx = await self.flyable_logic.on_kickoff(self._fly_ctx) + self._fly_ctx = await self.logic.on_kickoff(self._fly_ctx) self._fly_stage = _FlyStage.KICKED_OFF @WatchableAsyncStatus.wrap async def complete(self) -> AsyncIterator[WatcherUpdate]: """Block until the fly scan is done. - If the logic is also a `MovableLogic` (e.g. a flying `Motor`), report - progress to watchers by observing its readback while the fly scan runs, - reusing the same watcher-update stream as `StandardMovable.set`. Flyers - whose logic is not movable simply block with no progress updates. + A logic whose `on_complete` yields [](#WatcherUpdate) reports its own + progress, and those updates are passed straight on to watchers. One that + just blocks reports progress only if it is also a `MovableLogic` (e.g. a + flying `Motor`), by observing its readback while the fly scan runs, + reusing the same watcher-update stream as `StandardMovable.set`. Any + other flyer simply blocks with no progress updates. """ if self._fly_stage is not _FlyStage.KICKED_OFF: raise RuntimeError( f"{self.name}: kickoff() must be called before complete()" ) - logic = self.flyable_logic - if isinstance(logic, MovableLogic): + logic = self.logic + completing = logic.on_complete(self._fly_ctx) + if isinstance(completing, AsyncIterator): + # Progress that is neither a readback nor a setpoint -- a detector's + # is "collections written out of collections requested" -- so the + # logic reports it itself + async for update in completing: + yield update + elif isinstance(logic, MovableLogic): initial, target, (units, precision) = await asyncio.gather( logic.readback.get_value(), logic.setpoint.get_value(), logic.get_units_precision(), ) - async with AsyncStatus(logic.on_complete(self._fly_ctx)) as completing: + async with AsyncStatus(completing) as completed: async for current_position in observe_value( - logic.readback, done_status=completing + logic.readback, done_status=completed ): yield WatcherUpdate( current=current_position, @@ -241,5 +272,22 @@ async def complete(self) -> AsyncIterator[WatcherUpdate]: precision=precision, ) else: - await self.flyable_logic.on_complete(self._fly_ctx) + await completing self._fly_stage = _FlyStage.IDLE + + +class _EphemeralFlyable(StandardFlyable[PrepareT, CtxT]): + """A concrete `StandardFlyable` built around a logic object. + + `StandardFlyable.logic` is abstract, so the class itself cannot be + instantiated; `FlyableLogic.with_device` needs something concrete to wrap a + logic object in. + """ + + def __init__(self, logic: FlyableLogic[PrepareT, CtxT], name: str = "") -> None: + self._logic = logic + super().__init__(name=name) + + @cached_property + def logic(self) -> FlyableLogic[PrepareT, CtxT]: + return self._logic diff --git a/src/ophyd_async/core/_movable.py b/src/ophyd_async/core/_movable.py index beeb1a1ee8..9d5d293f2a 100644 --- a/src/ophyd_async/core/_movable.py +++ b/src/ophyd_async/core/_movable.py @@ -1,10 +1,8 @@ import asyncio import time -from abc import abstractmethod from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from functools import cached_property from typing import Generic from bluesky.protocols import ( @@ -26,6 +24,7 @@ CalculatableTimeout, Callback, WatcherUpdate, + abstract_cached_property, ) @@ -111,9 +110,9 @@ async def connect(self, device: "StandardMovable") -> None: """Mock signals to do an instant move on setpoint write.""" def _instant_move(value): - set_mock_value(device.movable_logic.readback, value) # Arrive instantly + set_mock_value(device.logic.readback, value) # Arrive instantly - callback_on_mock_put(device.movable_logic.setpoint, _instant_move) + callback_on_mock_put(device.logic.setpoint, _instant_move) class StopState(Enum): @@ -133,7 +132,7 @@ class StandardMovable( ): """Device that provides standard logic for moving. - This class must be inherited and have a `movable_logic` @cached_property. + This class must be inherited and have a `logic` @cached_property. If stop is called while moving, it will raise a RuntimeError. """ @@ -142,20 +141,26 @@ class StandardMovable( _stop_state: Enum = StopState.NONE _move_status: AsyncStatus | None = None - @cached_property - @abstractmethod - def movable_logic(self) -> MovableLogic: + @abstract_cached_property + def logic(self) -> MovableLogic[SignalDatatypeT]: """The logic object that describes how this device moves. This is intentionally public so that mock helpers (e.g. `InstantMovableMock`) and subclasses can access the `setpoint` and `readback` signals directly. Subclasses must implement this as a - `@cached_property` that returns a `MovableLogic` instance. + `@cached_property` returning a `MovableLogic`. + + A Device that is movable *and* flyable (e.g. `Motor`) implements this + once with a logic object that inherits both `MovableLogic` and + `FlyableLogic`; each mix-in declares `logic` with its own + required type, so the type checker verifies the one implementation + against both. """ + raise NotImplementedError async def check_value(self, value: SignalDatatypeT) -> None: """Check the move is valid before doing it.""" - await self.movable_logic.check_move(value) + await self.logic.check_move(value) @WatchableAsyncStatus.wrap async def set( @@ -166,13 +171,13 @@ async def set( """Move to the given value.""" self._stop_state = StopState.NONE old_position, (units, precision) = await asyncio.gather( - self.movable_logic.readback.get_value(), - self.movable_logic.get_units_precision(), + self.logic.readback.get_value(), + self.logic.get_units_precision(), ) - await self.movable_logic.check_move(new_position) + await self.logic.check_move(new_position) if timeout == CALCULATE_TIMEOUT: - move_timeout = await self.movable_logic.calculate_timeout( + move_timeout = await self.logic.calculate_timeout( old_position, new_position ) else: @@ -180,12 +185,12 @@ async def set( try: async with AsyncStatus( - self.movable_logic.move( + self.logic.move( new_position=new_position, timeout=MoveTimeout(move_timeout) ) ) as self._move_status: async for current_position in observe_value( - self.movable_logic.readback, + self.logic.readback, done_status=self._move_status, ): yield WatcherUpdate( @@ -215,20 +220,20 @@ async def set( async def stop(self, success=False): """Request to stop moving and return immediately.""" self._stop_state = StopState.USER_SUCCESS if success else StopState.USER_FAILURE - await self.movable_logic.stop() + await self.logic.stop() if self._move_status: self._move_status.task.cancel() def set_name(self, name: str, *, child_name_separator: str | None = None) -> None: super().set_name(name, child_name_separator=child_name_separator) # Readback should be named the same as its parent in read() - self.movable_logic.readback.set_name(name) + self.logic.readback.set_name(name) async def locate(self) -> Location[SignalDatatypeT]: """Return the current setpoint and readback of the device.""" setpoint, readback = await asyncio.gather( - self.movable_logic.setpoint.get_value(), - self.movable_logic.readback.get_value(), + self.logic.setpoint.get_value(), + self.logic.readback.get_value(), ) return Location(setpoint=setpoint, readback=readback) @@ -236,7 +241,7 @@ def subscribe_reading( self, function: Callback[dict[str, Reading[SignalDatatypeT]]] ) -> None: """Subscribe to reading.""" - self.movable_logic.readback.subscribe_reading(function) + self.logic.readback.subscribe_reading(function) subscribe = subscribe_reading @@ -244,4 +249,4 @@ def clear_sub( self, function: Callback[dict[str, Reading[SignalDatatypeT]]] ) -> None: """Unsubscribe.""" - self.movable_logic.readback.clear_sub(function) + self.logic.readback.clear_sub(function) diff --git a/src/ophyd_async/core/_readable.py b/src/ophyd_async/core/_readable.py index efb18d966a..8a2fe0e7db 100644 --- a/src/ophyd_async/core/_readable.py +++ b/src/ophyd_async/core/_readable.py @@ -1,15 +1,25 @@ +import asyncio +import itertools import warnings -from collections.abc import Awaitable, Callable, Generator, Sequence +from collections.abc import ( + Awaitable, + Callable, + Generator, + Iterator, + MutableMapping, + Sequence, +) from contextlib import contextmanager from enum import Enum -from typing import Any, cast +from functools import cached_property +from typing import Literal, cast from bluesky.protocols import HasHints, Hints, Reading from event_model import DataKey from ._device import Device, DeviceMap, DeviceVector from ._protocol import AsyncConfigurable, AsyncReadable, AsyncStageable -from ._signal import SignalR +from ._signal import SignalR, walk_devices from ._standard_base import _StandardBase from ._utils import merge_gathered_dicts @@ -49,38 +59,101 @@ class StandardReadableFormat(Enum): def __call__(self, parent: Device, child: Device): if not isinstance(parent, StandardReadable): raise TypeError(f"Expected parent to be StandardReadable, got {parent}") - parent.add_readables([child], self) + parent.set_readable_format(child, self) -# Back compat -class _WarningMatcher: - def __init__(self, name: str, target: StandardReadableFormat): - self._name = name - self._target = target +# Short alias for the case statements below +Format = StandardReadableFormat - def __eq__(self, value: object) -> bool: - warnings.warn( - DeprecationWarning( - f"Use `StandardReadableFormat.{self._target.name}` " - f"instead of `{self._name}`" - ), - stacklevel=2, - ) - return value == self._target +_SignalFormat = Literal[ + Format.CONFIG_SIGNAL, + Format.HINTED_SIGNAL, + Format.UNCACHED_SIGNAL, + Format.HINTED_UNCACHED_SIGNAL, +] +_ReadableEntry = tuple[Device, Literal[Format.CHILD]] | tuple[SignalR, _SignalFormat] -def _compat_format(name: str, target: StandardReadableFormat) -> StandardReadableFormat: - return cast(StandardReadableFormat, _WarningMatcher(name, target)) +class _Readables(MutableMapping[Device, StandardReadableFormat]): + """The registered children and their formats, in registration order. + Only a `SignalR` may be given a signal format, which is what lets + `entries()` hand each one to the verbs already typed. + """ -ConfigSignal = _compat_format("ConfigSignal", StandardReadableFormat.CONFIG_SIGNAL) -HintedSignal: Any = _compat_format("HintedSignal", StandardReadableFormat.HINTED_SIGNAL) -HintedSignal.uncached = _compat_format( - "HintedSignal.uncached", StandardReadableFormat.HINTED_UNCACHED_SIGNAL -) + def __init__(self) -> None: + self._formats: dict[Device, StandardReadableFormat] = {} + + def __getitem__(self, device: Device) -> StandardReadableFormat: + return self._formats[device] + + def __setitem__(self, device: Device, format: StandardReadableFormat) -> None: + if not isinstance(format, StandardReadableFormat): + raise TypeError(f"{format} is not a StandardReadableFormat") + match format: + case Format.CHILD: + pass + case ( + Format.CONFIG_SIGNAL + | Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL + ): + # Fails type checking unless these are exactly _SignalFormat, + # which is what makes the cast in entries() true + _: _SignalFormat = format + if not isinstance(device, SignalR): + raise TypeError(f"{device} is not a SignalR") + self._formats[device] = format + + def __delitem__(self, device: Device) -> None: + del self._formats[device] + + def __iter__(self) -> Iterator[Device]: + return iter(self._formats) + + def __len__(self) -> int: + return len(self._formats) + + def entries(self) -> list[_ReadableEntry]: + """Each child and its format, typed by what `__setitem__` allowed.""" + return cast(list[_ReadableEntry], list(self._formats.items())) + + +class _SealDefaultFormatsMeta(type(_StandardBase)): # type: ignore[misc] + """Record the formats a `StandardReadable` was constructed with. + + `type.__call__` runs `__new__` and then the *whole* `__init__` chain before + returning, which is the only point that is after every place a format can + be declared: + + - annotations, applied by `create_children_from_annotations` inside + `Device.__init__` + - registration a subclass does *before* calling `super().__init__()`, which + is what all 24 in-tree Devices do + - registration a subclass does *after* it, which nothing in tree does but + which is perfectly legal + + Snapshotting at the end of `StandardReadable.__init__` would silently miss + the third, and the symptom would be + [](#StandardReadable.reset_readable_formats) dropping a child the class had + declared. Devices are already built on a metaclass (`_ProtocolMeta`, from + the `HasName` Protocol), so this adds no new machinery to the hierarchy. + """ + + def __call__(cls, *args, **kwargs): + self = super().__call__(*args, **kwargs) + self._default_readables = dict(self._readables) + return self -class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints): +class StandardReadable( + _StandardBase, + AsyncReadable, + AsyncConfigurable, + HasHints, + metaclass=_SealDefaultFormatsMeta, +): """Device that provides selected child Device values in `read()`. Provides the ability for children to be registered to: @@ -89,16 +162,177 @@ class StandardReadable(_StandardBase, AsyncReadable, AsyncConfigurable, HasHints - Provide their value in `read_configuration()` and `describe_configuration() - Select a value to appear in `hints` - The behavior is customized with a [](#StandardReadableFormat) + The behavior is customized with a [](#StandardReadableFormat), which can be + changed at runtime with [](#StandardReadable.set_readable_format). """ - # These must be immutable types to avoid accidental sharing between - # different instances of the class - _describe_config_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () - _read_config_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () - _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + @cached_property + def _readables(self) -> _Readables: + """The registered children and their formats, in registration order. + + A `cached_property` rather than an attribute set in `__init__`, because + `add_children_as_readables` and `set_readable_format` are routinely + called *before* `super().__init__()` (e.g. `AreaDetector`), so anything + `__init__` assigned would discard them. It also keeps each instance's + registry its own, where a mutable class attribute would be shared. + + Keying by Device is identity keying, which is what a registry of + children needs: `Device` does not override `__eq__`, and while + `DeviceVector` and `DeviceMap` inherit a value based `__eq__` from + `Mapping`, they hash by `id()`. That is injective over live objects, so + two distinct devices never share a hash and `__eq__` is never reached. + """ + return _Readables() + + # The formats declared during construction, sealed by the metaclass once + # __init__ has fully returned. Only ever replaced, never mutated in place, + # so the shared class level default is safe. + _default_readables: dict[Device, StandardReadableFormat] = {} + + # What contributes to each verb, gathered in parallel by the verb methods. + # Immutable defaults to avoid accidental sharing between instances. A + # subclass producing data from somewhere other than a registered child adds + # to these rather than overriding the verbs, so registered children keep + # working; `StandardDetector` does it for its data logics. _read_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () - _has_hints: tuple[HasHints, ...] = () + _describe_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + _read_config_funcs: tuple[Callable[[], Awaitable[dict[str, Reading]]], ...] = () + _describe_config_funcs: tuple[Callable[[], Awaitable[dict[str, DataKey]]], ...] = () + _hint_funcs: tuple[Callable[[], Iterator[Hints]], ...] = () + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # Each of these derives from _readables on every call rather than being + # registered up-front, so a format changed between runs is picked up on + # the next stage() or the next descriptor. + self._stage_funcs += (self._stage_readables,) + self._unstage_funcs += (self._unstage_readables,) + self._read_funcs += (self._formatted_read,) + self._describe_funcs += (self._formatted_describe,) + self._read_config_funcs += (self._formatted_read_configuration,) + self._describe_config_funcs += (self._formatted_describe_configuration,) + self._hint_funcs += (self._formatted_hints,) + + async def _stage_readables(self) -> None: + await asyncio.gather(*(sig.stage().task for sig in self._signals_to_stage())) + + async def _unstage_readables(self) -> None: + await asyncio.gather(*(sig.unstage().task for sig in self._signals_to_stage())) + + def _signals_to_stage(self) -> Iterator[AsyncStageable]: + # HINTED_SIGNAL stages so that caching is set up and read() is fast; a + # CHILD stages because it may have staging of its own. The uncached and + # config formats deliberately do not. + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncStageable): + yield device + case (signal, Format.HINTED_SIGNAL): + yield signal + case ( + _, + Format.CONFIG_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass + + async def _formatted_read(self) -> dict[str, Reading]: + """What the registered children contribute to `read()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncReadable): + coros.append(device.read()) + case (signal, Format.HINTED_SIGNAL): + coros.append(signal.read()) + case ( + signal, + Format.UNCACHED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, + ): + coros.append(signal.read(cached=False)) + case (_, Format.CONFIG_SIGNAL): + pass + return await merge_gathered_dicts(coros) + + async def _formatted_describe(self) -> dict[str, DataKey]: + """What the registered children contribute to `describe()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncReadable): + coros.append(device.describe()) + case ( + signal, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + coros.append(signal.describe()) + case (_, Format.CONFIG_SIGNAL): + pass + return await merge_gathered_dicts(coros) + + async def _formatted_read_configuration(self) -> dict[str, Reading]: + """What the registered children contribute to `read_configuration()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncConfigurable): + coros.append(device.read_configuration()) + case (signal, Format.CONFIG_SIGNAL): + coros.append(signal.read()) + case ( + _, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass + return await merge_gathered_dicts(coros) + + async def _formatted_describe_configuration(self) -> dict[str, DataKey]: + """What the registered children contribute to `describe_configuration()`.""" + coros = [] + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, AsyncConfigurable): + coros.append(device.describe_configuration()) + case (signal, Format.CONFIG_SIGNAL): + coros.append(signal.describe()) + case ( + _, + Format.HINTED_SIGNAL + | Format.UNCACHED_SIGNAL + | Format.HINTED_UNCACHED_SIGNAL, + ): + pass + return await merge_gathered_dicts(coros) + + def _formatted_hints(self) -> Iterator[Hints]: + """What the registered children contribute to `hints`.""" + for entry in self._readables.entries(): + match entry: + case (device, Format.CHILD): + if isinstance(device, HasHints): + yield device.hints + case ( + signal, + Format.HINTED_SIGNAL | Format.HINTED_UNCACHED_SIGNAL, + ): + # An unnamed signal has nothing to put in `fields`, but is + # still hinted, so it contributes the key and no name + yield {"fields": [signal.name] if signal.name else []} + case ( + _, + Format.CONFIG_SIGNAL | Format.UNCACHED_SIGNAL, + ): + pass async def describe_configuration(self) -> dict[str, DataKey]: return await merge_gathered_dicts( @@ -117,12 +351,14 @@ async def read(self) -> dict[str, Reading]: @property def hints(self) -> Hints: hints: Hints = {} - for new_hint in self._has_hints: + for new_hint in itertools.chain.from_iterable( + func() for func in self._hint_funcs + ): # Merge the existing and new hints, based on the type of the value. # This avoids default dict merge behavior that overrides the values; # we want to combine them when they are Sequences, and ensure they are # identical when string values. - for key, value in new_hint.hints.items(): + for key, value in new_hint.items(): # fail early for unkwon types if isinstance(value, str): if key in hints: @@ -144,8 +380,7 @@ def hints(self) -> Hints: hints[key] = value # type: ignore[literal-required] else: msg = ( - f"{new_hint.name}: Unknown type for value '{value}'" - f" for key '{key}'" + f"{self.name}: Unknown type for value '{value}' for key '{key}'" ) raise TypeError(msg) @@ -156,11 +391,11 @@ def add_children_as_readables( self, format: StandardReadableFormat = StandardReadableFormat.CHILD, ) -> Generator[None, None, None]: - """Context manager that calls [](#add_readables) on child Devices added within. + """Register child Devices added within with the given format. Scans `self.children()` on entry and exit to context manager, and calls - `add_readables()` on any that are added with the provided - `StandardReadableFormat`. + [](#StandardReadable.set_readable_format) on any that are added with the + provided `StandardReadableFormat`. """ dict_copy = dict(self.children()) @@ -191,9 +426,88 @@ def add_children_as_readables( else: flattened_values.append(value) - new_devices = list(filter(lambda x: isinstance(x, Device), flattened_values)) - self.add_readables(new_devices, format) + for device in flattened_values: + if isinstance(device, Device): + self.set_readable_format(device, format) + + def set_readable_format( + self, device: Device, format: StandardReadableFormat | None + ) -> None: + """Set how a child Device contributes to this Device's bluesky verbs. + + This is the runtime equivalent of the `Kind` attribute in ophyd v1: it + can be called after construction to move a child between configuration, + hinted and uncached reads, or to drop it from the verbs entirely. + + Call it **between runs**. The descriptor for a run is emitted at its + start, and [](#StandardReadableFormat.HINTED_SIGNAL) sets up caching in + `stage()`, so changing a format part way through a run would make + `describe()` and `read()` disagree. + + :param device: The Device to set the format of, normally a child of this one + :param format: + The format to give it, or `None` to stop it contributing at all. + Replaces any format the device already has, rather than adding a + second contribution. + :raises TypeError: If a signal-only format is given a non-`SignalR` + """ + if format is None: + self._readables.pop(device, None) + else: + # Re-formatting an already registered child keeps its position, so + # changing a format does not reorder `hints` + self._readables[device] = format + + def reset_readable_formats(self) -> None: + """Undo every runtime format change, back to how the class declared them. + + The baseline is everything registered by the time construction + finished: annotations, `add_children_as_readables`, and any + [](#StandardReadable.set_readable_format) an `__init__` made. Anything + done after that -- switching a child between config and hinted, + dropping one with a format of `None`, registering a new one -- is + undone. + + Intended for a Device that is retuned per technique, so a scan can put + it back without knowing what it changed. + + This resets **this Device only**, not its children. To reset a whole + tree use `apply_readable_formats(device, {})`. + + The baseline is the *class declaration*, not the last stored settings + file, so this discards a technique loaded by [](#apply_settings) too. + """ + # Mutated rather than replaced, so the cached_property stays the same + # object and nothing holding a reference sees a stale registry + self._readables.clear() + self._readables.update(self._default_readables) + + def get_readable_formats(self) -> dict[Device, StandardReadableFormat]: + """Return the registered children and their formats, in registration order. + + A Device that does not contribute is absent, so + `get_readable_formats().get(child)` gives its format or `None`. + """ + return dict(self._readables) + + def get_readable_format_changes( + self, + ) -> dict[Device, StandardReadableFormat | None]: + """Return the formats that differ from those declared at construction. + A child registered at construction but since dropped has a format of `None`. + """ + changes: dict[Device, StandardReadableFormat | None] = { + child: format + for child, format in self._readables.items() + if self._default_readables.get(child) is not format + } + for child in self._default_readables: + if child not in self._readables: + changes[child] = None + return changes + + # Back compat - delete before 1.0 def add_readables( self, devices: Sequence[Device], @@ -201,78 +515,148 @@ def add_readables( ) -> None: """Add devices to contribute to various bluesky verbs. - Use output from the given devices to contribute to the verbs of the following - interfaces: - - - [](#bluesky.protocols.Readable) - - [](#bluesky.protocols.Configurable) - - [](#bluesky.protocols.Stageable) - - [](#bluesky.protocols.HasHints) - :param devices: The devices to be added :param format: Determines which of the devices functions are added to which verb as per the [](#StandardReadableFormat) documentation """ - - def assert_device_is_signalr(device: Device) -> SignalR: - if not isinstance(device, SignalR): - raise TypeError(f"{device} is not a SignalR") - return device - + warnings.warn( + DeprecationWarning( + "Use `set_readable_format(device, format)` on each device " + "instead of `add_readables(devices, format)`" + ), + stacklevel=2, + ) for device in devices: - match format: - case StandardReadableFormat.CHILD: - if isinstance(device, AsyncConfigurable): - self._describe_config_funcs += (device.describe_configuration,) - self._read_config_funcs += (device.read_configuration,) - if isinstance(device, AsyncReadable): - self._describe_funcs += (device.describe,) - self._read_funcs += (device.read,) - if isinstance(device, AsyncStageable): - self._stage_funcs += (device.stage,) - self._unstage_funcs += (device.unstage,) - if isinstance(device, HasHints): - self._has_hints += (device,) - case StandardReadableFormat.CONFIG_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_config_funcs += (signalr_device.describe,) - self._read_config_funcs += (signalr_device.read,) - case StandardReadableFormat.HINTED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (signalr_device.read,) - self._stage_funcs += (signalr_device.stage,) - self._unstage_funcs += (signalr_device.unstage,) - self._has_hints += (_HintsFromName(signalr_device),) - case StandardReadableFormat.UNCACHED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (_UncachedRead(signalr_device),) - case StandardReadableFormat.HINTED_UNCACHED_SIGNAL: - signalr_device = assert_device_is_signalr(device=device) - self._describe_funcs += (signalr_device.describe,) - self._read_funcs += (_UncachedRead(signalr_device),) - self._has_hints += (_HintsFromName(signalr_device),) - - -class _UncachedRead: - def __init__(self, signal: SignalR) -> None: - self.signal = signal - - async def __call__(self) -> dict[str, Reading]: - return await self.signal.read(cached=False) - - -class _HintsFromName(HasHints): - def __init__(self, device: Device) -> None: - self.device = device - - @property - def name(self) -> str: - return self.device.name - - @property - def hints(self) -> Hints: - fields = [self.name] if self.name else [] - return {"fields": fields} + self.set_readable_format(device, format) + + +#: Reserved key under which [](#store_settings) writes readable formats. Chosen +#: so that it cannot be a dotted attribute path: `<` cannot start a Python +#: identifier, so no attribute assignment can produce a colliding key, and +#: unlike `*FORMATS*` it needs no quoting in yaml. +READABLE_FORMATS_KEY = "" + +#: The key standing in for the root Device itself in a [](#ReadableFormats), +#: where every other key is a dotted attribute path. Not a valid Python +#: identifier, so no attribute assignment can produce a colliding path, and +#: unlike `""` it does not make yaml fall back to its hard-to-read explicit +#: key syntax. +ROOT_DEVICE_KEY = "" + +#: The changes to the readable formats of a Device tree since construction, in +#: a form that can be stored and retrieved: +#: `{path of the StandardReadable: {path of the child: format}}`, where a format +#: of `None` means "stop this child contributing". +#: The owner path is a dotted attribute path from the root Device, as produced +#: by [](#walk_devices), with [](#ROOT_DEVICE_KEY) meaning the root Device +#: itself. The child path is a dotted attribute path from the owner. +#: +#: The outer key is needed because a format is a fact about an (owner, child) +#: *pair*, not about a single Device: the same child can be registered on more +#: than one `StandardReadable` with a different format each time. +ReadableFormats = dict[str, dict[str, StandardReadableFormat | None]] + + +def _paths_to_devices(device: Device) -> dict[str, Device]: + return {ROOT_DEVICE_KEY: device, **walk_devices(device)} + + +def walk_readable_formats(device: Device) -> ReadableFormats | None: + """Retrieve how the readable formats in a Device tree differ from construction. + + Used as part of saving a Device, so that which signals are hinted or + config can be restored later. Only the differences from what each + `StandardReadable` registered during construction are returned, with a + child it has since dropped given a format of `None`. + + :param device: The root Device to walk. + :return: + A [](#ReadableFormats) suitable for storing, or `None` if there is no + `StandardReadable` in the tree. + """ + paths_to_devices = _paths_to_devices(device) + readables = { + path: dev + for path, dev in paths_to_devices.items() + if isinstance(dev, StandardReadable) + } + if not readables: + return None + formats: ReadableFormats = {} + for path, dev in readables.items(): + device_to_path = {child: p for p, child in walk_devices(dev).items()} + entries: dict[str, StandardReadableFormat | None] = {} + for child, format in dev.get_readable_format_changes().items(): + child_path = device_to_path.get(child) + if child_path is None: + # Not below its owner, so it has no stable path to store it + # against. See https://github.com/bluesky/ophyd-async/issues/1402 + warnings.warn( + f"{dev.name or dev}: {child.name or child} is not within " + f"{dev.name or dev}, so the change to its readable format " + "will not be stored", + stacklevel=2, + ) + else: + entries[child_path] = format + if entries: + formats[path] = entries + return formats + + +def apply_readable_formats(device: Device, formats: ReadableFormats) -> None: + """Reset the readable formats in a Device tree, then apply the given changes. + + Every `StandardReadable` in the tree is first put back to what it + registered during construction, as + [](#StandardReadable.reset_readable_formats) does, then the changes in + `formats` are applied on top. So an empty `formats` restores the class + defaults, and a child added to a Device since `formats` was stored keeps + the format its class gives it. + + To stop a child contributing, give it a format of `None`, which in a stored + file is a null: + + ```yaml + : + : + energy: HINTED_SIGNAL + temperature: null + ``` + + :param device: The root Device the owner paths in `formats` are relative to. + :param formats: A [](#ReadableFormats), e.g. from [](#walk_readable_formats). + :raises KeyError: If a path does not name a Device in the tree. + :raises TypeError: If an owner path does not name a `StandardReadable`. + """ + devices = _paths_to_devices(device) + resolved: list[ + tuple[StandardReadable, list[tuple[Device, StandardReadableFormat | None]]] + ] = [] + # Resolve everything before changing anything, so a bad path cannot leave + # the tree half applied + for owner_path, entries in formats.items(): + owner = devices.get(owner_path) + if owner is None: + raise KeyError(f"No Device at {owner_path!r} in {device.name or device}") + if not isinstance(owner, StandardReadable): + raise TypeError(f"{owner_path!r} is not a StandardReadable, got {owner}") + children = walk_devices(owner) + readables: list[tuple[Device, StandardReadableFormat | None]] = [] + for child_path, format in entries.items(): + child = children.get(child_path) + if child is None: + raise KeyError( + f"No Device at {child_path!r} in {owner.name or owner_path}" + ) + readables.append( + (child, None if format is None else StandardReadableFormat(format)) + ) + resolved.append((owner, readables)) + for dev in devices.values(): + if isinstance(dev, StandardReadable): + dev.reset_readable_formats() + for owner, readables in resolved: + for child, format in readables: + owner.set_readable_format(child, format) diff --git a/src/ophyd_async/core/_settings.py b/src/ophyd_async/core/_settings.py index 4edc87b2e5..41ef871e54 100644 --- a/src/ophyd_async/core/_settings.py +++ b/src/ophyd_async/core/_settings.py @@ -5,6 +5,7 @@ from typing import Any, Generic from ._device import Device, DeviceT +from ._readable import ReadableFormats from ._signal import SignalRW from ._signal_backend import SignalDatatypeT @@ -33,11 +34,26 @@ class Settings(MutableMapping[SignalRW[Any], Any], Generic[DeviceT]): """ def __init__( - self, device: DeviceT, settings: MutableMapping[SignalRW, Any] | None = None + self, + device: DeviceT, + settings: MutableMapping[SignalRW, Any] | None = None, + readable_formats: ReadableFormats | None = None, ): self.device = device self._settings = {} self.update(settings or {}) + self.readable_formats: ReadableFormats | None = ( + None + if readable_formats is None + else {owner: dict(entries) for owner, entries in readable_formats.items()} + ) + """How the Device's children contribute to its bluesky verbs. + + Stored and applied alongside the signal values, as the two change on + the same cadence. Applied by [](#apply_settings) with + [](#apply_readable_formats), so a dict (even an empty one) first resets + the tree to its class defaults, while `None` leaves formats alone. + """ def __getitem__(self, key: SignalRW[SignalDatatypeT]) -> SignalDatatypeT: return self._settings[key] @@ -70,10 +86,37 @@ def __len__(self) -> int: return len(self._settings) def __or__(self, other: MutableMapping[SignalRW, Any]) -> Settings[DeviceT]: - """Create a new Settings that is the union of self overridden by other.""" - if isinstance(other, Settings) and not self._is_in_device(other.device): - raise ValueError(f"{other.device} is not a child of {self.device}") - return Settings(self.device, self._settings | dict(other)) + """Create a new Settings that is the union of self overridden by other. + + Readable formats merge per (owner, child), the same granularity that + values merge at, so `other` only replaces the children it mentions. + + :raises ValueError: + If `other` is a `Settings` for a Device outside this one, or for a + Device below this one while carrying readable formats. + """ + formats = self.readable_formats + if isinstance(other, Settings): + if other.device is not self.device: + if not self._is_in_device(other.device): + msg = f"{other.device} is not a child of {self.device}" + raise ValueError(msg) + if other.readable_formats is not None: + # Re-rooting the owner paths is easy, but applying resets + # the whole tree, so a sub-device's formats would reset + # Devices it knows nothing about + msg = ( + f"Cannot merge readable formats from {other.device}, whose " + f"paths are relative to it rather than to {self.device}" + ) + raise ValueError(msg) + if other.readable_formats is not None: + formats = { + owner: dict(entries) for owner, entries in (formats or {}).items() + } + for owner, entries in other.readable_formats.items(): + formats.setdefault(owner, {}).update(entries) + return Settings(self.device, self._settings | dict(other), formats) def partition( self, predicate: Callable[[SignalRW], bool] @@ -88,13 +131,19 @@ def partition( The first contains the signals for which the predicate returned True, and the second contains the signals for which the predicate returned False. + Both halves carry the *same* `readable_formats`, as a device specific + apply plan normally applies only the halves (see + `apply_panda_settings`). Applying them twice is harmless: it writes to + no hardware and gives the same result. + :example: ```python settings = Settings(device, {device.special: 1, device.sig: 2}) specials, others = settings.partition(lambda sig: "special" in sig.name) ``` """ - where_true, where_false = Settings(self.device), Settings(self.device) + where_true = Settings(self.device, readable_formats=self.readable_formats) + where_false = Settings(self.device, readable_formats=self.readable_formats) for signal, value in self.items(): dest = where_true if predicate(signal) else where_false dest[signal] = value diff --git a/src/ophyd_async/core/_standard_base.py b/src/ophyd_async/core/_standard_base.py index 3c4d69a221..836f4b1b04 100644 --- a/src/ophyd_async/core/_standard_base.py +++ b/src/ophyd_async/core/_standard_base.py @@ -1,9 +1,9 @@ import asyncio -from collections.abc import Callable +from collections.abc import Awaitable, Callable from ._device import Device from ._protocol import AsyncStageable -from ._status import AsyncStatus, AsyncStatusBase +from ._status import AsyncStatus class _StandardBase(Device, AsyncStageable): @@ -17,13 +17,13 @@ class _StandardBase(Device, AsyncStageable): """ # Immutable defaults to avoid accidental sharing between instances - _stage_funcs: tuple[Callable[[], AsyncStatusBase], ...] = () - _unstage_funcs: tuple[Callable[[], AsyncStatusBase], ...] = () + _stage_funcs: tuple[Callable[[], Awaitable[None]], ...] = () + _unstage_funcs: tuple[Callable[[], Awaitable[None]], ...] = () @AsyncStatus.wrap async def stage(self) -> None: - await asyncio.gather(*(func().task for func in self._stage_funcs)) + await asyncio.gather(*(func() for func in self._stage_funcs)) @AsyncStatus.wrap async def unstage(self) -> None: - await asyncio.gather(*(func().task for func in self._unstage_funcs)) + await asyncio.gather(*(func() for func in self._unstage_funcs)) diff --git a/src/ophyd_async/core/_utils.py b/src/ophyd_async/core/_utils.py index 6acd93551c..477ba8da5e 100644 --- a/src/ophyd_async/core/_utils.py +++ b/src/ophyd_async/core/_utils.py @@ -13,7 +13,7 @@ ) from dataclasses import dataclass from enum import Enum, EnumMeta, StrEnum -from functools import lru_cache +from functools import cached_property, lru_cache from inspect import isawaitable from typing import ( Any, @@ -439,6 +439,27 @@ def non_zero(value): return int(value) != 0 +def abstract_cached_property(func: Callable[[Any], T]) -> cached_property[T]: + """A `cached_property` that `ABCMeta` treats as abstract. + + `functools.cached_property` does not forward `__isabstractmethod__` from the + function it wraps, so `@cached_property` stacked on `@abstractmethod` + registers nothing: the class stays instantiable and a missing implementation + surfaces as an `AttributeError` on `None` from deep inside `__init__`. Setting + the flag on the descriptor itself makes `Device`'s metaclass collect it, so + the omission is a `TypeError` at instantiation instead. + + The declared return type is deliberately `cached_property[T]` rather than a + subclass, so that a subclass overriding it with a plain `@cached_property` + still gets checked for covariance by the type checker. + + :param func: The property body, which should raise `NotImplementedError`. + """ + prop = cached_property(func) + prop.__isabstractmethod__ = True # type: ignore[attr-defined] + return prop + + async def _wait_for(coro: Awaitable[T], timeout: float | None, source: str) -> T: try: return await asyncio.wait_for(coro, timeout) diff --git a/src/ophyd_async/epics/adandor.py b/src/ophyd_async/epics/adandor.py index 6ceb2cdf3e..66d72c7c30 100644 --- a/src/ophyd_async/epics/adandor.py +++ b/src/ophyd_async/epics/adandor.py @@ -21,7 +21,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from ophyd_async.epics.core import PvSuffix @@ -79,7 +79,7 @@ async def prepare_edge(self, num: int, livetime: float): await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE, livetime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class AndorDetector(AreaDetector[Andor2DriverIO]): diff --git a/src/ophyd_async/epics/adaravis.py b/src/ophyd_async/epics/adaravis.py index dcd360363a..228364846f 100644 --- a/src/ophyd_async/epics/adaravis.py +++ b/src/ophyd_async/epics/adaravis.py @@ -23,7 +23,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .adgenicam import get_camera_deadtime from .core import PvSuffix @@ -59,9 +59,6 @@ class AravisTriggerLogic(DetectorTriggerLogic): driver: AravisDriverIO override_deadtime: float | None = None - def config_sigs(self) -> set[SignalR]: - return {self.driver.model} - def get_deadtime(self, config_values: SignalDict) -> float: return get_camera_deadtime( model=config_values[self.driver.model], @@ -83,7 +80,7 @@ async def prepare_edge(self, num: int, livetime: float): await prepare_exposures(self.driver, num, livetime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class AravisDetector(AreaDetector[AravisDriverIO]): diff --git a/src/ophyd_async/epics/adcore/__init__.py b/src/ophyd_async/epics/adcore/__init__.py index bf8a872021..5614203953 100644 --- a/src/ophyd_async/epics/adcore/__init__.py +++ b/src/ophyd_async/epics/adcore/__init__.py @@ -5,11 +5,11 @@ from ._acquire_logic import ADAcquireLogic, ADContAcqAcquireLogic from ._data_logic import ( + ADHDFDataCtx, ADHDFDataLogic, ADMultipartDataLogic, ADWriterFactory, NDArrayDescription, - PluginSignalDataLogic, ) from ._detector import AreaDetector, ContAcqDetector from ._io import ( @@ -31,6 +31,7 @@ NDROIStatIO, NDROIStatNIO, NDStatsIO, + NDStatsTSAcquireMode, ) from ._ndattribute import ( NDAttributeDataType, @@ -40,10 +41,11 @@ ndattributes_to_xml, ) from ._plan_stubs import setup_ndattributes, setup_ndstats_sum +from ._stats_time_series import StatsTimeSeriesDataLogic from ._trigger_logic import ( ADContAcqTriggerLogic, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) __all__ = [ @@ -57,6 +59,7 @@ "NDPluginBaseIO", "NDROIIO", "NDStatsIO", + "NDStatsTSAcquireMode", "NDROIStatNIO", "NDROIStatIO", "NDCBFlushOnSoftTrgMode", @@ -69,16 +72,17 @@ # TriggerLogic "prepare_exposures", "ADContAcqTriggerLogic", - "trigger_info_from_num_images", + "trigger_info_from_driver", # AcquireLogic "ADAcquireLogic", "ADContAcqAcquireLogic", # DataLogic "NDArrayDescription", - "PluginSignalDataLogic", + "ADHDFDataCtx", "ADHDFDataLogic", "ADMultipartDataLogic", "ADWriterFactory", + "StatsTimeSeriesDataLogic", # Detector "AreaDetector", "ContAcqDetector", diff --git a/src/ophyd_async/epics/adcore/_acquire_logic.py b/src/ophyd_async/epics/adcore/_acquire_logic.py index a5c0aaeabd..3ad36fd3d4 100644 --- a/src/ophyd_async/epics/adcore/_acquire_logic.py +++ b/src/ophyd_async/epics/adcore/_acquire_logic.py @@ -12,16 +12,46 @@ class ADAcquireLogic(DetectorAcquireLogic): + """Arm and disarm an areaDetector driver. + + :param driver: The areaDetector driver to arm. + :param driver_armed_signal: + The signal to watch for the driver having armed, if it is not `acquire`. + :param wait_for_plugins: + Whether the detector should wait for its plugins to drain before it + reports acquisition as complete. `WaitForPlugins` makes the driver's + `AcquireBusy` wait for the number of queued arrays to reach zero, and + `wait_for_idle` blocks on the `acquire` put callback, so turning it on + is what makes a step scan read plugin scalars for the frame it just + took rather than the one before. Defaults to on, since racing is + silent and the wait costs nothing when a file writer is the slowest + thing in the chain anyway. Turn it off for a detector whose plugin + chain must not gate acquisition. + """ + def __init__( - self, driver: ADBaseIO, driver_armed_signal: SignalR[bool] | None = None + self, + driver: ADBaseIO, + driver_armed_signal: SignalR[bool] | None = None, + wait_for_plugins: bool = True, ): self.driver = driver if driver_armed_signal is not None: self.driver_armed_signal = driver_armed_signal else: self.driver_armed_signal = driver.acquire + self.wait_for_plugins = wait_for_plugins self.acquire_status: AsyncStatus | None = None + async def ensure_ready(self): + # Stop first: WaitForPlugins is latched and has no _RBV, so set it once + # per scan while the driver is known to be idle rather than on the + # latency path before every arm. Always written, never left at whatever + # the IOC happened to boot with, so a scan does not depend on who + # touched the PV last. + await self.ensure_stopped() + await self.driver.wait_for_plugins.set(self.wait_for_plugins) + async def start_acquiring(self): self.acquire_status = await set_and_wait_for_other_value( set_signal=self.driver.acquire, @@ -46,6 +76,15 @@ async def ensure_stopped(self): class ADContAcqAcquireLogic(DetectorAcquireLogic): + """Start and stop capture on a circular buffer, leaving the driver acquiring. + + Deliberately has no `wait_for_plugins` equivalent to [](#ADAcquireLogic). + That setting gates the driver's `AcquireBusy` going to 0 at the end of an + acquisition, and here the driver is always acquiring -- there is no such + transition to wait on. `wait_for_idle` waits on the circular buffer's + capture instead. + """ + def __init__(self, driver: ADBaseIO, cb_plugin: NDCircularBuffIO): self.driver = driver self.cb_plugin = cb_plugin diff --git a/src/ophyd_async/epics/adcore/_data_logic.py b/src/ophyd_async/epics/adcore/_data_logic.py index 9beb01237d..13cd087d8f 100644 --- a/src/ophyd_async/epics/adcore/_data_logic.py +++ b/src/ophyd_async/epics/adcore/_data_logic.py @@ -1,5 +1,5 @@ import asyncio -from collections.abc import Callable, Sequence +from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass from pathlib import PureWindowsPath from typing import Any, Generic @@ -8,11 +8,11 @@ import numpy as np from ophyd_async.core import ( + Array1D, DetectorDataLogic, EnableDisable, PathInfo, PathProvider, - SignalDataProvider, SignalR, StreamableDataProvider, StreamResourceDataProvider, @@ -29,26 +29,13 @@ NDArrayBaseIO, NDFileHDF5IO, NDPluginBaseIO, + NDPluginBaseIOT, NDPluginFileIO, - NDPluginFileIOT, + NDStatsIO, + plugin_is_enabled, ) from ._ndattribute import NDAttributeDataType, NDAttributePvDbrType - - -@dataclass -class PluginSignalDataLogic(DetectorDataLogic): - driver: ADBaseIO - signal: SignalR - hinted: bool = True - - async def prepare_single(self, datakey_name: str) -> SignalDataProvider: - # Need to wait for all the plugins to have finished before we can read - # the plugin signal - await self.driver.wait_for_plugins.set(True) - return SignalDataProvider(self.signal) - - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return [self.signal.name] if self.hinted else [] +from ._stats_time_series import StatsTimeSeriesDataLogic @dataclass @@ -162,7 +149,15 @@ async def prepare_file_paths( @dataclass -class ADHDFDataLogic(DetectorDataLogic): +class ADHDFDataCtx: + """What `ADHDFDataLogic.make_data_provider` worked out for `start` to write.""" + + path_info: PathInfo + frames_per_chunk: int + + +@dataclass +class ADHDFDataLogic(DetectorDataLogic[ADHDFDataCtx]): """Data logic for AreaDetector HDF5 writer plugin. :param array_description: Signals describing the NDArray shape and data type. @@ -181,34 +176,35 @@ class ADHDFDataLogic(DetectorDataLogic): plugins: Sequence[NDPluginBaseIO] = () datakey_suffix: str = "" hinted: bool = True + #: Whether to switch the plugin on when starting. Left True, the plugin is + #: enabled as part of starting, which is the behaviour beamlines rely on. + #: Set False to follow whatever the plugin is set to instead: a disabled + #: plugin then makes no provider, so another data logic can serve the scan. + enable_callbacks: bool = True - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, ADHDFDataCtx] | None: + # A file writer captures for as long as it is told to, so it does not + # need to know how many collections the scan will take. + if not self.enable_callbacks and not await plugin_is_enabled(self.writer): + return None # Work out where to write path_info = self.path_provider(datakey_name) - # Determine number of frames that will be saved per HDF chunk. - # On a fresh IOC startup, this is set to zero until the first capture, - # so if it is zero, set it to 1. - frames_per_chunk = await self.writer.num_frames_chunks.get_value() - if frames_per_chunk == 0: - frames_per_chunk = 1 - await self.writer.num_frames_chunks.set(frames_per_chunk) - # Setup the HDF writer - await asyncio.gather( - self.writer.chunk_size_auto.set(True), - self.writer.num_extra_dims.set(0), - self.writer.lazy_open.set(True), - self.writer.swmr_mode.set(True), - self.writer.xml_file_name.set(""), - self.writer.enable_callbacks.set(EnableDisable.ENABLE), - prepare_file_paths( - path_info=path_info, file_template="%s%s.h5", writer=self.writer - ), - ) - # Start capturing - await set_and_wait_for_value( - self.writer.capture, True, wait_for_set_completion=False - ) - # Return a provider that reflects what we have made + # Size the HDF chunk from the frame period and the target flush period: + # e.g. 400 Hz frames (period 2.5 ms) with a 0.5 s flush period gives 200 + # frames per chunk, so the file is flushed at ~2 Hz (#1309). Only when + # both are known (non-zero); otherwise fall back to reading the chunk + # size back from the IOC, forcing a fresh-startup 0 to 1. + if flush_period > 0 and period > 0: + frames_per_chunk = max(1, round(flush_period / period)) + else: + frames_per_chunk = await self.writer.num_frames_chunks.get_value() or 1 + # Describe what we would write main_dataset = await get_ndarray_resource_info( array_description=self.array_description, data_key=datakey_name, @@ -231,13 +227,38 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: ) for name, (dtype_numpy, source) in ndattribute_dtype_sources.items() ] - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[main_dataset] + ndattribute_datasets, mimetype="application/x-hdf5", collections_written_signal=self.writer.num_captured, flush_signal=self.writer.flush_now, ) + return provider, ADHDFDataCtx(path_info, frames_per_chunk) + + async def start(self, ctx: ADHDFDataCtx) -> None: + path_info = ctx.path_info + # Write back the chunk size the provider was described with, so the file + # matches the chunk_shape in its stream resource either way + await self.writer.num_frames_chunks.set(ctx.frames_per_chunk) + # Setup the HDF writer + coros: list[Awaitable] = [ + self.writer.chunk_size_auto.set(True), + self.writer.num_extra_dims.set(0), + self.writer.lazy_open.set(True), + self.writer.swmr_mode.set(True), + self.writer.xml_file_name.set(""), + prepare_file_paths( + path_info=path_info, file_template="%s%s.h5", writer=self.writer + ), + ] + if self.enable_callbacks: + coros.append(self.writer.enable_callbacks.set(EnableDisable.ENABLE)) + await asyncio.gather(*coros) + # Start capturing + await set_and_wait_for_value( + self.writer.capture, True, wait_for_set_completion=False + ) async def stop(self) -> None: await stop_busy_record(self.writer.capture) @@ -247,7 +268,7 @@ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @dataclass -class ADMultipartDataLogic(DetectorDataLogic): +class ADMultipartDataLogic(DetectorDataLogic[PathInfo]): """Data logic for multipart AreaDetector file writers (e.g. JPEG, TIFF). :param array_description: Signals describing the NDArray shape and data type. @@ -268,29 +289,24 @@ class ADMultipartDataLogic(DetectorDataLogic): datakey_suffix: str = "" hinted: bool = True - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, PathInfo]: + # A multipart writer writes one file per frame, so there is nothing to + # size from the period, and it writes for as long as it is told to. # Work out where to write path_info = self.path_provider(datakey_name) - # Setup the file writer - await asyncio.gather( - self.writer.lazy_open.set(True), - prepare_file_paths( - path_info=path_info, - file_template="%s%s_%6.6d" + self.extension, - writer=self.writer, - ), - ) - # Start capturing - await set_and_wait_for_value( - self.writer.capture, True, wait_for_set_completion=False - ) - # Return a provider that reflects what we have made + # Describe what we would write main_dataset = await get_ndarray_resource_info( array_description=self.array_description, data_key=datakey_name, parameters={"template": path_info.filename + "_{:06d}" + self.extension}, ) - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( # TODO: remove the type ignore after # https://github.com/bluesky/ophyd-async/issues/1186 uri=path_info.directory_uri, @@ -298,6 +314,22 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: mimetype=self.mimetype, collections_written_signal=self.writer.num_captured, ) + return provider, path_info + + async def start(self, ctx: PathInfo) -> None: + # Setup the file writer + await asyncio.gather( + self.writer.lazy_open.set(True), + prepare_file_paths( + path_info=ctx, + file_template="%s%s_%6.6d" + self.extension, + writer=self.writer, + ), + ) + # Start capturing + await set_and_wait_for_value( + self.writer.capture, True, wait_for_set_completion=False + ) async def stop(self) -> None: await stop_busy_record(self.writer.capture) @@ -307,17 +339,17 @@ def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: @dataclass -class ADWriterFactory(Generic[NDPluginFileIOT]): - """Factory that creates a file-writer plugin and its matching data logic. +class ADWriterFactory(Generic[NDPluginBaseIOT]): + """Factory that creates a writer plugin and its matching data logic. - Construct using the classmethods `hdf`, `jpeg`, or `tiff`, then pass one - or more instances to `AreaDetector` as positional `*writer_factories` + Construct using the classmethods `hdf`, `jpeg`, `tiff` or `stats`, then pass + one or more instances to `AreaDetector` as positional `*writer_factories` arguments. When the detector is initialised `__call__` is invoked with the detector's PV `prefix`, its `driver`, and the flat list of extra `plugins`; it returns the writer device and the corresponding `DetectorDataLogic`. - :param writer_cls: Concrete `NDPluginFileIO` subclass to instantiate. + :param writer_cls: Concrete `NDPluginBaseIO` subclass to instantiate. :param writer_suffix: PV suffix appended to *prefix* to form the writer's PV prefix. :param writer_name: Attribute name under which the writer device is stored on the @@ -336,7 +368,7 @@ class ADWriterFactory(Generic[NDPluginFileIOT]): that builds the data logic given the already-constructed writer. """ - writer_cls: type[NDPluginFileIOT] + writer_cls: type[NDPluginBaseIOT] writer_suffix: str writer_name: str datakey_suffix: str @@ -344,7 +376,7 @@ class ADWriterFactory(Generic[NDPluginFileIOT]): NDArrayDescription | Callable[[ADBaseIO], NDArrayDescription] | None ) data_logic_factory: Callable[ - [NDPluginFileIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], + [NDPluginBaseIOT, NDArrayDescription, ADBaseIO, Sequence[NDPluginBaseIO]], DetectorDataLogic, ] @@ -353,7 +385,7 @@ def __call__( prefix: str, driver: ADBaseIO, plugins: Sequence[NDPluginBaseIO], - ) -> tuple[NDPluginFileIOT, DetectorDataLogic]: + ) -> tuple[NDPluginBaseIOT, DetectorDataLogic]: """Instantiate the writer plugin and build the data logic. :param prefix: EPICS PV prefix for the detector (same as `AreaDetector.prefix`). @@ -390,6 +422,7 @@ def hdf( | Callable[[ADBaseIO], NDArrayDescription] | None = None, hinted: bool = True, + enable_callbacks: bool = True, ) -> "ADWriterFactory[NDFileHDF5IO]": """Create a factory for an HDF5 file writer. @@ -406,6 +439,10 @@ def hdf( when the shape/type comes from a plugin rather than the main driver (e.g. an ROI plugin). :param hinted: Include the image in hints. Set False for BestEffortCallback. + :param enable_callbacks: Whether to switch the plugin on when starting, + defaults to ``True``. Set ``False`` to follow whatever the plugin is + set to instead: a disabled plugin then writes nothing and another + data logic can serve the scan. """ return ADWriterFactory( writer_cls=NDFileHDF5IO, @@ -421,6 +458,7 @@ def hdf( plugins=list(plugins), datakey_suffix=datakey_suffix, hinted=hinted, + enable_callbacks=enable_callbacks, ), ) @@ -513,3 +551,50 @@ def tiff( ) ), ) + + @staticmethod + def stats( + writer_suffix: str = "STAT:", + writer_name: str = "stats", + datakey_suffix: str = "", + stat_signals: Callable[ + [NDStatsIO], Sequence[tuple[str, SignalR[Array1D[np.float64]]]] + ] + | None = None, + enable_callbacks: bool = True, + ) -> "ADWriterFactory[NDStatsIO]": + """Create a factory for an NDPluginStats time series. + + Unlike `hdf`, `jpeg` and `tiff`, this writes no file: the stats plugin's + time series is a fixed-length buffer read back as event pages, for + detectors that cannot write assets (see `StatsTimeSeriesDataLogic`). + + :param enable_callbacks: Whether to switch the plugin on when starting, + defaults to ``True``. Set ``False`` to follow whatever the plugin is + set to instead: a disabled plugin then produces nothing. + :param writer_suffix: PV suffix for the NDPluginStats plugin, defaults to + ``STAT:``. + :param writer_name: + Attribute name for the plugin on the detector, defaults to ``"stats"``. + :param datakey_suffix: Suffix appended to the datakey name, defaults to ``""``. + :param stat_signals: + Callable ``(stats) → [(suffix, array_signal), ...]`` choosing which + statistics to expose, given the constructed plugin. Left as ``None`` + (the default), only the ``Total`` series is exposed under the bare + datakey name. + """ + return ADWriterFactory( + writer_cls=NDStatsIO, + writer_suffix=writer_suffix, + writer_name=writer_name, + datakey_suffix=datakey_suffix, + array_description=None, + data_logic_factory=lambda writer, desc, driver, plugins: ( + StatsTimeSeriesDataLogic( + stats=writer, + stat_signals=list(stat_signals(writer)) if stat_signals else [], + datakey_suffix=datakey_suffix, + enable_callbacks=enable_callbacks, + ) + ), + ) diff --git a/src/ophyd_async/epics/adcore/_detector.py b/src/ophyd_async/epics/adcore/_detector.py index f624db83d4..198fa5dc7e 100644 --- a/src/ophyd_async/epics/adcore/_detector.py +++ b/src/ophyd_async/epics/adcore/_detector.py @@ -1,12 +1,16 @@ from collections.abc import Mapping, Sequence +from functools import cached_property from typing import Generic from ophyd_async.core import ( DetectorAcquireLogic, + DetectorDataLogic, + DetectorLogic, DetectorTriggerLogic, SignalR, StandardDetector, ) +from ophyd_async.core import StandardReadableFormat as Format from ._acquire_logic import ADContAcqAcquireLogic from ._data_logic import ADWriterFactory @@ -30,10 +34,13 @@ def __init__( if plugins is not None: for plugin_name, plugin in plugins.items(): setattr(self, plugin_name, plugin) + logics: list[ + DetectorTriggerLogic | DetectorAcquireLogic | DetectorDataLogic + ] = [] if trigger_logic: - self.add_detector_logics(trigger_logic) + logics.append(trigger_logic) if acquire_logic: - self.add_detector_logics(acquire_logic) + logics.append(acquire_logic) if writer_factories: if prefix is None: raise ValueError("prefix is required when writer_factories are given") @@ -47,12 +54,22 @@ def __init__( for factory in writer_factories: writer, data_logic = factory(prefix, driver, plugin_list) setattr(self, factory.writer_name, writer) - self.add_detector_logics(data_logic) - self.add_config_signals( - self.driver.acquire_period, self.driver.acquire_time, *config_sigs - ) + logics.append(data_logic) + # The driver is always wired up, so register it whole and let ADBaseIO + # declare which of its signals are configuration. Plugins are + # deliberately *not* registered: a detector normally carries far more + # plugins than are wired into the chain, so opt in per plugin with + # `set_readable_format(det.stats, Format.CHILD)` instead. + self.set_readable_format(self.driver, Format.CHILD) + for signal in config_sigs: + self.set_readable_format(signal, Format.CONFIG_SIGNAL) + self._logic = DetectorLogic(*logics) super().__init__(name=name) + @cached_property + def logic(self) -> DetectorLogic: + return self._logic + def get_plugin( self, name: str, plugin_type: type[NDPluginBaseIOT] = NDPluginBaseIO ) -> NDPluginBaseIOT: @@ -68,7 +85,7 @@ def get_plugin( class ContAcqDetector(AreaDetector[ADBaseIO]): - """Create an ADSimDetector AreaDetector instance. + """Create an continuously acquiring AreaDetector instance. :param prefix: EPICS PV prefix for the detector :param writer_factories: Factories for file writer plugins and their data logics @@ -82,21 +99,22 @@ def __init__( self, prefix: str, *writer_factories: ADWriterFactory, - driver_suffix="cam1:", - cb_suffix="CB1:", + driver_suffix: str = "cam1:", + cb_suffix: str = "CB1:", + cb_plugin_name: str = "cb", plugins: dict[str, NDPluginBaseIO] | None = None, config_sigs: Sequence[SignalR] = (), name: str = "", ) -> None: driver = ADBaseIO(prefix + driver_suffix) - cb_plugin = NDCircularBuffIO(prefix + cb_suffix) + cb_plugin = NDCircularBuffIO(prefix + cb_suffix, name=cb_plugin_name) super().__init__( driver, prefix, *writer_factories, acquire_logic=ADContAcqAcquireLogic(driver, cb_plugin), trigger_logic=ADContAcqTriggerLogic(driver, cb_plugin), - plugins=(plugins or {}) | {"cb": cb_plugin}, + plugins=(plugins or {}) | {cb_plugin.name: cb_plugin}, config_sigs=config_sigs, name=name, ) diff --git a/src/ophyd_async/epics/adcore/_io.py b/src/ophyd_async/epics/adcore/_io.py index b48a0e6fef..91a8347883 100644 --- a/src/ophyd_async/epics/adcore/_io.py +++ b/src/ophyd_async/epics/adcore/_io.py @@ -1,16 +1,21 @@ from typing import Annotated as A from typing import TypeVar +import numpy as np + from ophyd_async.core import ( + Array1D, DeviceVector, EnableDisable, SignalR, SignalRW, + StandardReadable, StrictEnum, SubsetEnum, SupersetEnum, non_zero, ) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.epics.core import EpicsDevice, EpicsOptions, PvSuffix # Common classes for drivers and plugins @@ -107,7 +112,7 @@ class ADState(StrictEnum): ADBaseIOT = TypeVar("ADBaseIOT", bound="ADBaseIO") -class ADBaseIO(NDArrayBaseIO): +class ADBaseIO(StandardReadable, NDArrayBaseIO): """Base class from which areaDetector drivers are derived. This mirrors the interface provided by ADCore/db/ADBase.template. @@ -118,10 +123,14 @@ class ADBaseIO(NDArrayBaseIO): compatibility with changes made in https://github.com/areaDetector/ADCore/blob/master/RELEASE.md#asynndarraydriver-addriver """ - acquire_time: A[SignalRW[float], PvSuffix.rbv("AcquireTime")] - acquire_period: A[SignalRW[float], PvSuffix.rbv("AcquirePeriod")] - num_images: A[SignalRW[int], PvSuffix.rbv("NumImages")] - image_mode: A[SignalRW[ADImageMode], PvSuffix.rbv("ImageMode")] + acquire_time: A[SignalRW[float], PvSuffix.rbv("AcquireTime"), Format.CONFIG_SIGNAL] + acquire_period: A[ + SignalRW[float], PvSuffix.rbv("AcquirePeriod"), Format.CONFIG_SIGNAL + ] + num_images: A[SignalRW[int], PvSuffix.rbv("NumImages"), Format.CONFIG_SIGNAL] + image_mode: A[ + SignalRW[ADImageMode], PvSuffix.rbv("ImageMode"), Format.CONFIG_SIGNAL + ] detector_state: A[SignalR[ADState], PvSuffix("DetectorState_RBV")] # The following signals have been moved from NDArrayBaseIO for backwards @@ -131,11 +140,13 @@ class ADBaseIO(NDArrayBaseIO): # There is no _RBV for this one wait_for_plugins: A[SignalRW[bool], PvSuffix("WaitForPlugins")] - manufacturer: A[SignalR[str], PvSuffix("Manufacturer_RBV")] - model: A[SignalR[str], PvSuffix("Model_RBV")] - serial_number: A[SignalR[str], PvSuffix("SerialNumber_RBV")] - sdk_version: A[SignalR[str], PvSuffix("SDKVersion_RBV")] - firmware_version: A[SignalR[str], PvSuffix("FirmwareVersion_RBV")] + manufacturer: A[SignalR[str], PvSuffix("Manufacturer_RBV"), Format.CONFIG_SIGNAL] + model: A[SignalR[str], PvSuffix("Model_RBV"), Format.CONFIG_SIGNAL] + serial_number: A[SignalR[str], PvSuffix("SerialNumber_RBV"), Format.CONFIG_SIGNAL] + sdk_version: A[SignalR[str], PvSuffix("SDKVersion_RBV"), Format.CONFIG_SIGNAL] + firmware_version: A[ + SignalR[str], PvSuffix("FirmwareVersion_RBV"), Format.CONFIG_SIGNAL + ] # Classes for plugins @@ -156,15 +167,28 @@ class NDPluginBaseIO(NDArrayBaseIO): queue_size: A[SignalRW[int], PvSuffix.rbv("QueueSize")] -class NDROIIO(NDPluginBaseIO): +async def plugin_is_enabled(plugin: NDPluginBaseIO) -> bool: + """Whether a plugin is switched on, so is receiving NDArrays. + + The nearest thing to "is this plugin in the chain": a plugin with callbacks + disabled certainly is not, though an enabled one may still have its + `nd_array_port` pointed somewhere else. Parsing the whole plugin graph was + rejected in ADR 0020 as too large. + """ + return await plugin.enable_callbacks.get_value() is EnableDisable.ENABLE + + +class NDROIIO(StandardReadable, NDPluginBaseIO): """Plugin for taking a region of an NDArray. This mirrors the interface provided by ADCore/db/NDROI.template. See HTML docs at https://areadetector.github.io/areaDetector/ADCore/NDPluginROI.html """ - size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] - size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] + min_x: A[SignalRW[int], PvSuffix.rbv("MinX"), Format.CONFIG_SIGNAL] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY"), Format.CONFIG_SIGNAL] + size_x: A[SignalRW[int], PvSuffix.rbv("SizeX"), Format.CONFIG_SIGNAL] + size_y: A[SignalRW[int], PvSuffix.rbv("SizeY"), Format.CONFIG_SIGNAL] size_z: A[SignalRW[int], PvSuffix.rbv("SizeZ")] bin_x: A[SignalRW[int], PvSuffix.rbv("BinX")] @@ -183,7 +207,18 @@ class NDProcessIO(NDPluginBaseIO): scale: A[SignalRW[float], PvSuffix.rbv("Scale")] -class NDStatsIO(NDPluginBaseIO): +class NDStatsTSAcquireMode(StrictEnum): + """Whether an NDPluginStats time series stops when full or wraps around. + + Mirrors the ``TSAcquireMode`` record in + ADCore/db/NDPluginTimeSeries.template. + """ + + FIXED_LENGTH = "Fixed length" + CIRCULAR_BUFFER = "Circ. buffer" + + +class NDStatsIO(StandardReadable, NDPluginBaseIO): """Plugin for computing statistics from an image or ROI within an image. This mirrors the interface provided by ADCore/db/NDStats.template. @@ -191,58 +226,94 @@ class NDStatsIO(NDPluginBaseIO): """ # Basic statistics - compute_statistics: A[SignalRW[bool], PvSuffix.rbv("ComputeStatistics")] - bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth")] - total: A[SignalR[float], PvSuffix("Total_RBV")] + compute_statistics: A[ + SignalRW[bool], PvSuffix.rbv("ComputeStatistics"), Format.CONFIG_SIGNAL + ] + bgd_width: A[SignalRW[int], PvSuffix.rbv("BgdWidth"), Format.CONFIG_SIGNAL] + min_value: A[ + SignalR[float], PvSuffix("MinValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + min_x: A[SignalR[float], PvSuffix("MinX_RBV"), Format.UNCACHED_SIGNAL] + min_y: A[SignalR[float], PvSuffix("MinY_RBV"), Format.UNCACHED_SIGNAL] + max_value: A[ + SignalR[float], PvSuffix("MaxValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + max_x: A[SignalR[float], PvSuffix("MaxX_RBV"), Format.UNCACHED_SIGNAL] + max_y: A[SignalR[float], PvSuffix("MaxY_RBV"), Format.UNCACHED_SIGNAL] + mean_value: A[ + SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + sigma: A[SignalR[float], PvSuffix("Sigma_RBV")] + total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] + net: A[SignalR[float], PvSuffix("Net_RBV")] + # Time series. NDStats delegates its time series to an embedded + # NDPluginTimeSeries instance, so the controls live under an inner "TS:" + # prefix while the per-statistic arrays are named by NDStats itself at the + # plugin's own prefix. ts_num_points sizes the buffer, ts_acquire erases and + # starts it (writing 1 clears the arrays and resets ts_current_point to 0), + # ts_current_point reports progress, ts_total holds the Total series and + # ts_timestamp holds the per-point acquisition times. These are not + # registered as readables: the scalars above serve read(), and the arrays + # are consumed by StatsTimeSeriesDataLogic as event pages. + ts_acquire: A[SignalRW[bool], PvSuffix("TS:TSAcquire")] + ts_num_points: A[SignalRW[int], PvSuffix("TS:TSNumPoints")] + ts_current_point: A[SignalR[int], PvSuffix("TS:TSCurrentPoint")] + ts_acquiring: A[SignalR[bool], PvSuffix("TS:TSAcquiring")] + ts_acquire_mode: A[SignalRW[NDStatsTSAcquireMode], PvSuffix.rbv("TS:TSAcquireMode")] + ts_total: A[SignalR[Array1D[np.float64]], PvSuffix("TSTotal")] + ts_timestamp: A[SignalR[Array1D[np.float64]], PvSuffix("TSTimestamp")] # Centroid statistics - compute_centroid: A[SignalRW[bool], PvSuffix.rbv("ComputeCentroid")] - centroid_threshold: A[SignalRW[float], PvSuffix.rbv("CentroidThreshold")] + compute_centroid: A[ + SignalRW[bool], PvSuffix.rbv("ComputeCentroid"), Format.CONFIG_SIGNAL + ] + centroid_threshold: A[ + SignalRW[float], PvSuffix.rbv("CentroidThreshold"), Format.CONFIG_SIGNAL + ] + centroid_x: A[SignalR[float], PvSuffix("CentroidX_RBV")] + centroid_y: A[SignalR[float], PvSuffix("CentroidY_RBV")] + sigma_x: A[SignalR[float], PvSuffix("SigmaX_RBV")] + sigma_y: A[SignalR[float], PvSuffix("SigmaY_RBV")] + sigma_xy: A[SignalR[float], PvSuffix("SigmaXY_RBV")] # X and Y Profiles - compute_profiles: A[SignalRW[bool], PvSuffix.rbv("ComputeProfiles")] + compute_profiles: A[ + SignalRW[bool], PvSuffix.rbv("ComputeProfiles"), Format.CONFIG_SIGNAL + ] profile_size_x: A[SignalR[int], PvSuffix("ProfileSizeX_RBV")] profile_size_y: A[SignalR[int], PvSuffix("ProfileSizeY_RBV")] - cursor_x: A[SignalRW[int], PvSuffix.rbv("CursorX")] - cursor_y: A[SignalRW[int], PvSuffix.rbv("CursorY")] + cursor_x: A[SignalRW[int], PvSuffix.rbv("CursorX"), Format.CONFIG_SIGNAL] + cursor_y: A[SignalRW[int], PvSuffix.rbv("CursorY"), Format.CONFIG_SIGNAL] # Array Histogram - compute_histogram: A[SignalRW[bool], PvSuffix.rbv("ComputeHistogram")] - hist_size: A[SignalRW[int], PvSuffix.rbv("HistSize")] - hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin")] - hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax")] + compute_histogram: A[ + SignalRW[bool], PvSuffix.rbv("ComputeHistogram"), Format.CONFIG_SIGNAL + ] + hist_size: A[SignalRW[int], PvSuffix.rbv("HistSize"), Format.CONFIG_SIGNAL] + hist_min: A[SignalRW[float], PvSuffix.rbv("HistMin"), Format.CONFIG_SIGNAL] + hist_max: A[SignalRW[float], PvSuffix.rbv("HistMax"), Format.CONFIG_SIGNAL] -class NDROIStatNIO(EpicsDevice): +class NDROIStatNIO(StandardReadable, EpicsDevice): """Defines the parameters for a single ROI used for statistics calculation. - Each instance represents a single ROI, with attributes for its position - (min_x, min_y) and size (size_x, size_y), as well as a name and use status. - This mirrors the interface provided by ADCore/db/NDROIStatN.template. See definition in ADApp/pluginSrc/NDPluginROIStat.h in https://github.com/areaDetector/ADCore. - - Attributes: - name: The name of the ROI. - use: Flag indicating whether the ROI is used. - min_x: The start X-coordinate of the ROI. - min_y: The start Y-coordinate of the ROI. - size_x: The width of the ROI. - size_y: The height of the ROI. - min_value: Minimum count value in the ROI. - max_value: Maximum count value in the ROI. - mean_value: Mean counts value in the ROI. - total: Total counts in the ROI. """ name_: A[SignalRW[str], PvSuffix("Name")] - use: A[SignalRW[bool], PvSuffix.rbv("Use")] - min_x: A[SignalRW[int], PvSuffix.rbv("MinX")] - min_y: A[SignalRW[int], PvSuffix.rbv("MinY")] - size_x: A[SignalRW[int], PvSuffix.rbv("SizeX")] - size_y: A[SignalRW[int], PvSuffix.rbv("SizeY")] - # stats - min_value: A[SignalR[float], PvSuffix("MinValue_RBV")] - max_value: A[SignalR[float], PvSuffix("MaxValue_RBV")] - mean_value: A[SignalR[float], PvSuffix("MeanValue_RBV")] - total: A[SignalR[float], PvSuffix("Total_RBV")] + use: A[SignalRW[bool], PvSuffix.rbv("Use"), Format.CONFIG_SIGNAL] + min_x: A[SignalRW[int], PvSuffix.rbv("MinX"), Format.CONFIG_SIGNAL] + min_y: A[SignalRW[int], PvSuffix.rbv("MinY"), Format.CONFIG_SIGNAL] + size_x: A[SignalRW[int], PvSuffix.rbv("SizeX"), Format.CONFIG_SIGNAL] + size_y: A[SignalRW[int], PvSuffix.rbv("SizeY"), Format.CONFIG_SIGNAL] + min_value: A[ + SignalR[float], PvSuffix("MinValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + max_value: A[ + SignalR[float], PvSuffix("MaxValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + mean_value: A[ + SignalR[float], PvSuffix("MeanValue_RBV"), Format.HINTED_UNCACHED_SIGNAL + ] + total: A[SignalR[float], PvSuffix("Total_RBV"), Format.HINTED_UNCACHED_SIGNAL] class NDROIStatIO(NDPluginBaseIO): diff --git a/src/ophyd_async/epics/adcore/_stats_time_series.py b/src/ophyd_async/epics/adcore/_stats_time_series.py new file mode 100644 index 0000000000..4e8a4c1cb1 --- /dev/null +++ b/src/ophyd_async/epics/adcore/_stats_time_series.py @@ -0,0 +1,96 @@ +import asyncio +from collections.abc import Awaitable, Sequence +from dataclasses import dataclass, field + +import numpy as np + +from ophyd_async.core import ( + Array1D, + DetectorDataLogic, + EnableDisable, + EventPageDataProvider, + SignalR, +) + +from ._io import NDStatsIO, NDStatsTSAcquireMode, plugin_is_enabled + + +@dataclass +class StatsTimeSeriesDataLogic(DetectorDataLogic[int]): + """Bounded data logic for an `NDPluginStats` time series. + + For detectors that write no file: the stats plugin holds each statistic in a + fixed-length buffer that must be sized before acquisition, so this is a + data logic that emits event pages. One plugin has + one time-series control (`ts_acquire`, `ts_num_points`) shared across many + arrays, so one logic covers many arrays with the control embedded. + + The buffer is sized and armed in `start` by writing 1 to + ``ts_acquire``, which erases the arrays and resets the current point before + the detector's acquire logic drives the camera; its frames then feed the time + series via NDArray callbacks. A detector that writes a file should pull stats + into the file as NDAttributes instead (see `ADHDFDataLogic`). + + :param stats: the stats plugin whose time series to read + :param stat_signals: datakey suffix to the array signal for each statistic to + expose; defaults to the ``Total`` series under the bare datakey name + """ + + stats: NDStatsIO + stat_signals: Sequence[tuple[str, SignalR[Array1D[np.float64]]]] = field( + default_factory=list + ) + datakey_suffix: str = "" + #: Whether to switch the plugin on when starting. Left True, the plugin is + #: enabled as part of starting. Set False to follow whatever the plugin is + #: set to instead: a disabled plugin then makes no provider. + enable_callbacks: bool = True + + def _arrays(self, datakey_name: str) -> dict[str, SignalR[Array1D[np.float64]]]: + signals = self.stat_signals or [("", self.stats.ts_total)] + return {datakey_name + suffix: signal for suffix, signal in signals} + + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[EventPageDataProvider, int] | None: + # The buffer is sized by count, not rate, so neither period is used. + if num_collections == 0: + # A finite buffer cannot serve an unbounded scan + return None + if not self.enable_callbacks and not await plugin_is_enabled(self.stats): + return None + # The datakeys are known from the arrays we were configured with, so + # nothing has to be armed to describe them + provider = EventPageDataProvider( + self._arrays(datakey_name), + self.stats.ts_current_point, + self.stats.ts_timestamp, + ) + return provider, num_collections + + async def start(self, ctx: int) -> None: + # Size the buffer to the collection count make_data_provider was given, + # and put it in fixed-length mode, before arming it. + coros: list[Awaitable] = [ + self.stats.compute_statistics.set(True), + self.stats.ts_num_points.set(ctx), + self.stats.ts_acquire_mode.set(NDStatsTSAcquireMode.FIXED_LENGTH), + ] + if self.enable_callbacks: + coros.append(self.stats.enable_callbacks.set(EnableDisable.ENABLE)) + await asyncio.gather(*coros) + # Writing 1 to ts_acquire clears the arrays and resets ts_current_point to + # 0, so the buffer is armed and empty before the detector's frames arrive. + # This is the data logic performing the erase itself, which is why + # trigger()'s zero baseline is correct (see ADR 0022). + await self.stats.ts_acquire.set(True) + + async def stop(self) -> None: + await self.stats.ts_acquire.set(False) + + def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: + return list(self._arrays(datakey_name)) diff --git a/src/ophyd_async/epics/adcore/_trigger_logic.py b/src/ophyd_async/epics/adcore/_trigger_logic.py index 72a3f5a0f4..49fb3cab54 100644 --- a/src/ophyd_async/epics/adcore/_trigger_logic.py +++ b/src/ophyd_async/epics/adcore/_trigger_logic.py @@ -30,10 +30,25 @@ async def prepare_exposures( await asyncio.gather(*coros) -async def trigger_info_from_num_images(driver: ADBaseIO) -> TriggerInfo: - """Default TriggerInfo for AD detectors, reading num_images from the driver.""" - num = await driver.num_images.get_value() - return TriggerInfo(collections_per_event=max(1, num)) +async def trigger_info_from_driver(driver: ADBaseIO) -> TriggerInfo: + """Default TriggerInfo for AD detectors, read back from the driver. + + Reads num_images, acquire_time and acquire_period so the returned + TriggerInfo carries the current livetime and deadtime as well as the + collections per event. This lets prepare() fill in a livetime of 0 ("use + current") with the real exposure period, which a data logic needs to size + its chunks or buffer. + """ + num, livetime, period = await asyncio.gather( + driver.num_images.get_value(), + driver.acquire_time.get_value(), + driver.acquire_period.get_value(), + ) + return TriggerInfo( + collections_per_event=max(1, num), + livetime=livetime, + deadtime=max(0.0, period - livetime), + ) @dataclass diff --git a/src/ophyd_async/epics/adkinetix.py b/src/ophyd_async/epics/adkinetix.py index 1d3774f0d2..72a81037d4 100644 --- a/src/ophyd_async/epics/adkinetix.py +++ b/src/ophyd_async/epics/adkinetix.py @@ -22,7 +22,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .core import PvSuffix @@ -81,7 +81,7 @@ async def prepare_level(self, num: int): await prepare_exposures(self.driver, num) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class KinetixDetector(AreaDetector[KinetixDriverIO]): diff --git a/src/ophyd_async/epics/admerlin.py b/src/ophyd_async/epics/admerlin.py index 04bba91b0e..401f6dbded 100644 --- a/src/ophyd_async/epics/admerlin.py +++ b/src/ophyd_async/epics/admerlin.py @@ -26,7 +26,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) __all__ = [ @@ -106,7 +106,7 @@ async def prepare_edge(self, num: int, livetime: float): await prepare_exposures(self.driver, num, livetime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class MerlinDetector(AreaDetector[MerlinDriverIO]): diff --git a/src/ophyd_async/epics/adpilatus.py b/src/ophyd_async/epics/adpilatus.py index 2b038cea81..6008daba78 100644 --- a/src/ophyd_async/epics/adpilatus.py +++ b/src/ophyd_async/epics/adpilatus.py @@ -25,7 +25,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .core import PvSuffix @@ -122,7 +122,7 @@ async def prepare_level(self, num: int): await prepare_exposures(self.driver, num or _MAX_NUM_IMAGE) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class PilatusDetector(AreaDetector[PilatusDriverIO]): diff --git a/src/ophyd_async/epics/adsimdetector.py b/src/ophyd_async/epics/adsimdetector.py index 1a47d7c895..c5ed33bb16 100644 --- a/src/ophyd_async/epics/adsimdetector.py +++ b/src/ophyd_async/epics/adsimdetector.py @@ -18,7 +18,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) __all__ = [ @@ -37,7 +37,7 @@ async def prepare_internal(self, num: int, livetime: float, deadtime: float): await prepare_exposures(self.driver, num, livetime, deadtime) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class SimDetector(AreaDetector[ADBaseIO]): diff --git a/src/ophyd_async/epics/advimba.py b/src/ophyd_async/epics/advimba.py index b44566b852..ac76100473 100644 --- a/src/ophyd_async/epics/advimba.py +++ b/src/ophyd_async/epics/advimba.py @@ -26,7 +26,7 @@ AreaDetector, NDPluginBaseIO, prepare_exposures, - trigger_info_from_num_images, + trigger_info_from_driver, ) from .adgenicam import get_camera_deadtime @@ -96,9 +96,6 @@ class VimbaTriggerLogic(DetectorTriggerLogic): driver: VimbaDriverIO override_deadtime: float | None = None - def config_sigs(self) -> set[SignalR]: - return {self.driver.model} - def get_deadtime(self, config_values: SignalDict) -> float: return get_camera_deadtime( model=config_values[self.driver.model], @@ -130,7 +127,7 @@ async def prepare_level(self, num: int): await prepare_exposures(self.driver, num) async def default_trigger_info(self): - return await trigger_info_from_num_images(self.driver) + return await trigger_info_from_driver(self.driver) class VimbaDetector(AreaDetector[VimbaDriverIO]): diff --git a/src/ophyd_async/epics/demo/_motor.py b/src/ophyd_async/epics/demo/_motor.py index 308bc4f99a..40be2edcfa 100644 --- a/src/ophyd_async/epics/demo/_motor.py +++ b/src/ophyd_async/epics/demo/_motor.py @@ -55,7 +55,7 @@ class DemoMotor(EpicsDevice, StandardReadable, StandardMovable[float]): stop_: A[TriggerableCommand, PvSuffix("Stop.PROC")] @cached_property - def movable_logic(self) -> MovableLogic: + def logic(self) -> MovableLogic: return DemoMotorMoveLogic( readback=self.readback, setpoint=self.setpoint, diff --git a/src/ophyd_async/epics/motor.py b/src/ophyd_async/epics/motor.py index bb89b38982..b4a1903bb9 100644 --- a/src/ophyd_async/epics/motor.py +++ b/src/ophyd_async/epics/motor.py @@ -103,7 +103,7 @@ class MotorFlyableMovableLogic( ): """Combined move + fly logic for a motor record. - A single logic object backs a `Motor`'s `movable_logic` and `flyable_logic`: + A single logic object backs a `Motor`'s `logic`: it implements `MovableLogic` (the motor-record move, limit and timeout logic) and `FlyableLogic` (the fly hooks, which reuse `move`/`check_move` and carry all per-scan state in a `MotorFlyCtx` rather than on the logic). @@ -343,8 +343,8 @@ def __init__(self, prefix: str, name="") -> None: super().__init__(name) @cached_property - def _logic(self) -> MotorFlyableMovableLogic: - """The combined move + fly logic, shared by movable_logic and flyable_logic.""" + def logic(self) -> MotorFlyableMovableLogic: + """The combined move + fly logic, satisfying both mix-ins at once.""" return MotorFlyableMovableLogic( readback=self.user_readback, setpoint=self.user_setpoint, @@ -359,11 +359,3 @@ def _logic(self) -> MotorFlyableMovableLogic: max_velocity=self.max_velocity, motor_egu=self.motor_egu, ) - - @cached_property - def movable_logic(self) -> MotorFlyableMovableLogic: - return self._logic - - @cached_property - def flyable_logic(self) -> MotorFlyableMovableLogic: - return self._logic diff --git a/src/ophyd_async/fastcs/eiger/_detector.py b/src/ophyd_async/fastcs/eiger/_detector.py index 7257bde6a7..3b51efe1c2 100644 --- a/src/ophyd_async/fastcs/eiger/_detector.py +++ b/src/ophyd_async/fastcs/eiger/_detector.py @@ -1,7 +1,11 @@ +from functools import cached_property + from ophyd_async.core import ( + DetectorLogic, PathProvider, SignalR, StandardDetector, + StandardReadableFormat, TriggerableCommand, soft_signal_rw, ) @@ -33,7 +37,7 @@ def __init__( # Need to do this first so the type hints are filled in connector = fastcs_connector(prefix, self) - self.add_detector_logics( + self._logic = DetectorLogic( EigerTriggerLogic(self.detector), EigerAcquireLogic(self.detector, self.arm_when_ready), odin.OdinDataLogic( @@ -44,6 +48,10 @@ def __init__( ), ) - self.add_config_signals(self.pixel_mask) + self.set_readable_format(self.pixel_mask, StandardReadableFormat.CONFIG_SIGNAL) super().__init__(name=name, connector=connector) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/fastcs/jungfrau/_detector.py b/src/ophyd_async/fastcs/jungfrau/_detector.py index 0adb10db3c..a1a0c9c832 100644 --- a/src/ophyd_async/fastcs/jungfrau/_detector.py +++ b/src/ophyd_async/fastcs/jungfrau/_detector.py @@ -1,4 +1,11 @@ -from ophyd_async.core import PathProvider, StandardDetector, soft_signal_rw +from functools import cached_property + +from ophyd_async.core import ( + DetectorLogic, + PathProvider, + StandardDetector, + soft_signal_rw, +) from ophyd_async.fastcs import odin from ophyd_async.fastcs.core import fastcs_connector @@ -29,7 +36,7 @@ def __init__( self.acquisition_type = soft_signal_rw( AcquisitionType, AcquisitionType.STANDARD ) - self.add_detector_logics( + self._logic = DetectorLogic( JungfrauTriggerLogic(self.detector, self.acquisition_type), JungfrauAcquireLogic(self.detector), odin.OdinDataLogic( @@ -39,3 +46,7 @@ def __init__( ), ) super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/fastcs/odin/_data_logic.py b/src/ophyd_async/fastcs/odin/_data_logic.py index 282e707f6f..c25b7134a1 100644 --- a/src/ophyd_async/fastcs/odin/_data_logic.py +++ b/src/ophyd_async/fastcs/odin/_data_logic.py @@ -18,7 +18,7 @@ from ._io import OdinIO -class OdinDataLogic(DetectorDataLogic): +class OdinDataLogic(DetectorDataLogic[None]): def __init__( self, path_provider: PathProvider, @@ -31,7 +31,20 @@ def __init__( self.detector_bit_depth = detector_bit_depth self.pixel_mask = pixel_mask - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, None]: + # Odin sizes its own chunks and writes for as long as it is told to, so + # neither the periods nor the count are used here yet. + # Unlike other data logics this one cannot describe its data without + # starting: the frame shape is only readable from the file processor + # once it is writing. So it does its own writes here and has nothing + # left for start(). That is safe while it is the only kind of provider a + # detector carrying it can produce, since it is then never discarded. # Work out where to write path_info = self.path_provider(datakey_name) # Get the current bit depth @@ -73,13 +86,14 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: dtype_numpy=np.dtype(datatype).str, parameters={"dataset": "/data"}, ) - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( # Should be _vds instead of _000001, see https://github.com/bluesky/ophyd-async/issues/1272 uri=f"{path_info.directory_uri}{filename}_000001.h5", resources=[resource], mimetype="application/x-hdf5", collections_written_signal=self.odin.fp.frames_written, ) + return provider, None async def stop(self) -> None: await asyncio.gather( diff --git a/src/ophyd_async/fastcs/panda/_data_logic.py b/src/ophyd_async/fastcs/panda/_data_logic.py index ef8ec675e2..ac304a4aa4 100644 --- a/src/ophyd_async/fastcs/panda/_data_logic.py +++ b/src/ophyd_async/fastcs/panda/_data_logic.py @@ -1,10 +1,12 @@ import asyncio import logging +from dataclasses import dataclass import numpy as np from ophyd_async.core import ( DetectorDataLogic, + PathInfo, PathProvider, StreamableDataProvider, StreamResourceDataProvider, @@ -16,7 +18,15 @@ logger = logging.getLogger("ophyd_async") -class PandaHDFDataLogic(DetectorDataLogic): +@dataclass +class PandaHDFDataCtx: + """What `PandaHDFDataLogic.make_data_provider` worked out for `start`.""" + + path_info: PathInfo + flush_period: float + + +class PandaHDFDataLogic(DetectorDataLogic[PandaHDFDataCtx]): def __init__( self, path_provider: PathProvider, @@ -25,31 +35,20 @@ def __init__( self.path_provider = path_provider self.data_block = data_block - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, PandaHDFDataCtx]: + # TODO: size the chunk from `period` once the IOC exposes the chunk-size + # signal (see the chunk_shape TODO below). The PandA captures forever, so + # the number of collections is not needed. # Work out where to write path_info = self.path_provider(datakey_name) - # Set create dir depth first to guarantee that callback when setting - # directory path has correct value - await self.data_block.create_directory.set(path_info.create_dir_depth) - # Setup the HDF writer - await asyncio.gather( - self.data_block.flush_period.set(0), - self.data_block.hdf_directory.set(str(path_info.directory_path)), - self.data_block.hdf_file_name.set( - f"{path_info.filename}.h5", - ), - self.data_block.capture_mode.set(PandaCaptureMode.FOREVER), - ) - # Make sure that directory exists or has been created. - if not await self.data_block.directory_exists.get_value() == 1: - raise OSError( - f"Directory {path_info.directory_path} does not exist or " - "is not writable by the PandABlocks-ioc!" - ) - # Start capturing - await self.data_block.capture.set(True) - # Load data from the datasets PV on the panda, update internal - # representation of datasets that the panda will write. + # Load data from the datasets PV on the panda, which tells us which + # datasets it will write capture_table = await self.data_block.datasets.get_value() if len(capture_table) == 0: logger.warning( @@ -70,12 +69,38 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: ) for dataset_name in capture_table.name ] - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=resources, mimetype="application/x-hdf5", collections_written_signal=self.data_block.num_captured, ) + return provider, PandaHDFDataCtx(path_info, flush_period) + + async def start(self, ctx: PandaHDFDataCtx) -> None: + path_info = ctx.path_info + # Set create dir depth first to guarantee that callback when setting + # directory path has correct value + await self.data_block.create_directory.set(path_info.create_dir_depth) + # Setup the HDF writer. The PandA flushes on a timer rather than by + # chunk size, so it takes the flush period directly; 0 means it flushes + # only when its buffer fills. + await asyncio.gather( + self.data_block.flush_period.set(ctx.flush_period), + self.data_block.hdf_directory.set(str(path_info.directory_path)), + self.data_block.hdf_file_name.set( + f"{path_info.filename}.h5", + ), + self.data_block.capture_mode.set(PandaCaptureMode.FOREVER), + ) + # Make sure that directory exists or has been created. + if not await self.data_block.directory_exists.get_value() == 1: + raise OSError( + f"Directory {path_info.directory_path} does not exist or " + "is not writable by the PandABlocks-ioc!" + ) + # Start capturing + await self.data_block.capture.set(True) async def stop(self) -> None: await self.data_block.capture.set(False) diff --git a/src/ophyd_async/fastcs/panda/_detector.py b/src/ophyd_async/fastcs/panda/_detector.py index c365d56fa8..d0197e7e1f 100644 --- a/src/ophyd_async/fastcs/panda/_detector.py +++ b/src/ophyd_async/fastcs/panda/_detector.py @@ -1,6 +1,8 @@ from __future__ import annotations -from ophyd_async.core import PathProvider, StandardDetector +from functools import cached_property + +from ophyd_async.core import DetectorLogic, PathProvider, StandardDetector from ophyd_async.fastcs.core import fastcs_connector from ._acquire_logic import PandaAcquireLogic @@ -23,9 +25,13 @@ def __init__( error_hint = f"Is PandABlocks-ioc at least version {MINIMUM_PANDA_IOC}?" # This has to be first so we make self.pcap connector = fastcs_connector(prefix, self, error_hint) - self.add_detector_logics( + self._logic = DetectorLogic( PandaTriggerLogic(self.pcap), PandaAcquireLogic(self.pcap), PandaHDFDataLogic(path_provider, self.data), ) super().__init__(name=name, connector=connector) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/plan_stubs/_settings.py b/src/ophyd_async/plan_stubs/_settings.py index 5e75c6eb83..638d6a9536 100644 --- a/src/ophyd_async/plan_stubs/_settings.py +++ b/src/ophyd_async/plan_stubs/_settings.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import warnings from collections.abc import Callable, Mapping from typing import Any @@ -9,12 +10,17 @@ from bluesky.utils import MsgGenerator, plan from ophyd_async.core import ( + READABLE_FORMATS_KEY, Device, + ReadableFormats, Settings, SettingsProvider, SignalRW, + StandardReadableFormat, Table, + apply_readable_formats, walk_config_signals, + walk_readable_formats, walk_rw_signals, ) @@ -53,21 +59,39 @@ def get_current_settings( def store_settings( provider: SettingsProvider, name: str, device: Device, only_config: bool = False ) -> MsgGenerator[None]: - """Walk a Device for SignalRWs and store their values. + """Walk a Device for SignalRWs and store their values and readable formats. If `only_config` is True, store only configuration settings on `Configurable`. + Readable formats are stored in both cases, since they are what *determines* + which signals are configuration. + + Values are stored flat, keyed by dotted attribute path. The changes to + readable formats since construction go under the reserved + [](#READABLE_FORMATS_KEY), which is not a valid Python identifier and so + cannot collide with a path. It is written, even if empty, whenever the tree + contains a `StandardReadable`, as applying it resets formats to the class + defaults. :param provider: The provider to store the settings with. :param name: The name to store the settings under. :param device: The Device to walk for SignalRWs. - :param only_config: If True, store only configuration settings. + :param only_config: If True, store only configuration signal values. """ if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: signals = walk_rw_signals(device) - named_values = yield from _get_values_of_signals(signals) - yield from wait_for_awaitable(provider.store(name, named_values)) + data: dict[str, Any] = yield from _get_values_of_signals(signals) + formats = walk_readable_formats(device) + if formats is not None: + data[READABLE_FORMATS_KEY] = { + owner: { + child: None if format is None else format.value + for child, format in entries.items() + } + for owner, entries in formats.items() + } + yield from wait_for_awaitable(provider.store(name, data)) @plan @@ -78,26 +102,64 @@ def retrieve_settings( If `only_config` is True, retrieve only configuration settings on `Configurable`. + A file with no [](#READABLE_FORMATS_KEY), such as one stored before + readable formats existed, gives Settings that leave formats alone. + :param provider: The provider to retrieve the settings from. :param name: The name of the settings to retrieve. :param device: The Device to retrieve the settings for. :param only_config: If True, retrieve only configuration settings. """ - named_values = yield from wait_for_awaitable(provider.retrieve(name)) + # Copy, as the provider may hand back a dict it holds on to and the reserved + # keys are popped out of it below + data = dict((yield from wait_for_awaitable(provider.retrieve(name)))) + # Reserved keys first, so what is left is signal paths. Any other + # key is dropped with a warning: it is either a typo of a reserved key or + # written by a newer version, and a path can never look like that. + # Present but null, as a hand written ":" would be, still + # means reset to the class defaults + stored_formats = ( + data.pop(READABLE_FORMATS_KEY) or {} if READABLE_FORMATS_KEY in data else None + ) + unknown_reserved = sorted(k for k in data if k.startswith("<") and k.endswith(">")) + for key in unknown_reserved: + del data[key] + if unknown_reserved: + warnings.warn( + f"Ignoring unknown reserved keys {unknown_reserved} in settings {name!r}", + stacklevel=2, + ) + readable_formats: ReadableFormats | None = ( + None + if stored_formats is None + else { + owner: { + # A null means "stop this child contributing", so is not coerced + child: None if format is None else StandardReadableFormat(format) + for child, format in entries.items() + } + for owner, entries in stored_formats.items() + } + ) if only_config: signals = yield from wait_for_awaitable(walk_config_signals(device)) else: signals = walk_rw_signals(device) - unknown_names = set(named_values) - set(signals) + unknown_names = set(data) - set(signals) if unknown_names: raise NameError(f"Unknown signal names {sorted(unknown_names)}") - signal_values = {signals[name]: value for name, value in named_values.items()} - return Settings(device, signal_values) + signal_values = {signals[name]: value for name, value in data.items()} + return Settings(device, signal_values, readable_formats) @plan def apply_settings(settings: Settings) -> MsgGenerator[None]: - """Set every SignalRW to the given value in Settings. If value is None ignore it.""" + """Set every SignalRW to the given value in Settings. If value is None ignore it. + + Then apply any readable formats the Settings carries with + [](#apply_readable_formats). Formats go last because they touch no hardware + and cannot fail, so a failed value write leaves them untouched. + """ signal_values = { signal: value for signal, value in settings.items() if value is not None } @@ -105,6 +167,8 @@ def apply_settings(settings: Settings) -> MsgGenerator[None]: for signal, value in signal_values.items(): yield from bps.abs_set(signal, value, group="apply_settings") yield from bps.wait("apply_settings") + if settings.readable_formats is not None: + apply_readable_formats(settings.device, settings.readable_formats) @plan diff --git a/src/ophyd_async/sim/_blob_data_logic.py b/src/ophyd_async/sim/_blob_data_logic.py index 01073f0f62..f927126e49 100644 --- a/src/ophyd_async/sim/_blob_data_logic.py +++ b/src/ophyd_async/sim/_blob_data_logic.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from pathlib import PurePath import numpy as np @@ -16,7 +17,7 @@ HEIGHT = 240 -class BlobDataLogic(DetectorDataLogic): +class BlobDataLogic(DetectorDataLogic[PurePath]): def __init__( self, path_provider: PathProvider, @@ -25,13 +26,19 @@ def __init__( self.path_provider = path_provider self.pattern_generator = pattern_generator - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, PurePath]: + # The sim blob writer uses a fixed chunk shape and writes for as long as + # it is told to, so neither the periods nor the count are needed. # Work out where to write path_info = self.path_provider(datakey_name) - # Open the file write_path = path_info.directory_path / f"{path_info.filename}.h5" - self.pattern_generator.open_file(write_path, WIDTH, HEIGHT) - # Return a provider that reflects what we have made + # Describe what we would write data_resource = StreamResourceInfo( data_key=datakey_name, shape=(HEIGHT, WIDTH), @@ -50,12 +57,16 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: dtype_numpy=np.dtype(np.int64).str, parameters={"dataset": SUM_PATH}, ) - return StreamResourceDataProvider( + provider = StreamResourceDataProvider( uri=f"{path_info.directory_uri}{path_info.filename}.h5", resources=[data_resource, sum_resource], mimetype="application/x-hdf5", collections_written_signal=self.pattern_generator.images_written, ) + return provider, write_path + + async def start(self, ctx: PurePath) -> None: + self.pattern_generator.open_file(ctx, WIDTH, HEIGHT) async def stop(self) -> None: self.pattern_generator.close_file() diff --git a/src/ophyd_async/sim/_blob_detector.py b/src/ophyd_async/sim/_blob_detector.py index ed21968784..42e1be3cb0 100644 --- a/src/ophyd_async/sim/_blob_detector.py +++ b/src/ophyd_async/sim/_blob_detector.py @@ -1,6 +1,13 @@ from collections.abc import Sequence +from functools import cached_property -from ophyd_async.core import PathProvider, SignalR, StandardDetector +from ophyd_async.core import ( + DetectorLogic, + PathProvider, + SignalR, + StandardDetector, +) +from ophyd_async.core import StandardReadableFormat as Format from ._blob_acquire_logic import BlobAcquireLogic from ._blob_data_logic import BlobDataLogic @@ -19,12 +26,17 @@ def __init__( name: str = "", ) -> None: self.pattern_generator = pattern_generator or PatternGenerator() - self.add_detector_logics( + for signal in config_sigs: + self.set_readable_format(signal, Format.CONFIG_SIGNAL) + self._logic = DetectorLogic( BlobTriggerLogic(pattern_generator=self.pattern_generator), BlobAcquireLogic(pattern_generator=self.pattern_generator), BlobDataLogic( path_provider=path_provider, pattern_generator=self.pattern_generator ), ) - self.add_config_signals(*config_sigs) super().__init__(name=name) + + @cached_property + def logic(self) -> DetectorLogic: + return self._logic diff --git a/src/ophyd_async/sim/_motor.py b/src/ophyd_async/sim/_motor.py index 41697fd2e1..ba31504658 100644 --- a/src/ophyd_async/sim/_motor.py +++ b/src/ophyd_async/sim/_motor.py @@ -83,7 +83,7 @@ def __init__( super().__init__(name=name) @cached_property - def movable_logic(self): + def logic(self): return SimMotorMoveLogic( readback=self.user_readback, readback_set=self._user_readback_set, diff --git a/src/ophyd_async/tango/core/_tango_transport.py b/src/ophyd_async/tango/core/_tango_transport.py index c202f2c9f5..017fe3f364 100644 --- a/src/ophyd_async/tango/core/_tango_transport.py +++ b/src/ophyd_async/tango/core/_tango_transport.py @@ -749,6 +749,8 @@ def make_converter(info: AttributeConfig | CommandConfig, datatype) -> TangoConv AttrDataFormat.IMAGE, ]: return TangoDevStateArrayConverter() + case _: + pass else: # command info match info.in_type: case CmdArgType.DevState: @@ -765,6 +767,8 @@ def make_converter(info: AttributeConfig | CommandConfig, datatype) -> TangoConv return TangoLongStringTableConverter() case CmdArgType.DevVarDoubleStringArray: return TangoDoubleStringTableConverter() + case _: + pass # default case return trivial converter return TangoConverter() diff --git a/src/ophyd_async/tango/demo/_motor.py b/src/ophyd_async/tango/demo/_motor.py index f1194763f0..6fd504d4df 100644 --- a/src/ophyd_async/tango/demo/_motor.py +++ b/src/ophyd_async/tango/demo/_motor.py @@ -52,7 +52,7 @@ class DemoMotor(TangoDevice, StandardReadable, StandardMovable[float]): stop_: TriggerableCommand @cached_property - def movable_logic(self) -> MovableLogic: + def logic(self) -> MovableLogic: return DemoMotorMoveLogic( readback=self.readback, setpoint=self.setpoint, diff --git a/tests/container_tests/test_adsim_system.py b/tests/container_tests/test_adsim_system.py index 26fc83f3b7..a89c742052 100644 --- a/tests/container_tests/test_adsim_system.py +++ b/tests/container_tests/test_adsim_system.py @@ -34,6 +34,7 @@ import bluesky.plan_stubs as bps import bluesky.plans as bp +import bluesky.preprocessors as bpp import pytest from aioca import purge_channel_caches from bluesky.run_engine import RunEngine @@ -111,6 +112,24 @@ def adsim(RE: RunEngine, shared_tmp_path: Path) -> AreaDetector: return adsim +@pytest.fixture +def adsimstat(RE: RunEngine) -> AreaDetector: + """A writer-less SimDetector whose only data logic is the STAT time series. + + It writes no file: the NDPluginStats time series is a fixed-length buffer read + back as event pages, so this exercises the bounded data-logic tier against a + real IOC (the `ADWriterFactory.stats` factory and NDStatsIO's `TS:*` signals). + """ + with init_devices(): + adsimstat = SimDetector( + "BL01T-DI-CAM-01:", + adcore.ADWriterFactory.stats(writer_suffix="STAT:"), + driver_suffix="DET:", + ) + + return adsimstat + + @pytest.fixture(autouse=True) def reset_adsim_to_baseline(RE: RunEngine, adsim: SimDetector) -> None: """Put the detector back to a known state before every test. @@ -196,10 +215,24 @@ def test_software_triggering( "data": { "adsim-driver-acquire_period": 0.005, "adsim-driver-acquire_time": 0.1, + "adsim-driver-num_images": 1, + "adsim-driver-image_mode": "Multiple", + "adsim-driver-manufacturer": "Simulated detector", + "adsim-driver-model": "Basic simulator", + "adsim-driver-serial_number": "No serial number", + "adsim-driver-sdk_version": ANY, + "adsim-driver-firmware_version": "No firmware", }, "timestamps": { "adsim-driver-acquire_period": ANY, "adsim-driver-acquire_time": ANY, + "adsim-driver-num_images": ANY, + "adsim-driver-image_mode": ANY, + "adsim-driver-manufacturer": ANY, + "adsim-driver-model": ANY, + "adsim-driver-serial_number": ANY, + "adsim-driver-sdk_version": ANY, + "adsim-driver-firmware_version": ANY, }, "data_keys": { "adsim-driver-acquire_period": { @@ -218,6 +251,50 @@ def test_software_triggering( "units": "", "precision": 3, }, + "adsim-driver-num_images": { + "dtype": "integer", + "shape": [], + "dtype_numpy": " None: + """A writer-less stats detector arms the TS buffer, acquires, and reads the + per-frame Total series back as a single event holding the whole array. + + This drives the real STAT plugin's ``TS:*`` records end to end: prepare sizes + and arms ``TS:TSNumPoints``/``TS:TSAcquire``, the sim driver takes the frames, + and read() slices ``TS:TSTotal`` (timestamped by ``TS:TSTimestamp``) into a + single 5-element reading. + """ + collections = 5 + + @bpp.stage_decorator([adsimstat]) + @bpp.run_decorator() + def prepare_then_count() -> MsgGenerator[None]: + # prepare() must come after stage(), which resets the detector + yield from bps.prepare( + adsimstat, + TriggerInfo(collections_per_event=collections, livetime=0.1), + wait=True, + ) + yield from bps.trigger_and_read([adsimstat]) + + named_docs: list[tuple[str, DocumentType]] = [] + RE(prepare_then_count(), lambda name, doc: named_docs.append((name, doc))) + + # A bounded logic exposes collect_pages, never collect_asset_docs. + assert hasattr(adsimstat, "collect_pages") + assert not hasattr(adsimstat, "collect_asset_docs") + + descriptor = next(doc for name, doc in named_docs if name == "descriptor") + data_key = descriptor["data_keys"]["adsimstat"] + assert data_key["shape"] == [collections] + assert data_key["dtype"] == "array" + assert data_key["dtype_numpy"] == "= 0 + + def run_plan_and_get_documents( RE: RunEngine, plan: MsgGenerator[Any] ) -> list[DocumentType]: diff --git a/tests/system_tests/tango/core/test_yaml_save.yaml b/tests/system_tests/tango/core/test_yaml_save.yaml index c7103cfd3b..a6b0ac7f08 100644 --- a/tests/system_tests/tango/core/test_yaml_save.yaml +++ b/tests/system_tests/tango/core/test_yaml_save.yaml @@ -1,3 +1,4 @@ +: {} a_bool: true a_str: test_string float64: 1.234 diff --git a/tests/unit_tests/core/test_data_providers.py b/tests/unit_tests/core/test_data_providers.py new file mode 100644 index 0000000000..6072bf45e5 --- /dev/null +++ b/tests/unit_tests/core/test_data_providers.py @@ -0,0 +1,72 @@ +import numpy as np +import pytest + +from ophyd_async.core import ( + Array1D, + EventPageDataProvider, + soft_signal_r_and_setter, +) + + +@pytest.fixture +def provider() -> EventPageDataProvider: + """A 5-collection buffer, filled, with one timestamp per collection.""" + data, _ = soft_signal_r_and_setter( + Array1D[np.float64], np.array([10.0, 11.0, 12.0, 13.0, 14.0]) + ) + timestamps, _ = soft_signal_r_and_setter( + Array1D[np.float64], np.array([100.0, 101.0, 102.0, 103.0, 104.0]) + ) + written, _ = soft_signal_r_and_setter(int, 5) + return EventPageDataProvider({"det-stats": data}, written, timestamps) + + +@pytest.mark.parametrize( + "collections_per_event,expected_data,expected_times", + [ + # step scan: one event holds the whole 5-collection buffer, timed by its + # last collection + (5, [[10.0, 11.0, 12.0, 13.0, 14.0]], [104.0]), + # fly scan: five events of one collection each, timed one by one + ( + 1, + [[10.0], [11.0], [12.0], [13.0], [14.0]], + [100.0, 101.0, 102.0, 103.0, 104.0], + ), + ], +) +async def test_slices_the_arrays_into_events( + provider: EventPageDataProvider, + collections_per_event: int, + expected_data: list[list[float]], + expected_times: list[float], +): + pages = [ + page + async for page in provider.make_pages( + collections_written=5, collections_per_event=collections_per_event + ) + ] + (page,) = pages + np.testing.assert_array_equal(page["data"]["det-stats"], expected_data) + assert page["time"] == expected_times + + +async def test_datakeys_take_dtype_from_the_signal(provider: EventPageDataProvider): + datakeys = await provider.make_datakeys(5) + assert list(datakeys) == ["det-stats"] + assert datakeys["det-stats"]["shape"] == [5] + assert datakeys["det-stats"]["dtype"] == "array" + assert datakeys["det-stats"]["dtype_numpy"] == " MovableFlyableLogic: + def logic(self) -> MovableFlyableLogic: return MovableFlyableLogic(setpoint=self.setpoint, readback=self.readback) - @cached_property - def movable_logic(self) -> MovableFlyableLogic: - return self._logic - - @cached_property - def flyable_logic(self) -> MovableFlyableLogic: - return self._logic - @pytest.fixture async def recording_flyer(): @@ -114,7 +106,7 @@ async def test_ephemeral_flyable_drives_logic_hooks(recording_flyer): assert flyer.name == "flyer" assert isinstance(flyer, StandardFlyable) # The ephemeral device exposes the same logic instance - assert flyer.flyable_logic is logic + assert flyer.logic is logic await flyer.prepare(5) await flyer.kickoff() @@ -250,7 +242,7 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def flyable_logic(self) -> FlyableLogic[int, None]: + def logic(self) -> FlyableLogic[int, None]: return logic async with init_devices(mock=True): diff --git a/tests/unit_tests/core/test_movable.py b/tests/unit_tests/core/test_movable.py index c3df8fd096..7ac786f6b0 100644 --- a/tests/unit_tests/core/test_movable.py +++ b/tests/unit_tests/core/test_movable.py @@ -50,7 +50,7 @@ def __init__(self, name: str = ""): super().__init__(name=name) @cached_property - def movable_logic(self) -> MovableLogic[float]: + def logic(self) -> MovableLogic[float]: return MovableLogic(setpoint=self.setpoint, readback=self.readback) @@ -62,16 +62,16 @@ async def movable() -> StandardMovableImpl: def test_movable_logic_is_cached(movable: StandardMovableImpl): - logic = movable.movable_logic - logic2 = movable.movable_logic + logic = movable.logic + logic2 = movable.logic assert logic == logic2 async def test_movable_check_value(movable: StandardMovableImpl): - movable.movable_logic.check_move = AsyncMock() + movable.logic.check_move = AsyncMock() await movable.check_value(5) - movable.movable_logic.check_move.assert_awaited_once_with(5) + movable.logic.check_move.assert_awaited_once_with(5) async def test_locatable(movable: StandardMovableImpl) -> None: @@ -133,12 +133,12 @@ async def test_movable_moving_stopped(movable: StandardMovableImpl): async def test_movable_set_calls_movable_logic_check_move_and_calculate_timeout( movable: StandardMovableImpl, ): - mock_check_move = movable.movable_logic.check_move = AsyncMock() + mock_check_move = movable.logic.check_move = AsyncMock() timeout = 5 - mock_calculate_timeout = movable.movable_logic.calculate_timeout = AsyncMock( + mock_calculate_timeout = movable.logic.calculate_timeout = AsyncMock( return_value=timeout ) - mock_move = movable.movable_logic.move = AsyncMock() + mock_move = movable.logic.move = AsyncMock() with patch("ophyd_async.core._movable.MoveTimeout") as move_timeout: pos = 10 diff --git a/tests/unit_tests/core/test_multi_derived_signal.py b/tests/unit_tests/core/test_multi_derived_signal.py index dd252af6fa..3f8b038301 100644 --- a/tests/unit_tests/core/test_multi_derived_signal.py +++ b/tests/unit_tests/core/test_multi_derived_signal.py @@ -298,8 +298,8 @@ async def _put(val: float) -> None: derived_sig = derived_signal_rw(_get, _put, m1=m1, m2=m2) await derived_sig.set(6) - assert await m1.movable_logic.readback.get_value() == 3 - assert await m2.movable_logic.readback.get_value() == 3 + assert await m1.logic.readback.get_value() == 3 + assert await m2.logic.readback.get_value() == 3 assert await derived_sig.get_value() == 6 diff --git a/tests/unit_tests/core/test_readable.py b/tests/unit_tests/core/test_readable.py index 75a98b424b..469ff39d1f 100644 --- a/tests/unit_tests/core/test_readable.py +++ b/tests/unit_tests/core/test_readable.py @@ -4,14 +4,9 @@ from bluesky.protocols import HasHints from ophyd_async.core import ( - AsyncConfigurable, - AsyncReadable, - AsyncStageable, - ConfigSignal, Device, DeviceMap, DeviceVector, - HintedSignal, MockSignalBackend, SignalR, StandardReadable, @@ -21,11 +16,10 @@ from ophyd_async.core import StandardReadableFormat as Format -@pytest.mark.parametrize("wrapper", [HintedSignal, HintedSignal.uncached, ConfigSignal]) -def test_standard_readable_wrappers_raise_deprecation_warning(wrapper): +def test_standard_readable_rejects_a_non_format(): sr = StandardReadable() - with pytest.deprecated_call(): - sr.add_readables([soft_signal_rw(int)], wrapper) + with pytest.raises(TypeError, match="is not a StandardReadableFormat"): + sr.set_readable_format(soft_signal_rw(int), "CONFIG_SIGNAL") # type: ignore def test_standard_readable_hints(): @@ -42,7 +36,9 @@ def test_standard_readable_hints(): hint3 = MagicMock(spec=HasHints) hint3.hints = {"fields": ["jkl"], "gridding": "rectilinear_nonsequential"} - sr.add_readables([hint1, hint2, hint3]) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) + sr.set_readable_format(hint3, Format.CHILD) assert sr.hints == { "fields": ["abc", "def", "ghi", "jkl"], @@ -60,10 +56,8 @@ def test_standard_readable_hints_raises_when_overriding_string_literal(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"gridding": "a different string"} - sr._has_hints = ( - hint1, - hint2, - ) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) with pytest.raises(RuntimeError, match=r"Hints key .* value may not be overridden"): sr.hints # noqa: B018 @@ -78,10 +72,8 @@ def test_standard_readable_hints_raises_when_overriding_sequence(): hint2 = MagicMock(spec=HasHints) hint2.hints = {"fields": ["field2"]} - sr._has_hints = ( - hint1, - hint2, - ) + sr.set_readable_format(hint1, Format.CHILD) + sr.set_readable_format(hint2, Format.CHILD) with pytest.raises(RuntimeError, match=r"Hint fields .* overrides existing hint"): sr.hints # noqa: B018 @@ -94,7 +86,7 @@ def test_standard_readable_hints_invalid_types(invalid_type): hint1 = MagicMock(spec=HasHints) hint1.hints = {"test": invalid_type} - sr._has_hints = (hint1,) + sr.set_readable_format(hint1, Format.CHILD) with pytest.raises(TypeError, match=r"Unknown type for value .* for key .*"): sr.hints # noqa: B018 @@ -102,17 +94,12 @@ def test_standard_readable_hints_invalid_types(invalid_type): def test_standard_readable_add_children_context_manager(): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock with sr.add_children_as_readables(): sr.a = MagicMock(spec=SignalR) sr.b = MagicMock(spec=SignalR) sr.c = MagicMock(spec=SignalR) - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == {sr.a, sr.b, sr.c} + assert sr.get_readable_formats() == dict.fromkeys((sr.a, sr.b, sr.c), Format.CHILD) @pytest.mark.parametrize( @@ -124,26 +111,19 @@ def test_standard_readable_add_children_context_manager(): ) def test_standard_readable_add_children_cm_device_with_mappings(device_type, keys): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock - # Create a mock for the DeviceVector/DeviceMap.children() call + # A DeviceVector/DeviceMap registers its values, not itself devices = [MagicMock(spec=SignalR) for _ in range(3)] device = device_type(dict(zip(keys, devices, strict=True))) with sr.add_children_as_readables(): sr.a = device - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == set(devices) + assert sr.get_readable_formats() == dict.fromkeys(devices, Format.CHILD) def test_standard_readable_add_children_cm_filters_non_devices(): sr = StandardReadable() - mock = MagicMock() - sr.add_readables = mock with sr.add_children_as_readables(): sr.a = MagicMock(spec=SignalR) @@ -152,76 +132,73 @@ def test_standard_readable_add_children_cm_filters_non_devices(): sr.d = "abc" sr.e = MagicMock(spec=MockSignalBackend) - # Can't use assert_called_once_with() as the order of items returned from - # internal dict comprehension is not guaranteed - mock.assert_called_once() - assert set(mock.call_args.args[0]) == {sr.a, sr.b} - - -def assert_sr_has_attrs(sr: StandardReadable, expected_attrs: dict[str, tuple]): - attrs_to_check = ( - "_describe_config_funcs", - "_read_config_funcs", - "_describe_funcs", - "_read_funcs", - "_stage_funcs", - "_unstage_funcs", - "_has_hints", - ) - actual = {attr: getattr(sr, attr) for attr in attrs_to_check} - expected = {attr: expected_attrs.get(attr, ()) for attr in attrs_to_check} - assert actual == expected + # Only the Devices are registered + assert sr.get_readable_formats() == dict.fromkeys((sr.a, sr.b), Format.CHILD) -signal_r = MagicMock(spec=SignalR) -async_readable = MagicMock(spec=AsyncReadable) -async_configurable = MagicMock(spec=AsyncConfigurable) -async_stageable = MagicMock(spec=AsyncStageable) -has_hints = MagicMock(spec=HasHints) +async def assert_contributes( + sr: StandardReadable, + *, + read: set[str] = frozenset(), + config: set[str] = frozenset(), + hints: set[str] = frozenset(), +) -> None: + """Assert which keys a StandardReadable contributes to each verb.""" + assert set(await sr.read()) == set(read) + assert set(await sr.describe()) == set(read) + assert set(await sr.read_configuration()) == set(config) + assert set(await sr.describe_configuration()) == set(config) + assert set(sr.hints.get("fields", [])) == set(hints) @pytest.mark.parametrize( - "readable, expected_attrs", + "format, in_read, in_config, in_hints", [ - ( - signal_r, - { - "_read_funcs": (signal_r.read,), - "_describe_funcs": (signal_r.describe,), - "_stage_funcs": (signal_r.stage,), - "_unstage_funcs": (signal_r.unstage,), - }, - ), - ( - async_readable, - { - "_read_funcs": (async_readable.read,), - "_describe_funcs": (async_readable.describe,), - }, - ), - ( - async_configurable, - { - "_read_config_funcs": (async_configurable.read_configuration,), - "_describe_config_funcs": (async_configurable.describe_configuration,), - }, - ), - ( - async_stageable, - { - "_stage_funcs": (async_stageable.stage,), - "_unstage_funcs": (async_stageable.unstage,), - }, - ), - (has_hints, {"_has_hints": (has_hints,)}), + (Format.CONFIG_SIGNAL, False, True, False), + (Format.HINTED_SIGNAL, True, False, True), + (Format.UNCACHED_SIGNAL, True, False, False), + (Format.HINTED_UNCACHED_SIGNAL, True, False, True), ], ) -def test_standard_readable_add_readables_adds_to_expected_attrs( - readable, expected_attrs: dict[str, tuple] +async def test_standard_readable_signal_formats_route_to_expected_verbs( + format, in_read, in_config, in_hints ): - sr = StandardReadable() - sr.add_readables([readable]) - assert_sr_has_attrs(sr, expected_attrs) + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + sr.set_readable_format(sr.sig, format) + name = {sr.sig.name} + await assert_contributes( + sr, + read=name if in_read else set(), + config=name if in_config else set(), + hints=name if in_hints else set(), + ) + + +async def test_standard_readable_child_format_uses_child_verbs(): + inner = StandardReadable(name="inner") + inner.conf, _ = soft_signal_r_and_setter(int, 0, name="conf") + inner.val, _ = soft_signal_r_and_setter(int, 0, name="val") + inner.set_readable_format(inner.conf, Format.CONFIG_SIGNAL) + inner.set_readable_format(inner.val, Format.HINTED_SIGNAL) + + outer = StandardReadable(name="outer") + outer.inner = inner + outer.set_readable_format(inner, Format.CHILD) + + await assert_contributes( + outer, + read={inner.val.name}, + config={inner.conf.name}, + hints={inner.val.name}, + ) + + +async def test_standard_readable_child_format_ignores_plain_device(): + sr = StandardReadable(name="sr") + sr.child = Device(name="child") + sr.set_readable_format(sr.child, Format.CHILD) + await assert_contributes(sr) @pytest.mark.parametrize( @@ -240,43 +217,48 @@ def test_standard_readable_add_readables_raises_signalr_typeerror(format) -> Non # Ensure it raises TypeError with pytest.raises(TypeError, match=f"{mock_device} is not a SignalR"): - sr.add_readables([mock_device], format=format) - - -def test_standard_readable_config_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.CONFIG_SIGNAL) - assert sr._describe_config_funcs == (signal_r.describe,) - assert sr._read_config_funcs == (signal_r.read,) + sr.set_readable_format(mock_device, format) -def test_standard_readable_hinted_signal(): - signal_r = MagicMock(spec=SignalR) +def test_add_readables_is_deprecated() -> None: sr = StandardReadable() - sr.add_readables([signal_r], Format.HINTED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs == (signal_r.read,) - assert sr._stage_funcs == (signal_r.stage,) - assert sr._unstage_funcs == (signal_r.unstage,) - assert sr._has_hints[0].device == signal_r + sig = soft_signal_rw(int) + with pytest.warns(DeprecationWarning, match="set_readable_format"): + sr.add_readables([sig], Format.CONFIG_SIGNAL) + # Still does the job it always did + assert sr.get_readable_formats() == {sig: Format.CONFIG_SIGNAL} -def test_standard_readable_uncached_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.UNCACHED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs[0].signal == signal_r +@pytest.mark.parametrize( + "format, staged", + [ + (Format.HINTED_SIGNAL, True), + (Format.CHILD, True), + (Format.CONFIG_SIGNAL, False), + (Format.UNCACHED_SIGNAL, False), + (Format.HINTED_UNCACHED_SIGNAL, False), + ], +) +async def test_standard_readable_stage_starts_caching_only_for_cached_formats( + format, staged +): + # stage() exists to start monitoring, and reading cached raises if it isn't + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + await sr.sig.connect(mock=False) + sr.set_readable_format(sr.sig, format) + await sr.stage() + if staged: + assert await sr.sig.read(cached=True) + else: + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) -def test_standard_readable_hinted_uncached_signal(): - signal_r = MagicMock(spec=SignalR) - sr = StandardReadable() - sr.add_readables([signal_r], Format.HINTED_UNCACHED_SIGNAL) - assert sr._describe_funcs == (signal_r.describe,) - assert sr._read_funcs[0].signal == signal_r - assert sr._has_hints[0].device == signal_r + # unstage tears the monitor back down + await sr.unstage() + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) def test_standard_readable_add_children_multi_nested(): @@ -305,3 +287,186 @@ def __init__(self, name): with pytest.raises(KeyError): DummyDerivedDevice("test_duplicates") + + +async def test_set_readable_format_changes_verb_at_runtime(): + # The ophyd v1 `kind` use case from #1394: a signal that is config for one + # technique and hinted for another, without redefining the Device. + sr = StandardReadable(name="sr") + sr.energy, _ = soft_signal_r_and_setter(float, 0.0, name="energy") + + sr.set_readable_format(sr.energy, Format.CONFIG_SIGNAL) + await assert_contributes(sr, config={sr.energy.name}) + + sr.set_readable_format(sr.energy, Format.HINTED_SIGNAL) + await assert_contributes(sr, read={sr.energy.name}, hints={sr.energy.name}) + + sr.set_readable_format(sr.energy, None) + await assert_contributes(sr) + + +async def test_set_readable_format_replaces_rather_than_duplicates(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + + assert sr.hints == {"fields": [sr.sig.name]} + assert sr.get_readable_formats() == {sr.sig: Format.HINTED_SIGNAL} + + +async def test_get_readable_formats_omits_unregistered_children(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + sr.other, _ = soft_signal_r_and_setter(int, 0, name="other") + assert sr.get_readable_formats() == {} + sr.set_readable_format(sr.sig, Format.CONFIG_SIGNAL) + sr.set_readable_format(sr.other, Format.HINTED_SIGNAL) + # Registration order is preserved, and an unregistered child is absent + assert sr.get_readable_formats() == { + sr.sig: Format.CONFIG_SIGNAL, + sr.other: Format.HINTED_SIGNAL, + } + assert list(sr.get_readable_formats()) == [sr.sig, sr.other] + sr.set_readable_format(sr.sig, None) + assert sr.get_readable_formats() == {sr.other: Format.HINTED_SIGNAL} + + +async def test_reset_readable_formats_undoes_runtime_changes(): + class Dev(StandardReadable): + def __init__(self, name=""): + self.energy, _ = soft_signal_r_and_setter(float, 0.0, name="energy") + self.temp, _ = soft_signal_r_and_setter(float, 0.0, name="temp") + self.set_readable_format(self.energy, Format.CONFIG_SIGNAL) + self.set_readable_format(self.temp, Format.CONFIG_SIGNAL) + super().__init__(name=name) + + dev = Dev(name="dev") + dev.extra, _ = soft_signal_r_and_setter(float, 0.0, name="extra") + declared = dev.get_readable_formats() + + # Change one, drop one, add one that was never declared + dev.set_readable_format(dev.energy, Format.HINTED_SIGNAL) + dev.set_readable_format(dev.temp, None) + dev.set_readable_format(dev.extra, Format.HINTED_SIGNAL) + assert dev.get_readable_formats() == { + dev.energy: Format.HINTED_SIGNAL, + dev.extra: Format.HINTED_SIGNAL, + } + assert dev.get_readable_format_changes() == { + dev.energy: Format.HINTED_SIGNAL, + dev.extra: Format.HINTED_SIGNAL, + dev.temp: None, + } + + dev.reset_readable_formats() + assert dev.get_readable_formats() == declared + assert dev.get_readable_format_changes() == {} + # Repeatable, and the baseline is not consumed + dev.set_readable_format(dev.energy, None) + dev.reset_readable_formats() + assert dev.get_readable_formats() == declared + + +async def test_reset_readable_formats_keeps_registrations_after_super_init(): + # The reason the baseline is sealed by the metaclass rather than at the end + # of StandardReadable.__init__: this registration happens after super() and + # is still part of what the class declares + class Dev(StandardReadable): + def __init__(self, name=""): + self.before, _ = soft_signal_r_and_setter(int, 0, name="before") + self.set_readable_format(self.before, Format.CONFIG_SIGNAL) + super().__init__(name=name) + self.after, _ = soft_signal_r_and_setter(int, 0, name="after") + self.set_readable_format(self.after, Format.HINTED_SIGNAL) + + dev = Dev(name="dev") + expected = {dev.before: Format.CONFIG_SIGNAL, dev.after: Format.HINTED_SIGNAL} + assert dev.get_readable_formats() == expected + + dev.set_readable_format(dev.after, None) + dev.reset_readable_formats() + assert dev.get_readable_formats() == expected + + +async def test_reset_readable_formats_restores_annotated_declarations(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + # Nothing declared, so a runtime registration resets away entirely + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + sr.reset_readable_formats() + assert sr.get_readable_formats() == {} + + +async def test_default_readable_formats_are_not_shared_between_instances(): + class Dev(StandardReadable): + def __init__(self, name=""): + self.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + self.set_readable_format(self.sig, Format.CONFIG_SIGNAL) + super().__init__(name=name) + + a, b = Dev(name="a"), Dev(name="b") + a.set_readable_format(a.sig, None) + a.reset_readable_formats() + b.reset_readable_formats() + assert a.get_readable_formats() == {a.sig: Format.CONFIG_SIGNAL} + assert b.get_readable_formats() == {b.sig: Format.CONFIG_SIGNAL} + assert StandardReadable._default_readables == {} + + +async def test_readables_registered_before_super_init_survive(): + # AreaDetector registers config signals before calling super().__init__(), + # so the registry must not be created there. It must also be per instance. + class Dev(StandardReadable): + def __init__(self, name=""): + self.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + self.set_readable_format(self.sig, Format.HINTED_SIGNAL) + super().__init__(name=name) + + a, b = Dev(name="a"), Dev(name="b") + assert a.get_readable_formats() == {a.sig: Format.HINTED_SIGNAL} + assert b.get_readable_formats() == {b.sig: Format.HINTED_SIGNAL} + assert a.get_readable_formats() != b.get_readable_formats() + + b.set_readable_format(b.sig, None) + assert a.get_readable_formats() == {a.sig: Format.HINTED_SIGNAL} + assert b.get_readable_formats() == {} + + +async def test_set_readable_format_keeps_position_when_reformatting(): + # Changing a format must not reorder hints + sr = StandardReadable(name="sr") + sr.one, _ = soft_signal_r_and_setter(int, 0, name="one") + sr.two, _ = soft_signal_r_and_setter(int, 0, name="two") + sr.set_readable_format(sr.one, Format.HINTED_SIGNAL) + sr.set_readable_format(sr.two, Format.HINTED_SIGNAL) + assert sr.hints == {"fields": [sr.one.name, sr.two.name]} + + sr.set_readable_format(sr.one, Format.HINTED_UNCACHED_SIGNAL) + assert sr.hints == {"fields": [sr.one.name, sr.two.name]} + + +async def test_set_readable_format_affects_staging_of_later_runs(): + sr = StandardReadable(name="sr") + sr.sig, _ = soft_signal_r_and_setter(int, 0, name="sig") + await sr.sig.connect(mock=False) + sr.set_readable_format(sr.sig, Format.HINTED_SIGNAL) + + await sr.stage() + assert await sr.sig.read(cached=True) + await sr.unstage() + + # Dropping the signal means the next run does not monitor it + sr.set_readable_format(sr.sig, None) + await sr.stage() + with pytest.raises(RuntimeError, match="not being monitored"): + await sr.sig.read(cached=True) + await sr.unstage() + + +def test_set_readable_format_rejects_non_signal_for_signal_format(): + sr = StandardReadable(name="sr") + sr.child = Device(name="child") + with pytest.raises(TypeError, match="is not a SignalR"): + sr.set_readable_format(sr.child, Format.CONFIG_SIGNAL) diff --git a/tests/unit_tests/core/test_standard_detector.py b/tests/unit_tests/core/test_standard_detector.py index 02e8be2ac2..e0927c9d4b 100644 --- a/tests/unit_tests/core/test_standard_detector.py +++ b/tests/unit_tests/core/test_standard_detector.py @@ -5,19 +5,22 @@ import numpy as np import pytest +from bluesky.protocols import EventPageCollectable, WritesStreamAssets +from event_model import DataKey +from event_model.documents import PartialEventPage from ophyd_async.core import ( Array1D, DetectorAcquireLogic, DetectorDataLogic, + DetectorLogic, DetectorTrigger, DetectorTriggerLogic, - ReadableDataProvider, + PageableDataProvider, Settings, - SignalDataProvider, SignalDict, SignalR, - StandardDetector, + StandardReadable, StreamableDataProvider, StreamResourceDataProvider, StreamResourceInfo, @@ -25,6 +28,7 @@ soft_signal_r_and_setter, soft_signal_rw, ) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.testing import ( assert_configuration, assert_reading, @@ -97,12 +101,12 @@ class DeadtimeTriggerLogic(DetectorTriggerLogic): def __init__(self, deadtime_signal: SignalR[float]): self.deadtime_signal = deadtime_signal - def config_sigs(self) -> set[SignalR]: - """Return the deadtime signal as a config signal.""" - return {self.deadtime_signal} - def get_deadtime(self, config_values: SignalDict) -> float: - """Return the deadtime from the signal value.""" + """Return the deadtime from the signal value. + + The signal reaches us because the detector registers it as + CONFIG_SIGNAL, not because this logic nominates it. + """ return config_values[self.deadtime_signal] async def prepare_internal(self, num: int, livetime: float, deadtime: float): @@ -129,28 +133,22 @@ async def ensure_stopped(self): self.disarm_count += 1 -class ReadableOnlyDataLogic(DetectorDataLogic): - """Produces only readable (non-streaming) data.""" - - def __init__(self): - self.signal = soft_signal_rw(int, initial_value=42, name="foo-value") - - async def prepare_single(self, datakey_name: str) -> ReadableDataProvider: - return SignalDataProvider(self.signal) - - def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: - return ["foo-value"] - - -class StreamableOnlyDataLogic(DetectorDataLogic): +class StreamableOnlyDataLogic(DetectorDataLogic[None]): """Produces only streamable (file-based) data.""" - def __init__(self, tmp_path): + def __init__(self, tmp_path, datakey_suffix: str = ""): self.collections_written = soft_signal_rw(int) self.stop_count = 0 self.tmp_path = tmp_path - - async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: + self.datakey_suffix = datakey_suffix + + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[StreamableDataProvider, None]: resource = StreamResourceInfo( data_key=datakey_name, shape=(10, 15), @@ -164,7 +162,7 @@ async def prepare_unbounded(self, datakey_name: str) -> StreamableDataProvider: mimetype="application/x-hdf5", collections_written_signal=self.collections_written, ) - return provider + return provider, None async def stop(self) -> None: self.stop_count += 1 @@ -200,6 +198,71 @@ async def test_stream_resource_data_provider_1d_single_collection_is_array(): } +class MockPageableProvider(PageableDataProvider): + """A finite buffer emitted as event pages, one value per collection.""" + + def __init__(self, datakey_name: str, collections_written_signal: SignalR[int]): + self.datakey_name = datakey_name + self.collections_written_signal = collections_written_signal + self.last_emitted = 0 + + async def make_datakeys(self, collections_per_event: int) -> dict[str, DataKey]: + return { + self.datakey_name: DataKey( + source="mock", + shape=[collections_per_event], + dtype="array", + dtype_numpy=" self.last_emitted: + new = range(self.last_emitted, events) + page: PartialEventPage = { + "data": {self.datakey_name: [[0] * collections_per_event for _ in new]}, + "time": [0.0 for _ in new], + "timestamps": {self.datakey_name: [0.0 for _ in new]}, + } + self.last_emitted = events + yield page + + +class BoundedOnlyDataLogic(DetectorDataLogic[tuple[int, float]]): + """Produces bounded data held in a finite buffer, sized when it is armed.""" + + def __init__(self, datakey_suffix: str = ""): + self.datakey_suffix = datakey_suffix + self.collections_written = soft_signal_rw(int) + self.prepare_calls: list[tuple[int, float]] = [] + self.stop_count = 0 + + async def make_data_provider( + self, + datakey_name: str, + num_collections: int, + period: float, + flush_period: float, + ) -> tuple[PageableDataProvider, tuple[int, float]] | None: + if num_collections == 0: + # A finite buffer cannot serve an unbounded scan + return None + provider = MockPageableProvider(datakey_name, self.collections_written) + return provider, (num_collections, period) + + async def start(self, ctx: tuple[int, float]) -> None: + self.prepare_calls.append(ctx) + # A real buffer clears its progress counter when armed + await self.collections_written.set(0) + + async def stop(self) -> None: + self.stop_count += 1 + + def get_hinted_fields(self, datakey_name: str) -> Sequence[str]: + return [datakey_name] + + # Parameterized Tests for Logic Combinations @@ -228,9 +291,11 @@ async def test_get_trigger_deadtime( trigger_logic, expected_triggers, expected_deadtime ): """Test get_trigger_deadtime with various trigger logic implementations.""" - det = StandardDetector() - if trigger_logic: - det.add_detector_logics(trigger_logic) + det = DetectorLogic(*([trigger_logic] if trigger_logic else [])).with_device() + if isinstance(trigger_logic, DeadtimeTriggerLogic): + # A logic that needs a signal declares it as configuration; the logic + # no longer nominates signals separately + det.set_readable_format(trigger_logic.deadtime_signal, Format.CONFIG_SIGNAL) triggers, deadtime = await det.get_trigger_deadtime() assert triggers == expected_triggers assert deadtime == expected_deadtime @@ -242,10 +307,9 @@ async def test_get_trigger_deadtime_with_settings(): deadtime_signal = soft_signal_rw(float, 0.02) # Create detector with DeadtimeTriggerLogic - det = StandardDetector() + det = DetectorLogic(DeadtimeTriggerLogic(deadtime_signal)).with_device() det.sig = deadtime_signal - tl = DeadtimeTriggerLogic(deadtime_signal) - det.add_detector_logics(tl) + det.set_readable_format(deadtime_signal, Format.CONFIG_SIGNAL) # Verify initial deadtime from signal triggers, deadtime = await det.get_trigger_deadtime() @@ -273,9 +337,8 @@ async def test_get_trigger_deadtime_with_settings(): ) async def test_prepare_trigger_types(trigger_type): """Test each trigger type is properly delegated to trigger logic.""" - det = StandardDetector() trigger_logic = AllTriggerTypesLogic() - det.add_detector_logics(trigger_logic) + det = DetectorLogic(trigger_logic).with_device() trigger_info = TriggerInfo( trigger=trigger_type, livetime=0.5, deadtime=0.1, number_of_events=10 @@ -289,8 +352,7 @@ async def test_prepare_trigger_types(trigger_type): async def test_prepare_unsupported_trigger_type(): """Test that preparing with unsupported trigger type raises error.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() with pytest.raises(ValueError, match="Trigger type.*EXTERNAL_EDGE not supported"): await det.prepare(TriggerInfo(trigger=DetectorTrigger.EXTERNAL_EDGE)) @@ -327,9 +389,8 @@ async def test_trigger_info_calculations( @pytest.mark.parametrize("exposures_per_collection", [1, 2, 5, 10]) async def test_exposures_per_collection(exposures_per_collection): """Test exposure averaging configuration.""" - det = StandardDetector() tl = AveragingTriggerLogic() - det.add_detector_logics(tl) + det = DetectorLogic(tl).with_device() await det.prepare( TriggerInfo( @@ -344,8 +405,7 @@ async def test_exposures_per_collection(exposures_per_collection): async def test_exposures_per_collection_not_supported(): """Test that exposures_per_collection > 1 fails without supporting logic.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) # Doesn't support averaging + det = DetectorLogic(JustInternalTriggerLogic()).with_device() with pytest.raises( ValueError, match="Multiple exposures per collection not supported" @@ -363,11 +423,10 @@ async def test_exposures_per_collection_not_supported(): ) async def test_arm_timing(trigger_type, arm_timing, tmp_path): """Verify detector is armed at the correct time based on trigger type.""" - det = StandardDetector() tl = AllTriggerTypesLogic() al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, al, dl) + det = DetectorLogic(tl, al, dl).with_device() # Prepare the detector await det.prepare(TriggerInfo(trigger=trigger_type, number_of_events=2)) @@ -394,23 +453,34 @@ async def test_arm_timing(trigger_type, arm_timing, tmp_path): assert al.armed is True -async def test_trigger_arms_detector(tmp_path): - """Test that trigger() arms the detector when arm logic is present.""" - det = StandardDetector() +@pytest.mark.parametrize( + "trigger_type,arms_at_trigger", + [ + (DetectorTrigger.INTERNAL, True), + (DetectorTrigger.EXTERNAL_EDGE, False), + (DetectorTrigger.EXTERNAL_LEVEL, False), + ], +) +async def test_trigger_arms_only_an_internally_triggered_detector( + trigger_type, arms_at_trigger, tmp_path +): + """trigger() arms the detector, unless prepare() already did. + + An externally triggered detector was armed by prepare() and is sitting + waiting on its trigger source, so arming it again would restart it. + """ al = MockAcquireLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), al, dl) - - await det.prepare(TriggerInfo()) + det = DetectorLogic(AllTriggerTypesLogic(), al, dl).with_device() - # Should not be armed yet - assert al.armed is False - assert al.arm_count == 0 + await det.prepare(TriggerInfo(trigger=trigger_type)) + assert al.arm_count == (0 if arms_at_trigger else 1) + assert al.armed is not arms_at_trigger - # Trigger should arm it status = det.trigger() # Give it a moment to arm await wait_for_pending_wakeups(raise_if_exceeded=False) + # Armed either way by now, but only ever once assert al.armed is True assert al.arm_count == 1 @@ -421,9 +491,8 @@ async def test_trigger_arms_detector(tmp_path): async def test_arm_logic_called_on_stage(): """Test that acquire logic is stopped on stage.""" - det = StandardDetector() al = MockAcquireLogic() - det.add_detector_logics(al) + det = DetectorLogic(al).with_device() al.armed = True # Simulate being armed await det.stage() @@ -432,28 +501,25 @@ async def test_arm_logic_called_on_stage(): assert al.armed is False -async def test_describe_before_prepare_raises(): +async def test_describe_before_prepare_raises(tmp_path): """Test that describe() fails before prepare().""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device() - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe() async def test_describe_collect_before_prepare_raises(tmp_path): """Test that describe_collect() fails before prepare().""" - det = StandardDetector() - det.add_detector_logics(StreamableOnlyDataLogic(tmp_path)) + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device() - with pytest.raises(RuntimeError, match="Prepare not run"): + with pytest.raises(RuntimeError, match="prepare.. must be called first"): await det.describe_collect() async def test_trigger_after_multi_event_prepare_raises(): """Test that trigger() after prepare with multiple events fails.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() await det.prepare(TriggerInfo(number_of_events=5)) @@ -466,9 +532,8 @@ async def test_preserve_detector_state_requires_default_trigger_info( ): """Test PRESERVE_DETECTOR_STATE=YES errors without default_trigger_info.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="mydet") + det = DetectorLogic(AllTriggerTypesLogic()).with_device("mydet") # AllTriggerTypesLogic intentionally does not implement default_trigger_info - det.add_detector_logics(AllTriggerTypesLogic()) await det.stage() with pytest.raises( @@ -486,12 +551,12 @@ async def test_preserve_detector_state_no_trigger_logic_falls_back( trigger() silently falls back to a bare TriggerInfo() rather than raising. A detector with no trigger logic has no hardware state to preserve.""" monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="nodet") + det = DetectorLogic().with_device("nodet") await det.stage() # Should not raise — no trigger logic means nothing to preserve await det.trigger() - assert det._prepare_ctx is not None - assert det._prepare_ctx.trigger_info == TriggerInfo() + assert det.logic.data is not None + assert det.logic.data is not None async def test_preserve_detector_state_multi_collection_watcher_and_assets( @@ -510,9 +575,8 @@ async def default_trigger_info(self) -> TriggerInfo: return TriggerInfo(collections_per_event=5) monkeypatch.setenv("OPHYD_ASYNC_PRESERVE_DETECTOR_STATE", "YES") - det = StandardDetector(name="multidet") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(MultiCollectionTriggerLogic(), dl) + det = DetectorLogic(MultiCollectionTriggerLogic(), dl).with_device("multidet") await det.stage() # Collect watcher updates via watch() callback @@ -549,12 +613,11 @@ async def test_collect_asset_docs_uses_trigger_observed_event( It could be the case that ``dl.collections_written`` changes between ``trigger()`` and ``collect_asset_docs``.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) dl.collections_written, set_collections_written = soft_signal_r_and_setter( int, getter=lambda: 0 ) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("det") await det.prepare(TriggerInfo(collections_per_event=5, exposure_timeout=0.1)) status = det.trigger() @@ -567,78 +630,64 @@ async def test_collect_asset_docs_uses_trigger_observed_event( assert docs[1][1]["indices"] == {"start": 0, "stop": 1} -async def test_kickoff_respects_prepare_bounds(tmp_path): - """Test that multiple kickoff() calls respect prepared bounds.""" - det = StandardDetector() - tl = JustInternalTriggerLogic() +async def test_one_kickoff_per_prepare(tmp_path): + """A prepare serves one kickoff/complete cycle; the next needs a new prepare.""" dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, dl) + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") - # Prepare for 5 events await det.prepare(TriggerInfo(number_of_events=5)) - - # Update collections_written signal to simulate data being written - - # First kickoff for 3 events - await det.events_to_kickoff.set(3) await det.kickoff() - await dl.collections_written.set(3) - # Second kickoff for 2 events should work (total = 5) - await det.events_to_kickoff.set(2) - await det.kickoff() + # A second kickoff without an intervening complete is rejected + with pytest.raises(RuntimeError, match="prepare.* before kickoff"): + await det.kickoff() + + status = det.complete() await dl.collections_written.set(5) + await status - # Third kickoff should fail (would exceed 5) - await det.events_to_kickoff.set(1) - with pytest.raises( - RuntimeError, - match="Kickoff requested 5:6, but detector was only prepared up to 5", - ): + # ...and so is one after complete, until prepare() runs again + with pytest.raises(RuntimeError, match="prepare.* before kickoff"): await det.kickoff() async def test_stage_resets_state(): """Test that stage() resets detector state.""" - det = StandardDetector() - det.add_detector_logics(JustInternalTriggerLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device() await det.prepare(TriggerInfo(number_of_events=5)) - await det.events_to_kickoff.set(3) # Stage should reset everything await det.stage() - assert det._prepare_ctx is None - assert det._kickoff_ctx is None - assert await det.events_to_kickoff.get_value() == 0 + assert det.logic.data is None + with pytest.raises(RuntimeError, match="prepare.* must be called first"): + await det.describe() + with pytest.raises(RuntimeError, match="prepare.* before kickoff"): + await det.kickoff() -async def test_hints_from_single_data_logic(): +async def test_hints_from_single_data_logic(tmp_path): """Test that hints come from data logic.""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) + det = DetectorLogic(StreamableOnlyDataLogic(tmp_path)).with_device("bar") await det.prepare(TriggerInfo()) - assert det.hints == { - "fields": ["foo-value"] - } # ReadableOnlyDataLogic uses foo-value + assert det.hints == {"fields": ["bar"]} async def test_hints_from_multiple_data_logics(tmp_path): """Test that hints are aggregated from multiple data logics.""" - det = StandardDetector(name="bar") - dl1 = ReadableOnlyDataLogic() - dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(dl1, dl2) + dl1 = StreamableOnlyDataLogic(tmp_path) + dl2 = StreamableOnlyDataLogic(tmp_path, datakey_suffix="-extra") + det = DetectorLogic(dl1, dl2).with_device("bar") await det.prepare(TriggerInfo()) # Should include hints from both logics hints = det.hints assert "fields" in hints - assert hints["fields"] == ["foo-value", "bar"] + assert hints["fields"] == ["bar", "bar-extra"] @pytest.mark.parametrize( @@ -653,11 +702,11 @@ async def test_config_signals_in_describe_configuration( signal_type, initial_value, expected_dtype, expected_dtype_numpy, expected_shape ): """Test that added config signals appear in describe_configuration.""" - det = StandardDetector() + det = DetectorLogic().with_device() signal = soft_signal_rw( signal_type, initial_value=initial_value, name="test-config" ) - det.add_config_signals(signal) + det.set_readable_format(signal, Format.CONFIG_SIGNAL) await det.stage() @@ -683,33 +732,32 @@ async def test_config_signals_in_describe_configuration( async def test_kickoff_without_streamable_data_raises(): """Test that kickoff() without streamable data fails.""" - det = StandardDetector(name="foo") - det.add_detector_logics(JustInternalTriggerLogic(), ReadableOnlyDataLogic()) + det = DetectorLogic(JustInternalTriggerLogic()).with_device("foo") - # Single event prepare for readable-only logic + # A detector with no data logic can still be triggered for a step scan await det.prepare(TriggerInfo()) await det.trigger() # This works - # Readable-only logic doesn't support kickoff + # ...but there is nothing to collect, so it cannot be flown await det.prepare(TriggerInfo(number_of_events=5)) with pytest.raises( - ValueError, match="Detector foo is not streamable, so cannot kickoff" + ValueError, match="Detector foo has no collectable data, so cannot kickoff" ): await det.kickoff() async def test_streamable_supports_both_step_and_fly(tmp_path): """Test that streamable data logic supports both step and fly scanning.""" - det = StandardDetector(name="foo") tl = JustInternalTriggerLogic() dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(tl, dl) + det = DetectorLogic(tl, dl).with_device("foo") # Step scan should work status = det.trigger() # Yield so detector can get collections written, then set it so we complete await wait_for_pending_wakeups(raise_if_exceeded=False) await dl.collections_written.set(1) + await wait_for_pending_wakeups(raise_if_exceeded=False) assert status.done assert status.success docs = [doc async for doc in det.collect_asset_docs()] @@ -777,18 +825,18 @@ async def test_streamable_supports_both_step_and_fly(tmp_path): async def test_read_returns_correct_values(): - """Test that read() returns values from readable providers.""" - det = StandardDetector(name="det") - dl = ReadableOnlyDataLogic() - det.add_detector_logics(dl) + """Test that read() returns the values of registered signals.""" + det = DetectorLogic().with_device("det") + det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") + det.set_readable_format(det.counts, Format.HINTED_SIGNAL) await det.trigger() - await assert_reading(det, {"foo-value": {"value": 42}}) + await assert_reading(det, {"det-counts": {"value": 42}}) async def test_detector_with_no_logics(): """Test that detector works with no logics for basic internal triggering.""" - det = StandardDetector() + det = DetectorLogic().with_device() # Should support only INTERNAL triggering triggers, deadtime = await det.get_trigger_deadtime() @@ -804,7 +852,7 @@ async def test_detector_with_no_logics(): async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_triggering(): """Test that detector without trigger logic cannot set livetime/deadtime/trigger.""" - det = StandardDetector(name="foo") + det = DetectorLogic().with_device("foo") msg = "Detector foo has no trigger logic, so " with pytest.raises(ValueError, match=msg + "cannot set livetime or deadtime"): @@ -814,39 +862,41 @@ async def test_detector_without_trigger_logic_cannot_set_timing_or_exteral_trigg await det.prepare(TriggerInfo(deadtime=0.1)) -async def test_cannot_add_two_trigger_logics(): - """Test that adding two trigger logics raises an error.""" - det = StandardDetector() - tl1 = JustInternalTriggerLogic() - tl2 = AllTriggerTypesLogic() - - det.add_detector_logics(tl1) - +async def test_cannot_have_two_trigger_logics(): + """Test that two trigger logics raises an error.""" with pytest.raises(RuntimeError, match="Detector already has trigger logic"): - det.add_detector_logics(tl2) - - -async def test_cannot_add_two_arm_logics(): - """Test that adding two acquire logics raises an error.""" - det = StandardDetector() - al1 = MockAcquireLogic() - al2 = MockAcquireLogic() + DetectorLogic(JustInternalTriggerLogic(), AllTriggerTypesLogic()).with_device() - det.add_detector_logics(al1) +async def test_cannot_have_two_arm_logics(): + """Test that two acquire logics raises an error.""" with pytest.raises(RuntimeError, match="Detector already has acquire logic"): - det.add_detector_logics(al2) + DetectorLogic(MockAcquireLogic(), MockAcquireLogic()).with_device() -async def test_add_unknown_logic_type_raises(): - """Test that adding an unknown logic type raises TypeError.""" - det = StandardDetector() +async def test_unknown_logic_type_raises(): + """Test that an unknown logic type raises TypeError.""" class UnknownLogic: pass with pytest.raises(TypeError, match="Unknown logic type"): - det.add_detector_logics(UnknownLogic()) + DetectorLogic(UnknownLogic()).with_device() # type: ignore[arg-type] + + +async def test_logic_filling_two_roles_raises(): + """An object satisfying two logic protocols must not be silently half-registered.""" + + class BothAcquireAndData(DetectorAcquireLogic, DetectorDataLogic): + async def start_acquiring(self): ... + async def wait_for_idle(self): ... + async def ensure_stopped(self): ... + + with pytest.raises( + TypeError, + match="is both DetectorAcquireLogic, DetectorDataLogic", + ): + DetectorLogic(BothAcquireAndData()).with_device() @pytest.mark.parametrize("initial_shutter_closed", [True, False]) @@ -876,9 +926,8 @@ async def wait_for_idle(self): async def ensure_stopped(self): self.shutter_closed = True # close shutter at end of scan - det = StandardDetector() al = ShutterAcquireLogic(initial_shutter_closed) - det.add_detector_logics(al) + det = DetectorLogic(al).with_device() await det.stage() assert ( @@ -892,37 +941,131 @@ async def ensure_stopped(self): assert al.shutter_closed is True # unstage() must close the shutter -async def test_multiple_collections_with_single_only_logic_warns(caplog): - """Test that requesting multiple collections warns, leads to empty prepare ctx.""" - det = StandardDetector() - det.add_detector_logics(ReadableOnlyDataLogic()) +async def test_bounded_step_scan_reads_derived_from_page(): + """A bounded buffer describes as an array and read() derives one reading.""" + dl = BoundedOnlyDataLogic() + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") - await det.prepare(TriggerInfo(number_of_events=5)) + ti = TriggerInfo( + number_of_events=1, collections_per_event=5, livetime=0.1, deadtime=0.0 + ) + await det.prepare(ti) + desc = await det.describe() + assert desc["foo"]["shape"] == [5] + # prepare sized the buffer for 5 collections at the 0.1s period + assert dl.prepare_calls == [(5, pytest.approx(0.1))] + + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(5) + assert status.done + assert status.success + # prepare() armed the buffer for this point, so trigger() does not re-arm it + assert dl.prepare_calls == [(5, pytest.approx(0.1))] + assert dl.stop_count == 0 + reading = await det.read() + assert reading["foo"]["value"] == [0, 0, 0, 0, 0] + + # The next point does re-arm, as the buffer holds the last point's data + status = det.trigger() + await wait_for_pending_wakeups(raise_if_exceeded=False) + await dl.collections_written.set(5) + assert status.done + assert status.success + assert dl.prepare_calls == [(5, pytest.approx(0.1)), (5, pytest.approx(0.1))] + assert dl.stop_count == 1 + + +async def test_bounded_fly_scan_accumulates_across_kickoffs(): + """A bounded buffer is armed once at prepare and not re-armed per kickoff.""" + dl = BoundedOnlyDataLogic() + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") + + await det.prepare( + TriggerInfo(number_of_events=10, collections_per_event=1, livetime=0.1) + ) + assert dl.prepare_calls == [(10, pytest.approx(0.1))] - assert "only supports a single collection" in caplog.text - assert det._prepare_ctx is not None - assert det._prepare_ctx.readable_data_providers == [] - assert det._prepare_ctx.streamable_data_providers == [] + await det.kickoff() + status = det.complete() + # The buffer fills as the scan runs, and pages come out as it goes + await dl.collections_written.set(5) + pages = [page async for page in det._collect_pages()] + assert pages[0]["data"]["foo"] == [[0]] * 5 + assert not status.done + + await dl.collections_written.set(10) + pages = [page async for page in det._collect_pages()] + assert pages[0]["data"]["foo"] == [[0]] * 5 + await status + # kickoff() never re-arms: the buffer was only ever armed once + assert dl.prepare_calls == [(10, pytest.approx(0.1))] + + +@pytest.mark.parametrize( + "requested_livetime,default,expected_period", + [ + # livetime unset -> resolved from the trigger logic's current state + (0.0, TriggerInfo(livetime=0.1, deadtime=0.02), 0.12), + # livetime given -> used as-is, no readback substitution + (0.05, TriggerInfo(livetime=0.1, deadtime=0.02), 0.05), + # livetime unset but trigger logic has nothing set -> stays 0 + (0.0, TriggerInfo(), 0.0), + ], +) +async def test_prepare_resolves_zero_livetime_for_bounded_period( + requested_livetime, default, expected_period +): + """A livetime of 0 is filled in from the trigger logic before sizing a buffer.""" + + class PeriodTriggerLogic(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): + pass + + async def default_trigger_info(self) -> TriggerInfo: + return default + + dl = BoundedOnlyDataLogic() + det = DetectorLogic(PeriodTriggerLogic(), dl).with_device("foo") + + await det.prepare( + TriggerInfo( + number_of_events=1, collections_per_event=3, livetime=requested_livetime + ) + ) + assert dl.prepare_calls == [(3, pytest.approx(expected_period))] + + +async def test_bounded_dropped_for_infinite_events(): + """A bounded buffer cannot serve an infinite scan, so it makes no provider.""" + dl = BoundedOnlyDataLogic() + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("foo") + + await det.prepare(TriggerInfo(number_of_events=0)) + ctx = det.logic.data + assert ctx is not None + assert ctx.pageable == [] + # Sitting the scan out is not an error, and nothing was armed + assert dl.prepare_calls == [] -async def test_data_logic_with_no_prepare_methods_raises(): - """Test error when DataLogic doesn't override any prepare methods.""" + +async def test_data_logic_with_no_make_data_provider_raises(): + """Test error when a DataLogic doesn't say what it would make.""" class EmptyDataLogic(DetectorDataLogic): pass - det = StandardDetector() - det.add_detector_logics(EmptyDataLogic()) + det = DetectorLogic(EmptyDataLogic()).with_device() - with pytest.raises(RuntimeError, match="hasn't overridden any prepare_\\* methods"): + with pytest.raises(NotImplementedError): await det.prepare(TriggerInfo()) async def test_unstage_disarms_detector(): """Test that unstage() calls disarm on the detector.""" - det = StandardDetector() al = MockAcquireLogic() - det.add_detector_logics(al) + det = DetectorLogic(al).with_device() al.armed = True await det.unstage() @@ -933,9 +1076,8 @@ async def test_unstage_disarms_detector(): async def test_prepare_stops_data_logic_when_recreating_providers(tmp_path): """Test that prepare() calls stop() on data logic when recreating providers.""" - det = StandardDetector(name="det") dl = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl) + det = DetectorLogic(JustInternalTriggerLogic(), dl).with_device("det") # First prepare with collections_per_event=2 await det.prepare(TriggerInfo(number_of_events=3, collections_per_event=2)) @@ -948,10 +1090,9 @@ async def test_prepare_stops_data_logic_when_recreating_providers(tmp_path): async def test_different_collections_written_raises(tmp_path): """Test that different collections_written values from providers raises error.""" - det = StandardDetector(name="det") dl1 = StreamableOnlyDataLogic(tmp_path) dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl1, dl2) + det = DetectorLogic(JustInternalTriggerLogic(), dl1, dl2).with_device("det") await det.prepare(TriggerInfo(number_of_events=5)) @@ -969,16 +1110,17 @@ async def test_different_collections_written_raises(tmp_path): await det.kickoff() -async def test_multiple_data_logics(tmp_path): - """Test detector with multiple data logics.""" - det = StandardDetector(name="det") - dl1 = ReadableOnlyDataLogic() - dl2 = StreamableOnlyDataLogic(tmp_path) - det.add_detector_logics(JustInternalTriggerLogic(), dl1, dl2) +async def test_data_logic_and_registered_signal(tmp_path): + """A registered signal and a data logic both describe(), only one collects.""" + det = DetectorLogic( + JustInternalTriggerLogic(), StreamableOnlyDataLogic(tmp_path) + ).with_device("det") + det.counts, _ = soft_signal_r_and_setter(int, 42, name="det-counts") + det.set_readable_format(det.counts, Format.HINTED_SIGNAL) await det.prepare(TriggerInfo()) - # Should have data from both logics + # Should have data from the data logic and the registered signal description = await det.describe() assert description == { "det": { @@ -988,11 +1130,11 @@ async def test_multiple_data_logics(tmp_path): "shape": [1, 10, 15], "source": f"file://localhost/{tmp_path.as_posix().lstrip('/')}/test.h5", }, - "foo-value": { + "det-counts": { "dtype": "integer", "dtype_numpy": " adcore.AreaDetector[adcore.ADBaseIO]: driver = adcore.ADBaseIO("PREFIX:DRV:") async with init_devices(mock=True): - det = adcore.AreaDetector(driver=driver) - det.add_detector_logics(adcore.ADAcquireLogic(driver)) + det = adcore.AreaDetector( + driver=driver, acquire_logic=adcore.ADAcquireLogic(driver) + ) return det diff --git a/tests/unit_tests/epics/adcore/test_data_logic.py b/tests/unit_tests/epics/adcore/test_data_logic.py index 4696f9ee98..34bccecd40 100644 --- a/tests/unit_tests/epics/adcore/test_data_logic.py +++ b/tests/unit_tests/epics/adcore/test_data_logic.py @@ -14,6 +14,7 @@ set_mock_value, ) from ophyd_async.epics import adcore, adsimdetector +from ophyd_async.epics.adcore import ADHDFDataLogic, NDArrayDescription from ophyd_async.testing import assert_has_calls @@ -33,6 +34,80 @@ async def hdf_det( return detector +async def test_make_data_provider_does_not_write( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, +): + """Describing the data must not open the file, since it may be discarded.""" + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + ) + + made = await logic.make_data_provider( + "det", num_collections=5, period=0.1, flush_period=0.0 + ) + assert made is not None + assert await writer.capture.get_value() is False + assert await writer.file_name.get_value() == "" + + await logic.start(made[1]) + assert await writer.capture.get_value() is True + assert await writer.file_name.get_value() != "" + + +@pytest.mark.parametrize( + "enable_callbacks,plugin_enabled,makes_provider", + [ + # The default switches the plugin on, whatever it was set to + (True, EnableDisable.DISABLE, True), + # Following the plugin, a disabled one writes nothing... + (False, EnableDisable.DISABLE, False), + # ...and an enabled one still writes + (False, EnableDisable.ENABLE, True), + ], +) +async def test_hdf_follows_the_plugin_when_not_enabling_it( + hdf_det: adcore.AreaDetector[adcore.ADBaseIO], + static_path_provider: StaticPathProvider, + enable_callbacks: bool, + plugin_enabled: EnableDisable, + makes_provider: bool, +): + writer = hdf_det.get_plugin("hdf", adcore.NDFileHDF5IO) + set_mock_value(writer.file_path_exists, True) + set_mock_value(writer.enable_callbacks, plugin_enabled) + logic = ADHDFDataLogic( + array_description=NDArrayDescription( + shape_signals=[hdf_det.driver.array_size_y, hdf_det.driver.array_size_x], + data_type_signal=hdf_det.driver.data_type, + color_mode_signal=hdf_det.driver.color_mode, + ), + path_provider=static_path_provider, + driver=hdf_det.driver, + writer=writer, + enable_callbacks=enable_callbacks, + ) + + made = await logic.make_data_provider( + "det", num_collections=5, period=0.1, flush_period=0.0 + ) + assert (made is not None) is makes_provider + + if made is not None: + await logic.start(made[1]) + expected = EnableDisable.ENABLE if enable_callbacks else plugin_enabled + assert await writer.enable_callbacks.get_value() is expected + + async def test_hdf_writer_file_not_found(hdf_det: adcore.AreaDetector[adcore.ADBaseIO]): with pytest.raises( FileNotFoundError, match=r"Path .* doesn't exist or not writable!" @@ -125,6 +200,52 @@ async def test_prepare_multipart( ) +@pytest.mark.parametrize( + "flush_period,livetime,deadtime,expected_frames_per_chunk", + [ + # 0.5s flush / 0.01s period -> 50 frames per chunk (#1309) + (0.5, 0.01, 0.0, 50), + # 0.5s flush / 0.05s period -> 10 frames per chunk + (0.5, 0.04, 0.01, 10), + # A sub-frame flush period floors to a single frame per chunk + (0.001, 0.05, 0.0, 1), + ], +) +async def test_hdf_chunk_sized_from_flush_period( + static_path_provider: StaticPathProvider, + flush_period: float, + livetime: float, + deadtime: float, + expected_frames_per_chunk: int, +): + """A TriggerInfo flush_period sizes the HDF chunk from the frame period.""" + async with init_devices(mock=True): + det = adsimdetector.SimDetector( + "PREFIX:", + adcore.ADWriterFactory.hdf(static_path_provider), + ) + set_mock_value(det.driver.array_size_x, 1024) + set_mock_value(det.driver.array_size_y, 768) + set_mock_value(det.driver.data_type, adcore.ADBaseDataType.UINT16) + writer = det.get_plugin("hdf", adcore.NDPluginFileIO) + set_mock_value(writer.file_path_exists, True) + await det.prepare( + TriggerInfo( + livetime=livetime, + deadtime=deadtime, + number_of_events=3, + flush_period=flush_period, + ) + ) + assert await writer.num_frames_chunks.get_value() == expected_frames_per_chunk + (sr, *_) = [doc async for doc in det.collect_asset_docs(3)] + assert sr[1]["parameters"]["chunk_shape"] == ( + expected_frames_per_chunk, + 768, + 1024, + ) + + @pytest.mark.parametrize( "factory_cls,is_hdf", [ @@ -282,14 +403,12 @@ async def test_describe_different_color_modes( # Expected to give the right shape in the descriptor await hdf_det.prepare(TriggerInfo()) describe = await hdf_det.describe() - assert describe == { - "detector": { - "dtype": "array", - "dtype_numpy": " adcore.NDStatsIO: + async with init_devices(mock=True): + plugin = adcore.NDStatsIO("PREFIX:STATS:") + return plugin + + +async def test_start_sizes_and_arms_the_buffer(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + made = await logic.make_data_provider( + "det-stats", num_collections=5, period=0.1, flush_period=0.0 + ) + assert made is not None + provider, ctx = made + assert isinstance(provider, EventPageDataProvider) + # Describing the buffer does not arm it + assert await stats.ts_num_points.get_value() == 0 + + await logic.start(ctx) + assert await stats.ts_num_points.get_value() == 5 + assert await stats.ts_acquire_mode.get_value() == NDStatsTSAcquireMode.FIXED_LENGTH + # ts_acquire=1 arms and clears the buffer + assert await stats.ts_acquire.get_value() is True + # The datakey defaults to the Total series under the bare name + datakeys = await provider.make_datakeys(5) + assert list(datakeys) == ["det-stats"] + assert datakeys["det-stats"]["shape"] == [5] + assert logic.get_hinted_fields("det-stats") == ["det-stats"] + + +async def test_unbounded_scan_makes_no_provider(stats: adcore.NDStatsIO): + """A finite buffer cannot serve an unbounded scan, so it sits it out.""" + logic = StatsTimeSeriesDataLogic(stats) + assert ( + await logic.make_data_provider( + "det", num_collections=0, period=0.1, flush_period=0.0 + ) + is None + ) + + +@pytest.mark.parametrize( + "enable_callbacks,plugin_enabled,makes_provider", + [ + # The default switches the plugin on, whatever it was set to + (True, EnableDisable.DISABLE, True), + # Following the plugin, a disabled one produces nothing... + (False, EnableDisable.DISABLE, False), + # ...and an enabled one still works + (False, EnableDisable.ENABLE, True), + ], +) +async def test_follows_the_plugin_when_not_enabling_it( + stats: adcore.NDStatsIO, + enable_callbacks: bool, + plugin_enabled: EnableDisable, + makes_provider: bool, +): + set_mock_value(stats.enable_callbacks, plugin_enabled) + logic = StatsTimeSeriesDataLogic(stats, enable_callbacks=enable_callbacks) + + made = await logic.make_data_provider( + "det", num_collections=5, period=0.1, flush_period=0.0 + ) + assert (made is not None) is makes_provider + + if made is not None: + await logic.start(made[1]) + # Only the default reaches out and switches the plugin on + expected = EnableDisable.ENABLE if enable_callbacks else plugin_enabled + assert await stats.enable_callbacks.get_value() is expected + + +async def test_stop_stops_the_time_series(stats: adcore.NDStatsIO): + logic = StatsTimeSeriesDataLogic(stats) + set_mock_value(stats.ts_acquire, True) + await logic.stop() + assert await stats.ts_acquire.get_value() is False + + +class _JustInternal(DetectorTriggerLogic): + async def prepare_internal(self, num: int, livetime: float, deadtime: float): ... + + async def default_trigger_info(self) -> TriggerInfo: + return TriggerInfo() + + +class _FillStatsAcquireLogic(DetectorAcquireLogic): + """Fills the mock time series when acquisition starts.""" + + def __init__(self, stats: adcore.NDStatsIO, values: np.ndarray, times: np.ndarray): + self.stats = stats + self.values = values + self.times = times + + async def start_acquiring(self): + set_mock_value(self.stats.ts_total, self.values) + set_mock_value(self.stats.ts_timestamp, self.times) + set_mock_value(self.stats.ts_current_point, len(self.values)) + + async def wait_for_idle(self): ... + + async def ensure_stopped(self): ... + + +@pytest.fixture +def stats_detector(stats: adcore.NDStatsIO) -> StandardDetector: + """A writer-less detector whose only data logic is a stats time series.""" + values = np.array([5.0, 6.0, 7.0, 8.0]) + times = np.array([200.0, 201.0, 202.0, 203.0]) + return DetectorLogic( + _JustInternal(), + _FillStatsAcquireLogic(stats, values, times), + StatsTimeSeriesDataLogic(stats), + ).with_device("det") + + +def test_step_scan_through_run_engine(RE: RunEngine, stats_detector: StandardDetector): + """A writer-less detector describes and reads as internal data. + + Driven through the RunEngine so the descriptor its datakeys make is the one + a real scan would emit, and so has to pass event-model validation. + """ + det = stats_detector + docs: dict[str, list] = defaultdict(list) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + # prepare() comes after stage(), which resets the detector + yield from bps.prepare(det, TriggerInfo(collections_per_event=4), wait=True) + yield from bps.trigger_and_read([det]) + + RE(plan(), lambda name, doc: docs[name].append(doc)) + + (descriptor,) = docs["descriptor"] + assert descriptor["data_keys"]["det"]["shape"] == [4] + assert "external" not in descriptor["data_keys"]["det"] + (event,) = docs["event"] + np.testing.assert_array_equal(event["data"]["det"], [5.0, 6.0, 7.0, 8.0]) + assert event["timestamps"]["det"] == 203.0 + + +def test_fly_scan_emits_event_pages(RE: RunEngine, stats_detector: StandardDetector): + """A writer-less detector collects its buffer as event pages.""" + det = stats_detector + docs: dict[str, list] = defaultdict(list) + + @bpp.stage_decorator([det]) + @bpp.run_decorator() + def plan(): + yield from bps.prepare(det, TriggerInfo(number_of_events=4), wait=True) + yield from bps.declare_stream(det, name="primary") + yield from bps.kickoff(det, wait=True) + yield from bps.collect_while_completing( + flyers=[det], dets=[det], flush_period=0.05 + ) + + RE(plan(), lambda name, doc: docs[name].append(doc)) + + (page,) = docs["event_page"] + np.testing.assert_array_equal(page["data"]["det"], [[5.0], [6.0], [7.0], [8.0]]) + assert docs["stop"][0]["num_events"] == {"primary": 4} + + +async def test_step_scan_read_derives_single_reading(stats_detector: StandardDetector): + """A step-scan read derives one reading holding the whole buffer, timed by + the last point's ts_timestamp.""" + det = stats_detector + await det.stage() + await det.prepare(TriggerInfo(collections_per_event=4)) + await det.trigger() + reading = await det.read() + np.testing.assert_array_equal(reading["det"]["value"], [5.0, 6.0, 7.0, 8.0]) + assert reading["det"]["timestamp"] == 203.0 + await det.unstage() diff --git a/tests/unit_tests/epics/adcore/test_use_cases.py b/tests/unit_tests/epics/adcore/test_use_cases.py index 9658fd1cab..e4139a7f1b 100644 --- a/tests/unit_tests/epics/adcore/test_use_cases.py +++ b/tests/unit_tests/epics/adcore/test_use_cases.py @@ -2,6 +2,7 @@ import pytest +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.core import ( StaticPathProvider, TriggerInfo, @@ -11,7 +12,6 @@ ) from ophyd_async.epics import adaravis, adcore, adsimdetector from ophyd_async.epics.core import epics_signal_r -from ophyd_async.testing import assert_configuration, assert_reading async def test_step_scan_hdf_detector_with_stats_and_temp( @@ -25,6 +25,8 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( plugins={"stats": stat}, ) temp = epics_signal_r(float, "SAMP:TEMP:RBV") + # Plugins are not registered automatically; opt this one in + det.set_readable_format(stat, Format.CHILD) ndattributes = [ adcore.NDAttributeParam( name="det-sum", @@ -47,9 +49,10 @@ async def test_step_scan_hdf_detector_with_stats_and_temp( set_mock_value(det.driver.array_size_y, 768) await det.stage() - assert await det.driver.wait_for_plugins.get_value() is False + # ensure_ready sets WaitForPlugins so read() sees this frame's scalars + assert await det.driver.wait_for_plugins.get_value() is True config_description = await det.describe_configuration() - assert config_description == { + assert { "det-driver-acquire_period": { "dtype": "number", "dtype_numpy": " DetectorLogic: + return self._logic + @pytest.fixture def odin_det(RE: RunEngine, tmp_path) -> OdinDet: diff --git a/tests/unit_tests/fastcs/panda/test_hdf_panda.py b/tests/unit_tests/fastcs/panda/test_hdf_panda.py index 7b9e87b4e0..489a38d2a8 100644 --- a/tests/unit_tests/fastcs/panda/test_hdf_panda.py +++ b/tests/unit_tests/fastcs/panda/test_hdf_panda.py @@ -187,7 +187,9 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): "source": uri, }, } - assert hdf_panda.hints == {"fields": []} + # No hinted fields is {} rather than {"fields": []}, as for any + # StandardReadable -- the detector no longer special cases this + assert hdf_panda.hints == {} # Kick it off and start it completing await hdf_panda.kickoff() status = hdf_panda.complete() @@ -274,7 +276,7 @@ async def test_flyscan_documents(hdf_panda: HDFPanda, tmp_path): # Check that another kickoff without prepare is not allowed with pytest.raises( RuntimeError, - match="Kickoff requested 15:30, but detector was only prepared up to 15", + match="prepare.* must be called before kickoff", ): await hdf_panda.kickoff() # But preparing again is ok diff --git a/tests/unit_tests/plan_stubs/test_settings.py b/tests/unit_tests/plan_stubs/test_settings.py index c64f2a36c3..2f056739c6 100644 --- a/tests/unit_tests/plan_stubs/test_settings.py +++ b/tests/unit_tests/plan_stubs/test_settings.py @@ -6,7 +6,17 @@ import pytest import yaml -from ophyd_async.core import Settings, YamlSettingsProvider, get_mock +from ophyd_async.core import ( + Device, + Settings, + StandardReadable, + YamlSettingsProvider, + apply_readable_formats, + get_mock, + soft_signal_rw, + walk_readable_formats, +) +from ophyd_async.core import StandardReadableFormat as Format from ophyd_async.plan_stubs import ( apply_settings, apply_settings_if_different, @@ -179,3 +189,433 @@ def my_plan(): assert m.mock_calls == [call.sig_rw.put("foo")] RE(my_plan()) + + +@pytest.fixture +def technique_device() -> StandardReadable: + """A Device whose energy signal is config for one technique, hinted for another.""" + device = StandardReadable(name="mono") + device.energy = soft_signal_rw(float, 7.0) + device.temperature = soft_signal_rw(float, 20.0) + device.set_name("mono") + device.set_readable_format(device.energy, Format.CONFIG_SIGNAL) + device.set_readable_format(device.temperature, Format.CONFIG_SIGNAL) + return device + + +async def test_store_settings_writes_formats_under_reserved_key( + RE, technique_device, tmp_path +): + provider = YamlSettingsProvider(tmp_path) + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + with open(tmp_path / "fixed_energy.yaml") as f: + stored = yaml.safe_load(f) + # Values stay flat, formats live under a key that cannot be a path + assert stored == { + "energy": 7.0, + "temperature": 20.0, + "": { + "": { + "energy": "CONFIG_SIGNAL", + "temperature": "CONFIG_SIGNAL", + } + }, + } + + RE(my_plan()) + + +async def test_reserved_keys_need_no_quoting_in_yaml(RE, technique_device, tmp_path): + provider = YamlSettingsProvider(tmp_path) + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + text = (tmp_path / "fixed_energy.yaml").read_text() + assert ":" in text + assert ":" in text + # An empty string root would make yaml fall back to "? ''" explicit keys + assert "?" not in text + + RE(my_plan()) + + +async def test_settings_round_trip_switches_technique(RE, technique_device, tmp_path): + provider = YamlSettingsProvider(tmp_path) + energy, temperature = technique_device.energy, technique_device.temperature + + def my_plan(): + yield from store_settings(provider, "fixed_energy", technique_device) + + # Switch to a technique that scans energy and drops temperature entirely + technique_device.set_readable_format(energy, Format.HINTED_SIGNAL) + technique_device.set_readable_format(temperature, None) + yield from bps.abs_set(energy, 9.0, wait=True) + yield from store_settings(provider, "scan_energy", technique_device) + + # Going back restores both the value and the formats in one apply + fixed = yield from retrieve_settings(provider, "fixed_energy", technique_device) + yield from apply_settings(fixed) + assert ( + technique_device.get_readable_formats().get(energy) is Format.CONFIG_SIGNAL + ) + assert ( + technique_device.get_readable_formats().get(temperature) + is Format.CONFIG_SIGNAL + ) + assert (yield from bps.rd(energy)) == 7.0 + + # And forward again drops temperature rather than merging the two + scanning = yield from retrieve_settings( + provider, "scan_energy", technique_device + ) + yield from apply_settings(scanning) + assert ( + technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL + ) + assert technique_device.get_readable_formats().get(temperature) is None + assert (yield from bps.rd(energy)) == 9.0 + + RE(my_plan()) + + +async def test_settings_file_without_formats_leaves_them_alone( + RE, technique_device, tmp_path +): + """A file stored before formats existed must not clear them.""" + provider = YamlSettingsProvider(tmp_path) + energy = technique_device.energy + (tmp_path / "old_style.yaml").write_text("energy: 3.0\ntemperature: 4.0\n") + + def my_plan(): + settings = yield from retrieve_settings(provider, "old_style", technique_device) + assert settings.readable_formats is None + yield from apply_settings(settings) + # Values applied, formats untouched + assert (yield from bps.rd(energy)) == 3.0 + assert ( + technique_device.get_readable_formats().get(energy) is Format.CONFIG_SIGNAL + ) + + RE(my_plan()) + + +async def test_formats_only_file_applies_without_touching_values( + RE, technique_device, tmp_path +): + """A hand written formats only profile changes no hardware. + + Also covers the explicit null, which is the only way to unregister a child, + since applying otherwise merges. + """ + provider = YamlSettingsProvider(tmp_path) + energy = technique_device.energy + (tmp_path / "hinted.yaml").write_text( + ":\n" + " :\n" + " energy: HINTED_SIGNAL\n" + " temperature: null\n" + ) + + def my_plan(): + settings = yield from retrieve_settings(provider, "hinted", technique_device) + assert dict(settings) == {} + yield from apply_settings(settings) + assert ( + technique_device.get_readable_formats().get(energy) is Format.HINTED_SIGNAL + ) + # An explicit null drops it, where omitting it would have left it alone + assert ( + technique_device.get_readable_formats().get(technique_device.temperature) + is None + ) + assert (yield from bps.rd(energy)) == 7.0 + + RE(my_plan()) + + +class _Mono(StandardReadable): + def __init__(self, name=""): + with self.add_children_as_readables(Format.CONFIG_SIGNAL): + self.energy = soft_signal_rw(float, 7.0) + self.temperature = soft_signal_rw(float, 20.0) + super().__init__(name=name) + + +async def test_store_settings_writes_only_changes_from_class_defaults(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + mono = _Mono(name="mono") + + def stored_formats(name: str): + with open(tmp_path / f"{name}.yaml") as f: + return yaml.safe_load(f)[""] + + def my_plan(): + yield from store_settings(provider, "defaults", mono) + mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + mono.set_readable_format(mono.temperature, None) + yield from store_settings(provider, "scan", mono) + assert stored_formats("defaults") == {} + # A dropped default is written as a null + assert stored_formats("scan") == { + "": {"energy": "HINTED_SIGNAL", "temperature": None} + } + # The defaults file undoes changes it does not mention + defaults = yield from retrieve_settings(provider, "defaults", mono) + yield from apply_settings(defaults) + assert mono.get_readable_formats() == { + mono.energy: Format.CONFIG_SIGNAL, + mono.temperature: Format.CONFIG_SIGNAL, + } + scan = yield from retrieve_settings(provider, "scan", mono) + yield from apply_settings(scan) + assert mono.get_readable_formats() == {mono.energy: Format.HINTED_SIGNAL} + + RE(my_plan()) + + +async def test_null_formats_key_resets_to_class_defaults(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + mono = _Mono(name="mono") + mono.set_readable_format(mono.energy, Format.HINTED_SIGNAL) + (tmp_path / "reset.yaml").write_text(":\n") + + def my_plan(): + settings = yield from retrieve_settings(provider, "reset", mono) + assert settings.readable_formats == {} + yield from apply_settings(settings) + assert mono.get_readable_formats().get(mono.energy) is Format.CONFIG_SIGNAL + + RE(my_plan()) + + +async def test_retrieve_settings_warns_on_unknown_reserved_keys( + RE, technique_device, tmp_path +): + """A typo'd or newer reserved key is skipped with a warning, not a NameError.""" + provider = YamlSettingsProvider(tmp_path) + (tmp_path / "newer.yaml").write_text( + "energy: 3.0\n:\n : {}\n: 1\n" + ) + + def my_plan(): + match = r"unknown reserved keys \['', ''\]" + with pytest.warns(UserWarning, match=match): + settings = yield from retrieve_settings(provider, "newer", technique_device) + assert dict(settings) == {technique_device.energy: 3.0} + yield from apply_settings(settings) + assert (yield from bps.rd(technique_device.energy)) == 3.0 + + RE(my_plan()) + + +async def test_retrieve_settings_still_rejects_a_genuine_typo( + RE, technique_device, tmp_path +): + provider = YamlSettingsProvider(tmp_path) + (tmp_path / "typo.yaml").write_text("energyy: 3.0\n") + + def my_plan(): + with pytest.raises(NameError, match=r"Unknown signal names \['energyy'\]"): + yield from retrieve_settings(provider, "typo", technique_device) + + RE(my_plan()) + + +async def test_store_settings_walks_nested_devices(RE, tmp_path): + provider = YamlSettingsProvider(tmp_path) + inner = StandardReadable(name="inner") + inner.sig = soft_signal_rw(float, 1.0) + outer = StandardReadable(name="outer") + outer.inner = inner + outer.top = soft_signal_rw(float, 2.0) + outer.set_name("outer") + inner.set_readable_format(inner.sig, Format.HINTED_SIGNAL) + outer.set_readable_format(outer.top, Format.CONFIG_SIGNAL) + outer.set_readable_format(inner, Format.CHILD) + # The same signal registered on a second owner, from deeper in its tree + outer.set_readable_format(inner.sig, Format.CONFIG_SIGNAL) + + def my_plan(): + yield from store_settings(provider, "nested", outer) + with open(tmp_path / "nested.yaml") as f: + assert yaml.safe_load(f) == { + "top": 2.0, + "inner.sig": 1.0, + "": { + "": { + "top": "CONFIG_SIGNAL", + "inner": "CHILD", + "inner.sig": "CONFIG_SIGNAL", + }, + # Child paths are relative to their owner + "inner": {"sig": "HINTED_SIGNAL"}, + }, + } + + RE(my_plan()) + + +def test_apply_readable_formats_rejects_unknown_path(technique_device): + with pytest.raises(KeyError, match="No Device at 'nope'"): + apply_readable_formats(technique_device, {"nope": {}}) + + +def test_apply_readable_formats_rejects_non_readable(technique_device): + with pytest.raises(TypeError, match="is not a StandardReadable"): + apply_readable_formats(technique_device, {"energy": {}}) + + +def test_settings_or_merges_formats_per_child(technique_device): + # Values merge per signal and apply_readable_formats merges per child, so + # `other` must only replace the children it mentions -- overriding a whole + # owner would drop children it has nothing to say about + base = Settings( + technique_device, + {technique_device.energy: 1.0}, + { + "": { + "energy": Format.CONFIG_SIGNAL, + "temperature": Format.CHILD, + } + }, + ) + override = Settings( + technique_device, + {technique_device.temperature: 2.0}, + {"": {"energy": Format.HINTED_SIGNAL}}, + ) + merged = base | override + assert merged.readable_formats == { + "": { + "energy": Format.HINTED_SIGNAL, # overridden + "temperature": Format.CHILD, # untouched, not dropped + } + } + assert dict(merged) == { + technique_device.energy: 1.0, + technique_device.temperature: 2.0, + } + # Neither operand is mutated + assert base.readable_formats[""]["energy"] is Format.CONFIG_SIGNAL + assert override.readable_formats[""] == { + "energy": Format.HINTED_SIGNAL + } + + +def test_settings_or_keeps_owners_the_other_does_not_mention(technique_device): + base = Settings(technique_device, {}, {"a": {"x": Format.CONFIG_SIGNAL}}) + override = Settings(technique_device, {}, {"b": {"y": Format.HINTED_SIGNAL}}) + assert (base | override).readable_formats == { + "a": {"x": Format.CONFIG_SIGNAL}, + "b": {"y": Format.HINTED_SIGNAL}, + } + + +def test_settings_or_rejects_formats_from_a_sub_device(technique_device): + # A sub-device's paths are relative to itself, so would silently come + # to mean the parent + parent = StandardReadable(name="parent") + parent.child = technique_device + parent.set_name("parent") + base = Settings(parent) + sub = Settings( + technique_device, {}, {"": {"energy": Format.HINTED_SIGNAL}} + ) + with pytest.raises(ValueError, match="paths are relative to it"): + base | sub + # Without formats it is just a value merge, which is fine + assert (base | Settings(technique_device)).readable_formats is None + + +def test_settings_partition_carries_formats_to_both_halves(technique_device): + # apply_panda_settings applies only the halves, never the original, so + # formats must survive on whichever half is applied + settings = Settings( + technique_device, + {technique_device.energy: 1.0, technique_device.temperature: 2.0}, + {"": {"energy": Format.CONFIG_SIGNAL}}, + ) + a, b = settings.partition(lambda sig: "energy" in sig.name) + assert a.readable_formats == settings.readable_formats + assert b.readable_formats == settings.readable_formats + + +async def test_store_settings_writes_empty_formats_for_an_unchanged_readable( + RE, tmp_path +): + """The key is still written, as applying it resets runtime changes. + + This is the shape of TangoTestDevice. + """ + provider = YamlSettingsProvider(tmp_path) + device = StandardReadable(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + + def my_plan(): + yield from store_settings(provider, "empty", device) + with open(tmp_path / "empty.yaml") as f: + assert yaml.safe_load(f) == {"sig": 1.0, "": {}} + + RE(my_plan()) + + +async def test_store_settings_warns_and_skips_a_readable_outside_the_tree(RE, tmp_path): + """A child registered from outside the tree has no path to store it against. + + Its value is not stored either, since store_settings only walks the tree, so + it cannot survive a round trip at all. See + https://github.com/bluesky/ophyd-async/issues/1402. + """ + provider = YamlSettingsProvider(tmp_path) + device = StandardReadable(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + outsider = soft_signal_rw(float, 2.0, name="outsider") + device.set_readable_format(device.sig, Format.CONFIG_SIGNAL) + device.set_readable_format(outsider, Format.CONFIG_SIGNAL) + + match = "dev: outsider is not within dev" + with pytest.warns(UserWarning, match=match): + assert walk_readable_formats(device) == { + "": {"sig": Format.CONFIG_SIGNAL} + } + + def my_plan(): + with pytest.warns(UserWarning, match=match): + yield from store_settings(provider, "outsider", device) + with open(tmp_path / "outsider.yaml") as f: + # Neither the outsider's format nor its value is stored + assert yaml.safe_load(f) == { + "sig": 1.0, + "": {"": {"sig": "CONFIG_SIGNAL"}}, + } + + RE(my_plan()) + + +async def test_store_settings_omits_the_key_for_a_device_with_no_readables( + RE, tmp_path +): + """A Device with no StandardReadable in its tree has nothing to clear. + + This is the shape of EpicsTestCaDevice, and is why the EPICS golden files + needed no regeneration. + """ + provider = YamlSettingsProvider(tmp_path) + + class Plain(Device): + pass + + device = Plain(name="dev") + device.sig = soft_signal_rw(float, 1.0) + device.set_name("dev") + + def my_plan(): + yield from store_settings(provider, "plain", device) + with open(tmp_path / "plain.yaml") as f: + assert yaml.safe_load(f) == {"sig": 1.0} + + RE(my_plan()) diff --git a/tests/unit_tests/test_data/test_yaml_config_save.yaml b/tests/unit_tests/test_data/test_yaml_config_save.yaml index 99fb4ac43c..1c981f1ef5 100644 --- a/tests/unit_tests/test_data/test_yaml_config_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_config_save.yaml @@ -1,3 +1,4 @@ +: {} a_bool: true a_enum: Bbb a_float: 1.234 diff --git a/tests/unit_tests/test_data/test_yaml_save.yaml b/tests/unit_tests/test_data/test_yaml_save.yaml index 900837389f..1d2d7047bb 100644 --- a/tests/unit_tests/test_data/test_yaml_save.yaml +++ b/tests/unit_tests/test_data/test_yaml_save.yaml @@ -1,3 +1,4 @@ +: {} _sig_rw: Top level private SignalRW child.a_bool: true child.a_enum: Bbb