diff --git a/docs/explanations/beamlines.md b/docs/explanations/beamlines.md new file mode 100644 index 00000000..47e64241 --- /dev/null +++ b/docs/explanations/beamlines.md @@ -0,0 +1,8 @@ +# Beamlines + +```{toctree} +:maxdepth: 1 +:glob: + +i15-1 +``` diff --git a/docs/explanations/beamlines/i15-1/auxiliary_scans.md b/docs/explanations/beamlines/i15-1/auxiliary_scans.md new file mode 100644 index 00000000..adf40b41 --- /dev/null +++ b/docs/explanations/beamlines/i15-1/auxiliary_scans.md @@ -0,0 +1,25 @@ +# i15-1 Auxiliary Scans + +I15-1 PDF experiments require some scans to be run before hand, for use during analysis. These are added to the queue automatically by the i15-1 queue plugin. For each experiment, we need: + +- an air scan, at room temperature +- an empty capillary scan, matching the capillary type and temperature range to the experiment +- a standard sample scan, at room temperature, matching the capillary type to the experiment + +These must happen in this order, as the analysis process is layered - i.e, the empty capillary scan needs the air scan to be processed, the standard sample needs the empty capillary, and therefore the air, and the actual experiment needs all three. Additionally, to save time, we can merge these auxiliary scans together, to avoid queuing up three additional scans for every experiment. For example, experiments can always share an air scan (as it's always the same); empty capillary scans can be shared as long as the temperature range spans the range of the experiment, and the capillary type is the same; and standard sample scans can be shared as long as the capillary type is the same. + +The above requirements lead to the following logic: + +- An experiment will require the three auxiliary scans outlined above, the details of which will be a function of the experiment parameters. See `I151Converter._get_required_auxiliary_scans` +- An `AuxiliaryScan` has a `requires` field, which details the other required auxiliary scans that must proceed it. The `AuxiliaryScan` objects produced by `I151Converter._get_required_auxiliary_scans` have each other as requirements. The empty capillary scan will be required by the standard sample scan, and the air scan will be required by both the other two. + +To add auxiliary scans, we build a new queue that includes the tasks along with their auxiliary scans. For each task in the queue, we will: + +- Gather the required auxiliary scans +- For each required auxiliary scan: +- Check whether a matching auxiliary scan already exists in the queue or tiled. For an existing scan to be considered suitable, all of the required auxiliary scan's own required auxiliary scans must also be required by the existing scan. +- If a suitable existing does not exist, we will see if we can combine the required auxiliary scan with an existing scan. This, for example, could involve expanding the temperature range of a queued auxiliary scan. Combining these auxiliary scans also combines the requirements of both scans, so no requirement information is lost, but we also don't duplicate similar requirements. We then go to the beginning of this checklist, and repeat for the required auxiliary scans of the combined auxiliary scan, inserting the results before the combined scan in the queue. +- If we cannot combine the required auxiliary scan, we will add the auxiliary scan to the queue. However, right before we do this, we will go through the required auxiliary scans of the auxiliary scan we added, and repeat this checklist for them. This ensures the auxiliary scan is added behind its requirements. +- Once all the recursive adding of required auxiliary scans is complete, we add the task itself to the new queue. + +![Auxiliary scan logic flowchart](./i15-1_aux_scans.drawio.png) diff --git a/docs/explanations/beamlines/i15-1/backgrounds_logic.drawio.png b/docs/explanations/beamlines/i15-1/backgrounds_logic.drawio.png deleted file mode 100644 index 8670a22c..00000000 Binary files a/docs/explanations/beamlines/i15-1/backgrounds_logic.drawio.png and /dev/null differ diff --git a/docs/explanations/beamlines/i15-1/i15-1.md b/docs/explanations/beamlines/i15-1/i15-1.md new file mode 100644 index 00000000..63aaf926 --- /dev/null +++ b/docs/explanations/beamlines/i15-1/i15-1.md @@ -0,0 +1,8 @@ +# i15-1 + +```{toctree} +:maxdepth: 1 +:glob: + +Auxiliary Scans +``` diff --git a/docs/explanations/beamlines/i15-1/i15-1_aux_scans.drawio.png b/docs/explanations/beamlines/i15-1/i15-1_aux_scans.drawio.png new file mode 100644 index 00000000..a94aae81 Binary files /dev/null and b/docs/explanations/beamlines/i15-1/i15-1_aux_scans.drawio.png differ diff --git a/docs/reference/rest_api.json b/docs/reference/rest_api.json index c93877bf..0785aa54 100644 --- a/docs/reference/rest_api.json +++ b/docs/reference/rest_api.json @@ -1072,29 +1072,9 @@ "OIDCConfig": { "properties": { "well_known_url": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], + "type": "string", "title": "Well Known Url", - "description": "URL to fetch OIDC config from the provider", - "deprecated": true - }, - "issuer": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Issuer", - "description": "URL of OIDC provider" + "description": "URL to fetch OIDC config from the provider" }, "client_id": { "type": "string", @@ -1117,6 +1097,7 @@ "additionalProperties": false, "type": "object", "required": [ + "well_known_url", "client_id" ], "title": "OIDCConfig" diff --git a/src/daq_queuing_service/plugins/i15_1/auxiliary.py b/src/daq_queuing_service/plugins/i15_1/auxiliary.py index 061336fd..24234850 100644 --- a/src/daq_queuing_service/plugins/i15_1/auxiliary.py +++ b/src/daq_queuing_service/plugins/i15_1/auxiliary.py @@ -1,8 +1,10 @@ +from collections.abc import Sequence from enum import StrEnum from pathlib import Path +from typing import Self from daq_config_server.models.i15_1.standards_puck import StandardsPin -from pydantic import BaseModel, ConfigDict, computed_field +from pydantic import BaseModel, ConfigDict, Field, computed_field from daq_queuing_service.task_queue.task import Experiment @@ -32,6 +34,25 @@ def get_background_temperatures( ) +def combine_auxiliary_requirements( + required_auxiliaries_1: Sequence["AuxiliaryScan"], + required_auxiliaries_2: list["AuxiliaryScan"], +) -> list["AuxiliaryScan"]: + result: list[AuxiliaryScan] = [] + remaining = required_auxiliaries_2.copy() + for aux1 in required_auxiliaries_1: + for aux2 in remaining: + if combined := aux1.attempt_to_combine_with(aux2): + result.append(combined) + remaining.remove(aux2) + + break + else: + result.append(aux1) + + return result + remaining + + class AuxiliaryScan(BaseModel): """Auxiliary scans are scans added automatically by the queue, such as background scans and standard sample calibration scans. They are needed for analysis of user @@ -43,6 +64,7 @@ class AuxiliaryScan(BaseModel): pin: StandardsPin | None time_per_pdf: float list_of_temperatures: list[int] | None = None + requires: list[Self] = Field(default_factory=lambda: [], exclude=True) @computed_field @property @@ -59,7 +81,9 @@ def is_suitable( temperature_step: int = DEFAULT_TEMPERATURE_STEP, ) -> bool: """Determine if this background is suitable compared to an experiment's required - background. + background. This includes checking that scans in required_background.requires + have a suitable pairing in self.requires, to ensure nested requirements are + taken into account. Args: required_background (AuxiliaryScan): The required background @@ -75,6 +99,10 @@ def is_suitable( elif self.list_of_temperatures: return False + for required in required_background.requires: + if not any(r.is_suitable(required) for r in self.requires): + return False + return ( self.instrument_session == required_background.instrument_session and self.pin == required_background.pin @@ -109,7 +137,8 @@ def attempt_to_combine_with( temperature_step: int = DEFAULT_TEMPERATURE_STEP, ) -> "AuxiliaryScan | None": """Creates a background that combines the requirements of this background object - and a provided required background, if possible. + and a provided required background, if possible. Anything that previously deemed + this scan as suitable should also deem the combined scan suitable. Args: required_background (AuxiliaryScan): The required background @@ -142,6 +171,9 @@ def attempt_to_combine_with( pin=self.pin, time_per_pdf=max(self.time_per_pdf, required_background.time_per_pdf), list_of_temperatures=list_of_temperatures, + requires=combine_auxiliary_requirements( + self.requires, required_background.requires + ), ) assert result.is_suitable(required_background) return result @@ -167,6 +199,9 @@ def from_experiment(cls, experiment: Experiment) -> "AuxiliaryScan": list_of_temperatures=experiment.experiment_definition.data.get( "list_of_temperatures" ), + requires=experiment.experiment_definition.data.get( + "required_auxiliary_scans", [] + ), ) diff --git a/src/daq_queuing_service/plugins/i15_1/i15_1_converter.py b/src/daq_queuing_service/plugins/i15_1/i15_1_converter.py index 5819c899..14e0beb1 100644 --- a/src/daq_queuing_service/plugins/i15_1/i15_1_converter.py +++ b/src/daq_queuing_service/plugins/i15_1/i15_1_converter.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass, replace from functools import cached_property from typing import Any @@ -52,9 +53,16 @@ ) -def _filter_auxiliary_scans(tasks: list[Task]) -> list[tuple[int, AuxiliaryScan]]: +@dataclass(frozen=True) +class QueuedAuxiliary: + index: int + task_id: str + auxiliary: AuxiliaryScan + + +def _filter_auxiliary_scans(tasks: list[Task]) -> list[QueuedAuxiliary]: return [ - (i, AuxiliaryScan.from_experiment(task.experiment)) + QueuedAuxiliary(i, task.id, AuxiliaryScan.from_experiment(task.experiment)) for i, task in enumerate(tasks) if isinstance(task.experiment, Experiment) and is_auxiliary_experiment(task.experiment) @@ -200,9 +208,7 @@ def _add_required_auxiliary_scans( for task in tasks: with add_session_to_logger(task.experiment.instrument_session): experiment = task.experiment - if isinstance(experiment, Experiment) and not is_auxiliary_experiment( - experiment - ): + if isinstance(experiment, Experiment): instrument_session = experiment.instrument_session required_auxiliary_scans = self._get_required_auxiliary_scans( @@ -241,8 +247,8 @@ def _ensure_auxiliary_in_queue_or_tiled( queued_auxiliary_scans = _filter_auxiliary_scans(new_tasks) if any( - queued_auxiliary.is_suitable(required_auxiliary) - for _, queued_auxiliary in queued_auxiliary_scans + queued_auxiliary.auxiliary.is_suitable(required_auxiliary) + for queued_auxiliary in queued_auxiliary_scans ): return new_tasks @@ -260,6 +266,7 @@ def _ensure_auxiliary_in_queue_or_tiled( ) return self._add_or_replace_auxiliary_scan( required_auxiliary, + current_task, new_tasks, queued_auxiliary_scans, instrument_session, @@ -268,32 +275,66 @@ def _ensure_auxiliary_in_queue_or_tiled( def _add_or_replace_auxiliary_scan( self, required_auxiliary: AuxiliaryScan, + current_task: TaskWithPosition | None, new_tasks: list[Task], - queued_auxiliary_scans: list[tuple[int, AuxiliaryScan]], + queued_auxiliary_scans: list[QueuedAuxiliary], instrument_session: str, ) -> list[Task]: - combined_auxiliary_scan = None - index = None - - for i, auxiliary_scan in queued_auxiliary_scans: - if combined_auxiliary_scan := auxiliary_scan.attempt_to_combine_with( - required_auxiliary - ): - index = i - break + # It is assumed every scan in queued_auxiliary_scans has already added their + # requirements before for them + combined_auxiliary = self._find_combinable_auxiliary( + required_auxiliary, queued_auxiliary_scans + ) aux_experiment = self._construct_auxiliary_experiment( - combined_auxiliary_scan or required_auxiliary, instrument_session + combined_auxiliary.auxiliary if combined_auxiliary else required_auxiliary, + instrument_session, ) - if index is None: - new_tasks.append(Task(experiment=aux_experiment)) + + if combined_auxiliary is None: + aux_task = Task(experiment=aux_experiment) + prefix = new_tasks + suffix = [] + additional_required = required_auxiliary.requires else: - new_tasks[index] = Task(experiment=aux_experiment) + aux_task = Task(experiment=aux_experiment, id=combined_auxiliary.task_id) + prefix = new_tasks[: combined_auxiliary.index] + suffix = new_tasks[combined_auxiliary.index + 1 :] + additional_required = combined_auxiliary.auxiliary.requires + + self._tiled_auxiliary_scans[aux_task.id] = {} + for required in additional_required: + prefix = self._ensure_auxiliary_in_queue_or_tiled( + required, + current_task, + prefix, + aux_task.id, + instrument_session, + ) + new_tasks = prefix + [aux_task] + suffix + return new_tasks + def _find_combinable_auxiliary( + self, + required_auxiliary: AuxiliaryScan, + queued_auxiliary_scans: list[QueuedAuxiliary], + ): + for queued_auxiliary in queued_auxiliary_scans: + if ( + combined_auxiliary_scan + := queued_auxiliary.auxiliary.attempt_to_combine_with( + required_auxiliary + ) + ): + return replace(queued_auxiliary, auxiliary=combined_auxiliary_scan) + def _get_required_auxiliary_scans( self, experiment: Experiment ) -> list[AuxiliaryScan]: + if is_auxiliary_experiment(experiment): + return experiment.experiment_definition.data["required_auxiliary_scans"] + time_per_pdf = experiment.experiment_definition.data["time_per_pdf"] list_of_temperatures = experiment.experiment_definition.data.get( "list_of_temperatures" @@ -302,34 +343,40 @@ def _get_required_auxiliary_scans( list_of_temperatures = get_background_temperatures(list_of_temperatures) assert experiment.sample - return [ - AuxiliaryScan( - instrument_session=experiment.instrument_session, - pin=None, - time_per_pdf=time_per_pdf, - ), - AuxiliaryScan( - instrument_session=experiment.instrument_session, - pin=StandardsPin( - capillary=experiment.sample.data["capillary"], contents=None - ), - time_per_pdf=time_per_pdf, - list_of_temperatures=list_of_temperatures, + + air = AuxiliaryScan( + instrument_session=experiment.instrument_session, + pin=None, + time_per_pdf=time_per_pdf, + ) + empty_capillary = AuxiliaryScan( + instrument_session=experiment.instrument_session, + pin=StandardsPin( + capillary=experiment.sample.data["capillary"], contents=None ), - AuxiliaryScan( - instrument_session=experiment.instrument_session, - pin=StandardsPin( - capillary=experiment.sample.data["capillary"], contents="Silicon" - ), - time_per_pdf=time_per_pdf, + time_per_pdf=time_per_pdf, + list_of_temperatures=list_of_temperatures, + requires=[air], + ) + standard_sample = AuxiliaryScan( + instrument_session=experiment.instrument_session, + pin=StandardsPin( + capillary=experiment.sample.data["capillary"], contents="Silicon" ), - ] + time_per_pdf=time_per_pdf, + requires=[air, empty_capillary], + ) + + return [standard_sample, empty_capillary, air] def _construct_auxiliary_experiment( self, auxiliary_scan: AuxiliaryScan, instrument_session: str ) -> Experiment: LOGGER.debug(f"Constructing experiment for auxiliary scan: {auxiliary_scan}") - data: dict[str, Any] = {"time_per_pdf": auxiliary_scan.time_per_pdf} + data: dict[str, Any] = { + "time_per_pdf": auxiliary_scan.time_per_pdf, + "required_auxiliary_scans": auxiliary_scan.requires, + } if auxiliary_scan.list_of_temperatures: settle_time = ( self._queue_params.blower_auxiliary_collection_params.settle_time_s diff --git a/src/daq_queuing_service/plugins/i15_1/tiled_interaction.py b/src/daq_queuing_service/plugins/i15_1/tiled_interaction.py index 9e00504a..7ab54478 100644 --- a/src/daq_queuing_service/plugins/i15_1/tiled_interaction.py +++ b/src/daq_queuing_service/plugins/i15_1/tiled_interaction.py @@ -39,59 +39,22 @@ def _query_tiled(instrument_session: str) -> list[TiledAuxiliary]: .search(KeyPresent("start.experiment_definition.data.time_per_pdf")) ) - items = sorted( - ((key, value) for key, value in result.items()), - key=lambda item: item[1].metadata["start"]["time"], - reverse=True, - ) + tiled_ids = [ + item[0] + for item in sorted( + result.items(), + key=lambda item: item[1].metadata["start"]["time"], + reverse=True, + ) + ] auxiliary_scans: list[TiledAuxiliary] = [] - for item in items: - tiled_id = item[0] - start_doc = item[1].metadata["start"] - filename = f"{start_doc['scan_file']}.nxs" - instrument_session_directory = Path(start_doc["data_session_directory"]) - filepath = instrument_session_directory / filename - - sample = start_doc.get("sample_info") - - if sample is None: - pin = None - else: - capillary = sample["data"].get("capillary") - if not capillary: - LOGGER.warning( - f"No capillary found for tiled scan '{tiled_id}'. Skipping." - ) - continue - pin = StandardsPin( - capillary=sample["data"]["capillary"], - contents=sample["data"].get("composition"), - ) - time_per_pdf = start_doc["experiment_definition"]["data"]["time_per_pdf"] - list_of_temperatures = start_doc["experiment_definition"]["data"].get( - "list_of_temperatures" - ) - - auxiliary_scan = TiledAuxiliary( - tiled_id=tiled_id, - instrument_session=instrument_session, - filename=filename, - instrument_session_directory=instrument_session_directory, - filepath=filepath, - pin=pin, - time_per_pdf=time_per_pdf, - list_of_temperatures=list_of_temperatures, - ) - if auxiliary_scan.kind != start_doc["scan_type"]: - LOGGER.warning( - f"Inferred auxiliary type: '{auxiliary_scan.kind}' does not match " - + f"scan type in metadata: '{start_doc['scan_type']}' for auxiliary" - + f" scan '{tiled_id}'. Skipping." - ) - continue - auxiliary_scans.append(auxiliary_scan) + for tiled_id in tiled_ids: + if auxiliary_scan := construct_tiled_auxiliary( + result, tiled_id, instrument_session + ): + auxiliary_scans.append(auxiliary_scan) LOGGER.debug( f"Found {len(auxiliary_scans)} auxiliary scans in tiled since " @@ -107,3 +70,57 @@ def _query_tiled(instrument_session: str) -> list[TiledAuxiliary]: LOGGER.info( f"Found no suitable auxiliary scans in tiled matching: {required_scan}." ) + + +def construct_tiled_auxiliary( + container: Container, tiled_id: str, instrument_session: str +) -> TiledAuxiliary | None: + start_doc = container[tiled_id].metadata["start"] + filename = f"{start_doc['scan_file']}.nxs" + instrument_session_directory = Path(start_doc["data_session_directory"]) + filepath = instrument_session_directory / filename + + sample = start_doc.get("sample_info") + + if sample is None: + pin = None + else: + capillary = sample["data"].get("capillary") + if not capillary: + LOGGER.warning(f"No capillary found for tiled scan '{tiled_id}'. Skipping.") + return + pin = StandardsPin( + capillary=sample["data"]["capillary"], + contents=sample["data"].get("composition"), + ) + time_per_pdf = start_doc["experiment_definition"]["data"]["time_per_pdf"] + list_of_temperatures = start_doc["experiment_definition"]["data"].get( + "list_of_temperatures" + ) + + requires = [ + construct_tiled_auxiliary( + container, auxiliary_scan["tiled_id"], instrument_session + ) + for auxiliary_scan in start_doc.get("auxiliary_scans", {}).values() + ] + + auxiliary_scan = TiledAuxiliary( + tiled_id=tiled_id, + instrument_session=instrument_session, + filename=filename, + instrument_session_directory=instrument_session_directory, + filepath=filepath, + pin=pin, + time_per_pdf=time_per_pdf, + list_of_temperatures=list_of_temperatures, + requires=[scan for scan in requires if scan is not None], + ) + if auxiliary_scan.kind != start_doc["scan_type"]: + LOGGER.warning( + f"Inferred auxiliary type: '{auxiliary_scan.kind}' does not match " + + f"scan type in metadata: '{start_doc['scan_type']}' for auxiliary" + + f" scan '{tiled_id}'. Skipping." + ) + return + return auxiliary_scan diff --git a/tests/unit_tests/plugins/i15_1/test_auxiliary.py b/tests/unit_tests/plugins/i15_1/test_auxiliary.py index e4784002..d17afcb5 100644 --- a/tests/unit_tests/plugins/i15_1/test_auxiliary.py +++ b/tests/unit_tests/plugins/i15_1/test_auxiliary.py @@ -1,7 +1,10 @@ import pytest from daq_config_server.models.i15_1.standards_puck import StandardsPin -from daq_queuing_service.plugins.i15_1.auxiliary import AuxiliaryScan +from daq_queuing_service.plugins.i15_1.auxiliary import ( + AuxiliaryScan, + combine_auxiliary_requirements, +) @pytest.mark.parametrize( @@ -219,3 +222,158 @@ def test_attempt_to_combine_returns_none_when_auxiliaries_cannot_be_combined( required: AuxiliaryScan, ): assert auxiliary.attempt_to_combine_with(required) is None + + +@pytest.mark.parametrize( + "auxiliaries_1, auxiliaries_2, expected_combined_auxiliaries", + [ + ( + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=None, + ), + ], + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=None, + ), + ], + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=None, + ), + ], + ), + ( + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=[100, 150], + ), + ], + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=None, + ), + ], + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=[100, 150], + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=None, + ), + ], + ), + ( + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=[100, 200], + ), + ], + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=[400, 500], + ), + ], + [ + AuxiliaryScan( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=20.0, + list_of_temperatures=None, + ), + AuxiliaryScan( + instrument_session="cm12345-1", + pin=StandardsPin(capillary="fq1.0", contents=None), + time_per_pdf=20.0, + list_of_temperatures=[100, 200, 300, 400, 500], + ), + ], + ), + ], +) +def test_combine_auxiliary_requirements_combines_auxiliaries_as_expected( + auxiliaries_1: list[AuxiliaryScan], + auxiliaries_2: list[AuxiliaryScan], + expected_combined_auxiliaries: list[AuxiliaryScan], +): + + assert ( + combine_auxiliary_requirements(auxiliaries_1, auxiliaries_2) + == expected_combined_auxiliaries + ) diff --git a/tests/unit_tests/plugins/i15_1/test_i15_1_converter.py b/tests/unit_tests/plugins/i15_1/test_i15_1_converter.py index 934eb708..8d90ec21 100644 --- a/tests/unit_tests/plugins/i15_1/test_i15_1_converter.py +++ b/tests/unit_tests/plugins/i15_1/test_i15_1_converter.py @@ -6,7 +6,7 @@ import pytest from blueapi.service.model import TaskRequest -from blueapi.worker.event import TaskError +from blueapi.worker.event import TaskError, TaskResult from daq_config_server.models.i15_1.queue_parameters import QueueParameters from daq_config_server.models.i15_1.standards_puck import ( STANDARD_CAPILLARY, @@ -66,7 +66,7 @@ def i15_1_tasks(): }, ), ), - id="c5786b54-fd88-4088-8952-8d087cb47028", + id="0", blueapi_calls=[], user=None, ), @@ -94,7 +94,7 @@ def i15_1_tasks(): }, ), ), - id="7a23c7f6-eecf-4376-a5f3-40b0f221635b", + id="1", blueapi_calls=[], user=None, ), @@ -122,7 +122,7 @@ def i15_1_tasks(): }, ), ), - id="c96a85c3-cc4b-41e4-b0b5-7292956b6f7c", + id="2", blueapi_calls=[], user=None, ), @@ -143,7 +143,7 @@ def i15_1_tasks(): name="", id="", data={"time_per_pdf": 20} ), ), - id="5e2571d9-2f87-45b2-ac3b-b486e83a92d6", + id="3", blueapi_calls=[], user=None, ), @@ -171,7 +171,7 @@ def i15_1_tasks(): }, ), ), - id="098d3c5a-5bcb-43a3-acc6-c7e2ae050587", + id="4", blueapi_calls=[], user=None, ), @@ -226,7 +226,10 @@ def air_task( "experiment_definition": { "name": "Air", "id": "", - "data": {"time_per_pdf": time_per_pdf}, + "data": { + "time_per_pdf": time_per_pdf, + "required_auxiliary_scans": [], + }, }, }, ) @@ -240,7 +243,18 @@ def empty_capillary_task( capillary: STANDARD_CAPILLARY, standards_puck: StandardsPuck, ) -> dict[str, Any]: - data: dict[str, Any] = {"time_per_pdf": time_per_pdf} + data: dict[str, Any] = { + "time_per_pdf": time_per_pdf, + "required_auxiliary_scans": [ + { + "instrument_session": instrument_session, + "pin": None, + "time_per_pdf": time_per_pdf, + "list_of_temperatures": None, + "kind": AuxiliaryScanType.AIR, + }, + ], + } if list_of_temperatures is not None: data.update( { @@ -279,6 +293,34 @@ def empty_capillary_task( ) +def air_auxiliary_scan(instrument_session: str, time_per_pdf: float) -> dict[str, Any]: + return { + "instrument_session": instrument_session, + "pin": None, + "time_per_pdf": time_per_pdf, + "list_of_temperatures": None, + "kind": AuxiliaryScanType.AIR, + } + + +def empty_capillary_auxiliary_scan( + instrument_session: str, + time_per_pdf: float, + list_of_temperatures: list[float] | None, + capillary: STANDARD_CAPILLARY, +) -> dict[str, Any]: + return { + "instrument_session": instrument_session, + "pin": { + "capillary": capillary, + "contents": None, + }, + "time_per_pdf": time_per_pdf, + "list_of_temperatures": list_of_temperatures, + "kind": AuxiliaryScanType.EMPTY_CAPILLARY, + } + + def standard_sample_task( task_id: str, instrument_session: str, @@ -286,7 +328,9 @@ def standard_sample_task( capillary: STANDARD_CAPILLARY, contents: STANDARD_SAMPLE, standards_puck: StandardsPuck, + required_empty_capillaries: list[dict[str, Any]], ) -> dict[str, Any]: + return create_task_dict( task_id, { @@ -311,7 +355,13 @@ def standard_sample_task( "experiment_definition": { "name": "Standard Sample", "id": "", - "data": {"time_per_pdf": time_per_pdf}, + "data": { + "time_per_pdf": time_per_pdf, + "required_auxiliary_scans": [ + air_auxiliary_scan(instrument_session, time_per_pdf) + ] + + required_empty_capillaries, + }, }, }, ) @@ -351,7 +401,7 @@ def background_found_in_tiled(): def background_not_found_in_tiled(): with patch( "daq_queuing_service.plugins.i15_1.i15_1_converter.get_suitable_tiled_scan", - MagicMock(return_value=None), + return_value=None, ) as mock_get_suitable_tiled_background: yield mock_get_suitable_tiled_background @@ -693,9 +743,16 @@ def test_if_no_background_found_in_tiled_then_background_scan_added_to_tasks( tasks[1].id, "cm12345-1", 100.0, None, "fq1.0", standards_puck ) assert tasks[2].model_dump() == standard_sample_task( - tasks[2].id, "cm12345-1", 100.0, "fq1.0", "Silicon", standards_puck + tasks[2].id, + "cm12345-1", + 100.0, + "fq1.0", + "Silicon", + standards_puck, + [empty_capillary_auxiliary_scan("cm12345-1", 100.0, None, "fq1.0")], ) - assert converter._tiled_auxiliary_scans == {"1": {}} + + assert list(converter._tiled_auxiliary_scans.values()) == [{}, {}, {}, {}] def test_add_required_background_scans_does_not_add_the_same_background_twice( @@ -781,19 +838,19 @@ def test_add_required_background_scans_combines_similar_background_requirements( assert ( isinstance(new_tasks[2].experiment, Experiment) and new_tasks[2].experiment.sample - and new_tasks[2].experiment.sample.name == "Silicon fq1.0" + and new_tasks[2].experiment.sample.name == "Empty fq1.0" ) assert ( - isinstance(new_tasks[6].experiment, Experiment) - and new_tasks[6].experiment.sample - and new_tasks[6].experiment.sample.name == "Empty fq1.0" + isinstance(new_tasks[3].experiment, Experiment) + and new_tasks[3].experiment.sample + and new_tasks[3].experiment.sample.name == "Silicon fq1.0" ) # Should have the maximum time_per_pdf of i15_1_tasks assert new_tasks[0].experiment.experiment_definition.data["time_per_pdf"] == 25 assert new_tasks[1].experiment.experiment_definition.data["time_per_pdf"] == 25 - assert new_tasks[2].experiment.experiment_definition.data["time_per_pdf"] == 25 + assert new_tasks[2].experiment.experiment_definition.data["time_per_pdf"] == 20 + assert new_tasks[3].experiment.experiment_definition.data["time_per_pdf"] == 25 # This one isn't a temperature experiment so needs a separate empty capillary - assert new_tasks[6].experiment.experiment_definition.data["time_per_pdf"] == 20 def test_same_experiment_in_different_instrument_sessions_will_add_backgrounds_in_each( @@ -820,29 +877,52 @@ def test_same_experiment_in_different_instrument_sessions_will_add_backgrounds_i "fq1.0", standards_puck, ) - assert new_tasks[2].model_dump() == standard_sample_task( - new_tasks[2].id, "cm12345-1", 25.0, "fq1.0", "Silicon", standards_puck + assert new_tasks[2].model_dump() == empty_capillary_task( + new_tasks[2].id, "cm12345-1", 20.0, None, "fq1.0", standards_puck + ) + assert new_tasks[3].model_dump() == standard_sample_task( + new_tasks[3].id, + "cm12345-1", + 25.0, + "fq1.0", + "Silicon", + standards_puck, + [ + empty_capillary_auxiliary_scan( + "cm12345-1", 25.0, [0, 100, 200, 300, 400, 500], "fq1.0" + ), + empty_capillary_auxiliary_scan("cm12345-1", 20.0, None, "fq1.0"), + ], ) - assert new_tasks[4].model_dump() == air_task(new_tasks[4].id, "different", 10.0) - assert new_tasks[5].model_dump() == empty_capillary_task( - new_tasks[5].id, "different", 10.0, [100, 200], "fq1.0", standards_puck + assert new_tasks[5].model_dump() == air_task(new_tasks[5].id, "different", 10.0) + assert new_tasks[6].model_dump() == empty_capillary_task( + new_tasks[6].id, "different", 10.0, [100, 200], "fq1.0", standards_puck ) - assert new_tasks[6].model_dump() == standard_sample_task( - new_tasks[6].id, "different", 10.0, "fq1.0", "Silicon", standards_puck + assert new_tasks[7].model_dump() == standard_sample_task( + new_tasks[7].id, + "different", + 10.0, + "fq1.0", + "Silicon", + standards_puck, + [empty_capillary_auxiliary_scan("different", 10.0, [100, 200], "fq1.0")], ) - assert new_tasks[8].model_dump() == air_task( - new_tasks[8].id, "also_different", 10.0 + assert new_tasks[9].model_dump() == air_task( + new_tasks[9].id, "also_different", 10.0 ) - assert new_tasks[9].model_dump() == empty_capillary_task( - new_tasks[9].id, "also_different", 10.0, [200, 300], "fq1.0", standards_puck + assert new_tasks[10].model_dump() == empty_capillary_task( + new_tasks[10].id, "also_different", 10.0, [200, 300], "fq1.0", standards_puck ) - assert new_tasks[10].model_dump() == standard_sample_task( - new_tasks[10].id, "also_different", 10.0, "fq1.0", "Silicon", standards_puck - ) - assert new_tasks[12].model_dump() == empty_capillary_task( - new_tasks[12].id, "cm12345-1", 20.0, None, "fq1.0", standards_puck + assert new_tasks[11].model_dump() == standard_sample_task( + new_tasks[11].id, + "also_different", + 10.0, + "fq1.0", + "Silicon", + standards_puck, + [empty_capillary_auxiliary_scan("also_different", 10.0, [200, 300], "fq1.0")], ) @@ -874,6 +954,60 @@ def test_add_required_background_scans_if_found_in_tiled_then_no_background_adde } +async def test_auxiliary_scans_have_their_own_required_tiled_auxiliaries_added_to_data( + queue_with_i15_1_plugin: TaskQueue, + queue_params: QueueParameters, + background_not_found_in_tiled: MagicMock, +): + queue_params.do_wait_for_beam = False + queue = await queue_with_i15_1_plugin.get_queue() + assert [task.experiment.experiment_definition.name for task in queue[:4]] == [ # type: ignore + # Auxiliaries added to queue + ScanType.AIR, + ScanType.EMPTY_CAPILLARY, + ScanType.EMPTY_CAPILLARY, + ScanType.STANDARD_SAMPLE, + ] + + air_scan = await queue_with_i15_1_plugin.get_next_call_once_available() + air_scan.put_in_progress() + + air_tiled_scan = TiledAuxiliary( + instrument_session="cm12345-1", + tiled_id="fake_tiled_id", + pin=None, + time_per_pdf=25, + filepath=Path(""), + instrument_session_directory=Path(""), + filename="air_scan.nxs", + ) + background_not_found_in_tiled.return_value = air_tiled_scan + + await queue_with_i15_1_plugin.complete_call( + air_scan, TaskResult(outcome="success", result=None, type="None") + ) + async with queue_with_i15_1_plugin._modifying: + queue_with_i15_1_plugin._sync() + + queue = await queue_with_i15_1_plugin.get_queue() + + assert [task.experiment.experiment_definition.name for task in queue[:3]] == [ # type: ignore + ScanType.EMPTY_CAPILLARY, + ScanType.EMPTY_CAPILLARY, + ScanType.STANDARD_SAMPLE, + ] + converter = queue_with_i15_1_plugin._converter + assert isinstance(converter, I151Converter) + for task in queue: + assert ( + converter._tiled_auxiliary_scans[task.id][AuxiliaryScanType.AIR] + == air_tiled_scan + ) + assert task.blueapi_calls[2].task_request.params["metadata"][ + "auxiliary_scans" + ] == {AuxiliaryScanType.AIR: air_tiled_scan.model_dump()} + + async def test_queue_with_i15_1_converter_can_sync(queue_with_i15_1_plugin: TaskQueue): for i, task in enumerate(queue_with_i15_1_plugin._tasks.values()): task.experiment.instrument_session = str(i) @@ -993,12 +1127,10 @@ def test__ensure_background_in_queue_or_tiled_saves_tiled_info_if_exists( # 5 is wait_for_beam (6, ScanType.EMPTY_CAPILLARY), # 7 is robot unload - # 8 is robot load - # 9 is wait_for_beam - # 10 is centring - # 11 is wait_for_beam - (12, ScanType.STANDARD_SAMPLE), - (18, ScanType.DATA_COLLECTION), + # etc + (12, ScanType.EMPTY_CAPILLARY), + (18, ScanType.STANDARD_SAMPLE), + (24, ScanType.DATA_COLLECTION), ], ) async def test_queued_data_collections_have_correct_scan_type_parameter( diff --git a/tests/unit_tests/plugins/i15_1/test_tiled_interaction.py b/tests/unit_tests/plugins/i15_1/test_tiled_interaction.py index 1cd01b66..ddb59ee5 100644 --- a/tests/unit_tests/plugins/i15_1/test_tiled_interaction.py +++ b/tests/unit_tests/plugins/i15_1/test_tiled_interaction.py @@ -10,6 +10,7 @@ from daq_queuing_service.plugins.i15_1.auxiliary import ( AuxiliaryScan, + AuxiliaryScanType, TiledAuxiliary, ) from daq_queuing_service.plugins.i15_1.tiled_interaction import ( @@ -34,7 +35,7 @@ def mock_tiled_searches( result_1.metadata = { "start": { "time": 1, - "experiment_definition": {"data": {"time_per_pdf": 10}}, + "experiment_definition": {"data": {"time_per_pdf": 25}}, "sample_info": None, "data_session_directory": "/path/to/data/2026/cm12345-1", "scan_file": "i15-1-10000", @@ -50,6 +51,18 @@ def mock_tiled_searches( "data_session_directory": "/path/to/data/2026/cm12345-1", "scan_file": "i15-1-10001", "scan_type": "Empty Capillary", + "auxiliary_scans": { + AuxiliaryScanType.AIR.value: { + "instrument_session": "cm12345-1", + "pin": None, + "time_per_pdf": 25.0, + "list_of_temperatures": None, + "tiled_id": "tiled_id_1", + "filename": "i15-1-10000.nxs", + "instrument_session_directory": "/path/to/data/2026/cm12345-1", + "filepath": "/path/to/data/2026/cm12345-1/i15-1-10000.nxs", + } + }, } } result_3 = MagicMock() @@ -61,6 +74,18 @@ def mock_tiled_searches( "data_session_directory": "/path/to/data/2026/cm12345-1", "scan_file": "i15-1-10002", "scan_type": "Empty Capillary", + "auxiliary_scans": { + AuxiliaryScanType.AIR.value: { + "instrument_session": "cm12345-1", + "pin": None, + "time_per_pdf": 25.0, + "list_of_temperatures": None, + "tiled_id": "tiled_id_1", + "filename": "i15-1-10000.nxs", + "instrument_session_directory": "/path/to/data/2026/cm12345-1", + "filepath": "/path/to/data/2026/cm12345-1/i15-1-10000.nxs", + } + }, } } result_4 = MagicMock() @@ -72,6 +97,18 @@ def mock_tiled_searches( "data_session_directory": "/path/to/data/2026/cm12345-1", "scan_file": "i15-1-10003", "scan_type": "Empty Capillary", + "auxiliary_scans": { + AuxiliaryScanType.AIR.value: { + "instrument_session": "cm12345-1", + "pin": None, + "time_per_pdf": 25.0, + "list_of_temperatures": None, + "tiled_id": "tiled_id_1", + "filename": "i15-1-10000.nxs", + "instrument_session_directory": "/path/to/data/2026/cm12345-1", + "filepath": "/path/to/data/2026/cm12345-1/i15-1-10000.nxs", + } + }, } } result_5 = MagicMock() @@ -90,6 +127,18 @@ def mock_tiled_searches( "data_session_directory": "/path/to/data/2026/cm12345-1", "scan_file": "i15-1-10004", "scan_type": "Empty Capillary", + "auxiliary_scans": { + AuxiliaryScanType.AIR.value: { + "instrument_session": "cm12345-1", + "pin": None, + "time_per_pdf": 25.0, + "list_of_temperatures": None, + "tiled_id": "tiled_id_1", + "filename": "i15-1-10000.nxs", + "instrument_session_directory": "/path/to/data/2026/cm12345-1", + "filepath": "/path/to/data/2026/cm12345-1/i15-1-10000.nxs", + } + }, } } @@ -170,6 +219,21 @@ def test_get_suitable_tiled_background_makes_expected_searches( filename="i15-1-10001.nxs", instrument_session_directory=Path("/path/to/data/2026/cm12345-1"), filepath=Path("/path/to/data/2026/cm12345-1/i15-1-10001.nxs"), + requires=[ + TiledAuxiliary( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=25.0, + list_of_temperatures=None, + requires=[], + tiled_id="tiled_id_1", + filename="i15-1-10000.nxs", + filepath=Path("/path/to/data/2026/cm12345-1/i15-1-10000.nxs"), + instrument_session_directory=Path( + "/path/to/data/2026/cm12345-1" + ), + ) + ], ), ), ( @@ -188,6 +252,21 @@ def test_get_suitable_tiled_background_makes_expected_searches( instrument_session_directory=Path("/path/to/data/2026/cm12345-1"), filepath=Path("/path/to/data/2026/cm12345-1/i15-1-10004.nxs"), list_of_temperatures=[100, 200, 300], + requires=[ + TiledAuxiliary( + instrument_session="cm12345-1", + pin=None, + time_per_pdf=25.0, + list_of_temperatures=None, + requires=[], + tiled_id="tiled_id_1", + filename="i15-1-10000.nxs", + filepath=Path("/path/to/data/2026/cm12345-1/i15-1-10000.nxs"), + instrument_session_directory=Path( + "/path/to/data/2026/cm12345-1" + ), + ) + ], ), ), ],